diff --git a/docs/guides/android/android-analysis.md b/docs/guides/android/android-analysis.md index 0ed6ba06ec..668745826e 100644 --- a/docs/guides/android/android-analysis.md +++ b/docs/guides/android/android-analysis.md @@ -101,10 +101,9 @@ complete decoded manifest. Component, filter, data and metadata `location` value use one-based lines/columns and UTF-16 offsets into that manifest. A component's `declaration_index` preserves its original declaration ordinal, including gaps. -`declaration_coverage` distinguishes complete, partial and unknown observations. -Historical Evidence omits newly introduced fields and defaults coverage to -`unknown`; omission never establishes absence. Current records contain the fields, -with null for absent or unresolved scalar declarations. The inventory retains at +`declaration_coverage` distinguishes complete and partial observations. Every +record contains the declaration fields, with null for absent or unresolved scalar +declarations. The inventory retains at most 32 filters and 64 nested action/category/data/metadata records per component, and shares a 1 MiB structured-value copy budget across components. Omitted values or unsupported filter constructs mark coverage partial and include a limitation; @@ -783,5 +782,4 @@ means the producer did not establish instruction coverage. Using-node totals and coverage do not imply complete instruction evidence. `direct_invokes_coverage` reports counts of using nodes with complete, partial, or unknown positions, plus the number of retained instruction sites. This -aggregate makes position truncation visible without scanning every caller; -older retained Evidence may omit it, which means the aggregate was unexamined. +aggregate makes position truncation visible without scanning every caller. diff --git a/docs/guides/native/binary-diagnostics.md b/docs/guides/native/binary-diagnostics.md index be327912ef..80de3e54e1 100644 --- a/docs/guides/native/binary-diagnostics.md +++ b/docs/guides/native/binary-diagnostics.md @@ -445,23 +445,21 @@ rea inspect-native-exports ./libexample.so --detail summary --evidence_output ./ - ELF rows list defined `.dynsym` symbols with global, weak or GNU-unique binding and default or protected visibility. Each row keeps `value` (`st_value` as stored; bit 0 marks Thumb code on 32-bit ARM), `size`, `type`, - `binding` and `visibility` (`"default"` or `"protected"`; `"unknown"` in older - historical Evidence where visibility was not recorded). ELF string tables do + `binding` and `visibility` (`"default"` or `"protected"`). ELF string tables do not declare a text encoding. Names that contain non-ASCII bytes are decoded as UTF-8 when those bytes form valid UTF-8; `name_utf8_decodable` records whether the non-ASCII name bytes are valid UTF-8 and `name_bytes_base64` retains those exact bytes (`null` for ASCII names). When they are not valid UTF-8, `name` uses a byte-preserving - Latin-1 mapping and the raw bytes remain available. Older retained results - report null for the newly added encoding fields. `version` is null for an + Latin-1 mapping and the raw bytes remain available. `version` is null for an unversioned symbol. A versioned symbol reports its GNU index, a name resolved from `.gnu.version_d` or `.gnu.version_r`, the corresponding `name_bytes_base64` and - `name_utf8_decodable` facts, and the `hidden` flag from `.gnu.version`. + `name_utf8_decodable` facts (both null when the version name is unresolved), + and the `hidden` flag from `.gnu.version`. Version names use the same UTF-8-when-valid, otherwise byte-preserving Latin-1 display mapping as symbol names; ELF does not declare either - string's text encoding. Older retained results report null for the added - version-name encoding fields. `"unknown"` + string's text encoding. `"unknown"` means the symbol-version table could not be read. A hidden version is not the default version that a by-name `dlsym` lookup selects. Definition names take precedence when both tables name the same index. diff --git a/docs/guides/native/native-investigation.md b/docs/guides/native/native-investigation.md index 05fa2cf2b8..e6bd5b3e39 100644 --- a/docs/guides/native/native-investigation.md +++ b/docs/guides/native/native-investigation.md @@ -187,15 +187,14 @@ address tokens, typed references and flow destinations. Mid-instruction, data, outside-memory and unlisted addresses have separate outcomes. For Ghidra, `not-in-listing` means the address is in memory without a listed instruction or defined data. REA does not attempt a decode there: the memory may be uninitialized -or contain valid instructions. Older Evidence may report `undecodable` for this -same ambiguous case; it does not establish that a decode failed. Effective +or contain valid instructions. Effective memory base/index/displacement roles remain unavailable. Ghidra reports its program language variant in `mode`; other providers retain their own mode label. Ghidra MIPS results additionally expose `decoder_context` with the observed `ISA_MODE` register at the queried address and its provider source. A null value is unavailable, not mode zero; an empty array reports no -available context register, and omission preserves older captures or providers -that do not expose this facet. The context is an analysis observation, not proof +available context register, and omission means the provider does not expose +this facet for the program's architecture. The context is an analysis observation, not proof of a runtime processor state. `inspect_native_instruction` also returns `incoming_references` to the exact @@ -229,8 +228,8 @@ profile produce a profile mismatch rather than replaying the former generic terminal result. The old snapshot's `evidence_bundle` remains valid retained Evidence: copy that object to a separate JSON file and use `rea evidence-import` to validate it. Run analysis without the old snapshot and save a fresh one -under the current profile. Other providers and older captures may omit the -classification evidence field. +under the current profile. Other providers may omit the classification +evidence field. Call resolution reports direct, resolved indirect, ambiguous, unresolved and non-call outcomes from static call references. It does not establish runtime diff --git a/docs/guides/web/browser-observation.md b/docs/guides/web/browser-observation.md index 1e2909a74b..66297c43cf 100644 --- a/docs/guides/web/browser-observation.md +++ b/docs/guides/web/browser-observation.md @@ -54,7 +54,7 @@ than inferring completeness from the array. Passive DOM comparison checks complete node inventories and complete frame parent/child structure independently. Node value lengths are comparable only when both nonempty snapshots declare the same `node_value_length_unit`. -Missing or different units leave length comparison unknown; independent node +Different units leave length comparison unknown; independent node or frame structural differences can still prove change. Empty inventories have no value lengths to compare. Frame IDs and sibling enumeration order are capture-local and do not establish a change; duplicate IDs, missing parents, @@ -429,8 +429,8 @@ contextual evidence with `timestamp: null`, accompanied by a limitation stating that it does not establish activity in the selected window. Main-frame scope enforcement still applies to contextual navigation events. Network redirect events retain their paired prior-hop response context when the timestamped -next-request event falls inside the window. A redirect row's optional -`redirect_response` contains the prior response `url`, CDP-reported `status`, +next-request event falls inside the window. Every redirect row, and only a redirect row, has a +`redirect_response` that contains the prior response `url`, CDP-reported `status`, and `scope`. The URL has transport credentials removed; URL and status are withheld unless its origin is approved. Missing or malformed response URLs have null scope and add `invalid_protocol_value` timeline coverage; a valid URL with an @@ -438,8 +438,9 @@ unsupported protocol has `unsupported` scope. Outside-policy and unsupported responses mark the timeline as policy-filtered with `disallowed_origin` or `unsupported_url`. Missing status remains null rather than zero. A supplied non-numeric status is also null and records `invalid_protocol_value` coverage; -the other valid response facts remain available. Historical rows may omit this -field; omission does not establish that no prior response existed. Response +the other valid response facts remain available. A redirect row without this +field is malformed and refused; recreate such observations with the current +release. Response headers and bodies are discarded. The row timestamp belongs to the next request, not an exact prior response time. The pre-arm timeline buffer is bounded by original CDP message bytes. If clock sampling completes after that budget is @@ -490,7 +491,7 @@ not bound the whole process heap. REA retains local observation data and removes only URL userinfo credentials, explicitly declared scenario secrets, and values from structured credential/storage fields: - URL observations preserve the complete local URL, including query values, fragments, duplicate parameters, and their order. Only username/password userinfo is removed; malformed original URL strings remain available with `origin: null`. -- DOM snapshots retain node types, node names, value lengths, and attribute names, but not text or attribute values. `dom.node_value_length_unit` is `utf16-code-units` for web observations and applies to every node's `node_value_length`; it counts JavaScript string code units, not Unicode code points or UTF-8 bytes. Electron observations declare `utf8-bytes` instead. Historical captures without the unit leave it unknown; reading them does not infer a unit or convert their stored lengths. +- DOM snapshots retain node types, node names, value lengths, and attribute names, but not text or attribute values. `dom.node_value_length_unit` is `utf16-code-units` for web observations and applies to every node's `node_value_length`; it counts JavaScript string code units, not Unicode code points or UTF-8 bytes. Electron observations declare `utf8-bytes` instead. A snapshot without the unit is malformed; REA does not infer a unit or convert stored lengths. - Supported DOM URL attributes are projected separately as resolved destinations. Empty `action` on HTML or XHTML forms and empty `formaction` on HTML or XHTML submit controls resolve to the document URL; other relative URL attributes resolve against the captured document base. - Accessibility structure and reported roles are retained by default, while names and descriptions require `include_accessibility_text: true`. The browser protocol may omit an individual role; that role remains unknown for comparison. Other observed accessibility differences can still establish a change. - Network observations retain method, status, MIME type, size, type, initiator stack location, and complete URLs with userinfo credentials removed. Redirects reported by CDP are attached as ordered `redirects` on the one request ID, with the prior request URL, CDP response URL, method/type, response status/MIME/encoded length, request timestamp, and the next `requestWillBeSent` event timestamp. CDP delivers `redirectResponse` with that next request event, so the latter is an event boundary rather than an exact response timestamp. Redirect response headers and bodies are discarded; hops outside the selected origins are discarded with the request chain. Headers are discarded after an allowlisted projection of length/encoding, structured CSP/Link/policy fields, and untrusted agent hints. Cookies and authorization headers are never retained. diff --git a/docs/providers/ghidra-provider.md b/docs/providers/ghidra-provider.md index e3a07fd7dd..f7553abea8 100644 --- a/docs/providers/ghidra-provider.md +++ b/docs/providers/ghidra-provider.md @@ -230,8 +230,7 @@ status and limitations state. Relocation rows retain Ghidra's `symbol` value, including an empty string or null when returned by Ghidra. It is a provider observation, not proof of the -runtime-resolved address. Older Evidence may not have recorded this field; -a relocation page requesting it reports that uncertainty. +runtime-resolved address. Every relocation row records the field. For large relocation inventories, select MCP `inspect_native_load_image` with `detail: "summary"`. It returns verification state, observed identity and counts, diff --git a/docs/reference/mcp-contracts.md b/docs/reference/mcp-contracts.md index 9d0d7d90f4..a344554b1e 100644 --- a/docs/reference/mcp-contracts.md +++ b/docs/reference/mcp-contracts.md @@ -1013,7 +1013,11 @@ whether it was never recorded, cleared, or retained by another connection. Supply complete inline Evidence, repeat its producer, or import an exported Evidence bundle before referencing that imported record. Export a bundle or use `reset_investigation(save_path)` before ending the process if the investigation -needs it later. Retention has no automatic persistence or silent eviction. +needs it later. Exported bundles carry `evidence_format`, the Evidence format revision +(currently `1`). Import, comparison and every other bundle reader refuse a +bundle without it, or with another revision, as an integrity mismatch whose +recovery is to recreate the observations with the current release; +re-exporting an earlier bundle does not update its format or identities. Retention has no automatic persistence or silent eviction. `binary_session.investigation` reports record/revision/head counts, estimated retained representation bytes and the fixed allowance derived from initial V8 heap headroom. A capacity refusal preserves the existing investigation; checkpoint diff --git a/docs/reference/reconstruction-obligation-ledgers.md b/docs/reference/reconstruction-obligation-ledgers.md index 66f3bf5028..5e3310dafb 100644 --- a/docs/reference/reconstruction-obligation-ledgers.md +++ b/docs/reference/reconstruction-obligation-ledgers.md @@ -36,6 +36,7 @@ never claims closure: rea build-reconstruction-obligation-ledger \ '{ "evidence_bundle": { + "evidence_format": 1, "artifacts": [], "providers": [], "environments": [], diff --git a/src/android/AndroidManifestComponents.ts b/src/android/AndroidManifestComponents.ts index 03f63f3d46..7ae622a346 100644 --- a/src/android/AndroidManifestComponents.ts +++ b/src/android/AndroidManifestComponents.ts @@ -9,8 +9,8 @@ import { createAndroidManifestDetailsReader } from "./AndroidManifestComponentDe import { ANDROID_COMPONENT_KINDS, type AndroidComponentKind, - type ObservedAndroidManifestComponent, - type ObservedAndroidManifestComponentInventory, + type AndroidManifestComponent, + type AndroidManifestComponentInventory, } from "../domain/android/androidManifestComponents.js"; /** Bounds keep a hostile manifest from exhausting the inspection. */ @@ -35,7 +35,7 @@ const PARTIAL_LIMITATION = const unavailable = ( limitation: string, -): ObservedAndroidManifestComponentInventory => ({ +): AndroidManifestComponentInventory => ({ status: "unavailable", items: [], limitations: [limitation], @@ -152,7 +152,7 @@ export const parseAndroidManifestComponents = ( coverage: "complete" | "partial", declaredTargetSdk: string | null = null, minimumSdk: string | null = null, -): ObservedAndroidManifestComponentInventory => { +): AndroidManifestComponentInventory => { if (coverage === "partial") return { status: "partial", @@ -185,7 +185,7 @@ export const parseAndroidManifestComponents = ( const packageNameBytes = packageName === null ? 0 : Buffer.byteLength(packageName); const limitations: string[] = []; - const items: ObservedAndroidManifestComponent[] = []; + const items: AndroidManifestComponent[] = []; let truncated = false; let applicationSeen = false; let unnamed = 0; diff --git a/src/android/JadxAnalysis.ts b/src/android/JadxAnalysis.ts index 736353d927..b5da2323db 100644 --- a/src/android/JadxAnalysis.ts +++ b/src/android/JadxAnalysis.ts @@ -1,7 +1,6 @@ import type { z } from "zod"; import type { AndroidRequest } from "../domain/android/androidAnalysis.js"; import { androidResultSchemas } from "../domain/android/androidAnalysis.js"; -import { observedAndroidManifestComponentsSchema } from "../domain/android/androidManifestComponents.js"; import { type AndroidMappingQuery, type AndroidMappingSource, @@ -64,11 +63,6 @@ export interface JadxAnalysisContext { }; } -const observedPackageSchema = - androidResultSchemas.inspect_android_package.extend({ - components: observedAndroidManifestComponentsSchema, - }); - const methodsOf = (summary: Pick) => { const seen = new Map(); return summary.methods.map((method) => { @@ -276,7 +270,7 @@ const inspectPackage = async ( const manifest = normalizeJadxText( await tools.text("get_android_manifest", {}), ); - return observedPackageSchema.parse({ + return androidResultSchemas.inspect_android_package.parse({ engine, package_name: app.package || null, version_name: app.version_name || null, diff --git a/src/application/analysisView/NativeLoadImageViews.test.ts b/src/application/analysisView/NativeLoadImageViews.test.ts index cf8643147c..174ad3b1b7 100644 --- a/src/application/analysisView/NativeLoadImageViews.test.ts +++ b/src/application/analysisView/NativeLoadImageViews.test.ts @@ -2,10 +2,7 @@ import { expect, it } from "vitest"; import { createEvidence } from "../../domain/evidence.js"; import { inspectAnalysisViewInputSchema } from "../../domain/analysisView/analysisViewRequest.js"; import { analysisViewResultSchema } from "../../domain/analysisView/analysisViewResult.js"; -import { - RELOCATION_SYMBOL_UNRECORDED, - projectNativeLoadImageView, -} from "../../domain/analysisView/nativeLoadImageView.js"; +import { projectNativeLoadImageView } from "../../domain/analysisView/nativeLoadImageView.js"; import type { NativeLoadImage } from "../../domain/native/nativeLoadImage.js"; import { nativeDosLoadImageFixture, @@ -27,6 +24,7 @@ const source = (): Extract => ({ offset: null, status: "APPLIED", type: 8, + symbol: null, values: [16], original_bytes_hex: "0000", memory_bytes_hex: "1000", @@ -37,6 +35,7 @@ const source = (): Extract => ({ offset: null, status: "UNSUPPORTED", type: 9, + symbol: null, values: [], original_bytes_hex: null, memory_bytes_hex: null, @@ -68,7 +67,7 @@ const view = async (report: NativeLoadImage, request: unknown) => { return analysisViewResultSchema.parse(result.value.normalized_result); }; -it("distinguishes recorded, absent, and historical unrecorded relocation symbols", async () => { +it("pages recorded and absent relocation symbols", async () => { const report = source(); report.observations.relocations[0]!.symbol = "external_symbol"; report.observations.relocations[1]!.symbol = null; @@ -81,7 +80,6 @@ it("distinguishes recorded, absent, and historical unrecorded relocation symbols { record: { symbol: "external_symbol" } }, { record: { symbol: null } }, ]); - expect(current.unknowns).not.toContain(RELOCATION_SYMBOL_UNRECORDED); report.observations.relocations[1]!.symbol = ""; const emptyName = await view(report, request); @@ -89,15 +87,11 @@ it("distinguishes recorded, absent, and historical unrecorded relocation symbols if (emptyName.items.collection !== "relocations") throw new Error("Expected relocations"); expect(emptyName.items.rows?.[1]?.record).toMatchObject({ symbol: "" }); - expect(emptyName.unknowns).not.toContain(RELOCATION_SYMBOL_UNRECORDED); - delete report.observations.relocations[0]!.symbol; - const historical = await view(report, request); - if (historical.kind !== "native-load-image") throw new Error("Expected page"); - expect(historical.items.rows?.[0]?.record).not.toHaveProperty("symbol"); - expect(historical.unknowns).toContain(RELOCATION_SYMBOL_UNRECORDED); - const omitted = await view(report, { ...request, fields: ["address"] }); - expect(omitted.unknowns).not.toContain(RELOCATION_SYMBOL_UNRECORDED); + delete (report.observations.relocations[0] as { symbol?: unknown }).symbol; + await expect(view(report, request)).rejects.toMatchObject({ + _tag: "AnalysisInputError", + }); }); it("summarizes measured ELF facts without upgrading unsupported verification", async () => { @@ -145,6 +139,7 @@ it("preserves same-address alternatives, padding, nulls and byte facts over page "offset", "status", "type", + "symbol", "values", "original_bytes_hex", "memory_bytes_hex", @@ -301,6 +296,7 @@ it("keeps byte and potentially large comparison payloads opt-in while retaining offset: null, status: "APPLIED", type: 8, + symbol: null, values: [16], }); const report = nativeDosLoadImageFixture(); diff --git a/src/application/artifacts/ArtifactComparisonEvidence.test.ts b/src/application/artifacts/ArtifactComparisonEvidence.test.ts index 9c0d7f16a4..62fb67152a 100644 --- a/src/application/artifacts/ArtifactComparisonEvidence.test.ts +++ b/src/application/artifacts/ArtifactComparisonEvidence.test.ts @@ -1,3 +1,4 @@ +import { RECREATE_EVIDENCE_GUIDANCE } from "../../domain/evidenceFormat.js"; import { expect, it } from "vitest"; import { inspectAnalysisView } from "../analysisView/AnalysisViewService.js"; @@ -270,7 +271,20 @@ const summaryAtEra = async (era: SummaryEra, aliasPath?: string) => { }; }; -it.each(["pre-ce1f4b1bb", "pre-b395190e4", "current"])( +// The Evidence format owner no longer rebuilds 6.7-era summary identities. +it.each(["pre-ce1f4b1bb", "pre-b395190e4"])( + "refuses the %s summary with recreate guidance", + async (era) => { + const { parent, summary } = await summaryAtEra(era); + const resolved = resolveArtifactComparisonEvidence(summary, () => parent); + expect(resolved.ok).toBe(false); + if (resolved.ok) return; + expect(resolved.error).toBeInstanceOf(EvidenceIntegrityError); + expect(resolved.error.message).toContain(RECREATE_EVIDENCE_GUIDANCE); + }, +); + +it.each(["current"])( "authenticates the exact %s summary against its retained inventory", async (era) => { const { parent, summary } = await summaryAtEra(era); @@ -281,7 +295,7 @@ it.each(["pre-ce1f4b1bb", "pre-b395190e4", "current"])( }, ); -it.each(["pre-ce1f4b1bb", "pre-b395190e4", "current"])( +it.each(["current"])( "authenticates a %s summary projected from the caller alias against the original retained parent", async (era) => { const { parent, summary } = await summaryAtEra(era, "/selected/alias.bin"); @@ -297,7 +311,7 @@ it.each(["pre-ce1f4b1bb", "pre-b395190e4", "current"])( }, ); -it.each(["pre-ce1f4b1bb", "pre-b395190e4", "current"])( +it.each(["current"])( "authenticates a %s summary after its retained parent is rebound to an alias path", async (era) => { const { parent, summary } = await summaryAtEra(era); @@ -426,7 +440,7 @@ it("authenticates a retained parent's content instead of trusting its original I expect(resolved.error).toBeInstanceOf(EvidenceIntegrityError); }); -it.each(["pre-ce1f4b1bb", "pre-b395190e4", "current"])( +it.each(["current"])( "preserves both immutable Evidence commitments when resolving %s", async (era) => { const { parent, summary } = await summaryAtEra(era, "/selected/alias.bin"); diff --git a/src/application/artifacts/ArtifactComparisonEvidence.ts b/src/application/artifacts/ArtifactComparisonEvidence.ts index dd3d4c727e..bfcc50d4e3 100644 --- a/src/application/artifacts/ArtifactComparisonEvidence.ts +++ b/src/application/artifacts/ArtifactComparisonEvidence.ts @@ -25,7 +25,7 @@ import { type Evidence, } from "../../domain/evidence.js"; import { readEvidence } from "../../domain/evidenceValidation.js"; -import { artifactInventorySummaryProjectionIds } from "../../domain/analysisView/artifactInventorySummaryProjection.js"; +import { RECREATE_EVIDENCE_GUIDANCE } from "../../domain/evidenceFormat.js"; import { err, ok, type Result } from "../../domain/result.js"; const OPERATION = "compare_artifacts"; @@ -178,18 +178,15 @@ export const resolveArtifactComparisonEvidence = ( (evidenceId) => (evidenceId === parentId ? presentation : undefined), ); if (!reprojected.ok) return reprojected; - if ( - !artifactInventorySummaryProjectionIds(reprojected.value).includes( - evidence.evidence_id, - ) - ) - return mismatch(); + // Only the current format's projection identifies the summary; the format + // owner refuses earlier projections instead of rebuilding them. + if (reprojected.value.evidence_id !== evidence.evidence_id) return mismatch(); return ok(owner); }; const mismatch = (): Result => err( new EvidenceIntegrityError( - "Artifact summary does not match its retained parent inventory.", + `Artifact summary does not match its retained parent inventory. A summary from an earlier Evidence format no longer matches. ${RECREATE_EVIDENCE_GUIDANCE}`, ), ); diff --git a/src/application/desktop/DesktopApplicationState.test.ts b/src/application/desktop/DesktopApplicationState.test.ts index 8e8992cffe..7332e01a9f 100644 --- a/src/application/desktop/DesktopApplicationState.test.ts +++ b/src/application/desktop/DesktopApplicationState.test.ts @@ -1,3 +1,4 @@ +import { DESKTOP_VIEW_DEFAULTS } from "../../domain/desktop/desktopViewPolicy.js"; import { chmod, open, readFile, stat, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { DatabaseSync } from "node:sqlite"; @@ -184,6 +185,25 @@ describe("desktop state view admission", () => { }); }); +it("records the complete effective view policy and refuses state without it", async () => { + const evidence = await inspectDesktopApplicationState( + await writeCaptureFolder(capture()), + readers, + ); + if (!("normalized_result" in evidence)) throw evidence; + const state = desktopApplicationStateSchema.parse(evidence.normalized_result); + expect(state.effective_view_policy).toEqual(DESKTOP_VIEW_DEFAULTS); + const { effective_view_policy: policy, ...unrecorded } = state; + const { sqlite_view_bytes: _bytes, ...partial } = policy; + for (const refused of [ + unrecorded, + { ...state, effective_view_policy: partial }, + ]) + expect(desktopApplicationStateSchema.safeParse(refused).success).toBe( + false, + ); +}); + describe("inspectDesktopApplicationState", () => { it("assigns roles and reads each version into typed views and changes", async () => { const folder = await writeCaptureFolder(capture()); diff --git a/src/application/desktop/DesktopSqliteState.test.ts b/src/application/desktop/DesktopSqliteState.test.ts index 0de2ce6575..7188fc2a11 100644 --- a/src/application/desktop/DesktopSqliteState.test.ts +++ b/src/application/desktop/DesktopSqliteState.test.ts @@ -40,6 +40,7 @@ const report = (selected: string | undefined): SqliteDatabase => ({ artifact: { path: "/staged/state.db", sha256: "a".repeat(64), bytes: 4096 }, wal: null, engine: { name: "SQLite", version: "3.50.2" }, + worker_timeout_ms: 30_000, database_encoding: "UTF-8", schema: { scope: "database", diff --git a/src/application/desktop/DesktopTaskDraft.ts b/src/application/desktop/DesktopTaskDraft.ts index 80b91ceb78..91f75b4c4a 100644 --- a/src/application/desktop/DesktopTaskDraft.ts +++ b/src/application/desktop/DesktopTaskDraft.ts @@ -120,9 +120,7 @@ const draft = async ( }); const environment = exportEnvironmentName(capture, undefined); const result = { - ...(state.effective_view_policy === undefined - ? {} - : { effective_view_policy: state.effective_view_policy }), + effective_view_policy: state.effective_view_policy, application: { environment, title: capture.application.desktop_entry?.name ?? environment, diff --git a/src/application/managed/ManagedNativeVerificationService.platform.test.ts b/src/application/managed/ManagedNativeVerificationService.platform.test.ts index 49e6520c61..f006e01e8a 100644 --- a/src/application/managed/ManagedNativeVerificationService.platform.test.ts +++ b/src/application/managed/ManagedNativeVerificationService.platform.test.ts @@ -25,6 +25,7 @@ const native = (format: NativeFormat, path: string, decorated: boolean) => { items: [ { name: symbolTableName, + runtime_name: name, address: "0x1000", weak: false, reexport: false, diff --git a/src/application/native/NativeValueTrace.fixture.ts b/src/application/native/NativeValueTrace.fixture.ts index e68bd7342d..42c16fe295 100644 --- a/src/application/native/NativeValueTrace.fixture.ts +++ b/src/application/native/NativeValueTrace.fixture.ts @@ -75,6 +75,12 @@ export const nativeFlowDossier = ( native_value_flow: { available: true, provenance: "ghidra-high-pcode", + limits: { + max_operations: 3_000, + max_inputs_per_operation: 64, + max_edges: 12_000, + }, + limits_reached: [], operations, def_use: [], effects: [], diff --git a/src/application/native/NativeValueTrace.test.ts b/src/application/native/NativeValueTrace.test.ts index b87023f29f..c2c560fbf8 100644 --- a/src/application/native/NativeValueTrace.test.ts +++ b/src/application/native/NativeValueTrace.test.ts @@ -30,6 +30,12 @@ import { valueTraceSubject as subject, } from "./NativeValueTrace.fixture.js"; +const valueFlowLimits = { + max_operations: 3_000, + max_inputs_per_operation: 64, + max_edges: 12_000, +}; + const traceNativeValues: typeof traceNativeValuesCore = async ( source, parameters, @@ -95,6 +101,8 @@ const analysis: AnalysisOperationPort = { native_value_flow: { available: true, provenance: "ghidra-high-pcode", + limits: valueFlowLimits, + limits_reached: [], operations: callee ? [node("return", "0x2000", "RETURN")] : [node("call", "0x1000", "CALL")], @@ -243,6 +251,8 @@ describe("native value parameter-use node membership", () => { const largeFlow = { available: true as const, provenance: "ghidra-high-pcode" as const, + limits: valueFlowLimits, + limits_reached: [], operations: Array.from({ length: operationCount }, (_, index) => ({ ...node(`operation-${String(index)}`, procedureAddress, "MULTIEQUAL"), inputs: [scalar, scalar, scalar, scalar], @@ -261,10 +271,7 @@ describe("native value parameter-use node membership", () => { known_omitted_edges: 0, limitations: [], }; - const largeDossier = { - ...base, - native_value_flow: largeFlow, - }; + const largeDossier = { ...base, native_value_flow: largeFlow }; const largeAnalysis: AnalysisOperationPort = { execute: async () => ok(createAnalysisExecution(largeDossier, provider, { subject })), @@ -332,6 +339,8 @@ describe("native value parameter-use node membership", () => { native_value_flow: { available: true, provenance: "ghidra-high-pcode", + limits: valueFlowLimits, + limits_reached: [], operations: [ node("parameter:0", address, "COPY"), node("parameter:0#1", address, "COPY"), diff --git a/src/application/sqlite/SqliteDatabaseService.test.ts b/src/application/sqlite/SqliteDatabaseService.test.ts index 9926f2fa1c..3fd7f42278 100644 --- a/src/application/sqlite/SqliteDatabaseService.test.ts +++ b/src/application/sqlite/SqliteDatabaseService.test.ts @@ -177,10 +177,11 @@ it("accepts sqlite_schema rows without requiring a fabricated declaration", asyn ).toMatchObject({ ok: false, error: { _tag: "AnalysisOutputError" } }); }); -it("parses historical report omissions as unknown without supplying a default", () => { +// Evidence format 1 requires the reported worker deadline; a report from an +// earlier format is refused rather than read as unknown. +it("refuses a report without its worker deadline", () => { const { worker_timeout_ms: _default, ...historical } = fixture(); - const parsed = sqliteDatabaseSchema.parse(historical); - expect(parsed).not.toHaveProperty("worker_timeout_ms"); + expect(sqliteDatabaseSchema.safeParse(historical).success).toBe(false); }); it("preserves producer failures and rejects pre-cancelled requests before effects", async () => { diff --git a/src/contracts/android/androidToolContracts.ts b/src/contracts/android/androidToolContracts.ts index 3be5889ac3..e1a21edc35 100644 --- a/src/contracts/android/androidToolContracts.ts +++ b/src/contracts/android/androidToolContracts.ts @@ -53,7 +53,7 @@ export const ANDROID_TOOL_CONTRACTS = [ name: "inspect_android_package", ...toolContractMetadata("inspect_android_package"), description: - "Inspect APK package/SDK/permission declarations, decoded manifest, static components and class/resource counts without executing it or verifying signatures. Requires caller-supplied JADX MCP JAR and JDK. Default complete returns full Evidence; detail summary retains it and omits manifest text/raw duplicates. With summary, view kind android selects facet components (exact filters, fields, paging), component-summary or manifest (UTF-16 text pages); complete detail rejects view. Use normalized_result.parent_evidence_id with inspect_analysis_view for later views. Component source_index identifies the original declaration, and declaration_coverage preserves unknowns.", + "Inspect APK package/SDK/permission declarations, decoded manifest, static components and class/resource counts without executing it or verifying signatures. Requires caller-supplied JADX MCP JAR and JDK. Default complete returns full Evidence; detail summary retains it and omits manifest text/raw duplicates. With summary, view kind android selects facet components (exact filters, fields, paging), component-summary or manifest (UTF-16 text pages); complete detail rejects view. Use normalized_result.parent_evidence_id with inspect_analysis_view for later views. Component source_index identifies the original declaration, and declaration_coverage marks partial declarations.", kind: "android-provider", inputSchema: androidPackageInspectionInputSchema, outputSchema: evidenceResultOf( diff --git a/src/contracts/applicationToolContracts.ts b/src/contracts/applicationToolContracts.ts index 2e26a4a0bb..13f0ba7626 100644 --- a/src/contracts/applicationToolContracts.ts +++ b/src/contracts/applicationToolContracts.ts @@ -296,6 +296,7 @@ export const APPLICATION_TOOL_CONTRACTS = [ title: "Generate the complete obligation ledger", input: { evidence_bundle: { + evidence_format: 1, artifacts: [], providers: [], environments: [], @@ -327,6 +328,7 @@ export const APPLICATION_TOOL_CONTRACTS = [ input: { coverage: { evidence_bundle: { + evidence_format: 1, artifacts: [], providers: [], environments: [], diff --git a/src/contracts/javascript/javascriptRuntimeReconciliationExample.ts b/src/contracts/javascript/javascriptRuntimeReconciliationExample.ts index 9af0980a76..74830c478d 100644 --- a/src/contracts/javascript/javascriptRuntimeReconciliationExample.ts +++ b/src/contracts/javascript/javascriptRuntimeReconciliationExample.ts @@ -253,7 +253,11 @@ const runtimeEvidence = createEvidence( }, completeness: browserCompleteness(), frames: [], - dom: { total_nodes: 0, nodes: [] }, + dom: { + total_nodes: 0, + node_value_length_unit: "utf8-bytes", + nodes: [], + }, scripts: { total: 1, items: [ diff --git a/src/domain/analysisView/analysisViewResult.ts b/src/domain/analysisView/analysisViewResult.ts index 98918bf728..0f1459b308 100644 --- a/src/domain/analysisView/analysisViewResult.ts +++ b/src/domain/analysisView/analysisViewResult.ts @@ -774,11 +774,11 @@ export const androidMethodSummaryViewResultSchema = z.strictObject({ summary: androidMethodSummarySchema, }); -/** Legacy manifest views keep their original shape; new facets are explicit. */ +/** One page of the retained manifest text. */ export const androidManifestViewResultSchema = z.strictObject({ ...viewResultBase, kind: z.literal("android"), - facet: z.literal("manifest").optional(), + facet: z.literal("manifest"), view: androidManifestViewRequestSchema, item: androidManifestViewItemSchema, parent_operation: z.literal("inspect_android_package"), diff --git a/src/domain/analysisView/androidPackageView.test.ts b/src/domain/analysisView/androidPackageView.test.ts index 53bb2f050a..f6ed9217b7 100644 --- a/src/domain/analysisView/androidPackageView.test.ts +++ b/src/domain/analysisView/androidPackageView.test.ts @@ -24,6 +24,16 @@ const component = (name: string, extra: Record = {}) => ({ foreground_service_type: null, authorities: null, target_activity: null, + declaration_coverage: "complete", + declaration_index: 0, + location: null, + enabled: null, + enabled_declaration: null, + process: null, + grant_uri_permissions: null, + grant_uri_permissions_declaration: null, + intent_filters: [], + meta_data: [], ...extra, }); const report = ( @@ -148,19 +158,37 @@ it("keeps parent ordinals, duplicate declarations and coverage after filtering, expect(JSON.stringify(source)).toBe(before); }); -it("keeps omitted historical facts unknown even with empty or expanded field selections", () => { - const source = parent(report([component("legacy.Class")])); - for (const fields of [[], ["enabled", "intent_filters", "location"]]) { - const page = androidComponentsViewResultSchema.parse( - project(source, { kind: "android", facet: "components", fields }), - ); - expect(page.items).toEqual([ - { source_index: 0, declaration_coverage: "unknown" }, - ]); - } +it("refuses a component without its declaration facets instead of reporting them unknown", () => { + const { declaration_coverage: _coverage, ...unrecorded } = + component("unrecorded.Class"); + const result = projectAnalysisView( + parent({ + ...report(), + components: { status: "complete", items: [unrecorded], limitations: [] }, + } as unknown as ReturnType), + analysisViewRequestSchema.parse({ kind: "android", facet: "components" }), + ); + expect(result).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisInputError", + issues: [ + { + path: [ + "source", + "normalized_result", + "components", + "items", + 0, + "declaration_coverage", + ], + }, + ], + }, + }); }); -it("counts explicit exported declarations separately from defaults and unknown historical coverage", () => { +it("counts explicit exported declarations separately from defaults and declaration coverage", () => { const source = parent(); const summary = androidComponentSummaryViewResultSchema.parse( project(source, { kind: "android", facet: "component-summary" }), @@ -176,7 +204,7 @@ it("counts explicit exported declarations separately from defaults and unknown h }, exported_declarations: { true: 1, false: 1, unknown: 1 }, exported_defaults: { true: 1, false: 0, unknown: 2 }, - declaration_coverage: { complete: 1, partial: 1, unknown: 1 }, + declaration_coverage: { complete: 2, partial: 1 }, }); const explicit = androidComponentsViewResultSchema.parse( project(source, { @@ -237,7 +265,7 @@ it("bounds selected row serialization and lets field selection recover without c project(source, { kind: "android", facet: "components", fields: ["name"] }), ); expect(small.items).toEqual([ - { source_index: 0, name: "large.Class", declaration_coverage: "unknown" }, + { source_index: 0, name: "large.Class", declaration_coverage: "complete" }, ]); }); diff --git a/src/domain/analysisView/androidPackageView.ts b/src/domain/analysisView/androidPackageView.ts index db0cf0437b..c455398d2b 100644 --- a/src/domain/analysisView/androidPackageView.ts +++ b/src/domain/analysisView/androidPackageView.ts @@ -83,7 +83,6 @@ export const androidComponentSummarySchema = z.strictObject({ declaration_coverage: z.strictObject({ complete: z.int().nonnegative(), partial: z.int().nonnegative(), - unknown: z.int().nonnegative(), }), }); @@ -188,6 +187,7 @@ export const projectAndroidPackageView = ( if (!page.ok) return page; return ok({ kind: "android" as const, + facet: "manifest" as const, view, ...shared, item: { @@ -212,7 +212,7 @@ export const projectAndroidPackageView = ( unknowns: boundedViewMessages( [ ...shared.unknowns, - "Components are static declarations. Omitted historical declaration fields remain unknown; exported declarations and SDK defaults are distinct.", + "Components are static declarations; exported declarations and SDK defaults are distinct.", ...(report.components.status === "complete" ? [] : [ @@ -275,7 +275,7 @@ export const projectAndroidPackageView = ( }, exported_declarations: { true: 0, false: 0, unknown: 0 }, exported_defaults: { true: 0, false: 0, unknown: 0 }, - declaration_coverage: { complete: 0, partial: 0, unknown: 0 }, + declaration_coverage: { complete: 0, partial: 0 }, }; for (const row of rows) { if (!matches(row)) continue; diff --git a/src/domain/analysisView/artifactInventorySummaryProjection.ts b/src/domain/analysisView/artifactInventorySummaryProjection.ts deleted file mode 100644 index 7ad2a10074..0000000000 --- a/src/domain/analysisView/artifactInventorySummaryProjection.ts +++ /dev/null @@ -1,82 +0,0 @@ -import { sealAnalysisView } from "./analysisView.js"; -import { artifactInventorySummaryViewResultSchema } from "./analysisViewResult.js"; -import { ARTIFACT_INVENTORY_VIEW_CAVEATS } from "./artifactInventoryViewCaveats.js"; -import { digestCanonicalValue } from "../canonicalDigest.js"; -import { evidenceSemanticContent, type Evidence } from "../evidence.js"; - -// The ordinary-root formats actually admitted before ce1f4b1bb/b395190e4. -// Godot/PyInstaller summaries already disclosed coverage in those revisions. -// Freeze this historical set: newly introduced formats have no older aliases. -const HISTORICAL_ORDINARY_ROOT_FORMATS: ReadonlySet = new Set([ - "dmg", - "zip", - "pkg", - "asar", - "javascript-bundle", - "source-map", - "mach-o-universal", - "mach-o", - "elf", - "pe", - "dos-mz", - "dos-com", - "raw-binary", - "hopper", - "analysis-database", - "ipa", - "apk", - "msix", - "appx", - "hap", - "hsp", - "app-pack", - "plist", - "entitlements", - "directory", - "file", - "unknown", -]); - -/** Exact shipped projections, rather than arbitrary omission or prose tolerance. */ -export const artifactInventorySummaryProjectionIds = ( - current: Evidence, -): readonly string[] => { - const result = artifactInventorySummaryViewResultSchema.parse( - current.normalized_result, - ); - const format = result.summary.manifest.root_format; - if (!HISTORICAL_ORDINARY_ROOT_FORMATS.has(format)) - return [current.evidence_id]; - const { view_digest: _digest, ...projection } = result; - // b395190e4 began disclosing unavailable hashes for ordinary root formats. - const beforeHashDisclosure = { - ...projection, - unknowns: projection.unknowns.filter( - (value) => value !== ARTIFACT_INVENTORY_VIEW_CAVEATS.unknownContentHashes, - ), - }; - // ce1f4b1bb began carrying traversal coverage and its caveat for these roots. - const { source_coverage: _coverage, ...summary } = projection.summary; - const beforeTraversalDisclosure = { - ...beforeHashDisclosure, - summary, - unknowns: beforeHashDisclosure.unknowns.filter( - (value) => value !== ARTIFACT_INVENTORY_VIEW_CAVEATS.unknownTraversal, - ), - }; - return [ - current.evidence_id, - ...[beforeHashDisclosure, beforeTraversalDisclosure].map( - (variant) => - `ev_${digestCanonicalValue( - evidenceSemanticContent({ - ...current, - normalized_result: artifactInventorySummaryViewResultSchema.parse( - sealAnalysisView(variant), - ), - }), - "Evidence", - )}`, - ), - ]; -}; diff --git a/src/domain/analysisView/gpuCodeObjectsView.ts b/src/domain/analysisView/gpuCodeObjectsView.ts index f8d82e6b29..828dfe2f8d 100644 --- a/src/domain/analysisView/gpuCodeObjectsView.ts +++ b/src/domain/analysisView/gpuCodeObjectsView.ts @@ -62,24 +62,16 @@ export const gpuCodeObjectsSummarySchema = gpuCodeObjectsObjectSchema }); export const gpuCodeObjectsItemsSchema = z.union([ z.array( - gpuObjectSchema - .partial() - .required({ - id: true, - kind: true, - target: true, - location: true, - metadata_location: true, - payload_target: true, - target_consistency: true, - inspection: true, - }) - .extend({ - // Source defaults qualify historical full rows, never omitted page columns. - amd_target_metadata: gpuObjectSchema.shape.amd_target_metadata - .removeDefault() - .optional(), - }), + gpuObjectSchema.partial().required({ + id: true, + kind: true, + target: true, + location: true, + metadata_location: true, + payload_target: true, + target_consistency: true, + inspection: true, + }), ), z.array( gpuKernelSchema.partial().required({ diff --git a/src/domain/analysisView/machoView.test.ts b/src/domain/analysisView/machoView.test.ts index a9818507df..f5a1d44f5c 100644 --- a/src/domain/analysisView/machoView.test.ts +++ b/src/domain/analysisView/machoView.test.ts @@ -44,63 +44,37 @@ it("binds every Mach-O facet to its rows, including empty pages", () => { } }); -it("keeps historical symbol defaults scoped to their selected Mach-O facet", () => { - const { runtime_name: _runtimeName, ...historical } = symbol("_malloc"); - const projected = projectAnalysisView( - parent({ - ...report, - imports: coverage([historical]), - exports: coverage([historical]), - }), - { kind: "macho", facet: "exports", offset: 0, limit: 64 }, - ); - if (!projected.ok || projected.value.kind !== "macho") - throw new Error("Expected Mach-O page"); - expect(projected.value.item).toEqual([{ ...historical, runtime_name: null }]); - expect(projected.value.item[0]).not.toHaveProperty("raw_name"); - expect( - analysisViewResultSchema.safeParse({ - ...projected.value, - item: undefined, - }).success, - ).toBe(false); -}); - -it.each(["not-encrypted", "malformed"] as const)( - "projects historical encryption rows with %s status", - (status) => { - const current = inspectMachoSchema.parse(report); - const { malformed_kind: _unrecorded, ...recorded } = - current.encryption.items[0]!; - const historical = { - ...recorded, - status, - ...(status === "malformed" - ? { reason: "slice has 3 encryption load commands", range: null } - : {}), - }; - const source = { ...current, encryption: coverage([historical]) }; - expect(inspectMachoSchema.safeParse(source).success).toBe(false); - const projected = projectAnalysisView(parent(source), { - kind: "macho", - facet: "encryption", - offset: 0, - limit: 10, - }); - if (!projected.ok || projected.value.kind !== "macho") - throw new Error("Expected a historical Mach-O encryption page"); - expect(projected.value.item).toEqual([ - { - ...historical, - malformed_kind: status === "malformed" ? "unrecorded" : null, - }, - ]); - expect(analysisViewResultSchema.safeParse(projected.value).success).toBe( - true, +it("refuses symbols and encryption rows missing fields every producer writes", () => { + const { runtime_name: _runtimeName, ...withoutRuntimeName } = + symbol("_malloc"); + const current = inspectMachoSchema.parse(report); + const { malformed_kind: _malformedKind, ...withoutCause } = + current.encryption.items[0]!; + for (const [source, path] of [ + [ + { ...report, exports: coverage([withoutRuntimeName]) }, + ["exports", "items", 0, "runtime_name"], + ], + [ + { ...report, encryption: coverage([withoutCause]) }, + ["encryption", "items", 0, "malformed_kind"], + ], + [{ ...report, encryption: undefined }, ["encryption"]], + ] as const) { + const parsed = inspectMachoSchema.safeParse(source); + expect(parsed.error?.issues.map((issue) => issue.path)).toContainEqual( + path, ); - expect(historical).not.toHaveProperty("malformed_kind"); - }, -); + expect( + projectAnalysisView(parent(source), { + kind: "macho", + facet: "encryption", + offset: 0, + limit: 10, + }).ok, + ).toBe(false); + } +}); const symbol = (name: string) => ({ name, @@ -110,6 +84,12 @@ const symbol = (name: string) => ({ reexport: null, source: "libSystem", }); +const importSymbol = (name: string) => ({ + ...symbol(name), + raw_name: null, + name_status: "exact" as const, + addend: null, +}); const coverage = (items: T[], exhaustive = true) => ({ items, total: exhaustive ? items.length : null, @@ -148,7 +128,7 @@ const report = { }, ]), imports: coverage( - ["_malloc", "_free", "_open"].map((name) => symbol(name)), + ["_malloc", "_free", "_open"].map((name) => importSymbol(name)), false, ), exports: coverage([symbol("__mh_execute_header")]), @@ -317,7 +297,7 @@ it("retains binary names and exact numbers alongside partial command fields", () true, ); } - // Historical records do not contain the newly added exact-byte fields. + // Names that decode as UTF-8 carry no exact-byte fields. expect(inspectMachoSchema.safeParse(report).success).toBe(true); expect(report.segments.items[0]).not.toHaveProperty("name_bytes_hex"); }); @@ -387,7 +367,7 @@ describe("Mach-O analysis views", () => { if (!first.ok) throw first.error; expect(first.value).toMatchObject({ kind: "macho", - item: [symbol("_free")], + item: [importSymbol("_free")], source_total: null, source_exhaustive: false, coverage: { status: "page", examined: 1, total: 3, next_offset: 2 }, @@ -436,25 +416,6 @@ describe("Mach-O analysis views", () => { }); describe("Mach-O view uncertainty", () => { - it("keeps encryption unknown for Evidence retained before it was reported", () => { - const { encryption: _omitted, ...legacy } = report; - const projected = projectAnalysisView(parent(legacy), { kind: "summary" }); - if (!projected.ok) throw projected.error; - expect(projected.value).toMatchObject({ - summary: { - collections: { - encryption: { returned: 0, total: null, exhaustive: false }, - }, - }, - unknowns: [expect.stringContaining("exhaustive imports, encryption;")], - }); - expect(projected.value.limitations).toContainEqual( - expect.stringMatching( - /inspect_macho Evidence.*predates encryption reporting.*slice encryption status is unknown/iu, - ), - ); - }); - it("keeps a missing subject unknown instead of inventing identity", () => { const projected = projectAnalysisView(parent(report, null), { kind: "summary", diff --git a/src/domain/analysisView/machoView.ts b/src/domain/analysisView/machoView.ts index 5a016cf8dc..51194934e6 100644 --- a/src/domain/analysisView/machoView.ts +++ b/src/domain/analysisView/machoView.ts @@ -4,10 +4,7 @@ import type { AnalysisError } from "../analysisErrorBase.js"; import { uniqueSorted } from "../canonicalOrdering.js"; import { isMachOFormat } from "../binaryTargetTypes.js"; import { machoSliceEncryptionViewSchema } from "../apple/machoEncryption.js"; -import { - inspectMachoSchema, - retainedInspectMachoSchema, -} from "../native/nativeInspection.js"; +import { inspectMachoSchema } from "../native/nativeInspection.js"; import { err, ok, type Result } from "../result.js"; import { parseAnalysisViewResult } from "./analysisViewParent.js"; import type { AnalysisViewParent } from "./analysisViewParent.js"; @@ -91,13 +88,13 @@ export const machoSummarySchema = z.strictObject({ limitation_count: z.int().nonnegative(), }); -type MachoReport = z.output; +type MachoReport = z.output; /** Admit a well-formed report whose recorded subject, if any, is a Mach-O. */ const parseMachoReport = ( parent: AnalysisViewParent, ): Result => { - const parsed = parseAnalysisViewResult(retainedInspectMachoSchema, parent); + const parsed = parseAnalysisViewResult(inspectMachoSchema, parent); if (!parsed.ok) return parsed; if (parent.subject != null && !isMachOFormat(parent.subject.format)) return err( diff --git a/src/domain/analysisView/nativeExportsView.ts b/src/domain/analysisView/nativeExportsView.ts index 2dad3ee5cd..fce8e42268 100644 --- a/src/domain/analysisView/nativeExportsView.ts +++ b/src/domain/analysisView/nativeExportsView.ts @@ -57,19 +57,7 @@ export const nativeExportsViewRequestSchema = z.discriminatedUnion( export const nativeExportsViewItemsSchema = z.union([ z.array(peRow.partial().extend(sourceIndex)), - z.array( - elfRow.partial().extend({ - ...sourceIndex, - // Historical defaults normalize complete source rows, never omitted columns. - name_bytes_base64: elfRow.shape.name_bytes_base64 - .removeDefault() - .optional(), - name_utf8_decodable: elfRow.shape.name_utf8_decodable - .removeDefault() - .optional(), - visibility: elfRow.shape.visibility.removeDefault().optional(), - }), - ), + z.array(elfRow.partial().extend(sourceIndex)), z.array(issue.extend(sourceIndex)), ]); @@ -194,21 +182,6 @@ export const projectNativeExportsView = ( : []; }), ); - if ( - names !== undefined && - names.size > 0 && - report.format === "elf" && - report.exports.some( - (row) => - row.name_utf8_decodable === null || - (row.name_utf8_decodable && - row.name_bytes_base64 === null && - !/^[\x00-\x7f]*$/u.test(row.name)), - ) - ) - shared.unknowns.push( - "Some ELF names lack retained encoding facts and cannot be matched by UTF-8 identity; select their source indices. A no-match page does not establish their absence.", - ); // A projected decoded name still carries its exact bytes and encoding qualification. const columns = new Set(fields); if (report.format === "elf" && columns.has("name")) { @@ -244,7 +217,7 @@ export const projectNativeExportsView = ( ? row.name_utf8_decodable === true && (row.name_bytes_base64 !== null ? encodedNames!.has(row.name_bytes_base64) - : /^[\x00-\x7f]*$/u.test(row.name) && names.has(row.name)) + : names.has(row.name)) : row.name !== null && names.has(row.name))), projectRow, ); diff --git a/src/domain/analysisView/nativeLoadImageView.ts b/src/domain/analysisView/nativeLoadImageView.ts index c381349228..cfa808c302 100644 --- a/src/domain/analysisView/nativeLoadImageView.ts +++ b/src/domain/analysisView/nativeLoadImageView.ts @@ -198,11 +198,7 @@ const selectRows = (source: readonly (Row | string)[], view: PageRequest) => { }; }; -/** Query the owner's parsed historical observations without reopening the target. */ -/** A selected historical relocation row omitted its symbol field. */ -export const RELOCATION_SYMBOL_UNRECORDED = - "The relocation symbol was not recorded for at least one selected row."; - +/** Query the owner's parsed observations without reopening the target. */ export const projectNativeLoadImageView = ( parent: AnalysisViewParent, admitted: AdmittedView, @@ -296,16 +292,8 @@ export const projectNativeLoadImageView = ( : undefined : observed[view.collection]; const page = selected === undefined ? undefined : selectRows(selected, view); - const symbolUnrecorded = - view.collection === "relocations" && - (view.fields === undefined || - view.fields.some((field: string) => field === "symbol")) && - (page?.rows.some(({ record }) => !("symbol" in record)) ?? false); return ok({ ...shared, - unknowns: symbolUnrecorded - ? [...shared.unknowns, RELOCATION_SYMBOL_UNRECORDED] - : shared.unknowns, kind: "native-load-image" as const, verification_status: report.status, view, diff --git a/src/domain/analysisView/pcapCaptureView.test.ts b/src/domain/analysisView/pcapCaptureView.test.ts index 1f0152d4a9..8f108ed46a 100644 --- a/src/domain/analysisView/pcapCaptureView.test.ts +++ b/src/domain/analysisView/pcapCaptureView.test.ts @@ -7,7 +7,7 @@ const capture = ( tsharkStderr: string, dnsQueries: { query: string; - query_labels_hex?: string[] | null; + query_labels_hex: string[] | null; response_addresses: string[]; response_address_count: number; }[] = [], @@ -22,9 +22,15 @@ const capture = ( tshark_command: "/usr/bin/tshark", tshark_source: "path", tshark_version: "4.6.6", + tshark_resolved_path: null, + tshark_sha256: null, + tshark_identity_status: "unresolved", capinfos_command: "/usr/bin/capinfos", capinfos_source: "path", capinfos_version: "4.6.6", + capinfos_resolved_path: null, + capinfos_sha256: null, + capinfos_identity_status: "unresolved", }, target, file_type: "pcap", @@ -81,23 +87,6 @@ const diagnostics = (offset: number, limit: number) => limit, }) as const; -it("reads historical capture Evidence without inventing executable identity", () => { - const result = projectAnalysisView(capture(""), { kind: "summary" }); - if (!result.ok) throw result.error; - if (result.value.kind !== "summary") - throw new Error("Expected a packet summary."); - if (!("client" in result.value.summary)) - throw new Error("Expected packet client facts."); - expect(result.value.summary.client).toMatchObject({ - tshark_resolved_path: null, - tshark_sha256: null, - tshark_identity_status: "unresolved", - capinfos_resolved_path: null, - capinfos_sha256: null, - capinfos_identity_status: "unresolved", - }); -}); - it("pages diagnostics without splitting a surrogate pair", () => { const parent = capture("ab😀cd"); expect(projectAnalysisView(parent, diagnostics(3, 2))).toMatchObject({ @@ -165,21 +154,39 @@ it("selects colliding DNS displays by label identity and retains unknown occurre ).toMatchObject({ ok: true, value: { page: { items: ["192.0.2.4"] } } }); }); -it("keeps historical DNS records accessible without inventing label identity", () => { - const parent = capture("", [ - { - query: "old.example", - response_addresses: ["192.0.2.9"], - response_address_count: 1, - }, - ]); - expect( - projectAnalysisView(parent, { - kind: "pcap", - facet: "dns-addresses", - query: "old.example", - offset: 0, - limit: 64, - }), - ).toMatchObject({ ok: true, value: { page: { items: ["192.0.2.9"] } } }); -}); +it.each([ + ["executable identity", ["client", "tshark_identity_status"]], + ["DNS label identity", ["dns_queries", 0, "query_labels_hex"]], +] as const)( + "refuses capture Evidence that omits %s instead of defaulting it", + (_label, path) => { + const parent = capture("", [ + { + query: "a.example", + query_labels_hex: ["61", "6578616d706c65"], + response_addresses: ["192.0.2.9"], + response_address_count: 1, + }, + ]); + const report = structuredClone(parent.normalizedResult) as Record< + string, + unknown + >; + let owner: unknown = report; + for (const key of path.slice(0, -1)) + owner = (owner as Record)[key]; + delete (owner as Record)[path.at(-1)!]; + expect( + projectAnalysisView( + { ...parent, normalizedResult: report }, + { kind: "summary" }, + ), + ).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisInputError", + issues: [{ path: ["source", "normalized_result", ...path] }], + }, + }); + }, +); diff --git a/src/domain/analysisView/pcapCaptureView.ts b/src/domain/analysisView/pcapCaptureView.ts index cc7f05b9b7..d42268dd70 100644 --- a/src/domain/analysisView/pcapCaptureView.ts +++ b/src/domain/analysisView/pcapCaptureView.ts @@ -1,6 +1,6 @@ import { analysisViewInputError } from "./analysisViewErrors.js"; import { z } from "zod"; -import { pcapEvidenceSchemas } from "../pcap/pcapAnalysis.js"; +import { pcapResultSchemas } from "../pcap/pcapAnalysis.js"; import { parseAnalysisViewResult } from "./analysisViewParent.js"; import type { AnalysisError } from "../analysisErrorBase.js"; import { err, ok, type Result } from "../result.js"; @@ -16,7 +16,7 @@ import { } from "./analysisViewCoverage.js"; import { analysisTextPage } from "./analysisViewText.js"; -const captureSchema = pcapEvidenceSchemas.inspect_pcap_capture; +const captureSchema = pcapResultSchemas.inspect_pcap_capture; const fields = captureSchema.shape; const pageShape = { kind: z.literal("pcap"), @@ -39,9 +39,10 @@ export const pcapViewRequestSchema = z.discriminatedUnion("facet", [ ...pageShape, facet: z.literal("dns-addresses"), query: z.string().min(1), - query_labels_hex: - fields.dns_queries.element.shape.query_labels_hex.describe( - "Select the exact retained wire label bytes for this query display. Null selects unknown identity, including historical records with the field omitted; omit this selector to match any identity.", + query_labels_hex: fields.dns_queries.element.shape.query_labels_hex + .optional() + .describe( + "Select the exact retained wire label bytes for this query display. Null selects unknown identity; omit this selector to match any identity.", ), query_index: z .int() @@ -226,7 +227,7 @@ const selectRows = ( query.query === view.query && (view.query_index === undefined || index === view.query_index) && (view.query_labels_hex === undefined || - JSON.stringify(query.query_labels_hex ?? null) === + JSON.stringify(query.query_labels_hex) === JSON.stringify(view.query_labels_hex)), ); if (matches.length !== 1) diff --git a/src/domain/android/androidAnalysis.ts b/src/domain/android/androidAnalysis.ts index 2e7be99b49..a80289e3cd 100644 --- a/src/domain/android/androidAnalysis.ts +++ b/src/domain/android/androidAnalysis.ts @@ -233,15 +233,7 @@ export const androidResultSchemas = { target_sdk: z.string().nullable(), permissions: z.array(z.string()), manifest: textCoverage, - // Evidence retained before the component projection existed re-parses - // with an explicit unavailable inventory; current results always carry it. - components: androidManifestComponentsSchema.default(() => ({ - status: "unavailable" as const, - items: [], - limitations: [ - "This Evidence predates the manifest component projection; run inspect_android_package again to obtain it.", - ], - })), + components: androidManifestComponentsSchema, class_count: z.int().nonnegative(), resource_count: z.int().nonnegative(), signature_verification: z.literal("not_performed"), @@ -288,8 +280,7 @@ export const androidResultSchemas = { }), trace_android_references: z.strictObject({ ...common, - /** Older retained Evidence omits this fact; omission does not mean external. */ - target_definition: z.enum(["defined", "external"]).exactOptional(), + target_definition: z.enum(["defined", "external"]), /** External results inventory directly encoded invokes, not reflection or dispatch. */ reference_scope: z.literal("direct_dex_invokes").exactOptional(), class_name: z.string(), @@ -298,16 +289,13 @@ export const androidResultSchemas = { reported_signature: z.string().nullable(), dex_descriptor: dexDescriptorSchema.nullable(), total: z.int().nonnegative(), - /** Position coverage is separate from the complete using-node inventory. - * Historical Evidence may omit this aggregate; omission is unexamined. */ - direct_invokes_coverage: z - .strictObject({ - complete_using_nodes: z.int().nonnegative(), - partial_using_nodes: z.int().nonnegative(), - unknown_using_nodes: z.int().nonnegative(), - recorded_sites: z.int().nonnegative(), - }) - .exactOptional(), + /** Position coverage is separate from the complete using-node inventory. */ + direct_invokes_coverage: z.strictObject({ + complete_using_nodes: z.int().nonnegative(), + partial_using_nodes: z.int().nonnegative(), + unknown_using_nodes: z.int().nonnegative(), + recorded_sites: z.int().nonnegative(), + }), references: z.array( z.strictObject({ kind: z.string(), diff --git a/src/domain/android/androidManifestComponents.ts b/src/domain/android/androidManifestComponents.ts index 9219e9b8eb..519dc2a896 100644 --- a/src/domain/android/androidManifestComponents.ts +++ b/src/domain/android/androidManifestComponents.ts @@ -61,20 +61,16 @@ export const androidManifestComponentSchema = z.strictObject({ foreground_service_type: z.string().nullable(), authorities: z.array(z.string()).nullable(), target_activity: z.string().nullable(), - // Historical records never observed these facets. Omission stays omission; - // the default describes unknown coverage, never an absent declaration. - declaration_coverage: z - .enum(["complete", "partial", "unknown"]) - .default("unknown"), - declaration_index: z.int().nonnegative().optional(), - location: androidManifestLocationSchema.nullable().optional(), - enabled: z.boolean().nullable().optional(), - enabled_declaration: z.string().nullable().optional(), - process: z.string().nullable().optional(), - grant_uri_permissions: z.boolean().nullable().optional(), - grant_uri_permissions_declaration: z.string().nullable().optional(), - intent_filters: z.array(androidIntentFilterSchema).optional(), - meta_data: z.array(androidComponentMetadataSchema).optional(), + declaration_coverage: z.enum(["complete", "partial"]), + declaration_index: z.int().nonnegative(), + location: androidManifestLocationSchema.nullable(), + enabled: z.boolean().nullable(), + enabled_declaration: z.string().nullable(), + process: z.string().nullable(), + grant_uri_permissions: z.boolean().nullable(), + grant_uri_permissions_declaration: z.string().nullable(), + intent_filters: z.array(androidIntentFilterSchema), + meta_data: z.array(androidComponentMetadataSchema), }); export type AndroidManifestComponent = Readonly< z.output @@ -84,17 +80,6 @@ export const androidManifestComponentsSchema = z.strictObject({ items: z.array(androidManifestComponentSchema), limitations: z.array(z.string()), }); - -/** Current producers observe every declaration facet; historical readers accept omissions. */ -export const observedAndroidManifestComponentSchema = - androidManifestComponentSchema.required(); -export type ObservedAndroidManifestComponent = Readonly< - z.output ->; -export const observedAndroidManifestComponentsSchema = - androidManifestComponentsSchema.extend({ - items: z.array(observedAndroidManifestComponentSchema), - }); -export type ObservedAndroidManifestComponentInventory = Readonly< - z.output +export type AndroidManifestComponentInventory = Readonly< + z.output >; diff --git a/src/domain/apple/machoEncryption.ts b/src/domain/apple/machoEncryption.ts index 4e84cadecc..0615436741 100644 --- a/src/domain/apple/machoEncryption.ts +++ b/src/domain/apple/machoEncryption.ts @@ -111,13 +111,10 @@ const sliceEncryptionFields = { }; const validateMalformedKind = ( - slice: { status: string; malformed_kind?: string | null | undefined }, + slice: { status: string; malformed_kind: string | null }, context: z.RefinementCtx, ): void => { - if ( - slice.malformed_kind !== undefined && - (slice.status === "malformed") !== (slice.malformed_kind !== null) - ) + if ((slice.status === "malformed") !== (slice.malformed_kind !== null)) context.addIssue({ code: "custom", path: ["malformed_kind"], @@ -130,36 +127,11 @@ export const machoSliceEncryptionSchema = z .object(sliceEncryptionFields) .superRefine(validateMalformedKind); -/** View rows also report when a historical malformed cause was not recorded. */ +/** View rows carry exactly the producer's slice fields. */ export const machoSliceEncryptionViewSchema = z - .strictObject({ - ...sliceEncryptionFields, - malformed_kind: z - .enum([...malformedKinds, "unrecorded"]) - .nullable() - .describe( - "Structural cause or duplicate-command ambiguity. Unrecorded means the historical inspection did not classify the cause. Null for other statuses.", - ), - }) + .strictObject(sliceEncryptionFields) .superRefine(validateMalformedKind); -/** Interpret earlier slice records without inferring a cause from their command count. */ -export const retainedMachoSliceEncryptionSchema = z - .object({ - ...sliceEncryptionFields, - malformed_kind: malformedKind.exactOptional(), - }) - .superRefine(validateMalformedKind) - .transform((slice): z.output => ({ - ...slice, - malformed_kind: - slice.malformed_kind === undefined - ? slice.status === "malformed" - ? "unrecorded" - : null - : slice.malformed_kind, - })); - export type MachoEncryptionCommand = z.infer< typeof machoEncryptionCommandSchema >; diff --git a/src/domain/browserSession.ts b/src/domain/browserSession.ts index 24d5a9f4d2..8009a0d006 100644 --- a/src/domain/browserSession.ts +++ b/src/domain/browserSession.ts @@ -30,44 +30,53 @@ const destinationScopeSchema = z.enum([ "outside_policy", "unsupported", ]); -const timelineEventSchema = z.object({ - sequence: z.int().min(1), - type: z.enum([ - "navigation_requested", - "navigation_committed", - "same_origin_reload", - "same_document_navigation", - "redirect", - "load_failed", - "lifecycle", - "target_terminated", - "session_detached", - "transport_disconnected", - ]), - timestamp: z - .number() - .min(0) - .nullable() - .describe( - "CDP monotonic seconds when supplied; null when the event has no valid comparable producer timestamp.", - ), - frame_id: z.string().nullable(), - loader_id: z.string().nullable(), - request_id: z.string().nullable(), - url: z.string().nullable(), - destination_scope: destinationScopeSchema.nullable(), - detail: z.string().nullable(), - redirect_response: z - .object({ - url: z.string().nullable(), - status: z.number().nullable(), - scope: destinationScopeSchema.nullable(), - }) - .optional() - .describe( - "Prior-hop response metadata. URL and status require an approved response origin; null scope is unknown. Omitted in historical Evidence.", - ), -}); +const timelineEventSchema = z + .object({ + sequence: z.int().min(1), + type: z.enum([ + "navigation_requested", + "navigation_committed", + "same_origin_reload", + "same_document_navigation", + "redirect", + "load_failed", + "lifecycle", + "target_terminated", + "session_detached", + "transport_disconnected", + ]), + timestamp: z + .number() + .min(0) + .nullable() + .describe( + "CDP monotonic seconds when supplied; null when the event has no valid comparable producer timestamp.", + ), + frame_id: z.string().nullable(), + loader_id: z.string().nullable(), + request_id: z.string().nullable(), + url: z.string().nullable(), + destination_scope: destinationScopeSchema.nullable(), + detail: z.string().nullable(), + redirect_response: z + .object({ + url: z.string().nullable(), + status: z.number().nullable(), + scope: destinationScopeSchema.nullable(), + }) + .optional() + .describe( + "Prior-hop response metadata, present exactly on redirect events. URL and status require an approved response origin; null scope is unknown.", + ), + }) + .refine( + (event) => + (event.type === "redirect") === (event.redirect_response !== undefined), + { + path: ["redirect_response"], + message: "redirect_response is present exactly on redirect events.", + }, + ); /** Navigation-aware session result for external user actions. */ export const webObservationSessionSchema = z.object({ diff --git a/src/domain/cdpObservationSchemas.ts b/src/domain/cdpObservationSchemas.ts index 6e705d1192..6b3b39dc2f 100644 --- a/src/domain/cdpObservationSchemas.ts +++ b/src/domain/cdpObservationSchemas.ts @@ -24,15 +24,12 @@ export const cdpDomNodeSchema = z.object({ attribute_names: z.array(z.string()), }); -/** Unit applies to every retained node; historical omission remains unknown. */ +/** Unit applies to every retained node. */ export const cdpDomSnapshotSchema = z.object({ total_nodes: z.int().min(0), node_value_length_unit: z .enum(["utf16-code-units", "utf8-bytes"]) - .describe( - "Unit for all node_value_length values; absent in historical captures means unknown.", - ) - .exactOptional(), + .describe("Unit for all node_value_length values."), nodes: z.array(cdpDomNodeSchema), }); diff --git a/src/domain/desktop/desktopState.ts b/src/domain/desktop/desktopState.ts index 64f772751c..bb2d3d5812 100644 --- a/src/domain/desktop/desktopState.ts +++ b/src/domain/desktop/desktopState.ts @@ -106,14 +106,8 @@ export const versionAt = ( /** The normalized result of `inspect_desktop_application_state`. */ export const desktopApplicationStateSchema = z.strictObject({ - /** Policy actually used to produce these views; older Evidence may omit it. */ - effective_view_policy: desktopViewPolicyInputSchema - .required() - // Historical policy records predate the additive table-view byte control. - .extend({ - sqlite_view_bytes: desktopViewPolicyInputSchema.shape.sqlite_view_bytes, - }) - .optional(), + /** Policy actually used to produce these views. */ + effective_view_policy: desktopViewPolicyInputSchema.required(), capture: z.strictObject({ evidence_id: z.string().min(1), home: z.string().min(1), diff --git a/src/domain/evidence.ts b/src/domain/evidence.ts index 96f9d76e1b..be2c9ecf4a 100644 --- a/src/domain/evidence.ts +++ b/src/domain/evidence.ts @@ -1,3 +1,4 @@ +import { RECREATE_EVIDENCE_GUIDANCE } from "./evidenceFormat.js"; import { isDeepStrictEqual } from "node:util"; import { basename } from "node:path"; import { z } from "zod"; @@ -160,7 +161,7 @@ const authenticatedEvidenceSchema = evidenceSchema.superRefine( code: "custom", path: ["evidence_id"], message: usesPre66NullCoverageIdentity(withoutId, evidenceId) - ? "Evidence uses an earlier relationship_coverage identity commitment: explicit null was omitted from that digest. Recreate this observation with the current REA release; re-exporting the old bundle cannot update its identity." + ? `Evidence uses an earlier relationship_coverage identity commitment: explicit null was omitted from that digest. ${RECREATE_EVIDENCE_GUIDANCE}` : "Evidence semantic identifier does not match its record", }); }, diff --git a/src/domain/evidenceBundle.ts b/src/domain/evidenceBundle.ts index 362e273617..7c952c09f9 100644 --- a/src/domain/evidenceBundle.ts +++ b/src/domain/evidenceBundle.ts @@ -23,6 +23,13 @@ import { type ResidualUnknown, } from "./residualUnknown.js"; import { freezeJsonSnapshot } from "./immutableJson.js"; +import { + admitEvidenceBundleFormat, + EVIDENCE_FORMAT_REVISION, + evidenceFormatSchema, + RECREATE_EVIDENCE_GUIDANCE, + UnsupportedEvidenceFormatError, +} from "./evidenceFormat.js"; const artifactManifestBaseSchema = z.strictObject({ digest: z.strictObject({ sha256: digestSchema }), @@ -43,6 +50,7 @@ const captureManifestSchema = z.strictObject({ }); export const evidenceBundleSchema = z.strictObject({ + evidence_format: evidenceFormatSchema, artifacts: z.array(artifactManifestSchema), providers: z.array(providerManifestSchema), environments: z.array(environmentManifestSchema), @@ -109,6 +117,7 @@ export const createEvidenceBundle = ( compareUnicodeCodePoints(left.evidence_id, right.evidence_id), ); return { + evidence_format: EVIDENCE_FORMAT_REVISION, artifacts: uniqueSorted( sortedRecords.flatMap(({ subject }) => subject === null @@ -257,6 +266,7 @@ export const referencedEvidenceIds = ( ]; type EvidenceBundleValidationFailure = + | UnsupportedEvidenceFormatError | z.ZodError | DuplicateEvidenceRecordError | InvalidResidualUnknownGraphError @@ -264,7 +274,7 @@ type EvidenceBundleValidationFailure = /** Expected source rejection, including authenticated standalone Evidence. */ export type EvidenceBundleFailure = { - readonly kind: "single-record" | "invalid-bundle"; + readonly kind: "single-record" | "unsupported-format" | "invalid-bundle"; readonly message: string; readonly cause: EvidenceBundleValidationFailure; }; @@ -282,12 +292,16 @@ const describeEvidenceBundleFailure = ( }; } catch (recordFailure: unknown) { if (!(recordFailure instanceof z.ZodError)) throw recordFailure; + // The format owner decides earlier bundles once; its message carries the + // recreate guidance instead of per-field schema failures. + if (cause instanceof UnsupportedEvidenceFormatError) + return { kind: "unsupported-format", cause, message: cause.message }; return { kind: "invalid-bundle", cause, message: cause instanceof z.ZodError - ? `Evidence bundle does not match the bundle schema ${describeValidationFailure(cause)}. Supply an unmodified bundle produced by this REA identity format. Recreate observations from earlier formats; re-exporting an old bundle does not update its identities.` + ? `Evidence bundle does not match the bundle schema ${describeValidationFailure(cause)}. Supply an unmodified bundle produced by this REA release. ${RECREATE_EVIDENCE_GUIDANCE}` : `Evidence bundle validation failed: ${describeValidationFailure(cause)}. Recreate or re-export the bundle, then try again.`, }; } @@ -317,6 +331,7 @@ const parseBundle = (input: unknown, seal: boolean): EvidenceBundle => { const owned = immutableBundles.get(input); if (owned !== undefined) return owned; } + admitEvidenceBundleFormat(input); return authenticateBundle(evidenceBundleSchema.parse(input), seal); }; @@ -357,6 +372,7 @@ const readBundle = ( ); } catch (cause: unknown) { if ( + !(cause instanceof UnsupportedEvidenceFormatError) && !(cause instanceof z.ZodError) && !(cause instanceof DuplicateEvidenceRecordError) && !(cause instanceof InvalidResidualUnknownGraphError) && diff --git a/src/domain/evidenceFormat.test.ts b/src/domain/evidenceFormat.test.ts new file mode 100644 index 0000000000..b5e0c4832a --- /dev/null +++ b/src/domain/evidenceFormat.test.ts @@ -0,0 +1,60 @@ +import { expect, it } from "vitest"; + +import { createEvidence } from "./evidence.js"; +import { + createEvidenceBundle, + readEvidenceBundle, + safeParseEvidenceBundle, +} from "./evidenceBundle.js"; +import { + EVIDENCE_FORMAT_REVISION, + RECREATE_EVIDENCE_GUIDANCE, +} from "./evidenceFormat.js"; + +const record = createEvidence( + undefined, + { id: "fixture", name: "Fixture", version: "1" }, + { operation: "observe", parameters: {}, result: { observed: true } }, +); + +it("writes the current format revision and reads it back", () => { + const bundle = createEvidenceBundle([record]); + expect(bundle.evidence_format).toBe(EVIDENCE_FORMAT_REVISION); + expect(safeParseEvidenceBundle(JSON.parse(JSON.stringify(bundle)))).toEqual({ + ok: true, + value: bundle, + }); +}); + +it.each([ + ["an unmarked (6.x) bundle", undefined], + ["another revision", EVIDENCE_FORMAT_REVISION + 1], + ["a malformed revision", "1"], +])("refuses %s once, with the recreate guidance", (_name, format) => { + const { evidence_format: _current, ...unmarked } = createEvidenceBundle([ + record, + ]); + const earlier = + format === undefined ? unmarked : { ...unmarked, evidence_format: format }; + const parsed = safeParseEvidenceBundle(earlier); + if (parsed.ok) throw new Error("Expected the earlier format to be refused"); + expect(parsed.error.kind).toBe("unsupported-format"); + expect(parsed.error.message).toContain(RECREATE_EVIDENCE_GUIDANCE); + const read = readEvidenceBundle(earlier, { + operation: "import_evidence_bundle", + path: ["evidence_bundle"], + }); + expect(read).toMatchObject({ + ok: false, + error: { + _tag: "EvidenceIntegrityError", + userMessage: expect.stringContaining(RECREATE_EVIDENCE_GUIDANCE), + }, + }); +}); + +it("still names a standalone Evidence record as the wrong input kind", () => { + const parsed = safeParseEvidenceBundle(record); + if (parsed.ok) throw new Error("Expected a record to be refused"); + expect(parsed.error.kind).toBe("single-record"); +}); diff --git a/src/domain/evidenceFormat.ts b/src/domain/evidenceFormat.ts new file mode 100644 index 0000000000..0a5d9e6e9d --- /dev/null +++ b/src/domain/evidenceFormat.ts @@ -0,0 +1,62 @@ +import { z } from "zod"; + +/** + * The Evidence format revision this REA release writes and reads. + * + * This module owns Evidence format evolution. A bundle declares its revision + * in `evidence_format`, and readers admit only the current one; anything + * older is refused with {@link RECREATE_EVIDENCE_GUIDANCE} rather than + * admitted one optional field at a time. A field added after a revision is + * optional only while its revision note below names it, and becomes required + * when the revision advances. + * + * Revision 1: the first marked format. Bundles without `evidence_format` + * (REA 6.x and earlier) are refused, and the 6.7-era artifact inventory + * summary identities are no longer rebuilt. + */ +export const EVIDENCE_FORMAT_REVISION = 1; + +/** The single recovery for Evidence from an earlier format. */ +export const RECREATE_EVIDENCE_GUIDANCE = + "Recreate observations with the current REA release; re-exporting an earlier bundle does not update its format or identities."; + +/** A bundle member naming the format revision it was written in. */ +export const evidenceFormatSchema = z.literal(EVIDENCE_FORMAT_REVISION, { + error: `evidence_format must be ${String(EVIDENCE_FORMAT_REVISION)}. ${RECREATE_EVIDENCE_GUIDANCE}`, +}); + +/** A bundle written in a format this release does not read. */ +export class UnsupportedEvidenceFormatError extends TypeError { + constructor( + /** The declared revision, or `undefined` for an unmarked bundle. */ + readonly found: unknown, + ) { + super( + `${ + found === undefined + ? "This Evidence bundle predates format revisions" + : `Evidence bundle format ${JSON.stringify(found)} is not format ${String(EVIDENCE_FORMAT_REVISION)}` + }, which this REA release reads. ${RECREATE_EVIDENCE_GUIDANCE}`, + ); + this.name = "UnsupportedEvidenceFormatError"; + } +} + +/** + * Refuse a bundle from another format before reading its shape, so an earlier + * bundle is reported as one format decision instead of per-field schema + * failures. Input without a `records` member is not a bundle and is left to + * the bundle schema (including a standalone Evidence record). + */ +export const admitEvidenceBundleFormat = (input: unknown): void => { + if ( + typeof input !== "object" || + input === null || + Array.isArray(input) || + !Object.hasOwn(input, "records") + ) + return; + const found: unknown = Reflect.get(input, "evidence_format"); + if (found !== EVIDENCE_FORMAT_REVISION) + throw new UnsupportedEvidenceFormatError(found); +}; diff --git a/src/domain/evidenceRecordSummary.test.ts b/src/domain/evidenceRecordSummary.test.ts index b0571be4b1..32eafcb22d 100644 --- a/src/domain/evidenceRecordSummary.test.ts +++ b/src/domain/evidenceRecordSummary.test.ts @@ -7,8 +7,14 @@ import { summarizeEvidenceBundle } from "./evidenceRecordSummary.js"; it("preserves native identity independently of missing or incompatible value-flow metadata", () => { const records = [ undefined, + { + available: true, + truncated: true, + omitted_operations_lower_bound: 1, + known_omitted_inputs: 0, + known_omitted_edges: 2, + }, { available: true, truncated: true }, - { unexpected: "legacy" }, { available: false, reason: "Decompiler unavailable" }, ].map((flow) => createEvidence( @@ -39,9 +45,9 @@ it("preserves native identity independently of missing or incompatible value-flo value_flow: { status: "observed", truncated: true, - omitted_operations_lower_bound: null, - known_omitted_inputs: null, - known_omitted_edges: null, + omitted_operations_lower_bound: 1, + known_omitted_inputs: 0, + known_omitted_edges: 2, }, }), expect.objectContaining({ @@ -62,7 +68,7 @@ it.each([ omitted_operations_lower_bound: 2, known_omitted_inputs: 3, known_omitted_edges: 4, - operations: "historical collection representation", + operations: "full collections are not parsed", }, expected: { status: "observed", @@ -74,7 +80,7 @@ it.each([ }, { flow: { available: false, reason: "" }, - expected: { status: "unavailable", reason: "" }, + expected: { status: "unknown", reason: expect.any(String) }, }, { flow: { available: true, truncated: false, known_omitted_edges: -1 }, diff --git a/src/domain/evidenceRecordSummary.ts b/src/domain/evidenceRecordSummary.ts index caa2438448..870d2bf49c 100644 --- a/src/domain/evidenceRecordSummary.ts +++ b/src/domain/evidenceRecordSummary.ts @@ -31,9 +31,9 @@ const nativeSummarySchema = z.discriminatedUnion("available", [ z.strictObject({ status: z.literal("observed"), truncated: z.boolean(), - omitted_operations_lower_bound: countSchema.nullable(), - known_omitted_inputs: countSchema.nullable(), - known_omitted_edges: countSchema.nullable(), + omitted_operations_lower_bound: countSchema, + known_omitted_inputs: countSchema, + known_omitted_edges: countSchema, }), z.strictObject({ status: z.literal("unavailable"), reason: z.string() }), z.strictObject({ status: z.literal("not-recorded") }), @@ -106,9 +106,9 @@ const nativeSummary = ( status: "observed", truncated: flow.data.truncated, omitted_operations_lower_bound: - flow.data.omitted_operations_lower_bound ?? null, - known_omitted_inputs: flow.data.known_omitted_inputs ?? null, - known_omitted_edges: flow.data.known_omitted_edges ?? null, + flow.data.omitted_operations_lower_bound, + known_omitted_inputs: flow.data.known_omitted_inputs, + known_omitted_edges: flow.data.known_omitted_edges, } : { status: "unavailable", reason: flow.data.reason }, }; diff --git a/src/domain/flutter/flutterBuildAnalysis.ts b/src/domain/flutter/flutterBuildAnalysis.ts index 487327bb1c..1859a0bcb1 100644 --- a/src/domain/flutter/flutterBuildAnalysis.ts +++ b/src/domain/flutter/flutterBuildAnalysis.ts @@ -233,11 +233,7 @@ export const flutterResultSchemas = { ), }), string_pool: z.strictObject({ - policy: dartPoolPolicySchema - .optional() - .describe( - "Effective selection and budgets; absent from older Evidence", - ), + policy: dartPoolPolicySchema.describe("Effective selection and budgets"), coverage: z .enum(["complete", "partial"]) .describe( diff --git a/src/domain/ghidraValues.fixture.ts b/src/domain/ghidraValues.fixture.ts index 904de4e462..01087a6498 100644 --- a/src/domain/ghidraValues.fixture.ts +++ b/src/domain/ghidraValues.fixture.ts @@ -192,6 +192,12 @@ export const ghidraFunctionDossier = (): JsonValue => { native_value_flow: { available: true, provenance: "ghidra-high-pcode", + limits: { + max_operations: 3_000, + max_inputs_per_operation: 64, + max_edges: 12_000, + }, + limits_reached: [], operations: [ { id: "0x401000#0", diff --git a/src/domain/gpu/gpuCodeObjects.test.ts b/src/domain/gpu/gpuCodeObjects.test.ts index 4ad9eca74d..e937aed35f 100644 --- a/src/domain/gpu/gpuCodeObjects.test.ts +++ b/src/domain/gpu/gpuCodeObjects.test.ts @@ -1,6 +1,6 @@ import { createHash } from "node:crypto"; import { expect, it } from "vitest"; -import { gpuPtxArtifactSchema } from "./gpuCodeObjects.js"; +import { gpuObjectSchema, gpuPtxArtifactSchema } from "./gpuCodeObjects.js"; it("binds retained PTX identity to its actual UTF-8 bytes and source extent", () => { const text = "\uFEFF// 雪🧪\n.version 8.8\n", size = Buffer.byteLength(text); @@ -32,3 +32,9 @@ it("binds retained PTX identity to its actual UTF-8 bytes and source extent", () gpuPtxArtifactSchema.safeParse({ ...artifact, ...patch }).success, ).toBe(false); }); + +it("requires the AMD target metadata fact that every inventory row records", () => { + const field = gpuObjectSchema.shape.amd_target_metadata; + expect(field.safeParse(null).success).toBe(true); + expect(field.safeParse(undefined).success).toBe(false); +}); diff --git a/src/domain/gpu/gpuCodeObjects.ts b/src/domain/gpu/gpuCodeObjects.ts index ae00de1ec3..cee0531682 100644 --- a/src/domain/gpu/gpuCodeObjects.ts +++ b/src/domain/gpu/gpuCodeObjects.ts @@ -53,8 +53,7 @@ export const gpuObjectSchema = z.strictObject({ }), ), }) - .nullable() - .default(null), + .nullable(), target_consistency: z.enum([ "unexamined", "single-source", diff --git a/src/domain/javascript/electronPageInspection.fixture.ts b/src/domain/javascript/electronPageInspection.fixture.ts index e5fe9d0dcb..3f96da7d08 100644 --- a/src/domain/javascript/electronPageInspection.fixture.ts +++ b/src/domain/javascript/electronPageInspection.fixture.ts @@ -26,7 +26,11 @@ export const emptyElectronPageCoverage = (observationMs = 0) => ({ }, }), frames: [], - dom: { total_nodes: 0, nodes: [] }, + dom: { + total_nodes: 0, + node_value_length_unit: "utf8-bytes" as const, + nodes: [], + }, scripts: { total: 0, items: [] }, resources: [], workers: [], diff --git a/src/domain/managed/managedNativeVerificationSource.test.ts b/src/domain/managed/managedNativeVerificationSource.test.ts index 003aedc97c..340dcbd9e7 100644 --- a/src/domain/managed/managedNativeVerificationSource.test.ts +++ b/src/domain/managed/managedNativeVerificationSource.test.ts @@ -6,19 +6,18 @@ import { inspectMachoSchema } from "../native/nativeInspection.js"; import { readManagedNativeVerificationSource } from "./managedNativeVerificationSource.js"; describe("managed/native verification source", () => { - it("accepts inspect_macho Evidence retained before encryption reporting", () => { + it("refuses inspect_macho Evidence without encryption reporting", () => { const original = MANAGED_NATIVE_VERIFICATION_EXAMPLE.native_observations[0]; if (original === undefined) throw new Error( "Managed verification example is missing native Evidence", ); - const { encryption: _omitted, ...legacy } = inspectMachoSchema.parse( - original.normalized_result, - ); + const { encryption: _omitted, ...withoutEncryption } = + inspectMachoSchema.parse(original.normalized_result); const evidence = createEvidence(undefined, original.provider, { operation: "inspect_macho", parameters: original.parameters, - result: legacy, + result: withoutEncryption, rawResult: null, limitations: [], }); @@ -27,12 +26,9 @@ describe("managed/native verification source", () => { MANAGED_NATIVE_VERIFICATION_EXAMPLE.managed_boundaries, native_observations: [evidence], }); - if (!source.ok) throw source.error; - const [observation] = source.value.nativeObservations; - expect(observation?.supported).toBe(true); - expect(observation?.platform).toBeNull(); - expect(observation?.symbols.map(({ name }) => name)).toEqual([ - "open_native", - ]); + expect(source).toMatchObject({ + ok: false, + error: { _tag: "EvidenceIntegrityError" }, + }); }); }); diff --git a/src/domain/managed/managedNativeVerificationSource.ts b/src/domain/managed/managedNativeVerificationSource.ts index a9c5414e9c..f8e15c3ce5 100644 --- a/src/domain/managed/managedNativeVerificationSource.ts +++ b/src/domain/managed/managedNativeVerificationSource.ts @@ -10,7 +10,7 @@ import { analysisInputErrorFromIssues } from "../inputIssueProjection.js"; import { readEvidence, readEvidencePayload } from "../evidenceValidation.js"; import { err, ok, type Result } from "../result.js"; import { functionDossierSchema } from "../native/functionDossier.js"; -import { retainedInspectMachoSchema } from "../native/nativeInspection.js"; +import { inspectMachoSchema } from "../native/nativeInspection.js"; import { managedNativeBoundaryInspectionSchema, type ManagedNativeBoundaryInspection, @@ -71,7 +71,7 @@ const readNativeObservation = ( : null; if (evidence.operation === "inspect_macho") { const payload = readEvidencePayload( - retainedInspectMachoSchema, + inspectMachoSchema, evidence.normalized_result, ); if (!payload.ok) return payload; diff --git a/src/domain/native/kernelModule.test.ts b/src/domain/native/kernelModule.test.ts index d0d0f46808..a2e4b52087 100644 --- a/src/domain/native/kernelModule.test.ts +++ b/src/domain/native/kernelModule.test.ts @@ -1,5 +1,8 @@ import { expect, it } from "vitest"; -import { inspectKernelModuleInputSchema } from "./kernelModule.js"; +import { + KERNEL_MODULE_MODINFO_ENTRY_REASONS, + inspectKernelModuleInputSchema, +} from "./kernelModule.js"; it("rejects extra caller arguments and applies kernel-module budgets", () => { expect( @@ -15,3 +18,9 @@ it("rejects extra caller arguments and applies kernel-module budgets", () => { max_entries: 8192, }); }); + +it("keeps modinfo field omission separate from entry validity", () => { + expect(KERNEL_MODULE_MODINFO_ENTRY_REASONS).not.toContain( + "modinfo_field_exceeds_limit", + ); +}); diff --git a/src/domain/native/kernelModule.ts b/src/domain/native/kernelModule.ts index 9d2c40e752..5b4e8a216e 100644 --- a/src/domain/native/kernelModule.ts +++ b/src/domain/native/kernelModule.ts @@ -51,8 +51,6 @@ export const KERNEL_MODULE_MODINFO_ENTRY_REASONS = [ "unterminated_modinfo_entry", "modinfo_missing_separator", "modinfo_empty_key", - // Accepted when decoding older Evidence; new observations separate omission from validity. - "modinfo_field_exceeds_limit", ] as const; export const KERNEL_MODULE_MODINFO_SECTION_REASONS = [ diff --git a/src/domain/native/nativeExports.ts b/src/domain/native/nativeExports.ts index 715d5b15f8..70d8988c03 100644 --- a/src/domain/native/nativeExports.ts +++ b/src/domain/native/nativeExports.ts @@ -57,33 +57,32 @@ const peExportSchema = z.strictObject({ /** * One defined, default- or protected-visibility global, weak or unique - * dynamic symbol. `visibility` is "unknown" in historical Evidence where - * dynamic symbol visibility was not recorded. `version` is null when the + * dynamic symbol. `version` is null when the * symbol carries no version and "unknown" when the version table could not be * read; a hidden version is not the default one a by-name lookup selects. */ const elfExportSchema = z.strictObject({ name: z.string().min(1), - /** Exact non-ASCII name bytes; ELF string tables do not declare text encoding. */ - name_bytes_base64: canonicalBase64Schema.nullable().default(null), - /** Whether `name` was decoded as UTF-8; null means this field was not recorded in older Evidence. */ - name_utf8_decodable: z.boolean().nullable().default(null), + /** Exact name bytes when any is non-ASCII; null means `name` is ASCII. ELF string tables do not declare text encoding. */ + name_bytes_base64: canonicalBase64Schema.nullable(), + /** Whether the name bytes are valid UTF-8; otherwise `name` is their Latin-1 display. */ + name_utf8_decodable: z.boolean(), /** `st_value` as stored; on 32-bit ARM, bit 0 of a function marks Thumb code. */ value: address, size: z.string().regex(/^[0-9]+$/u), type: z.enum(["function", "ifunc", "object", "tls", "other"]), binding: z.enum(["global", "weak", "unique"]), - visibility: z.enum(["default", "protected", "unknown"]).default("unknown"), + visibility: z.enum(["default", "protected"]), version: z.union([ z.null(), z.literal("unknown"), z.strictObject({ index: z.number().int().min(2).max(0x7fff), name: z.string().min(1).nullable(), - /** Exact non-ASCII version-name bytes; ELF strings do not declare encoding. */ - name_bytes_base64: canonicalBase64Schema.nullable().default(null), - /** Null means older Evidence did not retain version-name encoding facts. */ - name_utf8_decodable: z.boolean().nullable().default(null), + /** Exact version-name bytes when any is non-ASCII; ELF strings do not declare encoding. */ + name_bytes_base64: canonicalBase64Schema.nullable(), + /** Whether the version-name bytes are valid UTF-8; null exactly when `name` is unresolved. */ + name_utf8_decodable: z.boolean().nullable(), hidden: z.boolean(), }), ]), @@ -168,15 +167,25 @@ export const nativeExportsSchema = z }, path: (string | number)[], ) => { - // Historical omissions remain unknown. Complete encoding facts must agree - // with the producer's UTF-8-or-Latin-1 display interpretation. - if (row.name_bytes_base64 === null || row.name_utf8_decodable === null) - return; - const bytes = Buffer.from(row.name_bytes_base64, "base64"); - const decoded = decodeElfExportName(bytes); + // Encoding facts must agree with the producer's UTF-8-or-Latin-1 display + // interpretation; an unresolved version name has none. + const expected = + row.name === null + ? { name: null, name_bytes_base64: null, name_utf8_decodable: null } + : row.name_bytes_base64 === null + ? /^[\x00-\x7f]*$/u.test(row.name) + ? { + name: row.name, + name_bytes_base64: null, + name_utf8_decodable: true, + } + : undefined + : decodeElfExportName(Buffer.from(row.name_bytes_base64, "base64")); if ( - decoded.name_utf8_decodable !== row.name_utf8_decodable || - decoded.name !== row.name + expected === undefined || + expected.name !== row.name || + expected.name_bytes_base64 !== row.name_bytes_base64 || + expected.name_utf8_decodable !== row.name_utf8_decodable ) context.addIssue({ code: "custom", diff --git a/src/domain/native/nativeInspection.ts b/src/domain/native/nativeInspection.ts index 5599819802..cf12a3f3ec 100644 --- a/src/domain/native/nativeInspection.ts +++ b/src/domain/native/nativeInspection.ts @@ -2,10 +2,7 @@ import { z } from "zod"; import { bundleResourceRootSchema } from "../apple/bundleResourceRoot.js"; import { jsonRecordSchema } from "../jsonRecord.js"; import { jsonValueSchema } from "../jsonValue.js"; -import { - machoSliceEncryptionSchema, - retainedMachoSliceEncryptionSchema, -} from "../apple/machoEncryption.js"; +import { machoSliceEncryptionSchema } from "../apple/machoEncryption.js"; const hexAddress = z.string().regex(/^0x[a-fA-F0-9]+$/u); const bytesHex = z.string().regex(/^(?:[0-9a-f]{2})*$/u); @@ -55,13 +52,13 @@ const sectionSchema = z.object({ segment_bytes_hex: bytesHex .optional() .describe( - "Original segment-name bytes before the first NUL; omitted in older Evidence.", + "Original segment-name bytes before the first NUL, present when the name is not valid UTF-8 and `segment` is null.", ), name: z.string().nullable(), name_bytes_hex: bytesHex .optional() .describe( - "Original section-name bytes before the first NUL; omitted in older Evidence.", + "Original section-name bytes before the first NUL, present when the name is not valid UTF-8 and `name` is null.", ), address: hexAddress.nullable(), size: z.int().min(0).nullable(), @@ -90,7 +87,7 @@ const segmentSchema = z.object({ name_bytes_hex: bytesHex .optional() .describe( - "Original segment-name bytes before the first NUL; omitted in older Evidence.", + "Original segment-name bytes before the first NUL, present when the name is not valid UTF-8 and `name` is null.", ), vm_address: hexAddress.nullable(), vm_size: z.int().min(0).nullable(), @@ -123,9 +120,8 @@ const segmentSchema = z.object({ const runtimeNameSchema = z .string() .nullable() - .default(null) .describe( - "Symbol-table `name` without the one leading underscore Darwin adds to language-level symbols (`_main` is `main`, `__ZN3rea3addEii` is `_ZN3rea3addEii`), as LLDB and dlsym name it and observe_native_calls function breakpoints accept; a name without that underscore, such as `-[Class selector]`, is unchanged. Null when `name` is null, nothing remains, or the Evidence predates this field.", + "Symbol-table `name` without the one leading underscore Darwin adds to language-level symbols (`_main` is `main`, `__ZN3rea3addEii` is `_ZN3rea3addEii`), as LLDB and dlsym name it and observe_native_calls function breakpoints accept; a name without that underscore, such as `-[Class selector]`, is unchanged. Null when `name` is null or nothing remains.", ); const symbolSchema = z.object({ /** Exact Mach-O symbol-table name, the source fact. */ @@ -141,10 +137,10 @@ const importSymbolSchema = z.object({ name: z.string().nullable(), runtime_name: runtimeNameSchema, /** Exact import text retained when a chained suffix makes identity ambiguous. */ - raw_name: z.string().nullable().default(null), - name_status: z.enum(["exact", "ambiguous"]).nullable().default(null), + raw_name: z.string().nullable(), + name_status: z.enum(["exact", "ambiguous"]), /** Addend is unknown when only the ambiguous text expression is available. */ - addend: hexAddress.nullable().default(null), + addend: hexAddress.nullable(), address: hexAddress.nullable(), weak: z.boolean().nullable(), reexport: z.boolean().nullable(), @@ -155,7 +151,7 @@ const dependencySchema = z.object({ path_bytes_hex: bytesHex .optional() .describe( - "Original load-command path bytes before the first NUL; omitted in older Evidence.", + "Original load-command path bytes before the first NUL, present when the path is not valid UTF-8 and `path` is null.", ), kind: z.string(), current_version: z.string().nullable(), @@ -220,27 +216,6 @@ export const inspectMachoSchema = z.object({ limitations: z.array(z.string()), }); -/** - * Retained inspect_macho Evidence, including results recorded before - * `encryption` was reported. Not recorded is unknown, never evidence that no - * slice is encrypted. - */ -export const retainedInspectMachoSchema = inspectMachoSchema - .extend({ - encryption: coverage(retainedMachoSliceEncryptionSchema).optional(), - }) - .transform(({ encryption, ...report }) => ({ - ...report, - encryption: encryption ?? { - items: [], - total: null, - exhaustive: false, - limitations: [ - "This inspect_macho Evidence predates encryption reporting; slice encryption status is unknown.", - ], - }, - })); - export const inspectSignatureSchema = z.object({ signed: z.boolean(), identifier: z.string().nullable(), diff --git a/src/domain/native/nativeInstruction.test.ts b/src/domain/native/nativeInstruction.test.ts new file mode 100644 index 0000000000..2f77ce0217 --- /dev/null +++ b/src/domain/native/nativeInstruction.test.ts @@ -0,0 +1,9 @@ +import { expect, it } from "vitest"; + +import { nativeInstructionSchema } from "./nativeInstruction.js"; + +it("admits only instruction statuses a provider reports", () => { + const status = nativeInstructionSchema.shape.status; + expect(status.safeParse("not-in-listing").success).toBe(true); + expect(status.safeParse("undecodable").success).toBe(false); +}); diff --git a/src/domain/native/nativeInstruction.ts b/src/domain/native/nativeInstruction.ts index c7f7b73d49..8b09faa0a5 100644 --- a/src/domain/native/nativeInstruction.ts +++ b/src/domain/native/nativeInstruction.ts @@ -50,11 +50,10 @@ export const nativeInstructionSchema = z.strictObject({ "not-instruction-boundary", "data", "not-in-listing", - "undecodable", "outside-memory", ]) .describe( - "decoded is an instruction at the address; not-instruction-boundary is inside a listed instruction; data is defined data; outside-memory is outside the program's memory. Ghidra's not-in-listing means the address is in memory with no listed instruction or defined data; no decode was attempted, and memory may be uninitialized. The legacy undecodable value in older Evidence has the same ambiguous meaning and does not prove decoding failed.", + "decoded is an instruction at the address; not-instruction-boundary is inside a listed instruction; data is defined data; outside-memory is outside the program's memory. Ghidra's not-in-listing means the address is in memory with no listed instruction or defined data; no decode was attempted, and memory may be uninitialized.", ), procedure: z.string().nullable(), containing_instruction: z @@ -67,7 +66,7 @@ export const nativeInstructionSchema = z.strictObject({ }) .optional() .describe( - "Provider-observed listed instruction containing the exact requested interior address. Its address and length describe the containing instruction; the query address remains unchanged. Omission means not recorded or not applicable, including older Evidence. This does not imply a function, alternate decode, reachability, or runtime execution.", + "Provider-observed listed instruction containing the exact requested interior address. Its address and length describe the containing instruction; the query address remains unchanged. Present when status is not-instruction-boundary. This does not imply a function, alternate decode, reachability, or runtime execution.", ), containing_data: z .strictObject({ @@ -83,7 +82,7 @@ export const nativeInstructionSchema = z.strictObject({ }) .optional() .describe( - "Provider-observed containing defined data object, including its start and type identity; the selected address remains unchanged. Inspect its start with inspect_native_data_type for its layout. This does not choose a field or union member or prove runtime use. Omission means not recorded or not applicable, including older Evidence.", + "Provider-observed containing defined data object, including its start and type identity; the selected address remains unchanged. Inspect its start with inspect_native_data_type for its layout. This does not choose a field or union member or prove runtime use. Present when status is data.", ), architecture: z.string(), mode: z @@ -101,7 +100,7 @@ export const nativeInstructionSchema = z.strictObject({ ) .optional() .describe( - "Observed per-address decoder context. Null means no value was available; omission means the provider or older capture does not expose this facet. The separate mode field retains the program language variant.", + "Observed per-address decoder context. Null means no value was available; omission means the provider does not expose this facet for the program architecture. The separate mode field retains the program language variant.", ), bytes: z.string().nullable(), length: z.int().positive().nullable(), @@ -144,7 +143,7 @@ export const nativeInstructionSchema = z.strictObject({ ) .optional() .describe( - "Observed provider facts used to classify flow; older captures or other providers may omit them.", + "Observed provider facts used to classify flow; a provider that does not expose them omits the field.", ), }), references: z.array(reference), diff --git a/src/domain/native/nativeInvestigationGraph.test.ts b/src/domain/native/nativeInvestigationGraph.test.ts index 14099becc9..f51a022b1a 100644 --- a/src/domain/native/nativeInvestigationGraph.test.ts +++ b/src/domain/native/nativeInvestigationGraph.test.ts @@ -331,20 +331,11 @@ describe("Interface Builder dispatch joins", () => { describe("Objective-C category ownership joins", () => { it.each([ - { - className: "BuildViewController", - category: "BuilderActions", - legacy: false, - }, - { - className: "BuildViewController(Literal)", - category: undefined, - legacy: false, - }, - { className: "BuildViewController", category: undefined, legacy: true }, + { className: "BuildViewController", category: "BuilderActions" }, + { className: "BuildViewController(Literal)", category: undefined }, ])( "matches exact instance owner $className with category $category", - ({ className, category, legacy }) => { + ({ className, category }) => { const ui = buildInterfaceBuilderAnalysis({ targetSha256: "b".repeat(64), toolVersion: "test", @@ -390,27 +381,17 @@ describe("Objective-C category ownership joins", () => { evidence, }, { - class_name: legacy ? `${className}(BuilderActions)` : className, + class_name: className, ...(category === undefined ? {} : { category }), selector: "buildTapped:", method_type: "instance" as const, implementation_address: "0x1000", location: { address: "0x1000", file_offset: 0 }, - decode: legacy - ? { status: "partial" as const, reason: "symbol_name_only" } - : { status: "decoded" as const, reason: null }, - evidence: legacy - ? [ - { - ...evidence[0]!, - kind: "symbol" as const, - description: `Provider symbol -[${className}(BuilderActions) buildTapped:]`, - }, - ] - : evidence, + decode: { status: "decoded" as const, reason: null }, + evidence, }, { - // A historical or binary class identity does not declare a category. + // A binary class identity does not declare a category. class_name: `${className}(Legacy)`, selector: "buildTapped:", method_type: "instance" as const, @@ -432,7 +413,7 @@ describe("Objective-C category ownership joins", () => { expect(joined.edges).toContainEqual( expect.objectContaining({ relation: "objc_dispatch", - resolution: legacy ? "inferred" : "resolved", + resolution: "resolved", to: "native:function:0x1000", }), ); diff --git a/src/domain/native/nativeInvestigationGraph.ts b/src/domain/native/nativeInvestigationGraph.ts index a1654d5ab6..5991c37f06 100644 --- a/src/domain/native/nativeInvestigationGraph.ts +++ b/src/domain/native/nativeInvestigationGraph.ts @@ -1,7 +1,6 @@ import { z } from "zod"; import { - objcImplementationClassName, nativeMetadataEvidenceSchema, nativeMetadataLocationSchema, type ObjcSwiftMetadata, @@ -367,8 +366,7 @@ export const joinInterfaceBuilderDispatch = ( (candidate) => candidate.method_type === "instance" && candidate.selector === selector && - (className === null || - objcImplementationClassName(candidate) === className), + (className === null || candidate.class_name === className), ); const implementation = candidates.length === 1 ? candidates[0] : undefined; const edgeId = `dispatch:${selectorNodeId}:${className ?? owner?.id ?? "unknown"}:${selector}`; diff --git a/src/domain/native/nativeLoadImage.ts b/src/domain/native/nativeLoadImage.ts index 12857e89f6..b723089756 100644 --- a/src/domain/native/nativeLoadImage.ts +++ b/src/domain/native/nativeLoadImage.ts @@ -33,10 +33,9 @@ export const nativeLoadImageObservationSchema = z.strictObject({ initialized: z.boolean(), loaded: z.boolean(), overlay: z.boolean(), - // Optional so inspection snapshots saved before permissions were measured still parse. - readable: z.boolean().optional(), - writable: z.boolean().optional(), - executable: z.boolean().optional(), + readable: z.boolean(), + writable: z.boolean(), + executable: z.boolean(), length: integer.min(1), source_file_index: integer.nullable(), file_offset: integer.nullable(), @@ -50,8 +49,8 @@ export const nativeLoadImageObservationSchema = z.strictObject({ offset: integer.max(0xffff).nullable(), status: z.string().min(1), type: z.int(), - // Missing on retained observations made before the provider recorded it. - symbol: z.string().nullable().exactOptional(), + /** Ghidra's relocation symbol name; null when the relocation has none. */ + symbol: z.string().nullable(), values: z.array( z.int().min(Number.MIN_SAFE_INTEGER).max(Number.MAX_SAFE_INTEGER), ), diff --git a/src/domain/native/nativeValueFlow.ts b/src/domain/native/nativeValueFlow.ts index ff6cd88b56..1f8488b40d 100644 --- a/src/domain/native/nativeValueFlow.ts +++ b/src/domain/native/nativeValueFlow.ts @@ -46,7 +46,7 @@ export const nativePcodeEffectSchema = z.strictObject({ value_input: z.int().nonnegative().nullable(), }); -/** Accepted projection budgets; historical observations may omit these facts. */ +/** Accepted projection budgets. */ export const nativeValueFlowLimitsSchema = z.strictObject({ max_operations: z.int().min(1).max(MAXIMUM_GHIDRA_VALUE_FLOW_RECORDS), max_inputs_per_operation: z @@ -61,10 +61,10 @@ export type NativeValueFlowLimits = z.infer; export const availableNativeValueFlowSchema = z.strictObject({ available: z.literal(true), provenance: z.literal("ghidra-high-pcode"), - limits: nativeValueFlowLimitsSchema.exactOptional(), - limits_reached: z - .array(z.enum(["max_operations", "max_inputs_per_operation", "max_edges"])) - .exactOptional(), + limits: nativeValueFlowLimitsSchema, + limits_reached: z.array( + z.enum(["max_operations", "max_inputs_per_operation", "max_edges"]), + ), operations: z.array(nativePcodeOperationSchema), def_use: z.array(nativePcodeDefUseSchema), effects: z @@ -102,10 +102,7 @@ export const nativeValueFlowSchema = z.discriminatedUnion("available", [ unavailableNativeValueFlowSchema, ]); -/** Lightweight retained metadata; historical records may omit newer omission counts. - * Strip full flow collections without parsing or copying them. An empty recorded - * unavailable reason remains source evidence even though new producers forbid it. - */ +/** Lightweight retained metadata: strip full flow collections without parsing or copying them. */ export const nativeValueFlowMetadataSchema = z.union([ availableNativeValueFlowSchema .pick({ @@ -115,14 +112,8 @@ export const nativeValueFlowMetadataSchema = z.union([ known_omitted_inputs: true, known_omitted_edges: true, }) - .partial({ - omitted_operations_lower_bound: true, - known_omitted_inputs: true, - known_omitted_edges: true, - }) .strip(), unavailableNativeValueFlowSchema .pick({ available: true, reason: true }) - .extend({ reason: z.string() }) .strip(), ]); diff --git a/src/domain/native/objcSwiftMetadata.test.ts b/src/domain/native/objcSwiftMetadata.test.ts index cf5c8d8067..be84abaa54 100644 --- a/src/domain/native/objcSwiftMetadata.test.ts +++ b/src/domain/native/objcSwiftMetadata.test.ts @@ -2,12 +2,12 @@ import { describe, expect, it } from "vitest"; import { inspectNativeDispatchMetadata, - objcImplementationClassName, + swiftConformanceSchema, } from "./objcSwiftMetadata.js"; describe("symbol-only Objective-C implementation identity", () => { it.each([" ", " ", "\t"])( - "interprets legacy symbol ownership with separator %j without rewriting retained facts", + "separates category ownership with separator %j", (separator) => { const metadata = inspectNativeDispatchMetadata( [ @@ -18,37 +18,13 @@ describe("symbol-only Objective-C implementation identity", () => { ], 1, ); - const current = metadata.objc_dispatch_implementations[0]!; - const { category: _category, ...record } = current; - const legacy = { ...record, class_name: "StoreController(Actions)" }; - expect(objcImplementationClassName(legacy)).toBe("StoreController"); - expect(legacy.class_name).toBe("StoreController(Actions)"); - expect(legacy.evidence).toBe(current.evidence); - expect(objcImplementationClassName({ ...legacy, evidence: [] })).toBe( - legacy.class_name, - ); - expect( - objcImplementationClassName({ - ...legacy, - evidence: [ - ...legacy.evidence, - { - ...legacy.evidence[0]!, - description: - "Provider symbol -[OtherController(Actions) buildTapped:]", - }, - ], - }), - ).toBe(legacy.class_name); - expect(objcImplementationClassName({ ...legacy, category: null })).toBe( - legacy.class_name, - ); - expect( - objcImplementationClassName({ - ...legacy, - decode: { status: "decoded", reason: null }, - }), - ).toBe(legacy.class_name); + expect(metadata.objc_dispatch_implementations).toMatchObject([ + { + class_name: "StoreController", + category: "Actions", + selector: "buildTapped:", + }, + ]); }, ); @@ -370,3 +346,9 @@ it("derives classes only from complete class symbol leaves", () => { }), ); }); + +it("requires the Swift ABI type-reference kind on every conformance", () => { + const field = swiftConformanceSchema.shape.type_reference_kind; + expect(field.safeParse(0).success).toBe(true); + expect(field.safeParse(undefined).success).toBe(false); +}); diff --git a/src/domain/native/objcSwiftMetadata.ts b/src/domain/native/objcSwiftMetadata.ts index 75e1a15c5b..015e767c8e 100644 --- a/src/domain/native/objcSwiftMetadata.ts +++ b/src/domain/native/objcSwiftMetadata.ts @@ -213,7 +213,7 @@ const objcMethodSymbolOwner = (owner: string) => { }; }; -/** The accepted provider spelling, shared by new and retained symbol records. */ +/** The accepted provider method-symbol spelling. */ const parseObjcMethodSymbol = (symbol: string) => { const match = /^([+-])\[([^\s\]]+)\s+([^\]]+)\]$/u.exec(symbol); if (match?.[1] === undefined || !match[2] || !match[3]) return undefined; @@ -225,39 +225,6 @@ const parseObjcMethodSymbol = (symbol: string) => { }; }; -/** Retained symbol-only records predate separate category ownership. */ -export const objcImplementationClassName = ( - implementation: ObjcDispatchImplementation, -): string => { - if ( - implementation.category !== undefined || - implementation.decode.reason !== "symbol_name_only" - ) - return implementation.class_name; - const symbols = implementation.evidence.filter( - ({ kind }) => kind === "symbol", - ); - // Conflicting or absent source facts cannot establish category qualification. - if ( - symbols.length === 0 || - symbols.some((item) => { - const prefix = "Provider symbol "; - if (!item.description.startsWith(prefix)) return true; - const parsed = parseObjcMethodSymbol( - item.description.slice(prefix.length), - ); - return ( - parsed === undefined || - parsed.reported_owner !== implementation.class_name || - parsed.selector !== implementation.selector || - parsed.method_type !== implementation.method_type - ); - }) - ) - return implementation.class_name; - return objcMethodSymbolOwner(implementation.class_name).class_name; -}; - /** Project class declarations without selecting an ambiguous or unrepresentable address. */ export const projectObjcClassMethods = ( implementations: readonly ObjcDispatchImplementation[], @@ -315,8 +282,8 @@ export type MachoImportBinding = z.infer; /** Swift protocol conformance and its associated witness-table location. */ export const swiftConformanceSchema = z.strictObject({ type_name: z.string().min(1).nullable(), - /** Raw three-bit Swift ABI type-reference kind; absent in older Evidence. */ - type_reference_kind: z.number().int().min(0).max(7).optional(), + /** Raw three-bit Swift ABI type-reference kind. */ + type_reference_kind: z.number().int().min(0).max(7), /** Address of the descriptor, class-name string, or resolved ObjC class reference. */ type_reference_target: nativeMetadataLocationSchema.optional(), type_binding: machoImportBindingSchema.nullable().optional(), @@ -587,8 +554,8 @@ export const inspectNativeDispatchMetadata = ( category, } = method; // The provider's method-symbol grammar owns category qualification. Binary - // class names and historical records are identities, not display strings - // for downstream consumers to split. Keep the original symbol in Evidence. + // class names are identities, not display strings for downstream + // consumers to split. Keep the original symbol in Evidence. const key = parsedAddress.success ? compositeKey([ className, diff --git a/src/domain/pcap/pcapAnalysis.ts b/src/domain/pcap/pcapAnalysis.ts index 6359781777..6861049120 100644 --- a/src/domain/pcap/pcapAnalysis.ts +++ b/src/domain/pcap/pcapAnalysis.ts @@ -61,20 +61,6 @@ const client = z.strictObject({ capinfos_identity_status: z.enum(["verified", "unresolved"]), }); -/** Defaults for historical report Evidence written before executable identity was recorded. */ -const legacyClientIdentity = client.extend({ - tshark_resolved_path: localPathStringSchema.nullable().default(null), - tshark_sha256: digestSchema.nullable().default(null), - tshark_identity_status: z - .enum(["verified", "unresolved"]) - .default("unresolved"), - capinfos_resolved_path: localPathStringSchema.nullable().default(null), - capinfos_sha256: digestSchema.nullable().default(null), - capinfos_identity_status: z - .enum(["verified", "unresolved"]) - .default("unresolved"), -}); - const endpoint = z.strictObject({ address: z.string().min(1), port: z.int().positive().nullable(), @@ -86,9 +72,8 @@ const dnsQuery = z.strictObject({ query_labels_hex: z .array(z.string().regex(/^(?:[0-9a-f]{2}){1,63}$/u)) .nullable() - .optional() .describe( - "Original DNS wire label bytes in order, as lowercase hex strings; the root terminator is excluded. Null or omission means unknown wire identity; display text alone does not establish identity.", + "Original DNS wire label bytes in order, as lowercase hex strings; the root terminator is excluded. Null means unknown wire identity; display text alone does not establish identity.", ), response_addresses: z.array(z.string().min(1)), response_address_count: z.int().nonnegative(), @@ -144,10 +129,3 @@ export const pcapResultSchemas = { .optional(), }), } as const; - -/** Read compatibility for retained capture Evidence from earlier REA versions. */ -export const pcapEvidenceSchemas = { - inspect_pcap_capture: pcapResultSchemas.inspect_pcap_capture.extend({ - client: legacyClientIdentity, - }), -} as const; diff --git a/src/domain/sqlite/sqliteDatabase.ts b/src/domain/sqlite/sqliteDatabase.ts index c776d0471b..904e2e4488 100644 --- a/src/domain/sqlite/sqliteDatabase.ts +++ b/src/domain/sqlite/sqliteDatabase.ts @@ -150,11 +150,9 @@ export const sqliteDatabaseSchema = z name: z.literal("SQLite"), version: z.string().min(1), }), - worker_timeout_ms: timerDurationMsSchema - .exactOptional() - .describe( - "Actual parent-supervised worker deadline in milliseconds; absent in historical Evidence means unknown, not the default.", - ), + worker_timeout_ms: timerDurationMsSchema.describe( + "Actual parent-supervised worker deadline in milliseconds.", + ), database_encoding: encoding, schema: z.strictObject({ scope: z.enum(["inventory", "selected_table", "database"]), diff --git a/src/domain/webCaptureDiff.ts b/src/domain/webCaptureDiff.ts index 61a93e97d6..e0bb4c087a 100644 --- a/src/domain/webCaptureDiff.ts +++ b/src/domain/webCaptureDiff.ts @@ -1082,8 +1082,7 @@ const domValueLengthsComparable = ( after: WebPageInspection["dom"], ): boolean => (before.nodes.length === 0 && after.nodes.length === 0) || - (before.node_value_length_unit !== undefined && - before.node_value_length_unit === after.node_value_length_unit); + before.node_value_length_unit === after.node_value_length_unit; /** Compare parsed node facts without serializing or copying the inventories. */ const domNodesDiffer = ( diff --git a/src/ghidra/GhidraDolLoadImage.test.ts b/src/ghidra/GhidraDolLoadImage.test.ts index 8d074cfa78..ec1de9ecba 100644 --- a/src/ghidra/GhidraDolLoadImage.test.ts +++ b/src/ghidra/GhidraDolLoadImage.test.ts @@ -384,23 +384,6 @@ describe("GameCube DOL block permissions and state", () => { }); }); - it("does not verify a DOL block whose permissions were never measured", async () => { - const image = await attestMutated((mappings) => { - const text = mappings.find((m) => m.block === ".text0"); - if (text === undefined) throw new Error("Missing text mapping"); - delete text.readable; - delete text.writable; - delete text.executable; - }); - expect(image.status).toBe("mismatch"); - expect( - checkNamed(image.checks, "block_permissions_measured"), - ).toMatchObject({ matched: false, observed: [".text0"] }); - expect(blockCheck(image.checks, ".text0", TEXT_ADDRESS)).toMatchObject({ - matched: false, - }); - }); - it("flags a text block that is not readable", async () => { const image = await attestMutated((mappings) => { const text = mappings.find((m) => m.block === ".text0"); diff --git a/src/ghidra/GhidraDolLoadImage.ts b/src/ghidra/GhidraDolLoadImage.ts index e993b41ec7..ed8813c9c6 100644 --- a/src/ghidra/GhidraDolLoadImage.ts +++ b/src/ghidra/GhidraDolLoadImage.ts @@ -49,9 +49,9 @@ type FileFacts = { readonly loaded: boolean; readonly overlay: boolean; readonly address_space: string; - readonly readable?: boolean | undefined; - readonly writable?: boolean | undefined; - readonly executable?: boolean | undefined; + readonly readable: boolean; + readonly writable: boolean; + readonly executable: boolean; }; type BssFacts = Omit; const hex = (value: number): string => `0x${value.toString(16)}`; @@ -102,9 +102,9 @@ export const attestGhidraDolLoadImage = ( loaded: mapping.loaded, overlay: mapping.overlay, address_space: mapping.address_space, - readable: mapping.readable ?? null, - writable: mapping.writable ?? null, - executable: mapping.executable ?? null, + readable: mapping.readable, + writable: mapping.writable, + executable: mapping.executable, }); const bssFacts = (mapping: BssFacts) => ({ start: mapping.start, @@ -114,9 +114,9 @@ export const attestGhidraDolLoadImage = ( loaded: mapping.loaded, overlay: mapping.overlay, address_space: mapping.address_space, - readable: mapping.readable ?? null, - writable: mapping.writable ?? null, - executable: mapping.executable ?? null, + readable: mapping.readable, + writable: mapping.writable, + executable: mapping.executable, }); const expectedFileMappings = header.sections @@ -171,18 +171,6 @@ export const attestGhidraDolLoadImage = ( const observedBss = observedBssBlocks .map((mapping) => bssFacts(mapping)) .sort(byStart); - // Permissions are attested only when measured; an older observation without them fails closed. - const unmeasured = observations.mappings - .filter( - (mapping) => - (mapping.source_file_index !== null || - observedBssBlocks.includes(mapping)) && - (mapping.readable === undefined || - mapping.writable === undefined || - mapping.executable === undefined), - ) - .map((mapping) => mapping.block); - const source = observations.source_files[0]; check("executable_format", "Raw Binary", observations.executable_format); check("language_id", language.languageId, observations.language_id); @@ -236,8 +224,6 @@ export const attestGhidraDolLoadImage = ( (entry) => entry === hex(header.entryPoint), ) ?? null, ); - if (unmeasured.length > 0) - check("block_permissions_measured", [], unmeasured); check("relocation_records", [], observations.relocations); return ok( nativeLoadImageSchema.parse({ diff --git a/src/ghidra/GhidraDosComLoadImage.test.ts b/src/ghidra/GhidraDosComLoadImage.test.ts index 0885a1f9c3..d241227850 100644 --- a/src/ghidra/GhidraDosComLoadImage.test.ts +++ b/src/ghidra/GhidraDosComLoadImage.test.ts @@ -80,6 +80,7 @@ const mutations: ReadonlyArray< offset: 0x100, status: "APPLIED", type: 0, + symbol: null, values: [], original_bytes_hex: null, memory_bytes_hex: null, @@ -124,27 +125,39 @@ describe("COM independent load-image checks", () => { value: { status: "mismatch", observations: f.observation }, }); }); - it("accepts an older observation without measured permissions", () => { + it("refuses an observation without measured permissions or relocation symbols", () => { const f = fixtureDosComLoadImage(); - for (const mapping of f.observation.mappings) { - delete mapping.readable; - delete mapping.writable; - delete mapping.executable; - } - expect( - nativeLoadImageObservationSchema.safeParse(f.observation).success, - ).toBe(true); - const result = attestGhidraDosComLoadImage( - f.bytes, - f.sha256, - f.observation, - ); - expect(result).toMatchObject({ - ok: true, - value: { status: "verified", format: "dos-com" }, + const { readable: _r, ...unmeasured } = f.observation.mappings[0]!; + const withoutPermissions = nativeLoadImageObservationSchema.safeParse({ + ...f.observation, + mappings: [unmeasured, ...f.observation.mappings.slice(1)], }); - if (!result.ok) throw new Error(result.error); - expect(nativeLoadImageSchema.safeParse(result.value).success).toBe(true); + expect(withoutPermissions.success).toBe(false); + expect(withoutPermissions.error?.issues[0]?.path).toEqual([ + "mappings", + 0, + "readable", + ]); + const withoutSymbol = nativeLoadImageObservationSchema.safeParse({ + ...f.observation, + relocations: [ + { + address: "0x10100", + segment: 0x1000, + offset: 0x100, + status: "APPLIED", + type: 0, + values: [], + original_bytes_hex: null, + memory_bytes_hex: null, + }, + ], + }); + expect(withoutSymbol.error?.issues[0]?.path).toEqual([ + "relocations", + 0, + "symbol", + ]); }); it("rejects stale snapshot identity", () => { diff --git a/src/ghidra/GhidraLoadImage.fixture.ts b/src/ghidra/GhidraLoadImage.fixture.ts index 1d665473b4..cf07e61847 100644 --- a/src/ghidra/GhidraLoadImage.fixture.ts +++ b/src/ghidra/GhidraLoadImage.fixture.ts @@ -142,6 +142,7 @@ export const fixtureDosLoadImage = ( offset: 9, status: "APPLIED", type: 0, + symbol: null, values: [0, 9, 0x1004], original_bytes_hex: "0400", memory_bytes_hex: "0410", @@ -200,6 +201,7 @@ export const fixtureDosOverlapLoadImage = () => { offset: 9, status: "APPLIED", type: 0, + symbol: null, values: [0, 9, 0x1004], original_bytes_hex: "0400", memory_bytes_hex: "1410", @@ -210,6 +212,7 @@ export const fixtureDosOverlapLoadImage = () => { offset: 8, status: "APPLIED", type: 0, + symbol: null, values: [0, 8, 0x1400], original_bytes_hex: "0004", memory_bytes_hex: "0014", diff --git a/src/ghidra/GhidraLoadImageValues.ts b/src/ghidra/GhidraLoadImageValues.ts index 198b5deed2..b2c58a2ba5 100644 --- a/src/ghidra/GhidraLoadImageValues.ts +++ b/src/ghidra/GhidraLoadImageValues.ts @@ -27,7 +27,11 @@ export type GhidraDosImage = { readonly header: DosMzHeader; readonly loadSegment: number; readonly modified: Buffer; - readonly expectedRelocations: NativeLoadImageObservation["relocations"]; + /** The independent DOS check does not attest Ghidra's symbol name. */ + readonly expectedRelocations: readonly Omit< + NativeLoadImageObservation["relocations"][number], + "symbol" + >[]; }; /** Prepare initialized bytes and relocation facts without copying overlays. */ @@ -137,7 +141,11 @@ export const attestGhidraDosLoadImage = ( ); check("entry_points", [entryAddress], [...observations.entry_points].sort()); // A multiset preserves duplicate relocations and aliases; iteration order is not semantic. - const rows = (values: NativeLoadImageObservation["relocations"]): string[] => + const rows = ( + values: readonly (GhidraDosImage["expectedRelocations"][number] & { + readonly symbol?: string | null; + })[], + ): string[] => values // The independent DOS check does not attest Ghidra's symbol name. .map( @@ -270,17 +278,9 @@ const checkMappings = ( check( "initialized_unbacked_mappings", [], - observations.mappings - .filter( - (mapping) => mapping.initialized && mapping.source_file_index === null, - ) - // Unmeasured permissions serialize as null so the observed value stays JSON. - .map((mapping) => ({ - ...mapping, - readable: mapping.readable ?? null, - writable: mapping.writable ?? null, - executable: mapping.executable ?? null, - })), + observations.mappings.filter( + (mapping) => mapping.initialized && mapping.source_file_index === null, + ), ); }; diff --git a/src/native/exports/NativeExportParser.test.ts b/src/native/exports/NativeExportParser.test.ts index efc19fdc6b..aaed2262e4 100644 --- a/src/native/exports/NativeExportParser.test.ts +++ b/src/native/exports/NativeExportParser.test.ts @@ -740,9 +740,9 @@ describe("ELF version index 0 unversioned exports", () => { }); }); -describe("historical ELF export evidence", () => { - it("normalizes historical ELF export evidence lacking visibility to unknown", () => { - const historical = { +describe("ELF export evidence without visibility", () => { + it("refuses an ELF export row without visibility", () => { + const withoutVisibility = { artifact, format: "elf" as const, architecture: "amd64", @@ -769,10 +769,9 @@ describe("historical ELF export evidence", () => { issues: [], limitations: [], }; - const parsed = nativeExportContracts.nativeExportsSchema.parse(historical); - expect(parsed.format).toBe("elf"); - if (parsed.format !== "elf") throw new Error("Expected ELF result"); - expect(parsed.exports[0]?.visibility).toBe("unknown"); + const parsed = + nativeExportContracts.nativeExportsSchema.safeParse(withoutVisibility); + expect(parsed.error?.issues[0]?.path).toEqual(["exports", 0, "visibility"]); }); }); diff --git a/src/native/exports/NativeExportsView.test.ts b/src/native/exports/NativeExportsView.test.ts index d8cc4eb001..ae242267ac 100644 --- a/src/native/exports/NativeExportsView.test.ts +++ b/src/native/exports/NativeExportsView.test.ts @@ -83,7 +83,7 @@ it("keeps every ELF same-name/version alternative, source order and matching-pag }); }); -it("qualifies projected names with exact encoding facts and normalizes historical omissions", () => { +it("qualifies projected names with exact encoding facts and refuses rows without them", () => { const report = elf(); expect( project(report, { @@ -102,35 +102,30 @@ it("qualifies projected names with exact encoding facts and normalizes historica }, ], }); - const historical = { + const withoutEncoding = { ...report, exports: report.exports.map( ({ name_bytes_base64: _bytes, name_utf8_decodable: _utf8, ...row }) => row, ), }; - freezeJsonSnapshot(historical); - const input = { ...parent(historical), sealedResult: true }; - for (let attempt = 0; attempt < 2; attempt++) { - const result = projectAnalysisView( - input, - analysisViewRequestSchema.parse({ - kind: "native-exports", - collection: "exports", - indices: [3], - fields: ["name"], - }), - ); - if (!result.ok) throw result.error; - expect(result.value).toMatchObject({ - items: [ - { name: "café", name_bytes_base64: null, name_utf8_decodable: null }, - ], - }); - } + freezeJsonSnapshot(withoutEncoding); + const refused = projectAnalysisView( + { ...parent(withoutEncoding), sealedResult: true }, + analysisViewRequestSchema.parse({ + kind: "native-exports", + collection: "exports", + indices: [3], + fields: ["name"], + }), + ); + expect(refused).toMatchObject({ + ok: false, + error: { _tag: "AnalysisInputError" }, + }); }); -it("matches ELF UTF-8 identities without conflating Latin-1 display names or unknown encodings", () => { +it("matches ELF UTF-8 identities without conflating Latin-1 display names", () => { const report = parseNativeExports( elfExportImage({ symbols: [{ name: "café" }, { name: "café" }, { name: "same" }], @@ -149,48 +144,24 @@ it("matches ELF UTF-8 identities without conflating Latin-1 display names or unk coverage: { total: 1 }, }); if (report.format !== "elf") throw new Error("Expected ELF fixture"); - const historical = { - ...report, - exports: report.exports.map( - ({ name_bytes_base64: _bytes, name_utf8_decodable: _utf8, ...row }) => - row, - ), - }; - expect( - project(historical, { - kind: "native-exports", - collection: "exports", - names: ["café"], - }), - ).toMatchObject({ - items: [], - unknowns: [expect.stringContaining("lack retained encoding facts")], - }); - expect( - project(historical, { - kind: "native-exports", - collection: "exports", - names: [], - }), - ).toMatchObject({ items: [], unknowns: [] }); const missingBytes = { ...report, exports: report.exports.map((row) => ({ ...row, name_bytes_base64: null })), }; expect( - project(missingBytes, { - kind: "native-exports", - collection: "exports", - names: ["café", "same"], - fields: [], - }), - ).toMatchObject({ - items: [{ source_index: 2 }], - unknowns: [expect.stringContaining("lack retained encoding facts")], - }); + projectAnalysisView( + parent(missingBytes), + analysisViewRequestSchema.parse({ + kind: "native-exports", + collection: "exports", + names: ["café", "same"], + fields: [], + }), + ).ok, + ).toBe(false); }); -it("does not fill historical name defaults into a projection that omits names", () => { +it("does not add name encoding facts to a projection that omits names", () => { const result = project(elf(), { kind: "native-exports", collection: "exports", @@ -205,7 +176,7 @@ it("does not fill historical name defaults into a projection that omits names", expect(view_digest).toBe(digestCanonicalValue(content, "Analysis view")); }); -it("projects ELF visibility when selected and normalizes historical records without visibility to unknown", () => { +it("projects ELF visibility when selected and refuses rows without it", () => { const report = parseNativeExports( elfExportImage({ symbols: [ @@ -243,22 +214,20 @@ it("projects ELF visibility when selected and normalizes historical records with expect(omitted.items[0]).not.toHaveProperty("visibility"); if (report.format !== "elf") throw new Error("Expected ELF fixture"); - const historical = { + const withoutVisibility = { ...report, exports: report.exports.map(({ visibility: _vis, ...row }) => row), }; - freezeJsonSnapshot(historical); - const historicalResult = project(historical, { - kind: "native-exports", - collection: "exports", - fields: ["name", "visibility"], - }); - expect(historicalResult).toMatchObject({ - items: [ - { source_index: 0, name: "default_fn", visibility: "unknown" }, - { source_index: 1, name: "protected_fn", visibility: "unknown" }, - ], - }); + expect( + projectAnalysisView( + parent(withoutVisibility), + analysisViewRequestSchema.parse({ + kind: "native-exports", + collection: "exports", + fields: ["name", "visibility"], + }), + ).ok, + ).toBe(false); }); it.each(["Y2Fmw6k", "Y2Fmw6l=", "not base64"])( diff --git a/src/pcap/PcapOutput.ts b/src/pcap/PcapOutput.ts index f454cca368..d0f3925b17 100644 --- a/src/pcap/PcapOutput.ts +++ b/src/pcap/PcapOutput.ts @@ -146,7 +146,7 @@ export interface TsharkFields { readonly endpointCount: number; readonly dnsQueries: { query: string; - query_labels_hex?: string[] | null; + query_labels_hex: string[] | null; response_addresses: string[]; response_address_count: number; }[]; diff --git a/src/server/registerApplicationTools/obligationLedger.ts b/src/server/registerApplicationTools/obligationLedger.ts index fd04c437df..873664d582 100644 --- a/src/server/registerApplicationTools/obligationLedger.ts +++ b/src/server/registerApplicationTools/obligationLedger.ts @@ -7,7 +7,7 @@ import { resolveReconstructionObligationLedgerRequest } from "../../application/ import { ok } from "../../domain/result.js"; import { toolContract } from "../../contracts/toolContracts.js"; import { logToolExecution } from "../toolLogging.js"; -import { toolRegistrationOptions } from "../toolRegistrationOptions.js"; +import { applicationToolRegistrationOptions } from "../toolRegistrationOptions.js"; import { recordResult } from "./helpers.js"; import type { ApplicationToolRegistration } from "./types.js"; @@ -20,7 +20,7 @@ export const registerReconstructionObligationLedgerTool = ( ): void => { server.registerTool( contract.name, - toolRegistrationOptions(contract), + applicationToolRegistrationOptions(contract), async (input, context) => runAdmittedToolOperation( server, diff --git a/tests/acceptance/artifactComparisonHistoricalSummary.test.ts b/tests/acceptance/artifactComparisonHistoricalSummary.test.ts index f3860482a3..0c1a4ee99f 100644 --- a/tests/acceptance/artifactComparisonHistoricalSummary.test.ts +++ b/tests/acceptance/artifactComparisonHistoricalSummary.test.ts @@ -5,6 +5,7 @@ import { expect, it } from "vitest"; import { sealAnalysisView } from "../../src/domain/analysisView/analysisView.js"; import { artifactInventorySummaryViewResultSchema } from "../../src/domain/analysisView/analysisViewResult.js"; import { artifactComparisonResultSchema } from "../../src/domain/artifactComparison.js"; +import { RECREATE_EVIDENCE_GUIDANCE } from "../../src/domain/evidenceFormat.js"; import { digestCanonicalValue } from "../../src/domain/canonicalDigest.js"; import { evidenceSemanticContent, @@ -34,7 +35,7 @@ const resealSummary = ( }); }; -it("compares a historical alias summary through its retained complete MCP parent and rejects resealed false facts", async () => { +it("refuses a historical alias summary with recreate guidance and rejects resealed false facts", async () => { const root = await createTestTempDirectory("rea-historical-summary-mcp-"); const path = join(root, "original.plist"); await writeFile( @@ -87,21 +88,19 @@ it("compares a historical alias summary through its retained complete MCP parent }); expect(historical.evidence_id).not.toBe(current.evidence_id); const originalHistorical = JSON.stringify(historical); + // The Evidence format owner no longer rebuilds 6.7-era summary identities: + // the earlier projection is refused with the single recreate guidance. const compared = await call("compare_artifacts", { left: historical, right: parent, }); - expect(compared.isError, JSON.stringify(compared)).not.toBe(true); - const comparison = parseEvidence(compared.structuredContent); - expect( - artifactComparisonResultSchema.parse(comparison.normalized_result), - ).toMatchObject({ - status: "unchanged", - summary: { unchanged: 1, unknown: 0 }, - changes: [], + expect(compared.isError, JSON.stringify(compared)).toBe(true); + expect(parseMcpToolError(compared)).toMatchObject({ + error: { + code: "evidence_integrity_mismatch", + details: { reason: expect.stringContaining(RECREATE_EVIDENCE_GUIDANCE) }, + }, }); - expect(comparison.evidence_links).toContain(parent.evidence_id); - expect(comparison.evidence_links).not.toContain(historical.evidence_id); const historicalResult = artifactInventorySummaryViewResultSchema.parse( historical.normalized_result, diff --git a/tests/acceptance/webSessionRedirectContext.test.ts b/tests/acceptance/webSessionRedirectContext.test.ts index e5e28925fb..ccfa6b7d1e 100644 --- a/tests/acceptance/webSessionRedirectContext.test.ts +++ b/tests/acceptance/webSessionRedirectContext.test.ts @@ -7,7 +7,7 @@ import { connectLocalToolsMcp } from "../fixtures/localToolsMcp.js"; import { cliTest } from "../support/cli/cliFixture.js"; cliTest.for(["CLI", "stdio MCP"] as const)( - "retains admitted redirect response facts and historical omission through %s", + "retains admitted redirect response facts exactly on redirect events through %s", async (surface, { cli }) => { const browser = await startFakeCdpBrowser({ eventsBeforeReply: ["Page.setLifecycleEventsEnabled"], @@ -86,15 +86,31 @@ cliTest.for(["CLI", "stdio MCP"] as const)( }), ); expect(JSON.stringify(source)).not.toContain("discard-redirect-"); - const historical = { + const omitted = { ...result, timeline: result.timeline.map(({ redirect_response, ...row }) => { void redirect_response; return row; }), }; - expect(webObservationSessionSchema.parse(historical)).toStrictEqual( - historical, - ); + const misplaced = { + ...result, + timeline: result.timeline.map((row) => ({ + ...row, + type: "lifecycle" as const, + })), + }; + for (const malformed of [omitted, misplaced]) + expect(webObservationSessionSchema.safeParse(malformed)).toMatchObject({ + success: false, + error: { + issues: expect.arrayContaining([ + expect.objectContaining({ + message: + "redirect_response is present exactly on redirect events.", + }), + ]), + }, + }); }, ); diff --git a/tests/boundary/android/JadxAnalysis.test.ts b/tests/boundary/android/JadxAnalysis.test.ts index 1fb29abcf2..3dc9862242 100644 --- a/tests/boundary/android/JadxAnalysis.test.ts +++ b/tests/boundary/android/JadxAnalysis.test.ts @@ -393,12 +393,19 @@ it("accepts an examined empty inventory without requiring cursor progress", asyn it("requires explicit direct invoke coverage state in result evidence", () => { const result = { engine, + target_definition: "defined", class_name: "fixture.Target", method_name: null, overload_index: null, reported_signature: null, dex_descriptor: null, total: 1, + direct_invokes_coverage: { + complete_using_nodes: 0, + partial_using_nodes: 0, + unknown_using_nodes: 1, + recorded_sites: 0, + }, references: [ { kind: "method", diff --git a/tests/boundary/android/JadxExternalReferences.test.ts b/tests/boundary/android/JadxExternalReferences.test.ts index 4049f3af6b..6b45c1634b 100644 --- a/tests/boundary/android/JadxExternalReferences.test.ts +++ b/tests/boundary/android/JadxExternalReferences.test.ts @@ -115,7 +115,7 @@ describe("external DEX method-reference projection", () => { }); }); - it("counts complete positions and preserves historical aggregate omission", async () => { + it("counts complete positions and refuses results without the aggregate or target definition", async () => { const result = androidResultSchemas.trace_android_references.parse( await analyze(), ); @@ -125,10 +125,16 @@ describe("external DEX method-reference projection", () => { unknown_using_nodes: 0, recorded_sites: 2, }); - const { direct_invokes_coverage: _coverage, ...historical } = result; - expect( - androidResultSchemas.trace_android_references.parse(historical), - ).not.toHaveProperty("direct_invokes_coverage"); + for (const field of [ + "direct_invokes_coverage", + "target_definition", + ] as const) { + const { [field]: _omitted, ...unrecorded } = result; + expect( + androidResultSchemas.trace_android_references.safeParse(unrecorded) + .error?.issues, + ).toEqual([expect.objectContaining({ path: [field] })]); + } }); it("lists exact observed DEX descriptors when an external overload is ambiguous", async () => { diff --git a/tests/boundary/android/androidManifestComponents.test.ts b/tests/boundary/android/androidManifestComponents.test.ts index 618169cfdd..3fb954d615 100644 --- a/tests/boundary/android/androidManifestComponents.test.ts +++ b/tests/boundary/android/androidManifestComponents.test.ts @@ -35,8 +35,8 @@ it.each([ }, ); -it("keeps pre-projection package parses independent", () => { - const older = { +it("refuses package results without the component projection", () => { + const unprojected = { engine: { name: "jadx", version: null, @@ -60,15 +60,10 @@ it("keeps pre-projection package parses independent", () => { resource_count: 0, signature_verification: "not_performed", }; - const first = androidResultSchemas.inspect_android_package.parse(older); - expect(first.components).toMatchObject({ status: "unavailable", items: [] }); - first.components.limitations.push("local parse mutation"); - try { - const second = androidResultSchemas.inspect_android_package.parse(older); - expect(second.components.limitations).not.toContain("local parse mutation"); - } finally { - first.components.limitations.pop(); - } + expect( + androidResultSchemas.inspect_android_package.safeParse(unprojected).error + ?.issues, + ).toEqual([expect.objectContaining({ path: ["components"] })]); }); it("projects exact declared component attributes without inferring runtime defaults", () => { @@ -151,9 +146,9 @@ it("enforces the process copy budget across individually valid components", () = ).toBeLessThanOrEqual(ANDROID_MANIFEST_LIMITS.processBytes); }); -it("keeps historical declaration omissions unknown rather than absent", () => { +it("refuses components without their declaration facets", () => { const parsed = - androidResultSchemas.inspect_android_package.shape.components.parse({ + androidResultSchemas.inspect_android_package.shape.components.safeParse({ status: "complete", items: [ { @@ -174,17 +169,20 @@ it("keeps historical declaration omissions unknown rather than absent", () => { ], limitations: [], }); - expect(parsed.status).toBe("complete"); - expect(parsed.items[0]?.declaration_coverage).toBe("unknown"); - for (const field of [ - "enabled", - "process", - "grant_uri_permissions", - "intent_filters", - "meta_data", - "location", - ]) - expect(parsed.items[0]).not.toHaveProperty(field); + expect(new Set(parsed.error?.issues.map(({ path }) => path.at(-1)))).toEqual( + new Set([ + "declaration_coverage", + "declaration_index", + "location", + "enabled", + "enabled_declaration", + "process", + "grant_uri_permissions", + "grant_uri_permissions_declaration", + "intent_filters", + "meta_data", + ]), + ); }); const MANIFEST = ` diff --git a/tests/boundary/browser/cdpJsonShapeComparison.test.ts b/tests/boundary/browser/cdpJsonShapeComparison.test.ts index 9ac82331e3..a700d50718 100644 --- a/tests/boundary/browser/cdpJsonShapeComparison.test.ts +++ b/tests/boundary/browser/cdpJsonShapeComparison.test.ts @@ -494,7 +494,11 @@ const captureBody = ( }, completeness: events.completeness.snapshot(), frames: [], - dom: { total_nodes: 0, nodes: [] }, + dom: { + total_nodes: 0, + node_value_length_unit: "utf16-code-units", + nodes: [], + }, accessibility: { total_nodes: 0, text_capture: { diff --git a/tests/boundary/cli/domValueLengthUnits.test.ts b/tests/boundary/cli/domValueLengthUnits.test.ts index d1583bb9a0..4ee10422fe 100644 --- a/tests/boundary/cli/domValueLengthUnits.test.ts +++ b/tests/boundary/cli/domValueLengthUnits.test.ts @@ -14,7 +14,7 @@ import { webCaptureDiffSchema } from "../../../src/domain/webCaptureDiffSchemas. for (const electron of [false, true]) cliTest( - `declares ${electron ? "Electron UTF-8" : "web UTF-16"} DOM value lengths through CLI and MCP without reinterpreting historical captures`, + `declares ${electron ? "Electron UTF-8" : "web UTF-16"} DOM value lengths through CLI and MCP and refuses snapshots without a unit`, async ({ cli, onTestFinished }) => { const root = await createTestTempDirectory("rea-dom-value-units-"); const path = join(root, "index.html"); @@ -92,19 +92,15 @@ for (const electron of [false, true]) "node_value_length_unit", ); expect(JSON.stringify(report.dom)).not.toContain("ééé😀"); - const legacyDom = { + const unmarked = { total_nodes: report.dom.total_nodes, nodes: report.dom.nodes, }; - expect(schema.parse({ ...report, dom: legacyDom }).dom).toEqual( - legacyDom, - ); - expect( - schema.safeParse({ - ...report, - dom: { ...legacyDom, node_value_length_unit: "characters" }, - }).success, - ).toBe(false); + for (const dom of [ + unmarked, + { ...unmarked, node_value_length_unit: "characters" }, + ]) + expect(schema.safeParse({ ...report, dom }).success).toBe(false); }; check(result.json); const call = await cli.sharedMcpCall!(); @@ -141,20 +137,16 @@ cliTest( for (const [beforeUnit, afterUnit, length, variation, status] of [ ["utf16-code-units", "utf16-code-units", 5, "none", "unchanged"], ["utf8-bytes", "utf8-bytes", 10, "none", "changed"], - [undefined, undefined, 5, "none", "unknown"], - [undefined, "utf16-code-units", 10, "none", "unknown"], ["utf16-code-units", "utf8-bytes", 5, "none", "unknown"], ["utf16-code-units", "utf8-bytes", 10, "none", "unknown"], - [undefined, undefined, 5, "node", "changed"], + ["utf16-code-units", "utf8-bytes", 5, "node", "changed"], ["utf16-code-units", "utf8-bytes", 10, "frame", "changed"], - [undefined, undefined, 5, "empty", "unchanged"], + ["utf16-code-units", "utf8-bytes", 5, "empty", "unchanged"], ] as const) { const before = passiveNetworkCapture(); before.dom = { total_nodes: 2, - ...(beforeUnit === undefined - ? {} - : { node_value_length_unit: beforeUnit }), + node_value_length_unit: beforeUnit, nodes: [ { index: 0, @@ -175,8 +167,7 @@ cliTest( ], }; const after = structuredClone(before); - if (afterUnit === undefined) delete after.dom.node_value_length_unit; - else after.dom.node_value_length_unit = afterUnit; + after.dom.node_value_length_unit = afterUnit; after.dom.nodes[1]!.node_value_length = length; if (variation === "node") { after.dom.nodes[1]!.node_type = 8; @@ -190,8 +181,16 @@ cliTest( origin: "https://app.example.test", }); if (variation === "empty") { - before.dom = { total_nodes: 0, nodes: [] }; - after.dom = { total_nodes: 0, nodes: [] }; + before.dom = { + total_nodes: 0, + node_value_length_unit: beforeUnit, + nodes: [], + }; + after.dom = { + total_nodes: 0, + node_value_length_unit: afterUnit, + nodes: [], + }; } const result = await cli.run({ arguments: [ @@ -222,5 +221,17 @@ cliTest( expect(diff.dimensions.dom_structure.reason).toContain("unit"); } } + const unmarked = passiveNetworkCapture(); + const { node_value_length_unit: _unit, ...dom } = unmarked.dom; + const refused = await cli.run({ + arguments: [ + "compare-web-captures", + JSON.stringify({ inspection: { ...unmarked, dom } }), + JSON.stringify({ inspection: unmarked }), + "--json", + ], + }); + expect(refused.exitCode).not.toBe(0); + expect(refused.stderr + refused.stdout).toContain("node_value_length_unit"); }, ); diff --git a/tests/boundary/cli/evidenceBundleCommands.test.ts b/tests/boundary/cli/evidenceBundleCommands.test.ts index 2e3b3a504b..fd06319e8f 100644 --- a/tests/boundary/cli/evidenceBundleCommands.test.ts +++ b/tests/boundary/cli/evidenceBundleCommands.test.ts @@ -1,3 +1,4 @@ +import { EVIDENCE_FORMAT_REVISION } from "../../../src/domain/evidenceFormat.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { PRE_66_NULL_COVERAGE_EVIDENCE } from "../../../src/domain/evidenceLegacy.fixture.js"; import { @@ -487,7 +488,10 @@ cliTest( const root = await createTestTempDirectory("rea-legacy-coverage-identity-"); const path = join(root, "legacy.json"); const record = PRE_66_NULL_COVERAGE_EVIDENCE; + // A current-format bundle carrying the earlier record reaches the + // record-level identity diagnosis; an unmarked one is refused earlier. const bundle = { + evidence_format: EVIDENCE_FORMAT_REVISION, artifacts: [], providers: [record.provider], environments: [], diff --git a/tests/boundary/cli/strictPositionals.test.ts b/tests/boundary/cli/strictPositionals.test.ts index 2a85bf19c9..258cc2fe8b 100644 --- a/tests/boundary/cli/strictPositionals.test.ts +++ b/tests/boundary/cli/strictPositionals.test.ts @@ -44,6 +44,7 @@ it("rejects surplus public CLI inputs before importing and retains the one-path await writeFile( path, JSON.stringify({ + evidence_format: 1, artifacts: [], captures: [], environments: [], diff --git a/tests/boundary/filesystem/evidenceExportCleanupRetry.test.ts b/tests/boundary/filesystem/evidenceExportCleanupRetry.test.ts index 2a7d99f71a..13f934f769 100644 --- a/tests/boundary/filesystem/evidenceExportCleanupRetry.test.ts +++ b/tests/boundary/filesystem/evidenceExportCleanupRetry.test.ts @@ -1,3 +1,4 @@ +import { EVIDENCE_FORMAT_REVISION } from "../../../src/domain/evidenceFormat.js"; import { readdir, rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; @@ -40,6 +41,7 @@ vi.mock("node:fs/promises", async (importOriginal) => { }); const emptyBundle = JSON.stringify({ + evidence_format: EVIDENCE_FORMAT_REVISION, artifacts: [], providers: [], environments: [], diff --git a/tests/boundary/filesystem/nativeExports.test.ts b/tests/boundary/filesystem/nativeExports.test.ts index ee1f37c34a..3ca8c8f1ca 100644 --- a/tests/boundary/filesystem/nativeExports.test.ts +++ b/tests/boundary/filesystem/nativeExports.test.ts @@ -347,35 +347,23 @@ describe("native export encoding and failures", () => { }, ], }); - const olderRows = JSON.parse( + const withoutEncoding = JSON.parse( JSON.stringify(result.value.normalized_result), ) as { exports: Record[] }; - for (const row of olderRows.exports) { + for (const row of withoutEncoding.exports) { delete row.name_bytes_base64; delete row.name_utf8_decodable; - const version = row.version; - if (version !== null && typeof version === "object") { - Reflect.deleteProperty(version, "name_bytes_base64"); - Reflect.deleteProperty(version, "name_utf8_decodable"); - } } - expect(nativeExportsSchema.parse(olderRows)).toMatchObject({ - exports: [ - { - name: "café", - name_bytes_base64: null, - name_utf8_decodable: null, - version: { - index: 2, - name: "libcafé.so", - name_bytes_base64: null, - name_utf8_decodable: null, - hidden: false, - }, - }, - { name: "ÿ", name_bytes_base64: null, name_utf8_decodable: null }, - ], - }); + expect( + nativeExportsSchema + .safeParse(withoutEncoding) + .error?.issues.map(({ path }) => path.join(".")), + ).toEqual( + expect.arrayContaining([ + "exports.0.name_bytes_base64", + "exports.0.name_utf8_decodable", + ]), + ); await service.close(); }); diff --git a/tests/boundary/mcp/analysisViewMcp.test.ts b/tests/boundary/mcp/analysisViewMcp.test.ts index dd6561c597..a3b8919ce4 100644 --- a/tests/boundary/mcp/analysisViewMcp.test.ts +++ b/tests/boundary/mcp/analysisViewMcp.test.ts @@ -1610,6 +1610,7 @@ it("advertises Mach-O pages whose selected facet owns their exact rows", async ( }); const symbol = { name: "_entry", + runtime_name: "entry", address: null, weak: null, reexport: null, @@ -1632,6 +1633,7 @@ it("advertises Mach-O pages whose selected facet owns their exact rows", async ( { ...symbol, name: null, + runtime_name: null, raw_name: "_entry+0x4", name_status: "ambiguous", addend: null, @@ -1656,20 +1658,13 @@ it("advertises Mach-O pages whose selected facet owns their exact rows", async ( provenance: [], limitations: [], }); - const { malformed_kind: _unrecorded, ...recordedEncryption } = - report.encryption.items[0]!; - const historicalEncryption = { - ...recordedEncryption, - status: "malformed" as const, - reason: "slice has 3 encryption load commands", - }; const parent = createEvidence( { path: "/fixtures/tool", format: "mach-o", sha256: "a".repeat(64) }, { id: "native", name: "Native fixture", version: "1" }, { operation: "inspect_macho", parameters: {}, - result: { ...report, encryption: coverage([historicalEncryption]) }, + result: report, }, ); const original = structuredClone(parent); @@ -1699,11 +1694,7 @@ it("advertises Mach-O pages whose selected facet owns their exact rows", async ( const delivered = parseEvidence(response.structuredContent); const page = analysisViewResultSchema.parse(delivered.normalized_result); if (page.kind !== "macho") throw new Error("Expected Mach-O page"); - const rows = - facet === "encryption" - ? [{ ...historicalEncryption, malformed_kind: "unrecorded" }] - : report[facet].items; - expect(page.item).toEqual(rows.slice(offset, offset + 64)); + expect(page.item).toEqual(report[facet].items.slice(offset, offset + 64)); if (page.item.length > 0) { const mismatched = { ...delivered, diff --git a/tests/boundary/mcp/androidPackageSummaryMcp.test.ts b/tests/boundary/mcp/androidPackageSummaryMcp.test.ts index 305d729b25..a84269e9d4 100644 --- a/tests/boundary/mcp/androidPackageSummaryMcp.test.ts +++ b/tests/boundary/mcp/androidPackageSummaryMcp.test.ts @@ -416,7 +416,7 @@ it.skipIf(process.platform === "win32")( }, ); -it("preserves partial/unknown declarations and Unicode text while rejecting missing APK identity and malformed source", async () => { +it("preserves partial manifest text and Unicode while rejecting missing APK identity and malformed source", async () => { const report = { engine: { name: "jadx", @@ -433,6 +433,7 @@ it("preserves partial/unknown declarations and Unicode text while rejecting miss target_sdk: null, permissions: [], manifest: { status: "partial", text: "a😀b", reported_total_bytes: null }, + components: { status: "unavailable", items: [], limitations: [] }, class_count: 0, resource_count: 0, signature_verification: "not_performed", @@ -471,6 +472,7 @@ it("preserves partial/unknown declarations and Unicode text while rejecting miss const result = analysisViewResultSchema.parse(page.value.normalized_result); expect(result).toMatchObject({ kind: "android", + facet: "manifest", item: { text: "a😀b", source_status: "partial", @@ -617,6 +619,16 @@ it("delivers the selected retained page in one analysis and reads it later witho foreground_service_type: null, authorities: null, target_activity: null, + declaration_coverage: "complete", + declaration_index: 0, + location: null, + enabled: null, + enabled_declaration: null, + process: null, + grant_uri_permissions: null, + grant_uri_permissions_declaration: null, + intent_filters: [], + meta_data: [], }, ], }, @@ -660,7 +672,7 @@ it("delivers the selected retained page in one analysis and reads it later witho selected.value.normalized_result, ); expect(first.items).toEqual([ - { source_index: 0, name: "example.Main", declaration_coverage: "unknown" }, + { source_index: 0, name: "example.Main", declaration_coverage: "complete" }, ]); expect( session.evidenceById(first.parent_evidence_id)?.normalized_result, diff --git a/tests/boundary/mcp/dartAotSummaryMcp.test.ts b/tests/boundary/mcp/dartAotSummaryMcp.test.ts index 5e88924df4..6cb72cb4d8 100644 --- a/tests/boundary/mcp/dartAotSummaryMcp.test.ts +++ b/tests/boundary/mcp/dartAotSummaryMcp.test.ts @@ -213,6 +213,7 @@ it("preserves null snapshot facts and pool lower bounds after exhaustion, reject sections: [], }, string_pool: { + policy: {}, coverage: "partial", package_uris: ["package:opaque/a.dart"], package_uri_count_at_least: 1000, diff --git a/tests/boundary/mcp/flutterMcp.test.ts b/tests/boundary/mcp/flutterMcp.test.ts index 88225ed103..6e1bb2e303 100644 --- a/tests/boundary/mcp/flutterMcp.test.ts +++ b/tests/boundary/mcp/flutterMcp.test.ts @@ -142,6 +142,7 @@ const inspectExecution = () => ], }, string_pool: { + policy: {}, coverage: "complete", package_uris: [ "package:adaptive_breakpoints/src/adaptive_breakpoints.dart", diff --git a/tests/boundary/mcp/pcapMcp.test.ts b/tests/boundary/mcp/pcapMcp.test.ts index 8678cac2df..e36c0087df 100644 --- a/tests/boundary/mcp/pcapMcp.test.ts +++ b/tests/boundary/mcp/pcapMcp.test.ts @@ -62,6 +62,7 @@ it("advertises exact valid pcap schemas and records inline evidence through MCP" dns_queries: [ { query: "example.com", + query_labels_hex: ["6578616d706c65", "636f6d"], response_addresses: ["93.184.216.34"], response_address_count: 1, }, diff --git a/tests/boundary/mcp/pcapParentIssuePath.test.ts b/tests/boundary/mcp/pcapParentIssuePath.test.ts index bef8b25916..cd294e844b 100644 --- a/tests/boundary/mcp/pcapParentIssuePath.test.ts +++ b/tests/boundary/mcp/pcapParentIssuePath.test.ts @@ -14,9 +14,15 @@ it("locates malformed pcap parent fields beneath the analysis-view source", asyn tshark_command: "tshark", tshark_source: "path", tshark_version: "4.6.6", + tshark_resolved_path: null, + tshark_sha256: null, + tshark_identity_status: "unresolved", capinfos_command: "capinfos", capinfos_source: "path", capinfos_version: "4.6.6", + capinfos_resolved_path: null, + capinfos_sha256: null, + capinfos_identity_status: "unresolved", }, target, file_type: "pcap", @@ -47,8 +53,8 @@ it("locates malformed pcap parent fields beneath the analysis-view source", asyn createEvidence( { ...target, format: "file" }, { - id: "historical-pcap", - name: "Historical packet capture", + id: "retained-pcap", + name: "Retained packet capture", version: "1", }, { diff --git a/tests/boundary/mcp/reconstructionCoverageMcp.test.ts b/tests/boundary/mcp/reconstructionCoverageMcp.test.ts index d851df0c87..0a71feefce 100644 --- a/tests/boundary/mcp/reconstructionCoverageMcp.test.ts +++ b/tests/boundary/mcp/reconstructionCoverageMcp.test.ts @@ -15,6 +15,7 @@ import { updateResidualUnknown, } from "../../../src/domain/residualUnknown.js"; import { createEvidenceBundle } from "../../../src/domain/evidenceBundle.js"; +import { RECREATE_EVIDENCE_GUIDANCE } from "../../../src/domain/evidenceFormat.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; import { createServer } from "../../../src/server/createServer.js"; import { observed } from "../../fixtures/analysisExecution.js"; @@ -40,6 +41,29 @@ it("rejects malformed boundary identifiers at the MCP request boundary", async ( }); }); +it("refuses an unmarked Evidence bundle through the format owner, not the SDK", async () => { + await withCoverageMcp(async ({ client }) => { + const coverage = currentFixtureCoverage(); + const { evidence_format: _current, ...unmarked } = coverage.evidence_bundle; + const result = await client.callTool({ + name: "evaluate_reconstruction_coverage", + arguments: { + coverage: { ...coverage, evidence_bundle: unmarked }, + boundary_id: "replacement.cli", + }, + }); + expect(result.isError).toBe(true); + expect(parseMcpToolError(result)).toMatchObject({ + error: { + code: "evidence_integrity_mismatch", + details: { + reason: expect.stringContaining(RECREATE_EVIDENCE_GUIDANCE), + }, + }, + }); + }); +}); + it("uses the latest listed unknown revision when evaluating a boundary", async () => { await withCoverageMcp(async ({ client }) => { const base = currentFixtureCoverage(); diff --git a/tests/boundary/mcp/reconstructionObligationLedgerMcp.test.ts b/tests/boundary/mcp/reconstructionObligationLedgerMcp.test.ts index ca55bae794..fa5440c24d 100644 --- a/tests/boundary/mcp/reconstructionObligationLedgerMcp.test.ts +++ b/tests/boundary/mcp/reconstructionObligationLedgerMcp.test.ts @@ -8,6 +8,10 @@ import { describe, expect, it } from "vitest"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { createEvidenceBundle } from "../../../src/domain/evidenceBundle.js"; +import { + EVIDENCE_FORMAT_REVISION, + RECREATE_EVIDENCE_GUIDANCE, +} from "../../../src/domain/evidenceFormat.js"; import { createEvidence, evidenceSchema, @@ -243,6 +247,31 @@ describe("reconstruction obligation ledger process provider admission", () => { ); }); +describe("reconstruction obligation ledger Evidence format", () => { + it.each([ + ["an unmarked", undefined], + ["a later", EVIDENCE_FORMAT_REVISION + 1], + ])( + "refuses %s Evidence format through the format owner, not the SDK", + async (_label, revision) => { + const { evidence_format: _current, ...unmarked } = createEvidenceBundle( + [], + ); + const bundle = + revision === undefined + ? unmarked + : { ...unmarked, evidence_format: revision }; + await withLedgerClient(async (request) => { + const result = await request(bundle); + expect(result.isError).toBe(true); + expect(parseMcpToolError(result)).toMatchObject( + integrityError(RECREATE_EVIDENCE_GUIDANCE), + ); + }); + }, + ); +}); + describe("reconstruction obligation ledger MCP parity", () => { it("rejects noncanonical source bundles before ledger generation", async () => { const records = ["first", "second"].map((label) => diff --git a/tests/conformance/providers/managed/managedNativeVerification.test.ts b/tests/conformance/providers/managed/managedNativeVerification.test.ts index 2b0110568e..bd15f689bf 100644 --- a/tests/conformance/providers/managed/managedNativeVerification.test.ts +++ b/tests/conformance/providers/managed/managedNativeVerification.test.ts @@ -1,4 +1,5 @@ import { readManagedPeLayout } from "../../../../src/dotnet/ManagedPeReader.js"; +import { machoRuntimeSymbolName } from "../../../../src/domain/native/machoSymbolName.js"; import { metadataRowOffset, readManagedMetadataLayout, @@ -66,6 +67,7 @@ const nativeEvidenceWithExports = ( ...macho.exports, items: names.map((name, index) => ({ name, + runtime_name: machoRuntimeSymbolName(name), address: `0x${(0x1000 + index).toString(16)}`, weak: false, reexport: false, diff --git a/tests/fixtures/javascriptRuntimeReconciliation.ts b/tests/fixtures/javascriptRuntimeReconciliation.ts index 96faa251ab..3471dd007c 100644 --- a/tests/fixtures/javascriptRuntimeReconciliation.ts +++ b/tests/fixtures/javascriptRuntimeReconciliation.ts @@ -87,7 +87,11 @@ export const electronRuntimeEvidence = ( file_path: join(root, "index.html"), }, ], - dom: { total_nodes: 0, nodes: [] }, + dom: { + total_nodes: 0, + node_value_length_unit: "utf8-bytes", + nodes: [], + }, scripts: { total: 1, items: [ diff --git a/tests/fixtures/nativeLoadImage.ts b/tests/fixtures/nativeLoadImage.ts index c60ef2adbb..56e8ae1ff3 100644 --- a/tests/fixtures/nativeLoadImage.ts +++ b/tests/fixtures/nativeLoadImage.ts @@ -28,6 +28,9 @@ export const nativeLoadImageObservationFixture = initialized: true, loaded: true, overlay: false, + readable: true, + writable: false, + executable: true, length: 16, source_file_index: 0, file_offset: 32, diff --git a/tests/fixtures/passiveNetworkCapture.ts b/tests/fixtures/passiveNetworkCapture.ts index 1f563bdafd..c2ff254207 100644 --- a/tests/fixtures/passiveNetworkCapture.ts +++ b/tests/fixtures/passiveNetworkCapture.ts @@ -67,7 +67,11 @@ export const passiveNetworkCapture = (paths: readonly string[] = ["/a"]) => { prior_activity_available: false, }, frames: [], - dom: { total_nodes: 0, nodes: [] }, + dom: { + total_nodes: 0, + node_value_length_unit: "utf16-code-units", + nodes: [], + }, accessibility: { total_nodes: 0, nodes: [], diff --git a/tests/support/browser/webPageInspection.ts b/tests/support/browser/webPageInspection.ts index e14124af0f..7fd101a3f0 100644 --- a/tests/support/browser/webPageInspection.ts +++ b/tests/support/browser/webPageInspection.ts @@ -55,7 +55,11 @@ export const createWebPageInspection = ( }, }, frames: [], - dom: { total_nodes: 0, nodes: [] }, + dom: { + total_nodes: 0, + node_value_length_unit: "utf16-code-units", + nodes: [], + }, accessibility: { total_nodes: 0, text_capture: {