From b6fced40a3917168d3dcd3faece150707a6f8072 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 21 Nov 2026 03:42:05 -0500 Subject: [PATCH] test(snapshots): verify ambiguous provenance through direct replay (#13046) --- .../binary/AnalysisSnapshotCache.test.ts | 32 ------ src/domain/analysisSnapshot.test.ts | 105 +++++------------- .../directAnalysisSnapshotReplay.test.ts | 87 ++++++++++++--- 3 files changed, 98 insertions(+), 126 deletions(-) diff --git a/src/application/binary/AnalysisSnapshotCache.test.ts b/src/application/binary/AnalysisSnapshotCache.test.ts index 25cefddb2e..442231b226 100644 --- a/src/application/binary/AnalysisSnapshotCache.test.ts +++ b/src/application/binary/AnalysisSnapshotCache.test.ts @@ -1581,38 +1581,6 @@ it("replays source-bearing entries only when their complete retained closure exi ) ?.sourceEvidence?.map((record) => record.evidence_id), ).toEqual([source.evidence_id, parent.evidence_id]); - - const otherSource = immutableEvidence( - createEvidence(target, ANALYSIS_SNAPSHOT_PROVIDER, { - operation: "other_source", - parameters: {}, - result: "different source", - }), - ); - const otherParent = immutableEvidence( - createAnalysisExecutionEvidence( - "procedure_info", - parameters, - { ...execution, sourceEvidence: [otherSource] }, - "observed", - ), - ); - const records = [parent, movedSource, leaf, otherParent, otherSource]; - const ambiguous = new AnalysisSnapshotCache(); - ambiguous.stage( - parseAnalysisSnapshot({ - ...exported.value, - evidence_bundle: createEvidenceBundle(records), - }), - ); - const allRecords = new Map( - records.map((record) => [record.evidence_id, record]), - ); - expect( - ambiguous.lookup(target, profile, "procedure_info", parameters, (id) => - allRecords.get(id), - ), - ).toBeUndefined(); }); describe("executions that report truncated references", () => { diff --git a/src/domain/analysisSnapshot.test.ts b/src/domain/analysisSnapshot.test.ts index fc82c16da6..7a500c103e 100644 --- a/src/domain/analysisSnapshot.test.ts +++ b/src/domain/analysisSnapshot.test.ts @@ -16,6 +16,7 @@ import { ANALYSIS_SNAPSHOT_PROVIDER, ANALYSIS_SNAPSHOT_TARGET, } from "./analysisSnapshot.fixture.js"; +import { createAnalysisProfile } from "./analysisProfile.js"; import { createEvidence, evidenceSubject } from "./evidence.js"; import { unknownTarget } from "./targetIdentity.js"; import { createEvidenceBundle } from "./evidenceBundle.js"; @@ -504,51 +505,6 @@ describe("analysis snapshot canonical ordering and scope", () => { }); }); -it("does not select arbitrary provenance when multiple authenticated records match one replay entry", () => { - const { current, snapshot } = snapshotWithHistoricalEvidence(); - const source = snapshot.evidence_bundle.records.find( - (record) => record.operation === "unprofiled_query", - )!; - const alternative = createEvidence( - ANALYSIS_SNAPSHOT_TARGET, - ANALYSIS_SNAPSHOT_PROVIDER, - { - operation: current.operation, - parameters: current.parameters, - result: current.normalized_result, - analysisProfile: ANALYSIS_SNAPSHOT_PROFILE, - evidenceLinks: [source.evidence_id], - }, - ); - expect(alternative.evidence_id).not.toBe(current.evidence_id); - const ambiguous = parseAnalysisSnapshot({ - ...snapshot, - evidence_bundle: createEvidenceBundle([ - ...snapshot.evidence_bundle.records, - alternative, - ]), - }); - expect(ambiguous.entries).toEqual(snapshot.entries); - const reversedInsertion = parseAnalysisSnapshot({ - ...ambiguous, - evidence_bundle: createEvidenceBundle( - [...ambiguous.evidence_bundle.records].reverse(), - ), - }); - expect(reversedInsertion).toEqual(ambiguous); - for (const candidate of [ambiguous, reversedInsertion]) - expect( - snapshotEvidenceForQuery(candidate, { - target: ANALYSIS_SNAPSHOT_TARGET, - bindingProfile: ANALYSIS_SNAPSHOT_PROFILE, - operation: current.operation, - parameters: current.parameters, - provider: ANALYSIS_SNAPSHOT_PROVIDER, - evidenceProfile: ANALYSIS_SNAPSHOT_PROFILE, - }), - ).toBeUndefined(); -}); - it("retains linked foreign Evidence without treating it as selected-target replay", () => { const { current, snapshot } = snapshotWithHistoricalEvidence(); const foreignTarget = { ...ANALYSIS_SNAPSHOT_TARGET, sha256: "b".repeat(64) }; @@ -580,16 +536,37 @@ it("retains linked foreign Evidence without treating it as selected-target repla ...snapshot, evidence_bundle: bundle, }); + const query = { + target: ANALYSIS_SNAPSHOT_TARGET, + bindingProfile: ANALYSIS_SNAPSHOT_PROFILE, + operation: linked.operation, + parameters: linked.parameters, + provider: ANALYSIS_SNAPSHOT_PROVIDER, + evidenceProfile: ANALYSIS_SNAPSHOT_PROFILE, + }; + expect(snapshotEvidenceForQuery(admitted, query)).toEqual(linked); + const differentProfile = createAnalysisProfile(ANALYSIS_SNAPSHOT_PROVIDER, { + fixture: "other", + }); expect( snapshotEvidenceForQuery(admitted, { - target: ANALYSIS_SNAPSHOT_TARGET, - bindingProfile: ANALYSIS_SNAPSHOT_PROFILE, - operation: linked.operation, - parameters: linked.parameters, - provider: ANALYSIS_SNAPSHOT_PROVIDER, - evidenceProfile: ANALYSIS_SNAPSHOT_PROFILE, + ...query, + evidenceProfile: differentProfile, }), - ).toEqual(linked); + ).toBeUndefined(); + expect( + snapshotEvidenceForQuery(admitted, { + ...query, + bindingProfile: differentProfile, + }), + ).toBeUndefined(); + expect( + snapshotEvidenceForQuery(admitted, { + ...query, + operation: "unprofiled_query", + parameters: {}, + }), + ).toBeUndefined(); expect(snapshotMatchesTarget(admitted.target, foreignTarget)).toBe(false); const unrelated = createEvidence(foreignTarget, ANALYSIS_SNAPSHOT_PROVIDER, { operation: "unrelated", @@ -672,30 +649,6 @@ it("binds raw Evidence only when its entry point matches the snapshot target", ( }); describe("analysis snapshot Evidence binding", () => { - it("finds only Evidence committed to the exact binding and profile", () => { - const { current, snapshot } = snapshotWithHistoricalEvidence(); - expect( - snapshotEvidenceForQuery(snapshot, { - target: ANALYSIS_SNAPSHOT_TARGET, - bindingProfile: ANALYSIS_SNAPSHOT_PROFILE, - operation: "analyze_function", - parameters: { procedure: "main" }, - provider: ANALYSIS_SNAPSHOT_PROVIDER, - evidenceProfile: ANALYSIS_SNAPSHOT_PROFILE, - }), - ).toEqual(current); - expect( - snapshotEvidenceForQuery(snapshot, { - target: ANALYSIS_SNAPSHOT_TARGET, - bindingProfile: ANALYSIS_SNAPSHOT_PROFILE, - operation: "unprofiled_query", - parameters: {}, - provider: ANALYSIS_SNAPSHOT_PROVIDER, - evidenceProfile: ANALYSIS_SNAPSHOT_PROFILE, - }), - ).toBeUndefined(); - }); - it("rejects cached execution data that differs from its bundled Evidence", () => { const target = snapshotTarget(ANALYSIS_SNAPSHOT_TARGET); const binding = snapshotBinding(ANALYSIS_SNAPSHOT_PROFILE); diff --git a/tests/boundary/filesystem/directAnalysisSnapshotReplay.test.ts b/tests/boundary/filesystem/directAnalysisSnapshotReplay.test.ts index e8b0ff4517..7bed459442 100644 --- a/tests/boundary/filesystem/directAnalysisSnapshotReplay.test.ts +++ b/tests/boundary/filesystem/directAnalysisSnapshotReplay.test.ts @@ -35,6 +35,7 @@ import { import { createEvidence, evidenceSubject, + evidenceSubjectTarget, parseEvidence, type Evidence, } from "../../../src/domain/evidence.js"; @@ -42,6 +43,7 @@ import { REA_WORKFLOW_PROVIDER } from "../../../src/domain/investigationProvider import { workflowAnalysisProfile } from "../../../src/application/WorkflowAnalysisProfile.js"; import { analysisQueryId, + evidenceMatchesEntry, createAnalysisSnapshotWorkflowEntry, type AnalysisSnapshot, } from "../../../src/domain/analysisSnapshot.js"; @@ -363,7 +365,7 @@ describe("one-shot Ghidra pseudocode snapshots", () => { }); it.skipIf(process.platform === "win32")( - "replays the exact selected-target answer without opening another provider", + "replays the exact selected-target answer and reruns when provenance is ambiguous", async () => { const fixture = await createSnapshotReplayFixture("rea-one-shot-code-"); const profile = createAnalysisProfile( @@ -376,24 +378,18 @@ describe("one-shot Ghidra pseudocode snapshots", () => { supportedOperations: ["procedure_pseudo_code"], }), ); - const options = { snapshotPath: fixture.snapshotPath }; const parameters = { procedure: "fixture" }; - const first = await runDirectAnalysis( - dependencies, - fixture.path, - "procedure_pseudo_code", - parameters, - options, - ); - if (!first.ok) throw new Error(JSON.stringify(first.error)); - const second = await runDirectAnalysis( - dependencies, - fixture.path, - "procedure_pseudo_code", - parameters, - options, - ); - expect(second).toEqual(first); + const query = (selected = dependencies) => + runDirectAnalysis( + selected, + fixture.path, + "procedure_pseudo_code", + parameters, + { snapshotPath: fixture.snapshotPath }, + ); + const first = await query(); + if (!first.ok) throw first.error; + expect(await query()).toEqual(first); expect(fixture.starts).toHaveLength(1); const saved = await readAnalysisSnapshot(fixture.snapshotPath); if (!saved.ok) throw saved.error; @@ -406,6 +402,61 @@ describe("one-shot Ghidra pseudocode snapshots", () => { ({ operation }) => operation === "procedure_pseudo_code", ), ).toEqual([first.value]); + + const parent = first.value; + const target = evidenceSubjectTarget(parent.subject); + if (target === undefined) + throw new Error("Expected a file Evidence subject"); + const alternative = createEvidence(target, parent.provider, { + operation: parent.operation, + parameters: parent.parameters, + result: parent.normalized_result, + rawResult: parent.raw_result, + analysisProfile: profile, + limitations: parent.limitations, + ...(parent.relationship_coverage === null + ? {} + : { relationshipCoverage: parent.relationship_coverage }), + locations: parent.locations, + evidenceLinks: [parent.evidence_id], + }); + expect(alternative.evidence_id).not.toBe(parent.evidence_id); + const entry = saved.value.entries[0]!; + expect(evidenceMatchesEntry(alternative, entry, saved.value)).toBe(true); + const written = await withArtifactResourceScope((resources) => + writeAnalysisSnapshot( + { + ...saved.value, + evidence_bundle: createEvidenceBundle([ + ...saved.value.evidence_bundle.records, + alternative, + ]), + }, + fixture.snapshotPath, + true, + resources, + ), + ); + if (!written.ok) throw written.error; + const fresh = await query( + fixture.dependencies( + fixture.createProvider({ + profile, + supportedOperations: ["procedure_pseudo_code"], + limitations: ["Observed after ambiguous snapshot"], + }), + ), + ); + if (!fresh.ok) throw fresh.error; + expect(fresh.value.limitations).toContain( + "Observed after ambiguous snapshot", + ); + expect( + fixture.calls.filter( + (operation) => operation === "procedure_pseudo_code", + ), + ).toHaveLength(2); + expect(fixture.starts).toHaveLength(2); }, );