test(snapshots): verify ambiguous provenance through direct replay (#13046)

This commit is contained in:
morluto authored and GitHub committed 2026-11-21 08:42:05 +00:00
1 parent eb84f90417
commit b6fced40a3
3 files changed
+98 -126

No files matched your search

@@ -1581,38 +1581,6 @@ it("replays source-bearing entries only when their complete retained closure exi
)
?.sourceEvidence?.map((record) => record.evidence_id),
).toEqual([source.evidence_id, parent.evidence_id]);
const otherSource = immutableEvidence(
createEvidence(target, ANALYSIS_SNAPSHOT_PROVIDER, {
operation: "other_source",
parameters: {},
result: "different source",
}),
);
const otherParent = immutableEvidence(
createAnalysisExecutionEvidence(
"procedure_info",
parameters,
{ ...execution, sourceEvidence: [otherSource] },
"observed",
),
);
const records = [parent, movedSource, leaf, otherParent, otherSource];
const ambiguous = new AnalysisSnapshotCache();
ambiguous.stage(
parseAnalysisSnapshot({
...exported.value,
evidence_bundle: createEvidenceBundle(records),
}),
);
const allRecords = new Map(
records.map((record) => [record.evidence_id, record]),
);
expect(
ambiguous.lookup(target, profile, "procedure_info", parameters, (id) =>
allRecords.get(id),
),
).toBeUndefined();
});
describe("executions that report truncated references", () => {
+29 -76
View File
@@ -16,6 +16,7 @@ import {
ANALYSIS_SNAPSHOT_PROVIDER,
ANALYSIS_SNAPSHOT_TARGET,
} from "./analysisSnapshot.fixture.js";
import { createAnalysisProfile } from "./analysisProfile.js";
import { createEvidence, evidenceSubject } from "./evidence.js";
import { unknownTarget } from "./targetIdentity.js";
import { createEvidenceBundle } from "./evidenceBundle.js";
@@ -504,51 +505,6 @@ describe("analysis snapshot canonical ordering and scope", () => {
});
});
it("does not select arbitrary provenance when multiple authenticated records match one replay entry", () => {
const { current, snapshot } = snapshotWithHistoricalEvidence();
const source = snapshot.evidence_bundle.records.find(
(record) => record.operation === "unprofiled_query",
)!;
const alternative = createEvidence(
ANALYSIS_SNAPSHOT_TARGET,
ANALYSIS_SNAPSHOT_PROVIDER,
{
operation: current.operation,
parameters: current.parameters,
result: current.normalized_result,
analysisProfile: ANALYSIS_SNAPSHOT_PROFILE,
evidenceLinks: [source.evidence_id],
},
);
expect(alternative.evidence_id).not.toBe(current.evidence_id);
const ambiguous = parseAnalysisSnapshot({
...snapshot,
evidence_bundle: createEvidenceBundle([
...snapshot.evidence_bundle.records,
alternative,
]),
});
expect(ambiguous.entries).toEqual(snapshot.entries);
const reversedInsertion = parseAnalysisSnapshot({
...ambiguous,
evidence_bundle: createEvidenceBundle(
[...ambiguous.evidence_bundle.records].reverse(),
),
});
expect(reversedInsertion).toEqual(ambiguous);
for (const candidate of [ambiguous, reversedInsertion])
expect(
snapshotEvidenceForQuery(candidate, {
target: ANALYSIS_SNAPSHOT_TARGET,
bindingProfile: ANALYSIS_SNAPSHOT_PROFILE,
operation: current.operation,
parameters: current.parameters,
provider: ANALYSIS_SNAPSHOT_PROVIDER,
evidenceProfile: ANALYSIS_SNAPSHOT_PROFILE,
}),
).toBeUndefined();
});
it("retains linked foreign Evidence without treating it as selected-target replay", () => {
const { current, snapshot } = snapshotWithHistoricalEvidence();
const foreignTarget = { ...ANALYSIS_SNAPSHOT_TARGET, sha256: "b".repeat(64) };
@@ -580,16 +536,37 @@ it("retains linked foreign Evidence without treating it as selected-target repla
...snapshot,
evidence_bundle: bundle,
});
const query = {
target: ANALYSIS_SNAPSHOT_TARGET,
bindingProfile: ANALYSIS_SNAPSHOT_PROFILE,
operation: linked.operation,
parameters: linked.parameters,
provider: ANALYSIS_SNAPSHOT_PROVIDER,
evidenceProfile: ANALYSIS_SNAPSHOT_PROFILE,
};
expect(snapshotEvidenceForQuery(admitted, query)).toEqual(linked);
const differentProfile = createAnalysisProfile(ANALYSIS_SNAPSHOT_PROVIDER, {
fixture: "other",
});
expect(
snapshotEvidenceForQuery(admitted, {
target: ANALYSIS_SNAPSHOT_TARGET,
bindingProfile: ANALYSIS_SNAPSHOT_PROFILE,
operation: linked.operation,
parameters: linked.parameters,
provider: ANALYSIS_SNAPSHOT_PROVIDER,
evidenceProfile: ANALYSIS_SNAPSHOT_PROFILE,
...query,
evidenceProfile: differentProfile,
}),
).toEqual(linked);
).toBeUndefined();
expect(
snapshotEvidenceForQuery(admitted, {
...query,
bindingProfile: differentProfile,
}),
).toBeUndefined();
expect(
snapshotEvidenceForQuery(admitted, {
...query,
operation: "unprofiled_query",
parameters: {},
}),
).toBeUndefined();
expect(snapshotMatchesTarget(admitted.target, foreignTarget)).toBe(false);
const unrelated = createEvidence(foreignTarget, ANALYSIS_SNAPSHOT_PROVIDER, {
operation: "unrelated",
@@ -672,30 +649,6 @@ it("binds raw Evidence only when its entry point matches the snapshot target", (
});
describe("analysis snapshot Evidence binding", () => {
it("finds only Evidence committed to the exact binding and profile", () => {
const { current, snapshot } = snapshotWithHistoricalEvidence();
expect(
snapshotEvidenceForQuery(snapshot, {
target: ANALYSIS_SNAPSHOT_TARGET,
bindingProfile: ANALYSIS_SNAPSHOT_PROFILE,
operation: "analyze_function",
parameters: { procedure: "main" },
provider: ANALYSIS_SNAPSHOT_PROVIDER,
evidenceProfile: ANALYSIS_SNAPSHOT_PROFILE,
}),
).toEqual(current);
expect(
snapshotEvidenceForQuery(snapshot, {
target: ANALYSIS_SNAPSHOT_TARGET,
bindingProfile: ANALYSIS_SNAPSHOT_PROFILE,
operation: "unprofiled_query",
parameters: {},
provider: ANALYSIS_SNAPSHOT_PROVIDER,
evidenceProfile: ANALYSIS_SNAPSHOT_PROFILE,
}),
).toBeUndefined();
});
it("rejects cached execution data that differs from its bundled Evidence", () => {
const target = snapshotTarget(ANALYSIS_SNAPSHOT_TARGET);
const binding = snapshotBinding(ANALYSIS_SNAPSHOT_PROFILE);
@@ -35,6 +35,7 @@ import {
import {
createEvidence,
evidenceSubject,
evidenceSubjectTarget,
parseEvidence,
type Evidence,
} from "../../../src/domain/evidence.js";
@@ -42,6 +43,7 @@ import { REA_WORKFLOW_PROVIDER } from "../../../src/domain/investigationProvider
import { workflowAnalysisProfile } from "../../../src/application/WorkflowAnalysisProfile.js";
import {
analysisQueryId,
evidenceMatchesEntry,
createAnalysisSnapshotWorkflowEntry,
type AnalysisSnapshot,
} from "../../../src/domain/analysisSnapshot.js";
@@ -363,7 +365,7 @@ describe("one-shot Ghidra pseudocode snapshots", () => {
});
it.skipIf(process.platform === "win32")(
"replays the exact selected-target answer without opening another provider",
"replays the exact selected-target answer and reruns when provenance is ambiguous",
async () => {
const fixture = await createSnapshotReplayFixture("rea-one-shot-code-");
const profile = createAnalysisProfile(
@@ -376,24 +378,18 @@ describe("one-shot Ghidra pseudocode snapshots", () => {
supportedOperations: ["procedure_pseudo_code"],
}),
);
const options = { snapshotPath: fixture.snapshotPath };
const parameters = { procedure: "fixture" };
const first = await runDirectAnalysis(
dependencies,
fixture.path,
"procedure_pseudo_code",
parameters,
options,
);
if (!first.ok) throw new Error(JSON.stringify(first.error));
const second = await runDirectAnalysis(
dependencies,
fixture.path,
"procedure_pseudo_code",
parameters,
options,
);
expect(second).toEqual(first);
const query = (selected = dependencies) =>
runDirectAnalysis(
selected,
fixture.path,
"procedure_pseudo_code",
parameters,
{ snapshotPath: fixture.snapshotPath },
);
const first = await query();
if (!first.ok) throw first.error;
expect(await query()).toEqual(first);
expect(fixture.starts).toHaveLength(1);
const saved = await readAnalysisSnapshot(fixture.snapshotPath);
if (!saved.ok) throw saved.error;
@@ -406,6 +402,61 @@ describe("one-shot Ghidra pseudocode snapshots", () => {
({ operation }) => operation === "procedure_pseudo_code",
),
).toEqual([first.value]);
const parent = first.value;
const target = evidenceSubjectTarget(parent.subject);
if (target === undefined)
throw new Error("Expected a file Evidence subject");
const alternative = createEvidence(target, parent.provider, {
operation: parent.operation,
parameters: parent.parameters,
result: parent.normalized_result,
rawResult: parent.raw_result,
analysisProfile: profile,
limitations: parent.limitations,
...(parent.relationship_coverage === null
? {}
: { relationshipCoverage: parent.relationship_coverage }),
locations: parent.locations,
evidenceLinks: [parent.evidence_id],
});
expect(alternative.evidence_id).not.toBe(parent.evidence_id);
const entry = saved.value.entries[0]!;
expect(evidenceMatchesEntry(alternative, entry, saved.value)).toBe(true);
const written = await withArtifactResourceScope((resources) =>
writeAnalysisSnapshot(
{
...saved.value,
evidence_bundle: createEvidenceBundle([
...saved.value.evidence_bundle.records,
alternative,
]),
},
fixture.snapshotPath,
true,
resources,
),
);
if (!written.ok) throw written.error;
const fresh = await query(
fixture.dependencies(
fixture.createProvider({
profile,
supportedOperations: ["procedure_pseudo_code"],
limitations: ["Observed after ambiguous snapshot"],
}),
),
);
if (!fresh.ok) throw fresh.error;
expect(fresh.value.limitations).toContain(
"Observed after ambiguous snapshot",
);
expect(
fixture.calls.filter(
(operation) => operation === "procedure_pseudo_code",
),
).toHaveLength(2);
expect(fixture.starts).toHaveLength(2);
},
);