From 34da2cce1f75a3fff8532d9b7cc6d8082c7cbcb6 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 21 Nov 2026 03:51:24 -0500 Subject: [PATCH] test(snapshots): verify live source metadata refresh (#13048) --- .../binary/AnalysisSnapshotCache.test.ts | 51 ++----- .../mcp/workflowSnapshotReplayMcp.test.ts | 127 +++++++++++------- 2 files changed, 88 insertions(+), 90 deletions(-) diff --git a/src/application/binary/AnalysisSnapshotCache.test.ts b/src/application/binary/AnalysisSnapshotCache.test.ts index 442231b226..e6babe8d1f 100644 --- a/src/application/binary/AnalysisSnapshotCache.test.ts +++ b/src/application/binary/AnalysisSnapshotCache.test.ts @@ -252,11 +252,6 @@ it.each(["provider", "workflow"] as const)( name: "display-only-name", }), }; - const nameImport = ledger.import(createEvidenceBundle([nameOnly])); - if (!nameImport.ok) throw nameImport.error; - cache.refreshEvidenceMetadata(nameImport.value.replacements); - expect(cache.entries()).toEqual(before.value.entries); - expect(cache.workflowEntries()).toEqual(before.value.workflow_entries); // A different observation for this exact query is not the entry's dependency. const alternative = createEvidence(target, provider, { operation, @@ -277,19 +272,17 @@ it.each(["provider", "workflow"] as const)( }), }; }; - const foreign = ledger.import( - createEvidenceBundle([relocate(alternative)]), + // Name-only changes, a different result for this query, and source + // changes do not rewrite the cached entry's own subject. + const unrelated = ledger.import( + createEvidenceBundle([ + nameOnly, + relocate(alternative), + ...dependencies.map(relocate), + ]), ); - if (!foreign.ok) throw foreign.error; - cache.refreshEvidenceMetadata(foreign.value.replacements); - expect(cache.entries()).toEqual(before.value.entries); - expect(cache.workflowEntries()).toEqual(before.value.workflow_entries); - // A result may cite several observations; updating their envelopes does not change its payload. - const dependentImport = ledger.import( - createEvidenceBundle(dependencies.map(relocate)), - ); - if (!dependentImport.ok) throw dependentImport.error; - cache.refreshEvidenceMetadata(dependentImport.value.replacements); + if (!unrelated.ok) throw unrelated.error; + cache.refreshEvidenceMetadata(unrelated.value.replacements); expect(cache.entries()).toEqual(before.value.entries); expect(cache.workflowEntries()).toEqual(before.value.workflow_entries); const relocated = relocate(evidence); @@ -1527,25 +1520,9 @@ it("replays source-bearing entries only when their complete retained closure exi .lookup(target, profile, "procedure_info", parameters) ?.sourceEvidence?.map((record) => record.evidence_id), ).toEqual([source.evidence_id, parent.evidence_id]); - if (source.subject.identity.kind !== "file") - throw new Error("Expected source subject"); const ledger = new EvidenceLedger(); for (const record of [parent, source, leaf]) expect(ledger.record(record).ok).toBe(true); - const movedSource = { - ...source, - subject: displayedEvidenceSubject(source.subject, { - local_path: "/relocated/source", - name: "source", - }), - }; - const replacement = ledger.import(createEvidenceBundle([movedSource])); - if (!replacement.ok) throw replacement.error; - cache.refreshEvidenceMetadata(replacement.value.replacements); - expect( - cache.lookup(target, profile, "procedure_info", parameters) - ?.sourceEvidence?.[0]?.subject?.local_path, - ).toBe("/relocated/source"); const exported = cache.export(target, profile, ledger.export()); if (!exported.ok) throw exported.error; const imported = new AnalysisSnapshotCache(); @@ -1555,18 +1532,14 @@ it("replays source-bearing entries only when their complete retained closure exi .lookup(target, profile, "procedure_info", parameters) ?.sourceEvidence?.map((record) => record.evidence_id), ).toEqual([source.evidence_id, parent.evidence_id]); - expect( - imported.lookup(target, profile, "procedure_info", parameters) - ?.sourceEvidence?.[0]?.subject?.local_path, - ).toBe("/relocated/source"); const pruned = parseAnalysisSnapshot({ ...exported.value, - evidence_bundle: createEvidenceBundle([parent, movedSource]), + evidence_bundle: createEvidenceBundle([parent, source]), }); const incomplete = new AnalysisSnapshotCache(); incomplete.stage(pruned); const partial = new Map( - [parent, movedSource].map((record) => [record.evidence_id, record]), + [parent, source].map((record) => [record.evidence_id, record]), ); expect( incomplete.lookup(target, profile, "procedure_info", parameters, (id) => diff --git a/tests/boundary/mcp/workflowSnapshotReplayMcp.test.ts b/tests/boundary/mcp/workflowSnapshotReplayMcp.test.ts index 19a8d5ae66..62b90b5963 100644 --- a/tests/boundary/mcp/workflowSnapshotReplayMcp.test.ts +++ b/tests/boundary/mcp/workflowSnapshotReplayMcp.test.ts @@ -23,7 +23,8 @@ import { createTestBinarySession } from "../../fixtures/binarySession.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { createAnalysisProfile } from "../../../src/domain/analysisProfile.js"; -import { createEvidence } from "../../../src/domain/evidence.js"; +import { createEvidence, type Evidence } from "../../../src/domain/evidence.js"; +import { displayedEvidenceSubject } from "../../fixtures/evidenceSubject.js"; import { readAnalysisSnapshot } from "../../../src/application/binary/AnalysisSnapshotFiles.js"; import { ok } from "../../../src/domain/result.js"; import { @@ -571,6 +572,21 @@ describe("MCP investigation history snapshot replay", () => { ); }); +const expectSourceReplay = ( + evidence: Evidence, + parentId: string, + sourceId: string, +) => { + expect(evidence.evidence_links.toSorted()).toEqual( + [parentId, sourceId].toSorted(), + ); + expect(evidence.limitations).toEqual( + expect.arrayContaining([ + expect.stringContaining("did not re-run the provider"), + ]), + ); +}; + it("replays a source-bearing observation through an exported MCP snapshot", async () => { const { targetPaths, snapshotPath } = await createWorkflowFiles( "rea-mcp-source-replay-", @@ -604,37 +620,66 @@ it("replays a source-bearing observation through an exported MCP snapshot", asyn }; }, }; - const { mcp } = await connectWorkflowMcp( - createWorkflowSession(provider), - "source-replay", - ); + const session = createWorkflowSession(provider); + const { mcp } = await connectWorkflowMcp(session, "source-replay"); await openWorkflowTarget(mcp, targetPath); - const read = (client: Client) => - client.callTool({ - name: "list_strings", - arguments: { document: "fixture" }, - }); - const first = await read(mcp); - const firstEvidence = toolContract("list_strings").outputSchema.parse( - first.structuredContent, - ); + const read = async (client: Client) => + toolContract("list_strings").outputSchema.parse( + ( + await client.callTool({ + name: "list_strings", + arguments: { document: "fixture" }, + }) + ).structuredContent, + ); + const firstEvidence = await read(mcp); expect(firstEvidence.evidence_links).toHaveLength(1); - const second = await read(mcp); - const secondEvidence = toolContract("list_strings").outputSchema.parse( - second.structuredContent, + const secondEvidence = await read(mcp); + const sourceId = firstEvidence.evidence_links[0]!; + expectSourceReplay(secondEvidence, firstEvidence.evidence_id, sourceId); + const original = await session.execute("list_strings", { + document: "fixture", + }); + if (!original.ok) throw original.error; + const source = original.value.sourceEvidence?.find( + (record) => record.evidence_id === sourceId, ); - expect(secondEvidence.evidence_links).toEqual( - expect.arrayContaining([ - ...firstEvidence.evidence_links, - firstEvidence.evidence_id, - ]), + if (!source) throw new Error("Expected the cached source"); + const relocatedPath = join(dirname(targetPath), "relocated-source.hop"); + const sourcePath = join(dirname(snapshotPath), "display-update.json"); + await writeFile( + sourcePath, + JSON.stringify( + createEvidenceBundle([ + { + ...source, + subject: displayedEvidenceSubject(source.subject, { + local_path: relocatedPath, + name: "relocated-source.hop", + }), + }, + ]), + ), ); - expect(secondEvidence.evidence_links).toHaveLength(2); - expect(secondEvidence.limitations).toEqual( - expect.arrayContaining([ - expect.stringContaining("did not re-run the provider"), - ]), + expect( + ( + await mcp.callTool({ + name: "import_evidence_bundle", + arguments: { path: sourcePath }, + }) + ).isError, + ).not.toBe(true); + const refreshed = await session.execute("list_strings", { + document: "fixture", + }); + if (!refreshed.ok) throw refreshed.error; + expect(refreshed.value.sourceEvidence).toContainEqual( + expect.objectContaining({ + evidence_id: sourceId, + subject: expect.objectContaining({ local_path: relocatedPath }), + }), ); + expect(source.subject).toMatchObject({ local_path: targetPath }); expect( calls.filter((operation) => operation === "list_strings"), ).toHaveLength(1); @@ -658,21 +703,8 @@ it("replays a source-bearing observation through an exported MCP snapshot", asyn }) ).isError, ).not.toBe(true); - const replayed = toolContract("list_strings").outputSchema.parse( - (await read(reopened)).structuredContent, - ); - expect(replayed.evidence_links).toEqual( - expect.arrayContaining([ - ...firstEvidence.evidence_links, - firstEvidence.evidence_id, - ]), - ); - expect(replayed.evidence_links).toHaveLength(2); - expect(replayed.limitations).toEqual( - expect.arrayContaining([ - expect.stringContaining("did not re-run the provider"), - ]), - ); + const replayed = await read(reopened); + expectSourceReplay(replayed, firstEvidence.evidence_id, sourceId); expect( calls.filter((operation) => operation === "list_strings"), ).toHaveLength(1); @@ -693,6 +725,7 @@ it("replays a source-bearing observation through an exported MCP snapshot", asyn { evidence_id: firstEvidence.evidence_links[0], normalized_result: "historical source", + subject: { local_path: relocatedPath }, }, ], }); @@ -760,15 +793,7 @@ const expectRestoredSourceReplay = async ( record.evidence_links.includes(source.evidence_id), ); if (parent === undefined) throw new Error("Expected original observation"); - expect(replayed.evidence_links).toEqual( - expect.arrayContaining([source.evidence_id, parent.evidence_id]), - ); - expect(replayed.evidence_links).toHaveLength(2); - expect(replayed.limitations).toEqual( - expect.arrayContaining([ - expect.stringContaining("did not re-run the provider"), - ]), - ); + expectSourceReplay(replayed, parent.evidence_id, source.evidence_id); expect( calls.filter((operation) => operation === "list_strings"), ).toHaveLength(1);