diff --git a/docs/process-capture.md b/docs/process-capture.md index 2bf0d06da7..17e69f3119 100644 --- a/docs/process-capture.md +++ b/docs/process-capture.md @@ -425,7 +425,9 @@ results for unaffected dimensions and mark affected dimensions unknown; an incomplete capture cannot locate the first divergence across all dimensions. Trace assertions use coverage for the sources they select, so an assertion about complete raw terminal output need not fail because rendered snapshots -were omitted. Captures missing `truncation_details` are rejected on import; +were omitted. A trace difference applies to the dimensions identified by its +diagnostic; unrelated dimensions retain their own comparison, including unknown +coverage. Captures missing `truncation_details` are rejected on import; preserve them as historical files and recapture for current comparisons. `limits.file_bytes` is already the cumulative whole-file hashing budget, independent of `limits.output_bytes`, which bounds retained terminal content. diff --git a/src/domain/process/processComparison.ts b/src/domain/process/processComparison.ts index 06e1340616..71f3b9952d 100644 --- a/src/domain/process/processComparison.ts +++ b/src/domain/process/processComparison.ts @@ -330,7 +330,8 @@ const applyTraceVerdict = ( const sources = new Set(specification.events.map(({ source }) => source)); const covered = dimensionsForTraceSources(sources); // A different verdict names the failing events and observations; attribute - // it only to their dimensions so equal observations keep their exact status. + // it only to their dimensions; unrelated comparisons keep their own status, + // including unknown coverage and independently observed additions/removals. const diagnosticSources = new Set([ ...specification.events .filter(({ id }) => trace.diagnostic?.event_ids.includes(id) === true) @@ -350,8 +351,7 @@ const applyTraceVerdict = ( if (!covered.has(dimension)) return exact; if (trace.verdict === "nonconforming") return exact; if (trace.verdict === "different") - return traceShowsDifference && - (exact !== "unchanged" || attributed.has(dimension)) + return traceShowsDifference && attributed.has(dimension) ? "changed" : exact; if ( diff --git a/src/domain/process/processTraceComparison.test.ts b/src/domain/process/processTraceComparison.test.ts index 5966e4ef10..da3767aedf 100644 --- a/src/domain/process/processTraceComparison.test.ts +++ b/src/domain/process/processTraceComparison.test.ts @@ -360,6 +360,50 @@ describe("unknown process trace evidence", () => { }); describe("process trace dimension attribution", () => { + it.each([false, true])( + "preserves an independently observed terminal addition/removal (reversed: %s)", + (reversed) => { + const absent = capture({ + frames: [], + process_samples: [], + event_journal: [], + }); + const present = capture(values(["terminal", "process"])); + const specification = partialSpecification(); + const result = compareProcessCaptures( + reversed ? present : absent, + reversed ? absent : present, + { + traceSpecification: { + ...specification, + events: specification.events.map((event) => + event.id === "ready" + ? { ...event, cardinality: { kind: "optional" } } + : event, + ), + language: { + kind: "partial_order", + happens_before: [], + not_before: [], + unordered_groups: [{ events: ["ready", "worker"] }], + prefix: [], + suffix: [], + }, + }, + }, + ); + expect(result).toMatchObject({ + terminal: reversed ? "removed" : "added", + process: "changed", + status: "changed", + trace: { + verdict: "different", + diagnostic: { event_ids: ["worker"], locations: [] }, + }, + }); + }, + ); + it("applies passing filesystem trace timestamp omissions to exact comparison", () => { const base = capture(values(["terminal", "process"])); const left = base; diff --git a/tests/boundary/mcp/processComparisonDiagnosticsMcp.test.ts b/tests/boundary/mcp/processComparisonDiagnosticsMcp.test.ts index bda4672def..a67373e0c5 100644 --- a/tests/boundary/mcp/processComparisonDiagnosticsMcp.test.ts +++ b/tests/boundary/mcp/processComparisonDiagnosticsMcp.test.ts @@ -8,6 +8,17 @@ import { expect, it, onTestFinished } from "vitest"; import { compareProcessEvidenceFiles } from "../../../src/application/process/ProcessCli.js"; import { PROCESS_PROVIDER } from "../../../src/domain/process/processEvidenceProvider.js"; import { EMPTY_PROCESS_CAPTURE_EXAMPLE } from "../../../src/domain/process/processCaptureExample.js"; +import { + capture, + partialSpecification, + processStarted, + terminal, +} from "../../../src/domain/process/processTraceComparison.fixture.js"; +import { + parseProcessCapture, + type ProcessCapture, +} from "../../../src/domain/process/processCaptureParsing.js"; +import type { ProcessTraceSpecification } from "../../../src/domain/process/processTraceSpecification.js"; import { createEvidence, parseEvidence, @@ -23,15 +34,18 @@ const captureEvidence = ( side: "left" | "right", comparisonContract: Readonly> = {}, provider: EvidenceProvider = PROCESS_PROVIDER, + observation: ProcessCapture = parseProcessCapture( + EMPTY_PROCESS_CAPTURE_EXAMPLE, + ), ) => createEvidence(undefined, provider, { predicateType: "rea.process-capture", operation: "capture_process_scenario", parameters: { side }, result: jsonValueSchema.parse({ - ...EMPTY_PROCESS_CAPTURE_EXAMPLE, + ...observation, manifest: { - ...EMPTY_PROCESS_CAPTURE_EXAMPLE.manifest, + ...observation.manifest, comparison_contract: comparisonContract, comparison_contract_sha256: digestProcessCommitment(comparisonContract), }, @@ -64,6 +78,7 @@ const connect = async () => { const compareViaCli = async ( left: ReturnType, right: ReturnType, + specification?: ProcessTraceSpecification, ) => { const root = await createTestTempDirectory("rea-process-provider-"); const leftPath = join(root, "left.json"); @@ -72,9 +87,100 @@ const compareViaCli = async ( writeFile(leftPath, JSON.stringify(left)), writeFile(rightPath, JSON.stringify(right)), ]); - return compareProcessEvidenceFiles(leftPath, rightPath); + const tracePath = join(root, "trace.json"); + if (specification !== undefined) + await writeFile(tracePath, JSON.stringify(specification)); + return compareProcessEvidenceFiles( + leftPath, + rightPath, + specification === undefined ? undefined : tracePath, + ); }; +it.each( + (["left", "right"] as const).flatMap((missingWorker) => + (["left", "right", "both", "neither"] as const).map((omittedRendered) => ({ + missingWorker, + omittedRendered, + })), + ), +)( + "preserves terminal coverage when $missingWorker lacks a worker and $omittedRendered omits rendered output", + async ({ missingWorker, omittedRendered }) => { + const observation = (side: "left" | "right") => { + const hasWorker = side !== missingWorker; + const complete = capture({ + frames: [terminal], + process_samples: hasWorker ? [processStarted] : [], + event_journal: [ + { collection: "frames", index: 0 }, + ...(hasWorker + ? [{ collection: "process_samples" as const, index: 0 }] + : []), + ], + }); + const omitted = omittedRendered === "both" || omittedRendered === side; + return captureEvidence( + side, + {}, + PROCESS_PROVIDER, + omitted + ? parseProcessCapture({ + ...complete, + truncated: true, + truncation_details: { + ...complete.truncation_details, + rendered_terminal: { + budget_bytes: 5, + observed_bytes: 0, + retained_bytes: 0, + observed_frames: 1, + retained_frames: 0, + unknown_size_frames: 1, + }, + }, + }) + : complete, + ); + }; + const left = observation("left"); + const right = observation("right"); + const terminalStatus = + omittedRendered === "neither" ? "unchanged" : "unknown"; + const plain = parseEvidence(await compareViaCli(left, right)); + expect(plain.normalized_result).toMatchObject({ + terminal: terminalStatus, + process: missingWorker === "left" ? "added" : "removed", + status: "changed", + }); + + const specification = partialSpecification(); + const cli = parseEvidence(await compareViaCli(left, right, specification)); + const client = await connect(); + const response = await client.callTool({ + name: "compare_process_captures", + arguments: { left, right, trace_spec: specification }, + }); + expect(response.isError).not.toBe(true); + expect(parseEvidence(response.structuredContent)).toEqual(cli); + expect(cli.evidence_links).toEqual([left.evidence_id, right.evidence_id]); + expect(cli.normalized_result).toMatchObject({ + terminal: terminalStatus, + process: "changed", + status: "changed", + trace: { + verdict: "different", + diagnostic: { + kind: "cardinality", + side: missingWorker, + event_ids: ["worker"], + locations: [], + }, + }, + }); + }, +); + it.each( (["left", "right"] as const).flatMap((side) => [