fix(process): preserve unrelated trace comparison uncertainty (#4178)

Only apply a different trace verdict to its attributed dimensions. Preserve
unrelated unknown coverage and independently observed additions/removals in
the shared CLI/MCP comparison owner.

Fixes #3931

Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com>
Co-authored-by: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com>
This commit is contained in:
authored and GitHub committed 2026-10-16 00:56:55 +08:00
1 parent 62bf09809e
commit 7b86d5caf6
4 files changed
+159 -7

No files matched your search

+3 -1
View File
@@ -425,7 +425,9 @@ results for unaffected dimensions and mark affected dimensions unknown; an
incomplete capture cannot locate the first divergence across all dimensions.
Trace assertions use coverage for the sources they select, so an assertion
about complete raw terminal output need not fail because rendered snapshots
were omitted. Captures missing `truncation_details` are rejected on import;
were omitted. A trace difference applies to the dimensions identified by its
diagnostic; unrelated dimensions retain their own comparison, including unknown
coverage. Captures missing `truncation_details` are rejected on import;
preserve them as historical files and recapture for current comparisons.
`limits.file_bytes` is already the cumulative whole-file hashing budget,
independent of `limits.output_bytes`, which bounds retained terminal content.
+3 -3
View File
@@ -330,7 +330,8 @@ const applyTraceVerdict = (
const sources = new Set(specification.events.map(({ source }) => source));
const covered = dimensionsForTraceSources(sources);
// A different verdict names the failing events and observations; attribute
// it only to their dimensions so equal observations keep their exact status.
// it only to their dimensions; unrelated comparisons keep their own status,
// including unknown coverage and independently observed additions/removals.
const diagnosticSources = new Set<ProcessObservationSource>([
...specification.events
.filter(({ id }) => trace.diagnostic?.event_ids.includes(id) === true)
@@ -350,8 +351,7 @@ const applyTraceVerdict = (
if (!covered.has(dimension)) return exact;
if (trace.verdict === "nonconforming") return exact;
if (trace.verdict === "different")
return traceShowsDifference &&
(exact !== "unchanged" || attributed.has(dimension))
return traceShowsDifference && attributed.has(dimension)
? "changed"
: exact;
if (
@@ -360,6 +360,50 @@ describe("unknown process trace evidence", () => {
});
describe("process trace dimension attribution", () => {
it.each([false, true])(
"preserves an independently observed terminal addition/removal (reversed: %s)",
(reversed) => {
const absent = capture({
frames: [],
process_samples: [],
event_journal: [],
});
const present = capture(values(["terminal", "process"]));
const specification = partialSpecification();
const result = compareProcessCaptures(
reversed ? present : absent,
reversed ? absent : present,
{
traceSpecification: {
...specification,
events: specification.events.map((event) =>
event.id === "ready"
? { ...event, cardinality: { kind: "optional" } }
: event,
),
language: {
kind: "partial_order",
happens_before: [],
not_before: [],
unordered_groups: [{ events: ["ready", "worker"] }],
prefix: [],
suffix: [],
},
},
},
);
expect(result).toMatchObject({
terminal: reversed ? "removed" : "added",
process: "changed",
status: "changed",
trace: {
verdict: "different",
diagnostic: { event_ids: ["worker"], locations: [] },
},
});
},
);
it("applies passing filesystem trace timestamp omissions to exact comparison", () => {
const base = capture(values(["terminal", "process"]));
const left = base;
@@ -8,6 +8,17 @@ import { expect, it, onTestFinished } from "vitest";
import { compareProcessEvidenceFiles } from "../../../src/application/process/ProcessCli.js";
import { PROCESS_PROVIDER } from "../../../src/domain/process/processEvidenceProvider.js";
import { EMPTY_PROCESS_CAPTURE_EXAMPLE } from "../../../src/domain/process/processCaptureExample.js";
import {
capture,
partialSpecification,
processStarted,
terminal,
} from "../../../src/domain/process/processTraceComparison.fixture.js";
import {
parseProcessCapture,
type ProcessCapture,
} from "../../../src/domain/process/processCaptureParsing.js";
import type { ProcessTraceSpecification } from "../../../src/domain/process/processTraceSpecification.js";
import {
createEvidence,
parseEvidence,
@@ -23,15 +34,18 @@ const captureEvidence = (
side: "left" | "right",
comparisonContract: Readonly<Record<string, string>> = {},
provider: EvidenceProvider = PROCESS_PROVIDER,
observation: ProcessCapture = parseProcessCapture(
EMPTY_PROCESS_CAPTURE_EXAMPLE,
),
) =>
createEvidence(undefined, provider, {
predicateType: "rea.process-capture",
operation: "capture_process_scenario",
parameters: { side },
result: jsonValueSchema.parse({
...EMPTY_PROCESS_CAPTURE_EXAMPLE,
...observation,
manifest: {
...EMPTY_PROCESS_CAPTURE_EXAMPLE.manifest,
...observation.manifest,
comparison_contract: comparisonContract,
comparison_contract_sha256: digestProcessCommitment(comparisonContract),
},
@@ -64,6 +78,7 @@ const connect = async () => {
const compareViaCli = async (
left: ReturnType<typeof captureEvidence>,
right: ReturnType<typeof captureEvidence>,
specification?: ProcessTraceSpecification,
) => {
const root = await createTestTempDirectory("rea-process-provider-");
const leftPath = join(root, "left.json");
@@ -72,9 +87,100 @@ const compareViaCli = async (
writeFile(leftPath, JSON.stringify(left)),
writeFile(rightPath, JSON.stringify(right)),
]);
return compareProcessEvidenceFiles(leftPath, rightPath);
const tracePath = join(root, "trace.json");
if (specification !== undefined)
await writeFile(tracePath, JSON.stringify(specification));
return compareProcessEvidenceFiles(
leftPath,
rightPath,
specification === undefined ? undefined : tracePath,
);
};
it.each(
(["left", "right"] as const).flatMap((missingWorker) =>
(["left", "right", "both", "neither"] as const).map((omittedRendered) => ({
missingWorker,
omittedRendered,
})),
),
)(
"preserves terminal coverage when $missingWorker lacks a worker and $omittedRendered omits rendered output",
async ({ missingWorker, omittedRendered }) => {
const observation = (side: "left" | "right") => {
const hasWorker = side !== missingWorker;
const complete = capture({
frames: [terminal],
process_samples: hasWorker ? [processStarted] : [],
event_journal: [
{ collection: "frames", index: 0 },
...(hasWorker
? [{ collection: "process_samples" as const, index: 0 }]
: []),
],
});
const omitted = omittedRendered === "both" || omittedRendered === side;
return captureEvidence(
side,
{},
PROCESS_PROVIDER,
omitted
? parseProcessCapture({
...complete,
truncated: true,
truncation_details: {
...complete.truncation_details,
rendered_terminal: {
budget_bytes: 5,
observed_bytes: 0,
retained_bytes: 0,
observed_frames: 1,
retained_frames: 0,
unknown_size_frames: 1,
},
},
})
: complete,
);
};
const left = observation("left");
const right = observation("right");
const terminalStatus =
omittedRendered === "neither" ? "unchanged" : "unknown";
const plain = parseEvidence(await compareViaCli(left, right));
expect(plain.normalized_result).toMatchObject({
terminal: terminalStatus,
process: missingWorker === "left" ? "added" : "removed",
status: "changed",
});
const specification = partialSpecification();
const cli = parseEvidence(await compareViaCli(left, right, specification));
const client = await connect();
const response = await client.callTool({
name: "compare_process_captures",
arguments: { left, right, trace_spec: specification },
});
expect(response.isError).not.toBe(true);
expect(parseEvidence(response.structuredContent)).toEqual(cli);
expect(cli.evidence_links).toEqual([left.evidence_id, right.evidence_id]);
expect(cli.normalized_result).toMatchObject({
terminal: terminalStatus,
process: "changed",
status: "changed",
trace: {
verdict: "different",
diagnostic: {
kind: "cardinality",
side: missingWorker,
event_ids: ["worker"],
locations: [],
},
},
});
},
);
it.each(
(["left", "right"] as const).flatMap((side) =>
[