mirror of
https://github.com/morluto/rea.git
synced 2026-10-26 17:35:11 +00:00
fix(process): preserve unrelated trace comparison uncertainty (#4178)
Only apply a different trace verdict to its attributed dimensions. Preserve unrelated unknown coverage and independently observed additions/removals in the shared CLI/MCP comparison owner. Fixes #3931 Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Co-authored-by: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com>
This commit is contained in:
4 files changed
+159
-7
No files matched your search
@@ -425,7 +425,9 @@ results for unaffected dimensions and mark affected dimensions unknown; an
|
||||
incomplete capture cannot locate the first divergence across all dimensions.
|
||||
Trace assertions use coverage for the sources they select, so an assertion
|
||||
about complete raw terminal output need not fail because rendered snapshots
|
||||
were omitted. Captures missing `truncation_details` are rejected on import;
|
||||
were omitted. A trace difference applies to the dimensions identified by its
|
||||
diagnostic; unrelated dimensions retain their own comparison, including unknown
|
||||
coverage. Captures missing `truncation_details` are rejected on import;
|
||||
preserve them as historical files and recapture for current comparisons.
|
||||
`limits.file_bytes` is already the cumulative whole-file hashing budget,
|
||||
independent of `limits.output_bytes`, which bounds retained terminal content.
|
||||
|
||||
@@ -330,7 +330,8 @@ const applyTraceVerdict = (
|
||||
const sources = new Set(specification.events.map(({ source }) => source));
|
||||
const covered = dimensionsForTraceSources(sources);
|
||||
// A different verdict names the failing events and observations; attribute
|
||||
// it only to their dimensions so equal observations keep their exact status.
|
||||
// it only to their dimensions; unrelated comparisons keep their own status,
|
||||
// including unknown coverage and independently observed additions/removals.
|
||||
const diagnosticSources = new Set<ProcessObservationSource>([
|
||||
...specification.events
|
||||
.filter(({ id }) => trace.diagnostic?.event_ids.includes(id) === true)
|
||||
@@ -350,8 +351,7 @@ const applyTraceVerdict = (
|
||||
if (!covered.has(dimension)) return exact;
|
||||
if (trace.verdict === "nonconforming") return exact;
|
||||
if (trace.verdict === "different")
|
||||
return traceShowsDifference &&
|
||||
(exact !== "unchanged" || attributed.has(dimension))
|
||||
return traceShowsDifference && attributed.has(dimension)
|
||||
? "changed"
|
||||
: exact;
|
||||
if (
|
||||
|
||||
@@ -360,6 +360,50 @@ describe("unknown process trace evidence", () => {
|
||||
});
|
||||
|
||||
describe("process trace dimension attribution", () => {
|
||||
it.each([false, true])(
|
||||
"preserves an independently observed terminal addition/removal (reversed: %s)",
|
||||
(reversed) => {
|
||||
const absent = capture({
|
||||
frames: [],
|
||||
process_samples: [],
|
||||
event_journal: [],
|
||||
});
|
||||
const present = capture(values(["terminal", "process"]));
|
||||
const specification = partialSpecification();
|
||||
const result = compareProcessCaptures(
|
||||
reversed ? present : absent,
|
||||
reversed ? absent : present,
|
||||
{
|
||||
traceSpecification: {
|
||||
...specification,
|
||||
events: specification.events.map((event) =>
|
||||
event.id === "ready"
|
||||
? { ...event, cardinality: { kind: "optional" } }
|
||||
: event,
|
||||
),
|
||||
language: {
|
||||
kind: "partial_order",
|
||||
happens_before: [],
|
||||
not_before: [],
|
||||
unordered_groups: [{ events: ["ready", "worker"] }],
|
||||
prefix: [],
|
||||
suffix: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
);
|
||||
expect(result).toMatchObject({
|
||||
terminal: reversed ? "removed" : "added",
|
||||
process: "changed",
|
||||
status: "changed",
|
||||
trace: {
|
||||
verdict: "different",
|
||||
diagnostic: { event_ids: ["worker"], locations: [] },
|
||||
},
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("applies passing filesystem trace timestamp omissions to exact comparison", () => {
|
||||
const base = capture(values(["terminal", "process"]));
|
||||
const left = base;
|
||||
|
||||
@@ -8,6 +8,17 @@ import { expect, it, onTestFinished } from "vitest";
|
||||
import { compareProcessEvidenceFiles } from "../../../src/application/process/ProcessCli.js";
|
||||
import { PROCESS_PROVIDER } from "../../../src/domain/process/processEvidenceProvider.js";
|
||||
import { EMPTY_PROCESS_CAPTURE_EXAMPLE } from "../../../src/domain/process/processCaptureExample.js";
|
||||
import {
|
||||
capture,
|
||||
partialSpecification,
|
||||
processStarted,
|
||||
terminal,
|
||||
} from "../../../src/domain/process/processTraceComparison.fixture.js";
|
||||
import {
|
||||
parseProcessCapture,
|
||||
type ProcessCapture,
|
||||
} from "../../../src/domain/process/processCaptureParsing.js";
|
||||
import type { ProcessTraceSpecification } from "../../../src/domain/process/processTraceSpecification.js";
|
||||
import {
|
||||
createEvidence,
|
||||
parseEvidence,
|
||||
@@ -23,15 +34,18 @@ const captureEvidence = (
|
||||
side: "left" | "right",
|
||||
comparisonContract: Readonly<Record<string, string>> = {},
|
||||
provider: EvidenceProvider = PROCESS_PROVIDER,
|
||||
observation: ProcessCapture = parseProcessCapture(
|
||||
EMPTY_PROCESS_CAPTURE_EXAMPLE,
|
||||
),
|
||||
) =>
|
||||
createEvidence(undefined, provider, {
|
||||
predicateType: "rea.process-capture",
|
||||
operation: "capture_process_scenario",
|
||||
parameters: { side },
|
||||
result: jsonValueSchema.parse({
|
||||
...EMPTY_PROCESS_CAPTURE_EXAMPLE,
|
||||
...observation,
|
||||
manifest: {
|
||||
...EMPTY_PROCESS_CAPTURE_EXAMPLE.manifest,
|
||||
...observation.manifest,
|
||||
comparison_contract: comparisonContract,
|
||||
comparison_contract_sha256: digestProcessCommitment(comparisonContract),
|
||||
},
|
||||
@@ -64,6 +78,7 @@ const connect = async () => {
|
||||
const compareViaCli = async (
|
||||
left: ReturnType<typeof captureEvidence>,
|
||||
right: ReturnType<typeof captureEvidence>,
|
||||
specification?: ProcessTraceSpecification,
|
||||
) => {
|
||||
const root = await createTestTempDirectory("rea-process-provider-");
|
||||
const leftPath = join(root, "left.json");
|
||||
@@ -72,9 +87,100 @@ const compareViaCli = async (
|
||||
writeFile(leftPath, JSON.stringify(left)),
|
||||
writeFile(rightPath, JSON.stringify(right)),
|
||||
]);
|
||||
return compareProcessEvidenceFiles(leftPath, rightPath);
|
||||
const tracePath = join(root, "trace.json");
|
||||
if (specification !== undefined)
|
||||
await writeFile(tracePath, JSON.stringify(specification));
|
||||
return compareProcessEvidenceFiles(
|
||||
leftPath,
|
||||
rightPath,
|
||||
specification === undefined ? undefined : tracePath,
|
||||
);
|
||||
};
|
||||
|
||||
it.each(
|
||||
(["left", "right"] as const).flatMap((missingWorker) =>
|
||||
(["left", "right", "both", "neither"] as const).map((omittedRendered) => ({
|
||||
missingWorker,
|
||||
omittedRendered,
|
||||
})),
|
||||
),
|
||||
)(
|
||||
"preserves terminal coverage when $missingWorker lacks a worker and $omittedRendered omits rendered output",
|
||||
async ({ missingWorker, omittedRendered }) => {
|
||||
const observation = (side: "left" | "right") => {
|
||||
const hasWorker = side !== missingWorker;
|
||||
const complete = capture({
|
||||
frames: [terminal],
|
||||
process_samples: hasWorker ? [processStarted] : [],
|
||||
event_journal: [
|
||||
{ collection: "frames", index: 0 },
|
||||
...(hasWorker
|
||||
? [{ collection: "process_samples" as const, index: 0 }]
|
||||
: []),
|
||||
],
|
||||
});
|
||||
const omitted = omittedRendered === "both" || omittedRendered === side;
|
||||
return captureEvidence(
|
||||
side,
|
||||
{},
|
||||
PROCESS_PROVIDER,
|
||||
omitted
|
||||
? parseProcessCapture({
|
||||
...complete,
|
||||
truncated: true,
|
||||
truncation_details: {
|
||||
...complete.truncation_details,
|
||||
rendered_terminal: {
|
||||
budget_bytes: 5,
|
||||
observed_bytes: 0,
|
||||
retained_bytes: 0,
|
||||
observed_frames: 1,
|
||||
retained_frames: 0,
|
||||
unknown_size_frames: 1,
|
||||
},
|
||||
},
|
||||
})
|
||||
: complete,
|
||||
);
|
||||
};
|
||||
const left = observation("left");
|
||||
const right = observation("right");
|
||||
const terminalStatus =
|
||||
omittedRendered === "neither" ? "unchanged" : "unknown";
|
||||
const plain = parseEvidence(await compareViaCli(left, right));
|
||||
expect(plain.normalized_result).toMatchObject({
|
||||
terminal: terminalStatus,
|
||||
process: missingWorker === "left" ? "added" : "removed",
|
||||
status: "changed",
|
||||
});
|
||||
|
||||
const specification = partialSpecification();
|
||||
const cli = parseEvidence(await compareViaCli(left, right, specification));
|
||||
const client = await connect();
|
||||
const response = await client.callTool({
|
||||
name: "compare_process_captures",
|
||||
arguments: { left, right, trace_spec: specification },
|
||||
});
|
||||
expect(response.isError).not.toBe(true);
|
||||
expect(parseEvidence(response.structuredContent)).toEqual(cli);
|
||||
expect(cli.evidence_links).toEqual([left.evidence_id, right.evidence_id]);
|
||||
expect(cli.normalized_result).toMatchObject({
|
||||
terminal: terminalStatus,
|
||||
process: "changed",
|
||||
status: "changed",
|
||||
trace: {
|
||||
verdict: "different",
|
||||
diagnostic: {
|
||||
kind: "cardinality",
|
||||
side: missingWorker,
|
||||
event_ids: ["worker"],
|
||||
locations: [],
|
||||
},
|
||||
},
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it.each(
|
||||
(["left", "right"] as const).flatMap((side) =>
|
||||
[
|
||||
|
||||
Reference in new issue
Block a user