diff --git a/docs/guides/runtime/process-capture.md b/docs/guides/runtime/process-capture.md index ed10c87301..2eae6781ad 100644 --- a/docs/guides/runtime/process-capture.md +++ b/docs/guides/runtime/process-capture.md @@ -112,6 +112,21 @@ event includes that field. Semantic fields such as output, interaction outcome, exit status, finalization delivery, process identity/command, and filesystem contents still participate in the default comparison. +A located divergence names the compared fields that differed, so a report does +not have to be diffed by hand: `first_divergence.fields` lists sorted leaf names +such as `files[0].path`, `pid`, or `finalization.signals[0].delivery`, with array +positions as bracketed indices, `$` for a whole observation, and at most eight +names for a wide difference. The names describe only the projection the +comparison actually judged, so capture-local timing is never named unless a +`trace_spec` made it part of the question. `left` and `right` still retain the +complete observations. `fields` is optional in the contract because records +authenticated before it existed remain readable; new reports always populate it. +Naming a volatile value is not a claim of equivalence: when a per-run temporary +path or PID is not behavior, exclude it at capture time with +`normalization.paths`, `normalization.pids`, or an explicit +`normalization.patterns` replacement, and recapture both sides under the same +rules. Comparison refuses captures whose normalization rules differ. + For scenarios where independent scheduling can change event order, the CLI also accepts an optional trace-specification JSON file as the third argument. That specification must state the exact events and ordering constraints to diff --git a/src/domain/process/processCapture.comparison.test.ts b/src/domain/process/processCapture.comparison.test.ts index 806c60b0ca..be305ca673 100644 --- a/src/domain/process/processCapture.comparison.test.ts +++ b/src/domain/process/processCapture.comparison.test.ts @@ -281,12 +281,163 @@ it("does not report timing ignored by one terminal trace source", () => { dimension: "terminal", left_at_ms: 50, right_at_ms: 60, + // The trace made the raw at_ms part of the question, so the field list + // names exactly that timing value and nothing else. + fields: ["at_ms"], left: { kind: "raw" }, right: { kind: "raw" }, }, }); }); +it("names compared lifecycle timing and retains ignored settlement timing as evidence", () => { + const base = parseProcessCapture(FINALIZED_PROCESS_CAPTURE_EXAMPLE); + const finalization = base.exit.finalization; + if (finalization === undefined || finalization.elapsed_ms === null) + throw new Error("Finalization fixture must retain its deadline outcome"); + const elapsedMs = finalization.elapsed_ms; + const capture = (offset: number) => ({ + ...base, + exit: { + ...base.exit, + finalization: { + ...finalization, + elapsed_ms: elapsedMs + offset, + }, + }, + settlement: { + ...base.settlement, + elapsed_ms: base.settlement.elapsed_ms + offset, + }, + event_journal: [ + { + capture_order: 0, + collection: "filesystem_checkpoints" as const, + index: 0, + }, + { capture_order: 1, collection: "lifecycle" as const, index: 0 }, + { capture_order: 2, collection: "lifecycle" as const, index: 1 }, + { + capture_order: 3, + collection: "filesystem_checkpoints" as const, + index: 1, + }, + ], + }); + const left = capture(0); + const right = capture(10); + const { elapsed_ms: _ignoredElapsedMs, ...settlementExact } = left.settlement; + + expect( + compareFullyObserved(left, right, { + traceSpecification: { + events: [ + { + id: "exit", + source: "lifecycle", + exact: jsonValueSchema.parse({ event: "exit", ...left.exit }), + ignore_fields: ["elapsed_ms"], + cardinality: { kind: "required" }, + }, + { + id: "settlement", + source: "lifecycle", + exact: jsonValueSchema.parse({ + event: "settlement", + ...settlementExact, + }), + ignore_fields: ["elapsed_ms"], + cardinality: { kind: "required" }, + }, + ], + language: { + kind: "finite_traces", + variants: [{ id: "lifecycle", trace: ["exit", "settlement"] }], + }, + }, + }), + ).toMatchObject({ + trace: { verdict: "different" }, + exit: "changed", + first_divergence: { + status: "found", + dimension: "exit", + fields: ["finalization.elapsed_ms"], + left: { settlement: left.settlement }, + right: { settlement: right.settlement }, + }, + }); +}); + +it("reports compared dispatch timing when the trace ignores the interaction schedule", () => { + const base = emptyCapture(); + const interaction = { + sequence: 0, + type: "input" as const, + data: "request", + outcome: "dispatched" as const, + }; + const capture = (scheduledMs: number, dispatchedMs: number) => ({ + ...base, + interaction_events: [ + { + ...interaction, + scheduled_at_ms: scheduledMs, + dispatched_at_ms: dispatchedMs, + }, + ], + event_journal: ( + [ + { collection: "filesystem_checkpoints", index: 0 }, + { collection: "interaction_events", index: 0 }, + { collection: "lifecycle", index: 0 }, + { collection: "lifecycle", index: 1 }, + { collection: "filesystem_checkpoints", index: 1 }, + ] as const + ).map(({ collection, index }, capture_order) => ({ + collection, + index, + capture_order, + })), + }); + const left = capture(5, 10); + const right = capture(8, 20); + expect(compareFullyObserved(left, right).first_divergence).toMatchObject({ + status: "found", + dimension: "interaction", + fields: ["scheduled_at_ms"], + }); + expect( + compareFullyObserved(left, right, { + traceSpecification: { + events: [ + { + id: "request", + source: "interaction", + exact: { ...interaction, dispatched_at_ms: 10 }, + ignore_fields: ["scheduled_at_ms"], + cardinality: { kind: "required" }, + }, + ], + language: { + kind: "finite_traces", + variants: [{ id: "interaction", trace: ["request"] }], + }, + }, + }), + ).toMatchObject({ + trace: { verdict: "different" }, + interaction: "changed", + first_divergence: { + status: "found", + dimension: "interaction", + fields: ["dispatched_at_ms"], + left: { scheduled_at_ms: 5 }, + right: { scheduled_at_ms: 8 }, + }, + }); +}); + it("does not turn unresolved filesystem effects into equality or a definite change", () => { const capture = emptyCapture(); const file = { @@ -809,6 +960,156 @@ it("compares raw terminal and process observations while honoring normalization ).toMatchObject({ process: "changed", status: "changed" }); }); +it("names the compared field that differed inside a filesystem checkpoint", () => { + const capture = (root: string) => { + const base = emptyCapture(); + const file = { + path: `${root}:out.txt`, + type: "file" as const, + mode: 0o644, + size: 4, + sha256: "a".repeat(64), + symlink_target: null, + }; + return { + ...base, + files_after: [file], + filesystem_checkpoints: base.filesystem_checkpoints.map((checkpoint) => ({ + ...checkpoint, + files: checkpoint.name === "before" ? [] : [file], + })), + }; + }; + + expect( + compareFullyObserved(capture("root_0"), capture("root_1")), + ).toMatchObject({ + filesystem: "changed", + first_divergence: { + status: "found", + dimension: "filesystem", + index: 1, + fields: ["files[0].path"], + }, + }); +}); + +it("names every differing field of one observation in sorted order", () => { + const capture = (side: "left" | "right") => ({ + ...emptyCapture(), + process_samples: [ + { + at_ms: 10, + pid: side === "left" ? 2 : 3, + parent_pid: side === "left" ? 1 : 4, + process_group_id: side === "left" ? 2 : 5, + session_id: side === "left" ? 2 : 6, + command: side === "left" ? "worker-a" : "worker-b", + }, + ], + }); + + expect(compareFullyObserved(capture("left"), capture("right"))).toMatchObject( + { + process: "changed", + first_divergence: { + status: "found", + dimension: "process", + index: 0, + fields: [ + "command", + "parent_pid", + "pid", + "process_group_id", + "session_id", + ], + }, + }, + ); +}); + +it("bounds the field list when a wide observation differs throughout", () => { + const capture = (root: string) => { + const base = emptyCapture(); + const files = Array.from({ length: 10 }, (_entry, index) => ({ + path: `${root}:f${index}.txt`, + type: "file" as const, + mode: 0o644, + size: 1, + sha256: "a".repeat(64), + symlink_target: null, + })); + return { + ...base, + files_after: files, + filesystem_checkpoints: base.filesystem_checkpoints.map((checkpoint) => ({ + ...checkpoint, + files: checkpoint.name === "before" ? [] : files, + })), + }; + }; + + const divergence = compareFullyObserved( + capture("root_0"), + capture("root_1"), + ).first_divergence; + if (divergence.status !== "found") + throw new Error("A differing file set must locate a divergence"); + expect(divergence.fields).toStrictEqual( + Array.from({ length: 8 }, (_entry, index) => `files[${index}].path`), + ); +}); + +it("reports an observation that is present on one side only as the whole field", () => { + const sample = { + at_ms: 10, + pid: 1, + parent_pid: 0, + process_group_id: 1, + session_id: 1, + command: "worker", + }; + const withSample = { ...emptyCapture(), process_samples: [sample] }; + + expect(compareFullyObserved(withSample, emptyCapture())).toMatchObject({ + process: "removed", + first_divergence: { + status: "found", + dimension: "process", + index: 0, + fields: ["$"], + }, + }); +}); + +it("accepts a divergence recorded before the compared field list existed", () => { + const capture = (pid: number) => ({ + ...emptyCapture(), + process_samples: [ + { + at_ms: 10, + pid, + parent_pid: 0, + process_group_id: 1, + session_id: 1, + command: "worker", + }, + ], + }); + const comparison = compareFullyObserved(capture(1), capture(2)); + const divergence = comparison.first_divergence; + if (divergence.status !== "found") + throw new Error("A differing PID must locate a divergence"); + const { fields: _compared, ...legacyDivergence } = divergence; + + expect( + processCaptureComparisonSchema.parse({ + ...comparison, + first_divergence: legacyDivergence, + }), + ).toMatchObject({ status: "changed", process: "changed" }); +}); + it("keeps filesystem comparison unknown without coverage and reports observed changes", () => { const capture = emptyCapture(); const incomplete = { diff --git a/src/domain/process/processComparison.ts b/src/domain/process/processComparison.ts index 05a5459d9c..15278d2bab 100644 --- a/src/domain/process/processComparison.ts +++ b/src/domain/process/processComparison.ts @@ -7,7 +7,10 @@ import type { } from "./processObservation.js"; import type { ProcessCapture } from "./processCaptureParsing.js"; -import { comparableTerminalFrame } from "./processObservation.js"; +import { + comparableProcessObservationPayload, + comparableTerminalFrame, +} from "./processObservation.js"; import { AnalysisInputError } from "../analysisErrorCore.js"; import { jsonValueSchema } from "../jsonValue.js"; import type { ProcessTraceSpecification } from "./processTraceSpecification.js"; @@ -23,6 +26,12 @@ import { } from "./processTraceComparison.js"; const OPERATION = "compare_process_captures"; +/** + * A divergence report names the fields that differed, not a second copy of the + * observations, so the list is bounded to keep the report readable. + */ +const DIFFERING_FIELD_LIMIT = 8; + /** Comparison classification that never equates incomplete evidence. */ export const comparisonStatusSchema = z.enum([ "unchanged", @@ -85,6 +94,15 @@ export const processCaptureComparisonSchema = z index: z.int().nonnegative(), left_at_ms: z.int().nonnegative().nullable(), right_at_ms: z.int().nonnegative().nullable(), + // Absent on records authenticated before the field list existed; the + // divergence itself is still fully described by `left` and `right`. + fields: z + .array(z.string().min(1)) + .max(DIFFERING_FIELD_LIMIT) + .optional() + .describe( + "Sorted names of the fields the selected projection compared and that differ, with array positions as bracketed indices and `$` for a whole observation. Bounded to eight names; `left` and `right` retain the complete observations.", + ), left: jsonValueSchema.nullable(), right: jsonValueSchema.nullable(), }), @@ -132,6 +150,67 @@ const sameJsonValue = (left: unknown, right: unknown): boolean => { return encodedLeft !== undefined && encodedLeft === canonicalize(right); }; +const containerEntries = (value: unknown): readonly string[] | null => + Array.isArray(value) + ? value.map((_entry, index) => String(index)) + : value !== null && typeof value === "object" + ? Object.keys(value) + : null; + +const differingFieldPaths = ( + left: unknown, + right: unknown, +): readonly string[] => { + const output: string[] = []; + const walk = ( + leftValue: unknown, + rightValue: unknown, + path: string, + ): void => { + if (output.length >= DIFFERING_FIELD_LIMIT) return; + if (sameJsonValue(leftValue, rightValue)) return; + const leftKeys = containerEntries(leftValue); + const rightKeys = containerEntries(rightValue); + if ( + leftKeys === null || + rightKeys === null || + Array.isArray(leftValue) !== Array.isArray(rightValue) + ) { + output.push(path.length === 0 ? "$" : path); + return; + } + for (const key of [...new Set([...leftKeys, ...rightKeys])].sort()) + walk( + Reflect.get(leftValue as object, key), + Reflect.get(rightValue as object, key), + Array.isArray(leftValue) + ? `${path}[${key}]` + : path.length === 0 + ? key + : `${path}.${key}`, + ); + }; + walk(left, right, ""); + return output.sort(); +}; + +const exitComparisonObservation = ( + capture: ProcessCapture, + ignoredFields?: readonly string[], +): unknown => + comparableProcessObservationPayload( + { + ...capture.exit, + // A trace matches settlement separately before the diagnostic nests it + // under exit. Apply ignored fields at each record's own top level. + settlement: comparableProcessObservationPayload( + capture.settlement, + ignoredFields, + ), + }, + ignoredFields, + ); + /** Retained observations for diagnostics and explicit trace matching. */ export const processDimensionObservations = ( capture: ProcessCapture, @@ -143,7 +222,7 @@ export const processDimensionObservations = ( case "interaction": return capture.interaction_events; case "exit": - return [{ ...capture.exit, settlement: capture.settlement }]; + return [exitComparisonObservation(capture)]; case "filesystem": return capture.filesystem_checkpoints; case "process": @@ -357,6 +436,9 @@ const firstCollectionDivergence = ( index, left_at_ms: eventTime(dimension, leftValue), right_at_ms: eventTime(dimension, rightValue), + fields: [ + ...differingFieldPaths(leftComparisonValue, rightComparisonValue), + ], left: jsonValueSchema.parse(leftValue), right: jsonValueSchema.parse(rightValue), }; @@ -367,7 +449,7 @@ const firstCollectionDivergence = ( const firstTerminalDivergence = ( left: ProcessCapture, right: ProcessCapture, - timedSources: ReadonlySet | null, + timedSources: TimedSources | null, ): Extract< ProcessCaptureComparison["first_divergence"], { readonly status: "found" } @@ -444,39 +526,55 @@ const CAPTURE_LOCAL_TIMING_FIELDS = new Set([ "sent_at_ms", ]); -const includesCaptureLocalTiming = ( - value: unknown, - ignoredTopLevelFields: ReadonlySet, - topLevel = true, -): boolean => { +const includesCaptureLocalTiming = (value: unknown): boolean => { if (typeof value !== "object" || value === null) return false; - if (Array.isArray(value)) - return value.some((item) => - includesCaptureLocalTiming(item, ignoredTopLevelFields, false), - ); + if (Array.isArray(value)) return value.some(includesCaptureLocalTiming); return Object.entries(value).some( ([field, nested]) => - !(topLevel && ignoredTopLevelFields.has(field)) && - (CAPTURE_LOCAL_TIMING_FIELDS.has(field) || - includesCaptureLocalTiming(nested, ignoredTopLevelFields, false)), + CAPTURE_LOCAL_TIMING_FIELDS.has(field) || + includesCaptureLocalTiming(nested), ); }; +type TimedSources = ReadonlyMap< + ProcessObservationSource, + readonly string[] | undefined +>; + +// Trace validation requires every event from one source to use the same +// ignored fields, regardless of declaration order. const explicitlyTimedSources = ( - specification: ProcessTraceSpecification, -): ReadonlySet => - new Set( - specification.events.flatMap((event) => - includesCaptureLocalTiming( + specification: ProcessTraceSpecification | undefined, +): TimedSources => + new Map( + specification?.events.flatMap((event) => { + const exact = comparableProcessObservationPayload( event.exact, - // optional-fact-default: a trace specification option, not an observation. - new Set(event.ignore_fields ?? []), - ) - ? [event.source] - : [], - ), + event.ignore_fields, + ); + return includesCaptureLocalTiming(exact) + ? [[event.source, event.ignore_fields] as const] + : []; + }), ); +const timedProcessDimensionObservations = ( + capture: ProcessCapture, + dimension: Exclude, + observed: readonly unknown[], + timedSources: TimedSources, +): readonly unknown[] => { + if (dimension === "exit") + return [exitComparisonObservation(capture, timedSources.get("lifecycle"))]; + if (dimension !== "interaction") return observed; + const ignoredFields = timedSources.get("interaction"); + return ignoredFields === undefined + ? observed + : observed.map((value) => + comparableProcessObservationPayload(value, ignoredFields), + ); +}; + const assertComparable = ( left: ProcessCapture, right: ProcessCapture, @@ -698,26 +796,16 @@ export const compareProcessCaptures = ( traceEvaluation, [left, right], ); - const timedSources = - options.traceSpecification === undefined - ? new Set() - : explicitlyTimedSources(options.traceSpecification); - const timedDimensions = dimensionsForTraceSources(timedSources); + const timedSources = explicitlyTimedSources(options.traceSpecification); + const timedDimensions = dimensionsForTraceSources( + new Set(timedSources.keys()), + ); + const ignoredInteractionFields = timedSources.get("interaction"); const divergenceCandidates = PROCESS_COMPARISON_DIMENSIONS.map( (dimension) => { if (dimensions[dimension] === "unknown") return null; const observedLeft = processDimensionObservations(left, dimension); const observedRight = processDimensionObservations(right, dimension); - const semanticCandidate = - dimension === "terminal" - ? firstTerminalDivergence(left, right, null) - : firstCollectionDivergence( - dimension, - comparableProcessDimensionObservations(left, dimension), - comparableProcessDimensionObservations(right, dimension), - observedLeft, - observedRight, - ); // A trace may explicitly make retained observer timing part of the // question even though the default semantic projection omits it. Keep // that candidate alongside semantic changes so an earlier selected @@ -728,12 +816,43 @@ export const compareProcessCaptures = ( ? firstTerminalDivergence(left, right, timedSources) : firstCollectionDivergence( dimension, - observedLeft, - observedRight, + timedProcessDimensionObservations( + left, + dimension, + observedLeft, + timedSources, + ), + timedProcessDimensionObservations( + right, + dimension, + observedRight, + timedSources, + ), observedLeft, observedRight, ) : null; + // The timed interaction projection already includes every semantic + // field selected by the trace. Use it when the trace ignores fields + // that a default semantic candidate could otherwise report. + if ( + dimension === "interaction" && + timedCandidate !== null && + trace?.verdict === "different" && + ignoredInteractionFields !== undefined && + ignoredInteractionFields.length > 0 + ) + return timedCandidate; + const semanticCandidate = + dimension === "terminal" + ? firstTerminalDivergence(left, right, null) + : firstCollectionDivergence( + dimension, + comparableProcessDimensionObservations(left, dimension), + comparableProcessDimensionObservations(right, dimension), + observedLeft, + observedRight, + ); const firstCandidate = chooseFirstDivergence([ semanticCandidate, timedCandidate, diff --git a/src/domain/process/processTraceSpecification.test.ts b/src/domain/process/processTraceSpecification.test.ts index 2eec3c207d..9c7300832c 100644 --- a/src/domain/process/processTraceSpecification.test.ts +++ b/src/domain/process/processTraceSpecification.test.ts @@ -272,6 +272,41 @@ describe("malformed process trace declarations", () => { }); describe("finite trace specification validation", () => { + it.each([false, true])( + "rejects different ignored fields within lifecycle (reverse declarations: %s)", + (reverseDeclarations) => { + const events = [ + { + id: "exit", + source: "lifecycle", + exact: { event: "exit" }, + ignore_fields: ["elapsed_ms"], + }, + { + id: "settlement", + source: "lifecycle", + exact: { event: "settlement", elapsed_ms: 50 }, + }, + ]; + const result = processTraceSpecificationSchema.safeParse({ + events: reverseDeclarations ? events.toReversed() : events, + language: { + kind: "finite_traces", + variants: [{ id: "lifecycle", trace: ["exit", "settlement"] }], + }, + }); + if (result.success) + throw new Error("Expected one ignore policy per source"); + expect(result.error.issues).toStrictEqual([ + expect.objectContaining({ + path: ["events", 1, "ignore_fields"], + message: + "events from one source must use the same ignored schedule fields", + }), + ]); + }, + ); + it("rejects invalid ignores and unsatisfiable variants", () => { const base = specification(); expect(