mirror of
https://github.com/morluto/rea.git
synced 2026-11-09 16:26:36 +00:00
fix(process): preserve semantic comparison across capture-local timing (#9644)
fix(process): stabilize capture comparisons Co-authored-by: Sonai Biswas <sonai.bsws124@gmail.com>
This commit is contained in:
1 parent
3230f50b16
commit
c5ee066022
7 files changed
+608
-41
No files matched your search
@@ -41,6 +41,12 @@ behavior unknown. Record truncation, unmatched breakpoints, changed target
|
||||
identity and cleanup uncertainty as limitations, rather than treating no retained
|
||||
event as proof that the behavior never occurs.
|
||||
|
||||
For same-session MCP comparisons, pass each capture inline or reuse it as
|
||||
`{"kind":"retained-evidence","evidence_id":"ev_..."}`. Default comparison
|
||||
keeps capture-local timing as provenance without treating scheduler or observer
|
||||
latency as a behavior change; use an exact `trace_spec` event when timing is part
|
||||
of the analyst question.
|
||||
|
||||
Detailed contracts and prerequisites: [Frida](https://github.com/morluto/rea/blob/main/docs/guides/runtime/frida-instrumentation.md),
|
||||
[GDB/Rizin/Cutter](https://github.com/morluto/rea/blob/main/docs/reverse-engineering-tools.md),
|
||||
[process capture](https://github.com/morluto/rea/blob/main/docs/guides/runtime/process-capture.md),
|
||||
|
||||
@@ -97,6 +97,18 @@ process observations under a shared comparison contract. It reports observed
|
||||
differences with their locations, while residual unknowns or truncated
|
||||
observations prevent a complete-equivalence claim. Matching scenario
|
||||
commitments do not reveal whether redacted sensitive inputs were equal.
|
||||
The MCP tool accepts each complete capture inline or as an exact
|
||||
`{"kind":"retained-evidence","evidence_id":"ev_..."}` reference to capture
|
||||
Evidence already retained by the current session.
|
||||
|
||||
Capture-local observer timing remains in Evidence for provenance, but scheduler
|
||||
and observation latency alone do not make behavior changed: terminal,
|
||||
interaction-dispatch, settlement/finalization, process-sample, and filesystem
|
||||
checkpoint times are excluded from the default semantic equality projection.
|
||||
To make a timing value part of the question, supply a `trace_spec` whose exact
|
||||
event includes that field. Semantic fields such as output, interaction outcome,
|
||||
exit status, finalization delivery, process identity/command, and filesystem
|
||||
contents still participate in the default comparison.
|
||||
|
||||
For scenarios where independent scheduling can change event order, the CLI
|
||||
also accepts an optional trace-specification JSON file as the third argument.
|
||||
@@ -508,8 +520,9 @@ a representability limit, not a timer limit. The nominal lifecycle budget is tha
|
||||
sum; polling, snapshotting and cleanup lie outside it. Scenario `events` keep
|
||||
being dispatched during finalization and are recorded as usual. `finalization_ms`
|
||||
of `0`, the default, keeps today's behaviour and the committed identity of every
|
||||
existing scenario. `elapsed_ms` and `sent_at_ms` are timing data and take part in
|
||||
exit comparison like `settlement.elapsed_ms`.
|
||||
existing scenario. `elapsed_ms` and `sent_at_ms` are retained timing provenance
|
||||
like `settlement.elapsed_ms`. The default semantic comparison excludes these
|
||||
capture-local times; an exact trace event can assert them explicitly.
|
||||
|
||||
Every capture requires `truncation_details`, with separate accounting for:
|
||||
|
||||
|
||||
@@ -108,7 +108,7 @@ export const SESSION_TOOL_CONTRACTS = [
|
||||
),
|
||||
session(
|
||||
"compare_process_captures",
|
||||
"Compare two compatible process capture observations across terminal, interaction, lifecycle, process, filesystem, command-shim, HTTP, and WebSocket evidence. Optional trace_spec validates exact events against an explicit partial order or finite trace language; concurrency is never inferred from timestamps or broad sorting. Missing, journal-free, or truncated observations are never treated as equivalent.",
|
||||
'Compare two compatible process capture observations, supplied inline or as exact {kind: "retained-evidence", evidence_id} references from this session, across terminal, interaction, lifecycle, process, filesystem, command-shim, HTTP, and WebSocket evidence. Capture-local observer timing is retained as provenance but does not by itself establish a behavior change. Optional trace_spec validates exact events, including caller-selected timing assertions, against an explicit partial order or finite trace language; concurrency is never inferred from timestamps or broad sorting. Missing, journal-free, or truncated observations are never treated as equivalent.',
|
||||
processComparisonInputSchema,
|
||||
),
|
||||
session(
|
||||
|
||||
@@ -19,6 +19,7 @@ import {
|
||||
accountFullyObservedProcessCapture,
|
||||
compareUnverifiedProcessCaptures as compareProcessCaptures,
|
||||
emptyProcessCapture as emptyCapture,
|
||||
FINALIZED_PROCESS_CAPTURE_EXAMPLE,
|
||||
} from "./processCapture.fixture.js";
|
||||
|
||||
const compareFullyObserved = (
|
||||
@@ -32,6 +33,261 @@ const compareFullyObserved = (
|
||||
options,
|
||||
);
|
||||
|
||||
it("ignores capture-local timing when every observed behavior is identical", () => {
|
||||
const base = parseProcessCapture(FINALIZED_PROCESS_CAPTURE_EXAMPLE);
|
||||
const finalization = base.exit.finalization;
|
||||
if (finalization === undefined)
|
||||
throw new Error("Finalization fixture must retain its deadline outcome");
|
||||
const capture = (atMs: number) => ({
|
||||
...base,
|
||||
frames: [{ sequence: 0, at_ms: atMs, data: "deterministic output" }],
|
||||
rendered_frames: [
|
||||
{
|
||||
sequence: 0,
|
||||
at_ms: atMs,
|
||||
columns: 20,
|
||||
rows: 1,
|
||||
cursor_x: 20,
|
||||
cursor_y: 0,
|
||||
active_buffer: "normal" as const,
|
||||
lines: ["deterministic output"],
|
||||
serialized_state: "deterministic output",
|
||||
},
|
||||
],
|
||||
interaction_events: [
|
||||
{
|
||||
sequence: 0,
|
||||
scheduled_at_ms: 5,
|
||||
dispatched_at_ms: atMs + 5,
|
||||
type: "input" as const,
|
||||
data: "request",
|
||||
outcome: "dispatched" as const,
|
||||
},
|
||||
],
|
||||
exit: {
|
||||
...base.exit,
|
||||
finalization: {
|
||||
...finalization,
|
||||
elapsed_ms: atMs + 12,
|
||||
signals: finalization.signals.map((signal) => ({
|
||||
...signal,
|
||||
sent_at_ms: atMs,
|
||||
})),
|
||||
},
|
||||
},
|
||||
settlement: { ...base.settlement, elapsed_ms: atMs + 50 },
|
||||
process_samples: [
|
||||
{
|
||||
at_ms: atMs + 10,
|
||||
pid: 1,
|
||||
parent_pid: 0,
|
||||
process_group_id: 1,
|
||||
session_id: 1,
|
||||
command: "deterministic-cli",
|
||||
},
|
||||
],
|
||||
filesystem_checkpoints: base.filesystem_checkpoints.map(
|
||||
(checkpoint, index) => ({
|
||||
...checkpoint,
|
||||
at_ms: index === 0 ? 0 : atMs + 50,
|
||||
}),
|
||||
),
|
||||
});
|
||||
|
||||
expect(compareFullyObserved(capture(10), capture(30))).toMatchObject({
|
||||
status: "unchanged",
|
||||
terminal: "unchanged",
|
||||
interaction: "unchanged",
|
||||
exit: "unchanged",
|
||||
filesystem: "unchanged",
|
||||
process: "unchanged",
|
||||
first_divergence: { status: "none" },
|
||||
});
|
||||
});
|
||||
|
||||
it("still reports semantic differences when capture-local timing also differs", () => {
|
||||
const base = emptyCapture();
|
||||
const file = (sha256: string) => ({
|
||||
path: "root_0:result.txt",
|
||||
type: "file" as const,
|
||||
mode: 0o644,
|
||||
size: 6,
|
||||
sha256,
|
||||
symlink_target: null,
|
||||
});
|
||||
const capture = (side: "left" | "right") => {
|
||||
const atMs = side === "left" ? 10 : 30;
|
||||
const observedFile = file((side === "left" ? "a" : "b").repeat(64));
|
||||
return {
|
||||
...base,
|
||||
frames: [
|
||||
{
|
||||
sequence: 0,
|
||||
at_ms: atMs,
|
||||
data: side === "left" ? "left output" : "right output",
|
||||
},
|
||||
],
|
||||
interaction_events: [
|
||||
{
|
||||
sequence: 0,
|
||||
scheduled_at_ms: 5,
|
||||
dispatched_at_ms: atMs + 5,
|
||||
type: "input" as const,
|
||||
data: "request",
|
||||
outcome:
|
||||
side === "left" ? ("dispatched" as const) : ("failed" as const),
|
||||
},
|
||||
],
|
||||
exit: { ...base.exit, code: side === "left" ? 0 : 2 },
|
||||
settlement: { ...base.settlement, elapsed_ms: atMs + 50 },
|
||||
process_samples: [
|
||||
{
|
||||
at_ms: atMs + 10,
|
||||
pid: 1,
|
||||
parent_pid: 0,
|
||||
process_group_id: 1,
|
||||
session_id: 1,
|
||||
command: side === "left" ? "worker-a" : "worker-b",
|
||||
},
|
||||
],
|
||||
files_before: [observedFile],
|
||||
files_after: [observedFile],
|
||||
filesystem_checkpoints: base.filesystem_checkpoints.map(
|
||||
(checkpoint, index) => ({
|
||||
...checkpoint,
|
||||
at_ms: index === 0 ? 0 : atMs + 50,
|
||||
files: [observedFile],
|
||||
}),
|
||||
),
|
||||
};
|
||||
};
|
||||
|
||||
expect(compareFullyObserved(capture("left"), capture("right"))).toMatchObject(
|
||||
{
|
||||
status: "changed",
|
||||
terminal: "changed",
|
||||
interaction: "changed",
|
||||
exit: "changed",
|
||||
filesystem: "changed",
|
||||
process: "changed",
|
||||
first_divergence: { status: "found" },
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("reports an earlier rendered divergence before a later raw divergence", () => {
|
||||
const base = emptyCapture();
|
||||
const rendered = (side: "left" | "right") => ({
|
||||
sequence: 0,
|
||||
at_ms: 10,
|
||||
columns: 5,
|
||||
rows: 1,
|
||||
cursor_x: 5,
|
||||
cursor_y: 0,
|
||||
active_buffer: "normal" as const,
|
||||
lines: [side],
|
||||
serialized_state: side,
|
||||
});
|
||||
const capture = (side: "left" | "right") => ({
|
||||
...base,
|
||||
frames: [{ sequence: 0, at_ms: 50, data: `raw-${side}` }],
|
||||
rendered_frames: [rendered(side)],
|
||||
});
|
||||
|
||||
expect(compareFullyObserved(capture("left"), capture("right"))).toMatchObject(
|
||||
{
|
||||
terminal: "changed",
|
||||
first_divergence: {
|
||||
status: "found",
|
||||
dimension: "terminal",
|
||||
index: 0,
|
||||
left_at_ms: 10,
|
||||
right_at_ms: 10,
|
||||
left: { kind: "rendered", serialized_state: "left" },
|
||||
right: { kind: "rendered", serialized_state: "right" },
|
||||
},
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("does not report timing ignored by one terminal trace source", () => {
|
||||
const base = emptyCapture();
|
||||
const capture = (rawAtMs: number, renderedAtMs: number) => ({
|
||||
...base,
|
||||
frames: [{ sequence: 0, at_ms: rawAtMs, data: "ready" }],
|
||||
rendered_frames: [
|
||||
{
|
||||
sequence: 0,
|
||||
at_ms: renderedAtMs,
|
||||
columns: 5,
|
||||
rows: 1,
|
||||
cursor_x: 5,
|
||||
cursor_y: 0,
|
||||
active_buffer: "normal" as const,
|
||||
lines: ["ready"],
|
||||
serialized_state: "ready",
|
||||
},
|
||||
],
|
||||
event_journal: [
|
||||
{
|
||||
capture_order: 0,
|
||||
collection: "filesystem_checkpoints" as const,
|
||||
index: 0,
|
||||
},
|
||||
{ capture_order: 1, collection: "frames" as const, index: 0 },
|
||||
{ capture_order: 2, collection: "rendered_frames" as const, index: 0 },
|
||||
{ capture_order: 3, collection: "lifecycle" as const, index: 0 },
|
||||
{ capture_order: 4, collection: "lifecycle" as const, index: 1 },
|
||||
{
|
||||
capture_order: 5,
|
||||
collection: "filesystem_checkpoints" as const,
|
||||
index: 1,
|
||||
},
|
||||
],
|
||||
});
|
||||
const left = capture(50, 10);
|
||||
const right = capture(60, 20);
|
||||
const { at_ms: _ignoredAtMs, ...renderedExact } =
|
||||
left.rendered_frames[0] ?? {};
|
||||
|
||||
expect(
|
||||
compareFullyObserved(left, right, {
|
||||
traceSpecification: {
|
||||
events: [
|
||||
{
|
||||
id: "raw",
|
||||
source: "terminal_raw",
|
||||
exact: jsonValueSchema.parse(left.frames[0]),
|
||||
cardinality: { kind: "required" },
|
||||
},
|
||||
{
|
||||
id: "rendered",
|
||||
source: "terminal_rendered",
|
||||
exact: jsonValueSchema.parse(renderedExact),
|
||||
ignore_fields: ["at_ms"],
|
||||
cardinality: { kind: "required" },
|
||||
},
|
||||
],
|
||||
language: {
|
||||
kind: "finite_traces",
|
||||
variants: [{ id: "terminal", trace: ["raw", "rendered"] }],
|
||||
},
|
||||
},
|
||||
}),
|
||||
).toMatchObject({
|
||||
trace: { verdict: "different" },
|
||||
terminal: "changed",
|
||||
first_divergence: {
|
||||
status: "found",
|
||||
dimension: "terminal",
|
||||
left_at_ms: 50,
|
||||
right_at_ms: 60,
|
||||
left: { kind: "raw" },
|
||||
right: { kind: "raw" },
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("does not turn unresolved filesystem effects into equality or a definite change", () => {
|
||||
const capture = emptyCapture();
|
||||
const file = {
|
||||
|
||||
@@ -109,29 +109,31 @@ export type ProcessCaptureComparison = z.infer<
|
||||
typeof processCaptureComparisonSchema
|
||||
>;
|
||||
|
||||
const terminalObservations = (capture: ProcessCapture): readonly unknown[] =>
|
||||
[
|
||||
...capture.frames.map((frame) => ({
|
||||
kind: "raw" as const,
|
||||
...comparableTerminalFrame(frame),
|
||||
})),
|
||||
...capture.rendered_frames.map((frame) => ({
|
||||
kind: "rendered" as const,
|
||||
...frame,
|
||||
})),
|
||||
].sort((left, right) => {
|
||||
const time = left.at_ms - right.at_ms;
|
||||
if (time !== 0) return time;
|
||||
if (left.kind !== right.kind) return left.kind === "raw" ? -1 : 1;
|
||||
return left.sequence - right.sequence;
|
||||
});
|
||||
const rawTerminalObservations = (capture: ProcessCapture): readonly unknown[] =>
|
||||
capture.frames.map((frame) => ({
|
||||
kind: "raw" as const,
|
||||
...comparableTerminalFrame(frame),
|
||||
}));
|
||||
|
||||
const renderedTerminalObservations = (
|
||||
capture: ProcessCapture,
|
||||
): readonly unknown[] =>
|
||||
capture.rendered_frames.map((frame) => ({
|
||||
kind: "rendered" as const,
|
||||
...frame,
|
||||
}));
|
||||
|
||||
const terminalObservations = (capture: ProcessCapture): readonly unknown[] => [
|
||||
...rawTerminalObservations(capture),
|
||||
...renderedTerminalObservations(capture),
|
||||
];
|
||||
|
||||
const sameJsonValue = (left: unknown, right: unknown): boolean => {
|
||||
const encodedLeft = canonicalize(left);
|
||||
return encodedLeft !== undefined && encodedLeft === canonicalize(right);
|
||||
};
|
||||
|
||||
/** Canonical observations for one process-comparison dimension. */
|
||||
/** Retained observations for diagnostics and explicit trace matching. */
|
||||
export const processDimensionObservations = (
|
||||
capture: ProcessCapture,
|
||||
dimension: (typeof PROCESS_COMPARISON_DIMENSIONS)[number],
|
||||
@@ -150,6 +152,66 @@ export const processDimensionObservations = (
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Project semantic comparison facts while retaining capture-local timing in
|
||||
* the source Evidence and first-divergence diagnostics. Scheduler and observer
|
||||
* latency describe when a fact was collected, not a behavioral difference.
|
||||
* Callers that need timing assertions can select them explicitly with a trace.
|
||||
*/
|
||||
const comparableProcessDimensionObservations = (
|
||||
capture: ProcessCapture,
|
||||
dimension: ProcessComparisonDimension,
|
||||
): readonly unknown[] => {
|
||||
switch (dimension) {
|
||||
case "terminal":
|
||||
return [
|
||||
...capture.frames.map(({ sequence, data }) => ({
|
||||
kind: "raw" as const,
|
||||
sequence,
|
||||
data,
|
||||
})),
|
||||
...capture.rendered_frames.map(({ at_ms: _atMs, ...frame }) => ({
|
||||
kind: "rendered" as const,
|
||||
...frame,
|
||||
})),
|
||||
];
|
||||
case "interaction":
|
||||
return capture.interaction_events.map(
|
||||
({ dispatched_at_ms: _dispatchedAtMs, ...event }) => event,
|
||||
);
|
||||
case "exit": {
|
||||
const { finalization, ...exit } = capture.exit;
|
||||
const { elapsed_ms: _settlementElapsedMs, ...settlement } =
|
||||
capture.settlement;
|
||||
return [
|
||||
{
|
||||
...exit,
|
||||
...(finalization === undefined
|
||||
? {}
|
||||
: {
|
||||
finalization: {
|
||||
requested_ms: finalization.requested_ms,
|
||||
exit_code: finalization.exit_code,
|
||||
signals: finalization.signals.map(
|
||||
({ sent_at_ms: _sentAtMs, ...signal }) => signal,
|
||||
),
|
||||
},
|
||||
}),
|
||||
settlement,
|
||||
},
|
||||
];
|
||||
}
|
||||
case "filesystem":
|
||||
return capture.filesystem_checkpoints.map(
|
||||
({ at_ms: _atMs, ...checkpoint }) => checkpoint,
|
||||
);
|
||||
case "process":
|
||||
return capture.process_samples.map(
|
||||
({ at_ms: _atMs, ...sample }) => sample,
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
const classifyCollection = (
|
||||
left: readonly unknown[],
|
||||
right: readonly unknown[],
|
||||
@@ -284,17 +346,21 @@ const eventTime = (
|
||||
|
||||
const firstCollectionDivergence = (
|
||||
dimension: DivergenceDimension,
|
||||
left: readonly unknown[],
|
||||
right: readonly unknown[],
|
||||
leftComparable: readonly unknown[],
|
||||
rightComparable: readonly unknown[],
|
||||
leftObserved: readonly unknown[],
|
||||
rightObserved: readonly unknown[],
|
||||
): Extract<
|
||||
ProcessCaptureComparison["first_divergence"],
|
||||
{ readonly status: "found" }
|
||||
> | null => {
|
||||
const length = Math.max(left.length, right.length);
|
||||
const length = Math.max(leftComparable.length, rightComparable.length);
|
||||
for (let index = 0; index < length; index += 1) {
|
||||
const leftValue = left[index] ?? null;
|
||||
const rightValue = right[index] ?? null;
|
||||
if (sameJsonValue(leftValue, rightValue)) continue;
|
||||
const leftComparisonValue = leftComparable[index] ?? null;
|
||||
const rightComparisonValue = rightComparable[index] ?? null;
|
||||
if (sameJsonValue(leftComparisonValue, rightComparisonValue)) continue;
|
||||
const leftValue = leftObserved[index] ?? null;
|
||||
const rightValue = rightObserved[index] ?? null;
|
||||
return {
|
||||
status: "found",
|
||||
dimension,
|
||||
@@ -308,6 +374,54 @@ const firstCollectionDivergence = (
|
||||
return null;
|
||||
};
|
||||
|
||||
const firstTerminalDivergence = (
|
||||
left: ProcessCapture,
|
||||
right: ProcessCapture,
|
||||
timedSources: ReadonlySet<ProcessObservationSource> | null,
|
||||
): Extract<
|
||||
ProcessCaptureComparison["first_divergence"],
|
||||
{ readonly status: "found" }
|
||||
> | null => {
|
||||
const leftRaw = rawTerminalObservations(left);
|
||||
const rightRaw = rawTerminalObservations(right);
|
||||
const leftRendered = renderedTerminalObservations(left);
|
||||
const rightRendered = renderedTerminalObservations(right);
|
||||
const semanticRaw = (capture: ProcessCapture): readonly unknown[] =>
|
||||
capture.frames.map(({ sequence, data }) => ({
|
||||
kind: "raw" as const,
|
||||
sequence,
|
||||
data,
|
||||
}));
|
||||
const semanticRendered = (capture: ProcessCapture): readonly unknown[] =>
|
||||
capture.rendered_frames.map(({ at_ms: _atMs, ...frame }) => ({
|
||||
kind: "rendered" as const,
|
||||
...frame,
|
||||
}));
|
||||
const candidate = chooseFirstDivergence([
|
||||
firstCollectionDivergence(
|
||||
"terminal",
|
||||
timedSources?.has("terminal_raw") === true ? leftRaw : semanticRaw(left),
|
||||
timedSources?.has("terminal_raw") === true
|
||||
? rightRaw
|
||||
: semanticRaw(right),
|
||||
leftRaw,
|
||||
rightRaw,
|
||||
),
|
||||
firstCollectionDivergence(
|
||||
"terminal",
|
||||
timedSources?.has("terminal_rendered") === true
|
||||
? leftRendered
|
||||
: semanticRendered(left),
|
||||
timedSources?.has("terminal_rendered") === true
|
||||
? rightRendered
|
||||
: semanticRendered(right),
|
||||
leftRendered,
|
||||
rightRendered,
|
||||
),
|
||||
]);
|
||||
return candidate.status === "found" ? candidate : null;
|
||||
};
|
||||
|
||||
const chooseFirstDivergence = (
|
||||
candidates: ReadonlyArray<Extract<
|
||||
ProcessCaptureComparison["first_divergence"],
|
||||
@@ -333,6 +447,45 @@ const chooseFirstDivergence = (
|
||||
);
|
||||
};
|
||||
|
||||
const CAPTURE_LOCAL_TIMING_FIELDS = new Set([
|
||||
"at_ms",
|
||||
"dispatched_at_ms",
|
||||
"elapsed_ms",
|
||||
"sent_at_ms",
|
||||
]);
|
||||
|
||||
const includesCaptureLocalTiming = (
|
||||
value: unknown,
|
||||
ignoredTopLevelFields: ReadonlySet<string>,
|
||||
topLevel = true,
|
||||
): boolean => {
|
||||
if (typeof value !== "object" || value === null) return false;
|
||||
if (Array.isArray(value))
|
||||
return value.some((item) =>
|
||||
includesCaptureLocalTiming(item, ignoredTopLevelFields, false),
|
||||
);
|
||||
return Object.entries(value).some(
|
||||
([field, nested]) =>
|
||||
!(topLevel && ignoredTopLevelFields.has(field)) &&
|
||||
(CAPTURE_LOCAL_TIMING_FIELDS.has(field) ||
|
||||
includesCaptureLocalTiming(nested, ignoredTopLevelFields, false)),
|
||||
);
|
||||
};
|
||||
|
||||
const explicitlyTimedSources = (
|
||||
specification: ProcessTraceSpecification,
|
||||
): ReadonlySet<ProcessObservationSource> =>
|
||||
new Set(
|
||||
specification.events.flatMap((event) =>
|
||||
includesCaptureLocalTiming(
|
||||
event.exact,
|
||||
new Set(event.ignore_fields ?? []),
|
||||
)
|
||||
? [event.source]
|
||||
: [],
|
||||
),
|
||||
);
|
||||
|
||||
const assertComparable = (
|
||||
left: ProcessCapture,
|
||||
right: ProcessCapture,
|
||||
@@ -503,8 +656,8 @@ const compareDimensions = (
|
||||
processSourceTruncated(right, source)
|
||||
? "unknown"
|
||||
: classifyCollection(
|
||||
processDimensionObservations(left, dimension),
|
||||
processDimensionObservations(right, dimension),
|
||||
comparableProcessDimensionObservations(left, dimension),
|
||||
comparableProcessDimensionObservations(right, dimension),
|
||||
);
|
||||
return {
|
||||
terminal: classify("terminal", "terminal_rendered"),
|
||||
@@ -513,8 +666,8 @@ const compareDimensions = (
|
||||
hasUnknown(left, "exit") || hasUnknown(right, "exit")
|
||||
? "unknown"
|
||||
: sameJsonValue(
|
||||
processDimensionObservations(left, "exit"),
|
||||
processDimensionObservations(right, "exit"),
|
||||
comparableProcessDimensionObservations(left, "exit"),
|
||||
comparableProcessDimensionObservations(right, "exit"),
|
||||
)
|
||||
? "unchanged"
|
||||
: "changed",
|
||||
@@ -554,14 +707,48 @@ export const compareProcessCaptures = (
|
||||
traceEvaluation,
|
||||
[left, right],
|
||||
);
|
||||
const divergenceCandidates = PROCESS_COMPARISON_DIMENSIONS.map((dimension) =>
|
||||
dimensions[dimension] === "unknown"
|
||||
? null
|
||||
: firstCollectionDivergence(
|
||||
dimension,
|
||||
processDimensionObservations(left, dimension),
|
||||
processDimensionObservations(right, dimension),
|
||||
),
|
||||
const timedSources =
|
||||
options.traceSpecification === undefined
|
||||
? new Set<ProcessObservationSource>()
|
||||
: explicitlyTimedSources(options.traceSpecification);
|
||||
const timedDimensions = dimensionsForTraceSources(timedSources);
|
||||
const divergenceCandidates = PROCESS_COMPARISON_DIMENSIONS.map(
|
||||
(dimension) => {
|
||||
if (dimensions[dimension] === "unknown") return null;
|
||||
const observedLeft = processDimensionObservations(left, dimension);
|
||||
const observedRight = processDimensionObservations(right, dimension);
|
||||
const semanticCandidate =
|
||||
dimension === "terminal"
|
||||
? firstTerminalDivergence(left, right, null)
|
||||
: firstCollectionDivergence(
|
||||
dimension,
|
||||
comparableProcessDimensionObservations(left, dimension),
|
||||
comparableProcessDimensionObservations(right, dimension),
|
||||
observedLeft,
|
||||
observedRight,
|
||||
);
|
||||
// A trace may explicitly make retained observer timing part of the
|
||||
// question even though the default semantic projection omits it. Keep
|
||||
// that candidate alongside semantic changes so an earlier selected
|
||||
// timing mismatch is not hidden by a later behavioral difference.
|
||||
const timedCandidate =
|
||||
dimensions[dimension] === "changed" && timedDimensions.has(dimension)
|
||||
? dimension === "terminal"
|
||||
? firstTerminalDivergence(left, right, timedSources)
|
||||
: firstCollectionDivergence(
|
||||
dimension,
|
||||
observedLeft,
|
||||
observedRight,
|
||||
observedLeft,
|
||||
observedRight,
|
||||
)
|
||||
: null;
|
||||
const firstCandidate = chooseFirstDivergence([
|
||||
semanticCandidate,
|
||||
timedCandidate,
|
||||
]);
|
||||
return firstCandidate.status === "found" ? firstCandidate : null;
|
||||
},
|
||||
);
|
||||
const observedFirstDivergence = chooseFirstDivergence(
|
||||
divergenceCandidates.map((candidate) =>
|
||||
|
||||
@@ -416,7 +416,7 @@ describe("process trace dimension attribution", () => {
|
||||
},
|
||||
);
|
||||
|
||||
it("applies passing filesystem trace timestamp omissions to exact comparison", () => {
|
||||
it("keeps filesystem observer timing non-semantic with or without a trace", () => {
|
||||
const base = capture(values(["terminal", "process"]));
|
||||
const left = base;
|
||||
const right = parseProcessCapture({
|
||||
@@ -441,8 +441,8 @@ describe("process trace dimension attribution", () => {
|
||||
);
|
||||
|
||||
expect(compareProcessCaptures(left, right)).toMatchObject({
|
||||
filesystem: "changed",
|
||||
first_divergence: { status: "found", dimension: "filesystem" },
|
||||
filesystem: "unchanged",
|
||||
first_divergence: { status: "none" },
|
||||
});
|
||||
expect(
|
||||
compareProcessCaptures(left, right, {
|
||||
@@ -520,6 +520,97 @@ describe("process trace dimension attribution", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("locates timing selected explicitly by a filesystem trace", () => {
|
||||
const left = capture(values(["terminal", "process"]));
|
||||
const right = parseProcessCapture({
|
||||
...left,
|
||||
filesystem_checkpoints: left.filesystem_checkpoints.map((checkpoint) =>
|
||||
checkpoint.name === "after_settlement"
|
||||
? { ...checkpoint, at_ms: checkpoint.at_ms + 10 }
|
||||
: checkpoint,
|
||||
),
|
||||
});
|
||||
const specification = finiteTraceSpecification(
|
||||
left.filesystem_checkpoints.map((checkpoint) => ({
|
||||
id: checkpoint.name,
|
||||
source: "filesystem" as const,
|
||||
exact: jsonValueSchema.parse(checkpoint),
|
||||
cardinality: { kind: "required" as const },
|
||||
})),
|
||||
[{ id: "timed_checkpoints", trace: ["before", "after_settlement"] }],
|
||||
);
|
||||
|
||||
expect(
|
||||
compareProcessCaptures(left, right, {
|
||||
traceSpecification: specification,
|
||||
}),
|
||||
).toMatchObject({
|
||||
trace: { verdict: "different" },
|
||||
filesystem: "changed",
|
||||
first_divergence: {
|
||||
status: "found",
|
||||
dimension: "filesystem",
|
||||
index: 1,
|
||||
left_at_ms: left.filesystem_checkpoints[1]?.at_ms,
|
||||
right_at_ms: right.filesystem_checkpoints[1]?.at_ms,
|
||||
left: left.filesystem_checkpoints[1],
|
||||
right: right.filesystem_checkpoints[1],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("prefers an earlier selected timing divergence over a later semantic change", () => {
|
||||
const early = { sequence: 0, at_ms: 10, data: "same" };
|
||||
const leftLater = { sequence: 1, at_ms: 50, data: "left" };
|
||||
const rightEarly = { ...early, at_ms: 20 };
|
||||
const rightLater = { ...leftLater, data: "right" };
|
||||
const left = capture({
|
||||
frames: [early, leftLater],
|
||||
process_samples: [],
|
||||
event_journal: [
|
||||
{ collection: "frames", index: 0 },
|
||||
{ collection: "frames", index: 1 },
|
||||
],
|
||||
});
|
||||
const right = capture({
|
||||
frames: [rightEarly, rightLater],
|
||||
process_samples: [],
|
||||
event_journal: [
|
||||
{ collection: "frames", index: 0 },
|
||||
{ collection: "frames", index: 1 },
|
||||
],
|
||||
});
|
||||
const specification = finiteTraceSpecification(
|
||||
[
|
||||
{
|
||||
id: "early",
|
||||
source: "terminal_raw",
|
||||
exact: early,
|
||||
cardinality: { kind: "required" },
|
||||
},
|
||||
],
|
||||
[{ id: "timed_frames", trace: ["early"] }],
|
||||
);
|
||||
|
||||
expect(
|
||||
compareProcessCaptures(left, right, {
|
||||
traceSpecification: specification,
|
||||
}),
|
||||
).toMatchObject({
|
||||
trace: { verdict: "different" },
|
||||
terminal: "changed",
|
||||
first_divergence: {
|
||||
status: "found",
|
||||
dimension: "terminal",
|
||||
index: 0,
|
||||
left_at_ms: early.at_ms,
|
||||
right_at_ms: rightEarly.at_ms,
|
||||
left: { kind: "raw", ...early },
|
||||
right: { kind: "raw", ...rightEarly },
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
describe("finite trace dimension attribution", () => {
|
||||
it("attributes a finite-trace omission to the omitted event dimension", () => {
|
||||
const left = capture({
|
||||
|
||||
@@ -170,6 +170,20 @@ it("feeds the exact public CLI/MCP process comparison into all three MCP consume
|
||||
await expectConsumerResults(client, comparison);
|
||||
});
|
||||
|
||||
it("compares exact retained capture Evidence without resending its payloads", async () => {
|
||||
const { client, left, right, comparison } = await connectedComparison();
|
||||
const response = await client.callTool({
|
||||
name: "compare_process_captures",
|
||||
arguments: {
|
||||
left: { kind: "retained-evidence", evidence_id: left.evidence_id },
|
||||
right: { kind: "retained-evidence", evidence_id: right.evidence_id },
|
||||
},
|
||||
});
|
||||
|
||||
expect(response.isError, JSON.stringify(response)).not.toBe(true);
|
||||
expect(inlineEvidence(response)).toEqual(comparison);
|
||||
});
|
||||
|
||||
it("advertises the same process comparison Evidence that its example inputs produce", async () => {
|
||||
const { client } = await connectedComparison();
|
||||
const response = await client.callTool({
|
||||
|
||||
Reference in new issue
Block a user