diff --git a/experiments/analysis-efficiency/process-trace-validation.md b/experiments/analysis-efficiency/process-trace-validation.md new file mode 100644 index 0000000000..0a4f9b0f6f --- /dev/null +++ b/experiments/analysis-efficiency/process-trace-validation.md @@ -0,0 +1,59 @@ +# Process trace validation + +A 6,000-event directed cycle previously overflowed the JavaScript stack inside +the schema refinement. Cycle detection now uses iterative topological traversal. +Reachability is computed once per queried origin; cyclic inputs retain +independent declaration diagnostics without expanding all event pairs. Unordered +declarations retain group membership, with one diagnostic per overlapping group, +instead of allocating a pair set and repeating the same error for every pair. +Valid partial orders still receive complete pair validation. + +## Bounded comparison + +Flameox ran three randomized paired blocks (seed 917), each limited to 20 seconds +and 512 MiB. Node 24.18.0 used a 384-MiB old-space limit on macOS arm64. Each +process validated the same 2,500-event acyclic chain four times, then checked the +known 6,000-event cycle failure with per-event not-before constraints and one +unordered group containing every event. All six executions passed the embedded answer +digest and expected rejection checks. + +The baseline source is from `bb540826d`; the candidate includes this trace fix. +Both modules had TypeScript annotations removed with Node's identical built-in +transform and used the same copied runtime dependencies. This isolates schema +validation rather than comparing complete packaged CLI/MCP processes. + +| Median measurement | Baseline | Candidate | +| -------------------------------------------- | -------: | --------: | +| First validation elapsed milliseconds | 335.80 | 177.25 | +| Warm validation elapsed milliseconds | 313.36 | 165.23 | +| First validation CPU milliseconds | 366.85 | 218.39 | +| Warm validation CPU milliseconds | 335.32 | 194.08 | +| Whole-process elapsed milliseconds | 1,344.13 | 787.00 | +| Process CPU milliseconds before final report | 1,457.67 | 941.05 | +| Peak process-family MiB | 328.36 | 395.52 | + +The input is identical: 233,095 JSON bytes, estimated at 77,010 tokens by tokenx +1.3.0. The valid parsed result digest is +`28bd99ce83f0da8e45600ef20f114cb52bb26b5f5e6bfc1601ca3e712eaf3147`. +Actual agent tokens, catalog injection, transport, model reasoning and provider +costs were not measured. This is a component measurement, not an end-to-end +investigation speed claim. + +First validation excludes module import and fixture preparation; warm runs reuse +the process but create fresh per-validation graph state. Whole-process elapsed +includes startup, imports, all four validations, the cyclic-input check, token +estimation and reporting. CPU includes startup through metrics publication; +peak RSS covers the complete single-process workload, with no children. Host +caches were not flushed. Peak memory increased in this cohort. The remaining +valid-order closure and pair validation can still require quadratic resources. +Three pairs describe this fixture only. + +Preserved Flameox evidence: +`d6211639446e8b6ec2e82636913eaf471def96673e28f03e9c7f98bec07f8e13`; +analysis `e663b6bd8f5693975542d5b5e2b18437b3c167a0be939bb7386eb68549936231`. +An earlier cohort lost modules during another checkout build; its three failed +executions and timing results are excluded. The successful cohort uses an +isolated runtime copy. Source-module SHA-256 values are +`a7729faa2183b0e493b9ac4fc55f6feaee5b2e5294be13d1b29b57a44b899788` +(baseline) and `725409ce84902125941251037a8a214ccc3c1eec577e484c164c361ca580d693` +(candidate). diff --git a/src/domain/process/processTraceSpecification.test.ts b/src/domain/process/processTraceSpecification.test.ts index aeedf4d31d..9484e9075b 100644 --- a/src/domain/process/processTraceSpecification.test.ts +++ b/src/domain/process/processTraceSpecification.test.ts @@ -39,27 +39,61 @@ const specification = (): ProcessTraceSpecification => ({ }); describe("process trace specification", () => { - it("summarizes an incomplete language with its exact missing pair count", () => { - const events = Array.from({ length: 200 }, (_, index) => ({ - id: `e${String(index)}`, - source: "lifecycle" as const, - exact: { fixture: index }, - cardinality: { kind: "optional" as const }, - })); + it("reports independent declaration errors alongside a cycle", () => { + const base = specification(); const result = processTraceSpecificationSchema.safeParse({ - events, - language: { kind: "partial_order" }, + ...base, + language: { + ...base.language, + happens_before: [ + { before: "ready", after: "worker" }, + { before: "worker", after: "ready" }, + ], + not_before: [{ event: "missing", anchor: "ready" }], + unordered_groups: [{ events: ["missing", "missing"] }], + prefix: ["missing"], + suffix: ["missing"], + }, }); - expect(result.success).toBe(false); - if (result.success) - throw new Error("Expected incomplete pair declarations"); - expect(result.error.issues).toEqual([ - expect.objectContaining({ - path: ["language"], - message: - "19900 event pairs must be explicitly ordered or unordered; first undeclared pair: e0/e1", - }), - ]); + if (result.success) throw new Error("Expected invalid trace declarations"); + expect(result.error.issues).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + message: "happens-before graph must be acyclic", + }), + expect.objectContaining({ + path: ["language", "not_before", 0, "event"], + }), + expect.objectContaining({ + path: ["language", "unordered_groups", 0, "events"], + }), + expect.objectContaining({ path: ["language", "prefix", 0] }), + expect.objectContaining({ path: ["language", "suffix", 0] }), + ]), + ); + }); + + it("accepts an acyclic order with shared successors and explicit unordered peers", () => { + const result = processTraceSpecificationSchema.parse({ + events: ["a", "b", "c", "d"].map((id) => ({ + id, + source: "lifecycle", + exact: { marker: id }, + })), + language: { + kind: "partial_order", + happens_before: [ + { before: "a", after: "b" }, + { before: "a", after: "c" }, + { before: "b", after: "d" }, + { before: "c", after: "d" }, + ], + unordered_groups: [{ events: ["b", "c"] }], + prefix: ["a"], + suffix: ["d"], + }, + }); + expect(result.language.kind).toBe("partial_order"); }); it("counts only undeclared pairs after transitive, not-before and unordered relations", () => { @@ -91,7 +125,7 @@ describe("process trace specification", () => { ]); }); - it("retains duplicate unordered-pair diagnostics for reversed declarations", () => { + it("reports one duplicate-pair diagnostic per overlapping group", () => { const result = processTraceSpecificationSchema.safeParse({ events: ["z", "a", "b"].map((id) => ({ id, @@ -101,7 +135,10 @@ describe("process trace specification", () => { })), language: { kind: "partial_order", - unordered_groups: [{ events: ["z", "a", "b"] }, { events: ["a", "z"] }], + unordered_groups: [ + { events: ["z", "a", "b"] }, + { events: ["b", "a", "z"] }, + ], }, }); expect(result.success).toBe(false); @@ -114,26 +151,8 @@ describe("process trace specification", () => { ]); }); - it("rejects cycles, implicit ordering gaps, invalid ignores, and unsatisfiable variants", () => { + it("rejects invalid ignores and unsatisfiable variants", () => { const base = specification(); - expect( - processTraceSpecificationSchema.safeParse({ - ...base, - language: { - kind: "partial_order", - happens_before: [ - { before: "ready", after: "worker" }, - { before: "worker", after: "ready" }, - ], - }, - }).success, - ).toBe(false); - expect( - processTraceSpecificationSchema.safeParse({ - ...base, - language: { kind: "partial_order" }, - }).success, - ).toBe(false); expect( processTraceSpecificationSchema.safeParse({ events: [ diff --git a/src/domain/process/processTraceSpecification.ts b/src/domain/process/processTraceSpecification.ts index 61ace7ca1f..8e4e6fca6e 100644 --- a/src/domain/process/processTraceSpecification.ts +++ b/src/domain/process/processTraceSpecification.ts @@ -105,6 +105,7 @@ export type ProcessTraceSpecification = z.infer< typeof specificationShapeSchema >; type PartialOrderLanguage = z.infer; +type Reachability = (origin: string) => ReadonlySet; export const canonicalTraceJson = (value: unknown): string => { const serialized = canonicalize(value); @@ -170,7 +171,6 @@ const validateNotBefore = ( language: PartialOrderLanguage, context: z.RefinementCtx, known: ReadonlySet, - reachable: ReadonlyMap>, ): ReadonlySet => { const pairs = new Set(); for (const [index, constraint] of language.not_before.entries()) { @@ -191,63 +191,81 @@ const validateNotBefore = ( path, }); pairs.add(key); - if (reachable.get(constraint.event)?.has(constraint.anchor) === true) + } + return pairs; +}; + +const validateNotBeforeOrder = ( + language: PartialOrderLanguage, + context: z.RefinementCtx, + reachable: Reachability, +): void => { + for (const [index, constraint] of language.not_before.entries()) + if (reachable(constraint.event).has(constraint.anchor)) context.addIssue({ code: "custom", message: "not-before constraint conflicts with happens-before order", - path, + path: ["language", "not_before", index], }); - } - return pairs; }; const graphHasCycle = ( ids: readonly string[], adjacency: ReadonlyMap>, ): boolean => { - const active = new Set(); - const complete = new Set(); - const visit = (current: string): boolean => { - if (active.has(current)) return true; - if (complete.has(current)) return false; - active.add(current); - const cyclic = [...(adjacency.get(current) ?? [])].some(visit); - active.delete(current); - complete.add(current); - return cyclic; - }; - return ids.some(visit); + const incoming = new Map(ids.map((id) => [id, 0])); + for (const successors of adjacency.values()) + for (const successor of successors) { + const count = incoming.get(successor); + if (count !== undefined) incoming.set(successor, count + 1); + } + const ready = [...incoming].flatMap(([id, count]) => + count === 0 ? [id] : [], + ); + let examined = 0; + while (ready.length > 0) { + const current = ready.pop(); + if (current === undefined) break; + examined++; + for (const successor of adjacency.get(current) ?? []) { + const count = incoming.get(successor); + if (count === undefined) continue; + incoming.set(successor, count - 1); + if (count === 1) ready.push(successor); + } + } + return examined !== incoming.size; }; -const computeReachability = ( - ids: readonly string[], +const createReachability = ( adjacency: ReadonlyMap>, -): ReadonlyMap> => { +): Reachability => { const reachable = new Map>(); - for (const origin of ids) { + return (origin) => { + const cached = reachable.get(origin); + if (cached !== undefined) return cached; const seen = new Set(); const pending = [...(adjacency.get(origin) ?? [])]; while (pending.length > 0) { const current = pending.pop(); if (current === undefined || seen.has(current)) continue; seen.add(current); - pending.push(...(adjacency.get(current) ?? [])); + for (const successor of adjacency.get(current) ?? []) + pending.push(successor); } reachable.set(origin, seen); - } - return reachable; + return seen; + }; }; const isOrdered = ( - reachable: ReadonlyMap>, + reachable: Reachability, left: string, right: string, -): boolean => - reachable.get(left)?.has(right) === true || - reachable.get(right)?.has(left) === true; +): boolean => reachable(left).has(right) || reachable(right).has(left); type OrderingRelations = { - readonly reachable: ReadonlyMap>; + readonly reachable: Reachability; readonly notBeforePairs: ReadonlySet; }; @@ -255,12 +273,12 @@ const validateUnorderedGroups = ( language: PartialOrderLanguage, context: z.RefinementCtx, known: ReadonlySet, - relations: OrderingRelations, -): ReadonlySet => { - const pairs = new Set(); +): ((left: string, right: string) => boolean) => { + const memberships = new Map>(); for (const [groupIndex, group] of language.unordered_groups.entries()) { const path = ["language", "unordered_groups", groupIndex]; - if (new Set(group.events).size !== group.events.length) + const uniqueIds = new Set(group.events); + if (uniqueIds.size !== group.events.length) context.addIssue({ code: "custom", message: "unordered groups must not repeat event ids", @@ -268,6 +286,35 @@ const validateUnorderedGroups = ( }); for (const [eventIndex, id] of group.events.entries()) addReferenceIssue(context, known, id, [...path, "events", eventIndex]); + const overlaps = new Map(); + for (const id of uniqueIds) { + const priorGroups = memberships.get(id) ?? new Set(); + for (const prior of priorGroups) + overlaps.set(prior, (overlaps.get(prior) ?? 0) + 1); + priorGroups.add(groupIndex); + memberships.set(id, priorGroups); + } + if ([...overlaps.values()].some((count) => count > 1)) + context.addIssue({ + code: "custom", + message: "unordered event pairs must be declared once", + path, + }); + } + return (left, right) => { + const rightGroups = memberships.get(right); + for (const group of memberships.get(left) ?? []) + if (rightGroups?.has(group) === true) return true; + return false; + }; +}; + +const validateUnorderedOrder = ( + language: PartialOrderLanguage, + context: z.RefinementCtx, + relations: OrderingRelations, +): void => { + for (const [groupIndex, group] of language.unordered_groups.entries()) for (const [leftIndex, leftId] of group.events.entries()) { for ( let rightIndex = leftIndex + 1; @@ -277,13 +324,6 @@ const validateUnorderedGroups = ( const rightId = group.events[rightIndex]; if (rightId === undefined) continue; const key = pairKey(leftId, rightId); - if (pairs.has(key)) - context.addIssue({ - code: "custom", - message: "unordered event pairs must be declared once", - path, - }); - pairs.add(key); if ( isOrdered(relations.reachable, leftId, rightId) || relations.notBeforePairs.has(key) @@ -291,18 +331,16 @@ const validateUnorderedGroups = ( context.addIssue({ code: "custom", message: "ordered events cannot also be declared unordered", - path, + path: ["language", "unordered_groups", groupIndex], }); } } - } - return pairs; }; const validateCompletePairDeclarations = ( ids: readonly string[], relations: OrderingRelations, - unorderedPairs: ReadonlySet, + unorderedPair: (left: string, right: string) => boolean, context: z.RefinementCtx, ): void => { let missingPairCount = 0; @@ -317,7 +355,10 @@ const validateCompletePairDeclarations = ( if (rightId === undefined) continue; if (isOrdered(relations.reachable, leftId, rightId)) continue; const key = pairKey(leftId, rightId); - if (!relations.notBeforePairs.has(key) && !unorderedPairs.has(key)) { + if ( + !relations.notBeforePairs.has(key) && + !unorderedPair(leftId, rightId) + ) { if (missingPairCount === 0) firstMissingPair = `${leftId}/${rightId}`; missingPairCount++; } @@ -342,27 +383,32 @@ const validatePartialOrder = ( const ids = specification.events.map(({ id }) => id); const known = new Set(ids); const adjacency = buildAdjacency(specification.language, context, known); - if (graphHasCycle(ids, adjacency)) + const cyclic = graphHasCycle(ids, adjacency); + if (cyclic) context.addIssue({ code: "custom", message: "happens-before graph must be acyclic", path: ["language"], }); - const reachable = computeReachability(ids, adjacency); + const reachable = createReachability(adjacency); const notBeforePairs = validateNotBefore( specification.language, context, known, - reachable, ); const relations = { reachable, notBeforePairs }; - const unorderedPairs = validateUnorderedGroups( + const unorderedPair = validateUnorderedGroups( specification.language, context, known, - relations, ); - validateCompletePairDeclarations(ids, relations, unorderedPairs, context); + // Preserve independent declaration diagnostics on cyclic input. Computing + // all pair relations cannot establish a valid partial order in that case. + if (!cyclic) { + validateNotBeforeOrder(specification.language, context, reachable); + validateUnorderedOrder(specification.language, context, relations); + validateCompletePairDeclarations(ids, relations, unorderedPair, context); + } for (const [kind, values] of [ ["prefix", specification.language.prefix], ["suffix", specification.language.suffix], diff --git a/tests/boundary/mcp/processComparisonDiagnosticsMcp.test.ts b/tests/boundary/mcp/processComparisonDiagnosticsMcp.test.ts index 69fdc39444..bda4672def 100644 --- a/tests/boundary/mcp/processComparisonDiagnosticsMcp.test.ts +++ b/tests/boundary/mcp/processComparisonDiagnosticsMcp.test.ts @@ -252,6 +252,56 @@ it("reports CLI trace specification issues like the MCP trace_spec input", async }); }); +it("rejects a deep trace cycle as an input error through CLI and MCP", async () => { + const events = Array.from({ length: 6000 }, (_, index) => ({ + id: `e${String(index)}`, + source: "lifecycle", + exact: { marker: index }, + })); + const trace = { + events, + language: { + kind: "partial_order", + happens_before: events.map((event, index) => ({ + before: event.id, + after: events[(index + 1) % events.length]!.id, + })), + not_before: events.map((event, index) => ({ + event: event.id, + anchor: events[(index + 1) % events.length]!.id, + })), + unordered_groups: [{ events: events.map(({ id }) => id) }], + }, + }; + const issue = { + path: ["trace_spec", "language"], + message: "happens-before graph must be acyclic", + }; + const capture = captureEvidence("left"); + const root = await createTestTempDirectory("rea-process-trace-cycle-"); + const capturePath = join(root, "capture.json"); + const specPath = join(root, "spec.json"); + await writeFile(capturePath, JSON.stringify(capture)); + await writeFile(specPath, JSON.stringify(trace)); + expect( + await compareProcessEvidenceFiles(capturePath, capturePath, specPath), + ).toMatchObject({ + code: "invalid_request", + details: { issues: [issue] }, + }); + + const client = await connect(); + const response = await client.callTool({ + name: "compare_process_captures", + arguments: { left: capture, right: capture, trace_spec: trace }, + }); + expect(response.isError).toBe(true); + expect(response.content).toEqual([ + { type: "text", text: expect.stringContaining(issue.message) }, + ]); + await client.ping(); +}); + it("names the side and constraint when Evidence is not a usable capture", async () => { const client = await connect(); const capture = captureEvidence("right");