From 19b2972dfb9716fc03e3378304081f95718bb421 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Tue, 27 Oct 2026 05:08:14 +0800 Subject: [PATCH] fix(environment): separate request order from completion Retain valid request starts independently of duration and require observed completion for interval-based state and write inferences. Bound indexed interference checks across captured writes while preserving ordinal fallback and unmodified Evidence. Validated state, comparison, CLI and MCP workflows with known, unknown, overlapping and redacted timing controls; check:fast passed. --- .../EnvironmentStateAccumulator.ts | 94 ++++---- .../EnvironmentStateExtraction.ts | 45 ++-- .../EnvironmentStateWriteSimulation.test.ts | 220 ++++++++++++++++-- .../EnvironmentStateWriteSimulation.ts | 44 +++- .../analysisView/StateRecordChronology.ts | 36 +++ .../analysisView/environmentStateGate.test.ts | 3 +- .../environmentStateQueryCalls.test.ts | 140 ++++++++--- .../environmentStateQueryCalls.ts | 63 +++-- .../environmentStateRewrites.test.ts | 112 +++++++++ .../analysisView/environmentStateRewrites.ts | 36 +-- .../analysisView/environmentStateRules.ts | 58 ++++- .../analysisView/environmentStateView.test.ts | 157 +++++++++++-- .../environmentStateWriteAttribution.test.ts | 44 ++++ .../environmentStateWriteAttribution.ts | 24 +- .../environmentStateWriteCompletion.test.ts | 112 +++++++++ .../environmentStateWriteCompletion.ts | 149 ++++++++++++ .../environmentStateWriteCutoffs.test.ts | 14 +- .../environmentStateWriteEffects.ts | 12 +- .../environmentStateWriteLinks.test.ts | 75 +++++- .../environmentStateWriteLinks.ts | 20 +- .../environmentStateWriteObserve.ts | 10 +- .../analysisView/environmentStateWrites.ts | 14 +- tests/boundary/mcp/analysisViewMcp.test.ts | 89 ++++++- 23 files changed, 1333 insertions(+), 238 deletions(-) create mode 100644 src/domain/analysisView/StateRecordChronology.ts create mode 100644 src/domain/analysisView/environmentStateRewrites.test.ts create mode 100644 src/domain/analysisView/environmentStateWriteAttribution.test.ts create mode 100644 src/domain/analysisView/environmentStateWriteCompletion.test.ts create mode 100644 src/domain/analysisView/environmentStateWriteCompletion.ts diff --git a/src/domain/analysisView/EnvironmentStateAccumulator.ts b/src/domain/analysisView/EnvironmentStateAccumulator.ts index 2d0f2fce15..dff8df6a44 100644 --- a/src/domain/analysisView/EnvironmentStateAccumulator.ts +++ b/src/domain/analysisView/EnvironmentStateAccumulator.ts @@ -5,7 +5,6 @@ import { MAX_JSON_DEPTH, type JsonValue } from "../jsonValue.js"; import { compareOffsetDateTimeInstants, readRecordedDateTime, - type OffsetDateTimeInstant, } from "../offsetDateTime.js"; import { ok, type Result } from "../result.js"; import { @@ -86,10 +85,8 @@ export const stateBodyJson = ( : value; }; -const timingSchema = z.object({ - startedDateTime: z.string(), - time: z.number().nonnegative(), -}); +const startSchema = z.object({ startedDateTime: z.string() }); +const durationSchema = z.object({ time: z.number().nonnegative() }); const redactedWriteSchema = z.object({ request: z.object({ method: z.string() }), response: z.object({ status: z.number() }).optional(), @@ -100,7 +97,6 @@ interface RecordEntry { readonly origin: string | null; readonly contentType: string; /** Exact ordering fact; request interval coordinates remain milliseconds. */ - readonly startInstant: OffsetDateTimeInstant | undefined; readonly record: WebNetworkCaptureRecord; } @@ -143,44 +139,46 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator record: WebNetworkCaptureRecord, line: RecordedRequestLine, ): Result { - const timing = timingSchema.safeParse(record.reported); + const started = startSchema.safeParse(record.reported); + const duration = durationSchema.safeParse(record.reported); // Admit normalization and fraction parsing before materializing their strings. if ( - timing.success && + started.success && !this.scratch( RECORD_SCRATCH_BYTES + - textBytes(timing.data.startedDateTime.length + 6) * 3, + textBytes(started.data.startedDateTime.length + 6) * 3, ) ) return this.overBudget(); - const startInstant = timing.success - ? readRecordedDateTime(timing.data.startedDateTime)?.instant + const startInstant = started.success + ? readRecordedDateTime(started.data.startedDateTime)?.instant : undefined; // Keep the established millisecond interval policy independently of sorting. - const start = + const startMs = startInstant === undefined - ? record.ordinal + ? undefined : startInstant.epochSecond * 1000 + Number(startInstant.fraction.slice(0, 3).padEnd(3, "0")); + const end = + startMs === undefined || !duration.success + ? undefined + : startMs + duration.data.time; + const timing: StateRecordFacts["timing"] = { + start: + startInstant === undefined || startMs === undefined + ? undefined + : { instant: startInstant, ms: startMs }, + endMs: end !== undefined && Number.isFinite(end) ? end : undefined, + }; // A fraction substring may retain its normalized timestamp's backing store, // including trailing zeroes removed from the exact comparison value. const instantBytes = - timing.success && startInstant !== undefined - ? ENTRY_BYTES + textBytes(timing.data.startedDateTime.length + 6) + started.success && startInstant !== undefined + ? ENTRY_BYTES + textBytes(started.data.startedDateTime.length + 6) : 0; if (line.kind === "excluded") return line.reason === "url-redacted" - ? this.#redactedWrite( - record, - timing.success && startInstant !== undefined - ? { - start, - end: start + timing.data.time, - startInstant, - instantBytes, - } - : undefined, - ) + ? this.#redactedWrite(record, timing, instantBytes) : ok(undefined); // The entry keeps the line's strings and a reference to the already admitted record. if ( @@ -204,15 +202,10 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator path: line.path, query: line.query === "" ? undefined : line.query, status: line.status, - start, - end: - timing.success && startInstant !== undefined - ? start + timing.data.time - : record.ordinal, + timing, }, origin: line.origin, contentType: line.rawMime, - startInstant, record, }); return ok(undefined); @@ -221,14 +214,8 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator /** Keep a successful write whose URL was redacted, with an empty path whose root relates to every type. */ #redactedWrite( record: WebNetworkCaptureRecord, - timing: - | { - readonly start: number; - readonly end: number; - readonly startInstant: OffsetDateTimeInstant; - readonly instantBytes: number; - } - | undefined, + timing: StateRecordFacts["timing"], + instantBytes: number, ): Result { const parsed = redactedWriteSchema.safeParse(record.reported); if (!parsed.success) return ok(undefined); @@ -239,15 +226,12 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator path: "", query: undefined, status: status > 0 ? status : null, - start: timing?.start ?? record.ordinal, - end: timing?.end ?? record.ordinal, + timing, }; if (!isStateWrite(facts)) return ok(undefined); if ( !this.retain( - ENTRY_BYTES * 8 + - textBytes(facts.method.length) + - (timing?.instantBytes ?? 0), + ENTRY_BYTES * 8 + textBytes(facts.method.length) + instantBytes, ) ) return this.overBudget(); @@ -255,7 +239,6 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator facts, origin: null, contentType: "", - startInstant: timing?.startInstant, record, }); return ok(undefined); @@ -358,7 +341,7 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator ).length; const timeline: StateTimeline = entries.length > 0 && - entries.every((entry) => entry.startInstant !== undefined) + entries.every((entry) => entry.facts.timing.start !== undefined) ? "recorded-start-time" : "record-order"; const records = entries @@ -366,16 +349,11 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator timeline === "record-order" ? left.facts.ordinal - right.facts.ordinal : compareOffsetDateTimeInstants( - left.startInstant!, - right.startInstant!, + left.facts.timing.start!.instant, + right.facts.timing.start!.instant, ) || left.facts.ordinal - right.facts.ordinal, ) - .map((entry) => entry.facts) - .map((facts) => - timeline === "record-order" - ? { ...facts, start: facts.ordinal, end: facts.ordinal } - : facts, - ); + .map((entry) => entry.facts); const state = new EnvironmentStateExtraction({ tools: manifest.value.tools, records, @@ -401,6 +379,12 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator "Some records carry no start time, so the state's timeline is record order and overlapping requests are not detected.", ] : []), + ...(timeline === "recorded-start-time" && + records.some((record) => record.timing.endMs === undefined) + ? [ + "Some records carry no usable request duration; their completion and overlap remain unknown.", + ] + : []), ...(state.views.some((view) => view.shape === "recorded") ? [ "Tools whose recordings do not reproduce from the extracted state are listed as recorded views with the reason; they replay their recordings.", diff --git a/src/domain/analysisView/EnvironmentStateExtraction.ts b/src/domain/analysisView/EnvironmentStateExtraction.ts index 18e7bbbd1e..25d0f60f7a 100644 --- a/src/domain/analysisView/EnvironmentStateExtraction.ts +++ b/src/domain/analysisView/EnvironmentStateExtraction.ts @@ -1,3 +1,4 @@ +import { StateRecordChronology } from "./StateRecordChronology.js"; import { jsonByteLength } from "../jsonSerialization.js"; import type { JsonValue } from "../jsonValue.js"; import { @@ -107,14 +108,13 @@ const memberNames = (body: JsonValue, pointer: string): string[] => { export class EnvironmentStateExtraction { readonly #tools: readonly StateToolFacts[]; readonly #records: readonly StateRecordFacts[]; - readonly #timeline: StateTimeline; + readonly #chronology: StateRecordChronology; readonly #budget: StateBudget; - readonly #position: ReadonlyMap; readonly #byOrdinal: ReadonlyMap; /** POST tools whose calls the capture shows read state rather than change it. */ #readOnly: ReadonlySet = new Set(); #writes: readonly StateRecordFacts[] = []; - #writeWithin: ReturnType = writeSpanCheck([]); + #writeWithin: ReturnType = () => false; readonly #bases: ReadonlyMap; #writeCutoffs: StateWriteCutoffs | undefined; readonly #initLimits = new Map(); @@ -146,11 +146,8 @@ export class EnvironmentStateExtraction { }) { this.#tools = input.tools; this.#records = input.records; - this.#timeline = input.timeline; + this.#chronology = new StateRecordChronology(input.records, input.timeline); this.#budget = input.budget; - this.#position = new Map( - input.records.map((record, index) => [record.ordinal, index]), - ); this.#byOrdinal = new Map( input.records.map((record) => [record.ordinal, record]), ); @@ -169,9 +166,10 @@ export class EnvironmentStateExtraction { request: (ordinal) => this.#requestBody(ordinal), retain: (bytes) => this.#retain(bytes), }, + this.#chronology, )); release(); - this.#writeWithin = writeSpanCheck(this.#writes); + this.#writeWithin = writeSpanCheck(this.#writes, this.#chronology); } /** The extracted state, or null when the representation budget is exhausted. */ @@ -183,8 +181,8 @@ export class EnvironmentStateExtraction { this.#writes, this.#bases, (left, right) => - this.#position.get(left.ordinal)! < - this.#position.get(right.ordinal)!, + this.#chronology.positions.get(left.ordinal)! < + this.#chronology.positions.get(right.ordinal)!, (bytes) => this.#retain(bytes), ); this.#creations(); @@ -240,7 +238,7 @@ export class EnvironmentStateExtraction { const split = withoutVariants(plan, { shapeOf: (record) => shapes.get(record.ordinal), queryOf: (ordinal) => this.#byOrdinal.get(ordinal)?.query, - position: (ordinal) => this.#position.get(ordinal)!, + position: (ordinal) => this.#chronology.positions.get(ordinal)!, retain: (bytes) => this.#retain(bytes), checkpoint: () => this.#budget.checkpoint(), }); @@ -259,6 +257,7 @@ export class EnvironmentStateExtraction { this.#spans.filter((span) => this.#rewritten.has(span.type)), this.#writes, this.#byOrdinal, + this.#chronology, ); if (this.#cuts.size === 0) return false; for (const map of [this.#initLimits, this.#identity, this.#views]) @@ -282,7 +281,7 @@ export class EnvironmentStateExtraction { tools: this.#tools, records: this.#records, readOnly: this.#readOnly, - timed: this.#timeline === "recorded-start-time", + chronology: this.#chronology, volatileOf: (type) => this.#volatileOf(type), io: { withBody: (ordinal, use) => this.#withBody(ordinal, use), @@ -355,9 +354,10 @@ export class EnvironmentStateExtraction { this.#creations(), { type: template, + chronology: this.#chronology, tools: this.#tools, records: this.#byOrdinal, - position: (record) => this.#position.get(record.ordinal)!, + position: (record) => this.#chronology.positions.get(record.ordinal)!, withBody: (ordinal, use) => this.#withBody(ordinal, use), retain: (bytes) => this.#retain(bytes), }, @@ -367,7 +367,8 @@ export class EnvironmentStateExtraction { const limit = cut !== undefined && (bound === undefined || - this.#position.get(cut.ordinal)! < this.#position.get(bound.ordinal)!) + this.#chronology.positions.get(cut.ordinal)! < + this.#chronology.positions.get(bound.ordinal)!) ? cut : bound; this.#retain(ENTRY_BYTES + textBytes(template.length)); @@ -391,9 +392,9 @@ export class EnvironmentStateExtraction { ): boolean { return ( limit === undefined || - (this.#position.get(record.ordinal)! < - this.#position.get(limit.ordinal)! && - (this.#timeline === "record-order" || record.end <= limit.start)) + (this.#chronology.positions.get(record.ordinal)! < + this.#chronology.positions.get(limit.ordinal)! && + this.#chronology.completedBefore(record, limit)) ); } @@ -412,8 +413,8 @@ export class EnvironmentStateExtraction { ) .sort( (left, right) => - this.#position.get(left.ordinal)! - - this.#position.get(right.ordinal)!, + this.#chronology.positions.get(left.ordinal)! - + this.#chronology.positions.get(right.ordinal)!, ); } @@ -553,7 +554,9 @@ export class EnvironmentStateExtraction { this.#identity.set(type, { field, basis: "path-parameter" }); const late = this.#addLateProjections(tool, type, projected); projected.sort( - (left, right) => this.#position.get(left)! - this.#position.get(right)!, + (left, right) => + this.#chronology.positions.get(left)! - + this.#chronology.positions.get(right)!, ); this.#addPlan(candidate, pointer, kept, { projected, late }); } @@ -709,7 +712,7 @@ export class EnvironmentStateExtraction { if (id === undefined) return; const collection = collectionPathOf(record, plan.item); const key = entityKey(plan.type, collection, id); - const position = this.#position.get(record.ordinal)!; + const position = this.#chronology.positions.get(record.ordinal)!; let entity = this.#entities.get(key); if (entity === undefined) { this.#retain(ENTRY_BYTES * 4 + textBytes(key.length)); diff --git a/src/domain/analysisView/EnvironmentStateWriteSimulation.test.ts b/src/domain/analysisView/EnvironmentStateWriteSimulation.test.ts index 2ee9a0d8e7..279818275d 100644 --- a/src/domain/analysisView/EnvironmentStateWriteSimulation.test.ts +++ b/src/domain/analysisView/EnvironmentStateWriteSimulation.test.ts @@ -1,9 +1,12 @@ +import { StateRecordChronology } from "./StateRecordChronology.js"; import { OmittedFields } from "./environmentStateWriteOmitted.js"; import { checkedLines, servedLine } from "./environmentStateCheckedLines.js"; import { describe, expect, it, vi } from "vitest"; import { timedCaptureEvidence, + analysisViewCaptureEvidence, + environmentManifestCaptureFixture, type TimedCaptureRow, } from "../../../tests/fixtures/analysisView.js"; import { @@ -1298,8 +1301,7 @@ describe("encoded parent spellings", () => { path: "/api/orgs/uid/acme1/books", query: undefined, status: 200, - start: 0, - end: 0, + timing: { start: undefined, endMs: undefined }, }; const lines = checkedLines({ tools: [tool], @@ -1502,8 +1504,7 @@ describe("resolved encoded item observations", () => { path: "/api/orgs/acme1/books/%61", query: undefined, status: 204, - start: 0, - end: 0, + timing: { start: undefined, endMs: undefined }, }; const target = itemTarget(state, "/api/orgs/{org_id}/books", write); const itemView = { @@ -1516,19 +1517,20 @@ describe("resolved encoded item observations", () => { checked_record_ordinals: [1], envelope_ordinal: 1, }; + const records = [ + write, + { + ordinal: 1, + method: "GET", + path: "/api/orgs/acme1/books/%61", + query: undefined, + status: 200, + timing: { start: undefined, endMs: undefined }, + }, + ]; const search: ObservationSearch = { - records: [ - write, - { - ordinal: 1, - method: "GET", - path: "/api/orgs/acme1/books/%61", - query: undefined, - status: 200, - start: 1, - end: 1, - }, - ], + records, + chronology: new StateRecordChronology(records, "record-order"), from: 0, viewOf: () => itemView, withBody: (_ordinal, use) => use({ id: "%61", name: "After" }), @@ -2130,7 +2132,54 @@ describe("what an unmodelled write explains", () => { }); }); +it("leaves the global served clock unknown when any record completion is unknown", () => { + const rows = (duration: number): TimedCaptureRow[] => [ + ["GET", "/api/books", [FIRST_BOOK, SECOND_BOOK]], + // Even an earlier record with an unknown end may finish after the final known record. + ["GET", "/api/books/1", FIRST_BOOK, 200, undefined, duration], + ["GET", "/api/books/2", SECOND_BOOK], + ]; + expect( + environmentStateViewOf(timedCaptureEvidence(rows(5))).write_clock, + ).toBe(new Date(bookClock(3)).toISOString()); + expect( + environmentStateViewOf(timedCaptureEvidence(rows(-1))).write_clock, + ).toBeNull(); +}); + describe("reads and calls on an entity an unmodelled write left unknown", () => { + it.each([ + { duration: 5, target: "1", served: true }, + { duration: -1, target: "1", served: false }, + { duration: -1, target: "2", served: true }, + ])( + "requires unmodelled completion before resync and later confirmation: $target duration=$duration", + ({ duration, target, served }) => { + const rows: TimedCaptureRow[] = [ + [ + "GET", + "/api/books", + [ + { id: 1, name: "A" }, + { id: 2, name: "B" }, + ], + ], + ["GET", "/api/books/1", { id: 1, name: "A" }], + ["GET", "/api/books/2", { id: 2, name: "B" }], + ["POST", `/api/books/${target}/archive`, null, 204, {}, duration], + // This read may re-establish a baseline only after the related action finished. + ["GET", "/api/books/1", { id: 1, name: "A" }], + ["PATCH", "/api/books/1", { id: 1, name: "Z" }, 200, { name: "Z" }], + ["GET", "/api/books/1", { id: 1, name: "Z" }], + ]; + expect(gated(rows, "patch_books_by_id")).toMatchObject({ + served, + checked_record_ordinals: served ? [6] : [], + contradicted_by: null, + }); + }, + ); + const named = (name: string) => ({ id: 1, name }); const other = { id: 2, name: "B" }; const ARCHIVE: TimedCaptureRow = [ @@ -2307,8 +2356,7 @@ describe("entities an unmodelled write made", () => { path: "/api/books/3", query: undefined, status: 200, - start: 1, - end: 1, + timing: { start: undefined, endMs: undefined }, }; expect( unseen.adopt(view as unknown as StateItemView, { record, index: 1 }), @@ -2333,8 +2381,7 @@ describe("entities an unmodelled write made", () => { path: "/api/books/3", query: undefined, status: 200, - start: index, - end: index, + timing: { start: undefined, endMs: undefined }, }, index, }); @@ -3042,8 +3089,7 @@ it("learns omitted fields using the raw ID resolved by the item owner", () => { path: "/api/books/%63", query: undefined, status: 200, - start: 1, - end: 1, + timing: { start: undefined, endMs: undefined }, }; const target = itemTarget(state, view.type, record); expect(target.id).toBe("%63"); @@ -3066,3 +3112,133 @@ it("learns omitted fields using the raw ID resolved by the item owner", () => { required_fields: ["id", "name"], }); }); + +it.each([ + { + label: "point read with simultaneous unknown action", + archiveStart: 4, + duration: 0, + archiveDuration: -1, + target: "1", + served: false, + }, + { + label: "point read with simultaneous point action", + archiveStart: 4, + duration: 0, + archiveDuration: 0, + target: "1", + served: true, + }, + { + label: "point read with unrelated unknown action", + archiveStart: 4, + duration: 0, + archiveDuration: -1, + target: "2", + served: true, + }, + { + label: "same-start later ordinal", + archiveStart: 4, + duration: 5, + target: "1", + served: false, + }, + { + label: "later start within response interval", + archiveStart: 5, + duration: 1500, + target: "1", + served: false, + }, + { + label: "unknown response completion", + archiveStart: 5, + duration: -1, + target: "1", + served: false, + }, + { + label: "known disjoint response interval", + archiveStart: 6, + duration: 1500, + target: "1", + served: true, + }, + { + label: "unrelated overlapping write", + archiveStart: 5, + duration: 1500, + target: "2", + served: true, + }, +])( + "requires a read free of captured write overlap: $label", + ({ archiveStart, duration, target, served, ...options }) => { + const rows: TimedCaptureRow[] = [ + [ + "GET", + "/api/books", + [ + { id: 1, name: "A" }, + { id: 2, name: "B" }, + ], + ], + ["GET", "/api/books/1", { id: 1, name: "A" }], + ["GET", "/api/books/2", { id: 2, name: "B" }], + ["PATCH", "/api/books/1", { id: 1, name: "Z" }, 200, { name: "Z" }], + ["GET", "/api/books/1", { id: 1, name: "Z" }, 200, undefined, duration], + [ + "POST", + `/api/books/${target}/archive`, + null, + 204, + {}, + "archiveDuration" in options ? options.archiveDuration : 5, + ], + ]; + const entries = rows.map( + ( + [method, path, body = null, status = 200, request, time = 5], + index, + ) => ({ + startedDateTime: new Date( + Date.parse(bookClock(index === 5 ? archiveStart : index)), + ).toISOString(), + time, + request: { + method, + url: `http://app.example${path}`, + bodySize: + request === undefined + ? 0 + : Buffer.byteLength(JSON.stringify(request)), + ...(request === undefined + ? {} + : { + postData: { + mimeType: "application/json", + text: JSON.stringify(request), + }, + }), + }, + response: { + status, + content: { mimeType: "application/json", text: JSON.stringify(body) }, + }, + }), + ); + const capture = analysisViewCaptureEvidence( + environmentManifestCaptureFixture(entries), + ); + const plan = environmentStateViewOf(capture).writes.find( + (write) => write.tool === "patch_books_by_id", + ); + expect(plan).toMatchObject({ + served, + checked_record_ordinals: served ? [4] : [], + contradicted_by: null, + }); + }, +); diff --git a/src/domain/analysisView/EnvironmentStateWriteSimulation.ts b/src/domain/analysisView/EnvironmentStateWriteSimulation.ts index 2b524a14fe..ddfa17f13a 100644 --- a/src/domain/analysisView/EnvironmentStateWriteSimulation.ts +++ b/src/domain/analysisView/EnvironmentStateWriteSimulation.ts @@ -1,3 +1,4 @@ +import { CapturedWriteCompletion } from "./environmentStateWriteCompletion.js"; import type { JsonValue } from "../jsonValue.js"; import { at, @@ -114,6 +115,7 @@ export class EnvironmentStateWriteSimulation { readonly #touchedSince: PendingTargets = new Map(); /** Timeline indexes of writes the gate does not model. */ readonly #unmodelledAt: number[] = []; + readonly #writeCompletion: CapturedWriteCompletion; readonly #history: WriteHistory = { touched: this.#touched, pending: this.#touchedSince, @@ -135,6 +137,7 @@ export class EnvironmentStateWriteSimulation { constructor(input: WriteSimulationInput) { this.#input = input; + this.#writeCompletion = new CapturedWriteCompletion(input.io.retain); this.#state = new SimulatedState(input.entities, input.documents); this.#unseen = new UnseenEntities(input.entities, { state: this.#state, @@ -224,6 +227,15 @@ export class EnvironmentStateWriteSimulation { linked: link.collection, }); } + // Reads may finish after writes with the same or a later start: index the + // complete capture before replay so ordinal tie-breaking cannot create proof. + for (const record of this.#input.records) + if (this.#isWrite(record)) + this.#writeCompletion.write( + record.path, + this.#input.chronology.endMs(record), + this.#input.chronology.startMs(record), + ); this.#input.records.forEach((record, index) => this.#step(record, index)); const views = new Map( this.views([...this.#input.views.values()]).map((view) => [ @@ -280,7 +292,7 @@ export class EnvironmentStateWriteSimulation { : this.#input.io.requestBody(ordinal), }, records: this.#input.records, - timed: this.#input.timed, + chronology: this.#input.chronology, viewOf: (ordinal) => { const view = this.#input.views.get( this.#toolOf.get(ordinal)?.name ?? "", @@ -354,7 +366,7 @@ export class EnvironmentStateWriteSimulation { index, tool: tool.name, path: record.path, - end: record.end, + end: this.#input.chronology.endMs(record), }); }; // Item deletes use their path; calls outside a built plan's query never interpret their body. @@ -370,7 +382,7 @@ export class EnvironmentStateWriteSimulation { readonly index: number; readonly tool: string; readonly path: string; - readonly end: number; + readonly end: number | undefined; }, ): void { if (result.kind === "unknown") { @@ -427,6 +439,7 @@ export class EnvironmentStateWriteSimulation { /** Whether an item read shows an entity an unmodelled write made, or one left unknown, again rather than checks it. */ #resynced(view: StateView, record: StateRecordFacts, index: number): boolean { if (view.shape !== "item") return false; + if (this.#writeOverlapsRead(record)) return true; const adopted = this.#unseen.adopt(view, { record, index }); if (adopted !== undefined) { this.#taint.known(adopted.type, adopted.collection, adopted.id); @@ -436,7 +449,14 @@ export class EnvironmentStateWriteSimulation { if (!this.#taint.unknown(view.type, collection, id)) return false; // A read begun while a modelled write to the entity still ran may show either state, so it neither checks nor shows it. const target = entityTarget(view.type, collection, id); - if (runningAt(this.#touchedSince, target, record.start)) return true; + if ( + runningAt( + this.#touchedSince, + target, + this.#input.chronology.startMs(record), + ) + ) + return true; // A failed read that is no absence shows nothing of the entity, so it stays unknown for a usable read. if (!succeeded(record) && !isAbsence(record.status)) return true; const entity = this.#state.entity(view.type, collection, id); @@ -530,6 +550,7 @@ export class EnvironmentStateWriteSimulation { if (original === undefined) return; let view = this.#sorted.comparisonView(this.omitted.view(original)); if (projectedRead(view, record, this.#input.io)) return; + if (this.#writeOverlapsRead(record)) return; const borrowers = this.#observations.get(record.ordinal); if ( this.#viewOf(original).shape !== "recorded" && @@ -544,7 +565,11 @@ export class EnvironmentStateWriteSimulation { targets.some( (key) => this.#taint.tainted(key) || - runningAt(this.#touchedSince, key, record.start), + runningAt( + this.#touchedSince, + key, + this.#input.chronology.startMs(record), + ), ); if (blocked(keys)) return; // A read the gate counts can show the view lists a written array sorted, or rule that out. @@ -650,6 +675,15 @@ export class EnvironmentStateWriteSimulation { return shown.unseen; } + /** An overlapping captured write cannot supply a stable read baseline. */ + #writeOverlapsRead(read: StateRecordFacts): boolean { + return this.#writeCompletion.runningAt( + read.path, + this.#input.chronology.startMs(read), + this.#input.chronology.endMs(read), + ); + } + #unmodelled(path: string, index: number): void { this.#taint.write(path); this.#unmodelledAt.push(index); diff --git a/src/domain/analysisView/StateRecordChronology.ts b/src/domain/analysisView/StateRecordChronology.ts new file mode 100644 index 0000000000..dbc38a986a --- /dev/null +++ b/src/domain/analysisView/StateRecordChronology.ts @@ -0,0 +1,36 @@ +import type { + StateRecordFacts, + StateTimeline, +} from "./environmentStateRules.js"; + +/** Own recorded interval facts separately from the explicit ordinal fallback policy. */ +export class StateRecordChronology { + readonly positions: ReadonlyMap; + + constructor( + records: readonly StateRecordFacts[], + readonly timeline: StateTimeline, + ) { + this.positions = new Map( + records.map((record, index) => [record.ordinal, index]), + ); + } + + startMs(record: StateRecordFacts): number | undefined { + return this.timeline === "record-order" + ? this.positions.get(record.ordinal) + : record.timing.start?.ms; + } + + endMs(record: StateRecordFacts): number | undefined { + return this.timeline === "record-order" + ? this.positions.get(record.ordinal) + : record.timing.endMs; + } + + completedBefore(left: StateRecordFacts, right: StateRecordFacts): boolean { + const end = this.endMs(left); + const start = this.startMs(right); + return end !== undefined && start !== undefined && end <= start; + } +} diff --git a/src/domain/analysisView/environmentStateGate.test.ts b/src/domain/analysisView/environmentStateGate.test.ts index 50a29b4863..3fe1781780 100644 --- a/src/domain/analysisView/environmentStateGate.test.ts +++ b/src/domain/analysisView/environmentStateGate.test.ts @@ -21,8 +21,7 @@ const check = ( path: `/api/books/${ordinal + 1}`, query: undefined, status: 200, - start: ordinal, - end: ordinal, + timing: { start: undefined, endMs: undefined }, })); const bodies: JsonObject[] = clocks.map(([, stamp], index) => ({ id: index + 1, diff --git a/src/domain/analysisView/environmentStateQueryCalls.test.ts b/src/domain/analysisView/environmentStateQueryCalls.test.ts index f8ed3e0ef2..ff63fd447c 100644 --- a/src/domain/analysisView/environmentStateQueryCalls.test.ts +++ b/src/domain/analysisView/environmentStateQueryCalls.test.ts @@ -1,3 +1,5 @@ +import { StateRecordChronology } from "./StateRecordChronology.js"; +import { parseOffsetDateTimeInstant } from "../offsetDateTime.js"; import { describe, expect, it } from "vitest"; import type { JsonValue } from "../jsonValue.js"; @@ -7,6 +9,7 @@ import { } from "./environmentStateQueryCalls.js"; import type { StateRecordFacts, + StateTimeline, StateToolFacts, } from "./environmentStateRules.js"; @@ -20,7 +23,7 @@ interface Call { readonly tool: string; readonly template?: string; readonly start?: number; - readonly end?: number; + readonly end?: number | null; } /** How the read-only pass parses response bodies and charges what it keeps of them, when a test watches either. */ @@ -30,15 +33,26 @@ interface Bodies { } /** The read-only tools of a timeline of calls, ordinals in order; a call spans its index unless it gives its times. */ -const readOnlyOf = (calls: readonly Call[], bodies: Bodies = {}): string[] => { +const readOnlyOf = ( + calls: readonly Call[], + bodies: Bodies = {}, + timeline: StateTimeline = "recorded-start-time", +): string[] => { const records: StateRecordFacts[] = calls.map((call, index) => ({ ordinal: index, method: call.method, path: call.path, query: call.query, status: call.status === undefined ? 200 : call.status, - start: call.start ?? index, - end: call.end ?? index, + timing: { + start: { + instant: parseOffsetDateTimeInstant( + new Date(call.start ?? index).toISOString(), + ), + ms: call.start ?? index, + }, + endMs: call.end === null ? undefined : (call.end ?? index), + }, })); const tools = new Map(); calls.forEach((call, index) => { @@ -53,11 +67,16 @@ const readOnlyOf = (calls: readonly Call[], bodies: Bodies = {}): string[] => { const withBody: QueryCallIo["withBody"] = bodies.withBody ?? ((ordinal, use) => use(calls[ordinal]?.response)); return [ - ...readOnlyTools([...tools.values()], records, { - withBody, - request: (ordinal) => calls[ordinal]?.request, - retain: bodies.retain ?? (() => undefined), - }), + ...readOnlyTools( + [...tools.values()], + records, + { + withBody, + request: (ordinal) => calls[ordinal]?.request, + retain: bodies.retain ?? (() => undefined), + }, + new StateRecordChronology(records, timeline), + ), ].sort(); }; @@ -316,6 +335,31 @@ describe("POSTs that read state but stay writes", () => { }); describe("the reads either side of a POST", () => { + it("requires known read completion without discarding independent known intervals", () => { + const known: Call = { + method: "GET", + path: "/api/members", + tool: "get_members", + response: JOBS, + start: 0, + end: 5, + }; + const unknown = { ...known, start: 6, end: null }; + const post: Call = { + method: "POST", + path: "/api/members", + tool: "post_members", + response: JOBS, + start: 10, + end: 11, + }; + expect(readOnlyOf([unknown, post])).toEqual([]); + expect(readOnlyOf([known, unknown, post])).toEqual(["post_members"]); + expect(readOnlyOf([unknown, post], {}, "record-order")).toEqual([ + "post_members", + ]); + }); + it("compares a POST only with reads that finished before it began or began after it ended", () => { const note = (title: string): Call => ({ method: "GET", @@ -454,8 +498,15 @@ describe("read-only decisions over many calls", () => { path: read ? "/api/members" : "/api/members/ids", query: undefined, status: 200, - start: index, - end: index, + timing: { + start: { + instant: parseOffsetDateTimeInstant( + new Date(index).toISOString(), + ), + ms: index, + }, + endMs: index, + }, }; const method = read ? "GET" : "POST"; return Object.defineProperty(record, "method", { @@ -487,11 +538,16 @@ describe("read-only decisions over many calls", () => { ]; visits = 0; expect([ - ...readOnlyTools(tools, records, { - withBody: (_ordinal, use) => use([]), - request: () => undefined, - retain: () => undefined, - }), + ...readOnlyTools( + tools, + records, + { + withBody: (_ordinal, use) => use([]), + request: () => undefined, + retain: () => undefined, + }, + new StateRecordChronology(records, "recorded-start-time"), + ), ]).toEqual(["post_ids"]); expect(visits).toBeLessThan(50 * records.length); }); @@ -510,8 +566,15 @@ describe("read-only decisions over a long run of calls", () => { path: "/v1/jobs/statuses", query: undefined, status: 200, - start: index, - end: index, + timing: { + start: { + instant: parseOffsetDateTimeInstant( + new Date(index).toISOString(), + ), + ms: index, + }, + endMs: index, + }, }; const method = read ? "GET" : "POST"; return Object.defineProperty(record, "method", { @@ -543,11 +606,16 @@ describe("read-only decisions over a long run of calls", () => { ]; visits = 0; expect([ - ...readOnlyTools(tools, records, { - withBody: (_ordinal, use) => use(JOBS), - request: () => ({ jobs: ["night-watch"] }), - retain: () => undefined, - }), + ...readOnlyTools( + tools, + records, + { + withBody: (_ordinal, use) => use(JOBS), + request: () => ({ jobs: ["night-watch"] }), + retain: () => undefined, + }, + new StateRecordChronology(records, "recorded-start-time"), + ), ]).toEqual(["post_statuses"]); expect(visits).toBeLessThan(50 * records.length); }); @@ -566,8 +634,15 @@ describe("read-only decisions over many lines", () => { method: read ? "GET" : "POST", path: read ? "/api/members" : "/api/members/ids", status: 200, - start: index, - end: index, + timing: { + start: { + instant: parseOffsetDateTimeInstant( + new Date(index).toISOString(), + ), + ms: index, + }, + endMs: index, + }, }; const query = read ? `?page=${String(index)}` : undefined; return Object.defineProperty(record, "query", { @@ -596,11 +671,16 @@ describe("read-only decisions over many lines", () => { }, ]; expect([ - ...readOnlyTools(tools, records, { - withBody: (_ordinal, use) => use([]), - request: () => undefined, - retain: () => undefined, - }), + ...readOnlyTools( + tools, + records, + { + withBody: (_ordinal, use) => use([]), + request: () => undefined, + retain: () => undefined, + }, + new StateRecordChronology(records, "recorded-start-time"), + ), ]).toEqual(["post_ids"]); expect(reads).toBeLessThan(20 * records.length); }); diff --git a/src/domain/analysisView/environmentStateQueryCalls.ts b/src/domain/analysisView/environmentStateQueryCalls.ts index 97ed34a184..c9b0794e3b 100644 --- a/src/domain/analysisView/environmentStateQueryCalls.ts +++ b/src/domain/analysisView/environmentStateQueryCalls.ts @@ -1,3 +1,4 @@ +import type { StateRecordChronology } from "./StateRecordChronology.js"; import { digestCanonicalValue } from "../canonicalDigest.js"; import { identifierWords } from "./environmentStateWords.js"; import type { JsonValue } from "../jsonValue.js"; @@ -176,10 +177,9 @@ export const readOnlyTools = ( tools: readonly StateToolFacts[], records: readonly StateRecordFacts[], io: QueryCallIo, + chronology: StateRecordChronology, ): Set => { - const position = new Map( - records.map((record, index) => [record.ordinal, index]), - ); + const position = chronology.positions; const response = responseDigests(io); /** The timeline indexes of the nearest state writes either side of an index, the skipped calls aside: -1 and the length when none, found in one pass each way. */ const nearestWrites = ( @@ -220,6 +220,7 @@ export const readOnlyTools = ( if (!calls.some(succeeded) || calls.some(unresolved)) continue; const timeline = { position, + chronology, ...nearestWrites(new Set(tool.record_ordinals)), response, }; @@ -234,6 +235,7 @@ export const readOnlyTools = ( for (let vetoed = true; vetoed;) { const timeline = { position, + chronology, ...nearestWrites( new Set([...candidates.values()].flat().map((call) => call.ordinal)), ), @@ -250,6 +252,7 @@ export const readOnlyTools = ( }; interface Timeline { + readonly chronology: StateRecordChronology; readonly position: ReadonlyMap; /** The nearest other writes either side of a timeline index. */ readonly writes: (at: number) => { @@ -266,12 +269,14 @@ interface Timeline { const lastFinishedBefore = ( reads: readonly StateRecordFacts[], call: StateRecordFacts, + chronology: StateRecordChronology, ): StateRecordFacts | undefined => { let latest: StateRecordFacts | undefined; for (const read of reads) if ( - read.end <= call.start && - (latest === undefined || read.end >= latest.end) + chronology.completedBefore(read, call) && + (latest === undefined || + chronology.endMs(read)! >= chronology.endMs(latest)!) ) latest = read; return latest; @@ -281,10 +286,10 @@ const lastFinishedBefore = ( const repeatsRead = ( calls: readonly StateRecordFacts[], records: readonly StateRecordFacts[], - { position, isWrite, response }: Timeline, + { position, isWrite, response, chronology }: Timeline, ): boolean => { const answering = calls.filter(succeeded); - const reads = lastReads(answering, records, position, isWrite); + const reads = lastReads(answering, records, position, isWrite, chronology); return answering.every((call) => { const read = reads.get(call.ordinal); if (read === undefined) return false; @@ -302,6 +307,7 @@ const lastReads = ( records: readonly StateRecordFacts[], position: ReadonlyMap, isWrite: (index: number) => boolean, + chronology: StateRecordChronology, ): Map => { const paths = new Set(calls.map((call) => call.path)); const callAt = new Map( @@ -317,12 +323,12 @@ const lastReads = ( for (let index = 0; index <= lastCall; index += 1) { const record = records[index]!; const call = callAt.get(index); - if (call !== undefined) { + if (call !== undefined && chronology.startMs(call) !== undefined) { const ready = waiting.get(call.path); for ( - let read = ready?.popEndedBy(call.start); + let read = ready?.popEndedBy(chronology.startMs(call)!); read !== undefined; - read = ready?.popEndedBy(call.start) + read = ready?.popEndedBy(chronology.startMs(call)!) ) latest.set(call.path, read); const read = latest.get(call.path); @@ -333,9 +339,11 @@ const lastReads = ( } else if ( record.method === "GET" && succeeded(record) && - paths.has(record.path) + paths.has(record.path) && + chronology.endMs(record) !== undefined ) { - const heap = waiting.get(record.path) ?? new ReadsByEnd(position); + const heap = + waiting.get(record.path) ?? new ReadsByEnd(position, chronology); heap.push(record); waiting.set(record.path, heap); } @@ -347,12 +355,15 @@ const lastReads = ( class ReadsByEnd { readonly #reads: StateRecordFacts[] = []; - constructor(private readonly position: ReadonlyMap) {} + constructor( + private readonly position: ReadonlyMap, + private readonly chronology: StateRecordChronology, + ) {} #before(left: StateRecordFacts, right: StateRecordFacts): boolean { return ( - left.end < right.end || - (left.end === right.end && + this.chronology.endMs(left)! < this.chronology.endMs(right)! || + (this.chronology.endMs(left) === this.chronology.endMs(right) && this.position.get(left.ordinal)! < this.position.get(right.ordinal)!) ); } @@ -372,7 +383,8 @@ class ReadsByEnd { popEndedBy(time: number): StateRecordFacts | undefined { const reads = this.#reads; const first = reads[0]; - if (first === undefined || first.end > time) return undefined; + if (first === undefined || this.chronology.endMs(first)! > time) + return undefined; const last = reads.pop()!; if (reads.length > 0) { reads[0] = last; @@ -431,7 +443,7 @@ const lastMeaningfulWord = (segment: string): string | undefined => const changesReadsAcross = ( calls: readonly StateRecordFacts[], records: readonly StateRecordFacts[], - { position, writes, response }: Timeline, + { position, writes, response, chronology }: Timeline, ): boolean => { const stretches = new Map< string, @@ -451,7 +463,8 @@ const changesReadsAcross = ( else stretch.calls.push(call); } for (const stretch of stretches.values()) - if (changesInStretch(stretch, records, { position, response })) return true; + if (changesInStretch(stretch, records, { position, response, chronology })) + return true; return false; }; @@ -464,7 +477,11 @@ const changesInStretch = ( readonly calls: readonly StateRecordFacts[]; }, records: readonly StateRecordFacts[], - { position, response }: Pick, + { + position, + response, + chronology, + }: Pick, ): boolean => { const shown = (record: StateRecordFacts): string | symbol => succeeded(record) ? (response(record.ordinal) ?? UNKNOWN) : ABSENT; @@ -493,9 +510,12 @@ const changesInStretch = ( const earlier = lastFinishedBefore( reads.filter((read) => position.get(read.ordinal)! < at), call, + chronology, ); const later = reads.find( - (read) => position.get(read.ordinal)! > at && read.start >= call.end, + (read) => + position.get(read.ordinal)! > at && + chronology.completedBefore(call, read), ); if (earlier === undefined || later === undefined) return false; const [was, now] = [shown(earlier), shown(later)]; @@ -515,11 +535,12 @@ export const stateWritesOf = ( tools: readonly StateToolFacts[], records: readonly StateRecordFacts[], io: QueryCallIo, + chronology: StateRecordChronology, ): { readonly readOnly: ReadonlySet; readonly writes: readonly StateRecordFacts[]; } => { - const readOnly = readOnlyTools(tools, records, io); + const readOnly = readOnlyTools(tools, records, io, chronology); const reads = new Set( tools .filter((tool) => readOnly.has(tool.name)) diff --git a/src/domain/analysisView/environmentStateRewrites.test.ts b/src/domain/analysisView/environmentStateRewrites.test.ts new file mode 100644 index 0000000000..68e8e9b6dd --- /dev/null +++ b/src/domain/analysisView/environmentStateRewrites.test.ts @@ -0,0 +1,112 @@ +import { expect, it } from "vitest"; +import { parseOffsetDateTimeInstant } from "../offsetDateTime.js"; +import { StateRecordChronology } from "./StateRecordChronology.js"; +import { rewriteCuts } from "./environmentStateRewrites.js"; +import { + writeSpanCheck, + writeSpanIndex, + type StateRecordFacts, +} from "./environmentStateRules.js"; + +const record = ( + ordinal: number, + method: string, + start: number, + endMs: number | undefined, +): StateRecordFacts => ({ + ordinal, + method, + path: "/api/items/1", + query: undefined, + status: 200, + timing: { + start: { + instant: parseOffsetDateTimeInstant(new Date(start).toISOString()), + ms: start, + }, + endMs, + }, +}); + +it.each([ + { label: "unknown duration", start: 10, end: undefined, possible: true }, + { label: "completed before the reads", start: 10, end: 19, possible: false }, + { + label: "completed at the first read's start", + start: 10, + end: 20, + possible: false, + }, + { + label: "known zero duration at the boundary", + start: 20, + end: 20, + possible: false, + }, + { label: "known overlap", start: 10, end: 21, possible: true }, +])( + "keeps rewrite cuts conservative for an earlier write with $label", + ({ start, end, possible }) => { + const write = record(0, "PUT", start, end); + const earlier = record(1, "GET", 20, 25); + const later = record(2, "GET", 30, 35); + const records = [write, earlier, later]; + const chronology = new StateRecordChronology( + records, + "recorded-start-time", + ); + const cuts = rewriteCuts( + [{ type: "/api/items", earlier: earlier.ordinal, later: later.ordinal }], + [write], + new Map(records.map((value) => [value.ordinal, value])), + chronology, + ); + expect(writeSpanCheck([write], chronology)(earlier, later)).toBe(possible); + expect(cuts.get("/api/items")).toBe(possible ? write : undefined); + if (end === undefined) { + // A possible explanation prevents an independence proof; it does not prove + // that the write completed, or that it caused the observed difference. + expect(chronology.completedBefore(write, earlier)).toBe(false); + } + }, +); + +it("selects the earliest possibly interfering write without replacing a known completed predecessor", () => { + const finished = record(0, "PUT", 0, 5); + const unresolved = record(1, "PUT", 10, undefined); + const earlier = record(2, "GET", 20, 25); + const later = record(3, "GET", 30, 35); + const records = [finished, unresolved, earlier, later]; + const chronology = new StateRecordChronology(records, "recorded-start-time"); + const writes = [finished, unresolved]; + expect(writeSpanIndex(writes, chronology)(earlier, later)).toBe(1); + expect( + rewriteCuts( + [{ type: "/api/items", earlier: earlier.ordinal, later: later.ordinal }], + writes, + new Map(records.map((value) => [value.ordinal, value])), + chronology, + ).get("/api/items"), + ).toBe(unresolved); +}); + +it.each([35, undefined])( + "does not invent a later read's completion when locating possible writes (end=%s)", + (end) => { + const earlier = record(0, "GET", 20, 25); + const later = record(1, "GET", 30, end); + const write = record(2, "PUT", 40, 45); + const records = [earlier, later, write]; + const chronology = new StateRecordChronology( + records, + "recorded-start-time", + ); + const cuts = rewriteCuts( + [{ type: "/api/items", earlier: earlier.ordinal, later: later.ordinal }], + [write], + new Map(records.map((value) => [value.ordinal, value])), + chronology, + ); + expect(cuts.get("/api/items")).toBe(end === undefined ? write : undefined); + }, +); diff --git a/src/domain/analysisView/environmentStateRewrites.ts b/src/domain/analysisView/environmentStateRewrites.ts index b80bd83418..5a11662b98 100644 --- a/src/domain/analysisView/environmentStateRewrites.ts +++ b/src/domain/analysisView/environmentStateRewrites.ts @@ -1,4 +1,8 @@ -import type { StateRecordFacts } from "./environmentStateRules.js"; +import { + writeSpanIndex, + type StateRecordFacts, +} from "./environmentStateRules.js"; +import type { StateRecordChronology } from "./StateRecordChronology.js"; /** Two recordings of a type that disagree across a write. */ export interface RewriteSpan { @@ -8,40 +12,24 @@ export interface RewriteSpan { } /** - * Per type whose reads disagree across writes, the earliest write within any such span: one that - * began before the later recording ended and ended after the earlier began, in timeline order. + * Per type whose reads disagree, the earliest write that may intersect the span. + * Unknown interval bounds cannot establish that the observations were independent + * of a write; choosing a conservative cutoff does not prove the write caused a change. */ export const rewriteCuts = ( spans: readonly RewriteSpan[], writes: readonly StateRecordFacts[], byOrdinal: ReadonlyMap, + chronology: StateRecordChronology, ): Map => { - // The latest end among each prefix of writes, so the first write ending after a time is found by bisection. - const reach: number[] = []; - for (const write of writes) - reach.push(Math.max(reach.at(-1) ?? Number.NEGATIVE_INFINITY, write.end)); - const starts = writes.map((write) => write.start); - /** The first index whose value passes `test`, which once true stays true. */ - const first = ( - values: readonly number[], - test: (value: number) => boolean, - ) => { - let [low, high] = [0, values.length]; - while (low < high) { - const middle = (low + high) >>> 1; - if (test(values[middle]!)) high = middle; - else low = middle + 1; - } - return low; - }; + const possibleWrite = writeSpanIndex(writes, chronology); const cuts = new Map(); for (const span of spans) { const earlier = byOrdinal.get(span.earlier); const later = byOrdinal.get(span.later); if (earlier === undefined || later === undefined) continue; - // Writes come in start order: those begun before the later recording ended are a prefix. - const index = first(reach, (end) => end > earlier.start); - if (index < first(starts, (start) => start >= later.end)) + const index = possibleWrite(earlier, later); + if (index !== undefined) cuts.set(span.type, Math.min(index, cuts.get(span.type) ?? index)); } return new Map([...cuts].map(([type, index]) => [type, writes[index]!])); diff --git a/src/domain/analysisView/environmentStateRules.ts b/src/domain/analysisView/environmentStateRules.ts index e4746e88ad..979a878b46 100644 --- a/src/domain/analysisView/environmentStateRules.ts +++ b/src/domain/analysisView/environmentStateRules.ts @@ -1,3 +1,5 @@ +import type { OffsetDateTimeInstant } from "../offsetDateTime.js"; +import type { StateRecordChronology } from "./StateRecordChronology.js"; import type { ListQuery } from "./environmentStateListQuery.js"; import { canonicalJson } from "../comparisonSemantics.js"; import { compareUnicodeCodePoints } from "../unicodeCodePointOrder.js"; @@ -21,8 +23,12 @@ export interface StateRecordFacts { /** WHATWG query spelling including `?`, or undefined when the request had none. */ readonly query: string | undefined; readonly status: number | null; - readonly start: number; - readonly end: number; + readonly timing: { + readonly start: + | { readonly instant: OffsetDateTimeInstant; readonly ms: number } + | undefined; + readonly endMs: number | undefined; + }; } /** The manifest tool facts extraction uses. */ @@ -392,26 +398,54 @@ export const isStateWrite = (record: StateRecordFacts): boolean => changesState(record.method, record.path) && (record.status === null || (record.status >= 200 && record.status < 400)); -/** Whether any write overlaps the span from one record's start to a later record's end; writes come in timeline order. */ -export const writeSpanCheck = ( +/** Index possible write interference without inventing completion for unknown intervals. */ +export const writeSpanIndex = ( writes: readonly StateRecordFacts[], -): ((earlier: StateRecordFacts, later: StateRecordFacts) => boolean) => { - // The latest end among each prefix of writes, so one search answers a span. - const reach: number[] = []; - for (const write of writes) - reach.push(Math.max(reach.at(-1) ?? Number.NEGATIVE_INFINITY, write.end)); + chronology: StateRecordChronology, +): (( + earlier: StateRecordFacts, + later: StateRecordFacts, +) => number | undefined) => { + const reach: (number | undefined)[] = []; + const starts: (number | undefined)[] = []; + let maximum: number | undefined; + let unknown = false; + for (const write of writes) { + const end = chronology.endMs(write); + if (end === undefined) unknown = true; + else maximum = maximum === undefined ? end : Math.max(maximum, end); + reach.push(unknown ? undefined : maximum); + starts.push(chronology.startMs(write)); + } return (earlier, later) => { + const beginning = chronology.startMs(earlier); + const ending = chronology.endMs(later); let low = 0; let high = writes.length; while (low < high) { const middle = (low + high) >>> 1; - if (writes[middle]!.start < later.end) low = middle + 1; - else high = middle; + const end = reach[middle]; + if (beginning === undefined || end === undefined || end > beginning) + high = middle; + else low = middle + 1; } - return low > 0 && reach[low - 1]! > earlier.start; + const start = starts[low]; + return low < writes.length && + (ending === undefined || start === undefined || start < ending) + ? low + : undefined; }; }; +/** Whether a write may overlap the span; unknown completion cannot prove absence. */ +export const writeSpanCheck = ( + writes: readonly StateRecordFacts[], + chronology: StateRecordChronology, +): ((earlier: StateRecordFacts, later: StateRecordFacts) => boolean) => { + const index = writeSpanIndex(writes, chronology); + return (earlier, later) => index(earlier, later) !== undefined; +}; + /** Literal leading segments shared by every template with the same first segment: each path's API base. */ export const apiBaseLengths = ( templates: readonly string[], diff --git a/src/domain/analysisView/environmentStateView.test.ts b/src/domain/analysisView/environmentStateView.test.ts index a10be1cc3c..3d21159ffd 100644 --- a/src/domain/analysisView/environmentStateView.test.ts +++ b/src/domain/analysisView/environmentStateView.test.ts @@ -41,6 +41,8 @@ import { type LineShape, } from "./environmentStateVariants.js"; import { rewriteCuts } from "./environmentStateRewrites.js"; +import { StateRecordChronology } from "./StateRecordChronology.js"; +import { offsetDateTimeInstantFromEpochMilliseconds } from "../offsetDateTime.js"; import { ENTRY_BYTES } from "./webNetworkRouteTemplates.js"; type State = Extract; @@ -839,8 +841,7 @@ it("leaves every line checked rather than compare more lines than the variant pa path: "/api/items", query: `?view=${String(ordinal)}`, status: 200, - start: ordinal, - end: ordinal, + timing: { start: undefined, endMs: undefined }, })); const shape = { entities: new Map( @@ -867,8 +868,7 @@ it("tells apart member names that hold the separator, so a line spelling them ot path: "/api/items/1", query, status: 200, - start: ordinal, - end: ordinal, + timing: { start: undefined, endMs: undefined }, }); const observed = (ordinal: number, object: JsonObject) => ({ position: ordinal, @@ -908,8 +908,7 @@ it("leaves every line checked when comparing many mostly empty lines would excee method: "GET", path: "/api/items", status: 200, - start: ordinal, - end: ordinal, + timing: { start: undefined, endMs: undefined }, }; return Object.defineProperty(record, "query", { enumerable: true, @@ -950,8 +949,7 @@ const expandedItemLines = (items: number) => { path: `/api/items/${String(ordinal % items)}`, query: ordinal < items ? undefined : "?expand=shelf", status: 200, - start: ordinal, - end: ordinal, + timing: { start: undefined, endMs: undefined }, }), ); const shapes = new Map( @@ -1137,11 +1135,21 @@ it("finds each span's earliest write without scanning the writes before it", () path: "/api/log", query: undefined, status: 200, + timing: { + start: { + instant: offsetDateTimeInstantFromEpochMilliseconds(start), + get ms() { + reads += 1; + return start; + }, + }, + get endMs() { + reads += 1; + return end; + }, + }, }; - return Object.defineProperties(record, { - start: { enumerable: true, get: () => ((reads += 1), start) }, - end: { enumerable: true, get: () => ((reads += 1), end) }, - }) as typeof record & { readonly start: number; readonly end: number }; + return record; }; const count = 2000; const writes = Array.from({ length: count }, (_, index) => @@ -1162,7 +1170,11 @@ it("finds each span's earliest write without scanning the writes before it", () later: earlier, })); reads = 0; - const cuts = rewriteCuts(spans, writes, reads_); + const chronology = new StateRecordChronology( + [...writes, ...reads_.values()], + "recorded-start-time", + ); + const cuts = rewriteCuts(spans, writes, reads_, chronology); expect(cuts.size).toBe(count - 1); expect(reads).toBeLessThan(200 * count); }); @@ -1423,6 +1435,125 @@ it.each<{ }, ); +it.each([ + { duration: undefined, redactedWrite: false }, + { duration: null, redactedWrite: false }, + { duration: -1, redactedWrite: false }, + { duration: "0", redactedWrite: false }, + { duration: undefined, redactedWrite: true }, + { duration: null, redactedWrite: true }, +])( + "retains chronological initial state with duration $duration and redacted write $redactedWrite", + ({ duration, redactedWrite }) => { + const entry = ( + method: string, + path: string | null, + second: string, + body: JsonValue, + ) => ({ + startedDateTime: `2026-10-20T09:00:${second}Z`, + time: 0, + request: { + method, + url: path === null ? null : `http://app.example${path}`, + }, + response: { + status: 200, + content: { mimeType: "application/json", text: JSON.stringify(body) }, + }, + }); + const { time: _duration, ...laterRead } = entry( + "GET", + "/api/settings", + "02", + { theme: "dark" }, + ); + const entries = [ + entry("GET", "/api/settings", "00", { theme: "light" }), + { ...laterRead, ...(duration === undefined ? {} : { time: duration }) }, + entry("PATCH", redactedWrite ? null : "/api/settings", "01", { + theme: "dark", + }), + entry("GET", "/api/health", "00.1", { healthy: true }), + entry("GET", "/api/version", "00.2", { version: "1" }), + ]; + const capture = webNetworkCaptureSchema.parse( + environmentManifestCaptureFixture(entries), + ); + const before = structuredClone(capture); + const state = stateOf(parentOf(analysisViewCaptureEvidence(capture))); + expect(state.timeline).toBe("recorded-start-time"); + expect(state.limitations).toContain( + "Some records carry no usable request duration; their completion and overlap remain unknown.", + ); + expect(state.documents).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + path: "/api/settings", + value: { theme: "light" }, + record_ordinal: 0, + }), + expect.objectContaining({ + path: "/api/health", + value: { healthy: true }, + record_ordinal: 3, + }), + expect.objectContaining({ + path: "/api/version", + value: { version: "1" }, + record_ordinal: 4, + }), + ]), + ); + expect(capture).toEqual(before); + if (redactedWrite) + expect(state.limitations).toContain( + "1 of the recorded writes had a redacted URL; each ends the initial state of every type, since its target is unknown.", + ); + }, +); + +it("does not confirm completion of an untimed delete while preserving unrelated known reads", () => { + const source = captureOf([ + ["GET", "/api/items", [{ id: 1 }, { id: 2 }]], + ["GET", "/api/items/2", { id: 2 }], + ["GET", "/api/health", { healthy: true }], + ["GET", "/api/version", { version: "1" }], + ["DELETE", "/api/items/2", "", 204], + ["GET", "/api/items/2", null, 404], + ]); + const known = stateOf(source); + expect(known.writes).toContainEqual( + expect.objectContaining({ tool: "delete_items_by_id", served: true }), + ); + const result = structuredClone(source.normalizedResult) as { + records: { reported: { time?: JsonValue } }[]; + }; + delete result.records[4]!.reported.time; + const before = structuredClone(result); + const state = stateOf({ ...source, normalizedResult: result }); + expect(state.timeline).toBe("recorded-start-time"); + expect(state.limitations).toContain( + "Some records carry no usable request duration; their completion and overlap remain unknown.", + ); + expect(state.writes).toContainEqual( + expect.objectContaining({ tool: "delete_items_by_id", served: false }), + ); + expect(state.documents).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + path: "/api/health", + value: { healthy: true }, + }), + expect.objectContaining({ + path: "/api/version", + value: { version: "1" }, + }), + ]), + ); + expect(result).toEqual(before); +}); + it("bounds a parameterized namespace by concrete writes before publishing new items", () => { const result = projectAnalysisView( captureOf([ diff --git a/src/domain/analysisView/environmentStateWriteAttribution.test.ts b/src/domain/analysisView/environmentStateWriteAttribution.test.ts new file mode 100644 index 0000000000..1c7bc7c2eb --- /dev/null +++ b/src/domain/analysisView/environmentStateWriteAttribution.test.ts @@ -0,0 +1,44 @@ +import { expect, it } from "vitest"; +import { + forgetChecked, + runningAt, + touchTarget, + WriteEffects, + type PendingTargets, + type TouchedTargets, +} from "./environmentStateWriteAttribution.js"; + +it("keeps an unknown write completion pending alongside later known completions", () => { + const touched: TouchedTargets = new Map(); + const pending: PendingTargets = new Map(); + for (const [index, end] of [undefined, 20].entries()) + touchTarget(touched, pending, "entity", { + index, + tool: `write-${String(index)}`, + end, + fields: ["name"], + }); + expect(runningAt(pending, "entity", 30)).toBe(true); + expect(runningAt(pending, "entity", undefined)).toBe(true); + expect(runningAt(pending, "absent", undefined)).toBe(false); + forgetChecked( + { touched, since: pending, effects: new WriteEffects() }, + "entity", + undefined, + ); + expect(runningAt(pending, "entity", 30)).toBe(false); +}); + +it("requires a known read start to establish a known pending write completed", () => { + const touched: TouchedTargets = new Map(); + const pending: PendingTargets = new Map(); + touchTarget(touched, pending, "entity", { + index: 0, + tool: "write", + end: 20, + fields: ["name"], + }); + expect(runningAt(pending, "entity", undefined)).toBe(true); + expect(runningAt(pending, "entity", 19)).toBe(true); + expect(runningAt(pending, "entity", 20)).toBe(false); +}); diff --git a/src/domain/analysisView/environmentStateWriteAttribution.ts b/src/domain/analysisView/environmentStateWriteAttribution.ts index 119ccca5f4..cba2d2e655 100644 --- a/src/domain/analysisView/environmentStateWriteAttribution.ts +++ b/src/domain/analysisView/environmentStateWriteAttribution.ts @@ -28,12 +28,14 @@ export type TouchedTargets = Map>; /** Per target, when the latest modelled write since its last check ended, and each tool's latest call there. */ export type PendingTargets = Map< string, - { until: number; readonly at: Map } + { until: number | undefined; readonly at: Map } >; /** Whether a write that ended at `until` was still running when a read began at `start`, so the read may show either state. */ -export const overlapsWrite = (until: number, start: number): boolean => - until > start; +export const overlapsWrite = ( + until: number | undefined, + start: number | undefined, +): boolean => until === undefined || start === undefined || until > start; /** What the gate tracks of the writes since each target's last check, and when each unmodelled write ran. */ export interface WriteHistory { @@ -76,8 +78,11 @@ export const interference = ( export const runningAt = ( pending: PendingTargets, key: string, - start: number, -): boolean => overlapsWrite(pending.get(key)?.until ?? -Infinity, start); + start: number | undefined, +): boolean => { + const held = pending.get(key); + return held !== undefined && overlapsWrite(held.until, start); +}; /** The tools whose writes a read exercises: those that changed its targets since they were last checked, less those that borrowed the read to learn their effect. */ export const exercisedBy = ( @@ -100,13 +105,18 @@ export const touchTarget = ( write: { readonly index: number; readonly tool: string; - readonly end: number; + readonly end: number | undefined; readonly fields: readonly string[] | undefined; }, ): void => { const held = pending.get(target); pending.set(target, { - until: Math.max(held?.until ?? -Infinity, write.end), + // One unknown completion keeps this target pending until a real check clears it. + until: + write.end === undefined || + (held !== undefined && held.until === undefined) + ? undefined + : Math.max(held?.until ?? -Infinity, write.end), at: (held?.at ?? new Map()).set(write.tool, write.index), }); const tools = touched.get(target) ?? new Map(); diff --git a/src/domain/analysisView/environmentStateWriteCompletion.test.ts b/src/domain/analysisView/environmentStateWriteCompletion.test.ts new file mode 100644 index 0000000000..68e59028fa --- /dev/null +++ b/src/domain/analysisView/environmentStateWriteCompletion.test.ts @@ -0,0 +1,112 @@ +import { expect, it, vi } from "vitest"; +import * as paths from "./environmentStateRules.js"; +import { prefixRelated } from "./environmentStateWriteRules.js"; +import { CapturedWriteCompletion } from "./environmentStateWriteCompletion.js"; + +it("preserves the existing prefix relation, including non-transitive decoding and parameters", () => { + const spellings = [ + "", + "api", + "a", + "%61", + "%2561", + "%2F", + "%252F", + "{id}", + "%ZZ", + ]; + const candidates = [ + "/", + ...spellings.flatMap((value) => [ + `/api/${value}`, + `/api/${value}/child`, + `/${value}/other`, + ]), + ]; + for (const write of candidates) { + const completion = new CapturedWriteCompletion(() => {}); + completion.write(write, undefined); + for (const read of candidates) + expect(completion.runningAt(read, 100), `${write} -> ${read}`).toBe( + prefixRelated(write, read), + ); + } +}); + +it("retains unknown completion and isolates unrelated writes while aggregating known ends", () => { + const completion = new CapturedWriteCompletion(() => {}); + completion.write("/api/books/1", 20); + completion.write("/api/books/1", 30); + completion.write("/api/books/2", undefined); + completion.write("/api/books/2", 10); + expect(completion.runningAt("/api/books/1", 29)).toBe(true); + expect(completion.runningAt("/api/books/1", 30)).toBe(false); + expect(completion.runningAt("/api/books/1", undefined)).toBe(true); + expect(completion.runningAt("/api/books/2", 100)).toBe(true); + expect(completion.runningAt("/api/books", 100)).toBe(true); + expect(completion.runningAt("/api/authors", undefined)).toBe(false); + completion.write("/", undefined); + expect(completion.runningAt("/api/authors", 100)).toBe(true); +}); + +it("uses explicit record-order coordinates without treating absent writes as pending", () => { + const completion = new CapturedWriteCompletion(() => {}); + expect(completion.runningAt("/api/books", undefined)).toBe(false); + completion.write("/api/books", 1); + expect(completion.runningAt("/api/books/1", 2)).toBe(false); +}); + +it("looks up a concrete path without reinterpreting every retained write path", () => { + const completion = new CapturedWriteCompletion(() => {}); + for (let id = 0; id < 128; id += 1) + completion.write(`/api/books/${String(id)}`, undefined); + const decoded = vi.spyOn(paths, "decodedSegment"); + try { + expect(completion.runningAt("/api/books/64", 100)).toBe(true); + // Matching interprets only the three read segments; prior write segments were indexed once. + expect(decoded).toHaveBeenCalledTimes(3); + } finally { + decoded.mockRestore(); + } +}); + +it("reserves retained prefix growth before publishing an unbudgeted edge", () => { + const refused = new Error("state retained-memory budget exhausted"); + let remaining = Infinity; + const completion = new CapturedWriteCompletion((bytes) => { + if (bytes > remaining) throw refused; + remaining -= bytes; + }); + completion.write("/api/books/1", 10); + remaining = 0; + // Updating an indexed path retains no new keys or nodes. + completion.write("/api/books/1", 20); + expect(completion.runningAt("/api/books/1", 19)).toBe(true); + expect(() => completion.write("/api/authors/1", undefined)).toThrow(refused); + expect(completion.runningAt("/api/authors/1", 100)).toBe(false); + expect(completion.runningAt("/api", 100)).toBe(false); + expect(completion.runningAt("/", 100)).toBe(false); + expect(completion.runningAt("/api/{collection}", 100)).toBe(false); + expect(completion.runningAt("/api/books/1", 20)).toBe(false); +}); + +it("queries both interval boundaries across future writes in chronology order", () => { + const completion = new CapturedWriteCompletion(() => {}); + completion.write("/api/books/1", 15, 10); + completion.write("/api/books/1", undefined, 30); + expect(completion.runningAt("/api/books/1", 5, 10)).toBe(false); + expect(completion.runningAt("/api/books/1", 5, 11)).toBe(true); + expect(completion.runningAt("/api/books/1", 15, 30)).toBe(false); + expect(completion.runningAt("/api/books/1", 15, 31)).toBe(true); + expect(completion.runningAt("/api/books/1", 15, undefined)).toBe(true); + expect(completion.runningAt("/api/authors", 5, undefined)).toBe(false); +}); + +it("keeps a simultaneous unknown write ambiguous for a point read, unlike a known point write", () => { + const completion = new CapturedWriteCompletion(() => {}); + completion.write("/api/books/1", undefined, 10); + completion.write("/api/books/2", 10, 10); + expect(completion.runningAt("/api/books/1", 10, 10)).toBe(true); + expect(completion.runningAt("/api/books/2", 10, 10)).toBe(false); + expect(completion.runningAt("/api/books/3", 10, 10)).toBe(false); +}); diff --git a/src/domain/analysisView/environmentStateWriteCompletion.ts b/src/domain/analysisView/environmentStateWriteCompletion.ts new file mode 100644 index 0000000000..dec7dde716 --- /dev/null +++ b/src/domain/analysisView/environmentStateWriteCompletion.ts @@ -0,0 +1,149 @@ +import { ENTRY_BYTES, textBytes } from "./webNetworkRouteTemplates.js"; +import { + decodedSegment, + isParam, + segmentsOf, +} from "./environmentStateRules.js"; + +/** Ordered write starts and the latest completion among writes through each start. */ +class CompletionHistory { + readonly #starts: number[] = []; + readonly #ends: (number | undefined)[] = []; + + constructor(private readonly retain: (bytes: number) => void) {} + + reserve(start: number): void { + if (this.#starts.at(-1) !== start) this.retain(ENTRY_BYTES); + } + + append(start: number, end: number | undefined): void { + const previous = this.#ends.at(-1); + const maximum = + end === undefined || (this.#ends.length > 0 && previous === undefined) + ? undefined + : Math.max(previous ?? -Infinity, end); + if (this.#starts.at(-1) === start) + this.#ends[this.#ends.length - 1] = maximum; + else { + this.#starts.push(start); + this.#ends.push(maximum); + } + } + + overlaps(start: number | undefined, end: number | undefined): boolean { + // Admit same-start writes for a point observation; a known zero-length write + // is still excluded by its completion below. Otherwise starts at read end are disjoint. + // Find writes begun before the read completed. Unknown read completion + // admits every captured write; all histories are built in chronology order. + let low = 0; + let high = this.#starts.length; + while (low < high) { + const middle = Math.floor((low + high) / 2); + if ( + end === undefined || + this.#starts[middle]! < end || + (start === end && this.#starts[middle] === end) + ) + low = middle + 1; + else high = middle; + } + if (low === 0) return false; + const completion = this.#ends[low - 1]; + return ( + completion === undefined || start === undefined || completion > start + ); + } +} + +class CompletionPrefix { + terminal: CompletionHistory | undefined; + subtree: CompletionHistory | undefined; + readonly children: Map; + readonly decoded: Map>; + parameter: CompletionPrefix | undefined; + + constructor(private readonly retain: (bytes: number) => void) { + // Node, child maps, parameter link and both history owners with their arrays. + retain(ENTRY_BYTES * 10); + this.children = new Map(); + this.decoded = new Map(); + } + + child(segment: string): CompletionPrefix { + if (isParam(segment)) + return (this.parameter ??= new CompletionPrefix(this.retain)); + const held = this.children.get(segment); + if (held !== undefined) return held; + // Reserve edge entries, inverse-match membership and both retained spellings + // before decoding or allocating. A once-decoded segment cannot grow in length. + this.retain(ENTRY_BYTES * 4 + textBytes(segment.length) * 2); + const child = new CompletionPrefix(this.retain); + const decoded = decodedSegment(segment); + this.children.set(segment, child); + this.decoded.set( + decoded, + (this.decoded.get(decoded) ?? new Set()).add(child), + ); + return child; + } + + /** Keep verbatim, once-decoded and inverse-decoded matches distinct: decoding is not transitive. */ + matching(segment: string): Set { + if (isParam(segment)) + return new Set([ + ...this.children.values(), + ...(this.parameter === undefined ? [] : [this.parameter]), + ]); + const matches = new Set(this.decoded.get(segment)); + for (const spelling of [segment, decodedSegment(segment)]) { + const child = this.children.get(spelling); + if (child !== undefined) matches.add(child); + } + if (this.parameter !== undefined) matches.add(this.parameter); + return matches; + } +} + +/** Index captured write intervals by the same segment-prefix relation as write attribution. */ +export class CapturedWriteCompletion { + readonly #root: CompletionPrefix; + + constructor(private readonly retain: (bytes: number) => void) { + this.#root = new CompletionPrefix(retain); + } + + /** Admit writes in nondecreasing chronology start order. */ + write(path: string, end: number | undefined, start = -Infinity): void { + let node = this.#root; + const ancestors = [node]; + // Finish budgeted path admission before publishing any completion summary. + // A refused edge may leave reserved empty prefixes, but no write facts. + for (const segment of segmentsOf(path)) { + node = node.child(segment); + ancestors.push(node); + } + const histories = ancestors.map( + (ancestor) => (ancestor.subtree ??= new CompletionHistory(this.retain)), + ); + histories.push((node.terminal ??= new CompletionHistory(this.retain))); + for (const history of histories) history.reserve(start); + for (const history of histories) history.append(start, end); + } + + /** Unknown completion or read start cannot prove that a related write finished. */ + runningAt(path: string, start: number | undefined, end = Infinity): boolean { + const running = (history: CompletionHistory | undefined): boolean => + history?.overlaps(start, end) === true; + let nodes = new Set([this.#root]); + for (const segment of segmentsOf(path)) { + const next = new Set(); + for (const node of nodes) { + if (running(node.terminal)) return true; + for (const child of node.matching(segment)) next.add(child); + } + if (next.size === 0) return false; + nodes = next; + } + return [...nodes].some((node) => running(node.subtree)); + } +} diff --git a/src/domain/analysisView/environmentStateWriteCutoffs.test.ts b/src/domain/analysisView/environmentStateWriteCutoffs.test.ts index 664a90034d..09fedb7c96 100644 --- a/src/domain/analysisView/environmentStateWriteCutoffs.test.ts +++ b/src/domain/analysisView/environmentStateWriteCutoffs.test.ts @@ -4,6 +4,7 @@ import { rootOf, type StateRecordFacts, } from "./environmentStateRules.js"; +import { offsetDateTimeInstantFromEpochMilliseconds } from "../offsetDateTime.js"; import { StateWriteCutoffs } from "./environmentStateWriteCutoffs.js"; const write = (path: string, ordinal: number): StateRecordFacts => ({ @@ -12,8 +13,13 @@ const write = (path: string, ordinal: number): StateRecordFacts => ({ method: "POST", query: undefined, status: 200, - start: ordinal, - end: ordinal, + timing: { + start: { + instant: offsetDateTimeInstantFromEpochMilliseconds(ordinal), + ms: ordinal, + }, + endMs: ordinal, + }, }); it("matches prefix and parameter semantics against an independent linear oracle", () => { @@ -46,12 +52,12 @@ it("matches prefix and parameter semantics against an independent linear oracle" const index = new StateWriteCutoffs( selection, bases, - (a, b) => a.start < b.start, + (a, b) => a.timing.start!.ms < b.timing.start!.ms, () => {}, ); for (const root of patterns) { const expected = [...selection] - .sort((a, b) => a.start - b.start) + .sort((a, b) => a.timing.start!.ms - b.timing.start!.ms) .find((record) => { const candidate = rootOf(record.path, bases); const shared = Math.min(root.length, candidate.length); diff --git a/src/domain/analysisView/environmentStateWriteEffects.ts b/src/domain/analysisView/environmentStateWriteEffects.ts index a5ca517de0..254438dee2 100644 --- a/src/domain/analysisView/environmentStateWriteEffects.ts +++ b/src/domain/analysisView/environmentStateWriteEffects.ts @@ -1,3 +1,4 @@ +import type { StateRecordChronology } from "./StateRecordChronology.js"; import { jsonTextCodeUnits } from "../jsonSerialization.js"; import type { JsonValue } from "../jsonValue.js"; import { @@ -133,7 +134,7 @@ export interface WriteSimulationInput { readonly records: readonly StateRecordFacts[]; /** POST tools whose calls read state rather than change it. */ readonly readOnly: ReadonlySet; - readonly timed: boolean; + readonly chronology: StateRecordChronology; readonly views: ReadonlyMap; readonly types: readonly StateType[]; readonly entities: readonly StateEntity[]; @@ -147,7 +148,7 @@ export interface EffectContext { readonly state: SimulatedState; readonly io: WriteIo; readonly records: readonly StateRecordFacts[]; - readonly timed: boolean; + readonly chronology: StateRecordChronology; /** Whether a record may change state: a write the capture does not show reading only. */ readonly isWrite: (record: StateRecordFacts) => boolean; readonly viewOf: (ordinal: number) => StateView | undefined; @@ -276,12 +277,15 @@ export const referencedIn = export const startOf = ( context: EffectContext, step: WriteStep, -): number | undefined => (context.timed ? step.record.start : undefined); +): number | undefined => + context.chronology.timeline === "recorded-start-time" + ? step.record.timing.start?.ms + : undefined; export const searchFrom = (context: EffectContext, step: WriteStep) => ({ records: context.records, from: step.index, - ...(context.timed ? { after: step.record.end } : {}), + chronology: context.chronology, viewOf: context.viewOf, withBody: context.io.withBody, queryOf: context.io.queryOf, diff --git a/src/domain/analysisView/environmentStateWriteLinks.test.ts b/src/domain/analysisView/environmentStateWriteLinks.test.ts index 85b40df3d4..8ac06c376f 100644 --- a/src/domain/analysisView/environmentStateWriteLinks.test.ts +++ b/src/domain/analysisView/environmentStateWriteLinks.test.ts @@ -1,3 +1,5 @@ +import { StateRecordChronology } from "./StateRecordChronology.js"; +import { parseOffsetDateTimeInstant } from "../offsetDateTime.js"; import { describe, expect, it } from "vitest"; import type { JsonValue } from "../jsonValue.js"; @@ -15,8 +17,13 @@ const record = ( path, query: undefined, status: 200, - start, - end, + timing: { + start: { + instant: parseOffsetDateTimeInstant(new Date(start).toISOString()), + ms: start, + }, + endMs: end, + }, }); /** The creation limit of `/api/projects` over list reads and a POST returning `created`, in ordinal order. */ @@ -35,6 +42,10 @@ const projectsLimit = ( creationLimit( [{ record: post, ids: created }], { + chronology: new StateRecordChronology( + [...reads, post], + "recorded-start-time", + ), type: "/api/projects", tools: [ { @@ -52,6 +63,52 @@ const projectsLimit = ( undefined, ); +describe("short-id creation limits", () => { + it("does not use an unknown completion as the prior read needed to link a short id", () => { + const prior = record(0, "GET", "/api/projects", [0, 5]); + const unknown = { + ...prior, + timing: { ...prior.timing, endMs: undefined }, + }; + const post = record(1, "POST", "/api/imports", [10, 11]); + const later = record(2, "GET", "/api/projects/7", [20, 21]); + const limit = (earlier: StateRecordFacts) => + creationLimit( + [{ record: post, ids: ["7"] }], + { + type: "/api/projects", + chronology: new StateRecordChronology( + [earlier, post, later], + "recorded-start-time", + ), + tools: [ + { + name: "list", + method: "GET", + path_template: "/api/projects", + record_ordinals: [0], + }, + { + name: "item", + method: "GET", + path_template: "/api/projects/{id}", + record_ordinals: [2], + }, + ], + records: new Map( + [earlier, post, later].map((entry) => [entry.ordinal, entry]), + ), + position: (entry) => entry.ordinal, + withBody: (ordinal, use) => use(ordinal === 0 ? [] : { id: "7" }), + retain: () => undefined, + }, + undefined, + ); + expect(limit(unknown)).toBeUndefined(); + expect(limit(prior)).toBe(post); + }); +}); + describe("creation limits", () => { it("reads the ids of a list body longer than a call's arguments can spread", () => { // A million elements overflow the argument limit of a spread, here and in the test workers' larger stack. @@ -67,6 +124,10 @@ describe("creation limits", () => { const limit = creationLimit( [{ record: post, ids: ["1003"] }], { + chronology: new StateRecordChronology( + [list, post, later], + "recorded-start-time", + ), type: "/api/metrics", tools: [ { @@ -107,7 +168,9 @@ describe("creation limits", () => { }), ).toBeUndefined(); }); +}); +describe("the entity identities creation limits use", () => { it("reads a list's ids from a suffix-named field when no identity name identifies its entities", () => { const [list, post, later] = [ record(0, "GET", "/api/projects"), @@ -163,6 +226,10 @@ describe("creation limits", () => { const limit = creationLimit( [{ record: post, ids: ["proj-0002"] }], { + chronology: new StateRecordChronology( + [item, post, later], + "recorded-start-time", + ), type: "/api/projects", tools: [ { @@ -282,6 +349,10 @@ describe("the reads creation limits parse", () => { const limit = creationLimit( [{ record: post, ids: ["proj-0002"] }], { + chronology: new StateRecordChronology( + [...reads, post, until], + "recorded-start-time", + ), type: "/api/projects", tools: [ { diff --git a/src/domain/analysisView/environmentStateWriteLinks.ts b/src/domain/analysisView/environmentStateWriteLinks.ts index e7c7c340ff..3c428a20b3 100644 --- a/src/domain/analysisView/environmentStateWriteLinks.ts +++ b/src/domain/analysisView/environmentStateWriteLinks.ts @@ -1,3 +1,4 @@ +import type { StateRecordChronology } from "./StateRecordChronology.js"; import type { JsonValue } from "../jsonValue.js"; import { ownMember, @@ -140,7 +141,7 @@ const keptIds = ( /** One read of a type: where it ran, when it finished, whether it reads one item, and the ids it shows. */ interface ShownRead { readonly at: number; - readonly end: number; + readonly end: number | undefined; readonly item: boolean; readonly ids: ReadonlySet; } @@ -154,11 +155,14 @@ interface ShownRead { const freshIds = ( ids: readonly string[], shown: readonly ShownRead[], - post: { readonly at: number; readonly start: number }, + post: { readonly at: number; readonly start: number | undefined }, until: number, ): string[] => { const before = (read: ShownRead): boolean => - read.at < post.at && read.end <= post.start; + read.at < post.at && + read.end !== undefined && + post.start !== undefined && + read.end <= post.start; const seen = shown.some(before); return ids.filter((id) => { const short = id.length < MIN_ID_LENGTH; @@ -215,6 +219,7 @@ export const creationLimit = ( creations: readonly CreatedIds[], reads: { readonly type: string; + readonly chronology: StateRecordChronology; readonly tools: readonly StateToolFacts[]; readonly records: ReadonlyMap; readonly position: (record: StateRecordFacts) => number; @@ -249,7 +254,7 @@ export const creationLimit = ( return [ { at, - end: record.end, + end: reads.chronology.endMs(record), item, ids: keptIds(ids, returned, reads.retain), }, @@ -261,7 +266,7 @@ export const creationLimit = ( )) { const at = reads.position(creation.record); if (at >= end) break; - const post = { at, start: creation.record.start }; + const post = { at, start: reads.chronology.startMs(creation.record) }; if (freshIds(creation.ids, shown, post, end).length > 0) return creation.record; } @@ -276,6 +281,7 @@ export const creationLimit = ( */ export const linkedCreate = (input: { readonly tool: StateToolFacts; + readonly chronology: StateRecordChronology; readonly records: readonly StateRecordFacts[]; readonly isWrite: (record: StateRecordFacts) => boolean; readonly viewOf: (ordinal: number) => StateView | undefined; @@ -296,7 +302,7 @@ export const linkedCreate = (input: { // Only a read finished before the call began shows what was there before it, as `freshIds` decides. const before = records .slice(0, first) - .filter((record) => record.end <= post.start); + .filter((record) => input.chronology.completedBefore(record, post)); for (const record of records.slice(first + 1)) { // Another write may have made what a later read shows, so the search ends there. if (!own.has(record.ordinal) && input.isWrite(record)) return undefined; @@ -417,6 +423,7 @@ export const linkedTarget = ( classified: { readonly effect: WriteEffect; readonly target: string | null }, input: { readonly records: readonly StateRecordFacts[]; + readonly chronology: StateRecordChronology; readonly isWrite: (record: StateRecordFacts) => boolean; readonly viewOf: (ordinal: number) => StateView | undefined; readonly entities: readonly StateEntity[]; @@ -437,6 +444,7 @@ export const linkedTarget = ( ? linkedCreate({ tool, records: input.records, + chronology: input.chronology, isWrite: input.isWrite, viewOf: input.viewOf, entities: input.entities, diff --git a/src/domain/analysisView/environmentStateWriteObserve.ts b/src/domain/analysisView/environmentStateWriteObserve.ts index 1b228a0d9c..8e1d981e02 100644 --- a/src/domain/analysisView/environmentStateWriteObserve.ts +++ b/src/domain/analysisView/environmentStateWriteObserve.ts @@ -1,5 +1,5 @@ import type { JsonValue } from "../jsonValue.js"; -import { overlapsWrite } from "./environmentStateWriteAttribution.js"; +import type { StateRecordChronology } from "./StateRecordChronology.js"; import { ownMember, at, @@ -126,8 +126,8 @@ export const shownBefore = ( export interface ObservationSearch { readonly records: readonly StateRecordFacts[]; readonly from: number; - /** A timed write's completion. A read begun while it still ran may show the prior state. */ - readonly after?: number; + /** Completion must be established before a read may supply a write observation. */ + readonly chronology: StateRecordChronology; readonly viewOf: (ordinal: number) => StateView | undefined; readonly withBody: ( ordinal: number, @@ -192,7 +192,9 @@ export const laterObservation = ( for (const record of search.records.slice(search.from + 1)) { if (target.isWrite(record)) return undefined; // The same overlap the check path ignores: the write was still running when this read began. - if (search.after !== undefined && overlapsWrite(search.after, record.start)) + if ( + !search.chronology.completedBefore(search.records[search.from]!, record) + ) continue; const view = search.viewOf(record.ordinal); if ( diff --git a/src/domain/analysisView/environmentStateWrites.ts b/src/domain/analysisView/environmentStateWrites.ts index e57d505844..88ed34f6e5 100644 --- a/src/domain/analysisView/environmentStateWrites.ts +++ b/src/domain/analysisView/environmentStateWrites.ts @@ -8,14 +8,15 @@ const LATEST_END_MS = 8_640_000_000_000_000 - 1000; /** The instant a served clock starts from: the end of the last timed record a date can hold, to the next whole second. */ const clockStart = ( records: WriteSimulationInput["records"], - timed: boolean, ): string | null => { - if (!timed) return null; + // The clock is global: any unknown completion may extend beyond every known end. + if (records.some((record) => record.timing.endMs === undefined)) return null; // A record timed past the range of a date carries no usable time, so the clock starts from the others. const end = records.reduce( (latest, record) => - Math.abs(record.end) <= LATEST_END_MS - ? Math.max(latest, record.end) + record.timing.endMs !== undefined && + Math.abs(record.timing.endMs) <= LATEST_END_MS + ? Math.max(latest, record.timing.endMs) : latest, Number.NEGATIVE_INFINITY, ); @@ -56,7 +57,10 @@ export const extractWrites = ( } return { writes, - write_clock: clockStart(input.records, input.timed), + write_clock: + input.chronology.timeline === "recorded-start-time" + ? clockStart(input.records) + : null, views, types: state.types.filter((type) => backedTypes.has(type.collection_template), diff --git a/tests/boundary/mcp/analysisViewMcp.test.ts b/tests/boundary/mcp/analysisViewMcp.test.ts index aeea895809..b03f77f9a9 100644 --- a/tests/boundary/mcp/analysisViewMcp.test.ts +++ b/tests/boundary/mcp/analysisViewMcp.test.ts @@ -22,7 +22,10 @@ import { BinaryLayoutService } from "../../../src/application/binaryDiagnostics/ import { toolContract } from "../../../src/contracts/toolContracts.js"; import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; import { createAnalysisProfile } from "../../../src/domain/analysisProfile.js"; -import { jsonObjectSchema } from "../../../src/domain/jsonValue.js"; +import { + jsonObjectSchema, + type JsonValue, +} from "../../../src/domain/jsonValue.js"; import { functionDossierSchema } from "../../../src/domain/hopperValues.js"; import { ghidraFunctionDossier } from "../../../src/domain/ghidraValues.fixture.js"; import { ok } from "../../../src/domain/result.js"; @@ -1390,6 +1393,90 @@ it.each(PUBLISHED_EXAMPLES)( }, ); +it.each([false, true])( + "keeps known chronology and unrelated facts with unknown delete completion %s through MCP", + async (unknownCompletion) => { + const { client, session } = await connect(); + const entry = ( + method: string, + path: string, + second: string, + body: JsonValue, + status = 200, + ) => ({ + startedDateTime: `2026-10-20T09:00:${second}Z`, + time: 0, + request: { method, url: `http://app.example${path}` }, + response: { + status, + content: { mimeType: "application/json", text: JSON.stringify(body) }, + }, + }); + const { time: _readDuration, ...laterRead } = entry( + "GET", + "/api/settings", + "02", + { theme: "dark" }, + ); + const deletion = entry("DELETE", "/api/items/2", "03", "", 204); + const { time: _deleteDuration, ...unknownDeletion } = deletion; + const capture = environmentManifestCaptureFixture([ + entry("GET", "/api/settings", "00", { theme: "light" }), + laterRead, + entry("PATCH", "/api/settings", "01", { theme: "dark" }), + entry("GET", "/api/health", "00.1", { healthy: true }), + entry("GET", "/api/version", "00.2", { version: "1" }), + entry("GET", "/api/items", "00.3", [{ id: 1 }, { id: 2 }]), + entry("GET", "/api/items/2", "00.4", { id: 2 }), + unknownCompletion ? unknownDeletion : deletion, + entry("GET", "/api/items/2", "04", null, 404), + ]); + const parent = analysisViewCaptureEvidence(capture); + const before = structuredClone(parent); + expect(session.recordEvidence(parent).ok).toBe(true); + const response = await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source: { kind: "retained-evidence", evidence_id: parent.evidence_id }, + view: { kind: "environment-state" }, + }, + }); + expect(response.isError, JSON.stringify(response.content)).not.toBe(true); + const state = analysisViewResultSchema.parse( + parseEvidence(response.structuredContent).normalized_result, + ); + expect(state).toMatchObject({ + kind: "environment-state", + timeline: "recorded-start-time", + documents: expect.arrayContaining([ + expect.objectContaining({ + path: "/api/settings", + value: { theme: "light" }, + }), + expect.objectContaining({ + path: "/api/health", + value: { healthy: true }, + }), + expect.objectContaining({ + path: "/api/version", + value: { version: "1" }, + }), + ]), + writes: expect.arrayContaining([ + expect.objectContaining({ + tool: "delete_items_by_id", + served: !unknownCompletion, + }), + ]), + limitations: expect.arrayContaining([ + "Some records carry no usable request duration; their completion and overlap remain unknown.", + ]), + }); + expect(parent).toEqual(before); + expect(session.evidenceById(parent.evidence_id)).toEqual(before); + }, +); + it("extracts an environment state identically from retained, portable and file MCP Evidence", async () => { const { client, session } = await connect(); const { ajv, inputSchema, outputSchema } = await advertisedMcpToolSchemas(