diff --git a/src/domain/analysisView/EnvironmentStateAccumulator.ts b/src/domain/analysisView/EnvironmentStateAccumulator.ts index 2d0f2fce15..dff8df6a44 100644 --- a/src/domain/analysisView/EnvironmentStateAccumulator.ts +++ b/src/domain/analysisView/EnvironmentStateAccumulator.ts @@ -5,7 +5,6 @@ import { MAX_JSON_DEPTH, type JsonValue } from "../jsonValue.js"; import { compareOffsetDateTimeInstants, readRecordedDateTime, - type OffsetDateTimeInstant, } from "../offsetDateTime.js"; import { ok, type Result } from "../result.js"; import { @@ -86,10 +85,8 @@ export const stateBodyJson = ( : value; }; -const timingSchema = z.object({ - startedDateTime: z.string(), - time: z.number().nonnegative(), -}); +const startSchema = z.object({ startedDateTime: z.string() }); +const durationSchema = z.object({ time: z.number().nonnegative() }); const redactedWriteSchema = z.object({ request: z.object({ method: z.string() }), response: z.object({ status: z.number() }).optional(), @@ -100,7 +97,6 @@ interface RecordEntry { readonly origin: string | null; readonly contentType: string; /** Exact ordering fact; request interval coordinates remain milliseconds. */ - readonly startInstant: OffsetDateTimeInstant | undefined; readonly record: WebNetworkCaptureRecord; } @@ -143,44 +139,46 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator record: WebNetworkCaptureRecord, line: RecordedRequestLine, ): Result { - const timing = timingSchema.safeParse(record.reported); + const started = startSchema.safeParse(record.reported); + const duration = durationSchema.safeParse(record.reported); // Admit normalization and fraction parsing before materializing their strings. if ( - timing.success && + started.success && !this.scratch( RECORD_SCRATCH_BYTES + - textBytes(timing.data.startedDateTime.length + 6) * 3, + textBytes(started.data.startedDateTime.length + 6) * 3, ) ) return this.overBudget(); - const startInstant = timing.success - ? readRecordedDateTime(timing.data.startedDateTime)?.instant + const startInstant = started.success + ? readRecordedDateTime(started.data.startedDateTime)?.instant : undefined; // Keep the established millisecond interval policy independently of sorting. - const start = + const startMs = startInstant === undefined - ? record.ordinal + ? undefined : startInstant.epochSecond * 1000 + Number(startInstant.fraction.slice(0, 3).padEnd(3, "0")); + const end = + startMs === undefined || !duration.success + ? undefined + : startMs + duration.data.time; + const timing: StateRecordFacts["timing"] = { + start: + startInstant === undefined || startMs === undefined + ? undefined + : { instant: startInstant, ms: startMs }, + endMs: end !== undefined && Number.isFinite(end) ? end : undefined, + }; // A fraction substring may retain its normalized timestamp's backing store, // including trailing zeroes removed from the exact comparison value. const instantBytes = - timing.success && startInstant !== undefined - ? ENTRY_BYTES + textBytes(timing.data.startedDateTime.length + 6) + started.success && startInstant !== undefined + ? ENTRY_BYTES + textBytes(started.data.startedDateTime.length + 6) : 0; if (line.kind === "excluded") return line.reason === "url-redacted" - ? this.#redactedWrite( - record, - timing.success && startInstant !== undefined - ? { - start, - end: start + timing.data.time, - startInstant, - instantBytes, - } - : undefined, - ) + ? this.#redactedWrite(record, timing, instantBytes) : ok(undefined); // The entry keeps the line's strings and a reference to the already admitted record. if ( @@ -204,15 +202,10 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator path: line.path, query: line.query === "" ? undefined : line.query, status: line.status, - start, - end: - timing.success && startInstant !== undefined - ? start + timing.data.time - : record.ordinal, + timing, }, origin: line.origin, contentType: line.rawMime, - startInstant, record, }); return ok(undefined); @@ -221,14 +214,8 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator /** Keep a successful write whose URL was redacted, with an empty path whose root relates to every type. */ #redactedWrite( record: WebNetworkCaptureRecord, - timing: - | { - readonly start: number; - readonly end: number; - readonly startInstant: OffsetDateTimeInstant; - readonly instantBytes: number; - } - | undefined, + timing: StateRecordFacts["timing"], + instantBytes: number, ): Result { const parsed = redactedWriteSchema.safeParse(record.reported); if (!parsed.success) return ok(undefined); @@ -239,15 +226,12 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator path: "", query: undefined, status: status > 0 ? status : null, - start: timing?.start ?? record.ordinal, - end: timing?.end ?? record.ordinal, + timing, }; if (!isStateWrite(facts)) return ok(undefined); if ( !this.retain( - ENTRY_BYTES * 8 + - textBytes(facts.method.length) + - (timing?.instantBytes ?? 0), + ENTRY_BYTES * 8 + textBytes(facts.method.length) + instantBytes, ) ) return this.overBudget(); @@ -255,7 +239,6 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator facts, origin: null, contentType: "", - startInstant: timing?.startInstant, record, }); return ok(undefined); @@ -358,7 +341,7 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator ).length; const timeline: StateTimeline = entries.length > 0 && - entries.every((entry) => entry.startInstant !== undefined) + entries.every((entry) => entry.facts.timing.start !== undefined) ? "recorded-start-time" : "record-order"; const records = entries @@ -366,16 +349,11 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator timeline === "record-order" ? left.facts.ordinal - right.facts.ordinal : compareOffsetDateTimeInstants( - left.startInstant!, - right.startInstant!, + left.facts.timing.start!.instant, + right.facts.timing.start!.instant, ) || left.facts.ordinal - right.facts.ordinal, ) - .map((entry) => entry.facts) - .map((facts) => - timeline === "record-order" - ? { ...facts, start: facts.ordinal, end: facts.ordinal } - : facts, - ); + .map((entry) => entry.facts); const state = new EnvironmentStateExtraction({ tools: manifest.value.tools, records, @@ -401,6 +379,12 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator "Some records carry no start time, so the state's timeline is record order and overlapping requests are not detected.", ] : []), + ...(timeline === "recorded-start-time" && + records.some((record) => record.timing.endMs === undefined) + ? [ + "Some records carry no usable request duration; their completion and overlap remain unknown.", + ] + : []), ...(state.views.some((view) => view.shape === "recorded") ? [ "Tools whose recordings do not reproduce from the extracted state are listed as recorded views with the reason; they replay their recordings.", diff --git a/src/domain/analysisView/EnvironmentStateExtraction.ts b/src/domain/analysisView/EnvironmentStateExtraction.ts index 18e7bbbd1e..25d0f60f7a 100644 --- a/src/domain/analysisView/EnvironmentStateExtraction.ts +++ b/src/domain/analysisView/EnvironmentStateExtraction.ts @@ -1,3 +1,4 @@ +import { StateRecordChronology } from "./StateRecordChronology.js"; import { jsonByteLength } from "../jsonSerialization.js"; import type { JsonValue } from "../jsonValue.js"; import { @@ -107,14 +108,13 @@ const memberNames = (body: JsonValue, pointer: string): string[] => { export class EnvironmentStateExtraction { readonly #tools: readonly StateToolFacts[]; readonly #records: readonly StateRecordFacts[]; - readonly #timeline: StateTimeline; + readonly #chronology: StateRecordChronology; readonly #budget: StateBudget; - readonly #position: ReadonlyMap; readonly #byOrdinal: ReadonlyMap; /** POST tools whose calls the capture shows read state rather than change it. */ #readOnly: ReadonlySet = new Set(); #writes: readonly StateRecordFacts[] = []; - #writeWithin: ReturnType = writeSpanCheck([]); + #writeWithin: ReturnType = () => false; readonly #bases: ReadonlyMap; #writeCutoffs: StateWriteCutoffs | undefined; readonly #initLimits = new Map(); @@ -146,11 +146,8 @@ export class EnvironmentStateExtraction { }) { this.#tools = input.tools; this.#records = input.records; - this.#timeline = input.timeline; + this.#chronology = new StateRecordChronology(input.records, input.timeline); this.#budget = input.budget; - this.#position = new Map( - input.records.map((record, index) => [record.ordinal, index]), - ); this.#byOrdinal = new Map( input.records.map((record) => [record.ordinal, record]), ); @@ -169,9 +166,10 @@ export class EnvironmentStateExtraction { request: (ordinal) => this.#requestBody(ordinal), retain: (bytes) => this.#retain(bytes), }, + this.#chronology, )); release(); - this.#writeWithin = writeSpanCheck(this.#writes); + this.#writeWithin = writeSpanCheck(this.#writes, this.#chronology); } /** The extracted state, or null when the representation budget is exhausted. */ @@ -183,8 +181,8 @@ export class EnvironmentStateExtraction { this.#writes, this.#bases, (left, right) => - this.#position.get(left.ordinal)! < - this.#position.get(right.ordinal)!, + this.#chronology.positions.get(left.ordinal)! < + this.#chronology.positions.get(right.ordinal)!, (bytes) => this.#retain(bytes), ); this.#creations(); @@ -240,7 +238,7 @@ export class EnvironmentStateExtraction { const split = withoutVariants(plan, { shapeOf: (record) => shapes.get(record.ordinal), queryOf: (ordinal) => this.#byOrdinal.get(ordinal)?.query, - position: (ordinal) => this.#position.get(ordinal)!, + position: (ordinal) => this.#chronology.positions.get(ordinal)!, retain: (bytes) => this.#retain(bytes), checkpoint: () => this.#budget.checkpoint(), }); @@ -259,6 +257,7 @@ export class EnvironmentStateExtraction { this.#spans.filter((span) => this.#rewritten.has(span.type)), this.#writes, this.#byOrdinal, + this.#chronology, ); if (this.#cuts.size === 0) return false; for (const map of [this.#initLimits, this.#identity, this.#views]) @@ -282,7 +281,7 @@ export class EnvironmentStateExtraction { tools: this.#tools, records: this.#records, readOnly: this.#readOnly, - timed: this.#timeline === "recorded-start-time", + chronology: this.#chronology, volatileOf: (type) => this.#volatileOf(type), io: { withBody: (ordinal, use) => this.#withBody(ordinal, use), @@ -355,9 +354,10 @@ export class EnvironmentStateExtraction { this.#creations(), { type: template, + chronology: this.#chronology, tools: this.#tools, records: this.#byOrdinal, - position: (record) => this.#position.get(record.ordinal)!, + position: (record) => this.#chronology.positions.get(record.ordinal)!, withBody: (ordinal, use) => this.#withBody(ordinal, use), retain: (bytes) => this.#retain(bytes), }, @@ -367,7 +367,8 @@ export class EnvironmentStateExtraction { const limit = cut !== undefined && (bound === undefined || - this.#position.get(cut.ordinal)! < this.#position.get(bound.ordinal)!) + this.#chronology.positions.get(cut.ordinal)! < + this.#chronology.positions.get(bound.ordinal)!) ? cut : bound; this.#retain(ENTRY_BYTES + textBytes(template.length)); @@ -391,9 +392,9 @@ export class EnvironmentStateExtraction { ): boolean { return ( limit === undefined || - (this.#position.get(record.ordinal)! < - this.#position.get(limit.ordinal)! && - (this.#timeline === "record-order" || record.end <= limit.start)) + (this.#chronology.positions.get(record.ordinal)! < + this.#chronology.positions.get(limit.ordinal)! && + this.#chronology.completedBefore(record, limit)) ); } @@ -412,8 +413,8 @@ export class EnvironmentStateExtraction { ) .sort( (left, right) => - this.#position.get(left.ordinal)! - - this.#position.get(right.ordinal)!, + this.#chronology.positions.get(left.ordinal)! - + this.#chronology.positions.get(right.ordinal)!, ); } @@ -553,7 +554,9 @@ export class EnvironmentStateExtraction { this.#identity.set(type, { field, basis: "path-parameter" }); const late = this.#addLateProjections(tool, type, projected); projected.sort( - (left, right) => this.#position.get(left)! - this.#position.get(right)!, + (left, right) => + this.#chronology.positions.get(left)! - + this.#chronology.positions.get(right)!, ); this.#addPlan(candidate, pointer, kept, { projected, late }); } @@ -709,7 +712,7 @@ export class EnvironmentStateExtraction { if (id === undefined) return; const collection = collectionPathOf(record, plan.item); const key = entityKey(plan.type, collection, id); - const position = this.#position.get(record.ordinal)!; + const position = this.#chronology.positions.get(record.ordinal)!; let entity = this.#entities.get(key); if (entity === undefined) { this.#retain(ENTRY_BYTES * 4 + textBytes(key.length)); diff --git a/src/domain/analysisView/EnvironmentStateWriteSimulation.test.ts b/src/domain/analysisView/EnvironmentStateWriteSimulation.test.ts index 2ee9a0d8e7..279818275d 100644 --- a/src/domain/analysisView/EnvironmentStateWriteSimulation.test.ts +++ b/src/domain/analysisView/EnvironmentStateWriteSimulation.test.ts @@ -1,9 +1,12 @@ +import { StateRecordChronology } from "./StateRecordChronology.js"; import { OmittedFields } from "./environmentStateWriteOmitted.js"; import { checkedLines, servedLine } from "./environmentStateCheckedLines.js"; import { describe, expect, it, vi } from "vitest"; import { timedCaptureEvidence, + analysisViewCaptureEvidence, + environmentManifestCaptureFixture, type TimedCaptureRow, } from "../../../tests/fixtures/analysisView.js"; import { @@ -1298,8 +1301,7 @@ describe("encoded parent spellings", () => { path: "/api/orgs/uid/acme1/books", query: undefined, status: 200, - start: 0, - end: 0, + timing: { start: undefined, endMs: undefined }, }; const lines = checkedLines({ tools: [tool], @@ -1502,8 +1504,7 @@ describe("resolved encoded item observations", () => { path: "/api/orgs/acme1/books/%61", query: undefined, status: 204, - start: 0, - end: 0, + timing: { start: undefined, endMs: undefined }, }; const target = itemTarget(state, "/api/orgs/{org_id}/books", write); const itemView = { @@ -1516,19 +1517,20 @@ describe("resolved encoded item observations", () => { checked_record_ordinals: [1], envelope_ordinal: 1, }; + const records = [ + write, + { + ordinal: 1, + method: "GET", + path: "/api/orgs/acme1/books/%61", + query: undefined, + status: 200, + timing: { start: undefined, endMs: undefined }, + }, + ]; const search: ObservationSearch = { - records: [ - write, - { - ordinal: 1, - method: "GET", - path: "/api/orgs/acme1/books/%61", - query: undefined, - status: 200, - start: 1, - end: 1, - }, - ], + records, + chronology: new StateRecordChronology(records, "record-order"), from: 0, viewOf: () => itemView, withBody: (_ordinal, use) => use({ id: "%61", name: "After" }), @@ -2130,7 +2132,54 @@ describe("what an unmodelled write explains", () => { }); }); +it("leaves the global served clock unknown when any record completion is unknown", () => { + const rows = (duration: number): TimedCaptureRow[] => [ + ["GET", "/api/books", [FIRST_BOOK, SECOND_BOOK]], + // Even an earlier record with an unknown end may finish after the final known record. + ["GET", "/api/books/1", FIRST_BOOK, 200, undefined, duration], + ["GET", "/api/books/2", SECOND_BOOK], + ]; + expect( + environmentStateViewOf(timedCaptureEvidence(rows(5))).write_clock, + ).toBe(new Date(bookClock(3)).toISOString()); + expect( + environmentStateViewOf(timedCaptureEvidence(rows(-1))).write_clock, + ).toBeNull(); +}); + describe("reads and calls on an entity an unmodelled write left unknown", () => { + it.each([ + { duration: 5, target: "1", served: true }, + { duration: -1, target: "1", served: false }, + { duration: -1, target: "2", served: true }, + ])( + "requires unmodelled completion before resync and later confirmation: $target duration=$duration", + ({ duration, target, served }) => { + const rows: TimedCaptureRow[] = [ + [ + "GET", + "/api/books", + [ + { id: 1, name: "A" }, + { id: 2, name: "B" }, + ], + ], + ["GET", "/api/books/1", { id: 1, name: "A" }], + ["GET", "/api/books/2", { id: 2, name: "B" }], + ["POST", `/api/books/${target}/archive`, null, 204, {}, duration], + // This read may re-establish a baseline only after the related action finished. + ["GET", "/api/books/1", { id: 1, name: "A" }], + ["PATCH", "/api/books/1", { id: 1, name: "Z" }, 200, { name: "Z" }], + ["GET", "/api/books/1", { id: 1, name: "Z" }], + ]; + expect(gated(rows, "patch_books_by_id")).toMatchObject({ + served, + checked_record_ordinals: served ? [6] : [], + contradicted_by: null, + }); + }, + ); + const named = (name: string) => ({ id: 1, name }); const other = { id: 2, name: "B" }; const ARCHIVE: TimedCaptureRow = [ @@ -2307,8 +2356,7 @@ describe("entities an unmodelled write made", () => { path: "/api/books/3", query: undefined, status: 200, - start: 1, - end: 1, + timing: { start: undefined, endMs: undefined }, }; expect( unseen.adopt(view as unknown as StateItemView, { record, index: 1 }), @@ -2333,8 +2381,7 @@ describe("entities an unmodelled write made", () => { path: "/api/books/3", query: undefined, status: 200, - start: index, - end: index, + timing: { start: undefined, endMs: undefined }, }, index, }); @@ -3042,8 +3089,7 @@ it("learns omitted fields using the raw ID resolved by the item owner", () => { path: "/api/books/%63", query: undefined, status: 200, - start: 1, - end: 1, + timing: { start: undefined, endMs: undefined }, }; const target = itemTarget(state, view.type, record); expect(target.id).toBe("%63"); @@ -3066,3 +3112,133 @@ it("learns omitted fields using the raw ID resolved by the item owner", () => { required_fields: ["id", "name"], }); }); + +it.each([ + { + label: "point read with simultaneous unknown action", + archiveStart: 4, + duration: 0, + archiveDuration: -1, + target: "1", + served: false, + }, + { + label: "point read with simultaneous point action", + archiveStart: 4, + duration: 0, + archiveDuration: 0, + target: "1", + served: true, + }, + { + label: "point read with unrelated unknown action", + archiveStart: 4, + duration: 0, + archiveDuration: -1, + target: "2", + served: true, + }, + { + label: "same-start later ordinal", + archiveStart: 4, + duration: 5, + target: "1", + served: false, + }, + { + label: "later start within response interval", + archiveStart: 5, + duration: 1500, + target: "1", + served: false, + }, + { + label: "unknown response completion", + archiveStart: 5, + duration: -1, + target: "1", + served: false, + }, + { + label: "known disjoint response interval", + archiveStart: 6, + duration: 1500, + target: "1", + served: true, + }, + { + label: "unrelated overlapping write", + archiveStart: 5, + duration: 1500, + target: "2", + served: true, + }, +])( + "requires a read free of captured write overlap: $label", + ({ archiveStart, duration, target, served, ...options }) => { + const rows: TimedCaptureRow[] = [ + [ + "GET", + "/api/books", + [ + { id: 1, name: "A" }, + { id: 2, name: "B" }, + ], + ], + ["GET", "/api/books/1", { id: 1, name: "A" }], + ["GET", "/api/books/2", { id: 2, name: "B" }], + ["PATCH", "/api/books/1", { id: 1, name: "Z" }, 200, { name: "Z" }], + ["GET", "/api/books/1", { id: 1, name: "Z" }, 200, undefined, duration], + [ + "POST", + `/api/books/${target}/archive`, + null, + 204, + {}, + "archiveDuration" in options ? options.archiveDuration : 5, + ], + ]; + const entries = rows.map( + ( + [method, path, body = null, status = 200, request, time = 5], + index, + ) => ({ + startedDateTime: new Date( + Date.parse(bookClock(index === 5 ? archiveStart : index)), + ).toISOString(), + time, + request: { + method, + url: `http://app.example${path}`, + bodySize: + request === undefined + ? 0 + : Buffer.byteLength(JSON.stringify(request)), + ...(request === undefined + ? {} + : { + postData: { + mimeType: "application/json", + text: JSON.stringify(request), + }, + }), + }, + response: { + status, + content: { mimeType: "application/json", text: JSON.stringify(body) }, + }, + }), + ); + const capture = analysisViewCaptureEvidence( + environmentManifestCaptureFixture(entries), + ); + const plan = environmentStateViewOf(capture).writes.find( + (write) => write.tool === "patch_books_by_id", + ); + expect(plan).toMatchObject({ + served, + checked_record_ordinals: served ? [4] : [], + contradicted_by: null, + }); + }, +); diff --git a/src/domain/analysisView/EnvironmentStateWriteSimulation.ts b/src/domain/analysisView/EnvironmentStateWriteSimulation.ts index 2b524a14fe..ddfa17f13a 100644 --- a/src/domain/analysisView/EnvironmentStateWriteSimulation.ts +++ b/src/domain/analysisView/EnvironmentStateWriteSimulation.ts @@ -1,3 +1,4 @@ +import { CapturedWriteCompletion } from "./environmentStateWriteCompletion.js"; import type { JsonValue } from "../jsonValue.js"; import { at, @@ -114,6 +115,7 @@ export class EnvironmentStateWriteSimulation { readonly #touchedSince: PendingTargets = new Map(); /** Timeline indexes of writes the gate does not model. */ readonly #unmodelledAt: number[] = []; + readonly #writeCompletion: CapturedWriteCompletion; readonly #history: WriteHistory = { touched: this.#touched, pending: this.#touchedSince, @@ -135,6 +137,7 @@ export class EnvironmentStateWriteSimulation { constructor(input: WriteSimulationInput) { this.#input = input; + this.#writeCompletion = new CapturedWriteCompletion(input.io.retain); this.#state = new SimulatedState(input.entities, input.documents); this.#unseen = new UnseenEntities(input.entities, { state: this.#state, @@ -224,6 +227,15 @@ export class EnvironmentStateWriteSimulation { linked: link.collection, }); } + // Reads may finish after writes with the same or a later start: index the + // complete capture before replay so ordinal tie-breaking cannot create proof. + for (const record of this.#input.records) + if (this.#isWrite(record)) + this.#writeCompletion.write( + record.path, + this.#input.chronology.endMs(record), + this.#input.chronology.startMs(record), + ); this.#input.records.forEach((record, index) => this.#step(record, index)); const views = new Map( this.views([...this.#input.views.values()]).map((view) => [ @@ -280,7 +292,7 @@ export class EnvironmentStateWriteSimulation { : this.#input.io.requestBody(ordinal), }, records: this.#input.records, - timed: this.#input.timed, + chronology: this.#input.chronology, viewOf: (ordinal) => { const view = this.#input.views.get( this.#toolOf.get(ordinal)?.name ?? "", @@ -354,7 +366,7 @@ export class EnvironmentStateWriteSimulation { index, tool: tool.name, path: record.path, - end: record.end, + end: this.#input.chronology.endMs(record), }); }; // Item deletes use their path; calls outside a built plan's query never interpret their body. @@ -370,7 +382,7 @@ export class EnvironmentStateWriteSimulation { readonly index: number; readonly tool: string; readonly path: string; - readonly end: number; + readonly end: number | undefined; }, ): void { if (result.kind === "unknown") { @@ -427,6 +439,7 @@ export class EnvironmentStateWriteSimulation { /** Whether an item read shows an entity an unmodelled write made, or one left unknown, again rather than checks it. */ #resynced(view: StateView, record: StateRecordFacts, index: number): boolean { if (view.shape !== "item") return false; + if (this.#writeOverlapsRead(record)) return true; const adopted = this.#unseen.adopt(view, { record, index }); if (adopted !== undefined) { this.#taint.known(adopted.type, adopted.collection, adopted.id); @@ -436,7 +449,14 @@ export class EnvironmentStateWriteSimulation { if (!this.#taint.unknown(view.type, collection, id)) return false; // A read begun while a modelled write to the entity still ran may show either state, so it neither checks nor shows it. const target = entityTarget(view.type, collection, id); - if (runningAt(this.#touchedSince, target, record.start)) return true; + if ( + runningAt( + this.#touchedSince, + target, + this.#input.chronology.startMs(record), + ) + ) + return true; // A failed read that is no absence shows nothing of the entity, so it stays unknown for a usable read. if (!succeeded(record) && !isAbsence(record.status)) return true; const entity = this.#state.entity(view.type, collection, id); @@ -530,6 +550,7 @@ export class EnvironmentStateWriteSimulation { if (original === undefined) return; let view = this.#sorted.comparisonView(this.omitted.view(original)); if (projectedRead(view, record, this.#input.io)) return; + if (this.#writeOverlapsRead(record)) return; const borrowers = this.#observations.get(record.ordinal); if ( this.#viewOf(original).shape !== "recorded" && @@ -544,7 +565,11 @@ export class EnvironmentStateWriteSimulation { targets.some( (key) => this.#taint.tainted(key) || - runningAt(this.#touchedSince, key, record.start), + runningAt( + this.#touchedSince, + key, + this.#input.chronology.startMs(record), + ), ); if (blocked(keys)) return; // A read the gate counts can show the view lists a written array sorted, or rule that out. @@ -650,6 +675,15 @@ export class EnvironmentStateWriteSimulation { return shown.unseen; } + /** An overlapping captured write cannot supply a stable read baseline. */ + #writeOverlapsRead(read: StateRecordFacts): boolean { + return this.#writeCompletion.runningAt( + read.path, + this.#input.chronology.startMs(read), + this.#input.chronology.endMs(read), + ); + } + #unmodelled(path: string, index: number): void { this.#taint.write(path); this.#unmodelledAt.push(index); diff --git a/src/domain/analysisView/StateRecordChronology.ts b/src/domain/analysisView/StateRecordChronology.ts new file mode 100644 index 0000000000..dbc38a986a --- /dev/null +++ b/src/domain/analysisView/StateRecordChronology.ts @@ -0,0 +1,36 @@ +import type { + StateRecordFacts, + StateTimeline, +} from "./environmentStateRules.js"; + +/** Own recorded interval facts separately from the explicit ordinal fallback policy. */ +export class StateRecordChronology { + readonly positions: ReadonlyMap; + + constructor( + records: readonly StateRecordFacts[], + readonly timeline: StateTimeline, + ) { + this.positions = new Map( + records.map((record, index) => [record.ordinal, index]), + ); + } + + startMs(record: StateRecordFacts): number | undefined { + return this.timeline === "record-order" + ? this.positions.get(record.ordinal) + : record.timing.start?.ms; + } + + endMs(record: StateRecordFacts): number | undefined { + return this.timeline === "record-order" + ? this.positions.get(record.ordinal) + : record.timing.endMs; + } + + completedBefore(left: StateRecordFacts, right: StateRecordFacts): boolean { + const end = this.endMs(left); + const start = this.startMs(right); + return end !== undefined && start !== undefined && end <= start; + } +} diff --git a/src/domain/analysisView/environmentStateGate.test.ts b/src/domain/analysisView/environmentStateGate.test.ts index 50a29b4863..3fe1781780 100644 --- a/src/domain/analysisView/environmentStateGate.test.ts +++ b/src/domain/analysisView/environmentStateGate.test.ts @@ -21,8 +21,7 @@ const check = ( path: `/api/books/${ordinal + 1}`, query: undefined, status: 200, - start: ordinal, - end: ordinal, + timing: { start: undefined, endMs: undefined }, })); const bodies: JsonObject[] = clocks.map(([, stamp], index) => ({ id: index + 1, diff --git a/src/domain/analysisView/environmentStateQueryCalls.test.ts b/src/domain/analysisView/environmentStateQueryCalls.test.ts index f8ed3e0ef2..ff63fd447c 100644 --- a/src/domain/analysisView/environmentStateQueryCalls.test.ts +++ b/src/domain/analysisView/environmentStateQueryCalls.test.ts @@ -1,3 +1,5 @@ +import { StateRecordChronology } from "./StateRecordChronology.js"; +import { parseOffsetDateTimeInstant } from "../offsetDateTime.js"; import { describe, expect, it } from "vitest"; import type { JsonValue } from "../jsonValue.js"; @@ -7,6 +9,7 @@ import { } from "./environmentStateQueryCalls.js"; import type { StateRecordFacts, + StateTimeline, StateToolFacts, } from "./environmentStateRules.js"; @@ -20,7 +23,7 @@ interface Call { readonly tool: string; readonly template?: string; readonly start?: number; - readonly end?: number; + readonly end?: number | null; } /** How the read-only pass parses response bodies and charges what it keeps of them, when a test watches either. */ @@ -30,15 +33,26 @@ interface Bodies { } /** The read-only tools of a timeline of calls, ordinals in order; a call spans its index unless it gives its times. */ -const readOnlyOf = (calls: readonly Call[], bodies: Bodies = {}): string[] => { +const readOnlyOf = ( + calls: readonly Call[], + bodies: Bodies = {}, + timeline: StateTimeline = "recorded-start-time", +): string[] => { const records: StateRecordFacts[] = calls.map((call, index) => ({ ordinal: index, method: call.method, path: call.path, query: call.query, status: call.status === undefined ? 200 : call.status, - start: call.start ?? index, - end: call.end ?? index, + timing: { + start: { + instant: parseOffsetDateTimeInstant( + new Date(call.start ?? index).toISOString(), + ), + ms: call.start ?? index, + }, + endMs: call.end === null ? undefined : (call.end ?? index), + }, })); const tools = new Map(); calls.forEach((call, index) => { @@ -53,11 +67,16 @@ const readOnlyOf = (calls: readonly Call[], bodies: Bodies = {}): string[] => { const withBody: QueryCallIo["withBody"] = bodies.withBody ?? ((ordinal, use) => use(calls[ordinal]?.response)); return [ - ...readOnlyTools([...tools.values()], records, { - withBody, - request: (ordinal) => calls[ordinal]?.request, - retain: bodies.retain ?? (() => undefined), - }), + ...readOnlyTools( + [...tools.values()], + records, + { + withBody, + request: (ordinal) => calls[ordinal]?.request, + retain: bodies.retain ?? (() => undefined), + }, + new StateRecordChronology(records, timeline), + ), ].sort(); }; @@ -316,6 +335,31 @@ describe("POSTs that read state but stay writes", () => { }); describe("the reads either side of a POST", () => { + it("requires known read completion without discarding independent known intervals", () => { + const known: Call = { + method: "GET", + path: "/api/members", + tool: "get_members", + response: JOBS, + start: 0, + end: 5, + }; + const unknown = { ...known, start: 6, end: null }; + const post: Call = { + method: "POST", + path: "/api/members", + tool: "post_members", + response: JOBS, + start: 10, + end: 11, + }; + expect(readOnlyOf([unknown, post])).toEqual([]); + expect(readOnlyOf([known, unknown, post])).toEqual(["post_members"]); + expect(readOnlyOf([unknown, post], {}, "record-order")).toEqual([ + "post_members", + ]); + }); + it("compares a POST only with reads that finished before it began or began after it ended", () => { const note = (title: string): Call => ({ method: "GET", @@ -454,8 +498,15 @@ describe("read-only decisions over many calls", () => { path: read ? "/api/members" : "/api/members/ids", query: undefined, status: 200, - start: index, - end: index, + timing: { + start: { + instant: parseOffsetDateTimeInstant( + new Date(index).toISOString(), + ), + ms: index, + }, + endMs: index, + }, }; const method = read ? "GET" : "POST"; return Object.defineProperty(record, "method", { @@ -487,11 +538,16 @@ describe("read-only decisions over many calls", () => { ]; visits = 0; expect([ - ...readOnlyTools(tools, records, { - withBody: (_ordinal, use) => use([]), - request: () => undefined, - retain: () => undefined, - }), + ...readOnlyTools( + tools, + records, + { + withBody: (_ordinal, use) => use([]), + request: () => undefined, + retain: () => undefined, + }, + new StateRecordChronology(records, "recorded-start-time"), + ), ]).toEqual(["post_ids"]); expect(visits).toBeLessThan(50 * records.length); }); @@ -510,8 +566,15 @@ describe("read-only decisions over a long run of calls", () => { path: "/v1/jobs/statuses", query: undefined, status: 200, - start: index, - end: index, + timing: { + start: { + instant: parseOffsetDateTimeInstant( + new Date(index).toISOString(), + ), + ms: index, + }, + endMs: index, + }, }; const method = read ? "GET" : "POST"; return Object.defineProperty(record, "method", { @@ -543,11 +606,16 @@ describe("read-only decisions over a long run of calls", () => { ]; visits = 0; expect([ - ...readOnlyTools(tools, records, { - withBody: (_ordinal, use) => use(JOBS), - request: () => ({ jobs: ["night-watch"] }), - retain: () => undefined, - }), + ...readOnlyTools( + tools, + records, + { + withBody: (_ordinal, use) => use(JOBS), + request: () => ({ jobs: ["night-watch"] }), + retain: () => undefined, + }, + new StateRecordChronology(records, "recorded-start-time"), + ), ]).toEqual(["post_statuses"]); expect(visits).toBeLessThan(50 * records.length); }); @@ -566,8 +634,15 @@ describe("read-only decisions over many lines", () => { method: read ? "GET" : "POST", path: read ? "/api/members" : "/api/members/ids", status: 200, - start: index, - end: index, + timing: { + start: { + instant: parseOffsetDateTimeInstant( + new Date(index).toISOString(), + ), + ms: index, + }, + endMs: index, + }, }; const query = read ? `?page=${String(index)}` : undefined; return Object.defineProperty(record, "query", { @@ -596,11 +671,16 @@ describe("read-only decisions over many lines", () => { }, ]; expect([ - ...readOnlyTools(tools, records, { - withBody: (_ordinal, use) => use([]), - request: () => undefined, - retain: () => undefined, - }), + ...readOnlyTools( + tools, + records, + { + withBody: (_ordinal, use) => use([]), + request: () => undefined, + retain: () => undefined, + }, + new StateRecordChronology(records, "recorded-start-time"), + ), ]).toEqual(["post_ids"]); expect(reads).toBeLessThan(20 * records.length); }); diff --git a/src/domain/analysisView/environmentStateQueryCalls.ts b/src/domain/analysisView/environmentStateQueryCalls.ts index 97ed34a184..c9b0794e3b 100644 --- a/src/domain/analysisView/environmentStateQueryCalls.ts +++ b/src/domain/analysisView/environmentStateQueryCalls.ts @@ -1,3 +1,4 @@ +import type { StateRecordChronology } from "./StateRecordChronology.js"; import { digestCanonicalValue } from "../canonicalDigest.js"; import { identifierWords } from "./environmentStateWords.js"; import type { JsonValue } from "../jsonValue.js"; @@ -176,10 +177,9 @@ export const readOnlyTools = ( tools: readonly StateToolFacts[], records: readonly StateRecordFacts[], io: QueryCallIo, + chronology: StateRecordChronology, ): Set => { - const position = new Map( - records.map((record, index) => [record.ordinal, index]), - ); + const position = chronology.positions; const response = responseDigests(io); /** The timeline indexes of the nearest state writes either side of an index, the skipped calls aside: -1 and the length when none, found in one pass each way. */ const nearestWrites = ( @@ -220,6 +220,7 @@ export const readOnlyTools = ( if (!calls.some(succeeded) || calls.some(unresolved)) continue; const timeline = { position, + chronology, ...nearestWrites(new Set(tool.record_ordinals)), response, }; @@ -234,6 +235,7 @@ export const readOnlyTools = ( for (let vetoed = true; vetoed;) { const timeline = { position, + chronology, ...nearestWrites( new Set([...candidates.values()].flat().map((call) => call.ordinal)), ), @@ -250,6 +252,7 @@ export const readOnlyTools = ( }; interface Timeline { + readonly chronology: StateRecordChronology; readonly position: ReadonlyMap; /** The nearest other writes either side of a timeline index. */ readonly writes: (at: number) => { @@ -266,12 +269,14 @@ interface Timeline { const lastFinishedBefore = ( reads: readonly StateRecordFacts[], call: StateRecordFacts, + chronology: StateRecordChronology, ): StateRecordFacts | undefined => { let latest: StateRecordFacts | undefined; for (const read of reads) if ( - read.end <= call.start && - (latest === undefined || read.end >= latest.end) + chronology.completedBefore(read, call) && + (latest === undefined || + chronology.endMs(read)! >= chronology.endMs(latest)!) ) latest = read; return latest; @@ -281,10 +286,10 @@ const lastFinishedBefore = ( const repeatsRead = ( calls: readonly StateRecordFacts[], records: readonly StateRecordFacts[], - { position, isWrite, response }: Timeline, + { position, isWrite, response, chronology }: Timeline, ): boolean => { const answering = calls.filter(succeeded); - const reads = lastReads(answering, records, position, isWrite); + const reads = lastReads(answering, records, position, isWrite, chronology); return answering.every((call) => { const read = reads.get(call.ordinal); if (read === undefined) return false; @@ -302,6 +307,7 @@ const lastReads = ( records: readonly StateRecordFacts[], position: ReadonlyMap, isWrite: (index: number) => boolean, + chronology: StateRecordChronology, ): Map => { const paths = new Set(calls.map((call) => call.path)); const callAt = new Map( @@ -317,12 +323,12 @@ const lastReads = ( for (let index = 0; index <= lastCall; index += 1) { const record = records[index]!; const call = callAt.get(index); - if (call !== undefined) { + if (call !== undefined && chronology.startMs(call) !== undefined) { const ready = waiting.get(call.path); for ( - let read = ready?.popEndedBy(call.start); + let read = ready?.popEndedBy(chronology.startMs(call)!); read !== undefined; - read = ready?.popEndedBy(call.start) + read = ready?.popEndedBy(chronology.startMs(call)!) ) latest.set(call.path, read); const read = latest.get(call.path); @@ -333,9 +339,11 @@ const lastReads = ( } else if ( record.method === "GET" && succeeded(record) && - paths.has(record.path) + paths.has(record.path) && + chronology.endMs(record) !== undefined ) { - const heap = waiting.get(record.path) ?? new ReadsByEnd(position); + const heap = + waiting.get(record.path) ?? new ReadsByEnd(position, chronology); heap.push(record); waiting.set(record.path, heap); } @@ -347,12 +355,15 @@ const lastReads = ( class ReadsByEnd { readonly #reads: StateRecordFacts[] = []; - constructor(private readonly position: ReadonlyMap) {} + constructor( + private readonly position: ReadonlyMap, + private readonly chronology: StateRecordChronology, + ) {} #before(left: StateRecordFacts, right: StateRecordFacts): boolean { return ( - left.end < right.end || - (left.end === right.end && + this.chronology.endMs(left)! < this.chronology.endMs(right)! || + (this.chronology.endMs(left) === this.chronology.endMs(right) && this.position.get(left.ordinal)! < this.position.get(right.ordinal)!) ); } @@ -372,7 +383,8 @@ class ReadsByEnd { popEndedBy(time: number): StateRecordFacts | undefined { const reads = this.#reads; const first = reads[0]; - if (first === undefined || first.end > time) return undefined; + if (first === undefined || this.chronology.endMs(first)! > time) + return undefined; const last = reads.pop()!; if (reads.length > 0) { reads[0] = last; @@ -431,7 +443,7 @@ const lastMeaningfulWord = (segment: string): string | undefined => const changesReadsAcross = ( calls: readonly StateRecordFacts[], records: readonly StateRecordFacts[], - { position, writes, response }: Timeline, + { position, writes, response, chronology }: Timeline, ): boolean => { const stretches = new Map< string, @@ -451,7 +463,8 @@ const changesReadsAcross = ( else stretch.calls.push(call); } for (const stretch of stretches.values()) - if (changesInStretch(stretch, records, { position, response })) return true; + if (changesInStretch(stretch, records, { position, response, chronology })) + return true; return false; }; @@ -464,7 +477,11 @@ const changesInStretch = ( readonly calls: readonly StateRecordFacts[]; }, records: readonly StateRecordFacts[], - { position, response }: Pick, + { + position, + response, + chronology, + }: Pick, ): boolean => { const shown = (record: StateRecordFacts): string | symbol => succeeded(record) ? (response(record.ordinal) ?? UNKNOWN) : ABSENT; @@ -493,9 +510,12 @@ const changesInStretch = ( const earlier = lastFinishedBefore( reads.filter((read) => position.get(read.ordinal)! < at), call, + chronology, ); const later = reads.find( - (read) => position.get(read.ordinal)! > at && read.start >= call.end, + (read) => + position.get(read.ordinal)! > at && + chronology.completedBefore(call, read), ); if (earlier === undefined || later === undefined) return false; const [was, now] = [shown(earlier), shown(later)]; @@ -515,11 +535,12 @@ export const stateWritesOf = ( tools: readonly StateToolFacts[], records: readonly StateRecordFacts[], io: QueryCallIo, + chronology: StateRecordChronology, ): { readonly readOnly: ReadonlySet; readonly writes: readonly StateRecordFacts[]; } => { - const readOnly = readOnlyTools(tools, records, io); + const readOnly = readOnlyTools(tools, records, io, chronology); const reads = new Set( tools .filter((tool) => readOnly.has(tool.name)) diff --git a/src/domain/analysisView/environmentStateRewrites.test.ts b/src/domain/analysisView/environmentStateRewrites.test.ts new file mode 100644 index 0000000000..68e8e9b6dd --- /dev/null +++ b/src/domain/analysisView/environmentStateRewrites.test.ts @@ -0,0 +1,112 @@ +import { expect, it } from "vitest"; +import { parseOffsetDateTimeInstant } from "../offsetDateTime.js"; +import { StateRecordChronology } from "./StateRecordChronology.js"; +import { rewriteCuts } from "./environmentStateRewrites.js"; +import { + writeSpanCheck, + writeSpanIndex, + type StateRecordFacts, +} from "./environmentStateRules.js"; + +const record = ( + ordinal: number, + method: string, + start: number, + endMs: number | undefined, +): StateRecordFacts => ({ + ordinal, + method, + path: "/api/items/1", + query: undefined, + status: 200, + timing: { + start: { + instant: parseOffsetDateTimeInstant(new Date(start).toISOString()), + ms: start, + }, + endMs, + }, +}); + +it.each([ + { label: "unknown duration", start: 10, end: undefined, possible: true }, + { label: "completed before the reads", start: 10, end: 19, possible: false }, + { + label: "completed at the first read's start", + start: 10, + end: 20, + possible: false, + }, + { + label: "known zero duration at the boundary", + start: 20, + end: 20, + possible: false, + }, + { label: "known overlap", start: 10, end: 21, possible: true }, +])( + "keeps rewrite cuts conservative for an earlier write with $label", + ({ start, end, possible }) => { + const write = record(0, "PUT", start, end); + const earlier = record(1, "GET", 20, 25); + const later = record(2, "GET", 30, 35); + const records = [write, earlier, later]; + const chronology = new StateRecordChronology( + records, + "recorded-start-time", + ); + const cuts = rewriteCuts( + [{ type: "/api/items", earlier: earlier.ordinal, later: later.ordinal }], + [write], + new Map(records.map((value) => [value.ordinal, value])), + chronology, + ); + expect(writeSpanCheck([write], chronology)(earlier, later)).toBe(possible); + expect(cuts.get("/api/items")).toBe(possible ? write : undefined); + if (end === undefined) { + // A possible explanation prevents an independence proof; it does not prove + // that the write completed, or that it caused the observed difference. + expect(chronology.completedBefore(write, earlier)).toBe(false); + } + }, +); + +it("selects the earliest possibly interfering write without replacing a known completed predecessor", () => { + const finished = record(0, "PUT", 0, 5); + const unresolved = record(1, "PUT", 10, undefined); + const earlier = record(2, "GET", 20, 25); + const later = record(3, "GET", 30, 35); + const records = [finished, unresolved, earlier, later]; + const chronology = new StateRecordChronology(records, "recorded-start-time"); + const writes = [finished, unresolved]; + expect(writeSpanIndex(writes, chronology)(earlier, later)).toBe(1); + expect( + rewriteCuts( + [{ type: "/api/items", earlier: earlier.ordinal, later: later.ordinal }], + writes, + new Map(records.map((value) => [value.ordinal, value])), + chronology, + ).get("/api/items"), + ).toBe(unresolved); +}); + +it.each([35, undefined])( + "does not invent a later read's completion when locating possible writes (end=%s)", + (end) => { + const earlier = record(0, "GET", 20, 25); + const later = record(1, "GET", 30, end); + const write = record(2, "PUT", 40, 45); + const records = [earlier, later, write]; + const chronology = new StateRecordChronology( + records, + "recorded-start-time", + ); + const cuts = rewriteCuts( + [{ type: "/api/items", earlier: earlier.ordinal, later: later.ordinal }], + [write], + new Map(records.map((value) => [value.ordinal, value])), + chronology, + ); + expect(cuts.get("/api/items")).toBe(end === undefined ? write : undefined); + }, +); diff --git a/src/domain/analysisView/environmentStateRewrites.ts b/src/domain/analysisView/environmentStateRewrites.ts index b80bd83418..5a11662b98 100644 --- a/src/domain/analysisView/environmentStateRewrites.ts +++ b/src/domain/analysisView/environmentStateRewrites.ts @@ -1,4 +1,8 @@ -import type { StateRecordFacts } from "./environmentStateRules.js"; +import { + writeSpanIndex, + type StateRecordFacts, +} from "./environmentStateRules.js"; +import type { StateRecordChronology } from "./StateRecordChronology.js"; /** Two recordings of a type that disagree across a write. */ export interface RewriteSpan { @@ -8,40 +12,24 @@ export interface RewriteSpan { } /** - * Per type whose reads disagree across writes, the earliest write within any such span: one that - * began before the later recording ended and ended after the earlier began, in timeline order. + * Per type whose reads disagree, the earliest write that may intersect the span. + * Unknown interval bounds cannot establish that the observations were independent + * of a write; choosing a conservative cutoff does not prove the write caused a change. */ export const rewriteCuts = ( spans: readonly RewriteSpan[], writes: readonly StateRecordFacts[], byOrdinal: ReadonlyMap, + chronology: StateRecordChronology, ): Map => { - // The latest end among each prefix of writes, so the first write ending after a time is found by bisection. - const reach: number[] = []; - for (const write of writes) - reach.push(Math.max(reach.at(-1) ?? Number.NEGATIVE_INFINITY, write.end)); - const starts = writes.map((write) => write.start); - /** The first index whose value passes `test`, which once true stays true. */ - const first = ( - values: readonly number[], - test: (value: number) => boolean, - ) => { - let [low, high] = [0, values.length]; - while (low < high) { - const middle = (low + high) >>> 1; - if (test(values[middle]!)) high = middle; - else low = middle + 1; - } - return low; - }; + const possibleWrite = writeSpanIndex(writes, chronology); const cuts = new Map(); for (const span of spans) { const earlier = byOrdinal.get(span.earlier); const later = byOrdinal.get(span.later); if (earlier === undefined || later === undefined) continue; - // Writes come in start order: those begun before the later recording ended are a prefix. - const index = first(reach, (end) => end > earlier.start); - if (index < first(starts, (start) => start >= later.end)) + const index = possibleWrite(earlier, later); + if (index !== undefined) cuts.set(span.type, Math.min(index, cuts.get(span.type) ?? index)); } return new Map([...cuts].map(([type, index]) => [type, writes[index]!])); diff --git a/src/domain/analysisView/environmentStateRules.ts b/src/domain/analysisView/environmentStateRules.ts index e4746e88ad..979a878b46 100644 --- a/src/domain/analysisView/environmentStateRules.ts +++ b/src/domain/analysisView/environmentStateRules.ts @@ -1,3 +1,5 @@ +import type { OffsetDateTimeInstant } from "../offsetDateTime.js"; +import type { StateRecordChronology } from "./StateRecordChronology.js"; import type { ListQuery } from "./environmentStateListQuery.js"; import { canonicalJson } from "../comparisonSemantics.js"; import { compareUnicodeCodePoints } from "../unicodeCodePointOrder.js"; @@ -21,8 +23,12 @@ export interface StateRecordFacts { /** WHATWG query spelling including `?`, or undefined when the request had none. */ readonly query: string | undefined; readonly status: number | null; - readonly start: number; - readonly end: number; + readonly timing: { + readonly start: + | { readonly instant: OffsetDateTimeInstant; readonly ms: number } + | undefined; + readonly endMs: number | undefined; + }; } /** The manifest tool facts extraction uses. */ @@ -392,26 +398,54 @@ export const isStateWrite = (record: StateRecordFacts): boolean => changesState(record.method, record.path) && (record.status === null || (record.status >= 200 && record.status < 400)); -/** Whether any write overlaps the span from one record's start to a later record's end; writes come in timeline order. */ -export const writeSpanCheck = ( +/** Index possible write interference without inventing completion for unknown intervals. */ +export const writeSpanIndex = ( writes: readonly StateRecordFacts[], -): ((earlier: StateRecordFacts, later: StateRecordFacts) => boolean) => { - // The latest end among each prefix of writes, so one search answers a span. - const reach: number[] = []; - for (const write of writes) - reach.push(Math.max(reach.at(-1) ?? Number.NEGATIVE_INFINITY, write.end)); + chronology: StateRecordChronology, +): (( + earlier: StateRecordFacts, + later: StateRecordFacts, +) => number | undefined) => { + const reach: (number | undefined)[] = []; + const starts: (number | undefined)[] = []; + let maximum: number | undefined; + let unknown = false; + for (const write of writes) { + const end = chronology.endMs(write); + if (end === undefined) unknown = true; + else maximum = maximum === undefined ? end : Math.max(maximum, end); + reach.push(unknown ? undefined : maximum); + starts.push(chronology.startMs(write)); + } return (earlier, later) => { + const beginning = chronology.startMs(earlier); + const ending = chronology.endMs(later); let low = 0; let high = writes.length; while (low < high) { const middle = (low + high) >>> 1; - if (writes[middle]!.start < later.end) low = middle + 1; - else high = middle; + const end = reach[middle]; + if (beginning === undefined || end === undefined || end > beginning) + high = middle; + else low = middle + 1; } - return low > 0 && reach[low - 1]! > earlier.start; + const start = starts[low]; + return low < writes.length && + (ending === undefined || start === undefined || start < ending) + ? low + : undefined; }; }; +/** Whether a write may overlap the span; unknown completion cannot prove absence. */ +export const writeSpanCheck = ( + writes: readonly StateRecordFacts[], + chronology: StateRecordChronology, +): ((earlier: StateRecordFacts, later: StateRecordFacts) => boolean) => { + const index = writeSpanIndex(writes, chronology); + return (earlier, later) => index(earlier, later) !== undefined; +}; + /** Literal leading segments shared by every template with the same first segment: each path's API base. */ export const apiBaseLengths = ( templates: readonly string[], diff --git a/src/domain/analysisView/environmentStateView.test.ts b/src/domain/analysisView/environmentStateView.test.ts index a10be1cc3c..3d21159ffd 100644 --- a/src/domain/analysisView/environmentStateView.test.ts +++ b/src/domain/analysisView/environmentStateView.test.ts @@ -41,6 +41,8 @@ import { type LineShape, } from "./environmentStateVariants.js"; import { rewriteCuts } from "./environmentStateRewrites.js"; +import { StateRecordChronology } from "./StateRecordChronology.js"; +import { offsetDateTimeInstantFromEpochMilliseconds } from "../offsetDateTime.js"; import { ENTRY_BYTES } from "./webNetworkRouteTemplates.js"; type State = Extract; @@ -839,8 +841,7 @@ it("leaves every line checked rather than compare more lines than the variant pa path: "/api/items", query: `?view=${String(ordinal)}`, status: 200, - start: ordinal, - end: ordinal, + timing: { start: undefined, endMs: undefined }, })); const shape = { entities: new Map( @@ -867,8 +868,7 @@ it("tells apart member names that hold the separator, so a line spelling them ot path: "/api/items/1", query, status: 200, - start: ordinal, - end: ordinal, + timing: { start: undefined, endMs: undefined }, }); const observed = (ordinal: number, object: JsonObject) => ({ position: ordinal, @@ -908,8 +908,7 @@ it("leaves every line checked when comparing many mostly empty lines would excee method: "GET", path: "/api/items", status: 200, - start: ordinal, - end: ordinal, + timing: { start: undefined, endMs: undefined }, }; return Object.defineProperty(record, "query", { enumerable: true, @@ -950,8 +949,7 @@ const expandedItemLines = (items: number) => { path: `/api/items/${String(ordinal % items)}`, query: ordinal < items ? undefined : "?expand=shelf", status: 200, - start: ordinal, - end: ordinal, + timing: { start: undefined, endMs: undefined }, }), ); const shapes = new Map( @@ -1137,11 +1135,21 @@ it("finds each span's earliest write without scanning the writes before it", () path: "/api/log", query: undefined, status: 200, + timing: { + start: { + instant: offsetDateTimeInstantFromEpochMilliseconds(start), + get ms() { + reads += 1; + return start; + }, + }, + get endMs() { + reads += 1; + return end; + }, + }, }; - return Object.defineProperties(record, { - start: { enumerable: true, get: () => ((reads += 1), start) }, - end: { enumerable: true, get: () => ((reads += 1), end) }, - }) as typeof record & { readonly start: number; readonly end: number }; + return record; }; const count = 2000; const writes = Array.from({ length: count }, (_, index) => @@ -1162,7 +1170,11 @@ it("finds each span's earliest write without scanning the writes before it", () later: earlier, })); reads = 0; - const cuts = rewriteCuts(spans, writes, reads_); + const chronology = new StateRecordChronology( + [...writes, ...reads_.values()], + "recorded-start-time", + ); + const cuts = rewriteCuts(spans, writes, reads_, chronology); expect(cuts.size).toBe(count - 1); expect(reads).toBeLessThan(200 * count); }); @@ -1423,6 +1435,125 @@ it.each<{ }, ); +it.each([ + { duration: undefined, redactedWrite: false }, + { duration: null, redactedWrite: false }, + { duration: -1, redactedWrite: false }, + { duration: "0", redactedWrite: false }, + { duration: undefined, redactedWrite: true }, + { duration: null, redactedWrite: true }, +])( + "retains chronological initial state with duration $duration and redacted write $redactedWrite", + ({ duration, redactedWrite }) => { + const entry = ( + method: string, + path: string | null, + second: string, + body: JsonValue, + ) => ({ + startedDateTime: `2026-10-20T09:00:${second}Z`, + time: 0, + request: { + method, + url: path === null ? null : `http://app.example${path}`, + }, + response: { + status: 200, + content: { mimeType: "application/json", text: JSON.stringify(body) }, + }, + }); + const { time: _duration, ...laterRead } = entry( + "GET", + "/api/settings", + "02", + { theme: "dark" }, + ); + const entries = [ + entry("GET", "/api/settings", "00", { theme: "light" }), + { ...laterRead, ...(duration === undefined ? {} : { time: duration }) }, + entry("PATCH", redactedWrite ? null : "/api/settings", "01", { + theme: "dark", + }), + entry("GET", "/api/health", "00.1", { healthy: true }), + entry("GET", "/api/version", "00.2", { version: "1" }), + ]; + const capture = webNetworkCaptureSchema.parse( + environmentManifestCaptureFixture(entries), + ); + const before = structuredClone(capture); + const state = stateOf(parentOf(analysisViewCaptureEvidence(capture))); + expect(state.timeline).toBe("recorded-start-time"); + expect(state.limitations).toContain( + "Some records carry no usable request duration; their completion and overlap remain unknown.", + ); + expect(state.documents).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + path: "/api/settings", + value: { theme: "light" }, + record_ordinal: 0, + }), + expect.objectContaining({ + path: "/api/health", + value: { healthy: true }, + record_ordinal: 3, + }), + expect.objectContaining({ + path: "/api/version", + value: { version: "1" }, + record_ordinal: 4, + }), + ]), + ); + expect(capture).toEqual(before); + if (redactedWrite) + expect(state.limitations).toContain( + "1 of the recorded writes had a redacted URL; each ends the initial state of every type, since its target is unknown.", + ); + }, +); + +it("does not confirm completion of an untimed delete while preserving unrelated known reads", () => { + const source = captureOf([ + ["GET", "/api/items", [{ id: 1 }, { id: 2 }]], + ["GET", "/api/items/2", { id: 2 }], + ["GET", "/api/health", { healthy: true }], + ["GET", "/api/version", { version: "1" }], + ["DELETE", "/api/items/2", "", 204], + ["GET", "/api/items/2", null, 404], + ]); + const known = stateOf(source); + expect(known.writes).toContainEqual( + expect.objectContaining({ tool: "delete_items_by_id", served: true }), + ); + const result = structuredClone(source.normalizedResult) as { + records: { reported: { time?: JsonValue } }[]; + }; + delete result.records[4]!.reported.time; + const before = structuredClone(result); + const state = stateOf({ ...source, normalizedResult: result }); + expect(state.timeline).toBe("recorded-start-time"); + expect(state.limitations).toContain( + "Some records carry no usable request duration; their completion and overlap remain unknown.", + ); + expect(state.writes).toContainEqual( + expect.objectContaining({ tool: "delete_items_by_id", served: false }), + ); + expect(state.documents).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + path: "/api/health", + value: { healthy: true }, + }), + expect.objectContaining({ + path: "/api/version", + value: { version: "1" }, + }), + ]), + ); + expect(result).toEqual(before); +}); + it("bounds a parameterized namespace by concrete writes before publishing new items", () => { const result = projectAnalysisView( captureOf([ diff --git a/src/domain/analysisView/environmentStateWriteAttribution.test.ts b/src/domain/analysisView/environmentStateWriteAttribution.test.ts new file mode 100644 index 0000000000..1c7bc7c2eb --- /dev/null +++ b/src/domain/analysisView/environmentStateWriteAttribution.test.ts @@ -0,0 +1,44 @@ +import { expect, it } from "vitest"; +import { + forgetChecked, + runningAt, + touchTarget, + WriteEffects, + type PendingTargets, + type TouchedTargets, +} from "./environmentStateWriteAttribution.js"; + +it("keeps an unknown write completion pending alongside later known completions", () => { + const touched: TouchedTargets = new Map(); + const pending: PendingTargets = new Map(); + for (const [index, end] of [undefined, 20].entries()) + touchTarget(touched, pending, "entity", { + index, + tool: `write-${String(index)}`, + end, + fields: ["name"], + }); + expect(runningAt(pending, "entity", 30)).toBe(true); + expect(runningAt(pending, "entity", undefined)).toBe(true); + expect(runningAt(pending, "absent", undefined)).toBe(false); + forgetChecked( + { touched, since: pending, effects: new WriteEffects() }, + "entity", + undefined, + ); + expect(runningAt(pending, "entity", 30)).toBe(false); +}); + +it("requires a known read start to establish a known pending write completed", () => { + const touched: TouchedTargets = new Map(); + const pending: PendingTargets = new Map(); + touchTarget(touched, pending, "entity", { + index: 0, + tool: "write", + end: 20, + fields: ["name"], + }); + expect(runningAt(pending, "entity", undefined)).toBe(true); + expect(runningAt(pending, "entity", 19)).toBe(true); + expect(runningAt(pending, "entity", 20)).toBe(false); +}); diff --git a/src/domain/analysisView/environmentStateWriteAttribution.ts b/src/domain/analysisView/environmentStateWriteAttribution.ts index 119ccca5f4..cba2d2e655 100644 --- a/src/domain/analysisView/environmentStateWriteAttribution.ts +++ b/src/domain/analysisView/environmentStateWriteAttribution.ts @@ -28,12 +28,14 @@ export type TouchedTargets = Map>; /** Per target, when the latest modelled write since its last check ended, and each tool's latest call there. */ export type PendingTargets = Map< string, - { until: number; readonly at: Map } + { until: number | undefined; readonly at: Map } >; /** Whether a write that ended at `until` was still running when a read began at `start`, so the read may show either state. */ -export const overlapsWrite = (until: number, start: number): boolean => - until > start; +export const overlapsWrite = ( + until: number | undefined, + start: number | undefined, +): boolean => until === undefined || start === undefined || until > start; /** What the gate tracks of the writes since each target's last check, and when each unmodelled write ran. */ export interface WriteHistory { @@ -76,8 +78,11 @@ export const interference = ( export const runningAt = ( pending: PendingTargets, key: string, - start: number, -): boolean => overlapsWrite(pending.get(key)?.until ?? -Infinity, start); + start: number | undefined, +): boolean => { + const held = pending.get(key); + return held !== undefined && overlapsWrite(held.until, start); +}; /** The tools whose writes a read exercises: those that changed its targets since they were last checked, less those that borrowed the read to learn their effect. */ export const exercisedBy = ( @@ -100,13 +105,18 @@ export const touchTarget = ( write: { readonly index: number; readonly tool: string; - readonly end: number; + readonly end: number | undefined; readonly fields: readonly string[] | undefined; }, ): void => { const held = pending.get(target); pending.set(target, { - until: Math.max(held?.until ?? -Infinity, write.end), + // One unknown completion keeps this target pending until a real check clears it. + until: + write.end === undefined || + (held !== undefined && held.until === undefined) + ? undefined + : Math.max(held?.until ?? -Infinity, write.end), at: (held?.at ?? new Map()).set(write.tool, write.index), }); const tools = touched.get(target) ?? new Map(); diff --git a/src/domain/analysisView/environmentStateWriteCompletion.test.ts b/src/domain/analysisView/environmentStateWriteCompletion.test.ts new file mode 100644 index 0000000000..68e59028fa --- /dev/null +++ b/src/domain/analysisView/environmentStateWriteCompletion.test.ts @@ -0,0 +1,112 @@ +import { expect, it, vi } from "vitest"; +import * as paths from "./environmentStateRules.js"; +import { prefixRelated } from "./environmentStateWriteRules.js"; +import { CapturedWriteCompletion } from "./environmentStateWriteCompletion.js"; + +it("preserves the existing prefix relation, including non-transitive decoding and parameters", () => { + const spellings = [ + "", + "api", + "a", + "%61", + "%2561", + "%2F", + "%252F", + "{id}", + "%ZZ", + ]; + const candidates = [ + "/", + ...spellings.flatMap((value) => [ + `/api/${value}`, + `/api/${value}/child`, + `/${value}/other`, + ]), + ]; + for (const write of candidates) { + const completion = new CapturedWriteCompletion(() => {}); + completion.write(write, undefined); + for (const read of candidates) + expect(completion.runningAt(read, 100), `${write} -> ${read}`).toBe( + prefixRelated(write, read), + ); + } +}); + +it("retains unknown completion and isolates unrelated writes while aggregating known ends", () => { + const completion = new CapturedWriteCompletion(() => {}); + completion.write("/api/books/1", 20); + completion.write("/api/books/1", 30); + completion.write("/api/books/2", undefined); + completion.write("/api/books/2", 10); + expect(completion.runningAt("/api/books/1", 29)).toBe(true); + expect(completion.runningAt("/api/books/1", 30)).toBe(false); + expect(completion.runningAt("/api/books/1", undefined)).toBe(true); + expect(completion.runningAt("/api/books/2", 100)).toBe(true); + expect(completion.runningAt("/api/books", 100)).toBe(true); + expect(completion.runningAt("/api/authors", undefined)).toBe(false); + completion.write("/", undefined); + expect(completion.runningAt("/api/authors", 100)).toBe(true); +}); + +it("uses explicit record-order coordinates without treating absent writes as pending", () => { + const completion = new CapturedWriteCompletion(() => {}); + expect(completion.runningAt("/api/books", undefined)).toBe(false); + completion.write("/api/books", 1); + expect(completion.runningAt("/api/books/1", 2)).toBe(false); +}); + +it("looks up a concrete path without reinterpreting every retained write path", () => { + const completion = new CapturedWriteCompletion(() => {}); + for (let id = 0; id < 128; id += 1) + completion.write(`/api/books/${String(id)}`, undefined); + const decoded = vi.spyOn(paths, "decodedSegment"); + try { + expect(completion.runningAt("/api/books/64", 100)).toBe(true); + // Matching interprets only the three read segments; prior write segments were indexed once. + expect(decoded).toHaveBeenCalledTimes(3); + } finally { + decoded.mockRestore(); + } +}); + +it("reserves retained prefix growth before publishing an unbudgeted edge", () => { + const refused = new Error("state retained-memory budget exhausted"); + let remaining = Infinity; + const completion = new CapturedWriteCompletion((bytes) => { + if (bytes > remaining) throw refused; + remaining -= bytes; + }); + completion.write("/api/books/1", 10); + remaining = 0; + // Updating an indexed path retains no new keys or nodes. + completion.write("/api/books/1", 20); + expect(completion.runningAt("/api/books/1", 19)).toBe(true); + expect(() => completion.write("/api/authors/1", undefined)).toThrow(refused); + expect(completion.runningAt("/api/authors/1", 100)).toBe(false); + expect(completion.runningAt("/api", 100)).toBe(false); + expect(completion.runningAt("/", 100)).toBe(false); + expect(completion.runningAt("/api/{collection}", 100)).toBe(false); + expect(completion.runningAt("/api/books/1", 20)).toBe(false); +}); + +it("queries both interval boundaries across future writes in chronology order", () => { + const completion = new CapturedWriteCompletion(() => {}); + completion.write("/api/books/1", 15, 10); + completion.write("/api/books/1", undefined, 30); + expect(completion.runningAt("/api/books/1", 5, 10)).toBe(false); + expect(completion.runningAt("/api/books/1", 5, 11)).toBe(true); + expect(completion.runningAt("/api/books/1", 15, 30)).toBe(false); + expect(completion.runningAt("/api/books/1", 15, 31)).toBe(true); + expect(completion.runningAt("/api/books/1", 15, undefined)).toBe(true); + expect(completion.runningAt("/api/authors", 5, undefined)).toBe(false); +}); + +it("keeps a simultaneous unknown write ambiguous for a point read, unlike a known point write", () => { + const completion = new CapturedWriteCompletion(() => {}); + completion.write("/api/books/1", undefined, 10); + completion.write("/api/books/2", 10, 10); + expect(completion.runningAt("/api/books/1", 10, 10)).toBe(true); + expect(completion.runningAt("/api/books/2", 10, 10)).toBe(false); + expect(completion.runningAt("/api/books/3", 10, 10)).toBe(false); +}); diff --git a/src/domain/analysisView/environmentStateWriteCompletion.ts b/src/domain/analysisView/environmentStateWriteCompletion.ts new file mode 100644 index 0000000000..dec7dde716 --- /dev/null +++ b/src/domain/analysisView/environmentStateWriteCompletion.ts @@ -0,0 +1,149 @@ +import { ENTRY_BYTES, textBytes } from "./webNetworkRouteTemplates.js"; +import { + decodedSegment, + isParam, + segmentsOf, +} from "./environmentStateRules.js"; + +/** Ordered write starts and the latest completion among writes through each start. */ +class CompletionHistory { + readonly #starts: number[] = []; + readonly #ends: (number | undefined)[] = []; + + constructor(private readonly retain: (bytes: number) => void) {} + + reserve(start: number): void { + if (this.#starts.at(-1) !== start) this.retain(ENTRY_BYTES); + } + + append(start: number, end: number | undefined): void { + const previous = this.#ends.at(-1); + const maximum = + end === undefined || (this.#ends.length > 0 && previous === undefined) + ? undefined + : Math.max(previous ?? -Infinity, end); + if (this.#starts.at(-1) === start) + this.#ends[this.#ends.length - 1] = maximum; + else { + this.#starts.push(start); + this.#ends.push(maximum); + } + } + + overlaps(start: number | undefined, end: number | undefined): boolean { + // Admit same-start writes for a point observation; a known zero-length write + // is still excluded by its completion below. Otherwise starts at read end are disjoint. + // Find writes begun before the read completed. Unknown read completion + // admits every captured write; all histories are built in chronology order. + let low = 0; + let high = this.#starts.length; + while (low < high) { + const middle = Math.floor((low + high) / 2); + if ( + end === undefined || + this.#starts[middle]! < end || + (start === end && this.#starts[middle] === end) + ) + low = middle + 1; + else high = middle; + } + if (low === 0) return false; + const completion = this.#ends[low - 1]; + return ( + completion === undefined || start === undefined || completion > start + ); + } +} + +class CompletionPrefix { + terminal: CompletionHistory | undefined; + subtree: CompletionHistory | undefined; + readonly children: Map; + readonly decoded: Map>; + parameter: CompletionPrefix | undefined; + + constructor(private readonly retain: (bytes: number) => void) { + // Node, child maps, parameter link and both history owners with their arrays. + retain(ENTRY_BYTES * 10); + this.children = new Map(); + this.decoded = new Map(); + } + + child(segment: string): CompletionPrefix { + if (isParam(segment)) + return (this.parameter ??= new CompletionPrefix(this.retain)); + const held = this.children.get(segment); + if (held !== undefined) return held; + // Reserve edge entries, inverse-match membership and both retained spellings + // before decoding or allocating. A once-decoded segment cannot grow in length. + this.retain(ENTRY_BYTES * 4 + textBytes(segment.length) * 2); + const child = new CompletionPrefix(this.retain); + const decoded = decodedSegment(segment); + this.children.set(segment, child); + this.decoded.set( + decoded, + (this.decoded.get(decoded) ?? new Set()).add(child), + ); + return child; + } + + /** Keep verbatim, once-decoded and inverse-decoded matches distinct: decoding is not transitive. */ + matching(segment: string): Set { + if (isParam(segment)) + return new Set([ + ...this.children.values(), + ...(this.parameter === undefined ? [] : [this.parameter]), + ]); + const matches = new Set(this.decoded.get(segment)); + for (const spelling of [segment, decodedSegment(segment)]) { + const child = this.children.get(spelling); + if (child !== undefined) matches.add(child); + } + if (this.parameter !== undefined) matches.add(this.parameter); + return matches; + } +} + +/** Index captured write intervals by the same segment-prefix relation as write attribution. */ +export class CapturedWriteCompletion { + readonly #root: CompletionPrefix; + + constructor(private readonly retain: (bytes: number) => void) { + this.#root = new CompletionPrefix(retain); + } + + /** Admit writes in nondecreasing chronology start order. */ + write(path: string, end: number | undefined, start = -Infinity): void { + let node = this.#root; + const ancestors = [node]; + // Finish budgeted path admission before publishing any completion summary. + // A refused edge may leave reserved empty prefixes, but no write facts. + for (const segment of segmentsOf(path)) { + node = node.child(segment); + ancestors.push(node); + } + const histories = ancestors.map( + (ancestor) => (ancestor.subtree ??= new CompletionHistory(this.retain)), + ); + histories.push((node.terminal ??= new CompletionHistory(this.retain))); + for (const history of histories) history.reserve(start); + for (const history of histories) history.append(start, end); + } + + /** Unknown completion or read start cannot prove that a related write finished. */ + runningAt(path: string, start: number | undefined, end = Infinity): boolean { + const running = (history: CompletionHistory | undefined): boolean => + history?.overlaps(start, end) === true; + let nodes = new Set([this.#root]); + for (const segment of segmentsOf(path)) { + const next = new Set(); + for (const node of nodes) { + if (running(node.terminal)) return true; + for (const child of node.matching(segment)) next.add(child); + } + if (next.size === 0) return false; + nodes = next; + } + return [...nodes].some((node) => running(node.subtree)); + } +} diff --git a/src/domain/analysisView/environmentStateWriteCutoffs.test.ts b/src/domain/analysisView/environmentStateWriteCutoffs.test.ts index 664a90034d..09fedb7c96 100644 --- a/src/domain/analysisView/environmentStateWriteCutoffs.test.ts +++ b/src/domain/analysisView/environmentStateWriteCutoffs.test.ts @@ -4,6 +4,7 @@ import { rootOf, type StateRecordFacts, } from "./environmentStateRules.js"; +import { offsetDateTimeInstantFromEpochMilliseconds } from "../offsetDateTime.js"; import { StateWriteCutoffs } from "./environmentStateWriteCutoffs.js"; const write = (path: string, ordinal: number): StateRecordFacts => ({ @@ -12,8 +13,13 @@ const write = (path: string, ordinal: number): StateRecordFacts => ({ method: "POST", query: undefined, status: 200, - start: ordinal, - end: ordinal, + timing: { + start: { + instant: offsetDateTimeInstantFromEpochMilliseconds(ordinal), + ms: ordinal, + }, + endMs: ordinal, + }, }); it("matches prefix and parameter semantics against an independent linear oracle", () => { @@ -46,12 +52,12 @@ it("matches prefix and parameter semantics against an independent linear oracle" const index = new StateWriteCutoffs( selection, bases, - (a, b) => a.start < b.start, + (a, b) => a.timing.start!.ms < b.timing.start!.ms, () => {}, ); for (const root of patterns) { const expected = [...selection] - .sort((a, b) => a.start - b.start) + .sort((a, b) => a.timing.start!.ms - b.timing.start!.ms) .find((record) => { const candidate = rootOf(record.path, bases); const shared = Math.min(root.length, candidate.length); diff --git a/src/domain/analysisView/environmentStateWriteEffects.ts b/src/domain/analysisView/environmentStateWriteEffects.ts index a5ca517de0..254438dee2 100644 --- a/src/domain/analysisView/environmentStateWriteEffects.ts +++ b/src/domain/analysisView/environmentStateWriteEffects.ts @@ -1,3 +1,4 @@ +import type { StateRecordChronology } from "./StateRecordChronology.js"; import { jsonTextCodeUnits } from "../jsonSerialization.js"; import type { JsonValue } from "../jsonValue.js"; import { @@ -133,7 +134,7 @@ export interface WriteSimulationInput { readonly records: readonly StateRecordFacts[]; /** POST tools whose calls read state rather than change it. */ readonly readOnly: ReadonlySet; - readonly timed: boolean; + readonly chronology: StateRecordChronology; readonly views: ReadonlyMap; readonly types: readonly StateType[]; readonly entities: readonly StateEntity[]; @@ -147,7 +148,7 @@ export interface EffectContext { readonly state: SimulatedState; readonly io: WriteIo; readonly records: readonly StateRecordFacts[]; - readonly timed: boolean; + readonly chronology: StateRecordChronology; /** Whether a record may change state: a write the capture does not show reading only. */ readonly isWrite: (record: StateRecordFacts) => boolean; readonly viewOf: (ordinal: number) => StateView | undefined; @@ -276,12 +277,15 @@ export const referencedIn = export const startOf = ( context: EffectContext, step: WriteStep, -): number | undefined => (context.timed ? step.record.start : undefined); +): number | undefined => + context.chronology.timeline === "recorded-start-time" + ? step.record.timing.start?.ms + : undefined; export const searchFrom = (context: EffectContext, step: WriteStep) => ({ records: context.records, from: step.index, - ...(context.timed ? { after: step.record.end } : {}), + chronology: context.chronology, viewOf: context.viewOf, withBody: context.io.withBody, queryOf: context.io.queryOf, diff --git a/src/domain/analysisView/environmentStateWriteLinks.test.ts b/src/domain/analysisView/environmentStateWriteLinks.test.ts index 85b40df3d4..8ac06c376f 100644 --- a/src/domain/analysisView/environmentStateWriteLinks.test.ts +++ b/src/domain/analysisView/environmentStateWriteLinks.test.ts @@ -1,3 +1,5 @@ +import { StateRecordChronology } from "./StateRecordChronology.js"; +import { parseOffsetDateTimeInstant } from "../offsetDateTime.js"; import { describe, expect, it } from "vitest"; import type { JsonValue } from "../jsonValue.js"; @@ -15,8 +17,13 @@ const record = ( path, query: undefined, status: 200, - start, - end, + timing: { + start: { + instant: parseOffsetDateTimeInstant(new Date(start).toISOString()), + ms: start, + }, + endMs: end, + }, }); /** The creation limit of `/api/projects` over list reads and a POST returning `created`, in ordinal order. */ @@ -35,6 +42,10 @@ const projectsLimit = ( creationLimit( [{ record: post, ids: created }], { + chronology: new StateRecordChronology( + [...reads, post], + "recorded-start-time", + ), type: "/api/projects", tools: [ { @@ -52,6 +63,52 @@ const projectsLimit = ( undefined, ); +describe("short-id creation limits", () => { + it("does not use an unknown completion as the prior read needed to link a short id", () => { + const prior = record(0, "GET", "/api/projects", [0, 5]); + const unknown = { + ...prior, + timing: { ...prior.timing, endMs: undefined }, + }; + const post = record(1, "POST", "/api/imports", [10, 11]); + const later = record(2, "GET", "/api/projects/7", [20, 21]); + const limit = (earlier: StateRecordFacts) => + creationLimit( + [{ record: post, ids: ["7"] }], + { + type: "/api/projects", + chronology: new StateRecordChronology( + [earlier, post, later], + "recorded-start-time", + ), + tools: [ + { + name: "list", + method: "GET", + path_template: "/api/projects", + record_ordinals: [0], + }, + { + name: "item", + method: "GET", + path_template: "/api/projects/{id}", + record_ordinals: [2], + }, + ], + records: new Map( + [earlier, post, later].map((entry) => [entry.ordinal, entry]), + ), + position: (entry) => entry.ordinal, + withBody: (ordinal, use) => use(ordinal === 0 ? [] : { id: "7" }), + retain: () => undefined, + }, + undefined, + ); + expect(limit(unknown)).toBeUndefined(); + expect(limit(prior)).toBe(post); + }); +}); + describe("creation limits", () => { it("reads the ids of a list body longer than a call's arguments can spread", () => { // A million elements overflow the argument limit of a spread, here and in the test workers' larger stack. @@ -67,6 +124,10 @@ describe("creation limits", () => { const limit = creationLimit( [{ record: post, ids: ["1003"] }], { + chronology: new StateRecordChronology( + [list, post, later], + "recorded-start-time", + ), type: "/api/metrics", tools: [ { @@ -107,7 +168,9 @@ describe("creation limits", () => { }), ).toBeUndefined(); }); +}); +describe("the entity identities creation limits use", () => { it("reads a list's ids from a suffix-named field when no identity name identifies its entities", () => { const [list, post, later] = [ record(0, "GET", "/api/projects"), @@ -163,6 +226,10 @@ describe("creation limits", () => { const limit = creationLimit( [{ record: post, ids: ["proj-0002"] }], { + chronology: new StateRecordChronology( + [item, post, later], + "recorded-start-time", + ), type: "/api/projects", tools: [ { @@ -282,6 +349,10 @@ describe("the reads creation limits parse", () => { const limit = creationLimit( [{ record: post, ids: ["proj-0002"] }], { + chronology: new StateRecordChronology( + [...reads, post, until], + "recorded-start-time", + ), type: "/api/projects", tools: [ { diff --git a/src/domain/analysisView/environmentStateWriteLinks.ts b/src/domain/analysisView/environmentStateWriteLinks.ts index e7c7c340ff..3c428a20b3 100644 --- a/src/domain/analysisView/environmentStateWriteLinks.ts +++ b/src/domain/analysisView/environmentStateWriteLinks.ts @@ -1,3 +1,4 @@ +import type { StateRecordChronology } from "./StateRecordChronology.js"; import type { JsonValue } from "../jsonValue.js"; import { ownMember, @@ -140,7 +141,7 @@ const keptIds = ( /** One read of a type: where it ran, when it finished, whether it reads one item, and the ids it shows. */ interface ShownRead { readonly at: number; - readonly end: number; + readonly end: number | undefined; readonly item: boolean; readonly ids: ReadonlySet; } @@ -154,11 +155,14 @@ interface ShownRead { const freshIds = ( ids: readonly string[], shown: readonly ShownRead[], - post: { readonly at: number; readonly start: number }, + post: { readonly at: number; readonly start: number | undefined }, until: number, ): string[] => { const before = (read: ShownRead): boolean => - read.at < post.at && read.end <= post.start; + read.at < post.at && + read.end !== undefined && + post.start !== undefined && + read.end <= post.start; const seen = shown.some(before); return ids.filter((id) => { const short = id.length < MIN_ID_LENGTH; @@ -215,6 +219,7 @@ export const creationLimit = ( creations: readonly CreatedIds[], reads: { readonly type: string; + readonly chronology: StateRecordChronology; readonly tools: readonly StateToolFacts[]; readonly records: ReadonlyMap; readonly position: (record: StateRecordFacts) => number; @@ -249,7 +254,7 @@ export const creationLimit = ( return [ { at, - end: record.end, + end: reads.chronology.endMs(record), item, ids: keptIds(ids, returned, reads.retain), }, @@ -261,7 +266,7 @@ export const creationLimit = ( )) { const at = reads.position(creation.record); if (at >= end) break; - const post = { at, start: creation.record.start }; + const post = { at, start: reads.chronology.startMs(creation.record) }; if (freshIds(creation.ids, shown, post, end).length > 0) return creation.record; } @@ -276,6 +281,7 @@ export const creationLimit = ( */ export const linkedCreate = (input: { readonly tool: StateToolFacts; + readonly chronology: StateRecordChronology; readonly records: readonly StateRecordFacts[]; readonly isWrite: (record: StateRecordFacts) => boolean; readonly viewOf: (ordinal: number) => StateView | undefined; @@ -296,7 +302,7 @@ export const linkedCreate = (input: { // Only a read finished before the call began shows what was there before it, as `freshIds` decides. const before = records .slice(0, first) - .filter((record) => record.end <= post.start); + .filter((record) => input.chronology.completedBefore(record, post)); for (const record of records.slice(first + 1)) { // Another write may have made what a later read shows, so the search ends there. if (!own.has(record.ordinal) && input.isWrite(record)) return undefined; @@ -417,6 +423,7 @@ export const linkedTarget = ( classified: { readonly effect: WriteEffect; readonly target: string | null }, input: { readonly records: readonly StateRecordFacts[]; + readonly chronology: StateRecordChronology; readonly isWrite: (record: StateRecordFacts) => boolean; readonly viewOf: (ordinal: number) => StateView | undefined; readonly entities: readonly StateEntity[]; @@ -437,6 +444,7 @@ export const linkedTarget = ( ? linkedCreate({ tool, records: input.records, + chronology: input.chronology, isWrite: input.isWrite, viewOf: input.viewOf, entities: input.entities, diff --git a/src/domain/analysisView/environmentStateWriteObserve.ts b/src/domain/analysisView/environmentStateWriteObserve.ts index 1b228a0d9c..8e1d981e02 100644 --- a/src/domain/analysisView/environmentStateWriteObserve.ts +++ b/src/domain/analysisView/environmentStateWriteObserve.ts @@ -1,5 +1,5 @@ import type { JsonValue } from "../jsonValue.js"; -import { overlapsWrite } from "./environmentStateWriteAttribution.js"; +import type { StateRecordChronology } from "./StateRecordChronology.js"; import { ownMember, at, @@ -126,8 +126,8 @@ export const shownBefore = ( export interface ObservationSearch { readonly records: readonly StateRecordFacts[]; readonly from: number; - /** A timed write's completion. A read begun while it still ran may show the prior state. */ - readonly after?: number; + /** Completion must be established before a read may supply a write observation. */ + readonly chronology: StateRecordChronology; readonly viewOf: (ordinal: number) => StateView | undefined; readonly withBody: ( ordinal: number, @@ -192,7 +192,9 @@ export const laterObservation = ( for (const record of search.records.slice(search.from + 1)) { if (target.isWrite(record)) return undefined; // The same overlap the check path ignores: the write was still running when this read began. - if (search.after !== undefined && overlapsWrite(search.after, record.start)) + if ( + !search.chronology.completedBefore(search.records[search.from]!, record) + ) continue; const view = search.viewOf(record.ordinal); if ( diff --git a/src/domain/analysisView/environmentStateWrites.ts b/src/domain/analysisView/environmentStateWrites.ts index e57d505844..88ed34f6e5 100644 --- a/src/domain/analysisView/environmentStateWrites.ts +++ b/src/domain/analysisView/environmentStateWrites.ts @@ -8,14 +8,15 @@ const LATEST_END_MS = 8_640_000_000_000_000 - 1000; /** The instant a served clock starts from: the end of the last timed record a date can hold, to the next whole second. */ const clockStart = ( records: WriteSimulationInput["records"], - timed: boolean, ): string | null => { - if (!timed) return null; + // The clock is global: any unknown completion may extend beyond every known end. + if (records.some((record) => record.timing.endMs === undefined)) return null; // A record timed past the range of a date carries no usable time, so the clock starts from the others. const end = records.reduce( (latest, record) => - Math.abs(record.end) <= LATEST_END_MS - ? Math.max(latest, record.end) + record.timing.endMs !== undefined && + Math.abs(record.timing.endMs) <= LATEST_END_MS + ? Math.max(latest, record.timing.endMs) : latest, Number.NEGATIVE_INFINITY, ); @@ -56,7 +57,10 @@ export const extractWrites = ( } return { writes, - write_clock: clockStart(input.records, input.timed), + write_clock: + input.chronology.timeline === "recorded-start-time" + ? clockStart(input.records) + : null, views, types: state.types.filter((type) => backedTypes.has(type.collection_template), diff --git a/tests/boundary/mcp/analysisViewMcp.test.ts b/tests/boundary/mcp/analysisViewMcp.test.ts index aeea895809..b03f77f9a9 100644 --- a/tests/boundary/mcp/analysisViewMcp.test.ts +++ b/tests/boundary/mcp/analysisViewMcp.test.ts @@ -22,7 +22,10 @@ import { BinaryLayoutService } from "../../../src/application/binaryDiagnostics/ import { toolContract } from "../../../src/contracts/toolContracts.js"; import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; import { createAnalysisProfile } from "../../../src/domain/analysisProfile.js"; -import { jsonObjectSchema } from "../../../src/domain/jsonValue.js"; +import { + jsonObjectSchema, + type JsonValue, +} from "../../../src/domain/jsonValue.js"; import { functionDossierSchema } from "../../../src/domain/hopperValues.js"; import { ghidraFunctionDossier } from "../../../src/domain/ghidraValues.fixture.js"; import { ok } from "../../../src/domain/result.js"; @@ -1390,6 +1393,90 @@ it.each(PUBLISHED_EXAMPLES)( }, ); +it.each([false, true])( + "keeps known chronology and unrelated facts with unknown delete completion %s through MCP", + async (unknownCompletion) => { + const { client, session } = await connect(); + const entry = ( + method: string, + path: string, + second: string, + body: JsonValue, + status = 200, + ) => ({ + startedDateTime: `2026-10-20T09:00:${second}Z`, + time: 0, + request: { method, url: `http://app.example${path}` }, + response: { + status, + content: { mimeType: "application/json", text: JSON.stringify(body) }, + }, + }); + const { time: _readDuration, ...laterRead } = entry( + "GET", + "/api/settings", + "02", + { theme: "dark" }, + ); + const deletion = entry("DELETE", "/api/items/2", "03", "", 204); + const { time: _deleteDuration, ...unknownDeletion } = deletion; + const capture = environmentManifestCaptureFixture([ + entry("GET", "/api/settings", "00", { theme: "light" }), + laterRead, + entry("PATCH", "/api/settings", "01", { theme: "dark" }), + entry("GET", "/api/health", "00.1", { healthy: true }), + entry("GET", "/api/version", "00.2", { version: "1" }), + entry("GET", "/api/items", "00.3", [{ id: 1 }, { id: 2 }]), + entry("GET", "/api/items/2", "00.4", { id: 2 }), + unknownCompletion ? unknownDeletion : deletion, + entry("GET", "/api/items/2", "04", null, 404), + ]); + const parent = analysisViewCaptureEvidence(capture); + const before = structuredClone(parent); + expect(session.recordEvidence(parent).ok).toBe(true); + const response = await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source: { kind: "retained-evidence", evidence_id: parent.evidence_id }, + view: { kind: "environment-state" }, + }, + }); + expect(response.isError, JSON.stringify(response.content)).not.toBe(true); + const state = analysisViewResultSchema.parse( + parseEvidence(response.structuredContent).normalized_result, + ); + expect(state).toMatchObject({ + kind: "environment-state", + timeline: "recorded-start-time", + documents: expect.arrayContaining([ + expect.objectContaining({ + path: "/api/settings", + value: { theme: "light" }, + }), + expect.objectContaining({ + path: "/api/health", + value: { healthy: true }, + }), + expect.objectContaining({ + path: "/api/version", + value: { version: "1" }, + }), + ]), + writes: expect.arrayContaining([ + expect.objectContaining({ + tool: "delete_items_by_id", + served: !unknownCompletion, + }), + ]), + limitations: expect.arrayContaining([ + "Some records carry no usable request duration; their completion and overlap remain unknown.", + ]), + }); + expect(parent).toEqual(before); + expect(session.evidenceById(parent.evidence_id)).toEqual(before); + }, +); + it("extracts an environment state identically from retained, portable and file MCP Evidence", async () => { const { client, session } = await connect(); const { ajv, inputSchema, outputSchema } = await advertisedMcpToolSchemas(