diff --git a/src/application/environment/environmentStateAnswer.ts b/src/application/environment/environmentStateAnswer.ts index ff55679687..176961b390 100644 --- a/src/application/environment/environmentStateAnswer.ts +++ b/src/application/environment/environmentStateAnswer.ts @@ -25,6 +25,7 @@ import { import { pageBound } from "../../domain/analysisView/environmentStatePaging.js"; import { replayReason } from "../../domain/analysisView/environmentStateVariants.js"; import { isAbsence } from "../../domain/analysisView/environmentStateWriteChecks.js"; +import { updatedEntity } from "../../domain/analysisView/environmentStateWriteApply.js"; import { evaluateQuery, renderQueryList, @@ -291,16 +292,6 @@ export const softDeleteFlag = ( return undefined; }; -/** Fields as an application that soft-deletes shows them once deleted: with the type's deleted flag set, when it has one. */ -const deletedFields = ( - serving: StateServing, - type: string, - fields: JsonObject, -): JsonObject => { - const flag = softDeleteFlag(serving, type); - return flag === undefined ? fields : { ...fields, [flag]: true }; -}; - /** Fields as a delete's answer shows them: with the type's deleted flag as its recorded answer `shown` gave it, if it did. */ export const deleteAnswerFields = ( serving: StateServing, @@ -326,9 +317,11 @@ const deletedEntity = ( call, (collection, id) => store.deleted(type, collection, id), ); - return entity === undefined - ? undefined - : { ...entity, fields: deletedFields(serving, type, entity.fields) }; + if (entity === undefined) return undefined; + const flag = softDeleteFlag(serving, type); + return flag === undefined + ? entity + : updatedEntity(entity, { ...entity.fields, [flag]: true }); }; /** The answer for an entity the state does not hold: the tool's latest recorded 400, 404 or 410, else a tool error. */ diff --git a/src/application/environment/environmentStateDerivedAnswer.test.ts b/src/application/environment/environmentStateDerivedAnswer.test.ts index b464e4d70a..4aad818b19 100644 --- a/src/application/environment/environmentStateDerivedAnswer.test.ts +++ b/src/application/environment/environmentStateDerivedAnswer.test.ts @@ -33,6 +33,44 @@ const objectOf = (body: JsonValue | undefined): Record => { return body; }; +describe("served path-derived fields", () => { + it.each([false, true])( + "preserves repeated parameter occurrences (ambiguous: %s)", + (ambiguous) => { + const project = ambiguous ? 1 : 3; + const path = `/api/orgs/1/projects/${project}/tasks`; + const first = { id: 1, name: "Old", orgId: 1, projectId: project }; + const made = { id: 2, name: "Recorded", orgId: 1, projectId: project }; + const tasks = servedFrom([ + ["GET", path, [first]], + ["GET", `${path}/1`, first], + ["GET", "/api/orgs/2/projects/4/tasks", []], + ["POST", path, made, 201, { name: "Recorded" }], + ["GET", `${path}/2`, made], + ["GET", path, [first, made]], + ]); + const tool = "post_orgs_projects_tasks_by_id_id_2"; + const args = { id: "2", id_2: "4", body: { name: "Elsewhere" } }; + if (ambiguous) { + // Equal recorded values establish no choice between the path positions. + const before = tasks.snapshot(); + expect(tasks.call(tool, args).ok).toBe(false); + expect(tasks.snapshot()).toBe(before); + expect( + tasks.answer(tool, { id: "1", id_2: "1", body: { name: "Here" } }) + .body, + ).toMatchObject({ name: "Here", orgId: 1, projectId: 1 }); + } else { + expect(tasks.answer(tool, args).body).toMatchObject({ + name: "Elsewhere", + orgId: 2, + projectId: 4, + }); + } + }, + ); +}); + describe("served values an application derives from others", () => { it("stamps a created tag's clocks with trailing fraction zeros dropped, as the capture spelled them", () => { const tags = servedFrom(TAG_ROWS); diff --git a/src/application/environment/environmentStateLaterReadsAnswer.test.ts b/src/application/environment/environmentStateLaterReadsAnswer.test.ts index 78fa050912..2df9af718f 100644 --- a/src/application/environment/environmentStateLaterReadsAnswer.test.ts +++ b/src/application/environment/environmentStateLaterReadsAnswer.test.ts @@ -181,3 +181,47 @@ describe("served writes modelled on later reads", () => { }); }); }); + +it.each([ + ["item", SOFT_DELETED_ROWS, "delete_contacts_by_id", { id: "1" }], + ["bulk", STALE_BULK_DELETE_ROWS, "delete_contacts", { body: [{ id: 1 }] }], +] as const)( + "invalidates the deletion flag override after a served %s delete", + (_kind, rows, tool, args) => { + const contacts = servedFrom(rows); + const snapshot = contacts.store.snapshot(); + expect( + contacts.store.replace({ + rea_state: { + ...snapshot.rea_state, + entities: snapshot.rea_state.entities.map((entity) => + entity.id === "1" + ? { + ...entity, + view_fields: [ + { + tool: "get_contacts_by_id", + fields: { deleted: false, name: "Item display" }, + }, + ], + } + : entity, + ), + }, + }).ok, + ).toBe(true); + expect( + contacts.answer("get_contacts_by_id", { id: "1" }).body, + ).toMatchObject({ + deleted: false, + }); + contacts.answer(tool, args); + expect( + contacts.answer("get_contacts_by_id", { id: "1" }).body, + ).toMatchObject({ + id: 1, + name: "Item display", + deleted: true, + }); + }, +); diff --git a/src/application/environment/environmentStateWriteAnswer.ts b/src/application/environment/environmentStateWriteAnswer.ts index 895512e342..1e235795a3 100644 --- a/src/application/environment/environmentStateWriteAnswer.ts +++ b/src/application/environment/environmentStateWriteAnswer.ts @@ -32,6 +32,7 @@ import { } from "../../domain/analysisView/environmentStateWriteLinks.js"; import { pathValuesOf, + pathParameterKeys, type GeneratedRule, type StateWritePlan, type WriteCall, @@ -106,14 +107,12 @@ export const pathValues = ( args: ReplayArguments, ): Map => { const names = environmentToolArguments(tool).path; - const template = tool.path_template - .slice(tool.path_template.indexOf("/") + 1) - .split("/"); + const keys = pathParameterKeys(tool.path_template); const values = new Map(); for (const param of tool.params) { const value = ownMember(args, names.get(param.name) ?? param.name); - const segment = template[param.index] ?? ""; - if (typeof value === "string") values.set(segment.slice(1, -1), value); + const key = keys.get(param.index); + if (typeof value === "string" && key !== undefined) values.set(key, value); } return values; }; diff --git a/src/application/environment/environmentStateWriteDisclosure.test.ts b/src/application/environment/environmentStateWriteDisclosure.test.ts index dce366bb68..e4d59205fc 100644 --- a/src/application/environment/environmentStateWriteDisclosure.test.ts +++ b/src/application/environment/environmentStateWriteDisclosure.test.ts @@ -11,6 +11,8 @@ import { type Book, } from "../../../tests/fixtures/environmentStateWrites.js"; import { servedCatalog } from "./environmentState.js"; +import { environmentToolAnswer } from "./environmentToolAnswer.js"; +import { projectAnalysisError } from "../../domain/analysisErrorProjection.js"; const RENAMED = renamedBook(FIRST_BOOK, "Renamed Book", 3); @@ -72,6 +74,33 @@ const toolOf = (rows: readonly TimedCaptureRow[], name: string) => { }; describe("served write queries", () => { + it.each([ + "?notify=quiet&unmodeled=yes", + "?notify=quiet¬ify=quiet", + "?notify=quiet&unmodeled=yes&unmodeled=no", + ])("refuses unsupported observed query %s before changing state", (query) => { + const { served } = toolOf(PINNED_ROWS, "patch_books_by_id"); + const tool = served.source.manifest.tools.find( + (candidate) => candidate.name === "patch_books_by_id", + ); + if (tool === undefined) throw new Error("Expected patch tool"); + const before = served.snapshot(); + const pending = served.store.pendingWrite(); + const respond = environmentToolAnswer(served.source, served.store); + const result = respond( + tool, + { id: "2", notify: "quiet", body: { name: "Changed" } }, + query, + ); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected pinned-query refusal"); + expect(projectAnalysisError(result.error).message).toMatch( + /query|repeated/u, + ); + expect(served.snapshot()).toBe(before); + expect(served.store.pendingWrite()).toEqual(pending); + }); + it("advertises the recorded query as optional so omission reaches its default", () => { const { served, tool } = toolOf(PINNED_ROWS, "patch_books_by_id"); expect(tool.inputSchema.properties["notify"]).toMatchObject({ diff --git a/src/application/environment/environmentStateWriteQuery.ts b/src/application/environment/environmentStateWriteQuery.ts index ede0857e4c..f5f561cd29 100644 --- a/src/application/environment/environmentStateWriteQuery.ts +++ b/src/application/environment/environmentStateWriteQuery.ts @@ -1,5 +1,6 @@ import { pinnedQuery } from "../../domain/analysisView/environmentStateWriteStamps.js"; import type { AnalysisError } from "../../domain/analysisErrorBase.js"; +import { iterateUrlQueryParameters } from "../../domain/recordedTransportUrl.js"; import { err, ok, type Result } from "../../domain/result.js"; import { refused, type StateServing } from "./environmentStateAnswer.js"; import { callQuery } from "./environmentStateCallLines.js"; @@ -15,7 +16,16 @@ export const pinnedRequest = ( request: WriteRequest, ): Result => { const { tool, plan, args } = request; - if (hasRepeatedQueryArgument(tool, args)) + const supplied = callQuery(tool, args); + const observed = new Map(); + let repeated = hasRepeatedQueryArgument(tool, args); + for (const [name, value] of iterateUrlQueryParameters( + request.observedQuery ?? "", + )) { + if (observed.has(name)) repeated = true; + observed.set(name, value); + } + if (repeated) return err( refused( tool, @@ -24,7 +34,9 @@ export const pinnedRequest = ( ); const query = pinnedQuery( serving.source.exchanges.get(plan.exemplar_ordinal ?? -1)?.query, - callQuery(tool, args), + // Comparison supplies the raw query as well as translated known arguments. + // Keep unmodeled parameters visible without replacing those translations. + new Map([...observed, ...supplied]), ); if (!query.ok) return err(refused(tool, `Tool ${tool.name} ${query.error}`)); const names = environmentToolArguments(tool).query; diff --git a/src/domain/analysisView/EnvironmentManifestAccumulator.test.ts b/src/domain/analysisView/EnvironmentManifestAccumulator.test.ts index c113e8ee5f..600b959249 100644 --- a/src/domain/analysisView/EnvironmentManifestAccumulator.test.ts +++ b/src/domain/analysisView/EnvironmentManifestAccumulator.test.ts @@ -120,3 +120,48 @@ it("charges the group key before building it so an oversized request method fail }); expect(finished(accumulator).tools).toHaveLength(0); }); + +it("counts only admitted scenario transactions and refreshes their snapshot limitations", () => { + const accumulator = new EnvironmentManifestAccumulator(32 * 1024); + expect(finished(accumulator).origin).toBe(""); + expect(accumulator.add(sample(0, "/api/items"), "absent").ok).toBe(true); + const before = finished(accumulator); + expect(before.limitations.join(" ")).not.toContain("no primary origin"); + expect( + accumulator.add( + { + ...sample(1, "/api/items", `{"name":"${"x".repeat(2_000)}"}`), + limitations: ["rejected-only limitation"], + }, + "redacted", + ).ok, + ).toBe(false); + expect(finished(accumulator)).toEqual(before); + expect( + accumulator.add({ + ...sample(1, "/api/items"), + limitations: ["uncommitted limitation", "x".repeat(100_000)], + }).ok, + ).toBe(false); + expect(finished(accumulator)).toEqual(before); + expect( + accumulator.add( + { + ...sample(2, "/api/items", '{"name":"n"}'), + limitations: ["accepted record limitation"], + }, + "redacted", + ).ok, + ).toBe(true); + const after = finished(accumulator); + expect(after.limitations).toContain("accepted record limitation"); + expect(after.records.map((record) => record.ordinal)).toEqual([0, 2]); + expect( + after.limitations.filter((limitation) => + limitation.includes("scenario transactions with a usable request line"), + ), + ).toEqual([ + "1 of 2 scenario transactions with a usable request line retained a response body with declared secret values replaced by redaction markers; tools citing them replay the redacted bytes.", + "1 of 2 scenario transactions with a usable request line retained no response body; tools citing them replay only the recorded status and content type, either of which may be absent.", + ]); +}); diff --git a/src/domain/analysisView/EnvironmentManifestAccumulator.ts b/src/domain/analysisView/EnvironmentManifestAccumulator.ts index fb3018c064..ae1cabbf22 100644 --- a/src/domain/analysisView/EnvironmentManifestAccumulator.ts +++ b/src/domain/analysisView/EnvironmentManifestAccumulator.ts @@ -215,9 +215,11 @@ export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumula record: WebNetworkCaptureRecord, scenarioResponseBody?: ScenarioResponseBodyRetention, ): Result { - const limitations = this.addLimitations(record.limitations); - if (!limitations.ok) return limitations; + const rollbackAdmission = this.retainedCheckpoint(); + let committed = false; try { + const limitations = this.prepareLimitations(record.limitations); + if (!limitations.ok) return limitations; const line = readRecordedRequestLine( record.reported, this.includeQueryParameterNames, @@ -226,16 +228,14 @@ export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumula if (line === null) return this.overBudget(); const releaseGrouping = this.temporaryCheckpoint(); if (line.kind === "excluded") { - this.#unrepresented += 1; releaseGrouping(); - return this.onRecordedRequestLine(record, line); - } - // Coverage uses the same admitted request line as grouping, including origin and raw path. - if (scenarioResponseBody !== undefined) { - this.#scenarioTransactions += 1; - if (scenarioResponseBody === "absent") this.#scenarioWithoutBody += 1; - if (scenarioResponseBody === "redacted") - this.#scenarioRedactedBodies += 1; + const prepared = this.prepareRecordedRequestLine(record, line); + if (!prepared.ok) return prepared; + prepared.value(); + limitations.value(); + this.#unrepresented += 1; + committed = true; + return ok(undefined); } const route = scanRouteTemplate(line.path, this.scratch); if (route === null) return this.overBudget(); @@ -266,7 +266,7 @@ export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumula ? this.#bodyShape(record.reported) : ok(null); if (!bodyShape.ok) return bodyShape; - const grouped = this.#append( + const grouped = this.#prepareAppend( record.ordinal, line, route, @@ -277,18 +277,33 @@ export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumula if (!grouped.ok) return grouped; // Keep the parsed line's admission while releasing grouping-only scratch. releaseGrouping(); - return this.onRecordedRequestLine(record, line); + const prepared = this.prepareRecordedRequestLine(record, line); + if (!prepared.ok) return prepared; + // Both owners admitted their facts; neither commit can reject the record. + grouped.value(); + prepared.value(); + limitations.value(); + // Coverage uses the same admitted request line as grouping, including origin and raw path. + if (scenarioResponseBody !== undefined) { + this.#scenarioTransactions += 1; + if (scenarioResponseBody === "absent") this.#scenarioWithoutBody += 1; + if (scenarioResponseBody === "redacted") + this.#scenarioRedactedBodies += 1; + } + committed = true; + return ok(undefined); } finally { + if (!committed) rollbackAdmission(); this.releaseTemporary(); } } - /** Consume the owner's parsed request line after successful grouping, including exclusions. */ - protected onRecordedRequestLine( + /** Admit derived facts from the owner's parsed line, then commit only after all owners admit it. */ + protected prepareRecordedRequestLine( _record: WebNetworkCaptureRecord, _line: RecordedRequestLine, - ): Result { - return ok(undefined); + ): Result<() => void, AnalysisError> { + return ok(() => {}); } /** Read the top-level key types of the group's first recorded JSON-typed request body; nested values are never walked. */ @@ -316,14 +331,14 @@ export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumula return ok({ type: "object", properties: Object.fromEntries(entries) }); } - #append( + #prepareAppend( ordinal: number, line: RequestLine, route: RouteTemplate, queryNames: ReadonlySet | undefined, bucket: { readonly key: string; readonly group: GroupState | undefined }, exchange: { readonly shape: BodyShape | null; readonly empty: boolean }, - ): Result { + ): Result<() => void, AnalysisError> { const bodyShape = exchange.shape; const mime = mediaTypeEssence(line.rawMime); let group = bucket.group; @@ -366,48 +381,49 @@ export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumula bytes += ENTRY_BYTES + textBytes(name.length); // Reserve the whole mutation while route scratch is still live. if (!this.retain(bytes)) return this.overBudget(); - const position = this.#records.length; - this.#records.push({ - ordinal, - method: line.method, - path: line.path, - status: line.status, - contentType: mime, - }); - const jsonLike = isJsonLike(mime); - const stats = origin ?? { records: 0, jsonLike: 0 }; - stats.records += 1; - if (jsonLike) stats.jsonLike += 1; - this.#origins.set(line.origin, stats); - if (group === undefined) { - group = { + return ok(() => { + const position = this.#records.length; + this.#records.push({ + ordinal, method: line.method, - origin: line.origin, - api_surface_template: route.template, - parts: routeParts(route), - positions: [], - statuses: new Set(), - contentTypes: new Set(), - queryNames: new Set(), - bodyShape, - apiPath: isApiPath(route.template), - jsonLike: false, - bodiless: BODILESS_WRITE_METHODS.has(line.method), - }; - this.#groups.set(bucket.key, group); - } else { - for (const segment of route.segments) { - const part = group.parts[segment.segment_index]; - if (part?.kind === "param") part.values.add(segment.value); + path: line.path, + status: line.status, + contentType: mime, + }); + const jsonLike = isJsonLike(mime); + const stats = origin ?? { records: 0, jsonLike: 0 }; + stats.records += 1; + if (jsonLike) stats.jsonLike += 1; + this.#origins.set(line.origin, stats); + if (group === undefined) { + group = { + method: line.method, + origin: line.origin, + api_surface_template: route.template, + parts: routeParts(route), + positions: [], + statuses: new Set(), + contentTypes: new Set(), + queryNames: new Set(), + bodyShape, + apiPath: isApiPath(route.template), + jsonLike: false, + bodiless: BODILESS_WRITE_METHODS.has(line.method), + }; + this.#groups.set(bucket.key, group); + } else { + for (const segment of route.segments) { + const part = group.parts[segment.segment_index]; + if (part?.kind === "param") part.values.add(segment.value); + } } - } - group.positions.push(position); - group.jsonLike ||= jsonLike; - group.bodiless &&= exchange.empty; - if (line.status !== null) group.statuses.add(line.status); - if (mime !== null) group.contentTypes.add(mime); - for (const name of queryNames ?? []) group.queryNames.add(name); - return ok(undefined); + group.positions.push(position); + group.jsonLike ||= jsonLike; + group.bodiless &&= exchange.empty; + if (line.status !== null) group.statuses.add(line.status); + if (mime !== null) group.contentTypes.add(mime); + for (const name of queryNames ?? []) group.queryNames.add(name); + }); } #tools(groups: readonly ToolGroup[]): Tool[] { @@ -513,7 +529,8 @@ export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumula compare(left.method, right.method) || compare(left.reason, right.reason), ); - const limitations = this.addLimitations([ + // Derived limitations describe this snapshot, not an earlier finish. + const limitations = this.snapshotLimitations([ ...(this.reportsBodyShapes && tools.some((tool) => tool.body_shape !== null) ? [ @@ -521,16 +538,6 @@ export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumula ] : []), ...this.#segmentLimitLimitations(toolGroups), - ...(this.#scenarioWithoutBody > 0 - ? [ - `${String(this.#scenarioWithoutBody)} of ${String(this.#scenarioTransactions)} scenario transactions with a usable request line retained no response body; tools citing them replay only the recorded status and content type, either of which may be absent.`, - ] - : []), - ...(this.#scenarioRedactedBodies > 0 - ? [ - `${String(this.#scenarioRedactedBodies)} of ${String(this.#scenarioTransactions)} scenario transactions with a usable request line retained a response body with declared secret values replaced by redaction markers; tools citing them replay the redacted bytes.`, - ] - : []), ...(this.#unrepresented > 0 ? [ `${String(this.#unrepresented)} retained records without a usable request line are not represented in the manifest.`, @@ -541,6 +548,16 @@ export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumula "No retained record had a parseable origin; the manifest names no primary origin.", ] : []), + ...(this.#scenarioWithoutBody > 0 + ? [ + `${String(this.#scenarioWithoutBody)} of ${String(this.#scenarioTransactions)} scenario transactions with a usable request line retained no response body; tools citing them replay only the recorded status and content type, either of which may be absent.`, + ] + : []), + ...(this.#scenarioRedactedBodies > 0 + ? [ + `${String(this.#scenarioRedactedBodies)} of ${String(this.#scenarioTransactions)} scenario transactions with a usable request line retained a response body with declared secret values replaced by redaction markers; tools citing them replay the redacted bytes.`, + ] + : []), ]); if (!limitations.ok) return limitations; return ok({ @@ -567,7 +584,7 @@ export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumula tools.filter((tool) => tool.params.length > 0).length + excluded.filter((exclusion) => exclusion.templated).length, coverage: completeWithinViewCoverage(tools.length, tools.length), - limitations: this.sortedLimitations(), + limitations: limitations.value, }); } } diff --git a/src/domain/analysisView/EnvironmentStateAccumulator.ts b/src/domain/analysisView/EnvironmentStateAccumulator.ts index 4808d9506f..0750896d2f 100644 --- a/src/domain/analysisView/EnvironmentStateAccumulator.ts +++ b/src/domain/analysisView/EnvironmentStateAccumulator.ts @@ -3,6 +3,7 @@ import type { AnalysisError } from "../analysisErrorBase.js"; import { parseExactJson } from "../exactJsonNumber.js"; import { MAX_JSON_DEPTH, type JsonValue } from "../jsonValue.js"; import { + addOffsetDateTimeMilliseconds, compareOffsetDateTimeInstants, readRecordedStart, } from "../offsetDateTime.js"; @@ -29,6 +30,7 @@ import { base64PrefixChars, readRecordedResponseBody, readRecordedResponseDeclaration, + recordedResponseStatus, recordedResponseBodyIsEmpty, type RecordedResponseBody, } from "./recordedResponseBody.js"; @@ -45,6 +47,9 @@ import { * room under the JSON depth limit to be sealed and served. */ const STATE_NESTING_DEPTH = 16; +// Number.MIN_VALUE milliseconds needs this many decimal places in seconds; +// this also covers the whole-second digits of every finite number duration. +const DURATION_DIGITS = 3 - Math.floor(Math.log10(Number.MIN_VALUE)); /** A recorded response body as state JSON: a JSON-like media type, complete UTF-8 text within the state body limit, exact numbers; otherwise undefined. */ export const stateBodyJson = ( @@ -69,14 +74,14 @@ export const stateBodyJson = ( const startSchema = z.object({ startedDateTime: z.string() }); const durationSchema = z.object({ time: z.number().nonnegative() }); -const redactedWriteSchema = z.object({ +const unlocatedWriteSchema = z.object({ request: z.object({ method: z.string() }), - response: z.object({ status: z.number() }).optional(), }); interface RecordEntry { readonly facts: StateRecordFacts; - /** Null for a write whose URL was redacted: its target is unknown, so it bounds every type. */ + /** Null for a write without a usable URL: its unknown target bounds every type. */ readonly origin: string | null; + readonly targetExclusion?: "url-redacted" | "url-unparseable"; readonly contentType: string; readonly startInterpretation: | NonNullable>["interpretation"] @@ -119,10 +124,10 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator return false; } - protected override onRecordedRequestLine( + protected override prepareRecordedRequestLine( record: WebNetworkCaptureRecord, line: RecordedRequestLine, - ): Result { + ): Result<() => void, AnalysisError> { const started = startSchema.safeParse(record.reported); const duration = durationSchema.safeParse(record.reported); // Admit normalization and fraction parsing before materializing their strings. @@ -148,28 +153,52 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator startMs === undefined || !duration.success ? undefined : startMs + duration.data.time; + // Exact completion keeps arbitrary source precision. Reserve digit-array and + // string expansion before the shared decimal addition allocates them. + if ( + startInstant !== undefined && + duration.success && + duration.data.time !== 0 && + !this.scratch( + RECORD_SCRATCH_BYTES + + textBytes(Math.max(startInstant.fraction.length, DURATION_DIGITS)) * + 3, + ) + ) + return this.overBudget(); + const endInstant = + startInstant === undefined || !duration.success + ? undefined + : addOffsetDateTimeMilliseconds(startInstant, duration.data.time); const timing: StateRecordFacts["timing"] = { start: startInstant === undefined || startMs === undefined ? undefined : { instant: startInstant, ms: startMs }, + end: endInstant, endMs: end !== undefined && Number.isFinite(end) ? end : undefined, }; // A fraction substring may retain its normalized timestamp's backing store, // including trailing zeroes removed from the exact comparison value. const instantBytes = - started.success && startInstant !== undefined + (started.success && startInstant !== undefined ? ENTRY_BYTES + textBytes(started.data.startedDateTime.length + 6) - : 0; + : 0) + + (endInstant !== undefined && endInstant !== startInstant + ? ENTRY_BYTES + textBytes(endInstant.fraction.length) + : 0); if (line.kind === "excluded") - return line.reason === "url-redacted" - ? this.#redactedWrite( + return line.reason === "url-redacted" || + line.reason === "url-unparseable" || + line.reason === "request-unavailable" + ? this.#unlocatedWrite( record, + line.reason === "url-redacted" ? "url-redacted" : "url-unparseable", timing, instantBytes, reading?.interpretation, ) - : ok(undefined); + : ok(() => undefined); // The entry keeps the line's strings and a reference to the already admitted record. if ( !this.retain( @@ -185,56 +214,62 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator ) ) return this.overBudget(); - this.#entries.set(record.ordinal, { - facts: { - ordinal: record.ordinal, - method: line.method, - path: line.path, - query: line.query === "" ? undefined : line.query, - status: line.status, - timing, - }, - origin: line.origin, - contentType: line.rawMime, - startInterpretation: reading?.interpretation, - record, + return ok(() => { + this.#entries.set(record.ordinal, { + facts: { + ordinal: record.ordinal, + method: line.method, + path: line.path, + query: line.query === "" ? undefined : line.query, + status: line.status, + timing, + }, + origin: line.origin, + contentType: line.rawMime, + startInterpretation: reading?.interpretation, + record, + }); }); - return ok(undefined); } - /** Keep a successful write whose URL was redacted, with an empty path whose root relates to every type. */ - #redactedWrite( + /** Prepare a successful write with unknown target, whose empty root relates to every type. */ + #unlocatedWrite( record: WebNetworkCaptureRecord, + reason: "url-redacted" | "url-unparseable", timing: StateRecordFacts["timing"], instantBytes: number, startInterpretation: RecordEntry["startInterpretation"], - ): Result { - const parsed = redactedWriteSchema.safeParse(record.reported); - if (!parsed.success) return ok(undefined); - const status = parsed.data.response?.status ?? 0; + ): Result<() => void, AnalysisError> { + const parsed = unlocatedWriteSchema.safeParse(record.reported); + if (!parsed.success) return ok(() => undefined); + const status = recordedResponseStatus( + readRecordedResponseDeclaration(record.reported), + ); const facts: StateRecordFacts = { ordinal: record.ordinal, method: parsed.data.request.method, path: "", query: undefined, - status: status > 0 ? status : null, + status, timing, }; - if (!isStateWrite(facts)) return ok(undefined); + if (!isStateWrite(facts)) return ok(() => undefined); if ( !this.retain( ENTRY_BYTES * 8 + textBytes(facts.method.length) + instantBytes, ) ) return this.overBudget(); - this.#entries.set(record.ordinal, { - facts, - origin: null, - contentType: "", - startInterpretation, - record, + return ok(() => { + this.#entries.set(record.ordinal, { + facts, + origin: null, + targetExclusion: reason, + contentType: "", + startInterpretation, + record, + }); }); - return ok(undefined); } /** Parse one retained response body as state JSON, charging its decoding and tree as temporary work. */ @@ -330,7 +365,10 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator entry.origin === manifest.value.origin || entry.origin === null, ); const redactedWrites = entries.filter( - (entry) => entry.origin === null, + (entry) => entry.targetExclusion === "url-redacted", + ).length; + const unparseableWrites = entries.filter( + (entry) => entry.targetExclusion === "url-unparseable", ).length; const timeline: StateTimeline = entries.length > 0 && @@ -361,7 +399,8 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator }, }).run(); if (state === null) return this.overBudget(); - const limitations = this.addLimitations([ + const limitations = this.snapshotLimitations([ + ...manifest.value.limitations, ...(entries.some((entry) => entry.startInterpretation === "native-date") ? [ "Some recorded start spellings use native Date parsing at millisecond precision; spellings without an explicit zone depend on the host timezone.", @@ -372,6 +411,11 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator `${String(redactedWrites)} of the recorded writes had a redacted URL; each ends the initial state of every type, since its target is unknown.`, ] : []), + ...(unparseableWrites > 0 + ? [ + `${String(unparseableWrites)} of the recorded writes had an unusable URL; each ends the initial state of every type, since its target is unknown.`, + ] + : []), ...(timeline === "record-order" ? [ "Some records carry no start time, so the state's timeline is record order and overlapping requests are not detected.", @@ -397,7 +441,7 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator ...state, unknowns: UNKNOWNS.map((unknown) => ({ ...unknown })), coverage: manifest.value.coverage, - limitations: this.sortedLimitations(), + limitations: limitations.value, }); } } diff --git a/src/domain/analysisView/EnvironmentStateWriteSimulation.test.ts b/src/domain/analysisView/EnvironmentStateWriteSimulation.test.ts index fe0a31d855..fd022da534 100644 --- a/src/domain/analysisView/EnvironmentStateWriteSimulation.test.ts +++ b/src/domain/analysisView/EnvironmentStateWriteSimulation.test.ts @@ -1418,7 +1418,7 @@ describe("encoded parent spellings", () => { path: "/api/orgs/uid/acme1/books", query: undefined, status: 200, - timing: { start: undefined, endMs: undefined }, + timing: { start: undefined, end: undefined, endMs: undefined }, }; const lines = checkedLines({ tools: [tool], @@ -1621,7 +1621,7 @@ describe("resolved encoded item observations", () => { path: "/api/orgs/acme1/books/%61", query: undefined, status: 204, - timing: { start: undefined, endMs: undefined }, + timing: { start: undefined, end: undefined, endMs: undefined }, }; const target = itemTarget(state, "/api/orgs/{org_id}/books", write); const itemView = { @@ -1642,7 +1642,7 @@ describe("resolved encoded item observations", () => { path: "/api/orgs/acme1/books/%61", query: undefined, status: 200, - timing: { start: undefined, endMs: undefined }, + timing: { start: undefined, end: undefined, endMs: undefined }, }, ]; const search: ObservationSearch = { @@ -2308,23 +2308,53 @@ describe("reads and calls on an entity an unmodelled write left unknown", () => {}, ]; - it("keeps the entity unknown through a failed read that is no absence, for a later read to check", () => { - const rows: TimedCaptureRow[] = [ - ["GET", "/api/books", [named("A"), other]], - ["GET", "/api/books/1", named("A")], - ["GET", "/api/books/2", other], - ["PATCH", "/api/books/1", named("Z"), 200, { name: "Z" }], - ARCHIVE, - // The application was briefly unavailable, so this answer shows nothing of the book. - ["GET", "/api/books/1", { error: "unavailable" }, 503], - ["GET", "/api/books/1", named("Z")], - ]; - expect(gated(rows, "patch_books_by_id")).toMatchObject({ - served: true, - checked_record_ordinals: [6], - contradicted_by: null, - }); - }); + it.each([ + { label: "scalar", read: "unavailable", served: false }, + { label: "array", read: [], served: false }, + { label: "object", read: named("A"), served: true }, + ])( + "requires a usable item baseline after an unmodelled write ($label)", + ({ read, served }) => { + const rows: TimedCaptureRow[] = [ + ["GET", "/api/books", [named("A"), other]], + ["GET", "/api/books/1", named("A")], + ["GET", "/api/books/2", other], + ARCHIVE, + ["GET", "/api/books/1", read], + ["PATCH", "/api/books/1", named("Z"), 200, { name: "Z" }], + ["GET", "/api/books/1", named("Z")], + ]; + expect(gated(rows, "patch_books_by_id")).toMatchObject({ + served, + checked_record_ordinals: served ? [6] : [], + }); + }, + ); + + it.each([ + { status: 503, body: { error: "unavailable" } }, + { status: 200, body: "unavailable" }, + { status: 200, body: [] }, + ])( + "keeps pending effects through an unusable item read ($status, $body)", + ({ status, body }) => { + const rows: TimedCaptureRow[] = [ + ["GET", "/api/books", [named("A"), other]], + ["GET", "/api/books/1", named("A")], + ["GET", "/api/books/2", other], + ["PATCH", "/api/books/1", named("Z"), 200, { name: "Z" }], + ARCHIVE, + // Neither failure nor a successful non-object response shows the book. + ["GET", "/api/books/1", body, status], + ["GET", "/api/books/1", named("Z")], + ]; + expect(gated(rows, "patch_books_by_id")).toMatchObject({ + served: true, + checked_record_ordinals: [6], + contradicted_by: null, + }); + }, + ); it("confirms no update of a volatile field by a read that shows the entity again, as no other read does", () => { const seen = (id: number, count: number) => ({ @@ -2468,7 +2498,7 @@ describe("entities an unmodelled write made", () => { path: "/api/books/3", query: undefined, status: 200, - timing: { start: undefined, endMs: undefined }, + timing: { start: undefined, end: undefined, endMs: undefined }, }; expect( unseen.adopt(view as unknown as StateItemView, { record, index: 1 }), @@ -2493,7 +2523,7 @@ describe("entities an unmodelled write made", () => { path: "/api/books/3", query: undefined, status: 200, - timing: { start: undefined, endMs: undefined }, + timing: { start: undefined, end: undefined, endMs: undefined }, }, index, }); @@ -3201,7 +3231,7 @@ it("learns omitted fields using the raw ID resolved by the item owner", () => { path: "/api/books/%63", query: undefined, status: 200, - timing: { start: undefined, endMs: undefined }, + timing: { start: undefined, end: undefined, endMs: undefined }, }; const target = itemTarget(state, view.type, record); expect(target.id).toBe("%63"); diff --git a/src/domain/analysisView/EnvironmentStateWriteSimulation.ts b/src/domain/analysisView/EnvironmentStateWriteSimulation.ts index 178415eef0..247d408105 100644 --- a/src/domain/analysisView/EnvironmentStateWriteSimulation.ts +++ b/src/domain/analysisView/EnvironmentStateWriteSimulation.ts @@ -555,8 +555,6 @@ export class EnvironmentStateWriteSimulation { ): void { const target = entityTarget(entity.type, entity.collection, entity.id); const writes = this.#touched.get(target) ?? new Map(); - this.#taint.known(entity.type, entity.collection, entity.id); - this.#touched.delete(target); if ( isAbsence(record.status) || this.#input.io.withBody(record.ordinal, (body) => @@ -567,12 +565,16 @@ export class EnvironmentStateWriteSimulation { ), ) ) { + this.#taint.known(entity.type, entity.collection, entity.id); + this.#touched.delete(target); this.#removeEntity(entity); return; } const object = this.#checkFields(view, record, entity, writes); - if (object !== undefined) - this.#state.put(updatedEntity(entity, { ...entity.fields, ...object })); + if (object === undefined) return; + this.#taint.known(entity.type, entity.collection, entity.id); + this.#touched.delete(target); + this.#state.put(updatedEntity(entity, { ...entity.fields, ...object })); } /** Whether an item read shows an entity an unmodelled write made, or one left unknown, again rather than checks it. */ diff --git a/src/domain/analysisView/WebNetworkApiSurfaceAccumulator.test.ts b/src/domain/analysisView/WebNetworkApiSurfaceAccumulator.test.ts index a55cf2e15c..7d020d85a4 100644 --- a/src/domain/analysisView/WebNetworkApiSurfaceAccumulator.test.ts +++ b/src/domain/analysisView/WebNetworkApiSurfaceAccumulator.test.ts @@ -30,9 +30,9 @@ it("bounds query parser expansion before publishing or merging observed names", const accumulator = new WebNetworkApiSurfaceAccumulator(64 * 1024, true); add(accumulator, record(0, "/products?kept=1")); const before = accumulator.page(0, 1); - expect( - accumulator.add(record(1, `/products?${"x=1&".repeat(200)}`)), - ).toMatchObject({ + const rejected = record(1, `/products?${"x=1&".repeat(200)}`); + rejected.limitations = ["Uncommitted rejected record caveat"]; + expect(accumulator.add(rejected)).toMatchObject({ ok: false, error: { _tag: "AnalysisResourceConstraintError" }, }); diff --git a/src/domain/analysisView/WebNetworkApiSurfaceAccumulator.ts b/src/domain/analysisView/WebNetworkApiSurfaceAccumulator.ts index 41aeb8824d..888c5918a2 100644 --- a/src/domain/analysisView/WebNetworkApiSurfaceAccumulator.ts +++ b/src/domain/analysisView/WebNetworkApiSurfaceAccumulator.ts @@ -92,8 +92,10 @@ export class WebNetworkApiSurfaceAccumulator extends WebNetworkProjectionAccumul } add(record: WebNetworkCaptureRecord): Result { - const limitations = this.addLimitations(record.limitations); + const rollbackAdmission = this.retainedCheckpoint(); + const limitations = this.prepareLimitations(record.limitations); if (!limitations.ok) return limitations; + let committed = false; try { const line = readRecordedRequestLine( record.reported, @@ -101,14 +103,23 @@ export class WebNetworkApiSurfaceAccumulator extends WebNetworkProjectionAccumul this.scratch, ); if (line === null) return this.overBudget(); + let appended: Result; if (line.kind === "excluded") - return this.#exclude(record.ordinal, line.reason); - const route = scanRouteTemplate(line.path, this.scratch); - if (route === null) return this.overBudget(); - const queryNames = readQueryParameterNames(line.query, this.scratch); - if (queryNames === null) return this.overBudget(); - return this.#append(record.ordinal, line, route, queryNames); + appended = this.#exclude(record.ordinal, line.reason); + else { + const route = scanRouteTemplate(line.path, this.scratch); + if (route === null) return this.overBudget(); + const queryNames = readQueryParameterNames(line.query, this.scratch); + if (queryNames === null) return this.overBudget(); + appended = this.#append(record.ordinal, line, route, queryNames); + } + if (appended.ok) { + limitations.value(); + committed = true; + } + return appended; } finally { + if (!committed) rollbackAdmission(); this.releaseTemporary(); } } diff --git a/src/domain/analysisView/WebNetworkProjectionAccumulator.ts b/src/domain/analysisView/WebNetworkProjectionAccumulator.ts index 9080df2b2b..e503a8c857 100644 --- a/src/domain/analysisView/WebNetworkProjectionAccumulator.ts +++ b/src/domain/analysisView/WebNetworkProjectionAccumulator.ts @@ -105,15 +105,54 @@ export abstract class WebNetworkProjectionAccumulator { } addLimitations(limitations: readonly string[]): Result { - for (const limitation of limitations) { - if (this.#limitations.has(limitation)) continue; - if (!this.retain(ENTRY_BYTES + textBytes(limitation.length))) - return this.overBudget(); - this.#limitations.add(limitation); - } + const prepared = this.prepareLimitations(limitations); + if (!prepared.ok) return prepared; + prepared.value(); return ok(undefined); } + /** Admit record qualifications without publishing them before the record commits. */ + protected prepareLimitations( + limitations: readonly string[], + ): Result<() => void, AnalysisError> { + const rollbackAdmission = this.retainedCheckpoint(); + const pending = new Set(); + for (const limitation of limitations) { + if (this.#limitations.has(limitation) || pending.has(limitation)) + continue; + if (!this.retain(ENTRY_BYTES + textBytes(limitation.length))) { + rollbackAdmission(); + return this.overBudget(); + } + pending.add(limitation); + } + return ok(() => { + for (const limitation of pending) this.#limitations.add(limitation); + }); + } + + /** Derive budgeted qualifications for this snapshot without retaining old snapshots. */ + protected snapshotLimitations( + derived: readonly string[], + ): Result { + const release = this.temporaryCheckpoint(); + if ( + !this.scratch( + derived.reduce( + (bytes, limitation) => + bytes + ENTRY_BYTES + textBytes(limitation.length), + 0, + ), + ) + ) + return this.overBudget(); + try { + return ok([...new Set([...this.#limitations, ...derived])].sort(compare)); + } finally { + release(); + } + } + protected sortedLimitations(): string[] { return [...this.#limitations].sort(compare); } diff --git a/src/domain/analysisView/analysisView.test.ts b/src/domain/analysisView/analysisView.test.ts index 861a91b94f..8152b392a0 100644 --- a/src/domain/analysisView/analysisView.test.ts +++ b/src/domain/analysisView/analysisView.test.ts @@ -67,6 +67,19 @@ const nativeParent = ( artifact: { path: "/fixtures/example.exe", sha256: "a".repeat(64) }, }); +it("returns input errors for malformed parent Evidence identifiers before projection", () => { + for (const evidenceId of [`xx_${"a".repeat(64)}`, "ev_bad"]) + expect( + projectAnalysisView( + { ...nativeParent(), evidenceId }, + { kind: "summary" }, + ), + ).toMatchObject({ ok: false, error: { _tag: "AnalysisInputError" } }); + expect(projectAnalysisView(nativeParent(), { kind: "summary" }).ok).toBe( + true, + ); +}); + it("projects native procedure and bounded assembly and high-pcode with exact parent identity", () => { const parent = nativeParent(); const procedure = projectAnalysisView(parent, { diff --git a/src/domain/analysisView/analysisView.ts b/src/domain/analysisView/analysisView.ts index 353ff7f420..71f64171cb 100644 --- a/src/domain/analysisView/analysisView.ts +++ b/src/domain/analysisView/analysisView.ts @@ -53,7 +53,7 @@ import { analysisViewInputError, } from "./analysisViewErrors.js"; import { digestCanonicalValue } from "../canonicalDigest.js"; -import { digestSchema } from "../digests.js"; +import { prefixedDigestSchema } from "../digests.js"; import { err, ok, type Result } from "../result.js"; import type { AnalysisError } from "../analysisErrorBase.js"; import { safeParseOwnedBinaryLayout } from "../native/binaryLayout.js"; @@ -269,7 +269,7 @@ export const sealAnalysisView = ( view_digest: digestCanonicalValue(projection, "Analysis view"), }); -const parentDigest = (evidenceId: string): string => evidenceId.slice(3); +const parentEvidenceIdSchema = prefixedDigestSchema("ev"); const parseJavaScriptParent = ( input: unknown, @@ -473,8 +473,7 @@ export const projectAnalysisView = ( view: AnalysisViewRequest, accumulators: AnalysisViewAccumulators = {}, ): Result => { - const digest = parentDigest(parent.evidenceId); - if (!digestSchema.safeParse(digest).success) + if (!parentEvidenceIdSchema.safeParse(parent.evidenceId).success) return err( analysisViewInputError([ { diff --git a/src/domain/analysisView/environmentStateGate.test.ts b/src/domain/analysisView/environmentStateGate.test.ts index 3fe1781780..3e7b4563c7 100644 --- a/src/domain/analysisView/environmentStateGate.test.ts +++ b/src/domain/analysisView/environmentStateGate.test.ts @@ -3,6 +3,7 @@ import { checkItemPlan, type StateGateContext, } from "./environmentStateGate.js"; +import { locateItem, locateItems } from "./environmentStateRules.js"; import type { JsonObject, StateEntity, @@ -21,7 +22,7 @@ const check = ( path: `/api/books/${ordinal + 1}`, query: undefined, status: 200, - timing: { start: undefined, endMs: undefined }, + timing: { start: undefined, end: undefined, endMs: undefined }, })); const bodies: JsonObject[] = clocks.map(([, stamp], index) => ({ id: index + 1, @@ -116,3 +117,37 @@ describe("clock spelling facts at the read gate", () => { expect(large.bytes).toBe(small.bytes); }); }); + +describe("item response identity selection", () => { + it("uses the known identity field before choosing an envelope tier", () => { + const body = { request_id: "42", data: { id: "42", name: "Book" } }; + expect(locateItem(body, "42", "id")).toEqual({ + pointer: "/data", + keys: ["id"], + }); + expect(locateItem(body, "42")).toEqual({ + pointer: "", + keys: ["request_id"], + }); + expect(locateItem(body, "42", "uid")).toBeUndefined(); + }); + + it("retains ambiguity between objects carrying the known identity", () => { + const body = { request_id: "42", left: { id: "42" }, right: { id: "42" } }; + expect(locateItem(body, "42", "id")).toBeUndefined(); + expect(locateItems(body, "42", "id")).toEqual([ + { pointer: "/left", keys: ["id"] }, + { pointer: "/right", keys: ["id"] }, + ]); + expect(locateItems(body, "42", "uid")).toEqual([]); + }); + + it("admits an explicitly known custom identity in a sole array element", () => { + const body = { data: [{ bookNumber: "42", name: "Book" }] }; + expect(locateItem(body, "42", "bookNumber")).toEqual({ + pointer: "/data/0", + keys: ["bookNumber"], + }); + expect(locateItem(body, "42")).toBeUndefined(); + }); +}); diff --git a/src/domain/analysisView/environmentStateListCounters.test.ts b/src/domain/analysisView/environmentStateListCounters.test.ts index 7828890070..e9efa84be1 100644 --- a/src/domain/analysisView/environmentStateListCounters.test.ts +++ b/src/domain/analysisView/environmentStateListCounters.test.ts @@ -108,3 +108,75 @@ it("takes no nested member for a counter unless it equals the number listed", () }, ); }); + +it.each([{ nextCursor: "cursor-2" }, { has_more: true }, { nextPage: 2 }])( + "keeps membership recorded when paging metadata says more items follow: %j", + (pageInfo) => { + const state = environmentStateViewOf( + timedCaptureEvidence([ + ["GET", "/api/records", { list: SEEDED, pageInfo }], + ...SEEDED.map((item): TimedCaptureRow => [ + "GET", + `/api/records/${String(item["Id"])}`, + item, + ]), + ]), + ); + expect( + state.views.find((view) => view.tool === "get_records"), + ).toMatchObject({ + shape: "list", + membership: "recorded", + }); + }, +); + +it("keeps completed paging envelopes whole and does not read entity fields as continuation metadata", () => { + const items = SEEDED.map((item): JsonObject => ({ + ...item, + nextCursor: "entity-value", + })); + const state = environmentStateViewOf( + timedCaptureEvidence([ + [ + "GET", + "/api/records", + { list: items, pageInfo: { nextCursor: null, has_more: false } }, + ], + ...items.map((item): TimedCaptureRow => [ + "GET", + `/api/records/${String(item["Id"])}`, + item, + ]), + ]), + ); + expect(state.views.find((view) => view.tool === "get_records")).toMatchObject( + { + shape: "list", + membership: "all", + }, + ); +}); + +it("keeps a direction-only recording on its recorded members instead of learning an unused direction", () => { + const state = environmentStateViewOf( + timedCaptureEvidence([ + ["GET", "/api/records", SEEDED], + ["GET", "/api/records?direction=desc", SEEDED], + ...SEEDED.map((item): TimedCaptureRow => [ + "GET", + `/api/records/${String(item["Id"])}`, + item, + ]), + ]), + ); + const view = state.views.find( + (candidate) => candidate.tool === "get_records", + ); + expect(view).toMatchObject({ + shape: "list", + membership: "recorded", + checked_record_ordinals: [0, 1], + }); + expect(view).not.toHaveProperty("query.parameters.direction"); +}); diff --git a/src/domain/analysisView/environmentStateListFacts.ts b/src/domain/analysisView/environmentStateListFacts.ts index bc0a8362aa..22c1df8b7c 100644 --- a/src/domain/analysisView/environmentStateListFacts.ts +++ b/src/domain/analysisView/environmentStateListFacts.ts @@ -72,7 +72,7 @@ export const listFacts = ( counted.some( ([, value]) => recordedItemCount(value) !== items.length, ) || - Object.keys(parent).some((name) => continues(name, parent[name])) + hasContinuation(parent, plan.pointer) ) return; const ids = items.map((item) => scalarText(ownMember(item, field)) ?? ""); @@ -150,3 +150,15 @@ const continues = (name: string, value: JsonValue | undefined): boolean => value !== null && value !== false && value !== ""; + +/** Continuation facts in the envelope or its sibling paging objects, never inside the list payload. */ +const hasContinuation = (parent: JsonObject, listPointer: string): boolean => { + const root = pointerTokens(listPointer)[0]; + return Object.entries(parent).some( + ([name, value]) => + continues(name, value) || + (name !== root && + isObject(value) && + Object.entries(value).some(([inner, flag]) => continues(inner, flag))), + ); +}; diff --git a/src/domain/analysisView/environmentStateListQuery.test.ts b/src/domain/analysisView/environmentStateListQuery.test.ts index f4790ce5a4..92425f5ca2 100644 --- a/src/domain/analysisView/environmentStateListQuery.test.ts +++ b/src/domain/analysisView/environmentStateListQuery.test.ts @@ -376,6 +376,16 @@ describe("queries the state can always answer", () => { }); describe("parameters that ask for an order", () => { + it("cannot apply a direction without a sort field, even when the recorded order happens to match", () => { + expect(explainLine(line("?direction=desc", [1, 2, 3, 4]))).toBeUndefined(); + const parameters = { direction: { role: "direction" } } as const; + expect(ids("?direction=asc", parameters)).toBeUndefined(); + expect(ids("?direction=desc", parameters)).toBeUndefined(); + expect( + ids("?sort=id&direction=desc", { ...parameters, sort: { role: "sort" } }), + ).toEqual(["4", "3", "2", "1"]); + }); + it("reads a sort field in any spelling as a sort, never as a parameter without effect", () => { expect(explainLine(line("?_sort=id&_order=DESC", [4, 3, 2, 1]))).toEqual({ _sort: { role: "sort" }, diff --git a/src/domain/analysisView/environmentStateListQuery.ts b/src/domain/analysisView/environmentStateListQuery.ts index 6c008b6fde..54658a143c 100644 --- a/src/domain/analysisView/environmentStateListQuery.ts +++ b/src/domain/analysisView/environmentStateListQuery.ts @@ -206,7 +206,11 @@ const readCall = ( call.page = { index: number, first: parameter.first }; else return undefined; } - return call; + // A direction cannot identify the field being ordered. Leaving it unused + // would turn a recorded direction into permission to answer its opposite. + return call.direction !== undefined && call.sort === undefined + ? undefined + : call; }; /** Entities in the order a sort value names, ties in state order, and those whose place state order decides: all without a sort, else those tied with a neighbour; undefined when it names no field. */ diff --git a/src/domain/analysisView/environmentStateQueryCalls.test.ts b/src/domain/analysisView/environmentStateQueryCalls.test.ts index ff63fd447c..30ae2ad061 100644 --- a/src/domain/analysisView/environmentStateQueryCalls.test.ts +++ b/src/domain/analysisView/environmentStateQueryCalls.test.ts @@ -51,6 +51,12 @@ const readOnlyOf = ( ), ms: call.start ?? index, }, + end: + call.end === null + ? undefined + : parseOffsetDateTimeInstant( + new Date(call.end ?? index).toISOString(), + ), endMs: call.end === null ? undefined : (call.end ?? index), }, })); @@ -505,6 +511,7 @@ describe("read-only decisions over many calls", () => { ), ms: index, }, + end: parseOffsetDateTimeInstant(new Date(index).toISOString()), endMs: index, }, }; @@ -573,6 +580,7 @@ describe("read-only decisions over a long run of calls", () => { ), ms: index, }, + end: parseOffsetDateTimeInstant(new Date(index).toISOString()), endMs: index, }, }; @@ -641,6 +649,7 @@ describe("read-only decisions over many lines", () => { ), ms: index, }, + end: parseOffsetDateTimeInstant(new Date(index).toISOString()), endMs: index, }, }; diff --git a/src/domain/analysisView/environmentStateRewrites.test.ts b/src/domain/analysisView/environmentStateRewrites.test.ts index 68e8e9b6dd..fade955ac1 100644 --- a/src/domain/analysisView/environmentStateRewrites.test.ts +++ b/src/domain/analysisView/environmentStateRewrites.test.ts @@ -24,6 +24,10 @@ const record = ( instant: parseOffsetDateTimeInstant(new Date(start).toISOString()), ms: start, }, + end: + endMs === undefined + ? undefined + : parseOffsetDateTimeInstant(new Date(endMs).toISOString()), endMs, }, }); diff --git a/src/domain/analysisView/environmentStateRules.ts b/src/domain/analysisView/environmentStateRules.ts index 6270c74fa4..48a49dacf3 100644 --- a/src/domain/analysisView/environmentStateRules.ts +++ b/src/domain/analysisView/environmentStateRules.ts @@ -31,6 +31,8 @@ export interface StateRecordFacts { readonly start: | { readonly instant: OffsetDateTimeInstant; readonly ms: number } | undefined; + /** Exact completion, independent of the millisecond chronology policy. */ + readonly end: OffsetDateTimeInstant | undefined; readonly endMs: number | undefined; }; } @@ -630,19 +632,22 @@ export const replaceAt = ( const matchingKeys = (object: JsonObject, value: string): string[] => Object.keys(object).filter((key) => scalarText(object[key]) === value); -/** The entity object an item response carries with the keys repeating its path value: at the root or in one wrapper member, else as the element of a one-element array there. */ -export const locateItem = ( +/** Candidate entity objects at the first matching envelope tier. A known identity field constrains admission before choosing a tier; multiple candidates remain ambiguous. */ +export const locateItems = ( body: JsonValue, value: string, -): { readonly pointer: string; readonly keys: string[] } | undefined => { + idField?: string, +): readonly { readonly pointer: string; readonly keys: string[] }[] => { const located = ( object: JsonValue | undefined, pointer: string, named = false, ) => { const keys = isObject(object) - ? matchingKeys(object, value).filter( - (key) => !named || IDENTITY_NAMES.includes(key), + ? matchingKeys(object, value).filter((key) => + idField !== undefined + ? key === idField + : !named || IDENTITY_NAMES.includes(key), ) : []; return keys.length > 0 ? [{ pointer, keys }] : []; @@ -658,8 +663,17 @@ export const locateItem = ( located(soleElement(item), `${pointerTo(name)}/0`, true), ), ]; - const chosen = tiers.find((tier) => tier.length > 0); - return chosen?.length === 1 ? chosen[0] : undefined; + return tiers.find((tier) => tier.length > 0) ?? []; +}; + +/** The unique entity object at the first matching envelope tier; absent or ambiguous otherwise. */ +export const locateItem = ( + body: JsonValue, + value: string, + idField?: string, +): { readonly pointer: string; readonly keys: string[] } | undefined => { + const candidates = locateItems(body, value, idField); + return candidates.length === 1 ? candidates[0] : undefined; }; /** Whether every item carries `name` as a distinct string or number. */ diff --git a/src/domain/analysisView/environmentStateView.test.ts b/src/domain/analysisView/environmentStateView.test.ts index 7a068d9426..1e77daf221 100644 --- a/src/domain/analysisView/environmentStateView.test.ts +++ b/src/domain/analysisView/environmentStateView.test.ts @@ -12,7 +12,7 @@ import { scenarioCaptureFixture, } from "../../../tests/fixtures/browserScenarioCapture.js"; import type { Evidence } from "../evidence.js"; -import type { JsonValue } from "../jsonValue.js"; +import { ownedJsonObjectSchema, type JsonValue } from "../jsonValue.js"; import { webNetworkCaptureSchema } from "../webNetworkCapture.js"; import { analysisViewResultSchema, @@ -842,7 +842,7 @@ it("leaves every line checked rather than compare more lines than the variant pa path: "/api/items", query: `?view=${String(ordinal)}`, status: 200, - timing: { start: undefined, endMs: undefined }, + timing: { start: undefined, end: undefined, endMs: undefined }, })); const shape = { entities: new Map( @@ -869,7 +869,7 @@ it("tells apart member names that hold the separator, so a line spelling them ot path: "/api/items/1", query, status: 200, - timing: { start: undefined, endMs: undefined }, + timing: { start: undefined, end: undefined, endMs: undefined }, }); const observed = (ordinal: number, object: JsonObject) => ({ position: ordinal, @@ -909,7 +909,7 @@ it("leaves every line checked when comparing many mostly empty lines would excee method: "GET", path: "/api/items", status: 200, - timing: { start: undefined, endMs: undefined }, + timing: { start: undefined, end: undefined, endMs: undefined }, }; return Object.defineProperty(record, "query", { enumerable: true, @@ -950,7 +950,7 @@ const expandedItemLines = (items: number) => { path: `/api/items/${String(ordinal % items)}`, query: ordinal < items ? undefined : "?expand=shelf", status: 200, - timing: { start: undefined, endMs: undefined }, + timing: { start: undefined, end: undefined, endMs: undefined }, }), ); const shapes = new Map( @@ -1144,6 +1144,7 @@ it("finds each span's earliest write without scanning the writes before it", () return start; }, }, + end: offsetDateTimeInstantFromEpochMilliseconds(end), get endMs() { reads += 1; return end; @@ -2020,3 +2021,168 @@ it.each([ if (missingDuration) expect(state.write_clock).toBeNull(); }, ); + +it.each(["not a URL", "", 42, undefined])( + "bounds initial state at a successful write whose URL is unavailable: %j", + (url) => { + const source = captureOf([ + ["GET", "/api/items/1", { id: 1, title: "initial" }], + ["PUT", "/api/items/1", { id: 1 }], + ["GET", "/api/items/1", { id: 1, title: "after" }], + ]); + const capture = webNetworkCaptureSchema.parse(source.normalizedResult); + const reported = ownedJsonObjectSchema.parse(capture.records[1]!.reported); + const request = ownedJsonObjectSchema.parse(reported.request); + if (url === undefined) delete request.url; + else request.url = url; + const before = structuredClone(capture); + const state = stateOf(parentOf(analysisViewCaptureEvidence(capture))); + expect(state.types).toMatchObject([ + { init_before_ordinal: 1, volatile_fields: [] }, + ]); + expect(state.entities).toMatchObject([ + { fields: { title: "initial" }, record_ordinals: [0] }, + ]); + expect(state.limitations).toContainEqual( + expect.stringMatching(/1.*writes.*unusable URL.*target is unknown/iu), + ); + expect( + state.limitations.some((text) => text.includes("redacted URL")), + ).toBe(false); + expect(capture).toEqual(before); + }, +); + +it("commits neither a manifest line nor state facts when inherited state admission fails", () => { + const records = webNetworkCaptureSchema.parse( + environmentManifestCaptureFixture([ + { + startedDateTime: "2026-10-20T09:00:00Z", + time: 0, + request: { method: "GET", url: "https://manifest.test/api/health" }, + response: { + status: 200, + content: { mimeType: "application/json", text: '{"healthy":true}' }, + }, + }, + { + startedDateTime: "x".repeat(5000), + time: 0, + request: { method: "GET", url: "https://manifest.test/api/items" }, + response: { + status: 200, + content: { mimeType: "application/json", text: "[]" }, + }, + }, + ]), + ).records; + const accumulator = new EnvironmentStateAccumulator(32 * 1024); + expect(accumulator.add(records[0]!).ok).toBe(true); + expect(accumulator.add(records[1]!).ok).toBe(false); + const admitted = accumulator.finish(); + if (!admitted.ok) throw admitted.error; + expect(admitted.value.records.map(({ ordinal }) => ordinal)).toEqual([0]); + const retry = structuredClone(records[1]!); + if ( + retry.reported === null || + typeof retry.reported !== "object" || + Array.isArray(retry.reported) + ) + throw new Error("Expected fixture object"); + retry.reported.startedDateTime = "2026-10-20T09:00:01Z"; + expect(accumulator.add(retry).ok).toBe(true); + const state = accumulator.finishState(); + if (!state.ok) throw state.error; + expect(state.value.documents.map(({ path }) => path)).toEqual([ + "/api/health", + "/api/items", + ]); +}); + +it.each([ + { method: "GET", status: 200 }, + { method: "PUT", status: 500 }, +])( + "does not infer a successful state write from an excluded $method/$status record", + ({ method, status }) => { + const source = captureOf([ + ["GET", "/api/items/1", { id: 1, title: "initial" }], + [method, "/api/items/1", {}, status], + ["GET", "/api/items/1", { id: 1, title: "after" }], + ]); + const capture = webNetworkCaptureSchema.parse(source.normalizedResult); + const reported = ownedJsonObjectSchema.parse(capture.records[1]!.reported); + ownedJsonObjectSchema.parse(reported.request).url = "not a URL"; + const state = stateOf(parentOf(analysisViewCaptureEvidence(capture))); + expect(state.types).toMatchObject([ + { init_before_ordinal: null, volatile_fields: ["title"] }, + ]); + expect(state.entities).toMatchObject([{ fields: { title: "after" } }]); + expect( + state.limitations.some((text) => + text.includes("writes had an unusable URL"), + ), + ).toBe(false); + }, +); + +it.each([{}, { status: "unknown" }, { status: null }])( + "keeps a possible unlocated write when its response status is unknown: %j", + (response) => { + const source = captureOf([ + ["GET", "/api/items/1", { id: 1, title: "initial" }], + ["PUT", "/api/items/1", {}], + ["GET", "/api/items/1", { id: 1, title: "after" }], + ]); + const capture = webNetworkCaptureSchema.parse(source.normalizedResult); + const reported = ownedJsonObjectSchema.parse(capture.records[1]!.reported); + ownedJsonObjectSchema.parse(reported.request).url = "not a URL"; + reported.response = ownedJsonObjectSchema.parse(response); + const state = stateOf(parentOf(analysisViewCaptureEvidence(capture))); + expect(state.types).toMatchObject([ + { init_before_ordinal: 1, volatile_fields: [] }, + ]); + expect(state.entities).toMatchObject([ + { fields: { title: "initial" }, record_ordinals: [0] }, + ]); + }, +); + +it.each([true, false])( + "derives timeline limitations from the current state snapshot (timed: %s)", + (timed) => { + const accumulator = new EnvironmentStateAccumulator(); + const empty = accumulator.finishState(); + if (!empty.ok) throw empty.error; + expect(empty.value.timeline).toBe("record-order"); + const record = webNetworkCaptureSchema.parse( + environmentManifestCaptureFixture([ + { + ...(timed ? { startedDateTime: "2026-10-20T09:00:00Z" } : {}), + time: 0, + request: { method: "GET", url: "https://snapshot.test/api/items" }, + response: { + status: 200, + content: { mimeType: "application/json", text: "[]" }, + }, + }, + ]), + ).records[0]!; + expect(accumulator.add(record).ok).toBe(true); + const current = accumulator.finishState(); + if (!current.ok) throw current.error; + expect(current.value.timeline).toBe( + timed ? "recorded-start-time" : "record-order", + ); + expect( + current.value.limitations.some((text) => + text.includes("Some records carry no start time"), + ), + ).toBe(!timed); + expect( + current.value.limitations.some((text) => + text.includes("No retained record had a parseable origin"), + ), + ).toBe(false); + }, +); diff --git a/src/domain/analysisView/environmentStateWriteAddress.test.ts b/src/domain/analysisView/environmentStateWriteAddress.test.ts index 17ba45e74a..4da89058f8 100644 --- a/src/domain/analysisView/environmentStateWriteAddress.test.ts +++ b/src/domain/analysisView/environmentStateWriteAddress.test.ts @@ -195,6 +195,34 @@ describe("item actions", () => { }); }); + it("locates an action's entity by its known id field before response metadata", () => { + const response = { request_id: 1, data: ARCHIVED }; + expect( + planOf( + [ + [ + "GET", + "/api/books", + [shelved(FIRST_BOOK, false), shelved(SECOND_BOOK, false)], + ], + ["GET", "/api/books/1", { data: shelved(FIRST_BOOK, false) }], + ["GET", "/api/books/2", { data: shelved(SECOND_BOOK, false) }], + ["POST", "/api/books/1/archive", response], + ["GET", "/api/books/1", { data: ARCHIVED }], + ["POST", "/api/books/1/archive", response], + ["GET", "/api/books/1", { data: ARCHIVED }], + ], + "post_books_archive_by_id", + ), + ).toMatchObject({ + effect: "update", + served: true, + address: { by: "action", segment: "archive" }, + response: { shape: "entity", pointer: "/data" }, + rules: [{ field: "archived", rule: "set", value: true }], + }); + }); + it("serves no action seen changing its field only once, which may toggle it", () => { expect( planOf( diff --git a/src/domain/analysisView/environmentStateWriteAddress.ts b/src/domain/analysisView/environmentStateWriteAddress.ts index e50fa830fe..eae197c4e6 100644 --- a/src/domain/analysisView/environmentStateWriteAddress.ts +++ b/src/domain/analysisView/environmentStateWriteAddress.ts @@ -18,6 +18,7 @@ import { entityTarget, succeeded } from "./environmentStateWriteChecks.js"; import type { WriteIo } from "./environmentStateWriteEffects.js"; import { pathValuesOf, + pathParameterKeys, type BodyAddress, type StateWritePlan, type WriteAddress, @@ -95,19 +96,20 @@ export const itemAction = ( ) return undefined; const views = facts.itemViews(target); + const itemKey = pathParameterKeys(tool.path_template).get( + segments.length - 2, + )!; const showsEntity = (record: StateRecordFacts): boolean => { - const id = pathValuesOf(tool.path_template, record.path)?.get( - item.slice(1, -1), - ); + const id = pathValuesOf(tool.path_template, record.path)?.get(itemKey); if (id === undefined || record.status === 201) return false; if (facts.emptyResponse(record.ordinal)) return true; return facts.withBody(record.ordinal, (body) => { if (body === undefined || body === null) return true; - const located = locateItem(body, id); + const located = locateItem(body, id, idField); const object = located === undefined ? undefined : at(body, located.pointer); return ( - located?.keys.includes(idField) === true && + located !== undefined && isObject(object) && views.some( (view) => diff --git a/src/domain/analysisView/environmentStateWriteBlame.ts b/src/domain/analysisView/environmentStateWriteBlame.ts index d285b733c1..9a7a46eaa2 100644 --- a/src/domain/analysisView/environmentStateWriteBlame.ts +++ b/src/domain/analysisView/environmentStateWriteBlame.ts @@ -152,7 +152,7 @@ export class FailedReadBlame { const response = call === undefined ? undefined - : this.#responseObject(records[call]!, item.entity.id); + : this.#responseObject(records[call]!, item.entity); if (response !== undefined) responses.set(name, response); } return fieldBlame( @@ -169,10 +169,13 @@ export class FailedReadBlame { /** The entity a write's response shows, located by its id. */ #responseObject( record: StateRecordFacts, - id: string, + entity: StateEntity, ): JsonObject | undefined { return this.#facts.io.withBody(record.ordinal, (body) => { - const located = body === undefined ? undefined : locateItem(body, id); + const located = + body === undefined + ? undefined + : locateItem(body, entity.id, this.#facts.idField(entity.type)); const object = located === undefined ? undefined : at(body!, located.pointer); return isObject(object) ? object : undefined; diff --git a/src/domain/analysisView/environmentStateWriteCopies.test.ts b/src/domain/analysisView/environmentStateWriteCopies.test.ts index 9406f327f6..1ad8fe9585 100644 --- a/src/domain/analysisView/environmentStateWriteCopies.test.ts +++ b/src/domain/analysisView/environmentStateWriteCopies.test.ts @@ -119,6 +119,22 @@ it( ); describe("coincidences no member derives a created field from", () => { + it.each([ + ["before", (value: string) => `𐐀${value}`], + ["after", (value: string) => `${value}𐐀`], + ])( + "keeps a member accepted when an astral letter joins its text %s", + (_, label) => { + const value = "harbour-notes"; + const body = create( + { id: 9, name: value, label: label(value) }, + { name: value }, + ); + expect(body.accepted).toEqual(["name"]); + expect(body.fixed).toEqual([]); + }, + ); + it("keeps a reference accepted when a constant repeats another reference of the named entity", () => { const channel = "1bdxw3qoyj8zjyyhdjbdobdf4r"; const user = "c10c054bc2a4478bb78e0210549376b1"; diff --git a/src/domain/analysisView/environmentStateWriteCopies.ts b/src/domain/analysisView/environmentStateWriteCopies.ts index 6059c467d1..44f2dec475 100644 --- a/src/domain/analysisView/environmentStateWriteCopies.ts +++ b/src/domain/analysisView/environmentStateWriteCopies.ts @@ -8,7 +8,12 @@ import { } from "./environmentStateRules.js"; import { GENERATED } from "./environmentStateWriteGenerated.js"; import type { FieldRule } from "./environmentStateWriteRules.js"; -import { delimitedOffsets, readClock } from "./environmentStateWriteValues.js"; +import { + delimitedOffsets, + joinsAfter, + joinsBefore, + readClock, +} from "./environmentStateWriteValues.js"; /** Shorter body strings are too common to show that a field copied one. */ export const COPY_MIN_LENGTH = 8; @@ -52,12 +57,6 @@ export const storedMembers = ( }), ); -/** A letter or digit, which joins a token to the text around it. */ -const WORD = /[\p{L}\p{N}]/u; - -const joins = (character: string | undefined): boolean => - character !== undefined && WORD.test(character); - /** * The texts that hold each token delimited, read in one pass per text: from each start after a * delimiter, each length a token has that a delimiter ends, rather than a search per token and text. @@ -72,10 +71,10 @@ const holdersOf = ( const holders = new Map>(); for (const [field, text] of texts) for (let at = 0; at < text.length; at += 1) { - if (joins(text[at - 1])) continue; + if (joinsBefore(text, at)) continue; for (const length of lengths) { if (at + length > text.length) break; - if (joins(text[at + length])) continue; + if (joinsAfter(text, at + length)) continue; const token = text.slice(at, at + length); if (tokens.has(token)) holders.set(token, (holders.get(token) ?? new Set()).add(field)); diff --git a/src/domain/analysisView/environmentStateWriteCutoffs.test.ts b/src/domain/analysisView/environmentStateWriteCutoffs.test.ts index 09fedb7c96..56451c650a 100644 --- a/src/domain/analysisView/environmentStateWriteCutoffs.test.ts +++ b/src/domain/analysisView/environmentStateWriteCutoffs.test.ts @@ -18,6 +18,7 @@ const write = (path: string, ordinal: number): StateRecordFacts => ({ instant: offsetDateTimeInstantFromEpochMilliseconds(ordinal), ms: ordinal, }, + end: offsetDateTimeInstantFromEpochMilliseconds(ordinal), endMs: ordinal, }, }); diff --git a/src/domain/analysisView/environmentStateWriteEffects.test.ts b/src/domain/analysisView/environmentStateWriteEffects.test.ts new file mode 100644 index 0000000000..fd5c9f6288 --- /dev/null +++ b/src/domain/analysisView/environmentStateWriteEffects.test.ts @@ -0,0 +1,118 @@ +import { expect, it } from "vitest"; + +import { + timedCaptureEvidence, + type TimedCaptureRow, +} from "../../../tests/fixtures/analysisView.js"; +import { + FIRST_BOOK, + environmentStateViewOf, +} from "../../../tests/fixtures/environmentStateWrites.js"; + +it.each([ + { fraction: "123456790", served: true, label: "one nanosecond later" }, + { fraction: "123456788", served: false, label: "one nanosecond earlier" }, + { fraction: "123456789", served: false, label: "the same instant" }, + { fraction: "124456789", served: true, label: "one millisecond later" }, +])("checks an update clock at $label", ({ fraction, served }) => { + const first = { + ...FIRST_BOOK, + name: "Second title", + updated_at: "2026-10-17T18:00:03.123456789Z", + }; + const second = { + ...first, + name: "Third title", + updated_at: `2026-10-17T18:00:03.${fraction}Z`, + }; + const state = environmentStateViewOf( + timedCaptureEvidence([ + ["GET", "/api/books", [FIRST_BOOK]], + ["GET", "/api/books/1", FIRST_BOOK], + ["PATCH", "/api/books/1", first, 200, { name: first.name }], + ["GET", "/api/books/1", first], + ["PATCH", "/api/books/1", second, 200, { name: second.name }], + ["GET", "/api/books/1", second], + ]), + ); + expect( + state.writes.find((plan) => plan.tool === "patch_books_by_id"), + ).toMatchObject({ + served, + reason: served ? null : "unexplained-change", + checked_record_ordinals: served ? [3, 5] : [3], + rules: expect.arrayContaining([ + { + field: "updated_at", + rule: "clock", + format: { kind: "iso", separator: "T", fraction: 9, zone: "Z" }, + }, + ]), + }); +}); + +it.each([false, true])( + "learns only after an unknown entity is observed again (update while unknown: %s)", + (updateWhileUnknown) => { + const initial = { ...FIRST_BOOK, revision: 1 }; + const observed = { ...initial, name: "Second title", revision: 2 }; + const final = { ...observed, name: "Third title" }; + const unknownCall: TimedCaptureRow[] = updateWhileUnknown + ? [["PATCH", "/api/books/1", observed, 200, { name: observed.name }]] + : []; + const state = environmentStateViewOf( + timedCaptureEvidence([ + ["GET", "/api/books", [initial]], + ["GET", "/api/books/1", initial], + ["POST", "/api/books/1/import", { queued: true }, 302, {}], + ...unknownCall, + ["GET", "/api/books/1", observed], + ["PATCH", "/api/books/1", final, 200, { name: final.name }], + ["GET", "/api/books/1", final], + ]), + ); + expect( + state.writes.find((plan) => plan.tool === "patch_books_by_id"), + ).toMatchObject({ + served: true, + reason: null, + exemplar_ordinal: updateWhileUnknown ? 5 : 4, + checked_record_ordinals: [updateWhileUnknown ? 6 : 5], + rules: [{ field: "name", rule: "echo" }], + }); + }, +); + +it.each([ + { requestId: 1, ambiguous: false, served: true }, + { requestId: 2, ambiguous: false, served: true }, + { requestId: 1, ambiguous: true, served: false }, +])( + "uses the known identity field for response observation ($requestId, ambiguous: $ambiguous)", + ({ requestId, ambiguous, served }) => { + const changed = { ...FIRST_BOOK, name: "Second title" }; + const response = { + request_id: requestId, + data: changed, + ...(ambiguous ? { other: changed } : {}), + }; + const state = environmentStateViewOf( + timedCaptureEvidence([ + ["GET", "/api/books", [FIRST_BOOK]], + ["GET", "/api/books/1", FIRST_BOOK], + ["PATCH", "/api/books/1", response, 200, { name: changed.name }], + ["GET", "/api/books/1", changed], + ]), + ); + const plan = state.writes.find( + (candidate) => candidate.tool === "patch_books_by_id", + ); + expect(plan?.served).toBe(served); + if (served) + expect(plan).toMatchObject({ + observation: { ordinal: 2, pointer: "/data" }, + response: { ordinal: 2, shape: "entity", pointer: "/data" }, + checked_record_ordinals: [3], + }); + }, +); diff --git a/src/domain/analysisView/environmentStateWriteEffects.ts b/src/domain/analysisView/environmentStateWriteEffects.ts index 003255ba5b..e09d3c7a0a 100644 --- a/src/domain/analysisView/environmentStateWriteEffects.ts +++ b/src/domain/analysisView/environmentStateWriteEffects.ts @@ -28,6 +28,7 @@ import { type SimulatedState, } from "./environmentStateWriteChecks.js"; import { + compareValues, explainUpdate, responseShapeOf, } from "./environmentStateWriteExplain.js"; @@ -367,11 +368,12 @@ const updatedObservation = ( step.record.ordinal, (body): Observed | undefined => { const located = - body === undefined ? undefined : locateItem(body, entity.id); + body === undefined + ? undefined + : locateItem(body, entity.id, entity.idField); const object = located === undefined ? undefined : at(body!, located.pointer); - return located?.keys.includes(entity.idField) === true && - isObject(object) + return located !== undefined && isObject(object) ? { ordinal: step.record.ordinal, pointer: located.pointer, @@ -657,6 +659,9 @@ export const updateEntity = ( return UNKNOWN; } const { targets, target, recorded } = outcomes(context, entity, after.object); + // An unknown baseline cannot establish this call's changes. Keep its outcome + // without creating or refining rules from values another write may have changed. + if (context.unknown(type, collection, id)) return recorded(); // Only confirmed absence makes a bodyless action; malformed or unretained content stays unknown. const body = "body" in named @@ -670,7 +675,6 @@ export const updateEntity = ( fail(draft, step, "body-unusable"); return recorded(); } - const unknown = context.unknown(type, collection, id); const call = { ...callOf(step, body), body }; const fitted = fitUpdateClocks( draft, @@ -704,7 +708,7 @@ export const updateEntity = ( const explained = { ...fitted.explained, rules: later }; for (const member of keptOrders(call.body, after.object)) draft.kept.add(member); - if (explained.unexplained.length > 0 && !unknown) { + if (explained.unexplained.length > 0) { refuse(draft, step, "unexplained-change"); return recorded(); } @@ -713,16 +717,15 @@ export const updateEntity = ( searchFrom(context, step), itemReads(context, { type, idField, collection, id }), )?.object; - // Each call must agree with the plan; one on an unknown entity cannot be checked, so it adds nothing to the plan. + // Each call with a known baseline must agree with the plan. if ( built(draft) && - (unknown || - contradictsPlan( - { ...draft, plan: { ...draft.plan, rules } }, - entity.fields, - after.object, - explained, - ) || + (contradictsPlan( + { ...draft, plan: { ...draft.plan, rules } }, + entity.fields, + after.object, + explained, + ) || // A member calls left out is cleared by every one or by none. // A reference this call or an earlier one showed counts its member as the field's, whichever came first. defaultsDisagree(explained.body.defaults, { earlier: draft.updates }, [ @@ -735,7 +738,7 @@ export const updateEntity = ( [...rules, ...explained.rules], )) ) { - if (!unknown) refuse(draft, step, "unexplained-change"); + refuse(draft, step, "unexplained-change"); return recorded(); } draft.plan.rules = rules; @@ -778,9 +781,10 @@ export const updateEntity = ( after: after.object, kept, }); - if (!written.ok || (disagrees && !unknown)) + if (!written.ok || disagrees) { refuse(draft, step, "unexplained-change"); - if (!written.ok || disagrees || unknown) return recorded(); + return recorded(); + } rememberCall( context.io.retain, draft, @@ -893,7 +897,7 @@ const contradictsGeneratedClock = ( candidate.field === rule.field && candidate.rule === "clock", ) || next === undefined || - next.ms <= previous.ms + compareValues(after[rule.field], ownMember(before, rule.field))! <= 0 ); }); diff --git a/src/domain/analysisView/environmentStateWriteExplain.test.ts b/src/domain/analysisView/environmentStateWriteExplain.test.ts index 2cc663d846..2624c33e2d 100644 --- a/src/domain/analysisView/environmentStateWriteExplain.test.ts +++ b/src/domain/analysisView/environmentStateWriteExplain.test.ts @@ -13,6 +13,7 @@ import { type CreateEvidence, } from "./environmentStateWriteExplain.js"; import type { UpdateEvidence } from "./environmentStateWriteChanges.js"; +import { writtenResponse } from "./environmentStateWriteApply.js"; import { callRenewal, unexplainedRenewals, @@ -504,6 +505,35 @@ it.each<[string, string[]]>([ type Shaped = [WriteResponse["shape"], string | null]; +it.each([false, true])( + "keeps response entity selection distinct from repeated envelope metadata, ambiguous=%s", + (ambiguous) => { + const recorded = { + request_id: "42", + data: { id: "42", title: "Captured title" }, + ...(ambiguous ? { other: { id: "42", title: "Another entity" } } : {}), + }; + const shape = responseShapeOf(3, recorded, false, { + idField: "id", + id: "42", + }); + expect( + writtenResponse(shape, recorded, { + id: "43", + fields: { id: "43", title: "Authored title" }, + }), + ).toEqual({ + ok: true, + value: ambiguous + ? recorded + : { + request_id: "42", + data: { id: "43", title: "Authored title" }, + }, + }); + }, +); + it.each<[string, JsonValue | undefined, boolean, Shaped]>([ ["no body", undefined, true, ["empty", null]], ["an empty response with a body", { id: "42" }, true, ["empty", null]], diff --git a/src/domain/analysisView/environmentStateWriteExplain.ts b/src/domain/analysisView/environmentStateWriteExplain.ts index 80f519afa2..f10d0dcea6 100644 --- a/src/domain/analysisView/environmentStateWriteExplain.ts +++ b/src/domain/analysisView/environmentStateWriteExplain.ts @@ -4,7 +4,7 @@ import { compareUnicodeCodePoints as compare } from "../unicodeCodePointOrder.js import { ownMember, isObject, - locateItem, + locateItems, pointerTo, same, scalarText, @@ -174,7 +174,7 @@ const requestRule = ( index: BodyIndex, field: string, value: JsonValue, -): FieldRule | undefined => { +): FieldRule | "ambiguous-path" | undefined => { const { body, pathValues } = evidence.call; if (Object.hasOwn(body, field) && same(body[field], value)) return { field, rule: "echo" }; @@ -185,9 +185,11 @@ const requestRule = ( if (from !== undefined) return { field, rule: "copy", from }; } const text = scalarText(value); - for (const [name, pathValue] of pathValues) - if (text !== undefined && text === pathValue) - return { field, rule: "path", from: name }; + const paths = [...pathValues].filter( + ([, value]) => text !== undefined && text === value, + ); + if (paths.length > 1) return "ambiguous-path"; + if (paths.length === 1) return { field, rule: "path", from: paths[0]![0] }; const slugged = typeof value === "string" && value.length >= 3; const source = slugged ? index.slugs.get(value) : undefined; return source === undefined ? undefined : slugRule(field, source); @@ -357,8 +359,14 @@ export const explainCreate = ( const index = indexBody(evidence.call.body); // The member each slug the held entities' `&` spellings refused was slugged from. const refusedSlugs = new Map(); + let ambiguousPath = false; for (const [field, value] of Object.entries(evidence.entity)) { const requested = requestRule(evidence, index, field, value); + if (requested === "ambiguous-path") { + ambiguousPath = true; + unexplained.push(field); + continue; + } const held = heldSlugRule(requested, evidence.call.body, evidence.type); if (requested?.rule === "slug" && held === undefined) refusedSlugs.set(field, requested.from); @@ -468,12 +476,14 @@ export const explainCreate = ( rules, body: blank.length === 0 ? shaped : { ...shaped, blank }, // A template rebuilds only its generated value, so other text it keeps ties it like a constant. - tied: [ - ...[...constants, ...templates].map((field) => - ownMember(evidence.entity, field)!, - ), - ...literals, - ].some((text) => tokens.some((token) => containsDelimited(text, token))), + tied: + ambiguousPath || + [ + ...[...constants, ...templates].map((field) => + ownMember(evidence.entity, field)!, + ), + ...literals, + ].some((text) => tokens.some((token) => containsDelimited(text, token))), }; }; @@ -646,8 +656,11 @@ export const responseShapeOf = ( ): WriteResponse => { if (empty) return { ordinal, shape: "empty", pointer: null }; if (body === undefined) return { ordinal, shape: "recorded", pointer: null }; - const located = locateItem(body, entity.id); - if (located?.keys.includes(entity.idField) === true) + const candidates = locateItems(body, entity.id, entity.idField); + if (candidates.length > 1) + return { ordinal, shape: "recorded", pointer: null }; + const located = candidates[0]; + if (located !== undefined) return { ordinal, shape: "entity", pointer: located.pointer }; if (scalarText(body) === entity.id) return { ordinal, shape: "identity", pointer: "" }; diff --git a/src/domain/analysisView/environmentStateWriteLinks.test.ts b/src/domain/analysisView/environmentStateWriteLinks.test.ts index 8ac06c376f..14759a4cce 100644 --- a/src/domain/analysisView/environmentStateWriteLinks.test.ts +++ b/src/domain/analysisView/environmentStateWriteLinks.test.ts @@ -22,6 +22,7 @@ const record = ( instant: parseOffsetDateTimeInstant(new Date(start).toISOString()), ms: start, }, + end: parseOffsetDateTimeInstant(new Date(end).toISOString()), endMs: end, }, }); @@ -68,7 +69,7 @@ describe("short-id creation limits", () => { const prior = record(0, "GET", "/api/projects", [0, 5]); const unknown = { ...prior, - timing: { ...prior.timing, endMs: undefined }, + timing: { ...prior.timing, end: undefined, endMs: undefined }, }; const post = record(1, "POST", "/api/imports", [10, 11]); const later = record(2, "GET", "/api/projects/7", [20, 21]); diff --git a/src/domain/analysisView/environmentStateWriteLinks.ts b/src/domain/analysisView/environmentStateWriteLinks.ts index 9c751ccb94..6bf01558fc 100644 --- a/src/domain/analysisView/environmentStateWriteLinks.ts +++ b/src/domain/analysisView/environmentStateWriteLinks.ts @@ -23,6 +23,7 @@ import { import { ENTRY_BYTES, textBytes } from "./webNetworkRouteTemplates.js"; import { pathValuesOf, + pathParameterKeys, type FieldRule, type WriteEffect, } from "./environmentStateWriteRules.js"; @@ -414,13 +415,9 @@ export const anyLinkedSource = ( plan.collection !== null && normalizedPath(plan.collection) === plan.target && !segmentsOf(plan.target).some(isParam) && - segmentsOf(template) - .filter(isParam) - .every((segment) => - plan.rules.some( - (rule) => rule.rule === "path" && rule.from === segment.slice(1, -1), - ), - ); + [...pathParameterKeys(template).values()].every((name) => + plan.rules.some((rule) => rule.rule === "path" && rule.from === name), + ); /** * Whether a linked create's call sends the path values its exemplar did, each as recorded or in the diff --git a/src/domain/analysisView/environmentStateWriteRules.ts b/src/domain/analysisView/environmentStateWriteRules.ts index 093bf08b5f..9ca01c1f2f 100644 --- a/src/domain/analysisView/environmentStateWriteRules.ts +++ b/src/domain/analysisView/environmentStateWriteRules.ts @@ -407,6 +407,23 @@ export interface WriteCall { readonly body: JsonObject | undefined; } +/** Parameter lookup keys preserve repeated placeholder occurrences by segment position. */ +export const pathParameterKeys = (template: string): Map => { + const parameters = segmentsOf(template).flatMap((segment, index) => + isParam(segment) ? [[index, segment.slice(1, -1)] as const] : [], + ); + const repeated = + new Set(parameters.map(([, name]) => name)).size !== parameters.length; + // If any name repeats, encode every key so a literal name cannot collide with + // an indexed key. Unique-name templates retain their established lookup keys. + return new Map( + parameters.map(([index, name]) => [ + index, + repeated ? JSON.stringify([index, name]) : name, + ]), + ); +}; + /** The value each template parameter takes in a concrete path, decoded; undefined when the path does not instantiate it. */ export const pathValuesOf = ( template: string, @@ -416,13 +433,14 @@ export const pathValuesOf = ( const concrete = segmentsOf(path); if (pattern.length !== concrete.length) return undefined; const values = new Map(); + const keys = pathParameterKeys(template); for (const [index, segment] of pattern.entries()) { const value = concrete[index]!; if (!isParam(segment)) { if (segment !== value) return undefined; continue; } - values.set(segment.slice(1, -1), decodedSegment(value)); + values.set(keys.get(index)!, decodedSegment(value)); } return values; }; diff --git a/src/domain/analysisView/environmentStateWrites.test.ts b/src/domain/analysisView/environmentStateWrites.test.ts index 646cbdbecf..bb541fc7f8 100644 --- a/src/domain/analysisView/environmentStateWrites.test.ts +++ b/src/domain/analysisView/environmentStateWrites.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; -import type { JsonValue } from "../jsonValue.js"; +import { jsonObjectSchema, type JsonValue } from "../jsonValue.js"; import { webNetworkCaptureSchema } from "../webNetworkCapture.js"; import { EnvironmentStateAccumulator } from "./EnvironmentStateAccumulator.js"; import { @@ -86,6 +86,43 @@ it("retains learned clock rules but withholds their serving when the global cloc ).toEqual(plan); }); +it.each([ + ["000000000", 0, "2026-10-17T18:00:12.000Z"], + ["000001", 0, "2026-10-17T18:00:13.000Z"], + ["000000001", 0, "2026-10-17T18:00:13.000Z"], + ["123000000", 0, "2026-10-17T18:00:13.000Z"], + ["999999", 0.001, "2026-10-17T18:00:13.000Z"], + ["999999", 0.001001, "2026-10-17T18:00:14.000Z"], +])( + "starts the served clock after a final captured fractional instant .%s plus %sms", + (fraction, duration, expected) => { + const evidence = timedCaptureEvidence([ + ...BOOK_ROWS, + ["GET", "/api/health", { ready: true }, 200, undefined, 0], + ]); + const capture = webNetworkCaptureSchema.parse(evidence.normalized_result); + const state = environmentStateViewOf({ + ...evidence, + normalized_result: { + ...capture, + records: capture.records.map((record, index) => + index === capture.records.length - 1 + ? { + ...record, + reported: { + ...jsonObjectSchema.parse(record.reported), + startedDateTime: `2026-10-17T18:00:12.${fraction}Z`, + time: duration, + }, + } + : record, + ), + }, + }); + expect(state.write_clock).toBe(expected); + }, +); + it.each(["delete", "mismatched-query"])( "does not parse unused request content for %s under a bounded state allowance", (kind) => { diff --git a/src/domain/analysisView/environmentStateWrites.ts b/src/domain/analysisView/environmentStateWrites.ts index b45c37e599..469aefccb2 100644 --- a/src/domain/analysisView/environmentStateWrites.ts +++ b/src/domain/analysisView/environmentStateWrites.ts @@ -2,6 +2,10 @@ import type { ExtractedState } from "./environmentStateRules.js"; import { EnvironmentStateWriteSimulation } from "./EnvironmentStateWriteSimulation.js"; import type { WriteSimulationInput } from "./environmentStateWriteEffects.js"; import { usesWriteClock } from "./environmentStateWriteRules.js"; +import { + compareOffsetDateTimeInstants, + type OffsetDateTimeInstant, +} from "../offsetDateTime.js"; /** The latest end a record can have and still round up to a whole second a date can hold. */ const LATEST_END_MS = 8_640_000_000_000_000 - 1000; @@ -13,17 +17,23 @@ const clockStart = ( // The clock is global: any unknown completion may extend beyond every known end. if (records.some((record) => record.timing.endMs === undefined)) return null; // A record timed past the range of a date carries no usable time, so the clock starts from the others. - const end = records.reduce( - (latest, record) => - record.timing.endMs !== undefined && - Math.abs(record.timing.endMs) <= LATEST_END_MS - ? Math.max(latest, record.timing.endMs) - : latest, - Number.NEGATIVE_INFINITY, - ); - return Number.isFinite(end) - ? new Date(Math.ceil(end / 1000) * 1000).toISOString() - : null; + let latest: OffsetDateTimeInstant | undefined; + for (const record of records) { + if ( + record.timing.endMs === undefined || + Math.abs(record.timing.endMs) > LATEST_END_MS + ) + continue; + const end = record.timing.end; + if (end === undefined) return null; + if (latest === undefined || compareOffsetDateTimeInstants(end, latest) > 0) + latest = end; + } + return latest === undefined + ? null + : new Date( + (latest.epochSecond + (latest.fraction === "" ? 0 : 1)) * 1000, + ).toISOString(); }; /** diff --git a/src/domain/analysisView/recordedResponseBody.ts b/src/domain/analysisView/recordedResponseBody.ts index 643c49b9cb..981eb703e9 100644 --- a/src/domain/analysisView/recordedResponseBody.ts +++ b/src/domain/analysisView/recordedResponseBody.ts @@ -44,6 +44,14 @@ export type RecordedResponseDeclaration = ReturnType< typeof readRecordedResponseDeclaration >; +/** HAR's zero status sentinel and nonnumeric declarations do not prove a response status. */ +export const recordedResponseStatus = ( + declaration: RecordedResponseDeclaration, +): number | null => { + const status = declaration?.status; + return typeof status === "number" && status !== 0 ? status : null; +}; + /** Strict, and keeping a leading byte-order mark, so callers see the recorded text unchanged. */ const strictUtf8 = (): TextDecoder => new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); @@ -83,6 +91,28 @@ export const readRecordedResponseBody = ( ): RecordedResponseBody => { const content = declaration?.content; if (content == null) return { bodyText: null }; + const bodyCoordinates = [ + "/response/content/text", + "/response/content/encoding", + ]; + const within = (pointer: string, ancestor: string): boolean => + pointer.startsWith(ancestor) && + (pointer.length === ancestor.length || pointer[ancestor.length] === "/"); + // Explicit value exclusions also govern imported retained facts. Keep their + // source text in complete Evidence, but never promote it into replay/state. + if ( + record.redactions.some( + (redaction) => + !("scope" in redaction) && + bodyCoordinates.some((pointer) => within(pointer, redaction.pointer)), + ) || + record.binary_fields.some( + (field) => + field.state === "redacted" && + bodyCoordinates.some((pointer) => within(pointer, field.pointer)), + ) + ) + return { bodyText: null }; const { text, encoding } = content; if (typeof text !== "string") return { bodyText: null }; if (encoding === undefined) { diff --git a/src/domain/analysisView/webNetworkCaptureView.test.ts b/src/domain/analysisView/webNetworkCaptureView.test.ts index af31680954..f19fd0e09f 100644 --- a/src/domain/analysisView/webNetworkCaptureView.test.ts +++ b/src/domain/analysisView/webNetworkCaptureView.test.ts @@ -3,7 +3,11 @@ import { createHash } from "node:crypto"; import { createEvidence } from "../evidence.js"; import { freezeJsonSnapshot } from "../immutableJson.js"; import { webNetworkCaptureSchema } from "../webNetworkCapture.js"; -import { projectAnalysisView } from "./analysisView.js"; +import { + projectAnalysisView, + analysisViewRequestSchema, +} from "./analysisView.js"; +import { timedCaptureEvidence } from "../../../tests/fixtures/analysisView.js"; const capture = () => { const body = Buffer.from("original body"); @@ -800,18 +804,6 @@ it.each([ "bytes that are not its text", { binaryFields: [retainedBytes("/request/path", "/b")] }, ], - [ - "a byte count that is not its text", - { binaryFields: [{ ...retainedBytes("/request/path", "/a"), bytes: 3 }] }, - ], - [ - "a digest that is not its text", - { - binaryFields: [ - { ...retainedBytes("/request/path", "/a"), sha256: "0".repeat(64) }, - ], - }, - ], ] as const)( "keeps a mitmproxy path with %s out of the preview", (_label, facets) => { @@ -1040,3 +1032,111 @@ it.each(["scheme", "host", "port", "authority"])( expect(JSON.stringify(reported)).not.toContain("password"); }, ); + +it.each(["base64", "bytes", "sha256", "duplicate"] as const)( + "refuses inconsistent retained binary %s at parent admission", + (fault) => { + const { parent, report } = capture(); + const record = report.records[0]!; + const binary = record.binary_fields[0]!; + if (binary.state !== "retained") + throw new Error("Expected retained source bytes"); + const changed = { + ...binary, + ...(fault === "base64" + ? { content_base64: "!!!!" } + : fault === "bytes" + ? { bytes: binary.bytes + 1 } + : { sha256: "0".repeat(64) }), + }; + const invalid = { + ...report, + records: [ + { + ...record, + binary_fields: fault === "duplicate" ? [binary, binary] : [changed], + }, + ], + }; + const projected = projectAnalysisView( + { ...parent, normalizedResult: invalid }, + { kind: "capture", detail: "preview", offset: 0, limit: 1 }, + ); + expect(projected).toMatchObject({ + ok: false, + error: { _tag: "AnalysisInputError" }, + }); + }, +); + +it.each([ + "/response/content/text", + "/response/content", + "/response", + "", + "binary-redaction", + "control", +])( + "respects retained response-body authority without altering raw facts: %s", + (marker) => { + const seed = timedCaptureEvidence([ + ["GET", "/api/items", [{ id: 1, name: "withheld-value" }]], + ]); + const report = webNetworkCaptureSchema.parse(seed.normalized_result); + const record = report.records[0]!; + if (marker === "binary-redaction") + record.binary_fields.push({ + pointer: "/response/content/text", + representation: "har-base64-content", + state: "redacted", + content_base64: null, + bytes: null, + sha256: null, + }); + else if (marker !== "control") + record.redactions.push({ + pointer: marker, + reason: "explicit-sensitive-value", + }); + const evidence = createEvidence( + { + path: report.artifact.path, + sha256: report.artifact.sha256!, + format: "file", + }, + report.decoder, + { + operation: seed.operation, + parameters: seed.parameters, + result: report, + }, + ); + const parent = { + predicateType: evidence.predicate_type, + evidenceId: evidence.evidence_id, + operation: evidence.operation, + normalizedResult: evidence.normalized_result, + subject: evidence.subject, + parameters: evidence.parameters, + limitations: evidence.limitations, + }; + const state = projectAnalysisView( + parent, + analysisViewRequestSchema.parse({ kind: "environment-state" }), + ); + if (!state.ok || state.value.kind !== "environment-state") + throw new Error("Expected retained state projection"); + expect( + JSON.stringify(state.value.entities).includes("withheld-value"), + ).toBe(marker === "control"); + const complete = projectAnalysisView(parent, { + kind: "capture", + detail: "complete", + offset: 0, + limit: 1, + }); + expect( + complete.ok && JSON.stringify(complete.value).includes("withheld-value"), + ).toBe(true); + }, +); diff --git a/src/domain/analysisView/webNetworkRouteTemplates.ts b/src/domain/analysisView/webNetworkRouteTemplates.ts index 0ed154612b..53c55f0e57 100644 --- a/src/domain/analysisView/webNetworkRouteTemplates.ts +++ b/src/domain/analysisView/webNetworkRouteTemplates.ts @@ -1,5 +1,8 @@ import { z } from "zod"; -import { readRecordedResponseDeclaration } from "./recordedResponseBody.js"; +import { + readRecordedResponseDeclaration, + recordedResponseStatus, +} from "./recordedResponseBody.js"; import { parseRecordedTransportUrl, parseUrlQueryParameterNames, @@ -148,7 +151,6 @@ export const readRecordedRequestLine = ( return excluded("method-unavailable"); if (parsed.path === null) return excluded("url-unparseable"); const response = readRecordedResponseDeclaration(reported); - const status = response?.status; const mime = response?.content?.mimeType; const rawMime = typeof mime === "string" ? mime : ""; if (!temporary(textBytes(rawMime.length))) return null; @@ -158,7 +160,7 @@ export const readRecordedRequestLine = ( origin: parsed.origin, path: parsed.path, query: parsed.query, - status: typeof status === "number" && status !== 0 ? status : null, + status: recordedResponseStatus(response), rawMime, }; }; diff --git a/src/domain/offsetDateTime.test.ts b/src/domain/offsetDateTime.test.ts index 41ac64cf5e..7a53785930 100644 --- a/src/domain/offsetDateTime.test.ts +++ b/src/domain/offsetDateTime.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import { z } from "zod"; import { + addOffsetDateTimeMilliseconds, inspectRecordedDateTime, offsetDateTimeSchema, readRecordedDateTime, @@ -22,6 +23,34 @@ it.each([ expect(inspectRecordedDateTime(text).kind).toBe(kind); }); +it.each([ + ["000001", 0, 10, "000001"], + ["999999", 0.001, 11, ""], + ["999999", 0.001001, 11, "000000001"], + ["1234567890123456789", 1000.001, 11, "1234577890123456789"], + ["", 1e-7, 10, "0000000001"], + ["", 999.9999999999999, 10, "9999999999999999"], +])( + "adds duration %s + %sms without losing decimal precision", + (fraction, duration, epochSecond, expectedFraction) => { + expect( + addOffsetDateTimeMilliseconds({ epochSecond: 10, fraction }, duration), + ).toEqual({ epochSecond, fraction: expectedFraction }); + }, +); + +it.each([-1, Number.NaN, Number.POSITIVE_INFINITY, Number.MAX_VALUE])( + "keeps unsupported recorded duration %s unknown", + (duration) => { + expect( + addOffsetDateTimeMilliseconds( + { epochSecond: 10, fraction: "" }, + duration, + ), + ).toBeUndefined(); + }, +); + const ACCEPTED: readonly (readonly [string, string])[] = [ ["a UTC timestamp", "2026-10-09T04:00:00Z"], ["a positive offset", "2026-10-09T04:00:00+02:00"], diff --git a/src/domain/offsetDateTime.ts b/src/domain/offsetDateTime.ts index 6b61a44dba..e896cafd24 100644 --- a/src/domain/offsetDateTime.ts +++ b/src/domain/offsetDateTime.ts @@ -146,12 +146,14 @@ export const offsetDateTimeInstantFromEpochMilliseconds = ( const millisecondsWithinSecond = epochMilliseconds - epochSecond * 1000; return { epochSecond, - fraction: fractionalSecondFromMilliseconds(millisecondsWithinSecond), + fraction: secondsFromMilliseconds(millisecondsWithinSecond).fraction, }; }; -const fractionalSecondFromMilliseconds = (milliseconds: number): string => { - if (milliseconds === 0) return ""; +const secondsFromMilliseconds = ( + milliseconds: number, +): { readonly seconds: number; readonly fraction: string } => { + if (milliseconds === 0) return { seconds: 0, fraction: "" }; const [coefficient = "", exponentText] = String(milliseconds) .toLowerCase() .split("e"); @@ -166,9 +168,43 @@ const fractionalSecondFromMilliseconds = (milliseconds: number): string => { : decimalPosition < digits.length ? `${digits.slice(0, decimalPosition)}.${digits.slice(decimalPosition)}` : `${digits}${"0".repeat(decimalPosition - digits.length)}`; - return ( - fraction.includes(".") ? fraction.slice(fraction.indexOf(".") + 1) : "" - ).replace(/0+$/u, ""); + const separator = fraction.indexOf("."); + return { + seconds: Number(separator < 0 ? fraction : fraction.slice(0, separator)), + fraction: (separator < 0 ? "" : fraction.slice(separator + 1)).replace( + /0+$/u, + "", + ), + }; +}; + +/** Add a recorded nonnegative decimal millisecond duration without rounding away either fraction. */ +export const addOffsetDateTimeMilliseconds = ( + start: OffsetDateTimeInstant, + milliseconds: number, +): OffsetDateTimeInstant | undefined => { + if (!Number.isFinite(milliseconds) || milliseconds < 0) return undefined; + if (milliseconds === 0) return start; + const duration = secondsFromMilliseconds(milliseconds); + const wholeSeconds = start.epochSecond + duration.seconds; + if (!Number.isSafeInteger(wholeSeconds)) return undefined; + const length = Math.max(start.fraction.length, duration.fraction.length); + const digits: string[] = []; + let carry = 0; + for (let index = length - 1; index >= 0; index--) { + const sum = + (start.fraction.charCodeAt(index) || 48) - + 48 + + (duration.fraction.charCodeAt(index) || 48) - + 48 + + carry; + digits.push(String(sum % 10)); + carry = Math.floor(sum / 10); + } + const epochSecond = wholeSeconds + carry; + return Number.isSafeInteger(epochSecond) + ? { epochSecond, fraction: digits.reverse().join("").replace(/0+$/u, "") } + : undefined; }; /** Compare instants exactly while treating absent fraction digits as zeroes. */ diff --git a/src/domain/webNetworkCapture.test.ts b/src/domain/webNetworkCapture.test.ts index 990ca99274..e26a1d6af1 100644 --- a/src/domain/webNetworkCapture.test.ts +++ b/src/domain/webNetworkCapture.test.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import { expect, it } from "vitest"; import { freezeJsonSnapshot, @@ -116,3 +117,107 @@ it("retains schema normalization when an owned source contains a hidden reported Object.hasOwn(parsed.data.container.reported as object, "hidden"), ).toBe(false); }); + +it.each([ + { + name: "malformed base64", + content_base64: "!!!!", + bytes: 0, + sha256: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + }, + { + name: "noncanonical pad bits", + content_base64: "AR==", + bytes: 1, + sha256: createHash("sha256") + .update(Buffer.from([1])) + .digest("hex"), + }, + { + name: "mismatched byte count", + content_base64: "YQ==", + bytes: 2, + sha256: "ca978112ca1bbdcafac231b39a23dc4da786eff8147c4e72b9807785afee48bb", + }, + { + name: "mismatched digest", + content_base64: "YQ==", + bytes: 1, + sha256: "0".repeat(64), + }, +])( + "refuses $name in retained binary facts before projection", + ({ name: _name, ...binary }) => { + const report = capture(); + const record = { + ...report.records[0]!, + binary_fields: [ + { + pointer: "/response/content/text", + representation: "har-base64-content", + state: "retained", + ...binary, + }, + ], + }; + const parsed = webNetworkCaptureSchema.safeParse({ + ...report, + records: [record], + }); + expect(parsed.success).toBe(false); + }, +); + +it.each([ + { name: "empty", bytes: Buffer.alloc(0) }, + { name: "Unicode", bytes: Buffer.from("雪�") }, + { name: "multiple windows", bytes: Buffer.alloc(100_001, 0xff) }, +])( + "retains $name canonical binary identity across bounded digest windows", + ({ bytes }) => { + const report = capture(); + const source = { + ...report, + records: [ + { + ...report.records[0]!, + binary_fields: [ + { + pointer: "/body", + representation: "producer-bytes", + state: "retained", + content_base64: bytes.toString("base64"), + bytes: bytes.length, + sha256: createHash("sha256").update(bytes).digest("hex"), + }, + ], + }, + ], + }; + const parsed = webNetworkCaptureSchema.parse(source); + expect(parsed).toEqual(source); + }, +); + +it("refuses multiple binary declarations for one source coordinate", () => { + const report = capture(); + const binary = (text: string) => ({ + pointer: "/response/content/text", + representation: "har-base64-content", + state: "retained", + content_base64: Buffer.from(text).toString("base64"), + bytes: Buffer.byteLength(text), + sha256: createHash("sha256").update(text).digest("hex"), + }); + expect( + webNetworkCaptureSchema.safeParse({ + ...report, + records: [ + { + ...report.records[0]!, + binary_fields: [binary("first"), binary("second")], + }, + ], + }).success, + ).toBe(false); +}); diff --git a/src/domain/webNetworkCapture.ts b/src/domain/webNetworkCapture.ts index a2239d6a51..e0c3c1da45 100644 --- a/src/domain/webNetworkCapture.ts +++ b/src/domain/webNetworkCapture.ts @@ -1,3 +1,5 @@ +import { createHash } from "node:crypto"; +import { canonicalBase64Schema } from "./stringPatterns.js"; import { z } from "zod"; import { browserDeadlineIncrease, @@ -215,7 +217,35 @@ export const webNetworkCaptureBinaryDescriptorSchema = z.discriminatedUnion( [retainedBinaryDescriptorSchema, redactedBinaryDescriptorSchema], ); const binarySchema = z.discriminatedUnion("state", [ - retainedBinaryDescriptorSchema.extend({ content_base64: z.string() }), + retainedBinaryDescriptorSchema + .extend({ content_base64: canonicalBase64Schema }) + .superRefine((field, context) => { + const encoded = field.content_base64; + const decodedBytes = Buffer.byteLength(encoded, "base64"); + if (field.bytes !== decodedBytes) { + context.addIssue({ + code: "custom", + path: ["bytes"], + message: + "Retained binary byte count differs from its canonical content.", + }); + return; + } + // Quartet-aligned windows authenticate every retained byte without + // allocating a second whole body while the capture remains live. + const hash = createHash("sha256"); + for (let offset = 0; offset < encoded.length; offset += 65536) + hash.update( + Buffer.from(encoded.slice(offset, offset + 65536), "base64"), + ); + if (hash.digest("hex") !== field.sha256) + context.addIssue({ + code: "custom", + path: ["sha256"], + message: + "Retained binary SHA-256 differs from its canonical content.", + }); + }), redactedBinaryDescriptorSchema.extend({ content_base64: z.null() }), ]); const numberSchema = z.strictObject({ @@ -244,7 +274,21 @@ export const webNetworkCaptureRecordSchema = z.strictObject({ }), ]), reported: jsonValueSchema, - binary_fields: z.array(binarySchema), + binary_fields: z.array(binarySchema).superRefine((fields, context) => { + const pointers = new Set(); + for (const [index, field] of fields.entries()) { + if (pointers.has(field.pointer)) { + context.addIssue({ + code: "custom", + path: [index, "pointer"], + message: + "Each retained source coordinate can declare binary facts only once.", + }); + return; + } + pointers.add(field.pointer); + } + }), numeric_literals: z.array(numberSchema), redactions: z.array(redactionSchema), limitations: z.array(z.string()),