diff --git a/src/application/environment/environmentBundle.test.ts b/src/application/environment/environmentBundle.test.ts index d8487217d4..b6344466a1 100644 --- a/src/application/environment/environmentBundle.test.ts +++ b/src/application/environment/environmentBundle.test.ts @@ -22,7 +22,7 @@ import { } from "./environmentBundle.js"; import type { EnvironmentSource } from "./environmentSource.js"; import { prepareServedState } from "./environmentState.js"; -import { environmentToolAnswer } from "./environmentStateAnswer.js"; +import { environmentToolAnswer } from "./environmentToolAnswer.js"; import { EnvironmentStateStore } from "./environmentStateStore.js"; const fixture = environmentServeFixture(); diff --git a/src/application/environment/environmentBundle.ts b/src/application/environment/environmentBundle.ts index c5fce6f373..8b05b27d5e 100644 --- a/src/application/environment/environmentBundle.ts +++ b/src/application/environment/environmentBundle.ts @@ -7,7 +7,7 @@ import { servedCatalog, type PreparedState } from "./environmentState.js"; import { environmentToolAnswer, type EnvironmentToolAnswer, -} from "./environmentStateAnswer.js"; +} from "./environmentToolAnswer.js"; import { EnvironmentStateStore } from "./environmentStateStore.js"; import type { EnvironmentToolDefinition } from "./environmentTools.js"; @@ -332,7 +332,7 @@ export const planEnvironmentBundle = ( // The catalog and answers the bundle's server will have, so the verifier and README predict them. const catalog = servedCatalog( source, - state !== undefined, + state, name, ENVIRONMENT_EXPORT_OPERATION, ); diff --git a/src/application/environment/environmentState.ts b/src/application/environment/environmentState.ts index f2be336101..9211969fb8 100644 --- a/src/application/environment/environmentState.ts +++ b/src/application/environment/environmentState.ts @@ -6,6 +6,7 @@ import { type StateType, type StateView, } from "../../domain/analysisView/environmentStateRules.js"; +import type { StateWritePlan } from "../../domain/analysisView/environmentStateWriteRules.js"; import type { AnalysisError } from "../../domain/analysisErrorBase.js"; import type { AnalysisInputIssue, @@ -22,6 +23,7 @@ import { type EnvironmentManifestTool, type EnvironmentSource, } from "./environmentSource.js"; +import { describeWrite } from "./environmentStateWriteDescription.js"; import { environmentToolDefinitions, type EnvironmentDataPlane, @@ -41,11 +43,22 @@ export interface EnvironmentState { readonly views: ReadonlyMap; readonly entities: readonly StateEntity[]; readonly documents: readonly StateDocument[]; + /** Write plans keyed by tool name. */ + readonly writes: ReadonlyMap; + /** The instant the served clock starts from, or null when the capture has no times. */ + readonly writeClock: string | null; } -/** Serving from the state is read-only: tools whose requests may change state are not served. */ -const servedFromState = (tool: EnvironmentManifestTool): boolean => - !changesState(tool.method, tool.path_template); +/** The write plans a catalog serves from: none before a state is derived. */ +type ServedWrites = Pick; + +/** Serving from the state serves read tools and the write tools whose effect the capture confirmed. */ +const servedFromState = ( + tool: EnvironmentManifestTool, + state: ServedWrites, +): boolean => + !changesState(tool.method, tool.path_template) || + state.writes.get(tool.name)?.served === true; /** The tools an environment serves and the data plane its card declares. */ export interface ServedCatalog { @@ -53,26 +66,35 @@ export interface ServedCatalog { readonly dataPlane: EnvironmentDataPlane; } -/** The catalog an environment serves: every manifest tool for replay, the read tools and the data plane from a state. */ +/** The catalog an environment serves: every manifest tool for replay; from a state, the read tools, the confirmed write tools and the data plane. */ export const servedCatalog = ( source: EnvironmentSource, - fromState: boolean, + state: ServedWrites | undefined, name: string, operation: string = ENVIRONMENT_SERVE_OPERATION, ): Result => { - const dataPlane: EnvironmentDataPlane = fromState ? "state" : "none"; + const dataPlane: EnvironmentDataPlane = + state === undefined ? "none" : "state"; const tools = environmentToolDefinitions( - fromState - ? { + state === undefined + ? source + : { ...source, manifest: { ...source.manifest, - tools: source.manifest.tools.filter(servedFromState), + tools: source.manifest.tools.filter((tool) => + servedFromState(tool, state), + ), }, - } - : source, + }, name, - { operation, dataPlane }, + { + operation, + dataPlane, + ...(state === undefined + ? {} + : { describe: (tool) => describeWrite(state.writes.get(tool.name)) }), + }, ); return tools.ok ? ok({ tools: tools.value, dataPlane }) : tools; }; @@ -162,6 +184,8 @@ export const loadEnvironmentState = ( views: new Map(view.views.map((entry) => [entry.tool, entry])), entities: view.entities, documents: view.documents, + writes: new Map(view.writes.map((plan) => [plan.tool, plan])), + writeClock: view.write_clock, }); }; @@ -232,19 +256,18 @@ export const prepareServedState = ( } = {}, ): PreparedState => { const operation = options.operation ?? ENVIRONMENT_SERVE_OPERATION; - // The state catalog depends only on the manifest and name, so checking it first spares deriving a state that cannot be served. - const catalog = servedCatalog( - source, - true, - options.name ?? source.manifest.name, - operation, - ); - if (!catalog.ok) return unavailable(catalog.error); + const name = options.name ?? source.manifest.name; + // The read catalog depends only on the manifest and name, so checking it first spares deriving a state that cannot be served. + const reads = servedCatalog(source, { writes: new Map() }, name, operation); + if (!reads.ok) return unavailable(reads.error); const derived = deriveEnvironmentState(source, captureValue, operation); - return derived.ok + if (!derived.ok) return unavailable(derived.error); + // Served write tools join the catalog, so the whole catalog is admitted again. + const catalog = servedCatalog(source, derived.value, name, operation); + return catalog.ok ? { state: derived.value, report: { served: true, evidence_id: derived.value.evidenceId }, } - : unavailable(derived.error); + : unavailable(catalog.error); }; diff --git a/src/application/environment/environmentStateAnswer.ts b/src/application/environment/environmentStateAnswer.ts index dbd6b45535..2ffc0b93e4 100644 --- a/src/application/environment/environmentStateAnswer.ts +++ b/src/application/environment/environmentStateAnswer.ts @@ -2,6 +2,7 @@ import { stateBodyJson } from "../../domain/analysisView/EnvironmentStateAccumul import { collectionPathOf, listItems, + pagesOnly, renderStateEntity, renderStateList, replaceAt, @@ -14,6 +15,7 @@ import { type StateListView, type StateView, } from "../../domain/analysisView/environmentStateRules.js"; +import { isAbsence } from "../../domain/analysisView/environmentStateWriteChecks.js"; import type { AnalysisError } from "../../domain/analysisErrorBase.js"; import { AnalysisInputError, @@ -64,8 +66,16 @@ export type StateProvenance = } | { readonly source: "state"; - readonly view: "document"; + readonly view: "document" | "document-write"; readonly path: string; + } + | { + readonly source: "state"; + /** A write applied to the state, or the recorded answer for an entity it does not hold. */ + readonly view: "create" | "update" | "delete" | "missing"; + readonly type: string; + readonly collection: string; + readonly id: string; }; /** A replay answer that also says whether the state or a recording produced it. */ @@ -79,7 +89,7 @@ export interface StateServing { readonly store: EnvironmentStateStore; } -const refused = ( +export const refused = ( tool: EnvironmentManifestTool, userMessage: string, ): AnalysisError => @@ -122,11 +132,12 @@ const fromRecording = ( }; /** A recorded body the state renders into, usable exactly when extraction could use it. */ -const recordedJson = (exchange: RecordedExchange): JsonValue | undefined => - stateBodyJson(exchange.contentType, exchange); +export const recordedJson = ( + exchange: RecordedExchange, +): JsonValue | undefined => stateBodyJson(exchange.contentType, exchange); /** The rendered body on the base record's status, content type and provenance; never cut. */ -const stateAnswer = ( +export const stateAnswer = ( serving: StateServing, base: { readonly exchange: RecordedExchange; readonly exact: boolean }, body: JsonValue, @@ -157,22 +168,22 @@ const stateAnswer = ( }; /** One collection segment an item call names: a template literal, or a supplied path value. */ -interface CallSegment { +export interface CallSegment { readonly text: string; readonly supplied: boolean; } /** An item call's collection segments and identity value, kept as supplied rather than joined and re-split. */ -interface ItemCall { +export interface ItemCall { readonly segments: readonly CallSegment[]; readonly id: string; } -/** What an item call names, or undefined when a path argument is missing. */ -const itemCall = ( +/** A call's path segments: template literals and supplied values, or undefined when a path argument is missing. */ +export const callSegments = ( tool: EnvironmentManifestTool, args: ReplayArguments, -): ItemCall | undefined => { +): CallSegment[] | undefined => { const names = environmentToolArguments(tool).path; const params = new Map(tool.params.map((param) => [param.index, param])); const segments: CallSegment[] = []; @@ -189,12 +200,23 @@ const itemCall = ( if (typeof value !== "string") return undefined; segments.push({ text: value, supplied: true }); } - const last = segments.pop(); - return last?.supplied === true ? { segments, id: last.text } : undefined; + return segments; +}; + +/** What an item call names, or undefined when a path argument is missing. */ +export const itemCall = ( + tool: EnvironmentManifestTool, + args: ReplayArguments, +): ItemCall | undefined => { + const segments = callSegments(tool, args); + const last = segments?.pop(); + return segments !== undefined && last?.supplied === true + ? { segments, id: last.text } + : undefined; }; /** How many supplied segments a held collection matches verbatim, when it matches the call under replay's path rule. */ -const collectionMatch = ( +export const collectionMatch = ( collection: string, segments: readonly CallSegment[], ): number | undefined => { @@ -211,7 +233,7 @@ const collectionMatch = ( }; /** The entity an item call names, preferring verbatim collection spellings as replay does; the id is tried percent-decoded, then as written. */ -const namedEntity = ( +export const namedEntity = ( store: EnvironmentStateStore, type: string, call: ItemCall, @@ -232,7 +254,51 @@ const namedEntity = ( return undefined; }; -/** An item call renders the named entity into the recorded envelope; it never invents an entity. */ +/** The answer for an entity the state does not hold: the tool's latest recorded 400, 404 or 410, else a tool error. */ +export const missingAnswer = ( + serving: StateServing, + tool: EnvironmentManifestTool, + missing: { + readonly type: string; + readonly collection: string; + readonly id: string; + }, +): Result => { + const latestAbsence = (ordinals: readonly number[]) => + ordinals + .map((ordinal) => serving.source.exchanges.get(ordinal)) + .filter( + (exchange): exchange is RecordedExchange => + exchange !== undefined && isAbsence(exchange.status), + ) + .at(-1); + // The tool's own recorded absence first, else that of an item view of the type. + const itemTools = new Set( + [...serving.store.state.views.values()].flatMap((view) => + view.shape === "item" && view.type === missing.type ? [view.tool] : [], + ), + ); + const recorded = + latestAbsence(tool.record_ordinals) ?? + latestAbsence( + serving.source.manifest.tools + .filter((candidate) => itemTools.has(candidate.name)) + .flatMap((candidate) => candidate.record_ordinals), + ); + if (recorded === undefined) + return err( + refused( + tool, + `The state holds no ${missing.type} entity ${missing.id} in ${missing.collection}.`, + ), + ); + return ok({ + ...replayAnswer(serving.source, recorded, false), + state: { source: "state", view: "missing", ...missing }, + }); +}; + +/** An item call renders the named entity into the recorded envelope; it never invents an entity; one the state does not hold answers the tool's recorded absence. */ const answerItem = ( serving: StateServing, tool: EnvironmentManifestTool, @@ -245,12 +311,11 @@ const answerItem = ( return err(refused(tool, `Tool ${tool.name} needs every path argument.`)); const entity = namedEntity(store, view.type, call); if (entity === undefined) - return err( - refused( - tool, - `The state holds no ${view.type} entity ${call.id} in /${call.segments.map(({ text }) => text).join("/")}.`, - ), - ); + return missingAnswer(serving, tool, { + type: view.type, + collection: `/${call.segments.map(({ text }) => text).join("/")}`, + id: call.id, + }); const shown = shownEntity( tool, view, @@ -297,18 +362,37 @@ const checkedLine = ( return selectReplayExchange(tool, lines, args)?.exchange; }; +/** For a list of a whole collection, a call that supplies no query argument answers from a checked line that only pages it; a requested page answers from its recording. */ +const unfilteredLine = ( + source: EnvironmentSource, + tool: EnvironmentManifestTool, + checked: readonly number[], + args: ReplayArguments, +): RecordedExchange | undefined => { + const query = environmentToolArguments(tool).query; + if ([...query.values()].some((argument) => args[argument] !== undefined)) + return undefined; + return checked + .map((ordinal) => source.exchanges.get(ordinal)) + .find( + (exchange): exchange is RecordedExchange => + exchange !== undefined && pagesOnly(exchange.query ?? undefined), + ); +}; + type StateDocumentView = Extract; -/** One tool call: the tool and the arguments it was called with. */ +/** One tool call: the tool, the arguments it was called with, and whether they name the answering line exactly. */ interface ToolCall { readonly tool: EnvironmentManifestTool; readonly args: ReplayArguments; + readonly exact: boolean; } /** A list call renders the members of the checked line it names from the state. */ const answerList = ( serving: StateServing, - { tool, args }: ToolCall, + { tool, args, exact }: ToolCall, view: StateListView, base: RecordedExchange, ): Result => { @@ -340,7 +424,7 @@ const answerList = ( } return stateAnswer( serving, - { exchange: base, exact: true }, + { exchange: base, exact }, renderStateList(view, envelope, items), { source: "state", @@ -376,16 +460,27 @@ const answerCheckedLine = ( view: StateListView | StateDocumentView, args: ReplayArguments, ): Result => { - const base = checkedLine( + const named = checkedLine( serving.source, tool, view.checked_record_ordinals, args, ); + const base = + named ?? + (view.shape === "list" && view.membership === "all" + ? unfilteredLine(serving.source, tool, view.checked_record_ordinals, args) + : undefined); if (base === undefined) return fromRecording(serving.source, tool, args, "unrecorded-request"); + // An unfiltered call answers on a line it did not name, so its replay match is not exact. return view.shape === "list" - ? answerList(serving, { tool, args }, view, base) + ? answerList( + serving, + { tool, args, exact: named !== undefined }, + view, + base, + ) : answerDocument(serving, tool, base); }; @@ -408,18 +503,3 @@ export const answerFromState = ( return answerCheckedLine(serving, tool, view, args); } }; - -/** How a served environment answers one tool call. */ -export type EnvironmentToolAnswer = ( - tool: EnvironmentManifestTool, - args: ReplayArguments, -) => Result; - -/** The answer a served environment gives: replay without a state store, the live state with one. */ -export const environmentToolAnswer = ( - source: EnvironmentSource, - store: EnvironmentStateStore | undefined, -): EnvironmentToolAnswer => - store === undefined - ? (tool, args) => replayEnvironmentTool(source, tool, args) - : (tool, args) => answerFromState({ source, store }, tool, args); diff --git a/src/application/environment/environmentStateStore.ts b/src/application/environment/environmentStateStore.ts index 916b182bca..9e284abf66 100644 --- a/src/application/environment/environmentStateStore.ts +++ b/src/application/environment/environmentStateStore.ts @@ -3,6 +3,7 @@ import type { StateType, } from "../../domain/analysisView/environmentStateRules.js"; import type { AnalysisInputError } from "../../domain/analysisErrorCore.js"; +import type { JsonValue } from "../../domain/jsonValue.js"; import { ok, type Result } from "../../domain/result.js"; import { compositeKey } from "../../domain/unicodeCodePointOrder.js"; import type { EnvironmentState } from "./environmentState.js"; @@ -35,27 +36,117 @@ export class EnvironmentStateStore { #documents: ReadonlyMap = new Map(); #content: StateContent = { entities: [], documents: [] }; readonly #types: ReadonlyMap; + readonly #clockStart: number; + /** Served writes since the state was last reset or loaded: the clock's seconds and the generators' counter. */ + #writes = 0; + /** The largest value each sequence has generated since the last reset or load, so a deleted entity's value is not reused. */ + readonly #sequences = new Map(); constructor(readonly state: EnvironmentState) { this.#types = new Map( state.types.map((type) => [type.collection_template, type]), ); + this.#clockStart = + state.writeClock === null ? 0 : Date.parse(state.writeClock); this.reset(); } - /** Restore the initial state the Evidence holds. */ + /** Restore the initial state the Evidence holds; the clock and counter start again. */ reset(): void { this.#install(this.state); + this.#restart(); } - /** Validate a `rea-state/v1` document and, only when it is valid, make it the whole state. */ + /** Validate a `rea-state/v1` document and, only when it is valid, make it the whole state; the clock and counter start again. */ replace(document: unknown): Result { const content = parseStateDocument(this.state, document); if (!content.ok) return content; this.#install(content.value); + this.#restart(); return ok(undefined); } + /** Count one served write: its number since the last reset or load, and its clock instant one second per write after the state's start. */ + nextWrite(): { readonly count: number; readonly clockMs: number } { + this.#writes += 1; + return { + count: this.#writes, + clockMs: this.#clockStart + this.#writes * 1000, + }; + } + + /** The next value of a sequence: one past both the largest held value and the largest it generated. */ + nextSequence(key: string, held: number): number { + const value = Math.max(held, (this.#sequences.get(key) ?? 0) + 1); + this.#sequences.set(key, value); + return value; + } + + #restart(): void { + this.#writes = 0; + this.#sequences.clear(); + } + + /** Every held entity of a type, in state order. */ + ofType(type: string): readonly StateEntity[] { + return this.#content.entities.filter((entity) => entity.type === type); + } + + /** Add an entity at `index` of its collection's order. */ + insert(entity: StateEntity, index: number): void { + const entities = this.#content.entities; + const members = this.inCollection(entity.type, entity.collection); + const anchor = + index < members.length + ? entities.indexOf(members[index]!) + : members.length > 0 + ? entities.indexOf(members.at(-1)!) + 1 + : entities.length; + this.#install({ + ...this.#content, + entities: [ + ...entities.slice(0, anchor), + entity, + ...entities.slice(anchor), + ], + }); + } + + /** Replace a held entity's values in place. */ + update(entity: StateEntity): void { + const key = compositeKey([entity.type, entity.collection, entity.id]); + this.#install({ + ...this.#content, + entities: this.#content.entities.map((held) => + compositeKey([held.type, held.collection, held.id]) === key + ? entity + : held, + ), + }); + } + + remove(entity: StateEntity): void { + const key = compositeKey([entity.type, entity.collection, entity.id]); + this.#install({ + ...this.#content, + entities: this.#content.entities.filter( + (held) => compositeKey([held.type, held.collection, held.id]) !== key, + ), + }); + } + + /** Set a held document's value; a value that is not the Evidence's cites no record. */ + setDocument(tool: string, value: JsonValue): void { + this.#install({ + ...this.#content, + documents: this.#content.documents.map((document) => + document.tool === tool + ? { tool: document.tool, path: document.path, value } + : document, + ), + }); + } + type(collectionTemplate: string): StateType | undefined { return this.#types.get(collectionTemplate); } diff --git a/src/application/environment/environmentStateWriteAnswer.ts b/src/application/environment/environmentStateWriteAnswer.ts new file mode 100644 index 0000000000..40bb87a712 --- /dev/null +++ b/src/application/environment/environmentStateWriteAnswer.ts @@ -0,0 +1,445 @@ +import { + createdFields, + updatedEntity, + updatedFields, + valueAt, + writtenDocument, + writtenResponse, + type GenerateValue, +} from "../../domain/analysisView/environmentStateWriteApply.js"; +import { insertIndex } from "../../domain/analysisView/environmentStateWriteExplain.js"; +import type { + GeneratedRule, + StateWritePlan, + WriteCall, +} from "../../domain/analysisView/environmentStateWriteRules.js"; +import { + formatClock, + integerOf, + isUuid, + seededToken, + seededUuid, + spelledLike, +} from "../../domain/analysisView/environmentStateWriteValues.js"; +import { + isObject, + scalarText, + type JsonObject, + type StateEntity, +} from "../../domain/analysisView/environmentStateRules.js"; +import type { AnalysisError } from "../../domain/analysisErrorBase.js"; +import type { JsonValue } from "../../domain/jsonValue.js"; +import { err, ok, type Result } from "../../domain/result.js"; +import { + replayAnswer, + selectReplayExchange, + type ReplayArguments, +} from "./environmentReplay.js"; +import type { EnvironmentManifestTool } from "./environmentSource.js"; +import { + callSegments, + collectionMatch, + itemCall, + missingAnswer, + namedEntity, + recordedJson, + refused, + stateAnswer, + type StateAnswer, + type StateProvenance, + type StateServing, +} from "./environmentStateAnswer.js"; +import { environmentToolArguments } from "./environmentTools.js"; + +/** Attempts at a generated UUID or token before a collision with held values is refused. */ +const GENERATION_ATTEMPTS = 64; + +/** One served write call: the tool, its plan and its arguments. */ +interface WriteRequest { + readonly tool: EnvironmentManifestTool; + readonly plan: StateWritePlan; + readonly args: ReplayArguments; +} + +/** The path values a call supplies, keyed by template segment as write plans name them. */ +const pathValues = ( + tool: EnvironmentManifestTool, + args: ReplayArguments, +): Map => { + const names = environmentToolArguments(tool).path; + const template = tool.path_template + .slice(tool.path_template.indexOf("/") + 1) + .split("/"); + const values = new Map(); + for (const param of tool.params) { + const value = args[names.get(param.name) ?? param.name]; + const segment = template[param.index] ?? ""; + if (typeof value === "string") values.set(segment.slice(1, -1), value); + } + return values; +}; + +/** The call's body: an object, nothing, or a refusal for any other value. */ +const callBody = ({ + tool, + args, +}: WriteRequest): Result => { + const name = environmentToolArguments(tool).body; + const value = name === undefined ? undefined : args[name]; + if (value === undefined) return ok(undefined); + return isObject(value as JsonValue) + ? ok(value as JsonObject) + : err(refused(tool, `Tool ${tool.name} takes a JSON object body.`)); +}; + +/** The entity a recorded observation shows: the values a create keeps and an update's constants. */ +const observedEntity = ( + serving: StateServing, + plan: StateWritePlan, +): JsonObject | undefined => { + if (plan.observation === null) return undefined; + const exchange = serving.source.exchanges.get(plan.observation.ordinal); + const body = exchange === undefined ? undefined : recordedJson(exchange); + const object = + body === undefined ? undefined : valueAt(body, plan.observation.pointer); + return isObject(object) ? object : undefined; +}; + +/** The largest integer the entities hold in a field, plus one. */ +const nextInteger = (entities: readonly StateEntity[], field: string): number => + entities.reduce( + (largest, entity) => + Math.max(largest, integerOf(entity.fields[field]) ?? 0), + 0, + ) + 1; + +/** + * Values one served write generates: its clock instant, sequences one past the largest held + * value, and UUIDs and tokens seeded by the state Evidence, tool, field and write count, + * skipping values the type already holds. + */ +const generator = ( + serving: StateServing, + request: WriteRequest, + target: { + readonly name: string; + readonly collection: readonly StateEntity[]; + readonly observed: JsonObject; + }, +): GenerateValue => { + const { store } = serving; + const write = store.nextWrite(); + const ofType = store.ofType(request.plan.target ?? ""); + const unused = (field: string, make: (attempt: number) => string): string => { + const held = new Set( + ofType.map((entity) => scalarText(entity.fields[field])), + ); + for (let attempt = 0; attempt < GENERATION_ATTEMPTS; attempt += 1) { + const value = make(attempt); + if (!held.has(value)) return value; + } + return make(GENERATION_ATTEMPTS); + }; + return (rule: GeneratedRule): JsonValue => { + const exemplar = target.observed[rule.field] ?? ""; + const seed = (attempt: number) => + `${store.state.evidenceId}:${request.tool.name}:${rule.field}:${String(write.count)}:${String(attempt)}`; + switch (rule.rule) { + case "clock": + return formatClock(write.clockMs, rule.format); + case "sequence": + return spelledLike( + exemplar, + store.nextSequence( + `${request.tool.name} ${rule.field} ${rule.scope === "collection" ? target.name : ""}`, + nextInteger( + rule.scope === "collection" ? target.collection : ofType, + rule.field, + ), + ), + ); + case "unique": + return unused(rule.field, (attempt) => + seededUuid(seed(attempt), String(exemplar)), + ); + case "identity": + if (rule.generator === "sequence") + return spelledLike( + exemplar, + store.nextSequence( + `${request.plan.target ?? ""} ${rule.field} ${target.name}`, + nextInteger(target.collection, rule.field), + ), + ); + return unused(rule.field, (attempt) => + rule.generator === "uuid" || isUuid(exemplar) + ? seededUuid(seed(attempt), String(exemplar)) + : seededToken(seed(attempt), String(exemplar)), + ); + } + }; +}; + +/** A write's answer on its recorded response's status and content type: a rendered body, none, or the recorded one. */ +const writeAnswer = ( + serving: StateServing, + request: WriteRequest, + rendered: { readonly fields: JsonObject | undefined; readonly id: string }, + state: StateProvenance, +): Result => { + const { plan, tool, args } = request; + const exchange = + plan.response === null + ? undefined + : serving.source.exchanges.get(plan.response.ordinal); + if (plan.response === null || exchange === undefined) + return err(refused(tool, `Tool ${tool.name} has no recorded response.`)); + const exact = + selectReplayExchange(tool, new Map([[exchange.ordinal, exchange]]), args) + ?.exact === true; + if (plan.response.shape === "recorded") + return ok({ ...replayAnswer(serving.source, exchange, exact), state }); + const body = writtenResponse(plan.response, recordedJson(exchange), rendered); + if (!body.ok) return err(refused(tool, body.error)); + return stateAnswer(serving, { exchange, exact }, body.value ?? null, state); +}; + +/** The collection a create writes to: a held collection the call's path matches, else the path as supplied. */ +const createdCollection = ( + serving: StateServing, + request: WriteRequest, +): string | undefined => { + const segments = callSegments(request.tool, request.args); + if (segments === undefined) return undefined; + const held = serving.store + .collectionsOf(request.plan.target ?? "") + .find((collection) => collectionMatch(collection, segments) !== undefined); + return ( + held ?? + `/${segments.map(({ text, supplied }) => (supplied ? encodeURIComponent(text) : text)).join("/")}` + ); +}; + +const answerCreate = ( + serving: StateServing, + request: WriteRequest, + call: WriteCall, +): Result => { + const { tool, plan } = request; + const type = plan.target ?? ""; + const idField = serving.store.type(type)?.id_field ?? ""; + const collection = createdCollection(serving, request); + const observed = observedEntity(serving, plan); + if (collection === undefined || observed === undefined) + return err(refused(tool, `Tool ${tool.name} needs every path argument.`)); + if (plan.collection !== null && plan.collection !== collection) + return err( + refused( + tool, + `The recorded create's values are tied to ${plan.collection}; the environment models creates only there.`, + ), + ); + const members = serving.store.inCollection(type, collection); + const fields = createdFields( + plan, + observed, + call, + generator(serving, request, { + name: collection, + collection: members, + observed, + }), + ); + if (!fields.ok) return err(refused(tool, fields.error)); + const id = scalarText(fields.value[idField]); + if ( + id === undefined || + serving.store.entity(type, collection, id) !== undefined + ) + return err( + refused( + tool, + `The state already holds a ${type} entity ${id ?? ""} in ${collection}.`, + ), + ); + const state = { + source: "state", + view: "create", + type, + collection, + id, + } as const; + const answer = writeAnswer( + serving, + request, + { fields: fields.value, id }, + state, + ); + if (!answer.ok) return answer; + serving.store.insert( + { + type, + collection, + id, + fields: fields.value, + view_fields: [], + record_ordinals: [], + }, + insertIndex( + plan.insert ?? { rule: "last" }, + members.map((entity) => entity.fields), + fields.value, + ), + ); + return answer; +}; + +/** The held entity an update or delete names, or the answer for one the state does not hold. */ +const targetEntity = ( + serving: StateServing, + request: WriteRequest, +): Result> => { + const { tool, plan, args } = request; + const call = itemCall(tool, args); + if (call === undefined) + return err( + err(refused(tool, `Tool ${tool.name} needs every path argument.`)), + ); + const entity = namedEntity(serving.store, plan.target ?? "", call); + return entity === undefined + ? err( + missingAnswer(serving, tool, { + type: plan.target ?? "", + collection: `/${call.segments.map(({ text }) => text).join("/")}`, + id: call.id, + }), + ) + : ok(entity); +}; + +const answerUpdate = ( + serving: StateServing, + request: WriteRequest, + call: WriteCall, +): Result => { + const { tool, plan } = request; + const target = targetEntity(serving, request); + if (!target.ok) return target.error; + const entity = target.value; + const observed = observedEntity(serving, plan) ?? {}; + const written = updatedFields( + plan, + { before: entity.fields, observed }, + { ...call, id: entity.id }, + generator(serving, request, { + name: entity.collection, + collection: serving.store.inCollection(entity.type, entity.collection), + observed, + }), + ); + if (!written.ok) return err(refused(tool, written.error)); + const answer = writeAnswer( + serving, + request, + { fields: written.value, id: entity.id }, + { + source: "state", + view: "update", + type: entity.type, + collection: entity.collection, + id: entity.id, + }, + ); + if (answer.ok) serving.store.update(updatedEntity(entity, written.value)); + return answer; +}; + +const answerDelete = ( + serving: StateServing, + request: WriteRequest, +): Result => { + const target = targetEntity(serving, request); + if (!target.ok) return target.error; + const entity = target.value; + const answer = writeAnswer( + serving, + request, + { fields: entity.fields, id: entity.id }, + { + source: "state", + view: "delete", + type: entity.type, + collection: entity.collection, + id: entity.id, + }, + ); + if (answer.ok) serving.store.remove(entity); + return answer; +}; + +const answerDocument = ( + serving: StateServing, + request: WriteRequest, + call: WriteCall, +): Result => { + const { tool, plan } = request; + const held = serving.store.document(plan.target ?? ""); + if (held === undefined) + return err( + refused(tool, `The state holds no document for ${plan.target ?? ""}.`), + ); + const written = writtenDocument(plan, held.value, call); + if (!written.ok) return err(refused(tool, written.error)); + const state = { + source: "state", + view: "document-write", + path: held.path, + } as const; + const answer = + plan.response?.shape === "entity" + ? (() => { + const exchange = serving.source.exchanges.get(plan.response.ordinal); + return exchange === undefined + ? err(refused(tool, `Tool ${tool.name} has no recorded response.`)) + : stateAnswer( + serving, + { exchange, exact: true }, + written.value, + state, + ); + })() + : writeAnswer(serving, request, { fields: undefined, id: "" }, state); + if (answer.ok) serving.store.setDocument(plan.target ?? "", written.value); + return answer; +}; + +/** Apply one served write tool call to the live state and answer it; a refused call leaves the state unchanged. */ +export const answerWrite = ( + serving: StateServing, + request: WriteRequest, +): Result => { + const body = callBody(request); + if (!body.ok) return body; + const call: WriteCall = { + path: request.tool.path_template, + pathValues: pathValues(request.tool, request.args), + body: body.value, + }; + switch (request.plan.effect) { + case "create": + return answerCreate(serving, request, call); + case "update": + return answerUpdate(serving, request, call); + case "delete": + return answerDelete(serving, request); + case "document": + return answerDocument(serving, request, call); + case "none": + return err( + refused( + request.tool, + `Tool ${request.tool.name} is not served from the state.`, + ), + ); + } +}; diff --git a/src/application/environment/environmentStateWriteDescription.ts b/src/application/environment/environmentStateWriteDescription.ts new file mode 100644 index 0000000000..2f98711c35 --- /dev/null +++ b/src/application/environment/environmentStateWriteDescription.ts @@ -0,0 +1,50 @@ +import type { + StateWritePlan, + WriteBody, +} from "../../domain/analysisView/environmentStateWriteRules.js"; + +/** Members named in a description; the plan in the state Evidence holds every one. */ +const DESCRIBED_MEMBERS = 12; + +const members = (names: readonly string[]): string => + names.length <= DESCRIBED_MEMBERS + ? names.join(", ") + : `${names.slice(0, DESCRIBED_MEMBERS).join(", ")}, … (${String(names.length)} members; all in the state Evidence)`; + +/** The body members a served write accepts, as a description clause. */ +const bodyClause = ( + body: WriteBody, + effect: StateWritePlan["effect"], +): string => { + const parts: string[] = []; + if (body.accepted.length > 0) + parts.push(`body members ${members(body.accepted)}`); + if (body.fixed.length > 0) + parts.push( + `${members(body.fixed.map(({ member }) => member))} only with the value the capture sent`, + ); + if (body.required.length > 0) + parts.push(`required: ${members(body.required)}`); + else if (effect === "update") parts.push("send at least one"); + return parts.length === 0 ? "" : ` (${parts.join("; ")})`; +}; + +/** What a served write tool does to the state, for its description; undefined for a tool the state does not serve. */ +export const describeWrite = ( + plan: StateWritePlan | undefined, +): string | undefined => { + if (plan?.served !== true) return undefined; + const body = plan.body === null ? "" : bodyClause(plan.body, plan.effect); + switch (plan.effect) { + case "create": + return `creates an entity of ${plan.target ?? ""} in the served environment state${body}, modelled on the recorded calls`; + case "update": + return `changes the entity the path names in the served environment state${body}, modelled on the recorded calls`; + case "delete": + return "removes the entity the path names from the served environment state, modelled on the recorded calls"; + case "document": + return `changes a document of the served environment state${body}, modelled on the recorded calls`; + case "none": + return undefined; + } +}; diff --git a/src/application/environment/environmentToolAnswer.ts b/src/application/environment/environmentToolAnswer.ts new file mode 100644 index 0000000000..5b2e4bf010 --- /dev/null +++ b/src/application/environment/environmentToolAnswer.ts @@ -0,0 +1,34 @@ +import type { AnalysisError } from "../../domain/analysisErrorBase.js"; +import type { Result } from "../../domain/result.js"; +import { + replayEnvironmentTool, + type ReplayArguments, + type ReplayResult, +} from "./environmentReplay.js"; +import type { + EnvironmentManifestTool, + EnvironmentSource, +} from "./environmentSource.js"; +import { answerFromState } from "./environmentStateAnswer.js"; +import type { EnvironmentStateStore } from "./environmentStateStore.js"; +import { answerWrite } from "./environmentStateWriteAnswer.js"; + +/** How a served environment answers one tool call. */ +export type EnvironmentToolAnswer = ( + tool: EnvironmentManifestTool, + args: ReplayArguments, +) => Result; + +/** The answer a served environment gives: replay without a state store; with one, a served write applies to the live state and a read answers from it. */ +export const environmentToolAnswer = ( + source: EnvironmentSource, + store: EnvironmentStateStore | undefined, +): EnvironmentToolAnswer => + store === undefined + ? (tool, args) => replayEnvironmentTool(source, tool, args) + : (tool, args) => { + const plan = store.state.writes.get(tool.name); + return plan?.served === true + ? answerWrite({ source, store }, { tool, plan, args }) + : answerFromState({ source, store }, tool, args); + }; diff --git a/src/application/environment/environmentTools.ts b/src/application/environment/environmentTools.ts index 0194a533dc..1cf9d7956b 100644 --- a/src/application/environment/environmentTools.ts +++ b/src/application/environment/environmentTools.ts @@ -189,6 +189,15 @@ const describeResponses = ( return `${returns} ${describeJson(found.value)}; ${recorded}`; }; +/** One tool's description: method, path, response summary, what it does, the records it cites and the Evidence. */ +const toolDescription = ( + source: EnvironmentSource, + tool: EnvironmentManifestTool, + describe: EnvironmentCatalogOptions["describe"], + answers: string, +): string => + `${REPLAY_CATALOG_INSTRUCTIONS} ${describeText(tool.method, "method")} ${describeText(tool.path_template, "path")}; ${describeResponses(tool, source.exchanges)}; ${describe?.(tool) ?? answers} [${describeList(tool.record_ordinals, tool.record_ordinals.length)}]; evidence ${source.evidenceId}`; + /** Captured argument names longer than this are advertised under a short alias; the manifest keeps the name. */ export const MAX_ARGUMENT_NAME_CHARS = 64; @@ -314,6 +323,8 @@ export type EnvironmentDataPlane = "none" | "state"; export interface EnvironmentCatalogOptions { readonly operation?: string; readonly dataPlane?: EnvironmentDataPlane; + /** What a served tool does, in place of how it answers; undefined keeps the answering sentence. */ + readonly describe?: (tool: EnvironmentManifestTool) => string | undefined; } /** Admit complete discovery before binding; six bytes per UTF-16 unit covers JSON escaping and UTF-8 expansion. */ @@ -323,6 +334,7 @@ export const environmentToolDefinitions = ( { operation = ENVIRONMENT_SERVE_OPERATION, dataPlane = "none", + describe, }: EnvironmentCatalogOptions = {}, ): Result< readonly EnvironmentToolDefinition[], @@ -378,7 +390,7 @@ export const environmentToolDefinitions = ( // A lower bound rejects enormous argument lists before constructing their alias map. if (!reserve(count * EMPTY_PROPERTY_BYTES)) return refused(); // The response summary is bounded by its samples and charged with the whole description. - const description = `${REPLAY_CATALOG_INSTRUCTIONS} ${describeText(tool.method, "method")} ${describeText(tool.path_template, "path")}; ${describeResponses(tool, source.exchanges)}; ${answers} [${describeList(tool.record_ordinals, tool.record_ordinals.length)}]; evidence ${source.evidenceId}`; + const description = toolDescription(source, tool, describe, answers); if (!text(description)) return refused(); const args = environmentToolArguments(tool); const properties = new Map(); diff --git a/src/domain/analysisView/EnvironmentStateAccumulator.ts b/src/domain/analysisView/EnvironmentStateAccumulator.ts index d1c3711fd0..5dc2844ee2 100644 --- a/src/domain/analysisView/EnvironmentStateAccumulator.ts +++ b/src/domain/analysisView/EnvironmentStateAccumulator.ts @@ -45,6 +45,8 @@ type StateFields = Pick< | "entities" | "documents" | "views" + | "writes" + | "write_clock" | "unknowns" | "coverage" | "limitations" @@ -93,6 +95,17 @@ const redactedWriteSchema = z.object({ const contentSchema = z.object({ response: z.object({ content: z.object({ text: z.string() }) }), }); +const requestSchema = z.object({ + request: z.object({ + postData: z.object({ mimeType: z.string(), text: z.string() }), + }), +}); +const responseSizeSchema = z.object({ + response: z.object({ + status: z.number(), + content: z.object({ text: z.string().optional() }).optional(), + }), +}); interface RecordEntry { readonly facts: StateRecordFacts; @@ -117,7 +130,7 @@ const UNKNOWNS: StateView["unknowns"] = [ { kind: "state-transition", statement: - "The state is one snapshot from before the first related recorded write; it states no effects of writes.", + "The state is one snapshot from before the first related recorded write; write effects are modelled only from the calls the capture recorded, and served only where its later reads confirmed them.", }, ]; @@ -243,6 +256,43 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator return value === undefined ? { kind: "unusable" } : { kind: "json", value }; } + /** Parse one recorded request body as state JSON, as response bodies are admitted. */ + #requestBody(ordinal: number): ParsedBody | null { + const entry = this.#entries.get(ordinal); + const request = requestSchema.safeParse(entry?.record.reported); + if ( + entry === undefined || + !request.success || + !isJsonLike(mediaTypeEssence(request.data.request.postData.mimeType)) + ) + return { kind: "unusable" }; + const { mimeType, text } = request.data.request.postData; + if (Buffer.byteLength(text, "utf8") > STATE_BODY_LIMIT_BYTES) + return { kind: "unusable" }; + if (!this.temporary(RECORD_SCRATCH_BYTES + textBytes(text.length) * 4)) + return null; + try { + const value = stateBodyJson(mimeType, { bodyText: text }); + return value === undefined + ? { kind: "unusable" } + : { kind: "json", value }; + } finally { + this.releaseTemporary(); + } + } + + /** Whether a record's response carried no body: a 204, or empty content. */ + #emptyResponse(ordinal: number): boolean { + const parsed = responseSizeSchema.safeParse( + this.#entries.get(ordinal)?.record.reported, + ); + return ( + parsed.success && + (parsed.data.response.status === 204 || + (parsed.data.response.content?.text ?? "") === "") + ); + } + /** Project the init state of the manifest's primary origin; records keep timeline order. */ finishState(): Result { const manifest = this.finish(); @@ -276,6 +326,8 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator timeline, budget: { body: (ordinal) => this.#body(ordinal), + requestBody: (ordinal) => this.#requestBody(ordinal), + emptyResponse: (ordinal) => this.#emptyResponse(ordinal), retain: (bytes) => this.retain(bytes), release: () => this.releaseTemporary(), }, diff --git a/src/domain/analysisView/EnvironmentStateExtraction.ts b/src/domain/analysisView/EnvironmentStateExtraction.ts index 4e8daab8b8..a8aa548986 100644 --- a/src/domain/analysisView/EnvironmentStateExtraction.ts +++ b/src/domain/analysisView/EnvironmentStateExtraction.ts @@ -51,6 +51,7 @@ import { type StatePlan, } from "./environmentStateGate.js"; import { StateWriteCutoffs } from "./environmentStateWriteCutoffs.js"; +import { extractWrites } from "./environmentStateWrites.js"; class StateBudgetExhausted extends Error {} @@ -71,6 +72,7 @@ const requestLine = (record: StateRecordFacts): string => /** Derive the init state and render plans from manifest tools and timeline-ordered records. */ export class EnvironmentStateExtraction { readonly #tools: readonly StateToolFacts[]; + readonly #records: readonly StateRecordFacts[]; readonly #timeline: StateTimeline; readonly #budget: StateBudget; readonly #position: ReadonlyMap; @@ -101,6 +103,7 @@ export class EnvironmentStateExtraction { readonly budget: StateBudget; }) { this.#tools = input.tools; + this.#records = input.records; this.#timeline = input.timeline; this.#budget = input.budget; this.#position = new Map( @@ -143,13 +146,38 @@ export class EnvironmentStateExtraction { plan.tool.name, plan.item ? checkItemPlan(gate, plan) : checkListPlan(gate, plan), ); - return this.#result(this.#documents(candidates)); + return this.#withWrites(this.#result(this.#documents(candidates))); } catch (cause: unknown) { if (cause instanceof StateBudgetExhausted) return null; throw cause; } } + /** The write plans the capture's own timeline confirms against the extracted state. */ + #withWrites( + state: Omit, + ): ExtractedState { + return { + ...state, + ...extractWrites(state, { + tools: this.#tools, + records: this.#records, + timed: this.#timeline === "recorded-start-time", + volatileOf: (type) => this.#volatileOf(type), + io: { + withBody: (ordinal, use) => this.#withBody(ordinal, use), + requestBody: (ordinal) => { + const parsed = this.#budget.requestBody(ordinal); + if (parsed === null) throw new StateBudgetExhausted(); + return parsed.kind === "json" ? parsed.value : undefined; + }, + emptyResponse: (ordinal) => this.#budget.emptyResponse(ordinal), + retain: (bytes) => this.#retain(bytes), + }, + }), + }; + } + #retain(bytes: number): void { if (!this.#budget.retain(bytes)) throw new StateBudgetExhausted(); } @@ -553,7 +581,9 @@ export class EnvironmentStateExtraction { } /** Types and entities that at least one state-backed view renders; view values only for those views. */ - #result(documents: StateDocument[]): ExtractedState { + #result( + documents: StateDocument[], + ): Omit { const backed = new Set(); const backedTypes = new Set(); for (const view of this.#views.values()) diff --git a/src/domain/analysisView/EnvironmentStateWriteSimulation.ts b/src/domain/analysisView/EnvironmentStateWriteSimulation.ts new file mode 100644 index 0000000000..f488f2047e --- /dev/null +++ b/src/domain/analysisView/EnvironmentStateWriteSimulation.ts @@ -0,0 +1,508 @@ +import { + at, + changesState, + collectionPathOf, + isObject, + isStateWrite, + lastSegmentValue, + normalizedPath, + type StateDocument, + type StateEntity, + type StateRecordFacts, + type StateToolFacts, + type StateType, + type StateView, +} from "./environmentStateRules.js"; +import { updatedEntity } from "./environmentStateWriteApply.js"; +import { + checkDocumentRead, + checkItemRead, + checkListRead, + entityTarget, + isAbsence, + writtenFieldChecks, + SimulatedState, + type CheckOutcome, +} from "./environmentStateWriteChecks.js"; +import { applyCreate } from "./environmentStateWriteCreate.js"; +import { + applyDelete, + applyDocument, + applyUpdate, + type EffectContext, + type EffectResult, + type WriteDraft, + type WriteIo, + type WriteStep, +} from "./environmentStateWriteEffects.js"; +import { insertIndex } from "./environmentStateWriteExplain.js"; +import { + classifyWrite, + prefixRelated, + unservedPlan, + type StateWritePlan, +} from "./environmentStateWriteRules.js"; +import { WriteTaint } from "./environmentStateWriteTaint.js"; + +/** What the write gate reads: the manifest tools, the timeline, the read views and the initial state. */ +export interface WriteSimulationInput { + readonly tools: readonly StateToolFacts[]; + readonly records: readonly StateRecordFacts[]; + readonly timed: boolean; + readonly views: ReadonlyMap; + readonly types: readonly StateType[]; + readonly entities: readonly StateEntity[]; + readonly documents: readonly StateDocument[]; + readonly volatileOf: (type: string) => ReadonlySet; + readonly io: WriteIo; +} + +const requestLine = (record: StateRecordFacts): string => + record.path + (record.query ?? ""); + +const APPLY = { + create: applyCreate, + update: applyUpdate, + delete: applyDelete, + document: applyDocument, +} as const; + +/** + * Replay the capture's timeline on a copy of the initial state: modelled writes apply their + * effects, other writes taint related types, and each read the served state would answer is + * compared with its recording and attributed to the writes it exercises. + */ +export class EnvironmentStateWriteSimulation { + readonly #input: WriteSimulationInput; + readonly #state: SimulatedState; + readonly #drafts = new Map(); + readonly #toolOf = new Map(); + readonly #types = new Map(); + readonly #checkedLines = new Map>(); + readonly #observations = new Set(); + readonly #taint: WriteTaint; + /** Modelled writes per target since its last check, with the fields each wrote (null: all). */ + readonly #touched = new Map>(); + readonly #lastCheck = new Map(); + /** The earliest modelled write to each target since its last check. */ + readonly #touchedSince = new Map(); + /** Timeline indexes of writes the gate does not model. */ + readonly #unmodelledAt: number[] = []; + readonly #log: { + readonly index: number; + readonly tool: string; + readonly path: string; + }[] = []; + /** Entities a create inserted, by target, with the create's tool. */ + readonly #inserted = new Map(); + + constructor(input: WriteSimulationInput) { + this.#input = input; + this.#state = new SimulatedState(input.entities, input.documents); + for (const type of input.types) + this.#types.set(type.collection_template, type); + for (const tool of input.tools) + for (const ordinal of tool.record_ordinals) + this.#toolOf.set(ordinal, tool); + const byOrdinal = new Map( + input.records.map((record) => [record.ordinal, record]), + ); + for (const view of input.views.values()) + if (view.shape !== "recorded" && view.shape !== "item") + this.#checkedLines.set( + view.tool, + new Set( + view.checked_record_ordinals.flatMap((ordinal) => { + const record = byOrdinal.get(ordinal); + return record === undefined ? [] : [requestLine(record)]; + }), + ), + ); + this.#taint = new WriteTaint( + [...this.#types.keys()], + new Map( + input.documents.map((document) => [document.tool, document.path]), + ), + (type, collection, id) => + this.#state.entity(type, collection, id) !== undefined, + ); + } + + /** Every state-changing tool's plan, in manifest order, with its verdict. */ + run(): StateWritePlan[] { + const documents = new Map(); + for (const tool of this.#input.tools) + if (this.#input.views.get(tool.name)?.shape === "document") + documents.set(normalizedPath(tool.path_template), tool.name); + const types = new Set(this.#types.keys()); + for (const tool of this.#input.tools) { + if (!changesState(tool.method, tool.path_template)) continue; + const { effect, target } = classifyWrite(tool, types, documents); + this.#input.io.retain(512); + this.#drafts.set(tool.name, { + plan: { + ...unservedPlan(tool.name, effect, target, "no-state-target"), + reason: effect === "none" ? "no-state-target" : null, + }, + observed: undefined, + candidates: [], + confirmed: [], + contradictedBy: undefined, + sideEffect: false, + calls: 0, + }); + } + this.#input.records.forEach((record, index) => this.#step(record, index)); + return [...this.#drafts.values()].map((draft) => this.#verdict(draft)); + } + + #step(record: StateRecordFacts, index: number): void { + const tool = this.#toolOf.get(record.ordinal); + if (isStateWrite(record)) this.#write(record, index, tool); + else if (tool?.method === "GET" && !this.#observations.has(record.ordinal)) + this.#read(record, index, tool); + } + + #context(): EffectContext { + return { + state: this.#state, + io: this.#input.io, + records: this.#input.records, + timed: this.#input.timed, + viewOf: (ordinal) => { + const tool = this.#toolOf.get(ordinal); + return tool === undefined + ? undefined + : this.#input.views.get(tool.name); + }, + idField: (type) => this.#types.get(type)!.id_field, + observed: (ordinal) => this.#observations.add(ordinal), + unknown: (type, collection, id) => + this.#taint.unknown(type, collection, id), + }; + } + + #write( + record: StateRecordFacts, + index: number, + tool: StateToolFacts | undefined, + ): void { + const draft = tool === undefined ? undefined : this.#drafts.get(tool.name); + const effect = draft?.plan.effect; + if ( + draft === undefined || + tool === undefined || + effect === undefined || + effect === "none" + ) { + this.#unmodelled(record.path, index); + return; + } + const observeOnly = draft.plan.reason !== null; + if (!observeOnly) draft.calls += 1; + const step: WriteStep = { record, index, tool, observeOnly }; + this.#settle(APPLY[effect](this.#context(), draft, step), { + index, + tool: tool.name, + path: record.path, + }); + } + + /** Attribute an applied effect to its tool; a recorded outcome is the state's new baseline; an unknown one taints. */ + #settle( + result: EffectResult, + write: { + readonly index: number; + readonly tool: string; + readonly path: string; + }, + ): void { + if (result.kind === "unknown") { + this.#unmodelled(write.path, write.index); + return; + } + if (result.entity !== undefined) + this.#taint.known( + result.entity.type, + result.entity.collection, + result.entity.id, + ); + for (const target of result.targets) { + // A recorded outcome replaces the entity's values, not its collection's membership. + if (result.kind === "observed") { + if (target === result.targets[0]) this.#touched.delete(target); + continue; + } + if (!this.#touchedSince.has(target)) + this.#touchedSince.set(target, write.index); + const tools = this.#touched.get(target) ?? new Map(); + const before = tools.get(write.tool); + tools.set( + write.tool, + result.fields === undefined || before === null + ? null + : [...new Set([...(before ?? []), ...result.fields])], + ); + this.#touched.set(target, tools); + } + if (result.kind !== "applied") return; + this.#log.push(write); + if (result.inserted !== undefined) + this.#inserted.set(result.inserted, write.tool); + } + + /** + * An item read of an entity an unmodelled write may have changed shows it again: the writes that + * changed it are checked on their own fields, and the state takes what the read recorded. + */ + #resync( + view: Extract, + record: StateRecordFacts, + entity: StateEntity, + ): void { + const target = entityTarget(entity.type, entity.collection, entity.id); + const writes = this.#touched.get(target) ?? new Map(); + this.#taint.known(entity.type, entity.collection, entity.id); + this.#touched.delete(target); + if (isAbsence(record.status)) { + this.#state.remove(entity); + return; + } + const object = this.#input.io.withBody(record.ordinal, (body) => + body === undefined ? undefined : at(body, view.pointer), + ); + if (!isObject(object)) return; + const outcomes = writtenFieldChecks(entity, view, object, { + volatile: this.#input.volatileOf(view.type), + writes, + }); + for (const [name, pass] of outcomes) { + const draft = this.#drafts.get(name)!; + if (pass) draft.confirmed.push(record.ordinal); + else draft.contradictedBy ??= record.ordinal; + } + this.#state.put(updatedEntity(entity, { ...entity.fields, ...object })); + } + + /** Whether an item read is of an entity left unknown that it shows again rather than checks. */ + #resynced(view: StateView, record: StateRecordFacts): boolean { + if (view.shape !== "item") return false; + const collection = collectionPathOf(record, true); + const id = lastSegmentValue(record.path); + if (!this.#taint.unknown(view.type, collection, id)) return false; + const entity = this.#state.entity(view.type, collection, id); + // A recorded absence of an entity the state no longer holds is still a check of its delete. + if (entity === undefined) { + if (isAbsence(record.status)) return false; + this.#taint.known(view.type, collection, id); + return true; + } + this.#resync(view, record, entity); + return true; + } + + #check(view: StateView, record: StateRecordFacts): CheckOutcome | undefined { + if (view.shape === "recorded") return undefined; + if ( + view.shape !== "item" && + !this.#checkedLines.get(view.tool)?.has(requestLine(record)) + ) + return undefined; + if ( + (view.shape === "item" || view.shape === "list") && + this.#taint.under(collectionPathOf(record, view.shape === "item")) + ) + return undefined; + if (view.shape === "list") { + const collection = collectionPathOf(record, false); + const ids = this.#state + .members(view.type, collection) + .map((entity) => entity.id); + if (this.#taint.blocksList(view.type, collection, ids)) return undefined; + } + return this.#input.io.withBody(record.ordinal, (body) => { + if (view.shape === "document") + return checkDocumentRead(this.#state, view.tool, record, body); + const check = { + body, + idField: this.#types.get(view.type)?.id_field ?? "", + volatile: this.#input.volatileOf(view.type), + }; + return view.shape === "item" + ? checkItemRead(this.#state, view, record, check) + : checkListRead(this.#state, view, record, check); + }); + } + + #read(record: StateRecordFacts, index: number, tool: StateToolFacts): void { + const view = this.#input.views.get(tool.name); + if (view === undefined || this.#resynced(view, record)) return; + const outcome = this.#check(view, record); + if (outcome === undefined) return; + const keys = [...outcome.targets, ...outcome.unheld]; + if (keys.some((key) => this.#taint.tainted(key))) return; + const exercised = new Set(); + for (const key of keys) + for (const name of this.#touched.get(key)?.keys() ?? []) + exercised.add(name); + const pass = + outcome.pass || + (view.shape === "list" && this.#retryInsert(view, record, exercised)); + if (pass) + for (const name of exercised) + this.#drafts.get(name)!.confirmed.push(record.ordinal); + else if (this.#interfered(keys)) { + // An unmodelled write since these writes may explain the difference: check their own members only. + if (view.shape === "item") this.#checkMembers(view, record); + else for (const key of keys) this.#taint.fail(key); + } else this.#mismatch({ record, index, keys, exercised, view }); + if (!pass && view.shape === "item") this.#adopt(view, record); + for (const key of keys) { + this.#touched.delete(key); + this.#touchedSince.delete(key); + this.#lastCheck.set(key, index); + } + } + + #unmodelled(path: string, index: number): void { + this.#taint.write(path); + this.#unmodelledAt.push(index); + } + + /** Whether a write the gate does not model came after the earliest modelled write a failed check exercised. */ + #interfered(keys: readonly string[]): boolean { + const since = Math.min( + ...keys.map((key) => this.#touchedSince.get(key) ?? Infinity), + ); + return this.#unmodelledAt.some((index) => index > since); + } + + /** Check the writes to an item read's entity on the members they sent, as a resynchronising read does. */ + #checkMembers( + view: Extract, + record: StateRecordFacts, + ): void { + const entity = this.#state.entity( + view.type, + collectionPathOf(record, true), + lastSegmentValue(record.path), + ); + if (entity === undefined) return; + const writes = + this.#touched.get( + entityTarget(entity.type, entity.collection, entity.id), + ) ?? new Map(); + const object = this.#input.io.withBody(record.ordinal, (body) => + body === undefined || body === null ? undefined : at(body, view.pointer), + ); + if (!isObject(object)) return; + for (const [name, pass] of writtenFieldChecks(entity, view, object, { + volatile: this.#input.volatileOf(view.type), + writes, + })) { + const draft = this.#drafts.get(name)!; + if (pass) draft.confirmed.push(record.ordinal); + else draft.contradictedBy ??= record.ordinal; + } + } + + /** After a failed item check the state takes what the read recorded, so later checks start from it. */ + #adopt( + view: Extract, + record: StateRecordFacts, + ): void { + const collection = collectionPathOf(record, true); + const id = lastSegmentValue(record.path); + const entity = this.#state.entity(view.type, collection, id); + if (entity === undefined) { + this.#taint.fail(entityTarget(view.type, collection, id)); + return; + } + const object = this.#input.io.withBody(record.ordinal, (body) => + body === undefined || body === null ? undefined : at(body, view.pointer), + ); + if (isObject(object)) + this.#state.put(updatedEntity(entity, { ...entity.fields, ...object })); + else this.#state.remove(entity); + } + + /** Contradict the writes a failed check exercised, or, when none, the related writes since its targets were last checked; a failed list stops being checked. */ + #mismatch(failed: { + readonly record: StateRecordFacts; + readonly index: number; + readonly keys: readonly string[]; + readonly exercised: ReadonlySet; + readonly view: StateView; + }): void { + if (failed.exercised.size > 0) + for (const name of failed.exercised) { + const draft = this.#drafts.get(name)!; + draft.contradictedBy ??= failed.record.ordinal; + } + else { + const since = Math.min( + ...failed.keys.map((key) => this.#lastCheck.get(key) ?? -1), + ); + for (const write of this.#log) + if ( + write.index > since && + prefixRelated(write.path, failed.record.path) + ) + this.#drafts.get(write.tool)!.sideEffect = true; + } + if (failed.view.shape !== "item") + for (const key of failed.keys) this.#taint.fail(key); + } + + /** A list that fails only for where one create placed its entities passes under another of that create's insert candidates. */ + #retryInsert( + view: Extract, + record: StateRecordFacts, + exercised: ReadonlySet, + ): boolean { + if (view.membership !== "all") return false; + const collection = collectionPathOf(record, false); + const members = this.#state.members(view.type, collection); + const key = (entity: StateEntity) => + entityTarget(entity.type, entity.collection, entity.id); + const inserted = members.filter((entity) => + this.#inserted.has(key(entity)), + ); + const creates = new Set( + inserted.map((entity) => this.#inserted.get(key(entity))!), + ); + if (creates.size !== 1) return false; + const [name] = creates; + const draft = this.#drafts.get(name!)!; + if (!exercised.has(name!) || draft.confirmed.length > 0) return false; + for (const candidate of draft.candidates.slice(1)) { + for (const entity of inserted) this.#state.remove(entity); + for (const entity of inserted) { + const order = this.#state + .members(view.type, collection) + .map((member) => member.fields); + this.#state.put(entity, insertIndex(candidate, order, entity.fields)); + } + if (this.#check(view, record)?.pass === true) { + draft.plan.insert = candidate; + return true; + } + } + return false; + } + + #verdict(draft: WriteDraft): StateWritePlan { + const plan = draft.plan; + if (plan.reason === null) { + if (draft.calls === 0) plan.reason = "no-call"; + else if (draft.contradictedBy !== undefined) { + plan.reason = "contradicted"; + plan.contradicted_by = draft.contradictedBy; + } else if (draft.sideEffect) plan.reason = "side-effect"; + else if (draft.confirmed.length === 0) plan.reason = "unconfirmed"; + } + plan.served = plan.reason === null; + plan.checked_record_ordinals = [...new Set(draft.confirmed)]; + return { ...plan }; + } +} diff --git a/src/domain/analysisView/environmentStateGate.ts b/src/domain/analysisView/environmentStateGate.ts index 4df98d3506..c1ba4a7bb3 100644 --- a/src/domain/analysisView/environmentStateGate.ts +++ b/src/domain/analysisView/environmentStateGate.ts @@ -7,6 +7,7 @@ import { isObject, lastSegmentValue, listItems, + pagesOnly, renderStateEntity, renderStateList, replaceAt, @@ -173,8 +174,9 @@ const listFacts = ( body === undefined ? undefined : listItems(body, plan.pointer); if (body === undefined || items === undefined) return; items.forEach((item) => union.add(item)); - // Once a list cannot be the whole collection, its membership is not compared again. - if (all && record.query === undefined) { + // Once a list cannot be the whole collection, its membership is not compared again; a line + // that only pages its list can still hold all of it. + if (all && pagesOnly(record.query)) { const ids = items.map((item) => scalarText(item[field]) ?? ""); const expected = context .inCollection(plan.type, collectionPathOf(record, false)) diff --git a/src/domain/analysisView/environmentStateRules.ts b/src/domain/analysisView/environmentStateRules.ts index 0a2e9de412..cb18769f8e 100644 --- a/src/domain/analysisView/environmentStateRules.ts +++ b/src/domain/analysisView/environmentStateRules.ts @@ -1,5 +1,7 @@ import { canonicalJson } from "../comparisonSemantics.js"; +import { parseUrlQueryParameterNames } from "../recordedTransportUrl.js"; import type { JsonValue } from "../jsonValue.js"; +import type { StateWritePlan } from "./environmentStateWriteRules.js"; import { ENTRY_BYTES, textBytes } from "./webNetworkRouteTemplates.js"; /** How records are ordered: by recorded start time, or by record order when any start time is missing. */ @@ -33,6 +35,10 @@ export type ParsedBody = /** Budgeted access the accumulator lends extraction; null and false mean the budget is exhausted. */ export interface StateBudget { readonly body: (ordinal: number) => ParsedBody | null; + /** A record's request body, admitted as response bodies are. */ + readonly requestBody: (ordinal: number) => ParsedBody | null; + /** Whether a record's response had no body. */ + readonly emptyResponse: (ordinal: number) => boolean; readonly retain: (bytes: number) => boolean; /** Drop the temporary work of the body just read. */ readonly release: () => void; @@ -117,6 +123,9 @@ export interface ExtractedState { readonly entities: StateEntity[]; readonly documents: StateDocument[]; readonly views: StateView[]; + readonly writes: StateWritePlan[]; + /** The instant a served clock starts from: the end of the capture's last timed record, or null without times. */ + readonly write_clock: string | null; } const WRITE_METHODS = new Set(["POST", "PUT", "PATCH", "DELETE"]); @@ -135,6 +144,28 @@ const QUERY_SEGMENTS = new Set([ "_count", "_msearch", ]); +/** Query parameters that page or bound a list without filtering it. */ +const PAGING_PARAMETERS = new Set([ + "limit", + "offset", + "page", + "per_page", + "perpage", + "page_size", + "pagesize", + "size", + "start", + "_start", + "_end", +]); + +/** Whether a query only pages its list: no parameters, or paging parameters alone. */ +export const pagesOnly = (query: string | undefined): boolean => + query === undefined || + [...parseUrlQueryParameterNames(query)].every((name) => + PAGING_PARAMETERS.has(name.toLowerCase()), + ); + /** Identity field names tried, in order, when no path parameter proves one. */ export const IDENTITY_NAMES = [ "id", diff --git a/src/domain/analysisView/environmentStateView.ts b/src/domain/analysisView/environmentStateView.ts index 3a63ba7700..3d982ba3f9 100644 --- a/src/domain/analysisView/environmentStateView.ts +++ b/src/domain/analysisView/environmentStateView.ts @@ -86,6 +86,120 @@ const viewSchema = z.discriminatedUnion("shape", [ }), ]); +const ordinalSchema = z.number().int().nonnegative(); +const clockFormatSchema = z.discriminatedUnion("kind", [ + z.strictObject({ + kind: z.literal("iso"), + separator: z.enum(["T", " "]), + fraction: z.number().int().nonnegative(), + zone: z.string(), + }), + z.strictObject({ kind: z.literal("epoch"), unit: z.enum(["s", "ms"]) }), +]); +const ruled = { field: z.string() }; +const fieldRuleSchema = z.discriminatedUnion("rule", [ + z.strictObject({ + ...ruled, + rule: z.literal("identity"), + generator: z.enum(["sequence", "uuid", "token"]), + }), + z.strictObject({ + ...ruled, + rule: z.literal("sequence"), + scope: z.enum(["collection", "type"]), + }), + z.strictObject({ ...ruled, rule: z.literal("unique") }), + z.strictObject({ + ...ruled, + rule: z.literal("clock"), + format: clockFormatSchema, + }), + z.strictObject({ ...ruled, rule: z.literal("echo") }), + z.strictObject({ ...ruled, rule: z.literal("copy"), from: z.string() }), + z.strictObject({ ...ruled, rule: z.literal("path"), from: z.string() }), + z.strictObject({ ...ruled, rule: z.literal("slug"), from: z.string() }), + z.strictObject({ ...ruled, rule: z.literal("template"), from: z.string() }), + z.strictObject({ ...ruled, rule: z.literal("increment") }), + z.strictObject({ ...ruled, rule: z.literal("constant") }), +]); +const insertRuleSchema = z.union([ + z.strictObject({ rule: z.enum(["first", "last"]) }), + z.strictObject({ + rule: z.literal("sorted"), + field: z.string(), + direction: z.enum(["ascending", "descending"]), + }), +]); +const writePlanSchema = z.strictObject({ + tool: z.string(), + effect: z.enum(["create", "update", "delete", "document", "none"]), + target: z + .string() + .nullable() + .describe( + "The entity type an entity effect writes, or the document view a document write changes.", + ), + served: z.boolean(), + reason: z + .enum([ + "no-state-target", + "no-call", + "body-unusable", + "no-observation", + "unexplained-change", + "contradicted", + "side-effect", + "unconfirmed", + ]) + .nullable(), + exemplar_ordinal: ordinalSchema + .nullable() + .describe("The first successful recorded call the plan was built from."), + observation: z + .strictObject({ ordinal: ordinalSchema, pointer: z.string() }) + .nullable() + .describe( + "The record whose response showed the written entity or document, and a JSON pointer to it.", + ), + response: z + .strictObject({ + ordinal: ordinalSchema, + shape: z.enum(["entity", "identity", "empty", "recorded"]), + pointer: z.string().nullable(), + }) + .nullable(), + body: z + .strictObject({ + accepted: z.array(z.string()), + fixed: z.array( + z.strictObject({ member: z.string(), sent: jsonValueSchema }), + ), + required: z.array(z.string()), + identity: z.string().nullable(), + }) + .nullable() + .describe( + "Body members the application stored as sent (accepted), members accepted only with their recorded value (fixed), and members every call sends (required).", + ), + rules: z + .array(fieldRuleSchema) + .describe( + "How written fields get their values; unlisted fields of a created entity keep the observed value.", + ), + collection: z + .string() + .nullable() + .describe( + "For a create whose recorded values are tied to its collection, the only collection it writes to.", + ), + insert: insertRuleSchema.nullable(), + merge: z.enum(["merge", "replace"]).nullable(), + checked_record_ordinals: z + .array(ordinalSchema) + .describe("Later reads that reproduced from the state after this write."), + contradicted_by: ordinalSchema.nullable(), +}); + export const environmentStateFields = { origin: z .string() @@ -145,6 +259,17 @@ export const environmentStateFields = { }), ), views: z.array(viewSchema), + writes: z + .array(writePlanSchema) + .describe( + "One plan per tool whose method may change state: its modelled effect, and whether the capture's later reads confirmed it (served).", + ), + write_clock: z + .string() + .nullable() + .describe( + "The instant a served clock starts from: the end of the capture's last timed record.", + ), unknowns: z.array( z.strictObject({ kind: z.enum([ diff --git a/src/domain/analysisView/environmentStateWriteApply.ts b/src/domain/analysisView/environmentStateWriteApply.ts new file mode 100644 index 0000000000..241fc77fcc --- /dev/null +++ b/src/domain/analysisView/environmentStateWriteApply.ts @@ -0,0 +1,317 @@ +import type { JsonValue } from "../jsonValue.js"; +import { err, ok, type Result } from "../result.js"; +import { + at, + isObject, + pointerTokens, + replaceAt, + same, + scalarText, + type JsonObject, + type StateEntity, +} from "./environmentStateRules.js"; +import type { + FieldRule, + GeneratedRule, + StateWritePlan, + WriteBody, + WriteCall, + WriteResponse, +} from "./environmentStateWriteRules.js"; +import { + integerOf, + replaceDelimited, + slugOf, + spelledLike, +} from "./environmentStateWriteValues.js"; + +/** The value a generated rule takes: recorded values in the gate, the store's generators when served. */ +export type GenerateValue = (rule: GeneratedRule) => JsonValue; + +/** A refusal message naming what the call did wrong; the state is left unchanged. */ +type Applied = Result; + +const kindOf = (value: JsonValue | undefined): string => + value === null ? "null" : Array.isArray(value) ? "array" : typeof value; + +const listed = (members: readonly string[]): string => + members.length === 0 ? "none" : members.join(", "); + +/** What a body may hold, for refusal messages. */ +const bodyContract = (body: WriteBody): string => + `accepted members: ${listed(body.accepted)}; members accepted only with their recorded value: ${listed(body.fixed.map(({ member }) => member))}; required: ${listed(body.required)}`; + +/** Why a call's body does not fit the members the capture showed, if it does not; `shapes` gives each accepted member's JSON type. */ +export const bodyProblem = ( + body: WriteBody, + call: WriteCall, + shapes: { readonly entity: JsonObject; readonly id?: string }, +): string | undefined => { + const sent = call.body ?? {}; + const fixed = new Map(body.fixed.map((entry) => [entry.member, entry.sent])); + for (const [member, value] of Object.entries(sent)) { + if (member === body.identity) { + if (scalarText(value) !== shapes.id) + return `Member ${member} must repeat the target's identity ${shapes.id ?? ""}.`; + continue; + } + if (fixed.has(member)) { + if (!same(fixed.get(member), value)) + return `Member ${member} is accepted only with the value the capture sent, because the application stored it differently (${bodyContract(body)}).`; + continue; + } + if (!body.accepted.includes(member)) + return `Member ${member} is not one the capture showed stored (${bodyContract(body)}).`; + const recorded = shapes.entity[member]; + if ( + recorded !== undefined && + recorded !== null && + kindOf(recorded) !== kindOf(value) + ) + return `Member ${member} must be a JSON ${kindOf(recorded)}, as the capture stored it.`; + } + const missing = body.required.filter( + (member) => !Object.hasOwn(sent, member), + ); + if (missing.length > 0) + return `The call lacks required members ${missing.join(", ")} (${bodyContract(body)}).`; + return undefined; +}; + +/** The value at a JSON pointer inside a body, following array indexes and members. */ +export const valueAt = ( + body: JsonValue, + pointer: string, +): JsonValue | undefined => { + let value: JsonValue | undefined = body; + for (const token of pointerTokens(pointer)) { + if (Array.isArray(value)) value = value[Number(token)]; + else if (isObject(value) && Object.hasOwn(value, token)) + value = value[token]; + else return undefined; + } + return value; +}; + +/** A path value spelled as the observed value: a number when the observed value is one. */ +const pathValue = (observed: JsonValue | undefined, text: string): JsonValue => + typeof observed === "number" && integerOf(text) !== undefined + ? Number(text) + : text; + +/** One field's value under a rule that reads the call or generates. */ +const ruledValue = ( + rule: FieldRule, + observed: JsonObject, + call: WriteCall, + generate: GenerateValue, +): Applied => { + const body = call.body ?? {}; + switch (rule.rule) { + case "identity": + case "sequence": + case "unique": + case "clock": + return ok(generate(rule)); + case "echo": + return ok(Object.hasOwn(body, rule.field) ? body[rule.field] : undefined); + case "copy": { + const value = valueAt(body, rule.from); + return value === undefined + ? err( + `The call has no value at ${rule.from} in its body, which ${rule.field} copies.`, + ) + : ok(value); + } + case "path": { + const value = call.pathValues.get(rule.from); + return value === undefined + ? err( + `The call names no ${rule.from} path value, which ${rule.field} takes.`, + ) + : ok(pathValue(observed[rule.field], value)); + } + default: + return ok(undefined); + } +}; + +/** A string field rebuilt around the new value of the generated field it embeds. */ +const templated = ( + observed: JsonObject, + written: JsonObject, + rule: Extract, +): JsonValue | undefined => { + const text = observed[rule.field]; + const token = scalarText(observed[rule.from]); + const replacement = scalarText(written[rule.from]); + return typeof text === "string" && + token !== undefined && + replacement !== undefined + ? replaceDelimited(text, token, replacement) + : undefined; +}; + +/** Apply the rules that derive from fields already written: slugs and templates. */ +const derive = ( + rules: readonly FieldRule[], + observed: JsonObject, + written: JsonObject, + sent: JsonObject, +): void => { + for (const rule of rules) { + if (rule.rule === "slug") { + const from = sent[rule.from]; + if (typeof from === "string") written[rule.field] = slugOf(from); + } else if (rule.rule === "template") { + const value = templated(observed, written, rule); + if (value !== undefined) written[rule.field] = value; + } + } +}; + +/** The fields of a created entity: the observed entity with every ruled field taken from the call or generated. */ +export const createdFields = ( + plan: Pick, + observed: JsonObject, + call: WriteCall, + generate: GenerateValue, +): Applied => { + if (plan.body === null) return err("The tool has no modelled body."); + const problem = bodyProblem(plan.body, call, { entity: observed }); + if (problem !== undefined) return err(problem); + const written: JsonObject = { ...observed }; + for (const rule of plan.rules) { + const value = ruledValue(rule, observed, call, generate); + if (!value.ok) return value; + if (value.value !== undefined) written[rule.field] = value.value; + else if (rule.rule === "echo") delete written[rule.field]; + } + derive(plan.rules, observed, written, call.body ?? {}); + return ok(written); +}; + +/** The fields of an updated entity: sent echoed members, constants from the recorded update, generated and derived changes. */ +export const updatedFields = ( + plan: Pick, + entity: { readonly before: JsonObject; readonly observed: JsonObject }, + call: WriteCall & { readonly id: string }, + generate: GenerateValue, +): Applied => { + const { before, observed } = entity; + if (plan.body === null) return err("The tool has no modelled body."); + const sent = call.body ?? {}; + const problem = bodyProblem(plan.body, call, { + entity: { ...observed, ...before }, + id: call.id, + }); + if (problem !== undefined) return err(problem); + const members = Object.keys(sent).filter( + (member) => member !== plan.body?.identity, + ); + if (members.length === 0) + return err( + `The call sends none of the members an update applies (${bodyContract(plan.body)}).`, + ); + const written: JsonObject = { ...before }; + for (const rule of plan.rules) { + if (rule.rule === "echo" || rule.rule === "constant") { + if (!Object.hasOwn(sent, rule.field)) continue; + written[rule.field] = + rule.rule === "echo" ? sent[rule.field]! : observed[rule.field]!; + } else if (rule.rule === "increment") { + const was = integerOf(before[rule.field]); + if (was !== undefined) + written[rule.field] = spelledLike(before[rule.field]!, was + 1); + } else if (rule.rule === "clock") written[rule.field] = generate(rule); + } + derive( + plan.rules.filter( + (rule) => rule.rule !== "slug" || Object.hasOwn(sent, rule.from), + ), + observed, + written, + sent, + ); + return ok(written); +}; + +/** A document after a write: the body's members merged into, or replacing, the object at the pointer. */ +export const writtenDocument = ( + plan: Pick, + before: JsonValue, + call: WriteCall, +): Applied => { + const pointer = plan.observation?.pointer ?? ""; + const target = at(before, pointer); + if (!isObject(target) || call.body === undefined) + return err("The call's body does not fit the document."); + const value = + plan.merge === "replace" ? { ...call.body } : { ...target, ...call.body }; + return ok(replaceAt(before, pointer, value)); +}; + +/** A write's answer body from its recorded response: the entity or identity rendered in, nothing, or the recorded body. */ +export const writtenResponse = ( + response: WriteResponse, + recorded: JsonValue | undefined, + written: { readonly fields: JsonObject | undefined; readonly id: string }, +): Applied => { + if (response.shape === "empty") return ok(undefined); + if (response.shape === "recorded" || recorded === undefined) + return ok(recorded); + const pointer = response.pointer ?? ""; + const shown = at(recorded, pointer); + if (response.shape === "identity") + return ok( + replaceAt( + recorded, + pointer, + typeof shown === "number" && integerOf(written.id) !== undefined + ? Number(written.id) + : written.id, + ), + ); + if (!isObject(shown) || written.fields === undefined) + return err("The recorded response does not carry the entity."); + const missing = Object.keys(shown).filter( + (field) => !Object.hasOwn(written.fields!, field), + ); + if (missing.length > 0) + return err( + `The state holds no complete entity for this response: it lacks ${missing.join(", ")}.`, + ); + return ok( + replaceAt( + recorded, + pointer, + Object.fromEntries( + Object.keys(shown).map((field) => [field, written.fields![field]!]), + ), + ), + ); +}; + +/** An entity with updated fields; view-specific values of the fields that changed give way to the new values. */ +export const updatedEntity = ( + entity: StateEntity, + fields: JsonObject, +): StateEntity => { + const changed = new Set( + Object.keys(fields).filter( + (field) => !same(entity.fields[field], fields[field]), + ), + ); + return { + ...entity, + fields, + view_fields: entity.view_fields + .map((view) => ({ + tool: view.tool, + fields: Object.fromEntries( + Object.entries(view.fields).filter(([field]) => !changed.has(field)), + ), + })) + .filter((view) => Object.keys(view.fields).length > 0), + }; +}; diff --git a/src/domain/analysisView/environmentStateWriteChecks.ts b/src/domain/analysisView/environmentStateWriteChecks.ts new file mode 100644 index 0000000000..d8be87719e --- /dev/null +++ b/src/domain/analysisView/environmentStateWriteChecks.ts @@ -0,0 +1,269 @@ +import type { JsonValue } from "../jsonValue.js"; +import { compositeKey } from "../unicodeCodePointOrder.js"; +import { + at, + collectionPathOf, + isObject, + lastSegmentValue, + listItems, + renderStateEntity, + renderStateList, + same, + scalarText, + type JsonObject, + type StateDocument, + type StateEntity, + type StateItemView, + type StateListView, + type StateRecordFacts, +} from "./environmentStateRules.js"; + +export const entityTarget = (type: string, collection: string, id: string) => + `e${compositeKey([type, collection, id])}`; +export const collectionTarget = (type: string, collection: string) => + `c${compositeKey([type, collection])}`; +export const documentTarget = (tool: string) => `d${compositeKey([tool])}`; + +/** The gate's copy of the state: entities in collection order and documents, changed by modelled writes. */ +export class SimulatedState { + readonly #entities = new Map(); + readonly #collections = new Map(); + readonly #documents = new Map(); + + constructor( + entities: readonly StateEntity[], + documents: readonly StateDocument[], + ) { + for (const entity of entities) this.put(entity); + for (const document of documents) + this.#documents.set(document.tool, document.value); + } + + entity( + type: string, + collection: string, + id: string, + ): StateEntity | undefined { + return this.#entities.get(entityTarget(type, collection, id)); + } + + members(type: string, collection: string): StateEntity[] { + return ( + this.#collections.get(collectionTarget(type, collection)) ?? [] + ).map((key) => this.#entities.get(key)!); + } + + ofType(type: string): StateEntity[] { + return [...this.#entities.values()].filter( + (entity) => entity.type === type, + ); + } + + /** Replace an entity in place, or add it at `index` of its collection (the end when omitted). */ + put(entity: StateEntity, index?: number): void { + const key = entityTarget(entity.type, entity.collection, entity.id); + const known = this.#entities.has(key); + this.#entities.set(key, entity); + if (known) return; + const collection = collectionTarget(entity.type, entity.collection); + const order = this.#collections.get(collection) ?? []; + order.splice(index ?? order.length, 0, key); + this.#collections.set(collection, order); + } + + remove(entity: StateEntity): void { + const key = entityTarget(entity.type, entity.collection, entity.id); + this.#entities.delete(key); + const collection = collectionTarget(entity.type, entity.collection); + const order = this.#collections.get(collection) ?? []; + this.#collections.set( + collection, + order.filter((member) => member !== key), + ); + } + + document(tool: string): JsonValue | undefined { + return this.#documents.get(tool); + } + + setDocument(tool: string, value: JsonValue): void { + this.#documents.set(tool, value); + } +} + +/** An item read status that says the entity does not exist: 400, 404 or 410, as applications answer a missing id. */ +export const isAbsence = (status: number | null): boolean => + status === 400 || status === 404 || status === 410; + +const succeeded = (record: StateRecordFacts): boolean => + record.status !== null && record.status >= 200 && record.status < 300; + +/** A check's outcome and the targets it covered. */ +export interface CheckOutcome { + readonly pass: boolean; + readonly targets: readonly string[]; + /** Entities the recording lists that the state does not hold, by target. */ + readonly unheld: readonly string[]; +} + +/** An item read: the recorded entity object must render from the held entity; a recorded 404 or 410 agrees with none held. */ +export const checkItemRead = ( + state: SimulatedState, + view: StateItemView, + record: StateRecordFacts, + check: { + readonly body: JsonValue | undefined; + readonly volatile: ReadonlySet; + }, +): CheckOutcome | undefined => { + const collection = collectionPathOf(record, true); + const id = lastSegmentValue(record.path); + const entity = state.entity(view.type, collection, id); + const targets = [entityTarget(view.type, collection, id)]; + if (isAbsence(record.status) || (succeeded(record) && check.body === null)) + return { pass: entity === undefined, targets, unheld: [] }; + if (record.status === null || record.status < 200 || record.status >= 300) + return undefined; + const object = + check.body === undefined ? undefined : at(check.body, view.pointer); + if (!isObject(object)) return undefined; + if (entity === undefined) return { pass: false, targets, unheld: targets }; + return { + pass: same(renderStateEntity(entity, view, check.volatile, object), object), + targets, + unheld: [], + }; +}; + +/** A list read of every entity of a collection: membership, order, fields and counters must render from the state. */ +const checkAllList = ( + state: SimulatedState, + view: StateListView, + record: StateRecordFacts, + check: { + readonly body: JsonValue; + readonly items: readonly JsonObject[]; + readonly idField: string; + readonly volatile: ReadonlySet; + }, +): CheckOutcome => { + const collection = collectionPathOf(record, false); + const members = state.members(view.type, collection); + const observed = new Map( + check.items.map((item) => [scalarText(item[check.idField]) ?? "", item]), + ); + const rendered = members.map((entity) => + renderStateEntity(entity, view, check.volatile, observed.get(entity.id)), + ); + const held = new Set(members.map((entity) => entity.id)); + return { + pass: same(renderStateList(view, check.body, rendered), check.body), + targets: [ + collectionTarget(view.type, collection), + ...members.map((entity) => + entityTarget(view.type, collection, entity.id), + ), + ], + unheld: [...observed.keys()] + .filter((id) => !held.has(id)) + .map((id) => entityTarget(view.type, collection, id)), + }; +}; + +/** A list read of recorded members: every listed entity the state holds must render; others are not compared. */ +const checkRecordedList = ( + state: SimulatedState, + view: StateListView, + record: StateRecordFacts, + check: { + readonly items: readonly JsonObject[]; + readonly idField: string; + readonly volatile: ReadonlySet; + }, +): CheckOutcome => { + const collection = collectionPathOf(record, false); + const targets: string[] = []; + const unheld: string[] = []; + let pass = true; + for (const item of check.items) { + const id = scalarText(item[check.idField]) ?? ""; + const target = entityTarget(view.type, collection, id); + const entity = state.entity(view.type, collection, id); + if (entity === undefined) { + unheld.push(target); + continue; + } + targets.push(target); + if (!same(renderStateEntity(entity, view, check.volatile, item), item)) + pass = false; + } + return { pass, targets, unheld }; +}; + +/** A list read on a checked line, compared as the served environment answers it. */ +export const checkListRead = ( + state: SimulatedState, + view: StateListView, + record: StateRecordFacts, + check: { + readonly body: JsonValue | undefined; + readonly idField: string; + readonly volatile: ReadonlySet; + }, +): CheckOutcome | undefined => { + if (record.status === null || record.status < 200 || record.status >= 300) + return undefined; + const items = + check.body === undefined ? undefined : listItems(check.body, view.pointer); + if (check.body === undefined || items === undefined) return undefined; + return view.membership === "all" + ? checkAllList(state, view, record, { ...check, body: check.body, items }) + : checkRecordedList(state, view, record, { ...check, items }); +}; + +/** A document read on its checked line must equal the held document. */ +export const checkDocumentRead = ( + state: SimulatedState, + tool: string, + record: StateRecordFacts, + body: JsonValue | undefined, +): CheckOutcome | undefined => { + if (record.status === null || record.status < 200 || record.status >= 300) + return undefined; + const value = state.document(tool); + if (body === undefined || value === undefined) return undefined; + return { + pass: same(value, body), + targets: [documentTarget(tool)], + unheld: [], + }; +}; + +/** + * Check, field by field, the writes that changed an entity an unmodelled write also changed: + * each write's own fields must show as the state holds them. A create wrote every field, so it + * cannot be told apart from the unmodelled change and is not checked. + */ +export const writtenFieldChecks = ( + entity: StateEntity, + view: StateItemView, + observed: JsonObject, + check: { + readonly volatile: ReadonlySet; + readonly writes: ReadonlyMap; + }, +): Map => { + const shown = renderStateEntity(entity, view, check.volatile, observed); + const outcomes = new Map(); + for (const [tool, fields] of check.writes) { + const compared = (fields ?? []).filter( + (field) => Object.hasOwn(observed, field) || Object.hasOwn(shown, field), + ); + if (fields === null || compared.length === 0) continue; + outcomes.set( + tool, + compared.every((field) => same(shown[field], observed[field])), + ); + } + return outcomes; +}; diff --git a/src/domain/analysisView/environmentStateWriteCreate.ts b/src/domain/analysisView/environmentStateWriteCreate.ts new file mode 100644 index 0000000000..c84b3bf4a6 --- /dev/null +++ b/src/domain/analysisView/environmentStateWriteCreate.ts @@ -0,0 +1,211 @@ +import { + isStateWrite, + normalizedPath, + scalarText, + type JsonObject, + type StateEntity, +} from "./environmentStateRules.js"; +import { createdFields } from "./environmentStateWriteApply.js"; +import { + collectionTarget, + entityTarget, +} from "./environmentStateWriteChecks.js"; +import { + built, + callOf, + fail, + refuse, + requestObject, + searchFrom, + startOf, + UNKNOWN, + type EffectContext, + type EffectResult, + type WriteDraft, + type WriteStep, +} from "./environmentStateWriteEffects.js"; +import { + explainCreate, + insertCandidates, + insertIndex, + responseShapeOf, +} from "./environmentStateWriteExplain.js"; +import { + laterObservation, + newEntityIn, + type Observed, +} from "./environmentStateWriteObserve.js"; +import type { WriteCall } from "./environmentStateWriteRules.js"; + +/** The written entity a create's response or a later read shows. */ +const createdObservation = ( + context: EffectContext, + step: WriteStep, + target: { + readonly type: string; + readonly collection: string; + readonly idField: string; + }, +): Observed | undefined => { + const held = (id: string) => + context.state.entity(target.type, target.collection, id) !== undefined; + const own = context.io.withBody(step.record.ordinal, (body) => + newEntityIn(step.record.ordinal, body, target.idField, held), + ); + if (own !== undefined) return own; + const later = laterObservation(searchFrom(context, step), { + ...target, + held, + isWrite: isStateWrite, + }); + if (later !== undefined) context.observed(later.ordinal); + return later; +}; + +const fieldsOf = (entities: readonly StateEntity[]): JsonObject[] => + entities.map((entity) => entity.fields); + +/** Build a create plan from its exemplar's observation. */ +const buildCreate = ( + context: EffectContext, + draft: WriteDraft, + step: WriteStep, + evidence: { + readonly observed: Observed; + readonly call: WriteCall & { readonly body: JsonObject }; + readonly collection: string; + }, +): void => { + const type = draft.plan.target!; + const idField = context.idField(type); + const members = fieldsOf(context.state.members(type, evidence.collection)); + const explained = explainCreate({ + entity: evidence.observed.object, + call: evidence.call, + idField, + startMs: startOf(context, step), + collection: members, + type: fieldsOf(context.state.ofType(type)), + }); + const clocks = explained.rules.flatMap((rule) => + rule.rule === "clock" ? [rule.field] : [], + ); + // Identity and clock orders first, then any other scalar field the collection may be sorted by. + const scalars = Object.keys(members[0] ?? {}).filter((field) => + members.every((member) => + ["string", "number"].includes(typeof member[field]), + ), + ); + draft.candidates = insertCandidates(members, [ + ...new Set([idField, ...clocks, ...scalars]), + ]); + Object.assign(draft.plan, { + exemplar_ordinal: step.record.ordinal, + observation: { + ordinal: evidence.observed.ordinal, + pointer: evidence.observed.pointer, + }, + response: context.io.withBody(step.record.ordinal, (body) => + responseShapeOf( + step.record.ordinal, + body, + context.io.emptyResponse(step.record.ordinal), + { idField, id: evidence.observed.id }, + ), + ), + body: explained.body, + rules: explained.rules, + collection: explained.tied ? evidence.collection : null, + insert: draft.candidates[0]!, + }); + draft.observed = evidence.observed.object; +}; + +/** Add a created entity to the state at the plan's insert position, or last before the plan has one. */ +const insertCreated = ( + context: EffectContext, + draft: WriteDraft, + entity: StateEntity, +): string => { + const members = fieldsOf( + context.state.members(entity.type, entity.collection), + ); + context.io.retain(64 + JSON.stringify(entity.fields).length * 2); + context.state.put( + entity, + draft.plan.insert === null + ? members.length + : insertIndex(draft.plan.insert, members, entity.fields), + ); + return entityTarget(entity.type, entity.collection, entity.id); +}; + +/** Apply a recorded create, with the values it recorded for generated fields; an unexplained create adds what it recorded. */ +export const applyCreate = ( + context: EffectContext, + draft: WriteDraft, + step: WriteStep, +): EffectResult => { + const type = draft.plan.target!; + const idField = context.idField(type); + const collection = normalizedPath(step.record.path); + const body = requestObject(context, step.record); + const observed = createdObservation(context, step, { + type, + collection, + idField, + }); + if (observed === undefined) { + fail(draft, step, body === undefined ? "body-unusable" : "no-observation"); + return UNKNOWN; + } + const entity = (fields: JsonObject, id: string): StateEntity => ({ + type, + collection, + id, + fields, + view_fields: [], + record_ordinals: [], + }); + const recorded = (): EffectResult => { + const inserted = insertCreated( + context, + draft, + entity(observed.object, observed.id), + ); + return { + kind: "observed", + targets: [inserted, collectionTarget(type, collection)], + entity: { type, collection, id: observed.id }, + }; + }; + if (step.observeOnly || body === undefined) { + fail(draft, step, "body-unusable"); + return recorded(); + } + const call = { ...callOf(step, body), body }; + if (!built(draft)) + buildCreate(context, draft, step, { observed, call, collection }); + const fields = createdFields( + draft.plan, + draft.observed!, + call, + (rule) => observed.object[rule.field] ?? null, + ); + const id = fields.ok ? scalarText(fields.value[idField]) : undefined; + if ( + !fields.ok || + id === undefined || + context.state.entity(type, collection, id) !== undefined + ) { + refuse(draft, step, "unexplained-change"); + return recorded(); + } + const inserted = insertCreated(context, draft, entity(fields.value, id)); + return { + kind: "applied", + targets: [inserted, collectionTarget(type, collection)], + entity: { type, collection, id }, + inserted, + }; +}; diff --git a/src/domain/analysisView/environmentStateWriteEffects.ts b/src/domain/analysisView/environmentStateWriteEffects.ts new file mode 100644 index 0000000000..d4ffc5f9cf --- /dev/null +++ b/src/domain/analysisView/environmentStateWriteEffects.ts @@ -0,0 +1,431 @@ +import type { JsonValue } from "../jsonValue.js"; +import { + collectionPathOf, + isObject, + isStateWrite, + lastSegmentValue, + locateItem, + at, + same, + type JsonObject, + type StateRecordFacts, + type StateToolFacts, + type StateView, +} from "./environmentStateRules.js"; +import { + updatedEntity, + updatedFields, + writtenDocument, +} from "./environmentStateWriteApply.js"; +import { + collectionTarget, + documentTarget, + entityTarget, + type SimulatedState, +} from "./environmentStateWriteChecks.js"; +import { + documentPointerOf, + explainUpdate, + responseShapeOf, +} from "./environmentStateWriteExplain.js"; +import { + laterObservation, + type Observed, +} from "./environmentStateWriteObserve.js"; +import { + pathValuesOf, + type InsertRule, + type StateWritePlan, + type WriteCall, + type WriteReason, +} from "./environmentStateWriteRules.js"; + +type Mutable = { -readonly [K in keyof T]: T[K] }; + +/** A plan under construction and the evidence the walk gathers for its verdict. */ +export interface WriteDraft { + readonly plan: Mutable; + /** The exemplar's observed entity, whose unruled fields a create keeps. */ + observed: JsonObject | undefined; + candidates: InsertRule[]; + readonly confirmed: number[]; + contradictedBy: number | undefined; + sideEffect: boolean; + calls: number; +} + +/** Response bodies, request bodies and budget the effects read. */ +export interface WriteIo { + readonly withBody: ( + ordinal: number, + use: (body: JsonValue | undefined) => T, + ) => T; + readonly requestBody: (ordinal: number) => JsonValue | undefined; + readonly emptyResponse: (ordinal: number) => boolean; + readonly retain: (bytes: number) => void; +} + +/** What applying a recorded write reads besides the draft and the call. */ +export interface EffectContext { + readonly state: SimulatedState; + readonly io: WriteIo; + readonly records: readonly StateRecordFacts[]; + readonly timed: boolean; + readonly viewOf: (ordinal: number) => StateView | undefined; + readonly idField: (type: string) => string; + /** Mark a later read as a create's or update's observation, so it does not check that write. */ + readonly observed: (ordinal: number) => void; + /** Whether an unmodelled write may have changed the entity since the state last saw it. */ + readonly unknown: (type: string, collection: string, id: string) => boolean; +} + +/** One recorded write: the record, its timeline index, its tool, and whether its plan only takes the recorded outcome. */ +export interface WriteStep { + readonly record: StateRecordFacts; + readonly index: number; + readonly tool: StateToolFacts; + /** Set once the plan has a reason: the write only brings the state to its recorded outcome. */ + readonly observeOnly: boolean; +} + +/** + * What applying a recorded write did: the modelled effect (attributable to the tool), the + * recorded outcome when the model does not explain it (not attributable), or nothing known. + */ +export type EffectResult = + | { + readonly kind: "applied"; + readonly targets: readonly string[]; + readonly entity?: WrittenEntity; + /** The fields an update wrote; a create writes the whole entity. */ + readonly fields?: readonly string[]; + readonly inserted?: string; + } + | { + readonly kind: "observed"; + readonly targets: readonly string[]; + readonly entity?: WrittenEntity; + } + | { readonly kind: "unknown" }; + +/** The entity a write changed, by type, collection and id. */ +export interface WrittenEntity { + readonly type: string; + readonly collection: string; + readonly id: string; +} + +export const UNKNOWN: EffectResult = { kind: "unknown" }; + +export const built = (draft: WriteDraft): boolean => + draft.plan.exemplar_ordinal !== null; + +/** Give an unbuilt plan its reason; a built plan keeps its model. */ +export const fail = ( + draft: WriteDraft, + step: WriteStep, + reason: WriteReason, +): void => { + if (!built(draft) && !step.observeOnly) draft.plan.reason = reason; +}; + +/** A recorded call the built model does not explain disqualifies the plan. */ +export const refuse = ( + draft: WriteDraft, + step: WriteStep, + reason: WriteReason, +): void => { + if (!step.observeOnly) draft.plan.reason = reason; +}; + +export const requestObject = ( + context: EffectContext, + record: StateRecordFacts, +): JsonObject | undefined => { + const body = context.io.requestBody(record.ordinal); + return isObject(body) ? body : undefined; +}; + +export const callOf = ( + step: WriteStep, + body: JsonObject | undefined, +): WriteCall => ({ + path: step.record.path, + pathValues: + pathValuesOf(step.tool.path_template, step.record.path) ?? new Map(), + body, +}); + +export const startOf = ( + context: EffectContext, + step: WriteStep, +): number | undefined => (context.timed ? step.record.start : undefined); + +export const searchFrom = (context: EffectContext, step: WriteStep) => ({ + records: context.records, + from: step.index, + viewOf: context.viewOf, + withBody: context.io.withBody, +}); + +/** The entity an update's response shows, located by its identity, else a later item read of it. */ +const updatedObservation = ( + context: EffectContext, + step: WriteStep, + target: { + readonly type: string; + readonly collection: string; + readonly idField: string; + readonly id: string; + }, +): Observed | undefined => { + const own = context.io.withBody( + step.record.ordinal, + (body): Observed | undefined => { + const located = + body === undefined ? undefined : locateItem(body, target.id); + const object = + located === undefined ? undefined : at(body!, located.pointer); + return located?.keys.includes(target.idField) === true && isObject(object) + ? { + ordinal: step.record.ordinal, + pointer: located.pointer, + object, + id: target.id, + } + : undefined; + }, + ); + if (own !== undefined) return own; + const later = laterObservation(searchFrom(context, step), { + ...target, + held: () => true, + isWrite: isStateWrite, + }); + if (later !== undefined) context.observed(later.ordinal); + return later; +}; + +/** Add what a later call of an update showed to its plan: members, fixed values and rules not seen before. */ +const refineUpdate = ( + plan: WriteDraft["plan"], + explained: ReturnType, +): void => { + const body = plan.body!; + const fixed = new Set(body.fixed.map((entry) => entry.member)); + plan.body = { + ...body, + accepted: [ + ...body.accepted, + ...explained.body.accepted.filter( + (member) => !body.accepted.includes(member), + ), + ], + fixed: [ + ...body.fixed, + ...explained.body.fixed.filter((entry) => !fixed.has(entry.member)), + ], + }; + const ruled = new Set(plan.rules.map((rule) => rule.field)); + plan.rules = [ + ...plan.rules, + ...explained.rules.filter((rule) => !ruled.has(rule.field)), + ]; +}; + +/** Build an update plan from its exemplar, or add what a later call showed. */ +const buildUpdate = ( + context: EffectContext, + draft: WriteDraft, + step: WriteStep, + evidence: { + readonly after: Observed; + readonly explained: ReturnType; + readonly idField: string; + }, +): void => { + if (built(draft)) { + refineUpdate(draft.plan, evidence.explained); + return; + } + const { ordinal } = step.record; + Object.assign(draft.plan, { + exemplar_ordinal: ordinal, + observation: { + ordinal: evidence.after.ordinal, + pointer: evidence.after.pointer, + }, + response: context.io.withBody(ordinal, (response) => + responseShapeOf(ordinal, response, context.io.emptyResponse(ordinal), { + idField: evidence.idField, + id: evidence.after.id, + }), + ), + body: evidence.explained.body, + rules: evidence.explained.rules, + }); + draft.observed = evidence.after.object; +}; + +/** + * Apply a recorded update to the held entity, explaining every change it made. An update the + * model does not explain, or of an entity an unmodelled write may have changed, brings the + * entity to what it recorded instead and is not attributed to the tool. + */ +export const applyUpdate = ( + context: EffectContext, + draft: WriteDraft, + step: WriteStep, +): EffectResult => { + const type = draft.plan.target!; + const idField = context.idField(type); + const collection = collectionPathOf(step.record, true); + const id = lastSegmentValue(step.record.path); + const entity = context.state.entity(type, collection, id); + const after = + entity === undefined + ? undefined + : updatedObservation(context, step, { type, collection, idField, id }); + if (entity === undefined || after === undefined) { + fail(draft, step, "no-observation"); + return UNKNOWN; + } + const targets = [ + entityTarget(type, collection, id), + collectionTarget(type, collection), + ]; + const target = { type, collection, id }; + const recorded = (): EffectResult => { + context.state.put( + updatedEntity(entity, { ...entity.fields, ...after.object }), + ); + return { kind: "observed", targets, entity: target }; + }; + const body = requestObject(context, step.record); + if (step.observeOnly || body === undefined) { + fail(draft, step, "body-unusable"); + return recorded(); + } + const unknown = context.unknown(type, collection, id); + const call = { ...callOf(step, body), body }; + const explained = explainUpdate({ + before: entity.fields, + after: after.object, + call, + method: step.tool.method, + idField, + id, + startMs: startOf(context, step), + }); + if (explained.unexplained.length > 0 && !unknown) { + refuse(draft, step, "unexplained-change"); + return recorded(); + } + buildUpdate(context, draft, step, { after, explained, idField }); + const written = updatedFields( + draft.plan, + { before: entity.fields, observed: after.object }, + { ...call, id }, + (rule) => after.object[rule.field] ?? null, + ); + if (!written.ok) refuse(draft, step, "unexplained-change"); + if (!written.ok || unknown) return recorded(); + context.io.retain(64 + JSON.stringify(written.value).length * 2); + context.state.put(updatedEntity(entity, written.value)); + // A later read that also shows an unmodelled change checks only the members this call sent. + const sent = Object.keys(body).filter( + (member) => member !== draft.plan.body?.identity, + ); + return { kind: "applied", targets, entity: target, fields: sent }; +}; + +/** Apply a recorded delete: the held entity leaves its collection. */ +export const applyDelete = ( + context: EffectContext, + draft: WriteDraft, + step: WriteStep, +): EffectResult => { + const type = draft.plan.target!; + const collection = collectionPathOf(step.record, true); + const id = lastSegmentValue(step.record.path); + const entity = context.state.entity(type, collection, id); + if (entity === undefined) { + fail(draft, step, "no-observation"); + return UNKNOWN; + } + const { ordinal } = step.record; + if (!built(draft) && !step.observeOnly) + Object.assign(draft.plan, { + exemplar_ordinal: ordinal, + response: context.io.withBody(ordinal, (body) => + responseShapeOf(ordinal, body, context.io.emptyResponse(ordinal), { + idField: context.idField(type), + id, + }), + ), + }); + context.state.remove(entity); + const targets = [ + entityTarget(type, collection, id), + collectionTarget(type, collection), + ]; + return { + kind: step.observeOnly ? "observed" : "applied", + targets, + entity: { type, collection, id }, + }; +}; + +/** Apply a recorded document write: the body merged into, or replacing, the object holding most of its members. */ +export const applyDocument = ( + context: EffectContext, + draft: WriteDraft, + step: WriteStep, +): EffectResult => { + const tool = draft.plan.target!; + const value = context.state.document(tool); + const body = requestObject(context, step.record); + const pointer = + value === undefined || body === undefined + ? undefined + : documentPointerOf(value, body); + if ( + step.observeOnly || + value === undefined || + body === undefined || + pointer === undefined + ) { + fail(draft, step, body === undefined ? "body-unusable" : "no-observation"); + return UNKNOWN; + } + const { ordinal } = step.record; + if (!built(draft)) + Object.assign(draft.plan, { + exemplar_ordinal: ordinal, + observation: { ordinal, pointer }, + merge: step.tool.method === "PUT" ? "replace" : "merge", + body: { + accepted: Object.keys(body), + fixed: [], + required: [], + identity: null, + }, + }); + const written = writtenDocument(draft.plan, value, callOf(step, body)); + if (!written.ok) { + refuse(draft, step, "no-observation"); + return UNKNOWN; + } + draft.plan.response ??= context.io.withBody(ordinal, (response) => ({ + ordinal, + shape: context.io.emptyResponse(ordinal) + ? "empty" + : response !== undefined && same(response, written.value) + ? "entity" + : "recorded", + pointer: null, + })); + context.io.retain(64 + JSON.stringify(written.value).length * 2); + context.state.setDocument(tool, written.value); + return { kind: "applied", targets: [documentTarget(tool)] }; +}; diff --git a/src/domain/analysisView/environmentStateWriteExplain.ts b/src/domain/analysisView/environmentStateWriteExplain.ts new file mode 100644 index 0000000000..5c1ffd8ef5 --- /dev/null +++ b/src/domain/analysisView/environmentStateWriteExplain.ts @@ -0,0 +1,426 @@ +import type { JsonValue } from "../jsonValue.js"; +import { compareUnicodeCodePoints as compare } from "../unicodeCodePointOrder.js"; +import { + isObject, + locateItem, + pointerTo, + same, + scalarText, + type JsonObject, +} from "./environmentStateRules.js"; +import type { + FieldRule, + InsertRule, + WriteBody, + WriteCall, + WriteResponse, +} from "./environmentStateWriteRules.js"; +import { + containsDelimited, + delimitedOffsets, + integerOf, + isUuid, + readClock, + slugOf, +} from "./environmentStateWriteValues.js"; + +/** A clock value counts as written by the request when it lies this close to the request's start. */ +const CLOCK_WINDOW_MS = 10 * 60_000; +/** Shorter body strings are too common to show that a field copied one. */ +const COPY_MIN_LENGTH = 8; + +/** What explaining a create reads: the observed entity, the call that wrote it and the entities it joined. */ +export interface CreateEvidence { + readonly entity: JsonObject; + readonly call: WriteCall & { readonly body: JsonObject }; + readonly idField: string; + /** The request's start in epoch milliseconds, when the timeline is timed. */ + readonly startMs: number | undefined; + readonly collection: readonly JsonObject[]; + readonly type: readonly JsonObject[]; +} + +/** The JSON pointer of the first string in document order equal to `value`, searched inside the body. */ +const bodyPointerOf = (body: JsonValue, value: string): string | undefined => { + const pending: { readonly value: JsonValue; readonly pointer: string }[] = [ + { value: body, pointer: "" }, + ]; + for (let next = pending.shift(); next !== undefined; next = pending.shift()) { + if (next.value === value) return next.pointer; + const children: [string, JsonValue][] = Array.isArray(next.value) + ? next.value.map((item, index) => [String(index), item]) + : isObject(next.value) + ? Object.entries(next.value) + : []; + for (const [key, item] of children) + pending.push({ + value: item, + pointer: `${next.pointer}${pointerTo(key)}`, + }); + } + return undefined; +}; + +const isClockNear = ( + value: JsonValue, + startMs: number | undefined, +): boolean => { + const clock = readClock(value); + return ( + clock !== undefined && + startMs !== undefined && + Math.abs(clock.ms - startMs) <= CLOCK_WINDOW_MS + ); +}; + +/** One more than the largest integer the members hold in the field, or undefined when none holds one. */ +const nextInteger = ( + members: readonly JsonObject[], + field: string, +): number | undefined => { + let largest: number | undefined; + for (const member of members) { + const value = integerOf(member[field]); + if (value !== undefined && (largest === undefined || value > largest)) + largest = value; + } + return largest === undefined ? undefined : largest + 1; +}; + +const identityRule = (field: string, value: JsonValue): FieldRule => ({ + field, + rule: "identity", + generator: + integerOf(value) !== undefined + ? "sequence" + : isUuid(value) + ? "uuid" + : "token", +}); + +/** The rule a request explains a created field by, before generated values and templates are considered. */ +const requestRule = ( + evidence: CreateEvidence, + field: string, + value: JsonValue, +): FieldRule | undefined => { + const { body, pathValues } = evidence.call; + if (Object.hasOwn(body, field) && same(body[field], value)) + return { field, rule: "echo" }; + if (typeof value === "string" && value.length >= COPY_MIN_LENGTH) { + const from = bodyPointerOf(body, value); + if (from !== undefined) return { field, rule: "copy", from }; + } + const text = scalarText(value); + for (const [name, pathValue] of pathValues) + if (text !== undefined && text === pathValue) + return { field, rule: "path", from: name }; + if (typeof value === "string" && value.length >= 3) + for (const [member, sent] of Object.entries(body)) + if (typeof sent === "string" && sent !== value && slugOf(sent) === value) + return { field, rule: "slug", from: member }; + return undefined; +}; + +/** The rule a created field's generated value follows, if it follows one. */ +const generatedRule = ( + evidence: CreateEvidence, + field: string, + value: JsonValue, +): FieldRule | undefined => { + if (field === evidence.idField) return identityRule(field, value); + const clock = readClock(value); + if (clock !== undefined && isClockNear(value, evidence.startMs)) + return { field, rule: "clock", format: clock.format }; + const integer = integerOf(value); + if (integer !== undefined) { + if (nextInteger(evidence.collection, field) === integer) + return { field, rule: "sequence", scope: "collection" }; + if (nextInteger(evidence.type, field) === integer) + return { field, rule: "sequence", scope: "type" }; + } + if ( + isUuid(value) && + evidence.type.length > 0 && + evidence.type.every((member) => member[field] !== value) + ) + return { field, rule: "unique" }; + return undefined; +}; + +/** A string that embeds a generated value delimited is rebuilt around the new value. */ +const templateRule = ( + entity: JsonObject, + field: string, + generated: readonly FieldRule[], +): FieldRule | undefined => { + const value = entity[field]; + if (typeof value !== "string") return undefined; + for (const rule of generated) { + const token = scalarText(entity[rule.field]); + if ( + rule.field !== field && + token !== undefined && + token.length > 0 && + delimitedOffsets(value, token).length > 0 + ) + return { field, rule: "template", from: rule.field }; + } + return undefined; +}; + +/** Body members the application stored as sent, and the others with what they were sent. */ +const bodyOf = ( + body: JsonObject, + echoed: ReadonlySet, + required: readonly string[], +): WriteBody => ({ + accepted: Object.keys(body).filter((member) => echoed.has(member)), + fixed: Object.entries(body) + .filter(([member]) => !echoed.has(member)) + .map(([member, sent]) => ({ member, sent })), + required: [...required], + identity: null, +}); + +/** Explain every field of a created entity: echoed, copied, path, slug, generated, template or constant (unlisted). */ +export const explainCreate = ( + evidence: CreateEvidence, +): { + readonly rules: FieldRule[]; + readonly body: WriteBody; + readonly tied: boolean; +} => { + const rules: FieldRule[] = []; + const unexplained: string[] = []; + for (const [field, value] of Object.entries(evidence.entity)) { + const rule = + requestRule(evidence, field, value) ?? + generatedRule(evidence, field, value); + if (rule === undefined) unexplained.push(field); + else rules.push(rule); + } + const generated = rules.filter( + (rule) => + rule.rule === "identity" || + rule.rule === "sequence" || + rule.rule === "unique", + ); + const constants: string[] = []; + for (const field of unexplained) { + const template = templateRule(evidence.entity, field, generated); + if (template === undefined) constants.push(field); + else rules.push(template); + } + const echoed = new Set( + rules.filter((rule) => rule.rule === "echo").map((rule) => rule.field), + ); + const tokens = [...evidence.call.pathValues.values()].filter( + (value) => value.length > 0, + ); + return { + rules, + body: bodyOf(evidence.call.body, echoed, Object.keys(evidence.call.body)), + tied: constants.some((field) => + tokens.some((token) => containsDelimited(evidence.entity[field]!, token)), + ), + }; +}; + +/** What explaining an update reads: the entity before and after, the call and its method. */ +export interface UpdateEvidence { + readonly before: JsonObject; + readonly after: JsonObject; + readonly call: WriteCall & { readonly body: JsonObject }; + readonly method: string; + readonly idField: string; + readonly id: string; + readonly startMs: number | undefined; +} + +/** The rule a field the update changed without sending follows, if any. */ +const changeRule = ( + evidence: UpdateEvidence, + field: string, + echoed: ReadonlySet, +): FieldRule | undefined => { + const before = evidence.before[field]; + const after = evidence.after[field]!; + if (readClock(after) !== undefined && isClockNear(after, evidence.startMs)) + return { field, rule: "clock", format: readClock(after)!.format }; + const was = integerOf(before); + if (was !== undefined && integerOf(after) === was + 1) + return { field, rule: "increment" }; + if (typeof after === "string") + for (const member of echoed) { + const sent = evidence.call.body[member]; + if (typeof sent === "string" && slugOf(sent) === after) + return { field, rule: "slug", from: member }; + } + return undefined; +}; + +/** Explain an update: echoed members, fixed members whose stored value is constant, and the changes it made unsent. */ +export const explainUpdate = ( + evidence: UpdateEvidence, +): { + readonly rules: FieldRule[]; + readonly body: WriteBody; + readonly unexplained: string[]; +} => { + const { body } = evidence.call; + const rules: FieldRule[] = []; + const echoed = new Set(); + let identity: string | null = null; + for (const [member, sent] of Object.entries(body)) { + if (member === evidence.idField && scalarText(sent) === evidence.id) { + identity = member; + continue; + } + if (!Object.hasOwn(evidence.after, member)) continue; + if (same(evidence.after[member], sent)) { + echoed.add(member); + rules.push({ field: member, rule: "echo" }); + } else rules.push({ field: member, rule: "constant" }); + } + const unexplained: string[] = []; + for (const field of Object.keys(evidence.after)) { + if ( + Object.hasOwn(body, field) || + !Object.hasOwn(evidence.before, field) || + same(evidence.before[field], evidence.after[field]) + ) + continue; + const rule = changeRule(evidence, field, echoed); + if (rule === undefined) unexplained.push(field); + else rules.push(rule); + } + const shaped = bodyOf( + Object.fromEntries( + Object.entries(body).filter(([member]) => member !== identity), + ), + echoed, + evidence.method === "PUT" + ? Object.keys(body).filter((member) => member !== identity) + : [], + ); + return { rules, body: { ...shaped, identity }, unexplained }; +}; + +/** + * How a write's response carries the entity it wrote: an object with the type's identity field + * equal to the id (entity), the id alone at the root or in one member (identity), nothing + * (empty), or anything else (recorded). + */ +export const responseShapeOf = ( + ordinal: number, + body: JsonValue | undefined, + empty: boolean, + entity: { readonly idField: string; readonly id: string }, +): WriteResponse => { + if (empty) return { ordinal, shape: "empty", pointer: null }; + if (body === undefined) return { ordinal, shape: "recorded", pointer: null }; + const located = locateItem(body, entity.id); + if (located?.keys.includes(entity.idField) === true) + return { ordinal, shape: "entity", pointer: located.pointer }; + if (scalarText(body) === entity.id) + return { ordinal, shape: "identity", pointer: "" }; + if (isObject(body)) { + const members = Object.entries(body).filter( + ([, value]) => scalarText(value) === entity.id, + ); + if (members.length === 1) + return { + ordinal, + shape: "identity", + pointer: pointerTo(members[0]![0]), + }; + } + return { ordinal, shape: "recorded", pointer: null }; +}; + +/** Order two field values as numbers, instants or text; undefined when they are not comparable. */ +export const compareValues = ( + left: JsonValue | undefined, + right: JsonValue | undefined, +): number | undefined => { + const [a, b] = [integerOf(left), integerOf(right)]; + if (a !== undefined && b !== undefined) return a - b; + const [c, d] = [readClock(left), readClock(right)]; + if (c !== undefined && d !== undefined) return c.ms - d.ms; + return typeof left === "string" && typeof right === "string" + ? compare(left, right) + : undefined; +}; + +/** Whether the members are strictly ordered by the field in the direction. */ +const orderedBy = ( + members: readonly JsonObject[], + field: string, + direction: "ascending" | "descending", +): boolean => + members.every((member, index) => { + if (index === 0) return true; + const order = compareValues(members[index - 1]![field], member[field]); + return ( + order !== undefined && (direction === "ascending" ? order < 0 : order > 0) + ); + }); + +/** Insert rules consistent with a collection's recorded order: sorted by identity or clock fields when shown, then last and first. */ +export const insertCandidates = ( + members: readonly JsonObject[], + fields: readonly string[], +): InsertRule[] => { + const sorted: InsertRule[] = []; + if (members.length >= 2) + for (const field of fields) + for (const direction of ["ascending", "descending"] as const) + if (orderedBy(members, field, direction)) + sorted.push({ rule: "sorted", field, direction }); + return [...sorted, { rule: "last" }, { rule: "first" }]; +}; + +/** Where an entity with these fields enters a collection under an insert rule. */ +export const insertIndex = ( + rule: InsertRule, + members: readonly JsonObject[], + fields: JsonObject, +): number => { + if (rule.rule !== "sorted") return rule.rule === "first" ? 0 : members.length; + const index = members.findIndex((member) => { + const order = compareValues(fields[rule.field], member[rule.field]); + return ( + order !== undefined && + (rule.direction === "ascending" ? order < 0 : order > 0) + ); + }); + return index < 0 ? members.length : index; +}; + +/** The object in a document value, at the root or in one member, that holds most of a body's members. */ +export const documentPointerOf = ( + value: JsonValue, + body: JsonObject, +): string | undefined => { + const members = Object.keys(body); + const candidates: [string, JsonValue][] = [ + ["", value], + ...(isObject(value) + ? Object.entries(value).map(([name, item]): [string, JsonValue] => [ + pointerTo(name), + item, + ]) + : []), + ]; + let best: { readonly pointer: string; readonly shared: number } | undefined; + for (const [pointer, candidate] of candidates) { + if (!isObject(candidate)) continue; + const shared = members.filter((member) => + Object.hasOwn(candidate, member), + ).length; + if (shared > 0 && (best === undefined || shared > best.shared)) + best = { pointer, shared }; + } + return best?.pointer; +}; diff --git a/src/domain/analysisView/environmentStateWriteObserve.ts b/src/domain/analysisView/environmentStateWriteObserve.ts new file mode 100644 index 0000000000..0bd99018c7 --- /dev/null +++ b/src/domain/analysisView/environmentStateWriteObserve.ts @@ -0,0 +1,165 @@ +import type { JsonValue } from "../jsonValue.js"; +import { + collectionPathOf, + isObject, + lastSegmentValue, + listItems, + pointerTo, + scalarText, + type JsonObject, + type StateRecordFacts, + type StateView, +} from "./environmentStateRules.js"; + +/** Where an observed entity object was found: the record and a JSON pointer into its response. */ +export interface Observed { + readonly ordinal: number; + readonly pointer: string; + readonly object: JsonObject; + readonly id: string; +} + +/** Objects a response carries where an entity may be, in precedence tiers: the root, one member, then the element of a one-element array at the root or in one member. */ +const entitySlots = (body: JsonValue): [string, JsonValue][][] => { + const members = isObject(body) ? Object.entries(body) : []; + const elements: [string, JsonValue][] = + Array.isArray(body) && body.length === 1 ? [["/0", body[0]!]] : []; + for (const [name, item] of members) + if (Array.isArray(item) && item.length === 1) + elements.push([`${pointerTo(name)}/0`, item[0]!]); + return [ + [["", body]], + members.map(([name, item]): [string, JsonValue] => [pointerTo(name), item]), + elements, + ]; +}; + +/** The entity object a response carries with an identity value the state does not hold yet: the only one in the first tier that has one. */ +export const newEntityIn = ( + ordinal: number, + body: JsonValue | undefined, + idField: string, + held: (id: string) => boolean, +): Observed | undefined => { + if (body === undefined) return undefined; + for (const tier of entitySlots(body)) { + const found = tier.flatMap(([pointer, object]) => { + const id = isObject(object) ? scalarText(object[idField]) : undefined; + // An object holding only the identity names the entity without showing it. + return isObject(object) && + id !== undefined && + !held(id) && + Object.keys(object).some((key) => key !== idField) + ? [{ ordinal, pointer, object, id }] + : []; + }); + if (found.length > 0) return found.length === 1 ? found[0] : undefined; + } + return undefined; +}; + +/** What a forward search for an observation reads: records after a position, read views and response bodies. */ +export interface ObservationSearch { + readonly records: readonly StateRecordFacts[]; + readonly from: number; + readonly viewOf: (ordinal: number) => StateView | undefined; + readonly withBody: ( + ordinal: number, + use: (body: JsonValue | undefined) => T, + ) => T; +} + +const succeeded = (record: StateRecordFacts): boolean => + record.status !== null && record.status >= 200 && record.status < 300; + +/** + * The first later read, before the next write, that shows a written entity: an item read of + * `id` when given, else an item read of an unheld id or a collection read listing exactly one. + */ +export const laterObservation = ( + search: ObservationSearch, + target: { + readonly type: string; + readonly collection: string; + readonly idField: string; + readonly id?: string; + readonly held: (id: string) => boolean; + readonly isWrite: (record: StateRecordFacts) => boolean; + }, +): Observed | undefined => { + for (const record of search.records.slice(search.from + 1)) { + if (target.isWrite(record)) return undefined; + const view = search.viewOf(record.ordinal); + if ( + !succeeded(record) || + (view?.shape !== "item" && view?.shape !== "list") || + view.type !== target.type + ) + continue; + const found = + view.shape === "item" + ? itemObservation(search, record, view.pointer, target) + : listObservation(search, record, view.pointer, target); + if (found !== undefined) return found; + } + return undefined; +}; + +const itemObservation = ( + search: ObservationSearch, + record: StateRecordFacts, + pointer: string, + target: Parameters[1], +): Observed | undefined => { + const id = lastSegmentValue(record.path); + if ( + collectionPathOf(record, true) !== target.collection || + (target.id !== undefined ? id !== target.id : target.held(id)) + ) + return undefined; + return search.withBody(record.ordinal, (body) => { + const object = + body === undefined + ? undefined + : entitySlots(body) + .flat() + .find(([at]) => at === pointer)?.[1]; + return isObject(object) && scalarText(object[target.idField]) === id + ? { ordinal: record.ordinal, pointer, object, id } + : undefined; + }); +}; + +const listObservation = ( + search: ObservationSearch, + record: StateRecordFacts, + pointer: string, + target: Parameters[1], +): Observed | undefined => { + if ( + target.id !== undefined || + collectionPathOf(record, false) !== target.collection + ) + return undefined; + return search.withBody(record.ordinal, (body) => { + const items = body === undefined ? undefined : listItems(body, pointer); + const unheld = (items ?? []) + .map((object, index) => ({ + object, + index, + id: scalarText(object[target.idField]), + })) + .filter( + (entry): entry is { object: JsonObject; index: number; id: string } => + entry.id !== undefined && !target.held(entry.id), + ); + return unheld.length === 1 + ? { + ordinal: record.ordinal, + pointer: `${pointer}/${String(unheld[0]!.index)}`, + object: unheld[0]!.object, + id: unheld[0]!.id, + } + : undefined; + }); +}; diff --git a/src/domain/analysisView/environmentStateWriteRules.ts b/src/domain/analysisView/environmentStateWriteRules.ts new file mode 100644 index 0000000000..c228639663 --- /dev/null +++ b/src/domain/analysisView/environmentStateWriteRules.ts @@ -0,0 +1,199 @@ +import type { JsonValue } from "../jsonValue.js"; +import { + isParam, + normalizedPath, + segmentsOf, + type JsonObject, +} from "./environmentStateRules.js"; +import type { ClockFormat } from "./environmentStateWriteValues.js"; + +/** What a write tool does to the state, by method and the target its path names. */ +export type WriteEffect = "create" | "update" | "delete" | "document" | "none"; + +/** Why a write tool is not served. */ +export type WriteReason = + | "no-state-target" + | "no-call" + | "body-unusable" + | "no-observation" + | "unexplained-change" + | "contradicted" + | "side-effect" + | "unconfirmed"; + +/** Rules that generate a value when served; the gate takes the recorded value instead. */ +export type GeneratedRule = + | { + readonly field: string; + readonly rule: "identity"; + readonly generator: "sequence" | "uuid" | "token"; + } + | { + readonly field: string; + readonly rule: "sequence"; + readonly scope: "collection" | "type"; + } + | { readonly field: string; readonly rule: "unique" } + | { + readonly field: string; + readonly rule: "clock"; + readonly format: ClockFormat; + }; + +/** How one field of a written entity gets its value; fields no rule names keep the observed value. */ +export type FieldRule = + | GeneratedRule + | { readonly field: string; readonly rule: "echo" } + | { readonly field: string; readonly rule: "copy"; readonly from: string } + | { readonly field: string; readonly rule: "path"; readonly from: string } + | { readonly field: string; readonly rule: "slug"; readonly from: string } + | { readonly field: string; readonly rule: "template"; readonly from: string } + | { readonly field: string; readonly rule: "increment" } + | { readonly field: string; readonly rule: "constant" }; + +/** Where a create places its entity in its collection's order. */ +export type InsertRule = + | { readonly rule: "first" | "last" } + | { + readonly rule: "sorted"; + readonly field: string; + readonly direction: "ascending" | "descending"; + }; + +/** The body members a write accepts: echoed ones with any value, fixed ones only with the value they were sent. */ +export interface WriteBody { + readonly accepted: string[]; + readonly fixed: { readonly member: string; readonly sent: JsonValue }[]; + readonly required: string[]; + /** The member that may repeat the target's identity, which must then equal the path's. */ + readonly identity: string | null; +} + +/** What a write answers: the written entity or its identity rendered into a recorded body, no body, or the recorded body. */ +export interface WriteResponse { + readonly ordinal: number; + readonly shape: "entity" | "identity" | "empty" | "recorded"; + readonly pointer: string | null; +} + +/** One write tool's modelled effect and whether the capture confirmed it. */ +export interface StateWritePlan { + readonly tool: string; + readonly effect: WriteEffect; + /** The entity type for entity effects; the document tool for a document write. */ + readonly target: string | null; + readonly served: boolean; + readonly reason: WriteReason | null; + readonly exemplar_ordinal: number | null; + /** The record whose response showed the written entity or document, and where. */ + readonly observation: { + readonly ordinal: number; + readonly pointer: string; + } | null; + readonly response: WriteResponse | null; + readonly body: WriteBody | null; + readonly rules: FieldRule[]; + /** A create tied to the exemplar's collection writes only there. */ + readonly collection: string | null; + readonly insert: InsertRule | null; + readonly merge: "merge" | "replace" | null; + readonly checked_record_ordinals: number[]; + readonly contradicted_by: number | null; +} + +/** The tool facts classification reads. */ +interface WriteToolFacts { + readonly method: string; + readonly path_template: string; +} + +/** The effect a state-changing tool has and its target: a state type, or a document tool by template. */ +export const classifyWrite = ( + tool: WriteToolFacts, + types: ReadonlySet, + documents: ReadonlyMap, +): { readonly effect: WriteEffect; readonly target: string | null } => { + const segments = segmentsOf(tool.path_template); + const path = normalizedPath(tool.path_template); + const parent = `/${segments.slice(0, -1).join("/")}`; + const item = segments.length > 0 && isParam(segments.at(-1)!); + if (tool.method === "POST" && types.has(path)) + return { effect: "create", target: path }; + if (item && types.has(parent)) { + if (tool.method === "DELETE") return { effect: "delete", target: parent }; + if (["PUT", "PATCH", "POST"].includes(tool.method)) + return { effect: "update", target: parent }; + } + const document = documents.get(path); + if (document !== undefined && ["PUT", "PATCH", "POST"].includes(tool.method)) + return { effect: "document", target: document }; + return { effect: "none", target: null }; +}; + +/** A plan for a tool whose effect is not modelled or cannot be built. */ +export const unservedPlan = ( + tool: string, + effect: WriteEffect, + target: string | null, + reason: WriteReason, +): StateWritePlan => ({ + tool, + effect, + target, + served: false, + reason, + exemplar_ordinal: null, + observation: null, + response: null, + body: null, + rules: [], + collection: null, + insert: null, + merge: null, + checked_record_ordinals: [], + contradicted_by: null, +}); + +/** One write call: its concrete path, the values its path parameters took and its body. */ +export interface WriteCall { + readonly path: string; + readonly pathValues: ReadonlyMap; + readonly body: JsonObject | undefined; +} + +/** The value each template parameter takes in a concrete path, decoded; undefined when the path does not instantiate it. */ +export const pathValuesOf = ( + template: string, + path: string, +): Map | undefined => { + const pattern = segmentsOf(template); + const concrete = segmentsOf(path); + if (pattern.length !== concrete.length) return undefined; + const values = new Map(); + for (const [index, segment] of pattern.entries()) { + const value = concrete[index]!; + if (!isParam(segment)) { + if (segment !== value) return undefined; + continue; + } + let decoded = value; + try { + decoded = decodeURIComponent(value); + } catch { + decoded = value; + } + values.set(segment.slice(1, -1), decoded); + } + return values; +}; + +/** Whether two paths are segment prefixes of one another, a parameter on either side matching any segment. */ +export const prefixRelated = (left: string, right: string): boolean => { + const a = segmentsOf(left); + const b = segmentsOf(right); + const shared = Math.min(a.length, b.length); + for (let index = 0; index < shared; index += 1) + if (!isParam(a[index]!) && !isParam(b[index]!) && a[index] !== b[index]) + return false; + return true; +}; diff --git a/src/domain/analysisView/environmentStateWriteTaint.ts b/src/domain/analysisView/environmentStateWriteTaint.ts new file mode 100644 index 0000000000..644e31514c --- /dev/null +++ b/src/domain/analysisView/environmentStateWriteTaint.ts @@ -0,0 +1,103 @@ +import { + isParam, + lastSegmentValue, + normalizedPath, + segmentsOf, +} from "./environmentStateRules.js"; +import { + collectionTarget, + documentTarget, + entityTarget, +} from "./environmentStateWriteChecks.js"; +import { prefixRelated } from "./environmentStateWriteRules.js"; + +/** Whether the state holds an entity, so a write below its item path names it. */ +type Held = (type: string, collection: string, id: string) => boolean; + +/** + * What writes the gate does not model leave unknown: an entity below whose item path one wrote, + * or every entity of a collection when the path names none it holds (until a read shows each + * again); a collection one wrote at; everything under a path one wrote above collections; and + * targets whose check failed. + */ +export class WriteTaint { + readonly #prefixes: string[] = []; + readonly #targets = new Set(); + readonly #entities = new Set(); + /** Collections whose entities are all unknown, with the ones a read has shown since. */ + readonly #collections = new Map>(); + + constructor( + private readonly types: readonly string[], + private readonly documents: ReadonlyMap, + private readonly held: Held, + ) {} + + /** Record an unmodelled write's path against every related type and document. */ + write(path: string): void { + const segments = segmentsOf(path); + for (const type of this.types) { + if (!prefixRelated(type, path)) continue; + const depth = segmentsOf(type).length; + const collection = `/${segments.slice(0, depth).join("/")}`; + if (segments.length > depth) { + const id = lastSegmentValue(`/${segments[depth]!}`); + if (this.held(type, collection, id)) + this.#entities.add(entityTarget(type, collection, id)); + else + this.#collections.set(collectionTarget(type, collection), new Set()); + } else if (segments.length === depth) + this.#targets.add(collectionTarget(type, collection)); + else this.#prefixes.push(normalizedPath(path)); + } + for (const [tool, documentPath] of this.documents) + if (prefixRelated(documentPath, path)) + this.#targets.add(documentTarget(tool)); + } + + /** Stop checking a target whose check failed. */ + fail(target: string): void { + this.#targets.add(target); + } + + tainted(target: string): boolean { + return this.#targets.has(target); + } + + /** Whether an unmodelled write above a collection path left everything under it unknown. */ + under(collection: string): boolean { + const concrete = segmentsOf(collection); + return this.#prefixes.some((prefix) => + segmentsOf(prefix).every( + (segment, index) => isParam(segment) || segment === concrete[index], + ), + ); + } + + unknown(type: string, collection: string, id: string): boolean { + const target = entityTarget(type, collection, id); + const seen = this.#collections.get(collectionTarget(type, collection)); + return ( + this.#entities.has(target) || (seen !== undefined && !seen.has(target)) + ); + } + + /** Whether a list read of a collection lists, or holds, an entity left unknown. */ + blocksList( + type: string, + collection: string, + ids: readonly string[], + ): boolean { + return ( + this.#collections.has(collectionTarget(type, collection)) || + ids.some((id) => this.#entities.has(entityTarget(type, collection, id))) + ); + } + + /** The state saw the entity again, or a modelled write replaced it. */ + known(type: string, collection: string, id: string): void { + const target = entityTarget(type, collection, id); + this.#entities.delete(target); + this.#collections.get(collectionTarget(type, collection))?.add(target); + } +} diff --git a/src/domain/analysisView/environmentStateWriteValues.ts b/src/domain/analysisView/environmentStateWriteValues.ts new file mode 100644 index 0000000000..91cda985e4 --- /dev/null +++ b/src/domain/analysisView/environmentStateWriteValues.ts @@ -0,0 +1,226 @@ +import { createHash } from "node:crypto"; +import type { JsonValue } from "../jsonValue.js"; + +/** How a recorded clock value is spelled: ISO-8601 with its separator, fraction digits and zone, or an epoch number. */ +export type ClockFormat = + | { + readonly kind: "iso"; + readonly separator: "T" | " "; + readonly fraction: number; + readonly zone: string; + } + | { readonly kind: "epoch"; readonly unit: "s" | "ms" }; + +const ISO_INSTANT = + /^(\d{4})-(\d{2})-(\d{2})([T ])(\d{2}):(\d{2}):(\d{2})(?:\.(\d{1,9}))?(Z|[+-]\d{2}:?\d{2})?$/u; +const MINUTE_MS = 60_000; + +/** Minutes east of UTC that a zone suffix names; none and `Z` are UTC. */ +const zoneMinutes = (zone: string): number => { + if (zone === "" || zone === "Z") return 0; + const digits = zone.slice(1).replace(":", ""); + const minutes = + Number(digits.slice(0, 2)) * 60 + Number(digits.slice(2, 4) || "0"); + return zone.startsWith("-") ? -minutes : minutes; +}; + +/** The instant of an ISO-8601 date-time and how it is spelled, or undefined when it is not one. */ +const readIsoClock = ( + value: string, +): { readonly ms: number; readonly format: ClockFormat } | undefined => { + const match = ISO_INSTANT.exec(value); + if (match === null) return undefined; + const [year, month, day, separator, hour, minute, second] = match + .slice(1, 8) + .map((part) => (part === "T" || part === " " ? part : Number(part))); + const fraction = match[8] ?? ""; + const zone = match[9] ?? ""; + const local = Date.UTC( + year as number, + (month as number) - 1, + day as number, + hour as number, + minute as number, + second as number, + Number(`${fraction}000`.slice(0, 3)), + ); + // Date.UTC rolls invalid fields over; the round trip refuses them. + const rolled = new Date(local); + if ( + rolled.getUTCFullYear() !== year || + rolled.getUTCMonth() !== (month as number) - 1 || + rolled.getUTCDate() !== day || + rolled.getUTCHours() !== hour || + rolled.getUTCMinutes() !== minute + ) + return undefined; + return { + ms: local - zoneMinutes(zone) * MINUTE_MS, + format: { + kind: "iso", + separator: separator as "T" | " ", + fraction: fraction.length, + zone, + }, + }; +}; + +/** The instant a clock value names and its format: an ISO-8601 date-time, or epoch seconds or milliseconds. */ +export const readClock = ( + value: JsonValue | undefined, +): { readonly ms: number; readonly format: ClockFormat } | undefined => { + if (typeof value === "string") return readIsoClock(value); + if (typeof value !== "number" || !Number.isSafeInteger(value)) + return undefined; + if (value >= 1e9 && value < 1e10) + return { ms: value * 1000, format: { kind: "epoch", unit: "s" } }; + if (value >= 1e12 && value < 1e13) + return { ms: value, format: { kind: "epoch", unit: "ms" } }; + return undefined; +}; + +const pad = (value: number, width: number): string => + String(value).padStart(width, "0"); + +/** An instant spelled in a recorded clock format; fraction digits past milliseconds are zeros. */ +export const formatClock = (ms: number, format: ClockFormat): JsonValue => { + if (format.kind === "epoch") + return format.unit === "s" ? Math.floor(ms / 1000) : ms; + const at = new Date(ms + zoneMinutes(format.zone) * MINUTE_MS); + const fraction = + format.fraction === 0 + ? "" + : `.${`${pad(at.getUTCMilliseconds(), 3)}${"0".repeat(9)}`.slice(0, format.fraction)}`; + return `${pad(at.getUTCFullYear(), 4)}-${pad(at.getUTCMonth() + 1, 2)}-${pad(at.getUTCDate(), 2)}${format.separator}${pad(at.getUTCHours(), 2)}:${pad(at.getUTCMinutes(), 2)}:${pad(at.getUTCSeconds(), 2)}${fraction}${format.zone}`; +}; + +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/iu; + +export const isUuid = (value: JsonValue | undefined): value is string => + typeof value === "string" && UUID.test(value); + +/** An integer value as a number, from a JSON number or a string of digits. */ +export const integerOf = (value: JsonValue | undefined): number | undefined => + typeof value === "number" && Number.isSafeInteger(value) + ? value + : typeof value === "string" && /^\d{1,15}$/u.test(value) + ? Number(value) + : undefined; + +/** An integer spelled as the recorded value spells integers: a number, or a string of digits. */ +export const spelledLike = (exemplar: JsonValue, value: number): JsonValue => + typeof exemplar === "string" ? String(value) : value; + +/** Lower case, with every run of other characters joined by one `-` and none at either end. */ +export const slugOf = (text: string): string => + text + .normalize("NFKD") + .replace(/[̀-ͯ]/gu, "") + .toLowerCase() + .replace(/[^a-z0-9]+/gu, "-") + .replace(/^-+|-+$/gu, ""); + +const isAlphanumeric = (character: string | undefined): boolean => + character !== undefined && /[\p{L}\p{N}]/u.test(character); + +/** Start offsets of `token` in `text` where neither neighbour is a letter or digit. */ +export const delimitedOffsets = (text: string, token: string): number[] => { + const offsets: number[] = []; + if (token.length === 0) return offsets; + for ( + let at = text.indexOf(token); + at >= 0; + at = text.indexOf(token, at + token.length) + ) + if ( + !isAlphanumeric(text[at - 1]) && + !isAlphanumeric(text[at + token.length]) + ) + offsets.push(at); + return offsets; +}; + +/** `text` with every delimited occurrence of `token` replaced. */ +export const replaceDelimited = ( + text: string, + token: string, + replacement: string, +): string => { + let result = ""; + let from = 0; + for (const at of delimitedOffsets(text, token)) { + result += text.slice(from, at) + replacement; + from = at + token.length; + } + return result + text.slice(from); +}; + +/** Whether any string inside a value holds `token` delimited. */ +export const containsDelimited = (value: JsonValue, token: string): boolean => { + const pending: JsonValue[] = [value]; + for (let next = pending.pop(); next !== undefined; next = pending.pop()) { + if (typeof next === "string") { + if (delimitedOffsets(next, token).length > 0) return true; + } else if (Array.isArray(next)) pending.push(...next); + else if (typeof next === "object" && next !== null) + pending.push(...Object.values(next)); + } + return false; +}; + +/** SHA-256 bytes of a seed, extended by counter blocks to the requested length. */ +const seededBytes = (seed: string, length: number): Uint8Array => { + const bytes = new Uint8Array(length); + for (let block = 0; block * 32 < length; block += 1) { + const digest = createHash("sha256") + .update(`${seed}#${String(block)}`) + .digest(); + bytes.set( + digest.subarray(0, Math.min(32, length - block * 32)), + block * 32, + ); + } + return bytes; +}; + +/** A version 4 UUID derived from a seed, in the recorded value's letter case. */ +export const seededUuid = (seed: string, exemplar: string): string => { + const bytes = seededBytes(seed, 16); + bytes[6] = (bytes[6]! & 0x0f) | 0x40; + bytes[8] = (bytes[8]! & 0x3f) | 0x80; + const hex = Buffer.from(bytes).toString("hex"); + const uuid = `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`; + return exemplar === exemplar.toUpperCase() && /[A-F]/u.test(exemplar) + ? uuid.toUpperCase() + : uuid; +}; + +const DIGITS = "0123456789"; +const LOWER = "abcdefghijklmnopqrstuvwxyz"; +const UPPER = LOWER.toUpperCase(); + +/** The characters a token of this spelling draws from: hexadecimal, or the digit and letter cases it uses. */ +const tokenAlphabet = (exemplar: string): string => { + if (/^[0-9a-f]+$/u.test(exemplar) && /[0-9]/u.test(exemplar)) + return `${DIGITS}abcdef`; + if (/^[0-9A-F]+$/u.test(exemplar) && /[0-9]/u.test(exemplar)) + return `${DIGITS}ABCDEF`; + const alphabet = + (/[0-9]/u.test(exemplar) ? DIGITS : "") + + (/[a-z]/u.test(exemplar) ? LOWER : "") + + (/[A-Z]/u.test(exemplar) ? UPPER : ""); + return alphabet === "" ? LOWER : alphabet; +}; + +/** A token as long as the recorded one, from its alphabet, keeping its other characters in place. */ +export const seededToken = (seed: string, exemplar: string): string => { + const alphabet = tokenAlphabet(exemplar); + const bytes = seededBytes(seed, exemplar.length); + return [...exemplar] + .map((character, index) => + /[0-9A-Za-z]/u.test(character) + ? alphabet[bytes[index]! % alphabet.length]! + : character, + ) + .join(""); +}; diff --git a/src/domain/analysisView/environmentStateWrites.ts b/src/domain/analysisView/environmentStateWrites.ts new file mode 100644 index 0000000000..df01c43cf4 --- /dev/null +++ b/src/domain/analysisView/environmentStateWrites.ts @@ -0,0 +1,38 @@ +import type { ExtractedState } from "./environmentStateRules.js"; +import { + EnvironmentStateWriteSimulation, + type WriteSimulationInput, +} from "./EnvironmentStateWriteSimulation.js"; + +/** The instant a served clock starts from: the end of the last timed record, to the next whole second. */ +const clockStart = ( + records: WriteSimulationInput["records"], + timed: boolean, +): string | null => { + if (!timed || records.length === 0) return null; + const end = records.reduce( + (latest, record) => Math.max(latest, record.end), + Number.NEGATIVE_INFINITY, + ); + return Number.isFinite(end) + ? new Date(Math.ceil(end / 1000) * 1000).toISOString() + : null; +}; + +/** The write plans a capture's own timeline confirms against an extracted state, and the served clock's start. */ +export const extractWrites = ( + state: Pick, + input: Omit< + WriteSimulationInput, + "views" | "types" | "entities" | "documents" + >, +): Pick => ({ + writes: new EnvironmentStateWriteSimulation({ + ...input, + views: new Map(state.views.map((view) => [view.tool, view])), + types: state.types, + entities: state.entities, + documents: state.documents, + }).run(), + write_clock: clockStart(input.records, input.timed), +}); diff --git a/src/server/environment/serveEnvironment.ts b/src/server/environment/serveEnvironment.ts index 46118e9eaf..48e7d4a5ef 100644 --- a/src/server/environment/serveEnvironment.ts +++ b/src/server/environment/serveEnvironment.ts @@ -16,7 +16,7 @@ import { import { environmentToolAnswer, type EnvironmentToolAnswer, -} from "../../application/environment/environmentStateAnswer.js"; +} from "../../application/environment/environmentToolAnswer.js"; import { EnvironmentStateStore } from "../../application/environment/environmentStateStore.js"; import { environmentCard, @@ -334,7 +334,7 @@ export const serveEnvironment = async ( options.state === undefined ? undefined : new EnvironmentStateStore(options.state); - const catalog = servedCatalog(source, store !== undefined, name); + const catalog = servedCatalog(source, options.state, name); if (!catalog.ok) return catalog; const { tools, dataPlane } = catalog.value; const card = environmentCard(name, tools, dataPlane);