diff --git a/docs/agent-environments.md b/docs/agent-environments.md index 359fa0a4c7..4112bd87d5 100644 --- a/docs/agent-environments.md +++ b/docs/agent-environments.md @@ -424,7 +424,7 @@ Each read tool answers according to its view in the state: | View | Answer | | ---------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `item` | The entity the path arguments name, rendered into the recorded envelope of `envelope_ordinal`. Arguments match as replay matches them: each collection segment equals the held collection's segment as written or after percent-decoding, and the identity is the last argument percent-decoded, then as written; values are never re-split on `/`. Any entity the state holds answers, including one `data/add` loaded; query arguments only decide `replay.exact_match`. An entity the state does not hold answers the tool's latest recorded 400, 404 or 410 response with `state.view` `missing`, or is a tool error when none was recorded; one lacking a field every recording carried is a tool error; a body is never invented. | +| `item` | The entity the path arguments name, rendered into the recorded envelope of `envelope_ordinal`. Arguments match as replay matches them: each collection segment equals the held collection's segment as written or after percent-decoding, and the identity is the last argument percent-decoded, then as written; values are never re-split on `/`. Any entity the state holds answers, including one `data/add` loaded; query arguments only decide `replay.exact_match`. An entity the state does not hold answers the tool's latest recorded 400, 404 or 410 response with `state.view` `missing`, or is a tool error when none was recorded; one lacking a field every recording carried is a tool error, unless a read showed the application leaves that field out; a body is never invented. | | `list` | When the arguments name one of the lines the view checked exactly (its latest recordings before the first related write; every path and query value equal to the recording's, and every recorded query parameter supplied), that recorded envelope with its array rendered from the state: every entity of the collection in state order for `membership: "all"` or a line `all_record_ordinals` names, or the recorded members still held. A call with no query argument answers from such a line of the collection its path arguments name that only pages its list, with `replay.exact_match` false. On a line the view's `query` answers, and for a call whose every query argument has a role there (an argument-less call aside), the answer is the query evaluated over the state: filtered, searched, sorted and paged, with counters at the number listed and `query.totals` at the number matched, and `state.membership: "query"`; such a call answers on a query or whole line of its collection with `replay.exact_match` false. A value an `inert` parameter never recorded, or a parameter without a role, replays its recording. A member lacking a field every recording carried makes the call a tool error rather than a partial list. | | `document` | When the arguments name the document's checked line exactly, the state's value for it; a tool error when the state holds none. | | `recorded` | The recording, as replay answers it. | @@ -433,7 +433,7 @@ A list or document call that names no checked line, including a line recorded after a write or with other query values, answers from the recording. Every answer has the replay shape plus `state`: `{"source": "state", "view": …}` naming the entity, collection or document path, or `{"source": "recording", -"reason": …}` with the view's reason, `no-view`, `unrecorded-request` or `page-exceeded` (a call naming a page of a whole list that the collection has outgrown). A +"reason": …}` with the view's reason, `no-view`, `unrecorded-request`, `page-exceeded` (a call naming a page of a whole list that the collection has outgrown) or `projection` (a call to an item tool whose recordings asked for some fields only, naming such a parameter for an entity the state holds; the answer is a recording, so it does not show served writes). A state answer carries the base record's status, content type and `replay` provenance with `truncated: false`; one whose JSON would exceed 256 KiB is refused as a `resource_constraint` tool error rather than cut. Extraction does diff --git a/docs/specs/agent-environments/manifest-view/spec.md b/docs/specs/agent-environments/manifest-view/spec.md index 873685bb45..fc20092629 100644 --- a/docs/specs/agent-environments/manifest-view/spec.md +++ b/docs/specs/agent-environments/manifest-view/spec.md @@ -37,7 +37,11 @@ manifest are later increments. value anywhere in the result. - List records as pointers (`ordinal`, `method`, `path`, `status`, `content_type`) and excluded groups with reason `other-origin`, - `ui-page-or-asset` or `non-json`. + `ui-page-or-asset` or `non-json`. A write group (`POST`, `PUT`, `PATCH`, + `DELETE`) whose every response had no body (a 204 or 205, an empty text or a + recorded size of zero) and at least one succeeded is kept whatever its content + type: the empty reply is its answer. A missing body text alone does not count, + as it may be a body the capture did not keep. - Mark the Evidence inferred whenever any tool or excluded group is templated, derived otherwise; keep the parent's authority. - Admit the capture clone, the inline envelope, every retained pointer, group, diff --git a/docs/specs/agent-environments/state-view/spec.md b/docs/specs/agent-environments/state-view/spec.md index 18c0e46f74..24b9198203 100644 --- a/docs/specs/agent-environments/state-view/spec.md +++ b/docs/specs/agent-environments/state-view/spec.md @@ -29,6 +29,17 @@ later increments. - Prove an item type's identity field from the requested path value its responses repeat; otherwise infer it from a fixed list of identity names that every listed item carries distinctly, which needs at least one listed item. + An item read that asked for some fields only (`fields`, `fields[...]`, + `select`, `$select`) and does not return the item is left out of the proof + and of the view's checks; the view lists it in `projected_record_ordinals`, + and serving replays any call to that tool that names a projection parameter + for an entity the state holds. The write gate also leaves out any read that + names one and leaves out a field its view requires. +- A field every recording of a view carried is required: serving refuses an + entity lacking it. Once a successful read the write gate checks, naming no + projection parameter, shows the application leaves the field out of an entity + the state holds (APIs that omit empty or false members), no view of the type + requires it. Never treat an envelope that is itself an entity as a list. - Merge observations into one entity per collection and identity: item views' latest values win over list views', differing views keep their own values, diff --git a/docs/specs/agent-environments/state-writes/spec.md b/docs/specs/agent-environments/state-writes/spec.md index 68e37dda6d..d1b2b5411f 100644 --- a/docs/specs/agent-environments/state-writes/spec.md +++ b/docs/specs/agent-environments/state-writes/spec.md @@ -81,7 +81,11 @@ identity the client sends is an echo: format is kept: ISO-8601 with its separator, fraction digits and zone, or epoch seconds or milliseconds. 7. **Sequence.** An integer one greater than the largest value of the field in - the collection, else in the type. + the collection, else in the type. The integer may be written as `-`-separated + digit groups of one width of at least two digits (an asset number `000-005`; + not a date, version or address); a served value keeps the recorded value's + groups and zero padding, and digits that outgrow them lead. A later call must + record the value spelled that way. 8. **Unique.** A UUID-shaped string that no other entity of the type shares. 9. **Template.** A string containing a generated value as a delimited token (for example a URL that ends in the new id). It is rebuilt around the new diff --git a/docs/specs/agent-environments/write-coverage/plan.md b/docs/specs/agent-environments/write-coverage/plan.md new file mode 100644 index 0000000000..d18be7a89a --- /dev/null +++ b/docs/specs/agent-environments/write-coverage/plan.md @@ -0,0 +1,12 @@ +# Implementation plan + +1. **Bodiless writes.** The manifest accumulator tracks, per write group, whether every response was empty, and keeps + the group when one succeeded. +2. **Omitted fields.** The write simulation records required fields its successful reads show missing, and the + extracted state's views drop them. +3. **Projected reads.** Item planning sets aside unlocated projected reads and the view records their ordinals; the gate + skips them, and serving replays projected calls to such a view. +4. **Sequences.** A values helper reads a digit-group value as an integer and fills the next one into its mask; the + write rules, the gate and serving number sequences through it. +5. **Schema, tests and docs.** The view schema, manifest, gate, serving and unit tests, and the manifest, capture, + state-view and state-writes docs. diff --git a/docs/specs/agent-environments/write-coverage/spec.md b/docs/specs/agent-environments/write-coverage/spec.md new file mode 100644 index 0000000000..cfcb5094e7 --- /dev/null +++ b/docs/specs/agent-environments/write-coverage/spec.md @@ -0,0 +1,49 @@ +# Wider write coverage + +Issue: https://github.com/morluto/rea/issues/4933 + +## Outcome + +Four recording shapes kept write tools out of the manifest or left their writes unserved, although the recordings showed +what the writes did: + +- **Bodiless replies.** A write answered `204` with no body (Jellyfin's and Shlink's deletes, Miniflux's and Keycloak's + updates) had no JSON-like content type, so its group was excluded as `non-json` and never became a tool. +- **Omitted fields.** A field every recording of a view carried was required, so an application that leaves empty or + false members out of some entities (Go's `omitempty`) made the write gate refuse its own read-backs. +- **Projected reads.** An item read asking for some fields only (Directus's `/items/provisions/2?fields=title`) does not + repeat the requested id, so it blocked the identity proof of the whole item view, and the view's writes had no state + target. +- **Digit-group sequences.** A sequence spelled in digit groups (Homebox's asset number `000-005`) was not an integer, so + a create that numbers it had no rule for it and was contradicted. + +## Behaviour + +- **Manifest.** A write group (`POST`, `PUT`, `PATCH`, `DELETE`) is kept as a tool when every response had no body (a + `204` or `205`, an empty text, or a recorded size of zero) and one succeeded. A missing text alone does not count. +- **Omitted fields.** The write gate notes required fields missing from the entities the state holds that successful + reads it checks return, when the read names no projection parameter: the item whose id the path names, or the listed + items. No view of the type requires them any more. A read that checks nothing, such as one begun while a modelled + write to its targets still ran or a page the collection has outgrown, notes none. + Established omissions apply when classifying subsequent reads and choosing later write observations, not just + when serializing final views. A complete `fields=*` read is not partial solely for lacking a known optional field. +- **Projected reads.** An item read naming `fields`, `fields[...]`, `select` or `$select` that does not return the item + is left out of the identity proof and the view's checks. The view lists it in `projected_record_ordinals`, and serving + replays a call to that tool naming such a parameter, for an entity the state holds, with the reason `projection`; + the replay shows the recording, not served writes. The write gate neither checks nor learns a write's effect from a + successful read that names such a parameter and leaves out a field its view requires; a 400, 404 or 410 such a read + answers is still the absence it checks. + Identity-free projection evidence after the initial-state cutoff is retained too, without seeding later values + into initial entities. Failed responses are classified before the projection-replay shortcut, so a projected + absence can confirm a delete or contradict an unexplained disappearance. +- **Sequences.** A sequence value may be `-`-separated digit groups of one width of at least two digits, not a date, + version or address. The next value keeps the exemplar's groups and zero padding; digits that outgrow them lead. A + later create must record its value spelled that way. A held value whose digits outgrew the mask (`100-00` after + `99-99`) counts only when it is the exemplar's spelling of its number, so a reloaded state numbers on from it. + +## Acceptance + +- Over the 50 published captures, no write group is excluded as `non-json`, served write tools rise from 35 to 39, and + no write tool is served on the base and unserved here. +- The re-recorded Homebox capture's create is served. +- Every served write tool of those captures applies. diff --git a/docs/specs/agent-environments/write-coverage/tasks.md b/docs/specs/agent-environments/write-coverage/tasks.md new file mode 100644 index 0000000000..1df7c84182 --- /dev/null +++ b/docs/specs/agent-environments/write-coverage/tasks.md @@ -0,0 +1,10 @@ +# Tasks + +- [x] Keep write groups answered with no body as tools. +- [x] Stop requiring fields a successful read shows the application leaves out. +- [x] Set projected item reads aside from the identity proof and replay projected calls. +- [x] Number sequences spelled in digit groups. +- [x] Add manifest, gate, serving and unit tests, failing on the previous head. +- [x] Document the manifest rule, the view member, the replay reason and the sequence spelling. +- [x] Measure the 50 published captures, the re-recorded Homebox capture and the served smoke. +- [x] Run static, formatting, duplication, documentation and test gates. diff --git a/docs/specs/agent-environments/write-coverage/verification.md b/docs/specs/agent-environments/write-coverage/verification.md new file mode 100644 index 0000000000..f80493df64 --- /dev/null +++ b/docs/specs/agent-environments/write-coverage/verification.md @@ -0,0 +1,74 @@ +# Verification + +Executed on macOS with Node v22.23.1 on this branch, whose base is the write-plan branch's head `ad8f674d`. + +## Gates + +- `npm run build:cached`, `npm run check:fast` (typecheck and lint, no new warnings in the changed files), + `npm run format:check`, `npx knip`, `npm run scan:todos`, `npm run docs:check`, `npm run verify:test-discovery` and + `npm run mcp-catalog:check`: passed. +- `npm run jscpd`: 3.45% duplicated lines over 3776 files. +- `npx vitest run --project domain --project services`: 7808 passed, 5 pre-existing skips. Two unrelated heavy tests + (keyed-archive traversal, a 125,000-dependent Evidence graph) passed only with a longer timeout on a loaded machine. +- `npm run test:prepare`, then `npm run test:focused --` the environment serve and export CLI boundaries, the serve + and served-writes MCP boundaries, the analysis-view CLI and MCP boundaries, the compact input schema boundary and + tool schema validity: 8 files, 79 passed; the analysis-view MCP file needed a longer timeout on the loaded machine. + +## Tests + +- Manifest: a `204` write with no body is kept as a tool whatever its content type, and a write only redirected is not. +- Gate: + - a field a read-back leaves out stops being required, and a create lacking it is served only when its own read + leaves it out too; + - an item view proves its identity past a read that asked for some fields only, which is left to replay; + - creates numbered in digit groups are served. +- Serving: a created entity is served without the omitted field while seeded ones keep it; a call naming `fields` + replays with the reason `projection` while other calls answer from the state; creates take the next number in the + held value's digit groups. +- Unit: `-`-separated digit groups of one width read as a sequence, and a date, version, address or decimal does not; + the next value keeps the groups, overflow leading. +- The gate, serving and manifest tests fail on `ad8f674d`. + +Review fixes, each with a test that fails on `a4c4a23b`: + +- Gate: + - a read naming `fields`, or of a book the state no longer holds, notes no omission, and a create read back only + through `?fields=id,name` is not served; + - an omission an item read shows stops every view of the type requiring the field; + - a read naming `fields` that leaves a required field out neither resynchronises an entity nor serves as a write's + observation, so a `204` update read back first through `?fields=id,name` learns its effect from the whole read; + - a later create whose sequence value serving would spell otherwise (`00-04` or `4` after `000-003`) withholds the + plan. +- Serving: an empty projection argument replays; a projection of a deleted book answers as missing; a created book + without the omitted field lists. + +## Published captures + +Over the 50 published captures, against `ad8f674d`: + +- Write groups excluded as `non-json` go from 13 in five captures to none. Directus, Jellyfin, Keycloak, Miniflux and + Shlink gain those tools, so their manifests change. +- Served write tools go from 35 to 39; none is lost: + - Directus's provisions item view proves its identity past `GET /items/provisions/2?fields=title`, so its create and + update are served; + - Jellyfin's `DELETE /Items/{id}` and Shlink's `DELETE /rest/v3/short-urls/{slug}` are served. +- Airbyte's job item view stops requiring two fields its reads leave out; its verdicts are unchanged. +- Six states change: those five captures and Airbyte. +- A served smoke with manifests rebuilt by each tree applies all 39 served write tools here and all 35 on the base. + +## Re-recorded captures + +Captures re-recorded with a read-back after each first write and two scratch objects per write tool: + +- A Homebox pilot recording numbers assets in digit groups (`000-001`, `000-002`, …). Its create goes from + `contradicted` to served with a `sequence` rule on `assetId`, and the smoke applies it and the delete. +- Five published environments rebuilt from such recordings (Navidrome, Lychee, Homebox, Directus, Linkding) go from 9 + served write tools to 11, and the smoke applies all 11: + - Navidrome's playlist views stop requiring `imageAbsent`, which its new playlists leave out, so its playlist update is + served; + - Homebox's create is served, as in the pilot recording. +- Jellyfin and Memos, re-recorded and rebuilt end to end with this change (manifest, bundle, smoke and both evaluation + models on 12 retrieval tasks each), serve no new write: + - Jellyfin's four bodiless write groups become tools, all withheld. Its playlists are created through + `POST /Playlists` but read back only through `/Items/{id}`, which stops that item view rendering from the state. + - Memos has none of the four shapes; it names items `memos/`, which the identity proof does not match. diff --git a/docs/web-network-captures.md b/docs/web-network-captures.md index 305bdff1c5..a998083c8c 100644 --- a/docs/web-network-captures.md +++ b/docs/web-network-captures.md @@ -410,7 +410,8 @@ agent environment later serves from retained Evidence. The view reuses the api-surface grouping, then keeps the groups on the primary origin, the origin with the most JSON-like records (`application/json`, `application/*+json`, `text/plain`, `text/markdown`), whose path starts with `/api/` or `/apis/` or -whose content types are JSON-like. Every other group is listed under +whose content types are JSON-like, and the write groups answered with no body +(a bodiless `204 DELETE`). Every other group is listed under `excluded` with reason `other-origin`, `ui-page-or-asset` or `non-json`, and retained records without a usable request line are counted in a limitation. diff --git a/src/application/environment/environmentStateAnswer.ts b/src/application/environment/environmentStateAnswer.ts index 134ae8b204..408db4ff0a 100644 --- a/src/application/environment/environmentStateAnswer.ts +++ b/src/application/environment/environmentStateAnswer.ts @@ -1,7 +1,9 @@ +import { environmentToolArguments } from "./environmentTools.js"; import { stateBodyJson } from "../../domain/analysisView/EnvironmentStateAccumulator.js"; import { collectionMatch, collectionPathOf, + isProjectionParameter, decodedSegment, allLines, listItems, @@ -63,7 +65,8 @@ export type StateProvenance = | RecordedViewReason | "no-view" | "unrecorded-request" - | "page-exceeded"; + | "page-exceeded" + | "projection"; } | { readonly source: "state"; @@ -144,6 +147,18 @@ const fromRecording = ( : replayed; }; +/** Whether a call names a projection parameter of an item view with projected reads, which the state's whole entities cannot answer. */ +const asksProjection = ( + view: StateItemView, + tool: EnvironmentManifestTool, + args: ReplayArguments, +): boolean => + view.projected_record_ordinals !== undefined && + [...environmentToolArguments(tool).query].some( + ([name, argument]) => + isProjectionParameter(name) && args[argument] !== undefined, + ); + /** A recorded body the state renders into, usable exactly when extraction could use it. */ export const recordedJson = ( exchange: RecordedExchange, @@ -291,6 +306,9 @@ const answerItem = ( collection: `/${call.segments.map(({ text }) => text).join("/")}`, id: call.id, }); + // A projection of an entity the state holds replays as the capture's projected reads did. + if (asksProjection(view, tool, args)) + return fromRecording(source, tool, args, "projection"); const shown = shownEntity( tool, view, diff --git a/src/application/environment/environmentStateCoverage.test.ts b/src/application/environment/environmentStateCoverage.test.ts new file mode 100644 index 0000000000..ea5240c5ce --- /dev/null +++ b/src/application/environment/environmentStateCoverage.test.ts @@ -0,0 +1,127 @@ +import { describe, expect, it } from "vitest"; +import { servedFrom } from "../../../tests/fixtures/environmentStateServing.js"; +import type { TimedCaptureRow } from "../../../tests/fixtures/analysisView.js"; + +const first = { id: 1, name: "First", detail: "seeded" }; +const second = { id: 2, name: "Second", detail: "seeded" }; +const initial: readonly TimedCaptureRow[] = [ + ["GET", "/api/books", [first, second]], + ["GET", "/api/books/1", first], + ["GET", "/api/books/2", second], +]; + +describe("projection and omission coverage across writes", () => { + it("records a late identity-free projection without seeding its values into initial state", () => { + const renamed = { ...first, name: "Renamed" }; + const books = servedFrom([ + ...initial, + ["PATCH", "/api/books/1", renamed, 200, { name: "Renamed" }], + ["GET", "/api/books/1", renamed], + ["GET", "/api/books/1?fields=name", { name: "Renamed" }], + ]); + expect(books.store.state.views.get("get_books_by_id")).toMatchObject({ + shape: "item", + checked_record_ordinals: [1, 2], + projected_record_ordinals: [5], + }); + expect( + books.store + .snapshot() + .rea_state.entities.find((entity) => entity.id === "1")?.fields.name, + ).toBe("First"); + books.answer("patch_books_by_id", { id: "1", body: { name: "Changed" } }); + expect( + books.answer("get_books_by_id", { id: "1", fields: "name" }), + ).toMatchObject({ + body: { name: "Renamed" }, + replay: { record_ordinal: 5 }, + state: { source: "recording", reason: "projection" }, + }); + }); + + it("uses established omissions when a full projection is the only update observation", () => { + const third = { id: 3, name: "Third" }; + const books = servedFrom([ + ...initial, + ["POST", "/api/books", third, 201, { name: "Third" }], + ["GET", "/api/books/3", third], + [ + "PATCH", + "/api/books/3", + { ...third, name: "Renamed" }, + 200, + { name: "Renamed" }, + ], + ["GET", "/api/books/3?fields=*", { ...third, name: "Renamed" }], + ]); + expect(books.store.state.writes.get("patch_books_by_id")).toMatchObject({ + served: true, + checked_record_ordinals: [6], + }); + expect(books.store.state.views.get("get_books_by_id")).not.toMatchObject({ + required_fields: expect.arrayContaining(["detail"]), + }); + books.answer("post_books", { body: { name: "New" } }); + books.answer("patch_books_by_id", { id: "3", body: { name: "Changed" } }); + expect( + books.answer("get_books_by_id", { id: "3", fields: "*" }), + ).toMatchObject({ + body: { id: 3, name: "Changed" }, + state: { source: "state" }, + }); + }); + + it("uses established omissions when observing a later bodiless update through a full projection", () => { + const third = { id: 3, name: "Third" }; + const renamed = { ...third, name: "Renamed" }; + const again = { ...third, name: "Again" }; + const books = servedFrom([ + ...initial, + ["POST", "/api/books", third, 201, { name: "Third" }], + ["GET", "/api/books/3", third], + ["PATCH", "/api/books/3", renamed, 200, { name: "Renamed" }], + ["GET", "/api/books/3", renamed], + ["PATCH", "/api/books/3", "", 204, { name: "Again" }], + ["GET", "/api/books/3?fields=*", again], + ["GET", "/api/books", [first, second, again]], + ]); + expect(books.store.state.writes.get("patch_books_by_id")).toMatchObject({ + served: true, + checked_record_ordinals: [6, 9], + }); + }); + + it("does not hide an absence contradicting a confirmed update behind a projection", () => { + const renamed = { ...first, name: "Renamed" }; + const books = servedFrom([ + ...initial, + ["GET", "/api/books/2?fields=name", { name: "Second" }], + ["PATCH", "/api/books/1", renamed, 200, { name: "Renamed" }], + ["GET", "/api/books/1", renamed], + [ + "PATCH", + "/api/books/1", + { ...first, name: "Again" }, + 200, + { name: "Again" }, + ], + ["GET", "/api/books/1?fields=name", { detail: "Not found." }, 404], + ]); + expect(books.store.state.writes.get("patch_books_by_id")).toMatchObject({ + served: false, + }); + }); + + it("confirms a delete through a projected absence without needing a body", () => { + const books = servedFrom([ + ...initial, + ["GET", "/api/books/2?fields=name", { name: "Second" }], + ["DELETE", "/api/books/1", "", 204], + ["GET", "/api/books/1?fields=name", { detail: "Not found." }, 404], + ]); + expect(books.store.state.writes.get("delete_books_by_id")).toMatchObject({ + served: true, + checked_record_ordinals: [5], + }); + }); +}); diff --git a/src/application/environment/environmentStateWriteAnswer.test.ts b/src/application/environment/environmentStateWriteAnswer.test.ts index e12b2f49ee..a04f134da0 100644 --- a/src/application/environment/environmentStateWriteAnswer.test.ts +++ b/src/application/environment/environmentStateWriteAnswer.test.ts @@ -5,17 +5,23 @@ import { type Served, } from "../../../tests/fixtures/environmentStateServing.js"; import { + ASSET_ROWS, BOOK_INIT_ROWS, BOOK_NOT_FOUND, BOOK_ROWS, DELETE_NOT_FOUND, + DETAILED_LIST_ROWS, FIRST_BOOK, + OMITTED_FIELD_ROWS, + PROJECTED_ROWS, LIST_LINE_ROWS, QUERY_ROWS, SECOND_BOOK, THIRD_BOOK, + assetBook, book, bookClock, + detailedBook, bookPage, renamedBook, type Book, @@ -254,6 +260,108 @@ describe("served updates and deletes", () => { }); }); +describe("wider write coverage", () => { + it("serves a created book without the field the application leaves out of it, and keeps it on the seeded ones", () => { + const books = servedFrom(OMITTED_FIELD_ROWS); + const created = books.answer("post_books", CREATE_NEW_BOOK).body as Book; + expect(created).not.toHaveProperty("detail"); + expect(books.answer("get_books_by_id", { id: "3" }).body).toEqual(created); + expect(books.answer("get_books_by_id", { id: "1" }).body).toEqual( + detailedBook(FIRST_BOOK), + ); + }); + + it("replays a call that asks for some fields only, and answers the rest from the state", () => { + const books = servedFrom(PROJECTED_ROWS); + expect( + books.answer("get_books_by_id", { id: "2", fields: "name" }), + ).toMatchObject({ + body: { name: "Second Book" }, + state: { source: "recording", reason: "projection" }, + }); + expect(books.answer("get_books_by_id", { id: "2" })).toMatchObject({ + body: SECOND_BOOK, + state: { source: "state", view: "item" }, + }); + }); + + it("numbers created books after the largest held asset number, in its digit groups", () => { + const books = servedFrom(ASSET_ROWS); + const create = (name: string) => + books.answer("post_books", { body: { name } }).body as Book; + expect(create("Fifth Book")).toMatchObject({ asset: "000-003" }); + expect(create("Sixth Book")).toMatchObject({ asset: "000-004" }); + expect(books.answer("get_books_by_id", { id: "1" }).body).toEqual( + assetBook(FIRST_BOOK, "000-001"), + ); + }); +}); + +describe("write coverage limits", () => { + it("numbers past a mask's width again after the state is saved and loaded", () => { + const books = servedFrom([ + [ + "GET", + "/api/books", + [assetBook(FIRST_BOOK, "99-97"), assetBook(SECOND_BOOK, "99-98")], + ], + ["GET", "/api/books/1", assetBook(FIRST_BOOK, "99-97")], + ["GET", "/api/books/2", assetBook(SECOND_BOOK, "99-98")], + [ + "POST", + "/api/books", + assetBook(THIRD_BOOK, "99-99"), + 201, + { name: "Third Book" }, + ], + ["GET", "/api/books/3", assetBook(THIRD_BOOK, "99-99")], + ]); + const create = (name: string) => + books.answer("post_books", { body: { name } }).body; + expect(create("Fifth Book")).toMatchObject({ asset: "99-99" }); + expect(create("Sixth Book")).toMatchObject({ asset: "100-00" }); + // Loading what data/get wrote clears the sequence counter, so the held numbers alone carry it on. + const { rea_state } = books.store.snapshot(); + expect(books.store.replace({ rea_state }).ok).toBe(true); + expect(create("Seventh Book")).toMatchObject({ asset: "100-01" }); + }); + + it("replays a call naming a projection parameter, even an empty one", () => { + expect( + servedFrom(PROJECTED_ROWS).answer("get_books_by_id", { + id: "2", + fields: "", + }), + ).toMatchObject({ state: { source: "recording", reason: "projection" } }); + }); + + it("answers a projection of a book the state no longer holds as a missing one", () => { + const books = servedFrom([ + ...PROJECTED_ROWS, + ...BOOK_ROWS.slice(BOOK_INIT_ROWS.length), + ]); + books.answer("delete_books_by_id", { id: "2" }); + expect( + books.answer("get_books_by_id", { id: "2", fields: "name" }), + ).toMatchObject({ + status: 404, + body: BOOK_NOT_FOUND, + state: { source: "state", view: "missing", id: "2" }, + }); + }); + + it("lists a created book without the field an item read showed the application leaves out", () => { + const books = servedFrom(DETAILED_LIST_ROWS); + const created = books.answer("post_books", CREATE_NEW_BOOK).body as Book; + expect(created).not.toHaveProperty("detail"); + expect(books.answer("get_books").body).toEqual([ + detailedBook(FIRST_BOOK), + detailedBook(SECOND_BOOK), + created, + ]); + }); +}); + describe("refusals", () => { it("refuses a body the capture does not support and leaves the state unchanged", () => { const books = servedFrom(); @@ -1258,3 +1366,82 @@ describe("paged lines", () => { ); }); }); + +describe("omissions on query-backed read-backs", () => { + it("confirms a create through a filtered list and serves later omissions in that query", () => { + const alpha = { id: 1, name: "Alpha", detail: "x" }; + const beta = { id: 2, name: "Beta", detail: "x" }; + const gamma = { id: 3, name: "Gamma" }; + const books = servedFrom([ + ["GET", "/api/books", [alpha, beta]], + ["GET", "/api/books?name=Alpha", [alpha]], + ["GET", "/api/books/1", alpha], + ["GET", "/api/books/2", beta], + ["POST", "/api/books", gamma, 201, { name: "Gamma" }], + ["GET", "/api/books?name=Gamma", [gamma]], + ]); + const created = books.answer("post_books", { + body: { name: "Delta" }, + }).body; + expect(created).toMatchObject({ id: 3, name: "Delta" }); + expect(created).not.toHaveProperty("detail"); + expect(books.answer("get_books", { name: "Delta" })).toMatchObject({ + body: [created], + state: { source: "state", membership: "query", listed: 1 }, + }); + expect(books.answer("get_books", { name: "Alpha" }).body).toEqual([alpha]); + }); +}); + +describe("own JSON members in update bodies", () => { + it("accepts constructor added by a later recorded update without inheriting a function kind", () => { + const alpha = { id: 1, name: "Alpha" }; + const beta = { id: 2, name: "Beta", constructor: "known" }; + const renamed = { ...alpha, name: "Renamed" }; + const extended = { ...renamed, constructor: "stored" }; + const books = servedFrom([ + ["GET", "/api/books", [alpha, beta]], + ["GET", "/api/books/1", alpha], + ["GET", "/api/books/2", beta], + ["PATCH", "/api/books/1", renamed, 200, { name: "Renamed" }], + ["GET", "/api/books/1", renamed], + ["PATCH", "/api/books/1", extended, 200, { constructor: "stored" }], + ["GET", "/api/books/1", extended], + ]); + expect( + books.answer("patch_books_by_id", { + id: "1", + body: { constructor: "value" }, + }).body, + ).toMatchObject({ + id: 1, + name: "Alpha", + }); + expect(books.answer("get_books_by_id", { id: "1" }).body).toMatchObject({ + constructor: "value", + }); + }); +}); + +describe("asset sequence inference", () => { + it("answers the recorded first create consistently when a held asset has outgrown its mask", () => { + const alpha = { id: 1, name: "Alpha", asset: "99-98" }; + const beta = { id: 2, name: "Beta", asset: "100-00" }; + const gamma = { id: 3, name: "Gamma", asset: "99-99" }; + const books = servedFrom([ + ["GET", "/api/books", [alpha, beta]], + ["GET", "/api/books/1", alpha], + ["GET", "/api/books/2", beta], + ["POST", "/api/books", gamma, 201, { name: "Gamma" }], + ["GET", "/api/books/3", gamma], + ]); + expect(books.state.writes.get("post_books")?.rules).not.toContainEqual({ + field: "asset", + rule: "sequence", + scope: "collection", + }); + expect( + books.answer("post_books", { body: { name: "Gamma" } }).body, + ).toEqual(gamma); + }); +}); diff --git a/src/application/environment/environmentStateWriteAnswer.ts b/src/application/environment/environmentStateWriteAnswer.ts index 98fc683f45..39ac804de1 100644 --- a/src/application/environment/environmentStateWriteAnswer.ts +++ b/src/application/environment/environmentStateWriteAnswer.ts @@ -128,11 +128,16 @@ const observedEntity = ( return isObject(object) ? object : undefined; }; -/** One past the largest integer these entities hold in the field, or undefined when none holds one. */ -const nextHeld = (entities: readonly StateEntity[], field: string) => +/** One past the largest integer these entities hold in the field, read as the exemplar spells it, or undefined when none holds one. */ +const nextHeld = ( + entities: readonly StateEntity[], + field: string, + exemplar: JsonValue, +) => nextInteger( entities.map((entity) => entity.fields), field, + exemplar, ); /** @@ -197,6 +202,7 @@ const generator = ( ? [...target.collection, ...startedIn] : [...ofType, ...started], rule.field, + exemplar, ), ), ), @@ -212,7 +218,11 @@ const generator = ( exemplar, sequence( `${request.plan.target ?? ""} ${rule.field} ${target.name}`, - nextHeld([...target.collection, ...startedIn], rule.field), + nextHeld( + [...target.collection, ...startedIn], + rule.field, + exemplar, + ), ), ), ); diff --git a/src/domain/analysisView/EnvironmentManifestAccumulator.ts b/src/domain/analysisView/EnvironmentManifestAccumulator.ts index 5d69910188..8a9900bf29 100644 --- a/src/domain/analysisView/EnvironmentManifestAccumulator.ts +++ b/src/domain/analysisView/EnvironmentManifestAccumulator.ts @@ -91,8 +91,36 @@ interface GroupState extends ToolGroup { readonly queryNames: Set; readonly apiPath: boolean; jsonLike: boolean; + /** Every response of the write group had no body, as a 204 has. */ + bodiless: boolean; } +const responseTextSchema = z.object({ + response: z.object({ + status: z.unknown().optional(), + content: z + .object({ text: z.unknown().optional(), size: z.unknown().optional() }) + .nullable() + .optional(), + }), +}); +/** Whether a record's response had no body: a 204 or 205, an empty text, or a recorded size of zero; a missing text alone may be a body not kept. */ +const emptyResponse = (reported: unknown): boolean => { + const parsed = responseTextSchema.safeParse(reported); + if (!parsed.success) return false; + const { status, content } = parsed.data.response; + return ( + status === 204 || + status === 205 || + content?.text === "" || + ((content?.text === undefined || content.text === null) && + content?.size === 0) + ); +}; +/** A write group every response of which had no body, at least one of them a success. */ +const answeredEmpty = (group: GroupState): boolean => + group.bodiless && + [...group.statuses].some((status) => status >= 200 && status < 300); const postDataSchema = z.object({ request: z.object({ postData: z @@ -118,6 +146,8 @@ const ASSET_PREFIXES = ["image/", "font/", "audio/", "video/"]; const ASSET_PATH = /\.(?:m?js|css|map|html?|png|jpe?g|gif|svg|webp|ico|woff2?|ttf|otf)$/iu; const WRITE_METHODS = new Set(["POST", "PUT", "PATCH"]); +/** A write whose every response is empty is a tool whatever its content type: the empty reply is its answer. */ +const BODILESS_WRITE_METHODS = new Set([...WRITE_METHODS, "DELETE"]); /** Quotes, separators and brackets a four-part composite key adds around its texts. */ const KEY_OVERHEAD_CHARS = 32; const UNKNOWNS: Manifest["unknowns"] = [ @@ -262,7 +292,7 @@ export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumula route, queryNames, { key, group }, - bodyShape.value, + { shape: bodyShape.value, empty: emptyResponse(record.reported) }, ); } finally { this.releaseTemporary(); @@ -304,8 +334,9 @@ export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumula route: RouteTemplate, queryNames: ReadonlySet | undefined, bucket: { readonly key: string; readonly group: GroupState | undefined }, - bodyShape: BodyShape | null, + exchange: { readonly shape: BodyShape | null; readonly empty: boolean }, ): Result { + const bodyShape = exchange.shape; const mime = mediaTypeEssence(line.rawMime); let group = bucket.group; // One retained record pointer, including its projected output. @@ -373,6 +404,7 @@ export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumula bodyShape, apiPath: isApiPath(route.template), jsonLike: false, + bodiless: BODILESS_WRITE_METHODS.has(line.method), }; this.#groups.set(bucket.key, group); } else { @@ -383,6 +415,7 @@ export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumula } group.positions.push(position); group.jsonLike ||= jsonLike; + group.bodiless &&= exchange.empty; if (line.status !== null) group.statuses.add(line.status); if (mime !== null) group.contentTypes.add(mime); for (const name of queryNames ?? []) group.queryNames.add(name); @@ -457,7 +490,7 @@ export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumula const reason: Exclusion["reason"] | null = group.origin !== primary ? "other-origin" - : group.apiPath || group.jsonLike + : group.apiPath || group.jsonLike || answeredEmpty(group) ? null : isAsset(group) ? "ui-page-or-asset" diff --git a/src/domain/analysisView/EnvironmentStateExtraction.ts b/src/domain/analysisView/EnvironmentStateExtraction.ts index 610ef6e196..a05402a3e4 100644 --- a/src/domain/analysisView/EnvironmentStateExtraction.ts +++ b/src/domain/analysisView/EnvironmentStateExtraction.ts @@ -22,6 +22,7 @@ import { locateItem, locateList, normalizedPath, + projectsFields, rootOf, same, scalarText, @@ -230,14 +231,22 @@ export class EnvironmentStateExtraction { return first; } - /** Successful recordings that finished before the first related write began. */ - #initRecordings(tool: StateToolFacts, template: string): StateRecordFacts[] { - const limit = this.#initLimit(template); - const before = (record: StateRecordFacts): boolean => + /** Initial-state admission is shared with later projection-evidence selection. */ + #beforeInitLimit( + record: StateRecordFacts, + limit: StateRecordFacts | undefined, + ): boolean { + return ( limit === undefined || (this.#position.get(record.ordinal)! < this.#position.get(limit.ordinal)! && - (this.#timeline === "record-order" || record.end <= limit.start)); + (this.#timeline === "record-order" || record.end <= limit.start)) + ); + } + + /** Successful recordings that finished before the first related write began. */ + #initRecordings(tool: StateToolFacts, template: string): StateRecordFacts[] { + const limit = this.#initLimit(template); return tool.record_ordinals .map((ordinal) => this.#byOrdinal.get(ordinal)) .filter( @@ -246,7 +255,7 @@ export class EnvironmentStateExtraction { record.status !== null && record.status >= 200 && record.status < 300 && - before(record), + this.#beforeInitLimit(record, limit), ) .sort( (left, right) => @@ -276,15 +285,50 @@ export class EnvironmentStateExtraction { candidate: Candidate, pointer: string, recordings: readonly StateRecordFacts[], + projected: readonly number[] = [], ): void { this.#plans.push({ ...candidate, pointer, recordings, latest: this.#latestPerLine(recordings), + projected, }); } + /** Later identity-free projections are evidence of replay, never initial entities. Read only those query bodies. */ + #addLateProjections( + tool: StateToolFacts, + type: string, + projected: number[], + ): void { + const limit = this.#initLimit(type); + if (limit === undefined) return; + for (const ordinal of tool.record_ordinals) { + const record = this.#byOrdinal.get(ordinal); + if ( + record === undefined || + record.status === null || + record.status < 200 || + record.status >= 300 || + !projectsFields(record.query) || + this.#beforeInitLimit(record, limit) + ) + continue; + if ( + this.#withBody( + ordinal, + (body) => + body !== undefined && + locateItem(body, lastSegmentValue(record.path)) === undefined, + ) + ) { + this.#retain(ENTRY_BYTES); + projected.push(ordinal); + } + } + } + /** Item tools prove a type's identity field by the path value their bodies repeat. */ #planItem(candidate: Candidate): void { const { tool, type } = candidate; @@ -292,6 +336,8 @@ export class EnvironmentStateExtraction { if (recordings === undefined) return; let pointer: string | undefined; let keys: string[] | undefined; + const kept: StateRecordFacts[] = []; + const projected: number[] = []; for (const record of recordings) { const located = this.#withBody(record.ordinal, (body) => body === undefined @@ -302,6 +348,13 @@ export class EnvironmentStateExtraction { this.#recorded(tool, "body-unusable"); return; } + // A read that asked for some fields only may leave the identity out; replay answers it. + if (located === undefined && projectsFields(record.query)) { + this.#retain(ENTRY_BYTES); + projected.push(record.ordinal); + continue; + } + kept.push(record); if ( located === undefined || (pointer ?? located.pointer) !== located.pointer @@ -328,7 +381,11 @@ export class EnvironmentStateExtraction { return; } this.#identity.set(type, { field, basis: "path-parameter" }); - this.#addPlan(candidate, pointer, recordings); + this.#addLateProjections(tool, type, projected); + projected.sort( + (left, right) => this.#position.get(left)! - this.#position.get(right)!, + ); + this.#addPlan(candidate, pointer, kept, projected); } /** diff --git a/src/domain/analysisView/EnvironmentStateWriteSimulation.test.ts b/src/domain/analysisView/EnvironmentStateWriteSimulation.test.ts index b6600afc55..362b96a367 100644 --- a/src/domain/analysisView/EnvironmentStateWriteSimulation.test.ts +++ b/src/domain/analysisView/EnvironmentStateWriteSimulation.test.ts @@ -1,3 +1,4 @@ +import { OmittedFields } from "./environmentStateWriteOmitted.js"; import { checkedLines, servedLine } from "./environmentStateCheckedLines.js"; import { describe, expect, it } from "vitest"; @@ -19,7 +20,11 @@ import { } from "../../../tests/fixtures/environmentStateWrites.js"; import type { JsonValue } from "../jsonValue.js"; import { ok } from "../result.js"; -import { itemTarget, SimulatedState } from "./environmentStateWriteChecks.js"; +import { + checkItemRead, + itemTarget, + SimulatedState, +} from "./environmentStateWriteChecks.js"; import { laterObservation, type ObservationSearch, @@ -639,16 +644,23 @@ describe("reads the served answer no longer matches", () => { }); }); - it("confirms no create whose entity lacks a field every recording of its item view carried", () => { + it("confirms a create that lacks a field every item recording carried only when its own read leaves the field out too", () => { const detailed = (entry: Book): Book => ({ ...entry, detail: "x" }); - const rows: TimedCaptureRow[] = [ + const rows = (readBack: Book): TimedCaptureRow[] => [ ["GET", "/api/books", [FIRST_BOOK, SECOND_BOOK]], ["GET", "/api/books/1", detailed(FIRST_BOOK)], ["GET", "/api/books/2", detailed(SECOND_BOOK)], ["POST", "/api/books", THIRD_BOOK, 201, { name: "Third Book" }], - ["GET", "/api/books/3", THIRD_BOOK], + ["GET", "/api/books/3", readBack], ]; - expect(gated(rows, "post_books")).toMatchObject({ served: false }); + // The application omits the field for this book, so serving need not invent it. + expect(gated(rows(THIRD_BOOK), "post_books")).toMatchObject({ + served: true, + }); + // The read shows the field the create's own answer left out: the state cannot render it. + expect(gated(rows(detailed(THIRD_BOOK)), "post_books")).toMatchObject({ + served: false, + }); }); }); @@ -2813,3 +2825,82 @@ describe("path fields of a create tied to its collection", () => { }); }); }); + +it("withholds a create when a sibling read through an encoded collection reveals an extra change", () => { + const alpha = { id: 1, name: "Alpha" }; + const beta = { id: 2, name: "Beta" }; + const gamma = { id: 3, name: "Gamma" }; + const delta = { id: 4, name: "Delta" }; + const rows: TimedCaptureRow[] = [ + ["GET", "/api/orgs/%61cme1/books", [alpha]], + ["GET", "/api/orgs/beta2/books", [beta]], + ["GET", "/api/orgs/%61cme1/books/1", alpha], + ["GET", "/api/orgs/beta2/books/2", beta], + ["POST", "/api/orgs/acme1/books", gamma, 201, { name: "Gamma" }], + ["GET", "/api/orgs/acme1/books/3", gamma], + ["GET", "/api/orgs/%61cme1/books/1", { ...alpha, name: "Changed" }], + ["POST", "/api/orgs/beta2/books", delta, 201, { name: "Delta" }], + ]; + const state = environmentStateViewOf(timedCaptureEvidence(rows), [ + "sibling-variance", + ]); + expect(state.writes.find((plan) => plan.effect === "create")).toMatchObject({ + served: false, + reason: "side-effect", + }); +}); + +it("learns omitted fields using the raw ID resolved by the item owner", () => { + const state = new SimulatedState( + [ + { + type: "/api/books", + collection: "/api/books", + id: "%63", + fields: { id: "%63", name: "Created" }, + view_fields: [], + record_ordinals: [], + }, + ], + [], + ); + const view = { + tool: "get_books_by_id", + type: "/api/books", + shape: "item" as const, + pointer: "", + fields: ["id", "name", "detail"], + required_fields: ["id", "name", "detail"], + checked_record_ordinals: [0], + envelope_ordinal: 0, + }; + const record = { + ordinal: 1, + method: "GET", + path: "/api/books/%63", + query: undefined, + status: 200, + start: 1, + end: 1, + }; + const target = itemTarget(state, view.type, record); + expect(target.id).toBe("%63"); + const read = { + body: { id: "%63", name: "Created" }, + base: { id: "seed", name: "Seed", detail: "x" }, + volatile: new Set(), + idField: "id", + collection: target.collection, + itemId: target.id, + }; + const omitted = new OmittedFields(() => {}); + const outcome = omitted.check(state, view, record, read, (loosened) => + checkItemRead(state, loosened, record, read), + ); + expect(outcome).toMatchObject({ pass: true, omitted: ["detail"] }); + if (outcome === undefined) throw new Error("Expected a counted read"); + omitted.keep(view, outcome); + expect(omitted.views([view])[0]).toMatchObject({ + required_fields: ["id", "name"], + }); +}); diff --git a/src/domain/analysisView/EnvironmentStateWriteSimulation.ts b/src/domain/analysisView/EnvironmentStateWriteSimulation.ts index 4f59913ade..971e6d9cca 100644 --- a/src/domain/analysisView/EnvironmentStateWriteSimulation.ts +++ b/src/domain/analysisView/EnvironmentStateWriteSimulation.ts @@ -13,6 +13,8 @@ import { type StateView, } from "./environmentStateRules.js"; import { updatedEntity } from "./environmentStateWriteApply.js"; +import { projectedRead } from "./environmentStateWriteObserve.js"; +import { OmittedFields } from "./environmentStateWriteOmitted.js"; import { checkDocumentRead, checkItemRead, @@ -92,6 +94,8 @@ export class EnvironmentStateWriteSimulation { readonly #taint: WriteTaint; readonly #touched: TouchedTargets = new Map(); readonly #effects = new WriteEffects(); + /** Fields reads show the application leaves out, which serving then does not require. */ + readonly omitted = new OmittedFields((bytes) => this.#input.io.retain(bytes)); readonly #lastCheck = new Map(); /** When the latest modelled write to each target since its last check ended. */ readonly #touchedSince: PendingTargets = new Map(); @@ -178,8 +182,12 @@ export class EnvironmentStateWriteSimulation { io: this.#input.io, records: this.#input.records, timed: this.#input.timed, - viewOf: (ordinal) => - this.#input.views.get(this.#toolOf.get(ordinal)?.name ?? ""), + viewOf: (ordinal) => { + const view = this.#input.views.get( + this.#toolOf.get(ordinal)?.name ?? "", + ); + return view === undefined ? undefined : this.omitted.view(view); + }, idField: (type) => this.#types.get(type)!.id_field, observed: (ordinal) => this.#observations.set( @@ -311,9 +319,11 @@ export class EnvironmentStateWriteSimulation { checkDocumentRead(this.#state, view.tool, record, body), ); const item = view.shape === "item"; - const collection = item - ? itemTarget(this.#state, view.type, record).collection - : line!.collection; + const target = item + ? itemTarget(this.#state, view.type, record) + : undefined; + const collection = + target === undefined ? line!.collection : target.collection; if (this.#taint.under(collection)) return undefined; if (!item) { const ids = this.#state @@ -336,10 +346,16 @@ export class EnvironmentStateWriteSimulation { idField: this.#types.get(view.type)?.id_field ?? "", volatile: this.#input.volatileOf(view.type), collection, + ...(target === undefined ? {} : { itemId: target.id }), }; - return item - ? checkItemRead(this.#state, view, record, check) - : checkListRead(this.#state, view, record, { ...check, line: line! }); + return this.omitted.check(this.#state, view, record, check, (loosened) => + loosened.shape === "item" + ? checkItemRead(this.#state, loosened, record, check) + : checkListRead(this.#state, loosened, record, { + ...check, + line: line!, + }), + ); }); } @@ -349,10 +365,12 @@ export class EnvironmentStateWriteSimulation { } #read(record: StateRecordFacts, index: number, tool: StateToolFacts): void { - const view = this.#input.views.get(tool.name); + const original = this.#input.views.get(tool.name); + if (original === undefined) return; + const view = this.omitted.view(original); + if (projectedRead(view, record, this.#input.io)) return; const borrowers = this.#observations.get(record.ordinal); - if (view === undefined || (!borrowers && this.#resynced(view, record))) - return; + if (!borrowers && this.#resynced(view, record)) return; let outcome = this.#check(view, record); if (outcome === undefined) return; let keys = [...outcome.targets, ...outcome.unheld]; @@ -365,6 +383,7 @@ export class EnvironmentStateWriteSimulation { if (blocked(keys)) return; let exercised = exercisedBy(this.#touched, keys, borrowers); if (borrowers !== undefined) { + this.omitted.keep(view, outcome); if (!outcome.pass) this.#contradictOthers( exercised, @@ -384,6 +403,8 @@ export class EnvironmentStateWriteSimulation { exercised = exercisedBy(this.#touched, keys, borrowers); } } + // Retain omissions only after the final check's coverage passes the ownership guards. + this.omitted.keep(view, outcome); const interferes = interference(this.#history, outcome); const unseen = outcome.pass ? this.#confirm(view, record, outcome, exercised) diff --git a/src/domain/analysisView/environmentManifestView.test.ts b/src/domain/analysisView/environmentManifestView.test.ts index 0910f6e3fa..4c50886596 100644 --- a/src/domain/analysisView/environmentManifestView.test.ts +++ b/src/domain/analysisView/environmentManifestView.test.ts @@ -1062,6 +1062,46 @@ it("treats a +json suffix as JSON only on application media types", () => { expect(projected.tools[1]?.body_shape).toBeNull(); }); +it("keeps a write answered with no body as a tool whatever its content type, but not one only redirected", () => { + const capture = syntheticCapture([ + ["GET", "https://s.test/items/1", "application/json"], + ["GET", "https://s.test/items/2", "application/json"], + ["DELETE", "https://s.test/items/1", "text/html"], + ["DELETE", "https://s.test/items/2", "text/html"], + ["POST", "https://s.test/forms/1", "text/html"], + ["PUT", "https://s.test/pages/1", "text/html"], + ]); + const answer = (index: number, status: number, text?: string): void => { + const record = capture.records[index]!; + record.reported = { + ...(record.reported as Record), + response: { + status, + content: { + mimeType: "text/html", + ...(text === undefined ? {} : { text }), + }, + }, + }; + }; + answer(2, 204); + answer(3, 200, ""); + // A redirect is no answer, and a missing text may be a body the capture did not keep. + answer(4, 302, ""); + answer(5, 200); + const projected = manifest([], capture); + expect(projected.tools.map(line)).toEqual([ + "DELETE /items/{numeric}", + "GET /items/{numeric}", + ]); + expect( + projected.excluded.map((entry) => [entry.path_template, entry.reason]), + ).toEqual([ + ["/forms/{numeric}", "ui-page-or-asset"], + ["/pages/{numeric}", "ui-page-or-asset"], + ]); +}); + it("refuses a parent whose subject is absent or names another path or digest", () => { const parent = parentOf(); const refusal = { diff --git a/src/domain/analysisView/environmentStateGate.ts b/src/domain/analysisView/environmentStateGate.ts index 1acec652c7..317f9b090d 100644 --- a/src/domain/analysisView/environmentStateGate.ts +++ b/src/domain/analysisView/environmentStateGate.ts @@ -42,6 +42,8 @@ export interface StatePlan { readonly recordings: readonly StateRecordFacts[]; /** The latest of them per request line, in timeline order. */ readonly latest: readonly StateRecordFacts[]; + /** Item reads left to replay because they asked for some fields only. */ + readonly projected?: readonly number[]; } /** What the gate reads from an extraction; `withBody` drops each body's temporary work after `use`. */ @@ -166,6 +168,9 @@ export const checkItemPlan = ( required_fields: union.required, envelope_ordinal: plan.latest[0]!.ordinal, checked_record_ordinals: plan.latest.map((record) => record.ordinal), + ...(plan.projected === undefined || plan.projected.length === 0 + ? {} + : { projected_record_ordinals: [...plan.projected] }), }; }; diff --git a/src/domain/analysisView/environmentStateRules.ts b/src/domain/analysisView/environmentStateRules.ts index 1bc627deb3..9e0afd7be1 100644 --- a/src/domain/analysisView/environmentStateRules.ts +++ b/src/domain/analysisView/environmentStateRules.ts @@ -104,8 +104,26 @@ interface EntityViewPlan { export type StateItemView = EntityViewPlan & { readonly shape: "item"; readonly envelope_ordinal: number; + /** Recorded reads that asked for some fields only and did not return the item, which replay answers. */ + readonly projected_record_ordinals?: number[] | undefined; }; +/** A query parameter that asks for some fields only (`fields`, `fields[...]`, `select`, `$select`). */ +export const isProjectionParameter = (name: string): boolean => { + const lower = name.toLowerCase(); + return ( + lower === "fields" || + lower.startsWith("fields[") || + lower === "select" || + lower === "$select" + ); +}; + +/** Whether a recorded query string names a projection parameter. */ +export const projectsFields = (query: string | undefined): boolean => + query !== undefined && + [...new URLSearchParams(query).keys()].some(isProjectionParameter); + export type StateListView = EntityViewPlan & { readonly shape: "list"; readonly membership: "all" | "recorded"; diff --git a/src/domain/analysisView/environmentStateView.ts b/src/domain/analysisView/environmentStateView.ts index 5ca962a3a7..754a0a26e8 100644 --- a/src/domain/analysisView/environmentStateView.ts +++ b/src/domain/analysisView/environmentStateView.ts @@ -115,6 +115,12 @@ const viewSchema = z.discriminatedUnion("shape", [ .describe( "Record whose response, with the entity object replaced, every item response shares.", ), + projected_record_ordinals: z + .array(z.number().int().nonnegative()) + .optional() + .describe( + "Recorded reads that asked for some fields only (`fields`, `select`) and did not return the item; calls naming a projection replay.", + ), }), z.strictObject({ ...entityViewFields, diff --git a/src/domain/analysisView/environmentStateWriteApply.test.ts b/src/domain/analysisView/environmentStateWriteApply.test.ts index 3ea99ac31e..185efdfa53 100644 --- a/src/domain/analysisView/environmentStateWriteApply.test.ts +++ b/src/domain/analysisView/environmentStateWriteApply.test.ts @@ -4,6 +4,7 @@ import { err, ok } from "../result.js"; import type { JsonObject, StateEntity } from "./environmentStateRules.js"; import { bodyProblem, + storedKinds, createdFields, updatedEntity, updatedFields, @@ -532,3 +533,10 @@ it("drops view-specific values of the fields an update changed and cites no reco entity.view_fields, ); }); + +it("learns own stored JSON member kinds without inheriting prototype members", () => { + expect(storedKinds(["constructor"], {}, { constructor: "value" })).toEqual([ + { member: "constructor", kind: "string" }, + ]); + expect(storedKinds(["constructor", "toString"], {}, {})).toEqual([]); +}); diff --git a/src/domain/analysisView/environmentStateWriteApply.ts b/src/domain/analysisView/environmentStateWriteApply.ts index 92eaf9a64e..64ad47cb98 100644 --- a/src/domain/analysisView/environmentStateWriteApply.ts +++ b/src/domain/analysisView/environmentStateWriteApply.ts @@ -37,6 +37,13 @@ export type GenerateValue = ( const kindOf = (value: JsonValue | undefined): string => value === null ? "null" : Array.isArray(value) ? "array" : typeof value; +/** JSON objects contain only their own members, even when a name also exists on Object.prototype. */ +const ownMember = ( + object: JsonObject, + member: string, +): JsonValue | undefined => + Object.hasOwn(object, member) ? object[member] : undefined; + /** The JSON types `stored` holds members in, for those `observed` shows no value of. */ export const storedKinds = ( members: readonly string[], @@ -44,8 +51,8 @@ export const storedKinds = ( stored: JsonObject, ): WriteBody["kinds"] => members.flatMap((member) => { - const value = stored[member]; - return (observed[member] ?? null) === null && + const value = ownMember(stored, member); + return (ownMember(observed, member) ?? null) === null && value !== undefined && value !== null ? [{ member, kind: kindOf(value) as JsonKind }] @@ -100,7 +107,7 @@ export const bodyProblem = ( } if (!body.accepted.includes(member)) return `Member ${member} is not one the capture showed stored (${bodyContract(body)}).`; - const recorded = shapes.entity[member]; + const recorded = ownMember(shapes.entity, member); const kind = recorded === undefined || recorded === null ? kinds.get(member) @@ -361,7 +368,7 @@ export const updatedEntity = ( ): StateEntity => { const changed = new Set( Object.keys(fields).filter( - (field) => !same(entity.fields[field], fields[field]), + (field) => !same(ownMember(entity.fields, field), fields[field]), ), ); // Written values are no longer the ones the cited records show. diff --git a/src/domain/analysisView/environmentStateWriteChecks.ts b/src/domain/analysisView/environmentStateWriteChecks.ts index 40dbb942fa..7526160d67 100644 --- a/src/domain/analysisView/environmentStateWriteChecks.ts +++ b/src/domain/analysisView/environmentStateWriteChecks.ts @@ -192,6 +192,8 @@ export interface CheckOutcome { readonly unheld: readonly string[]; /** For a failed list read that differs only in some members' fields, those members' targets. */ readonly differing?: readonly string[]; + /** Required fields the read shows the application leaves out of entities the state holds. */ + readonly omitted?: readonly string[]; } /** Whether a passing check shows where a create placed its entity: a list's does unless a sort orders that entity, leaving it tied with none, or it matched alone. */ @@ -208,7 +210,7 @@ export const showsPlacement = ( )); /** What a check compares: the read's recorded body and the served base recording it would answer on. */ -interface ReadBodies { +export interface ReadBodies { readonly body: JsonValue | undefined; /** The recording whose envelope the served answer keeps: an item view's envelope, a list line's latest init recording. */ readonly base: JsonValue | undefined; diff --git a/src/domain/analysisView/environmentStateWriteCreate.ts b/src/domain/analysisView/environmentStateWriteCreate.ts index 09bde41eac..511e034474 100644 --- a/src/domain/analysisView/environmentStateWriteCreate.ts +++ b/src/domain/analysisView/environmentStateWriteCreate.ts @@ -51,10 +51,11 @@ import { type WriteCall, } from "./environmentStateWriteRules.js"; import { - integerOf, isUuid, nextSequence, readClock, + sequenceNumber, + spelledLike, } from "./environmentStateWriteValues.js"; /** The id a response names alone: a scalar body, or an object's id field or single scalar member. */ @@ -180,7 +181,7 @@ const sequencesOf = ( : [], ); -/** Whether a later call's sequences hold what serving generates: one past the largest value their scope holds or started with, and past the plan's last. */ +/** Whether a later call's sequences hold what serving generates: one past the largest value their scope holds or started with and past the plan's last, spelled as the exemplar spells it. */ const followsSequences = ( context: EffectContext, draft: WriteDraft, @@ -195,10 +196,13 @@ const followsSequences = ( rule.scope === "type" || entity.collection === created.collection, ); const next = nextSequence( - nextInteger(fieldsOf(scope), rule.field), + nextInteger(fieldsOf(scope), rule.field, draft.observed?.[rule.field]), draft.sequences.get(key), ); - return integerOf(created.fields[rule.field]) === next; + return ( + created.fields[rule.field] === + spelledLike(draft.observed?.[rule.field] ?? "", next) + ); }); /** Whether a later call's generated values take the forms serving generates them in: clocks in the learned format, unique values and UUID identities as UUIDs, and sequences in order. */ @@ -231,7 +235,10 @@ const takeSequences = ( created: Pick, ): void => { for (const { rule, key } of sequencesOf(draft, created.collection)) { - const value = integerOf(created.fields[rule.field]); + const value = sequenceNumber( + created.fields[rule.field], + draft.observed?.[rule.field], + ); if (value !== undefined) draft.sequences.set(key, value); } }; diff --git a/src/domain/analysisView/environmentStateWriteEffects.ts b/src/domain/analysisView/environmentStateWriteEffects.ts index dc34ab6bec..33b7528f39 100644 --- a/src/domain/analysisView/environmentStateWriteEffects.ts +++ b/src/domain/analysisView/environmentStateWriteEffects.ts @@ -477,7 +477,9 @@ export const applyUpdate = ( (member) => member !== draft.plan.body?.identity, ); const derived = Object.keys(written.value).filter( - (field) => !same(written.value[field], entity.fields[field]), + (field) => + !Object.hasOwn(entity.fields, field) || + !same(written.value[field], entity.fields[field]), ); const changed = { fields: [...new Set([...sent, ...derived])], diff --git a/src/domain/analysisView/environmentStateWriteExplain.test.ts b/src/domain/analysisView/environmentStateWriteExplain.test.ts index 0ab067888b..19f6298248 100644 --- a/src/domain/analysisView/environmentStateWriteExplain.test.ts +++ b/src/domain/analysisView/environmentStateWriteExplain.test.ts @@ -576,6 +576,10 @@ it.each<[string, string, boolean]>([ ["/users/42", "/users/43", false], ["/users", "/teams/{id}", false], ["/{tenant}/users", "/acme/teams", false], + ["/orgs/%61cme/books", "/orgs/acme/books/1", true], + ["/orgs/%2561cme/books", "/orgs/%61cme/books/1", true], + ["/orgs/%2561cme/books", "/orgs/acme/books/1", false], + ["/orgs/a%2Fb/books", "/orgs/a/b/books/1", false], ])("relates %s and %s by prefix: %s", (left, right, related) => { expect(prefixRelated(left, right)).toBe(related); expect(prefixRelated(right, left)).toBe(related); diff --git a/src/domain/analysisView/environmentStateWriteExplain.ts b/src/domain/analysisView/environmentStateWriteExplain.ts index 6cafd4522d..d41acaa0a4 100644 --- a/src/domain/analysisView/environmentStateWriteExplain.ts +++ b/src/domain/analysisView/environmentStateWriteExplain.ts @@ -23,6 +23,7 @@ import { integerOf, isUuid, readClock, + sequenceNumber, slugOf, } from "./environmentStateWriteValues.js"; @@ -100,10 +101,11 @@ const isClockNear = ( export const nextInteger = ( members: readonly JsonObject[], field: string, + exemplar?: JsonValue, ): number | undefined => { let largest: number | undefined; for (const member of members) { - const value = integerOf(member[field]); + const value = sequenceNumber(member[field], exemplar); if (value !== undefined && (largest === undefined || value > largest)) largest = value; } @@ -156,11 +158,11 @@ const generatedRule = ( const clock = readClock(value); if (clock !== undefined && isClockNear(value, evidence.startMs)) return { field, rule: "clock", format: clock.format }; - const integer = integerOf(value); + const integer = sequenceNumber(value); if (integer !== undefined) { - if (nextInteger(evidence.collection, field) === integer) + if (nextInteger(evidence.collection, field, value) === integer) return { field, rule: "sequence", scope: "collection" }; - if (nextInteger(evidence.type, field) === integer) + if (nextInteger(evidence.type, field, value) === integer) return { field, rule: "sequence", scope: "type" }; } if ( diff --git a/src/domain/analysisView/environmentStateWriteObserve.ts b/src/domain/analysisView/environmentStateWriteObserve.ts index ac79d7e01d..d7bf89768d 100644 --- a/src/domain/analysisView/environmentStateWriteObserve.ts +++ b/src/domain/analysisView/environmentStateWriteObserve.ts @@ -1,11 +1,13 @@ import type { JsonValue } from "../jsonValue.js"; import { overlapsWrite } from "./environmentStateWriteAttribution.js"; import { + at, collectionPathOf, isObject, lastSegmentValue, listItems, pointerTo, + projectsFields, scalarText, type JsonObject, type StateEntity, @@ -92,6 +94,36 @@ export interface ObservationSearch { ) => T; } +/** Whether a read asked for some fields only and shows no whole entity: an item view replays such reads, or the read leaves out a field its view requires. */ +export const projectedRead = ( + view: StateView, + record: StateRecordFacts, + io: Pick, +): boolean => { + if ( + (view.shape !== "item" && view.shape !== "list") || + !projectsFields(record.query) + ) + return false; + // An absence or error holds no entity, so it is never a partial one. + if (!succeeded(record)) return false; + if (view.shape === "item" && view.projected_record_ordinals !== undefined) + return true; + return io.withBody(record.ordinal, (body) => { + const objects = + body === undefined || body === null + ? [] + : view.shape === "item" + ? [at(body, view.pointer)] + : (listItems(body, view.pointer) ?? []); + return objects.some( + (object) => + !isObject(object) || + view.required_fields.some((field) => !Object.hasOwn(object, field)), + ); + }); +}; + const succeeded = (record: StateRecordFacts): boolean => record.status !== null && record.status >= 200 && record.status < 300; @@ -126,7 +158,8 @@ export const laterObservation = ( if ( !succeeded(record) || (view?.shape !== "item" && view?.shape !== "list") || - view.type !== target.type + view.type !== target.type || + projectedRead(view, record, search) ) continue; if (target.covers !== undefined && !target.covers(view)) continue; diff --git a/src/domain/analysisView/environmentStateWriteOmitted.ts b/src/domain/analysisView/environmentStateWriteOmitted.ts new file mode 100644 index 0000000000..10dc019bde --- /dev/null +++ b/src/domain/analysisView/environmentStateWriteOmitted.ts @@ -0,0 +1,146 @@ +import { + at, + isObject, + listItems, + projectsFields, + scalarText, + type JsonObject, + type StateItemView, + type StateListView, + type StateRecordFacts, + type StateView, +} from "./environmentStateRules.js"; +import { + succeeded, + type CheckOutcome, + type ReadBodies, + type SimulatedState, +} from "./environmentStateWriteChecks.js"; + +/** + * Fields a type's recordings always carried that a later successful whole read shows the + * application leaves out of an entity the state holds, as APIs that omit empty or false members + * do: no entity of the type must carry them, in any of its views. + */ +export class OmittedFields { + readonly #omitted = new Map>(); + + constructor(private readonly retain: (bytes: number) => void) {} + + /** Check an item or list read with the fields reads showed omitted, this one's included, no longer required, naming this one's in the outcome. */ + check( + state: SimulatedState, + view: View, + record: StateRecordFacts, + read: ReadBodies & { + readonly idField: string; + readonly collection: string; + readonly itemId?: string; + }, + compare: (view: View) => CheckOutcome | undefined, + ): CheckOutcome | undefined { + const omitted = lackedFields(state, view, record, read); + const noted = this.#omitted.get(view.type); + const loosened = relaxed( + view, + omitted.length === 0 ? noted : new Set([...(noted ?? []), ...omitted]), + ); + const outcome = compare(loosened); + return outcome === undefined ? undefined : { ...outcome, omitted }; + } + + /** Note the fields a read the gate counts shows left out, so no view of its type requires them. */ + keep(view: StateView, outcome: CheckOutcome): void { + if (view.shape !== "item" && view.shape !== "list") return; + for (const field of outcome.omitted ?? []) this.#note(view.type, field); + } + + /** The current required-field facts, shared by observations, checks and final output. */ + view(view: StateView): StateView { + return view.shape === "item" || view.shape === "list" + ? relaxed(view, this.#omitted.get(view.type)) + : view; + } + + /** The views with the fields reads showed omitted no longer required. */ + views(views: readonly StateView[]): StateView[] { + return views.map((view) => this.view(view)); + } + + #note(type: string, field: string): void { + let fields = this.#omitted.get(type); + if (fields === undefined) { + fields = new Set(); + this.#omitted.set(type, fields); + } + if (!fields.has(field)) { + this.retain(64 + field.length * 2); + fields.add(field); + } + } +} + +/** The required fields a successful whole read's held entities lack. */ +const lackedFields = ( + state: SimulatedState, + view: StateItemView | StateListView, + record: StateRecordFacts, + read: Pick & { + readonly idField: string; + readonly collection: string; + readonly itemId?: string; + }, +): string[] => { + if ( + !succeeded(record) || + read.body === undefined || + projectsFields(record.query) + ) + return []; + const lacked = new Set(); + for (const entity of heldEntities(state, view, read)) + for (const field of view.required_fields) + if (!Object.hasOwn(entity, field)) lacked.add(field); + return [...lacked]; +}; + +/** The entities a read shows that the state holds: an item read's object naming the path's id, a list read's items. */ +const heldEntities = ( + state: SimulatedState, + view: StateItemView | StateListView, + read: Pick & { + readonly idField: string; + readonly collection: string; + readonly itemId?: string; + }, +): JsonObject[] => { + const object = view.shape === "item" ? at(read.body!, view.pointer) : null; + const shown = + view.shape === "list" + ? (listItems(read.body!, view.pointer) ?? []) + : isObject(object) && + read.itemId !== undefined && + scalarText(object[read.idField]) === read.itemId + ? [object] + : []; + return shown.filter((entity) => { + const id = scalarText(entity[read.idField]); + return ( + id !== undefined && + state.entity(view.type, read.collection, id) !== undefined + ); + }); +}; + +const relaxed = ( + view: View, + omitted: ReadonlySet | undefined, +): View => + omitted === undefined + ? view + : { + ...view, + required_fields: view.required_fields.filter( + (field) => !omitted.has(field), + ), + }; diff --git a/src/domain/analysisView/environmentStateWriteRules.ts b/src/domain/analysisView/environmentStateWriteRules.ts index 58a598a669..6881dab31f 100644 --- a/src/domain/analysisView/environmentStateWriteRules.ts +++ b/src/domain/analysisView/environmentStateWriteRules.ts @@ -3,6 +3,7 @@ import { decodedSegment, isParam, normalizedPath, + pathSegmentMatches, segmentsOf, type JsonObject, } from "./environmentStateRules.js"; @@ -193,13 +194,17 @@ export const pathValuesOf = ( return values; }; -/** Whether two paths are segment prefixes of one another, a parameter on either side matching any segment. */ +/** Whether two paths are segment prefixes of one another, a parameter on either side matching any segment, with the shared single-decode spelling rule. */ export const prefixRelated = (left: string, right: string): boolean => { const a = segmentsOf(left); const b = segmentsOf(right); const shared = Math.min(a.length, b.length); for (let index = 0; index < shared; index += 1) - if (!isParam(a[index]!) && !isParam(b[index]!) && a[index] !== b[index]) + if ( + !isParam(a[index]!) && + !isParam(b[index]!) && + !pathSegmentMatches(a[index]!, b[index]!) + ) return false; return true; }; diff --git a/src/domain/analysisView/environmentStateWriteValues.test.ts b/src/domain/analysisView/environmentStateWriteValues.test.ts index acf0175a9b..6bb54dab69 100644 --- a/src/domain/analysisView/environmentStateWriteValues.test.ts +++ b/src/domain/analysisView/environmentStateWriteValues.test.ts @@ -10,6 +10,7 @@ import { replaceDelimited, seededToken, seededUuid, + sequenceNumber, slugOf, spelledLike, type ClockFormat, @@ -254,3 +255,30 @@ it("draws tokens from the whole alphabet the recorded token uses", () => { seededToken("seed-b", "k9x2m7q4z1w8"), ); }); + +it("reads a sequence written as digit groups of one width, but not a date, version or address", () => { + expect(sequenceNumber("000-005")).toBe(5); + expect(sequenceNumber("00-01-12")).toBe(112); + expect(sequenceNumber(7)).toBe(7); + for (const other of ["2026-10-21", "2026-08", "12.04", "1.2.3", "10.0.0.9"]) + expect(sequenceNumber(other)).toBeUndefined(); + expect(sequenceNumber("A-005")).toBeUndefined(); + expect(sequenceNumber("000-")).toBeUndefined(); +}); + +it("reads a count that outgrew its mask only as the exemplar's spelling of it", () => { + expect(sequenceNumber("100-00", "99-99")).toBe(10_000); + expect(sequenceNumber("1000-000", "000-005")).toBe(1_000_000); + expect(sequenceNumber("100-00")).toBeUndefined(); + for (const other of ["2026-10-21", "20261-02", "100-0"]) + expect(sequenceNumber(other, "000-005")).toBeUndefined(); + expect(sequenceNumber("100-00", "2026-10")).toBeUndefined(); +}); + +it("spells a sequence value in the recorded value's digit groups, the overflow leading", () => { + expect(spelledLike("000-005", 6)).toBe("000-006"); + expect(spelledLike("000-999", 1000)).toBe("001-000"); + expect(spelledLike("999-999", 1_000_000)).toBe("1000-000"); + expect(spelledLike("12", 13)).toBe("13"); + expect(spelledLike(12, 13)).toBe(13); +}); diff --git a/src/domain/analysisView/environmentStateWriteValues.ts b/src/domain/analysisView/environmentStateWriteValues.ts index 10e814f22a..b415ad0ff8 100644 --- a/src/domain/analysisView/environmentStateWriteValues.ts +++ b/src/domain/analysisView/environmentStateWriteValues.ts @@ -120,7 +120,58 @@ export const nextSequence = ( /** An integer spelled as the recorded value spells integers: a number, or a string of digits. */ export const spelledLike = (exemplar: JsonValue, value: number): JsonValue => - typeof exemplar === "string" ? String(value) : value; + maskedCount(exemplar) !== undefined + ? fillMask(exemplar as string, value) + : typeof exemplar === "string" + ? String(value) + : value; + +const MASKED_DIGITS = /^\d{2,}(?:-\d{2,})+$/u; + +/** The count `-`-separated digit groups of one width spell, such as an asset number `000-005`; dates, versions and addresses are not counts. */ +const maskedCount = (value: JsonValue | undefined): number | undefined => { + if (typeof value !== "string" || !MASKED_DIGITS.test(value)) return undefined; + const groups = value.split("-"); + if (groups.some((group) => group.length !== groups[0]!.length)) + return undefined; + const digits = groups.join(""); + return digits.length <= 15 ? Number(digits) : undefined; +}; + +/** A count that outgrew the exemplar's mask, read only when it is spelled as serving spells it, so dates never match. */ +const outgrownMask = ( + value: JsonValue | undefined, + exemplar: JsonValue | undefined, +): number | undefined => { + if (typeof value !== "string" || maskedCount(exemplar) === undefined) + return undefined; + const digits = value.replaceAll("-", ""); + if (!/^\d{1,15}$/u.test(digits)) return undefined; + const count = Number(digits); + return fillMask(exemplar as string, count) === value ? count : undefined; +}; + +/** A sequence's number: an integer, digit groups that count, or with the sequence's exemplar a count that outgrew its mask. */ +export const sequenceNumber = ( + value: JsonValue | undefined, + exemplar?: JsonValue, +): number | undefined => + integerOf(value) ?? maskedCount(value) ?? outgrownMask(value, exemplar); + +/** A number written into a mask's digit positions, zero-padded; digits that outgrow the mask lead it. */ +const fillMask = (mask: string, value: number): string => { + const padded = String(value).padStart(mask.replace(/\D/gu, "").length, "0"); + let next = padded.length - 1; + let out = ""; + for (let at = mask.length - 1; at >= 0; at -= 1) { + const character = mask[at]!; + if (/\d/u.test(character)) { + out = padded[next]! + out; + next -= 1; + } else out = character + out; + } + return padded.slice(0, next + 1) + out; +}; /** Lower case, with every run of other characters joined by one `-` and none at either end. */ export const slugOf = (text: string): string => diff --git a/src/domain/analysisView/environmentStateWrites.test.ts b/src/domain/analysisView/environmentStateWrites.test.ts index 1c3ba49824..43f40c1fe6 100644 --- a/src/domain/analysisView/environmentStateWrites.test.ts +++ b/src/domain/analysisView/environmentStateWrites.test.ts @@ -6,17 +6,24 @@ import { type TimedCaptureRow, } from "../../../tests/fixtures/analysisView.js"; import { + ASSET_ROWS, BOOK_INIT_ROWS, BOOK_NOT_FOUND, BOOK_ROWS, BOOK_WRITE_CLOCK, + DETAILED_LIST_ROWS, FIRST_BOOK, + OMITTED_FIELD_ROWS, + PROJECTED_ROWS, + RENAMED_BOOK, LIST_LINE_ROWS, QUERY_ROWS, SECOND_BOOK, THIRD_BOOK, + assetBook, book, bookClock, + detailedBook, environmentStateViewOf, rawCaptureEvidence, renamedBook, @@ -173,6 +180,216 @@ describe("served write plans", () => { }); }); +describe("wider write coverage", () => { + it("drops a field from an item view's required ones once a read shows the application leaves it out", () => { + const state = stateOf(OMITTED_FIELD_ROWS); + expect(planOf(state, "post_books")).toMatchObject({ served: true }); + expect( + state.views.find((view) => view.tool === "get_books_by_id"), + ).toMatchObject({ + shape: "item", + fields: expect.arrayContaining(["detail"]), + }); + expect( + state.views.find((view) => view.tool === "get_books_by_id"), + ).not.toMatchObject({ + required_fields: expect.arrayContaining(["detail"]), + }); + }); + + it("proves an item's identity past a read that asked for some fields only, leaving that read to replay", () => { + expect( + stateOf(PROJECTED_ROWS).views.find( + (view) => view.tool === "get_books_by_id", + ), + ).toMatchObject({ + shape: "item", + checked_record_ordinals: [1, 2], + projected_record_ordinals: [3], + }); + }); + + it("serves creates whose number follows the largest held one in its digit groups", () => { + expect(planOf(stateOf(ASSET_ROWS), "post_books")).toMatchObject({ + served: true, + rules: expect.arrayContaining([ + { field: "asset", rule: "sequence", scope: "collection" }, + ]), + }); + }); +}); + +describe("write coverage limits", () => { + const WHOLE_BOOK = ["slug", "created_at", "updated_at"]; + const itemView = (rows: readonly TimedCaptureRow[]) => + stateOf(rows).views.find((view) => view.tool === "get_books_by_id"); + + it("notes no omission from a read that asked for some fields, nor from one of a book the state no longer holds", () => { + const projected: TimedCaptureRow[] = [ + ...BOOK_INIT_ROWS, + ["PATCH", "/api/books/1", RENAMED_BOOK, 200, { name: "Renamed Book" }], + ["GET", "/api/books/1?fields=id,name", { id: 1, name: "Renamed Book" }], + ]; + const tombstone: TimedCaptureRow[] = [ + ...BOOK_INIT_ROWS, + ["POST", "/api/books", THIRD_BOOK, 201, { name: "Third Book" }], + ["GET", "/api/books/3", THIRD_BOOK], + ["DELETE", "/api/books/3", "", 204], + ["GET", "/api/books/3", { id: 3, deleted: true }], + ]; + for (const rows of [projected, tombstone]) + expect(itemView(rows)).toMatchObject({ + required_fields: expect.arrayContaining(WHOLE_BOOK), + }); + const partial = { id: 3, name: "Third Book" }; + const created: TimedCaptureRow[] = [ + ...BOOK_INIT_ROWS, + ["POST", "/api/books", partial, 201, { name: "Third Book" }], + ["GET", "/api/books/3?fields=id,name", partial], + ]; + expect(planOf(stateOf(created), "post_books")).toMatchObject({ + served: false, + }); + }); + + it("stops requiring a field in every view of the type once a read shows it left out", () => { + const state = stateOf(DETAILED_LIST_ROWS); + expect(planOf(state, "post_books")).toMatchObject({ served: true }); + const list = state.views.find((view) => view.tool === "get_books"); + expect(list).toMatchObject({ fields: expect.arrayContaining(["detail"]) }); + expect(list).not.toMatchObject({ + required_fields: expect.arrayContaining(["detail"]), + }); + }); + + it("leaves a read that asked for some fields only out of the gate, even where it would resynchronise a book", () => { + const rows: TimedCaptureRow[] = [ + ...BOOK_INIT_ROWS, + ["GET", "/api/books/2?fields=name", { name: "Second Book" }], + [ + "PATCH", + "/api/books/2", + renamedBook(SECOND_BOOK, "Renamed", 5), + 200, + { name: "Renamed" }, + ], + // A redirected write leaves book 2 unknown. + ["PATCH", "/api/books/2", "", 302, { name: "Renamed" }], + ["GET", "/api/books/2?fields=slug", { slug: "renamed" }], + ]; + expect(planOf(stateOf(rows), "patch_books_by_id")).toMatchObject({ + contradicted_by: null, + }); + }); + + it("learns no update's effect from a read-back that asked for some fields only", () => { + const rows: TimedCaptureRow[] = [ + ...BOOK_INIT_ROWS, + ["PATCH", "/api/books/1", "", 204, { name: "Renamed Book" }], + ["GET", "/api/books/1?fields=id,name", { id: 1, name: "Renamed Book" }], + ["GET", "/api/books/1", RENAMED_BOOK], + ["GET", "/api/books", [RENAMED_BOOK, SECOND_BOOK]], + ]; + expect(planOf(stateOf(rows), "patch_books_by_id")).toMatchObject({ + served: true, + observation: { ordinal: 5 }, + }); + }); + + it("confirms a delete from a read that asked for some fields and found no book", () => { + const rows: TimedCaptureRow[] = [ + ...BOOK_INIT_ROWS, + ["DELETE", "/api/books/2", "", 204], + ["GET", "/api/books/2?fields=id,name", BOOK_NOT_FOUND, 404], + ]; + expect(planOf(stateOf(rows), "delete_books_by_id")).toMatchObject({ + served: true, + checked_record_ordinals: [4], + }); + }); + + it("follows a sequence past its mask's width, as serving spells it", () => { + const numbered = (id: number, asset: string): Book => + assetBook(book(id, `Book ${String(id)}`, bookClock(id + 2)), asset); + const rows: TimedCaptureRow[] = [ + ["GET", "/api/books", [numbered(1, "99-97"), numbered(2, "99-98")]], + ["GET", "/api/books/1", numbered(1, "99-97")], + ["GET", "/api/books/2", numbered(2, "99-98")], + ]; + for (const [id, asset] of [ + [3, "99-99"], + [4, "100-00"], + [5, "100-01"], + ] as const) + rows.push( + [ + "POST", + "/api/books", + numbered(id, asset), + 201, + { name: `Book ${String(id)}` }, + ], + ["GET", `/api/books/${String(id)}`, numbered(id, asset)], + ); + expect(planOf(stateOf(rows), "post_books")).toMatchObject({ + served: true, + rules: expect.arrayContaining([ + { field: "asset", rule: "sequence", scope: "collection" }, + ]), + }); + }); + + it("withholds creates whose later call spells its sequence otherwise than serving would", () => { + const spelled = (asset: JsonValue): TimedCaptureRow[] => { + const fourth = { ...book(4, "Fourth Book", bookClock(5)), asset }; + return [ + ...ASSET_ROWS.slice(0, 5), + ["POST", "/api/books", fourth, 201, { name: "Fourth Book" }], + ["GET", "/api/books/4", fourth], + ]; + }; + for (const asset of ["00-04", 4]) + expect(planOf(stateOf(spelled(asset)), "post_books")).toMatchObject({ + served: false, + }); + expect(planOf(stateOf(spelled("000-004")), "post_books")).toMatchObject({ + served: true, + }); + }); +}); + +describe("omissions shown by reads the gate ignores", () => { + it("notes no omission from a read begun while the create still ran, nor from a page the collection outgrew", () => { + const first = detailedBook(FIRST_BOOK); + const second = detailedBook(SECOND_BOOK); + const running: TimedCaptureRow[] = [ + ["GET", "/api/books", [first, second]], + ["GET", "/api/books/1", first], + ["GET", "/api/books/2", second], + // The create runs from second 3 to 4.5, so the read at second 4 may show the book before its detail. + ["POST", "/api/books", THIRD_BOOK, 201, { name: "Third Book" }, 1500], + ["GET", "/api/books/3", THIRD_BOOK], + ]; + const outgrown: TimedCaptureRow[] = [ + ["GET", "/api/books?_start=0&_end=2", [second, first]], + ["GET", "/api/books/1", first], + ["GET", "/api/books/2", second], + ["POST", "/api/books", THIRD_BOOK, 201, { name: "Third Book" }], + // Serving answers the range from its recording once the collection outgrows it. + ["GET", "/api/books?_start=0&_end=2", [THIRD_BOOK, second]], + ]; + const requiresDetail = { + required_fields: expect.arrayContaining(["detail"]), + }; + const itemView = (rows: readonly TimedCaptureRow[]) => + stateOf(rows).views.find((view) => view.tool === "get_books_by_id"); + expect([running, outgrown].map(itemView)).toMatchObject([ + requiresDetail, + requiresDetail, + ]); + }); +}); + describe("unserved write plans", () => { it("names a write to no state type and a tool with no successful call", () => { expect( diff --git a/src/domain/analysisView/environmentStateWrites.ts b/src/domain/analysisView/environmentStateWrites.ts index b0c04eebdf..1628104a4f 100644 --- a/src/domain/analysisView/environmentStateWrites.ts +++ b/src/domain/analysisView/environmentStateWrites.ts @@ -24,20 +24,28 @@ const clockStart = ( : null; }; -/** The write plans a capture's own timeline confirms against an extracted state, and the served clock's start. */ +/** + * The write plans a capture's own timeline confirms against an extracted state, the served clock's + * start, and the views without the fields reads showed the application leaves out. + */ export const extractWrites = ( state: Pick, input: Omit< WriteSimulationInput, "views" | "types" | "entities" | "documents" >, -): Pick => ({ - writes: new EnvironmentStateWriteSimulation({ +): Pick => { + const simulation = new EnvironmentStateWriteSimulation({ ...input, views: new Map(state.views.map((view) => [view.tool, view])), types: state.types, entities: state.entities, documents: state.documents, - }).run(), - write_clock: clockStart(input.records, input.timed), -}); + }); + const writes = simulation.run(); + return { + writes, + write_clock: clockStart(input.records, input.timed), + views: simulation.omitted.views(state.views), + }; +}; diff --git a/tests/fixtures/environmentStateWrites.ts b/tests/fixtures/environmentStateWrites.ts index d646ca46bb..bc05618f12 100644 --- a/tests/fixtures/environmentStateWrites.ts +++ b/tests/fixtures/environmentStateWrites.ts @@ -94,6 +94,62 @@ export const BOOK_ROWS: readonly TimedCaptureRow[] = [ ["POST", "/api/exports", { queued: true }, 202, { format: "csv" }], ]; +/** A book whose item reads carry a `detail` member, which the application leaves out of books that have none. */ +export const detailedBook = (entity: Book): Book => ({ + ...entity, + detail: "x", +}); + +/** Seeded books read with `detail`; a created book read back without it. */ +export const OMITTED_FIELD_ROWS: readonly TimedCaptureRow[] = [ + ["GET", "/api/books", [FIRST_BOOK, SECOND_BOOK]], + ["GET", "/api/books/1", detailedBook(FIRST_BOOK)], + ["GET", "/api/books/2", detailedBook(SECOND_BOOK)], + ["POST", "/api/books", THIRD_BOOK, 201, { name: "Third Book" }], + ["GET", "/api/books/3", THIRD_BOOK], +]; + +/** Seeded books listed and read with `detail`; a created book read back by item alone, without it. */ +export const DETAILED_LIST_ROWS: readonly TimedCaptureRow[] = [ + ["GET", "/api/books", [detailedBook(FIRST_BOOK), detailedBook(SECOND_BOOK)]], + ["GET", "/api/books/1", detailedBook(FIRST_BOOK)], + ["GET", "/api/books/2", detailedBook(SECOND_BOOK)], + ["POST", "/api/books", THIRD_BOOK, 201, { name: "Third Book" }], + ["GET", "/api/books/3", THIRD_BOOK], +]; + +/** Item reads of both books, one of them asking for its name only, which leaves its id out. */ +export const PROJECTED_ROWS: readonly TimedCaptureRow[] = [ + ["GET", "/api/books", [FIRST_BOOK, SECOND_BOOK]], + ["GET", "/api/books/1", FIRST_BOOK], + ["GET", "/api/books/2", SECOND_BOOK], + ["GET", "/api/books/2?fields=name", { name: "Second Book" }], +]; + +/** A book numbered as an asset, in digit groups. */ +export const assetBook = (entity: Book, asset: string): Book => ({ + ...entity, + asset, +}); + +const ASSET_THIRD = assetBook(THIRD_BOOK, "000-003"); +const ASSET_FOURTH = assetBook(book(4, "Fourth Book", bookClock(5)), "000-004"); + +/** Two creates whose asset number follows the largest held one, in its digit groups. */ +export const ASSET_ROWS: readonly TimedCaptureRow[] = [ + [ + "GET", + "/api/books", + [assetBook(FIRST_BOOK, "000-001"), assetBook(SECOND_BOOK, "000-002")], + ], + ["GET", "/api/books/1", assetBook(FIRST_BOOK, "000-001")], + ["GET", "/api/books/2", assetBook(SECOND_BOOK, "000-002")], + ["POST", "/api/books", ASSET_THIRD, 201, { name: "Third Book" }], + ["GET", "/api/books/3", ASSET_THIRD], + ["POST", "/api/books", ASSET_FOURTH, 201, { name: "Fourth Book" }], + ["GET", "/api/books/4", ASSET_FOURTH], +]; + /** A page of books as a paging API wraps it, counting what it lists. */ export const bookPage = (books: readonly Book[]) => ({ content: [...books],