diff --git a/.agents/skills/reverse-engineer-anything/SKILL.md b/.agents/skills/reverse-engineer-anything/SKILL.md index 1eb5366684..3c6f4e7e06 100644 --- a/.agents/skills/reverse-engineer-anything/SKILL.md +++ b/.agents/skills/reverse-engineer-anything/SKILL.md @@ -2,7 +2,7 @@ name: reverse-engineer-anything description: Reverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. metadata: - version: "54" + version: "55" --- # REA diff --git a/.agents/skills/reverse-engineer-anything/references/environments.md b/.agents/skills/reverse-engineer-anything/references/environments.md index 7d2ededf46..0be64fbab2 100644 --- a/.agents/skills/reverse-engineer-anything/references/environments.md +++ b/.agents/skills/reverse-engineer-anything/references/environments.md @@ -161,8 +161,11 @@ locators to exactly one element. 5. Update both objects and read each back, changing the fields a person would change (a name, an email address, a status, a due date) rather than only a description: a served update accepts only the members a recorded - update stored. Delete both and read each back (expect 404, or the object - marked deleted) and the list. + update stored. Run an action that turns something on or off (star, + bookmark, archive) twice in a row on one object, reading it back after + each: the second call shows whether it sets or flips the field, and the + environment serves the action only once a call has shown it. Delete both + and read each back (expect 404, or the object marked deleted) and the list. 6. Vary what the two creates send, however you write them: the first every field the application takes, the second only what a person must give (its title or name), with the other fields left out or empty. A served create diff --git a/docs/agent-environments.md b/docs/agent-environments.md index 62dbb3d76e..8bcc62be86 100644 --- a/docs/agent-environments.md +++ b/docs/agent-environments.md @@ -739,14 +739,14 @@ tools are not served, so no call claims a change it does not make; each one's plan states why in its `reason`. A served write changes the live state, which every MCP session and `data/get` then see: -| Effect | Call | -| ------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `create` | Adds an entity to the collection the path names, built from the observed entity with each ruled field taken from the call or generated, at the plan's `insert` position. A plan tied to its exemplar's collection writes only there. | -| `update` | Sets the sent members on the entity the path names, constants for members accepted only with their recorded value, and the clock, nested clock, increment, slug, stamp, derived, composed, reference, unique and renewed changes the plan models. | -| `update` (action) | A `POST`, `PUT` or `PATCH` to an item path plus an action segment (`/{type}/{id}/archive`) sets, on the entity before the segment, each scalar the recorded calls showed the action set (`set`), with any sent members; it is served only when a recorded call found those fields already set, so a toggle never is. | -| `delete` | Removes the entity the path names. | -| bulk `update` or `delete` | A `DELETE`, `PUT` or `PATCH` at a collection, or a `POST` to it plus `delete`, `remove` or `destroy`, applies its effect to every entity the body names, as recorded: an array of ids or of objects carrying the id field, at the root or as the body's one array member (`{"IDs": […], "Read": true}`). | -| `document` | Merges the body into the plan's document object, or replaces it for `PUT`. | +| Effect | Call | +| ------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `create` | Adds an entity to the collection the path names, built from the observed entity with each ruled field taken from the call or generated, at the plan's `insert` position. A plan tied to its exemplar's collection writes only there. | +| `update` | Sets the sent members on the entity the path names, constants for members accepted only with their recorded value, and the clock, nested clock, increment, slug, stamp, derived, composed, reference, unique and renewed changes the plan models. | +| `update` (action) | A `POST`, `PUT` or `PATCH` to an item path plus an action segment (`/{type}/{id}/archive`) sets, on the entity before the segment, each scalar the recorded calls showed the action set (`set`), or flips each boolean they showed it flipping both ways (`toggle`), with any sent members; it is served only when a recorded call found what it sets already set, or flipped back what it toggles. | +| `delete` | Removes the entity the path names. | +| bulk `update` or `delete` | A `DELETE`, `PUT` or `PATCH` at a collection, or a `POST` to it plus `delete`, `remove` or `destroy`, applies its effect to every entity the body names, as recorded: an array of ids or of objects carrying the id field, at the root or as the body's one array member (`{"IDs": […], "Read": true}`). | +| `document` | Merges the body into the plan's document object, or replaces it for `PUT`. | An update's unsent changes follow the plan's rules. A `held-clock` is a clock the client sends back as it read it and the application stamps anew (Ghost's @@ -757,7 +757,9 @@ the entity already names under its `created…by` counterpart into a `modified…by` or `updated…by` field. A `derived` field takes a value showing the one sent member's text that the entity stores nowhere (a field Ghost derives from `html`), only with that member's recorded value. An action's -`set` rule sets a scalar to the value the recorded calls left it at. +`set` rule sets a scalar to the value the recorded calls left it at, and its +`toggle` rule flips a boolean, refusing a call on an entity that holds none +there (Miniflux's `PUT /v1/entries/{id}/bookmark` stars and unstars). Its answer has the status and content type of the plan's recorded response, with the written entity or its identity rendered in, no body, or the recorded diff --git a/docs/specs/agent-environments/write-sorts-references-toggles/plan.md b/docs/specs/agent-environments/write-sorts-references-toggles/plan.md new file mode 100644 index 0000000000..80cfd70c25 --- /dev/null +++ b/docs/specs/agent-environments/write-sorts-references-toggles/plan.md @@ -0,0 +1,11 @@ +# Implementation plan + +1. **Sorted creates.** Placed creates whose confirming read was a whole listing in collection order may be placed + again; narrowing keeps the sorts a later listing follows and adds those its items show, the type's id field last. +2. **Listed references.** `referenceRule` reads a sent list of ids; `many` on the reference rule and its schema; + `referencedValue` maps each id; the create merge lets an empty reference agree and frees the members it fixed. +3. **Hidden bodies.** Scenario records give a request with a body Playwright did not expose, and no `Content-Type`, a + `bodySize` of 0. +4. **Toggles.** `toggledRules` turns a plan's `set` and a later call's opposite `set` into `toggle`; the rule's schema, + serving, later-call checks and the action verdict. +5. **Skill, tests and docs.** diff --git a/docs/specs/agent-environments/write-sorts-references-toggles/spec.md b/docs/specs/agent-environments/write-sorts-references-toggles/spec.md new file mode 100644 index 0000000000..75e1fa64b3 --- /dev/null +++ b/docs/specs/agent-environments/write-sorts-references-toggles/spec.md @@ -0,0 +1,41 @@ +# Serve sorted creates, listed references and toggles + +Issue: https://github.com/morluto/rea/issues/4933 + +## Outcome + +Over 148 recorded captures (the 130 measured before and 18 from a fresh recording round), #7539 withholds 302 of 694 +write plans. A short diagnosis of the largest clusters found three gaps REA could close: + +- **A create's sort learned too late.** Grafana lists folders by title. A recording's first scratch folder sorted + after the one seeded folder, so the plan placed creates last; the second sorted before the first, the list after it + contradicted the plan, and the update and delete after that went unconfirmed. +- **Lists of references.** Homebox creates an item from `parentId` and `tagIds` and answers with the parent and tag + objects. REA modelled a parent but not a list of tags, and a create that sent `null` and `[]` disagreed with one that + named both. +- **Toggles.** Miniflux stars and unstars an entry with a body-less `PUT /v1/entries/{id}/bookmark`. Chromium exposes + no body for it, which REA read as unusable, and an action flipping a flag back was taken to contradict the call that + set it. + +## Behaviour + +- **Sorted creates.** A create a whole listing in collection order placed last is placed again by a field a later such + listing shows it sorted by, the type's id field last. A sort an earlier such listing did not follow stays ruled out, + though the entity that showed it is gone. A create another read placed keeps its place. +- **Listed references.** A field holding one object per id a sent list names, in that order, is a `reference` with + `many`. A create that sent the member as `null` or `[]` and stored the field so agrees with one that named entities, + whichever create first showed the reference; + serving stores the entities a call names, or the empty value as that create did. A `null` id stays refused where + no recorded create sent one, and a call leaving the member out stores no field where the observed create showed + none. +- **Hidden bodies.** A scenario request whose body Playwright did not expose and that names no `Content-Type` sent no + body. +- **Toggles.** An item action a later call shows flipping back a boolean an earlier call set, when no call found it + already set, has a `toggle` rule: serving flips the field, refusing an entity that holds no boolean there. +- **Skill.** Recordings run an action twice in a row on one object, reading it back after each (skill 55). + +Plans and views that need none of this are unchanged. + +## Acceptance + +- No write of the 148 captures goes from served to withheld, and no view changes. diff --git a/docs/specs/agent-environments/write-sorts-references-toggles/tasks.md b/docs/specs/agent-environments/write-sorts-references-toggles/tasks.md new file mode 100644 index 0000000000..3fb6efc9ea --- /dev/null +++ b/docs/specs/agent-environments/write-sorts-references-toggles/tasks.md @@ -0,0 +1,11 @@ +# Tasks + +- [x] Place a create again by the sort a later whole listing shows. +- [x] Learn and serve references a sent list of ids names, and let an empty reference agree. +- [x] Treat a hidden request body without a `Content-Type` as absent. +- [x] Learn, check and serve toggles. +- [x] Ask recordings to run an action twice on one object. +- [x] Add view, serving, record and unit tests. +- [x] Document the rules. +- [x] Measure the recorded captures. +- [x] Run static, formatting, documentation and test gates. diff --git a/docs/specs/agent-environments/write-sorts-references-toggles/verification.md b/docs/specs/agent-environments/write-sorts-references-toggles/verification.md new file mode 100644 index 0000000000..c499b2beca --- /dev/null +++ b/docs/specs/agent-environments/write-sorts-references-toggles/verification.md @@ -0,0 +1,88 @@ +# Verification + +Executed on macOS with Node v24.22.1 on this branch and on #7539 at `30e2a582`; the census and the base reproduction +were repeated after merging #7539 at `2817665d`, `174a0ff2`, `5691f97f` (which merges main with #7332) +and `3b800a10`. + +## Diagnosis + +Over 148 recorded captures, #7539 withholds 302 of 694 write plans. The largest clusters: Homebox item writes (36), +unconfirmed deletes (23), Grafana's feature-flag evaluation calls (15, which change no state), Linkding renumbering +bundles after a delete (7) and Miniflux's bookmark toggle (5). + +- **Grafana.** A fresh recording created "Scratch Pier 3", which the list after it showed after the one seeded folder, + then "Scratch Meadow Notes 4", which the list after it showed before the first, by title. The create plan had settled + on `last` when the first list confirmed it, so the second list contradicted it, and the folder update and delete + after it went unconfirmed. +- **Homebox.** Item creates send `parentId` and `tagIds` and answer with `parent` and `tags` objects. The recording + that created one item with neither (`null` and `[]`) and one with both was withheld: `tags` had no rule, and the + first create fixed both members to their empty values. +- **Miniflux.** `PUT /v1/entries/{id}/bookmark` sends no body and answers `204`. Chromium exposes no body for it and + the request names no `Content-Type`; REA read the body as unusable. Across the captures, every `POST`, `PUT` or + `PATCH` whose body was captured names `content-type` among its request header names (835 of 835). + +## Gates + +- `npm run -s check:fast`, `format:check`, `knip`, `scan:todos`, `docs:check`, `verify:test-discovery`, + `mcp-catalog:check` and `jscpd`: passed. +- `npx vitest run --project domain --project services`: 11,886 passed, 6 skipped, merged with `3b800a10`. An earlier run of the suite failed a + `SetupRegistrationPrewarm` runner test whose child process outran its 5-second timeout; alone it passes on this + branch and on #7539 (3.0 to 4.7 seconds). +- `npm run -s test:focused --` the environment build, export and serve CLI boundaries, the analysis-view CLI and MCP + boundaries and the environment serve MCP boundary: 7 files, 81 passed. `outputSchemaProfile`, `toolSchemaValidity`, + `mcpSchemaValidation`, `outputSchemaSemanticConstraints` and `analysisViewMcp`: 5 files, 54 passed; the compact + catalog stays under its frame. + +## Tests + +- View: a create placed last, then a create a later whole listing shows before it by title, learns `title` ascending + and its delete is served; an item create naming its parent and tags by id, after one naming none, learns + `reference` rules for `parent` and for `tags` (`many`) and is served; a body-less action two calls show starring + then unstarring an entry learns `toggle` and is served, and is not when its calls flip the flag one way only or a + later call leaves it unchanged. +- Serving: a create stores the parent and the tags its ids name, in the order sent; a create leaving out both, or + sending them as `null` and `[]`, stores `null` and `[]`; a tag the environment does not hold is refused, and so is a + `null` shelf where every recorded create named one; an entry the recording never starred flips on each call. +- Records: a hidden request body is absent when the request names no `Content-Type`, and stays unusable when it names + one. +- The nine behaviour tests fail on #7539 at `30e2a582`, `2817665d`, `174a0ff2`, `5691f97f` and `3b800a10`; the three that guard toggles and `null` ids + pass there too. + +## Write plans and views over recorded captures + +The 148 captures, built by #7539 and by this branch: 694 plans, 392 served and 399, both at `30e2a582` and after +merging `2817665d`, `174a0ff2`, `5691f97f` and +`3b800a10`. + +- No write goes from served to withheld, and no served plan changes. Newly served: Grafana's folder create, update and + delete in a fresh recording; Homebox's item create in another; Miniflux's bookmark toggle in three recordings. +- Five withheld plans explain more. Two Miniflux recordings learn the toggle but stay withheld: a list of starred + entries with no `order` parameter lists them by `published_at`, which REA answers in state order. Linkding's + unarchive learns its body-less call and stays `unconfirmed`, seen once. Two Homebox item writes learn their tag + references and stay withheld for other members. +- Views are unchanged: 3,188 views across the captures. + +## Review round + +Greptile's review of `4e777d28` raised five findings. Two were in this branch's code and are fixed: + +- A sort an earlier whole listing ruled out could come back from a later listing once the entity that ruled it out was + deleted, and a create's retry could adopt it. Each create now keeps the sorts its whole listings did not follow. +- A create sending a reference's member as `null` after a later create added the reference, the first having shown no + such field, was withheld: the empty-reference agreement now applies there too. + +Both tests fail on `4e777d28` and on #7539 at `174a0ff2`; the census is unchanged. The other three were in #7332's and +#7539's code: `308aebf0` (#7332), already here, covers the made-object count, and #7539's `174a0ff2`, merged here, +fixes the update default and the read clock spelling. + +Greptile's review of `f7c1bfe9` raised one more, fixed: once a later create sent a reference's member as `null`, a +served create leaving the member out stored `null`, though the create the plan observes left it out and stored no such +field. Serving now leaves the field out there, and still stores `null` for a call that sends it; the serving test fails +on `f7c1bfe9`. The census is unchanged. + +## Not verified + +- No environment was rebuilt and run with agents for this branch, and no recording was made with skill 55. +- The order an application gives a list read with no sort parameter (Miniflux's `published_at`) when it is not state + order. +- The rest of the Homebox cluster: `assetId` numbering and item updates; Linkding renumbering bundles after a delete. diff --git a/docs/web-network-captures.md b/docs/web-network-captures.md index 3f6573262c..cc8f7d9aa8 100644 --- a/docs/web-network-captures.md +++ b/docs/web-network-captures.md @@ -740,7 +740,9 @@ one-element array member) that a later item read of a type first shows (served only with the path values its recorded call sent, unless it creates into the type's own collection, whose path has no parameters, and a `path` rule stores each of its path values), and otherwise `none` with reason `no-state-target`. A plan is built from the tool's first successful call: its -request body must be a usable JSON object (as response bodies are usable), and +request body must be a usable JSON object (as response bodies are usable), or +absent for an item action (a scenario request is absent when Playwright exposed +no body and it names no `Content-Type`, which the bodies browsers send carry), and the written entity must show in that call's response or in a later read before the next write. Each field of a created entity gets the first rule that explains it: generated `identity` (a sequence, UUID or token, by the recorded @@ -780,15 +782,19 @@ with the entity leaves out a field only later reads showed, which no initial entity holds, when the entity does not hold it, as Vikunja's reads of tasks it did not create show no `subscription`. A `reference` is an object holding fields of the -entity a sent member names, as the state holds it (a parent beside `parentId`): +entity a sent member names, as the state holds it (a parent beside `parentId`), +or a list of them, one per id a sent list names, in its order (`many`, tags +beside `tagIds`): the member is accepted with any value, serving looks the entity up and refuses a call naming one the environment does not hold or holds in more than one collection, and the field is absent, from the entity and the create's answer, when the call does not send the member. An id naming entities of one type in several collections teaches no reference. Each later create widens its plan when, explained on its own, it agrees with it: both calls show the same fields -but for a reference whose member a call without it did not send (one an earlier -create added stays), every field both rule they rule alike (an echo one showed +but for a reference whose member a call without it did not send, or sent as +`null` or `[]` and stored so (one an earlier create added stays, and a call may +send the member empty as that create did; one leaving it out stores no such field +when the create the plan observes showed none), every field both rule they rule alike (an echo one showed sorted stays sorted, unless a call stored its strings out of order as sent), and a field neither rules holds one value, unless the later call's rule for it spells the earlier calls' value too: a `composed` or `template` string, or a @@ -868,14 +874,17 @@ is unchanged) must be a clock, a nested clock, an increment, a slug, a fresh `un value, a `stamp` of the acting user, a value `derived` from the one member it stores nowhere, a `reference` to the entity a sent member names (kept by a call that names the entity the field already repeats), a `composed` string whose parts read members it stored as sent or fields it left as they were and spell the value before the call too, or for -an item action a `set` of a scalar to the value it showed, or, for a `PUT`, a +an item action a `set` of a scalar to the value it showed (a `toggle` once a +later call flips back the boolean an earlier one set, when no call found it set +already), or, for a `PUT`, a member it left out that the application cleared to an empty value, which calls that leave it out then store (`body.defaults`) as long as every call that left it out stored that value (a call naming a `reference` field's entity by its member does not leave the field out), or the plan is `unexplained-change`; an increment, slug, stamp, derived -value or set must hold for every call of the tool, and an action is served -only once a call found what it sets already set. A write is served only +value, set or toggle must hold for every call of the tool, and an action is +served only once a call found what it sets already set, or flipped back what it +toggles. A write is served only with the query its first call sent, which serving applies to a call that leaves a parameter out and the catalog advertises as required. Every member every call sent is required, except by `PATCH` and `POST` updates and members a create sent empty. The state holds an update's @@ -919,7 +928,10 @@ contradicted it; otherwise its `reason` is the first of its build reason (`no-call`, `body-unusable`, `no-observation`, `unexplained-change`), `contradicted` (with `contradicted_by`), `side-effect`, `unconfirmed` and `incomplete-entities` (an update or delete that answers with the entity, when no initial entity holds every field its recorded answer shows but those only later reads showed, counting for an update the fields its rules write, so it could answer only for entities the environment created). An otherwise served plan whose learned rules generate clock values is withheld with `unknown-clock` when no global clock start is established; its learned rules and confirming reads remain retained. `checked_record_ordinals` lists the confirming reads, `insert` places created -entities (sorted by a field the collection's recorded order shows, else last), +entities (sorted by a field the collection's recorded order shows, else last; +a create a whole listing in collection order placed last is placed again by a +field a later such listing shows it sorted by and no earlier one ruled out, the +type's id field last), and `write_clock` is the end of the last timed record, to the next second, when all request completions are known; otherwise it is null. An update that writes the field its type's creates are sorted by moves its entity diff --git a/src/application/environment/environmentStateItemWritesAnswer.test.ts b/src/application/environment/environmentStateItemWritesAnswer.test.ts new file mode 100644 index 0000000000..fdc21025c5 --- /dev/null +++ b/src/application/environment/environmentStateItemWritesAnswer.test.ts @@ -0,0 +1,101 @@ +import { describe, expect, it } from "vitest"; + +import { + ADDED_EMPTY_REFERENCE_ROWS, + BOOKMARK_TOGGLE_ROWS, + ITEM_LOCATIONS, + ITEM_REFERENCES_ROWS, + ITEM_TAGS, +} from "../../../tests/fixtures/environmentStateItemWrites.js"; +import { FILED_ROWS } from "../../../tests/fixtures/environmentStateDerivedValues.js"; +import { servedFrom } from "../../../tests/fixtures/environmentStateServing.js"; + +describe("served item writes", () => { + it("answers a create with the parent and tags its ids name, in the order sent", () => { + const items = servedFrom(ITEM_REFERENCES_ROWS); + const body = items.answer("post_items", { + body: { + name: "Snow shovel", + parentId: ITEM_LOCATIONS[0]!["id"]!, + tagIds: [ITEM_TAGS[1]!["id"]!, ITEM_TAGS[0]!["id"]!], + }, + }).body; + expect(body).toMatchObject({ + name: "Snow shovel", + parent: ITEM_LOCATIONS[0], + tags: [ITEM_TAGS[1], ITEM_TAGS[0]], + }); + }); + + it("answers a create naming no parent and no tags with none", () => { + const items = servedFrom(ITEM_REFERENCES_ROWS); + expect( + items.answer("post_items", { body: { name: "Spare fuse" } }).body, + ).toMatchObject({ + name: "Spare fuse", + parent: null, + tags: [], + }); + }); + + it("answers a create sending no parent and no tags as a recorded create sent them", () => { + const items = servedFrom(ITEM_REFERENCES_ROWS); + expect( + items.answer("post_items", { + body: { name: "Spare fuse", parentId: null, tagIds: [] }, + }).body, + ).toMatchObject({ name: "Spare fuse", parent: null, tags: [] }); + }); + + it("answers a create leaving out a parent the first create left out with none, and one sending null with null", () => { + const items = servedFrom(ADDED_EMPTY_REFERENCE_ROWS); + const left = items.answer("post_items", { body: { name: "Spare fuse" } }) + .body as Record; + expect(left).toMatchObject({ name: "Spare fuse" }); + expect(Object.hasOwn(left, "parent")).toBe(false); + expect( + items.answer("post_items", { + body: { name: "Spare bulb", parentId: null }, + }).body, + ).toMatchObject({ name: "Spare bulb", parent: null }); + expect( + items.answer("post_items", { + body: { name: "Spare cord", parentId: ITEM_LOCATIONS[0]!["id"]! }, + }).body, + ).toMatchObject({ name: "Spare cord", parent: ITEM_LOCATIONS[0] }); + }); + + it("refuses a create naming no shelf when every recorded create named one", () => { + const books = servedFrom(FILED_ROWS); + expect( + books.refusal("post_books", { body: { name: "Loose", shelf_id: null } }), + ).toMatchObject({ message: expect.stringContaining("shelf_id") }); + }); + + it("refuses a create naming a tag the environment does not hold", () => { + const items = servedFrom(ITEM_REFERENCES_ROWS); + expect( + items.refusal("post_items", { + body: { + name: "Mystery", + tagIds: ["3b9f2a10-0009-4000-8000-000000000009"], + }, + }), + ).toMatchObject({ message: expect.stringContaining("tagIds") }); + }); + + it("flips an entry's flag on each body-less call", () => { + const entries = servedFrom(BOOKMARK_TOGGLE_ROWS); + const starred = () => + ( + entries.answer("get_entries_by_id", { id: "2" }).body as { + starred: boolean; + } + ).starred; + expect(starred()).toBe(false); + entries.answer("put_entries_bookmark_by_id", { id: "2" }); + expect(starred()).toBe(true); + entries.answer("put_entries_bookmark_by_id", { id: "2" }); + expect(starred()).toBe(false); + }); +}); diff --git a/src/domain/analysisView/EnvironmentStateWriteSimulation.ts b/src/domain/analysisView/EnvironmentStateWriteSimulation.ts index 488deec68b..28c6527cc4 100644 --- a/src/domain/analysisView/EnvironmentStateWriteSimulation.ts +++ b/src/domain/analysisView/EnvironmentStateWriteSimulation.ts @@ -109,6 +109,26 @@ const keptInOrder = ( ); }); +/** The sorts by a field every listed item holds as a string or number, and whether the items follow each. */ +const listedSorts = ( + items: readonly JsonObject[], +): { + readonly rule: Extract; + readonly follows: boolean; +}[] => + [...new Set(items.flatMap((item) => Object.keys(item)))] + .filter((field) => + items.every((item) => + ["string", "number"].includes(typeof ownMember(item, field)), + ), + ) + .flatMap((field) => + (["ascending", "descending"] as const).map((direction) => ({ + rule: { rule: "sorted", field, direction } as const, + follows: orderedBy(items, field, direction), + })), + ); + /** Whether a list read's query sets a parameter the view's query model reads as a sort or its direction. */ const sortedByQuery = ( view: Extract, @@ -182,6 +202,10 @@ export class EnvironmentStateWriteSimulation { readonly #refutedSorts = new Map>(); /** Creates whose insert rule a passing list check relied on, so it is no longer changed. */ readonly #placed = new Set(); + /** Placed creates a read other than a whole listing in collection order placed, whose insert stays as it is. */ + readonly #settledInserts = new Set(); + /** Sorts a whole listing that placed a create did not follow, by create, which no later listing adds back. */ + readonly #unfollowedSorts = new Map>(); /** The order a read showed a type's collections sorted in after an update, when no create of it showed one. */ readonly #learnedSorts = new Map(); /** Entities writes no plan models made, taken when an item read first shows them. */ @@ -797,7 +821,9 @@ export class EnvironmentStateWriteSimulation { this.#drafts.get(name)!.confirmed.push(record.ordinal); if (showsPlacement(view, outcome, this.#inserted, name)) { this.#placed.add(name); - if (view.shape === "list") this.#narrowInserts(view, record, name); + // Only whole listings in collection order narrow the inserts a later list may still choose among. + if (view.shape !== "list" || !this.#narrowInserts(view, record, name)) + this.#settledInserts.add(name); } } return shown.unseen; @@ -937,23 +963,44 @@ export class EnvironmentStateWriteSimulation { }); } - /** A whole-collection read in collection order that placed a create leaves it only the sorted inserts whose order the read follows; one its query sorts shows the query's order instead. */ + /** A whole-collection read in collection order that placed a create leaves it only the sorted inserts whose order the read follows, adding those by a field its items all hold, and says whether it did; one its query sorts shows the query's order instead. */ #narrowInserts( view: Extract, record: StateRecordFacts, name: string, - ): void { - if (sortedByQuery(view, record)) return; + ): boolean { + if (sortedByQuery(view, record)) return false; const draft = this.#drafts.get(name)!; - const kept = this.#wholeListing(view, record, (items) => - draft.candidates.filter( + const kept = this.#wholeListing(view, record, (items) => { + const followed = draft.candidates.filter( (candidate) => candidate.rule !== "sorted" || same(candidate, draft.plan.insert) || orderedBy(items, candidate.field, candidate.direction), - ), - ); - if (kept !== undefined) draft.candidates = kept; + ); + const sorts = listedSorts(items); + const unfollowed = this.#unfollowedSorts.get(name) ?? new Set(); + for (const { rule, follows } of sorts) + if (!follows) unfollowed.add(JSON.stringify(rule)); + this.#unfollowedSorts.set(name, unfollowed); + // A list shown in the order of the field that names its items says less than one in a field people read. + const idField = this.#types.get(view.type)?.id_field; + const learned = sorts + .map(({ rule }) => rule) + .filter( + (rule) => + !unfollowed.has(JSON.stringify(rule)) && + !followed.some((candidate) => same(candidate, rule)), + ) + .sort( + (left, right) => + Number(left.field === idField) - Number(right.field === idField), + ); + return [...followed, ...learned]; + }); + if (kept === undefined) return false; + draft.candidates = kept; + return true; } /** @@ -1110,9 +1157,25 @@ export class EnvironmentStateWriteSimulation { if (creates.size !== 1) return undefined; const [name] = creates; const draft = this.#drafts.get(name!)!; - // An item read confirms the entity, not where a list places it. - if (!exercised.has(name!) || this.#placed.has(name!)) return undefined; - for (const candidate of draft.candidates.slice(1)) { + // An item read confirms the entity, not where a list places it; a read that placed it otherwise than a whole + // listing did settles where it goes. + if (!exercised.has(name!) || this.#settledInserts.has(name!)) + return undefined; + // A create whole listings placed may still take a sort they followed: one placed last fits it before the next shows the order. + const options = this.#placed.has(name!) + ? draft.candidates.filter( + (candidate) => + candidate.rule === "sorted" && !same(candidate, draft.plan.insert), + ) + : draft.candidates.slice(1); + // A sort an earlier update's whole list ruled out must not become this create's order: serving would move those updates by it. + const refuted = this.#refutedSorts.get(view.type); + for (const candidate of options) { + if ( + candidate.rule === "sorted" && + refuted?.has(JSON.stringify(candidate)) === true + ) + continue; this.#state.reinsert(view.type, line.collection, inserted, candidate); const outcome = this.#check(view, record); if (outcome?.pass === true) { @@ -1120,6 +1183,13 @@ export class EnvironmentStateWriteSimulation { return outcome; } } + if (draft.plan.insert !== null) + this.#state.reinsert( + view.type, + line.collection, + inserted, + draft.plan.insert, + ); return undefined; } } diff --git a/src/domain/analysisView/environmentStateItemWrites.test.ts b/src/domain/analysisView/environmentStateItemWrites.test.ts new file mode 100644 index 0000000000..e1fccc2b61 --- /dev/null +++ b/src/domain/analysisView/environmentStateItemWrites.test.ts @@ -0,0 +1,137 @@ +import { describe, expect, it } from "vitest"; + +import { + timedCaptureEvidence, + type TimedCaptureRow, +} from "../../../tests/fixtures/analysisView.js"; +import { + ADDED_EMPTY_REFERENCE_ROWS, + BOOKMARK_TOGGLE_ROWS, + CREATES_BY_TITLE_ROWS, + ITEM_REFERENCES_ROWS, + NULL_THEN_STARRED_ROWS, + RANKED_AFTER_DELETE_ROWS, + REFUTED_TITLE_SORT_ROWS, + feedEntry, +} from "../../../tests/fixtures/environmentStateItemWrites.js"; +import { environmentStateViewOf } from "../../../tests/fixtures/environmentStateWrites.js"; + +const planOf = (rows: readonly TimedCaptureRow[], tool: string) => { + const plan = environmentStateViewOf(timedCaptureEvidence(rows)).writes.find( + (candidate) => candidate.tool === tool, + ); + if (plan === undefined) throw new Error(`No plan for ${tool}`); + return plan; +}; + +describe("item writes", () => { + it("learns a create's sort from a later list its full order follows, after one create was placed last", () => { + expect(planOf(CREATES_BY_TITLE_ROWS, "post_folders")).toMatchObject({ + served: true, + insert: { rule: "sorted", field: "title", direction: "ascending" }, + }); + expect(planOf(CREATES_BY_TITLE_ROWS, "delete_folders_by_id")).toMatchObject( + { served: true }, + ); + }); + + it("keeps a sort an earlier list ruled out away from a create, after the entity that ruled it out is deleted", () => { + expect(planOf(RANKED_AFTER_DELETE_ROWS, "post_folders")).toMatchObject({ + served: true, + insert: { rule: "sorted", field: "position", direction: "ascending" }, + }); + }); + + it("serves a create sending no parent after an earlier create added the parent reference", () => { + const plan = planOf(ADDED_EMPTY_REFERENCE_ROWS, "post_items"); + expect(plan).toMatchObject({ served: true }); + expect(plan.rules).toContainEqual( + expect.objectContaining({ + field: "parent", + rule: "reference", + member: "parentId", + }), + ); + }); + + it("serves a create whose parent and tags a call names by id, one call naming none", () => { + const plan = planOf(ITEM_REFERENCES_ROWS, "post_items"); + expect(plan).toMatchObject({ served: true }); + expect(plan.rules).toContainEqual( + expect.objectContaining({ + field: "parent", + rule: "reference", + member: "parentId", + type: "/api/locations", + }), + ); + expect(plan.rules).toContainEqual( + expect.objectContaining({ + field: "tags", + rule: "reference", + member: "tagIds", + type: "/api/tags", + many: true, + }), + ); + }); + + it("serves a body-less action that calls show flipping its flag both ways", () => { + expect( + planOf(BOOKMARK_TOGGLE_ROWS, "put_entries_bookmark_by_id"), + ).toMatchObject({ + served: true, + address: { by: "action", segment: "bookmark" }, + rules: expect.arrayContaining([{ field: "starred", rule: "toggle" }]), + }); + }); + + it("serves no action whose calls flipped its flag one way only", () => { + const starred = feedEntry(1, true, "2026-10-17T18:00:03.120000Z"); + expect( + planOf( + [ + ...BOOKMARK_TOGGLE_ROWS.slice(0, 4), + ["GET", "/api/entries/1", starred], + ], + "put_entries_bookmark_by_id", + ), + ).toMatchObject({ served: false }); + }); + + it("does not learn a title sort an earlier update's kept order ruled out", () => { + const ascending = { + rule: "sorted", + field: "title", + direction: "ascending", + }; + const update = planOf(REFUTED_TITLE_SORT_ROWS, "patch_folders_by_id"); + const created = planOf(REFUTED_TITLE_SORT_ROWS, "post_folders"); + expect(update).toMatchObject({ served: true }); + expect(update.insert).not.toEqual(ascending); + expect(created.insert).not.toEqual(ascending); + }); + + it("serves no toggle when an earlier call found the flag null", () => { + const plan = planOf(NULL_THEN_STARRED_ROWS, "put_entries_bookmark_by_id"); + expect(plan).toMatchObject({ served: false }); + expect(plan.rules).not.toContainEqual({ + field: "starred", + rule: "toggle", + }); + }); + + it("serves no toggle a call left its flag unchanged under", () => { + const unchanged = feedEntry(1, false, "2026-10-17T18:00:08.050000Z"); + expect( + planOf( + [ + ...BOOKMARK_TOGGLE_ROWS, + ["PUT", "/api/entries/1/bookmark", "", 204], + ["GET", "/api/entries/1", unchanged], + ], + "put_entries_bookmark_by_id", + ), + ).toMatchObject({ served: false }); + }); +}); diff --git a/src/domain/analysisView/environmentStateView.ts b/src/domain/analysisView/environmentStateView.ts index ce9f8dc6cc..82e0f0fb32 100644 --- a/src/domain/analysisView/environmentStateView.ts +++ b/src/domain/analysisView/environmentStateView.ts @@ -323,6 +323,7 @@ const fieldRuleSchema = z.discriminatedUnion("rule", [ member: z.string(), type: z.string(), fields: z.array(z.string()).min(1), + many: z.literal(true).exactOptional(), }), z.strictObject({ ...ruled, rule: z.literal("increment") }), z.strictObject({ @@ -343,6 +344,7 @@ const fieldRuleSchema = z.discriminatedUnion("rule", [ member: z.string(), }), z.strictObject({ ...ruled, rule: z.literal("set"), value: jsonValueSchema }), + z.strictObject({ ...ruled, rule: z.literal("toggle") }), ]); const insertRuleSchema = z.union([ z.strictObject({ rule: z.enum(["first", "last"]) }), diff --git a/src/domain/analysisView/environmentStateWriteApply.ts b/src/domain/analysisView/environmentStateWriteApply.ts index 4af66a7de3..255dd25d89 100644 --- a/src/domain/analysisView/environmentStateWriteApply.ts +++ b/src/domain/analysisView/environmentStateWriteApply.ts @@ -223,12 +223,20 @@ const ruledValue = ( observed: JsonObject, call: WriteCall, generate: GenerateValue, - referenced?: ReferencedEntity, + references: { + readonly referenced: ReferencedEntity | undefined; + readonly blank: WriteBody["blank"]; + }, ): Applied => { const body = call.body ?? {}; switch (rule.rule) { case "reference": - return referencedValue(rule, body, referenced); + return referencedValue( + rule, + body, + references.referenced, + references.blank, + ); case "sequence": return rule.sent === true && Object.hasOwn(body, rule.field) ? ok(body[rule.field]!) @@ -333,12 +341,17 @@ const copiedShapes = ( }), ); -/** A create's call with the members it left out that recorded calls sent empty, as the earliest sent them; one a call that left it out showed a default for keeps that default. */ -const withBlanks = (body: WriteBody, call: WriteCall): WriteCall => { +/** A create's call with the members it left out that recorded calls sent empty, as the earliest sent them; one a call that left it out showed a default for keeps that default, and one `unheld` names stays left out. */ +const withBlanks = ( + body: WriteBody, + call: WriteCall, + unheld: ReadonlySet, +): WriteCall => { const given = call.body ?? {}; const left = (body.blank ?? []).filter( ({ member }, index, blank) => !Object.hasOwn(given, member) && + !unheld.has(member) && !(body.defaults ?? []).some((entry) => entry.member === member) && blank.findIndex((entry) => entry.member === member) === index, ); @@ -363,6 +376,14 @@ export const createdFields = ( idField?: string, ): Applied => { if (plan.body === null) return err("The tool has no modelled body."); + // A reference the observed entity lacks is one a create left out and stored nothing for, as a call leaving it out does. + const unheld = new Set( + plan.rules.flatMap((rule) => + rule.rule === "reference" && !Object.hasOwn(observed, rule.field) + ? [rule.member] + : [], + ), + ); // A member a `sequence` with `sent` numbers when a call leaves it out takes the next number, not an empty value one sent. const numbered = new Set( plan.rules.flatMap((rule) => @@ -377,6 +398,7 @@ export const createdFields = ( ), }, sentCall, + unheld, ); const problem = bodyProblem(plan.body, call, { entity: { ...copiedShapes(plan.rules, observed), ...observed }, @@ -387,7 +409,10 @@ export const createdFields = ( (plan.body.defaults ?? []).map(({ member, value }) => [member, value]), ); for (const rule of plan.rules) { - const value = ruledValue(rule, observed, call, generate, referenced); + const value = ruledValue(rule, observed, call, generate, { + referenced, + blank: plan.body.blank, + }); if (!value.ok) return value; if (value.value !== undefined) setOwnMember(written, rule.field, value.value); @@ -482,8 +507,13 @@ export const updatedFields = ( for (const rule of plan.rules) { const stated = statedValue(rule, sent); if (stated !== undefined) setOwnMember(written, rule.field, stated); - else if (rule.rule === "reference" && Object.hasOwn(sent, rule.member)) { - const value = referencedValue(rule, sent, referenced); + else if (rule.rule === "toggle") { + const was = ownMember(before, rule.field); + if (typeof was !== "boolean") + return err(`The entity holds no flag ${rule.field} to flip.`); + setOwnMember(written, rule.field, !was); + } else if (rule.rule === "reference" && Object.hasOwn(sent, rule.member)) { + const value = referencedValue(rule, sent, referenced, plan.body.blank); if (!value.ok) return value; if (value.value !== undefined) setOwnMember(written, rule.field, value.value); diff --git a/src/domain/analysisView/environmentStateWriteCalls.ts b/src/domain/analysisView/environmentStateWriteCalls.ts index 2d6ab4087b..00fca10b4d 100644 --- a/src/domain/analysisView/environmentStateWriteCalls.ts +++ b/src/domain/analysisView/environmentStateWriteCalls.ts @@ -24,7 +24,8 @@ export const unsentChange = (rule: FieldRule): boolean => rule.rule === "composed" || rule.rule === "stamp" || rule.rule === "derived" || - rule.rule === "set"; + rule.rule === "set" || + rule.rule === "toggle"; /** Whether serving gives a rule's field a new value on every call, so no call may have kept it. */ export const regenerates = (rule: FieldRule): boolean => @@ -56,6 +57,34 @@ export const disagreesWithCall = ( (regenerates(rule) && call.kept.includes(rule.field)), ); +/** A plan's set rules a later call flipped back, and that call's own, as toggles; `steady` names fields a call found already set, which an action sets rather than toggles. Every earlier call must have started from a boolean too, or serving rejects the first one. */ +export const toggledRules = ( + rules: readonly FieldRule[], + later: readonly FieldRule[], + before: JsonObject, + steady: ReadonlySet, + earlier: readonly { readonly startedBoolean: ReadonlySet }[], +): { readonly rules: FieldRule[]; readonly later: FieldRule[] } => { + const flipped = new Set( + later.flatMap((rule) => { + const was = ownMember(before, rule.field); + const held = rules.find((other) => other.field === rule.field); + return rule.rule === "set" && + typeof was === "boolean" && + rule.value === !was && + !steady.has(rule.field) && + earlier.every((call) => call.startedBoolean.has(rule.field)) && + ((held?.rule === "set" && held.value === was) || + held?.rule === "toggle") + ? [rule.field] + : []; + }), + ); + const toggle = (rule: FieldRule): FieldRule => + flipped.has(rule.field) ? { field: rule.field, rule: "toggle" } : rule; + return { rules: rules.map(toggle), later: later.map(toggle) }; +}; + /** The fields a call found already at the value a set rule gives them: the action sets them rather than toggling them. */ export const steadyFields = ( rules: readonly FieldRule[], @@ -109,6 +138,8 @@ export interface UpdateCall { readonly sent: ReadonlyMap; readonly sentMembers: ReadonlySet; readonly kept: readonly string[]; + /** Fields whose value before the call was a boolean. A toggle requires this of every call. */ + readonly startedBoolean: ReadonlySet; } /** How a later update's entity and the plan's calls and held entities showed `&` in the slugs it learns. */ @@ -194,8 +225,8 @@ export const contradictsEarlierCalls = ( integerOf(observation.value) !== undefined ); if (rule.rule === "set") return !sameObserved(observation, expected!); - // Serving stamps a clock on every call, so a field an earlier call kept contradicts it. - if (rule.rule === "clock") return true; + // Serving stamps a clock and flips a toggle on every call, so a field an earlier call kept contradicts it. + if (rule.rule === "clock" || rule.rule === "toggle") return true; if (rule.rule === "stamp") return !sameObserved(observation, expected!); if (rule.rule === "derived") return sent.has(rule.member) && !sameObserved(observation, expected!); @@ -404,6 +435,11 @@ export const rememberCall = ( sent: scalars(call.sent), sentMembers: new Set(Object.keys(call.sent)), kept, + startedBoolean: new Set( + Object.entries(call.before).flatMap(([field, value]) => + typeof value === "boolean" ? [field] : [], + ), + ), }; retain( 64 + @@ -413,6 +449,7 @@ export const rememberCall = ( ...facts.sent, [...facts.sentMembers], kept, + [...facts.startedBoolean], ]) * 2, ); diff --git a/src/domain/analysisView/environmentStateWriteCreateCalls.ts b/src/domain/analysisView/environmentStateWriteCreateCalls.ts index f52b5fdf80..bbd92bbefb 100644 --- a/src/domain/analysisView/environmentStateWriteCreateCalls.ts +++ b/src/domain/analysisView/environmentStateWriteCreateCalls.ts @@ -9,6 +9,7 @@ import { import { storedKinds } from "./environmentStateWriteApply.js"; import { fittedClockFormat } from "./environmentStateWriteClockFormats.js"; import { composedValue } from "./environmentStateWriteComposed.js"; +import { emptyReference } from "./environmentStateWriteReferences.js"; import { alikeRules, isBlankValue, @@ -303,6 +304,8 @@ interface FieldCalls { readonly exemplar: JsonObject; readonly later: { readonly entity: JsonObject; readonly body: JsonObject }; readonly sentBefore: ReadonlySet; + /** The values the plan's calls sent members it fixes or saw sent empty. */ + readonly pinned: ReadonlyMap; /** Members an earlier call left out and the application stored a fixed value for (`body.defaults`). */ readonly defaulted: ReadonlySet; readonly required: readonly string[]; @@ -320,6 +323,12 @@ const addedReference = ( return Object.hasOwn(later.body, added.member) ? undefined : { rule: added }; + // A call that named nothing and stored nothing agrees, as it does with a reference the first create showed. + if ( + after === undefined && + emptyReference(later.body[added.member], later.entity[field]) + ) + return { rule: added }; const rule = after === undefined ? undefined @@ -408,6 +417,19 @@ const mergedField = ( ); return rule === undefined ? undefined : { rule }; } + // A reference one call showed and another that named nothing, storing nothing, agree. + if ( + before?.rule === "reference" && + after === undefined && + emptyReference(later.body[before.member], later.entity[field]) + ) + return { rule: before }; + if ( + before === undefined && + after?.rule === "reference" && + emptyReference(calls.pinned.get(after.member), exemplar[field]) + ) + return { rule: after }; // The plan echoes a member this call left out: what it stored there is the member's default. if (before !== undefined) return before.rule === "echo" && !Object.hasOwn(later.body, field) @@ -460,6 +482,16 @@ export const mergedCreatePlan = ( ...plan.body.fixed.map(({ member }) => member), ...plan.body.required, ]); + const pinned = new Map([ + ...(plan.body.blank ?? []).map(({ member, sent }): [string, JsonValue] => [ + member, + sent, + ]), + ...plan.body.fixed.map(({ member, sent }): [string, JsonValue] => [ + member, + sent, + ]), + ]); const merged = new Map(); const defaults: Default[] = []; for (const field of new Set([ @@ -473,6 +505,7 @@ export const mergedCreatePlan = ( exemplar, later, sentBefore, + pinned, defaulted: new Set( (plan.body.defaults ?? []).map(({ member }) => member), ), @@ -483,7 +516,20 @@ export const mergedCreatePlan = ( if (outcome.rule !== undefined) merged.set(field, outcome.rule); if (outcome.default !== undefined) defaults.push(outcome.default); } - const body = mergedBody(plan.body, later, exemplar); + // A member a reference reads is accepted with any id, though a call that named nothing showed it fixed. + const read = new Set( + [...merged.values()].flatMap((rule) => + rule.rule === "reference" ? [rule.member] : [], + ), + ); + const body = mergedBody( + released(plan.body, read), + { + ...later, + contract: released(later.contract, read), + }, + exemplar, + ); const joined = body === undefined ? undefined @@ -518,6 +564,30 @@ const joinedBlanks = ( return blank; }; +/** A body contract with the members references read accepted, and kept as sent empty, rather than fixed at the empty value a call sent. */ +const released = (body: WriteBody, read: ReadonlySet): WriteBody => { + const freed = body.fixed.filter( + ({ member, sent }) => read.has(member) && emptyReference(sent, sent), + ); + if (freed.length === 0) return body; + return { + ...body, + fixed: body.fixed.filter((entry) => !freed.includes(entry)), + blank: [ + ...(body.blank ?? []), + ...freed.filter( + ({ member }) => !body.blank?.some((entry) => entry.member === member), + ), + ], + accepted: [ + ...body.accepted, + ...freed + .map(({ member }) => member) + .filter((member) => !body.accepted.includes(member)), + ], + }; +}; + /** Two calls' body contracts joined, undefined when a member one accepts with any value the other fixes, or both fix it at different values. */ const mergedBody = ( body: WriteBody, diff --git a/src/domain/analysisView/environmentStateWriteEffects.ts b/src/domain/analysisView/environmentStateWriteEffects.ts index ef4696a2c6..53c9380954 100644 --- a/src/domain/analysisView/environmentStateWriteEffects.ts +++ b/src/domain/analysisView/environmentStateWriteEffects.ts @@ -44,6 +44,7 @@ import { conflictsWithPlan, refinedBody, reconciledUpdateRule, + toggledRules, } from "./environmentStateWriteCalls.js"; import { laterObservation, @@ -667,7 +668,7 @@ export const updateEntity = ( } const unknown = context.unknown(type, collection, id); const call = { ...callOf(step, body), body }; - const { explained, rules } = fitUpdateClocks( + const fitted = fitUpdateClocks( draft, after.object, explainUpdate({ @@ -688,6 +689,15 @@ export const updateEntity = ( references: draft.plan.rules.filter((rule) => rule.rule === "reference"), }), ); + // An action a later call shows flipping back what an earlier one set toggles it. + const { rules, later } = toggledRules( + fitted.rules, + fitted.explained.rules, + entity.fields, + draft.steady, + draft.updates, + ); + const explained = { ...fitted.explained, rules: later }; for (const member of keptOrders(call.body, after.object)) draft.kept.add(member); if (explained.unexplained.length > 0 && !unknown) { diff --git a/src/domain/analysisView/environmentStateWriteReferences.test.ts b/src/domain/analysisView/environmentStateWriteReferences.test.ts index 9c4d22fb22..e676f21700 100644 --- a/src/domain/analysisView/environmentStateWriteReferences.test.ts +++ b/src/domain/analysisView/environmentStateWriteReferences.test.ts @@ -116,6 +116,44 @@ describe("objects that repeat the entity a member names", () => { ).toBeUndefined(); }); + it("learns a list reference only when every element lists the same nonempty fields", () => { + const named = (id: string) => + id === "1" || id === "2" + ? [ + held("/api/tags", id, { + id, + name: id === "1" ? "Tools" : "Winter", + color: "#aa0000", + }), + ] + : []; + expect( + referenceRule( + "tags", + [ + { id: "1", name: "Tools" }, + { id: "2", name: "Winter" }, + ], + { tagIds: ["1", "2"] }, + named, + ), + ).toMatchObject({ many: true, fields: ["id", "name"] }); + expect( + referenceRule( + "tags", + [ + { id: "1", name: "Tools" }, + { id: "2", name: "Winter", color: "#aa0000" }, + ], + { tagIds: ["1", "2"] }, + named, + ), + ).toBeUndefined(); + expect( + referenceRule("tags", [{}], { tagIds: ["1"] }, named), + ).toBeUndefined(); + }); + it("refuses a call naming an entity the environment does not hold, or holds in several collections", () => { const rule: Extract = { field: "shelf", diff --git a/src/domain/analysisView/environmentStateWriteReferences.ts b/src/domain/analysisView/environmentStateWriteReferences.ts index 5d5a68525a..3bf24322ed 100644 --- a/src/domain/analysisView/environmentStateWriteReferences.ts +++ b/src/domain/analysisView/environmentStateWriteReferences.ts @@ -7,7 +7,7 @@ import { type JsonObject, type StateEntity, } from "./environmentStateRules.js"; -import type { FieldRule } from "./environmentStateWriteRules.js"; +import type { FieldRule, WriteBody } from "./environmentStateWriteRules.js"; type ReferenceRule = Extract; @@ -42,6 +42,7 @@ export const referenceRule = ( body: JsonObject, named: (id: string) => readonly StateEntity[], ): ReferenceRule | undefined => { + if (Array.isArray(value)) return listReferenceRule(field, value, body, named); if (!isObject(value) || Object.keys(value).length < 2) return undefined; const matches = new Map(); for (const [member, sent] of Object.entries(body)) { @@ -71,14 +72,101 @@ export const referenceRule = ( : undefined; }; +/** + * A created or updated list field holding, in order, the entities a sent list of ids names, as the state holds them: + * tags beside `tagIds`. The member is the one named like the field; every element must project one entity of one type. + */ +const listReferenceRule = ( + field: string, + value: readonly JsonValue[], + body: JsonObject, + named: (id: string) => readonly StateEntity[], +): ReferenceRule | undefined => { + if (value.length === 0 || !value.every(isObject)) return undefined; + // A list field is named for its items, as `tags` beside `tagIds`. + const stems = new Set([stemOf(field), stemOf(field).replace(/s$/u, "")]); + const members = Object.entries(body).filter( + ([member, sent]) => + stems.has(stemOf(member)) && + Array.isArray(sent) && + sent.length === value.length, + ); + if (members.length !== 1) return undefined; + const [member, sent] = members[0]! as [string, JsonValue[]]; + const types = new Set(); + for (const [index, element] of value.entries()) { + const id = scalarText(sent[index]); + const held = id === undefined ? [] : named(id); + const match = held.filter((entity) => + projects(element as JsonObject, entity.fields), + ); + if (match.length !== 1) return undefined; + types.add(match[0]!.type); + } + if (types.size !== 1) return undefined; + // Every recorded element must list the same fields. A union would add a key one element omitted, and an empty object lists none. + const [recorded, ...rest] = value.map((element) => + Object.keys(element as JsonObject), + ); + if ( + recorded === undefined || + recorded.length === 0 || + rest.some( + (fields) => + fields.length !== recorded.length || + fields.some((field) => !recorded.includes(field)), + ) + ) + return undefined; + return { + field, + rule: "reference", + member, + type: [...types][0]!, + fields: recorded, + many: true, + }; +}; + +/** Whether a call's member and the entity's field show a reference to nothing: a `null` id and field, or empty lists. */ +export const emptyReference = ( + sent: JsonValue | undefined, + stored: JsonValue | undefined, +): boolean => + (sent === null && stored === null) || + (Array.isArray(sent) && + sent.length === 0 && + Array.isArray(stored) && + stored.length === 0); + /** A reference's value for a call: the named entity's fields it lists, or undefined when the call does not send its member. */ export const referencedValue = ( rule: ReferenceRule, body: JsonObject, referenced: ReferencedEntity | undefined, + blank?: WriteBody["blank"], ): Result => { if (!Object.hasOwn(body, rule.member)) return ok(undefined); - const id = scalarText(body[rule.member]); + const sent = body[rule.member]!; + // A call naming nothing stores nothing, as a recorded call that sent the member `null` did. + if ( + sent === null && + blank?.some((entry) => entry.member === rule.member && entry.sent === null) + ) + return ok(null); + if (rule.many === true) { + if (!Array.isArray(sent)) + return err(`Member ${rule.member} sends no list of ids.`); + const { many: _many, ...single } = rule; + const values: JsonValue[] = []; + for (const one of sent) { + const value = referencedValue(single, { [rule.member]: one }, referenced); + if (!value.ok) return value; + values.push(value.value ?? null); + } + return ok(values); + } + const id = scalarText(sent); const held = id === undefined ? [] : (referenced?.(rule.type, id) ?? []); const [fields] = held; if (fields === undefined) diff --git a/src/domain/analysisView/environmentStateWriteRules.ts b/src/domain/analysisView/environmentStateWriteRules.ts index 7da1aeb34b..093bf08b5f 100644 --- a/src/domain/analysisView/environmentStateWriteRules.ts +++ b/src/domain/analysisView/environmentStateWriteRules.ts @@ -124,6 +124,8 @@ export type FieldRule = readonly member: string; readonly type: string; readonly fields: string[]; + /** The member sends a list of ids, and the field holds each named entity in that order. */ + readonly many?: true; } | { readonly field: string; readonly rule: "increment" } /** An update stores `value` whenever the call sends the member, as the call that showed it did. */ @@ -151,6 +153,11 @@ export type FieldRule = readonly field: string; readonly rule: "set"; readonly value: JsonValue; + } + /** An item action flips a boolean on every call. */ + | { + readonly field: string; + readonly rule: "toggle"; }; /** Clock rules spelled as a trimmed sibling is, since an application spells its clocks alike: fields whose `values`, and every `later` call's, show no fixed width. */ diff --git a/src/domain/analysisView/environmentStateWriteVerdict.ts b/src/domain/analysisView/environmentStateWriteVerdict.ts index d5834ee5d1..3ea6cd308a 100644 --- a/src/domain/analysisView/environmentStateWriteVerdict.ts +++ b/src/domain/analysisView/environmentStateWriteVerdict.ts @@ -56,13 +56,17 @@ const renders = ( ); }; -/** Whether an item action sets something, and a call showed each field it sets already at that value, so the action sets rather than toggles it. */ +/** Whether an item action sets or toggles something, and a call showed each field it sets already at that value, so it sets rather than toggles those. */ const steady = (plan: StateWritePlan, draft: WriteDraft): boolean => { if (plan.address?.by !== "action") return true; const set = plan.rules.flatMap((rule) => rule.rule === "set" ? [rule.field] : [], ); - return set.length > 0 && set.every((field) => draft.steady.has(field)); + // A toggle is learned only from calls that flipped its field both ways. + return ( + (set.length > 0 || plan.rules.some((rule) => rule.rule === "toggle")) && + set.every((field) => draft.steady.has(field)) + ); }; /** Whether a clock a call left as it was in its own second has no rule another call showed, so no call showed whether the write stamps it. */ diff --git a/src/domain/analysisView/scenarioCaptureRecords.test.ts b/src/domain/analysisView/scenarioCaptureRecords.test.ts index f63645f6ad..8b05021e83 100644 --- a/src/domain/analysisView/scenarioCaptureRecords.test.ts +++ b/src/domain/analysisView/scenarioCaptureRecords.test.ts @@ -86,6 +86,26 @@ describe("scenarioCaptureRecords", () => { ); }); + it("records a hidden request body as absent only when the request names no Content-Type", () => { + const toggle = { + method: "PUT", + url: "/api/entries/1/bookmark", + status: 204, + requestBodyHidden: true, + } as const; + const projected = scenarioCaptureRecords( + scenarioCaptureFixture([ + { ...toggle, requestHeaderNames: ["accept", "authorization"] }, + { ...toggle, requestHeaderNames: ["accept", "Content-Type"] }, + ]), + "manifest", + ); + expect(projected.records.map((entry) => entry.reported)).toMatchObject([ + { request: { bodySize: 0 } }, + { request: { bodySize: -1 } }, + ]); + }); + it("leaves a transaction that cannot be attributed to one request and response unrepresented", () => { const capture = scenarioCaptureFixture(SCENARIO_EXCHANGES.slice(1, 2)); const response = capture.events.items.find( diff --git a/src/domain/analysisView/scenarioCaptureRecords.ts b/src/domain/analysisView/scenarioCaptureRecords.ts index 2a5cc3f094..f43d8fedd0 100644 --- a/src/domain/analysisView/scenarioCaptureRecords.ts +++ b/src/domain/analysisView/scenarioCaptureRecords.ts @@ -88,6 +88,14 @@ export const scenarioContentMediaType = ( captured(content)?.media_type ?? headerValue(content?.headers, "content-type"); +/** Whether a request sent no body: Playwright exposed none and the request names no Content-Type, which the bodies browsers send carry. */ +const unsentBody = ( + request: RequestEvent, + content: ContentEvent | undefined, +): boolean => + content?.body.state === "not_exposed" && + !request.header_names.some((name) => name.toLowerCase() === "content-type"); + /** The request object HAR readers expect; a redacted URL is withheld so no path is templated from it. */ const reportedRequest = ( request: RequestEvent, @@ -97,9 +105,15 @@ const reportedRequest = ( const body = purpose === "manifest" ? captured(content) : undefined; const text = body === undefined ? undefined : utf8Text(body); const mimeType = scenarioContentMediaType(content); + const size = + body !== undefined && !body.redacted + ? body.bytes + : purpose === "manifest" && unsentBody(request, content) + ? 0 + : -1; return { method: request.method, - bodySize: body !== undefined && !body.redacted ? body.bytes : -1, + bodySize: size, url: request.url.redacted ? null : request.url.url, ...(text === undefined || mimeType === undefined ? {} diff --git a/src/generatedPackageMetadata.ts b/src/generatedPackageMetadata.ts index 5372dbba8b..9100e716d5 100644 --- a/src/generatedPackageMetadata.ts +++ b/src/generatedPackageMetadata.ts @@ -6,5 +6,5 @@ export const PACKAGE_METADATA = { serverSdkVersion: "2.3.1", clientSdkVersion: "2.3.1", coreSdkVersion: "2.3.1", - skillVersion: "54", + skillVersion: "55", } as const; diff --git a/tests/fixtures/browserScenarioCapture.ts b/tests/fixtures/browserScenarioCapture.ts index c5dfb24d56..7c7f9300ca 100644 --- a/tests/fixtures/browserScenarioCapture.ts +++ b/tests/fixtures/browserScenarioCapture.ts @@ -27,6 +27,10 @@ export interface ScenarioExchange { readonly body?: string | Buffer; readonly requestMediaType?: string; readonly requestBody?: string; + /** A request body Playwright did not expose. */ + readonly requestBodyHidden?: true; + /** The request event's header names. */ + readonly requestHeaderNames?: readonly string[]; /** Header values as `header_values` retains them; a null value is one the capture excluded. */ readonly requestHeaders?: readonly ScenarioHeader[]; readonly responseHeaders?: readonly ScenarioHeader[]; @@ -145,13 +149,15 @@ const exchangeEvents = ( method: exchange.method, url: exchange.redactedUrl === true ? { ...url, redacted: true } : url, resource_type: exchange.resourceType ?? "fetch", - header_names: [], + header_names: [...(exchange.requestHeaderNames ?? [])], }; const content = ( sequence: number, source: number, phase: "request" | "response", - body: ReturnType | { readonly state: "not_requested" }, + body: + | ReturnType + | { readonly state: "not_requested" | "not_exposed" }, ) => ({ sequence, step_index: 1, @@ -176,6 +182,7 @@ const exchangeEvents = ( let next = first + 1; if ( exchange.requestBody !== undefined || + exchange.requestBodyHidden === true || exchange.requestHeaders !== undefined ) { events.push( @@ -183,13 +190,15 @@ const exchangeEvents = ( next, first, "request", - exchange.requestBody === undefined - ? { state: "not_requested" } - : capturedBody( - exchange.requestBody, - "browser-exposed-request-bytes", - exchange.requestMediaType ?? null, - ), + exchange.requestBodyHidden === true + ? { state: "not_exposed" } + : exchange.requestBody === undefined + ? { state: "not_requested" } + : capturedBody( + exchange.requestBody, + "browser-exposed-request-bytes", + exchange.requestMediaType ?? null, + ), ), ); next += 1; diff --git a/tests/fixtures/environmentStateItemWrites.ts b/tests/fixtures/environmentStateItemWrites.ts new file mode 100644 index 0000000000..203cb03a30 --- /dev/null +++ b/tests/fixtures/environmentStateItemWrites.ts @@ -0,0 +1,297 @@ +import type { TimedCaptureRow } from "./analysisView.js"; +import type { JsonValue } from "../../src/domain/jsonValue.js"; + +type Entity = { readonly [key: string]: JsonValue }; + +const folder = (id: number, title: string): Entity => ({ + id, + title, + version: 1, +}); + +/** One seeded folder, a create a list shows after it, then one a list shows before that create by title, then a delete. */ +export const CREATES_BY_TITLE_ROWS: readonly TimedCaptureRow[] = (() => { + const rota = folder(1, "Lighthouse Rota"); + const pier = folder(2, "Scratch Pier 3"); + const meadow = folder(3, "Scratch Meadow 4"); + return [ + ["GET", "/api/folders", [rota]], + ["GET", "/api/folders/1", rota], + ["POST", "/api/folders", pier, 200, { title: "Scratch Pier 3" }], + ["GET", "/api/folders/2", pier], + ["GET", "/api/folders", [rota, pier]], + ["POST", "/api/folders", meadow, 200, { title: "Scratch Meadow 4" }], + ["GET", "/api/folders/3", meadow], + ["GET", "/api/folders", [rota, meadow, pier]], + ["DELETE", "/api/folders/2", { message: "Folder deleted" }, 200], + ["GET", "/api/folders/2", { message: "folder not found" }, 404], + ["GET", "/api/folders", [rota, meadow]], + ]; +})(); + +const LOCATIONS = [ + { + id: "7d1c0b6e-0001-4000-8000-000000000001", + name: "Garage", + kind: "location", + }, + { + id: "7d1c0b6e-0002-4000-8000-000000000002", + name: "Attic", + kind: "location", + }, +]; +const TAGS = [ + { + id: "3b9f2a10-0001-4000-8000-000000000001", + name: "Tools", + color: "#aa0000", + }, + { + id: "3b9f2a10-0002-4000-8000-000000000002", + name: "Winter", + color: "#0000aa", + }, +]; + +/** An item as the application answers it: its parent location and tags spelled out from the ids a call sent. */ +export const inventoryItem = ( + id: string, + name: string, + parent: Entity | null, + tags: readonly Entity[], +): Entity => ({ id, name, parent, tags: [...tags] }); + +/** Item creates naming a parent location and tags by id, the first with none, the second with one of each. */ +export const ITEM_REFERENCES_ROWS: readonly TimedCaptureRow[] = (() => { + const seeded = inventoryItem( + "5e5e0000-0001-4000-8000-000000000001", + "Ladder", + LOCATIONS[0]!, + [], + ); + const bare = inventoryItem( + "5e5e0000-0002-4000-8000-000000000002", + "Scratch box", + null, + [], + ); + const filed = inventoryItem( + "5e5e0000-0003-4000-8000-000000000003", + "Scratch drill", + LOCATIONS[1]!, + [TAGS[0]!], + ); + return [ + ["GET", "/api/locations", LOCATIONS], + ["GET", `/api/locations/${LOCATIONS[0]!["id"] as string}`, LOCATIONS[0]!], + ["GET", `/api/locations/${LOCATIONS[1]!["id"] as string}`, LOCATIONS[1]!], + ["GET", "/api/tags", TAGS], + ["GET", `/api/tags/${TAGS[0]!["id"] as string}`, TAGS[0]!], + ["GET", `/api/tags/${TAGS[1]!["id"] as string}`, TAGS[1]!], + ["GET", "/api/items", [seeded]], + ["GET", `/api/items/${seeded["id"] as string}`, seeded], + [ + "POST", + "/api/items", + bare, + 201, + { name: "Scratch box", parentId: null, tagIds: [] }, + ], + ["GET", `/api/items/${bare["id"] as string}`, bare], + ["GET", "/api/items", [seeded, bare]], + [ + "POST", + "/api/items", + filed, + 201, + { + name: "Scratch drill", + parentId: LOCATIONS[1]!["id"]!, + tagIds: [TAGS[0]!["id"]!], + }, + ], + ["GET", `/api/items/${filed["id"] as string}`, filed], + ["GET", "/api/items", [seeded, bare, filed]], + ]; +})(); + +export const ITEM_LOCATIONS = LOCATIONS; +export const ITEM_TAGS = TAGS; + +/** A feed entry with a `starred` flag a body-less action flips, and the instant it last changed. */ +export const feedEntry = ( + id: number, + starred: boolean, + changedAt: string, +): Entity => ({ + id, + title: `Entry ${String(id)}`, + starred, + changed_at: changedAt, +}); + +/** One entry starred, then unstarred, by the same body-less call, each read back after it. */ +export const BOOKMARK_TOGGLE_ROWS: readonly TimedCaptureRow[] = (() => { + const seeded = "2026-10-17T17:00:00.000000Z"; + const first = feedEntry(1, false, seeded); + const second = feedEntry(2, false, seeded); + const starred = feedEntry(1, true, "2026-10-17T18:00:03.120000Z"); + const unstarred = feedEntry(1, false, "2026-10-17T18:00:05.090000Z"); + return [ + ["GET", "/api/entries", [first, second]], + ["GET", "/api/entries/1", first], + ["GET", "/api/entries/2", second], + ["PUT", "/api/entries/1/bookmark", "", 204], + ["GET", "/api/entries/1", starred], + ["PUT", "/api/entries/1/bookmark", "", 204], + ["GET", "/api/entries/1", unstarred], + ["GET", "/api/entries", [unstarred, second]], + ]; +})(); + +const ranked = ( + id: number, + title: string, + position: number | null, +): Entity => ({ + id, + title, + position, +}); + +/** Folders a list shows by `position`, the seed holding none: the first create lists after the seed's higher title, and once the seed is deleted later lists alone would also fit a sort by title. */ +export const RANKED_AFTER_DELETE_ROWS: readonly TimedCaptureRow[] = (() => { + const zulu = ranked(1, "Zulu Base", null); + const bravo = ranked(2, "Bravo", 2); + const charlie = ranked(3, "Charlie", 3); + const able = ranked(4, "Able", 0); + return [ + ["GET", "/api/folders", [zulu]], + ["GET", "/api/folders/1", zulu], + ["POST", "/api/folders", bravo, 200, { title: "Bravo", position: 2 }], + ["GET", "/api/folders/2", bravo], + ["GET", "/api/folders", [zulu, bravo]], + ["DELETE", "/api/folders/1", { message: "Folder deleted" }, 200], + ["GET", "/api/folders/1", { message: "folder not found" }, 404], + ["GET", "/api/folders", [bravo]], + ["POST", "/api/folders", charlie, 200, { title: "Charlie", position: 3 }], + ["GET", "/api/folders/3", charlie], + ["GET", "/api/folders", [bravo, charlie]], + ["POST", "/api/folders", able, 200, { title: "Able", position: 0 }], + ["GET", "/api/folders/4", able], + ["GET", "/api/folders", [able, bravo, charlie]], + ]; +})(); + +/** Item creates: the first shows no parent at all, the second names one, the third sends `parentId: null` and stores none; lists show each item's id and name. */ +export const ADDED_EMPTY_REFERENCE_ROWS: readonly TimedCaptureRow[] = (() => { + const seeded = { id: "5e5e0000-0001-4000-8000-000000000001", name: "Ladder" }; + const loose = { + id: "5e5e0000-0002-4000-8000-000000000002", + name: "Scratch box", + }; + const filed = { + id: "5e5e0000-0003-4000-8000-000000000003", + name: "Scratch drill", + parent: LOCATIONS[1]!, + }; + const cleared = { + id: "5e5e0000-0004-4000-8000-000000000004", + name: "Scratch lamp", + parent: null, + }; + const summary = (item: Entity): Entity => ({ + id: item["id"]!, + name: item["name"]!, + }); + return [ + ["GET", "/api/locations", LOCATIONS], + ["GET", `/api/locations/${LOCATIONS[0]!["id"] as string}`, LOCATIONS[0]!], + ["GET", `/api/locations/${LOCATIONS[1]!["id"] as string}`, LOCATIONS[1]!], + ["GET", "/api/items", [seeded]], + ["GET", `/api/items/${seeded.id}`, seeded], + ["POST", "/api/items", loose, 201, { name: "Scratch box" }], + ["GET", `/api/items/${loose.id}`, loose], + ["GET", "/api/items", [seeded, loose].map(summary)], + [ + "POST", + "/api/items", + filed, + 201, + { name: "Scratch drill", parentId: LOCATIONS[1]!["id"]! }, + ], + ["GET", `/api/items/${filed.id}`, filed], + ["GET", "/api/items", [seeded, loose, filed].map(summary)], + [ + "POST", + "/api/items", + cleared, + 201, + { name: "Scratch lamp", parentId: null }, + ], + ["GET", `/api/items/${cleared.id}`, cleared], + ["GET", "/api/items", [seeded, loose, filed, cleared].map(summary)], + ]; +})(); + +/** Renames that keep collection order, which an ascending title sort would not replay, then a create a whole list could place by title. */ +export const REFUTED_TITLE_SORT_ROWS: readonly TimedCaptureRow[] = (() => { + const zulu = folder(1, "Zulu"); + const bravo = folder(2, "Bravo"); + const alpha = folder(3, "Alpha"); + const echo = folder(3, "Echo"); + const able = folder(1, "Able"); + const delta = folder(4, "Delta"); + return [ + ["GET", "/api/folders", [zulu, bravo, alpha]], + ["GET", "/api/folders/1", zulu], + ["GET", "/api/folders/2", bravo], + ["GET", "/api/folders/3", alpha], + ["PATCH", "/api/folders/3", echo, 200, { title: "Echo" }], + ["GET", "/api/folders/3", echo], + ["GET", "/api/folders", [zulu, bravo, echo]], + ["PATCH", "/api/folders/1", able, 200, { title: "Able" }], + ["GET", "/api/folders/1", able], + ["GET", "/api/folders", [able, bravo, echo]], + ["POST", "/api/folders", delta, 200, { title: "Delta" }], + ["GET", "/api/folders/4", delta], + ["GET", "/api/folders", [able, bravo, delta, echo]], + ]; +})(); + +/** A body-less bookmark action whose first call finds `starred` null, then flips the boolean it stored. */ +export const NULL_THEN_STARRED_ROWS: readonly TimedCaptureRow[] = (() => { + const seeded = { + id: 1, + title: "Entry 1", + starred: null, + changed_at: "2026-10-17T17:00:00.000000Z", + }; + const other = { + id: 2, + title: "Entry 2", + starred: false, + changed_at: "2026-10-17T17:00:00.000000Z", + }; + const starred = { + ...seeded, + starred: true, + changed_at: "2026-10-17T18:00:03.120000Z", + }; + const cleared = { + ...seeded, + starred: false, + changed_at: "2026-10-17T18:00:05.090000Z", + }; + return [ + ["GET", "/api/entries", [seeded, other]], + ["GET", "/api/entries/1", seeded], + ["GET", "/api/entries/2", other], + ["PUT", "/api/entries/1/bookmark", "", 204], + ["GET", "/api/entries/1", starred], + ["PUT", "/api/entries/1/bookmark", "", 204], + ["GET", "/api/entries/1", cleared], + ["GET", "/api/entries", [cleared, other]], + ]; +})();