From 51ad84516e55153c2d22704044f6d2b7de8a6fb4 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Thu, 29 Oct 2026 03:46:18 -0700 Subject: [PATCH] feat(environment): serve sorted creates, listed references and toggles (#7600) * feat(environment): serve the writes callers make without recorded form fields - A create member a recording sent empty ("", null, false, [] or {}) is no longer required; a call that leaves it out is applied as sending that value (body.blank). - List views render envelope members that list the listed entities' ids (Paperless's `all`), so creates in such collections are confirmed. - Updates move their entity in collections kept sorted by a field they write, and a failed whole-collection read after an update can teach that order. - A later create that stores an unruled field otherwise is withheld at the create, and an item read may fill values a create's answer had not loaded. - An unsent field respelled between null, [] and {} is unchanged. - The skill asks for one create with every field and one with only what a person must give. Co-Authored-By: Claude Opus 5.5 * docs(skill): show how to read signed-in pages before recording Agents guessed capture keys (artifacts, dom, final) to see pages behind sign-in, which inspect_web_page cannot reach. Co-Authored-By: Claude Opus 5.5 * fix(environment): accept every empty spelling recorded creates sent A create that fills only what a person must give sends `null` where the full create sent a string, and `null` or `false` for the same flag. The merged plan now accepts each recorded empty value, and a call that leaves the member out is applied as the earliest call sent it. Also adds the spec folder for this change. Co-Authored-By: Claude Opus 5.5 * docs(skill): keep the vary-fields step neutral on UI or API writes Agents read "the form" as a cue to write through the UI, where some applications' request bodies are not exposed; a body-unusable UI write now re-records once with request steps. Co-Authored-By: Claude Opus 5.5 * docs(environment): record verification for blank members and sorted writes Co-Authored-By: Claude Opus 5.5 * fix(environment): address review of blank members and sorted writes - Bulk updates move their entities in turn against the others' earlier values and install the final order once, as replay does. - A member's recorded empty values stay accepted once a later call teaches its default; a left-out member with a default keeps the default. - Write answers spell empty lists and maps as their recorded answers did. - A rename can teach a sort when the creates chose one by a field the update did not write, such as ids that ascend with titles. Co-Authored-By: Claude Opus 5.5 * feat(environment): serve the writes that later reads explain - An item read whose object is marked deleted (deleted, isDeleted or is_deleted true) agrees with an absent entity, as EspoCRM answers a deleted record; such an object never seeds or adopts an entity. - maxSize, max_results and maxResults are page sizes, for whole-collection list lines and the list query's limit role. - An item view also shows the fields only reads after its type's first write show, when no initial entity holds them (Vikunja's task subscription), without requiring them; write answers leave such a field out for entities without it, and it no longer withholds the write as incomplete-entities. - The skill asks updates to change the fields a person would change, and expects a deleted object to read back as 404 or marked deleted. Co-Authored-By: Claude Opus 5.5 * fix(environment): answer sent empty members as sent, and keep a placed create's order - A write answer spells an empty list or map the call did not send as its recorded answer did; one the call sent is answered as sent, so a create sending `[]` is not answered `null` because the first recorded create was. - A read listing the whole collection that placed a create leaves the create only the sorted inserts whose order that read follows, so a later rename cannot teach a sort an earlier read ruled out. Co-Authored-By: Claude Opus 5.5 * fix(environment): answer bulk updates without fields only later reads show A bulk update's answer renders each entity with the same allowance a single update's answer has: a field only reads after the type's first write showed, which no initial entity holds, is left out for an entity without it, so a bulk update of a seeded task is no longer refused for lacking it. Co-Authored-By: Claude Opus 5.5 * docs(environment): correct how the EspoCRM contact update task passes The agent trajectories show gpt-oss passes it by creating a replacement contact and deleting the original, since the served update refuses emailAddress; the update itself is refused in both builds. Co-Authored-By: Claude Opus 5.5 * fix(environment): answer a read of a served-deleted entity marked deleted An application that soft-deletes still answers a deleted record by id, marked deleted. Serving now keeps an entity a served delete removed from a type whose recorded item reads show that, until a reset or load and within the size data/add loads, and answers a read of it with its last fields and the flag true, instead of refusing the read. Co-Authored-By: Claude Opus 5.5 * fix(environment): keep unloaded create answers, check bulk answers with sent members, narrow inserts only on collection-order reads - A create whose item read loaded values its answer had not records them (`observation.read.unloaded`), and a served create's answer keeps them as the recorded answer showed them, a zero-time owner included, while the state holds the read's. - Replay checks a bulk update's recorded answers with each target's sent members, as serving answers them, so echoed `null` and `[]` no longer make the plan body-unusable. - A whole-collection read whose query sorts it shows the query's order, so it no longer narrows a create's sorted insert candidates. Co-Authored-By: Claude Opus 5.5 * docs(environment): record the census with main's unknown-clock rule merged Co-Authored-By: Claude Opus 5.5 * feat(environment): serve query numbers, counted values and made objects - A query parameter accepts a number or boolean, which serving sends as its JSON text, as agents send page sizes and flags. - A sequence may start at the value its first create took above every value its type held (Vikunja's per-project task index), and a later create's composed, template or started-sequence rule is adopted when it also spells the first create's value. - A made-object rule explains an object the application makes with each created entity: members the request stamped, members holding the entity's own id, and counts one past the held objects' values. - A create's answer names the id of the one object a one-element array member holds, so bulk creates link to their type. - The skill reads each listed object by id before writing. Co-Authored-By: Claude Opus 5.5 * feat(environment): let a linked create take path values it stores A create linked to its type's own collection, whose path has no parameters, accepts any path values when its plan stores each one in a field, as a Vikunja task stores its project; replay and serving apply it alike. Co-Authored-By: Claude Opus 5.5 * fix(environment): learn a sort from a rename only when moving its entities reaches it Serving moves only the entities a write changes in a collection kept sorted, so replay now tries a sort after a rename only when the entities no write the read exercised changed already follow it. Seeded folders listed out of title order no longer teach a title sort that serving would answer with another order. Co-Authored-By: Claude Opus 5.5 * fix(environment): keep unloaded create answers a sent value did not echo, and learn no sort from query-sorted reads - A served create keeps a value its recorded answer had not loaded unless the call sent that member and the plan stores it as sent, so a create sending `active: false` is answered `false` as recorded while the state holds the read's `true`. - A rename no longer teaches a sort from a whole-collection read its query sorts, whose ties need not follow the collection's own order. Co-Authored-By: Claude Opus 5.5 * docs(environment): verify query values, counted values and made objects Co-Authored-By: Claude Opus 5.5 * fix(environment): learn the soft-delete flag in the state view and serve by it Extraction now records the member later item reads set true on a record the application soft-deleted, and the item view carries it as `deleted_flag`. Serving reads that fact instead of scanning recorded bodies itself, so one owner decides what a soft delete looks like. Co-Authored-By: Claude Opus 5.5 * docs(environment): record the call-values checks at #7274 93821024 Co-Authored-By: Claude Opus 5.5 * fix(environment): learn the soft-delete flag from item reads with a query too A client may read a deleted record back only with a query, such as a cache-busting parameter. Every successful read of the item tool now shows the flag; only plain later reads still add fields. Co-Authored-By: Claude Opus 5.5 * docs(environment): record the call-values checks at #7274 45832406 Co-Authored-By: Claude Opus 5.5 * fix(environment): learn sorts only from orders a list read confirmed - A sort retry after a failed whole-collection read requires the entities no write changed since a list last showed them to follow it already, rather than excusing every entity the update's tool ever wrote: an earlier rename a list confirmed in place stays where it was. - An update takes its creates' sort only from a create a list read showed placed; replay still tries a sort a create guessed from the initial order. Co-Authored-By: Claude Opus 5.5 * fix(environment): drop entities read back as deleted and answer soft deletes as deleted - A held entity a later read shows marked deleted is dropped, not adopted with the marked object's fields. - Bulk-update replay renders answers with the optional and later-only fields serving uses, so a bulk update of created and seeded entities replays. - A served delete of a soft-deleting type answers with the entity marked deleted, single and bulk. - A deleted flag only query reads showed is a field of the item view, so a served tombstone keeps it where live records omit it. Co-Authored-By: Claude Opus 5.5 * fix(environment): check made objects as recorded and keep their ids and clocks honest - A made object's clock counts as a write clock, so a capture with no clock anchor withholds the create instead of stamping 1970. - Members holding the entity's identity follow the id written, generated or sent, through the type's id field. - A later create's made object is checked as recorded, before its own members are rewritten, so one naming another entity withholds the plan. - A later create's template is adopted only when it spells that create's value too, not just when the first create's text holds its source value. Co-Authored-By: Claude Opus 5.5 * fix(environment): move updated entities only by placed sorts and keep query-sorted reads from confirming order Replay moved an updated entity by a sort a create only guessed, while the update plan exported only placed sorts, so a read could confirm an update by a move serving would not make. Both now use the placed sort. A passing list read clears its entities from the moved set only when the state decides the order of every entity it lists, so a read whose query sorts it no longer hides an earlier move from a later sort check. Co-Authored-By: Claude Opus 5.5 * docs(environment): count six withheld plans that explain more, not seven The bulk task create counted among them is served on this branch. Co-Authored-By: Claude Opus 5.5 * fix(environment): try a learned sort by moving each pending write in turn, as serving does #retrySort sorted the whole collection, which a list could match while serving, moving each renamed entity in turn against the others' values before their own writes, gave another order. The state now records the values each write replaced, and the retry replays the writes since a list in collection order last showed their entities. Co-Authored-By: Claude Opus 5.5 * fix(environment): answer a served delete with the deleted flag its recorded answer showed An application may answer a delete with the record as it was and mark it deleted only on later reads. Delete answers, single and bulk, now take the flag's value from the recorded answer; reads of a served-deleted entity still show it marked deleted. Co-Authored-By: Claude Opus 5.5 * chore(environment): set aside two hunks next to lines main's saved-values change rewrites Main's #7267 changes the lines beside the blank-member docs and the empty-spelling check in explainUpdate. Setting them aside lets that merge apply cleanly; the commit after it puts them back on top of main's text. Co-Authored-By: Claude Opus 5.5 * chore(environment): restore the set-aside hunks on main and bump the skill to 51 Puts the blank-member docs and the empty-spelling check back beside main's slug and composed text and heldSlugRule. Main is at skill 50 with different text, so this branch's skill changes take 51. Co-Authored-By: Claude Opus 5.5 * chore(environment): bump the skill to 52 above #7260's 51 Co-Authored-By: Claude Opus 5.5 * chore(environment): put back the call-values hunks main's merge took its side for, and bump the skill to 53 The merge of main kept main's side where it overlapped this branch, so it could commit without a conflict. This restores this branch's side next to main's: made-object agreement beside the & slug spellings, sequence starts in the earlier evidence (now built by earlierOf), the combined value imports, and recording step 2 beside main's UI-first step 3. Main is at skill 50 and #7274 at 52, so this branch takes 53. Co-Authored-By: Claude Opus 5.5 * fix(environment): keep sorts a list showed in place ruled out, and created answers' nested members as recorded A whole list in collection order that showed a renamed entity where a sort by the renamed field would have moved it now rules that sort out, so a later rename of the same entity cannot teach it. A create's answer leaves out members its item read added inside an object the answer showed, as the recorded answer did; the state still stores them. Co-Authored-By: Claude Opus 5.5 * feat(environment): serve writes whose calls send different members Recordings now vary what a type's two creates and updates send, and REA withheld writes where a later call left out what the first sent. An update plan's required members narrow to those every call sent; a member one create sent and another left for the application to number is a sequence that takes the value a call sends; a PUT's left-out member the application cleared becomes that member's default; and an update's clock is held as the read after it spells it. The skill spaces creates (skill 54). Co-Authored-By: Claude Opus 5.5 * fix(environment): keep a later create's made object only with clocks its request stamped followsMadeObjects only checked that a made object's clocks parsed, so a later create whose object showed an older clock still passed, while serving stamps the write's clock. Its clocks must now be in the rule's format and within the stamp window of that create's request. Co-Authored-By: Claude Opus 5.5 * fix(environment): accept sent values either way, check defaults against every call, and store update clocks as reads spell them A sequence that takes sent values now accepts one a later create sent after the first left the member to be numbered. A default an update learned must agree with every update that left its member out, else the plan is withheld. An update plan whose read-back spells a stamped clock at a coarser precision keeps that read as observation.read, and serving stores the update's clocks in its spelling while answers keep the response's. Co-Authored-By: Claude Opus 5.5 * fix(environment): check recorded member and sequence values * fix(environment): compare soft-deleted read fields * fix(environment): reject contradictory learned sequences * fix(environment): preserve initially deleted item observations * fix(environment): index stored deleted entities separately * fix(environment): replay lists that include deleted records * feat(environment): serve sorted creates, listed references and toggles A create a whole listing placed last is placed again by the sort a later listing shows. A field holding the entities a sent list of ids names is a reference with `many`, and a create that sent the member empty agrees with one that named entities. A scenario request whose body Playwright did not expose and that names no Content-Type sent no body. An item action calls show flipping a boolean both ways learns a `toggle` rule. Skill 55 asks recordings to run an action twice on one object. Co-Authored-By: Claude Opus 5.5 * fix(environment): count a sequence number a create sends, as replay does Serving now records a number a call sends for a `sequence` with `sent` in the same count history it generates from, so after that entity is deleted the next generated number still follows it, as replay's takeSequences does. Co-Authored-By: Claude Opus 5.5 * docs(environment): record the final gate run in the verification Co-Authored-By: Claude Opus 5.5 * fix(environment): make no count of a made-object member earlier creates held alike A count only a later create shows starts at the exemplar's value, which is right only while the exemplar is the one create accepted: creates accepted after it showed the member unchanged, which a count would have moved. The merge now refuses such a count, as the top-level sequence check does. Co-Authored-By: Claude Opus 5.5 * docs(environment): restate the verification after merging #7539 Co-Authored-By: Claude Opus 5.5 * fix(environment): check PUT defaults against reads, and respell update clocks by serving's rule A default a PUT learns must also agree with what the read after an earlier or the current call showed for the member its answer left out, so a call whose read showed the member kept withholds a later clearing default. Replay and serving now choose an update clock's stored spelling by one rule, readSpellsCoarser: the read after it at a coarser precision in the same zone and separator. A read in another zone or separator is no longer adopted by replay, which serving could not store, and a create's plain clocks stay as answered, as serving makes them. Co-Authored-By: Claude Opus 5.5 * fix(environment): keep sorts earlier listings ruled out, and empty added references A sort a whole listing that placed a create did not follow stays ruled out for that create, though the entity that showed it is later deleted. A create that sends a reference's member empty after a later create added the reference agrees with it, as with a reference the first create showed. Co-Authored-By: Claude Opus 5.5 * docs(environment): restate the verification after the review round Co-Authored-By: Claude Opus 5.5 * fix(environment): leave out a reference the observed create left out A create the plan observes that left out a reference's member stored no such field, so a served call leaving the member out stores none either, though a later recorded create sent it empty; a call sending it empty still stores that. Co-Authored-By: Claude Opus 5.5 * docs(environment): restate the verification after #7539 merged main Co-Authored-By: Claude Opus 5.5 * fix(environment): keep the largest sequence value in replay, and store update clocks by the kept read Replay now keeps the largest value a sequence took, as serving's commitWrite does, so a lower sent value no longer lowers the next number. Serving stores every update's clocks in the spelling of the read the plan keeps as observation.read; replay now does the same through readSpelledClocks, instead of adopting each update's own read, so a later update whose read spells its clock otherwise contradicts the plan. The plain clock branch of restampedFields, which no caller used any more, is gone. Co-Authored-By: Claude Opus 5.5 * docs(environment): restate the verification after merging #7539 3b800a10 Co-Authored-By: Claude Opus 5.5 * fix(environment): keep learned sorts, references, and toggles faithful Do not adopt a sort an earlier update already ruled out. Learn a list reference only when every element shows the same nonempty fields. Promote a toggle only when every call started from a boolean. * test(environment): compare list-reference fields with matching values The mixed-field case uses the held tag's color, so rejection comes from the field set rather than a value the recording never stored. * test(mcp): assert bundle path issues by field The compare_bundles rejection reports each relative path as an invalid_format issue instead of one joined diagnostic sentence. * test(mcp): assert field issues for bundle paths and Dart ABI Relative bundle paths and rejected ABI selectors are reported as input issues on those fields, not as one joined diagnostic sentence. --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> Co-authored-by: Cursor Agent --- .../skills/reverse-engineer-anything/SKILL.md | 2 +- .../references/environments.md | 7 +- docs/agent-environments.md | 20 +- .../write-sorts-references-toggles/plan.md | 11 + .../write-sorts-references-toggles/spec.md | 41 +++ .../write-sorts-references-toggles/tasks.md | 11 + .../verification.md | 88 ++++++ docs/web-network-captures.md | 28 +- .../environmentStateItemWritesAnswer.test.ts | 101 ++++++ .../EnvironmentStateWriteSimulation.ts | 94 +++++- .../environmentStateItemWrites.test.ts | 137 ++++++++ .../analysisView/environmentStateView.ts | 2 + .../environmentStateWriteApply.ts | 44 ++- .../environmentStateWriteCalls.ts | 43 ++- .../environmentStateWriteCreateCalls.ts | 72 ++++- .../environmentStateWriteEffects.ts | 12 +- .../environmentStateWriteReferences.test.ts | 38 +++ .../environmentStateWriteReferences.ts | 92 +++++- .../environmentStateWriteRules.ts | 7 + .../environmentStateWriteVerdict.ts | 8 +- .../scenarioCaptureRecords.test.ts | 20 ++ .../analysisView/scenarioCaptureRecords.ts | 16 +- src/generatedPackageMetadata.ts | 2 +- tests/fixtures/browserScenarioCapture.ts | 27 +- tests/fixtures/environmentStateItemWrites.ts | 297 ++++++++++++++++++ 25 files changed, 1161 insertions(+), 59 deletions(-) create mode 100644 docs/specs/agent-environments/write-sorts-references-toggles/plan.md create mode 100644 docs/specs/agent-environments/write-sorts-references-toggles/spec.md create mode 100644 docs/specs/agent-environments/write-sorts-references-toggles/tasks.md create mode 100644 docs/specs/agent-environments/write-sorts-references-toggles/verification.md create mode 100644 src/application/environment/environmentStateItemWritesAnswer.test.ts create mode 100644 src/domain/analysisView/environmentStateItemWrites.test.ts create mode 100644 tests/fixtures/environmentStateItemWrites.ts diff --git a/.agents/skills/reverse-engineer-anything/SKILL.md b/.agents/skills/reverse-engineer-anything/SKILL.md index 1eb5366684..3c6f4e7e06 100644 --- a/.agents/skills/reverse-engineer-anything/SKILL.md +++ b/.agents/skills/reverse-engineer-anything/SKILL.md @@ -2,7 +2,7 @@ name: reverse-engineer-anything description: Reverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. metadata: - version: "54" + version: "55" --- # REA diff --git a/.agents/skills/reverse-engineer-anything/references/environments.md b/.agents/skills/reverse-engineer-anything/references/environments.md index 7d2ededf46..0be64fbab2 100644 --- a/.agents/skills/reverse-engineer-anything/references/environments.md +++ b/.agents/skills/reverse-engineer-anything/references/environments.md @@ -161,8 +161,11 @@ locators to exactly one element. 5. Update both objects and read each back, changing the fields a person would change (a name, an email address, a status, a due date) rather than only a description: a served update accepts only the members a recorded - update stored. Delete both and read each back (expect 404, or the object - marked deleted) and the list. + update stored. Run an action that turns something on or off (star, + bookmark, archive) twice in a row on one object, reading it back after + each: the second call shows whether it sets or flips the field, and the + environment serves the action only once a call has shown it. Delete both + and read each back (expect 404, or the object marked deleted) and the list. 6. Vary what the two creates send, however you write them: the first every field the application takes, the second only what a person must give (its title or name), with the other fields left out or empty. A served create diff --git a/docs/agent-environments.md b/docs/agent-environments.md index 62dbb3d76e..8bcc62be86 100644 --- a/docs/agent-environments.md +++ b/docs/agent-environments.md @@ -739,14 +739,14 @@ tools are not served, so no call claims a change it does not make; each one's plan states why in its `reason`. A served write changes the live state, which every MCP session and `data/get` then see: -| Effect | Call | -| ------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `create` | Adds an entity to the collection the path names, built from the observed entity with each ruled field taken from the call or generated, at the plan's `insert` position. A plan tied to its exemplar's collection writes only there. | -| `update` | Sets the sent members on the entity the path names, constants for members accepted only with their recorded value, and the clock, nested clock, increment, slug, stamp, derived, composed, reference, unique and renewed changes the plan models. | -| `update` (action) | A `POST`, `PUT` or `PATCH` to an item path plus an action segment (`/{type}/{id}/archive`) sets, on the entity before the segment, each scalar the recorded calls showed the action set (`set`), with any sent members; it is served only when a recorded call found those fields already set, so a toggle never is. | -| `delete` | Removes the entity the path names. | -| bulk `update` or `delete` | A `DELETE`, `PUT` or `PATCH` at a collection, or a `POST` to it plus `delete`, `remove` or `destroy`, applies its effect to every entity the body names, as recorded: an array of ids or of objects carrying the id field, at the root or as the body's one array member (`{"IDs": […], "Read": true}`). | -| `document` | Merges the body into the plan's document object, or replaces it for `PUT`. | +| Effect | Call | +| ------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `create` | Adds an entity to the collection the path names, built from the observed entity with each ruled field taken from the call or generated, at the plan's `insert` position. A plan tied to its exemplar's collection writes only there. | +| `update` | Sets the sent members on the entity the path names, constants for members accepted only with their recorded value, and the clock, nested clock, increment, slug, stamp, derived, composed, reference, unique and renewed changes the plan models. | +| `update` (action) | A `POST`, `PUT` or `PATCH` to an item path plus an action segment (`/{type}/{id}/archive`) sets, on the entity before the segment, each scalar the recorded calls showed the action set (`set`), or flips each boolean they showed it flipping both ways (`toggle`), with any sent members; it is served only when a recorded call found what it sets already set, or flipped back what it toggles. | +| `delete` | Removes the entity the path names. | +| bulk `update` or `delete` | A `DELETE`, `PUT` or `PATCH` at a collection, or a `POST` to it plus `delete`, `remove` or `destroy`, applies its effect to every entity the body names, as recorded: an array of ids or of objects carrying the id field, at the root or as the body's one array member (`{"IDs": […], "Read": true}`). | +| `document` | Merges the body into the plan's document object, or replaces it for `PUT`. | An update's unsent changes follow the plan's rules. A `held-clock` is a clock the client sends back as it read it and the application stamps anew (Ghost's @@ -757,7 +757,9 @@ the entity already names under its `created…by` counterpart into a `modified…by` or `updated…by` field. A `derived` field takes a value showing the one sent member's text that the entity stores nowhere (a field Ghost derives from `html`), only with that member's recorded value. An action's -`set` rule sets a scalar to the value the recorded calls left it at. +`set` rule sets a scalar to the value the recorded calls left it at, and its +`toggle` rule flips a boolean, refusing a call on an entity that holds none +there (Miniflux's `PUT /v1/entries/{id}/bookmark` stars and unstars). Its answer has the status and content type of the plan's recorded response, with the written entity or its identity rendered in, no body, or the recorded diff --git a/docs/specs/agent-environments/write-sorts-references-toggles/plan.md b/docs/specs/agent-environments/write-sorts-references-toggles/plan.md new file mode 100644 index 0000000000..80cfd70c25 --- /dev/null +++ b/docs/specs/agent-environments/write-sorts-references-toggles/plan.md @@ -0,0 +1,11 @@ +# Implementation plan + +1. **Sorted creates.** Placed creates whose confirming read was a whole listing in collection order may be placed + again; narrowing keeps the sorts a later listing follows and adds those its items show, the type's id field last. +2. **Listed references.** `referenceRule` reads a sent list of ids; `many` on the reference rule and its schema; + `referencedValue` maps each id; the create merge lets an empty reference agree and frees the members it fixed. +3. **Hidden bodies.** Scenario records give a request with a body Playwright did not expose, and no `Content-Type`, a + `bodySize` of 0. +4. **Toggles.** `toggledRules` turns a plan's `set` and a later call's opposite `set` into `toggle`; the rule's schema, + serving, later-call checks and the action verdict. +5. **Skill, tests and docs.** diff --git a/docs/specs/agent-environments/write-sorts-references-toggles/spec.md b/docs/specs/agent-environments/write-sorts-references-toggles/spec.md new file mode 100644 index 0000000000..75e1fa64b3 --- /dev/null +++ b/docs/specs/agent-environments/write-sorts-references-toggles/spec.md @@ -0,0 +1,41 @@ +# Serve sorted creates, listed references and toggles + +Issue: https://github.com/morluto/rea/issues/4933 + +## Outcome + +Over 148 recorded captures (the 130 measured before and 18 from a fresh recording round), #7539 withholds 302 of 694 +write plans. A short diagnosis of the largest clusters found three gaps REA could close: + +- **A create's sort learned too late.** Grafana lists folders by title. A recording's first scratch folder sorted + after the one seeded folder, so the plan placed creates last; the second sorted before the first, the list after it + contradicted the plan, and the update and delete after that went unconfirmed. +- **Lists of references.** Homebox creates an item from `parentId` and `tagIds` and answers with the parent and tag + objects. REA modelled a parent but not a list of tags, and a create that sent `null` and `[]` disagreed with one that + named both. +- **Toggles.** Miniflux stars and unstars an entry with a body-less `PUT /v1/entries/{id}/bookmark`. Chromium exposes + no body for it, which REA read as unusable, and an action flipping a flag back was taken to contradict the call that + set it. + +## Behaviour + +- **Sorted creates.** A create a whole listing in collection order placed last is placed again by a field a later such + listing shows it sorted by, the type's id field last. A sort an earlier such listing did not follow stays ruled out, + though the entity that showed it is gone. A create another read placed keeps its place. +- **Listed references.** A field holding one object per id a sent list names, in that order, is a `reference` with + `many`. A create that sent the member as `null` or `[]` and stored the field so agrees with one that named entities, + whichever create first showed the reference; + serving stores the entities a call names, or the empty value as that create did. A `null` id stays refused where + no recorded create sent one, and a call leaving the member out stores no field where the observed create showed + none. +- **Hidden bodies.** A scenario request whose body Playwright did not expose and that names no `Content-Type` sent no + body. +- **Toggles.** An item action a later call shows flipping back a boolean an earlier call set, when no call found it + already set, has a `toggle` rule: serving flips the field, refusing an entity that holds no boolean there. +- **Skill.** Recordings run an action twice in a row on one object, reading it back after each (skill 55). + +Plans and views that need none of this are unchanged. + +## Acceptance + +- No write of the 148 captures goes from served to withheld, and no view changes. diff --git a/docs/specs/agent-environments/write-sorts-references-toggles/tasks.md b/docs/specs/agent-environments/write-sorts-references-toggles/tasks.md new file mode 100644 index 0000000000..3fb6efc9ea --- /dev/null +++ b/docs/specs/agent-environments/write-sorts-references-toggles/tasks.md @@ -0,0 +1,11 @@ +# Tasks + +- [x] Place a create again by the sort a later whole listing shows. +- [x] Learn and serve references a sent list of ids names, and let an empty reference agree. +- [x] Treat a hidden request body without a `Content-Type` as absent. +- [x] Learn, check and serve toggles. +- [x] Ask recordings to run an action twice on one object. +- [x] Add view, serving, record and unit tests. +- [x] Document the rules. +- [x] Measure the recorded captures. +- [x] Run static, formatting, documentation and test gates. diff --git a/docs/specs/agent-environments/write-sorts-references-toggles/verification.md b/docs/specs/agent-environments/write-sorts-references-toggles/verification.md new file mode 100644 index 0000000000..c499b2beca --- /dev/null +++ b/docs/specs/agent-environments/write-sorts-references-toggles/verification.md @@ -0,0 +1,88 @@ +# Verification + +Executed on macOS with Node v24.22.1 on this branch and on #7539 at `30e2a582`; the census and the base reproduction +were repeated after merging #7539 at `2817665d`, `174a0ff2`, `5691f97f` (which merges main with #7332) +and `3b800a10`. + +## Diagnosis + +Over 148 recorded captures, #7539 withholds 302 of 694 write plans. The largest clusters: Homebox item writes (36), +unconfirmed deletes (23), Grafana's feature-flag evaluation calls (15, which change no state), Linkding renumbering +bundles after a delete (7) and Miniflux's bookmark toggle (5). + +- **Grafana.** A fresh recording created "Scratch Pier 3", which the list after it showed after the one seeded folder, + then "Scratch Meadow Notes 4", which the list after it showed before the first, by title. The create plan had settled + on `last` when the first list confirmed it, so the second list contradicted it, and the folder update and delete + after it went unconfirmed. +- **Homebox.** Item creates send `parentId` and `tagIds` and answer with `parent` and `tags` objects. The recording + that created one item with neither (`null` and `[]`) and one with both was withheld: `tags` had no rule, and the + first create fixed both members to their empty values. +- **Miniflux.** `PUT /v1/entries/{id}/bookmark` sends no body and answers `204`. Chromium exposes no body for it and + the request names no `Content-Type`; REA read the body as unusable. Across the captures, every `POST`, `PUT` or + `PATCH` whose body was captured names `content-type` among its request header names (835 of 835). + +## Gates + +- `npm run -s check:fast`, `format:check`, `knip`, `scan:todos`, `docs:check`, `verify:test-discovery`, + `mcp-catalog:check` and `jscpd`: passed. +- `npx vitest run --project domain --project services`: 11,886 passed, 6 skipped, merged with `3b800a10`. An earlier run of the suite failed a + `SetupRegistrationPrewarm` runner test whose child process outran its 5-second timeout; alone it passes on this + branch and on #7539 (3.0 to 4.7 seconds). +- `npm run -s test:focused --` the environment build, export and serve CLI boundaries, the analysis-view CLI and MCP + boundaries and the environment serve MCP boundary: 7 files, 81 passed. `outputSchemaProfile`, `toolSchemaValidity`, + `mcpSchemaValidation`, `outputSchemaSemanticConstraints` and `analysisViewMcp`: 5 files, 54 passed; the compact + catalog stays under its frame. + +## Tests + +- View: a create placed last, then a create a later whole listing shows before it by title, learns `title` ascending + and its delete is served; an item create naming its parent and tags by id, after one naming none, learns + `reference` rules for `parent` and for `tags` (`many`) and is served; a body-less action two calls show starring + then unstarring an entry learns `toggle` and is served, and is not when its calls flip the flag one way only or a + later call leaves it unchanged. +- Serving: a create stores the parent and the tags its ids name, in the order sent; a create leaving out both, or + sending them as `null` and `[]`, stores `null` and `[]`; a tag the environment does not hold is refused, and so is a + `null` shelf where every recorded create named one; an entry the recording never starred flips on each call. +- Records: a hidden request body is absent when the request names no `Content-Type`, and stays unusable when it names + one. +- The nine behaviour tests fail on #7539 at `30e2a582`, `2817665d`, `174a0ff2`, `5691f97f` and `3b800a10`; the three that guard toggles and `null` ids + pass there too. + +## Write plans and views over recorded captures + +The 148 captures, built by #7539 and by this branch: 694 plans, 392 served and 399, both at `30e2a582` and after +merging `2817665d`, `174a0ff2`, `5691f97f` and +`3b800a10`. + +- No write goes from served to withheld, and no served plan changes. Newly served: Grafana's folder create, update and + delete in a fresh recording; Homebox's item create in another; Miniflux's bookmark toggle in three recordings. +- Five withheld plans explain more. Two Miniflux recordings learn the toggle but stay withheld: a list of starred + entries with no `order` parameter lists them by `published_at`, which REA answers in state order. Linkding's + unarchive learns its body-less call and stays `unconfirmed`, seen once. Two Homebox item writes learn their tag + references and stay withheld for other members. +- Views are unchanged: 3,188 views across the captures. + +## Review round + +Greptile's review of `4e777d28` raised five findings. Two were in this branch's code and are fixed: + +- A sort an earlier whole listing ruled out could come back from a later listing once the entity that ruled it out was + deleted, and a create's retry could adopt it. Each create now keeps the sorts its whole listings did not follow. +- A create sending a reference's member as `null` after a later create added the reference, the first having shown no + such field, was withheld: the empty-reference agreement now applies there too. + +Both tests fail on `4e777d28` and on #7539 at `174a0ff2`; the census is unchanged. The other three were in #7332's and +#7539's code: `308aebf0` (#7332), already here, covers the made-object count, and #7539's `174a0ff2`, merged here, +fixes the update default and the read clock spelling. + +Greptile's review of `f7c1bfe9` raised one more, fixed: once a later create sent a reference's member as `null`, a +served create leaving the member out stored `null`, though the create the plan observes left it out and stored no such +field. Serving now leaves the field out there, and still stores `null` for a call that sends it; the serving test fails +on `f7c1bfe9`. The census is unchanged. + +## Not verified + +- No environment was rebuilt and run with agents for this branch, and no recording was made with skill 55. +- The order an application gives a list read with no sort parameter (Miniflux's `published_at`) when it is not state + order. +- The rest of the Homebox cluster: `assetId` numbering and item updates; Linkding renumbering bundles after a delete. diff --git a/docs/web-network-captures.md b/docs/web-network-captures.md index 3f6573262c..cc8f7d9aa8 100644 --- a/docs/web-network-captures.md +++ b/docs/web-network-captures.md @@ -740,7 +740,9 @@ one-element array member) that a later item read of a type first shows (served only with the path values its recorded call sent, unless it creates into the type's own collection, whose path has no parameters, and a `path` rule stores each of its path values), and otherwise `none` with reason `no-state-target`. A plan is built from the tool's first successful call: its -request body must be a usable JSON object (as response bodies are usable), and +request body must be a usable JSON object (as response bodies are usable), or +absent for an item action (a scenario request is absent when Playwright exposed +no body and it names no `Content-Type`, which the bodies browsers send carry), and the written entity must show in that call's response or in a later read before the next write. Each field of a created entity gets the first rule that explains it: generated `identity` (a sequence, UUID or token, by the recorded @@ -780,15 +782,19 @@ with the entity leaves out a field only later reads showed, which no initial entity holds, when the entity does not hold it, as Vikunja's reads of tasks it did not create show no `subscription`. A `reference` is an object holding fields of the -entity a sent member names, as the state holds it (a parent beside `parentId`): +entity a sent member names, as the state holds it (a parent beside `parentId`), +or a list of them, one per id a sent list names, in its order (`many`, tags +beside `tagIds`): the member is accepted with any value, serving looks the entity up and refuses a call naming one the environment does not hold or holds in more than one collection, and the field is absent, from the entity and the create's answer, when the call does not send the member. An id naming entities of one type in several collections teaches no reference. Each later create widens its plan when, explained on its own, it agrees with it: both calls show the same fields -but for a reference whose member a call without it did not send (one an earlier -create added stays), every field both rule they rule alike (an echo one showed +but for a reference whose member a call without it did not send, or sent as +`null` or `[]` and stored so (one an earlier create added stays, and a call may +send the member empty as that create did; one leaving it out stores no such field +when the create the plan observes showed none), every field both rule they rule alike (an echo one showed sorted stays sorted, unless a call stored its strings out of order as sent), and a field neither rules holds one value, unless the later call's rule for it spells the earlier calls' value too: a `composed` or `template` string, or a @@ -868,14 +874,17 @@ is unchanged) must be a clock, a nested clock, an increment, a slug, a fresh `un value, a `stamp` of the acting user, a value `derived` from the one member it stores nowhere, a `reference` to the entity a sent member names (kept by a call that names the entity the field already repeats), a `composed` string whose parts read members it stored as sent or fields it left as they were and spell the value before the call too, or for -an item action a `set` of a scalar to the value it showed, or, for a `PUT`, a +an item action a `set` of a scalar to the value it showed (a `toggle` once a +later call flips back the boolean an earlier one set, when no call found it set +already), or, for a `PUT`, a member it left out that the application cleared to an empty value, which calls that leave it out then store (`body.defaults`) as long as every call that left it out stored that value (a call naming a `reference` field's entity by its member does not leave the field out), or the plan is `unexplained-change`; an increment, slug, stamp, derived -value or set must hold for every call of the tool, and an action is served -only once a call found what it sets already set. A write is served only +value, set or toggle must hold for every call of the tool, and an action is +served only once a call found what it sets already set, or flipped back what it +toggles. A write is served only with the query its first call sent, which serving applies to a call that leaves a parameter out and the catalog advertises as required. Every member every call sent is required, except by `PATCH` and `POST` updates and members a create sent empty. The state holds an update's @@ -919,7 +928,10 @@ contradicted it; otherwise its `reason` is the first of its build reason (`no-call`, `body-unusable`, `no-observation`, `unexplained-change`), `contradicted` (with `contradicted_by`), `side-effect`, `unconfirmed` and `incomplete-entities` (an update or delete that answers with the entity, when no initial entity holds every field its recorded answer shows but those only later reads showed, counting for an update the fields its rules write, so it could answer only for entities the environment created). An otherwise served plan whose learned rules generate clock values is withheld with `unknown-clock` when no global clock start is established; its learned rules and confirming reads remain retained. `checked_record_ordinals` lists the confirming reads, `insert` places created -entities (sorted by a field the collection's recorded order shows, else last), +entities (sorted by a field the collection's recorded order shows, else last; +a create a whole listing in collection order placed last is placed again by a +field a later such listing shows it sorted by and no earlier one ruled out, the +type's id field last), and `write_clock` is the end of the last timed record, to the next second, when all request completions are known; otherwise it is null. An update that writes the field its type's creates are sorted by moves its entity diff --git a/src/application/environment/environmentStateItemWritesAnswer.test.ts b/src/application/environment/environmentStateItemWritesAnswer.test.ts new file mode 100644 index 0000000000..fdc21025c5 --- /dev/null +++ b/src/application/environment/environmentStateItemWritesAnswer.test.ts @@ -0,0 +1,101 @@ +import { describe, expect, it } from "vitest"; + +import { + ADDED_EMPTY_REFERENCE_ROWS, + BOOKMARK_TOGGLE_ROWS, + ITEM_LOCATIONS, + ITEM_REFERENCES_ROWS, + ITEM_TAGS, +} from "../../../tests/fixtures/environmentStateItemWrites.js"; +import { FILED_ROWS } from "../../../tests/fixtures/environmentStateDerivedValues.js"; +import { servedFrom } from "../../../tests/fixtures/environmentStateServing.js"; + +describe("served item writes", () => { + it("answers a create with the parent and tags its ids name, in the order sent", () => { + const items = servedFrom(ITEM_REFERENCES_ROWS); + const body = items.answer("post_items", { + body: { + name: "Snow shovel", + parentId: ITEM_LOCATIONS[0]!["id"]!, + tagIds: [ITEM_TAGS[1]!["id"]!, ITEM_TAGS[0]!["id"]!], + }, + }).body; + expect(body).toMatchObject({ + name: "Snow shovel", + parent: ITEM_LOCATIONS[0], + tags: [ITEM_TAGS[1], ITEM_TAGS[0]], + }); + }); + + it("answers a create naming no parent and no tags with none", () => { + const items = servedFrom(ITEM_REFERENCES_ROWS); + expect( + items.answer("post_items", { body: { name: "Spare fuse" } }).body, + ).toMatchObject({ + name: "Spare fuse", + parent: null, + tags: [], + }); + }); + + it("answers a create sending no parent and no tags as a recorded create sent them", () => { + const items = servedFrom(ITEM_REFERENCES_ROWS); + expect( + items.answer("post_items", { + body: { name: "Spare fuse", parentId: null, tagIds: [] }, + }).body, + ).toMatchObject({ name: "Spare fuse", parent: null, tags: [] }); + }); + + it("answers a create leaving out a parent the first create left out with none, and one sending null with null", () => { + const items = servedFrom(ADDED_EMPTY_REFERENCE_ROWS); + const left = items.answer("post_items", { body: { name: "Spare fuse" } }) + .body as Record; + expect(left).toMatchObject({ name: "Spare fuse" }); + expect(Object.hasOwn(left, "parent")).toBe(false); + expect( + items.answer("post_items", { + body: { name: "Spare bulb", parentId: null }, + }).body, + ).toMatchObject({ name: "Spare bulb", parent: null }); + expect( + items.answer("post_items", { + body: { name: "Spare cord", parentId: ITEM_LOCATIONS[0]!["id"]! }, + }).body, + ).toMatchObject({ name: "Spare cord", parent: ITEM_LOCATIONS[0] }); + }); + + it("refuses a create naming no shelf when every recorded create named one", () => { + const books = servedFrom(FILED_ROWS); + expect( + books.refusal("post_books", { body: { name: "Loose", shelf_id: null } }), + ).toMatchObject({ message: expect.stringContaining("shelf_id") }); + }); + + it("refuses a create naming a tag the environment does not hold", () => { + const items = servedFrom(ITEM_REFERENCES_ROWS); + expect( + items.refusal("post_items", { + body: { + name: "Mystery", + tagIds: ["3b9f2a10-0009-4000-8000-000000000009"], + }, + }), + ).toMatchObject({ message: expect.stringContaining("tagIds") }); + }); + + it("flips an entry's flag on each body-less call", () => { + const entries = servedFrom(BOOKMARK_TOGGLE_ROWS); + const starred = () => + ( + entries.answer("get_entries_by_id", { id: "2" }).body as { + starred: boolean; + } + ).starred; + expect(starred()).toBe(false); + entries.answer("put_entries_bookmark_by_id", { id: "2" }); + expect(starred()).toBe(true); + entries.answer("put_entries_bookmark_by_id", { id: "2" }); + expect(starred()).toBe(false); + }); +}); diff --git a/src/domain/analysisView/EnvironmentStateWriteSimulation.ts b/src/domain/analysisView/EnvironmentStateWriteSimulation.ts index 488deec68b..28c6527cc4 100644 --- a/src/domain/analysisView/EnvironmentStateWriteSimulation.ts +++ b/src/domain/analysisView/EnvironmentStateWriteSimulation.ts @@ -109,6 +109,26 @@ const keptInOrder = ( ); }); +/** The sorts by a field every listed item holds as a string or number, and whether the items follow each. */ +const listedSorts = ( + items: readonly JsonObject[], +): { + readonly rule: Extract; + readonly follows: boolean; +}[] => + [...new Set(items.flatMap((item) => Object.keys(item)))] + .filter((field) => + items.every((item) => + ["string", "number"].includes(typeof ownMember(item, field)), + ), + ) + .flatMap((field) => + (["ascending", "descending"] as const).map((direction) => ({ + rule: { rule: "sorted", field, direction } as const, + follows: orderedBy(items, field, direction), + })), + ); + /** Whether a list read's query sets a parameter the view's query model reads as a sort or its direction. */ const sortedByQuery = ( view: Extract, @@ -182,6 +202,10 @@ export class EnvironmentStateWriteSimulation { readonly #refutedSorts = new Map>(); /** Creates whose insert rule a passing list check relied on, so it is no longer changed. */ readonly #placed = new Set(); + /** Placed creates a read other than a whole listing in collection order placed, whose insert stays as it is. */ + readonly #settledInserts = new Set(); + /** Sorts a whole listing that placed a create did not follow, by create, which no later listing adds back. */ + readonly #unfollowedSorts = new Map>(); /** The order a read showed a type's collections sorted in after an update, when no create of it showed one. */ readonly #learnedSorts = new Map(); /** Entities writes no plan models made, taken when an item read first shows them. */ @@ -797,7 +821,9 @@ export class EnvironmentStateWriteSimulation { this.#drafts.get(name)!.confirmed.push(record.ordinal); if (showsPlacement(view, outcome, this.#inserted, name)) { this.#placed.add(name); - if (view.shape === "list") this.#narrowInserts(view, record, name); + // Only whole listings in collection order narrow the inserts a later list may still choose among. + if (view.shape !== "list" || !this.#narrowInserts(view, record, name)) + this.#settledInserts.add(name); } } return shown.unseen; @@ -937,23 +963,44 @@ export class EnvironmentStateWriteSimulation { }); } - /** A whole-collection read in collection order that placed a create leaves it only the sorted inserts whose order the read follows; one its query sorts shows the query's order instead. */ + /** A whole-collection read in collection order that placed a create leaves it only the sorted inserts whose order the read follows, adding those by a field its items all hold, and says whether it did; one its query sorts shows the query's order instead. */ #narrowInserts( view: Extract, record: StateRecordFacts, name: string, - ): void { - if (sortedByQuery(view, record)) return; + ): boolean { + if (sortedByQuery(view, record)) return false; const draft = this.#drafts.get(name)!; - const kept = this.#wholeListing(view, record, (items) => - draft.candidates.filter( + const kept = this.#wholeListing(view, record, (items) => { + const followed = draft.candidates.filter( (candidate) => candidate.rule !== "sorted" || same(candidate, draft.plan.insert) || orderedBy(items, candidate.field, candidate.direction), - ), - ); - if (kept !== undefined) draft.candidates = kept; + ); + const sorts = listedSorts(items); + const unfollowed = this.#unfollowedSorts.get(name) ?? new Set(); + for (const { rule, follows } of sorts) + if (!follows) unfollowed.add(JSON.stringify(rule)); + this.#unfollowedSorts.set(name, unfollowed); + // A list shown in the order of the field that names its items says less than one in a field people read. + const idField = this.#types.get(view.type)?.id_field; + const learned = sorts + .map(({ rule }) => rule) + .filter( + (rule) => + !unfollowed.has(JSON.stringify(rule)) && + !followed.some((candidate) => same(candidate, rule)), + ) + .sort( + (left, right) => + Number(left.field === idField) - Number(right.field === idField), + ); + return [...followed, ...learned]; + }); + if (kept === undefined) return false; + draft.candidates = kept; + return true; } /** @@ -1110,9 +1157,25 @@ export class EnvironmentStateWriteSimulation { if (creates.size !== 1) return undefined; const [name] = creates; const draft = this.#drafts.get(name!)!; - // An item read confirms the entity, not where a list places it. - if (!exercised.has(name!) || this.#placed.has(name!)) return undefined; - for (const candidate of draft.candidates.slice(1)) { + // An item read confirms the entity, not where a list places it; a read that placed it otherwise than a whole + // listing did settles where it goes. + if (!exercised.has(name!) || this.#settledInserts.has(name!)) + return undefined; + // A create whole listings placed may still take a sort they followed: one placed last fits it before the next shows the order. + const options = this.#placed.has(name!) + ? draft.candidates.filter( + (candidate) => + candidate.rule === "sorted" && !same(candidate, draft.plan.insert), + ) + : draft.candidates.slice(1); + // A sort an earlier update's whole list ruled out must not become this create's order: serving would move those updates by it. + const refuted = this.#refutedSorts.get(view.type); + for (const candidate of options) { + if ( + candidate.rule === "sorted" && + refuted?.has(JSON.stringify(candidate)) === true + ) + continue; this.#state.reinsert(view.type, line.collection, inserted, candidate); const outcome = this.#check(view, record); if (outcome?.pass === true) { @@ -1120,6 +1183,13 @@ export class EnvironmentStateWriteSimulation { return outcome; } } + if (draft.plan.insert !== null) + this.#state.reinsert( + view.type, + line.collection, + inserted, + draft.plan.insert, + ); return undefined; } } diff --git a/src/domain/analysisView/environmentStateItemWrites.test.ts b/src/domain/analysisView/environmentStateItemWrites.test.ts new file mode 100644 index 0000000000..e1fccc2b61 --- /dev/null +++ b/src/domain/analysisView/environmentStateItemWrites.test.ts @@ -0,0 +1,137 @@ +import { describe, expect, it } from "vitest"; + +import { + timedCaptureEvidence, + type TimedCaptureRow, +} from "../../../tests/fixtures/analysisView.js"; +import { + ADDED_EMPTY_REFERENCE_ROWS, + BOOKMARK_TOGGLE_ROWS, + CREATES_BY_TITLE_ROWS, + ITEM_REFERENCES_ROWS, + NULL_THEN_STARRED_ROWS, + RANKED_AFTER_DELETE_ROWS, + REFUTED_TITLE_SORT_ROWS, + feedEntry, +} from "../../../tests/fixtures/environmentStateItemWrites.js"; +import { environmentStateViewOf } from "../../../tests/fixtures/environmentStateWrites.js"; + +const planOf = (rows: readonly TimedCaptureRow[], tool: string) => { + const plan = environmentStateViewOf(timedCaptureEvidence(rows)).writes.find( + (candidate) => candidate.tool === tool, + ); + if (plan === undefined) throw new Error(`No plan for ${tool}`); + return plan; +}; + +describe("item writes", () => { + it("learns a create's sort from a later list its full order follows, after one create was placed last", () => { + expect(planOf(CREATES_BY_TITLE_ROWS, "post_folders")).toMatchObject({ + served: true, + insert: { rule: "sorted", field: "title", direction: "ascending" }, + }); + expect(planOf(CREATES_BY_TITLE_ROWS, "delete_folders_by_id")).toMatchObject( + { served: true }, + ); + }); + + it("keeps a sort an earlier list ruled out away from a create, after the entity that ruled it out is deleted", () => { + expect(planOf(RANKED_AFTER_DELETE_ROWS, "post_folders")).toMatchObject({ + served: true, + insert: { rule: "sorted", field: "position", direction: "ascending" }, + }); + }); + + it("serves a create sending no parent after an earlier create added the parent reference", () => { + const plan = planOf(ADDED_EMPTY_REFERENCE_ROWS, "post_items"); + expect(plan).toMatchObject({ served: true }); + expect(plan.rules).toContainEqual( + expect.objectContaining({ + field: "parent", + rule: "reference", + member: "parentId", + }), + ); + }); + + it("serves a create whose parent and tags a call names by id, one call naming none", () => { + const plan = planOf(ITEM_REFERENCES_ROWS, "post_items"); + expect(plan).toMatchObject({ served: true }); + expect(plan.rules).toContainEqual( + expect.objectContaining({ + field: "parent", + rule: "reference", + member: "parentId", + type: "/api/locations", + }), + ); + expect(plan.rules).toContainEqual( + expect.objectContaining({ + field: "tags", + rule: "reference", + member: "tagIds", + type: "/api/tags", + many: true, + }), + ); + }); + + it("serves a body-less action that calls show flipping its flag both ways", () => { + expect( + planOf(BOOKMARK_TOGGLE_ROWS, "put_entries_bookmark_by_id"), + ).toMatchObject({ + served: true, + address: { by: "action", segment: "bookmark" }, + rules: expect.arrayContaining([{ field: "starred", rule: "toggle" }]), + }); + }); + + it("serves no action whose calls flipped its flag one way only", () => { + const starred = feedEntry(1, true, "2026-10-17T18:00:03.120000Z"); + expect( + planOf( + [ + ...BOOKMARK_TOGGLE_ROWS.slice(0, 4), + ["GET", "/api/entries/1", starred], + ], + "put_entries_bookmark_by_id", + ), + ).toMatchObject({ served: false }); + }); + + it("does not learn a title sort an earlier update's kept order ruled out", () => { + const ascending = { + rule: "sorted", + field: "title", + direction: "ascending", + }; + const update = planOf(REFUTED_TITLE_SORT_ROWS, "patch_folders_by_id"); + const created = planOf(REFUTED_TITLE_SORT_ROWS, "post_folders"); + expect(update).toMatchObject({ served: true }); + expect(update.insert).not.toEqual(ascending); + expect(created.insert).not.toEqual(ascending); + }); + + it("serves no toggle when an earlier call found the flag null", () => { + const plan = planOf(NULL_THEN_STARRED_ROWS, "put_entries_bookmark_by_id"); + expect(plan).toMatchObject({ served: false }); + expect(plan.rules).not.toContainEqual({ + field: "starred", + rule: "toggle", + }); + }); + + it("serves no toggle a call left its flag unchanged under", () => { + const unchanged = feedEntry(1, false, "2026-10-17T18:00:08.050000Z"); + expect( + planOf( + [ + ...BOOKMARK_TOGGLE_ROWS, + ["PUT", "/api/entries/1/bookmark", "", 204], + ["GET", "/api/entries/1", unchanged], + ], + "put_entries_bookmark_by_id", + ), + ).toMatchObject({ served: false }); + }); +}); diff --git a/src/domain/analysisView/environmentStateView.ts b/src/domain/analysisView/environmentStateView.ts index ce9f8dc6cc..82e0f0fb32 100644 --- a/src/domain/analysisView/environmentStateView.ts +++ b/src/domain/analysisView/environmentStateView.ts @@ -323,6 +323,7 @@ const fieldRuleSchema = z.discriminatedUnion("rule", [ member: z.string(), type: z.string(), fields: z.array(z.string()).min(1), + many: z.literal(true).exactOptional(), }), z.strictObject({ ...ruled, rule: z.literal("increment") }), z.strictObject({ @@ -343,6 +344,7 @@ const fieldRuleSchema = z.discriminatedUnion("rule", [ member: z.string(), }), z.strictObject({ ...ruled, rule: z.literal("set"), value: jsonValueSchema }), + z.strictObject({ ...ruled, rule: z.literal("toggle") }), ]); const insertRuleSchema = z.union([ z.strictObject({ rule: z.enum(["first", "last"]) }), diff --git a/src/domain/analysisView/environmentStateWriteApply.ts b/src/domain/analysisView/environmentStateWriteApply.ts index 4af66a7de3..255dd25d89 100644 --- a/src/domain/analysisView/environmentStateWriteApply.ts +++ b/src/domain/analysisView/environmentStateWriteApply.ts @@ -223,12 +223,20 @@ const ruledValue = ( observed: JsonObject, call: WriteCall, generate: GenerateValue, - referenced?: ReferencedEntity, + references: { + readonly referenced: ReferencedEntity | undefined; + readonly blank: WriteBody["blank"]; + }, ): Applied => { const body = call.body ?? {}; switch (rule.rule) { case "reference": - return referencedValue(rule, body, referenced); + return referencedValue( + rule, + body, + references.referenced, + references.blank, + ); case "sequence": return rule.sent === true && Object.hasOwn(body, rule.field) ? ok(body[rule.field]!) @@ -333,12 +341,17 @@ const copiedShapes = ( }), ); -/** A create's call with the members it left out that recorded calls sent empty, as the earliest sent them; one a call that left it out showed a default for keeps that default. */ -const withBlanks = (body: WriteBody, call: WriteCall): WriteCall => { +/** A create's call with the members it left out that recorded calls sent empty, as the earliest sent them; one a call that left it out showed a default for keeps that default, and one `unheld` names stays left out. */ +const withBlanks = ( + body: WriteBody, + call: WriteCall, + unheld: ReadonlySet, +): WriteCall => { const given = call.body ?? {}; const left = (body.blank ?? []).filter( ({ member }, index, blank) => !Object.hasOwn(given, member) && + !unheld.has(member) && !(body.defaults ?? []).some((entry) => entry.member === member) && blank.findIndex((entry) => entry.member === member) === index, ); @@ -363,6 +376,14 @@ export const createdFields = ( idField?: string, ): Applied => { if (plan.body === null) return err("The tool has no modelled body."); + // A reference the observed entity lacks is one a create left out and stored nothing for, as a call leaving it out does. + const unheld = new Set( + plan.rules.flatMap((rule) => + rule.rule === "reference" && !Object.hasOwn(observed, rule.field) + ? [rule.member] + : [], + ), + ); // A member a `sequence` with `sent` numbers when a call leaves it out takes the next number, not an empty value one sent. const numbered = new Set( plan.rules.flatMap((rule) => @@ -377,6 +398,7 @@ export const createdFields = ( ), }, sentCall, + unheld, ); const problem = bodyProblem(plan.body, call, { entity: { ...copiedShapes(plan.rules, observed), ...observed }, @@ -387,7 +409,10 @@ export const createdFields = ( (plan.body.defaults ?? []).map(({ member, value }) => [member, value]), ); for (const rule of plan.rules) { - const value = ruledValue(rule, observed, call, generate, referenced); + const value = ruledValue(rule, observed, call, generate, { + referenced, + blank: plan.body.blank, + }); if (!value.ok) return value; if (value.value !== undefined) setOwnMember(written, rule.field, value.value); @@ -482,8 +507,13 @@ export const updatedFields = ( for (const rule of plan.rules) { const stated = statedValue(rule, sent); if (stated !== undefined) setOwnMember(written, rule.field, stated); - else if (rule.rule === "reference" && Object.hasOwn(sent, rule.member)) { - const value = referencedValue(rule, sent, referenced); + else if (rule.rule === "toggle") { + const was = ownMember(before, rule.field); + if (typeof was !== "boolean") + return err(`The entity holds no flag ${rule.field} to flip.`); + setOwnMember(written, rule.field, !was); + } else if (rule.rule === "reference" && Object.hasOwn(sent, rule.member)) { + const value = referencedValue(rule, sent, referenced, plan.body.blank); if (!value.ok) return value; if (value.value !== undefined) setOwnMember(written, rule.field, value.value); diff --git a/src/domain/analysisView/environmentStateWriteCalls.ts b/src/domain/analysisView/environmentStateWriteCalls.ts index 2d6ab4087b..00fca10b4d 100644 --- a/src/domain/analysisView/environmentStateWriteCalls.ts +++ b/src/domain/analysisView/environmentStateWriteCalls.ts @@ -24,7 +24,8 @@ export const unsentChange = (rule: FieldRule): boolean => rule.rule === "composed" || rule.rule === "stamp" || rule.rule === "derived" || - rule.rule === "set"; + rule.rule === "set" || + rule.rule === "toggle"; /** Whether serving gives a rule's field a new value on every call, so no call may have kept it. */ export const regenerates = (rule: FieldRule): boolean => @@ -56,6 +57,34 @@ export const disagreesWithCall = ( (regenerates(rule) && call.kept.includes(rule.field)), ); +/** A plan's set rules a later call flipped back, and that call's own, as toggles; `steady` names fields a call found already set, which an action sets rather than toggles. Every earlier call must have started from a boolean too, or serving rejects the first one. */ +export const toggledRules = ( + rules: readonly FieldRule[], + later: readonly FieldRule[], + before: JsonObject, + steady: ReadonlySet, + earlier: readonly { readonly startedBoolean: ReadonlySet }[], +): { readonly rules: FieldRule[]; readonly later: FieldRule[] } => { + const flipped = new Set( + later.flatMap((rule) => { + const was = ownMember(before, rule.field); + const held = rules.find((other) => other.field === rule.field); + return rule.rule === "set" && + typeof was === "boolean" && + rule.value === !was && + !steady.has(rule.field) && + earlier.every((call) => call.startedBoolean.has(rule.field)) && + ((held?.rule === "set" && held.value === was) || + held?.rule === "toggle") + ? [rule.field] + : []; + }), + ); + const toggle = (rule: FieldRule): FieldRule => + flipped.has(rule.field) ? { field: rule.field, rule: "toggle" } : rule; + return { rules: rules.map(toggle), later: later.map(toggle) }; +}; + /** The fields a call found already at the value a set rule gives them: the action sets them rather than toggling them. */ export const steadyFields = ( rules: readonly FieldRule[], @@ -109,6 +138,8 @@ export interface UpdateCall { readonly sent: ReadonlyMap; readonly sentMembers: ReadonlySet; readonly kept: readonly string[]; + /** Fields whose value before the call was a boolean. A toggle requires this of every call. */ + readonly startedBoolean: ReadonlySet; } /** How a later update's entity and the plan's calls and held entities showed `&` in the slugs it learns. */ @@ -194,8 +225,8 @@ export const contradictsEarlierCalls = ( integerOf(observation.value) !== undefined ); if (rule.rule === "set") return !sameObserved(observation, expected!); - // Serving stamps a clock on every call, so a field an earlier call kept contradicts it. - if (rule.rule === "clock") return true; + // Serving stamps a clock and flips a toggle on every call, so a field an earlier call kept contradicts it. + if (rule.rule === "clock" || rule.rule === "toggle") return true; if (rule.rule === "stamp") return !sameObserved(observation, expected!); if (rule.rule === "derived") return sent.has(rule.member) && !sameObserved(observation, expected!); @@ -404,6 +435,11 @@ export const rememberCall = ( sent: scalars(call.sent), sentMembers: new Set(Object.keys(call.sent)), kept, + startedBoolean: new Set( + Object.entries(call.before).flatMap(([field, value]) => + typeof value === "boolean" ? [field] : [], + ), + ), }; retain( 64 + @@ -413,6 +449,7 @@ export const rememberCall = ( ...facts.sent, [...facts.sentMembers], kept, + [...facts.startedBoolean], ]) * 2, ); diff --git a/src/domain/analysisView/environmentStateWriteCreateCalls.ts b/src/domain/analysisView/environmentStateWriteCreateCalls.ts index f52b5fdf80..bbd92bbefb 100644 --- a/src/domain/analysisView/environmentStateWriteCreateCalls.ts +++ b/src/domain/analysisView/environmentStateWriteCreateCalls.ts @@ -9,6 +9,7 @@ import { import { storedKinds } from "./environmentStateWriteApply.js"; import { fittedClockFormat } from "./environmentStateWriteClockFormats.js"; import { composedValue } from "./environmentStateWriteComposed.js"; +import { emptyReference } from "./environmentStateWriteReferences.js"; import { alikeRules, isBlankValue, @@ -303,6 +304,8 @@ interface FieldCalls { readonly exemplar: JsonObject; readonly later: { readonly entity: JsonObject; readonly body: JsonObject }; readonly sentBefore: ReadonlySet; + /** The values the plan's calls sent members it fixes or saw sent empty. */ + readonly pinned: ReadonlyMap; /** Members an earlier call left out and the application stored a fixed value for (`body.defaults`). */ readonly defaulted: ReadonlySet; readonly required: readonly string[]; @@ -320,6 +323,12 @@ const addedReference = ( return Object.hasOwn(later.body, added.member) ? undefined : { rule: added }; + // A call that named nothing and stored nothing agrees, as it does with a reference the first create showed. + if ( + after === undefined && + emptyReference(later.body[added.member], later.entity[field]) + ) + return { rule: added }; const rule = after === undefined ? undefined @@ -408,6 +417,19 @@ const mergedField = ( ); return rule === undefined ? undefined : { rule }; } + // A reference one call showed and another that named nothing, storing nothing, agree. + if ( + before?.rule === "reference" && + after === undefined && + emptyReference(later.body[before.member], later.entity[field]) + ) + return { rule: before }; + if ( + before === undefined && + after?.rule === "reference" && + emptyReference(calls.pinned.get(after.member), exemplar[field]) + ) + return { rule: after }; // The plan echoes a member this call left out: what it stored there is the member's default. if (before !== undefined) return before.rule === "echo" && !Object.hasOwn(later.body, field) @@ -460,6 +482,16 @@ export const mergedCreatePlan = ( ...plan.body.fixed.map(({ member }) => member), ...plan.body.required, ]); + const pinned = new Map([ + ...(plan.body.blank ?? []).map(({ member, sent }): [string, JsonValue] => [ + member, + sent, + ]), + ...plan.body.fixed.map(({ member, sent }): [string, JsonValue] => [ + member, + sent, + ]), + ]); const merged = new Map(); const defaults: Default[] = []; for (const field of new Set([ @@ -473,6 +505,7 @@ export const mergedCreatePlan = ( exemplar, later, sentBefore, + pinned, defaulted: new Set( (plan.body.defaults ?? []).map(({ member }) => member), ), @@ -483,7 +516,20 @@ export const mergedCreatePlan = ( if (outcome.rule !== undefined) merged.set(field, outcome.rule); if (outcome.default !== undefined) defaults.push(outcome.default); } - const body = mergedBody(plan.body, later, exemplar); + // A member a reference reads is accepted with any id, though a call that named nothing showed it fixed. + const read = new Set( + [...merged.values()].flatMap((rule) => + rule.rule === "reference" ? [rule.member] : [], + ), + ); + const body = mergedBody( + released(plan.body, read), + { + ...later, + contract: released(later.contract, read), + }, + exemplar, + ); const joined = body === undefined ? undefined @@ -518,6 +564,30 @@ const joinedBlanks = ( return blank; }; +/** A body contract with the members references read accepted, and kept as sent empty, rather than fixed at the empty value a call sent. */ +const released = (body: WriteBody, read: ReadonlySet): WriteBody => { + const freed = body.fixed.filter( + ({ member, sent }) => read.has(member) && emptyReference(sent, sent), + ); + if (freed.length === 0) return body; + return { + ...body, + fixed: body.fixed.filter((entry) => !freed.includes(entry)), + blank: [ + ...(body.blank ?? []), + ...freed.filter( + ({ member }) => !body.blank?.some((entry) => entry.member === member), + ), + ], + accepted: [ + ...body.accepted, + ...freed + .map(({ member }) => member) + .filter((member) => !body.accepted.includes(member)), + ], + }; +}; + /** Two calls' body contracts joined, undefined when a member one accepts with any value the other fixes, or both fix it at different values. */ const mergedBody = ( body: WriteBody, diff --git a/src/domain/analysisView/environmentStateWriteEffects.ts b/src/domain/analysisView/environmentStateWriteEffects.ts index ef4696a2c6..53c9380954 100644 --- a/src/domain/analysisView/environmentStateWriteEffects.ts +++ b/src/domain/analysisView/environmentStateWriteEffects.ts @@ -44,6 +44,7 @@ import { conflictsWithPlan, refinedBody, reconciledUpdateRule, + toggledRules, } from "./environmentStateWriteCalls.js"; import { laterObservation, @@ -667,7 +668,7 @@ export const updateEntity = ( } const unknown = context.unknown(type, collection, id); const call = { ...callOf(step, body), body }; - const { explained, rules } = fitUpdateClocks( + const fitted = fitUpdateClocks( draft, after.object, explainUpdate({ @@ -688,6 +689,15 @@ export const updateEntity = ( references: draft.plan.rules.filter((rule) => rule.rule === "reference"), }), ); + // An action a later call shows flipping back what an earlier one set toggles it. + const { rules, later } = toggledRules( + fitted.rules, + fitted.explained.rules, + entity.fields, + draft.steady, + draft.updates, + ); + const explained = { ...fitted.explained, rules: later }; for (const member of keptOrders(call.body, after.object)) draft.kept.add(member); if (explained.unexplained.length > 0 && !unknown) { diff --git a/src/domain/analysisView/environmentStateWriteReferences.test.ts b/src/domain/analysisView/environmentStateWriteReferences.test.ts index 9c4d22fb22..e676f21700 100644 --- a/src/domain/analysisView/environmentStateWriteReferences.test.ts +++ b/src/domain/analysisView/environmentStateWriteReferences.test.ts @@ -116,6 +116,44 @@ describe("objects that repeat the entity a member names", () => { ).toBeUndefined(); }); + it("learns a list reference only when every element lists the same nonempty fields", () => { + const named = (id: string) => + id === "1" || id === "2" + ? [ + held("/api/tags", id, { + id, + name: id === "1" ? "Tools" : "Winter", + color: "#aa0000", + }), + ] + : []; + expect( + referenceRule( + "tags", + [ + { id: "1", name: "Tools" }, + { id: "2", name: "Winter" }, + ], + { tagIds: ["1", "2"] }, + named, + ), + ).toMatchObject({ many: true, fields: ["id", "name"] }); + expect( + referenceRule( + "tags", + [ + { id: "1", name: "Tools" }, + { id: "2", name: "Winter", color: "#aa0000" }, + ], + { tagIds: ["1", "2"] }, + named, + ), + ).toBeUndefined(); + expect( + referenceRule("tags", [{}], { tagIds: ["1"] }, named), + ).toBeUndefined(); + }); + it("refuses a call naming an entity the environment does not hold, or holds in several collections", () => { const rule: Extract = { field: "shelf", diff --git a/src/domain/analysisView/environmentStateWriteReferences.ts b/src/domain/analysisView/environmentStateWriteReferences.ts index 5d5a68525a..3bf24322ed 100644 --- a/src/domain/analysisView/environmentStateWriteReferences.ts +++ b/src/domain/analysisView/environmentStateWriteReferences.ts @@ -7,7 +7,7 @@ import { type JsonObject, type StateEntity, } from "./environmentStateRules.js"; -import type { FieldRule } from "./environmentStateWriteRules.js"; +import type { FieldRule, WriteBody } from "./environmentStateWriteRules.js"; type ReferenceRule = Extract; @@ -42,6 +42,7 @@ export const referenceRule = ( body: JsonObject, named: (id: string) => readonly StateEntity[], ): ReferenceRule | undefined => { + if (Array.isArray(value)) return listReferenceRule(field, value, body, named); if (!isObject(value) || Object.keys(value).length < 2) return undefined; const matches = new Map(); for (const [member, sent] of Object.entries(body)) { @@ -71,14 +72,101 @@ export const referenceRule = ( : undefined; }; +/** + * A created or updated list field holding, in order, the entities a sent list of ids names, as the state holds them: + * tags beside `tagIds`. The member is the one named like the field; every element must project one entity of one type. + */ +const listReferenceRule = ( + field: string, + value: readonly JsonValue[], + body: JsonObject, + named: (id: string) => readonly StateEntity[], +): ReferenceRule | undefined => { + if (value.length === 0 || !value.every(isObject)) return undefined; + // A list field is named for its items, as `tags` beside `tagIds`. + const stems = new Set([stemOf(field), stemOf(field).replace(/s$/u, "")]); + const members = Object.entries(body).filter( + ([member, sent]) => + stems.has(stemOf(member)) && + Array.isArray(sent) && + sent.length === value.length, + ); + if (members.length !== 1) return undefined; + const [member, sent] = members[0]! as [string, JsonValue[]]; + const types = new Set(); + for (const [index, element] of value.entries()) { + const id = scalarText(sent[index]); + const held = id === undefined ? [] : named(id); + const match = held.filter((entity) => + projects(element as JsonObject, entity.fields), + ); + if (match.length !== 1) return undefined; + types.add(match[0]!.type); + } + if (types.size !== 1) return undefined; + // Every recorded element must list the same fields. A union would add a key one element omitted, and an empty object lists none. + const [recorded, ...rest] = value.map((element) => + Object.keys(element as JsonObject), + ); + if ( + recorded === undefined || + recorded.length === 0 || + rest.some( + (fields) => + fields.length !== recorded.length || + fields.some((field) => !recorded.includes(field)), + ) + ) + return undefined; + return { + field, + rule: "reference", + member, + type: [...types][0]!, + fields: recorded, + many: true, + }; +}; + +/** Whether a call's member and the entity's field show a reference to nothing: a `null` id and field, or empty lists. */ +export const emptyReference = ( + sent: JsonValue | undefined, + stored: JsonValue | undefined, +): boolean => + (sent === null && stored === null) || + (Array.isArray(sent) && + sent.length === 0 && + Array.isArray(stored) && + stored.length === 0); + /** A reference's value for a call: the named entity's fields it lists, or undefined when the call does not send its member. */ export const referencedValue = ( rule: ReferenceRule, body: JsonObject, referenced: ReferencedEntity | undefined, + blank?: WriteBody["blank"], ): Result => { if (!Object.hasOwn(body, rule.member)) return ok(undefined); - const id = scalarText(body[rule.member]); + const sent = body[rule.member]!; + // A call naming nothing stores nothing, as a recorded call that sent the member `null` did. + if ( + sent === null && + blank?.some((entry) => entry.member === rule.member && entry.sent === null) + ) + return ok(null); + if (rule.many === true) { + if (!Array.isArray(sent)) + return err(`Member ${rule.member} sends no list of ids.`); + const { many: _many, ...single } = rule; + const values: JsonValue[] = []; + for (const one of sent) { + const value = referencedValue(single, { [rule.member]: one }, referenced); + if (!value.ok) return value; + values.push(value.value ?? null); + } + return ok(values); + } + const id = scalarText(sent); const held = id === undefined ? [] : (referenced?.(rule.type, id) ?? []); const [fields] = held; if (fields === undefined) diff --git a/src/domain/analysisView/environmentStateWriteRules.ts b/src/domain/analysisView/environmentStateWriteRules.ts index 7da1aeb34b..093bf08b5f 100644 --- a/src/domain/analysisView/environmentStateWriteRules.ts +++ b/src/domain/analysisView/environmentStateWriteRules.ts @@ -124,6 +124,8 @@ export type FieldRule = readonly member: string; readonly type: string; readonly fields: string[]; + /** The member sends a list of ids, and the field holds each named entity in that order. */ + readonly many?: true; } | { readonly field: string; readonly rule: "increment" } /** An update stores `value` whenever the call sends the member, as the call that showed it did. */ @@ -151,6 +153,11 @@ export type FieldRule = readonly field: string; readonly rule: "set"; readonly value: JsonValue; + } + /** An item action flips a boolean on every call. */ + | { + readonly field: string; + readonly rule: "toggle"; }; /** Clock rules spelled as a trimmed sibling is, since an application spells its clocks alike: fields whose `values`, and every `later` call's, show no fixed width. */ diff --git a/src/domain/analysisView/environmentStateWriteVerdict.ts b/src/domain/analysisView/environmentStateWriteVerdict.ts index d5834ee5d1..3ea6cd308a 100644 --- a/src/domain/analysisView/environmentStateWriteVerdict.ts +++ b/src/domain/analysisView/environmentStateWriteVerdict.ts @@ -56,13 +56,17 @@ const renders = ( ); }; -/** Whether an item action sets something, and a call showed each field it sets already at that value, so the action sets rather than toggles it. */ +/** Whether an item action sets or toggles something, and a call showed each field it sets already at that value, so it sets rather than toggles those. */ const steady = (plan: StateWritePlan, draft: WriteDraft): boolean => { if (plan.address?.by !== "action") return true; const set = plan.rules.flatMap((rule) => rule.rule === "set" ? [rule.field] : [], ); - return set.length > 0 && set.every((field) => draft.steady.has(field)); + // A toggle is learned only from calls that flipped its field both ways. + return ( + (set.length > 0 || plan.rules.some((rule) => rule.rule === "toggle")) && + set.every((field) => draft.steady.has(field)) + ); }; /** Whether a clock a call left as it was in its own second has no rule another call showed, so no call showed whether the write stamps it. */ diff --git a/src/domain/analysisView/scenarioCaptureRecords.test.ts b/src/domain/analysisView/scenarioCaptureRecords.test.ts index f63645f6ad..8b05021e83 100644 --- a/src/domain/analysisView/scenarioCaptureRecords.test.ts +++ b/src/domain/analysisView/scenarioCaptureRecords.test.ts @@ -86,6 +86,26 @@ describe("scenarioCaptureRecords", () => { ); }); + it("records a hidden request body as absent only when the request names no Content-Type", () => { + const toggle = { + method: "PUT", + url: "/api/entries/1/bookmark", + status: 204, + requestBodyHidden: true, + } as const; + const projected = scenarioCaptureRecords( + scenarioCaptureFixture([ + { ...toggle, requestHeaderNames: ["accept", "authorization"] }, + { ...toggle, requestHeaderNames: ["accept", "Content-Type"] }, + ]), + "manifest", + ); + expect(projected.records.map((entry) => entry.reported)).toMatchObject([ + { request: { bodySize: 0 } }, + { request: { bodySize: -1 } }, + ]); + }); + it("leaves a transaction that cannot be attributed to one request and response unrepresented", () => { const capture = scenarioCaptureFixture(SCENARIO_EXCHANGES.slice(1, 2)); const response = capture.events.items.find( diff --git a/src/domain/analysisView/scenarioCaptureRecords.ts b/src/domain/analysisView/scenarioCaptureRecords.ts index 2a5cc3f094..f43d8fedd0 100644 --- a/src/domain/analysisView/scenarioCaptureRecords.ts +++ b/src/domain/analysisView/scenarioCaptureRecords.ts @@ -88,6 +88,14 @@ export const scenarioContentMediaType = ( captured(content)?.media_type ?? headerValue(content?.headers, "content-type"); +/** Whether a request sent no body: Playwright exposed none and the request names no Content-Type, which the bodies browsers send carry. */ +const unsentBody = ( + request: RequestEvent, + content: ContentEvent | undefined, +): boolean => + content?.body.state === "not_exposed" && + !request.header_names.some((name) => name.toLowerCase() === "content-type"); + /** The request object HAR readers expect; a redacted URL is withheld so no path is templated from it. */ const reportedRequest = ( request: RequestEvent, @@ -97,9 +105,15 @@ const reportedRequest = ( const body = purpose === "manifest" ? captured(content) : undefined; const text = body === undefined ? undefined : utf8Text(body); const mimeType = scenarioContentMediaType(content); + const size = + body !== undefined && !body.redacted + ? body.bytes + : purpose === "manifest" && unsentBody(request, content) + ? 0 + : -1; return { method: request.method, - bodySize: body !== undefined && !body.redacted ? body.bytes : -1, + bodySize: size, url: request.url.redacted ? null : request.url.url, ...(text === undefined || mimeType === undefined ? {} diff --git a/src/generatedPackageMetadata.ts b/src/generatedPackageMetadata.ts index 5372dbba8b..9100e716d5 100644 --- a/src/generatedPackageMetadata.ts +++ b/src/generatedPackageMetadata.ts @@ -6,5 +6,5 @@ export const PACKAGE_METADATA = { serverSdkVersion: "2.3.1", clientSdkVersion: "2.3.1", coreSdkVersion: "2.3.1", - skillVersion: "54", + skillVersion: "55", } as const; diff --git a/tests/fixtures/browserScenarioCapture.ts b/tests/fixtures/browserScenarioCapture.ts index c5dfb24d56..7c7f9300ca 100644 --- a/tests/fixtures/browserScenarioCapture.ts +++ b/tests/fixtures/browserScenarioCapture.ts @@ -27,6 +27,10 @@ export interface ScenarioExchange { readonly body?: string | Buffer; readonly requestMediaType?: string; readonly requestBody?: string; + /** A request body Playwright did not expose. */ + readonly requestBodyHidden?: true; + /** The request event's header names. */ + readonly requestHeaderNames?: readonly string[]; /** Header values as `header_values` retains them; a null value is one the capture excluded. */ readonly requestHeaders?: readonly ScenarioHeader[]; readonly responseHeaders?: readonly ScenarioHeader[]; @@ -145,13 +149,15 @@ const exchangeEvents = ( method: exchange.method, url: exchange.redactedUrl === true ? { ...url, redacted: true } : url, resource_type: exchange.resourceType ?? "fetch", - header_names: [], + header_names: [...(exchange.requestHeaderNames ?? [])], }; const content = ( sequence: number, source: number, phase: "request" | "response", - body: ReturnType | { readonly state: "not_requested" }, + body: + | ReturnType + | { readonly state: "not_requested" | "not_exposed" }, ) => ({ sequence, step_index: 1, @@ -176,6 +182,7 @@ const exchangeEvents = ( let next = first + 1; if ( exchange.requestBody !== undefined || + exchange.requestBodyHidden === true || exchange.requestHeaders !== undefined ) { events.push( @@ -183,13 +190,15 @@ const exchangeEvents = ( next, first, "request", - exchange.requestBody === undefined - ? { state: "not_requested" } - : capturedBody( - exchange.requestBody, - "browser-exposed-request-bytes", - exchange.requestMediaType ?? null, - ), + exchange.requestBodyHidden === true + ? { state: "not_exposed" } + : exchange.requestBody === undefined + ? { state: "not_requested" } + : capturedBody( + exchange.requestBody, + "browser-exposed-request-bytes", + exchange.requestMediaType ?? null, + ), ), ); next += 1; diff --git a/tests/fixtures/environmentStateItemWrites.ts b/tests/fixtures/environmentStateItemWrites.ts new file mode 100644 index 0000000000..203cb03a30 --- /dev/null +++ b/tests/fixtures/environmentStateItemWrites.ts @@ -0,0 +1,297 @@ +import type { TimedCaptureRow } from "./analysisView.js"; +import type { JsonValue } from "../../src/domain/jsonValue.js"; + +type Entity = { readonly [key: string]: JsonValue }; + +const folder = (id: number, title: string): Entity => ({ + id, + title, + version: 1, +}); + +/** One seeded folder, a create a list shows after it, then one a list shows before that create by title, then a delete. */ +export const CREATES_BY_TITLE_ROWS: readonly TimedCaptureRow[] = (() => { + const rota = folder(1, "Lighthouse Rota"); + const pier = folder(2, "Scratch Pier 3"); + const meadow = folder(3, "Scratch Meadow 4"); + return [ + ["GET", "/api/folders", [rota]], + ["GET", "/api/folders/1", rota], + ["POST", "/api/folders", pier, 200, { title: "Scratch Pier 3" }], + ["GET", "/api/folders/2", pier], + ["GET", "/api/folders", [rota, pier]], + ["POST", "/api/folders", meadow, 200, { title: "Scratch Meadow 4" }], + ["GET", "/api/folders/3", meadow], + ["GET", "/api/folders", [rota, meadow, pier]], + ["DELETE", "/api/folders/2", { message: "Folder deleted" }, 200], + ["GET", "/api/folders/2", { message: "folder not found" }, 404], + ["GET", "/api/folders", [rota, meadow]], + ]; +})(); + +const LOCATIONS = [ + { + id: "7d1c0b6e-0001-4000-8000-000000000001", + name: "Garage", + kind: "location", + }, + { + id: "7d1c0b6e-0002-4000-8000-000000000002", + name: "Attic", + kind: "location", + }, +]; +const TAGS = [ + { + id: "3b9f2a10-0001-4000-8000-000000000001", + name: "Tools", + color: "#aa0000", + }, + { + id: "3b9f2a10-0002-4000-8000-000000000002", + name: "Winter", + color: "#0000aa", + }, +]; + +/** An item as the application answers it: its parent location and tags spelled out from the ids a call sent. */ +export const inventoryItem = ( + id: string, + name: string, + parent: Entity | null, + tags: readonly Entity[], +): Entity => ({ id, name, parent, tags: [...tags] }); + +/** Item creates naming a parent location and tags by id, the first with none, the second with one of each. */ +export const ITEM_REFERENCES_ROWS: readonly TimedCaptureRow[] = (() => { + const seeded = inventoryItem( + "5e5e0000-0001-4000-8000-000000000001", + "Ladder", + LOCATIONS[0]!, + [], + ); + const bare = inventoryItem( + "5e5e0000-0002-4000-8000-000000000002", + "Scratch box", + null, + [], + ); + const filed = inventoryItem( + "5e5e0000-0003-4000-8000-000000000003", + "Scratch drill", + LOCATIONS[1]!, + [TAGS[0]!], + ); + return [ + ["GET", "/api/locations", LOCATIONS], + ["GET", `/api/locations/${LOCATIONS[0]!["id"] as string}`, LOCATIONS[0]!], + ["GET", `/api/locations/${LOCATIONS[1]!["id"] as string}`, LOCATIONS[1]!], + ["GET", "/api/tags", TAGS], + ["GET", `/api/tags/${TAGS[0]!["id"] as string}`, TAGS[0]!], + ["GET", `/api/tags/${TAGS[1]!["id"] as string}`, TAGS[1]!], + ["GET", "/api/items", [seeded]], + ["GET", `/api/items/${seeded["id"] as string}`, seeded], + [ + "POST", + "/api/items", + bare, + 201, + { name: "Scratch box", parentId: null, tagIds: [] }, + ], + ["GET", `/api/items/${bare["id"] as string}`, bare], + ["GET", "/api/items", [seeded, bare]], + [ + "POST", + "/api/items", + filed, + 201, + { + name: "Scratch drill", + parentId: LOCATIONS[1]!["id"]!, + tagIds: [TAGS[0]!["id"]!], + }, + ], + ["GET", `/api/items/${filed["id"] as string}`, filed], + ["GET", "/api/items", [seeded, bare, filed]], + ]; +})(); + +export const ITEM_LOCATIONS = LOCATIONS; +export const ITEM_TAGS = TAGS; + +/** A feed entry with a `starred` flag a body-less action flips, and the instant it last changed. */ +export const feedEntry = ( + id: number, + starred: boolean, + changedAt: string, +): Entity => ({ + id, + title: `Entry ${String(id)}`, + starred, + changed_at: changedAt, +}); + +/** One entry starred, then unstarred, by the same body-less call, each read back after it. */ +export const BOOKMARK_TOGGLE_ROWS: readonly TimedCaptureRow[] = (() => { + const seeded = "2026-10-17T17:00:00.000000Z"; + const first = feedEntry(1, false, seeded); + const second = feedEntry(2, false, seeded); + const starred = feedEntry(1, true, "2026-10-17T18:00:03.120000Z"); + const unstarred = feedEntry(1, false, "2026-10-17T18:00:05.090000Z"); + return [ + ["GET", "/api/entries", [first, second]], + ["GET", "/api/entries/1", first], + ["GET", "/api/entries/2", second], + ["PUT", "/api/entries/1/bookmark", "", 204], + ["GET", "/api/entries/1", starred], + ["PUT", "/api/entries/1/bookmark", "", 204], + ["GET", "/api/entries/1", unstarred], + ["GET", "/api/entries", [unstarred, second]], + ]; +})(); + +const ranked = ( + id: number, + title: string, + position: number | null, +): Entity => ({ + id, + title, + position, +}); + +/** Folders a list shows by `position`, the seed holding none: the first create lists after the seed's higher title, and once the seed is deleted later lists alone would also fit a sort by title. */ +export const RANKED_AFTER_DELETE_ROWS: readonly TimedCaptureRow[] = (() => { + const zulu = ranked(1, "Zulu Base", null); + const bravo = ranked(2, "Bravo", 2); + const charlie = ranked(3, "Charlie", 3); + const able = ranked(4, "Able", 0); + return [ + ["GET", "/api/folders", [zulu]], + ["GET", "/api/folders/1", zulu], + ["POST", "/api/folders", bravo, 200, { title: "Bravo", position: 2 }], + ["GET", "/api/folders/2", bravo], + ["GET", "/api/folders", [zulu, bravo]], + ["DELETE", "/api/folders/1", { message: "Folder deleted" }, 200], + ["GET", "/api/folders/1", { message: "folder not found" }, 404], + ["GET", "/api/folders", [bravo]], + ["POST", "/api/folders", charlie, 200, { title: "Charlie", position: 3 }], + ["GET", "/api/folders/3", charlie], + ["GET", "/api/folders", [bravo, charlie]], + ["POST", "/api/folders", able, 200, { title: "Able", position: 0 }], + ["GET", "/api/folders/4", able], + ["GET", "/api/folders", [able, bravo, charlie]], + ]; +})(); + +/** Item creates: the first shows no parent at all, the second names one, the third sends `parentId: null` and stores none; lists show each item's id and name. */ +export const ADDED_EMPTY_REFERENCE_ROWS: readonly TimedCaptureRow[] = (() => { + const seeded = { id: "5e5e0000-0001-4000-8000-000000000001", name: "Ladder" }; + const loose = { + id: "5e5e0000-0002-4000-8000-000000000002", + name: "Scratch box", + }; + const filed = { + id: "5e5e0000-0003-4000-8000-000000000003", + name: "Scratch drill", + parent: LOCATIONS[1]!, + }; + const cleared = { + id: "5e5e0000-0004-4000-8000-000000000004", + name: "Scratch lamp", + parent: null, + }; + const summary = (item: Entity): Entity => ({ + id: item["id"]!, + name: item["name"]!, + }); + return [ + ["GET", "/api/locations", LOCATIONS], + ["GET", `/api/locations/${LOCATIONS[0]!["id"] as string}`, LOCATIONS[0]!], + ["GET", `/api/locations/${LOCATIONS[1]!["id"] as string}`, LOCATIONS[1]!], + ["GET", "/api/items", [seeded]], + ["GET", `/api/items/${seeded.id}`, seeded], + ["POST", "/api/items", loose, 201, { name: "Scratch box" }], + ["GET", `/api/items/${loose.id}`, loose], + ["GET", "/api/items", [seeded, loose].map(summary)], + [ + "POST", + "/api/items", + filed, + 201, + { name: "Scratch drill", parentId: LOCATIONS[1]!["id"]! }, + ], + ["GET", `/api/items/${filed.id}`, filed], + ["GET", "/api/items", [seeded, loose, filed].map(summary)], + [ + "POST", + "/api/items", + cleared, + 201, + { name: "Scratch lamp", parentId: null }, + ], + ["GET", `/api/items/${cleared.id}`, cleared], + ["GET", "/api/items", [seeded, loose, filed, cleared].map(summary)], + ]; +})(); + +/** Renames that keep collection order, which an ascending title sort would not replay, then a create a whole list could place by title. */ +export const REFUTED_TITLE_SORT_ROWS: readonly TimedCaptureRow[] = (() => { + const zulu = folder(1, "Zulu"); + const bravo = folder(2, "Bravo"); + const alpha = folder(3, "Alpha"); + const echo = folder(3, "Echo"); + const able = folder(1, "Able"); + const delta = folder(4, "Delta"); + return [ + ["GET", "/api/folders", [zulu, bravo, alpha]], + ["GET", "/api/folders/1", zulu], + ["GET", "/api/folders/2", bravo], + ["GET", "/api/folders/3", alpha], + ["PATCH", "/api/folders/3", echo, 200, { title: "Echo" }], + ["GET", "/api/folders/3", echo], + ["GET", "/api/folders", [zulu, bravo, echo]], + ["PATCH", "/api/folders/1", able, 200, { title: "Able" }], + ["GET", "/api/folders/1", able], + ["GET", "/api/folders", [able, bravo, echo]], + ["POST", "/api/folders", delta, 200, { title: "Delta" }], + ["GET", "/api/folders/4", delta], + ["GET", "/api/folders", [able, bravo, delta, echo]], + ]; +})(); + +/** A body-less bookmark action whose first call finds `starred` null, then flips the boolean it stored. */ +export const NULL_THEN_STARRED_ROWS: readonly TimedCaptureRow[] = (() => { + const seeded = { + id: 1, + title: "Entry 1", + starred: null, + changed_at: "2026-10-17T17:00:00.000000Z", + }; + const other = { + id: 2, + title: "Entry 2", + starred: false, + changed_at: "2026-10-17T17:00:00.000000Z", + }; + const starred = { + ...seeded, + starred: true, + changed_at: "2026-10-17T18:00:03.120000Z", + }; + const cleared = { + ...seeded, + starred: false, + changed_at: "2026-10-17T18:00:05.090000Z", + }; + return [ + ["GET", "/api/entries", [seeded, other]], + ["GET", "/api/entries/1", seeded], + ["GET", "/api/entries/2", other], + ["PUT", "/api/entries/1/bookmark", "", 204], + ["GET", "/api/entries/1", starred], + ["PUT", "/api/entries/1/bookmark", "", 204], + ["GET", "/api/entries/1", cleared], + ["GET", "/api/entries", [cleared, other]], + ]; +})();