From f4cc4fdb140a3f99eb5f2e13154b24ecfc87e838 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Tue, 20 Oct 2026 23:08:25 -0700 Subject: [PATCH] test(environment): cover list queries; move call line selection to its own module; document queries - Unit tests for each role, merging and refusals; a gate test learning a filter, a search, a sort and an inert flag; serving tests for new filter, search and sort values and an unrecorded inert value. - A `-name` sort descended only after the name had matched loosely without its sign; the sign is now read first. - Search fields leave out URLs that merely repeat the term. - The functions that choose the line a call answers on move to environmentStateCallLines.ts. Co-Authored-By: Claude Opus 5.5 --- docs/agent-environments.md | 12 +- .../agent-environments/list-queries/plan.md | 14 + .../agent-environments/list-queries/spec.md | 60 +++++ .../agent-environments/list-queries/tasks.md | 11 + docs/web-network-captures.md | 28 ++ .../environment/environmentStateAnswer.ts | 252 +++++------------- .../environment/environmentStateCallLines.ts | 152 +++++++++++ .../environmentStateWriteAnswer.test.ts | 29 ++ .../environmentStateWriteAnswer.ts | 3 +- src/contracts/analysisViewToolContracts.ts | 2 +- .../environmentStateListQuery.test.ts | 167 ++++++++++++ .../analysisView/environmentStateListQuery.ts | 208 ++++++++++----- .../environmentStateWriteChecks.ts | 59 ++-- .../environmentStateWrites.test.ts | 30 +++ tests/fixtures/environmentStateWrites.ts | 28 ++ 15 files changed, 765 insertions(+), 290 deletions(-) create mode 100644 docs/specs/agent-environments/list-queries/plan.md create mode 100644 docs/specs/agent-environments/list-queries/spec.md create mode 100644 docs/specs/agent-environments/list-queries/tasks.md create mode 100644 src/application/environment/environmentStateCallLines.ts create mode 100644 src/domain/analysisView/environmentStateListQuery.test.ts diff --git a/docs/agent-environments.md b/docs/agent-environments.md index 1e7ac6bc86..9e969e661d 100644 --- a/docs/agent-environments.md +++ b/docs/agent-environments.md @@ -420,12 +420,12 @@ member reports which applied: Each read tool answers according to its view in the state: -| View | Answer | -| ---------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `item` | The entity the path arguments name, rendered into the recorded envelope of `envelope_ordinal`. Arguments match as replay matches them: each collection segment equals the held collection's segment as written or after percent-decoding, and the identity is the last argument percent-decoded, then as written; values are never re-split on `/`. Any entity the state holds answers, including one `data/add` loaded; query arguments only decide `replay.exact_match`. An entity the state does not hold answers the tool's latest recorded 400, 404 or 410 response with `state.view` `missing`, or is a tool error when none was recorded; one lacking a field every recording carried is a tool error; a body is never invented. | -| `list` | When the arguments name one of the lines the view checked exactly (its latest recordings before the first related write; every path and query value equal to the recording's, and every recorded query parameter supplied), that recorded envelope with its array rendered from the state: every entity of the collection in state order for `membership: "all"` or a line `all_record_ordinals` names, or the recorded members still held. A call with no query argument answers from such a line of the collection its path arguments name that only pages its list, with `replay.exact_match` false. A member lacking a field every recording carried makes the call a tool error rather than a partial list. | -| `document` | When the arguments name the document's checked line exactly, the state's value for it; a tool error when the state holds none. | -| `recorded` | The recording, as replay answers it. | +| View | Answer | +| ---------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `item` | The entity the path arguments name, rendered into the recorded envelope of `envelope_ordinal`. Arguments match as replay matches them: each collection segment equals the held collection's segment as written or after percent-decoding, and the identity is the last argument percent-decoded, then as written; values are never re-split on `/`. Any entity the state holds answers, including one `data/add` loaded; query arguments only decide `replay.exact_match`. An entity the state does not hold answers the tool's latest recorded 400, 404 or 410 response with `state.view` `missing`, or is a tool error when none was recorded; one lacking a field every recording carried is a tool error; a body is never invented. | +| `list` | When the arguments name one of the lines the view checked exactly (its latest recordings before the first related write; every path and query value equal to the recording's, and every recorded query parameter supplied), that recorded envelope with its array rendered from the state: every entity of the collection in state order for `membership: "all"` or a line `all_record_ordinals` names, or the recorded members still held. A call with no query argument answers from such a line of the collection its path arguments name that only pages its list, with `replay.exact_match` false. On a line the view's `query` answers, and for a call whose every query argument has a role there (an argument-less call aside), the answer is the query evaluated over the state: filtered, searched, sorted and paged, with counters at the number listed and `query.totals` at the number matched, and `state.membership: "query"`; such a call answers on a query or whole line of its collection with `replay.exact_match` false. A value an `inert` parameter never recorded, or a parameter without a role, replays its recording. A member lacking a field every recording carried makes the call a tool error rather than a partial list. | +| `document` | When the arguments name the document's checked line exactly, the state's value for it; a tool error when the state holds none. | +| `recorded` | The recording, as replay answers it. | A list or document call that names no checked line, including a line recorded after a write or with other query values, answers from the recording. Every diff --git a/docs/specs/agent-environments/list-queries/plan.md b/docs/specs/agent-environments/list-queries/plan.md new file mode 100644 index 0000000000..9387a09201 --- /dev/null +++ b/docs/specs/agent-environments/list-queries/plan.md @@ -0,0 +1,14 @@ +# Implementation plan + +1. **Query module.** A domain module holds the roles, explains one line (trying roles by parameter name, filters and + inert for any parameter, at most 256 combinations), merges lines, evaluates a query over entities, and renders an + answer with counters and totals. +2. **Gate.** The list plan learns the query from the lines that are not whole, reading each body once for its listed + identities, role choices and counts; demotes a line whose envelope or fields do not reproduce; and adds `query` to + the view. +3. **Write gate.** A read on a query line is checked against the query over the replayed state, covering the + collection only for sort- and page-only answers that hold every match. +4. **Serving.** Line selection moves to its own module and gains the query line; list answers choose whole, + query or recorded members and render totals. +5. **Tests and docs.** Unit, domain and serving tests; the environment-state and agent-environment docs and the + analysis-view tool description. diff --git a/docs/specs/agent-environments/list-queries/spec.md b/docs/specs/agent-environments/list-queries/spec.md new file mode 100644 index 0000000000..81ac30b08f --- /dev/null +++ b/docs/specs/agent-environments/list-queries/spec.md @@ -0,0 +1,60 @@ +# Answer list filters, search, sorting and paging from the state + +Issue: https://github.com/morluto/rea/issues/4933 + +## Outcome + +Agents list, filter, search and sort constantly. A list view answered a recorded request line from the state only when +that line listed the whole collection; any other line, such as `?shelf=a`, `?q=tide` or `?orderBy=name`, kept the +members it recorded, and a call with a value no line recorded replayed whichever recording was closest. A filtered +list therefore never showed what a write created or changed, and a search for anything new answered with an +unrelated recording. + +A list view now learns what each of its query parameters does, from the lines it recorded, and answers those lines, +and calls with new values, by evaluating the query over the live state. + +## Learning + +For each checked line that does not list the whole collection, the first combination of roles that lists exactly what +the line recorded, from the collection's initial entities, explains it: + +- `filter` on a field every listed item holds the value in: equal as text, or in an array directly or as an object's + `id`, `slug` or `name`; +- `search` over the string fields the term matched in the listed items, ignoring case (URL fields only for an item + nothing else matched); an empty term matches everything; +- `sort` by the field the value names, with `-` or an `asc`/`desc` suffix, and `direction` for a separate `asc`/`desc` + parameter: numbers by value, text ignoring case, null last ascending, ties in state order; +- `limit`, `offset` and `page` (counted from 1 or 0, whichever explains the line) for paging; zero or a negative limit + lists every match; +- `inert` for a parameter that changes nothing, valid only for the values it recorded. + +Parameter names decide which roles are tried (`sort` for `orderBy`, `search` for `q`), and any parameter may be a +filter or inert. The lines' roles merge: lines that disagree drop the parameter, inert values gather, search fields +unite. The merged query keeps the lines it reproduces, envelope and fields included; any other line keeps its +recorded members. The view's `query` names the roles, those lines, and the envelope members that count every match +before paging (`totals`); a `number`-of-items member such as `numberOfElements` counts the page. + +## Serving and the write gate + +- A call that names a query line exactly, or whose every query argument has a role (an argument-less call keeps + answering on a whole line), is answered by evaluating the query over the state, in the envelope of a query or whole + line of its collection, with counters at the number listed and totals at the number matched. The answer's state + provenance says `membership: "query"`. +- A value an inert parameter never recorded, or a parameter without a role, replays its recording as before. +- The write gate checks a read on a query line against the query over its copy of the state. Only a line whose + parameters all sort or page, and whose answer holds every match, covers the collection; any other covers the + entities it lists and those the line or the read listed that the state no longer holds, so a filtered list shows + whether a delete or create took effect without speaking for the rest of the collection. + +The roles are inferences from recorded behaviour and are stated as such. Nothing changes for a list whose lines all +list the whole collection, or that no role explains. + +## Acceptance + +- Unit tests: each role found from a line, including filters on arrays and object ids, search ignoring URLs, sort + spellings and nulls, paging by offset and by page, inert values; merging lines; values a role cannot apply. +- Domain: a tool recorded whole, filtered, searched, sorted and with a flag learns all four, and a create the filter + later lists is confirmed. +- Serving: after a served create, new filter, search and sort values list it; an unrecorded inert value replays. +- Measured: the fifty published captures' learned queries and write verdicts, and the six environments compared with + their applications, before and after. diff --git a/docs/specs/agent-environments/list-queries/tasks.md b/docs/specs/agent-environments/list-queries/tasks.md new file mode 100644 index 0000000000..7f5e34bf0a --- /dev/null +++ b/docs/specs/agent-environments/list-queries/tasks.md @@ -0,0 +1,11 @@ +# Tasks + +- [x] Explain recorded list lines with filter, search, sort, direction, paging and inert roles. +- [x] Merge roles across lines and keep the lines the query reproduces. +- [x] Count matches before paging in the envelope's totals. +- [x] Check reads on query lines in the write gate. +- [x] Serve query lines and calls with new values from the state. +- [x] Add unit, domain and serving tests. +- [x] Document list queries. +- [ ] Measure the published captures and the six environments before and after. +- [ ] Run static, formatting, duplication, documentation and test gates. diff --git a/docs/web-network-captures.md b/docs/web-network-captures.md index 3a9204eb8a..e252a13c27 100644 --- a/docs/web-network-captures.md +++ b/docs/web-network-captures.md @@ -601,6 +601,29 @@ when its content type is JSON-like, it is UTF-8 text of at most 256 KiB, every number survives native conversion exactly, and no member is named `__proto__`. +A list view also learns, from its checked lines that do not list the whole +collection, what each query parameter does, and names the result in `query`. +Each such line is explained by the first combination of roles that lists +exactly what it recorded from the collection's initial entities: + +| Role | Parameter | Effect | +| ----------- | ------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `filter` | any, on a `field` every listed item holds the value in | entities whose field equals the value as text, or whose array field holds it directly or as an object's `id`, `slug` or `name` | +| `search` | `q`, `query`, `search`, `searchTerm`, `term`, `keyword`(s) | entities with one of `fields` containing the value, ignoring case; the fields are those it matched in the listed items, URLs only where nothing else matched | +| `sort` | `order`, `orderBy`, `ordering`, `sort`, `sortBy` (and `_by` spellings) | ordered by the field the value names (`name`, `-name`, `name:desc`, `name-ASC`): numbers by value, text ignoring case, null last ascending, ties in state order | +| `direction` | `dir`, `direction`, `order`, `orderDirection`, `sortDir`, `sortOrder` with `asc` or `desc` | the sort's direction | +| `limit` | `limit`, `per_page`, `perPage`, `page_size`, `size`, `count`, `take`, `itemsPerPage` | page size; zero or a negative value lists every match | +| `offset` | `offset`, `start`, `_start`, `skip` | matches skipped before the page | +| `page` | `page` | page number, counted from `first` (1 or 0, whichever explains the line) | +| `inert` | any | changes nothing for its recorded `values`; other values are not computed | + +Lines that disagree on a parameter's role drop it, inert values gather and +search fields unite. `query.record_ordinals` are the lines the merged roles +reproduce, envelope and fields included, and `query.totals` the envelope +members that count every match before paging on each of them (a +`number`-of-items member counts the page). The roles are inferences from the +recorded lines; a line they do not reproduce keeps its recorded members. + `writes` lists one plan per tool whose request may change state, in tool order. Its `effect` is `create` for a `POST` to a type's `collection_template`, `update` for a `PUT`, `PATCH` or `POST` to that template @@ -624,6 +647,11 @@ sending must be a clock, an increment or a slug, or the plan is tool. Every member the first call sent is required, except by `PATCH` and `POST` updates. +A read on a line `query` answers is checked against the query evaluated over +the replayed state. It covers the collection only when every parameter sorts or +pages and the answer holds every match; otherwise it covers the entities it +lists, and those the line or the read listed that the state no longer holds. + The view then replays the timeline on a copy of the initial state. Modelled writes apply their effect; only a `2xx` write is modelled or an example, and any other write (a redirect, an unanswered or unmodelled one) leaves the entity diff --git a/src/application/environment/environmentStateAnswer.ts b/src/application/environment/environmentStateAnswer.ts index 2030dd4115..9fb7099368 100644 --- a/src/application/environment/environmentStateAnswer.ts +++ b/src/application/environment/environmentStateAnswer.ts @@ -4,13 +4,11 @@ import { allLines, listItems, listsAll, - pagesOnly, missingRequiredFields, renderStateEntity, renderStateList, replaceAt, scalarText, - segmentsOf, type JsonObject, type RecordedViewReason, type StateEntity, @@ -32,12 +30,10 @@ import type { JsonValue } from "../../domain/jsonValue.js"; import { err, ok, type Result } from "../../domain/result.js"; import { decodedPathSegment, - pathValueMatches, REPLAY_BODY_LIMIT_BYTES, replayAnswer, replayEnvironmentTool, selectReplayExchange, - selectsRecordedLine, type ReplayArguments, type ReplayResult, } from "./environmentReplay.js"; @@ -47,8 +43,16 @@ import { type EnvironmentSource, type RecordedExchange, } from "./environmentSource.js"; +import { + callQuery, + callSegments, + checkedLine, + collectionMatch, + queryLine, + unfilteredLine, + type CallSegment, +} from "./environmentStateCallLines.js"; import type { EnvironmentStateStore } from "./environmentStateStore.js"; -import { environmentToolArguments } from "./environmentTools.js"; /** Where a served answer came from: the live state, or a recording and why. */ export type StateProvenance = @@ -171,42 +175,12 @@ export const stateAnswer = ( }); }; -/** One collection segment an item call names: a template literal, or a supplied path value. */ -export interface CallSegment { - readonly text: string; - readonly supplied: boolean; -} - /** An item call's collection segments and identity value, kept as supplied rather than joined and re-split. */ export interface ItemCall { readonly segments: readonly CallSegment[]; readonly id: string; } -/** A call's path segments: template literals and supplied values, or undefined when a path argument is missing. */ -export const callSegments = ( - tool: EnvironmentManifestTool, - args: ReplayArguments, -): CallSegment[] | undefined => { - const names = environmentToolArguments(tool).path; - const params = new Map(tool.params.map((param) => [param.index, param])); - const segments: CallSegment[] = []; - const template = tool.path_template - .slice(tool.path_template.indexOf("/") + 1) - .split("/"); - for (const [index, text] of template.entries()) { - const param = params.get(index); - if (param === undefined) { - if (text !== "") segments.push({ text, supplied: false }); - continue; - } - const value = args[names.get(param.name) ?? param.name]; - if (typeof value !== "string") return undefined; - segments.push({ text: value, supplied: true }); - } - return segments; -}; - /** What an item call names, or undefined when a path argument is missing. */ export const itemCall = ( tool: EnvironmentManifestTool, @@ -219,23 +193,6 @@ export const itemCall = ( : undefined; }; -/** How many supplied segments a held collection matches verbatim, when it matches the call under replay's path rule. */ -export const collectionMatch = ( - collection: string, - segments: readonly CallSegment[], -): number | undefined => { - const held = segmentsOf(collection); - if (held.length !== segments.length) return undefined; - let verbatim = 0; - for (const [index, { text, supplied }] of segments.entries()) { - const segment = held[index]!; - if (supplied ? !pathValueMatches(segment, text) : segment !== text) - return undefined; - if (supplied && segment === text) verbatim += 1; - } - return verbatim; -}; - /** The entity an item call names, preferring verbatim collection spellings as replay does; the id is tried percent-decoded, then as written. */ export const namedEntity = ( store: EnvironmentStateStore, @@ -359,49 +316,6 @@ const answerItem = ( ); }; -/** The checked recorded line a list or document call names exactly, chosen among them as replay chooses; later or filtered recordings never qualify. */ -const checkedLine = ( - source: EnvironmentSource, - tool: EnvironmentManifestTool, - checked: readonly number[], - args: ReplayArguments, -): RecordedExchange | undefined => { - const lines = new Map(); - for (const ordinal of checked) { - const exchange = source.exchanges.get(ordinal); - if (exchange !== undefined && selectsRecordedLine(tool, exchange, args)) - lines.set(ordinal, exchange); - } - return selectReplayExchange(tool, lines, args)?.exchange; -}; - -/** For a list of a whole collection, a call that supplies no query argument answers from a checked line of the collection its path arguments name that only pages it, chosen as replay chooses; a requested page answers from its recording. */ -const unfilteredLine = ( - source: EnvironmentSource, - tool: EnvironmentManifestTool, - checked: readonly number[], - args: ReplayArguments, -): RecordedExchange | undefined => { - const query = environmentToolArguments(tool).query; - const segments = callSegments(tool, args); - if ( - segments === undefined || - [...query.values()].some((argument) => args[argument] !== undefined) - ) - return undefined; - const lines = new Map(); - for (const ordinal of checked) { - const exchange = source.exchanges.get(ordinal); - if ( - exchange !== undefined && - pagesOnly(exchange.query ?? undefined) && - collectionMatch(collectionPathOf(exchange, false), segments) !== undefined - ) - lines.set(ordinal, exchange); - } - return selectReplayExchange(tool, lines, args)?.exchange; -}; - type StateDocumentView = Extract; /** One tool call: the tool, the arguments it was called with, and whether they name the answering line exactly. */ @@ -411,117 +325,97 @@ interface ToolCall { readonly exact: boolean; } -/** A call's query arguments by parameter name. */ -const callQuery = ( - tool: EnvironmentManifestTool, - args: ReplayArguments, -): Map => - new Map( - [...environmentToolArguments(tool).query].flatMap( - ([name, argument]): [string, string][] => { - const value = args[argument]; - return value === undefined ? [] : [[name, String(value)]]; - }, - ), - ); - -/** For a list with a query, a call whose every query argument has a role answers on a query or whole line of the collection its path arguments name. */ -const queryLine = ( - source: EnvironmentSource, - tool: EnvironmentManifestTool, - view: StateListView, - args: ReplayArguments, -): RecordedExchange | undefined => { - const query = view.query; - const segments = callSegments(tool, args); - if ( - query === undefined || - segments === undefined || - [...callQuery(tool, args).keys()].some( - (name) => query.parameters[name] === undefined, - ) - ) - return undefined; - const lines = new Map(); - for (const ordinal of [...query.record_ordinals, ...allLines(view)]) { - const exchange = source.exchanges.get(ordinal); - if ( - exchange !== undefined && - collectionMatch(collectionPathOf(exchange, false), segments) !== undefined - ) - lines.set(ordinal, exchange); - } - return selectReplayExchange(tool, lines, args)?.exchange; -}; - /** - * A list call renders from the state on its line: every entity of the collection on a whole line, the query's matches - * on a line the query answers or for a call it computes, else the line's recorded members still held. + * The entities a list call shows on its line: every entity of the collection on a whole line, the query's matches on + * a line the query answers or for a call it computes, else the line's recorded members still held; undefined when the + * query cannot apply the call's values. */ -const answerList = ( - serving: StateServing, +const listSelection = ( + store: EnvironmentStateStore, { tool, args, exact }: ToolCall, view: StateListView, - base: RecordedExchange, -): Result => { - const { source, store } = serving; - const envelope = recordedJson(base); - const recorded = - envelope === undefined ? undefined : listItems(envelope, view.pointer); - if (envelope === undefined || recorded === undefined) - return fromRecording(source, tool, args, "body-unusable"); - const collection = collectionPathOf(base, false); - const type = store.type(view.type); + line: { + readonly base: RecordedExchange; + readonly recorded: readonly JsonObject[]; + }, +): + | { + readonly members: readonly StateEntity[]; + readonly membership: "all" | "query" | "recorded"; + readonly total: number; + } + | undefined => { + const collection = collectionPathOf(line.base, false); const values = callQuery(tool, args); // A named query line, or a call with query arguments answered on a line it did not name, evaluates the query. const queried = view.query !== undefined && (exact - ? view.query.record_ordinals.includes(base.ordinal) + ? view.query.record_ordinals.includes(line.base.ordinal) : values.size > 0); - const all = !queried && listsAll(view, base.ordinal); - const computed = queried - ? evaluateQuery( - view.query!.parameters, - values, - store.inCollection(view.type, collection), - ) - : undefined; - if (queried && computed === undefined) + if (queried) { + const computed = evaluateQuery( + view.query!.parameters, + values, + store.inCollection(view.type, collection), + ); + return computed === undefined + ? undefined + : { members: computed.page, membership: "query", total: computed.total }; + } + if (listsAll(view, line.base.ordinal)) { + const members = store.inCollection(view.type, collection); + return { members, membership: "all", total: members.length }; + } + const idField = store.type(view.type)?.id_field ?? ""; + const members = line.recorded.flatMap((item) => { + const id = scalarText(item[idField]); + const entity = + id === undefined ? undefined : store.entity(view.type, collection, id); + return entity === undefined ? [] : [entity]; + }); + return { members, membership: "recorded", total: members.length }; +}; + +/** A list call renders the entities its line selects from the state into that line's recorded envelope. */ +const answerList = ( + serving: StateServing, + call: ToolCall, + view: StateListView, + base: RecordedExchange, +): Result => { + const { source, store } = serving; + const { tool, args, exact } = call; + const envelope = recordedJson(base); + const recorded = + envelope === undefined ? undefined : listItems(envelope, view.pointer); + if (envelope === undefined || recorded === undefined) + return fromRecording(source, tool, args, "body-unusable"); + const selection = listSelection(store, call, view, { base, recorded }); + if (selection === undefined) return fromRecording(source, tool, args, "unrecorded-request"); - const members: readonly StateEntity[] = all - ? store.inCollection(view.type, collection) - : (computed?.page ?? - recorded.flatMap((item) => { - const id = scalarText(item[type?.id_field ?? ""]); - const entity = - id === undefined - ? undefined - : store.entity(view.type, collection, id); - return entity === undefined ? [] : [entity]; - })); - const volatile = new Set(type?.volatile_fields); + const volatile = new Set(store.type(view.type)?.volatile_fields); const items: JsonObject[] = []; - for (const entity of members) { + for (const entity of selection.members) { const shown = shownEntity(tool, view, entity, volatile); if (!shown.ok) return shown; items.push(shown.value); } const body = - view.query === undefined || (!all && computed === undefined) + view.query === undefined || selection.membership === "recorded" ? renderStateList(view, envelope, items) : renderQueryList( { ...view, query: view.query }, envelope, items, - computed?.total ?? items.length, + selection.total, ); return stateAnswer(serving, { exchange: base, exact }, body, { source: "state", view: "list", type: view.type, - collection, - membership: all ? "all" : computed === undefined ? "recorded" : "query", + collection: collectionPathOf(base, false), + membership: selection.membership, listed: items.length, }); }; diff --git a/src/application/environment/environmentStateCallLines.ts b/src/application/environment/environmentStateCallLines.ts new file mode 100644 index 0000000000..b44b8999f8 --- /dev/null +++ b/src/application/environment/environmentStateCallLines.ts @@ -0,0 +1,152 @@ +import { + allLines, + collectionPathOf, + pagesOnly, + segmentsOf, + type StateListView, +} from "../../domain/analysisView/environmentStateRules.js"; +import { + pathValueMatches, + selectReplayExchange, + selectsRecordedLine, + type ReplayArguments, +} from "./environmentReplay.js"; +import type { + EnvironmentManifestTool, + EnvironmentSource, + RecordedExchange, +} from "./environmentSource.js"; +import { environmentToolArguments } from "./environmentTools.js"; + +/** One collection segment an item call names: a template literal, or a supplied path value. */ +export interface CallSegment { + readonly text: string; + readonly supplied: boolean; +} + +/** A call's path segments: template literals and supplied values, or undefined when a path argument is missing. */ +export const callSegments = ( + tool: EnvironmentManifestTool, + args: ReplayArguments, +): CallSegment[] | undefined => { + const names = environmentToolArguments(tool).path; + const params = new Map(tool.params.map((param) => [param.index, param])); + const segments: CallSegment[] = []; + const template = tool.path_template + .slice(tool.path_template.indexOf("/") + 1) + .split("/"); + for (const [index, text] of template.entries()) { + const param = params.get(index); + if (param === undefined) { + if (text !== "") segments.push({ text, supplied: false }); + continue; + } + const value = args[names.get(param.name) ?? param.name]; + if (typeof value !== "string") return undefined; + segments.push({ text: value, supplied: true }); + } + return segments; +}; + +/** How many supplied segments a held collection matches verbatim, when it matches the call under replay's path rule. */ +export const collectionMatch = ( + collection: string, + segments: readonly CallSegment[], +): number | undefined => { + const held = segmentsOf(collection); + if (held.length !== segments.length) return undefined; + let verbatim = 0; + for (const [index, { text, supplied }] of segments.entries()) { + const segment = held[index]!; + if (supplied ? !pathValueMatches(segment, text) : segment !== text) + return undefined; + if (supplied && segment === text) verbatim += 1; + } + return verbatim; +}; + +/** The checked recorded line a list or document call names exactly, chosen among them as replay chooses; later or filtered recordings never qualify. */ +export const checkedLine = ( + source: EnvironmentSource, + tool: EnvironmentManifestTool, + checked: readonly number[], + args: ReplayArguments, +): RecordedExchange | undefined => { + const lines = new Map(); + for (const ordinal of checked) { + const exchange = source.exchanges.get(ordinal); + if (exchange !== undefined && selectsRecordedLine(tool, exchange, args)) + lines.set(ordinal, exchange); + } + return selectReplayExchange(tool, lines, args)?.exchange; +}; + +/** For a list of a whole collection, a call that supplies no query argument answers from a checked line of the collection its path arguments name that only pages it, chosen as replay chooses; a requested page answers from its recording. */ +export const unfilteredLine = ( + source: EnvironmentSource, + tool: EnvironmentManifestTool, + checked: readonly number[], + args: ReplayArguments, +): RecordedExchange | undefined => { + const query = environmentToolArguments(tool).query; + const segments = callSegments(tool, args); + if ( + segments === undefined || + [...query.values()].some((argument) => args[argument] !== undefined) + ) + return undefined; + const lines = new Map(); + for (const ordinal of checked) { + const exchange = source.exchanges.get(ordinal); + if ( + exchange !== undefined && + pagesOnly(exchange.query ?? undefined) && + collectionMatch(collectionPathOf(exchange, false), segments) !== undefined + ) + lines.set(ordinal, exchange); + } + return selectReplayExchange(tool, lines, args)?.exchange; +}; + +/** A call's query arguments by parameter name. */ +export const callQuery = ( + tool: EnvironmentManifestTool, + args: ReplayArguments, +): Map => + new Map( + [...environmentToolArguments(tool).query].flatMap( + ([name, argument]): [string, string][] => { + const value = args[argument]; + return value === undefined ? [] : [[name, String(value)]]; + }, + ), + ); + +/** For a list with a query, a call whose every query argument has a role answers on a query or whole line of the collection its path arguments name. */ +export const queryLine = ( + source: EnvironmentSource, + tool: EnvironmentManifestTool, + view: StateListView, + args: ReplayArguments, +): RecordedExchange | undefined => { + const query = view.query; + const segments = callSegments(tool, args); + if ( + query === undefined || + segments === undefined || + [...callQuery(tool, args).keys()].some( + (name) => query.parameters[name] === undefined, + ) + ) + return undefined; + const lines = new Map(); + for (const ordinal of [...query.record_ordinals, ...allLines(view)]) { + const exchange = source.exchanges.get(ordinal); + if ( + exchange !== undefined && + collectionMatch(collectionPathOf(exchange, false), segments) !== undefined + ) + lines.set(ordinal, exchange); + } + return selectReplayExchange(tool, lines, args)?.exchange; +}; diff --git a/src/application/environment/environmentStateWriteAnswer.test.ts b/src/application/environment/environmentStateWriteAnswer.test.ts index ada4612374..8d64904695 100644 --- a/src/application/environment/environmentStateWriteAnswer.test.ts +++ b/src/application/environment/environmentStateWriteAnswer.test.ts @@ -11,6 +11,7 @@ import { DELETE_NOT_FOUND, FIRST_BOOK, LIST_LINE_ROWS, + QUERY_ROWS, SECOND_BOOK, THIRD_BOOK, book, @@ -377,6 +378,34 @@ describe("list lines", () => { }); }); +describe("list queries", () => { + it("filters, searches and sorts the state for values no line recorded, created books included", () => { + const books = servedFrom(QUERY_ROWS); + const created = books.answer("post_books", { + body: { name: "New Book", shelf: "b" }, + }).body as Book; + const listed = (args: Record) => { + const answer = books.answer("get_books", args); + return { + ids: (answer.body as Book[]).map((entry) => entry["id"]), + state: answer.state, + }; + }; + expect(listed({ shelf: "b" })).toMatchObject({ + ids: [2, created["id"]], + state: { source: "state", membership: "query", listed: 2 }, + }); + expect(listed({ q: "NEW" }).ids).toEqual([created["id"]]); + expect(listed({ sort: "-id" }).ids).toEqual([created["id"], 2, 1]); + expect(listed({ sort: "-id", shelf: "a" }).ids).toEqual([1]); + // A value the flag never recorded, or a parameter the query has no role for, replays as before. + expect(listed({ include: "none" }).state).toMatchObject({ + source: "recording", + reason: "unrecorded-request", + }); + }); +}); + describe("path values", () => { const orgBook = (id: number, org: string, name: string): Book => ({ id, diff --git a/src/application/environment/environmentStateWriteAnswer.ts b/src/application/environment/environmentStateWriteAnswer.ts index 1bea1b2cfe..5e086b960c 100644 --- a/src/application/environment/environmentStateWriteAnswer.ts +++ b/src/application/environment/environmentStateWriteAnswer.ts @@ -38,9 +38,8 @@ import { type ReplayArguments, } from "./environmentReplay.js"; import type { EnvironmentManifestTool } from "./environmentSource.js"; +import { callSegments, collectionMatch } from "./environmentStateCallLines.js"; import { - callSegments, - collectionMatch, itemCall, missingAnswer, namedEntity, diff --git a/src/contracts/analysisViewToolContracts.ts b/src/contracts/analysisViewToolContracts.ts index 3adc35f12c..4a2323635b 100644 --- a/src/contracts/analysisViewToolContracts.ts +++ b/src/contracts/analysisViewToolContracts.ts @@ -17,7 +17,7 @@ export const ANALYSIS_VIEW_TOOL_CONTRACTS = [ ...toolContractMetadata("inspect_analysis_view"), kind: "application", description: - 'Inspect selected views of completed Frida session status, GDB/objdump command observations, Go build metadata, binary-layout, JavaScript application, native analyze_function, inspect_artifact, inventory_artifact, inspect_wasm_artifact, inspect_macho, decode_interface_builder, inspect_evm_interface, inspect_managed_artifact, inspect_managed_members, inspect_android_package, inspect_dart_aot, inspect_il2cpp_metadata, inspect_model_artifact or inspect_web_network_capture Evidence without repeating analysis. Command summary views preserve ordered GDB records or opaque objdump stdout/stderr once, source parameters, completion facts and parent identity. They do not establish current target state. Historical capture summaries return identity and counts without bodies or archive inventories. Capture views page retained records: view {kind: "capture", detail: "preview" | "complete", offset: 0, limit: 20}; preview selects recorded headline fields and binary descriptors, complete returns exact records with all sidecars. Offsets follow the selected parent record order, preserving original producer ordinals. Paging never reopens the source or reruns the decoder. Go metadata uses kind go, collection dependencies/settings, exact paths/keys or indices, fields and pagination. Row index refers to the original collection; empty selections return no rows, and dependency coverage remains unknown. For inspect_pcap_capture parents, kind pcap pages endpoints, dns-queries, tls-sni, protocols, http-uris or unattributed-addresses. Select an exact query for dns-addresses, or a tshark/capinfos stream for UTF-16 diagnostics text. Page totals count retained facts; source coverage remains explicit. Qlog summaries keep profile/scope and counts; kind qlog pages traces, streams, loss-declarations, incomplete-records, contributions, ranges or final-sizes. Except traces, select trace_index; stream-specific facets also require exact stream_id and direction. Facts retain all-recorded-events-before-selection scope, and interval overlap or endpoint loss declarations do not prove retransmission or network loss. The api-surface view groups retained HAR requests by method, origin and format-rule path template; templated segments are inferences. It reads no headers or bodies and reports no query values. The environment-manifest view projects the same HAR requests on the primary origin (most JSON-like records) into a replayable tool surface: view {kind: "environment-manifest", template_rules: ["after-uid-literal", "opaque-token", "sibling-variance"], include_query_parameter_names: true}. Each tool has a deterministic name, a path template and parameters from the four format rules plus the selected opt-in rules, observed query parameter names, a value-free body shape for POST/PUT/PATCH, status and content-type sets and record ordinals; records are pointers into the parent, excluded groups carry a reason and three fixed unknowns are always stated. Templates, parameters and names are inferences; the view copies no header, cookie or body values and refuses parents inspected with sensitive values. A capture_browser_scenario parent that selected response bodies or header values projects the same way, one record per network transaction in request order; it relies on capture-time redaction of declared secrets instead of that refusal, and a transaction whose URL was redacted is left unrepresented. The environment-state view extracts, from the same parents, the initial state the recorded reads observed: view {kind: "environment-state", template_rules: [...]} with the rules of the manifest it accompanies. It reads response bodies recorded before the first related write and reports entity types with an identity field (proved by a path parameter or inferred from a field name), entities in collection order with per-view values where views differ, documents for non-parameterized reads that are not entities and have one initial request line (item-shaped paths stay recorded), and a render plan per read tool: item and list views reproduce their latest recordings from the state, and every other read tool is a recorded view with a reason. It also reports a write plan per state-changing tool (create, update, delete or document, built from its first successful call) and write_clock; a plan is served only when replaying the capture shows a later read confirming its effect and none contradicting it. Types and identity are inferences; values are recorded data, so it refuses parents inspected with sensitive values, and a scenario parent must have selected response bodies; its values keep the capture redaction markers, and a write whose URL it redacted ends the state of every type. For a retained record use source: {kind: "retained-evidence", evidence_id: ""}. Use normalized_result.parent_evidence_id from a summary receipt. Source kind is required. Portable source uses {kind: "inline", evidence: }. A completed Evidence JSON file on this host uses {kind: "evidence-file", path: ""}: REA reads and authenticates it, then projects it like retained Evidence, so a large capture is bounded by the file reader rather than by inline-input accounting. For layout, JavaScript, inventory, WASM, Mach-O, Interface Builder, EVM, managed artifact, Android package and Dart AOT sources, request a summary with view: {kind: "summary"}; summary is a view kind and has no facet. The WASM summary gives validation, counts, WAT digest/length, tool profile and execution/network states without reading WAT text. Select a layout mitigations/linkage facet, one section/symbol/module, or a stable page. Module pages include assets, bundled and source modules; use node_id for ambiguous or unavailable paths. Typed module references use view {kind: "javascript", collection: "modules", projection: "references", where: {path: "main.js", kind: "javascript-module"}, offset: 0, limit: 64}; where matches exact retained strings (null selects unavailable paths). Rows include canonical identity, retained source-range count and observation support states. Source coverage and selection remain explicit; exhausting matching rows does not prove runtime loading or complete analysis. For exact file/native facts and processing gaps, use view {kind: "javascript-facts", where: {paths: ["package.json"]}, property_keys: ["name", "operation", "analysis_status"]}. Rows page original observations with canonical identities and Evidence; missing properties remain omitted. Paths match recorded artifact-path/source-range locations; optional kinds filter nodes. To discover field names without values, opt into include_available_property_keys: true with property_keys: []. Canonical identity and Evidence are separate from properties; keep each node_id/observation_id pair together when composing results. Inventory uses view: {kind: "inventory", collection: "occurrences", logical_paths: ["package.json"], offset: 0, limit: 64}. Inventory views page nodes, occurrences, edges or integrity contradictions. Occurrences accept exact logical_paths and occurrence_ids; nodes accept artifact_ids. Batch filters intersect (omitted means all available, [] means none); fields selects columns while retaining identity links. source_complete describes retained graph membership; source_coverage describes producer coverage, including unknown encrypted PCK directories. Page exhaustion does not prove absence in a partial source. WASM views use kind wasm and facet sections/imports/exports/candidates/glue/wat/headers/details/tool_profile; text facets use UTF-16 code units and source forms remain exact. Mach-O views use kind macho and facet entrypoints/architectures/build_metadata/load_commands/dependencies/imports/exports/segments/provenance; source_exhaustive false means unreturned rows may exist. Interface Builder views use kind interface-builder and collection documents/nodes/edges/coverage; nodes and edges accept an exact document relative_path. EVM summary returns all inferred interface candidates, carrier/decoded identities and explicit unknowns without hex or duplicate stdout. EVM views use kind evm and facet functions/bytecode_hex/stdout/stderr; function pages preserve the original candidate rows, and text offsets/limits count UTF-16 code units (hex has two characters per decoded byte). Exhausting candidate pages does not establish global interface completeness. Managed artifact summaries preserve complete identity/classification, metadata status, module/MVID, frameworks, coverage and limitations without unrelated inventories. Managed pages use kind managed and collection references/resources/attributes, retaining source coverage and build-local token scope; page exhaustion does not establish complete metadata. Managed member summaries keep identity, selection with unmatched names, coverage and per-collection counts; managed-members pages use collection types/exported_types/fields/methods/member_refs/call_edges/field_accesses/user_strings and carry the source selection and coverage. Android package summaries retain all declarations, engine identity and counts without decoded manifest text. Android manifest pages use kind android, facet manifest, offset and limit in UTF-16 code units; source_status preserves partial decoded coverage even when the retained text page is exhausted. Dart AOT summary preserves target/ABI/library identity, snapshot headers and pool lower bounds. String pages use kind dart-aot, collection package_uris/dart_uris/class_like_tokens; source coverage remains partial when the producer says partial. Protobuf views use kind protobuf, collection bodies/frames/fields, with record_ordinals, body_pointers and fields; field pages also select frame_indices, field_ids, field_numbers or schema_names. Byte identity, offset basis and body status remain present. Text and schema_value are opt-in page columns. Model views use kind model-artifact and collection metadata (keys), tensors (names/types) or types (derived totals), with indices, fields and paging. Header SHA-256 scopes declarations; full-file identity remains unknown unless full hashing was selected. Arrays retain typed eight-element previews with explicit omissions. IL2CPP metadata views use kind il2cpp-metadata and collection sections/images/types/methods/fields/properties/events/parameters/string_literals/code_gen_modules/method_addresses/return_types/field_types/parameter_types. Select indices, fields and pages; named rows accept names, types also accept image_indices/namespaces/declaring_types, and methods/members accept declaring_types. Parameters and parameter_types accept method_indices; code_gen_modules accept image_indices; method_addresses accept addresses/states, preserving every folded alternative. Row index is scoped to the metadata digest; native pages add binary_source. Uncollected native tables have null counts and remain unavailable. Desktop directory views use kind desktop-application and collection files/omissions/candidates/components/next_probes, with fields and pagination; files and omissions accept exact paths or their stable IDs, and candidates accept families. Source coverage and stable row IDs remain available. Native inventory Evidence from list_names, list/search_procedures or list/search_strings uses view: {kind: "native-inventory", offset: 0, limit: 64, fields: ["value"]}. Rows keep source_index and address even with selected fields. Summary counts describe retained rows; provider discovery coverage remains unknown. Native function Evidence requires kind native: view: {kind: "native", facet: "procedure"}. Facets select procedure, pseudocode, assembly, basic blocks, comments, callers/callees, references, unresolved calls, referenced strings/names, native API and high-pcode. Use nonnegative offset and a positive caller-selected limit (defaults 0 and 64); pseudocode uses UTF-16 code units, other native pages use rows. Returns artifact identity, parent Evidence ID, a distinct view digest, coverage, limitations and unknowns. Actual serialized size determines MCP transport admission; choose a smaller page or export the complete retained Evidence if needed.', + 'Inspect selected views of completed Frida session status, GDB/objdump command observations, Go build metadata, binary-layout, JavaScript application, native analyze_function, inspect_artifact, inventory_artifact, inspect_wasm_artifact, inspect_macho, decode_interface_builder, inspect_evm_interface, inspect_managed_artifact, inspect_managed_members, inspect_android_package, inspect_dart_aot, inspect_il2cpp_metadata, inspect_model_artifact or inspect_web_network_capture Evidence without repeating analysis. Command summary views preserve ordered GDB records or opaque objdump stdout/stderr once, source parameters, completion facts and parent identity. They do not establish current target state. Historical capture summaries return identity and counts without bodies or archive inventories. Capture views page retained records: view {kind: "capture", detail: "preview" | "complete", offset: 0, limit: 20}; preview selects recorded headline fields and binary descriptors, complete returns exact records with all sidecars. Offsets follow the selected parent record order, preserving original producer ordinals. Paging never reopens the source or reruns the decoder. Go metadata uses kind go, collection dependencies/settings, exact paths/keys or indices, fields and pagination. Row index refers to the original collection; empty selections return no rows, and dependency coverage remains unknown. For inspect_pcap_capture parents, kind pcap pages endpoints, dns-queries, tls-sni, protocols, http-uris or unattributed-addresses. Select an exact query for dns-addresses, or a tshark/capinfos stream for UTF-16 diagnostics text. Page totals count retained facts; source coverage remains explicit. Qlog summaries keep profile/scope and counts; kind qlog pages traces, streams, loss-declarations, incomplete-records, contributions, ranges or final-sizes. Except traces, select trace_index; stream-specific facets also require exact stream_id and direction. Facts retain all-recorded-events-before-selection scope, and interval overlap or endpoint loss declarations do not prove retransmission or network loss. The api-surface view groups retained HAR requests by method, origin and format-rule path template; templated segments are inferences. It reads no headers or bodies and reports no query values. The environment-manifest view projects the same HAR requests on the primary origin (most JSON-like records) into a replayable tool surface: view {kind: "environment-manifest", template_rules: ["after-uid-literal", "opaque-token", "sibling-variance"], include_query_parameter_names: true}. Each tool has a deterministic name, a path template and parameters from the four format rules plus the selected opt-in rules, observed query parameter names, a value-free body shape for POST/PUT/PATCH, status and content-type sets and record ordinals; records are pointers into the parent, excluded groups carry a reason and three fixed unknowns are always stated. Templates, parameters and names are inferences; the view copies no header, cookie or body values and refuses parents inspected with sensitive values. A capture_browser_scenario parent that selected response bodies or header values projects the same way, one record per network transaction in request order; it relies on capture-time redaction of declared secrets instead of that refusal, and a transaction whose URL was redacted is left unrepresented. The environment-state view extracts, from the same parents, the initial state the recorded reads observed: view {kind: "environment-state", template_rules: [...]} with the rules of the manifest it accompanies. It reads response bodies recorded before the first related write and reports entity types with an identity field (proved by a path parameter or inferred from a field name), entities in collection order with per-view values where views differ, documents for non-parameterized reads that are not entities and have one initial request line (item-shaped paths stay recorded), and a render plan per read tool: item and list views reproduce their latest recordings from the state (a list also learns the filters, search, sort and paging its recordings verify, and answers new values of them from the state), and every other read tool is a recorded view with a reason. It also reports a write plan per state-changing tool (create, update, delete or document, built from its first successful call) and write_clock; a plan is served only when replaying the capture shows a later read confirming its effect and none contradicting it. Types and identity are inferences; values are recorded data, so it refuses parents inspected with sensitive values, and a scenario parent must have selected response bodies; its values keep the capture redaction markers, and a write whose URL it redacted ends the state of every type. For a retained record use source: {kind: "retained-evidence", evidence_id: ""}. Use normalized_result.parent_evidence_id from a summary receipt. Source kind is required. Portable source uses {kind: "inline", evidence: }. A completed Evidence JSON file on this host uses {kind: "evidence-file", path: ""}: REA reads and authenticates it, then projects it like retained Evidence, so a large capture is bounded by the file reader rather than by inline-input accounting. For layout, JavaScript, inventory, WASM, Mach-O, Interface Builder, EVM, managed artifact, Android package and Dart AOT sources, request a summary with view: {kind: "summary"}; summary is a view kind and has no facet. The WASM summary gives validation, counts, WAT digest/length, tool profile and execution/network states without reading WAT text. Select a layout mitigations/linkage facet, one section/symbol/module, or a stable page. Module pages include assets, bundled and source modules; use node_id for ambiguous or unavailable paths. Typed module references use view {kind: "javascript", collection: "modules", projection: "references", where: {path: "main.js", kind: "javascript-module"}, offset: 0, limit: 64}; where matches exact retained strings (null selects unavailable paths). Rows include canonical identity, retained source-range count and observation support states. Source coverage and selection remain explicit; exhausting matching rows does not prove runtime loading or complete analysis. For exact file/native facts and processing gaps, use view {kind: "javascript-facts", where: {paths: ["package.json"]}, property_keys: ["name", "operation", "analysis_status"]}. Rows page original observations with canonical identities and Evidence; missing properties remain omitted. Paths match recorded artifact-path/source-range locations; optional kinds filter nodes. To discover field names without values, opt into include_available_property_keys: true with property_keys: []. Canonical identity and Evidence are separate from properties; keep each node_id/observation_id pair together when composing results. Inventory uses view: {kind: "inventory", collection: "occurrences", logical_paths: ["package.json"], offset: 0, limit: 64}. Inventory views page nodes, occurrences, edges or integrity contradictions. Occurrences accept exact logical_paths and occurrence_ids; nodes accept artifact_ids. Batch filters intersect (omitted means all available, [] means none); fields selects columns while retaining identity links. source_complete describes retained graph membership; source_coverage describes producer coverage, including unknown encrypted PCK directories. Page exhaustion does not prove absence in a partial source. WASM views use kind wasm and facet sections/imports/exports/candidates/glue/wat/headers/details/tool_profile; text facets use UTF-16 code units and source forms remain exact. Mach-O views use kind macho and facet entrypoints/architectures/build_metadata/load_commands/dependencies/imports/exports/segments/provenance; source_exhaustive false means unreturned rows may exist. Interface Builder views use kind interface-builder and collection documents/nodes/edges/coverage; nodes and edges accept an exact document relative_path. EVM summary returns all inferred interface candidates, carrier/decoded identities and explicit unknowns without hex or duplicate stdout. EVM views use kind evm and facet functions/bytecode_hex/stdout/stderr; function pages preserve the original candidate rows, and text offsets/limits count UTF-16 code units (hex has two characters per decoded byte). Exhausting candidate pages does not establish global interface completeness. Managed artifact summaries preserve complete identity/classification, metadata status, module/MVID, frameworks, coverage and limitations without unrelated inventories. Managed pages use kind managed and collection references/resources/attributes, retaining source coverage and build-local token scope; page exhaustion does not establish complete metadata. Managed member summaries keep identity, selection with unmatched names, coverage and per-collection counts; managed-members pages use collection types/exported_types/fields/methods/member_refs/call_edges/field_accesses/user_strings and carry the source selection and coverage. Android package summaries retain all declarations, engine identity and counts without decoded manifest text. Android manifest pages use kind android, facet manifest, offset and limit in UTF-16 code units; source_status preserves partial decoded coverage even when the retained text page is exhausted. Dart AOT summary preserves target/ABI/library identity, snapshot headers and pool lower bounds. String pages use kind dart-aot, collection package_uris/dart_uris/class_like_tokens; source coverage remains partial when the producer says partial. Protobuf views use kind protobuf, collection bodies/frames/fields, with record_ordinals, body_pointers and fields; field pages also select frame_indices, field_ids, field_numbers or schema_names. Byte identity, offset basis and body status remain present. Text and schema_value are opt-in page columns. Model views use kind model-artifact and collection metadata (keys), tensors (names/types) or types (derived totals), with indices, fields and paging. Header SHA-256 scopes declarations; full-file identity remains unknown unless full hashing was selected. Arrays retain typed eight-element previews with explicit omissions. IL2CPP metadata views use kind il2cpp-metadata and collection sections/images/types/methods/fields/properties/events/parameters/string_literals/code_gen_modules/method_addresses/return_types/field_types/parameter_types. Select indices, fields and pages; named rows accept names, types also accept image_indices/namespaces/declaring_types, and methods/members accept declaring_types. Parameters and parameter_types accept method_indices; code_gen_modules accept image_indices; method_addresses accept addresses/states, preserving every folded alternative. Row index is scoped to the metadata digest; native pages add binary_source. Uncollected native tables have null counts and remain unavailable. Desktop directory views use kind desktop-application and collection files/omissions/candidates/components/next_probes, with fields and pagination; files and omissions accept exact paths or their stable IDs, and candidates accept families. Source coverage and stable row IDs remain available. Native inventory Evidence from list_names, list/search_procedures or list/search_strings uses view: {kind: "native-inventory", offset: 0, limit: 64, fields: ["value"]}. Rows keep source_index and address even with selected fields. Summary counts describe retained rows; provider discovery coverage remains unknown. Native function Evidence requires kind native: view: {kind: "native", facet: "procedure"}. Facets select procedure, pseudocode, assembly, basic blocks, comments, callers/callees, references, unresolved calls, referenced strings/names, native API and high-pcode. Use nonnegative offset and a positive caller-selected limit (defaults 0 and 64); pseudocode uses UTF-16 code units, other native pages use rows. Returns artifact identity, parent Evidence ID, a distinct view digest, coverage, limitations and unknowns. Actual serialized size determines MCP transport admission; choose a smaller page or export the complete retained Evidence if needed.', inputSchema: inspectAnalysisViewInputSchema, outputSchema: evidenceResultOf(analysisViewResultSchema), examples: [ diff --git a/src/domain/analysisView/environmentStateListQuery.test.ts b/src/domain/analysisView/environmentStateListQuery.test.ts new file mode 100644 index 0000000000..82b9c35e61 --- /dev/null +++ b/src/domain/analysisView/environmentStateListQuery.test.ts @@ -0,0 +1,167 @@ +import { describe, expect, it } from "vitest"; + +import { + evaluateQuery, + explainLine, + learnQuery, + lineChoices, + queryValues, + type QueryLine, +} from "./environmentStateListQuery.js"; +import type { JsonObject, StateEntity } from "./environmentStateRules.js"; + +const entity = (fields: JsonObject): StateEntity => ({ + type: "/api/books", + collection: "/api/books", + id: String(fields["id"]), + fields, + view_fields: [], + record_ordinals: [], +}); + +const BOOKS = [ + { + id: 1, + name: "Tide Tables", + shelf: "a", + tags: ["sea"], + url: "https://x/tide", + }, + { id: 2, name: "Lamp Oil", shelf: "b", tags: [{ id: "t1", name: "oil" }] }, + { id: 3, name: "tidewater", shelf: "a", tags: [], url: "https://x/other" }, + { + id: 4, + name: "Kelp", + shelf: null, + tags: ["sea"], + url: "https://x/kelp-tide", + }, +].map(entity); + +/** A line as the gate reads it: its query, and the books it listed in order. */ +const line = ( + query: string, + listed: readonly number[], + ordinal = 0, +): QueryLine => { + const values = queryValues(query)!; + const items = listed.map( + (id) => BOOKS.find((book) => book.id === String(id))!.fields, + ); + return { + ordinal, + values, + listed: listed.map(String), + choices: lineChoices(values, items), + entities: BOOKS, + }; +}; + +const ids = (query: string, parameters: Parameters[0]) => + evaluateQuery(parameters, queryValues(query)!, BOOKS)?.page.map( + (book) => book.id, + ); + +describe("explaining a recorded line", () => { + it("finds a filter on the field every listed item holds, scalar or in an array", () => { + expect(explainLine(line("?shelf=a", [1, 3]))).toEqual({ + shelf: { role: "filter", field: "shelf" }, + }); + expect(explainLine(line("?tag=sea", [1, 4]))).toEqual({ + tag: { role: "filter", field: "tags" }, + }); + expect(explainLine(line("?tag=t1", [2]))).toEqual({ + tag: { role: "filter", field: "tags" }, + }); + }); + + it("searches the fields the term matched, ignoring case, and leaves out URLs that merely repeat it", () => { + expect(explainLine(line("?q=TIDE", [1, 3]))).toEqual({ + q: { role: "search", fields: ["name"] }, + }); + }); + + it("sorts by the field a value names, with a direction parameter, prefix or suffix", () => { + expect( + explainLine(line("?orderBy=name&orderDirection=desc", [3, 1, 2, 4])), + ).toEqual({ + orderBy: { role: "sort" }, + orderDirection: { role: "direction" }, + }); + expect(ids("?sort=-name", { sort: { role: "sort" } })).toEqual([ + "3", + "1", + "2", + "4", + ]); + expect(ids("?sort=NAME-asc", { sort: { role: "sort" } })).toEqual([ + "4", + "2", + "1", + "3", + ]); + // Null values sort last ascending. + expect(ids("?sort=shelf", { sort: { role: "sort" } })).toEqual([ + "1", + "3", + "2", + "4", + ]); + }); + + it("pages by limit and offset, or by page number from the first page it explains", () => { + expect(explainLine(line("?limit=2&offset=1", [2, 3]))).toEqual({ + limit: { role: "limit" }, + offset: { role: "offset" }, + }); + expect(explainLine(line("?page=2&per_page=2", [3, 4]))).toEqual({ + page: { role: "page", first: 1 }, + per_page: { role: "limit" }, + }); + expect(ids("?per_page=-1", { per_page: { role: "limit" } })).toEqual([ + "1", + "2", + "3", + "4", + ]); + }); + + it("takes a parameter that changes nothing as inert, for its recorded values only", () => { + const explained = explainLine(line("?includeDeleted=true", [1, 2, 3, 4])); + expect(explained).toEqual({ + includeDeleted: { role: "inert", values: ["true"] }, + }); + expect(ids("?includeDeleted=false", explained!)).toBeUndefined(); + }); + + it("explains nothing that the collection cannot reproduce", () => { + expect(explainLine(line("?shelf=a", [3, 1]))).toBeUndefined(); + }); +}); + +describe("learning a list's query", () => { + it("joins lines into one role per parameter and answers the lines it reproduces", () => { + const learned = learnQuery([ + line("?shelf=a", [1, 3], 10), + line("?shelf=b", [2], 11), + line("?q=kelp", [4], 12), + line("?q=oil", [2], 13), + ]); + expect(learned).toEqual({ + parameters: { + shelf: { role: "filter", field: "shelf" }, + q: { role: "search", fields: ["name"] }, + }, + record_ordinals: [10, 11, 12, 13], + }); + // A value no line recorded is evaluated the same way. + expect(ids("?q=TIDE&shelf=a", learned!.parameters)).toEqual(["1", "3"]); + }); + + it("refuses a value a parameter cannot apply, or a parameter it never learned", () => { + const parameters = { limit: { role: "limit" } } as const; + expect(ids("?limit=two", parameters)).toBeUndefined(); + expect(ids("?shelf=a", parameters)).toBeUndefined(); + expect(queryValues("?a=1&a=2")).toBeUndefined(); + }); +}); diff --git a/src/domain/analysisView/environmentStateListQuery.ts b/src/domain/analysisView/environmentStateListQuery.ts index d9adcbdbf3..9361e8e064 100644 --- a/src/domain/analysisView/environmentStateListQuery.ts +++ b/src/domain/analysisView/environmentStateListQuery.ts @@ -139,12 +139,14 @@ const sortKey = ( ); return found.length === 1 ? found[0] : undefined; }; - const whole = field(value); - if (whole !== undefined) return { field: whole, descending: false }; + if (names.includes(value)) return { field: value, descending: false }; + // A leading `-` descends; matching ignores separators, so it is read before the name is matched loosely. if (value.startsWith("-")) { const named = field(value.slice(1)); return named === undefined ? undefined : { field: named, descending: true }; } + const whole = field(value); + if (whole !== undefined) return { field: whole, descending: false }; const suffixed = /^(.+?)[\s:_-](asc|desc)$/iu.exec(value); const named = suffixed === null ? undefined : field(suffixed[1]!); return named === undefined @@ -171,64 +173,98 @@ const ascending = ( const integer = (value: string): number | undefined => /^-?\d{1,9}$/u.test(value) ? Number(value) : undefined; +/** What a call's values ask of the collection, or undefined when one has no role or cannot apply. */ +interface QueryCall { + readonly selections: ((entity: StateEntity) => boolean)[]; + sort?: string; + direction?: string; + limit?: number | undefined; + offset: number; + page?: { readonly index: number; readonly first: number }; +} + +const readCall = ( + parameters: Readonly>, + values: ReadonlyMap, +): QueryCall | undefined => { + const call: QueryCall = { selections: [], offset: 0 }; + for (const [name, value] of values) { + const parameter = parameters[name]; + const number = integer(value); + if (parameter === undefined) return undefined; + if (parameter.role === "filter") + call.selections.push((entity) => + holds(entity.fields[parameter.field], value), + ); + else if (parameter.role === "search") + call.selections.push((entity) => + contains(entity, parameter.fields, value), + ); + else if (parameter.role === "sort") call.sort = value; + else if (parameter.role === "direction") { + if (!DIRECTIONS.has(value.toLowerCase())) return undefined; + call.direction = value.toLowerCase(); + } else if (parameter.role === "inert") { + if (!parameter.values.includes(value)) return undefined; + } else if (number === undefined) return undefined; + else if (parameter.role === "limit") + call.limit = number > 0 ? number : undefined; + else if (parameter.role === "offset" && number >= 0) call.offset = number; + else if (parameter.role === "page" && number >= parameter.first) + call.page = { index: number, first: parameter.first }; + else return undefined; + } + return call; +}; + +/** Entities in the order a sort value names, ties in state order; undefined when it names no field. */ +const sorted = ( + matched: readonly StateEntity[], + entities: readonly StateEntity[], + call: QueryCall, +): StateEntity[] | undefined => { + if (call.sort === undefined) return [...matched]; + const names = [ + ...new Set(entities.flatMap((entity) => Object.keys(entity.fields))), + ]; + const key = sortKey(call.sort, names); + if (key === undefined) return undefined; + const sign = ( + call.direction === undefined ? key.descending : call.direction === "desc" + ) + ? -1 + : 1; + return matched + .map((entity, index) => ({ entity, index })) + .sort( + (left, right) => + sign * + ascending( + left.entity.fields[key.field], + right.entity.fields[key.field], + ) || left.index - right.index, + ) + .map(({ entity }) => entity); +}; + /** The entities a query selects, in its order, and how many matched before paging; undefined when a value cannot be applied. */ export const evaluateQuery = ( parameters: Readonly>, values: ReadonlyMap, entities: readonly StateEntity[], ): { readonly page: StateEntity[]; readonly total: number } | undefined => { - let matched = [...entities]; - let sort: string | undefined; - let direction: string | undefined; - let limit: number | undefined; - let offset = 0; - let page: { readonly index: number; readonly first: number } | undefined; - for (const [name, value] of values) { - const parameter = parameters[name]; - const number = integer(value); - if (parameter === undefined) return undefined; - if (parameter.role === "filter") - matched = matched.filter((entity) => - holds(entity.fields[parameter.field], value), - ); - else if (parameter.role === "search") - matched = matched.filter((entity) => - contains(entity, parameter.fields, value), - ); - else if (parameter.role === "sort") sort = value; - else if (parameter.role === "direction") { - if (!DIRECTIONS.has(value.toLowerCase())) return undefined; - direction = value.toLowerCase(); - } else if (parameter.role === "inert") { - if (!parameter.values.includes(value)) return undefined; - } else if (number === undefined) return undefined; - else if (parameter.role === "limit") - limit = number > 0 ? number : undefined; - else if (parameter.role === "offset" && number >= 0) offset = number; - else if (parameter.role === "page" && number >= parameter.first) - page = { index: number, first: parameter.first }; - else return undefined; - } - if (sort !== undefined) { - const names = [ - ...new Set(entities.flatMap((entity) => Object.keys(entity.fields))), - ]; - const key = sortKey(sort, names); - if (key === undefined) return undefined; - const descending = - direction === undefined ? key.descending : direction === "desc"; - matched = matched - .map((entity, index) => ({ entity, index })) - .sort( - (left, right) => - (descending ? -1 : 1) * - ascending( - left.entity.fields[key.field], - right.entity.fields[key.field], - ) || left.index - right.index, - ) - .map(({ entity }) => entity); - } + const call = readCall(parameters, values); + if (call === undefined) return undefined; + const matched = sorted( + entities.filter((entity) => + call.selections.every((selects) => selects(entity)), + ), + entities, + call, + ); + if (matched === undefined) return undefined; + let { offset } = call; + const { limit, page } = call; if (page !== undefined && page.index > page.first) { if (limit === undefined) return undefined; offset += (page.index - page.first) * limit; @@ -242,6 +278,32 @@ export const evaluateQuery = ( }; }; +/** A query's answer and whether it shows the whole collection: every parameter only sorts or pages, and the page holds every match. */ +export const answerQuery = ( + query: Pick, + values: ReadonlyMap, + entities: readonly StateEntity[], +): + | { + readonly page: StateEntity[]; + readonly total: number; + readonly whole: boolean; + } + | undefined => { + const result = evaluateQuery(query.parameters, values, entities); + return result === undefined + ? undefined + : { + ...result, + whole: + result.page.length === result.total && + [...values.keys()].every((name) => { + const role = query.parameters[name]?.role; + return role !== "filter" && role !== "search" && role !== "inert"; + }), + }; +}; + /** Fields every listed item holds the value in, those named like the parameter first. */ const filterFields = ( items: readonly JsonObject[], @@ -259,23 +321,27 @@ const filterFields = ( ) .slice(0, 4); -/** String fields that contain the term in at least one listed item. */ -const searchFields = (items: readonly JsonObject[], term: string): string[] => - term === "" - ? [] - : [ - ...new Set( - items.flatMap((item) => - Object.keys(item).filter((field) => { - const value = item[field]; - return ( - typeof value === "string" && - value.toLowerCase().includes(term.toLowerCase()) - ); - }), - ), - ), - ].sort(compare); +const isUrl = (value: string): boolean => + /^[a-z][a-z0-9+.-]*:\/\//iu.test(value); + +/** + * The string fields the term matched in the listed items: for each item the fields that contain it, ignoring case, + * other than URLs, which a name or slug often repeats; an item the term matched only in URLs adds those. + */ +const searchFields = (items: readonly JsonObject[], term: string): string[] => { + const needle = term.toLowerCase(); + if (needle === "") return []; + const fields = new Set(); + for (const item of items) { + const matching = Object.keys(item).filter((field) => { + const value = item[field]; + return typeof value === "string" && value.toLowerCase().includes(needle); + }); + const plain = matching.filter((field) => !isUrl(item[field] as string)); + for (const field of plain.length > 0 ? plain : matching) fields.add(field); + } + return [...fields].sort(compare); +}; /** The roles a parameter may play on a line that listed `items`, most specific first; inert last. */ const choicesFor = ( diff --git a/src/domain/analysisView/environmentStateWriteChecks.ts b/src/domain/analysisView/environmentStateWriteChecks.ts index d69a3fde69..91851df9fa 100644 --- a/src/domain/analysisView/environmentStateWriteChecks.ts +++ b/src/domain/analysisView/environmentStateWriteChecks.ts @@ -28,7 +28,7 @@ import { type StateToolFacts, type StateView, } from "./environmentStateRules.js"; -import { evaluateQuery, renderQueryList } from "./environmentStateListQuery.js"; +import { answerQuery, renderQueryList } from "./environmentStateListQuery.js"; /** A query as replay compares a call's: decoded values by name, in any order. A query repeating a name, which no call's arguments can carry, keeps its spelling. */ const queryKey = (query: string | undefined): string => { @@ -420,6 +420,25 @@ const listCheck = ( return same(answer, check.body); }; +/** The base line's and the read's listed entities by target: those the state holds, and those it no longer does. */ +const listedTargets = ( + state: SimulatedState, + view: StateListView, + check: ListBodies, +): { readonly targets: string[]; readonly unheld: string[] } => { + const targets: string[] = []; + const unheld: string[] = []; + const ids = [...check.baseItems, ...check.items].map( + (item) => scalarText(item[check.idField]) ?? "", + ); + for (const id of new Set(ids)) + (state.entity(view.type, check.collection, id) === undefined + ? unheld + : targets + ).push(entityTarget(view.type, check.collection, id)); + return { targets, unheld }; +}; + /** * A list read on a line the query answers: the query's matches in the state, in its order and page. Only a line whose * parameters all sort or page, and that shows every match, covers the collection; any other covers the entities it lists. @@ -431,40 +450,22 @@ const checkQueryList = ( values: ReadonlyMap, ): CheckOutcome | undefined => { const { collection } = check; - const result = evaluateQuery( - view.query!.parameters, + const result = answerQuery( + view.query!, values, state.members(view.type, collection), ); if (result === undefined) return undefined; - const selects = [...values.keys()].some((name) => { - const role = view.query!.parameters[name]?.role; - return role === "filter" || role === "search" || role === "inert"; - }); - const whole = !selects && result.page.length === result.total; - const held = new Set(result.page.map((entity) => entity.id)); return { pass: listCheck(view, check, result.page, result.total), targets: [ - ...(whole ? [collectionTarget(view.type, collection)] : []), + ...(result.whole ? [collectionTarget(view.type, collection)] : []), ...result.page.map((entity) => entityTarget(view.type, collection, entity.id), ), ], // What the base line or the read listed that the state no longer holds: the read shows whether it went. - unheld: [ - ...new Set( - [...check.baseItems, ...check.items].map( - (item) => scalarText(item[check.idField]) ?? "", - ), - ), - ] - .filter( - (id) => - !held.has(id) && - state.entity(view.type, collection, id) === undefined, - ) - .map((id) => entityTarget(view.type, collection, id)), + unheld: listedTargets(state, view, check).unheld, }; }; @@ -504,14 +505,10 @@ const checkRecordedList = ( const entity = state.entity(view.type, collection, idOf(item)); return entity === undefined ? [] : [entity]; }); - const targets: string[] = []; - const unheld: string[] = []; - for (const id of new Set([...check.baseItems, ...check.items].map(idOf))) - (state.entity(view.type, collection, id) === undefined - ? unheld - : targets - ).push(entityTarget(view.type, collection, id)); - return { pass: listCheck(view, check, members), targets, unheld }; + return { + pass: listCheck(view, check, members), + ...listedTargets(state, view, check), + }; }; /** A list read on a checked line, compared with the answer the served environment renders on that line's base recording from the collection it names: every entity when that line lists them all. */ diff --git a/src/domain/analysisView/environmentStateWrites.test.ts b/src/domain/analysisView/environmentStateWrites.test.ts index 6dba69076a..819403f576 100644 --- a/src/domain/analysisView/environmentStateWrites.test.ts +++ b/src/domain/analysisView/environmentStateWrites.test.ts @@ -12,6 +12,7 @@ import { BOOK_WRITE_CLOCK, FIRST_BOOK, LIST_LINE_ROWS, + QUERY_ROWS, SECOND_BOOK, THIRD_BOOK, book, @@ -539,3 +540,32 @@ describe("list lines", () => { ); }); }); + +describe("list queries", () => { + it("learns a filter, a search, a sort and an inert flag from the lines the whole collection does not explain", () => { + const state = stateOf(QUERY_ROWS); + expect(state.views.find((view) => view.tool === "get_books")).toMatchObject( + { + shape: "list", + membership: "recorded", + all_record_ordinals: [0], + query: { + parameters: { + shelf: { role: "filter", field: "shelf" }, + q: { role: "search", fields: ["name", "slug"] }, + sort: { role: "sort" }, + include: { role: "inert", values: ["all"] }, + }, + record_ordinals: [1, 2, 3, 4], + totals: [], + }, + checked_record_ordinals: [0, 1, 2, 3, 4], + }, + ); + // The shelf filter after the create lists the created book, which confirms it. + expect(planOf(state, "post_books")).toMatchObject({ + served: true, + checked_record_ordinals: [8], + }); + }); +}); diff --git a/tests/fixtures/environmentStateWrites.ts b/tests/fixtures/environmentStateWrites.ts index f89a56ad3e..d646ca46bb 100644 --- a/tests/fixtures/environmentStateWrites.ts +++ b/tests/fixtures/environmentStateWrites.ts @@ -114,6 +114,34 @@ export const LIST_LINE_ROWS: readonly TimedCaptureRow[] = [ ["GET", "/api/books", bookPage([FIRST_BOOK, SECOND_BOOK, THIRD_BOOK_AT_4])], ]; +const shelved = (entry: Book, shelf: string): Book => ({ ...entry, shelf }); +const SHELVED_FIRST = shelved(FIRST_BOOK, "a"); +const SHELVED_SECOND = shelved(SECOND_BOOK, "b"); +/** The book the create at exchange 7 of QUERY_ROWS made, on shelf a. */ +export const SHELVED_THIRD = shelved(book(3, "Third Book", bookClock(7)), "a"); + +/** + * One list tool recorded whole, filtered by shelf, searched, sorted and with a flag that changes nothing; item reads; + * then a create the shelf filter later lists. + */ +export const QUERY_ROWS: readonly TimedCaptureRow[] = [ + ["GET", "/api/books", [SHELVED_FIRST, SHELVED_SECOND]], + ["GET", "/api/books?shelf=a", [SHELVED_FIRST]], + ["GET", "/api/books?q=second", [SHELVED_SECOND]], + ["GET", "/api/books?sort=-id", [SHELVED_SECOND, SHELVED_FIRST]], + ["GET", "/api/books?include=all", [SHELVED_FIRST, SHELVED_SECOND]], + ["GET", "/api/books/1", SHELVED_FIRST], + ["GET", "/api/books/2", SHELVED_SECOND], + [ + "POST", + "/api/books", + SHELVED_THIRD, + 201, + { name: "Third Book", shelf: "a" }, + ], + ["GET", "/api/books?shelf=a", [SHELVED_FIRST, SHELVED_THIRD]], +]; + /** The end of the last exchange of BOOK_ROWS, to the next whole second: where the served clock starts. */ export const BOOK_WRITE_CLOCK = "2026-10-17T18:00:12.000Z";