diff --git a/docs/agent-environments.md b/docs/agent-environments.md index 2f149ef6db..0d68900781 100644 --- a/docs/agent-environments.md +++ b/docs/agent-environments.md @@ -68,6 +68,7 @@ where the answer came from: "exact_match": true, "recorded_path": "/api/users/2", "recorded_query": null, + "recorded_line_truncated": false, "body_recorded": true, "body_encoding": "utf-8", "truncated": false @@ -79,7 +80,9 @@ where the answer came from: equals the recorded one; it says nothing about parameters the caller omitted. A path value matches a recorded segment as written or after percent-decoding; when both kinds of match are available, the record whose segment equals the -value as written is chosen. +value as written is chosen. `recorded_query` keeps the `?` of an explicit empty +query, and a recorded path or query longer than 1 KiB is cut in the reply with +`recorded_line_truncated: true`; selection always uses the full recorded line. When no record matches, the tool still answers from the tool's first cited record and reports `exact_match: false`. `body` is parsed JSON when the recorded content type is JSON-like (`application/json`, `application/*+json`, @@ -194,8 +197,9 @@ values (sampled within the list budget above; the manifest keeps them all); query parameter names are optional strings; write methods accept an optional `body` object described by the recorded top-level key types. Each input keeps its own argument: a query name equal to a path parameter's name is -advertised as `query_`, and the write body as `request_body` when a query -parameter is named `body`. Every +advertised as `query_`, a query name longer than 64 characters as +`query_` (its position) with the captured name sampled in its description, +and the write body as `request_body` when a query parameter is named `body`. Every schema closes with `additionalProperties: false`, a key the echo example's schema does not carry. `extensions`, `operations` and `children_environments` are fixed to `null`, diff --git a/docs/specs/agent-environments/environment-serve/spec.md b/docs/specs/agent-environments/environment-serve/spec.md index bd210e8ec6..8c9ab64be7 100644 --- a/docs/specs/agent-environments/environment-serve/spec.md +++ b/docs/specs/agent-environments/environment-serve/spec.md @@ -28,7 +28,8 @@ later increment. every path parameter as a required string described by its segment index, rule and observed values, every observed query name as an optional string, and an optional `body` object for `POST`, `PUT` and `PATCH` (a query name - that collides with a path parameter becomes `query_`, and the body + that collides with a path parameter becomes `query_`, a query name + longer than 64 characters becomes `query_`, and the body becomes `request_body` when a query parameter is named `body`), closed with `additionalProperties: false` so unknown arguments are refused; the description reads `METHOD template; replays retained records [ordinals]; @@ -48,7 +49,8 @@ later increment. `content_type`, `body` (parsed JSON for JSON-like content types that parse without changing a number, else text; bodies over 256 KiB cut at a UTF-8 boundary with `truncated: true`) and `replay` with `record_ordinal`, `evidence_id`, - `exact_match`, `recorded_path`, `recorded_query`, `body_recorded` (false + `exact_match`, `recorded_path` and `recorded_query` (each cut to 1 KiB, with + `recorded_line_truncated`; an explicit empty query is `?`), `body_recorded` (false when the capture kept no response body, so a `null` body is not mistaken for a recorded one), `body_encoding` (`base64` when the retained bytes are not UTF-8, so they are returned exactly rather than decoded lossily) and diff --git a/src/application/environment/environmentReplay.test.ts b/src/application/environment/environmentReplay.test.ts index 22a84f11a7..2eaba171b0 100644 --- a/src/application/environment/environmentReplay.test.ts +++ b/src/application/environment/environmentReplay.test.ts @@ -180,6 +180,30 @@ describe("replayEnvironmentTool selection", () => { ); }); + it("replays an aliased query argument against the captured long name", () => { + const longName = "n".repeat(100); + const search = { + ...tool("get_search"), + query_parameter_names: [longName], + record_ordinals: [901, 902], + }; + const exchange = (ordinal: number, value: string): RecordedExchange => ({ + ordinal, + path: "/api/search", + query: `?${longName}=${value}`, + status: 200, + contentType: null, + bodyText: null, + }); + const exchanges = new Map([ + [901, exchange(901, "a")], + [902, exchange(902, "b")], + ]); + expect( + selectReplayExchange(search, exchanges, { query_1: "b" }), + ).toMatchObject({ exchange: { ordinal: 902 }, exact: true }); + }); + it("prefers path matches over query matches and keeps manifest order on ties", () => { const search = tool("get_search"); const [first, second] = search.record_ordinals; @@ -404,3 +428,38 @@ describe("replayEnvironmentTool bodies", () => { }); }); }); + +describe("replayEnvironmentTool reply metadata", () => { + it("cuts a long recorded path or query in the reply and flags it, while matching on the full line", () => { + const users = tool("get_users_by_id"); + const ordinal = users.record_ordinals[0] ?? 0; + const long: EnvironmentSource = { + ...source, + exchanges: new Map([ + [ + ordinal, + { + ordinal, + path: `/api/users/2/${"p".repeat(4000)}`, + query: `?${"q".repeat(4000)}`, + status: 200, + contentType: null, + bodyText: null, + }, + ], + ]), + }; + const cut = replay("get_users_by_id", { id: "2" }, long).replay; + expect(cut).toMatchObject({ + exact_match: true, + recorded_line_truncated: true, + }); + expect(Buffer.byteLength(cut.recorded_path)).toBeLessThanOrEqual(1024); + expect(Buffer.byteLength(cut.recorded_query ?? "")).toBeLessThanOrEqual( + 1024, + ); + expect( + replay("get_users_by_id", { id: "2" }).replay.recorded_line_truncated, + ).toBe(false); + }); +}); diff --git a/src/application/environment/environmentReplay.ts b/src/application/environment/environmentReplay.ts index f5a515452a..33f1c38809 100644 --- a/src/application/environment/environmentReplay.ts +++ b/src/application/environment/environmentReplay.ts @@ -29,6 +29,8 @@ export interface ReplayResult { readonly exact_match: boolean; readonly recorded_path: string; readonly recorded_query: string | null; + /** Set when the recorded path or query was cut to REPLAY_METADATA_LIMIT_BYTES; the Evidence record keeps it whole. */ + readonly recorded_line_truncated: boolean; readonly body_recorded: boolean; readonly body_encoding: "utf-8" | "base64"; readonly truncated: boolean; @@ -193,6 +195,9 @@ const parseExactJson = (text: string): unknown => { } }; +/** Recorded paths and queries are cut to this many UTF-8 bytes in replies. */ +export const REPLAY_METADATA_LIMIT_BYTES = 1024; + /** Replay one tool call from the retained record that best matches its arguments. */ export const replayEnvironmentTool = ( source: EnvironmentSource, @@ -212,6 +217,11 @@ export const replayEnvironmentTool = ( ); const { exchange, exact } = selected; const { body, truncated } = replayBody(exchange); + const recordedPath = utf8Prefix(exchange.path, REPLAY_METADATA_LIMIT_BYTES); + const recordedQuery = + exchange.query === null + ? null + : utf8Prefix(exchange.query, REPLAY_METADATA_LIMIT_BYTES); return ok({ status: exchange.status, content_type: exchange.contentType, @@ -220,8 +230,11 @@ export const replayEnvironmentTool = ( record_ordinal: exchange.ordinal, evidence_id: source.evidenceId, exact_match: exact, - recorded_path: exchange.path, - recorded_query: exchange.query, + recorded_path: recordedPath, + recorded_query: recordedQuery, + recorded_line_truncated: + recordedPath.length < exchange.path.length || + (recordedQuery?.length ?? 0) < (exchange.query?.length ?? 0), body_recorded: exchange.bodyText !== null, body_encoding: exchange.bodyEncoding ?? "utf-8", truncated, diff --git a/src/application/environment/environmentSource.test.ts b/src/application/environment/environmentSource.test.ts index 9ca62c8dec..614a822e79 100644 --- a/src/application/environment/environmentSource.test.ts +++ b/src/application/environment/environmentSource.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it } from "vitest"; import { analysisViewCaptureEvidence, analysisViewCaptureFixture, + environmentManifestCaptureFixture, } from "../../../tests/fixtures/analysisView.js"; import { environmentServeFixture, @@ -163,3 +164,33 @@ describe("loadEnvironmentSource", () => { }); }); }); + +describe("loadEnvironmentSource recorded request lines", () => { + it("keeps the delimiter of an explicit empty query in the recorded exchange", () => { + const capture = environmentManifestCaptureFixture(); + const record = capture.records.find((entry) => + JSON.stringify(entry.reported).includes("/api/users/2"), + ); + if (record === undefined) throw new Error("Fixture lacks /api/users/2"); + const reported = structuredClone(record.reported) as { + request: { url: string }; + }; + reported.request.url = `${reported.request.url}?`; + record.reported = reported; + const evidence = analysisViewCaptureEvidence(capture); + const manifest = inspectAnalysisView({ + source: { kind: "inline", evidence }, + view: { kind: "environment-manifest" }, + }); + if (!manifest.ok) throw manifest.error; + const loaded = loadEnvironmentSource(manifest.value, evidence); + if (!loaded.ok) throw loaded.error; + const exchanges = [...loaded.value.exchanges.values()]; + expect( + exchanges.find((entry) => entry.path === "/api/users/2")?.query, + ).toBe("?"); + expect( + exchanges.find((entry) => entry.path === "/api/users/3")?.query, + ).toBeNull(); + }); +}); diff --git a/src/application/environment/environmentSource.ts b/src/application/environment/environmentSource.ts index 1f6260add1..ac2942121e 100644 --- a/src/application/environment/environmentSource.ts +++ b/src/application/environment/environmentSource.ts @@ -135,6 +135,29 @@ const recordedBody = ( } }; +/** WHATWG reports an explicit empty query (`/api?`) as "", like no query; the recorded URL keeps the delimiter. */ +const recordedQuery = ( + record: WebNetworkCaptureRecord, + query: string | undefined, +): string | null => { + if (query === undefined) return null; + if (query !== "") return query; + const reported = record.reported; + const request = + typeof reported === "object" && + reported !== null && + !Array.isArray(reported) + ? reported.request + : undefined; + const url = + typeof request === "object" && request !== null && !Array.isArray(request) + ? request.url + : undefined; + return typeof url === "string" && url.split("#", 1)[0]?.includes("?") === true + ? "?" + : null; +}; + const recordedExchange = ( record: WebNetworkCaptureRecord, ): RecordedExchange | undefined => { @@ -143,7 +166,7 @@ const recordedExchange = ( return { ordinal: record.ordinal, path: line.path, - query: line.query === undefined || line.query === "" ? null : line.query, + query: recordedQuery(record, line.query), status: line.status, contentType: line.rawMime === "" ? null : line.rawMime, ...recordedBody(record), diff --git a/src/application/environment/environmentTools.test.ts b/src/application/environment/environmentTools.test.ts index 284c253b0d..c1da454b0e 100644 --- a/src/application/environment/environmentTools.test.ts +++ b/src/application/environment/environmentTools.test.ts @@ -4,6 +4,7 @@ import { environmentServeSource } from "../../../tests/fixtures/environmentServe import type { EnvironmentManifestTool } from "./environmentSource.js"; import { environmentCard, + environmentToolArguments, environmentToolDefinitions, environmentToolInputSchema, REPLAY_CATALOG_INSTRUCTIONS, @@ -148,6 +149,23 @@ describe("environment catalog description budgets", () => { expect(definition?.tool.path_template).toBe(long.path_template); }); + it("advertises an oversized query name under a short, collision-safe alias", () => { + const longName = "n".repeat(100_000); + const search = { + ...tool("get_search"), + query_parameter_names: [longName, "query_1"], + }; + const args = environmentToolArguments(search); + expect(args.query.get(longName)).toBe("query_1"); + expect(args.query.get("query_1")).toBe("query_query_1"); + const schema = environmentToolInputSchema(search); + expect(Buffer.byteLength(JSON.stringify(schema))).toBeLessThan(4096); + expect(schema.properties.query_1?.description).toContain( + "… (full name in manifest Evidence)", + ); + expect(search.query_parameter_names[0]).toBe(longName); + }); + it("reports an omitted oversized value instead of publishing an unbounded or partial observation", () => { const users = tool("get_users_by_id"); const schema = environmentToolInputSchema({ diff --git a/src/application/environment/environmentTools.ts b/src/application/environment/environmentTools.ts index d0ef287d76..3f49b6270e 100644 --- a/src/application/environment/environmentTools.ts +++ b/src/application/environment/environmentTools.ts @@ -61,18 +61,21 @@ const describeList = ( : `${listed}${sample.length === 0 ? "" : ", "}… (${String(sample.length)}/${String(values.length)} shown; full list in manifest Evidence)`; }; -/** The first DESCRIPTION_LIST_SAMPLE_BYTES of a captured path, cut on a character boundary, with a pointer to the rest. */ -const describePath = (path: string): string => { +/** The first DESCRIPTION_LIST_SAMPLE_BYTES of captured text, cut on a character boundary, with a pointer to the rest. */ +const describeText = (text: string, what: string): string => { // encodeInto stops before the first character that does not fit, so `read` ends on a boundary. const { read } = new TextEncoder().encodeInto( - path, + text, new Uint8Array(DESCRIPTION_LIST_SAMPLE_BYTES), ); - return read === path.length - ? path - : `${path.slice(0, read)}… (full path in manifest Evidence)`; + return read === text.length + ? text + : `${text.slice(0, read)}… (full ${what} in manifest Evidence)`; }; +/** Captured query names longer than this are advertised under a short alias; the manifest keeps the name. */ +export const MAX_QUERY_ARGUMENT_CHARS = 64; + const bodyDescription = (tool: EnvironmentManifestTool): string => tool.body_shape === null ? "Request body; the capture retained no JSON body shape for this tool." @@ -107,8 +110,12 @@ export const environmentToolArguments = ( ): EnvironmentToolArguments => { const taken = new Set(tool.params.map((param) => param.name)); const query = new Map(); - for (const name of tool.query_parameter_names) { - const argument = freeName(taken, name, "query_"); + for (const [index, name] of tool.query_parameter_names.entries()) { + const preferred = + name.length <= MAX_QUERY_ARGUMENT_CHARS + ? name + : `query_${String(index + 1)}`; + const argument = freeName(taken, preferred, "query_"); taken.add(argument); query.set(name, argument); } @@ -135,7 +142,7 @@ export const environmentToolInputSchema = ( for (const [name, argument] of args.query) properties.set(argument, { type: "string", - description: `Query parameter ${name}; optional, matched against the recorded query on replay.`, + description: `Query parameter ${describeText(name, "name")}; optional, matched against the recorded query on replay.`, }); if (args.body !== undefined) properties.set(args.body, { @@ -157,7 +164,7 @@ export const environmentToolDefinitions = ( ): readonly EnvironmentToolDefinition[] => manifest.tools.map((tool) => ({ name: tool.name, - description: `${REPLAY_CATALOG_INSTRUCTIONS} ${tool.method} ${describePath(tool.path_template)}; replays retained records [${describeList(tool.record_ordinals)}]; evidence ${evidenceId}`, + description: `${REPLAY_CATALOG_INSTRUCTIONS} ${tool.method} ${describeText(tool.path_template, "path")}; replays retained records [${describeList(tool.record_ordinals)}]; evidence ${evidenceId}`, inputSchema: environmentToolInputSchema(tool), tool, }));