Files
reverse-engineering-anything/src/application/environment/environmentReplay.test.ts
T
Edgar ArakelyanandClaude Opus 5.5 9196cf7302 fix(environment): mark every replay result, warn about the captured catalog and bound base64 decoding
- A transport wrapper marks every tools/call result as target content,
  including the argument errors the SDK returns before a replay runs.
- The server's instructions add that tool names, descriptions, parameter
  names and observed values come from the capture too, so the catalog is
  treated as data like the replies.
- Strict UTF-8 decoding keeps a leading byte-order mark instead of
  dropping it from the recorded text.
- A large retained base64 body is decoded only as far as the 256 KiB a
  reply can carry, and the replay reports it truncated.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-18 20:58:38 -07:00

353 lines
11 KiB
TypeScript

import { describe, expect, it } from "vitest";
import {
analysisViewCaptureEvidence,
environmentManifestCaptureFixture,
} from "../../../tests/fixtures/analysisView.js";
import {
environmentServeFixture,
environmentServeSource,
} from "../../../tests/fixtures/environmentServe.js";
import { projectAnalysisError } from "../../domain/analysisErrorProjection.js";
import { inspectAnalysisView } from "../analysisView/AnalysisViewService.js";
import {
REPLAY_BODY_LIMIT_BYTES,
replayEnvironmentTool,
selectReplayExchange,
} from "./environmentReplay.js";
import type {
EnvironmentSource,
RecordedExchange,
} from "./environmentSource.js";
const source = environmentServeSource();
const tool = (name: string) => {
const found = source.manifest.tools.find((entry) => entry.name === name);
if (found === undefined) throw new Error(`Fixture manifest lacks ${name}`);
return found;
};
const replay = (
name: string,
args: Readonly<Record<string, unknown>>,
from: EnvironmentSource = source,
) => {
const result = replayEnvironmentTool(from, tool(name), args);
if (!result.ok) throw result.error;
return result.value;
};
/** A source whose only exchange carries the given body and content type. */
const singleExchangeSource = (
bodyText: string | null,
contentType: string | null,
): EnvironmentSource => {
const users = tool("get_users_by_id");
const ordinal = users.record_ordinals[0] ?? 0;
const exchange: RecordedExchange = {
ordinal,
path: "/api/users/2",
query: null,
status: 200,
contentType,
bodyText,
};
return { ...source, exchanges: new Map([[ordinal, exchange]]) };
};
/** The fixture source with /api/users/2's response replaced by HAR base64 content the decoder retained. */
const base64Source = (bytes: Buffer): EnvironmentSource => {
const capture = environmentManifestCaptureFixture();
const record = capture.records.find(
(entry) =>
typeof entry.reported === "object" &&
entry.reported !== null &&
"request" in entry.reported &&
JSON.stringify(entry.reported).includes("/api/users/2"),
);
if (record === undefined) throw new Error("Fixture lacks /api/users/2");
const reported = structuredClone(record.reported) as {
response: { content: { text: string; encoding?: string } };
};
reported.response.content.text = bytes.toString("base64");
reported.response.content.encoding = "base64";
record.reported = reported;
record.binary_fields = [
{
pointer: "/response/content/text",
representation: "har-base64-content",
state: "retained",
content_base64: bytes.toString("base64"),
bytes: bytes.length,
sha256: "0".repeat(64),
},
];
const evidence = analysisViewCaptureEvidence(capture);
const manifest = inspectAnalysisView({
source: { kind: "inline", evidence },
view: { kind: "environment-manifest" },
});
if (!manifest.ok) throw manifest.error;
expect(environmentServeFixture().manifest.evidence_id).not.toBe(
manifest.value.evidence_id,
);
return environmentServeSource({
manifest: manifest.value,
capture: evidence,
});
};
describe("replayEnvironmentTool selection", () => {
it("replays the record whose path segments equal the supplied parameters as an exact match", () => {
const result = replay("get_users_by_id", { id: "2" });
expect(result).toMatchObject({
status: 200,
content_type: "application/json",
body: { id: 2, login: "probe" },
replay: {
evidence_id: source.evidenceId,
exact_match: true,
recorded_path: "/api/users/2",
recorded_query: null,
body_recorded: true,
truncated: false,
},
});
expect(replay("get_users_by_id", { id: "3" })).toMatchObject({
body: { login: "viewer" },
replay: { exact_match: true, recorded_path: "/api/users/3" },
});
expect(
replay("get_folders_by_uid", { uid: "general" }).replay,
).toMatchObject({
exact_match: true,
recorded_path: "/api/folders/general",
});
});
it("falls back to the tool's first record, flagged inexact, when no record matches", () => {
const users = tool("get_users_by_id");
const result = replay("get_users_by_id", { id: "999" });
expect(result.replay).toMatchObject({
record_ordinal: users.record_ordinals[0],
exact_match: false,
recorded_path: "/api/users/2",
});
expect(result.body).toMatchObject({ login: "probe" });
// Omitted parameters cannot mismatch: a call without arguments is exact on the first record.
expect(replay("get_users_by_id", {}).replay.exact_match).toBe(true);
});
it("matches query values after path segments and reports the recorded query", () => {
const both = replay("get_search", { query: "probe", type: "dash-db" });
expect(both.replay).toMatchObject({
exact_match: true,
recorded_path: "/api/search",
recorded_query: "?query=probe&type=dash-db",
});
expect(Array.isArray(both.body)).toBe(true);
expect(both.body).toMatchObject([{ title: "probe dashboard" }]);
const folder = replay("get_search", { type: "dash-folder" });
expect(folder.replay).toMatchObject({
exact_match: true,
recorded_query: "?query=&type=dash-folder",
});
const unmatched = replay("get_search", { type: "dash-nothing" });
expect(unmatched.replay.exact_match).toBe(false);
expect(unmatched.replay.recorded_query).toBe("?query=probe&type=dash-db");
});
it("prefers path matches over query matches and keeps manifest order on ties", () => {
const search = tool("get_search");
const [first, second] = search.record_ordinals;
if (first === undefined || second === undefined)
throw new Error("Fixture search tool must cite two records");
const exchanges = new Map<number, RecordedExchange>([
[
first,
{
ordinal: first,
path: "/api/search",
query: "?type=a",
status: 200,
contentType: null,
bodyText: null,
},
],
[
second,
{
ordinal: second,
path: "/api/search",
query: "?type=a",
status: 200,
contentType: null,
bodyText: null,
},
],
]);
expect(
selectReplayExchange(search, exchanges, { type: "a" }),
).toMatchObject({ exchange: { ordinal: first }, exact: true });
expect(
selectReplayExchange(search, new Map(), { type: "a" }),
).toBeUndefined();
const refused = replayEnvironmentTool(
{ ...source, exchanges: new Map() },
search,
{},
);
if (refused.ok) throw new Error("Expected a refusal without records");
expect(projectAnalysisError(refused.error)).toMatchObject({
code: "invalid_request",
details: { issues: [{ path: ["tool"], expected: "get_search" }] },
});
});
});
describe("replayEnvironmentTool argument names", () => {
it("matches a query value whose name collides with a path parameter under its renamed argument", () => {
const users = tool("get_users_by_id");
const exchange = (ordinal: number, query: string): RecordedExchange => ({
ordinal,
path: "/api/users/2",
query,
status: 200,
contentType: "application/json",
bodyText: JSON.stringify({ query }),
});
const colliding = {
...users,
query_parameter_names: ["id"],
record_ordinals: [901, 902],
};
const from: EnvironmentSource = {
...source,
exchanges: new Map([
[901, exchange(901, "id=3")],
[902, exchange(902, "id=4")],
]),
};
const result = replayEnvironmentTool(from, colliding, {
id: "2",
query_id: "4",
});
if (!result.ok) throw result.error;
expect(result.value.replay).toMatchObject({
record_ordinal: 902,
recorded_query: "id=4",
exact_match: true,
});
});
});
describe("replayEnvironmentTool bodies", () => {
it("reports a response whose body the capture did not retain instead of passing off null as recorded", () => {
const result = replay(
"get_users_by_id",
{ id: "2" },
singleExchangeSource(null, "application/json"),
);
expect(result.body).toBeNull();
expect(result.replay).toMatchObject({
body_recorded: false,
truncated: false,
});
});
it("returns text for non-JSON content types and for JSON-like content that does not parse", () => {
expect(
replay(
"get_users_by_id",
{ id: "2" },
singleExchangeSource("<p>", "text/html"),
),
).toMatchObject({ body: "<p>", content_type: "text/html" });
expect(
replay(
"get_users_by_id",
{ id: "2" },
singleExchangeSource("{not json", "application/json; charset=utf-8"),
).body,
).toBe("{not json");
expect(
replay(
"get_users_by_id",
{ id: "2" },
singleExchangeSource('{"ok":true}', "application/vnd.api+json"),
).body,
).toEqual({ ok: true });
expect(
replay("get_users_by_id", { id: "2" }, singleExchangeSource(null, null)),
).toMatchObject({ body: null, content_type: null });
});
it("truncates bodies over 256 KiB at a UTF-8 boundary and flags them", () => {
const text = "é".repeat(REPLAY_BODY_LIMIT_BYTES);
const result = replay(
"get_users_by_id",
{ id: "2" },
singleExchangeSource(text, "application/json"),
);
expect(result.replay.truncated).toBe(true);
expect(typeof result.body).toBe("string");
const body = result.body as string;
expect(Buffer.byteLength(body, "utf8")).toBe(REPLAY_BODY_LIMIT_BYTES);
expect(body).not.toContain("�");
const exact = "x".repeat(REPLAY_BODY_LIMIT_BYTES);
expect(
replay(
"get_users_by_id",
{ id: "2" },
singleExchangeSource(exact, "text/plain"),
),
).toMatchObject({ body: exact, replay: { truncated: false } });
});
it("decodes retained HAR base64 bytes that are UTF-8 and keeps other bytes as base64", () => {
const decoded = base64Source(Buffer.from('{"login":"decoded"}', "utf8"));
expect(replay("get_users_by_id", { id: "2" }, decoded)).toMatchObject({
body: { login: "decoded" },
replay: { body_encoding: "utf-8" },
});
const binary = Buffer.from([0xff, 0xfe, 0x00, 0x41]);
expect(
replay("get_users_by_id", { id: "2" }, base64Source(binary)),
).toMatchObject({
body: binary.toString("base64"),
replay: { body_encoding: "base64", body_recorded: true },
});
// A leading byte-order mark is recorded content, not decoding noise.
expect(
replay(
"get_users_by_id",
{ id: "2" },
base64Source(Buffer.from([0xef, 0xbb, 0xbf, 0x41])),
).body,
).toBe("\uFEFFA");
});
it("decodes only the replayable prefix of a large retained base64 body and flags it truncated", () => {
for (const [fill, encoding] of [
[0x61, "utf-8"],
[0xff, "base64"],
] as const) {
const large = Buffer.alloc(REPLAY_BODY_LIMIT_BYTES * 2, fill);
const result = replay(
"get_users_by_id",
{ id: "2" },
base64Source(large),
);
expect(result.replay).toMatchObject({
truncated: true,
body_encoding: encoding,
});
expect(typeof result.body).toBe("string");
expect(
Buffer.byteLength(result.body as string, "utf8"),
).toBeLessThanOrEqual(REPLAY_BODY_LIMIT_BYTES);
}
});
});