test(web): own verifier commands and cover filtered continuation

This commit is contained in:
gensokyo-dev committed 2026-10-26 17:20:37 +00:00
1 parent dd853c5c7a
commit 8f3203b1c3
4 files changed
+209 -146

No files matched your search

@@ -53,6 +53,15 @@ limitation union before display bounding, and distinct script-scoped unknowns.
These three corrections fail against the imported upstream production code;
explicit-null and explicit-origin controls continue to pass.
Automated review follow-up added an interleaved, intersecting-filter continuation
case: the second matching offset is `2`, while the joined original indices are
`2, 4, 5`. All 24 projection tests and both SDK boundary tests pass. Test temporary
directories now have the existing fixture as their sole cleanup owner. Both real
CLI launches in the verifier use `createVerifierCommands` and awaited shared
cleanup; real selected capture and both CLI collection readbacks pass after that
change. Command supervision is acquired lazily only for capture or explicit CLI
verification, outside the measured retained-query workflows above.
## Equivalent completed query costs
Both versions import the same already completed parent, discover the same 41
@@ -5,9 +5,10 @@ import { join, resolve } from "node:path";
import { tmpdir } from "node:os";
import { createRequire } from "node:module";
import { pathToFileURL } from "node:url";
import { execFile } from "node:child_process";
import { promisify, parseArgs } from "node:util";
import { parseArgs } from "node:util";
import { performance } from "node:perf_hooks";
import { createVerifierCommands } from "../../scripts/lib/verify-commands.mjs";
import { withVerifierCleanup } from "../../scripts/lib/verifier-cleanup.mjs";
// Public CLI capture and same-session retained MCP queries. All assets and output
// paths are explicitly supplied; this verifier installs no browser or application.
const { values } = parseArgs({
@@ -46,6 +47,10 @@ const env = {
REA_MCP_INPUT_SCHEMA_PROFILE: "compact",
REA_MCP_TOOL_FAMILIES: "application,session",
};
const commands =
mode === "capture" || values["verify-cli"]
? await createVerifierCommands()
: undefined;
if (mode === "capture") {
const { chromium } = require("playwright-core");
const assets = resolve(values.application);
@@ -82,7 +87,7 @@ if (mode === "capture") {
const origin = `http://127.0.0.1:${server.address().port}`;
const userData = await mkdtemp(join(tmpdir(), "rea-6198-chrome-"));
let context;
try {
await withVerifierCleanup(async () => {
context = await chromium.launchPersistentContext(userData, {
executablePath: resolve(values.browser),
headless: true,
@@ -105,7 +110,7 @@ if (mode === "capture") {
);
assert.ok(target);
const started = performance.now();
const { stdout, stderr } = await promisify(execFile)(
const { stdout, stderr } = await commands.exec(
process.execPath,
[
join(repo, "scripts/rea.mjs"),
@@ -168,11 +173,17 @@ if (mode === "capture") {
completeness: evidence.normalized_result.completeness,
}),
);
} finally {
await context?.close();
await new Promise((r) => server.close(r));
await rm(userData, { recursive: true, force: true });
}
}, [
() => commands.close(),
async () => {
await context?.close();
await rm(userData, { recursive: true, force: true });
},
() =>
new Promise((done, reject) =>
server.close((error) => (error ? reject(error) : done())),
),
]);
process.exit(0);
}
const parent = JSON.parse(await readFile(bundle, "utf8"));
@@ -213,7 +224,7 @@ async function invoke(name, args) {
assert.notEqual(result.isError, true, JSON.stringify(result));
return result.structuredContent;
}
try {
await withVerifierCleanup(async () => {
await client.connect(transport);
catalog = await client.listTools();
await writeFile(join(out, "catalog.json"), JSON.stringify(catalog));
@@ -322,7 +333,7 @@ try {
limit: 32,
},
});
const { stdout } = await promisify(execFile)(
const { stdout } = await commands.exec(
process.execPath,
[
join(repo, "scripts/rea.mjs"),
@@ -368,6 +379,4 @@ try {
),
);
console.log(JSON.stringify({ mode, task: values.task, runs }));
} finally {
await client.close();
}
}, [() => commands?.close(), () => client.close()]);
@@ -495,6 +495,56 @@ it("rejects non-web-bundle parent operation with invalid_value", () => {
expect(result.error.issues[0]!.reason).toBe("invalid_value");
});
it("continues filtered pages by matching ordinal while preserving original indices", () => {
const report = sampleAnalysis();
const row = report.observations.endpoints[0]!;
const endpoints = [
{ ...row, mechanism: "call:sendBeacon" },
{ ...row, location: { ...row.location, script_key: "vendor.js" } },
row,
{ ...row, value: "/api/orders" },
{ ...row, location: { ...row.location, line: 99 } },
{ ...row, location: { ...row.location, line: 100 } },
{ ...row, location: { ...row.location, script_key: "vendor.js" } },
];
const parent = makeParent({
observations: { ...report.observations, endpoints },
});
const selection = {
kind: "web-bundle",
collection: "endpoints",
script_keys: ["app.js"],
values: ["/api/users"],
mechanisms: ["call:fetch"],
limit: 2,
};
const first = query(parent, selection);
if (first.kind !== "web-bundle") throw new Error("Expected web-bundle page");
expect(first.coverage).toMatchObject({
total: 3,
examined: 2,
next_offset: 2,
exhausted: false,
});
const second = query(parent, {
...selection,
offset: first.coverage.next_offset,
});
if (second.kind !== "web-bundle") throw new Error("Expected web-bundle page");
expect(second.coverage).toMatchObject({
total: 3,
examined: 1,
next_offset: null,
exhausted: true,
});
expect([...first.items, ...second.items]).toEqual(
[2, 4, 5].map((source_index) => ({
source_index,
...endpoints[source_index],
})),
);
});
it("preserves parent artifact identity across summary and page views", () => {
const artifact = { path: "/app.js", sha256: "e".repeat(64) };
const parent: AnalysisViewParent = {
+127 -132
View File
@@ -1,5 +1,5 @@
import { Client, InMemoryTransport } from "@modelcontextprotocol/client";
import { writeFile, unlink, rm } from "node:fs/promises";
import { writeFile } from "node:fs/promises";
import { join } from "node:path";
import { expect, it, onTestFinished } from "vitest";
import { createServer } from "../../../src/server/createServer.js";
@@ -215,139 +215,134 @@ it("pages web bundle endpoints and inferences over MCP using evidence-file sourc
const evidenceFile = join(tempDir, "webbundle-evidence.json");
await writeFile(evidenceFile, JSON.stringify(parent));
try {
const page1Call = await client.callTool({
...schemas.toolDefinition,
arguments: {
source: { kind: "evidence-file", path: evidenceFile },
view: {
kind: "web-bundle",
collection: "endpoints",
offset: 0,
limit: 2,
},
const page1Call = await client.callTool({
...schemas.toolDefinition,
arguments: {
source: { kind: "evidence-file", path: evidenceFile },
view: {
kind: "web-bundle",
collection: "endpoints",
offset: 0,
limit: 2,
},
});
expect(page1Call.isError).not.toBe(true);
expect(
schemas.ajv.validate(schemas.outputSchema, page1Call.structuredContent),
JSON.stringify(schemas.ajv.errors),
).toBe(true);
const page1 = webBundlePageViewResultSchema.parse(
parseEvidence(page1Call.structuredContent).normalized_result,
);
expect(page1.items).toHaveLength(2);
expect(page1.items[0]).toMatchObject({
},
});
expect(page1Call.isError).not.toBe(true);
expect(
schemas.ajv.validate(schemas.outputSchema, page1Call.structuredContent),
JSON.stringify(schemas.ajv.errors),
).toBe(true);
const page1 = webBundlePageViewResultSchema.parse(
parseEvidence(page1Call.structuredContent).normalized_result,
);
expect(page1.items).toHaveLength(2);
expect(page1.items[0]).toMatchObject({
source_index: 0,
value: "/api/users",
});
expect(page1.items[1]).toMatchObject({
source_index: 1,
value: "/api/users",
});
expect(page1.coverage).toMatchObject({
status: "page",
examined: 2,
total: 3,
next_offset: 2,
exhausted: false,
});
expect(page1.completeness.status).toBe("partial");
const page2Call = await client.callTool({
...schemas.toolDefinition,
arguments: {
source: { kind: "evidence-file", path: evidenceFile },
view: {
kind: "web-bundle",
collection: "endpoints",
offset: 2,
limit: 2,
},
},
});
expect(page2Call.isError).not.toBe(true);
expect(
schemas.ajv.validate(schemas.outputSchema, page2Call.structuredContent),
JSON.stringify(schemas.ajv.errors),
).toBe(true);
const page2 = webBundlePageViewResultSchema.parse(
parseEvidence(page2Call.structuredContent).normalized_result,
);
expect(page2.items).toHaveLength(1);
expect(page2.items[0]).toMatchObject({
source_index: 2,
value: "/api/orders",
});
expect(page2.coverage).toMatchObject({
status: "page",
examined: 1,
total: 3,
next_offset: null,
exhausted: true,
});
const candidatesCall = await client.callTool({
...schemas.toolDefinition,
arguments: {
source: { kind: "evidence-file", path: evidenceFile },
view: {
kind: "web-bundle",
collection: "inferences",
kinds: ["endpoint_candidate"],
},
},
});
expect(candidatesCall.isError).not.toBe(true);
expect(
schemas.ajv.validate(
schemas.outputSchema,
candidatesCall.structuredContent,
),
JSON.stringify(schemas.ajv.errors),
).toBe(true);
const candidatesPage = webBundlePageViewResultSchema.parse(
parseEvidence(candidatesCall.structuredContent).normalized_result,
);
expect(candidatesPage.items).toEqual([
expect.objectContaining({
source_index: 0,
value: "/api/users",
});
expect(page1.items[1]).toMatchObject({
source_index: 1,
value: "/api/users",
});
expect(page1.coverage).toMatchObject({
status: "page",
examined: 2,
total: 3,
next_offset: 2,
exhausted: false,
});
expect(page1.completeness.status).toBe("partial");
kind: "endpoint_candidate",
value: "/api/dynamic-user",
confidence: "low",
}),
]);
const page2Call = await client.callTool({
...schemas.toolDefinition,
arguments: {
source: { kind: "evidence-file", path: evidenceFile },
view: {
kind: "web-bundle",
collection: "endpoints",
offset: 2,
limit: 2,
},
const emptyCall = await client.callTool({
...schemas.toolDefinition,
arguments: {
source: { kind: "evidence-file", path: evidenceFile },
view: {
kind: "web-bundle",
collection: "endpoints",
offset: 100,
limit: 10,
},
});
expect(page2Call.isError).not.toBe(true);
expect(
schemas.ajv.validate(schemas.outputSchema, page2Call.structuredContent),
JSON.stringify(schemas.ajv.errors),
).toBe(true);
const page2 = webBundlePageViewResultSchema.parse(
parseEvidence(page2Call.structuredContent).normalized_result,
);
expect(page2.items).toHaveLength(1);
expect(page2.items[0]).toMatchObject({
source_index: 2,
value: "/api/orders",
});
expect(page2.coverage).toMatchObject({
status: "page",
examined: 1,
total: 3,
next_offset: null,
exhausted: true,
});
const candidatesCall = await client.callTool({
...schemas.toolDefinition,
arguments: {
source: { kind: "evidence-file", path: evidenceFile },
view: {
kind: "web-bundle",
collection: "inferences",
kinds: ["endpoint_candidate"],
},
},
});
expect(candidatesCall.isError).not.toBe(true);
expect(
schemas.ajv.validate(
schemas.outputSchema,
candidatesCall.structuredContent,
),
JSON.stringify(schemas.ajv.errors),
).toBe(true);
const candidatesPage = webBundlePageViewResultSchema.parse(
parseEvidence(candidatesCall.structuredContent).normalized_result,
);
expect(candidatesPage.items).toEqual([
expect.objectContaining({
source_index: 0,
kind: "endpoint_candidate",
value: "/api/dynamic-user",
confidence: "low",
}),
]);
const emptyCall = await client.callTool({
...schemas.toolDefinition,
arguments: {
source: { kind: "evidence-file", path: evidenceFile },
view: {
kind: "web-bundle",
collection: "endpoints",
offset: 100,
limit: 10,
},
},
});
expect(emptyCall.isError).not.toBe(true);
expect(
schemas.ajv.validate(schemas.outputSchema, emptyCall.structuredContent),
JSON.stringify(schemas.ajv.errors),
).toBe(true);
const emptyPage = webBundlePageViewResultSchema.parse(
parseEvidence(emptyCall.structuredContent).normalized_result,
);
expect(emptyPage.items).toEqual([]);
expect(emptyPage.coverage).toMatchObject({
status: "empty",
examined: 0,
total: 3,
next_offset: null,
exhausted: true,
});
} finally {
await unlink(evidenceFile).catch(() => {});
await rm(tempDir, { recursive: true, force: true }).catch(() => {});
}
},
});
expect(emptyCall.isError).not.toBe(true);
expect(
schemas.ajv.validate(schemas.outputSchema, emptyCall.structuredContent),
JSON.stringify(schemas.ajv.errors),
).toBe(true);
const emptyPage = webBundlePageViewResultSchema.parse(
parseEvidence(emptyCall.structuredContent).normalized_result,
);
expect(emptyPage.items).toEqual([]);
expect(emptyPage.coverage).toMatchObject({
status: "empty",
examined: 0,
total: 3,
next_offset: null,
exhausted: true,
});
});