mirror of
https://github.com/morluto/rea.git
synced 2026-10-26 17:35:11 +00:00
test(web): own verifier commands and cover filtered continuation
This commit is contained in:
1 parent
dd853c5c7a
commit
8f3203b1c3
4 files changed
+209
-146
No files matched your search
@@ -53,6 +53,15 @@ limitation union before display bounding, and distinct script-scoped unknowns.
|
|||||||
These three corrections fail against the imported upstream production code;
|
These three corrections fail against the imported upstream production code;
|
||||||
explicit-null and explicit-origin controls continue to pass.
|
explicit-null and explicit-origin controls continue to pass.
|
||||||
|
|
||||||
|
Automated review follow-up added an interleaved, intersecting-filter continuation
|
||||||
|
case: the second matching offset is `2`, while the joined original indices are
|
||||||
|
`2, 4, 5`. All 24 projection tests and both SDK boundary tests pass. Test temporary
|
||||||
|
directories now have the existing fixture as their sole cleanup owner. Both real
|
||||||
|
CLI launches in the verifier use `createVerifierCommands` and awaited shared
|
||||||
|
cleanup; real selected capture and both CLI collection readbacks pass after that
|
||||||
|
change. Command supervision is acquired lazily only for capture or explicit CLI
|
||||||
|
verification, outside the measured retained-query workflows above.
|
||||||
|
|
||||||
## Equivalent completed query costs
|
## Equivalent completed query costs
|
||||||
|
|
||||||
Both versions import the same already completed parent, discover the same 41
|
Both versions import the same already completed parent, discover the same 41
|
||||||
|
|||||||
@@ -5,9 +5,10 @@ import { join, resolve } from "node:path";
|
|||||||
import { tmpdir } from "node:os";
|
import { tmpdir } from "node:os";
|
||||||
import { createRequire } from "node:module";
|
import { createRequire } from "node:module";
|
||||||
import { pathToFileURL } from "node:url";
|
import { pathToFileURL } from "node:url";
|
||||||
import { execFile } from "node:child_process";
|
import { parseArgs } from "node:util";
|
||||||
import { promisify, parseArgs } from "node:util";
|
|
||||||
import { performance } from "node:perf_hooks";
|
import { performance } from "node:perf_hooks";
|
||||||
|
import { createVerifierCommands } from "../../scripts/lib/verify-commands.mjs";
|
||||||
|
import { withVerifierCleanup } from "../../scripts/lib/verifier-cleanup.mjs";
|
||||||
// Public CLI capture and same-session retained MCP queries. All assets and output
|
// Public CLI capture and same-session retained MCP queries. All assets and output
|
||||||
// paths are explicitly supplied; this verifier installs no browser or application.
|
// paths are explicitly supplied; this verifier installs no browser or application.
|
||||||
const { values } = parseArgs({
|
const { values } = parseArgs({
|
||||||
@@ -46,6 +47,10 @@ const env = {
|
|||||||
REA_MCP_INPUT_SCHEMA_PROFILE: "compact",
|
REA_MCP_INPUT_SCHEMA_PROFILE: "compact",
|
||||||
REA_MCP_TOOL_FAMILIES: "application,session",
|
REA_MCP_TOOL_FAMILIES: "application,session",
|
||||||
};
|
};
|
||||||
|
const commands =
|
||||||
|
mode === "capture" || values["verify-cli"]
|
||||||
|
? await createVerifierCommands()
|
||||||
|
: undefined;
|
||||||
if (mode === "capture") {
|
if (mode === "capture") {
|
||||||
const { chromium } = require("playwright-core");
|
const { chromium } = require("playwright-core");
|
||||||
const assets = resolve(values.application);
|
const assets = resolve(values.application);
|
||||||
@@ -82,7 +87,7 @@ if (mode === "capture") {
|
|||||||
const origin = `http://127.0.0.1:${server.address().port}`;
|
const origin = `http://127.0.0.1:${server.address().port}`;
|
||||||
const userData = await mkdtemp(join(tmpdir(), "rea-6198-chrome-"));
|
const userData = await mkdtemp(join(tmpdir(), "rea-6198-chrome-"));
|
||||||
let context;
|
let context;
|
||||||
try {
|
await withVerifierCleanup(async () => {
|
||||||
context = await chromium.launchPersistentContext(userData, {
|
context = await chromium.launchPersistentContext(userData, {
|
||||||
executablePath: resolve(values.browser),
|
executablePath: resolve(values.browser),
|
||||||
headless: true,
|
headless: true,
|
||||||
@@ -105,7 +110,7 @@ if (mode === "capture") {
|
|||||||
);
|
);
|
||||||
assert.ok(target);
|
assert.ok(target);
|
||||||
const started = performance.now();
|
const started = performance.now();
|
||||||
const { stdout, stderr } = await promisify(execFile)(
|
const { stdout, stderr } = await commands.exec(
|
||||||
process.execPath,
|
process.execPath,
|
||||||
[
|
[
|
||||||
join(repo, "scripts/rea.mjs"),
|
join(repo, "scripts/rea.mjs"),
|
||||||
@@ -168,11 +173,17 @@ if (mode === "capture") {
|
|||||||
completeness: evidence.normalized_result.completeness,
|
completeness: evidence.normalized_result.completeness,
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
} finally {
|
}, [
|
||||||
await context?.close();
|
() => commands.close(),
|
||||||
await new Promise((r) => server.close(r));
|
async () => {
|
||||||
await rm(userData, { recursive: true, force: true });
|
await context?.close();
|
||||||
}
|
await rm(userData, { recursive: true, force: true });
|
||||||
|
},
|
||||||
|
() =>
|
||||||
|
new Promise((done, reject) =>
|
||||||
|
server.close((error) => (error ? reject(error) : done())),
|
||||||
|
),
|
||||||
|
]);
|
||||||
process.exit(0);
|
process.exit(0);
|
||||||
}
|
}
|
||||||
const parent = JSON.parse(await readFile(bundle, "utf8"));
|
const parent = JSON.parse(await readFile(bundle, "utf8"));
|
||||||
@@ -213,7 +224,7 @@ async function invoke(name, args) {
|
|||||||
assert.notEqual(result.isError, true, JSON.stringify(result));
|
assert.notEqual(result.isError, true, JSON.stringify(result));
|
||||||
return result.structuredContent;
|
return result.structuredContent;
|
||||||
}
|
}
|
||||||
try {
|
await withVerifierCleanup(async () => {
|
||||||
await client.connect(transport);
|
await client.connect(transport);
|
||||||
catalog = await client.listTools();
|
catalog = await client.listTools();
|
||||||
await writeFile(join(out, "catalog.json"), JSON.stringify(catalog));
|
await writeFile(join(out, "catalog.json"), JSON.stringify(catalog));
|
||||||
@@ -322,7 +333,7 @@ try {
|
|||||||
limit: 32,
|
limit: 32,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
const { stdout } = await promisify(execFile)(
|
const { stdout } = await commands.exec(
|
||||||
process.execPath,
|
process.execPath,
|
||||||
[
|
[
|
||||||
join(repo, "scripts/rea.mjs"),
|
join(repo, "scripts/rea.mjs"),
|
||||||
@@ -368,6 +379,4 @@ try {
|
|||||||
),
|
),
|
||||||
);
|
);
|
||||||
console.log(JSON.stringify({ mode, task: values.task, runs }));
|
console.log(JSON.stringify({ mode, task: values.task, runs }));
|
||||||
} finally {
|
}, [() => commands?.close(), () => client.close()]);
|
||||||
await client.close();
|
|
||||||
}
|
|
||||||
@@ -495,6 +495,56 @@ it("rejects non-web-bundle parent operation with invalid_value", () => {
|
|||||||
expect(result.error.issues[0]!.reason).toBe("invalid_value");
|
expect(result.error.issues[0]!.reason).toBe("invalid_value");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("continues filtered pages by matching ordinal while preserving original indices", () => {
|
||||||
|
const report = sampleAnalysis();
|
||||||
|
const row = report.observations.endpoints[0]!;
|
||||||
|
const endpoints = [
|
||||||
|
{ ...row, mechanism: "call:sendBeacon" },
|
||||||
|
{ ...row, location: { ...row.location, script_key: "vendor.js" } },
|
||||||
|
row,
|
||||||
|
{ ...row, value: "/api/orders" },
|
||||||
|
{ ...row, location: { ...row.location, line: 99 } },
|
||||||
|
{ ...row, location: { ...row.location, line: 100 } },
|
||||||
|
{ ...row, location: { ...row.location, script_key: "vendor.js" } },
|
||||||
|
];
|
||||||
|
const parent = makeParent({
|
||||||
|
observations: { ...report.observations, endpoints },
|
||||||
|
});
|
||||||
|
const selection = {
|
||||||
|
kind: "web-bundle",
|
||||||
|
collection: "endpoints",
|
||||||
|
script_keys: ["app.js"],
|
||||||
|
values: ["/api/users"],
|
||||||
|
mechanisms: ["call:fetch"],
|
||||||
|
limit: 2,
|
||||||
|
};
|
||||||
|
const first = query(parent, selection);
|
||||||
|
if (first.kind !== "web-bundle") throw new Error("Expected web-bundle page");
|
||||||
|
expect(first.coverage).toMatchObject({
|
||||||
|
total: 3,
|
||||||
|
examined: 2,
|
||||||
|
next_offset: 2,
|
||||||
|
exhausted: false,
|
||||||
|
});
|
||||||
|
const second = query(parent, {
|
||||||
|
...selection,
|
||||||
|
offset: first.coverage.next_offset,
|
||||||
|
});
|
||||||
|
if (second.kind !== "web-bundle") throw new Error("Expected web-bundle page");
|
||||||
|
expect(second.coverage).toMatchObject({
|
||||||
|
total: 3,
|
||||||
|
examined: 1,
|
||||||
|
next_offset: null,
|
||||||
|
exhausted: true,
|
||||||
|
});
|
||||||
|
expect([...first.items, ...second.items]).toEqual(
|
||||||
|
[2, 4, 5].map((source_index) => ({
|
||||||
|
source_index,
|
||||||
|
...endpoints[source_index],
|
||||||
|
})),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
it("preserves parent artifact identity across summary and page views", () => {
|
it("preserves parent artifact identity across summary and page views", () => {
|
||||||
const artifact = { path: "/app.js", sha256: "e".repeat(64) };
|
const artifact = { path: "/app.js", sha256: "e".repeat(64) };
|
||||||
const parent: AnalysisViewParent = {
|
const parent: AnalysisViewParent = {
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { Client, InMemoryTransport } from "@modelcontextprotocol/client";
|
import { Client, InMemoryTransport } from "@modelcontextprotocol/client";
|
||||||
import { writeFile, unlink, rm } from "node:fs/promises";
|
import { writeFile } from "node:fs/promises";
|
||||||
import { join } from "node:path";
|
import { join } from "node:path";
|
||||||
import { expect, it, onTestFinished } from "vitest";
|
import { expect, it, onTestFinished } from "vitest";
|
||||||
import { createServer } from "../../../src/server/createServer.js";
|
import { createServer } from "../../../src/server/createServer.js";
|
||||||
@@ -215,139 +215,134 @@ it("pages web bundle endpoints and inferences over MCP using evidence-file sourc
|
|||||||
const evidenceFile = join(tempDir, "webbundle-evidence.json");
|
const evidenceFile = join(tempDir, "webbundle-evidence.json");
|
||||||
await writeFile(evidenceFile, JSON.stringify(parent));
|
await writeFile(evidenceFile, JSON.stringify(parent));
|
||||||
|
|
||||||
try {
|
const page1Call = await client.callTool({
|
||||||
const page1Call = await client.callTool({
|
...schemas.toolDefinition,
|
||||||
...schemas.toolDefinition,
|
arguments: {
|
||||||
arguments: {
|
source: { kind: "evidence-file", path: evidenceFile },
|
||||||
source: { kind: "evidence-file", path: evidenceFile },
|
view: {
|
||||||
view: {
|
kind: "web-bundle",
|
||||||
kind: "web-bundle",
|
collection: "endpoints",
|
||||||
collection: "endpoints",
|
offset: 0,
|
||||||
offset: 0,
|
limit: 2,
|
||||||
limit: 2,
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
});
|
},
|
||||||
expect(page1Call.isError).not.toBe(true);
|
});
|
||||||
expect(
|
expect(page1Call.isError).not.toBe(true);
|
||||||
schemas.ajv.validate(schemas.outputSchema, page1Call.structuredContent),
|
expect(
|
||||||
JSON.stringify(schemas.ajv.errors),
|
schemas.ajv.validate(schemas.outputSchema, page1Call.structuredContent),
|
||||||
).toBe(true);
|
JSON.stringify(schemas.ajv.errors),
|
||||||
const page1 = webBundlePageViewResultSchema.parse(
|
).toBe(true);
|
||||||
parseEvidence(page1Call.structuredContent).normalized_result,
|
const page1 = webBundlePageViewResultSchema.parse(
|
||||||
);
|
parseEvidence(page1Call.structuredContent).normalized_result,
|
||||||
expect(page1.items).toHaveLength(2);
|
);
|
||||||
expect(page1.items[0]).toMatchObject({
|
expect(page1.items).toHaveLength(2);
|
||||||
|
expect(page1.items[0]).toMatchObject({
|
||||||
|
source_index: 0,
|
||||||
|
value: "/api/users",
|
||||||
|
});
|
||||||
|
expect(page1.items[1]).toMatchObject({
|
||||||
|
source_index: 1,
|
||||||
|
value: "/api/users",
|
||||||
|
});
|
||||||
|
expect(page1.coverage).toMatchObject({
|
||||||
|
status: "page",
|
||||||
|
examined: 2,
|
||||||
|
total: 3,
|
||||||
|
next_offset: 2,
|
||||||
|
exhausted: false,
|
||||||
|
});
|
||||||
|
expect(page1.completeness.status).toBe("partial");
|
||||||
|
|
||||||
|
const page2Call = await client.callTool({
|
||||||
|
...schemas.toolDefinition,
|
||||||
|
arguments: {
|
||||||
|
source: { kind: "evidence-file", path: evidenceFile },
|
||||||
|
view: {
|
||||||
|
kind: "web-bundle",
|
||||||
|
collection: "endpoints",
|
||||||
|
offset: 2,
|
||||||
|
limit: 2,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
expect(page2Call.isError).not.toBe(true);
|
||||||
|
expect(
|
||||||
|
schemas.ajv.validate(schemas.outputSchema, page2Call.structuredContent),
|
||||||
|
JSON.stringify(schemas.ajv.errors),
|
||||||
|
).toBe(true);
|
||||||
|
const page2 = webBundlePageViewResultSchema.parse(
|
||||||
|
parseEvidence(page2Call.structuredContent).normalized_result,
|
||||||
|
);
|
||||||
|
expect(page2.items).toHaveLength(1);
|
||||||
|
expect(page2.items[0]).toMatchObject({
|
||||||
|
source_index: 2,
|
||||||
|
value: "/api/orders",
|
||||||
|
});
|
||||||
|
expect(page2.coverage).toMatchObject({
|
||||||
|
status: "page",
|
||||||
|
examined: 1,
|
||||||
|
total: 3,
|
||||||
|
next_offset: null,
|
||||||
|
exhausted: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
const candidatesCall = await client.callTool({
|
||||||
|
...schemas.toolDefinition,
|
||||||
|
arguments: {
|
||||||
|
source: { kind: "evidence-file", path: evidenceFile },
|
||||||
|
view: {
|
||||||
|
kind: "web-bundle",
|
||||||
|
collection: "inferences",
|
||||||
|
kinds: ["endpoint_candidate"],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
expect(candidatesCall.isError).not.toBe(true);
|
||||||
|
expect(
|
||||||
|
schemas.ajv.validate(
|
||||||
|
schemas.outputSchema,
|
||||||
|
candidatesCall.structuredContent,
|
||||||
|
),
|
||||||
|
JSON.stringify(schemas.ajv.errors),
|
||||||
|
).toBe(true);
|
||||||
|
const candidatesPage = webBundlePageViewResultSchema.parse(
|
||||||
|
parseEvidence(candidatesCall.structuredContent).normalized_result,
|
||||||
|
);
|
||||||
|
expect(candidatesPage.items).toEqual([
|
||||||
|
expect.objectContaining({
|
||||||
source_index: 0,
|
source_index: 0,
|
||||||
value: "/api/users",
|
kind: "endpoint_candidate",
|
||||||
});
|
value: "/api/dynamic-user",
|
||||||
expect(page1.items[1]).toMatchObject({
|
confidence: "low",
|
||||||
source_index: 1,
|
}),
|
||||||
value: "/api/users",
|
]);
|
||||||
});
|
|
||||||
expect(page1.coverage).toMatchObject({
|
|
||||||
status: "page",
|
|
||||||
examined: 2,
|
|
||||||
total: 3,
|
|
||||||
next_offset: 2,
|
|
||||||
exhausted: false,
|
|
||||||
});
|
|
||||||
expect(page1.completeness.status).toBe("partial");
|
|
||||||
|
|
||||||
const page2Call = await client.callTool({
|
const emptyCall = await client.callTool({
|
||||||
...schemas.toolDefinition,
|
...schemas.toolDefinition,
|
||||||
arguments: {
|
arguments: {
|
||||||
source: { kind: "evidence-file", path: evidenceFile },
|
source: { kind: "evidence-file", path: evidenceFile },
|
||||||
view: {
|
view: {
|
||||||
kind: "web-bundle",
|
kind: "web-bundle",
|
||||||
collection: "endpoints",
|
collection: "endpoints",
|
||||||
offset: 2,
|
offset: 100,
|
||||||
limit: 2,
|
limit: 10,
|
||||||
},
|
|
||||||
},
|
},
|
||||||
});
|
},
|
||||||
expect(page2Call.isError).not.toBe(true);
|
});
|
||||||
expect(
|
expect(emptyCall.isError).not.toBe(true);
|
||||||
schemas.ajv.validate(schemas.outputSchema, page2Call.structuredContent),
|
expect(
|
||||||
JSON.stringify(schemas.ajv.errors),
|
schemas.ajv.validate(schemas.outputSchema, emptyCall.structuredContent),
|
||||||
).toBe(true);
|
JSON.stringify(schemas.ajv.errors),
|
||||||
const page2 = webBundlePageViewResultSchema.parse(
|
).toBe(true);
|
||||||
parseEvidence(page2Call.structuredContent).normalized_result,
|
const emptyPage = webBundlePageViewResultSchema.parse(
|
||||||
);
|
parseEvidence(emptyCall.structuredContent).normalized_result,
|
||||||
expect(page2.items).toHaveLength(1);
|
);
|
||||||
expect(page2.items[0]).toMatchObject({
|
expect(emptyPage.items).toEqual([]);
|
||||||
source_index: 2,
|
expect(emptyPage.coverage).toMatchObject({
|
||||||
value: "/api/orders",
|
status: "empty",
|
||||||
});
|
examined: 0,
|
||||||
expect(page2.coverage).toMatchObject({
|
total: 3,
|
||||||
status: "page",
|
next_offset: null,
|
||||||
examined: 1,
|
exhausted: true,
|
||||||
total: 3,
|
});
|
||||||
next_offset: null,
|
|
||||||
exhausted: true,
|
|
||||||
});
|
|
||||||
|
|
||||||
const candidatesCall = await client.callTool({
|
|
||||||
...schemas.toolDefinition,
|
|
||||||
arguments: {
|
|
||||||
source: { kind: "evidence-file", path: evidenceFile },
|
|
||||||
view: {
|
|
||||||
kind: "web-bundle",
|
|
||||||
collection: "inferences",
|
|
||||||
kinds: ["endpoint_candidate"],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
});
|
|
||||||
expect(candidatesCall.isError).not.toBe(true);
|
|
||||||
expect(
|
|
||||||
schemas.ajv.validate(
|
|
||||||
schemas.outputSchema,
|
|
||||||
candidatesCall.structuredContent,
|
|
||||||
),
|
|
||||||
JSON.stringify(schemas.ajv.errors),
|
|
||||||
).toBe(true);
|
|
||||||
const candidatesPage = webBundlePageViewResultSchema.parse(
|
|
||||||
parseEvidence(candidatesCall.structuredContent).normalized_result,
|
|
||||||
);
|
|
||||||
expect(candidatesPage.items).toEqual([
|
|
||||||
expect.objectContaining({
|
|
||||||
source_index: 0,
|
|
||||||
kind: "endpoint_candidate",
|
|
||||||
value: "/api/dynamic-user",
|
|
||||||
confidence: "low",
|
|
||||||
}),
|
|
||||||
]);
|
|
||||||
|
|
||||||
const emptyCall = await client.callTool({
|
|
||||||
...schemas.toolDefinition,
|
|
||||||
arguments: {
|
|
||||||
source: { kind: "evidence-file", path: evidenceFile },
|
|
||||||
view: {
|
|
||||||
kind: "web-bundle",
|
|
||||||
collection: "endpoints",
|
|
||||||
offset: 100,
|
|
||||||
limit: 10,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
});
|
|
||||||
expect(emptyCall.isError).not.toBe(true);
|
|
||||||
expect(
|
|
||||||
schemas.ajv.validate(schemas.outputSchema, emptyCall.structuredContent),
|
|
||||||
JSON.stringify(schemas.ajv.errors),
|
|
||||||
).toBe(true);
|
|
||||||
const emptyPage = webBundlePageViewResultSchema.parse(
|
|
||||||
parseEvidence(emptyCall.structuredContent).normalized_result,
|
|
||||||
);
|
|
||||||
expect(emptyPage.items).toEqual([]);
|
|
||||||
expect(emptyPage.coverage).toMatchObject({
|
|
||||||
status: "empty",
|
|
||||||
examined: 0,
|
|
||||||
total: 3,
|
|
||||||
next_offset: null,
|
|
||||||
exhausted: true,
|
|
||||||
});
|
|
||||||
} finally {
|
|
||||||
await unlink(evidenceFile).catch(() => {});
|
|
||||||
await rm(tempDir, { recursive: true, force: true }).catch(() => {});
|
|
||||||
}
|
|
||||||
});
|
});
|
||||||
Reference in new issue
Block a user