mirror of
https://github.com/morluto/rea.git
synced 2026-10-26 17:35:11 +00:00
test(web): own verifier commands and cover filtered continuation
This commit is contained in:
1 parent
dd853c5c7a
commit
8f3203b1c3
4 files changed
+209
-146
No files matched your search
@@ -53,6 +53,15 @@ limitation union before display bounding, and distinct script-scoped unknowns.
|
||||
These three corrections fail against the imported upstream production code;
|
||||
explicit-null and explicit-origin controls continue to pass.
|
||||
|
||||
Automated review follow-up added an interleaved, intersecting-filter continuation
|
||||
case: the second matching offset is `2`, while the joined original indices are
|
||||
`2, 4, 5`. All 24 projection tests and both SDK boundary tests pass. Test temporary
|
||||
directories now have the existing fixture as their sole cleanup owner. Both real
|
||||
CLI launches in the verifier use `createVerifierCommands` and awaited shared
|
||||
cleanup; real selected capture and both CLI collection readbacks pass after that
|
||||
change. Command supervision is acquired lazily only for capture or explicit CLI
|
||||
verification, outside the measured retained-query workflows above.
|
||||
|
||||
## Equivalent completed query costs
|
||||
|
||||
Both versions import the same already completed parent, discover the same 41
|
||||
|
||||
@@ -5,9 +5,10 @@ import { join, resolve } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { createRequire } from "node:module";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { execFile } from "node:child_process";
|
||||
import { promisify, parseArgs } from "node:util";
|
||||
import { parseArgs } from "node:util";
|
||||
import { performance } from "node:perf_hooks";
|
||||
import { createVerifierCommands } from "../../scripts/lib/verify-commands.mjs";
|
||||
import { withVerifierCleanup } from "../../scripts/lib/verifier-cleanup.mjs";
|
||||
// Public CLI capture and same-session retained MCP queries. All assets and output
|
||||
// paths are explicitly supplied; this verifier installs no browser or application.
|
||||
const { values } = parseArgs({
|
||||
@@ -46,6 +47,10 @@ const env = {
|
||||
REA_MCP_INPUT_SCHEMA_PROFILE: "compact",
|
||||
REA_MCP_TOOL_FAMILIES: "application,session",
|
||||
};
|
||||
const commands =
|
||||
mode === "capture" || values["verify-cli"]
|
||||
? await createVerifierCommands()
|
||||
: undefined;
|
||||
if (mode === "capture") {
|
||||
const { chromium } = require("playwright-core");
|
||||
const assets = resolve(values.application);
|
||||
@@ -82,7 +87,7 @@ if (mode === "capture") {
|
||||
const origin = `http://127.0.0.1:${server.address().port}`;
|
||||
const userData = await mkdtemp(join(tmpdir(), "rea-6198-chrome-"));
|
||||
let context;
|
||||
try {
|
||||
await withVerifierCleanup(async () => {
|
||||
context = await chromium.launchPersistentContext(userData, {
|
||||
executablePath: resolve(values.browser),
|
||||
headless: true,
|
||||
@@ -105,7 +110,7 @@ if (mode === "capture") {
|
||||
);
|
||||
assert.ok(target);
|
||||
const started = performance.now();
|
||||
const { stdout, stderr } = await promisify(execFile)(
|
||||
const { stdout, stderr } = await commands.exec(
|
||||
process.execPath,
|
||||
[
|
||||
join(repo, "scripts/rea.mjs"),
|
||||
@@ -168,11 +173,17 @@ if (mode === "capture") {
|
||||
completeness: evidence.normalized_result.completeness,
|
||||
}),
|
||||
);
|
||||
} finally {
|
||||
await context?.close();
|
||||
await new Promise((r) => server.close(r));
|
||||
await rm(userData, { recursive: true, force: true });
|
||||
}
|
||||
}, [
|
||||
() => commands.close(),
|
||||
async () => {
|
||||
await context?.close();
|
||||
await rm(userData, { recursive: true, force: true });
|
||||
},
|
||||
() =>
|
||||
new Promise((done, reject) =>
|
||||
server.close((error) => (error ? reject(error) : done())),
|
||||
),
|
||||
]);
|
||||
process.exit(0);
|
||||
}
|
||||
const parent = JSON.parse(await readFile(bundle, "utf8"));
|
||||
@@ -213,7 +224,7 @@ async function invoke(name, args) {
|
||||
assert.notEqual(result.isError, true, JSON.stringify(result));
|
||||
return result.structuredContent;
|
||||
}
|
||||
try {
|
||||
await withVerifierCleanup(async () => {
|
||||
await client.connect(transport);
|
||||
catalog = await client.listTools();
|
||||
await writeFile(join(out, "catalog.json"), JSON.stringify(catalog));
|
||||
@@ -322,7 +333,7 @@ try {
|
||||
limit: 32,
|
||||
},
|
||||
});
|
||||
const { stdout } = await promisify(execFile)(
|
||||
const { stdout } = await commands.exec(
|
||||
process.execPath,
|
||||
[
|
||||
join(repo, "scripts/rea.mjs"),
|
||||
@@ -368,6 +379,4 @@ try {
|
||||
),
|
||||
);
|
||||
console.log(JSON.stringify({ mode, task: values.task, runs }));
|
||||
} finally {
|
||||
await client.close();
|
||||
}
|
||||
}, [() => commands?.close(), () => client.close()]);
|
||||
@@ -495,6 +495,56 @@ it("rejects non-web-bundle parent operation with invalid_value", () => {
|
||||
expect(result.error.issues[0]!.reason).toBe("invalid_value");
|
||||
});
|
||||
|
||||
it("continues filtered pages by matching ordinal while preserving original indices", () => {
|
||||
const report = sampleAnalysis();
|
||||
const row = report.observations.endpoints[0]!;
|
||||
const endpoints = [
|
||||
{ ...row, mechanism: "call:sendBeacon" },
|
||||
{ ...row, location: { ...row.location, script_key: "vendor.js" } },
|
||||
row,
|
||||
{ ...row, value: "/api/orders" },
|
||||
{ ...row, location: { ...row.location, line: 99 } },
|
||||
{ ...row, location: { ...row.location, line: 100 } },
|
||||
{ ...row, location: { ...row.location, script_key: "vendor.js" } },
|
||||
];
|
||||
const parent = makeParent({
|
||||
observations: { ...report.observations, endpoints },
|
||||
});
|
||||
const selection = {
|
||||
kind: "web-bundle",
|
||||
collection: "endpoints",
|
||||
script_keys: ["app.js"],
|
||||
values: ["/api/users"],
|
||||
mechanisms: ["call:fetch"],
|
||||
limit: 2,
|
||||
};
|
||||
const first = query(parent, selection);
|
||||
if (first.kind !== "web-bundle") throw new Error("Expected web-bundle page");
|
||||
expect(first.coverage).toMatchObject({
|
||||
total: 3,
|
||||
examined: 2,
|
||||
next_offset: 2,
|
||||
exhausted: false,
|
||||
});
|
||||
const second = query(parent, {
|
||||
...selection,
|
||||
offset: first.coverage.next_offset,
|
||||
});
|
||||
if (second.kind !== "web-bundle") throw new Error("Expected web-bundle page");
|
||||
expect(second.coverage).toMatchObject({
|
||||
total: 3,
|
||||
examined: 1,
|
||||
next_offset: null,
|
||||
exhausted: true,
|
||||
});
|
||||
expect([...first.items, ...second.items]).toEqual(
|
||||
[2, 4, 5].map((source_index) => ({
|
||||
source_index,
|
||||
...endpoints[source_index],
|
||||
})),
|
||||
);
|
||||
});
|
||||
|
||||
it("preserves parent artifact identity across summary and page views", () => {
|
||||
const artifact = { path: "/app.js", sha256: "e".repeat(64) };
|
||||
const parent: AnalysisViewParent = {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { Client, InMemoryTransport } from "@modelcontextprotocol/client";
|
||||
import { writeFile, unlink, rm } from "node:fs/promises";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { expect, it, onTestFinished } from "vitest";
|
||||
import { createServer } from "../../../src/server/createServer.js";
|
||||
@@ -215,139 +215,134 @@ it("pages web bundle endpoints and inferences over MCP using evidence-file sourc
|
||||
const evidenceFile = join(tempDir, "webbundle-evidence.json");
|
||||
await writeFile(evidenceFile, JSON.stringify(parent));
|
||||
|
||||
try {
|
||||
const page1Call = await client.callTool({
|
||||
...schemas.toolDefinition,
|
||||
arguments: {
|
||||
source: { kind: "evidence-file", path: evidenceFile },
|
||||
view: {
|
||||
kind: "web-bundle",
|
||||
collection: "endpoints",
|
||||
offset: 0,
|
||||
limit: 2,
|
||||
},
|
||||
const page1Call = await client.callTool({
|
||||
...schemas.toolDefinition,
|
||||
arguments: {
|
||||
source: { kind: "evidence-file", path: evidenceFile },
|
||||
view: {
|
||||
kind: "web-bundle",
|
||||
collection: "endpoints",
|
||||
offset: 0,
|
||||
limit: 2,
|
||||
},
|
||||
});
|
||||
expect(page1Call.isError).not.toBe(true);
|
||||
expect(
|
||||
schemas.ajv.validate(schemas.outputSchema, page1Call.structuredContent),
|
||||
JSON.stringify(schemas.ajv.errors),
|
||||
).toBe(true);
|
||||
const page1 = webBundlePageViewResultSchema.parse(
|
||||
parseEvidence(page1Call.structuredContent).normalized_result,
|
||||
);
|
||||
expect(page1.items).toHaveLength(2);
|
||||
expect(page1.items[0]).toMatchObject({
|
||||
},
|
||||
});
|
||||
expect(page1Call.isError).not.toBe(true);
|
||||
expect(
|
||||
schemas.ajv.validate(schemas.outputSchema, page1Call.structuredContent),
|
||||
JSON.stringify(schemas.ajv.errors),
|
||||
).toBe(true);
|
||||
const page1 = webBundlePageViewResultSchema.parse(
|
||||
parseEvidence(page1Call.structuredContent).normalized_result,
|
||||
);
|
||||
expect(page1.items).toHaveLength(2);
|
||||
expect(page1.items[0]).toMatchObject({
|
||||
source_index: 0,
|
||||
value: "/api/users",
|
||||
});
|
||||
expect(page1.items[1]).toMatchObject({
|
||||
source_index: 1,
|
||||
value: "/api/users",
|
||||
});
|
||||
expect(page1.coverage).toMatchObject({
|
||||
status: "page",
|
||||
examined: 2,
|
||||
total: 3,
|
||||
next_offset: 2,
|
||||
exhausted: false,
|
||||
});
|
||||
expect(page1.completeness.status).toBe("partial");
|
||||
|
||||
const page2Call = await client.callTool({
|
||||
...schemas.toolDefinition,
|
||||
arguments: {
|
||||
source: { kind: "evidence-file", path: evidenceFile },
|
||||
view: {
|
||||
kind: "web-bundle",
|
||||
collection: "endpoints",
|
||||
offset: 2,
|
||||
limit: 2,
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(page2Call.isError).not.toBe(true);
|
||||
expect(
|
||||
schemas.ajv.validate(schemas.outputSchema, page2Call.structuredContent),
|
||||
JSON.stringify(schemas.ajv.errors),
|
||||
).toBe(true);
|
||||
const page2 = webBundlePageViewResultSchema.parse(
|
||||
parseEvidence(page2Call.structuredContent).normalized_result,
|
||||
);
|
||||
expect(page2.items).toHaveLength(1);
|
||||
expect(page2.items[0]).toMatchObject({
|
||||
source_index: 2,
|
||||
value: "/api/orders",
|
||||
});
|
||||
expect(page2.coverage).toMatchObject({
|
||||
status: "page",
|
||||
examined: 1,
|
||||
total: 3,
|
||||
next_offset: null,
|
||||
exhausted: true,
|
||||
});
|
||||
|
||||
const candidatesCall = await client.callTool({
|
||||
...schemas.toolDefinition,
|
||||
arguments: {
|
||||
source: { kind: "evidence-file", path: evidenceFile },
|
||||
view: {
|
||||
kind: "web-bundle",
|
||||
collection: "inferences",
|
||||
kinds: ["endpoint_candidate"],
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(candidatesCall.isError).not.toBe(true);
|
||||
expect(
|
||||
schemas.ajv.validate(
|
||||
schemas.outputSchema,
|
||||
candidatesCall.structuredContent,
|
||||
),
|
||||
JSON.stringify(schemas.ajv.errors),
|
||||
).toBe(true);
|
||||
const candidatesPage = webBundlePageViewResultSchema.parse(
|
||||
parseEvidence(candidatesCall.structuredContent).normalized_result,
|
||||
);
|
||||
expect(candidatesPage.items).toEqual([
|
||||
expect.objectContaining({
|
||||
source_index: 0,
|
||||
value: "/api/users",
|
||||
});
|
||||
expect(page1.items[1]).toMatchObject({
|
||||
source_index: 1,
|
||||
value: "/api/users",
|
||||
});
|
||||
expect(page1.coverage).toMatchObject({
|
||||
status: "page",
|
||||
examined: 2,
|
||||
total: 3,
|
||||
next_offset: 2,
|
||||
exhausted: false,
|
||||
});
|
||||
expect(page1.completeness.status).toBe("partial");
|
||||
kind: "endpoint_candidate",
|
||||
value: "/api/dynamic-user",
|
||||
confidence: "low",
|
||||
}),
|
||||
]);
|
||||
|
||||
const page2Call = await client.callTool({
|
||||
...schemas.toolDefinition,
|
||||
arguments: {
|
||||
source: { kind: "evidence-file", path: evidenceFile },
|
||||
view: {
|
||||
kind: "web-bundle",
|
||||
collection: "endpoints",
|
||||
offset: 2,
|
||||
limit: 2,
|
||||
},
|
||||
const emptyCall = await client.callTool({
|
||||
...schemas.toolDefinition,
|
||||
arguments: {
|
||||
source: { kind: "evidence-file", path: evidenceFile },
|
||||
view: {
|
||||
kind: "web-bundle",
|
||||
collection: "endpoints",
|
||||
offset: 100,
|
||||
limit: 10,
|
||||
},
|
||||
});
|
||||
expect(page2Call.isError).not.toBe(true);
|
||||
expect(
|
||||
schemas.ajv.validate(schemas.outputSchema, page2Call.structuredContent),
|
||||
JSON.stringify(schemas.ajv.errors),
|
||||
).toBe(true);
|
||||
const page2 = webBundlePageViewResultSchema.parse(
|
||||
parseEvidence(page2Call.structuredContent).normalized_result,
|
||||
);
|
||||
expect(page2.items).toHaveLength(1);
|
||||
expect(page2.items[0]).toMatchObject({
|
||||
source_index: 2,
|
||||
value: "/api/orders",
|
||||
});
|
||||
expect(page2.coverage).toMatchObject({
|
||||
status: "page",
|
||||
examined: 1,
|
||||
total: 3,
|
||||
next_offset: null,
|
||||
exhausted: true,
|
||||
});
|
||||
|
||||
const candidatesCall = await client.callTool({
|
||||
...schemas.toolDefinition,
|
||||
arguments: {
|
||||
source: { kind: "evidence-file", path: evidenceFile },
|
||||
view: {
|
||||
kind: "web-bundle",
|
||||
collection: "inferences",
|
||||
kinds: ["endpoint_candidate"],
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(candidatesCall.isError).not.toBe(true);
|
||||
expect(
|
||||
schemas.ajv.validate(
|
||||
schemas.outputSchema,
|
||||
candidatesCall.structuredContent,
|
||||
),
|
||||
JSON.stringify(schemas.ajv.errors),
|
||||
).toBe(true);
|
||||
const candidatesPage = webBundlePageViewResultSchema.parse(
|
||||
parseEvidence(candidatesCall.structuredContent).normalized_result,
|
||||
);
|
||||
expect(candidatesPage.items).toEqual([
|
||||
expect.objectContaining({
|
||||
source_index: 0,
|
||||
kind: "endpoint_candidate",
|
||||
value: "/api/dynamic-user",
|
||||
confidence: "low",
|
||||
}),
|
||||
]);
|
||||
|
||||
const emptyCall = await client.callTool({
|
||||
...schemas.toolDefinition,
|
||||
arguments: {
|
||||
source: { kind: "evidence-file", path: evidenceFile },
|
||||
view: {
|
||||
kind: "web-bundle",
|
||||
collection: "endpoints",
|
||||
offset: 100,
|
||||
limit: 10,
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(emptyCall.isError).not.toBe(true);
|
||||
expect(
|
||||
schemas.ajv.validate(schemas.outputSchema, emptyCall.structuredContent),
|
||||
JSON.stringify(schemas.ajv.errors),
|
||||
).toBe(true);
|
||||
const emptyPage = webBundlePageViewResultSchema.parse(
|
||||
parseEvidence(emptyCall.structuredContent).normalized_result,
|
||||
);
|
||||
expect(emptyPage.items).toEqual([]);
|
||||
expect(emptyPage.coverage).toMatchObject({
|
||||
status: "empty",
|
||||
examined: 0,
|
||||
total: 3,
|
||||
next_offset: null,
|
||||
exhausted: true,
|
||||
});
|
||||
} finally {
|
||||
await unlink(evidenceFile).catch(() => {});
|
||||
await rm(tempDir, { recursive: true, force: true }).catch(() => {});
|
||||
}
|
||||
},
|
||||
});
|
||||
expect(emptyCall.isError).not.toBe(true);
|
||||
expect(
|
||||
schemas.ajv.validate(schemas.outputSchema, emptyCall.structuredContent),
|
||||
JSON.stringify(schemas.ajv.errors),
|
||||
).toBe(true);
|
||||
const emptyPage = webBundlePageViewResultSchema.parse(
|
||||
parseEvidence(emptyCall.structuredContent).normalized_result,
|
||||
);
|
||||
expect(emptyPage.items).toEqual([]);
|
||||
expect(emptyPage.coverage).toMatchObject({
|
||||
status: "empty",
|
||||
examined: 0,
|
||||
total: 3,
|
||||
next_offset: null,
|
||||
exhausted: true,
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user