mirror of
https://github.com/morluto/rea.git
synced 2026-11-20 05:32:44 +00:00
test: harden verifier assertions and prune duplicate checks
This commit is contained in:
1 parent
f3734d90e7
commit
6bb2ad90ac
8 files changed
+113
-91
No files matched your search
@@ -20,6 +20,7 @@ import { inspectVerifierGhidraInstallation } from "./lib/ghidra-installation-ins
|
||||
import {
|
||||
requireMcpLifecycleResult,
|
||||
requireMcpOutput,
|
||||
requireMcpToolError,
|
||||
} from "./lib/mcp-verifier-results.mjs";
|
||||
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
|
||||
import {
|
||||
@@ -387,7 +388,16 @@ await withVerifierFixture(workspaceFixture, async () => {
|
||||
name: "search_bytes",
|
||||
arguments: { pattern: "c3", cursor: page.normalized_result.next_cursor },
|
||||
});
|
||||
assert.equal(wrongCursor.isError, true);
|
||||
const cursorError = requireMcpToolError(wrongCursor);
|
||||
assert.partialDeepStrictEqual(cursorError, {
|
||||
code: "invalid_request",
|
||||
details: {
|
||||
operation: "search_bytes",
|
||||
issues: [{ path: ["cursor"], reason: "invalid_value" }],
|
||||
diagnostics: { remote_code: "byte_search_cursor_mismatch" },
|
||||
},
|
||||
});
|
||||
assert.equal(cursorError.recovery, undefined);
|
||||
await call("close_binary");
|
||||
assert.deepEqual(await readFile(repeatedPath), repeatedBytes);
|
||||
|
||||
|
||||
@@ -2047,17 +2047,6 @@ describe("Ghidra byte-search cursor projection", () => {
|
||||
},
|
||||
});
|
||||
expect(projected).not.toHaveProperty("recovery");
|
||||
expect(projected.message).toContain("analysis profile");
|
||||
expect(projected.message).toContain("project");
|
||||
expect(projected.message).not.toContain("Analysis input is invalid");
|
||||
expect(projected.remediation.action).toContain("pattern");
|
||||
expect(projected.remediation.action).not.toContain("schema");
|
||||
expect(projected.remediation.action).not.toContain("tools/list");
|
||||
const bridge = await readFile(
|
||||
new URL("../../bridge/ghidra/ReaGhidraBridge.java", import.meta.url),
|
||||
"utf8",
|
||||
);
|
||||
expect(bridge).toContain(`"${CURSOR_MISMATCH}"`);
|
||||
await client.close();
|
||||
});
|
||||
|
||||
|
||||
@@ -344,7 +344,9 @@ it.each([
|
||||
{ setting: "REA_IDA_MCP_CONFIG", constraint },
|
||||
]);
|
||||
expect(parsed.error.message).toContain(constraint);
|
||||
// The guidance names a migration target the provider guide documents.
|
||||
});
|
||||
|
||||
it("points legacy protocol recovery to the documented migration", async () => {
|
||||
expect(LEGACY_PROTOCOL_REMOVED).toContain('"protocol": "native"');
|
||||
const guide = await fs.readFile("docs/providers/ida-provider.md", "utf8");
|
||||
expect(guide).toMatch(/^### Migrate a legacy attached registration$/mu);
|
||||
|
||||
@@ -48,12 +48,10 @@ it("retries without logging credential-bearing restore diagnostics", async () =>
|
||||
});
|
||||
try {
|
||||
await buildNativeAotFixture(run, options);
|
||||
expect(
|
||||
logged.mock.calls.flat().every((value) => typeof value === "string"),
|
||||
).toBe(true);
|
||||
expect(logged.mock.calls.flat().join("\n")).not.toContain(
|
||||
"fixture-password",
|
||||
);
|
||||
const output = logged.mock.calls.flat();
|
||||
expect(logged).toHaveBeenCalledWith(expect.stringContaining("attempt 2/2"));
|
||||
expect(output.every((value) => typeof value === "string")).toBe(true);
|
||||
expect(output.join("\n")).not.toContain("fixture-password");
|
||||
} finally {
|
||||
logged.mockRestore();
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { readFile, stat } from "node:fs/promises";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { RuleTester } from "oxlint/plugins-dev";
|
||||
import rule from "../../scripts/lint/rules/provider-failure-classifier.mjs";
|
||||
@@ -64,12 +64,12 @@ it("keeps the classifier baseline to existing source files", async () => {
|
||||
),
|
||||
) as { owners: string[]; counts: Record<string, number> };
|
||||
for (const owner of baseline.owners)
|
||||
await expect(readFile(owner, "utf8")).resolves.toBeTypeOf("string");
|
||||
expect((await stat(owner)).isFile(), owner).toBe(true);
|
||||
for (const [file, count] of Object.entries(baseline.counts)) {
|
||||
expect(file).toMatch(/^src\/.+\.ts$/u);
|
||||
expect(count).toBeGreaterThan(0);
|
||||
// The lint run reports an entry above a file's real count as stale; this
|
||||
// catches entries whose file no longer exists, which lint never visits.
|
||||
await expect(readFile(file, "utf8")).resolves.toBeTypeOf("string");
|
||||
expect((await stat(file)).isFile(), file).toBe(true);
|
||||
}
|
||||
});
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { readFile, stat } from "node:fs/promises";
|
||||
import { resolve } from "node:path";
|
||||
import { RuleTester } from "oxlint/plugins-dev";
|
||||
import rule from "../../scripts/lint/rules/typed-resource-refusal.mjs";
|
||||
@@ -50,7 +50,7 @@ new RuleTester({ languageOptions: { parserOptions: { lang: "ts" } } }).run(
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps the untyped-refusal baseline to files that still need migration", async () => {
|
||||
it("keeps the untyped-refusal baseline to existing source files", async () => {
|
||||
const baseline = JSON.parse(
|
||||
await readFile(
|
||||
"scripts/lint/rules/typed-resource-refusal-baseline.json",
|
||||
@@ -60,9 +60,7 @@ it("keeps the untyped-refusal baseline to files that still need migration", asyn
|
||||
for (const [file, count] of Object.entries(baseline)) {
|
||||
expect(file).toMatch(/^src\/.+\.ts$/u);
|
||||
expect(count).toBeGreaterThan(0);
|
||||
// A file whose untyped constructions are gone must leave the baseline.
|
||||
expect(await readFile(file, "utf8")).toContain(
|
||||
"new AnalysisResourceConstraintError(",
|
||||
);
|
||||
// The rule checks counts in files lint visits. A removed file gets no visit.
|
||||
expect((await stat(file)).isFile(), file).toBe(true);
|
||||
}
|
||||
});
|
||||
@@ -1,5 +1,7 @@
|
||||
import { readdir, readFile } from "node:fs/promises";
|
||||
import { join, relative, resolve } from "node:path";
|
||||
import { parse } from "@babel/parser";
|
||||
import { traverseFast, type Node } from "@babel/types";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { CdpConnection } from "../../src/browser/CdpConnection.js";
|
||||
import { startFakeCdpBrowser } from "../fixtures/fakeCdpBrowser.js";
|
||||
@@ -19,21 +21,92 @@ async function* scriptFiles(directory: string): AsyncGenerator<string> {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* JavaScript verifiers send through the same `CdpConnection` and Playwright
|
||||
* sessions as TypeScript, but the compiler does not check their method names.
|
||||
*
|
||||
* The structural check is at the send owners: `CdpConnection.send` and the
|
||||
* Playwright session refuse an unregistered command before it reaches the
|
||||
* browser, and `CdpConnection.onRegisteredEvent` and the Playwright session
|
||||
* refuse an unregistered event name when subscribing (see
|
||||
* `verifierCdpEventPayloads.test.ts`). Type-checking the verifier scripts
|
||||
* themselves is not practical (checkJs over them and the built modules they
|
||||
* import reports about 17,000 implicit `any` errors), so the text scan below
|
||||
* is only an early warning for verifiers that rarely run. It sees
|
||||
* string-literal sends only: a name built at run time or passed through a
|
||||
* variable is not scanned, and is caught only when that verifier runs.
|
||||
*/
|
||||
const cdpMethod = /^[A-Z][A-Za-z]*\.[a-z][A-Za-z]*$/u;
|
||||
|
||||
function staticString(node: Node | undefined): string | undefined {
|
||||
if (node?.type === "StringLiteral") return node.value;
|
||||
if (node?.type === "TemplateLiteral" && node.expressions.length === 0)
|
||||
return node.quasis[0]?.value.cooked ?? undefined;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// Guard literal sends in rarely run verifiers. The runtime send owners check
|
||||
// computed command names, and registered-event subscribers validate event names.
|
||||
function verifierCdpCalls(source: string) {
|
||||
const commands: { name: string; line: number }[] = [];
|
||||
const rawEvents: number[] = [];
|
||||
const tree = parse(source, {
|
||||
sourceType: "unambiguous",
|
||||
plugins: ["typescript"],
|
||||
});
|
||||
traverseFast(tree, (node) => {
|
||||
if (
|
||||
node.type !== "CallExpression" &&
|
||||
node.type !== "OptionalCallExpression"
|
||||
)
|
||||
return;
|
||||
const { callee } = node;
|
||||
if (
|
||||
callee.type !== "MemberExpression" &&
|
||||
callee.type !== "OptionalMemberExpression"
|
||||
)
|
||||
return;
|
||||
const method =
|
||||
!callee.computed && callee.property.type === "Identifier"
|
||||
? callee.property.name
|
||||
: staticString(callee.property);
|
||||
const line = callee.loc?.start.line ?? 1;
|
||||
if (method === "onEvent") {
|
||||
rawEvents.push(line);
|
||||
} else if (method === "send") {
|
||||
const name = staticString(node.arguments[0]);
|
||||
if (name !== undefined && cdpMethod.test(name))
|
||||
commands.push({ name, line });
|
||||
}
|
||||
});
|
||||
return { commands, rawEvents };
|
||||
}
|
||||
|
||||
describe("verifier CDP source references", () => {
|
||||
it("finds static calls independent of quoting and ignores source-like text", () => {
|
||||
const source = [
|
||||
'// session.send("Page.comment"); session.onEvent(handler);',
|
||||
`const example = 'session.send("Page.example"); session.onEvent(handler);';`,
|
||||
"session.send ('Page.notRegistered');",
|
||||
'session["send"](`DOM.getDocument`);',
|
||||
"session.onEvent (handler);",
|
||||
"session?.['onEvent'](handler);",
|
||||
"session.onRegisteredEvent('Page.lifecycleEvent', handler);",
|
||||
"session.send(method);",
|
||||
].join("\n");
|
||||
expect(verifierCdpCalls(source)).toEqual({
|
||||
commands: [
|
||||
{ name: "Page.notRegistered", line: 3 },
|
||||
{ name: "DOM.getDocument", line: 4 },
|
||||
],
|
||||
rawEvents: [5, 6],
|
||||
});
|
||||
});
|
||||
|
||||
it("uses registered commands and excludes raw event subscribers", async () => {
|
||||
const unregistered: string[] = [];
|
||||
const rawEvents: string[] = [];
|
||||
let sends = 0;
|
||||
for await (const path of scriptFiles(scriptsRoot)) {
|
||||
const calls = verifierCdpCalls(await readFile(path, "utf8"));
|
||||
const file = relative(scriptsRoot, path);
|
||||
sends += calls.commands.length;
|
||||
for (const { name, line } of calls.commands) {
|
||||
if (!isCdpCommand(name)) unregistered.push(`${file}:${line}: ${name}`);
|
||||
}
|
||||
for (const line of calls.rawEvents) rawEvents.push(`${file}:${line}`);
|
||||
}
|
||||
expect(sends).toBeGreaterThan(0);
|
||||
expect(unregistered).toEqual([]);
|
||||
expect(rawEvents).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("verifier CDP commands", () => {
|
||||
it("refuses an unregistered command at the connection before sending it", async () => {
|
||||
const browser = await startFakeCdpBrowser();
|
||||
@@ -57,23 +130,6 @@ describe("verifier CDP commands", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("sends only commands with registered result schemas", async () => {
|
||||
const unregistered: string[] = [];
|
||||
let sends = 0;
|
||||
for await (const path of scriptFiles(scriptsRoot)) {
|
||||
const source = await readFile(path, "utf8");
|
||||
for (const match of source.matchAll(
|
||||
/\.send\(\s*"([A-Z][A-Za-z]*\.[a-z][A-Za-z]*)"/gu,
|
||||
)) {
|
||||
sends += 1;
|
||||
if (!isCdpCommand(match[1]!))
|
||||
unregistered.push(`${relative(scriptsRoot, path)}: ${match[1]}`);
|
||||
}
|
||||
}
|
||||
expect(sends).toBeGreaterThan(0);
|
||||
expect(unregistered).toEqual([]);
|
||||
});
|
||||
|
||||
it("keeps the reply members the verifiers read", () => {
|
||||
expect(
|
||||
parseCdpCommandResult("Target.closeTarget", { success: true }),
|
||||
|
||||
@@ -1,27 +1,8 @@
|
||||
import { readdir, readFile } from "node:fs/promises";
|
||||
import { join, relative, resolve } from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { CdpConnection } from "../../src/browser/CdpConnection.js";
|
||||
import type { CdpEventName } from "../../src/browser/protocol/CdpEvents.js";
|
||||
import { startFakeCdpBrowser } from "../fixtures/fakeCdpBrowser.js";
|
||||
|
||||
const scriptsRoot = resolve(import.meta.dirname, "../../scripts");
|
||||
|
||||
async function* scriptFiles(directory: string): AsyncGenerator<string> {
|
||||
for (const entry of await readdir(directory, { withFileTypes: true })) {
|
||||
const path = join(directory, entry.name);
|
||||
if (entry.isDirectory()) yield* scriptFiles(path);
|
||||
else if (/\.m?js$/u.test(entry.name)) yield path;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* JavaScript verifiers subscribe to the same `CdpConnection` as TypeScript,
|
||||
* but the compiler does not check the event names they match. They subscribe
|
||||
* through `onRegisteredEvent`, which refuses an unregistered name when the
|
||||
* subscription is made, as `CdpConnection.send` does for commands; a raw
|
||||
* `onEvent` subscriber compares names no check would catch.
|
||||
*/
|
||||
describe("verifier CDP event subscribers", () => {
|
||||
it("refuses an unregistered event name before subscribing", async () => {
|
||||
const browser = await startFakeCdpBrowser();
|
||||
@@ -59,16 +40,4 @@ describe("verifier CDP event subscribers", () => {
|
||||
await browser.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("subscribe only through the registered-event check", async () => {
|
||||
const raw: string[] = [];
|
||||
for await (const path of scriptFiles(scriptsRoot)) {
|
||||
const source = await readFile(path, "utf8");
|
||||
for (const match of source.matchAll(/\.onEvent\(/gu))
|
||||
raw.push(
|
||||
`${relative(scriptsRoot, path)}:${String(source.slice(0, match.index).split("\n").length)}`,
|
||||
);
|
||||
}
|
||||
expect(raw).toEqual([]);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user