test: harden verifier assertions and prune duplicate checks

This commit is contained in:
morluto committed 2026-11-19 22:09:03 -05:00
1 parent f3734d90e7
commit 6bb2ad90ac
8 files changed
+113 -91

No files matched your search

+11 -1
View File
@@ -20,6 +20,7 @@ import { inspectVerifierGhidraInstallation } from "./lib/ghidra-installation-ins
import {
requireMcpLifecycleResult,
requireMcpOutput,
requireMcpToolError,
} from "./lib/mcp-verifier-results.mjs";
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
import {
@@ -387,7 +388,16 @@ await withVerifierFixture(workspaceFixture, async () => {
name: "search_bytes",
arguments: { pattern: "c3", cursor: page.normalized_result.next_cursor },
});
assert.equal(wrongCursor.isError, true);
const cursorError = requireMcpToolError(wrongCursor);
assert.partialDeepStrictEqual(cursorError, {
code: "invalid_request",
details: {
operation: "search_bytes",
issues: [{ path: ["cursor"], reason: "invalid_value" }],
diagnostics: { remote_code: "byte_search_cursor_mismatch" },
},
});
assert.equal(cursorError.recovery, undefined);
await call("close_binary");
assert.deepEqual(await readFile(repeatedPath), repeatedBytes);
@@ -2047,17 +2047,6 @@ describe("Ghidra byte-search cursor projection", () => {
},
});
expect(projected).not.toHaveProperty("recovery");
expect(projected.message).toContain("analysis profile");
expect(projected.message).toContain("project");
expect(projected.message).not.toContain("Analysis input is invalid");
expect(projected.remediation.action).toContain("pattern");
expect(projected.remediation.action).not.toContain("schema");
expect(projected.remediation.action).not.toContain("tools/list");
const bridge = await readFile(
new URL("../../bridge/ghidra/ReaGhidraBridge.java", import.meta.url),
"utf8",
);
expect(bridge).toContain(`"${CURSOR_MISMATCH}"`);
await client.close();
});
+3 -1
View File
@@ -344,7 +344,9 @@ it.each([
{ setting: "REA_IDA_MCP_CONFIG", constraint },
]);
expect(parsed.error.message).toContain(constraint);
// The guidance names a migration target the provider guide documents.
});
it("points legacy protocol recovery to the documented migration", async () => {
expect(LEGACY_PROTOCOL_REMOVED).toContain('"protocol": "native"');
const guide = await fs.readFile("docs/providers/ida-provider.md", "utf8");
expect(guide).toMatch(/^### Migrate a legacy attached registration$/mu);
@@ -48,12 +48,10 @@ it("retries without logging credential-bearing restore diagnostics", async () =>
});
try {
await buildNativeAotFixture(run, options);
expect(
logged.mock.calls.flat().every((value) => typeof value === "string"),
).toBe(true);
expect(logged.mock.calls.flat().join("\n")).not.toContain(
"fixture-password",
);
const output = logged.mock.calls.flat();
expect(logged).toHaveBeenCalledWith(expect.stringContaining("attempt 2/2"));
expect(output.every((value) => typeof value === "string")).toBe(true);
expect(output.join("\n")).not.toContain("fixture-password");
} finally {
logged.mockRestore();
}
@@ -1,5 +1,5 @@
import { describe, expect, it } from "vitest";
import { readFile } from "node:fs/promises";
import { readFile, stat } from "node:fs/promises";
import { fileURLToPath } from "node:url";
import { RuleTester } from "oxlint/plugins-dev";
import rule from "../../scripts/lint/rules/provider-failure-classifier.mjs";
@@ -64,12 +64,12 @@ it("keeps the classifier baseline to existing source files", async () => {
),
) as { owners: string[]; counts: Record<string, number> };
for (const owner of baseline.owners)
await expect(readFile(owner, "utf8")).resolves.toBeTypeOf("string");
expect((await stat(owner)).isFile(), owner).toBe(true);
for (const [file, count] of Object.entries(baseline.counts)) {
expect(file).toMatch(/^src\/.+\.ts$/u);
expect(count).toBeGreaterThan(0);
// The lint run reports an entry above a file's real count as stale; this
// catches entries whose file no longer exists, which lint never visits.
await expect(readFile(file, "utf8")).resolves.toBeTypeOf("string");
expect((await stat(file)).isFile(), file).toBe(true);
}
});
@@ -1,5 +1,5 @@
import { describe, expect, it } from "vitest";
import { readFile } from "node:fs/promises";
import { readFile, stat } from "node:fs/promises";
import { resolve } from "node:path";
import { RuleTester } from "oxlint/plugins-dev";
import rule from "../../scripts/lint/rules/typed-resource-refusal.mjs";
@@ -50,7 +50,7 @@ new RuleTester({ languageOptions: { parserOptions: { lang: "ts" } } }).run(
},
);
it("keeps the untyped-refusal baseline to files that still need migration", async () => {
it("keeps the untyped-refusal baseline to existing source files", async () => {
const baseline = JSON.parse(
await readFile(
"scripts/lint/rules/typed-resource-refusal-baseline.json",
@@ -60,9 +60,7 @@ it("keeps the untyped-refusal baseline to files that still need migration", asyn
for (const [file, count] of Object.entries(baseline)) {
expect(file).toMatch(/^src\/.+\.ts$/u);
expect(count).toBeGreaterThan(0);
// A file whose untyped constructions are gone must leave the baseline.
expect(await readFile(file, "utf8")).toContain(
"new AnalysisResourceConstraintError(",
);
// The rule checks counts in files lint visits. A removed file gets no visit.
expect((await stat(file)).isFile(), file).toBe(true);
}
});
+88 -32
View File
@@ -1,5 +1,7 @@
import { readdir, readFile } from "node:fs/promises";
import { join, relative, resolve } from "node:path";
import { parse } from "@babel/parser";
import { traverseFast, type Node } from "@babel/types";
import { describe, expect, it } from "vitest";
import { CdpConnection } from "../../src/browser/CdpConnection.js";
import { startFakeCdpBrowser } from "../fixtures/fakeCdpBrowser.js";
@@ -19,21 +21,92 @@ async function* scriptFiles(directory: string): AsyncGenerator<string> {
}
}
/**
* JavaScript verifiers send through the same `CdpConnection` and Playwright
* sessions as TypeScript, but the compiler does not check their method names.
*
* The structural check is at the send owners: `CdpConnection.send` and the
* Playwright session refuse an unregistered command before it reaches the
* browser, and `CdpConnection.onRegisteredEvent` and the Playwright session
* refuse an unregistered event name when subscribing (see
* `verifierCdpEventPayloads.test.ts`). Type-checking the verifier scripts
* themselves is not practical (checkJs over them and the built modules they
* import reports about 17,000 implicit `any` errors), so the text scan below
* is only an early warning for verifiers that rarely run. It sees
* string-literal sends only: a name built at run time or passed through a
* variable is not scanned, and is caught only when that verifier runs.
*/
const cdpMethod = /^[A-Z][A-Za-z]*\.[a-z][A-Za-z]*$/u;
function staticString(node: Node | undefined): string | undefined {
if (node?.type === "StringLiteral") return node.value;
if (node?.type === "TemplateLiteral" && node.expressions.length === 0)
return node.quasis[0]?.value.cooked ?? undefined;
return undefined;
}
// Guard literal sends in rarely run verifiers. The runtime send owners check
// computed command names, and registered-event subscribers validate event names.
function verifierCdpCalls(source: string) {
const commands: { name: string; line: number }[] = [];
const rawEvents: number[] = [];
const tree = parse(source, {
sourceType: "unambiguous",
plugins: ["typescript"],
});
traverseFast(tree, (node) => {
if (
node.type !== "CallExpression" &&
node.type !== "OptionalCallExpression"
)
return;
const { callee } = node;
if (
callee.type !== "MemberExpression" &&
callee.type !== "OptionalMemberExpression"
)
return;
const method =
!callee.computed && callee.property.type === "Identifier"
? callee.property.name
: staticString(callee.property);
const line = callee.loc?.start.line ?? 1;
if (method === "onEvent") {
rawEvents.push(line);
} else if (method === "send") {
const name = staticString(node.arguments[0]);
if (name !== undefined && cdpMethod.test(name))
commands.push({ name, line });
}
});
return { commands, rawEvents };
}
describe("verifier CDP source references", () => {
it("finds static calls independent of quoting and ignores source-like text", () => {
const source = [
'// session.send("Page.comment"); session.onEvent(handler);',
`const example = 'session.send("Page.example"); session.onEvent(handler);';`,
"session.send ('Page.notRegistered');",
'session["send"](`DOM.getDocument`);',
"session.onEvent (handler);",
"session?.['onEvent'](handler);",
"session.onRegisteredEvent('Page.lifecycleEvent', handler);",
"session.send(method);",
].join("\n");
expect(verifierCdpCalls(source)).toEqual({
commands: [
{ name: "Page.notRegistered", line: 3 },
{ name: "DOM.getDocument", line: 4 },
],
rawEvents: [5, 6],
});
});
it("uses registered commands and excludes raw event subscribers", async () => {
const unregistered: string[] = [];
const rawEvents: string[] = [];
let sends = 0;
for await (const path of scriptFiles(scriptsRoot)) {
const calls = verifierCdpCalls(await readFile(path, "utf8"));
const file = relative(scriptsRoot, path);
sends += calls.commands.length;
for (const { name, line } of calls.commands) {
if (!isCdpCommand(name)) unregistered.push(`${file}:${line}: ${name}`);
}
for (const line of calls.rawEvents) rawEvents.push(`${file}:${line}`);
}
expect(sends).toBeGreaterThan(0);
expect(unregistered).toEqual([]);
expect(rawEvents).toEqual([]);
});
});
describe("verifier CDP commands", () => {
it("refuses an unregistered command at the connection before sending it", async () => {
const browser = await startFakeCdpBrowser();
@@ -57,23 +130,6 @@ describe("verifier CDP commands", () => {
}
});
it("sends only commands with registered result schemas", async () => {
const unregistered: string[] = [];
let sends = 0;
for await (const path of scriptFiles(scriptsRoot)) {
const source = await readFile(path, "utf8");
for (const match of source.matchAll(
/\.send\(\s*"([A-Z][A-Za-z]*\.[a-z][A-Za-z]*)"/gu,
)) {
sends += 1;
if (!isCdpCommand(match[1]!))
unregistered.push(`${relative(scriptsRoot, path)}: ${match[1]}`);
}
}
expect(sends).toBeGreaterThan(0);
expect(unregistered).toEqual([]);
});
it("keeps the reply members the verifiers read", () => {
expect(
parseCdpCommandResult("Target.closeTarget", { success: true }),
@@ -1,27 +1,8 @@
import { readdir, readFile } from "node:fs/promises";
import { join, relative, resolve } from "node:path";
import { describe, expect, it } from "vitest";
import { CdpConnection } from "../../src/browser/CdpConnection.js";
import type { CdpEventName } from "../../src/browser/protocol/CdpEvents.js";
import { startFakeCdpBrowser } from "../fixtures/fakeCdpBrowser.js";
const scriptsRoot = resolve(import.meta.dirname, "../../scripts");
async function* scriptFiles(directory: string): AsyncGenerator<string> {
for (const entry of await readdir(directory, { withFileTypes: true })) {
const path = join(directory, entry.name);
if (entry.isDirectory()) yield* scriptFiles(path);
else if (/\.m?js$/u.test(entry.name)) yield path;
}
}
/**
* JavaScript verifiers subscribe to the same `CdpConnection` as TypeScript,
* but the compiler does not check the event names they match. They subscribe
* through `onRegisteredEvent`, which refuses an unregistered name when the
* subscription is made, as `CdpConnection.send` does for commands; a raw
* `onEvent` subscriber compares names no check would catch.
*/
describe("verifier CDP event subscribers", () => {
it("refuses an unregistered event name before subscribing", async () => {
const browser = await startFakeCdpBrowser();
@@ -59,16 +40,4 @@ describe("verifier CDP event subscribers", () => {
await browser.close();
}
});
it("subscribe only through the registered-event check", async () => {
const raw: string[] = [];
for await (const path of scriptFiles(scriptsRoot)) {
const source = await readFile(path, "utf8");
for (const match of source.matchAll(/\.onEvent\(/gu))
raw.push(
`${relative(scriptsRoot, path)}:${String(source.slice(0, match.index).split("\n").length)}`,
);
}
expect(raw).toEqual([]);
});
});