From 6bb2ad90ac85cae3571ed715c1c9fdaa5f58a3fe Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:09:03 -0500 Subject: [PATCH] test: harden verifier assertions and prune duplicate checks --- scripts/verify-real-ghidra-raw.mjs | 12 +- src/ghidra/GhidraProvider.behavior.test.ts | 11 -- src/ida/IdaConfiguration.test.ts | 4 +- .../evaluation/package/nativeAotBuild.test.ts | 10 +- .../providerFailureClassifierRule.test.ts | 6 +- .../typedResourceRefusalRule.test.ts | 10 +- tests/evaluation/verifierCdpCommands.test.ts | 120 +++++++++++++----- .../verifierCdpEventPayloads.test.ts | 31 ----- 8 files changed, 113 insertions(+), 91 deletions(-) diff --git a/scripts/verify-real-ghidra-raw.mjs b/scripts/verify-real-ghidra-raw.mjs index f98194f846..4c66c844cf 100644 --- a/scripts/verify-real-ghidra-raw.mjs +++ b/scripts/verify-real-ghidra-raw.mjs @@ -20,6 +20,7 @@ import { inspectVerifierGhidraInstallation } from "./lib/ghidra-installation-ins import { requireMcpLifecycleResult, requireMcpOutput, + requireMcpToolError, } from "./lib/mcp-verifier-results.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { @@ -387,7 +388,16 @@ await withVerifierFixture(workspaceFixture, async () => { name: "search_bytes", arguments: { pattern: "c3", cursor: page.normalized_result.next_cursor }, }); - assert.equal(wrongCursor.isError, true); + const cursorError = requireMcpToolError(wrongCursor); + assert.partialDeepStrictEqual(cursorError, { + code: "invalid_request", + details: { + operation: "search_bytes", + issues: [{ path: ["cursor"], reason: "invalid_value" }], + diagnostics: { remote_code: "byte_search_cursor_mismatch" }, + }, + }); + assert.equal(cursorError.recovery, undefined); await call("close_binary"); assert.deepEqual(await readFile(repeatedPath), repeatedBytes); diff --git a/src/ghidra/GhidraProvider.behavior.test.ts b/src/ghidra/GhidraProvider.behavior.test.ts index 40a6c4a01a..787b1cb32f 100644 --- a/src/ghidra/GhidraProvider.behavior.test.ts +++ b/src/ghidra/GhidraProvider.behavior.test.ts @@ -2047,17 +2047,6 @@ describe("Ghidra byte-search cursor projection", () => { }, }); expect(projected).not.toHaveProperty("recovery"); - expect(projected.message).toContain("analysis profile"); - expect(projected.message).toContain("project"); - expect(projected.message).not.toContain("Analysis input is invalid"); - expect(projected.remediation.action).toContain("pattern"); - expect(projected.remediation.action).not.toContain("schema"); - expect(projected.remediation.action).not.toContain("tools/list"); - const bridge = await readFile( - new URL("../../bridge/ghidra/ReaGhidraBridge.java", import.meta.url), - "utf8", - ); - expect(bridge).toContain(`"${CURSOR_MISMATCH}"`); await client.close(); }); diff --git a/src/ida/IdaConfiguration.test.ts b/src/ida/IdaConfiguration.test.ts index 0cd07ef55f..345399babb 100644 --- a/src/ida/IdaConfiguration.test.ts +++ b/src/ida/IdaConfiguration.test.ts @@ -344,7 +344,9 @@ it.each([ { setting: "REA_IDA_MCP_CONFIG", constraint }, ]); expect(parsed.error.message).toContain(constraint); - // The guidance names a migration target the provider guide documents. +}); + +it("points legacy protocol recovery to the documented migration", async () => { expect(LEGACY_PROTOCOL_REMOVED).toContain('"protocol": "native"'); const guide = await fs.readFile("docs/providers/ida-provider.md", "utf8"); expect(guide).toMatch(/^### Migrate a legacy attached registration$/mu); diff --git a/tests/evaluation/package/nativeAotBuild.test.ts b/tests/evaluation/package/nativeAotBuild.test.ts index 6e71db42a7..cfd36c8942 100644 --- a/tests/evaluation/package/nativeAotBuild.test.ts +++ b/tests/evaluation/package/nativeAotBuild.test.ts @@ -48,12 +48,10 @@ it("retries without logging credential-bearing restore diagnostics", async () => }); try { await buildNativeAotFixture(run, options); - expect( - logged.mock.calls.flat().every((value) => typeof value === "string"), - ).toBe(true); - expect(logged.mock.calls.flat().join("\n")).not.toContain( - "fixture-password", - ); + const output = logged.mock.calls.flat(); + expect(logged).toHaveBeenCalledWith(expect.stringContaining("attempt 2/2")); + expect(output.every((value) => typeof value === "string")).toBe(true); + expect(output.join("\n")).not.toContain("fixture-password"); } finally { logged.mockRestore(); } diff --git a/tests/evaluation/providerFailureClassifierRule.test.ts b/tests/evaluation/providerFailureClassifierRule.test.ts index f8576d8505..92282988c5 100644 --- a/tests/evaluation/providerFailureClassifierRule.test.ts +++ b/tests/evaluation/providerFailureClassifierRule.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { readFile } from "node:fs/promises"; +import { readFile, stat } from "node:fs/promises"; import { fileURLToPath } from "node:url"; import { RuleTester } from "oxlint/plugins-dev"; import rule from "../../scripts/lint/rules/provider-failure-classifier.mjs"; @@ -64,12 +64,12 @@ it("keeps the classifier baseline to existing source files", async () => { ), ) as { owners: string[]; counts: Record }; for (const owner of baseline.owners) - await expect(readFile(owner, "utf8")).resolves.toBeTypeOf("string"); + expect((await stat(owner)).isFile(), owner).toBe(true); for (const [file, count] of Object.entries(baseline.counts)) { expect(file).toMatch(/^src\/.+\.ts$/u); expect(count).toBeGreaterThan(0); // The lint run reports an entry above a file's real count as stale; this // catches entries whose file no longer exists, which lint never visits. - await expect(readFile(file, "utf8")).resolves.toBeTypeOf("string"); + expect((await stat(file)).isFile(), file).toBe(true); } }); diff --git a/tests/evaluation/typedResourceRefusalRule.test.ts b/tests/evaluation/typedResourceRefusalRule.test.ts index 58f479f73e..d60db867be 100644 --- a/tests/evaluation/typedResourceRefusalRule.test.ts +++ b/tests/evaluation/typedResourceRefusalRule.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { readFile } from "node:fs/promises"; +import { readFile, stat } from "node:fs/promises"; import { resolve } from "node:path"; import { RuleTester } from "oxlint/plugins-dev"; import rule from "../../scripts/lint/rules/typed-resource-refusal.mjs"; @@ -50,7 +50,7 @@ new RuleTester({ languageOptions: { parserOptions: { lang: "ts" } } }).run( }, ); -it("keeps the untyped-refusal baseline to files that still need migration", async () => { +it("keeps the untyped-refusal baseline to existing source files", async () => { const baseline = JSON.parse( await readFile( "scripts/lint/rules/typed-resource-refusal-baseline.json", @@ -60,9 +60,7 @@ it("keeps the untyped-refusal baseline to files that still need migration", asyn for (const [file, count] of Object.entries(baseline)) { expect(file).toMatch(/^src\/.+\.ts$/u); expect(count).toBeGreaterThan(0); - // A file whose untyped constructions are gone must leave the baseline. - expect(await readFile(file, "utf8")).toContain( - "new AnalysisResourceConstraintError(", - ); + // The rule checks counts in files lint visits. A removed file gets no visit. + expect((await stat(file)).isFile(), file).toBe(true); } }); diff --git a/tests/evaluation/verifierCdpCommands.test.ts b/tests/evaluation/verifierCdpCommands.test.ts index cbc58839e7..497b96740b 100644 --- a/tests/evaluation/verifierCdpCommands.test.ts +++ b/tests/evaluation/verifierCdpCommands.test.ts @@ -1,5 +1,7 @@ import { readdir, readFile } from "node:fs/promises"; import { join, relative, resolve } from "node:path"; +import { parse } from "@babel/parser"; +import { traverseFast, type Node } from "@babel/types"; import { describe, expect, it } from "vitest"; import { CdpConnection } from "../../src/browser/CdpConnection.js"; import { startFakeCdpBrowser } from "../fixtures/fakeCdpBrowser.js"; @@ -19,21 +21,92 @@ async function* scriptFiles(directory: string): AsyncGenerator { } } -/** - * JavaScript verifiers send through the same `CdpConnection` and Playwright - * sessions as TypeScript, but the compiler does not check their method names. - * - * The structural check is at the send owners: `CdpConnection.send` and the - * Playwright session refuse an unregistered command before it reaches the - * browser, and `CdpConnection.onRegisteredEvent` and the Playwright session - * refuse an unregistered event name when subscribing (see - * `verifierCdpEventPayloads.test.ts`). Type-checking the verifier scripts - * themselves is not practical (checkJs over them and the built modules they - * import reports about 17,000 implicit `any` errors), so the text scan below - * is only an early warning for verifiers that rarely run. It sees - * string-literal sends only: a name built at run time or passed through a - * variable is not scanned, and is caught only when that verifier runs. - */ +const cdpMethod = /^[A-Z][A-Za-z]*\.[a-z][A-Za-z]*$/u; + +function staticString(node: Node | undefined): string | undefined { + if (node?.type === "StringLiteral") return node.value; + if (node?.type === "TemplateLiteral" && node.expressions.length === 0) + return node.quasis[0]?.value.cooked ?? undefined; + return undefined; +} + +// Guard literal sends in rarely run verifiers. The runtime send owners check +// computed command names, and registered-event subscribers validate event names. +function verifierCdpCalls(source: string) { + const commands: { name: string; line: number }[] = []; + const rawEvents: number[] = []; + const tree = parse(source, { + sourceType: "unambiguous", + plugins: ["typescript"], + }); + traverseFast(tree, (node) => { + if ( + node.type !== "CallExpression" && + node.type !== "OptionalCallExpression" + ) + return; + const { callee } = node; + if ( + callee.type !== "MemberExpression" && + callee.type !== "OptionalMemberExpression" + ) + return; + const method = + !callee.computed && callee.property.type === "Identifier" + ? callee.property.name + : staticString(callee.property); + const line = callee.loc?.start.line ?? 1; + if (method === "onEvent") { + rawEvents.push(line); + } else if (method === "send") { + const name = staticString(node.arguments[0]); + if (name !== undefined && cdpMethod.test(name)) + commands.push({ name, line }); + } + }); + return { commands, rawEvents }; +} + +describe("verifier CDP source references", () => { + it("finds static calls independent of quoting and ignores source-like text", () => { + const source = [ + '// session.send("Page.comment"); session.onEvent(handler);', + `const example = 'session.send("Page.example"); session.onEvent(handler);';`, + "session.send ('Page.notRegistered');", + 'session["send"](`DOM.getDocument`);', + "session.onEvent (handler);", + "session?.['onEvent'](handler);", + "session.onRegisteredEvent('Page.lifecycleEvent', handler);", + "session.send(method);", + ].join("\n"); + expect(verifierCdpCalls(source)).toEqual({ + commands: [ + { name: "Page.notRegistered", line: 3 }, + { name: "DOM.getDocument", line: 4 }, + ], + rawEvents: [5, 6], + }); + }); + + it("uses registered commands and excludes raw event subscribers", async () => { + const unregistered: string[] = []; + const rawEvents: string[] = []; + let sends = 0; + for await (const path of scriptFiles(scriptsRoot)) { + const calls = verifierCdpCalls(await readFile(path, "utf8")); + const file = relative(scriptsRoot, path); + sends += calls.commands.length; + for (const { name, line } of calls.commands) { + if (!isCdpCommand(name)) unregistered.push(`${file}:${line}: ${name}`); + } + for (const line of calls.rawEvents) rawEvents.push(`${file}:${line}`); + } + expect(sends).toBeGreaterThan(0); + expect(unregistered).toEqual([]); + expect(rawEvents).toEqual([]); + }); +}); + describe("verifier CDP commands", () => { it("refuses an unregistered command at the connection before sending it", async () => { const browser = await startFakeCdpBrowser(); @@ -57,23 +130,6 @@ describe("verifier CDP commands", () => { } }); - it("sends only commands with registered result schemas", async () => { - const unregistered: string[] = []; - let sends = 0; - for await (const path of scriptFiles(scriptsRoot)) { - const source = await readFile(path, "utf8"); - for (const match of source.matchAll( - /\.send\(\s*"([A-Z][A-Za-z]*\.[a-z][A-Za-z]*)"/gu, - )) { - sends += 1; - if (!isCdpCommand(match[1]!)) - unregistered.push(`${relative(scriptsRoot, path)}: ${match[1]}`); - } - } - expect(sends).toBeGreaterThan(0); - expect(unregistered).toEqual([]); - }); - it("keeps the reply members the verifiers read", () => { expect( parseCdpCommandResult("Target.closeTarget", { success: true }), diff --git a/tests/evaluation/verifierCdpEventPayloads.test.ts b/tests/evaluation/verifierCdpEventPayloads.test.ts index f2acda440f..020b6cfffa 100644 --- a/tests/evaluation/verifierCdpEventPayloads.test.ts +++ b/tests/evaluation/verifierCdpEventPayloads.test.ts @@ -1,27 +1,8 @@ -import { readdir, readFile } from "node:fs/promises"; -import { join, relative, resolve } from "node:path"; import { describe, expect, it } from "vitest"; import { CdpConnection } from "../../src/browser/CdpConnection.js"; import type { CdpEventName } from "../../src/browser/protocol/CdpEvents.js"; import { startFakeCdpBrowser } from "../fixtures/fakeCdpBrowser.js"; -const scriptsRoot = resolve(import.meta.dirname, "../../scripts"); - -async function* scriptFiles(directory: string): AsyncGenerator { - for (const entry of await readdir(directory, { withFileTypes: true })) { - const path = join(directory, entry.name); - if (entry.isDirectory()) yield* scriptFiles(path); - else if (/\.m?js$/u.test(entry.name)) yield path; - } -} - -/** - * JavaScript verifiers subscribe to the same `CdpConnection` as TypeScript, - * but the compiler does not check the event names they match. They subscribe - * through `onRegisteredEvent`, which refuses an unregistered name when the - * subscription is made, as `CdpConnection.send` does for commands; a raw - * `onEvent` subscriber compares names no check would catch. - */ describe("verifier CDP event subscribers", () => { it("refuses an unregistered event name before subscribing", async () => { const browser = await startFakeCdpBrowser(); @@ -59,16 +40,4 @@ describe("verifier CDP event subscribers", () => { await browser.close(); } }); - - it("subscribe only through the registered-event check", async () => { - const raw: string[] = []; - for await (const path of scriptFiles(scriptsRoot)) { - const source = await readFile(path, "utf8"); - for (const match of source.matchAll(/\.onEvent\(/gu)) - raw.push( - `${relative(scriptsRoot, path)}:${String(source.slice(0, match.index).split("\n").length)}`, - ); - } - expect(raw).toEqual([]); - }); });