diff --git a/docs/development/testing.md b/docs/development/testing.md index 648ed9f0ed..90598d1ad6 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -969,9 +969,9 @@ cancels from inside final cleanup, and `controlledCleanup` gives children refused or confirmed outcomes with counted retries. `tests/support/providerLifecycleCoverage.ts` maps every registered provider id to its launch, handshake, operation, cancellation, cleanup and retry -regressions, a not-applicable reason, or an explicit gap. Name the test file -and literal title when a change adds or moves a provider lifecycle regression; -the shrink-only gap list closes as stages gain coverage. Check server +regressions or a reason a stage does not apply. Name the test file and the +title at its registration site when a change adds or moves a lifecycle +regression. The check rejects uncovered applicable stages. Check server settlement and retained resources even if the MCP SDK rejects the client promise before the handler finishes. A cancellation during active execution does not cover cancellation during final cleanup. diff --git a/src/ghidra/GhidraRiscvProfile.test.ts b/src/ghidra/GhidraRiscvProfile.test.ts index 2bc300a2ba..3002bb3fea 100644 --- a/src/ghidra/GhidraRiscvProfile.test.ts +++ b/src/ghidra/GhidraRiscvProfile.test.ts @@ -1,4 +1,4 @@ -import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { mkdir, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { parseConfig } from "../config/parseConfig.js"; @@ -15,7 +15,6 @@ import { IdaProvider } from "../ida/IdaProvider.js"; import { parseBinaryTarget } from "../application/BinaryTargetResolver.js"; import { GhidraProvider } from "./GhidraProvider.js"; import { - GHIDRA_RISCV32_DECODED_EXTENSIONS, GHIDRA_RISCV_CALLING_CONVENTIONS, GHIDRA_RISCV_CALLING_CONVENTION_IDS, GHIDRA_RISCV_LANES, @@ -516,34 +515,6 @@ describe("RISC-V lane and calling-convention tables", () => { "ilp32-prototype-v1": { abi: "ilp32", prototype: "__rea_ilp32" }, }); }); - - it("keeps the decision record in docs/providers/ghidra-riscv.md in step with the tables", async () => { - const docs = await readFile( - new URL("../../docs/providers/ghidra-riscv.md", import.meta.url), - "utf8", - ); - for (const lane of GHIDRA_RISCV_LANES) { - for (const loadModel of lane.loadModels) { - expect(docs).toContain( - `\`${ghidraRiscvSupportLane({ ...lane, loadModel })}\``, - ); - } - } - for (const id of GHIDRA_RISCV_CALLING_CONVENTION_IDS) { - const { abi, prototype } = GHIDRA_RISCV_CALLING_CONVENTIONS[id]; - expect(docs).toContain(`Policy \`${id}\`, prototype \`${prototype}\``); - expect(docs).toContain(`### ${abi}\n`); - } - const decoded = - /implements with\s+p-code: (.+?)\.\s+Any\s+other\s+extension/su.exec( - docs, - )?.[1]; - expect( - new Set( - [...(decoded ?? "").matchAll(/`([a-z0-9]+)`/gu)].map((m) => m[1]), - ), - ).toEqual(GHIDRA_RISCV32_DECODED_EXTENSIONS); - }); }); describe("bounded Ghidra RISC-V admission", () => { diff --git a/src/ghidra/GhidraRiscvProfile.ts b/src/ghidra/GhidraRiscvProfile.ts index 1bceee2d64..97c9ad6432 100644 --- a/src/ghidra/GhidraRiscvProfile.ts +++ b/src/ghidra/GhidraRiscvProfile.ts @@ -70,7 +70,7 @@ export const ghidraRiscvSupportLane = (lane: { * the names `Tag_RISCV_arch` records. Lanes with a calling-convention policy * admit an image only when it declares an RV32I base and only these. */ -export const GHIDRA_RISCV32_DECODED_EXTENSIONS: ReadonlySet = new Set([ +const GHIDRA_RISCV32_DECODED_EXTENSIONS: ReadonlySet = new Set([ "m", "a", "f", diff --git a/tests/boundary/providers/native/nativeProvider.test.ts b/tests/boundary/providers/native/nativeProvider.test.ts index e1108a9899..b65ebe1e2f 100644 --- a/tests/boundary/providers/native/nativeProvider.test.ts +++ b/tests/boundary/providers/native/nativeProvider.test.ts @@ -1428,8 +1428,6 @@ describe("native macOS provider inspection", () => { await mkdir(join(app, "Contents/MacOS"), { recursive: true }); await writeFile(executable, fat([0x01000007, 0x0100000c])); await writeFixturePlist(join(app, "Contents/Info.plist")); - const externalPlist = join(directory, "Outside.plist"); - await writeFixturePlist(externalPlist); const client = new NativeMacOSProvider( {}, new FixtureRunner(), @@ -1448,15 +1446,10 @@ describe("native macOS provider inspection", () => { entitlements: { "com.apple.security.app-sandbox": true }, }); const defaultPlist = await client.execute("inspect_plist", {}); - expect(defaultPlist.ok && defaultPlist.value.result).toMatchObject({ - source_path: join(app, "Contents/Info.plist"), - }); - const plist = await client.execute("inspect_plist", { - path: externalPlist, - }); - expect(plist.ok && plist.value.result).toMatchObject({ + if (!defaultPlist.ok) throw defaultPlist.error; + expect(defaultPlist.value.result).toMatchObject({ bundle: { identifier: "com.example.fixture", version: "42" }, - source_path: externalPlist, + source_path: join(app, "Contents/Info.plist"), }); const demangled = await client.execute("demangle_swift", { symbols: ["$s4Test3fooyyF", "plain_symbol"], diff --git a/tests/evaluation/providerLifecycleCoverage.test.ts b/tests/evaluation/providerLifecycleCoverage.test.ts index 26c36a7861..3f5467b656 100644 --- a/tests/evaluation/providerLifecycleCoverage.test.ts +++ b/tests/evaluation/providerLifecycleCoverage.test.ts @@ -1,10 +1,10 @@ import { readFile } from "node:fs/promises"; +import ts from "typescript6"; import { expect, it } from "vitest"; import { CAPABILITY_DECLARATIONS } from "../../src/composition/capabilityDeclarations.js"; import { LIFECYCLE_STAGES, PROVIDER_LIFECYCLE_COVERAGE, - PROVIDER_LIFECYCLE_GAPS, } from "../support/providerLifecycleCoverage.js"; const repositoryRoot = new URL("../../", import.meta.url); @@ -25,26 +25,133 @@ const tableGaps = () => ), ); +const fixtureRegistrars = new Map([ + ["cliFixture.js", "cliTest"], + ["mcpFixture.js", "mcpTest"], + ["workspaceCliFixture.js", "workspaceCliTest"], +]); +const registrationModifiers = new Set([ + "concurrent", + "each", + "fails", + "for", + "runIf", + "sequential", + "skipIf", +]); + +const testTitles = (path: string, source: string): ReadonlySet => { + const file = ts.createSourceFile(path, source, ts.ScriptTarget.Latest, true); + const registrars = new Set(); + for (const statement of file.statements) { + if ( + !ts.isImportDeclaration(statement) || + !ts.isStringLiteral(statement.moduleSpecifier) + ) + continue; + const bindings = statement.importClause?.namedBindings; + if (bindings === undefined || !ts.isNamedImports(bindings)) continue; + const module = statement.moduleSpecifier.text; + for (const binding of bindings.elements) { + const imported = (binding.propertyName ?? binding.name).text; + const fixture = fixtureRegistrars.get(module.split("/").at(-1) ?? ""); + if ( + (module === "vitest" && (imported === "it" || imported === "test")) || + (module.startsWith(".") && imported === fixture) + ) + registrars.add(binding.name.text); + } + } + + const isRegistrar = (expression: ts.Expression): boolean => { + if (ts.isIdentifier(expression)) return registrars.has(expression.text); + if (!ts.isCallExpression(expression)) return false; + const modifier = expression.expression; + return ( + ts.isPropertyAccessExpression(modifier) && + registrationModifiers.has(modifier.name.text) && + isRegistrar(modifier.expression) + ); + }; + + for (const statement of file.statements) { + if (!ts.isVariableStatement(statement)) continue; + for (const binding of statement.declarationList.declarations) { + if ( + ts.isIdentifier(binding.name) && + binding.initializer !== undefined && + isRegistrar(binding.initializer) + ) + registrars.add(binding.name.text); + } + } + + const titles = new Set(); + const visit = (node: ts.Node): void => { + if (ts.isCallExpression(node) && isRegistrar(node.expression)) { + const [title, callback] = node.arguments; + if ( + title !== undefined && + callback !== undefined && + (ts.isArrowFunction(callback) || ts.isFunctionExpression(callback)) + ) { + if (ts.isStringLiteralLike(title)) titles.add(title.text); + if (ts.isTemplateExpression(title)) + titles.add(title.getText(file).slice(1, -1)); + } + } + ts.forEachChild(node, visit); + }; + visit(file); + return titles; +}; + +it("counts test registrations instead of quoted source text", () => { + const source = [ + 'import { describe, it as test } from "vitest";', + 'import { workspaceCliTest as cli } from "../support/cli/workspaceCliFixture.js";', + '// test("removed", () => {});', + 'const data = "another removed test";', + 'describe("suite title", () => {});', + 'log("diagnostic", () => {});', + 'const posix = test.skipIf(process.platform === "win32");', + 'posix("conditional", () => {});', + 'test.each([1, 2])("table %d", () => {});', + 'test("with a snapshot", () => expect(value).toMatchInlineSnapshot("snapshot"));', + "cli(`inspect ${format}`, async () => {});", + ].join("\n"); + expect([...testTitles("fixture.test.ts", source)]).toEqual([ + "conditional", + "table %d", + "with a snapshot", + "inspect ${format}", + ]); +}); + it("has exactly one entry per registered provider id", () => { expect(Object.keys(PROVIDER_LIFECYCLE_COVERAGE).sort()).toEqual( registeredProviderIds(), ); }); -it("names an existing test file and literal test title for every covered stage", async () => { +it("names a test registration for every covered stage", async () => { const missing: string[] = []; - const sources = new Map>(); - const source = (path: string) => { - let pending = sources.get(path); + const registrations = new Map< + string, + Promise | undefined> + >(); + const titlesFor = (path: string) => { + let pending = registrations.get(path); if (pending === undefined) { - pending = readFile(new URL(path, repositoryRoot), "utf8").catch( + pending = readFile(new URL(path, repositoryRoot), "utf8").then( + (text) => testTitles(path, text), (cause: unknown) => { if ((cause as NodeJS.ErrnoException).code === "ENOENT") return undefined; throw cause; }, ); - sources.set(path, pending); + registrations.set(path, pending); } return pending; }; @@ -53,15 +160,12 @@ it("names an existing test file and literal test title for every covered stage", const coverage = entry.stages[stage]; if (coverage.kind !== "covered") continue; for (const { path, title } of coverage.tests) { - const text = await source(path); - if (!path.endsWith(".test.ts") || text === undefined) { + const titles = await titlesFor(path); + if (!path.endsWith(".test.ts") || titles === undefined) { missing.push(`${id}:${stage} missing file ${path}`); continue; } - const literal = ['"', "'", "`"].some((quote) => - text.includes(`${quote}${title}${quote}`), - ); - if (!literal) + if (!titles.has(title)) missing.push(`${id}:${stage} missing title in ${path}: ${title}`); } } @@ -69,6 +173,6 @@ it("names an existing test file and literal test title for every covered stage", expect(missing).toEqual([]); }); -it("lists exactly the gap stages in the shrink-only gap list", () => { - expect([...PROVIDER_LIFECYCLE_GAPS].sort()).toEqual(tableGaps().sort()); +it("has no uncovered applicable lifecycle stages", () => { + expect(tableGaps()).toEqual([]); }); diff --git a/tests/support/providerLifecycleCoverage.ts b/tests/support/providerLifecycleCoverage.ts index b9281ff876..1eab914ce9 100644 --- a/tests/support/providerLifecycleCoverage.ts +++ b/tests/support/providerLifecycleCoverage.ts @@ -2,11 +2,9 @@ * Lifecycle regression coverage for every provider id in * `CAPABILITY_DECLARATIONS`. * - * Each stage either names real regression tests (file and literal test title), - * is not applicable for a stated reason, or is an explicit gap. Gaps are also - * listed in `PROVIDER_LIFECYCLE_GAPS`; that list may only shrink, so closing a - * gap means naming its tests here and removing the gap entry. A newly - * registered provider must add its own entry. + * Each stage names a regression by file and title at its registration site, + * or states why the stage does not apply. The coverage check reports any + * applicable stage still marked as a gap. * * Stages: `launch` starts the owned process (including launch failure and * refusal at the launch boundary); `handshake` is the first readiness or @@ -34,7 +32,7 @@ export type LifecycleStage = (typeof LIFECYCLE_STAGES)[number]; export interface LifecycleRegression { /** Repository-relative test file. */ readonly path: string; - /** Literal title of a test in that file. */ + /** Title at the test registration site, before any parameter expansion. */ readonly title: string; } @@ -65,7 +63,6 @@ const notApplicable = (reason: string): StageCoverage => ({ kind: "not_applicable", reason, }); -const gap = (reason: string): StageCoverage => ({ kind: "gap", reason }); const ONE_SHOT = notApplicable( "One-shot command per request: there is no handshake after launch.", @@ -1665,7 +1662,7 @@ export const PROVIDER_LIFECYCLE_COVERAGE: Readonly< covered( test( "tests/acceptance/v8CodeCacheCli.test.ts", - "reports the observed header and payload through the public CLI", + "reports wide and synthetic narrow header extents through the public CLI", ), ), covered( @@ -2175,9 +2172,3 @@ export const PROVIDER_LIFECYCLE_COVERAGE: Readonly< }, }, }; - -/** - * Every stage marked as a gap above, as `providerId:stage`. This list may only - * shrink: close a gap by naming its regression tests and removing the entry. - */ -export const PROVIDER_LIFECYCLE_GAPS: readonly string[] = [];