diff --git a/scripts/lib/catalog-cli.mjs b/scripts/lib/catalog-cli.mjs index 5a63de0c40..58de1c6872 100644 --- a/scripts/lib/catalog-cli.mjs +++ b/scripts/lib/catalog-cli.mjs @@ -12,6 +12,8 @@ export const createCliInventory = (cli) => { if ("_alias" in entry) aliases.push({ name, target: entry.target }); else primary.push(name); } + if (primary.length === 0) + throw new Error("Incur exposed no primary REA CLI commands"); return { primary: primary.sort((left, right) => left.localeCompare(right)), aliases: aliases.sort((left, right) => left.name.localeCompare(right.name)), diff --git a/src/application/CapabilityInventory.test.ts b/src/application/CapabilityInventory.test.ts index d70e4e479f..d742d6c2f3 100644 --- a/src/application/CapabilityInventory.test.ts +++ b/src/application/CapabilityInventory.test.ts @@ -725,24 +725,6 @@ describe("binary_overview capability requirements", () => { }); }); - it("admits a provider without optional segment enumeration", () => { - const capabilities = [ - "current_document", - "list_procedures", - "list_strings", - ].map((operation) => ({ operation, available: true, reason: null })); - expect( - entry( - "binary_overview", - status({ open: true, kind: "executable", capabilities }), - ), - ).toMatchObject({ - available: true, - reason: "available", - remediation: null, - }); - }); - it("keeps binary overview available when a declared segment facet is unavailable", () => { const capabilities = [ { diff --git a/src/application/adb/AdbDeviceAnalysisService.test.ts b/src/application/adb/AdbDeviceAnalysisService.test.ts index 9cd84d3f2e..1760fafe26 100644 --- a/src/application/adb/AdbDeviceAnalysisService.test.ts +++ b/src/application/adb/AdbDeviceAnalysisService.test.ts @@ -3,7 +3,10 @@ import { expect, it } from "vitest"; import { AdbDeviceAnalysisService } from "./AdbDeviceAnalysisService.js"; import type { AdbDeviceAnalysisPort } from "./AdbDeviceAnalysisPort.js"; import { createAnalysisExecution } from "../AnalysisProvider.js"; -import { AnalysisCancelledError } from "../../domain/analysisErrorCore.js"; +import { + AnalysisCancelledError, + AnalysisInputError, +} from "../../domain/analysisErrorCore.js"; import { parseEvidence } from "../../domain/evidence.js"; import { ok } from "../../domain/result.js"; @@ -34,6 +37,7 @@ const refusedPort: AdbDeviceAnalysisPort = { }; it.each([ + [{}], [{ serial: "" }], [{ serial: "has space" }], [{ serial: "emulator-5554", scope: "unknown" }], @@ -48,9 +52,15 @@ it.each([ it("rejects an unknown operation before any effect", async () => { const service = new AdbDeviceAnalysisService(refusedPort); - expect( - await service.execute("inspect_adb_device" as never, {}), - ).toMatchObject({ ok: false, error: { _tag: "AnalysisInputError" } }); + const result = await service.execute("unknown_adb_operation" as never, {}); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected an unknown operation refusal"); + expect(result.error).toBeInstanceOf(AnalysisInputError); + if (!(result.error instanceof AnalysisInputError)) throw result.error; + expect(result.error.operation).toBe("unknown_adb_operation"); + expect(result.error.issues).toContainEqual( + expect.objectContaining({ path: ["operation"] }), + ); }); it("composes observed Evidence with the caller parameters inline", async () => { diff --git a/src/artifacts/pyinstaller/PyzArchive.test.ts b/src/artifacts/pyinstaller/PyzArchive.test.ts index d1f3948127..51084534c6 100644 --- a/src/artifacts/pyinstaller/PyzArchive.test.ts +++ b/src/artifacts/pyinstaller/PyzArchive.test.ts @@ -18,8 +18,9 @@ it("lists modules from a supported PYZ and leaves an unknown magic opaque", asyn modules: [{ name: "json.decoder", typeCode: 0, body }], }), ); - expect(parsed.ok).toBe(true); - if (!parsed.ok || parsed.value.kind !== "entries") return; + expect(parsed).toMatchObject({ ok: true, value: { kind: "entries" } }); + if (!parsed.ok || parsed.value.kind !== "entries") + throw new Error("Expected PYZ entries"); expect(parsed.value.pythonVersion).toBe("3.9"); expect(parsed.value.entries).toMatchObject([ { name: "json.decoder", typeCode: 0, role: "module" }, @@ -31,8 +32,9 @@ it("lists modules from a supported PYZ and leaves an unknown magic opaque", asyn modules: [{ name: "json.decoder", typeCode: 0, body }], }), ); - expect(opaque.ok).toBe(true); - if (!opaque.ok || opaque.value.kind !== "opaque") return; + expect(opaque).toMatchObject({ ok: true, value: { kind: "opaque" } }); + if (!opaque.ok || opaque.value.kind !== "opaque") + throw new Error("Expected an opaque PYZ"); expect(Buffer.compare(opaque.value.pythonMagic, Buffer.alloc(4))).toBe(0); expect(opaque.value.limitation).toContain("left opaque"); expect(opaque.value.pythonVersion).toBeUndefined(); @@ -45,8 +47,9 @@ it("leaves a PYZ opaque when the table uses an unsupported marshal type", async toc: Buffer.from([0x4e]), }); const parsed = await readImage(image); - expect(parsed.ok).toBe(true); - if (!parsed.ok || parsed.value.kind !== "opaque") return; + expect(parsed).toMatchObject({ ok: true, value: { kind: "opaque" } }); + if (!parsed.ok || parsed.value.kind !== "opaque") + throw new Error("Expected an opaque PYZ"); expect(parsed.value.limitation).toContain("left opaque"); expect(parsed.value.pythonMagic).toEqual(PYTHON_39_MAGIC); expect(parsed.value.pythonVersion).toBe("3.9"); diff --git a/src/browser/CdpBrowserProvider.admission.test.ts b/src/browser/CdpBrowserProvider.admission.test.ts index 4a698b92b6..c5f4a4d0c3 100644 --- a/src/browser/CdpBrowserProvider.admission.test.ts +++ b/src/browser/CdpBrowserProvider.admission.test.ts @@ -11,6 +11,14 @@ const input = analyzeWebBundleInputSchema.parse({ include_script_sources: true, }); +const assertSettled = (outcomes: PromiseSettledResult[]) => { + const failures = outcomes.flatMap((outcome) => + outcome.status === "rejected" ? [outcome.reason as unknown] : [], + ); + if (failures.length > 0) + throw new AggregateError(failures, "Browser admission cleanup failed"); +}; + it("shares a capture limit when browser providers use one admission", async () => { const admission = new WebBundleAnalysisAdmission(); const first = createBrowserObservationProvider(admission); @@ -28,8 +36,12 @@ it("shares a capture limit when browser providers use one admission", async () = }, }); } finally { - await pending; - await Promise.allSettled([first.close?.(), second.close?.()]); + const settled = await Promise.allSettled([pending]); + const closed = await Promise.allSettled([ + first.close?.(), + second.close?.(), + ]); + assertSettled([...settled, ...closed]); } }); @@ -46,11 +58,12 @@ it("does not share a capture limit across browser providers", async () => { "resource_constraint", ); } finally { - await Promise.allSettled([ + const settled = await Promise.allSettled([ pending, other, first.close?.(), second.close?.(), ]); + assertSettled(settled); } }); diff --git a/src/browser/PlaywrightElectronActiveFuses.test.ts b/src/browser/PlaywrightElectronActiveFuses.test.ts index eb641f3595..5037c8fb51 100644 --- a/src/browser/PlaywrightElectronActiveFuses.test.ts +++ b/src/browser/PlaywrightElectronActiveFuses.test.ts @@ -22,20 +22,15 @@ describe("assertNodeCliInspectArgumentsSupported", () => { const image = thinFuseImage(OBSIDIAN_FUSE_BYTES); // index 3 is disabled await writeFile(binaryPath, image.bytes); - await expect( - assertNodeCliInspectArgumentsSupported(binaryPath), - ).rejects.toThrow(AnalysisUnsupportedTargetError); - - try { - await assertNodeCliInspectArgumentsSupported(binaryPath); - } catch (error) { - expect(error).toBeInstanceOf(AnalysisUnsupportedTargetError); - const targetError = error as AnalysisUnsupportedTargetError; - expect(targetError.operation).toBe("capture_electron_scenario"); - expect(targetError.path).toBe(binaryPath); - expect(targetError.message).toContain("EnableNodeCliInspectArguments"); - expect(targetError.remediationAction).toContain("inspect_electron_page"); - } + const error: unknown = await assertNodeCliInspectArgumentsSupported( + binaryPath, + ).catch((cause: unknown) => cause); + if (!(error instanceof AnalysisUnsupportedTargetError)) + throw new Error("Expected unsupported target error", { cause: error }); + expect(error.operation).toBe("capture_electron_scenario"); + expect(error.path).toBe(binaryPath); + expect(error.message).toContain("EnableNodeCliInspectArguments"); + expect(error.remediationAction).toContain("inspect_electron_page"); }); it("passes when EnableNodeCliInspectArguments fuse is enabled", async () => { @@ -132,19 +127,14 @@ describe("assertNodeCliInspectArgumentsSupported", () => { const frameworkBinary = join(frameworkDir, "Electron Framework"); await writeFile(frameworkBinary, thinFuseImage(OBSIDIAN_FUSE_BYTES).bytes); - await expect( - assertNodeCliInspectArgumentsSupported(launcherBinary), - ).rejects.toThrow(AnalysisUnsupportedTargetError); - - try { - await assertNodeCliInspectArgumentsSupported(launcherBinary); - } catch (error) { - expect(error).toBeInstanceOf(AnalysisUnsupportedTargetError); - const targetError = error as AnalysisUnsupportedTargetError; - expect(targetError.operation).toBe("capture_electron_scenario"); - expect(targetError.path).toBe(launcherBinary); - expect(targetError.message).toContain("EnableNodeCliInspectArguments"); - } + const error: unknown = await assertNodeCliInspectArgumentsSupported( + launcherBinary, + ).catch((cause: unknown) => cause); + if (!(error instanceof AnalysisUnsupportedTargetError)) + throw new Error("Expected unsupported target error", { cause: error }); + expect(error.operation).toBe("capture_electron_scenario"); + expect(error.path).toBe(launcherBinary); + expect(error.message).toContain("EnableNodeCliInspectArguments"); }); it("evaluates matching architecture slice in universal Mach-O binary", async () => { diff --git a/src/browser/PlaywrightScenarioRequest.test.ts b/src/browser/PlaywrightScenarioRequest.test.ts index 60c3fa24ae..b962012c9c 100644 --- a/src/browser/PlaywrightScenarioRequest.test.ts +++ b/src/browser/PlaywrightScenarioRequest.test.ts @@ -1,5 +1,5 @@ import type { Page, Route } from "playwright-core"; -import { afterEach, describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it, onTestFinished, vi } from "vitest"; import { browserScenarioSchema } from "../domain/browserScenario.js"; import type { BrowserScenarioAction } from "../domain/browserScenarioValues.js"; @@ -1935,6 +1935,9 @@ describe("saved response limits and deadlines", () => { it.each(["array", "embedded", "headers"])( "refuses repeated large saved JSON values in %s before expansion or browser dispatch", async (kind) => { + onTestFinished(() => { + vi.unstubAllGlobals(); + }); const parsed = scenario([ requestStep({ save: [{ name: "large", json_pointer: "/value" }] }), requestStep({ diff --git a/src/domain/analysisView/EnvironmentStateWriteSimulation.test.ts b/src/domain/analysisView/EnvironmentStateWriteSimulation.test.ts index 46dcfa24eb..8d90e0f598 100644 --- a/src/domain/analysisView/EnvironmentStateWriteSimulation.test.ts +++ b/src/domain/analysisView/EnvironmentStateWriteSimulation.test.ts @@ -2301,14 +2301,8 @@ describe("entities an unmodelled write made", () => { ["GET", "/api/books/3", BOOK_NOT_FOUND, 404] as TimedCaptureRow, ]; - it("serves a delete of a book an unmodelled import made, which a later item read showed", () => { - // The rename ends the initial reads, so the third book is no initial one. - expect( - gated(deletedAfterRead([RENAME, IMPORT]), "delete_books_by_id"), - ).toMatchObject({ served: true, checked_record_ordinals: [7] }); - }); - it("takes a book the state never held only when a write the gate does not model came first", () => { + // The rename ends the initial reads, so the third book is no initial one. expect( gated(deletedAfterRead([RENAME, IMPORT]), "delete_books_by_id"), ).toMatchObject({ served: true, checked_record_ordinals: [7] }); diff --git a/src/domain/analysisView/analysisViewCycles.test.ts b/src/domain/analysisView/analysisViewCycles.test.ts index 791fc192ad..494e8814fb 100644 --- a/src/domain/analysisView/analysisViewCycles.test.ts +++ b/src/domain/analysisView/analysisViewCycles.test.ts @@ -27,6 +27,19 @@ const allowedCycles = [ ["environmentStateWriteAddress.ts", "environmentStateWriteEffects.ts"], ]; +const allowedCycleEdges = [ + "StateRecordChronology.ts -> environmentStateRules.ts", + "environmentStateListQuery.ts -> environmentStateRules.ts", + "environmentStateRules.ts -> StateRecordChronology.ts", + "environmentStateRules.ts -> environmentStateListQuery.ts", + "environmentStateRules.ts -> environmentStateWriteRules.ts", + "environmentStateWriteRules.ts -> environmentStateRules.ts", + "environmentStateGate.ts -> environmentStateListFacts.ts", + "environmentStateListFacts.ts -> environmentStateGate.ts", + "environmentStateWriteAddress.ts -> environmentStateWriteEffects.ts", + "environmentStateWriteEffects.ts -> environmentStateWriteAddress.ts", +].sort(); + const productionFiles = (directory: string): string[] => { const files: string[] = []; for (const name of readdirSync(directory)) { @@ -106,11 +119,18 @@ const stronglyConnected = ( describe("analysis view import cycles", () => { it("keeps projectors from importing the request and result hub", () => { - const cycles = stronglyConnected(importGraph()).map((cycle) => - cycle.join("\n"), - ); + const graph = importGraph(); + const cycles = stronglyConnected(graph).map((cycle) => cycle.join("\n")); expect(cycles).toEqual( allowedCycles.map((cycle) => [...cycle].sort().join("\n")), ); + const internalEdges = allowedCycles.flatMap((cycle) => + cycle.flatMap((source) => + [...(graph.get(source) ?? [])] + .filter((target) => cycle.includes(target)) + .map((target) => `${source} -> ${target}`), + ), + ); + expect(internalEdges.sort()).toEqual(allowedCycleEdges); }); }); diff --git a/src/domain/javascript/javascriptSemanticCallableResults.test.ts b/src/domain/javascript/javascriptSemanticCallableResults.test.ts index f2755e5e4e..ff1d2c9cd8 100644 --- a/src/domain/javascript/javascriptSemanticCallableResults.test.ts +++ b/src/domain/javascript/javascriptSemanticCallableResults.test.ts @@ -46,6 +46,18 @@ it("keeps identical source text in separate syntax trees distinct", () => { expect(semanticCallableResultExpressions(second)[0]?.node).toBe(second.body); }); +it("retains recovered yield syntax in a non-generator return", () => { + const parsed = parseJavaScriptSource("function f(){ return (yield 1); }"); + expect(parsed?.errors.length).toBeGreaterThan(0); + const callable = parsed?.program.body[0]; + if (!t.isFunctionDeclaration(callable)) + throw new Error("Expected recovered callable fixture"); + const results = semanticCallableResultExpressions(callable); + expect(results.map(({ kind }) => kind)).toEqual(["return", "yield"]); + expect(t.isYieldExpression(results[0]?.node)).toBe(true); + expect(t.isNumericLiteral(results[1]?.node, { value: 1 })).toBe(true); +}); + it("retains empty results and bare return/yield source sites", () => { const empty = t.functionExpression(null, [], t.blockStatement([])); const emptyResult = semanticCallableResultExpressions(empty); diff --git a/src/domain/javascript/javascriptSemanticCycles.test.ts b/src/domain/javascript/javascriptSemanticCycles.test.ts index fa2ed595a8..0019aae4ee 100644 --- a/src/domain/javascript/javascriptSemanticCycles.test.ts +++ b/src/domain/javascript/javascriptSemanticCycles.test.ts @@ -30,6 +30,18 @@ const allowedCycles = [ ["javascriptStaticAnalysisFindings.ts", "javascriptStaticAnalysisState.ts"], ]; +const allowedCycleEdges = [ + "electronStaticAnalysisTypes.ts -> javascriptStaticAnalysisTypes.ts", + "javascriptRouteFacts.ts -> javascriptSourceCoordinates.ts", + "javascriptSourceCoordinates.ts -> javascriptStaticAnalysisTypes.ts", + "javascriptStaticAnalysisTypes.ts -> electronStaticAnalysisTypes.ts", + "javascriptStaticAnalysisTypes.ts -> javascriptRouteFacts.ts", + "javascriptRuntimeReconciliationActive.ts -> javascriptRuntimeReconciliationCaptureParsing.ts", + "javascriptRuntimeReconciliationCaptureParsing.ts -> javascriptRuntimeReconciliationActive.ts", + "javascriptStaticAnalysisFindings.ts -> javascriptStaticAnalysisState.ts", + "javascriptStaticAnalysisState.ts -> javascriptStaticAnalysisFindings.ts", +].sort(); + const productionFiles = (directory: string): string[] => { const files: string[] = []; for (const name of readdirSync(directory)) { @@ -109,11 +121,18 @@ const stronglyConnected = ( describe("javascript semantic import cycles", () => { it("keeps the semantic modules out of one import cycle", () => { - const cycles = stronglyConnected(importGraph()).map((cycle) => - cycle.join("\n"), - ); + const graph = importGraph(); + const cycles = stronglyConnected(graph).map((cycle) => cycle.join("\n")); expect(cycles).toEqual( allowedCycles.map((cycle) => [...cycle].sort().join("\n")), ); + const internalEdges = allowedCycles.flatMap((cycle) => + cycle.flatMap((source) => + [...(graph.get(source) ?? [])] + .filter((target) => cycle.includes(target)) + .map((target) => `${source} -> ${target}`), + ), + ); + expect(internalEdges.sort()).toEqual(allowedCycleEdges); }); }); diff --git a/src/domain/javascript/javascriptSymbolicBindings.test.ts b/src/domain/javascript/javascriptSymbolicBindings.test.ts index 64366ec8e9..6ff6988d64 100644 --- a/src/domain/javascript/javascriptSymbolicBindings.test.ts +++ b/src/domain/javascript/javascriptSymbolicBindings.test.ts @@ -631,6 +631,7 @@ it("propagates branch guards through object properties and function calls", () = inspect(direct.action); inspect(called.action); `); + expect(inspected).toHaveLength(2); for (const value of inspected) { const functions = symbolicVariants(value).filter( (candidate) => candidate.kind === "function", diff --git a/src/domain/webBundleAnalyzer.test.ts b/src/domain/webBundleAnalyzer.test.ts index 58fe478171..1c086eb01c 100644 --- a/src/domain/webBundleAnalyzer.test.ts +++ b/src/domain/webBundleAnalyzer.test.ts @@ -1611,12 +1611,30 @@ it("keeps bare module locations unresolved without inventing URL-relative paths" importScripts("worker.js"); `), ); - for (const specifier of ["package-name", "@scope/package"]) - expect( - result.observations.chunks.edges - .filter((edge) => edge.specifier === specifier) - .every((edge) => edge.resolved_url === null), - ).toBe(true); + const bareEdges = result.observations.chunks.edges.filter( + ({ specifier }) => + specifier === "package-name" || specifier === "@scope/package", + ); + expect(bareEdges).toHaveLength(3); + expect(bareEdges).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + kind: "static_import", + specifier: "package-name", + resolved_url: null, + }), + expect.objectContaining({ + kind: "dynamic_import", + specifier: "@scope/package", + resolved_url: null, + }), + expect.objectContaining({ + kind: "require", + specifier: "package-name", + resolved_url: null, + }), + ]), + ); expect( result.observations.chunks.edges.find( (edge) => edge.specifier === "./actual.js", diff --git a/src/ghidra/GhidraLaunchFailure.test.ts b/src/ghidra/GhidraLaunchFailure.test.ts index 7f6d226de1..7a7bf7e5aa 100644 --- a/src/ghidra/GhidraLaunchFailure.test.ts +++ b/src/ghidra/GhidraLaunchFailure.test.ts @@ -96,30 +96,6 @@ describe("Ghidra launch failure diagnostics", () => { } }); - it("surfaces the retained launcher outcome in the session failure_cause", async () => { - const javaHome = await failingJavaHome( - "#!/bin/sh\necho 'jvm probe exploded' >&2\nexit 3\n", - ); - const runtimeRoot = await mkdtemp(join(tmpdir(), "rea-ghidra-launch-")); - try { - const launched = await probeLauncher(javaHome, runtimeRoot).launch( - launchSession(runtimeRoot), - ); - expect(launched.ok).toBe(false); - if (launched.ok) throw new Error("expected the launch to fail"); - const outcome = launchOutcomeOf(launched.error); - expect(outcome).toBeDefined(); - expect(outcome?.["killed"]).toBe(false); - expect(outcome?.["outputTruncated"]).toBe(false); - expect(outcome?.["stderrTruncated"]).toBe(false); - expect(outcome?.["stderr"]).toContain("jvm probe exploded"); - expect(outcome?.["outputTruncated"]).toBe(false); - } finally { - await rm(javaHome, { recursive: true, force: true }); - await rm(runtimeRoot, { recursive: true, force: true }); - } - }); - it("bounds projected launcher output and marks truncation explicitly", async () => { const oversized = `${"x".repeat(EXPECTED_OUTPUT_BOUND_BYTES)}UNRETAINED-TAIL`; const javaHome = await failingJavaHome( diff --git a/src/ida/IdaNativeProtocol.test.ts b/src/ida/IdaNativeProtocol.test.ts index e8194cbb5e..f22e53235f 100644 --- a/src/ida/IdaNativeProtocol.test.ts +++ b/src/ida/IdaNativeProtocol.test.ts @@ -976,15 +976,6 @@ describe("native IDA malformed observations", () => { }); describe("native IDA attached operation admission", () => { - it("forwards the native producer's explicit reference cap flag", async () => { - const { client } = await fixture(); - const result = await client.execute("xrefs", { address: "0x1000" }); - - if (!result.ok) throw result.error; - expect(result.value.referencesTruncated).toBe(false); - await client.close(); - }); - it("reports the bound document but not headless-only address resolution", async () => { const { client, producer, target } = await fixture(); const containing = await client.execute("resolve_containing_procedure", { diff --git a/src/native/NativeCommandDeadline.test.ts b/src/native/NativeCommandDeadline.test.ts index 421f47e350..dc391f9ce5 100644 --- a/src/native/NativeCommandDeadline.test.ts +++ b/src/native/NativeCommandDeadline.test.ts @@ -48,7 +48,11 @@ const execute = async ( const client = new NativeMacOSProvider({}, runner, "darwin").createClient( nativeMachoTarget("/fixture/app"), ); - return client.execute("list_architectures", parameters); + try { + return await client.execute("list_architectures", parameters); + } finally { + expect((await client.close()).ok).toBe(true); + } }; describe("native_command_timeout_ms", () => { @@ -65,6 +69,7 @@ describe("native_command_timeout_ms", () => { const runner = new RecordingRunner(); const result = await execute(runner, { native_command_timeout_ms: 1_500 }); expect(result.ok, JSON.stringify(result)).toBe(true); + expect(runner.timeouts.length).toBeGreaterThan(0); expect(runner.timeouts.every((timeout) => timeout === 1_500)).toBe(true); }); diff --git a/src/process/ExecFileOutput.test.ts b/src/process/ExecFileOutput.test.ts index 60066cb19a..4bf135651d 100644 --- a/src/process/ExecFileOutput.test.ts +++ b/src/process/ExecFileOutput.test.ts @@ -14,6 +14,28 @@ import { const realSetTimeout = globalThis.setTimeout; +const inlineJavaScriptCharMap: Record = { + "<": "\\u003C", + ">": "\\u003E", + "/": "\\u002F", + "\\": "\\\\", + "\b": "\\b", + "\f": "\\f", + "\n": "\\n", + "\r": "\\r", + "\t": "\\t", + "\0": "\\0", + "\u2028": "\\u2028", + "\u2029": "\\u2029", +}; + +function escapeUnsafeCharsForInlineJavaScript(value: string): string { + return value.replace( + /[<>/\\\b\f\n\r\t\0\u2028\u2029]/g, + (character) => inlineJavaScriptCharMap[character] ?? character, + ); +} + const waitForFile = async (path: string): Promise => { for (let attempt = 0; attempt < 500; attempt += 1) { if ( @@ -297,30 +319,34 @@ describe("execFileOutput cancellation lifecycle", () => { async () => { const directory = await createTestTempDirectory("rea-exec-double-stop-"); const ready = join(directory, "ready"); + const interrupted = join(directory, "interrupted"); const script = [ 'const fs = require("node:fs");', - 'for (const name of ["SIGINT", "SIGTERM"]) process.on(name, () => {});', - `fs.writeFileSync(${JSON.stringify(ready)}, "");`, + `process.on("SIGINT", () => fs.writeFileSync(${escapeUnsafeCharsForInlineJavaScript(JSON.stringify(interrupted))}, ""));`, + `fs.writeFileSync(${escapeUnsafeCharsForInlineJavaScript(JSON.stringify(ready))}, "");`, "setInterval(() => {}, 1000);", ].join("\n"); const controller = new AbortController(); + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); const execution = execFileOutput(process.execPath, ["-e", script], { signal: controller.signal, timeout: 10_000, stopSignal: "SIGINT", - }); - const startedAt = Date.now(); - expect(await waitForFile(ready)).toBe(true); - const remainingBeforeDeadline = Math.max( - 0, - 9_000 - (Date.now() - startedAt), - ); - await new Promise((resolve) => - setTimeout(resolve, remainingBeforeDeadline), - ); - controller.abort(); - - await expect(execution).rejects.toMatchObject({ name: "AbortError" }); + }).catch((cause: unknown) => cause); + try { + expect(await waitForFile(ready)).toBe(true); + await vi.advanceTimersByTimeAsync(9_000); + controller.abort(); + expect(await waitForFile(interrupted)).toBe(true); + await vi.advanceTimersByTimeAsync(1_000); + await vi.advanceTimersByTimeAsync(4_000); + expect(await execution).toMatchObject({ name: "AbortError" }); + } finally { + controller.abort(); + await vi.runAllTimersAsync(); + vi.useRealTimers(); + await execution; + } }, ); @@ -391,17 +417,22 @@ describe.skipIf(process.platform === "win32")( `fs.writeFileSync(${JSON.stringify(ready)}, "");`, "setInterval(() => {}, 1000);", ].join("\n"); + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + const execution = execFileOutput(process.execPath, ["-e", script], { + timeout: 10_000, + stopSignal: "SIGINT", + }).catch((cause: unknown) => cause); try { - const execution = execFileOutput(process.execPath, ["-e", script], { - timeout: 10_000, - stopSignal: "SIGINT", - }); expect(await waitForFile(ready)).toBe(true); - await expect(execution).rejects.toMatchObject({ + await vi.advanceTimersByTimeAsync(10_000); + expect(await execution).toMatchObject({ code: "ETIMEDOUT", signal: null, }); } finally { + await vi.runAllTimersAsync(); + vi.useRealTimers(); + await execution; await rm(directory, { recursive: true, force: true }); } }); diff --git a/tests/acceptance/applications/adbListingNumbers.test.ts b/tests/acceptance/applications/adbListingNumbers.test.ts index 297c171325..0fcaeab621 100644 --- a/tests/acceptance/applications/adbListingNumbers.test.ts +++ b/tests/acceptance/applications/adbListingNumbers.test.ts @@ -6,7 +6,7 @@ import { expect } from "vitest"; import { toolContract } from "../../../src/contracts/toolContracts.js"; import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; -import { cliTest } from "../../support/cli/cliFixture.js"; +import { cliTest, runFileCli } from "../../support/cli/cliFixture.js"; const SERIAL = "listing-numbers-device"; const UNSAFE = "9007199254740993"; @@ -220,10 +220,13 @@ process.stdout.write(require('node:fs').readFileSync(path)); ] as const) { await writeFile(path, listing.stdout); const contract = toolContract(operation); - const output = await cli.run({ + const invocation = { arguments: [operation.replaceAll("_", "-"), SERIAL, "--json"], environment, - }); + }; + const output = await (variant === "safe" + ? cli.run(invocation) + : runFileCli(cli, invocation)); expect(output.exitCode, output.stderr).toBe(0); const evidence = contract.outputSchema.parse(output.json); expect(evidence.parameters).toMatchObject({ serial: SERIAL }); diff --git a/tests/acceptance/applications/adbNumericDeclarations.test.ts b/tests/acceptance/applications/adbNumericDeclarations.test.ts index 9337fe5259..4c1af433ec 100644 --- a/tests/acceptance/applications/adbNumericDeclarations.test.ts +++ b/tests/acceptance/applications/adbNumericDeclarations.test.ts @@ -9,7 +9,7 @@ import { adbDisplayPartialObservationSchema } from "../../../src/domain/adb/adbD import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; -import { cliTest } from "../../support/cli/cliFixture.js"; +import { cliTest, runFileCli } from "../../support/cli/cliFixture.js"; const SERIAL = "numeric-declarations-device"; const PACKAGE = "com.example.numeric"; @@ -220,10 +220,13 @@ cliTest.skipIf(process.platform === "win32")( for (const scenario of displayScenarios) { await writeFile(fixture.size, scenario.size); await writeFile(fixture.density, scenario.density); - const output = await cli.run({ + const invocation = { arguments: ["inspect-adb-display", SERIAL, "--json"], environment: fixture.environment, - }); + }; + const output = await (scenario === displayScenarios[0] + ? cli.run(invocation) + : runFileCli(cli, invocation)); expect(output.exitCode, `${scenario.name}: ${output.stderr}`).toBe(0); const evidence = contract.outputSchema.parse(output.json); expect(evidence.parameters).toEqual({ serial: SERIAL }); @@ -274,10 +277,13 @@ cliTest.skipIf(process.platform === "win32")( "", ].join("\n"), ); - const output = await cli.run({ + const invocation = { arguments: ["inspect-adb-display", SERIAL, "--json"], environment: fixture.environment, - }); + }; + const output = await (value === null + ? cli.run(invocation) + : runFileCli(cli, invocation)); expect(output.exitCode, output.stderr).toBe(1); const projected = analysisErrorProjectionSchema.parse(output.json); expect(projected.code).toBe("unreadable_output"); @@ -444,10 +450,13 @@ cliTest.skipIf(process.platform === "win32")( "", ].join("\n"), ); - const output = await cli.run({ + const invocation = { arguments: ["inspect-adb-package", SERIAL, PACKAGE, "--json"], environment: fixture.environment, - }); + }; + const output = await (scenario === packageScenarios[0] + ? cli.run(invocation) + : runFileCli(cli, invocation)); expect(output.exitCode, `${scenario.name}: ${output.stderr}`).toBe(0); const evidence = contract.outputSchema.parse(output.json); expect(evidence.parameters).toEqual({ serial: SERIAL, package: PACKAGE }); diff --git a/tests/acceptance/applications/adbPackageInventoryCoverage.test.ts b/tests/acceptance/applications/adbPackageInventoryCoverage.test.ts index 59429c7e46..1b0fea8265 100644 --- a/tests/acceptance/applications/adbPackageInventoryCoverage.test.ts +++ b/tests/acceptance/applications/adbPackageInventoryCoverage.test.ts @@ -7,7 +7,7 @@ import { toolContract } from "../../../src/contracts/toolContracts.js"; import { parseEvidence } from "../../../src/domain/evidence.js"; import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; -import { cliTest } from "../../support/cli/cliFixture.js"; +import { cliTest, runFileCli } from "../../support/cli/cliFixture.js"; const PACKAGE_LINE = "package:/data/app/example/base.apk=com.example.app"; const unrecognized = (count: number) => @@ -60,10 +60,13 @@ process.exit(91); ["resolve-adb-packages", "resolve_adb_packages", ["example"]], ["list-adb-packages", "list_adb_packages", []], ] as const) { - const output = await cli.run({ + const invocation = { arguments: [command, "emulator-5554", ...tail, "--json"], environment, - }); + }; + const output = await (count === 0 + ? cli.run(invocation) + : runFileCli(cli, invocation)); expect(output.exitCode, output.stderr).toBe(0); const evidence = parseEvidence(output.json); expect( diff --git a/tests/acceptance/applications/applicationVersionMatchingCli.test.ts b/tests/acceptance/applications/applicationVersionMatchingCli.test.ts index c9c331a6da..cfa19663e0 100644 --- a/tests/acceptance/applications/applicationVersionMatchingCli.test.ts +++ b/tests/acceptance/applications/applicationVersionMatchingCli.test.ts @@ -7,7 +7,7 @@ import { parseEvidence } from "../../../src/domain/evidence.js"; import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; import { applicationVersionComparisonResultSchema } from "../../../src/domain/javascript/javascriptApplicationVersionComparisonSchemas.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; -import { cliTest } from "../../support/cli/cliFixture.js"; +import { cliTest, runFileCli } from "../../support/cli/cliFixture.js"; cliTest( "keeps unavailable source-selection scopes unknown in version comparisons", @@ -69,7 +69,7 @@ cliTest( ) => { const inputPath = join(root, `${name}.json`); await writeFile(inputPath, JSON.stringify({ left, right })); - const response = await cli.run({ + const response = await runFileCli(cli, { arguments: ["compare-application-versions", inputPath, "--json"], environment: { REA_LOG_LEVEL: "silent" }, }); @@ -493,7 +493,7 @@ cliTest( const compare = async (left: unknown, right: unknown) => { const inputPath = join(root, `comparison-${Math.random()}.json`); await writeFile(inputPath, JSON.stringify({ left, right })); - const response = await cli.run({ + const response = await runFileCli(cli, { arguments: ["compare-application-versions", inputPath, "--json"], environment: { REA_LOG_LEVEL: "silent" }, }); diff --git a/tests/acceptance/applications/interfaceBuilderPlistNumbers.test.ts b/tests/acceptance/applications/interfaceBuilderPlistNumbers.test.ts index 93378509d8..25f74d6f70 100644 --- a/tests/acceptance/applications/interfaceBuilderPlistNumbers.test.ts +++ b/tests/acceptance/applications/interfaceBuilderPlistNumbers.test.ts @@ -9,7 +9,11 @@ import { interfaceBuilderAnalysisSchema } from "../../../src/domain/apple/interf import { thinMach } from "../../../src/domain/binaryTarget.fixture.js"; import { parseEvidence } from "../../../src/domain/evidence.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; -import { cliTest } from "../../support/cli/cliFixture.js"; +import { + cliTest, + runFileCli, + type TestCli, +} from "../../support/cli/cliFixture.js"; const unknownMixedInteger = { $plist_type: "unknown", @@ -30,6 +34,14 @@ const scenarios = [ "segmented keys", ] as const; +const inspectArchive = (cli: TestCli, app: string, fresh: boolean) => { + const invocation = { + arguments: ["decode-interface-builder", app, "--json"], + environment: { REA_LOG_LEVEL: "silent", REA_ANALYSIS_PROVIDER: "auto" }, + }; + return fresh ? cli.run(invocation) : runFileCli(cli, invocation); +}; + cliTest.for(scenarios)( "preserves numeric meaning and graph identity for %s Interface Builder attributes", async (scenario, { cli }) => { @@ -113,10 +125,11 @@ cliTest.for(scenarios)( .replaceAll("safe", "sa"), ); await writeFile(join(resources, "Main.nib"), bytes); - const output = await cli.run({ - arguments: ["decode-interface-builder", app, "--json"], - environment: { REA_LOG_LEVEL: "silent", REA_ANALYSIS_PROVIDER: "auto" }, - }); + const output = await inspectArchive( + cli, + app, + scenario === "XML" || scenario === "binary", + ); expect(output.exitCode, output.stderr).toBe(0); const result = interfaceBuilderAnalysisSchema.parse( parseEvidence(output.json).normalized_result, diff --git a/tests/acceptance/applications/javascriptFeatureTraceSeedSelectionCli.test.ts b/tests/acceptance/applications/javascriptFeatureTraceSeedSelectionCli.test.ts index 477160d48b..b69d75dcca 100644 --- a/tests/acceptance/applications/javascriptFeatureTraceSeedSelectionCli.test.ts +++ b/tests/acceptance/applications/javascriptFeatureTraceSeedSelectionCli.test.ts @@ -13,6 +13,7 @@ import { findApplicationFeatureSeeds } from "../../../src/domain/javascript/java import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { cliTest, + runFileCli, type SharedMcpCall, type TestCli, } from "../../support/cli/cliFixture.js"; @@ -166,9 +167,10 @@ const traceThroughCli = async ( input, JSON.stringify({ application, seed: scenario.seed, direction: "both" }), ); - const response = await cli.run({ + const invocation = { arguments: ["trace-application-feature", input, "--json"], - }); + }; + const response = await runFileCli(cli, invocation); expect(response.exitCode, `${scenario.name} via CLI`).toBe(0); const evidence = toolContract("trace_application_feature").outputSchema.parse( response.json, diff --git a/tests/acceptance/applications/javascriptReturnedReferences.test.ts b/tests/acceptance/applications/javascriptReturnedReferences.test.ts index e5d00fcf40..d10ccf47ba 100644 --- a/tests/acceptance/applications/javascriptReturnedReferences.test.ts +++ b/tests/acceptance/applications/javascriptReturnedReferences.test.ts @@ -183,7 +183,9 @@ cliTest( cliTest( "analyzes wide receiver returns without losing escaped reference facts", ({ cli }) => { - const width = 160_000; + // Exceed the argument limit with the worker's 2048 KiB stack as well as + // Node's default stack, so a spread into push still fails this regression. + const width = 300_000; const object = Array.from( { length: width }, (_, index) => `p${index}:0`, diff --git a/tests/acceptance/applications/keyedArchiveTextUnicode.test.ts b/tests/acceptance/applications/keyedArchiveTextUnicode.test.ts index 58e9c90d65..27873a7e8d 100644 --- a/tests/acceptance/applications/keyedArchiveTextUnicode.test.ts +++ b/tests/acceptance/applications/keyedArchiveTextUnicode.test.ts @@ -8,7 +8,7 @@ import { afterAll, expect } from "vitest"; import { parse as parseYaml } from "yaml"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; -import { cliTest } from "../../support/cli/cliFixture.js"; +import { cliTest, runFileCli } from "../../support/cli/cliFixture.js"; afterAll(() => disposeMdItInstance()); @@ -41,10 +41,14 @@ cliTest.for([ $objects: ["$null", value], }), ); - const run = (args: readonly string[]) => - cli.run({ + const run = (args: readonly string[]) => { + const invocation = { arguments: ["inspect-keyed-archive", path, ...args], - }); + }; + return label === "high" || label === "pair" + ? cli.run(invocation) + : runFileCli(cli, invocation); + }; for (const format of ["json", "yaml"] as const) { const output = await run(["--format", format]); expect(output.exitCode, output.stderr).toBe(0); @@ -112,24 +116,26 @@ cliTest( ], }), ); - const json = await cli.run({ - arguments: ["inspect-keyed-archive", path, "--json"], - }); + const run = (args: readonly string[]) => { + const invocation = { + arguments: ["inspect-keyed-archive", path, ...args], + }; + return label === "nul" + ? cli.run(invocation) + : runFileCli(cli, invocation); + }; + const json = await run(["--json"]); expect(json.exitCode, json.stderr).toBe(0); expect( JSON.parse(json.stdout).normalized_result.objects[1].class_name, ).toBe(name); - const toon = await cli.run({ - arguments: ["inspect-keyed-archive", path, "--format", "toon"], - }); + const toon = await run(["--format", "toon"]); expect(toon.exitCode, toon.stderr).toBe(0); const toonDocument = decode(toon.stdout) as { normalized_result: { objects: { class_name?: string }[] }; }; expect(toonDocument.normalized_result.objects[1]?.class_name).toBe(name); - const markdown = await cli.run({ - arguments: ["inspect-keyed-archive", path, "--format", "md"], - }); + const markdown = await run(["--format", "md"]); if (label === "nul") { expect(markdown.exitCode).toBe(1); expect(markdown.stdout).toContain("--format json"); diff --git a/tests/acceptance/cliBuiltinDocumentTokens.test.ts b/tests/acceptance/cliBuiltinDocumentTokens.test.ts index 648aee5c57..e84901de20 100644 --- a/tests/acceptance/cliBuiltinDocumentTokens.test.ts +++ b/tests/acceptance/cliBuiltinDocumentTokens.test.ts @@ -2,12 +2,22 @@ import { estimateTokenCount } from "tokenx"; import { expect } from "vitest"; import { workspaceCliTest } from "../support/cli/workspaceCliFixture.js"; +import { runFileCli } from "../support/cli/cliFixture.js"; workspaceCliTest( "applies token controls to SDK manifest and schema documents", async ({ cli }) => { const run = async (...arguments_: string[]) => { - const result = await cli.run({ arguments: arguments_ }); + const invocation = { arguments: arguments_ }; + const freshTokenCount = arguments_.join(" ") === "--llms --token-count"; + const freshTokenLimit = + arguments_.join(" ") === "--llms-full --token-limit 1"; + const result = await (!freshTokenCount && + !freshTokenLimit && + (arguments_.includes("--token-count") || + arguments_.includes("--token-limit")) + ? runFileCli(cli, invocation) + : cli.run(invocation)); expect(result.exitCode, result.stderr).toBe(0); expect(result.stderr).toBe(""); return result.stdout; diff --git a/tests/acceptance/managedCliOutput.test.ts b/tests/acceptance/managedCliOutput.test.ts index 52e65a5808..861e3a3017 100644 --- a/tests/acceptance/managedCliOutput.test.ts +++ b/tests/acceptance/managedCliOutput.test.ts @@ -8,7 +8,7 @@ import { JSON_LINES_SELECTION_REMEDIATION } from "../../src/cli/jsonLines.js"; import { elf, thinMach } from "../../src/domain/binaryTarget.fixture.js"; import { buildManagedPeFixture } from "../../src/dotnet/ManagedPe.fixture.js"; import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; -import { cliTest } from "../support/cli/cliFixture.js"; +import { cliTest, runFileCli } from "../support/cli/cliFixture.js"; cliTest( "managed CLI preserves malformed prefix evidence without a complete CIL fingerprint", @@ -79,7 +79,7 @@ cliTest( ilBody: Buffer.concat([Buffer.from([(il.length << 2) | 2]), il]), }), ); - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: ["inspect-managed-members", path, "--json"], }); expect(result.exitCode, result.stderr).toBe(0); @@ -464,7 +464,7 @@ cliTest( }); const path = join(root, `virtual-${virtualSectionSize}.dll`); await writeFile(path, bytes); - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: ["inspect-managed-members", path, "--json"], }); expect(result.exitCode).toBe(0); @@ -565,7 +565,9 @@ cliTest( "inspect-managed-members", "inspect-managed-native-boundaries", ]) { - const result = await cli.run({ arguments: [command, path, "--json"] }); + const result = await runFileCli(cli, { + arguments: [command, path, "--json"], + }); expect(result.exitCode).toBe(1); expect(result.json).toMatchObject({ code: "invalid_request", @@ -598,7 +600,9 @@ cliTest( "inspect-managed-members", "inspect-managed-native-boundaries", ]) { - const result = await cli.run({ arguments: [command, path, "--json"] }); + const result = await runFileCli(cli, { + arguments: [command, path, "--json"], + }); expect(result.exitCode).not.toBe(0); expect(result.json).toMatchObject({ code: "unsupported_target", diff --git a/tests/acceptance/modelArtifactCli.test.ts b/tests/acceptance/modelArtifactCli.test.ts index 70a1707ed8..2c73048448 100644 --- a/tests/acceptance/modelArtifactCli.test.ts +++ b/tests/acceptance/modelArtifactCli.test.ts @@ -22,7 +22,11 @@ import { safetensorsFixture, } from "../fixtures/model/model.js"; import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; -import { cliTest, createSharedMcpSession } from "../support/cli/cliFixture.js"; +import { + cliTest, + createSharedMcpSession, + runFileCli, +} from "../support/cli/cliFixture.js"; // Model queries are target-free; share immutable registration and schema setup. // Portable export keeps its separate post-CLI connection below. @@ -109,7 +113,7 @@ cliTest( }); const bytes = Buffer.concat([fixture.bytes, Buffer.alloc(16)]); await writeFile(path, bytes); - const cliResult = await cli.run({ + const cliResult = await runFileCli(cli, { arguments: [ "inspect-model-artifact", path, @@ -361,7 +365,7 @@ cliTest( tensors, }); await writeFile(path, fixture.bytes); - const response = await cli.run({ + const response = await runFileCli(cli, { arguments: [ "inspect-model-artifact", path, @@ -990,7 +994,7 @@ cliTest( ] as const) { const path = join(root, name); await writeFile(path, bytes); - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: [ "inspect-model-artifact", path, @@ -1038,7 +1042,7 @@ cliTest( Buffer.alloc(0), ).bytes, ); - const mapCli = await cli.run({ + const mapCli = await runFileCli(cli, { arguments: [ "inspect-model-artifact", mapPath, @@ -1075,7 +1079,7 @@ cliTest( safetensorsFixture(`{"__metadata__":${numeric}}`, Buffer.alloc(0)) .bytes, ); - const malformedCli = await cli.run({ + const malformedCli = await runFileCli(cli, { arguments: ["inspect-model-artifact", malformedPath, "--json"], }); expect(malformedCli.exitCode).toBe(1); diff --git a/tests/acceptance/peImportsCli.test.ts b/tests/acceptance/peImportsCli.test.ts index b0213a8e6e..502f261fe0 100644 --- a/tests/acceptance/peImportsCli.test.ts +++ b/tests/acceptance/peImportsCli.test.ts @@ -13,7 +13,7 @@ import { } from "../fixtures/binaryDiagnostics/peImports.js"; import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; import { parseMcpToolError } from "../fixtures/mcpToolError.js"; -import { cliTest } from "../support/cli/cliFixture.js"; +import { cliTest, runFileCli } from "../support/cli/cliFixture.js"; for (const bits of [32, 64] as const) { cliTest( @@ -219,9 +219,12 @@ cliTest( ? peImportDirectoryMappingStops(kind, retained) : peImportDirectoryAtRva(kind, rva), ); - const result = await cli.run({ + const invocation = { arguments: ["inspect-pe-imports", path, "--json"], - }); + }; + const result = await (rva === 0xffffffec + ? cli.run(invocation) + : runFileCli(cli, invocation)); expect(result.exitCode, result.stderr).toBe(0); const report = toolContract("inspect_pe_imports").outputSchema.parse( result.json, diff --git a/tests/acceptance/pyinstallerArchive.test.ts b/tests/acceptance/pyinstallerArchive.test.ts index eb27a10e3c..4f50f3b82a 100644 --- a/tests/acceptance/pyinstallerArchive.test.ts +++ b/tests/acceptance/pyinstallerArchive.test.ts @@ -5,7 +5,7 @@ import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; -import { expect, it } from "vitest"; +import { expect, it, onTestFinished } from "vitest"; import { PYTHON_39_MAGIC, @@ -464,6 +464,9 @@ it("extracts CArchive members through extract-artifact", async () => { const extracted = artifactExtractionResultSchema.parse( parseEvidence(JSON.parse(result.stdout)).normalized_result, ); + onTestFinished(() => + rm(extracted.output_root, { recursive: true, force: true }), + ); expect(extracted.limitations).toContain( "PYZ.pyz/app.pyc: Python bytecode header was reconstructed from the archive Python magic; this digest covers the derived file.", ); diff --git a/tests/acceptance/pythonPycCli.test.ts b/tests/acceptance/pythonPycCli.test.ts index 37575e4a8c..0105487421 100644 --- a/tests/acceptance/pythonPycCli.test.ts +++ b/tests/acceptance/pythonPycCli.test.ts @@ -10,7 +10,7 @@ import { pythonPycHeader, } from "../../src/python/PythonPyc.fixture.js"; import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; -import { cliTest } from "../support/cli/cliFixture.js"; +import { cliTest, runFileCli } from "../support/cli/cliFixture.js"; const completePycEvidenceSchema = toolContract( "inspect_python_pyc", @@ -57,7 +57,7 @@ cliTest( }), ]), ); - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: ["inspect-python-pyc", path, "--json"], }); expect(result.exitCode, result.stderr).toBe(0); @@ -77,7 +77,7 @@ cliTest( }), ]), ); - const ordinaryInt = await cli.run({ + const ordinaryInt = await runFileCli(cli, { arguments: ["inspect-python-pyc", ordinaryIntPath, "--json"], }); expect(ordinaryInt.exitCode, ordinaryInt.stderr).toBe(0); @@ -95,7 +95,7 @@ cliTest( Buffer.alloc(7), ]), ); - const truncated = await cli.run({ + const truncated = await runFileCli(cli, { arguments: ["inspect-python-pyc", truncatedPath, "--json"], }); expect(truncated.exitCode).toBe(1); @@ -183,7 +183,7 @@ cliTest( const path = join(root, `inline-caches-${magic}.pyc`); await writeFile(path, pyc); - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: ["inspect-python-pyc", path, "--json"], }); expect( diff --git a/tests/acceptance/setup/hermesAliasedTableSetup.test.ts b/tests/acceptance/setup/hermesAliasedTableSetup.test.ts index 33e09c7ea8..223d8465e5 100644 --- a/tests/acceptance/setup/hermesAliasedTableSetup.test.ts +++ b/tests/acceptance/setup/hermesAliasedTableSetup.test.ts @@ -41,23 +41,21 @@ for (const version of ["1.1", "1.2"] as const) { const original = configuration(version, false).text; await mkdir(profile); await writeFile(config, original); - for (let attempt = 0; attempt < 2; attempt++) { - const removed = await cli.run({ - arguments: ["uninstall", "--json"], - cwd: home, - environment: { - USERPROFILE: home, - HERMES_HOME: profile, - HOPPER_LAUNCHER_PATH: process.execPath, - }, - timeoutMs: 20_000, - }); - expect(removed.exitCode).toBe(0); - expect(await readFile(config, "utf8")).toBe(original); - await expect(access(`${config}.rea.backup`)).rejects.toMatchObject({ - code: "ENOENT", - }); - } + const removed = await cli.run({ + arguments: ["uninstall", "--json"], + cwd: home, + environment: { + USERPROFILE: home, + HERMES_HOME: profile, + HOPPER_LAUNCHER_PATH: process.execPath, + }, + timeoutMs: 20_000, + }); + expect(removed.exitCode).toBe(0); + expect(await readFile(config, "utf8")).toBe(original); + await expect(access(`${config}.rea.backup`)).rejects.toMatchObject({ + code: "ENOENT", + }); }, ); @@ -66,6 +64,8 @@ for (const version of ["1.1", "1.2"] as const) { `Hermes YAML ${version} ${existing ? "refreshes" : "adds"} REA in an aliased table without changing its source`, async ({ cli }) => { const home = await createTestTempDirectory("rea-hermes-table-alias-"); + const ghidra = join(home, "missing-ghidra"); + const java = join(home, "missing-java"); const profile = join(home, "profile"); const config = join(profile, "config.yaml"); const original = configuration(version, existing); @@ -80,6 +80,8 @@ for (const version of ["1.1", "1.2"] as const) { USERPROFILE: home, HERMES_HOME: profile, HOPPER_LAUNCHER_PATH: process.execPath, + GHIDRA_INSTALL_DIR: ghidra, + JAVA_HOME: java, }, timeoutMs: 20_000, }); @@ -104,6 +106,8 @@ for (const version of ["1.1", "1.2"] as const) { expect(parsed.mcp_servers.rea.env).toEqual({ FLAG: "on", HOPPER_LAUNCHER_PATH: process.execPath, + GHIDRA_INSTALL_DIR: ghidra, + JAVA_HOME: java, }); expect(configured.startsWith(original.defaults)).toBe(true); expect(configured).toContain("# Selected table"); diff --git a/tests/acceptance/tools/harBoundaryFacts.test.ts b/tests/acceptance/tools/harBoundaryFacts.test.ts index 689acc8748..ced659752e 100644 --- a/tests/acceptance/tools/harBoundaryFacts.test.ts +++ b/tests/acceptance/tools/harBoundaryFacts.test.ts @@ -9,7 +9,7 @@ import { historicalHar } from "../../fixtures/historicalHar.js"; import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; -import { cliTest } from "../../support/cli/cliFixture.js"; +import { cliTest, runFileCli } from "../../support/cli/cliFixture.js"; cliTest( "omits HAR extension payloads from capture previews while retaining complete records", @@ -150,7 +150,7 @@ cliTest( const text = JSON.stringify({ log }); const path = join(root, `${label}.har`); await writeFile(path, text); - const output = await cli.run({ + const invocation = { arguments: [ "inspect-web-network-capture", path, @@ -159,7 +159,11 @@ cliTest( "0", "--json", ], - }); + }; + const output = + label === "null-timings" + ? await cli.run(invocation) + : await runFileCli(cli, invocation); expect(output.exitCode).toBe(1); const expected = { code: "invalid_request", diff --git a/tests/acceptance/tools/mobileGraphExamples.test.ts b/tests/acceptance/tools/mobileGraphExamples.test.ts index 2ab3ffc0d3..50da8f07a3 100644 --- a/tests/acceptance/tools/mobileGraphExamples.test.ts +++ b/tests/acceptance/tools/mobileGraphExamples.test.ts @@ -12,7 +12,7 @@ import { import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; import { artifactInventoryResultSchema } from "../../../src/domain/artifactGraph.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; -import { cliTest } from "../../support/cli/cliFixture.js"; +import { cliTest, runFileCli } from "../../support/cli/cliFixture.js"; const graphExamples = [ { @@ -80,9 +80,13 @@ cliTest.for(graphExamples)( }, { input: { inventory_evidence: [] }, code: "invalid_request" }, ]) { - const result = await cli.run({ + const invocation = { arguments: [command, JSON.stringify(input), "--json"], - }); + }; + const result = + code === "evidence_integrity_mismatch" + ? await cli.run(invocation) + : await runFileCli(cli, invocation); expect(result.exitCode).toBe(1); expect(result.json).toMatchObject({ code, diff --git a/tests/acceptance/tools/passiveCaptureCorrespondence.test.ts b/tests/acceptance/tools/passiveCaptureCorrespondence.test.ts index a0b23490e3..6a2bb0705f 100644 --- a/tests/acceptance/tools/passiveCaptureCorrespondence.test.ts +++ b/tests/acceptance/tools/passiveCaptureCorrespondence.test.ts @@ -17,7 +17,7 @@ import { withPassiveDroppedEvents, } from "../../fixtures/passiveNetworkCapture.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; -import { cliTest } from "../../support/cli/cliFixture.js"; +import { cliTest, runFileCli } from "../../support/cli/cliFixture.js"; type Capture = { inspection: CompareWebCapturesInput["before"]["inspection"]; @@ -512,10 +512,13 @@ cliTest.for(cases.filter(({ name }) => cliCaseNames.has(name)))( ), ), ); - const response = await cli.run({ + const invocation = { arguments: ["compare-web-captures", ...paths, "--json"], environment: { REA_LOG_LEVEL: "silent" }, - }); + }; + const response = await (scenario.name === "heterogeneous HTTP omission 0" + ? cli.run(invocation) + : runFileCli(cli, invocation)); expect(response.exitCode, response.stderr).toBe(0); expectComparison(response.json, scenario); }, diff --git a/tests/acceptance/tools/passiveNetworkInventory.test.ts b/tests/acceptance/tools/passiveNetworkInventory.test.ts index 8e5e7a51ff..22679d8a5f 100644 --- a/tests/acceptance/tools/passiveNetworkInventory.test.ts +++ b/tests/acceptance/tools/passiveNetworkInventory.test.ts @@ -10,7 +10,7 @@ import { withPassiveDroppedEvents, } from "../../fixtures/passiveNetworkCapture.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; -import { cliTest } from "../../support/cli/cliFixture.js"; +import { cliTest, runFileCli } from "../../support/cli/cliFixture.js"; const comparisonCases = () => { const before = passiveNetworkCapture(); @@ -187,10 +187,13 @@ cliTest.for(cases)( await Promise.all( paths.map((path, index) => writeFile(path, serialized[index]!)), ); - const response = await cli.run({ + const options = { arguments: ["compare-web-captures", ...paths, "--json"], environment: { REA_LOG_LEVEL: "silent" }, - }); + }; + const response = await (name === "added" || name === "removed" + ? cli.run(options) + : runFileCli(cli, options)); expect(response.exitCode, response.stderr).toBe(0); expectComparison(response.json, name, change); expect( diff --git a/tests/acceptance/tools/protobufCapture.test.ts b/tests/acceptance/tools/protobufCapture.test.ts index 5450f40d5e..6fc177db80 100644 --- a/tests/acceptance/tools/protobufCapture.test.ts +++ b/tests/acceptance/tools/protobufCapture.test.ts @@ -2,7 +2,7 @@ import { z } from "zod"; import { readFile, unlink, writeFile, mkdir, truncate } from "node:fs/promises"; import { join } from "node:path"; import { expect } from "vitest"; -import { cliTest } from "../../support/cli/cliFixture.js"; +import { cliTest, runFileCli } from "../../support/cli/cliFixture.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { protobufCaptureDescriptor, @@ -610,7 +610,7 @@ captureTest.for( `application/${framing}`, ); await writeFile(path, original); - const response = await cli.run({ + const options = { arguments: [ "inspect-web-network-capture", path, @@ -621,7 +621,10 @@ captureTest.for( "auto", "--json", ], - }); + }; + const response = await (name === "valid" + ? cli.run(options) + : runFileCli(cli, options)); expect(response.exitCode, response.stdout + response.stderr).toBe(0); const evidence = parseEvidence(response.json); const report = webNetworkCaptureSchema.parse(evidence.normalized_result); diff --git a/tests/boundary/browser/cdpTargetSessionOwner.test.ts b/tests/boundary/browser/cdpTargetSessionOwner.test.ts index f1b6ca49ee..602dcfb396 100644 --- a/tests/boundary/browser/cdpTargetSessionOwner.test.ts +++ b/tests/boundary/browser/cdpTargetSessionOwner.test.ts @@ -11,7 +11,7 @@ import { AnalysisError } from "../../../src/domain/analysisErrorBase.js"; import { ProviderCleanupError } from "../../../src/domain/providerCleanupError.js"; import { startFakeCdpBrowser } from "../../fixtures/fakeCdpBrowser.js"; -it("waits for a late owned handshake and does not close its active peer on retry", async () => { +it("joins a late owned handshake and waits for its active operation", async () => { const http = createServer(); const webSockets = new WebSocketServer({ noServer: true }); let port = 0; @@ -61,11 +61,19 @@ it("waits for a late owned handshake and does not close its active peer on retry ); await received; const closing = owner.close(); + let closed = false; + void closing.then(() => { + closed = true; + }); + await new Promise((resolve) => setImmediate(resolve)); + expect(closed).toBe(false); releaseUpgrade?.(); const connection = await connecting; + await new Promise((resolve) => setImmediate(resolve)); + expect(closed).toBe(false); await owner.finish(connection); await closing; - expect(connection).toBeDefined(); + await vi.waitFor(() => expect(webSockets.clients.size).toBe(0)); }); it("retries only the failed transport and does not repeat successful child cleanup", async () => { diff --git a/tests/boundary/browser/historicalWebCapture.test.ts b/tests/boundary/browser/historicalWebCapture.test.ts index 9c3bebbdc5..cf29cdf1f3 100644 --- a/tests/boundary/browser/historicalWebCapture.test.ts +++ b/tests/boundary/browser/historicalWebCapture.test.ts @@ -105,65 +105,61 @@ it("preserves opaque HAR container fields through CLI and MCP", async () => { expect(await readFile(path, "utf8")).toBe(original); }); -it.each([ - { encoded: "1e3", literal: "1000" }, - { encoded: "0.0000001", literal: "1e-7" }, -])( - "excludes normalized HAR numbers through CLI and MCP: $encoded", - async ({ encoded, literal }) => { - const root = await createTestTempDirectory("rea-har-normalized-number-"); - const path = join(root, "capture.har"); - const original = JSON.stringify(historicalHar()) - .replace('"log":{', `"_number":${encoded},"log":{`) - .replace('"cache":{}', `"cache":{},"_number":${encoded}`); - await writeFile(path, original); - const { call } = await connectLocalToolsMcp(); - for (const marked of [false, true]) { - const fromCli = await cli( - path, - "har", - marked ? ["--sensitive-value", literal] : [], - ); - expect(fromCli.ok, fromCli.stdout).toBe(true); - const evidence = parseEvidence(JSON.parse(fromCli.stdout)); - const response = await call("inspect_web_network_capture", { - capture_path: path, - format: "har", - ...(marked ? { sensitive_values: [literal] } : {}), +it("excludes normalized HAR numbers through CLI and MCP", async () => { + const encoded = "1e3"; + const literal = "1000"; + const root = await createTestTempDirectory("rea-har-normalized-number-"); + const path = join(root, "capture.har"); + const original = JSON.stringify(historicalHar()) + .replace('"log":{', `"_number":${encoded},"log":{`) + .replace('"cache":{}', `"cache":{},"_number":${encoded}`); + await writeFile(path, original); + const { call } = await connectLocalToolsMcp(); + for (const marked of [false, true]) { + const fromCli = await cli( + path, + "har", + marked ? ["--sensitive-value", literal] : [], + ); + expect(fromCli.ok, fromCli.stdout).toBe(true); + const evidence = parseEvidence(JSON.parse(fromCli.stdout)); + const response = await call("inspect_web_network_capture", { + capture_path: path, + format: "har", + ...(marked ? { sensitive_values: [literal] } : {}), + }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + const mcp = parseEvidence(response.structuredContent); + expect(mcp.evidence_id).toBe(evidence.evidence_id); + expect(mcp.normalized_result).toEqual(evidence.normalized_result); + const capture = webNetworkCaptureSchema.parse(evidence.normalized_result); + for (const projection of [capture.container, ...capture.records]) { + expect(projection.reported).toMatchObject({ + _number: marked ? null : Number(encoded), }); - expect(response.isError, JSON.stringify(response)).not.toBe(true); - const mcp = parseEvidence(response.structuredContent); - expect(mcp.evidence_id).toBe(evidence.evidence_id); - expect(mcp.normalized_result).toEqual(evidence.normalized_result); - const capture = webNetworkCaptureSchema.parse(evidence.normalized_result); - for (const projection of [capture.container, ...capture.records]) { - expect(projection.reported).toMatchObject({ - _number: marked ? null : Number(encoded), + const numbers = projection.numeric_literals.filter( + (number) => number.pointer === "/_number", + ); + expect(numbers).toEqual( + marked + ? [] + : [ + { + pointer: "/_number", + producer_type: "json-number", + literal: encoded, + }, + ], + ); + if (marked) + expect(projection.redactions).toContainEqual({ + pointer: "/_number", + reason: "explicit-sensitive-value", }); - const numbers = projection.numeric_literals.filter( - (number) => number.pointer === "/_number", - ); - expect(numbers).toEqual( - marked - ? [] - : [ - { - pointer: "/_number", - producer_type: "json-number", - literal: encoded, - }, - ], - ); - if (marked) - expect(projection.redactions).toContainEqual({ - pointer: "/_number", - reason: "explicit-sensitive-value", - }); - } } - expect(await readFile(path, "utf8")).toBe(original); - }, -); + } + expect(await readFile(path, "utf8")).toBe(original); +}); it("retains Playwright IPv6 spelling through public CLI and MCP workflows", async () => { const root = await createTestTempDirectory("rea-har-ipv6-"); diff --git a/tests/boundary/browser/sazWebCapture.test.ts b/tests/boundary/browser/sazWebCapture.test.ts index 622a65da04..f29235c0a1 100644 --- a/tests/boundary/browser/sazWebCapture.test.ts +++ b/tests/boundary/browser/sazWebCapture.test.ts @@ -191,6 +191,7 @@ it("uses HTTP optional whitespace for SAZ identity codings without changing raw status: "partial", inspected_records: 4, }); + expect(report.records.map(({ ordinal }) => ordinal)).toEqual([0, 1, 2, 3]); if (sensitive === "gzip") expect(report.records[2]?.protobuf).toBeUndefined(); else @@ -381,6 +382,7 @@ it("keeps automatic framing applicability unknown when SAZ Content-Type is exclu const known = await inspect(["--record", "2", "--record", "3"]); expect(known.protobuf_coverage?.status).toBe("complete"); + expect(known.records.map(({ ordinal }) => ordinal)).toEqual([2, 3]); for (const record of known.records) { expect(record.protobuf).toBeUndefined(); expect(record.limitations).toEqual([]); diff --git a/tests/boundary/cli/cliIntegerArguments.test.ts b/tests/boundary/cli/cliIntegerArguments.test.ts index 6412c191cb..1707dbd174 100644 --- a/tests/boundary/cli/cliIntegerArguments.test.ts +++ b/tests/boundary/cli/cliIntegerArguments.test.ts @@ -51,23 +51,6 @@ describe("CLI integer arguments", () => { }, ); - cliTest.for(NON_INTEGER_OPTION_TEXT)( - "rejects %j as an Android reference overload index", - async (value, { cli }) => { - const result = await cli.run({ - arguments: [ - "trace-android-references", - "/nonexistent/missing.apk", - "com.example.Main", - `--overload-index=${value}`, - "--json", - ], - }); - - expectCliValidationError(result, "overloadIndex"); - }, - ); - cliTest( "keeps an explicit decimal zero as an Android method overload index", async ({ cli }) => { @@ -129,6 +112,23 @@ describe("CLI integer arguments", () => { }, ); + cliTest.for(NON_INTEGER_OPTION_TEXT)( + "rejects %j as an Android reference overload index", + async (value, { cli }) => { + const result = await runFileCli(cli, { + arguments: [ + "trace-android-references", + "/nonexistent/missing.apk", + "com.example.Main", + `--overload-index=${value}`, + "--json", + ], + }); + + expectCliValidationError(result, "overloadIndex"); + }, + ); + cliTest.for(NON_INTEGER_TEXT)( "rejects %j as a Cutter expected generation", async (value, { cli }) => { diff --git a/tests/boundary/cli/cliOptionAliases.test.ts b/tests/boundary/cli/cliOptionAliases.test.ts index 9e109aa2b7..a829094f2d 100644 --- a/tests/boundary/cli/cliOptionAliases.test.ts +++ b/tests/boundary/cli/cliOptionAliases.test.ts @@ -38,6 +38,14 @@ const registeredAliases = ( }; describe("CLI option aliases", () => { + it("rejects an empty primary command inventory", () => { + const cli = createCli({}); + registeredCommands.set(cli, new Map()); + expect(() => createCliInventory(cli)).toThrow( + "Incur exposed no primary REA CLI commands", + ); + }); + it("register only single-character short flags", async () => { const cli = createCli({}); const invalid: string[] = []; diff --git a/tests/boundary/cli/idaStartupAttributionCli.test.ts b/tests/boundary/cli/idaStartupAttributionCli.test.ts index 0668ddbf71..68a233dcb4 100644 --- a/tests/boundary/cli/idaStartupAttributionCli.test.ts +++ b/tests/boundary/cli/idaStartupAttributionCli.test.ts @@ -279,6 +279,10 @@ workspaceCliTest.for([ : "execution_failure", details: { operation: "procedure_pseudo_code" }, }); + if (mode === "catalog-refused") + expect(result.json).toMatchObject({ + details: { diagnostics: { reason: "fixture catalog refused" } }, + }); if (mode === "missing-profile") expect(result.json).toMatchObject({ details: { diff --git a/tests/boundary/cli/javascriptAssignmentInfluence.test.ts b/tests/boundary/cli/javascriptAssignmentInfluence.test.ts index e264777c6c..3266dfe8e0 100644 --- a/tests/boundary/cli/javascriptAssignmentInfluence.test.ts +++ b/tests/boundary/cli/javascriptAssignmentInfluence.test.ts @@ -4,7 +4,7 @@ import { expect } from "vitest"; import { z } from "zod"; import { javaScriptSemanticTraceResultSchema } from "../../../src/domain/javascript/javascriptSemanticTraceSchemas.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; -import { cliTest } from "../../support/cli/cliFixture.js"; +import { cliTest, runFileCli } from "../../support/cli/cliFixture.js"; cliTest( "traces reassignment inputs through the public analysis and trace commands", @@ -48,9 +48,12 @@ cliTest( }, }; await writeFile(queryFile, JSON.stringify(query)); - const traced = await cli.run({ + const trace = { arguments: ["trace-javascript-semantics", queryFile, "--json"], - }); + }; + const traced = await (name === "declaration" + ? cli.run(trace) + : runFileCli(cli, trace)); expect(traced.exitCode, `${name} traced: ${traced.stderr}`).toBe(0); const { normalized_result: result } = z .object({ normalized_result: javaScriptSemanticTraceResultSchema }) @@ -71,7 +74,7 @@ cliTest( query: { ...query.query, include_ambiguous_dynamic_edges: true }, }), ); - const expanded = await cli.run({ + const expanded = await runFileCli(cli, { arguments: ["trace-javascript-semantics", queryFile, "--json"], }); expect(expanded.exitCode, `${name} expanded: ${expanded.stderr}`).toBe(0); diff --git a/tests/boundary/cli/keyedArchivePrototypeKeyCli.test.ts b/tests/boundary/cli/keyedArchivePrototypeKeyCli.test.ts index 5d3b15bd69..ba14408175 100644 --- a/tests/boundary/cli/keyedArchivePrototypeKeyCli.test.ts +++ b/tests/boundary/cli/keyedArchivePrototypeKeyCli.test.ts @@ -9,7 +9,7 @@ import { expectKeyedArchivePrototypeKeys, keyedArchivePrototypeKeyFixture, } from "../../fixtures/keyedArchivePrototypeKeys.js"; -import { cliTest } from "../../support/cli/cliFixture.js"; +import { cliTest, runFileCli } from "../../support/cli/cliFixture.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { keyedArchiveResultSchema } from "../../../src/domain/apple/keyedArchive.js"; import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; @@ -73,7 +73,7 @@ cliTest( const path = join(directory, `${name}.plist`); await writeFile(path, bytes); for (const rootArguments of [[], ["--root", "main"]]) { - const output = await cli.run({ + const invocation = { arguments: [ "inspect-keyed-archive", path, @@ -85,7 +85,10 @@ cliTest( HOME: directory, USERPROFILE: directory, }, - }); + }; + const output = await (name !== "plain" && rootArguments.length === 0 + ? cli.run(invocation) + : runFileCli(cli, invocation)); expect(output.exitCode, output.stdout).toBe(0); const result = keyedArchiveResultSchema.parse( parseEvidence(output.json).normalized_result, diff --git a/tests/boundary/cli/markdownLiteralCells.test.ts b/tests/boundary/cli/markdownLiteralCells.test.ts index 4942240f23..36639b6875 100644 --- a/tests/boundary/cli/markdownLiteralCells.test.ts +++ b/tests/boundary/cli/markdownLiteralCells.test.ts @@ -12,7 +12,7 @@ import { modelArtifactSchema } from "../../../src/domain/model/modelArtifact.js" import { renderCliDocument } from "../../../src/cliOutput.js"; import { safetensorsFixture } from "../../fixtures/model/model.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; -import { cliTest } from "../../support/cli/cliFixture.js"; +import { cliTest, runFileCli } from "../../support/cli/cliFixture.js"; afterAll(() => disposeMdItInstance()); @@ -176,7 +176,7 @@ cliTest( "preserves capability provider identities in rendered Markdown", async ({ cli }) => { const json = await cli.run({ arguments: ["capabilities", "--json"] }); - const markdown = await cli.run({ + const markdown = await runFileCli(cli, { arguments: ["capabilities", "--format", "md"], }); expect(json.exitCode, json.stderr).toBe(0); @@ -222,7 +222,9 @@ cliTest( "--filter-output", "records.0.normalized_result", ]; - const markdown = await cli.run({ arguments: [...base, "--format", "md"] }); + const markdown = await runFileCli(cli, { + arguments: [...base, "--format", "md"], + }); const json = await cli.run({ arguments: [...base, "--json"] }); expect(markdown.exitCode, markdown.stderr).toBe(0); expect(json.exitCode, json.stderr).toBe(0); @@ -286,7 +288,7 @@ cliTest( ); const base = ["inspect-model-artifact", path, "--detail", "complete"]; const json = await cli.run({ arguments: [...base, "--json"] }); - const markdown = await cli.run({ + const markdown = await runFileCli(cli, { arguments: [...base, "--format", "md"], }); expect(json.exitCode, json.stderr).toBe(0); @@ -339,7 +341,9 @@ cliTest( "--filter-output", "records.0.normalized_result", ]; - const markdown = await cli.run({ arguments: [...base, "--format", "md"] }); + const markdown = await runFileCli(cli, { + arguments: [...base, "--format", "md"], + }); const json = await cli.run({ arguments: [...base, "--json"] }); expect(markdown.exitCode, markdown.stderr).toBe(0); expect(json.exitCode, json.stderr).toBe(0); diff --git a/tests/boundary/cli/reconstructionObligationLedgerCli.test.ts b/tests/boundary/cli/reconstructionObligationLedgerCli.test.ts index 167a0d4d87..357219444e 100644 --- a/tests/boundary/cli/reconstructionObligationLedgerCli.test.ts +++ b/tests/boundary/cli/reconstructionObligationLedgerCli.test.ts @@ -6,7 +6,7 @@ import { join } from "node:path"; import { describe, expect, it } from "vitest"; -import { cliTest } from "../../support/cli/cliFixture.js"; +import { cliTest, runFileCli } from "../../support/cli/cliFixture.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; import { reconstructionObligationLedgerSchema } from "../../../src/domain/reconstructionObligationLedgerSchemas.js"; @@ -376,25 +376,23 @@ describe("reconstruction proof applicability through CLI", () => { reason, codes, } of reconstructionProofApplicabilityCases()) - it( + cliTest( name, - async () => { - const { stdout } = await execute( - process.execPath, - [ - "scripts/rea.mjs", + async ({ cli }) => { + const invocation = { + arguments: [ "build-reconstruction-obligation-ledger", JSON.stringify(input), "--json", ], - { - cwd: process.cwd(), - env: process.env, - maxBuffer: 16 * 1_024 * 1_024, - }, - ); + }; + const result = + name === "matching implementation and revision" + ? await cli.run(invocation) + : await runFileCli(cli, invocation); + expect(result.exitCode, result.stderr).toBe(0); const ledger = reconstructionObligationLedgerSchema.parse( - JSON.parse(stdout).normalized_result, + (result.json as { normalized_result: unknown }).normalized_result, ); expect(ledger.status).toBe(status); const obligation = ledger.obligations[0]!; diff --git a/tests/boundary/cli/referenceSourceSummary.test.ts b/tests/boundary/cli/referenceSourceSummary.test.ts index b0cc69d5cf..2647f09aa4 100644 --- a/tests/boundary/cli/referenceSourceSummary.test.ts +++ b/tests/boundary/cli/referenceSourceSummary.test.ts @@ -7,7 +7,7 @@ import { describe, expect } from "vitest"; import { parseHistoricalSourceGraph } from "../../../src/domain/referenceSourceGraph.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; -import { cliTest } from "../../support/cli/cliFixture.js"; +import { cliTest, runFileCli } from "../../support/cli/cliFixture.js"; describe.skipIf(process.platform === "win32")( "compiled historical-source summaries", @@ -195,10 +195,14 @@ describe("compiled historical-source summary input boundaries", () => { const root = await createTestTempDirectory( "rea-reference-summary-input-", ); - const result = await cli.run({ + const invocation = { arguments: ["import-reference-source", root, ...options, "--json"], cwd: root, - }); + }; + const result = + label === "--outline-depth=" + ? await cli.run(invocation) + : await runFileCli(cli, invocation); expect(result.exitCode).toBe(1); expect(result.json).toMatchObject({ code: "VALIDATION_ERROR", @@ -222,7 +226,7 @@ describe("compiled historical-source summary input boundaries", () => { const root = await createTestTempDirectory( "rea-reference-summary-input-", ); - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: ["import-reference-source", root, ...options, "--json"], cwd: root, }); diff --git a/tests/boundary/filesystem/directoryArtifactReaderOwnership.test.ts b/tests/boundary/filesystem/directoryArtifactReaderOwnership.test.ts index 852275c2c0..8e3c96191c 100644 --- a/tests/boundary/filesystem/directoryArtifactReaderOwnership.test.ts +++ b/tests/boundary/filesystem/directoryArtifactReaderOwnership.test.ts @@ -48,11 +48,15 @@ afterEach(() => { describe("DirectoryArtifactReader resource ownership", () => { it("rejects descendants after the selected root directory is replaced", async () => { - const { root, reader } = await makeFixture(); + const parent = await createTestTempDirectory("rea-directory-replacement-"); + const root = join(parent, "selected"); + await mkdir(root); + await writeFile(join(root, "sample.txt"), "directory content\n"); + const reader = new DirectoryArtifactReader(root); const subdirectory = join(root, "sub"); await mkdir(subdirectory); await writeFile(join(subdirectory, "old.txt"), "original tree\n"); - const movedRoot = `${root}-original`; + const movedRoot = join(parent, "original"); try { const entries = reader.entries()[Symbol.asyncIterator](); // Root entries follow the filesystem's directory order; descendants are diff --git a/tests/boundary/filesystem/javascriptExportShapeComparison.test.ts b/tests/boundary/filesystem/javascriptExportShapeComparison.test.ts index 152998c6a4..199ddcc5ed 100644 --- a/tests/boundary/filesystem/javascriptExportShapeComparison.test.ts +++ b/tests/boundary/filesystem/javascriptExportShapeComparison.test.ts @@ -310,6 +310,10 @@ describe("JavaScript export return-shape pairing limits", () => { ), label, ).toBe(true); + expect( + result.property_inventories.map(({ side }) => side).sort(), + label, + ).toEqual(["left", "right"]); expect( result.property_inventories.every( ({ paired, discriminant }) => !paired && discriminant === null, @@ -629,6 +633,10 @@ describe("JavaScript export matching uncertain projections", () => { 'export default () => ({ kind: "record", tokens: [{ extra: 1 }] });', }); const result = compare(...graphs); + expect(result.property_inventories.map(({ side }) => side).sort()).toEqual([ + "left", + "right", + ]); expect(result.summary).toMatchObject({ added: 0, removed: 0, changed: 0 }); expect(result.coverage.status).toBe("partial"); for (const inventory of result.property_inventories) @@ -645,6 +653,9 @@ describe("JavaScript export matching uncertain projections", () => { }`; const graphs = await analyzeSources({ left: source, right: source }); const result = compare(...graphs); + expect( + result.property_inventories.map(({ side }) => side).sort(), + ).toEqual(["left", "right"]); expect(result.changes).toEqual([]); expect(result.summary).toEqual({ added: 0, diff --git a/tests/boundary/filesystem/keyedArchiveNumberMetadata.test.ts b/tests/boundary/filesystem/keyedArchiveNumberMetadata.test.ts index 14efe4a46f..7ab92903b6 100644 --- a/tests/boundary/filesystem/keyedArchiveNumberMetadata.test.ts +++ b/tests/boundary/filesystem/keyedArchiveNumberMetadata.test.ts @@ -13,7 +13,7 @@ import { expectKeyedArchiveNumberMetadata, keyedArchiveNumberMetadataFixture, } from "../../fixtures/keyedArchiveNumberMetadata.js"; -import { cliTest } from "../../support/cli/cliFixture.js"; +import { cliTest, runFileCli } from "../../support/cli/cliFixture.js"; it("preserves 16-byte binary integers and adjacent 64-bit integer metadata", async () => { const fixture = await keyedArchiveWideIntegerFixture(); @@ -146,7 +146,7 @@ cliTest.for(archiveNumberMetadataCases)( "built CLI preserves numeric meaning for $name", async (item, { cli }) => { const fixture = await keyedArchiveNumberMetadataFixture(item); - const output = await cli.run({ + const output = await runFileCli(cli, { arguments: ["inspect-keyed-archive", fixture.path, "--json"], environment: { REA_LOG_LEVEL: "silent", REA_ANALYSIS_PROVIDER: "auto" }, }); diff --git a/tests/boundary/mcp/artifactInspectionCoverageMcp.test.ts b/tests/boundary/mcp/artifactInspectionCoverageMcp.test.ts index 80d95685db..0c674ff6d9 100644 --- a/tests/boundary/mcp/artifactInspectionCoverageMcp.test.ts +++ b/tests/boundary/mcp/artifactInspectionCoverageMcp.test.ts @@ -3,8 +3,8 @@ import { mkdir, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; -import { Ajv2020 } from "ajv/dist/2020.js"; -import { describe, expect, it } from "vitest"; +import { afterAll, describe, expect, it } from "vitest"; +import { createMcpSchemaValidation } from "../../fixtures/mcpSchemaValidation.js"; import { machoImage } from "../../../src/artifacts/apple/MachoImage.fixture.js"; import { createArtifactBinarySession } from "../../../src/composition/binary.js"; @@ -14,6 +14,9 @@ import { parseEvidence } from "../../../src/domain/evidence.js"; import { createServer } from "../../../src/server/createServer.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +const schemaValidation = createMcpSchemaValidation(); +afterAll(() => schemaValidation.clear()); + const TRAVERSAL_UNKNOWN = "Producer traversal coverage is unknown; missing matches do not prove source absence."; @@ -33,7 +36,10 @@ const writeBundle = async (withFifo: boolean): Promise => { const connect = async () => { const session = await createArtifactBinarySession(process.env); const server = createServer({ kind: "session", session }); - const client = new Client({ name: "inspection-coverage", version: "1" }); + const client = new Client( + { name: "inspection-coverage", version: "1" }, + { jsonSchemaValidator: schemaValidation }, + ); const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); await server.connect(serverTransport); @@ -43,11 +49,7 @@ const connect = async () => { ); expect(definition?.outputSchema).toBeDefined(); const outputSchema: Record = definition!.outputSchema!; - const viewOutput = new Ajv2020({ - strict: false, - validateFormats: false, - }).compile(outputSchema); - return { client, server, viewOutput }; + return { client, server, outputSchema }; }; describe.skipIf(process.platform === "win32")( @@ -68,7 +70,7 @@ describe.skipIf(process.platform === "win32")( "derives inspection and summary coverage from inventory traversal (FIFO: $withFifo)", async ({ withFifo, status, sourceCoverage }) => { const path = await writeBundle(withFifo); - const { client, server, viewOutput } = await connect(); + const { client, server, outputSchema } = await connect(); try { const opened = await client.callTool({ name: "open_binary", @@ -107,8 +109,11 @@ describe.skipIf(process.platform === "win32")( for (const reply of summaries) { expect(reply.isError, JSON.stringify(reply)).not.toBe(true); expect( - viewOutput(reply.structuredContent), - JSON.stringify(viewOutput.errors), + schemaValidation.assertions.validate( + outputSchema, + reply.structuredContent, + ), + JSON.stringify(schemaValidation.assertions.errors), ).toBe(true); const view = analysisViewResultSchema.parse( parseEvidence(reply.structuredContent).normalized_result, diff --git a/tests/boundary/mcp/artifactMcp.inlineInventory.test.ts b/tests/boundary/mcp/artifactMcp.inlineInventory.test.ts index c5550c7047..34a29e4cdb 100644 --- a/tests/boundary/mcp/artifactMcp.inlineInventory.test.ts +++ b/tests/boundary/mcp/artifactMcp.inlineInventory.test.ts @@ -4,9 +4,9 @@ import { readFile, unlink, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; -import { Ajv2020 } from "ajv/dist/2020.js"; import { TextReader, Uint8ArrayWriter, ZipWriter } from "@zip.js/zip.js"; -import { expect, it } from "vitest"; +import { afterAll, expect, it } from "vitest"; +import { createMcpSchemaValidation } from "../../fixtures/mcpSchemaValidation.js"; import { z } from "zod"; import { keyedArchiveResultSchema } from "../../../src/domain/apple/keyedArchive.js"; @@ -27,6 +27,9 @@ import { type AnalysisViewResult, } from "../../../src/domain/analysisView/analysisView.js"; +const schemaValidation = createMcpSchemaValidation(); +afterAll(() => schemaValidation.clear()); + const assertInventorySubjectValidation = async ( client: Client, source: Evidence, @@ -97,7 +100,10 @@ it("preserves keyed archive identities and rejects inconsistent inventory carrie ); const session = createTestBinarySession(new ArtifactProvider(process.env)); const server = createServer({ kind: "session", session }); - const client = new Client({ name: "keyed-mcp-test", version: "1" }); + const client = new Client( + { name: "keyed-mcp-test", version: "1" }, + { jsonSchemaValidator: schemaValidation }, + ); const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); try { @@ -199,16 +205,15 @@ const createInventoryViewReader = async ( ); if (advertised?.outputSchema === undefined) throw new Error("Missing advertised view schemas"); - const ajv = new Ajv2020({ strict: false, validateFormats: false }); + const ajv = schemaValidation.assertions; const inputSchema: Record = advertised.inputSchema; const outputSchema: Record = advertised.outputSchema; - const validateInput = ajv.compile(inputSchema); - const validateOutput = ajv.compile(outputSchema); const malformed = { source: { kind: "retained-evidence", evidence_id: parent.evidence_id }, view: { kind: "inventory", collection: "nodes", logical_paths: [] }, }; - if (profile === "full") expect(validateInput(malformed)).toBe(false); + if (profile === "full") + expect(ajv.validate(inputSchema, malformed)).toBe(false); const rejected = await client.callTool({ name: "inspect_analysis_view", arguments: malformed, @@ -225,8 +230,8 @@ const createInventoryViewReader = async ( ) => { const arguments_ = { source, view }; expect( - validateInput(arguments_), - JSON.stringify(validateInput.errors), + ajv.validate(inputSchema, arguments_), + JSON.stringify(ajv.errors), ).toBe(true); const reply = await client.callTool({ name: "inspect_analysis_view", @@ -235,8 +240,8 @@ const createInventoryViewReader = async ( if (reply.isError) throw new Error(JSON.stringify(parseMcpToolError(reply))); expect( - validateOutput(reply.structuredContent), - JSON.stringify(validateOutput.errors), + ajv.validate(outputSchema, reply.structuredContent), + JSON.stringify(ajv.errors), ).toBe(true); const evidence = parseEvidence(reply.structuredContent); return { @@ -358,7 +363,10 @@ it.each(["full", "compact"] as const)( }, }, ); - const client = new Client({ name: "artifact-inline-test", version: "1" }); + const client = new Client( + { name: "artifact-inline-test", version: "1" }, + { jsonSchemaValidator: schemaValidation }, + ); const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); try { @@ -497,7 +505,10 @@ it("refuses summary delivery without a retained Evidence owner and preserves com const opened = await session.open(archive); if (!opened.ok) throw opened.error; const server = createServer({ kind: "fixed", analysis: session }); - const client = new Client({ name: "artifact-fixed-test", version: "1" }); + const client = new Client( + { name: "artifact-fixed-test", version: "1" }, + { jsonSchemaValidator: schemaValidation }, + ); const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); try { diff --git a/tests/boundary/mcp/browserMcp.test.ts b/tests/boundary/mcp/browserMcp.test.ts index 2c426ba8ac..103df01633 100644 --- a/tests/boundary/mcp/browserMcp.test.ts +++ b/tests/boundary/mcp/browserMcp.test.ts @@ -4,7 +4,8 @@ import { parseEvidence } from "../../../src/domain/evidence.js"; import { ok as resultOk } from "../../../src/domain/result.js"; import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { Ajv2020 } from "ajv/dist/2020.js"; -import { afterEach, expect, it } from "vitest"; +import { afterAll, afterEach, expect, it } from "vitest"; +import { createMcpSchemaValidation } from "../../fixtures/mcpSchemaValidation.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; import { CdpBrowserProvider } from "../../../src/browser/CdpBrowserProvider.js"; @@ -24,6 +25,9 @@ import { type FakeCdpBrowser, } from "../../fixtures/fakeCdpBrowser.js"; +const schemaValidation = createMcpSchemaValidation(); +afterAll(() => schemaValidation.clear()); + const resources: Array<{ close(): Promise }> = []; const browsers: FakeCdpBrowser[] = []; const INTEGRATION_TEST_TIMEOUT_MS = 20_000; @@ -858,7 +862,10 @@ const connectBrowser = async (profile?: "full" | "compact") => { }), }, ); - const client = new Client({ name: "browser-mcp-test", version: "1" }); + const client = new Client( + { name: "browser-mcp-test", version: "1" }, + { jsonSchemaValidator: schemaValidation }, + ); const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); resources.push(client, server, session); diff --git a/tests/boundary/mcp/callRelationshipAvailability.test.ts b/tests/boundary/mcp/callRelationshipAvailability.test.ts index 97a2ede20d..cbc44f1bf3 100644 --- a/tests/boundary/mcp/callRelationshipAvailability.test.ts +++ b/tests/boundary/mcp/callRelationshipAvailability.test.ts @@ -106,6 +106,10 @@ describe("call relationship capability inventory at the public MCP boundary", () directory, () => relationshipProvider(scenario, calls), resources, + undefined, + // The full scenario retains default advertisement coverage; the + // remaining cases exercise availability and canonical call results. + scenario === FULL_SCENARIO ? undefined : "off", ); const toolsBefore = await mcp.listTools(); const before = await sessionInventory(mcp); diff --git a/tests/boundary/mcp/deferredRuntime.test.ts b/tests/boundary/mcp/deferredRuntime.test.ts index fe2834803a..52463522d9 100644 --- a/tests/boundary/mcp/deferredRuntime.test.ts +++ b/tests/boundary/mcp/deferredRuntime.test.ts @@ -577,37 +577,29 @@ console.log("LATE_PUBLICATION_REFUSED"); expect(child.stderr).toBe(""); }); -it.each([ - {}, - { REA_MCP_INPUT_SCHEMA_PROFILE: "compact" }, - { REA_MCP_INPUT_SCHEMA_PROFILE: "full" }, - { REA_MCP_OUTPUT_SCHEMA_PROFILE: "off" }, -])("preserves the configured catalog profile %j", async (environment) => { - const { client } = await connect("legacy", environment); - const before = await client.listTools(); - const catalog = JSON.parse( - await readFile( - environment.REA_MCP_INPUT_SCHEMA_PROFILE === "full" - ? "dist/mcp-catalog-full.json" - : "dist/mcp-catalog.json", - "utf8", - ), - ); - expect(before.tools).toHaveLength(catalog.tools.length); - const expected = - environment.REA_MCP_OUTPUT_SCHEMA_PROFILE === "off" - ? catalog.tools.map( - ({ outputSchema: _output, ...tool }: Record) => tool, - ) - : catalog.tools; - expect(before.tools).toEqual(expected); - const result = await client.callTool({ - name: "list_unknowns", - arguments: {}, - }); - expect(result.isError).not.toBe(true); - expect(await client.listTools()).toEqual(before); -}); +it.each([{}, { REA_MCP_OUTPUT_SCHEMA_PROFILE: "off" }])( + "preserves the configured catalog profile %j", + async (environment) => { + const { client } = await connect("legacy", environment); + const before = await client.listTools(); + const catalog = JSON.parse(await readFile("dist/mcp-catalog.json", "utf8")); + expect(before.tools).toHaveLength(catalog.tools.length); + const expected = + environment.REA_MCP_OUTPUT_SCHEMA_PROFILE === "off" + ? catalog.tools.map( + ({ outputSchema: _output, ...tool }: Record) => + tool, + ) + : catalog.tools; + expect(before.tools).toEqual(expected); + const result = await client.callTool({ + name: "list_unknowns", + arguments: {}, + }); + expect(result.isError).not.toBe(true); + expect(await client.listTools()).toEqual(before); + }, +); it("runs the real static artifact workflow after deferred initialization", async () => { const { client } = await connect("legacy"); diff --git a/tests/boundary/mcp/desktopApplicationMcp.test.ts b/tests/boundary/mcp/desktopApplicationMcp.test.ts index 34bd6756ac..8c20a74dee 100644 --- a/tests/boundary/mcp/desktopApplicationMcp.test.ts +++ b/tests/boundary/mcp/desktopApplicationMcp.test.ts @@ -1,10 +1,8 @@ -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { mkdir, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { execFileSync } from "node:child_process"; import { createPackageWithOptions } from "@electron/asar"; import { expect, it, onTestFinished } from "vitest"; -import { createServer } from "../../../src/server/createServer.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { advertisedMcpToolSchemas } from "../../fixtures/advertisedMcpTool.js"; @@ -101,17 +99,16 @@ it("advertises exact schemas and retains a compact directory answer with typed s const session = createTestBinarySession(() => { throw new Error("Directory inspection must not acquire a deep provider"); }); - const server = createServer({ kind: "session", session }); - const client = new Client({ name: "desktop-directory-test", version: "1" }); - onTestFinished(async () => { - await client.close(); - await server.close(); - await session.close(); + const { client } = await createApplicationMcpHarness({ + session, + clientIdentity: { name: "desktop-directory-test", version: "1" }, + serverOptions: { + environment: { + ...process.env, + REA_MCP_MAX_INPUT_ELEMENTS: process.env.REA_MCP_MAX_INPUT_ELEMENTS, + }, + }, }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - await server.connect(serverTransport); - await client.connect(clientTransport); const schemas = await advertisedMcpToolSchemas( client, "inspect_desktop_application", diff --git a/tests/boundary/mcp/electronFusesMcp.test.ts b/tests/boundary/mcp/electronFusesMcp.test.ts index 5ade4b4740..a18b02d893 100644 --- a/tests/boundary/mcp/electronFusesMcp.test.ts +++ b/tests/boundary/mcp/electronFusesMcp.test.ts @@ -23,7 +23,7 @@ import { createTestBinarySession } from "../../fixtures/binarySession.js"; import { mcpTest } from "../../support/mcp/mcpFixture.js"; const fuseMcpTest = mcpTest.extend<{ - fuse: { client: Client }; + fuse: { client: Client; session: ReturnType }; }>({ fuse: async ({ mcp }, use) => { const session = createTestBinarySession(() => { @@ -35,7 +35,7 @@ const fuseMcpTest = mcpTest.extend<{ const client = await mcp.connect( createServer({ kind: "session", session }), ); - await use({ client }); + await use({ client, session }); } finally { await session.close(); } @@ -44,141 +44,129 @@ const fuseMcpTest = mcpTest.extend<{ fuseMcpTest( "advertises fuse inspection and preserves unknown fuse bytes through MCP", - async ({ fuse: { client }, mcp }) => { - const session = createTestBinarySession(() => { - throw new Error( - "Electron fuse inspection must not start a deep provider", - ); + async ({ fuse: { client, session } }) => { + const status = await client.callTool({ + name: "binary_session", + arguments: {}, }); - try { - const connected = await mcp.connect( - createServer({ kind: "session", session }), - ); - const status = await connected.callTool({ - name: "binary_session", - arguments: {}, - }); - const availability = expandToolAvailability( - toolContract("binary_session").outputSchema.parse( - status.structuredContent, - ).result.tool_availability, - ); - expect(availability).toContainEqual( - expect.objectContaining({ - name: "inspect_electron_fuses", - available: true, - reason: "available", - }), - ); - const advertised = (await connected.listTools()).tools.find( - (tool) => tool.name === "inspect_electron_fuses", - ); - if (advertised?.outputSchema === undefined) - throw new Error("Electron fuse inspection must publish both schemas"); - const ajv = new Ajv2020({ strict: false, validateFormats: false }); - const inputSchema: Record = advertised.inputSchema; - const outputSchema: Record = advertised.outputSchema; - expect(ajv.validateSchema(inputSchema)).toBe(true); - expect(ajv.validateSchema(outputSchema)).toBe(true); - expect(advertised.annotations).toMatchObject({ - readOnlyHint: false, - destructiveHint: false, - idempotentHint: true, - openWorldHint: false, - }); - - const root = await createTestTempDirectory("rea-electron-fuses-mcp-"); - const elf = elfFuseImage([ELECTRON_FUSE_STATE_BYTES.removed, 0x99]); - const path = join(root, "application"); - await writeFile(path, elf.bytes); - const response = await connected.callTool({ + const availability = expandToolAvailability( + toolContract("binary_session").outputSchema.parse( + status.structuredContent, + ).result.tool_availability, + ); + expect(availability).toContainEqual( + expect.objectContaining({ name: "inspect_electron_fuses", - arguments: { path }, - }); - expect(response.isError, JSON.stringify(response.content)).not.toBe(true); - expect( - ajv.validate(outputSchema, response.structuredContent), - JSON.stringify(ajv.errors), - ).toBe(true); - const parsed = toolContract("inspect_electron_fuses").outputSchema.parse( - response.structuredContent, - ); - const evidence = parseEvidence(parsed); - expect(evidence.confidence).toBe("observed"); - expect(parsed.normalized_result.artifact.format).toBe("elf"); - expect( - parsed.normalized_result.slices[0]?.sentinels[0]?.fuses, - ).toMatchObject([ - { index: 0, name: "RunAsNode", state: "removed", raw_byte: 0x72 }, - { - index: 1, - name: "EnableCookieEncryption", - state: "unknown", - raw_byte: 0x99, + available: true, + reason: "available", + }), + ); + const advertised = (await client.listTools()).tools.find( + (tool) => tool.name === "inspect_electron_fuses", + ); + if (advertised?.outputSchema === undefined) + throw new Error("Electron fuse inspection must publish both schemas"); + const ajv = new Ajv2020({ strict: false, validateFormats: false }); + const inputSchema: Record = advertised.inputSchema; + const outputSchema: Record = advertised.outputSchema; + expect(ajv.validateSchema(inputSchema)).toBe(true); + expect(ajv.validateSchema(outputSchema)).toBe(true); + expect(advertised.annotations).toMatchObject({ + readOnlyHint: false, + destructiveHint: false, + idempotentHint: true, + openWorldHint: false, + }); + + const root = await createTestTempDirectory("rea-electron-fuses-mcp-"); + const elf = elfFuseImage([ELECTRON_FUSE_STATE_BYTES.removed, 0x99]); + const path = join(root, "application"); + await writeFile(path, elf.bytes); + const response = await client.callTool({ + name: "inspect_electron_fuses", + arguments: { path }, + }); + expect(response.isError, JSON.stringify(response.content)).not.toBe(true); + expect( + ajv.validate(outputSchema, response.structuredContent), + JSON.stringify(ajv.errors), + ).toBe(true); + const parsed = toolContract("inspect_electron_fuses").outputSchema.parse( + response.structuredContent, + ); + const evidence = parseEvidence(parsed); + expect(evidence.confidence).toBe("observed"); + expect(parsed.normalized_result.artifact.format).toBe("elf"); + expect( + parsed.normalized_result.slices[0]?.sentinels[0]?.fuses, + ).toMatchObject([ + { index: 0, name: "RunAsNode", state: "removed", raw_byte: 0x72 }, + { + index: 1, + name: "EnableCookieEncryption", + state: "unknown", + raw_byte: 0x99, + }, + ]); + expect(parsed.normalized_result.runtime_implications).toEqual([ + expect.objectContaining({ + subject: "electron_run_as_node", + relationship: "derivation", + outcome: "not_honored", + fuse_index: 0, + }), + ]); + expect(session.evidenceById(evidence.evidence_id)).toEqual(evidence); + + const duplicated = twoSentinelElf(); + const duplicatePath = join(root, "duplicated"); + await writeFile(duplicatePath, duplicated.bytes); + const duplicates = await client.callTool({ + name: "inspect_electron_fuses", + arguments: { path: duplicatePath }, + }); + expect(duplicates.isError).not.toBe(true); + const duplicateEvidence = toolContract( + "inspect_electron_fuses", + ).outputSchema.parse(duplicates.structuredContent); + expect( + duplicateEvidence.normalized_result.slices[0]?.sentinels.map( + (item) => item.sentinel_offset, + ), + ).toEqual([...duplicated.offsets]); + + const truncatedPath = join(root, "truncated"); + await writeFile(truncatedPath, truncatedFuseImage()); + const truncated = await client.callTool({ + name: "inspect_electron_fuses", + arguments: { path: truncatedPath }, + }); + expect(truncated.isError).not.toBe(true); + const truncatedEvidence = toolContract( + "inspect_electron_fuses", + ).outputSchema.parse(truncated.structuredContent); + expect( + truncatedEvidence.normalized_result.slices[0]?.sentinels[0], + ).toMatchObject({ + complete: false, + declared_length: 4, + wire_version: 1, + }); + + const relative = await client.callTool({ + name: "inspect_electron_fuses", + arguments: { path: "relative.bin" }, + }); + expect(relative.isError).toBe(true); + expect(parseMcpToolError(relative)).toMatchObject({ + error: { + code: "invalid_request", + details: { + operation: "inspect_electron_fuses", + issues: [expect.objectContaining({ path: ["path"] })], }, - ]); - expect(parsed.normalized_result.runtime_implications).toEqual([ - expect.objectContaining({ - subject: "electron_run_as_node", - relationship: "derivation", - outcome: "not_honored", - fuse_index: 0, - }), - ]); - expect(session.evidenceById(evidence.evidence_id)).toEqual(evidence); - - const duplicated = twoSentinelElf(); - const duplicatePath = join(root, "duplicated"); - await writeFile(duplicatePath, duplicated.bytes); - const duplicates = await connected.callTool({ - name: "inspect_electron_fuses", - arguments: { path: duplicatePath }, - }); - expect(duplicates.isError).not.toBe(true); - const duplicateEvidence = toolContract( - "inspect_electron_fuses", - ).outputSchema.parse(duplicates.structuredContent); - expect( - duplicateEvidence.normalized_result.slices[0]?.sentinels.map( - (item) => item.sentinel_offset, - ), - ).toEqual([...duplicated.offsets]); - - const truncatedPath = join(root, "truncated"); - await writeFile(truncatedPath, truncatedFuseImage()); - const truncated = await connected.callTool({ - name: "inspect_electron_fuses", - arguments: { path: truncatedPath }, - }); - expect(truncated.isError).not.toBe(true); - const truncatedEvidence = toolContract( - "inspect_electron_fuses", - ).outputSchema.parse(truncated.structuredContent); - expect( - truncatedEvidence.normalized_result.slices[0]?.sentinels[0], - ).toMatchObject({ - complete: false, - declared_length: 4, - wire_version: 1, - }); - - const relative = await client.callTool({ - name: "inspect_electron_fuses", - arguments: { path: "relative.bin" }, - }); - expect(relative.isError).toBe(true); - expect(parseMcpToolError(relative)).toMatchObject({ - error: { - code: "invalid_request", - details: { - operation: "inspect_electron_fuses", - issues: [expect.objectContaining({ path: ["path"] })], - }, - }, - }); - } finally { - await session.close(); - } + }, + }); }, ); diff --git a/tests/boundary/mcp/enhancedTools.06-rejects-incomplete-function-dossiers-at.test.ts b/tests/boundary/mcp/enhancedTools.06-rejects-incomplete-function-dossiers-at.test.ts index e839b544c1..2828e30768 100644 --- a/tests/boundary/mcp/enhancedTools.06-rejects-incomplete-function-dossiers-at.test.ts +++ b/tests/boundary/mcp/enhancedTools.06-rejects-incomplete-function-dossiers-at.test.ts @@ -1,3 +1,4 @@ +import type { CallToolResult } from "@modelcontextprotocol/server"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { afterEach, describe, expect, it } from "vitest"; @@ -8,6 +9,21 @@ import { closeEnhancedToolResources, connect } from "./enhancedToolsHarness.js"; afterEach(closeEnhancedToolResources); +const expectDossierFailure = (result: CallToolResult) => { + const error = parseMcpToolError(result); + expect(error.error).toMatchObject({ + code: "unreadable_output", + details: { + operation: "analyze_function", + reason: "provider output did not match the dossier contract", + }, + }); + expect(result.content).toContainEqual({ + type: "text", + text: JSON.stringify(error), + }); +}; + describe("analyze_function MCP producer validation", () => { it("rejects an incomplete function dossier", async () => { const client = await connect({ @@ -21,14 +37,10 @@ describe("analyze_function MCP producer validation", () => { }); const result = await client.callTool({ name: "analyze_function", - arguments: { procedure: "0x1" }, + arguments: { procedure: "0x1", detail: "complete" }, }); - expect(result.isError).toBe(true); - expect(result.content).toContainEqual({ - type: "text", - text: JSON.stringify(parseMcpToolError(result)), - }); + expectDossierFailure(result); }); it.each([ @@ -53,14 +65,10 @@ describe("analyze_function MCP producer validation", () => { }); const result = await client.callTool({ name: "analyze_function", - arguments: { procedure: "0x1" }, + arguments: { procedure: "0x1", detail: "complete" }, }); - expect(result.isError).toBe(true); - expect(result.content).toContainEqual({ - type: "text", - text: JSON.stringify(parseMcpToolError(result)), - }); + expectDossierFailure(result); }); it("rejects a malformed collection in an otherwise complete dossier", async () => { @@ -82,13 +90,9 @@ describe("analyze_function MCP producer validation", () => { }); const result = await client.callTool({ name: "analyze_function", - arguments: { procedure: "0x1" }, + arguments: { procedure: "0x1", detail: "complete" }, }); - expect(result.isError).toBe(true); - expect(result.content).toContainEqual({ - type: "text", - text: JSON.stringify(parseMcpToolError(result)), - }); + expectDossierFailure(result); }); }); diff --git a/tests/boundary/mcp/evmLimiterAvailability.test.ts b/tests/boundary/mcp/evmLimiterAvailability.test.ts index deb3e0bf16..f62a7079d0 100644 --- a/tests/boundary/mcp/evmLimiterAvailability.test.ts +++ b/tests/boundary/mcp/evmLimiterAvailability.test.ts @@ -1,12 +1,12 @@ import { join } from "node:path"; -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { expect, it } from "vitest"; import { EvmInterfaceService } from "../../../src/application/evm/EvmInterfaceService.js"; import { expandToolAvailability } from "../../../src/contracts/toolOutputSchemaPrimitives.js"; import { sessionOutputSchemas } from "../../../src/contracts/toolOutputSchemaGroups.js"; import { createServer } from "../../../src/server/createServer.js"; +import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; @@ -15,23 +15,17 @@ const linuxX64 = process.platform === "linux" && process.arch === "x64"; const connect = async ( options: Parameters[1], ): Promise<{ - readonly client: Client; - readonly server: ReturnType; readonly availability: () => Promise | undefined>; - readonly close: () => Promise; }> => { const session = createTestBinarySession(() => { throw new Error("binary analysis must not start during status discovery"); }); - const server = createServer({ kind: "session", session }, options); - const client = new Client({ name: "evm-limiter-availability", version: "1" }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - await server.connect(serverTransport); - await client.connect(clientTransport); + const { client } = await createApplicationMcpHarness({ + session, + serverOptions: options, + clientIdentity: { name: "evm-limiter-availability", version: "1" }, + }); return { - client, - server, availability: async () => { const response = await client.callTool({ name: "binary_session", @@ -45,13 +39,6 @@ const connect = async ( ({ name }) => name === "inspect_evm_interface", ); }, - close: async () => { - await Promise.allSettled([ - client.close(), - server.close(), - session.close(), - ]); - }, }; }; @@ -66,15 +53,11 @@ it.runIf(linuxX64)( REA_MCP_OUTPUT_SCHEMA_PROFILE: "off", }, }); - try { - expect(await harness.availability()).toMatchObject({ - available: false, - reason: "provider_missing", - remediation: expect.stringContaining(limiter), - }); - } finally { - await harness.close(); - } + expect(await harness.availability()).toMatchObject({ + available: false, + reason: "provider_missing", + remediation: expect.stringContaining(limiter), + }); }, ); @@ -94,13 +77,9 @@ it("keeps a caller-injected EVM service independent of the default prlimit prere }, evmInterface: service, }); - try { - expect(await harness.availability()).toMatchObject({ - available: true, - reason: "available", - remediation: null, - }); - } finally { - await harness.close(); - } + expect(await harness.availability()).toMatchObject({ + available: true, + reason: "available", + remediation: null, + }); }); diff --git a/tests/boundary/mcp/fridaMcp.test.ts b/tests/boundary/mcp/fridaMcp.test.ts index 1043df60d0..26d3e78cfe 100644 --- a/tests/boundary/mcp/fridaMcp.test.ts +++ b/tests/boundary/mcp/fridaMcp.test.ts @@ -2,7 +2,7 @@ import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { Ajv2020 } from "ajv/dist/2020.js"; import type { AnySchema } from "ajv"; import type { DeviceManager } from "frida"; -import { expect, it, onTestFinished } from "vitest"; +import { afterAll, expect, it, onTestFinished } from "vitest"; import type { FridaInstrumentationPort } from "../../../src/application/frida/FridaInstrumentationPort.js"; import { toolContract } from "../../../src/contracts/toolContracts.js"; @@ -15,6 +15,10 @@ import { fridaSessionStatusSchema } from "../../../src/domain/fridaSessionStatus import { evidenceBundleSummarySchema } from "../../../src/domain/evidenceRecordSummary.js"; import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; import { parseMcpEvidenceBundle } from "../../fixtures/mcpEvidenceBundle.js"; +import { createMcpSchemaValidation } from "../../fixtures/mcpSchemaValidation.js"; + +const schemaValidation = createMcpSchemaValidation(); +afterAll(() => schemaValidation.clear()); it("retains the remote address, but not authentication, in discovery failures", async () => { const session = createTestBinarySession(() => { @@ -138,7 +142,10 @@ const connectFridaTestClient = async ( session: ReturnType, name: string, ): Promise => { - const client = new Client({ name, version: "1" }); + const client = new Client( + { name, version: "1" }, + { jsonSchemaValidator: schemaValidation }, + ); onTestFinished(async () => { await client.close(); await server.close(); @@ -555,7 +562,10 @@ it("reports Frida tools as unavailable after the optional binding fails to load" { kind: "session", session }, { fridaInstrumentation: provider }, ); - const client = new Client({ name: "frida-optional-binding", version: "1" }); + const client = new Client( + { name: "frida-optional-binding", version: "1" }, + { jsonSchemaValidator: schemaValidation }, + ); onTestFinished(async () => { await client.close(); await server.close(); @@ -650,7 +660,10 @@ it("retains cancellation Evidence after the MCP client stops waiting", async () { kind: "session", session }, { fridaInstrumentation: provider }, ); - const client = new Client({ name: "frida-cancellation", version: "1" }); + const client = new Client( + { name: "frida-cancellation", version: "1" }, + { jsonSchemaValidator: schemaValidation }, + ); const controller = new AbortController(); onTestFinished(async () => { finishLoad?.(); diff --git a/tests/boundary/mcp/functionCallDirectionsMcp.test.ts b/tests/boundary/mcp/functionCallDirectionsMcp.test.ts index e72f9f575e..5d1bfa5159 100644 --- a/tests/boundary/mcp/functionCallDirectionsMcp.test.ts +++ b/tests/boundary/mcp/functionCallDirectionsMcp.test.ts @@ -1,6 +1,6 @@ import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; -import { Ajv2020 } from "ajv/dist/2020.js"; -import { expect, it } from "vitest"; +import { afterAll, expect, it } from "vitest"; +import { createMcpSchemaValidation } from "../../fixtures/mcpSchemaValidation.js"; import { FUNCTION_COMPARISON_EXAMPLE } from "../../../src/contracts/functionComparisonExample.js"; import { createEvidence } from "../../../src/domain/evidence.js"; @@ -33,6 +33,9 @@ const observe = (relationshipCoverage: RelationshipCoverage) => { ); }; +const schemaValidation = createMcpSchemaValidation(); +afterAll(() => schemaValidation.clear()); + const profiles = ["full", "compact"] as const; it.each(profiles)("preserves directional gaps (%s)", async (profile) => { const session = createTestBinarySession(() => { @@ -42,7 +45,10 @@ it.each(profiles)("preserves directional gaps (%s)", async (profile) => { { kind: "session", session }, { environment: { ...process.env, REA_MCP_INPUT_SCHEMA_PROFILE: profile } }, ); - const client = new Client({ name: "function-call-directions", version: "1" }); + const client = new Client( + { name: "function-call-directions", version: "1" }, + { jsonSchemaValidator: schemaValidation }, + ); const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); try { @@ -53,9 +59,8 @@ it.each(profiles)("preserves directional gaps (%s)", async (profile) => { ); if (advertised === undefined) throw new Error("Missing build_call_path contract"); - const ajv = new Ajv2020({ strict: false, validateFormats: false }); + const ajv = schemaValidation.assertions; expect(ajv.validateSchema(advertised.inputSchema)).toBe(true); - const validate = ajv.compile(advertised.inputSchema); for (const { directions, status, exhaustive } of [ { directions: ["callers"] as ["callers"], @@ -87,7 +92,10 @@ it.each(profiles)("preserves directional gaps (%s)", async (profile) => { start: { address: "0x1000" }, goal: { address: "0x9999" }, }; - expect(validate(input), JSON.stringify(validate.errors)).toBe(true); + expect( + ajv.validate(advertised.inputSchema, input), + JSON.stringify(ajv.errors), + ).toBe(true); for (const limited_directions of [ [], ["callers", "callers"], @@ -107,7 +115,8 @@ it.each(profiles)("preserves directional gaps (%s)", async (profile) => { }, ], }; - if (profile === "full") expect(validate(malformed)).toBe(false); + if (profile === "full") + expect(ajv.validate(advertised.inputSchema, malformed)).toBe(false); const rejected = await client.callTool({ name: "build_call_path", arguments: malformed, diff --git a/tests/boundary/mcp/godotPckMcp.test.ts b/tests/boundary/mcp/godotPckMcp.test.ts index 2ba1ec5c67..2a8eee83b0 100644 --- a/tests/boundary/mcp/godotPckMcp.test.ts +++ b/tests/boundary/mcp/godotPckMcp.test.ts @@ -1,9 +1,8 @@ import { writeFile, unlink } from "node:fs/promises"; import { join } from "node:path"; import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; -import { Ajv2020 } from "ajv/dist/2020.js"; -import type { ValidateFunction } from "ajv"; -import { expect, it } from "vitest"; +import { afterAll, expect, it } from "vitest"; +import { createMcpSchemaValidation } from "../../fixtures/mcpSchemaValidation.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; @@ -21,6 +20,9 @@ import { immutableJsonSnapshotMember, } from "../../../src/domain/immutableJson.js"; +const schemaValidation = createMcpSchemaValidation(); +afterAll(() => schemaValidation.clear()); + const viewValidators = async (client: Client) => { const catalog = await client.listTools(); const contract = catalog.tools.find( @@ -28,14 +30,12 @@ const viewValidators = async (client: Client) => { ); if (contract?.outputSchema === undefined) throw new Error("Missing named view output contract"); - const ajv = new Ajv2020({ strict: false, validateFormats: false }); + const ajv = schemaValidation.assertions; const inputSchema: Record = contract.inputSchema; const outputSchema: Record = contract.outputSchema; expect(ajv.validateSchema(inputSchema)).toBe(true); expect(ajv.validateSchema(outputSchema)).toBe(true); - const validateInput = ajv.compile(inputSchema); - const validateOutput = ajv.compile(outputSchema); - return { validateInput, validateOutput }; + return { ajv, inputSchema, outputSchema }; }; const retainedInventoryView = @@ -50,8 +50,8 @@ const retainedInventoryView = view: { kind: "inventory", ...query }, }; expect( - validators.validateInput(arguments_), - JSON.stringify(validators.validateInput.errors), + validators.ajv.validate(validators.inputSchema, arguments_), + JSON.stringify(validators.ajv.errors), ).toBe(true); const reply = await client.callTool({ name: "inspect_analysis_view", @@ -61,8 +61,8 @@ const retainedInventoryView = true, ); expect( - validators.validateOutput(reply.structuredContent), - JSON.stringify(validators.validateOutput.errors), + validators.ajv.validate(validators.outputSchema, reply.structuredContent), + JSON.stringify(validators.ajv.errors), ).toBe(true); const evidence = parseEvidence(reply.structuredContent); expect(evidence.evidence_links).toContain(evidenceId); @@ -72,14 +72,16 @@ const retainedInventoryView = const assertInvalidNodeField = async ( client: Client, evidenceId: string, - validateFullInput?: ValidateFunction, + validators?: Awaited>, ) => { const arguments_ = { source: { kind: "retained-evidence", evidence_id: evidenceId }, view: { kind: "inventory", collection: "nodes", fields: ["logical_path"] }, }; - if (validateFullInput !== undefined) - expect(validateFullInput(arguments_)).toBe(false); + if (validators !== undefined) + expect(validators.ajv.validate(validators.inputSchema, arguments_)).toBe( + false, + ); const rejected = await client.callTool({ name: "inspect_analysis_view", arguments: arguments_, @@ -113,7 +115,10 @@ it.each(inventoryCases)("retains PCK ($profile/v$version)", async (input) => { environment: { ...process.env, REA_MCP_INPUT_SCHEMA_PROFILE: profile }, }, ); - const client = new Client({ name: "pck-contract-test", version: "1" }); + const client = new Client( + { name: "pck-contract-test", version: "1" }, + { jsonSchemaValidator: schemaValidation }, + ); const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); try { @@ -157,7 +162,7 @@ it.each(inventoryCases)("retains PCK ($profile/v$version)", async (input) => { await assertInvalidNodeField( client, parent.evidence_id, - profile === "full" ? validators.validateInput : undefined, + profile === "full" ? validators : undefined, ); const wanted = graph.occurrences.filter((row) => row.logical_path !== "."); const occurrences = await view({ diff --git a/tests/boundary/mcp/jebMcp.test.ts b/tests/boundary/mcp/jebMcp.test.ts index 4755de535c..3db3f65a62 100644 --- a/tests/boundary/mcp/jebMcp.test.ts +++ b/tests/boundary/mcp/jebMcp.test.ts @@ -1,6 +1,6 @@ import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { Ajv2020 } from "ajv/dist/2020.js"; -import { expect, it, onTestFinished } from "vitest"; +import { afterAll, expect, it, onTestFinished } from "vitest"; import { z } from "zod"; import type { @@ -13,6 +13,7 @@ import { parseEvidence } from "../../../src/domain/evidence.js"; import { ok } from "../../../src/domain/result.js"; import { createServer } from "../../../src/server/createServer.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { createMcpSchemaValidation } from "../../fixtures/mcpSchemaValidation.js"; import { expandToolAvailability, toolAvailabilityInventory, @@ -25,6 +26,8 @@ import { } from "../../fixtures/jebMcp.js"; const identity = { id: "jeb", name: "JEB", version: "5.48.0" } as const; +const schemaValidation = createMcpSchemaValidation(); +afterAll(() => schemaValidation.clear()); it("discovers current JEB operation support and recovery through binary_session", async () => { let availability: JebAnalysisAvailability = { @@ -51,7 +54,10 @@ it("discovers current JEB operation support and recovery through binary_session" { kind: "session", session }, { environment: {}, jebAnalysis: provider }, ); - const client = new Client({ name: "jeb-availability", version: "1" }); + const client = new Client( + { name: "jeb-availability", version: "1" }, + { jsonSchemaValidator: schemaValidation }, + ); onTestFinished(async () => { await client.close(); await server.close(); @@ -148,7 +154,10 @@ it("advertises exact valid JEB schemas and records inline evidence through MCP", { kind: "session", session }, { jebAnalysis: provider }, ); - const client = new Client({ name: "jeb-contract", version: "1" }); + const client = new Client( + { name: "jeb-contract", version: "1" }, + { jsonSchemaValidator: schemaValidation }, + ); onTestFinished(async () => { await client.close(); await server.close(); @@ -236,7 +245,10 @@ it("preserves JEB provider failure reasons through the MCP boundary", async () = { kind: "session", session }, { jebAnalysis: unavailable }, ); - const client = new Client({ name: "jeb-failure", version: "1" }); + const client = new Client( + { name: "jeb-failure", version: "1" }, + { jsonSchemaValidator: schemaValidation }, + ); onTestFinished(async () => { await client.close(); await server.close(); @@ -266,7 +278,10 @@ it("keeps the MCP server usable when REA_JEB_MCP_URL is invalid", async () => { { kind: "session", session }, { environment: { REA_JEB_MCP_URL: "http://localhost:8425/mcp" } }, ); - const client = new Client({ name: "jeb-invalid-endpoint", version: "1" }); + const client = new Client( + { name: "jeb-invalid-endpoint", version: "1" }, + { jsonSchemaValidator: schemaValidation }, + ); onTestFinished(async () => { await client.close(); await server.close(); @@ -353,7 +368,10 @@ it.each( { kind: "session", session }, { jebAnalysis: provider, environment: {} }, ); - const client = new Client({ name: "jeb-attribution", version: "1" }); + const client = new Client( + { name: "jeb-attribution", version: "1" }, + { jsonSchemaValidator: schemaValidation }, + ); try { const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); diff --git a/tests/boundary/mcp/managedNativeVerificationAdapters.test.ts b/tests/boundary/mcp/managedNativeVerificationAdapters.test.ts index 995dd576f7..b9bd3d2292 100644 --- a/tests/boundary/mcp/managedNativeVerificationAdapters.test.ts +++ b/tests/boundary/mcp/managedNativeVerificationAdapters.test.ts @@ -16,7 +16,7 @@ import { managedNativeVerificationResultSchema } from "../../../src/domain/manag import { buildManagedPeFixture } from "../../../src/dotnet/ManagedPe.fixture.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; -import { cliTest } from "../../support/cli/cliFixture.js"; +import { cliTest, runFileCli } from "../../support/cli/cliFixture.js"; const client = new Client({ name: "managed-native-adapters", version: "1" }); const transport = new StdioClientTransport({ @@ -343,9 +343,13 @@ cliTest( ); const request = join(root, `verify-${String(index)}.json`); await writeFile(request, JSON.stringify(input)); - const cliRun = await cli.run({ + const invocation = { arguments: ["verify-managed-native-boundaries", request, "--json"], - }); + }; + const cliRun = + index === 0 + ? await cli.run(invocation) + : await runFileCli(cli, invocation); expect(cliRun.exitCode, cliRun.stderr).toBe(0); const cliResult = managedNativeVerificationResultSchema.parse( parseEvidence(cliRun.json).normalized_result, diff --git a/tests/boundary/mcp/nativeMachoSummaryMcp.test.ts b/tests/boundary/mcp/nativeMachoSummaryMcp.test.ts index 5d2337d070..d087c9893b 100644 --- a/tests/boundary/mcp/nativeMachoSummaryMcp.test.ts +++ b/tests/boundary/mcp/nativeMachoSummaryMcp.test.ts @@ -1,7 +1,8 @@ import { copyFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { describe, expect } from "vitest"; +import { afterAll, describe, expect } from "vitest"; +import { createMcpSchemaValidation } from "../../fixtures/mcpSchemaValidation.js"; import { z } from "zod"; import { inspectMachoSchema } from "../../../src/domain/native/nativeInspection.js"; @@ -17,6 +18,9 @@ import { openProductionTarget } from "../../support/mcp/productionSession.js"; import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; import { advertisedMcpToolSchemas } from "../../fixtures/advertisedMcpTool.js"; +const schemaValidation = createMcpSchemaValidation(); +afterAll(() => schemaValidation.clear()); + const evidenceOf = (schema: Schema) => z.object({ evidence_id: z.string(), @@ -166,8 +170,14 @@ mcpTest( environment: { ...process.env, REA_MCP_INPUT_SCHEMA_PROFILE: "full" }, }, ), + undefined, + schemaValidation, + ); + const tool = await advertisedMcpToolSchemas( + client, + "inspect_macho", + schemaValidation.assertions, ); - const tool = await advertisedMcpToolSchemas(client, "inspect_macho"); expect( tool.ajv.validate(tool.outputSchema, source), JSON.stringify(tool.ajv.errors), @@ -175,6 +185,7 @@ mcpTest( const view = await advertisedMcpToolSchemas( client, "inspect_analysis_view", + schemaValidation.assertions, ); for (const [facet, expected] of [ ["segments", segment], @@ -325,10 +336,13 @@ mcpTest( environment: { ...process.env, REA_MCP_INPUT_SCHEMA_PROFILE: "full" }, }, ), + undefined, + schemaValidation, ); const schemas = await advertisedMcpToolSchemas( client, "inspect_analysis_view", + schemaValidation.assertions, ); for (const [facet, expected] of [ ["imports", imported], diff --git a/tests/boundary/mcp/openBinaryFifo.test.ts b/tests/boundary/mcp/openBinaryFifo.test.ts index aedfc3d6c5..73311b3c36 100644 --- a/tests/boundary/mcp/openBinaryFifo.test.ts +++ b/tests/boundary/mcp/openBinaryFifo.test.ts @@ -2,12 +2,11 @@ import { execFile } from "node:child_process"; import { join } from "node:path"; import { promisify } from "node:util"; -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { expect, it } from "vitest"; import { ArtifactProvider } from "../../../src/artifacts/ArtifactProvider.js"; -import { createServer } from "../../../src/server/createServer.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; import { readWithoutFifoWriter } from "../../fixtures/fifoInput.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; @@ -19,33 +18,22 @@ it.skipIf(process.platform === "win32")( const fifoPath = join(root, "input.bin"); await promisify(execFile)("mkfifo", [fifoPath]); const session = createTestBinarySession(new ArtifactProvider(process.env)); - const server = createServer({ kind: "session", session }); - const client = new Client({ name: "fifo-target-test", version: "1" }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - - try { - await server.connect(serverTransport); - await client.connect(clientTransport); - const outcome = await readWithoutFifoWriter(fifoPath, () => - client.callTool({ name: "open_binary", arguments: { path: fifoPath } }), - ); - expect(outcome.state).toBe("completed"); - if (outcome.state !== "completed") - throw new Error("MCP open_binary waited for a FIFO writer"); - expect(outcome.result.isError).toBe(true); - expect(parseMcpToolError(outcome.result)).toMatchObject({ - error: { - code: "target_unavailable", - message: expect.stringContaining("not a regular file"), - }, - }); - } finally { - await Promise.allSettled([ - client.close(), - server.close(), - session.close(), - ]); - } + const { client } = await createApplicationMcpHarness({ + session, + clientIdentity: { name: "fifo-target-test", version: "1" }, + }); + const outcome = await readWithoutFifoWriter(fifoPath, () => + client.callTool({ name: "open_binary", arguments: { path: fifoPath } }), + ); + expect(outcome.state).toBe("completed"); + if (outcome.state !== "completed") + throw new Error("MCP open_binary waited for a FIFO writer"); + expect(outcome.result.isError).toBe(true); + expect(parseMcpToolError(outcome.result)).toMatchObject({ + error: { + code: "target_unavailable", + message: expect.stringContaining("not a regular file"), + }, + }); }, ); diff --git a/tests/boundary/mcp/peCertificatesMcp.test.ts b/tests/boundary/mcp/peCertificatesMcp.test.ts index ab106ed608..adfed26934 100644 --- a/tests/boundary/mcp/peCertificatesMcp.test.ts +++ b/tests/boundary/mcp/peCertificatesMcp.test.ts @@ -1,8 +1,7 @@ import { writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { expect } from "vitest"; -import { Ajv2020 } from "ajv/dist/2020.js"; -import type { AnySchema } from "ajv"; +import { afterAll, expect } from "vitest"; +import { createMcpSchemaValidation } from "../../fixtures/mcpSchemaValidation.js"; import { toolContract } from "../../../src/contracts/toolContracts.js"; import { createServer } from "../../../src/server/createServer.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; @@ -15,11 +14,18 @@ import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { mcpTest, type TestMcp } from "../../support/mcp/mcpFixture.js"; import { repeatedCertificateSignedData } from "../../fixtures/peCertificateAllowance.js"; +const schemaValidation = createMcpSchemaValidation(); +afterAll(() => schemaValidation.clear()); + const connect = async (mcp: TestMcp) => { const session = createTestBinarySession(() => { throw new Error("PE certificate inspection must not start a deep provider"); }); - const client = await mcp.connect(createServer({ kind: "session", session })); + const client = await mcp.connect( + createServer({ kind: "session", session }), + undefined, + schemaValidation, + ); return { client, session }; }; @@ -33,10 +39,9 @@ mcpTest( ); if (advertised?.outputSchema === undefined) throw new Error("PE certificates output schema is missing"); - const schema = advertised.outputSchema as AnySchema; - const ajv = new Ajv2020({ strict: false, validateFormats: false }); + const schema: Record = advertised.outputSchema; + const ajv = schemaValidation.assertions; expect(ajv.validateSchema(schema)).toBe(true); - const validate = ajv.compile(schema); const root = await createTestTempDirectory("rea-pe-cert-schema-"); const path = join(root, "timestamped.exe"); await writeFile( @@ -54,8 +59,8 @@ mcpTest( }); expect(result.isError).not.toBe(true); expect( - validate(result.structuredContent), - JSON.stringify(validate.errors), + ajv.validate(schema, result.structuredContent), + JSON.stringify(ajv.errors), ).toBe(true); } finally { await session.close(); diff --git a/tests/boundary/mcp/pyinstallerArtifactMcp.test.ts b/tests/boundary/mcp/pyinstallerArtifactMcp.test.ts index 86b40724db..f0a9c3f48e 100644 --- a/tests/boundary/mcp/pyinstallerArtifactMcp.test.ts +++ b/tests/boundary/mcp/pyinstallerArtifactMcp.test.ts @@ -3,8 +3,8 @@ import { createHash } from "node:crypto"; import { join } from "node:path"; import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; -import { Ajv2020 } from "ajv/dist/2020.js"; -import { expect, it } from "vitest"; +import { afterAll, expect, it } from "vitest"; +import { createMcpSchemaValidation } from "../../fixtures/mcpSchemaValidation.js"; import { createArtifactBinarySession } from "../../../src/composition/binary.js"; import { @@ -20,6 +20,9 @@ import { analysisViewResultSchema } from "../../../src/domain/analysisView/analy import { createServer } from "../../../src/server/createServer.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +const schemaValidation = createMcpSchemaValidation(); +afterAll(() => schemaValidation.clear()); + const assertMetadataQueries = async ( client: Client, parent: Evidence, @@ -29,11 +32,9 @@ const assertMetadataQueries = async ( (tool) => tool.name === "inspect_analysis_view", ); expect(definition?.outputSchema).toBeDefined(); - const ajv = new Ajv2020({ strict: false, validateFormats: false }); + const ajv = schemaValidation.assertions; const inputSchema: Record = definition!.inputSchema; const outputSchema: Record = definition!.outputSchema!; - const input = ajv.compile(inputSchema); - const output = ajv.compile(outputSchema); const ids: string[] = []; const selected: unknown[] = []; for (const offset of [0, 1, 2, 3]) { @@ -48,15 +49,18 @@ const assertMetadataQueries = async ( limit: 1, }, }; - expect(input(args), JSON.stringify(input.errors)).toBe(true); + expect(ajv.validate(inputSchema, args), JSON.stringify(ajv.errors)).toBe( + true, + ); const reply = await client.callTool({ name: "inspect_analysis_view", arguments: args, }); expect(reply.isError).not.toBe(true); - expect(output(reply.structuredContent), JSON.stringify(output.errors)).toBe( - true, - ); + expect( + ajv.validate(outputSchema, reply.structuredContent), + JSON.stringify(ajv.errors), + ).toBe(true); const evidence = parseEvidence(reply.structuredContent); const view = analysisViewResultSchema.parse(evidence.normalized_result); if (view.kind !== "inventory") throw new Error("Expected inventory page"); @@ -201,7 +205,10 @@ it.each([false, true])( const path = await writeExecutable(opaque); const session = await createArtifactBinarySession(process.env); const server = createServer({ kind: "session", session }); - const client = new Client({ name: "pyinstaller-mcp-test", version: "1" }); + const client = new Client( + { name: "pyinstaller-mcp-test", version: "1" }, + { jsonSchemaValidator: schemaValidation }, + ); const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); try { @@ -333,10 +340,13 @@ it("preserves legacy CArchive identities and modern bytecode derivation through await writeFile(path, fixture.archive); const session = await createArtifactBinarySession(process.env); const server = createServer({ kind: "session", session }); - const client = new Client({ - name: "pyinstaller-bytecode-mcp-test", - version: "1", - }); + const client = new Client( + { + name: "pyinstaller-bytecode-mcp-test", + version: "1", + }, + { jsonSchemaValidator: schemaValidation }, + ); const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); try { @@ -391,10 +401,13 @@ it("preserves surrogatepass PYZ module names through inspect_artifact", async () const path = await writeSurrogateNamespaceExecutable(); const session = await createArtifactBinarySession(process.env); const server = createServer({ kind: "session", session }); - const client = new Client({ - name: "pyinstaller-surrogate-mcp-test", - version: "1", - }); + const client = new Client( + { + name: "pyinstaller-surrogate-mcp-test", + version: "1", + }, + { jsonSchemaValidator: schemaValidation }, + ); const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); try { @@ -514,10 +527,13 @@ it.each([ ); const session = await createArtifactBinarySession(process.env); const server = createServer({ kind: "session", session }); - const client = new Client({ - name: "opaque-pyz-metadata-test", - version: "1", - }); + const client = new Client( + { + name: "opaque-pyz-metadata-test", + version: "1", + }, + { jsonSchemaValidator: schemaValidation }, + ); const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); try { @@ -643,7 +659,10 @@ it.each([false, true])( ); const session = await createArtifactBinarySession(process.env); const server = createServer({ kind: "session", session }); - const client = new Client({ name: "pyz-role-test", version: "1" }); + const client = new Client( + { name: "pyz-role-test", version: "1" }, + { jsonSchemaValidator: schemaValidation }, + ); const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); let outputRoot: string | undefined; diff --git a/tests/boundary/mcp/pythonPycMcp.test.ts b/tests/boundary/mcp/pythonPycMcp.test.ts index d42ef8944b..ac727d0c6d 100644 --- a/tests/boundary/mcp/pythonPycMcp.test.ts +++ b/tests/boundary/mcp/pythonPycMcp.test.ts @@ -1,7 +1,7 @@ import { writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { Ajv2020 } from "ajv/dist/2020.js"; -import { expect } from "vitest"; +import { createMcpSchemaValidation } from "../../fixtures/mcpSchemaValidation.js"; +import { afterAll, expect } from "vitest"; import { pythonPycSchema } from "../../../src/domain/python/pythonPyc.js"; import { toolContract } from "../../../src/contracts/toolContracts.js"; import { createServer } from "../../../src/server/createServer.js"; @@ -34,11 +34,18 @@ const marshalInt32 = (value: number): Buffer => { const marshalSingleItemTuple = (item: Buffer): Buffer => Buffer.concat([Buffer.from([0x29, 1]), item]); +const schemaValidation = createMcpSchemaValidation(); +afterAll(() => schemaValidation.clear()); + const connect = async (mcp: TestMcp) => { const session = createTestBinarySession(() => { throw new Error("Python pyc inspection must not start a deep provider"); }); - const client = await mcp.connect(createServer({ kind: "session", session })); + const client = await mcp.connect( + createServer({ kind: "session", session }), + undefined, + schemaValidation, + ); return { client, session }; }; @@ -148,7 +155,7 @@ mcpTest( if (advertised?.outputSchema === undefined) throw new Error("Missing inspect_python_pyc output schema"); const outputSchema: Record = advertised.outputSchema; - const ajv = new Ajv2020({ strict: false, validateFormats: false }); + const ajv = schemaValidation.assertions; expect(ajv.validateSchema(outputSchema)).toBe(true); const root = await createTestTempDirectory("rea-python-pyc-mcp-"); const path = join(root, "sample.pyc"); @@ -222,7 +229,7 @@ mcpTest( if (advertised?.outputSchema === undefined) throw new Error("Missing inspect_python_pyc output schema"); const outputSchema: Record = advertised.outputSchema; - const ajv = new Ajv2020({ strict: false, validateFormats: false }); + const ajv = schemaValidation.assertions; const root = await createTestTempDirectory("rea-python-pyc-mcp-flags-"); for (const testCase of [ { diff --git a/tests/boundary/mcp/retainedApplicationEvidenceMcp.test.ts b/tests/boundary/mcp/retainedApplicationEvidenceMcp.test.ts index f65250d81c..d4b59f73c3 100644 --- a/tests/boundary/mcp/retainedApplicationEvidenceMcp.test.ts +++ b/tests/boundary/mcp/retainedApplicationEvidenceMcp.test.ts @@ -1,6 +1,6 @@ import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; -import { afterEach, expect, it } from "vitest"; +import { afterAll, afterEach, expect, it } from "vitest"; import { Ajv2020 } from "ajv/dist/2020.js"; import type { AnySchema } from "ajv"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; @@ -12,8 +12,11 @@ import { } from "../../../src/contracts/javascript/javascriptApplicationWorkflowExamples.js"; import { JAVASCRIPT_EXPORT_SHAPE_COMPARISON_EXAMPLE } from "../../../src/contracts/javascript/javascriptExportShapeComparisonExample.js"; import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; +import { createMcpSchemaValidation } from "../../fixtures/mcpSchemaValidation.js"; const resources: Array<{ close(): Promise }> = []; +const schemaValidation = createMcpSchemaValidation(); +afterAll(() => schemaValidation.clear()); afterEach(async () => { for (const resource of resources.splice(0)) await resource.close(); }); @@ -33,7 +36,10 @@ async function connect(profile?: "full" | "compact") { { kind: "session", session }, { environment: { ...process.env, REA_MCP_INPUT_SCHEMA_PROFILE: profile } }, ); - const client = new Client({ name: "retained-evidence-test", version: "1" }); + const client = new Client( + { name: "retained-evidence-test", version: "1" }, + { jsonSchemaValidator: schemaValidation }, + ); resources.push(client, server, session); const [a, b] = InMemoryTransport.createLinkedPair(); await server.connect(b); diff --git a/tests/boundary/mcp/sessionAvailabilityRecovery.test.ts b/tests/boundary/mcp/sessionAvailabilityRecovery.test.ts index e0f3884718..c97c153106 100644 --- a/tests/boundary/mcp/sessionAvailabilityRecovery.test.ts +++ b/tests/boundary/mcp/sessionAvailabilityRecovery.test.ts @@ -1,4 +1,3 @@ -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { expect, it } from "vitest"; import { toolContract } from "../../../src/contracts/toolContracts.js"; @@ -8,8 +7,8 @@ import { ProviderAdapterError } from "../../../src/domain/providerAdapterError.j import { ProviderCleanupError } from "../../../src/domain/providerCleanupError.js"; import { err, ok } from "../../../src/domain/result.js"; import { silentLogger } from "../../../src/logger.js"; -import { createServer } from "../../../src/server/createServer.js"; import { observed } from "../../fixtures/analysisExecution.js"; +import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { createBinarySessionTargets, @@ -63,13 +62,11 @@ it("restores MCP tool availability after cancelling a target switch", async () = return resolveProfile(target, options); }, }); - const server = createServer( - { kind: "session", session }, - { environment: {}, logger: silentLogger }, - ); - const client = new Client({ name: "session-recovery", version: "1" }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); + const { client, close } = await createApplicationMcpHarness({ + session, + clientIdentity: { name: "session-recovery", version: "1" }, + serverOptions: { environment: {}, logger: silentLogger }, + }); const controller = new AbortController(); const readStatus = async () => { const reply = await client.callTool({ @@ -81,8 +78,6 @@ it("restores MCP tool availability after cancelling a target switch", async () = ).result; }; try { - await server.connect(serverTransport); - await client.connect(clientTransport); const opened = await client.callTool({ name: "open_binary", arguments: { path: first }, @@ -150,9 +145,7 @@ it("restores MCP tool availability after cancelling a target switch", async () = } finally { controller.abort(); release.resolve(); - await client.close(); - await server.close(); - await session.close(); + await close(); } }); @@ -190,13 +183,11 @@ it("keeps a retired run and its lineage visible over MCP until cleanup succeeds" }; }; const session = createTestBinarySession(provider); - const server = createServer( - { kind: "session", session }, - { environment: {}, logger: silentLogger }, - ); - const client = new Client({ name: "retained-run", version: "1" }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); + const { client, close } = await createApplicationMcpHarness({ + session, + clientIdentity: { name: "retained-run", version: "1" }, + serverOptions: { environment: {}, logger: silentLogger }, + }); const readStatus = async () => { const reply = await client.callTool({ name: "binary_session", @@ -207,8 +198,6 @@ it("keeps a retired run and its lineage visible over MCP until cleanup succeeds" ).result; }; try { - await server.connect(serverTransport); - await client.connect(clientTransport); expect((await readStatus()).analysis_run).toBeNull(); expect( (await client.callTool({ name: "open_binary", arguments: { path } })) @@ -249,9 +238,7 @@ it("keeps a retired run and its lineage visible over MCP until cleanup succeeds" }); } finally { allowed = true; - await client.close(); - await server.close(); - await session.close(); + await close(); } }); @@ -276,13 +263,11 @@ it("keeps provider tools available after one request reaches its deadline", asyn close: () => Promise.resolve(ok(null)), }); const session = createTestBinarySession(provider); - const server = createServer( - { kind: "session", session }, - { environment: {}, logger: silentLogger }, - ); - const client = new Client({ name: "request-timeout", version: "1" }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); + const { client, close } = await createApplicationMcpHarness({ + session, + clientIdentity: { name: "request-timeout", version: "1" }, + serverOptions: { environment: {}, logger: silentLogger }, + }); const availability = async () => { const reply = await client.callTool({ name: "binary_session", @@ -294,8 +279,6 @@ it("keeps provider tools available after one request reaches its deadline", asyn ); }; try { - await server.connect(serverTransport); - await client.connect(clientTransport); expect( (await client.callTool({ name: "open_binary", arguments: { path } })) .isError, @@ -321,8 +304,6 @@ it("keeps provider tools available after one request reaches its deadline", asyn normalized_result: "after-timeout", }); } finally { - await client.close(); - await server.close(); - await session.close(); + await close(); } }); diff --git a/tests/boundary/mcp/sessionMcpHarness.ts b/tests/boundary/mcp/sessionMcpHarness.ts index f718990681..28dcd12596 100644 --- a/tests/boundary/mcp/sessionMcpHarness.ts +++ b/tests/boundary/mcp/sessionMcpHarness.ts @@ -74,6 +74,7 @@ export const createSessionMcpHarness = async ( provider: (closed: string[]) => AnalysisProvider, resources: Array<{ close(): Promise }>, maximumInputElements?: number, + outputSchemaProfile?: "off", ): Promise => { const first = join(root, "first.hop"); const second = join(root, "second.hop"); @@ -90,12 +91,18 @@ export const createSessionMcpHarness = async ( adbDeviceAnalysis, apktoolAnalysis, logger: silentLogger, - ...(maximumInputElements === undefined + ...(maximumInputElements === undefined && + outputSchemaProfile === undefined ? {} : { environment: { ...process.env, - REA_MCP_MAX_INPUT_ELEMENTS: String(maximumInputElements), + ...(maximumInputElements === undefined + ? {} + : { REA_MCP_MAX_INPUT_ELEMENTS: String(maximumInputElements) }), + ...(outputSchemaProfile === undefined + ? {} + : { REA_MCP_OUTPUT_SCHEMA_PROFILE: outputSchemaProfile }), }, }), }, diff --git a/tests/boundary/mcp/sourceToBundleViewsMcp.test.ts b/tests/boundary/mcp/sourceToBundleViewsMcp.test.ts index d835e47f04..d43f830022 100644 --- a/tests/boundary/mcp/sourceToBundleViewsMcp.test.ts +++ b/tests/boundary/mcp/sourceToBundleViewsMcp.test.ts @@ -1,8 +1,8 @@ import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { mkdir, writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { expect, it, onTestFinished } from "vitest"; -import { Ajv2020 } from "ajv/dist/2020.js"; +import { afterAll, expect, it, onTestFinished } from "vitest"; +import { createMcpSchemaValidation } from "../../fixtures/mcpSchemaValidation.js"; import { createServer } from "../../../src/server/createServer.js"; import { parseEvidence } from "../../../src/domain/evidence.js"; import { sourceToBundleComparisonResultSchema } from "../../../src/domain/javascript/sourceToBundleComparisonSchemas.js"; @@ -12,6 +12,9 @@ import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { importReferenceSource } from "../../support/referenceSourceResourceScope.js"; import { analyzeJavaScriptApplication } from "../../support/javascriptApplicationScope.js"; +const schemaValidation = createMcpSchemaValidation(); +afterAll(() => schemaValidation.clear()); + it.each(["full", "compact"] as const)( "selects real source-map correspondence with stable identities and full parent retention (%s)", async (profile) => { @@ -54,7 +57,7 @@ it.each(["full", "compact"] as const)( summary: { summary: full.summary, source_coverage: full.coverage }, }); const { tools } = await client.listTools(); - const ajv = new Ajv2020({ strict: false, validateFormats: false }); + const ajv = schemaValidation.assertions; const comparisonTool = tools.find( (tool) => tool.name === "compare_source_to_bundle", )!; @@ -74,7 +77,7 @@ it.each(["full", "compact"] as const)( limit: 1, }; expect( - ajv.compile(comparisonTool.inputSchema)({ ...inputs, view: request }), + ajv.validate(comparisonTool.inputSchema, { ...inputs, view: request }), ).toBe(true); const selected = await call("compare_source_to_bundle", { ...inputs, @@ -82,7 +85,7 @@ it.each(["full", "compact"] as const)( view: request, }); const outputSchema: Record = comparisonTool.outputSchema!; - expect(ajv.compile(outputSchema)(selected)).toBe(true); + expect(ajv.validate(outputSchema, selected)).toBe(true); expect(selected.confidence).toBe("inferred"); const page = analysisViewResultSchema.parse(selected.normalized_result); expect(page).toMatchObject({ @@ -218,7 +221,10 @@ const comparisonFixture = async (profile: "full" | "compact") => { environment: { ...process.env, REA_MCP_INPUT_SCHEMA_PROFILE: profile }, }, ); - const client = new Client({ name: "source-view", version: "1" }); + const client = new Client( + { name: "source-view", version: "1" }, + { jsonSchemaValidator: schemaValidation }, + ); onTestFinished(async () => { await client.close(); await server.close(); diff --git a/tests/boundary/mcp/toolFamiliesMcp.test.ts b/tests/boundary/mcp/toolFamiliesMcp.test.ts index caa578f8c1..6deb9fd61d 100644 --- a/tests/boundary/mcp/toolFamiliesMcp.test.ts +++ b/tests/boundary/mcp/toolFamiliesMcp.test.ts @@ -1,4 +1,4 @@ -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import type { Client } from "@modelcontextprotocol/client"; import { STDIO_DEFAULT_MAX_BUFFER_SIZE } from "@modelcontextprotocol/server"; import { describe, expect, it } from "vitest"; @@ -14,6 +14,7 @@ import { EvidenceMcpServer } from "../../../src/server/EvidenceMcpServer.js"; import { jsonObjectSchema } from "../../../src/domain/jsonValue.js"; import { ToolResultDelivery } from "../../../src/server/ToolResultDelivery.js"; import { toolRegistrationOptions } from "../../../src/server/toolRegistrationOptions.js"; +import { createTestMcp } from "../../support/mcp/mcpFixture.js"; const scopedServer = ( advertisedToolFamilies: ReadonlySet | undefined, @@ -42,15 +43,15 @@ const withClient = async ( server: EvidenceMcpServer, use: (client: Client) => Promise, ): Promise => { - const client = new Client({ name: "tool-families", version: "0" }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); + const mcp = createTestMcp(); try { - await server.connect(serverTransport); - await client.connect(clientTransport); + const client = await mcp.connect(server, { + name: "tool-families", + version: "0", + }); await use(client); } finally { - await Promise.allSettled([client.close(), server.close()]); + await mcp.close(); } }; diff --git a/tests/boundary/mcp/toolInputAdmission.test.ts b/tests/boundary/mcp/toolInputAdmission.test.ts index 5b83809b36..87954814c0 100644 --- a/tests/boundary/mcp/toolInputAdmission.test.ts +++ b/tests/boundary/mcp/toolInputAdmission.test.ts @@ -1,7 +1,7 @@ -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import type { CallToolResult } from "@modelcontextprotocol/server"; import { STDIO_DEFAULT_MAX_BUFFER_SIZE } from "@modelcontextprotocol/server"; -import { describe, expect, it } from "vitest"; +import { describe, expect } from "vitest"; +import { mcpTest } from "../../support/mcp/mcpFixture.js"; import { TOOL_CONTRACTS } from "../../../src/contracts/toolContracts.js"; import { COMPACT_INPUT_SCHEMA_BUDGET_BYTES } from "../../../src/config/mcpInputSchemaProfile.js"; @@ -24,9 +24,9 @@ const createServer = (profile: "full" | "compact"): EvidenceMcpServer => // handlers. They establish canonical rejection before dispatch; they do not // exercise providers, targets, or the shape of SDK-generated error text. describe("catalog-wide MCP input admission", () => { - it.each(["full", "compact"] as const)( + mcpTest.for(["full", "compact"] as const)( "rejects unknown root keys before dispatch with the %s advertised profile", - async (profile) => { + async (profile, { mcp }) => { const server = createServer(profile); const dispatches = new Map(); for (const contract of TOOL_CONTRACTS) { @@ -46,49 +46,41 @@ describe("catalog-wide MCP input admission", () => { ); } - const client = new Client({ + const client = await mcp.connect(server, { name: "tool-input-admission-test", version: "0", }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - try { - await server.connect(serverTransport); - await client.connect(clientTransport); - expect(TOOL_CONTRACTS.length).toBeGreaterThan(0); - for (const contract of TOOL_CONTRACTS) { - const input = { - ...contract.examples[0]?.input, - __rea_unknown_admission_probe__: true, - }; - expect(contract.inputSchema.safeParse(input).success).toBe(false); + expect(TOOL_CONTRACTS.length).toBeGreaterThan(0); + for (const contract of TOOL_CONTRACTS) { + const input = { + ...contract.examples[0]?.input, + __rea_unknown_admission_probe__: true, + }; + expect(contract.inputSchema.safeParse(input).success).toBe(false); - const result = await client.callTool({ - name: contract.name, - arguments: input, - }); - expect(result.isError, contract.name).toBe(true); - expect(dispatches.has(contract.name), contract.name).toBe(false); - } - - const firstWithExample = TOOL_CONTRACTS.find( - (contract) => contract.examples[0] !== undefined, - ); - const liveExample = firstWithExample?.examples[0]; - if (firstWithExample === undefined || liveExample === undefined) - throw new Error( - "No canonical example is available for the liveness check", - ); - const live = await client.callTool({ - name: firstWithExample.name, - arguments: liveExample.input, + const result = await client.callTool({ + name: contract.name, + arguments: input, }); - expect(live.isError).toBe(true); - expect(dispatches.get(firstWithExample.name)).toBe(1); - } finally { - await Promise.allSettled([client.close(), server.close()]); + expect(result.isError, contract.name).toBe(true); + expect(dispatches.has(contract.name), contract.name).toBe(false); } + + const firstWithExample = TOOL_CONTRACTS.find( + (contract) => contract.examples[0] !== undefined, + ); + const liveExample = firstWithExample?.examples[0]; + if (firstWithExample === undefined || liveExample === undefined) + throw new Error( + "No canonical example is available for the liveness check", + ); + const live = await client.callTool({ + name: firstWithExample.name, + arguments: liveExample.input, + }); + expect(live.isError).toBe(true); + expect(dispatches.get(firstWithExample.name)).toBe(1); }, ); }); diff --git a/tests/boundary/mcp/wasmArtifactMcp.test.ts b/tests/boundary/mcp/wasmArtifactMcp.test.ts index 83e04f6cc0..f77eeff4da 100644 --- a/tests/boundary/mcp/wasmArtifactMcp.test.ts +++ b/tests/boundary/mcp/wasmArtifactMcp.test.ts @@ -1,10 +1,10 @@ -import { Ajv2020 } from "ajv/dist/2020.js"; import { advertisedMcpToolSchemas, assertRecordedMcpEvidence, } from "../../fixtures/advertisedMcpTool.js"; import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; -import { expect, it, onTestFinished, vi } from "vitest"; +import { afterAll, expect, it, onTestFinished, vi } from "vitest"; +import { createMcpSchemaValidation } from "../../fixtures/mcpSchemaValidation.js"; import { readFile, realpath, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { WasmArtifactService } from "../../../src/application/wasm/WasmArtifactService.js"; @@ -25,6 +25,9 @@ import { wasmArtifactExecution, } from "../../fixtures/wasm/artifact.js"; +const schemaValidation = createMcpSchemaValidation(); +afterAll(() => schemaValidation.clear()); + const assertFirstSummaryDelivery = async ( harness: Awaited>, value: ReturnType, @@ -34,7 +37,11 @@ const assertFirstSummaryDelivery = async ( ajv: wasmAjv, inputSchema, outputSchema, - } = await advertisedMcpToolSchemas(harness.client, "inspect_wasm_artifact"); + } = await advertisedMcpToolSchemas( + harness.client, + "inspect_wasm_artifact", + schemaValidation.assertions, + ); const summaryInput = { path: value.artifact.path, detail: "summary" }; expect(wasmAjv.validate(inputSchema, summaryInput)).toBe(true); const directSummary = await harness.client.callTool({ @@ -99,6 +106,7 @@ it.each(["bin", "hop"])( }); const session = createTestBinarySession(createClient); const harness = await createApplicationMcpHarness({ + schemaValidator: schemaValidation, session, serverOptions: { environment: {} }, }); @@ -107,6 +115,7 @@ it.each(["bin", "hop"])( const { name, ajv, inputSchema } = await advertisedMcpToolSchemas( client, "inspect_wasm_artifact", + schemaValidation.assertions, ); const input = { path }; const opening = await client.callTool({ @@ -192,6 +201,7 @@ it("records inline WASM evidence with compact inputs and canonical option checks throw new Error("deep provider must not start"); }); const harness = await createApplicationMcpHarness({ + schemaValidator: schemaValidation, session, serverOptions: { wasmArtifact: service }, }); @@ -200,6 +210,7 @@ it("records inline WASM evidence with compact inputs and canonical option checks const schemas = await advertisedMcpToolSchemas( client, "inspect_wasm_artifact", + schemaValidation.assertions, ); const { ajv, inputSchema } = schemas; expect(ajv.validate(inputSchema, { path: value.artifact.path })).toBe(true); @@ -282,6 +293,7 @@ it.each([undefined, "summary"] as const)( throw new Error("deep provider must not start"); }); const harness = await createApplicationMcpHarness({ + schemaValidator: schemaValidation, session, serverOptions: { wasmArtifact: service }, }); @@ -363,6 +375,7 @@ it("attributes rejected WASM paths to their selected fields before provider work Promise.resolve(ok(wasmArtifactExecution(value))), ); const harness = await createApplicationMcpHarness({ + schemaValidator: schemaValidation, serverOptions: { wasmArtifact: new WasmArtifactService({ close: () => Promise.resolve(), @@ -440,6 +453,7 @@ it("rejects a WASM provider response with a non-file subject before recording Ev subject: { ...execution.subject, format: "elf" }, } as unknown as typeof execution; const harness = await createApplicationMcpHarness({ + schemaValidator: schemaValidation, serverOptions: { wasmArtifact: new WasmArtifactService({ close: () => Promise.resolve(), @@ -484,6 +498,7 @@ it("reads an oversized completed WASM result through its reported retained refer inspect, }); const harness = await createApplicationMcpHarness({ + schemaValidator: schemaValidation, serverOptions: { wasmArtifact: service }, }); onTestFinished(harness.close); @@ -526,7 +541,7 @@ it("reads an oversized completed WASM result through its reported retained refer ); if (advertised?.outputSchema === undefined) throw new Error("Missing WASM view schemas"); - const ajv = new Ajv2020({ strict: false, validateFormats: false }); + const ajv = schemaValidation.assertions; const input: Record = advertised.inputSchema; const output: Record = advertised.outputSchema; for (const [view, expected] of [ @@ -590,6 +605,7 @@ it("reads an oversized completed WASM result through its reported retained refer it("rejects inconsistent decoded-WAT digests in portable Evidence without a provider run", async () => { const value = wasmArtifactFixture(); const harness = await createApplicationMcpHarness({ + schemaValidator: schemaValidation, serverOptions: { environment: {} }, }); onTestFinished(harness.close); @@ -632,6 +648,7 @@ it("rejects inconsistent decoded-WAT digests in portable Evidence without a prov it("checks WASM semantic identity while preserving rebound display paths", async () => { const value = wasmArtifactFixture(); const harness = await createApplicationMcpHarness({ + schemaValidator: schemaValidation, serverOptions: { environment: {} }, }); onTestFinished(harness.close); diff --git a/tests/boundary/mcp/webBundleViewsMcp.test.ts b/tests/boundary/mcp/webBundleViewsMcp.test.ts index b8d6b44be2..978ec282c2 100644 --- a/tests/boundary/mcp/webBundleViewsMcp.test.ts +++ b/tests/boundary/mcp/webBundleViewsMcp.test.ts @@ -1,8 +1,7 @@ -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { expect, it, onTestFinished } from "vitest"; -import { createServer } from "../../../src/server/createServer.js"; +import { afterAll, expect, it } from "vitest"; +import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; import { webBundlePageViewResultSchema, @@ -14,6 +13,9 @@ import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { createMcpSchemaValidation } from "../../fixtures/mcpSchemaValidation.js"; import { advertisedMcpToolSchemas } from "../../fixtures/advertisedMcpTool.js"; +const schemaValidation = createMcpSchemaValidation(); +afterAll(() => schemaValidation.clear()); + const sampleWebBundleAnalysis = (): WebBundleAnalysis => ({ capture: { target_url: "https://example.test/app", @@ -124,29 +126,17 @@ const makeParent = () => ); const connect = async () => { - const schemaValidation = createMcpSchemaValidation(); const session = createTestBinarySession(() => { throw new Error("selected views must not start a deep provider"); }); - const server = createServer( - { kind: "session", session }, - { + const { client } = await createApplicationMcpHarness({ + session, + serverOptions: { environment: { ...process.env, REA_MCP_INPUT_SCHEMA_PROFILE: "full" }, }, - ); - const client = new Client( - { name: "web-bundle-view-mcp", version: "1" }, - { jsonSchemaValidator: schemaValidation }, - ); - onTestFinished(async () => { - await client.close(); - await server.close(); - await session.close(); + clientIdentity: { name: "web-bundle-view-mcp", version: "1" }, + schemaValidator: schemaValidation, }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - await server.connect(serverTransport); - await client.connect(clientTransport); const schemas = await advertisedMcpToolSchemas( client, "inspect_analysis_view", diff --git a/tests/boundary/mcp/webModuleGrammarMcp.test.ts b/tests/boundary/mcp/webModuleGrammarMcp.test.ts index dbdde6cddd..c7a2c24181 100644 --- a/tests/boundary/mcp/webModuleGrammarMcp.test.ts +++ b/tests/boundary/mcp/webModuleGrammarMcp.test.ts @@ -1,7 +1,6 @@ -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { mkdir, readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { expect, it, onTestFinished } from "vitest"; +import { expect, it } from "vitest"; import { WebModuleTraceService } from "../../../src/application/WebModuleTraceService.js"; import { LocalWebModuleArtifacts } from "../../../src/browser/modules/WebModuleArtifacts.js"; @@ -10,7 +9,7 @@ import { webModuleTraceResultSchema, type WebModuleResolutionRequest, } from "../../../src/domain/webModuleTrace.js"; -import { createServer } from "../../../src/server/createServer.js"; +import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { @@ -59,20 +58,11 @@ it.each([ "Retained module tracing must not acquire a binary provider", ); }); - const server = createServer( - { kind: "session", session }, - { webModuleTrace: service }, - ); - const client = new Client({ name: "module-grammar", version: "1" }); - onTestFinished(async () => { - await client.close(); - await server.close(); - await session.close(); + const { client } = await createApplicationMcpHarness({ + session, + serverOptions: { webModuleTrace: service }, + clientIdentity: { name: "module-grammar", version: "1" }, }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - await server.connect(serverTransport); - await client.connect(clientTransport); const response = await client.callTool({ name: "trace_web_module_imports", arguments: { manifest_path: manifestPath, script_index: 0 }, diff --git a/tests/boundary/providers/GdbCommandQueue.test.ts b/tests/boundary/providers/GdbCommandQueue.test.ts index f424567df7..2ee4c02660 100644 --- a/tests/boundary/providers/GdbCommandQueue.test.ts +++ b/tests/boundary/providers/GdbCommandQueue.test.ts @@ -10,11 +10,19 @@ let directory: string | undefined; const managers: GdbSessionManager[] = []; afterEach(async () => { - await Promise.all( - managers - .splice(0) - .map((manager) => manager.closeAll().catch(() => undefined)), + const closing = managers.splice(0); + const outcomes = await Promise.allSettled( + closing.map((manager) => manager.closeAll()), ); + const failures: unknown[] = []; + outcomes.forEach((outcome, index) => { + if (outcome.status === "rejected") { + managers.push(closing[index]!); + failures.push(outcome.reason); + } + }); + if (failures.length > 0) + throw new AggregateError(failures, "GDB queue cleanup failed"); if (directory !== undefined) await rm(directory, { recursive: true, force: true }); directory = undefined; @@ -49,7 +57,7 @@ process.stdin.on("data", async (chunk) => { catch { await new Promise((resolve) => setTimeout(resolve, 10)); } } } - process.stdout.write(match[1] + "^done\\n"); + process.stdout.write(match[1] + (line.includes("-list-thread-groups") ? "^done,groups=[]\\n" : "^done\\n")); } }); `, diff --git a/tests/boundary/providers/ida/verifierObservations.test.ts b/tests/boundary/providers/ida/verifierObservations.test.ts index 56376174df..d9d0abe2ec 100644 --- a/tests/boundary/providers/ida/verifierObservations.test.ts +++ b/tests/boundary/providers/ida/verifierObservations.test.ts @@ -247,12 +247,50 @@ it.each([ ); cleanupConfirmed = true; const report = z - .object({ passed: z.boolean(), unverified: z.array(z.string()) }) + .object({ + passed: z.boolean(), + unverified: z.array(z.string()), + failure: z + .object({ phase: z.string(), message: z.string() }) + .optional(), + failed_cli: z.object({ stdout: z.string() }).optional(), + }) .parse(JSON.parse(await readFile(reportPath, "utf8"))); expect(report.passed, result.stdout.text + result.stderr.text).toBe( passed, ); expect(result.exitCode).toBe(passed ? 0 : 1); + if (empty.includes("list_funcs")) + expect(report.failure).toEqual({ + phase: "production MCP list_procedures", + message: "list_procedures omitted the resolved procedure.", + }); + else if (oracles && empty.includes("find_regex")) + expect(report.failure).toEqual({ + phase: "production MCP list_strings", + message: "list_strings omitted the expected literal.", + }); + else if (oracles && empty.includes("xrefs_to")) + expect(report.failure).toEqual({ + phase: "production MCP xrefs", + message: "xrefs omitted the expected source address.", + }); + else if (omitMore) { + expect(report.failure?.phase).toBe("production CLI function analysis"); + expect(JSON.parse(report.failed_cli!.stdout)).toMatchObject({ + code: "unreadable_output", + details: { + partial_observation: { + normalized_result: { + target_binding: { + reason: + "Native IDA callees omitted an explicit truncation marker.", + }, + }, + }, + }, + }); + } else expect(report.failure).toBeUndefined(); if (!oracles) expect(report.unverified).toEqual([ "String inventory/search correctness: no independent expected literal supplied.", diff --git a/tests/boundary/providers/kuna/kunaProvider.test.ts b/tests/boundary/providers/kuna/kunaProvider.test.ts index 89d719669c..4317cf7b7f 100644 --- a/tests/boundary/providers/kuna/kunaProvider.test.ts +++ b/tests/boundary/providers/kuna/kunaProvider.test.ts @@ -61,12 +61,12 @@ it("discovers configuration without probing and admits only verified targets", a diagnostics: { version_probed: false }, }); expect(await fakeKunaCalls(kuna)).toEqual([]); + const capabilities = configured.capabilities(); + expect(capabilities.length).toBeGreaterThan(0); expect( - configured - .capabilities() - .every( - ({ available, cachePolicy }) => available && cachePolicy === "live", - ), + capabilities.every( + ({ available, cachePolicy }) => available && cachePolicy === "live", + ), ).toBe(true); const executable = ( diff --git a/tests/boundary/providers/kuna/kunaSingleFunctionRuns.test.ts b/tests/boundary/providers/kuna/kunaSingleFunctionRuns.test.ts index 8c838f7d1a..e7c22bec3f 100644 --- a/tests/boundary/providers/kuna/kunaSingleFunctionRuns.test.ts +++ b/tests/boundary/providers/kuna/kunaSingleFunctionRuns.test.ts @@ -93,10 +93,13 @@ it("admits no more concurrent single-function runs than its slots", async () => (left, right) => left.at - right.at, ); const exits = await fakeKunaExits(kuna); - expect(first).toBeDefined(); - expect(second).toBeDefined(); + if (first === undefined || second === undefined) + throw new Error("Expected both single-function runs"); + const firstExit = exits.get(first.pid); + expect(firstExit).toBeDefined(); + if (firstExit === undefined) throw new Error("Missing first process exit"); // The second run starts only after the first one's process finished. - expect(second?.at).toBeGreaterThanOrEqual(exits.get(first?.pid ?? 0) ?? 0); + expect(second.at).toBeGreaterThanOrEqual(firstExit); } finally { expect(await client.close()).toEqual({ ok: true, value: null }); } diff --git a/tests/boundary/providers/native/artifactProvider.archiveInventory.test.ts b/tests/boundary/providers/native/artifactProvider.archiveInventory.test.ts index 59a192b9ca..9a37e91787 100644 --- a/tests/boundary/providers/native/artifactProvider.archiveInventory.test.ts +++ b/tests/boundary/providers/native/artifactProvider.archiveInventory.test.ts @@ -1,3 +1,4 @@ +import { createOwnedArtifactClient } from "../../../support/artifactClientScope.js"; import { projectCliAnalysisResult } from "../../../../src/cli/projectCliAnalysisResult.js"; import { mkdir, writeFile } from "node:fs/promises"; import { join } from "node:path"; @@ -9,7 +10,6 @@ import { describe, expect, it } from "vitest"; import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; import { createDirectAnalysis } from "../../../../src/composition/directAnalysis.js"; -import { ArtifactProvider } from "../../../../src/artifacts/ArtifactProvider.js"; import { artifactInventoryResultSchema } from "../../../../src/domain/artifactGraph.js"; import { artifactInspectionResultSchema } from "../../../../src/domain/artifactInspection.js"; import type { BinaryTarget } from "../../../../src/domain/binaryTargetTypes.js"; @@ -125,9 +125,9 @@ describe("artifact archive inventory", () => { join(`${unpackedPath}.unpacked`, "main.js"), "changed();\n", ); - const corrupted = await new ArtifactProvider(process.env) - .createClient(await target(unpackedPath)) - .execute("inventory_artifact", {}); + const corrupted = await createOwnedArtifactClient( + await target(unpackedPath), + ).execute("inventory_artifact", {}); expect(corrupted).toMatchObject({ ok: false, error: { @@ -139,9 +139,9 @@ describe("artifact archive inventory", () => { }, }, }); - const failedInspection = await new ArtifactProvider(process.env) - .createClient(await target(unpackedPath)) - .execute("inspect_artifact", {}); + const failedInspection = await createOwnedArtifactClient( + await target(unpackedPath), + ).execute("inspect_artifact", {}); expect(failedInspection).toMatchObject({ ok: false, error: { _tag: "ArtifactOperationError", reason: "integrity" }, @@ -149,9 +149,10 @@ describe("artifact archive inventory", () => { }); }); const inventory = async (targetValue: BinaryTarget) => { - const result = await new ArtifactProvider(process.env) - .createClient(targetValue) - .execute("inventory_artifact", {}); + const result = await createOwnedArtifactClient(targetValue).execute( + "inventory_artifact", + {}, + ); if (!result.ok) throw result.error; return artifactInventoryResultSchema.parse(result.value.result); }; diff --git a/tests/boundary/providers/native/artifactProvider.directoryErrors.test.ts b/tests/boundary/providers/native/artifactProvider.directoryErrors.test.ts index bcc8dfeb25..3adb8c1aa3 100644 --- a/tests/boundary/providers/native/artifactProvider.directoryErrors.test.ts +++ b/tests/boundary/providers/native/artifactProvider.directoryErrors.test.ts @@ -4,14 +4,16 @@ import { join } from "node:path"; import { describe, expect, it } from "vitest"; import { parseBinaryTarget } from "../../../../src/application/BinaryTargetResolver.js"; -import { ArtifactProvider } from "../../../../src/artifacts/ArtifactProvider.js"; import { artifactExtractionResultSchema, artifactInventoryResultSchema, } from "../../../../src/domain/artifactGraph.js"; import { thinMach } from "../../../../src/domain/binaryTarget.fixture.js"; import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; -import { withArtifactClient } from "../../../support/artifactClientScope.js"; +import { + createOwnedArtifactClient, + withArtifactClient, +} from "../../../support/artifactClientScope.js"; describe("artifact directory diagnostics", () => { it.skipIf(process.platform === "win32" || process.getuid?.() === 0)( @@ -101,9 +103,10 @@ describe("artifact directory diagnostics", () => { try { const target = await parseBinaryTarget(appPath); if (!target.ok) throw target.error; - const result = await new ArtifactProvider(process.env) - .createClient(target.value) - .execute("inventory_artifact", {}); + const result = await createOwnedArtifactClient(target.value).execute( + "inventory_artifact", + {}, + ); if (!result.ok) throw result.error; const inventory = artifactInventoryResultSchema.parse( result.value.result, diff --git a/tests/boundary/providers/native/artifactProvider.directoryInventory.test.ts b/tests/boundary/providers/native/artifactProvider.directoryInventory.test.ts index ae47f4e741..cdfbf40389 100644 --- a/tests/boundary/providers/native/artifactProvider.directoryInventory.test.ts +++ b/tests/boundary/providers/native/artifactProvider.directoryInventory.test.ts @@ -1,3 +1,4 @@ +import { createOwnedArtifactClient } from "../../../support/artifactClientScope.js"; import { mkdir, symlink, writeFile } from "node:fs/promises"; import { join } from "node:path"; @@ -10,7 +11,6 @@ import { CPU, machoImage, } from "../../../../src/artifacts/apple/MachoImage.fixture.js"; -import { ArtifactProvider } from "../../../../src/artifacts/ArtifactProvider.js"; import { artifactInventoryResultSchema } from "../../../../src/domain/artifactGraph.js"; import type { BinaryTarget } from "../../../../src/domain/binaryTargetTypes.js"; @@ -38,9 +38,7 @@ describe("artifact directory inventory", () => { await writeFile(join(app, "Contents", "Resources", "main.js"), "main();\n"); await writeFile(join(app, "Contents", "Resources", "main.js.MAP"), "{}\n"); await symlink("/etc/passwd", join(app, "Contents", "Resources", "outside")); - const client = new ArtifactProvider(process.env).createClient( - await target(app), - ); + const client = createOwnedArtifactClient(await target(app)); const input = {}; const first = await client.execute("inventory_artifact", input); const second = await client.execute("inventory_artifact", input); diff --git a/tests/boundary/providers/native/artifactProvider.extraction.test.ts b/tests/boundary/providers/native/artifactProvider.extraction.test.ts index de78ae6e22..49a3c4e45a 100644 --- a/tests/boundary/providers/native/artifactProvider.extraction.test.ts +++ b/tests/boundary/providers/native/artifactProvider.extraction.test.ts @@ -10,14 +10,16 @@ import { CPU, machoImage, } from "../../../../src/artifacts/apple/MachoImage.fixture.js"; -import { ArtifactProvider } from "../../../../src/artifacts/ArtifactProvider.js"; import { artifactExtractionExecutionSchema } from "../../../../src/contracts/artifactToolContracts.js"; import { artifactExtractionResultSchema, artifactInventoryResultSchema, } from "../../../../src/domain/artifactGraph.js"; import type { BinaryTarget } from "../../../../src/domain/binaryTargetTypes.js"; -import { withArtifactClient } from "../../../support/artifactClientScope.js"; +import { + createOwnedArtifactClient, + withArtifactClient, +} from "../../../support/artifactClientScope.js"; describe("artifact extraction", () => { it.skipIf(process.platform === "win32").each(["file/..", "missing"])( @@ -61,15 +63,13 @@ describe("artifact extraction", () => { await mkdir(source); await writeFile(join(source, "payload.txt"), "committed\n"); const output = join(root, "output"); - const client = new ArtifactProvider(process.env).createClient( - await target(source), - ); + const client = createOwnedArtifactClient(await target(source)); const result = await client.execute("extract_artifact", { output_root: output, }); if (!result.ok) throw result.error; - await client.close(); + expect((await client.close()).ok).toBe(true); expect(await readFile(join(output, "payload.txt"), "utf8")).toBe( "committed\n", ); @@ -91,29 +91,25 @@ describe("artifact extraction", () => { const cancelledOutput = join(root, "cancelled-output"); const controller = new AbortController(); controller.abort(); - const cancelled = await new ArtifactProvider(process.env) - .createClient(targetValue) - .execute( - "extract_artifact", - artifactExtractionExecutionSchema.parse({ - output_root: cancelledOutput, - }), - { signal: controller.signal }, - ); + const cancelled = await createOwnedArtifactClient(targetValue).execute( + "extract_artifact", + artifactExtractionExecutionSchema.parse({ + output_root: cancelledOutput, + }), + { signal: controller.signal }, + ); expect(cancelled).toMatchObject({ ok: false, error: { _tag: "ArtifactOperationError", reason: "cancelled" }, }); await expect(access(cancelledOutput)).rejects.toThrow(); const output = join(root, "output"); - const result = await new ArtifactProvider(process.env) - .createClient(targetValue) - .execute( - "extract_artifact", - artifactExtractionExecutionSchema.parse({ - output_root: output, - }), - ); + const result = await createOwnedArtifactClient(targetValue).execute( + "extract_artifact", + artifactExtractionExecutionSchema.parse({ + output_root: output, + }), + ); if (!result.ok) throw result.error; const firstExtraction = artifactExtractionResultSchema.parse( result.value.result, @@ -136,28 +132,24 @@ describe("artifact extraction", () => { ); const relocatedOutput = join(root, "relocated-output"); - const relocated = await new ArtifactProvider(process.env) - .createClient(targetValue) - .execute( - "extract_artifact", - artifactExtractionExecutionSchema.parse({ - output_root: relocatedOutput, - }), - ); + const relocated = await createOwnedArtifactClient(targetValue).execute( + "extract_artifact", + artifactExtractionExecutionSchema.parse({ + output_root: relocatedOutput, + }), + ); if (!relocated.ok) throw relocated.error; expect( artifactExtractionResultSchema.parse(relocated.value.result) .extraction_manifest, ).toEqual(firstExtraction.extraction_manifest); - const second = await new ArtifactProvider(process.env) - .createClient(targetValue) - .execute( - "extract_artifact", - artifactExtractionExecutionSchema.parse({ - output_root: output, - }), - ); + const second = await createOwnedArtifactClient(targetValue).execute( + "extract_artifact", + artifactExtractionExecutionSchema.parse({ + output_root: output, + }), + ); expect(second).toMatchObject({ ok: false, error: { _tag: "ArtifactOperationError", reason: "path" }, @@ -170,9 +162,10 @@ describe("artifact extraction", () => { }); }); const inventory = async (targetValue: BinaryTarget) => { - const result = await new ArtifactProvider(process.env) - .createClient(targetValue) - .execute("inventory_artifact", {}); + const result = await createOwnedArtifactClient(targetValue).execute( + "inventory_artifact", + {}, + ); if (!result.ok) throw result.error; return artifactInventoryResultSchema.parse(result.value.result); }; diff --git a/tests/boundary/providers/native/artifactProvider.extractionCompleteness.test.ts b/tests/boundary/providers/native/artifactProvider.extractionCompleteness.test.ts index 9b1568e6bf..bdeebb9144 100644 --- a/tests/boundary/providers/native/artifactProvider.extractionCompleteness.test.ts +++ b/tests/boundary/providers/native/artifactProvider.extractionCompleteness.test.ts @@ -1,3 +1,4 @@ +import { createOwnedArtifactClient } from "../../../support/artifactClientScope.js"; import { createHash } from "node:crypto"; import { access, @@ -13,7 +14,6 @@ import { createPackage, createPackageWithOptions } from "@electron/asar"; import { expect, it, onTestFinished, vi } from "vitest"; import { parseBinaryTarget } from "../../../../src/application/BinaryTargetResolver.js"; -import { ArtifactProvider } from "../../../../src/artifacts/ArtifactProvider.js"; import { materializeArtifactInventory, scanArtifactInventory, @@ -429,7 +429,7 @@ const extractDirectory = async (source: string, output: string) => { const extractArchive = async (source: string, output: string) => { const target = await parseBinaryTarget(source); if (!target.ok) throw target.error; - return new ArtifactProvider(process.env) - .createClient(target.value) - .execute("extract_artifact", { output_root: output }); + return createOwnedArtifactClient(target.value).execute("extract_artifact", { + output_root: output, + }); }; diff --git a/tests/boundary/providers/native/artifactProvider.extractionFormat.test.ts b/tests/boundary/providers/native/artifactProvider.extractionFormat.test.ts index e52ac5ed59..352fb102ff 100644 --- a/tests/boundary/providers/native/artifactProvider.extractionFormat.test.ts +++ b/tests/boundary/providers/native/artifactProvider.extractionFormat.test.ts @@ -7,10 +7,12 @@ import { elfExportImage } from "../../../fixtures/binaryDiagnostics/nativeExport import { writeOrderedZip } from "../../../fixtures/artifactEntryOrder.js"; import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; import { buildGodotPck, pckU64 } from "../../../fixtures/godotPck.js"; -import { withArtifactClient } from "../../../support/artifactClientScope.js"; +import { + createOwnedArtifactClient, + withArtifactClient, +} from "../../../support/artifactClientScope.js"; import { parseBinaryTarget } from "../../../../src/application/BinaryTargetResolver.js"; -import { ArtifactProvider } from "../../../../src/artifacts/ArtifactProvider.js"; import { CPU, machoImage, @@ -33,13 +35,11 @@ describe("artifact extraction format support", () => { const selected = await admittedTarget(alias); expect(selected.format).toBe("dmg"); const extract = (signal?: AbortSignal) => - new ArtifactProvider(process.env) - .createClient(selected) - .execute( - "extract_artifact", - artifactExtractionExecutionSchema.parse({ output_root: output }), - signal === undefined ? undefined : { signal }, - ); + createOwnedArtifactClient(selected).execute( + "extract_artifact", + artifactExtractionExecutionSchema.parse({ output_root: output }), + signal === undefined ? undefined : { signal }, + ); expect(await extract()).toMatchObject({ ok: false, error: { @@ -64,12 +64,10 @@ describe("artifact extraction format support", () => { await writeFile(binary, machoImage({ cpu: CPU.arm64e })); const output = join(root, "output"); expect( - await new ArtifactProvider(process.env) - .createClient(await admittedTarget(binary)) - .execute( - "extract_artifact", - artifactExtractionExecutionSchema.parse({ output_root: output }), - ), + await createOwnedArtifactClient(await admittedTarget(binary)).execute( + "extract_artifact", + artifactExtractionExecutionSchema.parse({ output_root: output }), + ), ).toMatchObject({ ok: false, error: { @@ -104,14 +102,14 @@ describe("artifact extraction format support", () => { }), ]), ); - const result = await new ArtifactProvider(process.env) - .createClient(await admittedTarget(binary)) - .execute( - "extract_artifact", - artifactExtractionExecutionSchema.parse({ - output_root: join(root, "output"), - }), - ); + const result = await createOwnedArtifactClient( + await admittedTarget(binary), + ).execute( + "extract_artifact", + artifactExtractionExecutionSchema.parse({ + output_root: join(root, "output"), + }), + ); if (!result.ok) throw result.error; const extracted = artifactExtractionResultSchema.parse(result.value.result); expect( @@ -170,14 +168,14 @@ describe("artifact extraction reader selection", () => { `Payload.${format === "app-pack" ? "app" : format}`, ); await writeOrderedZip(archive, ["payload.txt"]); - const result = await new ArtifactProvider(process.env) - .createClient(await admittedTarget(archive)) - .execute( - "extract_artifact", - artifactExtractionExecutionSchema.parse({ - output_root: join(root, "output"), - }), - ); + const result = await createOwnedArtifactClient( + await admittedTarget(archive), + ).execute( + "extract_artifact", + artifactExtractionExecutionSchema.parse({ + output_root: join(root, "output"), + }), + ); if (!result.ok) throw result.error; const extracted = artifactExtractionResultSchema.parse( result.value.result, diff --git a/tests/boundary/providers/native/artifactProvider.extractionIdentity.test.ts b/tests/boundary/providers/native/artifactProvider.extractionIdentity.test.ts index 99e45631f1..3360637984 100644 --- a/tests/boundary/providers/native/artifactProvider.extractionIdentity.test.ts +++ b/tests/boundary/providers/native/artifactProvider.extractionIdentity.test.ts @@ -1,3 +1,4 @@ +import { createOwnedArtifactClient } from "../../../support/artifactClientScope.js"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; @@ -6,7 +7,6 @@ import { writeOrderedZip } from "../../../fixtures/artifactEntryOrder.js"; import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; import { parseBinaryTarget } from "../../../../src/application/BinaryTargetResolver.js"; -import { ArtifactProvider } from "../../../../src/artifacts/ArtifactProvider.js"; import { artifactExtractionExecutionSchema } from "../../../../src/contracts/artifactToolContracts.js"; import { artifactExtractionResultSchema } from "../../../../src/domain/artifactGraph.js"; import type { BinaryTarget } from "../../../../src/domain/binaryTargetTypes.js"; @@ -16,14 +16,14 @@ describe("artifact extraction identity", () => { const root = await createTestTempDirectory("rea-extract-order-"); const source = join(root, "source.zip"); await writeOrderedZip(source, ["รค.js", "a.js", "Z.js"]); - const result = await new ArtifactProvider(process.env) - .createClient(await archiveTarget(source)) - .execute( - "extract_artifact", - artifactExtractionExecutionSchema.parse({ - output_root: join(root, "output"), - }), - ); + const result = await createOwnedArtifactClient( + await archiveTarget(source), + ).execute( + "extract_artifact", + artifactExtractionExecutionSchema.parse({ + output_root: join(root, "output"), + }), + ); if (!result.ok) throw result.error; const extracted = artifactExtractionResultSchema.parse(result.value.result); expect( diff --git a/tests/boundary/providers/native/artifactProvider.failureDetail.test.ts b/tests/boundary/providers/native/artifactProvider.failureDetail.test.ts index d1600d6469..94476e4753 100644 --- a/tests/boundary/providers/native/artifactProvider.failureDetail.test.ts +++ b/tests/boundary/providers/native/artifactProvider.failureDetail.test.ts @@ -1,3 +1,4 @@ +import { createOwnedArtifactClient } from "../../../support/artifactClientScope.js"; import { join } from "node:path"; import { expect, it } from "vitest"; @@ -10,7 +11,6 @@ import { destinationCaseCollision, ENGLISH_UNICODE_CASE_INVENTORY_LIMITATION, } from "../../../../src/artifacts/ArtifactPaths.js"; -import { ArtifactProvider } from "../../../../src/artifacts/ArtifactProvider.js"; import { artifactInventoryResultSchema } from "../../../../src/domain/artifactGraph.js"; import { ArtifactOperationError } from "../../../../src/domain/artifactOperationError.js"; import { projectAnalysisError } from "../../../../src/domain/analysisErrorProjection.js"; @@ -21,9 +21,10 @@ it("keeps logical names that differ only in case and records the limitation", as await writeOrderedZip(path, ["Main.js", "main.js"]); const target = await parseBinaryTarget(path); if (!target.ok) throw new Error("expected a ZIP target"); - const result = await new ArtifactProvider(process.env) - .createClient(target.value) - .execute("inventory_artifact", {}); + const result = await createOwnedArtifactClient(target.value).execute( + "inventory_artifact", + {}, + ); if (!result.ok) throw result.error; const inventory = artifactInventoryResultSchema.parse(result.value.result); expect(inventory.occurrences.map(({ logical_path }) => logical_path)).toEqual( @@ -47,9 +48,10 @@ it("keeps literal backslash ZIP names distinct from slash paths", async () => { await writeOrderedZip(path, ["a\\b.js", "a/b.js"]); const target = await parseBinaryTarget(path); if (!target.ok) throw new Error("expected a ZIP target"); - const result = await new ArtifactProvider(process.env) - .createClient(target.value) - .execute("inventory_artifact", {}); + const result = await createOwnedArtifactClient(target.value).execute( + "inventory_artifact", + {}, + ); if (!result.ok) throw result.error; const inventory = artifactInventoryResultSchema.parse(result.value.result); const literal = inventory.occurrences.find( @@ -69,9 +71,10 @@ it("reports an unsafe path as a hostile artifact path", async () => { await writeOrderedZip(path, ["../outside.js"]); const target = await parseBinaryTarget(path); if (!target.ok) throw new Error("expected a ZIP target"); - const result = await new ArtifactProvider(process.env) - .createClient(target.value) - .execute("inventory_artifact", {}); + const result = await createOwnedArtifactClient(target.value).execute( + "inventory_artifact", + {}, + ); if (result.ok) throw new Error("expected the path constraint to fail"); expect(projectAnalysisError(result.error)).toMatchObject({ code: "artifact_operation_failed", diff --git a/tests/boundary/providers/native/artifactProvider.universalExtraction.test.ts b/tests/boundary/providers/native/artifactProvider.universalExtraction.test.ts index c61e0a6bb3..53013e8b42 100644 --- a/tests/boundary/providers/native/artifactProvider.universalExtraction.test.ts +++ b/tests/boundary/providers/native/artifactProvider.universalExtraction.test.ts @@ -1,3 +1,4 @@ +import { createOwnedArtifactClient } from "../../../support/artifactClientScope.js"; import { access, writeFile } from "node:fs/promises"; import { join } from "node:path"; @@ -8,7 +9,6 @@ import { CPU_ARM64, universalMacho } from "../../../fixtures/universalMacho.js"; import { thinMach } from "../../../../src/domain/binaryTarget.fixture.js"; import { parseBinaryTarget } from "../../../../src/application/BinaryTargetResolver.js"; -import { ArtifactProvider } from "../../../../src/artifacts/ArtifactProvider.js"; import { ArtifactReaderFailure } from "../../../../src/artifacts/ArtifactReader.js"; import { artifactExtractionExecutionSchema } from "../../../../src/contracts/artifactToolContracts.js"; @@ -56,12 +56,10 @@ it("routes universal Mach-O extraction through slice inventory", async () => { const target = await parseBinaryTarget(image, { hostArchitecture: "arm64" }); if (!target.ok) throw target.error; const output = join(root, "output"); - const result = await new ArtifactProvider(process.env) - .createClient(target.value) - .execute( - "extract_artifact", - artifactExtractionExecutionSchema.parse({ output_root: output }), - ); + const result = await createOwnedArtifactClient(target.value).execute( + "extract_artifact", + artifactExtractionExecutionSchema.parse({ output_root: output }), + ); expect(result).toMatchObject({ ok: false, diff --git a/tests/boundary/providers/native/nativeSelectedArchitecture.test.ts b/tests/boundary/providers/native/nativeSelectedArchitecture.test.ts index 107623d5b9..3c52b8ee22 100644 --- a/tests/boundary/providers/native/nativeSelectedArchitecture.test.ts +++ b/tests/boundary/providers/native/nativeSelectedArchitecture.test.ts @@ -13,6 +13,7 @@ import { type NativeCommandRunner, } from "../../../../src/native/CommandRunner.js"; import { NativeMacOSProvider } from "../../../../src/native/NativeMacOSProvider.js"; +import type { AnalysisClient } from "../../../../src/application/AnalysisProvider.js"; import { inspectionImage } from "../../../../src/native/NativeMachoInspection.fixture.js"; import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; import { @@ -31,6 +32,14 @@ const cpus = { } as const; type SliceName = keyof typeof cpus; +const inspect = async (client: AnalysisClient) => { + try { + return await client.execute("inspect_macho", {}); + } finally { + expect((await client.close()).ok).toBe(true); + } +}; + const sliceImage = ( name: SliceName, uuid = "01234567-89AB-CDEF-0123-456789ABCDEF", @@ -109,9 +118,9 @@ it("does not attribute another universal slice's UUID or segments to the selecte lipo: fixture.lipo, dwarfdump: `UUID: AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA (x86_64) ${fixture.path}\nUUID: 01234567-89AB-CDEF-0123-456789ABCDEF (arm64) ${fixture.path}\n`, }); - const result = await new NativeMacOSProvider({}, runner, "darwin") - .createClient(fixture.target) - .execute("inspect_macho", {}); + const result = await inspect( + new NativeMacOSProvider({}, runner, "darwin").createClient(fixture.target), + ); if (!result.ok) throw result.error; expect(result.value.result).toMatchObject({ uuid: "01234567-89AB-CDEF-0123-456789ABCDEF", @@ -172,9 +181,11 @@ it.each([ dyld, calls, ); - const result = await new NativeMacOSProvider({}, runner, "darwin") - .createClient(fixture.target) - .execute("inspect_macho", {}); + const result = await inspect( + new NativeMacOSProvider({}, runner, "darwin").createClient( + fixture.target, + ), + ); if (!result.ok) throw result.error; expect(result.value.result).toMatchObject({ uuid: "01234567-89AB-CDEF-0123-456789ABCDEF", @@ -185,11 +196,11 @@ it.each([ }, architectures: { items: architectures.map((name) => ({ name })) }, }); - expect( - calls - .filter(({ tool }) => tool === "dyld_info") - .every(({ arguments_ }) => arguments_.includes(dyld)), - ).toBe(true); + const dyldCalls = calls.filter(({ tool }) => tool === "dyld_info"); + expect(dyldCalls.length).toBeGreaterThan(0); + expect(dyldCalls.every(({ arguments_ }) => arguments_.includes(dyld))).toBe( + true, + ); expect( calls.find(({ tool }) => tool === "dwarfdump")?.arguments_, ).toContain(`--arch=${dwarf}`); @@ -222,9 +233,11 @@ it.each([ "arm64e", "arm64e.v1", ); - const result = await new NativeMacOSProvider({}, runner, "darwin") - .createClient(fixture.target) - .execute("inspect_macho", {}); + const result = await inspect( + new NativeMacOSProvider({}, runner, "darwin").createClient( + fixture.target, + ), + ); if (!result.ok) throw result.error; expect(result.value.result).toMatchObject({ uuid: expected, @@ -241,13 +254,13 @@ it.each(["ordinary", "\nNon-fat file: forged", "\r\nNon-fat file: forged"])( "projects selected segment evidence with %s path framing", async (name) => { const fixture = await makeFat(["x86_64", "arm64"], name); - const result = await new NativeMacOSProvider( - {}, - new NativeFixtureRunner({ lipo: fixture.lipo }), - "darwin", - ) - .createClient(fixture.target) - .execute("inspect_macho", {}); + const result = await inspect( + new NativeMacOSProvider( + {}, + new NativeFixtureRunner({ lipo: fixture.lipo }), + "darwin", + ).createClient(fixture.target), + ); if (!result.ok) throw result.error; expect(result.value.locations).toContainEqual({ kind: "file-offset-range", @@ -274,13 +287,13 @@ it.each(["thin", "one-slice FAT"])( lipo = `Non-fat file: ${fixture.path} is architecture: arm64\n`; } const target = await nativeMachoTargetForFile(fixture.path); - const result = await new NativeMacOSProvider( - {}, - new NativeFixtureRunner({ lipo }), - "darwin", - ) - .createClient(target) - .execute("inspect_macho", {}); + const result = await inspect( + new NativeMacOSProvider( + {}, + new NativeFixtureRunner({ lipo }), + "darwin", + ).createClient(target), + ); if (!result.ok) throw result.error; const offset = format === "thin" ? 0 : 4096; expect(result.value.locations).toContainEqual({ @@ -297,13 +310,13 @@ it.each(["thin", "one-slice FAT"])( it("uses the uniquely matching FAT entry when lipo omits the selected slice offset", async () => { const fixture = await makeFat(["x86_64", "arm64"]); const lipo = fixture.lipo.replace("offset 8192\n", ""); - const result = await new NativeMacOSProvider( - {}, - new NativeFixtureRunner({ lipo }), - "darwin", - ) - .createClient(fixture.target) - .execute("inspect_macho", {}); + const result = await inspect( + new NativeMacOSProvider( + {}, + new NativeFixtureRunner({ lipo }), + "darwin", + ).createClient(fixture.target), + ); if (!result.ok) throw result.error; expect(result.value.result).toMatchObject({ selected_architecture: { container_file_offset: 8192 }, @@ -317,13 +330,13 @@ it("reports the selected 32-bit slice's word size and byte order", async () => { "i386", "i386", ); - const result = await new NativeMacOSProvider({}, runner, "darwin") - .createClient({ + const result = await inspect( + new NativeMacOSProvider({}, runner, "darwin").createClient({ ...fixture.target, architecture: "x86", availableArchitectures: ["x86_64", "x86"], - }) - .execute("inspect_macho", {}); + }), + ); if (!result.ok) throw result.error; expect(result.value.result).toMatchObject({ selected_architecture: { name: "i386", container_file_offset: 8192 }, diff --git a/tests/conformance/ida/providerContracts.test.ts b/tests/conformance/ida/providerContracts.test.ts index 10b1475eb2..465dba37e6 100644 --- a/tests/conformance/ida/providerContracts.test.ts +++ b/tests/conformance/ida/providerContracts.test.ts @@ -67,9 +67,18 @@ describe.each(["attached", "headless"] as const)( producer, ); try { - await client.execute(operation, parametersFor(operation)); + const execution = await client.execute( + operation, + parametersFor(operation), + ); + if (mode === "headless" && operation === "procedure_callers") + expect(execution).toMatchObject({ + ok: false, + error: { _tag: "AnalysisCapabilityUnavailableError" }, + }); + else if (!execution.ok) throw execution.error; } finally { - await client.close(); + expect((await client.close()).ok).toBe(true); } const descriptor = idaCapabilities( IDA_COMPATIBILITY_PROFILES[mode === "attached" ? "legacy" : "modern"], @@ -133,7 +142,7 @@ describe.each(["attached", "headless"] as const)( text: JSON.stringify(serialized.structuredContent), }); } finally { - await client.close(); + expect((await client.close()).ok).toBe(true); } }, ); diff --git a/tests/conformance/providers/hopper/realHopperExhaustiveSearch.test.ts b/tests/conformance/providers/hopper/realHopperExhaustiveSearch.test.ts index 70fc87b6ca..f02080a13e 100644 --- a/tests/conformance/providers/hopper/realHopperExhaustiveSearch.test.ts +++ b/tests/conformance/providers/hopper/realHopperExhaustiveSearch.test.ts @@ -18,12 +18,36 @@ describe("Hopper search verifier rejection", () => { symbolPrefix: "_rea_fixture_", stringPrefix: "REA_FIXTURE_", }), - ).rejects.toThrow(); + ).rejects.toThrow( + fault === "duplicate" + ? "search_procedures returned duplicate matches" + : "search_procedures did not return every expected match", + ); }, ); }); -const fixtureClient = (count: number, fault: "duplicate" | "missing") => { +it("accepts the complete fixture before testing rejection", async () => { + await expect( + openAndVerifyLargeFixture({ + client: fixtureClient(3, "none"), + options: {}, + normalizedResult: normalize, + path: "/fixture", + expectedCount: 3, + symbolPrefix: "_rea_fixture_", + stringPrefix: "REA_FIXTURE_", + }), + ).resolves.toEqual({ + procedures: { count: 3, calls: 1 }, + strings: { count: 3, calls: 1 }, + }); +}); + +const fixtureClient = ( + count: number, + fault: "duplicate" | "missing" | "none", +) => { return { callTool: async (request: unknown) => { if ( diff --git a/tests/conformance/readiness/report/reconstructionReadiness.test.ts b/tests/conformance/readiness/report/reconstructionReadiness.test.ts index d8746c1b6b..1604d9d771 100644 --- a/tests/conformance/readiness/report/reconstructionReadiness.test.ts +++ b/tests/conformance/readiness/report/reconstructionReadiness.test.ts @@ -256,28 +256,6 @@ describe("readiness closure history", () => { }, ); - it("does not count one obligation twice in a closure observation", () => { - const value = input(); - value.closure_history[0]!.required_open = 2; - value.closure_history[1]!.newly_verified_obligation_ids = [ - "obl.readiness", - "obl.readiness", - ]; - - const report = createReconstructionReadinessReport(value); - expect(report.status).toBe("fail"); - expect(report.summary.passed_required_stages).toBe(8); - expect(report.findings).toContainEqual( - expect.objectContaining({ - code: "invalid-obligation-closure-transition", - stage_id: "verify-reconstruction-closure", - status: "fail", - evidence_ids: value.closure_history[1]!.evidence_ids, - }), - ); - expect(report.snapshot.closure_history).toEqual(value.closure_history); - }); - it.each([false, true])( "checks distinct obligations across closure observations (repeated: %s)", (repeated) => { @@ -742,12 +720,16 @@ describe("readiness retained observation accounting", () => { ]; const report = createReconstructionReadinessReport(value); expect(report.status).toBe("fail"); + expect(report.summary.passed_required_stages).toBe(8); expect(report.findings).toContainEqual( expect.objectContaining({ code: "invalid-obligation-closure-transition", stage_id: "verify-reconstruction-closure", + status: "fail", + evidence_ids: value.closure_history[1]!.evidence_ids, }), ); + expect(report.snapshot.closure_history).toEqual(value.closure_history); expect( report.snapshot.closure_history[1]!.newly_verified_obligation_ids, ).toEqual(["obl.readiness", "obl.readiness"]); diff --git a/tests/fixtures/applicationMcpHarness.ts b/tests/fixtures/applicationMcpHarness.ts index 368a237d27..1a2c2ff494 100644 --- a/tests/fixtures/applicationMcpHarness.ts +++ b/tests/fixtures/applicationMcpHarness.ts @@ -11,6 +11,7 @@ import { createServer } from "../../src/server/createServer.js"; import { createTestMcp, type McpClientIdentity, + type TestMcp, } from "../support/mcp/mcpFixture.js"; /** Owned MCP SDK session shared by application workflow scenarios. */ @@ -26,6 +27,7 @@ export async function createApplicationMcpHarness( readonly session?: BinarySession; readonly serverOptions?: Parameters[1]; readonly clientIdentity?: McpClientIdentity; + readonly schemaValidator?: Parameters[2]; readonly registerCleanup?: (close: ApplicationMcpHarness["close"]) => void; } = {}, ): Promise { @@ -75,6 +77,7 @@ export async function createApplicationMcpHarness( name: "application-workflow-test", version: "1", }, + options.schemaValidator, ); return { client, session, close }; } catch (cause) { diff --git a/tests/support/artifactClientScope.ts b/tests/support/artifactClientScope.ts index ed5333ac58..9277a8f249 100644 --- a/tests/support/artifactClientScope.ts +++ b/tests/support/artifactClientScope.ts @@ -3,15 +3,23 @@ import type { AnalysisClient } from "../../src/application/AnalysisProvider.js"; import { ArtifactProvider } from "../../src/artifacts/ArtifactProvider.js"; import type { BinaryTarget } from "../../src/domain/binaryTargetTypes.js"; -export const withArtifactClient = async ( +/** Retain test ownership until every artifact client is closed. */ +export const createOwnedArtifactClient = ( target: BinaryTarget, - action: (client: AnalysisClient) => Promise, -): Promise => { +): AnalysisClient => { const client = new ArtifactProvider(process.env).createClient(target); onTestFinished(async () => { const closed = await client.close(); if (!closed.ok) throw closed.error; }); + return client; +}; + +export const withArtifactClient = async ( + target: BinaryTarget, + action: (client: AnalysisClient) => Promise, +): Promise => { + const client = createOwnedArtifactClient(target); let outcome: | { readonly ok: true; readonly value: T } | { readonly ok: false; readonly error: unknown };