From 2474e6ec6c9793684101b90d38beded4e8178bc9 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Thu, 19 Nov 2026 23:35:35 -0500 Subject: [PATCH] refactor(verify): run Hopper and Kuna verifier commands through the supervised owner (#12718) * refactor(verify): run Hopper and Kuna verifier commands through the supervised owner Replace bare execFile in the Hopper and Kuna real-provider verifiers with createVerifierCommands, and run pre-workspace toolchain probes through runOwnedCommand. Close the owner before lineage verification and keep the diagnostic workspace when cleanup, lineage, or the run fails. Replace isError-only and FAT64 prose refusals with the projected error code and structured issue or target details. * fix(verify): release a clean Kuna workspace after a verification failure A failed check whose provider, command, and lineage cleanup all completed still removes the root. The workspace stays only when that cleanup is unconfirmed, which is the contract the Kuna verifier failure-path tests already require. Co-authored-by: morluto * fix(verify): keep the Kuna report when workspace release fails release() closes cleanup registration before it removes the root, so retain() after a failed release throws and skips the JSON report. Register retention only when release has not started; a failed release already keeps the workspace. Co-authored-by: morluto * fix(verify): keep both Hopper verification and lineage failures When verification has already failed and lineage observation then fails, the retained-workspace error carried only the earlier cause. Aggregate both failures in the Hopper, FAT, and caller verifiers so the cleanup reason stays with the workspace. Co-authored-by: morluto * fix(verify): keep Hopper manifests inside the repository realpath() accepted REA_HOPPER_CONFORMANCE_MANIFEST_PATH as any filesystem path. Resolve the manifest against the repository and reject a lexical or symlink escape before it is read. Co-authored-by: morluto * fix(verify): acquire command owners before verifier workspaces * fix(verify): create the command owner before the verifier workspace If command-owner setup fails after a workspace is allocated, the temporary root is never released. Create the owner first and close it when workspace allocation fails. Co-authored-by: morluto * fix(verify): supervise Hopper helper CLI processes * fix(verify): run Hopper helper CLIs through the command owner Boundary and Linux owner-exit checks still launched the REA CLI with execFile, so a timeout could not stop those processes. Both helpers now require the verifier command owner and run those CLIs through it. Co-authored-by: morluto --------- Co-authored-by: Cursor Agent Co-authored-by: morluto --- scripts/lib/real-hopper-boundaries.mjs | 38 +- scripts/lib/real-hopper-owner-exit.mjs | 17 +- scripts/verify-real-hopper-callers.mjs | 184 ++- scripts/verify-real-hopper-fat.mjs | 249 +++- scripts/verify-real-hopper-linux.mjs | 22 +- scripts/verify-real-hopper.mjs | 1115 +++++++++-------- scripts/verify-real-kuna.mjs | 52 +- scripts/verify/apple/asset-catalog.mjs | 11 +- scripts/verify/apple/keyed-archive.mjs | 11 +- scripts/verify/apple/macos-dylib-e2e.mjs | 11 +- .../hopper/realHopperFixture.test.ts | 18 +- 11 files changed, 1068 insertions(+), 660 deletions(-) diff --git a/scripts/lib/real-hopper-boundaries.mjs b/scripts/lib/real-hopper-boundaries.mjs index 47c22e499f..55c3f6c964 100644 --- a/scripts/lib/real-hopper-boundaries.mjs +++ b/scripts/lib/real-hopper-boundaries.mjs @@ -1,11 +1,9 @@ import { canonicalOutputSchema } from "./canonical-output-schema.mjs"; import assert from "node:assert/strict"; -import { execFile } from "node:child_process"; import { copyFile, mkdtemp, readFile, realpath, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { basename, join } from "node:path"; import { fileURLToPath } from "node:url"; -import { promisify } from "node:util"; import Ajv from "ajv/dist/2020.js"; import addFormats from "ajv-formats"; import { @@ -403,7 +401,16 @@ export async function verifyHopperBoundaryContracts( } /** Prove actual document disposal and shared CLI/MCP behavior on disposable copies. */ -export async function verifyHopperLifecycleAndCli(client, options, targets) { +export async function verifyHopperLifecycleAndCli( + client, + options, + targets, + commands, +) { + if (commands === undefined) + throw new Error( + "Hopper boundary verification requires the supervised command owner", + ); const directory = await realpath( await mkdtemp(join(tmpdir(), "hopper-boundary-")), ); @@ -415,15 +422,22 @@ export async function verifyHopperLifecycleAndCli(client, options, targets) { await client.callTool({ name, arguments: args }, options), name, ); - const cli = promisify(execFile); - const runCli = (args) => { - const pending = cli(process.execPath, args, { - timeout: 180_000, - maxBuffer: 1_048_576, - }); - if (pending.child.pid !== undefined) - targets.ownedProcessIds.add(pending.child.pid); - return pending; + // Non-zero CLI exits keep the execFile-shaped code and stdout the refusal + // assertion reads. The process itself belongs to the command owner. + const runCli = async (args) => { + try { + return await commands.exec(process.execPath, args, { + timeout: 180_000, + maxBuffer: 1_048_576, + }); + } catch (error) { + if (error?.name !== "VerifierCommandExitError") throw error; + const failed = new Error(error.message, { cause: error }); + failed.code = error.code; + failed.stdout = error.snapshot.stdout.text; + failed.stderr = error.snapshot.stderr.text; + throw failed; + } }; try { await copyFile(targets.primary, primary); diff --git a/scripts/lib/real-hopper-owner-exit.mjs b/scripts/lib/real-hopper-owner-exit.mjs index 9cdf0d3cf8..6f6784e7be 100644 --- a/scripts/lib/real-hopper-owner-exit.mjs +++ b/scripts/lib/real-hopper-owner-exit.mjs @@ -1,7 +1,5 @@ import assert from "node:assert/strict"; -import { execFile } from "node:child_process"; import { setTimeout as delay } from "node:timers/promises"; -import { promisify } from "node:util"; import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; @@ -19,8 +17,6 @@ import { requireMcpLifecycleResult, } from "./mcp-verifier-results.mjs"; -const execFileAsync = promisify(execFile); - /** Prove actual MCP owner exit releases the demo singleton for subsequent CLI/MCP sessions. */ export async function verifyLinuxHopperOwnerExit({ primary, @@ -28,7 +24,12 @@ export async function verifyLinuxHopperOwnerExit({ serverCommand, serverArgs, environment, + commands, }) { + if (commands === undefined) + throw new Error( + "Linux Hopper owner-exit verification requires the supervised command owner", + ); const sentinel = await startUnrelatedHopperSentinel(); const outcomes = []; try { @@ -132,7 +133,7 @@ export async function verifyLinuxHopperOwnerExit({ assert.equal(cleanup.cleaned, true, JSON.stringify(cleanup)); // The SIGKILL case must recover the stale lease for both target identities. for (const target of [primary, secondary]) { - const result = await execFileAsync( + const result = await commands.exec( process.execPath, [ "scripts/rea.mjs", @@ -143,7 +144,11 @@ export async function verifyLinuxHopperOwnerExit({ "--format", "json", ], - { env: environment, timeout: 180_000, maxBuffer: 16 * 1024 * 1024 }, + { + env: environment, + timeout: 180_000, + maxBuffer: 16 * 1024 * 1024, + }, ); assert.ok(JSON.parse(result.stdout).normalized_result); } diff --git a/scripts/verify-real-hopper-callers.mjs b/scripts/verify-real-hopper-callers.mjs index bb6a238198..7a46aeec52 100644 --- a/scripts/verify-real-hopper-callers.mjs +++ b/scripts/verify-real-hopper-callers.mjs @@ -1,22 +1,43 @@ // Run from the repository root after `npm run build:cached` on macOS arm64, // with Hopper and the Xcode command-line tools installed. No target is executed. import assert from "node:assert/strict"; -import { execFile } from "node:child_process"; -import { createHash } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import { constants } from "node:fs"; import { access, readFile, writeFile } from "node:fs/promises"; import { createRequire } from "node:module"; import { join } from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; -import { promisify } from "node:util"; +import { runOwnedCommand } from "../dist/process/OwnedCommand.js"; import { artifactMcpCall } from "./lib/artifact-e2e.mjs"; import { snapshotHopperRuntime } from "./lib/real-hopper-cleanup.mjs"; import { closeVerifierResources } from "./lib/verifier-cleanup.mjs"; -import { HOPPER_TARGET_LEASE_DIRECTORY } from "../dist/process/HopperLease.js"; +import { createVerifierCommands } from "./lib/verify-commands.mjs"; import { - VerifierFixture, - withVerifierFixture, -} from "./lib/verifier-fixture.mjs"; + assertVerifierCleanupComplete, + completeVerifierRun, + createVerifierRun, +} from "./lib/verifier-run.mjs"; +import { HOPPER_TARGET_LEASE_DIRECTORY } from "../dist/process/HopperLease.js"; +import { VerifierFixture } from "./lib/verifier-fixture.mjs"; + +const verifierRun = createVerifierRun(); +// xcrun resolves the compiler before a workspace exists. A failed probe must +// not allocate a root, so it uses the same short-lived owner as requireLaneToolchain. +const probeOwnedCommand = async (command, arguments_) => { + const snapshot = await runOwnedCommand( + { + command, + arguments: arguments_, + cwd: process.cwd(), + env: process.env, + runId: `rea-hopper-callers-probe-${randomUUID()}`, + expectedCommand: null, + sweepTokenOwnedProcesses: true, + }, + { timeoutMs: null, diagnosticBytes: 1024 * 1024 }, + ); + return snapshot.stdout.text; +}; assert.equal(process.platform, "darwin", "requires macOS"); assert.equal( @@ -48,18 +69,32 @@ await access( ), constants.X_OK, ); -const run = promisify(execFile); -const clang = (await run("xcrun", ["--find", "clang"])).stdout.trim(); +const clang = (await probeOwnedCommand("xcrun", ["--find", "clang"])).trim(); const sdk = ( - await run("xcrun", ["--sdk", "macosx", "--show-sdk-path"]) -).stdout.trim(); + await probeOwnedCommand("xcrun", ["--sdk", "macosx", "--show-sdk-path"]) +).trim(); +// The command owner is created before the workspace. If allocation fails, +// the owner is still closed and no root is left behind. +const commands = await createVerifierCommands(); // A plain root under /tmp, where this verifier has always built its fixtures. const workspace = await VerifierFixture.open({ prefix: "rea-hopper-callers-verifier-", parent: "/tmp", private: false, +}).catch(async (cause) => { + await commands.close().catch((closeCause) => { + throw new AggregateError( + [cause, closeCause], + "Hopper caller workspace allocation and command cleanup both failed", + ); + }); + throw cause; }); const directory = workspace.path; +const retainWorkspace = (message, options) => { + workspace.retain(message); + return new Error(message, options); +}; const fixture = join(directory, "tail-call"); const large = join(directory, "long-scan"); const source = fileURLToPath( @@ -112,36 +147,9 @@ const matchingRef = (ref, site, from, to) => ref.source_procedure?.address === from && ref.target_procedure?.address === to && ref.call?.classification === "direct"; -// The root is removed only after these cleanups confirm. -workspace.releaseAfter(async () => { - try { - await closeVerifierResources([ - () => client.close(), - () => transport.close(), - async () => { - await new Promise((resolve) => setTimeout(resolve, 500)); - const leaked = [ - ...( - await snapshotHopperRuntime("/tmp", HOPPER_TARGET_LEASE_DIRECTORY) - ).paths, - ].filter((path) => !sessionsBefore.has(path)); - assert.deepEqual( - leaked, - [], - `Hopper resources remain: ${leaked.join(", ")}`, - ); - }, - ]); - } catch (cause) { - throw new Error(`Hopper cleanup failed; fixture retained at ${directory}`, { - cause, - }); - } finally { - if (stderr) process.stderr.write(stderr); - } -}); -await withVerifierFixture(workspace, async () => { - await run(clang, [ +let primaryFailure; +try { + await commands.exec(clang, [ "-isysroot", sdk, "-arch", @@ -161,7 +169,7 @@ await withVerifierFixture(workspace, async () => { lines.push("int main(void) { return rea_progress_0000(); }"); const largeSource = join(directory, "long-scan.c"); await writeFile(largeSource, lines.join("\n")); - await run(clang, [ + await commands.exec(clang, [ "-isysroot", sdk, "-arch", @@ -333,4 +341,94 @@ await withVerifierFixture(workspace, async () => { sha256: beforeLarge, }), ); -}); +} catch (cause) { + primaryFailure = cause; +} +let hopperCleanupFailure; +try { + await workspace.closeEarly([ + async () => { + try { + await closeVerifierResources([ + () => client.close(), + () => transport.close(), + async () => { + await new Promise((resolve) => setTimeout(resolve, 500)); + const leaked = [ + ...( + await snapshotHopperRuntime( + "/tmp", + HOPPER_TARGET_LEASE_DIRECTORY, + ) + ).paths, + ].filter((path) => !sessionsBefore.has(path)); + assert.deepEqual( + leaked, + [], + `Hopper resources remain: ${leaked.join(", ")}`, + ); + }, + ]); + } catch (cause) { + throw new Error( + `Hopper cleanup failed; fixture retained at ${directory}`, + { cause }, + ); + } finally { + if (stderr) process.stderr.write(stderr); + } + }, + ]); +} catch (cause) { + hopperCleanupFailure = cause; +} +if (hopperCleanupFailure !== undefined) + primaryFailure = + primaryFailure === undefined + ? hopperCleanupFailure + : new AggregateError( + [primaryFailure, hopperCleanupFailure], + "Hopper caller verification and Hopper cleanup both failed", + ); +let commandCleanupFailure; +try { + await workspace.closeEarly([() => commands.close()]); +} catch (cause) { + commandCleanupFailure = cause; +} +if (commandCleanupFailure !== undefined) + throw retainWorkspace( + `Hopper caller command cleanup failed; retained diagnostic workspace: ${directory}`, + { + cause: + primaryFailure === undefined + ? commandCleanupFailure + : new AggregateError( + [primaryFailure, commandCleanupFailure], + "Hopper caller verification and command cleanup both failed", + ), + }, + ); +const completedVerifierRun = await completeVerifierRun(verifierRun); +try { + assertVerifierCleanupComplete(completedVerifierRun.process_lineage); +} catch (cause) { + throw retainWorkspace( + `Cleanup unverified; retained diagnostic workspace: ${directory}`, + { + cause: + primaryFailure === undefined + ? cause + : new AggregateError( + [primaryFailure, cause], + "Hopper caller verification failed and cleanup was unverified", + ), + }, + ); +} +if (primaryFailure !== undefined) + throw retainWorkspace( + `Hopper caller verification failed; retained workspace: ${directory}`, + { cause: primaryFailure }, + ); +await workspace.release(); diff --git a/scripts/verify-real-hopper-fat.mjs b/scripts/verify-real-hopper-fat.mjs index 9ca3d07c1d..ab749376af 100644 --- a/scripts/verify-real-hopper-fat.mjs +++ b/scripts/verify-real-hopper-fat.mjs @@ -1,20 +1,25 @@ import { canonicalOutputSchema } from "./lib/canonical-output-schema.mjs"; import assert from "node:assert/strict"; -import { createHash } from "node:crypto"; -import { execFile } from "node:child_process"; +import { createHash, randomUUID } from "node:crypto"; import { access, readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { promisify } from "node:util"; import { fileURLToPath } from "node:url"; import { constants } from "node:fs"; import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import Ajv from "ajv/dist/2020.js"; import addFormats from "ajv-formats"; +import { runOwnedCommand } from "../dist/process/OwnedCommand.js"; import { - mcpTextValue, requireMcpOperationResult, + requireMcpToolError, } from "./lib/mcp-verifier-results.mjs"; +import { createVerifierCommands } from "./lib/verify-commands.mjs"; +import { + assertVerifierCleanupComplete, + completeVerifierRun, + createVerifierRun, +} from "./lib/verifier-run.mjs"; import { snapshotHopperRuntime } from "./lib/real-hopper-cleanup.mjs"; import { closeVerifierResources } from "./lib/verifier-cleanup.mjs"; import { HOPPER_TARGET_LEASE_DIRECTORY } from "../dist/process/HopperLease.js"; @@ -24,12 +29,26 @@ import { verifyHopperSourceBinding, relocateSingleFatSlice, } from "./lib/real-hopper-source-binding.mjs"; -import { - VerifierFixture, - withVerifierFixture, -} from "./lib/verifier-fixture.mjs"; +import { VerifierFixture } from "./lib/verifier-fixture.mjs"; -const run = promisify(execFile); +const verifierRun = createVerifierRun(); +// xcrun resolves clang and lipo before a workspace exists. A failed probe must +// not allocate a root, so it uses the same short-lived owner as requireLaneToolchain. +const probeOwnedCommand = async (command, arguments_) => { + const snapshot = await runOwnedCommand( + { + command, + arguments: arguments_, + cwd: process.cwd(), + env: process.env, + runId: `rea-hopper-fat-probe-${randomUUID()}`, + expectedCommand: null, + sweepTokenOwnedProcesses: true, + }, + { timeoutMs: null, diagnosticBytes: 1024 * 1024 }, + ); + return snapshot.stdout.text; +}; const verifyFat64 = process.argv[2] === "--fat64"; assert.deepEqual(process.argv.slice(2), verifyFat64 ? ["--fat64"] : []); if (process.platform !== "darwin" || !["arm64", "x64"].includes(process.arch)) @@ -52,24 +71,44 @@ await access(hopperLauncherPath, constants.X_OK).catch((cause) => { const toolPaths = {}; for (const command of ["clang", "lipo"]) toolPaths[command] = ( - await run("xcrun", ["--find", command]).catch((cause) => { + await probeOwnedCommand("xcrun", ["--find", command]).catch((cause) => { throw new Error(`Hopper FAT verification requires ${command}`, { cause }); }) - ).stdout.trim(); + ).trim(); const sdk = ( - await run("xcrun", ["--sdk", "macosx", "--show-sdk-path"]).catch((cause) => { + await probeOwnedCommand("xcrun", [ + "--sdk", + "macosx", + "--show-sdk-path", + ]).catch((cause) => { throw new Error( "Hopper FAT verification requires the installed macOS SDK", { cause }, ); }) -).stdout.trim(); +).trim(); +// The command owner is created before the workspace. If allocation fails, +// the owner is still closed and no root is left behind. +const commands = await createVerifierCommands(); const workspace = await VerifierFixture.open({ prefix: "hopper-fat-", private: false, +}).catch(async (cause) => { + await commands.close().catch((closeCause) => { + throw new AggregateError( + [cause, closeCause], + "Hopper FAT workspace allocation and command cleanup both failed", + ); + }); + throw cause; }); const directory = workspace.path; +const retainWorkspace = (message, options) => { + workspace.retain(message); + return new Error(message, options); +}; +const hostFatArchitecture = process.arch === "arm64" ? "arm64" : "x86_64"; const source = fileURLToPath( new URL("../tests/conformance/c/fixture.c", import.meta.url), ); @@ -100,33 +139,9 @@ const ajv = new Ajv({ strict: false }); addFormats(ajv); const observations = []; const sourceBinding = []; -// The root is removed only after these cleanups confirm. -workspace.releaseAfter(async () => { - try { - await closeVerifierResources([ - () => client.close(), - () => transport.close(), - ]); - } catch (cause) { - throw new Error(`Hopper cleanup failed; fixture retained at ${directory}`, { - cause, - }); - } - try { - const retained = [ - ...(await snapshotHopperRuntime("/tmp", HOPPER_TARGET_LEASE_DIRECTORY)) - .paths, - ].filter((path) => !before.has(path)); - assert.deepEqual(retained, [], "Hopper resources remain or are unresolved"); - } catch (cause) { - throw new Error( - `Hopper resource verification failed; fixture retained at ${directory}`, - { cause }, - ); - } -}); -await withVerifierFixture(workspace, async () => { - await run(toolPaths.clang, [ +let primaryFailure; +try { + await commands.exec(toolPaths.clang, [ "-isysroot", sdk, "-arch", @@ -137,7 +152,7 @@ await withVerifierFixture(workspace, async () => { "-o", arm, ]); - await run(toolPaths.clang, [ + await commands.exec(toolPaths.clang, [ "-isysroot", sdk, "-arch", @@ -148,9 +163,9 @@ await withVerifierFixture(workspace, async () => { "-o", intel, ]); - await run(toolPaths.lipo, ["-create", arm, intel, "-output", fat]); + await commands.exec(toolPaths.lipo, ["-create", arm, intel, "-output", fat]); if (verifyFat64) - await run(toolPaths.lipo, [ + await commands.exec(toolPaths.lipo, [ "-create", arm, intel, @@ -158,7 +173,7 @@ await withVerifierFixture(workspace, async () => { "-output", fat64, ]); - await run(toolPaths.lipo, [ + await commands.exec(toolPaths.lipo, [ "-create", process.arch === "arm64" ? arm : intel, "-output", @@ -186,7 +201,7 @@ await withVerifierFixture(workspace, async () => { await writeFile(little, swapped); extraFat64.push(little); const single = join(directory, "c-single-fat64"); - await run(toolPaths.lipo, [ + await commands.exec(toolPaths.lipo, [ "-create", process.arch === "arm64" ? arm : intel, "-fat64", @@ -194,25 +209,34 @@ await withVerifierFixture(workspace, async () => { single, ]); extraFat64.push(single); - for (const [name, reason, mutate] of [ + for (const [name, refusal, mutate] of [ [ "reserved", - /nonzero reserved field/u, + { + reason: "FAT64 architecture 0 has a nonzero reserved field", + }, (bytes) => bytes.writeUInt32BE(1, 36), ], [ "range", - /extends beyond the file/u, + { + reason: "FAT architecture 0 slice range extends beyond the file", + constraint: "file_format_rejected", + }, (bytes) => bytes.writeBigUInt64BE(BigInt(bytes.length), 24), ], [ "identity", - /disagrees with its Mach-O header/u, + { + reason: + "FAT architecture 0 CPU type/subtype disagrees with its Mach-O header", + constraint: "file_format_rejected", + }, (bytes) => bytes.writeUInt32BE(bytes.readUInt32BE(12) + 1, 12), ], [ "overlap", - /slices overlap/u, + { reason: "FAT slices overlap" }, (bytes) => { const firstOffset = bytes.readBigUInt64BE(16); const secondOffset = bytes.readBigUInt64BE(48); @@ -232,7 +256,7 @@ await withVerifierFixture(workspace, async () => { mutate(bytes); const path = join(directory, `c-fat64-invalid-${name}`); await writeFile(path, bytes); - invalidFat64.push({ path, reason }); + invalidFat64.push({ path, refusal }); } const hostBytes = await readFile(process.arch === "arm64" ? arm : intel); const secondOffset = Math.ceil((16384 + hostBytes.length) / 16384) * 16384; @@ -252,8 +276,10 @@ await withVerifierFixture(workspace, async () => { await writeFile(path, ambiguous); invalidFat64.push({ path, - reason: - /FAT binary declares several (?:arm64|x86_64) slices; no slice was selected/u, + refusal: { + reason: `FAT binary declares several ${hostFatArchitecture} slices; no slice was selected`, + constraint: "file_format_rejected", + }, }); } await client.connect(transport); @@ -391,13 +417,28 @@ await withVerifierFixture(workspace, async () => { }); await call("close_binary"); } - for (const { path, reason } of invalidFat64) { + for (const { path, refusal } of invalidFat64) { const reply = await client.callTool({ name: "open_binary", arguments: { path }, }); - assert.equal(reply.isError, true, "malformed FAT64 unexpectedly launched"); - assert.match(mcpTextValue(reply), reason); + // Admission failures are BinaryTargetError file_format_rejected. + // Reserved and overlap survive admission and fail in resolveHopperMachOImage, + // which returns BinaryTargetError with no constraint. Both classify as + // target_unavailable and keep the producer reason on details.reason. + const refused = requireMcpToolError(reply); + assert.partialDeepStrictEqual(refused, { + code: "target_unavailable", + details: { + path, + reason: refusal.reason, + ...(refusal.constraint === undefined + ? {} + : { constraint: refusal.constraint }), + }, + }); + if (refusal.constraint === undefined) + assert.equal(refused.details?.constraint, undefined); } if (verifyFat64) { await call("open_binary", { path: fat64 }); @@ -441,7 +482,7 @@ await withVerifierFixture(workspace, async () => { await assert.rejects(access(preparedImagePath), { code: "ENOENT" }); } for (const [path, expected] of cliMappings) { - const pending = run( + const cliResult = await commands.exec( process.execPath, [ "scripts/rea.mjs", @@ -458,11 +499,99 @@ await withVerifierFixture(workspace, async () => { env: { ...process.env, HOPPER_LOADER_ARGS_JSON: "[]" }, }, ); - assert.ok(pending.child.pid !== undefined); - const cli = JSON.parse((await pending).stdout); + const cli = JSON.parse(cliResult.stdout); assert.deepEqual(cli.normalized_result, expected.mapping); } -}); +} catch (cause) { + primaryFailure = cause; +} +let hopperCleanupFailure; +try { + await workspace.closeEarly([ + async () => { + try { + await closeVerifierResources([ + () => client.close(), + () => transport.close(), + ]); + } catch (cause) { + throw new Error( + `Hopper cleanup failed; fixture retained at ${directory}`, + { cause }, + ); + } + try { + const retained = [ + ...( + await snapshotHopperRuntime("/tmp", HOPPER_TARGET_LEASE_DIRECTORY) + ).paths, + ].filter((path) => !before.has(path)); + assert.deepEqual( + retained, + [], + "Hopper resources remain or are unresolved", + ); + } catch (cause) { + throw new Error( + `Hopper resource verification failed; fixture retained at ${directory}`, + { cause }, + ); + } + }, + ]); +} catch (cause) { + hopperCleanupFailure = cause; +} +if (hopperCleanupFailure !== undefined) + primaryFailure = + primaryFailure === undefined + ? hopperCleanupFailure + : new AggregateError( + [primaryFailure, hopperCleanupFailure], + "Hopper FAT verification and Hopper cleanup both failed", + ); +let commandCleanupFailure; +try { + await workspace.closeEarly([() => commands.close()]); +} catch (cause) { + commandCleanupFailure = cause; +} +if (commandCleanupFailure !== undefined) + throw retainWorkspace( + `Hopper FAT command cleanup failed; retained diagnostic workspace: ${directory}`, + { + cause: + primaryFailure === undefined + ? commandCleanupFailure + : new AggregateError( + [primaryFailure, commandCleanupFailure], + "Hopper FAT verification and command cleanup both failed", + ), + }, + ); +const completedVerifierRun = await completeVerifierRun(verifierRun); +try { + assertVerifierCleanupComplete(completedVerifierRun.process_lineage); +} catch (cause) { + throw retainWorkspace( + `Cleanup unverified; retained diagnostic workspace: ${directory}`, + { + cause: + primaryFailure === undefined + ? cause + : new AggregateError( + [primaryFailure, cause], + "Hopper FAT verification failed and cleanup was unverified", + ), + }, + ); +} +if (primaryFailure !== undefined) + throw retainWorkspace( + `Hopper FAT verification failed; retained workspace: ${directory}`, + { cause: primaryFailure }, + ); +await workspace.release(); console.log( JSON.stringify( { diff --git a/scripts/verify-real-hopper-linux.mjs b/scripts/verify-real-hopper-linux.mjs index 594391686a..045791eb87 100644 --- a/scripts/verify-real-hopper-linux.mjs +++ b/scripts/verify-real-hopper-linux.mjs @@ -1,15 +1,14 @@ +import { randomUUID } from "node:crypto"; import { access, symlink } from "node:fs/promises"; -import { execFile } from "node:child_process"; import { join } from "node:path"; -import { promisify } from "node:util"; +import { runOwnedCommand } from "../dist/process/OwnedCommand.js"; import { createVerifierRun } from "./lib/verifier-run.mjs"; import { VerifierFixture, withVerifierFixture, } from "./lib/verifier-fixture.mjs"; -const execFileAsync = promisify(execFile); createVerifierRun(); if (process.platform !== "linux") @@ -20,8 +19,21 @@ await access(launcher); await access("/usr/bin/Xvfb"); await access("/usr/bin/xauth"); await access("/usr/bin/python3"); -const linked = await execFileAsync("ldd", [launcher]); -if (`${linked.stdout}\n${linked.stderr}`.includes("not found")) +// ldd runs before a workspace exists, so it uses the same short-lived owner +// as requireLaneToolchain rather than a fixture-retained command owner. +const linked = await runOwnedCommand( + { + command: "ldd", + arguments: [launcher], + cwd: process.cwd(), + env: process.env, + runId: `rea-hopper-linux-ldd-${randomUUID()}`, + expectedCommand: null, + sweepTokenOwnedProcesses: true, + }, + { timeoutMs: null, diagnosticBytes: 1024 * 1024 }, +); +if (`${linked.stdout.text}\n${linked.stderr.text}`.includes("not found")) throw new Error("Hopper has unresolved shared-library dependencies"); const alternateDirectory = await VerifierFixture.open({ prefix: "rea-real-hopper-launcher-", diff --git a/scripts/verify-real-hopper.mjs b/scripts/verify-real-hopper.mjs index ca183ffd0b..0b19508437 100644 --- a/scripts/verify-real-hopper.mjs +++ b/scripts/verify-real-hopper.mjs @@ -1,6 +1,4 @@ -import { execFile } from "node:child_process"; import { tmpdir } from "node:os"; -import { promisify } from "node:util"; import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; @@ -38,7 +36,13 @@ import { verifyHopperFunctionBasics, } from "./lib/real-hopper-function-basics.mjs"; import { openAndVerifyLargeFixture } from "./lib/real-hopper-exhaustive-search.mjs"; -import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs"; +import { createVerifierCommands } from "./lib/verify-commands.mjs"; +import { + assertVerifierCleanupComplete, + completeVerifierRun, + createVerifierRun, +} from "./lib/verifier-run.mjs"; +import { VerifierFixture } from "./lib/verifier-fixture.mjs"; import { requireHopperSelection } from "./lib/real-hopper-selection.mjs"; import { verifyHopperInventories } from "./lib/real-hopper-inventory.mjs"; import { @@ -48,548 +52,625 @@ import { import { startUnrelatedHopperSentinel } from "./lib/unrelated-hopper-sentinel.mjs"; import { snapshotHopperRuntime } from "./lib/real-hopper-cleanup.mjs"; import { verifyLinuxHopperOwnerExit } from "./lib/real-hopper-owner-exit.mjs"; -const execFileAsync = promisify(execFile); const verifierRun = createVerifierRun(); -const timeout = 180_000; -const parseServerArgs = (encoded) => { - const parsed = JSON.parse(encoded); - if ( - !Array.isArray(parsed) || - parsed.some((value) => typeof value !== "string") - ) - throw new Error("REA_VERIFY_SERVER_ARGS_JSON must encode string arguments"); - return parsed; +// ps and any later host commands share one owner, created before a tool or +// workspace exists. A failed workspace allocation still closes the owner. +const commands = await createVerifierCommands(); +const workspaceFixture = await VerifierFixture.open({ + prefix: "rea-real-hopper-", +}).catch(async (cause) => { + await commands.close().catch((closeCause) => { + throw new AggregateError( + [cause, closeCause], + "Hopper workspace allocation and command cleanup both failed", + ); + }); + throw cause; +}); +const workspace = workspaceFixture.path; +const retainWorkspace = (message, options) => { + workspaceFixture.retain(message); + return new Error(message, options); }; +let primaryFailure; +let unrelatedHopperSurvived = false; +let summary; +try { + const timeout = 180_000; + const parseServerArgs = (encoded) => { + const parsed = JSON.parse(encoded); + if ( + !Array.isArray(parsed) || + parsed.some((value) => typeof value !== "string") + ) + throw new Error( + "REA_VERIFY_SERVER_ARGS_JSON must encode string arguments", + ); + return parsed; + }; -const runtimeParent = process.platform === "darwin" ? "/tmp" : tmpdir(); -const sessionsBefore = ( - await snapshotHopperRuntime(runtimeParent, HOPPER_TARGET_LEASE_DIRECTORY) -).paths; -const ownedProcessIds = new Set(); -const textValue = mcpTextValue; + const runtimeParent = process.platform === "darwin" ? "/tmp" : tmpdir(); + const sessionsBefore = ( + await snapshotHopperRuntime(runtimeParent, HOPPER_TARGET_LEASE_DIRECTORY) + ).paths; + const textValue = mcpTextValue; -const requireOverview = (value, operation) => { - if ( - value === null || - typeof value !== "object" || - typeof value.document !== "string" || - value.document.length === 0 || - !Number.isInteger(value.procedure_count) || - value.procedure_count < 1 || - !Number.isInteger(value.segment_count) || - value.segment_count < 1 || - !Array.isArray(value.segments) - ) { - throw new Error(`${operation} returned no analyzed binary overview`); - } - return value; -}; + const requireOverview = (value, operation) => { + if ( + value === null || + typeof value !== "object" || + typeof value.document !== "string" || + value.document.length === 0 || + !Number.isInteger(value.procedure_count) || + value.procedure_count < 1 || + !Number.isInteger(value.segment_count) || + value.segment_count < 1 || + !Array.isArray(value.segments) + ) { + throw new Error(`${operation} returned no analyzed binary overview`); + } + return value; + }; -const verifyRelationships = async (client, options, procedure) => { - const related = {}; - for (const operation of ["procedure_callers", "procedure_callees"]) { - related[operation] = requireAddressArray( - requireMcpEvidenceResult( - await client.callTool( - { name: operation, arguments: { procedure } }, - options, + const verifyRelationships = async (client, options, procedure) => { + const related = {}; + for (const operation of ["procedure_callers", "procedure_callees"]) { + related[operation] = requireAddressArray( + requireMcpEvidenceResult( + await client.callTool( + { name: operation, arguments: { procedure } }, + options, + ), + operation, ), operation, - ), - operation, - ); - } - related.xrefs = requireAddressArray( - requireMcpEvidenceResult( - await client.callTool( - { name: "xrefs", arguments: { address: procedure } }, - options, + ); + } + related.xrefs = requireAddressArray( + requireMcpEvidenceResult( + await client.callTool( + { name: "xrefs", arguments: { address: procedure } }, + options, + ), + "xrefs", ), "xrefs", - ), - "xrefs", - ); - return related; -}; - -const verifyByteAccess = async (client, options, firstAddress) => { - const byteRead = requireMcpEvidenceResult( - await client.callTool( - { name: "read_bytes", arguments: { address: firstAddress, length: 16 } }, - options, - ), - "read_bytes", - ); - if ( - byteRead?.address !== firstAddress || - byteRead.requested_bytes !== 16 || - byteRead.returned_bytes !== 16 || - byteRead.complete !== true || - !/^[a-f0-9]{32}$/u.test(byteRead.bytes_hex) - ) { - throw new Error("read_bytes returned an invalid bounded byte window"); - } - const mappedOffset = requireMcpEvidenceResult( - await client.callTool( - { - name: "address_to_file_offset", - arguments: { address: firstAddress }, - }, - options, - ), - "address_to_file_offset", - ); - if ( - mappedOffset?.address !== firstAddress || - !Number.isSafeInteger(mappedOffset.file_offset) || - mappedOffset.file_offset < 0 - ) { - throw new Error("address_to_file_offset returned an invalid mapping"); - } - return { - byteReadCount: byteRead.returned_bytes, - mappedFileOffset: mappedOffset.file_offset, + ); + return related; }; -}; -const verifyCurrentTarget = async (client, options) => { - const procedures = requireMcpEvidenceResult( - await client.callTool({ name: "list_procedures", arguments: {} }, options), - "list_procedures", - ); - const firstAddress = firstProcedureAddress(procedures); - const byteAccess = await verifyByteAccess(client, options, firstAddress); - const basics = await verifyHopperFunctionBasics( - client, - options, - firstAddress, - ); - const relationships = await verifyRelationships( - client, - options, - firstAddress, - ); - const boundedResult = requireMcpEvidenceResult( - await client.callTool( - { - name: "batch_decompile", - arguments: { addresses: [firstAddress] }, - }, - options, - ), - "batch_decompile", - ); - if ( - boundedResult === null || - typeof boundedResult !== "object" || - Array.isArray(boundedResult) || - boundedResult.total !== 1 || - boundedResult.succeeded !== 1 || - boundedResult.failed !== 0 || - !Array.isArray(boundedResult.items) || - boundedResult.items.length !== 1 || - boundedResult.items[0]?.address !== firstAddress || - boundedResult.items[0]?.status !== "ok" - ) { - throw new Error("batch_decompile returned an invalid result"); - } - const boundedPseudocode = requirePseudocode( - boundedResult.items[0].pseudocode, - "batch_decompile", - ); - const dossier = requireFunctionDossier( - requireMcpEvidenceResult( + const verifyByteAccess = async (client, options, firstAddress) => { + const byteRead = requireMcpEvidenceResult( await client.callTool( { - name: "analyze_function", - arguments: { procedure: firstAddress, detail: "complete" }, + name: "read_bytes", + arguments: { address: firstAddress, length: 16 }, }, options, ), - "analyze_function", - ), - firstAddress, - ); - return { - procedure: dossier.procedure, - procedureCount: procedures.length, - boundedPseudocodeChars: boundedPseudocode.length, - analyzedPseudocodeChars: [...dossier.pseudocode].length, - callerCount: relationships.procedure_callers.length, - calleeCount: relationships.procedure_callees.length, - xrefCount: relationships.xrefs.length, - outgoingReferenceCount: basics.outgoingReferenceCount, - instructionWindowCount: basics.instructionWindowCount, - ...byteAccess, + "read_bytes", + ); + if ( + byteRead?.address !== firstAddress || + byteRead.requested_bytes !== 16 || + byteRead.returned_bytes !== 16 || + byteRead.complete !== true || + !/^[a-f0-9]{32}$/u.test(byteRead.bytes_hex) + ) { + throw new Error("read_bytes returned an invalid bounded byte window"); + } + const mappedOffset = requireMcpEvidenceResult( + await client.callTool( + { + name: "address_to_file_offset", + arguments: { address: firstAddress }, + }, + options, + ), + "address_to_file_offset", + ); + if ( + mappedOffset?.address !== firstAddress || + !Number.isSafeInteger(mappedOffset.file_offset) || + mappedOffset.file_offset < 0 + ) { + throw new Error("address_to_file_offset returned an invalid mapping"); + } + return { + byteReadCount: byteRead.returned_bytes, + mappedFileOffset: mappedOffset.file_offset, + }; }; -}; -const fixtureManifestPath = - process.env.REA_HOPPER_CONFORMANCE_MANIFEST_PATH ?? - "build/conformance/manifest.json"; -const fixtureTargets = await loadRealHopperFixtureTargets(fixtureManifestPath); -const targetA = fixtureTargets.primary.path; -const targetB = fixtureTargets.secondary.path; -const largeTarget = fixtureTargets.large.path; -const targetHashA = fixtureTargets.primary.sha256; -const targetHashB = fixtureTargets.secondary.sha256; -const serverEnvironment = { ...process.env }; -serverEnvironment.REA_ANALYSIS_PROVIDER = "auto"; -delete serverEnvironment.HOPPER_TARGET_PATH; -delete serverEnvironment.HOPPER_SECOND_TARGET_PATH; -delete serverEnvironment.REA_VERIFY_SERVER_COMMAND; -delete serverEnvironment.REA_VERIFY_SERVER_ARGS_JSON; - -const serverCommand = process.env.REA_VERIFY_SERVER_COMMAND ?? process.execPath; -const serverArgs = parseServerArgs( - process.env.REA_VERIFY_SERVER_ARGS_JSON ?? '["dist/main.js"]', -); - -const transport = new StdioClientTransport({ - command: serverCommand, - args: serverArgs, - cwd: process.cwd(), - env: serverEnvironment, - stderr: "pipe", -}); -let stderrBytes = 0; -const stderrChunks = []; -transport.stderr?.on("data", (chunk) => { - stderrBytes += chunk.length; - stderrChunks.push(chunk.toString("utf8")); -}); -const client = new Client({ name: "real-hopper-verifier", version: "1.0.0" }); -const unrelatedHopper = await startUnrelatedHopperSentinel(); -let unrelatedHopperSurvived = false; -let summary; - -try { - await client.connect(transport); - if (transport.pid === null) - throw new Error("The verifier did not capture its MCP process identity"); - ownedProcessIds.add(transport.pid); - const progressUpdates = []; - const options = { - timeout, - onprogress: (update) => progressUpdates.push(update), - }; - const fullSessionStatus = () => - client.callTool({ name: "binary_session", arguments: {} }, options); - const initialSession = await fullSessionStatus(); - const initialStatus = requireMcpLifecycleResult( - initialSession, - "binary_session", - ); - const listed = await client.listTools(); - const availableTools = initialStatus.tool_availability; - if (!Array.isArray(availableTools)) - throw new Error("The real session omitted tool availability"); - const expectedNames = sortVerifierNames( - availableTools.map(({ name }) => name), - ); - const actualNames = sortVerifierNames(listed.tools.map(({ name }) => name)); - if (JSON.stringify(actualNames) !== JSON.stringify(expectedNames)) { - const expected = new Set(expectedNames); - const actual = new Set(actualNames); - throw new Error( - `The real server tool catalog diverged from session capability inventory (untracked tools: ${actualNames.filter((name) => !expected.has(name)).join(", ") || "none"}; missing catalog tools: ${expectedNames.filter((name) => !actual.has(name)).join(", ") || "none"})`, + const verifyCurrentTarget = async (client, options) => { + const procedures = requireMcpEvidenceResult( + await client.callTool( + { name: "list_procedures", arguments: {} }, + options, + ), + "list_procedures", ); - } - - if (initialStatus.open !== false) - throw new Error("The verifier did not start without a target"); - requireHopperSelection(initialStatus, { - targetSupport: "unknown", - selected: false, - }); - const opened = await client.callTool( - { name: "open_binary", arguments: { path: targetA } }, - options, - ); - if (opened.isError === true) throw new Error(textValue(opened)); - const firstStatus = requireMcpLifecycleResult( - await fullSessionStatus(), - "binary_session", - ); - const providerBinding = requireHopperSelection(firstStatus, { - targetSupport: "supported", - selected: true, - }); - const documents = await client.callTool( - { name: "list_documents", arguments: {} }, - options, - ); - requireMcpEvidenceResult(documents, "list_documents"); - requireBridgeProgress(progressUpdates); - const rejectedProcedure = await client.callTool( - { - name: "procedure_info", - arguments: { procedure: "__rea_missing_procedure__" }, - }, - options, - ); - if (rejectedProcedure.isError !== true) - throw new Error("The real Hopper bridge accepted an unknown procedure"); - const overview = await client.callTool( - { name: "binary_overview", arguments: {} }, - options, - ); - requireMcpEvidenceResult(overview, "binary_overview"); - const inventoryCounts = await verifyHopperInventories(client, options); - requireEvidenceProvider( - documents, - "list_documents", - HOPPER_PROVIDER_IDENTITY.id, - ); - requireWorkflowEvidenceProvider(overview, "binary_overview", { - workflowProviderId: REA_WORKFLOW_PROVIDER.id, - upstreamProviderId: HOPPER_PROVIDER_IDENTITY.id, - }); - const segments = await client.callTool( - { name: "list_segments", arguments: {} }, - options, - ); - requireTruthfulMemoryRegions( - requireMcpEvidenceResult(segments, "list_segments"), - ); - const firstDocuments = requireMcpEvidenceResult(documents, "list_documents"); - if ( - !Array.isArray(firstDocuments) || - firstDocuments.length === 0 || - typeof firstDocuments[0] !== "string" - ) - throw new Error("list_documents returned no real Hopper document"); - const currentDocument = await requireCurrentDocument( - client, - options, - firstDocuments, - requireMcpEvidenceResult, - ); - const firstOverview = requireOverview( - requireMcpEvidenceResult(overview, "binary_overview"), - "binary_overview", - ); - if (firstOverview.document !== currentDocument) - throw new Error( - "binary_overview did not identify the selected Hopper document", + const firstAddress = firstProcedureAddress(procedures); + const byteAccess = await verifyByteAccess(client, options, firstAddress); + const basics = await verifyHopperFunctionBasics( + client, + options, + firstAddress, ); - const firstAnalysis = await verifyCurrentTarget(client, options); - const fixtureAnalysis = await verifyRealHopperFixture({ - client, - options, - document: currentDocument, - oracle: fixtureTargets.oracle, - normalizedResult: requireMcpEvidenceResult, - }); - const boundaryContracts = await verifyHopperBoundaryContracts( - client, - options, - fixtureAnalysis.procedures.entry, - targetA, - ); - const switched = await client.callTool( - { name: "open_binary", arguments: { path: targetB } }, - options, - ); - if (switched.isError === true) throw new Error(textValue(switched)); - const secondSession = requireMcpLifecycleResult( - await fullSessionStatus(), - "binary_session", - ); - if (secondSession.path !== targetB) - throw new Error("The real session did not switch to target B"); - requireHopperSelection(secondSession, { - targetSupport: "supported", - selected: true, - }); - const secondOverview = await client.callTool( - { name: "binary_overview", arguments: {} }, - options, - ); - const verifiedSecondOverview = requireOverview( - requireMcpEvidenceResult(secondOverview, "binary_overview"), - "binary_overview after target switch", - ); - const secondAnalysis = await verifyCurrentTarget(client, options); - const documentsAfterTargetSwitch = requireMcpEvidenceResult( - await client.callTool({ name: "list_documents", arguments: {} }, options), - "list_documents", - ); - const secondDocument = await requireCurrentDocument( - client, - options, - documentsAfterTargetSwitch, - requireMcpEvidenceResult, - ); - if (verifiedSecondOverview.document !== secondDocument) - throw new Error( - "binary_overview retained another document after target switch", + const relationships = await verifyRelationships( + client, + options, + firstAddress, ); - const expectedDocumentsAfterTargetSwitch = firstDocuments.length; - if ( - !Array.isArray(documentsAfterTargetSwitch) || - documentsAfterTargetSwitch.length !== expectedDocumentsAfterTargetSwitch - ) - throw new Error( - "Target switching did not replace the previous Hopper document", - ); - const reopenedTarget = await client.callTool( - { name: "open_binary", arguments: { path: targetB } }, - options, - ); - if (reopenedTarget.isError === true) - throw new Error(textValue(reopenedTarget)); - const documentsAfterReopen = requireMcpEvidenceResult( - await client.callTool({ name: "list_documents", arguments: {} }, options), - "list_documents", - ); - if ( - !Array.isArray(documentsAfterReopen) || - documentsAfterReopen.length !== documentsAfterTargetSwitch.length - ) - throw new Error( - "Reopening the same target created a duplicate Hopper document", - ); - const largeInventory = await openAndVerifyLargeFixture({ - client, - options, - normalizedResult: requireMcpEvidenceResult, - path: largeTarget, - expectedCount: fixtureTargets.largeOracle.symbolCount, - symbolPrefix: fixtureTargets.largeOracle.symbolPrefix, - stringPrefix: fixtureTargets.largeOracle.stringPrefix, - }); - - const processList = await execFileAsync("ps", ["-ax", "-o", "command="]); - const bundledMcpRunning = processList.stdout - .split("\n") - .some((line) => line.trim().endsWith("/HopperMCPServer")); - if (bundledMcpRunning) { - throw new Error( - "Hopper's bundled MCP server was running during verification", - ); - } - const closed = await client.callTool( - { name: "close_binary", arguments: {} }, - options, - ); - if (closed.isError === true) throw new Error(textValue(closed)); - let unicodeStrings = null; - let terminalInstructions = null; - if (fixtureTargets.unicode !== undefined) { - const call = async (name, args = {}) => - requireMcpOperationResult( - await client.callTool({ name, arguments: args }, options), - name, - ); - await call("open_binary", { path: fixtureTargets.unicode.path }); - terminalInstructions = await verifyHopperTerminalInstructions(call); - unicodeStrings = await verifyHopperStringObjects(call, [ - { value: "REA_UTF16_é_😀", encoding: "utf-16-le" }, - { value: 'REA_UTF16_ESCAPED_é_"\\line\nend\t\r', encoding: "utf-16-le" }, - { value: "\nREA_UTF16_LEADING_é", encoding: "utf-16-le" }, - ]); - await call("close_binary"); - } - const lifecycleAndCli = await verifyHopperLifecycleAndCli(client, options, { - primary: targetA, - secondary: targetB, - ...(fixtureTargets.unicode === undefined - ? {} - : { unicode: fixtureTargets.unicode.path }), - ownedProcessIds, - }); - const closedSession = requireMcpLifecycleResult( - await fullSessionStatus(), - "binary_session", - ); - if (closedSession.open !== false) - throw new Error("The real session remained open after close_binary"); - const diagnosticCount = requireSafeDiagnostics( - stderrChunks, - [ - "procedure_info", + const boundedResult = requireMcpEvidenceResult( + await client.callTool( + { + name: "batch_decompile", + arguments: { addresses: [firstAddress] }, + }, + options, + ), "batch_decompile", - ...boundaryContracts.rejectedOperations, - ], - ["HopperRemoteError", "HopperCancelledError", "AnalysisCancelledError"], - ); - requireBridgeDiagnostic(stderrChunks); - - summary = { - toolCount: actualNames.length, - documentCount: firstDocuments.length, - overview: firstOverview, - segmentCount: firstOverview.segment_count, - analyses: [firstAnalysis, secondAnalysis], - fixtureAnalysis, - unicodeStrings, - terminalInstructions, - boundaryContracts, - lifecycleAndCli, - inventoryCounts, - largeInventory, - fixtureManifest: fixtureTargets.manifestPath, - bundledMcpRunning, - stderrBytes, - diagnosticCount, - bridgeProgressUpdates: progressUpdates.length, - dynamicSession: true, - providerBinding, - targets: [targetA, targetB, largeTarget], - targetHashes: [targetHashA, targetHashB, fixtureTargets.large.sha256], - switched: true, - secondOverview: verifiedSecondOverview, - documentCountAfterTargetSwitch: documentsAfterTargetSwitch.length, - documentCountAfterSameTargetReopen: documentsAfterReopen.length, + ); + if ( + boundedResult === null || + typeof boundedResult !== "object" || + Array.isArray(boundedResult) || + boundedResult.total !== 1 || + boundedResult.succeeded !== 1 || + boundedResult.failed !== 0 || + !Array.isArray(boundedResult.items) || + boundedResult.items.length !== 1 || + boundedResult.items[0]?.address !== firstAddress || + boundedResult.items[0]?.status !== "ok" + ) { + throw new Error("batch_decompile returned an invalid result"); + } + const boundedPseudocode = requirePseudocode( + boundedResult.items[0].pseudocode, + "batch_decompile", + ); + const dossier = requireFunctionDossier( + requireMcpEvidenceResult( + await client.callTool( + { + name: "analyze_function", + arguments: { procedure: firstAddress, detail: "complete" }, + }, + options, + ), + "analyze_function", + ), + firstAddress, + ); + return { + procedure: dossier.procedure, + procedureCount: procedures.length, + boundedPseudocodeChars: boundedPseudocode.length, + analyzedPseudocodeChars: [...dossier.pseudocode].length, + callerCount: relationships.procedure_callers.length, + calleeCount: relationships.procedure_callees.length, + xrefCount: relationships.xrefs.length, + outgoingReferenceCount: basics.outgoingReferenceCount, + instructionWindowCount: basics.instructionWindowCount, + ...byteAccess, + }; }; -} finally { - const keepAlive = setInterval(() => undefined, 100); - try { - await client.close(); - await transport.close(); - } finally { - clearInterval(keepAlive); - unrelatedHopperSurvived = await unrelatedHopper.verifyAndClose(); - } -} -await new Promise((resolve) => setTimeout(resolve, 500)); -const sessionsAfter = [ - ...(await snapshotHopperRuntime(runtimeParent, HOPPER_TARGET_LEASE_DIRECTORY)) - .paths, -] - .filter((path) => !sessionsBefore.has(path)) - .sort(); -if (sessionsAfter.length > 0) { - await writeVerificationReport({ - ...summary, - unrelatedHopperSurvived, - cleanShutdown: false, - retainedRuntimePaths: sessionsAfter, + const fixtureManifestPath = + process.env.REA_HOPPER_CONFORMANCE_MANIFEST_PATH ?? + "build/conformance/manifest.json"; + const fixtureTargets = + await loadRealHopperFixtureTargets(fixtureManifestPath); + const targetA = fixtureTargets.primary.path; + const targetB = fixtureTargets.secondary.path; + const largeTarget = fixtureTargets.large.path; + const targetHashA = fixtureTargets.primary.sha256; + const targetHashB = fixtureTargets.secondary.sha256; + const serverEnvironment = { ...process.env }; + serverEnvironment.REA_ANALYSIS_PROVIDER = "auto"; + delete serverEnvironment.HOPPER_TARGET_PATH; + delete serverEnvironment.HOPPER_SECOND_TARGET_PATH; + delete serverEnvironment.REA_VERIFY_SERVER_COMMAND; + delete serverEnvironment.REA_VERIFY_SERVER_ARGS_JSON; + + const serverCommand = + process.env.REA_VERIFY_SERVER_COMMAND ?? process.execPath; + const serverArgs = parseServerArgs( + process.env.REA_VERIFY_SERVER_ARGS_JSON ?? '["dist/main.js"]', + ); + + const transport = new StdioClientTransport({ + command: serverCommand, + args: serverArgs, + cwd: process.cwd(), + env: serverEnvironment, + stderr: "pipe", }); - throw new Error( - `Hopper resources remain or have unresolved ownership: ${sessionsAfter.join(", ")}`, + let stderrBytes = 0; + const stderrChunks = []; + transport.stderr?.on("data", (chunk) => { + stderrBytes += chunk.length; + stderrChunks.push(chunk.toString("utf8")); + }); + const client = new Client({ name: "real-hopper-verifier", version: "1.0.0" }); + const unrelatedHopper = await startUnrelatedHopperSentinel(); + + try { + await client.connect(transport); + if (transport.pid === null) + throw new Error("The verifier did not capture its MCP process identity"); + const progressUpdates = []; + const options = { + timeout, + onprogress: (update) => progressUpdates.push(update), + }; + const fullSessionStatus = () => + client.callTool({ name: "binary_session", arguments: {} }, options); + const initialSession = await fullSessionStatus(); + const initialStatus = requireMcpLifecycleResult( + initialSession, + "binary_session", + ); + const listed = await client.listTools(); + const availableTools = initialStatus.tool_availability; + if (!Array.isArray(availableTools)) + throw new Error("The real session omitted tool availability"); + const expectedNames = sortVerifierNames( + availableTools.map(({ name }) => name), + ); + const actualNames = sortVerifierNames(listed.tools.map(({ name }) => name)); + if (JSON.stringify(actualNames) !== JSON.stringify(expectedNames)) { + const expected = new Set(expectedNames); + const actual = new Set(actualNames); + throw new Error( + `The real server tool catalog diverged from session capability inventory (untracked tools: ${actualNames.filter((name) => !expected.has(name)).join(", ") || "none"}; missing catalog tools: ${expectedNames.filter((name) => !actual.has(name)).join(", ") || "none"})`, + ); + } + + if (initialStatus.open !== false) + throw new Error("The verifier did not start without a target"); + requireHopperSelection(initialStatus, { + targetSupport: "unknown", + selected: false, + }); + const opened = await client.callTool( + { name: "open_binary", arguments: { path: targetA } }, + options, + ); + if (opened.isError === true) throw new Error(textValue(opened)); + const firstStatus = requireMcpLifecycleResult( + await fullSessionStatus(), + "binary_session", + ); + const providerBinding = requireHopperSelection(firstStatus, { + targetSupport: "supported", + selected: true, + }); + const documents = await client.callTool( + { name: "list_documents", arguments: {} }, + options, + ); + requireMcpEvidenceResult(documents, "list_documents"); + requireBridgeProgress(progressUpdates); + const rejectedProcedure = await client.callTool( + { + name: "procedure_info", + arguments: { procedure: "__rea_missing_procedure__" }, + }, + options, + ); + if (rejectedProcedure.isError !== true) + throw new Error("The real Hopper bridge accepted an unknown procedure"); + const overview = await client.callTool( + { name: "binary_overview", arguments: {} }, + options, + ); + requireMcpEvidenceResult(overview, "binary_overview"); + const inventoryCounts = await verifyHopperInventories(client, options); + requireEvidenceProvider( + documents, + "list_documents", + HOPPER_PROVIDER_IDENTITY.id, + ); + requireWorkflowEvidenceProvider(overview, "binary_overview", { + workflowProviderId: REA_WORKFLOW_PROVIDER.id, + upstreamProviderId: HOPPER_PROVIDER_IDENTITY.id, + }); + const segments = await client.callTool( + { name: "list_segments", arguments: {} }, + options, + ); + requireTruthfulMemoryRegions( + requireMcpEvidenceResult(segments, "list_segments"), + ); + const firstDocuments = requireMcpEvidenceResult( + documents, + "list_documents", + ); + if ( + !Array.isArray(firstDocuments) || + firstDocuments.length === 0 || + typeof firstDocuments[0] !== "string" + ) + throw new Error("list_documents returned no real Hopper document"); + const currentDocument = await requireCurrentDocument( + client, + options, + firstDocuments, + requireMcpEvidenceResult, + ); + const firstOverview = requireOverview( + requireMcpEvidenceResult(overview, "binary_overview"), + "binary_overview", + ); + if (firstOverview.document !== currentDocument) + throw new Error( + "binary_overview did not identify the selected Hopper document", + ); + const firstAnalysis = await verifyCurrentTarget(client, options); + const fixtureAnalysis = await verifyRealHopperFixture({ + client, + options, + document: currentDocument, + oracle: fixtureTargets.oracle, + normalizedResult: requireMcpEvidenceResult, + }); + const boundaryContracts = await verifyHopperBoundaryContracts( + client, + options, + fixtureAnalysis.procedures.entry, + targetA, + ); + const switched = await client.callTool( + { name: "open_binary", arguments: { path: targetB } }, + options, + ); + if (switched.isError === true) throw new Error(textValue(switched)); + const secondSession = requireMcpLifecycleResult( + await fullSessionStatus(), + "binary_session", + ); + if (secondSession.path !== targetB) + throw new Error("The real session did not switch to target B"); + requireHopperSelection(secondSession, { + targetSupport: "supported", + selected: true, + }); + const secondOverview = await client.callTool( + { name: "binary_overview", arguments: {} }, + options, + ); + const verifiedSecondOverview = requireOverview( + requireMcpEvidenceResult(secondOverview, "binary_overview"), + "binary_overview after target switch", + ); + const secondAnalysis = await verifyCurrentTarget(client, options); + const documentsAfterTargetSwitch = requireMcpEvidenceResult( + await client.callTool({ name: "list_documents", arguments: {} }, options), + "list_documents", + ); + const secondDocument = await requireCurrentDocument( + client, + options, + documentsAfterTargetSwitch, + requireMcpEvidenceResult, + ); + if (verifiedSecondOverview.document !== secondDocument) + throw new Error( + "binary_overview retained another document after target switch", + ); + const expectedDocumentsAfterTargetSwitch = firstDocuments.length; + if ( + !Array.isArray(documentsAfterTargetSwitch) || + documentsAfterTargetSwitch.length !== expectedDocumentsAfterTargetSwitch + ) + throw new Error( + "Target switching did not replace the previous Hopper document", + ); + const reopenedTarget = await client.callTool( + { name: "open_binary", arguments: { path: targetB } }, + options, + ); + if (reopenedTarget.isError === true) + throw new Error(textValue(reopenedTarget)); + const documentsAfterReopen = requireMcpEvidenceResult( + await client.callTool({ name: "list_documents", arguments: {} }, options), + "list_documents", + ); + if ( + !Array.isArray(documentsAfterReopen) || + documentsAfterReopen.length !== documentsAfterTargetSwitch.length + ) + throw new Error( + "Reopening the same target created a duplicate Hopper document", + ); + const largeInventory = await openAndVerifyLargeFixture({ + client, + options, + normalizedResult: requireMcpEvidenceResult, + path: largeTarget, + expectedCount: fixtureTargets.largeOracle.symbolCount, + symbolPrefix: fixtureTargets.largeOracle.symbolPrefix, + stringPrefix: fixtureTargets.largeOracle.stringPrefix, + }); + + const processList = await commands.exec("ps", ["-ax", "-o", "command="]); + const bundledMcpRunning = processList.stdout + .split("\n") + .some((line) => line.trim().endsWith("/HopperMCPServer")); + if (bundledMcpRunning) { + throw new Error( + "Hopper's bundled MCP server was running during verification", + ); + } + const closed = await client.callTool( + { name: "close_binary", arguments: {} }, + options, + ); + if (closed.isError === true) throw new Error(textValue(closed)); + let unicodeStrings = null; + let terminalInstructions = null; + if (fixtureTargets.unicode !== undefined) { + const call = async (name, args = {}) => + requireMcpOperationResult( + await client.callTool({ name, arguments: args }, options), + name, + ); + await call("open_binary", { path: fixtureTargets.unicode.path }); + terminalInstructions = await verifyHopperTerminalInstructions(call); + unicodeStrings = await verifyHopperStringObjects(call, [ + { value: "REA_UTF16_é_😀", encoding: "utf-16-le" }, + { + value: 'REA_UTF16_ESCAPED_é_"\\line\nend\t\r', + encoding: "utf-16-le", + }, + { value: "\nREA_UTF16_LEADING_é", encoding: "utf-16-le" }, + ]); + await call("close_binary"); + } + const lifecycleAndCli = await verifyHopperLifecycleAndCli( + client, + options, + { + primary: targetA, + secondary: targetB, + ...(fixtureTargets.unicode === undefined + ? {} + : { unicode: fixtureTargets.unicode.path }), + }, + commands, + ); + const closedSession = requireMcpLifecycleResult( + await fullSessionStatus(), + "binary_session", + ); + if (closedSession.open !== false) + throw new Error("The real session remained open after close_binary"); + const diagnosticCount = requireSafeDiagnostics( + stderrChunks, + [ + "procedure_info", + "batch_decompile", + ...boundaryContracts.rejectedOperations, + ], + ["HopperRemoteError", "HopperCancelledError", "AnalysisCancelledError"], + ); + requireBridgeDiagnostic(stderrChunks); + + summary = { + toolCount: actualNames.length, + documentCount: firstDocuments.length, + overview: firstOverview, + segmentCount: firstOverview.segment_count, + analyses: [firstAnalysis, secondAnalysis], + fixtureAnalysis, + unicodeStrings, + terminalInstructions, + boundaryContracts, + lifecycleAndCli, + inventoryCounts, + largeInventory, + fixtureManifest: fixtureTargets.manifestPath, + bundledMcpRunning, + stderrBytes, + diagnosticCount, + bridgeProgressUpdates: progressUpdates.length, + dynamicSession: true, + providerBinding, + targets: [targetA, targetB, largeTarget], + targetHashes: [targetHashA, targetHashB, fixtureTargets.large.sha256], + switched: true, + secondOverview: verifiedSecondOverview, + documentCountAfterTargetSwitch: documentsAfterTargetSwitch.length, + documentCountAfterSameTargetReopen: documentsAfterReopen.length, + }; + } finally { + const keepAlive = setInterval(() => undefined, 100); + try { + await client.close(); + await transport.close(); + } finally { + clearInterval(keepAlive); + unrelatedHopperSurvived = await unrelatedHopper.verifyAndClose(); + } + } + + await new Promise((resolve) => setTimeout(resolve, 500)); + const sessionsAfter = [ + ...( + await snapshotHopperRuntime(runtimeParent, HOPPER_TARGET_LEASE_DIRECTORY) + ).paths, + ] + .filter((path) => !sessionsBefore.has(path)) + .sort(); + if (sessionsAfter.length > 0) { + await writeVerificationReport({ + ...summary, + unrelatedHopperSurvived, + cleanShutdown: false, + retainedRuntimePaths: sessionsAfter, + }); + throw new Error( + `Hopper resources remain or have unresolved ownership: ${sessionsAfter.join(", ")}`, + ); + } + if (summary === undefined) + throw new Error("Real-Hopper verification did not produce a summary"); + if (process.platform === "linux") + summary.ownerExit = await verifyLinuxHopperOwnerExit({ + primary: targetA, + secondary: targetB, + serverCommand, + serverArgs, + environment: serverEnvironment, + commands, + }); +} catch (cause) { + primaryFailure = cause; +} +let cleanupFailure; +try { + await workspaceFixture.closeEarly([() => commands.close()]); +} catch (cause) { + cleanupFailure = cause; +} +if (cleanupFailure !== undefined) + throw retainWorkspace( + `Hopper command cleanup failed; retained diagnostic workspace: ${workspace}`, + { + cause: + primaryFailure === undefined + ? cleanupFailure + : new AggregateError( + [primaryFailure, cleanupFailure], + "Hopper verification and command cleanup both failed", + ), + }, + ); +const completedVerifierRun = await completeVerifierRun(verifierRun); +try { + assertVerifierCleanupComplete(completedVerifierRun.process_lineage); +} catch (cause) { + throw retainWorkspace( + `Cleanup unverified; retained diagnostic workspace: ${workspace}`, + { + cause: + primaryFailure === undefined + ? cause + : new AggregateError( + [primaryFailure, cause], + "Hopper verification failed and cleanup was unverified", + ), + }, ); } -if (summary === undefined) - throw new Error("Real-Hopper verification did not produce a summary"); -if (process.platform === "linux") - summary.ownerExit = await verifyLinuxHopperOwnerExit({ - primary: targetA, - secondary: targetB, - serverCommand, - serverArgs, - environment: serverEnvironment, - }); -const completedVerifierRun = await completeVerifierRun(verifierRun); -if ( - completedVerifierRun.process_lineage.status !== "verified" || - completedVerifierRun.process_lineage.descendants.length !== 0 -) - throw new Error( - "The verifier's process lineage did not prove owned child cleanup", +if (primaryFailure !== undefined) + throw retainWorkspace( + `Real-Hopper verification failed; retained workspace: ${workspace}`, + { cause: primaryFailure }, ); +await workspaceFixture.release(); await writeVerificationReport({ verifier_run: completedVerifierRun, ...summary, diff --git a/scripts/verify-real-kuna.mjs b/scripts/verify-real-kuna.mjs index db1a424c81..17791c8aca 100644 --- a/scripts/verify-real-kuna.mjs +++ b/scripts/verify-real-kuna.mjs @@ -8,12 +8,11 @@ // `--compare-ghidra` runs the same functions through a bring-your-own Ghidra // and reports latency, CPU, memory, output size and simple quality signals. import assert from "node:assert/strict"; -import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; import { mkdir, readFile, readdir, stat, writeFile } from "node:fs/promises"; import { join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; -import { parseArgs, promisify } from "node:util"; +import { parseArgs } from "node:util"; import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { inspectKunaInstallation } from "../dist/kuna/KunaInstallation.js"; @@ -44,9 +43,8 @@ import { completeVerifierRun, createVerifierRun, } from "./lib/verifier-run.mjs"; +import { createVerifierCommands } from "./lib/verify-commands.mjs"; import { VerifierFixture } from "./lib/verifier-fixture.mjs"; - -const exec = promisify(execFile); const { values } = parseArgs({ options: { targets: { type: "string" }, @@ -87,10 +85,21 @@ const resourceScope = { }), run: (operation) => inspectionResources.run(operation), }; +// nm, ps, cc and strip share one owner, created before the workspace. If +// allocation fails, the owner is still closed and no root is left behind. +const commands = await createVerifierCommands(); // Kuna children cache under the root; a plain root admits them on every host. const workspaceFixture = await VerifierFixture.open({ prefix: "rea-kuna-lane-", private: false, +}).catch(async (cause) => { + await commands.close().catch((closeCause) => { + throw new AggregateError( + [cause, closeCause], + "Kuna workspace allocation and command cleanup both failed", + ); + }); + throw cause; }); const workspace = workspaceFixture.path; const cacheRoot = join(workspace, "cache"); @@ -135,7 +144,7 @@ int main(int argc, char **argv) { /** Resolve function entry addresses from a symbol-bearing build. */ const symbolAddresses = async (binary, names) => { - const { stdout } = await exec(process.env.REA_NM ?? "nm", [binary]); + const { stdout } = await commands.exec(process.env.REA_NM ?? "nm", [binary]); const found = new Map(); for (const line of stdout.split("\n")) { const [address, type, symbol] = line.trim().split(/\s+/u); @@ -150,7 +159,11 @@ const symbolAddresses = async (binary, names) => { /** Sum the resident memory and CPU time of a process and its descendants. */ const sampleTree = async (root) => { - const { stdout } = await exec("ps", ["-A", "-o", "pid=,ppid=,rss=,time="]); + const { stdout } = await commands.exec("ps", [ + "-A", + "-o", + "pid=,ppid=,rss=,time=", + ]); const rows = stdout .trim() .split("\n") @@ -778,10 +791,10 @@ try { const source = join(fixtureDir, "fixture.c"); await writeFile(source, FIXTURE_SOURCE); const built = join(fixtureDir, "rea-kuna-fixture"); - await exec(process.env.REA_CC ?? "cc", ["-O2", "-o", built, source]); + await commands.exec(process.env.REA_CC ?? "cc", ["-O2", "-o", built, source]); const stripped = `${built}.stripped`; await writeFile(stripped, await readFile(built)); - await exec(process.env.REA_STRIP ?? "strip", [stripped]); + await commands.exec(process.env.REA_STRIP ?? "strip", [stripped]); const fixtureFunctions = await symbolAddresses(built, [ "rea_checksum", "rea_classify", @@ -829,24 +842,43 @@ try { inspectionResourcesClean = false; failures.push(cause); } +let commandCleanupFailure; +try { + await workspaceFixture.closeEarly([() => commands.close()]); +} catch (cause) { + commandCleanupFailure = cause; + failures.push(cause); +} const lifecycle = await completeVerifierRun(run); let cleanupComplete = pendingServers.size === 0 && retainedNative.size === 0 && - inspectionResourcesClean; + inspectionResourcesClean && + commandCleanupFailure === undefined; try { assertVerifierCleanupComplete(lifecycle.process_lineage); } catch (cause) { cleanupComplete = false; failures.push(cause); } +let workspaceReleased = false; +// A verification failure with successful provider, command, and lineage +// cleanup still releases the root. Only an unconfirmed close retains it. +// release() closes cleanup registration before it removes the root, so a +// failed release already keeps the workspace and retain() would throw +// before this report is written. if (cleanupComplete) try { await workspaceFixture.release(); + workspaceReleased = true; } catch (cause) { cleanupComplete = false; failures.push(cause); } +else + workspaceFixture.retain( + `Kuna cleanup was not confirmed; retained diagnostic workspace: ${workspace}`, + ); const report = { lane: "kuna", status: failures.length === 0 ? "passed" : "failed", @@ -867,7 +899,7 @@ const report = { "Response bytes are UTF-8 serialization sizes. Model token usage, tool schema costs and agent task completion were not measured.", ], cleanup: cleanupComplete ? "complete" : "incomplete", - ...(cleanupComplete ? {} : { retained_workspace: workspace }), + ...(workspaceReleased ? {} : { retained_workspace: workspace }), ...(failures.length === 0 ? {} : { failures: failures.map(describeFailure) }), }; if (values.report !== undefined) diff --git a/scripts/verify/apple/asset-catalog.mjs b/scripts/verify/apple/asset-catalog.mjs index 9912072827..97492e6bb0 100644 --- a/scripts/verify/apple/asset-catalog.mjs +++ b/scripts/verify/apple/asset-catalog.mjs @@ -12,6 +12,7 @@ import { withArtifactMcp, withBundleResourcesMcp, } from "../../lib/artifact-e2e.mjs"; +import { requireMcpToolError } from "../../lib/mcp-verifier-results.mjs"; import { VerifierFixture, withVerifierFixture, @@ -106,7 +107,15 @@ await withVerifierFixture(fixture, async () => { name: "inspect_asset_catalog", arguments: { offset: -1 }, }); - assert.equal(invalid.isError, true); + // appleAssetCatalogInputSchema requires an inclusive offset minimum of 0. + // projectInputIssues reports that bound as out_of_range. + assert.partialDeepStrictEqual(requireMcpToolError(invalid), { + code: "invalid_request", + details: { + operation: "inspect_asset_catalog", + issues: [{ path: ["offset"], reason: "out_of_range", minimum: 0 }], + }, + }); }); const { resource_root, target_sha256, ...resourcePayload } = cli; assert.equal(resource_root.executable_sha256, null); diff --git a/scripts/verify/apple/keyed-archive.mjs b/scripts/verify/apple/keyed-archive.mjs index d963dcb04c..028b92e8e2 100644 --- a/scripts/verify/apple/keyed-archive.mjs +++ b/scripts/verify/apple/keyed-archive.mjs @@ -11,6 +11,7 @@ import { artifactMcpCall, withArtifactMcp, } from "../../lib/artifact-e2e.mjs"; +import { requireMcpToolError } from "../../lib/mcp-verifier-results.mjs"; import { assertKeyedArchiveIntegerResult, verifyKeyedArchiveIntegerArchives, @@ -62,7 +63,15 @@ await withVerifierFixture(fixture, async () => { name: "inspect_keyed_archive", arguments: { path: "../other.plist" }, }); - assert.equal(invalid.isError, true); + // Standalone archives reject a path other than the admitted basename. + // KeyedArchiveReader throws AnalysisInputError; classification is invalid_request. + assert.partialDeepStrictEqual(requireMcpToolError(invalid), { + code: "invalid_request", + details: { + operation: "inspect_keyed_archive", + issues: [{ path: ["path"], reason: "invalid_value" }], + }, + }); }); const xml = join(root, "model.xml.plist"); await promisify(execFile)("/usr/bin/plutil", [ diff --git a/scripts/verify/apple/macos-dylib-e2e.mjs b/scripts/verify/apple/macos-dylib-e2e.mjs index 0ea4a674cc..e9e60a2241 100644 --- a/scripts/verify/apple/macos-dylib-e2e.mjs +++ b/scripts/verify/apple/macos-dylib-e2e.mjs @@ -10,6 +10,7 @@ import { artifactMcpCall, withArtifactMcp, } from "../../lib/artifact-e2e.mjs"; +import { requireMcpToolError } from "../../lib/mcp-verifier-results.mjs"; const exec = promisify(execFile); @@ -185,7 +186,15 @@ export async function verifyDylibResolution(app) { name: "trace_dylib_resolution", arguments: { roots: ["../outside"] }, }); - assert.equal(invalid.isError, true); + // dylibResolutionInputSchema rejects a root that is not a normalized + // relative path. projectInputIssues maps that regex failure to invalid_format. + assert.partialDeepStrictEqual(requireMcpToolError(invalid), { + code: "invalid_request", + details: { + operation: "trace_dylib_resolution", + issues: [{ path: ["roots", 0], reason: "invalid_format" }], + }, + }); // Selecting the observed roots removes discovery uncertainty while retaining // complete dependency-resolution coverage for this known fixture. const roots = [...new Set(trace.roots.map(({ image }) => image))]; diff --git a/tests/conformance/providers/hopper/realHopperFixture.test.ts b/tests/conformance/providers/hopper/realHopperFixture.test.ts index 1bee8774d5..b325246af2 100644 --- a/tests/conformance/providers/hopper/realHopperFixture.test.ts +++ b/tests/conformance/providers/hopper/realHopperFixture.test.ts @@ -24,13 +24,23 @@ describe("real Hopper fixture binding", () => { ); }); - it("binds exact source-owned artifacts and their semantic oracle", async () => { + it("binds a caller-selected local manifest and its exact artifact digests", async () => { const manifest = await fixtureManifest(); await expect(loadRealHopperFixtureTargets(manifest)).resolves.toMatchObject( { - primary: { path: join(root ?? "", "c") }, - secondary: { path: join(root ?? "", "version-v2") }, - large: { path: join(root ?? "", "large") }, + manifestPath: manifest, + primary: { + path: join(root ?? "", "c"), + sha256: digest(Buffer.from("primary")), + }, + secondary: { + path: join(root ?? "", "version-v2"), + sha256: digest(Buffer.from("secondary")), + }, + large: { + path: join(root ?? "", "large"), + sha256: digest(Buffer.from("large")), + }, largeOracle: { symbolPrefix: "_rea_inventory_", symbolCount: 1_205,