diff --git a/scripts/lib/real-hopper-boundaries.mjs b/scripts/lib/real-hopper-boundaries.mjs index 47c22e499f..55c3f6c964 100644 --- a/scripts/lib/real-hopper-boundaries.mjs +++ b/scripts/lib/real-hopper-boundaries.mjs @@ -1,11 +1,9 @@ import { canonicalOutputSchema } from "./canonical-output-schema.mjs"; import assert from "node:assert/strict"; -import { execFile } from "node:child_process"; import { copyFile, mkdtemp, readFile, realpath, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { basename, join } from "node:path"; import { fileURLToPath } from "node:url"; -import { promisify } from "node:util"; import Ajv from "ajv/dist/2020.js"; import addFormats from "ajv-formats"; import { @@ -403,7 +401,16 @@ export async function verifyHopperBoundaryContracts( } /** Prove actual document disposal and shared CLI/MCP behavior on disposable copies. */ -export async function verifyHopperLifecycleAndCli(client, options, targets) { +export async function verifyHopperLifecycleAndCli( + client, + options, + targets, + commands, +) { + if (commands === undefined) + throw new Error( + "Hopper boundary verification requires the supervised command owner", + ); const directory = await realpath( await mkdtemp(join(tmpdir(), "hopper-boundary-")), ); @@ -415,15 +422,22 @@ export async function verifyHopperLifecycleAndCli(client, options, targets) { await client.callTool({ name, arguments: args }, options), name, ); - const cli = promisify(execFile); - const runCli = (args) => { - const pending = cli(process.execPath, args, { - timeout: 180_000, - maxBuffer: 1_048_576, - }); - if (pending.child.pid !== undefined) - targets.ownedProcessIds.add(pending.child.pid); - return pending; + // Non-zero CLI exits keep the execFile-shaped code and stdout the refusal + // assertion reads. The process itself belongs to the command owner. + const runCli = async (args) => { + try { + return await commands.exec(process.execPath, args, { + timeout: 180_000, + maxBuffer: 1_048_576, + }); + } catch (error) { + if (error?.name !== "VerifierCommandExitError") throw error; + const failed = new Error(error.message, { cause: error }); + failed.code = error.code; + failed.stdout = error.snapshot.stdout.text; + failed.stderr = error.snapshot.stderr.text; + throw failed; + } }; try { await copyFile(targets.primary, primary); diff --git a/scripts/lib/real-hopper-owner-exit.mjs b/scripts/lib/real-hopper-owner-exit.mjs index 9cdf0d3cf8..6f6784e7be 100644 --- a/scripts/lib/real-hopper-owner-exit.mjs +++ b/scripts/lib/real-hopper-owner-exit.mjs @@ -1,7 +1,5 @@ import assert from "node:assert/strict"; -import { execFile } from "node:child_process"; import { setTimeout as delay } from "node:timers/promises"; -import { promisify } from "node:util"; import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; @@ -19,8 +17,6 @@ import { requireMcpLifecycleResult, } from "./mcp-verifier-results.mjs"; -const execFileAsync = promisify(execFile); - /** Prove actual MCP owner exit releases the demo singleton for subsequent CLI/MCP sessions. */ export async function verifyLinuxHopperOwnerExit({ primary, @@ -28,7 +24,12 @@ export async function verifyLinuxHopperOwnerExit({ serverCommand, serverArgs, environment, + commands, }) { + if (commands === undefined) + throw new Error( + "Linux Hopper owner-exit verification requires the supervised command owner", + ); const sentinel = await startUnrelatedHopperSentinel(); const outcomes = []; try { @@ -132,7 +133,7 @@ export async function verifyLinuxHopperOwnerExit({ assert.equal(cleanup.cleaned, true, JSON.stringify(cleanup)); // The SIGKILL case must recover the stale lease for both target identities. for (const target of [primary, secondary]) { - const result = await execFileAsync( + const result = await commands.exec( process.execPath, [ "scripts/rea.mjs", @@ -143,7 +144,11 @@ export async function verifyLinuxHopperOwnerExit({ "--format", "json", ], - { env: environment, timeout: 180_000, maxBuffer: 16 * 1024 * 1024 }, + { + env: environment, + timeout: 180_000, + maxBuffer: 16 * 1024 * 1024, + }, ); assert.ok(JSON.parse(result.stdout).normalized_result); } diff --git a/scripts/verify-real-hopper-callers.mjs b/scripts/verify-real-hopper-callers.mjs index bb6a238198..7a46aeec52 100644 --- a/scripts/verify-real-hopper-callers.mjs +++ b/scripts/verify-real-hopper-callers.mjs @@ -1,22 +1,43 @@ // Run from the repository root after `npm run build:cached` on macOS arm64, // with Hopper and the Xcode command-line tools installed. No target is executed. import assert from "node:assert/strict"; -import { execFile } from "node:child_process"; -import { createHash } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import { constants } from "node:fs"; import { access, readFile, writeFile } from "node:fs/promises"; import { createRequire } from "node:module"; import { join } from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; -import { promisify } from "node:util"; +import { runOwnedCommand } from "../dist/process/OwnedCommand.js"; import { artifactMcpCall } from "./lib/artifact-e2e.mjs"; import { snapshotHopperRuntime } from "./lib/real-hopper-cleanup.mjs"; import { closeVerifierResources } from "./lib/verifier-cleanup.mjs"; -import { HOPPER_TARGET_LEASE_DIRECTORY } from "../dist/process/HopperLease.js"; +import { createVerifierCommands } from "./lib/verify-commands.mjs"; import { - VerifierFixture, - withVerifierFixture, -} from "./lib/verifier-fixture.mjs"; + assertVerifierCleanupComplete, + completeVerifierRun, + createVerifierRun, +} from "./lib/verifier-run.mjs"; +import { HOPPER_TARGET_LEASE_DIRECTORY } from "../dist/process/HopperLease.js"; +import { VerifierFixture } from "./lib/verifier-fixture.mjs"; + +const verifierRun = createVerifierRun(); +// xcrun resolves the compiler before a workspace exists. A failed probe must +// not allocate a root, so it uses the same short-lived owner as requireLaneToolchain. +const probeOwnedCommand = async (command, arguments_) => { + const snapshot = await runOwnedCommand( + { + command, + arguments: arguments_, + cwd: process.cwd(), + env: process.env, + runId: `rea-hopper-callers-probe-${randomUUID()}`, + expectedCommand: null, + sweepTokenOwnedProcesses: true, + }, + { timeoutMs: null, diagnosticBytes: 1024 * 1024 }, + ); + return snapshot.stdout.text; +}; assert.equal(process.platform, "darwin", "requires macOS"); assert.equal( @@ -48,18 +69,32 @@ await access( ), constants.X_OK, ); -const run = promisify(execFile); -const clang = (await run("xcrun", ["--find", "clang"])).stdout.trim(); +const clang = (await probeOwnedCommand("xcrun", ["--find", "clang"])).trim(); const sdk = ( - await run("xcrun", ["--sdk", "macosx", "--show-sdk-path"]) -).stdout.trim(); + await probeOwnedCommand("xcrun", ["--sdk", "macosx", "--show-sdk-path"]) +).trim(); +// The command owner is created before the workspace. If allocation fails, +// the owner is still closed and no root is left behind. +const commands = await createVerifierCommands(); // A plain root under /tmp, where this verifier has always built its fixtures. const workspace = await VerifierFixture.open({ prefix: "rea-hopper-callers-verifier-", parent: "/tmp", private: false, +}).catch(async (cause) => { + await commands.close().catch((closeCause) => { + throw new AggregateError( + [cause, closeCause], + "Hopper caller workspace allocation and command cleanup both failed", + ); + }); + throw cause; }); const directory = workspace.path; +const retainWorkspace = (message, options) => { + workspace.retain(message); + return new Error(message, options); +}; const fixture = join(directory, "tail-call"); const large = join(directory, "long-scan"); const source = fileURLToPath( @@ -112,36 +147,9 @@ const matchingRef = (ref, site, from, to) => ref.source_procedure?.address === from && ref.target_procedure?.address === to && ref.call?.classification === "direct"; -// The root is removed only after these cleanups confirm. -workspace.releaseAfter(async () => { - try { - await closeVerifierResources([ - () => client.close(), - () => transport.close(), - async () => { - await new Promise((resolve) => setTimeout(resolve, 500)); - const leaked = [ - ...( - await snapshotHopperRuntime("/tmp", HOPPER_TARGET_LEASE_DIRECTORY) - ).paths, - ].filter((path) => !sessionsBefore.has(path)); - assert.deepEqual( - leaked, - [], - `Hopper resources remain: ${leaked.join(", ")}`, - ); - }, - ]); - } catch (cause) { - throw new Error(`Hopper cleanup failed; fixture retained at ${directory}`, { - cause, - }); - } finally { - if (stderr) process.stderr.write(stderr); - } -}); -await withVerifierFixture(workspace, async () => { - await run(clang, [ +let primaryFailure; +try { + await commands.exec(clang, [ "-isysroot", sdk, "-arch", @@ -161,7 +169,7 @@ await withVerifierFixture(workspace, async () => { lines.push("int main(void) { return rea_progress_0000(); }"); const largeSource = join(directory, "long-scan.c"); await writeFile(largeSource, lines.join("\n")); - await run(clang, [ + await commands.exec(clang, [ "-isysroot", sdk, "-arch", @@ -333,4 +341,94 @@ await withVerifierFixture(workspace, async () => { sha256: beforeLarge, }), ); -}); +} catch (cause) { + primaryFailure = cause; +} +let hopperCleanupFailure; +try { + await workspace.closeEarly([ + async () => { + try { + await closeVerifierResources([ + () => client.close(), + () => transport.close(), + async () => { + await new Promise((resolve) => setTimeout(resolve, 500)); + const leaked = [ + ...( + await snapshotHopperRuntime( + "/tmp", + HOPPER_TARGET_LEASE_DIRECTORY, + ) + ).paths, + ].filter((path) => !sessionsBefore.has(path)); + assert.deepEqual( + leaked, + [], + `Hopper resources remain: ${leaked.join(", ")}`, + ); + }, + ]); + } catch (cause) { + throw new Error( + `Hopper cleanup failed; fixture retained at ${directory}`, + { cause }, + ); + } finally { + if (stderr) process.stderr.write(stderr); + } + }, + ]); +} catch (cause) { + hopperCleanupFailure = cause; +} +if (hopperCleanupFailure !== undefined) + primaryFailure = + primaryFailure === undefined + ? hopperCleanupFailure + : new AggregateError( + [primaryFailure, hopperCleanupFailure], + "Hopper caller verification and Hopper cleanup both failed", + ); +let commandCleanupFailure; +try { + await workspace.closeEarly([() => commands.close()]); +} catch (cause) { + commandCleanupFailure = cause; +} +if (commandCleanupFailure !== undefined) + throw retainWorkspace( + `Hopper caller command cleanup failed; retained diagnostic workspace: ${directory}`, + { + cause: + primaryFailure === undefined + ? commandCleanupFailure + : new AggregateError( + [primaryFailure, commandCleanupFailure], + "Hopper caller verification and command cleanup both failed", + ), + }, + ); +const completedVerifierRun = await completeVerifierRun(verifierRun); +try { + assertVerifierCleanupComplete(completedVerifierRun.process_lineage); +} catch (cause) { + throw retainWorkspace( + `Cleanup unverified; retained diagnostic workspace: ${directory}`, + { + cause: + primaryFailure === undefined + ? cause + : new AggregateError( + [primaryFailure, cause], + "Hopper caller verification failed and cleanup was unverified", + ), + }, + ); +} +if (primaryFailure !== undefined) + throw retainWorkspace( + `Hopper caller verification failed; retained workspace: ${directory}`, + { cause: primaryFailure }, + ); +await workspace.release(); diff --git a/scripts/verify-real-hopper-fat.mjs b/scripts/verify-real-hopper-fat.mjs index 9ca3d07c1d..ab749376af 100644 --- a/scripts/verify-real-hopper-fat.mjs +++ b/scripts/verify-real-hopper-fat.mjs @@ -1,20 +1,25 @@ import { canonicalOutputSchema } from "./lib/canonical-output-schema.mjs"; import assert from "node:assert/strict"; -import { createHash } from "node:crypto"; -import { execFile } from "node:child_process"; +import { createHash, randomUUID } from "node:crypto"; import { access, readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { promisify } from "node:util"; import { fileURLToPath } from "node:url"; import { constants } from "node:fs"; import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import Ajv from "ajv/dist/2020.js"; import addFormats from "ajv-formats"; +import { runOwnedCommand } from "../dist/process/OwnedCommand.js"; import { - mcpTextValue, requireMcpOperationResult, + requireMcpToolError, } from "./lib/mcp-verifier-results.mjs"; +import { createVerifierCommands } from "./lib/verify-commands.mjs"; +import { + assertVerifierCleanupComplete, + completeVerifierRun, + createVerifierRun, +} from "./lib/verifier-run.mjs"; import { snapshotHopperRuntime } from "./lib/real-hopper-cleanup.mjs"; import { closeVerifierResources } from "./lib/verifier-cleanup.mjs"; import { HOPPER_TARGET_LEASE_DIRECTORY } from "../dist/process/HopperLease.js"; @@ -24,12 +29,26 @@ import { verifyHopperSourceBinding, relocateSingleFatSlice, } from "./lib/real-hopper-source-binding.mjs"; -import { - VerifierFixture, - withVerifierFixture, -} from "./lib/verifier-fixture.mjs"; +import { VerifierFixture } from "./lib/verifier-fixture.mjs"; -const run = promisify(execFile); +const verifierRun = createVerifierRun(); +// xcrun resolves clang and lipo before a workspace exists. A failed probe must +// not allocate a root, so it uses the same short-lived owner as requireLaneToolchain. +const probeOwnedCommand = async (command, arguments_) => { + const snapshot = await runOwnedCommand( + { + command, + arguments: arguments_, + cwd: process.cwd(), + env: process.env, + runId: `rea-hopper-fat-probe-${randomUUID()}`, + expectedCommand: null, + sweepTokenOwnedProcesses: true, + }, + { timeoutMs: null, diagnosticBytes: 1024 * 1024 }, + ); + return snapshot.stdout.text; +}; const verifyFat64 = process.argv[2] === "--fat64"; assert.deepEqual(process.argv.slice(2), verifyFat64 ? ["--fat64"] : []); if (process.platform !== "darwin" || !["arm64", "x64"].includes(process.arch)) @@ -52,24 +71,44 @@ await access(hopperLauncherPath, constants.X_OK).catch((cause) => { const toolPaths = {}; for (const command of ["clang", "lipo"]) toolPaths[command] = ( - await run("xcrun", ["--find", command]).catch((cause) => { + await probeOwnedCommand("xcrun", ["--find", command]).catch((cause) => { throw new Error(`Hopper FAT verification requires ${command}`, { cause }); }) - ).stdout.trim(); + ).trim(); const sdk = ( - await run("xcrun", ["--sdk", "macosx", "--show-sdk-path"]).catch((cause) => { + await probeOwnedCommand("xcrun", [ + "--sdk", + "macosx", + "--show-sdk-path", + ]).catch((cause) => { throw new Error( "Hopper FAT verification requires the installed macOS SDK", { cause }, ); }) -).stdout.trim(); +).trim(); +// The command owner is created before the workspace. If allocation fails, +// the owner is still closed and no root is left behind. +const commands = await createVerifierCommands(); const workspace = await VerifierFixture.open({ prefix: "hopper-fat-", private: false, +}).catch(async (cause) => { + await commands.close().catch((closeCause) => { + throw new AggregateError( + [cause, closeCause], + "Hopper FAT workspace allocation and command cleanup both failed", + ); + }); + throw cause; }); const directory = workspace.path; +const retainWorkspace = (message, options) => { + workspace.retain(message); + return new Error(message, options); +}; +const hostFatArchitecture = process.arch === "arm64" ? "arm64" : "x86_64"; const source = fileURLToPath( new URL("../tests/conformance/c/fixture.c", import.meta.url), ); @@ -100,33 +139,9 @@ const ajv = new Ajv({ strict: false }); addFormats(ajv); const observations = []; const sourceBinding = []; -// The root is removed only after these cleanups confirm. -workspace.releaseAfter(async () => { - try { - await closeVerifierResources([ - () => client.close(), - () => transport.close(), - ]); - } catch (cause) { - throw new Error(`Hopper cleanup failed; fixture retained at ${directory}`, { - cause, - }); - } - try { - const retained = [ - ...(await snapshotHopperRuntime("/tmp", HOPPER_TARGET_LEASE_DIRECTORY)) - .paths, - ].filter((path) => !before.has(path)); - assert.deepEqual(retained, [], "Hopper resources remain or are unresolved"); - } catch (cause) { - throw new Error( - `Hopper resource verification failed; fixture retained at ${directory}`, - { cause }, - ); - } -}); -await withVerifierFixture(workspace, async () => { - await run(toolPaths.clang, [ +let primaryFailure; +try { + await commands.exec(toolPaths.clang, [ "-isysroot", sdk, "-arch", @@ -137,7 +152,7 @@ await withVerifierFixture(workspace, async () => { "-o", arm, ]); - await run(toolPaths.clang, [ + await commands.exec(toolPaths.clang, [ "-isysroot", sdk, "-arch", @@ -148,9 +163,9 @@ await withVerifierFixture(workspace, async () => { "-o", intel, ]); - await run(toolPaths.lipo, ["-create", arm, intel, "-output", fat]); + await commands.exec(toolPaths.lipo, ["-create", arm, intel, "-output", fat]); if (verifyFat64) - await run(toolPaths.lipo, [ + await commands.exec(toolPaths.lipo, [ "-create", arm, intel, @@ -158,7 +173,7 @@ await withVerifierFixture(workspace, async () => { "-output", fat64, ]); - await run(toolPaths.lipo, [ + await commands.exec(toolPaths.lipo, [ "-create", process.arch === "arm64" ? arm : intel, "-output", @@ -186,7 +201,7 @@ await withVerifierFixture(workspace, async () => { await writeFile(little, swapped); extraFat64.push(little); const single = join(directory, "c-single-fat64"); - await run(toolPaths.lipo, [ + await commands.exec(toolPaths.lipo, [ "-create", process.arch === "arm64" ? arm : intel, "-fat64", @@ -194,25 +209,34 @@ await withVerifierFixture(workspace, async () => { single, ]); extraFat64.push(single); - for (const [name, reason, mutate] of [ + for (const [name, refusal, mutate] of [ [ "reserved", - /nonzero reserved field/u, + { + reason: "FAT64 architecture 0 has a nonzero reserved field", + }, (bytes) => bytes.writeUInt32BE(1, 36), ], [ "range", - /extends beyond the file/u, + { + reason: "FAT architecture 0 slice range extends beyond the file", + constraint: "file_format_rejected", + }, (bytes) => bytes.writeBigUInt64BE(BigInt(bytes.length), 24), ], [ "identity", - /disagrees with its Mach-O header/u, + { + reason: + "FAT architecture 0 CPU type/subtype disagrees with its Mach-O header", + constraint: "file_format_rejected", + }, (bytes) => bytes.writeUInt32BE(bytes.readUInt32BE(12) + 1, 12), ], [ "overlap", - /slices overlap/u, + { reason: "FAT slices overlap" }, (bytes) => { const firstOffset = bytes.readBigUInt64BE(16); const secondOffset = bytes.readBigUInt64BE(48); @@ -232,7 +256,7 @@ await withVerifierFixture(workspace, async () => { mutate(bytes); const path = join(directory, `c-fat64-invalid-${name}`); await writeFile(path, bytes); - invalidFat64.push({ path, reason }); + invalidFat64.push({ path, refusal }); } const hostBytes = await readFile(process.arch === "arm64" ? arm : intel); const secondOffset = Math.ceil((16384 + hostBytes.length) / 16384) * 16384; @@ -252,8 +276,10 @@ await withVerifierFixture(workspace, async () => { await writeFile(path, ambiguous); invalidFat64.push({ path, - reason: - /FAT binary declares several (?:arm64|x86_64) slices; no slice was selected/u, + refusal: { + reason: `FAT binary declares several ${hostFatArchitecture} slices; no slice was selected`, + constraint: "file_format_rejected", + }, }); } await client.connect(transport); @@ -391,13 +417,28 @@ await withVerifierFixture(workspace, async () => { }); await call("close_binary"); } - for (const { path, reason } of invalidFat64) { + for (const { path, refusal } of invalidFat64) { const reply = await client.callTool({ name: "open_binary", arguments: { path }, }); - assert.equal(reply.isError, true, "malformed FAT64 unexpectedly launched"); - assert.match(mcpTextValue(reply), reason); + // Admission failures are BinaryTargetError file_format_rejected. + // Reserved and overlap survive admission and fail in resolveHopperMachOImage, + // which returns BinaryTargetError with no constraint. Both classify as + // target_unavailable and keep the producer reason on details.reason. + const refused = requireMcpToolError(reply); + assert.partialDeepStrictEqual(refused, { + code: "target_unavailable", + details: { + path, + reason: refusal.reason, + ...(refusal.constraint === undefined + ? {} + : { constraint: refusal.constraint }), + }, + }); + if (refusal.constraint === undefined) + assert.equal(refused.details?.constraint, undefined); } if (verifyFat64) { await call("open_binary", { path: fat64 }); @@ -441,7 +482,7 @@ await withVerifierFixture(workspace, async () => { await assert.rejects(access(preparedImagePath), { code: "ENOENT" }); } for (const [path, expected] of cliMappings) { - const pending = run( + const cliResult = await commands.exec( process.execPath, [ "scripts/rea.mjs", @@ -458,11 +499,99 @@ await withVerifierFixture(workspace, async () => { env: { ...process.env, HOPPER_LOADER_ARGS_JSON: "[]" }, }, ); - assert.ok(pending.child.pid !== undefined); - const cli = JSON.parse((await pending).stdout); + const cli = JSON.parse(cliResult.stdout); assert.deepEqual(cli.normalized_result, expected.mapping); } -}); +} catch (cause) { + primaryFailure = cause; +} +let hopperCleanupFailure; +try { + await workspace.closeEarly([ + async () => { + try { + await closeVerifierResources([ + () => client.close(), + () => transport.close(), + ]); + } catch (cause) { + throw new Error( + `Hopper cleanup failed; fixture retained at ${directory}`, + { cause }, + ); + } + try { + const retained = [ + ...( + await snapshotHopperRuntime("/tmp", HOPPER_TARGET_LEASE_DIRECTORY) + ).paths, + ].filter((path) => !before.has(path)); + assert.deepEqual( + retained, + [], + "Hopper resources remain or are unresolved", + ); + } catch (cause) { + throw new Error( + `Hopper resource verification failed; fixture retained at ${directory}`, + { cause }, + ); + } + }, + ]); +} catch (cause) { + hopperCleanupFailure = cause; +} +if (hopperCleanupFailure !== undefined) + primaryFailure = + primaryFailure === undefined + ? hopperCleanupFailure + : new AggregateError( + [primaryFailure, hopperCleanupFailure], + "Hopper FAT verification and Hopper cleanup both failed", + ); +let commandCleanupFailure; +try { + await workspace.closeEarly([() => commands.close()]); +} catch (cause) { + commandCleanupFailure = cause; +} +if (commandCleanupFailure !== undefined) + throw retainWorkspace( + `Hopper FAT command cleanup failed; retained diagnostic workspace: ${directory}`, + { + cause: + primaryFailure === undefined + ? commandCleanupFailure + : new AggregateError( + [primaryFailure, commandCleanupFailure], + "Hopper FAT verification and command cleanup both failed", + ), + }, + ); +const completedVerifierRun = await completeVerifierRun(verifierRun); +try { + assertVerifierCleanupComplete(completedVerifierRun.process_lineage); +} catch (cause) { + throw retainWorkspace( + `Cleanup unverified; retained diagnostic workspace: ${directory}`, + { + cause: + primaryFailure === undefined + ? cause + : new AggregateError( + [primaryFailure, cause], + "Hopper FAT verification failed and cleanup was unverified", + ), + }, + ); +} +if (primaryFailure !== undefined) + throw retainWorkspace( + `Hopper FAT verification failed; retained workspace: ${directory}`, + { cause: primaryFailure }, + ); +await workspace.release(); console.log( JSON.stringify( { diff --git a/scripts/verify-real-hopper-linux.mjs b/scripts/verify-real-hopper-linux.mjs index 594391686a..045791eb87 100644 --- a/scripts/verify-real-hopper-linux.mjs +++ b/scripts/verify-real-hopper-linux.mjs @@ -1,15 +1,14 @@ +import { randomUUID } from "node:crypto"; import { access, symlink } from "node:fs/promises"; -import { execFile } from "node:child_process"; import { join } from "node:path"; -import { promisify } from "node:util"; +import { runOwnedCommand } from "../dist/process/OwnedCommand.js"; import { createVerifierRun } from "./lib/verifier-run.mjs"; import { VerifierFixture, withVerifierFixture, } from "./lib/verifier-fixture.mjs"; -const execFileAsync = promisify(execFile); createVerifierRun(); if (process.platform !== "linux") @@ -20,8 +19,21 @@ await access(launcher); await access("/usr/bin/Xvfb"); await access("/usr/bin/xauth"); await access("/usr/bin/python3"); -const linked = await execFileAsync("ldd", [launcher]); -if (`${linked.stdout}\n${linked.stderr}`.includes("not found")) +// ldd runs before a workspace exists, so it uses the same short-lived owner +// as requireLaneToolchain rather than a fixture-retained command owner. +const linked = await runOwnedCommand( + { + command: "ldd", + arguments: [launcher], + cwd: process.cwd(), + env: process.env, + runId: `rea-hopper-linux-ldd-${randomUUID()}`, + expectedCommand: null, + sweepTokenOwnedProcesses: true, + }, + { timeoutMs: null, diagnosticBytes: 1024 * 1024 }, +); +if (`${linked.stdout.text}\n${linked.stderr.text}`.includes("not found")) throw new Error("Hopper has unresolved shared-library dependencies"); const alternateDirectory = await VerifierFixture.open({ prefix: "rea-real-hopper-launcher-", diff --git a/scripts/verify-real-hopper.mjs b/scripts/verify-real-hopper.mjs index ca183ffd0b..0b19508437 100644 --- a/scripts/verify-real-hopper.mjs +++ b/scripts/verify-real-hopper.mjs @@ -1,6 +1,4 @@ -import { execFile } from "node:child_process"; import { tmpdir } from "node:os"; -import { promisify } from "node:util"; import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; @@ -38,7 +36,13 @@ import { verifyHopperFunctionBasics, } from "./lib/real-hopper-function-basics.mjs"; import { openAndVerifyLargeFixture } from "./lib/real-hopper-exhaustive-search.mjs"; -import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs"; +import { createVerifierCommands } from "./lib/verify-commands.mjs"; +import { + assertVerifierCleanupComplete, + completeVerifierRun, + createVerifierRun, +} from "./lib/verifier-run.mjs"; +import { VerifierFixture } from "./lib/verifier-fixture.mjs"; import { requireHopperSelection } from "./lib/real-hopper-selection.mjs"; import { verifyHopperInventories } from "./lib/real-hopper-inventory.mjs"; import { @@ -48,548 +52,625 @@ import { import { startUnrelatedHopperSentinel } from "./lib/unrelated-hopper-sentinel.mjs"; import { snapshotHopperRuntime } from "./lib/real-hopper-cleanup.mjs"; import { verifyLinuxHopperOwnerExit } from "./lib/real-hopper-owner-exit.mjs"; -const execFileAsync = promisify(execFile); const verifierRun = createVerifierRun(); -const timeout = 180_000; -const parseServerArgs = (encoded) => { - const parsed = JSON.parse(encoded); - if ( - !Array.isArray(parsed) || - parsed.some((value) => typeof value !== "string") - ) - throw new Error("REA_VERIFY_SERVER_ARGS_JSON must encode string arguments"); - return parsed; +// ps and any later host commands share one owner, created before a tool or +// workspace exists. A failed workspace allocation still closes the owner. +const commands = await createVerifierCommands(); +const workspaceFixture = await VerifierFixture.open({ + prefix: "rea-real-hopper-", +}).catch(async (cause) => { + await commands.close().catch((closeCause) => { + throw new AggregateError( + [cause, closeCause], + "Hopper workspace allocation and command cleanup both failed", + ); + }); + throw cause; +}); +const workspace = workspaceFixture.path; +const retainWorkspace = (message, options) => { + workspaceFixture.retain(message); + return new Error(message, options); }; +let primaryFailure; +let unrelatedHopperSurvived = false; +let summary; +try { + const timeout = 180_000; + const parseServerArgs = (encoded) => { + const parsed = JSON.parse(encoded); + if ( + !Array.isArray(parsed) || + parsed.some((value) => typeof value !== "string") + ) + throw new Error( + "REA_VERIFY_SERVER_ARGS_JSON must encode string arguments", + ); + return parsed; + }; -const runtimeParent = process.platform === "darwin" ? "/tmp" : tmpdir(); -const sessionsBefore = ( - await snapshotHopperRuntime(runtimeParent, HOPPER_TARGET_LEASE_DIRECTORY) -).paths; -const ownedProcessIds = new Set(); -const textValue = mcpTextValue; + const runtimeParent = process.platform === "darwin" ? "/tmp" : tmpdir(); + const sessionsBefore = ( + await snapshotHopperRuntime(runtimeParent, HOPPER_TARGET_LEASE_DIRECTORY) + ).paths; + const textValue = mcpTextValue; -const requireOverview = (value, operation) => { - if ( - value === null || - typeof value !== "object" || - typeof value.document !== "string" || - value.document.length === 0 || - !Number.isInteger(value.procedure_count) || - value.procedure_count < 1 || - !Number.isInteger(value.segment_count) || - value.segment_count < 1 || - !Array.isArray(value.segments) - ) { - throw new Error(`${operation} returned no analyzed binary overview`); - } - return value; -}; + const requireOverview = (value, operation) => { + if ( + value === null || + typeof value !== "object" || + typeof value.document !== "string" || + value.document.length === 0 || + !Number.isInteger(value.procedure_count) || + value.procedure_count < 1 || + !Number.isInteger(value.segment_count) || + value.segment_count < 1 || + !Array.isArray(value.segments) + ) { + throw new Error(`${operation} returned no analyzed binary overview`); + } + return value; + }; -const verifyRelationships = async (client, options, procedure) => { - const related = {}; - for (const operation of ["procedure_callers", "procedure_callees"]) { - related[operation] = requireAddressArray( - requireMcpEvidenceResult( - await client.callTool( - { name: operation, arguments: { procedure } }, - options, + const verifyRelationships = async (client, options, procedure) => { + const related = {}; + for (const operation of ["procedure_callers", "procedure_callees"]) { + related[operation] = requireAddressArray( + requireMcpEvidenceResult( + await client.callTool( + { name: operation, arguments: { procedure } }, + options, + ), + operation, ), operation, - ), - operation, - ); - } - related.xrefs = requireAddressArray( - requireMcpEvidenceResult( - await client.callTool( - { name: "xrefs", arguments: { address: procedure } }, - options, + ); + } + related.xrefs = requireAddressArray( + requireMcpEvidenceResult( + await client.callTool( + { name: "xrefs", arguments: { address: procedure } }, + options, + ), + "xrefs", ), "xrefs", - ), - "xrefs", - ); - return related; -}; - -const verifyByteAccess = async (client, options, firstAddress) => { - const byteRead = requireMcpEvidenceResult( - await client.callTool( - { name: "read_bytes", arguments: { address: firstAddress, length: 16 } }, - options, - ), - "read_bytes", - ); - if ( - byteRead?.address !== firstAddress || - byteRead.requested_bytes !== 16 || - byteRead.returned_bytes !== 16 || - byteRead.complete !== true || - !/^[a-f0-9]{32}$/u.test(byteRead.bytes_hex) - ) { - throw new Error("read_bytes returned an invalid bounded byte window"); - } - const mappedOffset = requireMcpEvidenceResult( - await client.callTool( - { - name: "address_to_file_offset", - arguments: { address: firstAddress }, - }, - options, - ), - "address_to_file_offset", - ); - if ( - mappedOffset?.address !== firstAddress || - !Number.isSafeInteger(mappedOffset.file_offset) || - mappedOffset.file_offset < 0 - ) { - throw new Error("address_to_file_offset returned an invalid mapping"); - } - return { - byteReadCount: byteRead.returned_bytes, - mappedFileOffset: mappedOffset.file_offset, + ); + return related; }; -}; -const verifyCurrentTarget = async (client, options) => { - const procedures = requireMcpEvidenceResult( - await client.callTool({ name: "list_procedures", arguments: {} }, options), - "list_procedures", - ); - const firstAddress = firstProcedureAddress(procedures); - const byteAccess = await verifyByteAccess(client, options, firstAddress); - const basics = await verifyHopperFunctionBasics( - client, - options, - firstAddress, - ); - const relationships = await verifyRelationships( - client, - options, - firstAddress, - ); - const boundedResult = requireMcpEvidenceResult( - await client.callTool( - { - name: "batch_decompile", - arguments: { addresses: [firstAddress] }, - }, - options, - ), - "batch_decompile", - ); - if ( - boundedResult === null || - typeof boundedResult !== "object" || - Array.isArray(boundedResult) || - boundedResult.total !== 1 || - boundedResult.succeeded !== 1 || - boundedResult.failed !== 0 || - !Array.isArray(boundedResult.items) || - boundedResult.items.length !== 1 || - boundedResult.items[0]?.address !== firstAddress || - boundedResult.items[0]?.status !== "ok" - ) { - throw new Error("batch_decompile returned an invalid result"); - } - const boundedPseudocode = requirePseudocode( - boundedResult.items[0].pseudocode, - "batch_decompile", - ); - const dossier = requireFunctionDossier( - requireMcpEvidenceResult( + const verifyByteAccess = async (client, options, firstAddress) => { + const byteRead = requireMcpEvidenceResult( await client.callTool( { - name: "analyze_function", - arguments: { procedure: firstAddress, detail: "complete" }, + name: "read_bytes", + arguments: { address: firstAddress, length: 16 }, }, options, ), - "analyze_function", - ), - firstAddress, - ); - return { - procedure: dossier.procedure, - procedureCount: procedures.length, - boundedPseudocodeChars: boundedPseudocode.length, - analyzedPseudocodeChars: [...dossier.pseudocode].length, - callerCount: relationships.procedure_callers.length, - calleeCount: relationships.procedure_callees.length, - xrefCount: relationships.xrefs.length, - outgoingReferenceCount: basics.outgoingReferenceCount, - instructionWindowCount: basics.instructionWindowCount, - ...byteAccess, + "read_bytes", + ); + if ( + byteRead?.address !== firstAddress || + byteRead.requested_bytes !== 16 || + byteRead.returned_bytes !== 16 || + byteRead.complete !== true || + !/^[a-f0-9]{32}$/u.test(byteRead.bytes_hex) + ) { + throw new Error("read_bytes returned an invalid bounded byte window"); + } + const mappedOffset = requireMcpEvidenceResult( + await client.callTool( + { + name: "address_to_file_offset", + arguments: { address: firstAddress }, + }, + options, + ), + "address_to_file_offset", + ); + if ( + mappedOffset?.address !== firstAddress || + !Number.isSafeInteger(mappedOffset.file_offset) || + mappedOffset.file_offset < 0 + ) { + throw new Error("address_to_file_offset returned an invalid mapping"); + } + return { + byteReadCount: byteRead.returned_bytes, + mappedFileOffset: mappedOffset.file_offset, + }; }; -}; -const fixtureManifestPath = - process.env.REA_HOPPER_CONFORMANCE_MANIFEST_PATH ?? - "build/conformance/manifest.json"; -const fixtureTargets = await loadRealHopperFixtureTargets(fixtureManifestPath); -const targetA = fixtureTargets.primary.path; -const targetB = fixtureTargets.secondary.path; -const largeTarget = fixtureTargets.large.path; -const targetHashA = fixtureTargets.primary.sha256; -const targetHashB = fixtureTargets.secondary.sha256; -const serverEnvironment = { ...process.env }; -serverEnvironment.REA_ANALYSIS_PROVIDER = "auto"; -delete serverEnvironment.HOPPER_TARGET_PATH; -delete serverEnvironment.HOPPER_SECOND_TARGET_PATH; -delete serverEnvironment.REA_VERIFY_SERVER_COMMAND; -delete serverEnvironment.REA_VERIFY_SERVER_ARGS_JSON; - -const serverCommand = process.env.REA_VERIFY_SERVER_COMMAND ?? process.execPath; -const serverArgs = parseServerArgs( - process.env.REA_VERIFY_SERVER_ARGS_JSON ?? '["dist/main.js"]', -); - -const transport = new StdioClientTransport({ - command: serverCommand, - args: serverArgs, - cwd: process.cwd(), - env: serverEnvironment, - stderr: "pipe", -}); -let stderrBytes = 0; -const stderrChunks = []; -transport.stderr?.on("data", (chunk) => { - stderrBytes += chunk.length; - stderrChunks.push(chunk.toString("utf8")); -}); -const client = new Client({ name: "real-hopper-verifier", version: "1.0.0" }); -const unrelatedHopper = await startUnrelatedHopperSentinel(); -let unrelatedHopperSurvived = false; -let summary; - -try { - await client.connect(transport); - if (transport.pid === null) - throw new Error("The verifier did not capture its MCP process identity"); - ownedProcessIds.add(transport.pid); - const progressUpdates = []; - const options = { - timeout, - onprogress: (update) => progressUpdates.push(update), - }; - const fullSessionStatus = () => - client.callTool({ name: "binary_session", arguments: {} }, options); - const initialSession = await fullSessionStatus(); - const initialStatus = requireMcpLifecycleResult( - initialSession, - "binary_session", - ); - const listed = await client.listTools(); - const availableTools = initialStatus.tool_availability; - if (!Array.isArray(availableTools)) - throw new Error("The real session omitted tool availability"); - const expectedNames = sortVerifierNames( - availableTools.map(({ name }) => name), - ); - const actualNames = sortVerifierNames(listed.tools.map(({ name }) => name)); - if (JSON.stringify(actualNames) !== JSON.stringify(expectedNames)) { - const expected = new Set(expectedNames); - const actual = new Set(actualNames); - throw new Error( - `The real server tool catalog diverged from session capability inventory (untracked tools: ${actualNames.filter((name) => !expected.has(name)).join(", ") || "none"}; missing catalog tools: ${expectedNames.filter((name) => !actual.has(name)).join(", ") || "none"})`, + const verifyCurrentTarget = async (client, options) => { + const procedures = requireMcpEvidenceResult( + await client.callTool( + { name: "list_procedures", arguments: {} }, + options, + ), + "list_procedures", ); - } - - if (initialStatus.open !== false) - throw new Error("The verifier did not start without a target"); - requireHopperSelection(initialStatus, { - targetSupport: "unknown", - selected: false, - }); - const opened = await client.callTool( - { name: "open_binary", arguments: { path: targetA } }, - options, - ); - if (opened.isError === true) throw new Error(textValue(opened)); - const firstStatus = requireMcpLifecycleResult( - await fullSessionStatus(), - "binary_session", - ); - const providerBinding = requireHopperSelection(firstStatus, { - targetSupport: "supported", - selected: true, - }); - const documents = await client.callTool( - { name: "list_documents", arguments: {} }, - options, - ); - requireMcpEvidenceResult(documents, "list_documents"); - requireBridgeProgress(progressUpdates); - const rejectedProcedure = await client.callTool( - { - name: "procedure_info", - arguments: { procedure: "__rea_missing_procedure__" }, - }, - options, - ); - if (rejectedProcedure.isError !== true) - throw new Error("The real Hopper bridge accepted an unknown procedure"); - const overview = await client.callTool( - { name: "binary_overview", arguments: {} }, - options, - ); - requireMcpEvidenceResult(overview, "binary_overview"); - const inventoryCounts = await verifyHopperInventories(client, options); - requireEvidenceProvider( - documents, - "list_documents", - HOPPER_PROVIDER_IDENTITY.id, - ); - requireWorkflowEvidenceProvider(overview, "binary_overview", { - workflowProviderId: REA_WORKFLOW_PROVIDER.id, - upstreamProviderId: HOPPER_PROVIDER_IDENTITY.id, - }); - const segments = await client.callTool( - { name: "list_segments", arguments: {} }, - options, - ); - requireTruthfulMemoryRegions( - requireMcpEvidenceResult(segments, "list_segments"), - ); - const firstDocuments = requireMcpEvidenceResult(documents, "list_documents"); - if ( - !Array.isArray(firstDocuments) || - firstDocuments.length === 0 || - typeof firstDocuments[0] !== "string" - ) - throw new Error("list_documents returned no real Hopper document"); - const currentDocument = await requireCurrentDocument( - client, - options, - firstDocuments, - requireMcpEvidenceResult, - ); - const firstOverview = requireOverview( - requireMcpEvidenceResult(overview, "binary_overview"), - "binary_overview", - ); - if (firstOverview.document !== currentDocument) - throw new Error( - "binary_overview did not identify the selected Hopper document", + const firstAddress = firstProcedureAddress(procedures); + const byteAccess = await verifyByteAccess(client, options, firstAddress); + const basics = await verifyHopperFunctionBasics( + client, + options, + firstAddress, ); - const firstAnalysis = await verifyCurrentTarget(client, options); - const fixtureAnalysis = await verifyRealHopperFixture({ - client, - options, - document: currentDocument, - oracle: fixtureTargets.oracle, - normalizedResult: requireMcpEvidenceResult, - }); - const boundaryContracts = await verifyHopperBoundaryContracts( - client, - options, - fixtureAnalysis.procedures.entry, - targetA, - ); - const switched = await client.callTool( - { name: "open_binary", arguments: { path: targetB } }, - options, - ); - if (switched.isError === true) throw new Error(textValue(switched)); - const secondSession = requireMcpLifecycleResult( - await fullSessionStatus(), - "binary_session", - ); - if (secondSession.path !== targetB) - throw new Error("The real session did not switch to target B"); - requireHopperSelection(secondSession, { - targetSupport: "supported", - selected: true, - }); - const secondOverview = await client.callTool( - { name: "binary_overview", arguments: {} }, - options, - ); - const verifiedSecondOverview = requireOverview( - requireMcpEvidenceResult(secondOverview, "binary_overview"), - "binary_overview after target switch", - ); - const secondAnalysis = await verifyCurrentTarget(client, options); - const documentsAfterTargetSwitch = requireMcpEvidenceResult( - await client.callTool({ name: "list_documents", arguments: {} }, options), - "list_documents", - ); - const secondDocument = await requireCurrentDocument( - client, - options, - documentsAfterTargetSwitch, - requireMcpEvidenceResult, - ); - if (verifiedSecondOverview.document !== secondDocument) - throw new Error( - "binary_overview retained another document after target switch", + const relationships = await verifyRelationships( + client, + options, + firstAddress, ); - const expectedDocumentsAfterTargetSwitch = firstDocuments.length; - if ( - !Array.isArray(documentsAfterTargetSwitch) || - documentsAfterTargetSwitch.length !== expectedDocumentsAfterTargetSwitch - ) - throw new Error( - "Target switching did not replace the previous Hopper document", - ); - const reopenedTarget = await client.callTool( - { name: "open_binary", arguments: { path: targetB } }, - options, - ); - if (reopenedTarget.isError === true) - throw new Error(textValue(reopenedTarget)); - const documentsAfterReopen = requireMcpEvidenceResult( - await client.callTool({ name: "list_documents", arguments: {} }, options), - "list_documents", - ); - if ( - !Array.isArray(documentsAfterReopen) || - documentsAfterReopen.length !== documentsAfterTargetSwitch.length - ) - throw new Error( - "Reopening the same target created a duplicate Hopper document", - ); - const largeInventory = await openAndVerifyLargeFixture({ - client, - options, - normalizedResult: requireMcpEvidenceResult, - path: largeTarget, - expectedCount: fixtureTargets.largeOracle.symbolCount, - symbolPrefix: fixtureTargets.largeOracle.symbolPrefix, - stringPrefix: fixtureTargets.largeOracle.stringPrefix, - }); - - const processList = await execFileAsync("ps", ["-ax", "-o", "command="]); - const bundledMcpRunning = processList.stdout - .split("\n") - .some((line) => line.trim().endsWith("/HopperMCPServer")); - if (bundledMcpRunning) { - throw new Error( - "Hopper's bundled MCP server was running during verification", - ); - } - const closed = await client.callTool( - { name: "close_binary", arguments: {} }, - options, - ); - if (closed.isError === true) throw new Error(textValue(closed)); - let unicodeStrings = null; - let terminalInstructions = null; - if (fixtureTargets.unicode !== undefined) { - const call = async (name, args = {}) => - requireMcpOperationResult( - await client.callTool({ name, arguments: args }, options), - name, - ); - await call("open_binary", { path: fixtureTargets.unicode.path }); - terminalInstructions = await verifyHopperTerminalInstructions(call); - unicodeStrings = await verifyHopperStringObjects(call, [ - { value: "REA_UTF16_é_😀", encoding: "utf-16-le" }, - { value: 'REA_UTF16_ESCAPED_é_"\\line\nend\t\r', encoding: "utf-16-le" }, - { value: "\nREA_UTF16_LEADING_é", encoding: "utf-16-le" }, - ]); - await call("close_binary"); - } - const lifecycleAndCli = await verifyHopperLifecycleAndCli(client, options, { - primary: targetA, - secondary: targetB, - ...(fixtureTargets.unicode === undefined - ? {} - : { unicode: fixtureTargets.unicode.path }), - ownedProcessIds, - }); - const closedSession = requireMcpLifecycleResult( - await fullSessionStatus(), - "binary_session", - ); - if (closedSession.open !== false) - throw new Error("The real session remained open after close_binary"); - const diagnosticCount = requireSafeDiagnostics( - stderrChunks, - [ - "procedure_info", + const boundedResult = requireMcpEvidenceResult( + await client.callTool( + { + name: "batch_decompile", + arguments: { addresses: [firstAddress] }, + }, + options, + ), "batch_decompile", - ...boundaryContracts.rejectedOperations, - ], - ["HopperRemoteError", "HopperCancelledError", "AnalysisCancelledError"], - ); - requireBridgeDiagnostic(stderrChunks); - - summary = { - toolCount: actualNames.length, - documentCount: firstDocuments.length, - overview: firstOverview, - segmentCount: firstOverview.segment_count, - analyses: [firstAnalysis, secondAnalysis], - fixtureAnalysis, - unicodeStrings, - terminalInstructions, - boundaryContracts, - lifecycleAndCli, - inventoryCounts, - largeInventory, - fixtureManifest: fixtureTargets.manifestPath, - bundledMcpRunning, - stderrBytes, - diagnosticCount, - bridgeProgressUpdates: progressUpdates.length, - dynamicSession: true, - providerBinding, - targets: [targetA, targetB, largeTarget], - targetHashes: [targetHashA, targetHashB, fixtureTargets.large.sha256], - switched: true, - secondOverview: verifiedSecondOverview, - documentCountAfterTargetSwitch: documentsAfterTargetSwitch.length, - documentCountAfterSameTargetReopen: documentsAfterReopen.length, + ); + if ( + boundedResult === null || + typeof boundedResult !== "object" || + Array.isArray(boundedResult) || + boundedResult.total !== 1 || + boundedResult.succeeded !== 1 || + boundedResult.failed !== 0 || + !Array.isArray(boundedResult.items) || + boundedResult.items.length !== 1 || + boundedResult.items[0]?.address !== firstAddress || + boundedResult.items[0]?.status !== "ok" + ) { + throw new Error("batch_decompile returned an invalid result"); + } + const boundedPseudocode = requirePseudocode( + boundedResult.items[0].pseudocode, + "batch_decompile", + ); + const dossier = requireFunctionDossier( + requireMcpEvidenceResult( + await client.callTool( + { + name: "analyze_function", + arguments: { procedure: firstAddress, detail: "complete" }, + }, + options, + ), + "analyze_function", + ), + firstAddress, + ); + return { + procedure: dossier.procedure, + procedureCount: procedures.length, + boundedPseudocodeChars: boundedPseudocode.length, + analyzedPseudocodeChars: [...dossier.pseudocode].length, + callerCount: relationships.procedure_callers.length, + calleeCount: relationships.procedure_callees.length, + xrefCount: relationships.xrefs.length, + outgoingReferenceCount: basics.outgoingReferenceCount, + instructionWindowCount: basics.instructionWindowCount, + ...byteAccess, + }; }; -} finally { - const keepAlive = setInterval(() => undefined, 100); - try { - await client.close(); - await transport.close(); - } finally { - clearInterval(keepAlive); - unrelatedHopperSurvived = await unrelatedHopper.verifyAndClose(); - } -} -await new Promise((resolve) => setTimeout(resolve, 500)); -const sessionsAfter = [ - ...(await snapshotHopperRuntime(runtimeParent, HOPPER_TARGET_LEASE_DIRECTORY)) - .paths, -] - .filter((path) => !sessionsBefore.has(path)) - .sort(); -if (sessionsAfter.length > 0) { - await writeVerificationReport({ - ...summary, - unrelatedHopperSurvived, - cleanShutdown: false, - retainedRuntimePaths: sessionsAfter, + const fixtureManifestPath = + process.env.REA_HOPPER_CONFORMANCE_MANIFEST_PATH ?? + "build/conformance/manifest.json"; + const fixtureTargets = + await loadRealHopperFixtureTargets(fixtureManifestPath); + const targetA = fixtureTargets.primary.path; + const targetB = fixtureTargets.secondary.path; + const largeTarget = fixtureTargets.large.path; + const targetHashA = fixtureTargets.primary.sha256; + const targetHashB = fixtureTargets.secondary.sha256; + const serverEnvironment = { ...process.env }; + serverEnvironment.REA_ANALYSIS_PROVIDER = "auto"; + delete serverEnvironment.HOPPER_TARGET_PATH; + delete serverEnvironment.HOPPER_SECOND_TARGET_PATH; + delete serverEnvironment.REA_VERIFY_SERVER_COMMAND; + delete serverEnvironment.REA_VERIFY_SERVER_ARGS_JSON; + + const serverCommand = + process.env.REA_VERIFY_SERVER_COMMAND ?? process.execPath; + const serverArgs = parseServerArgs( + process.env.REA_VERIFY_SERVER_ARGS_JSON ?? '["dist/main.js"]', + ); + + const transport = new StdioClientTransport({ + command: serverCommand, + args: serverArgs, + cwd: process.cwd(), + env: serverEnvironment, + stderr: "pipe", }); - throw new Error( - `Hopper resources remain or have unresolved ownership: ${sessionsAfter.join(", ")}`, + let stderrBytes = 0; + const stderrChunks = []; + transport.stderr?.on("data", (chunk) => { + stderrBytes += chunk.length; + stderrChunks.push(chunk.toString("utf8")); + }); + const client = new Client({ name: "real-hopper-verifier", version: "1.0.0" }); + const unrelatedHopper = await startUnrelatedHopperSentinel(); + + try { + await client.connect(transport); + if (transport.pid === null) + throw new Error("The verifier did not capture its MCP process identity"); + const progressUpdates = []; + const options = { + timeout, + onprogress: (update) => progressUpdates.push(update), + }; + const fullSessionStatus = () => + client.callTool({ name: "binary_session", arguments: {} }, options); + const initialSession = await fullSessionStatus(); + const initialStatus = requireMcpLifecycleResult( + initialSession, + "binary_session", + ); + const listed = await client.listTools(); + const availableTools = initialStatus.tool_availability; + if (!Array.isArray(availableTools)) + throw new Error("The real session omitted tool availability"); + const expectedNames = sortVerifierNames( + availableTools.map(({ name }) => name), + ); + const actualNames = sortVerifierNames(listed.tools.map(({ name }) => name)); + if (JSON.stringify(actualNames) !== JSON.stringify(expectedNames)) { + const expected = new Set(expectedNames); + const actual = new Set(actualNames); + throw new Error( + `The real server tool catalog diverged from session capability inventory (untracked tools: ${actualNames.filter((name) => !expected.has(name)).join(", ") || "none"}; missing catalog tools: ${expectedNames.filter((name) => !actual.has(name)).join(", ") || "none"})`, + ); + } + + if (initialStatus.open !== false) + throw new Error("The verifier did not start without a target"); + requireHopperSelection(initialStatus, { + targetSupport: "unknown", + selected: false, + }); + const opened = await client.callTool( + { name: "open_binary", arguments: { path: targetA } }, + options, + ); + if (opened.isError === true) throw new Error(textValue(opened)); + const firstStatus = requireMcpLifecycleResult( + await fullSessionStatus(), + "binary_session", + ); + const providerBinding = requireHopperSelection(firstStatus, { + targetSupport: "supported", + selected: true, + }); + const documents = await client.callTool( + { name: "list_documents", arguments: {} }, + options, + ); + requireMcpEvidenceResult(documents, "list_documents"); + requireBridgeProgress(progressUpdates); + const rejectedProcedure = await client.callTool( + { + name: "procedure_info", + arguments: { procedure: "__rea_missing_procedure__" }, + }, + options, + ); + if (rejectedProcedure.isError !== true) + throw new Error("The real Hopper bridge accepted an unknown procedure"); + const overview = await client.callTool( + { name: "binary_overview", arguments: {} }, + options, + ); + requireMcpEvidenceResult(overview, "binary_overview"); + const inventoryCounts = await verifyHopperInventories(client, options); + requireEvidenceProvider( + documents, + "list_documents", + HOPPER_PROVIDER_IDENTITY.id, + ); + requireWorkflowEvidenceProvider(overview, "binary_overview", { + workflowProviderId: REA_WORKFLOW_PROVIDER.id, + upstreamProviderId: HOPPER_PROVIDER_IDENTITY.id, + }); + const segments = await client.callTool( + { name: "list_segments", arguments: {} }, + options, + ); + requireTruthfulMemoryRegions( + requireMcpEvidenceResult(segments, "list_segments"), + ); + const firstDocuments = requireMcpEvidenceResult( + documents, + "list_documents", + ); + if ( + !Array.isArray(firstDocuments) || + firstDocuments.length === 0 || + typeof firstDocuments[0] !== "string" + ) + throw new Error("list_documents returned no real Hopper document"); + const currentDocument = await requireCurrentDocument( + client, + options, + firstDocuments, + requireMcpEvidenceResult, + ); + const firstOverview = requireOverview( + requireMcpEvidenceResult(overview, "binary_overview"), + "binary_overview", + ); + if (firstOverview.document !== currentDocument) + throw new Error( + "binary_overview did not identify the selected Hopper document", + ); + const firstAnalysis = await verifyCurrentTarget(client, options); + const fixtureAnalysis = await verifyRealHopperFixture({ + client, + options, + document: currentDocument, + oracle: fixtureTargets.oracle, + normalizedResult: requireMcpEvidenceResult, + }); + const boundaryContracts = await verifyHopperBoundaryContracts( + client, + options, + fixtureAnalysis.procedures.entry, + targetA, + ); + const switched = await client.callTool( + { name: "open_binary", arguments: { path: targetB } }, + options, + ); + if (switched.isError === true) throw new Error(textValue(switched)); + const secondSession = requireMcpLifecycleResult( + await fullSessionStatus(), + "binary_session", + ); + if (secondSession.path !== targetB) + throw new Error("The real session did not switch to target B"); + requireHopperSelection(secondSession, { + targetSupport: "supported", + selected: true, + }); + const secondOverview = await client.callTool( + { name: "binary_overview", arguments: {} }, + options, + ); + const verifiedSecondOverview = requireOverview( + requireMcpEvidenceResult(secondOverview, "binary_overview"), + "binary_overview after target switch", + ); + const secondAnalysis = await verifyCurrentTarget(client, options); + const documentsAfterTargetSwitch = requireMcpEvidenceResult( + await client.callTool({ name: "list_documents", arguments: {} }, options), + "list_documents", + ); + const secondDocument = await requireCurrentDocument( + client, + options, + documentsAfterTargetSwitch, + requireMcpEvidenceResult, + ); + if (verifiedSecondOverview.document !== secondDocument) + throw new Error( + "binary_overview retained another document after target switch", + ); + const expectedDocumentsAfterTargetSwitch = firstDocuments.length; + if ( + !Array.isArray(documentsAfterTargetSwitch) || + documentsAfterTargetSwitch.length !== expectedDocumentsAfterTargetSwitch + ) + throw new Error( + "Target switching did not replace the previous Hopper document", + ); + const reopenedTarget = await client.callTool( + { name: "open_binary", arguments: { path: targetB } }, + options, + ); + if (reopenedTarget.isError === true) + throw new Error(textValue(reopenedTarget)); + const documentsAfterReopen = requireMcpEvidenceResult( + await client.callTool({ name: "list_documents", arguments: {} }, options), + "list_documents", + ); + if ( + !Array.isArray(documentsAfterReopen) || + documentsAfterReopen.length !== documentsAfterTargetSwitch.length + ) + throw new Error( + "Reopening the same target created a duplicate Hopper document", + ); + const largeInventory = await openAndVerifyLargeFixture({ + client, + options, + normalizedResult: requireMcpEvidenceResult, + path: largeTarget, + expectedCount: fixtureTargets.largeOracle.symbolCount, + symbolPrefix: fixtureTargets.largeOracle.symbolPrefix, + stringPrefix: fixtureTargets.largeOracle.stringPrefix, + }); + + const processList = await commands.exec("ps", ["-ax", "-o", "command="]); + const bundledMcpRunning = processList.stdout + .split("\n") + .some((line) => line.trim().endsWith("/HopperMCPServer")); + if (bundledMcpRunning) { + throw new Error( + "Hopper's bundled MCP server was running during verification", + ); + } + const closed = await client.callTool( + { name: "close_binary", arguments: {} }, + options, + ); + if (closed.isError === true) throw new Error(textValue(closed)); + let unicodeStrings = null; + let terminalInstructions = null; + if (fixtureTargets.unicode !== undefined) { + const call = async (name, args = {}) => + requireMcpOperationResult( + await client.callTool({ name, arguments: args }, options), + name, + ); + await call("open_binary", { path: fixtureTargets.unicode.path }); + terminalInstructions = await verifyHopperTerminalInstructions(call); + unicodeStrings = await verifyHopperStringObjects(call, [ + { value: "REA_UTF16_é_😀", encoding: "utf-16-le" }, + { + value: 'REA_UTF16_ESCAPED_é_"\\line\nend\t\r', + encoding: "utf-16-le", + }, + { value: "\nREA_UTF16_LEADING_é", encoding: "utf-16-le" }, + ]); + await call("close_binary"); + } + const lifecycleAndCli = await verifyHopperLifecycleAndCli( + client, + options, + { + primary: targetA, + secondary: targetB, + ...(fixtureTargets.unicode === undefined + ? {} + : { unicode: fixtureTargets.unicode.path }), + }, + commands, + ); + const closedSession = requireMcpLifecycleResult( + await fullSessionStatus(), + "binary_session", + ); + if (closedSession.open !== false) + throw new Error("The real session remained open after close_binary"); + const diagnosticCount = requireSafeDiagnostics( + stderrChunks, + [ + "procedure_info", + "batch_decompile", + ...boundaryContracts.rejectedOperations, + ], + ["HopperRemoteError", "HopperCancelledError", "AnalysisCancelledError"], + ); + requireBridgeDiagnostic(stderrChunks); + + summary = { + toolCount: actualNames.length, + documentCount: firstDocuments.length, + overview: firstOverview, + segmentCount: firstOverview.segment_count, + analyses: [firstAnalysis, secondAnalysis], + fixtureAnalysis, + unicodeStrings, + terminalInstructions, + boundaryContracts, + lifecycleAndCli, + inventoryCounts, + largeInventory, + fixtureManifest: fixtureTargets.manifestPath, + bundledMcpRunning, + stderrBytes, + diagnosticCount, + bridgeProgressUpdates: progressUpdates.length, + dynamicSession: true, + providerBinding, + targets: [targetA, targetB, largeTarget], + targetHashes: [targetHashA, targetHashB, fixtureTargets.large.sha256], + switched: true, + secondOverview: verifiedSecondOverview, + documentCountAfterTargetSwitch: documentsAfterTargetSwitch.length, + documentCountAfterSameTargetReopen: documentsAfterReopen.length, + }; + } finally { + const keepAlive = setInterval(() => undefined, 100); + try { + await client.close(); + await transport.close(); + } finally { + clearInterval(keepAlive); + unrelatedHopperSurvived = await unrelatedHopper.verifyAndClose(); + } + } + + await new Promise((resolve) => setTimeout(resolve, 500)); + const sessionsAfter = [ + ...( + await snapshotHopperRuntime(runtimeParent, HOPPER_TARGET_LEASE_DIRECTORY) + ).paths, + ] + .filter((path) => !sessionsBefore.has(path)) + .sort(); + if (sessionsAfter.length > 0) { + await writeVerificationReport({ + ...summary, + unrelatedHopperSurvived, + cleanShutdown: false, + retainedRuntimePaths: sessionsAfter, + }); + throw new Error( + `Hopper resources remain or have unresolved ownership: ${sessionsAfter.join(", ")}`, + ); + } + if (summary === undefined) + throw new Error("Real-Hopper verification did not produce a summary"); + if (process.platform === "linux") + summary.ownerExit = await verifyLinuxHopperOwnerExit({ + primary: targetA, + secondary: targetB, + serverCommand, + serverArgs, + environment: serverEnvironment, + commands, + }); +} catch (cause) { + primaryFailure = cause; +} +let cleanupFailure; +try { + await workspaceFixture.closeEarly([() => commands.close()]); +} catch (cause) { + cleanupFailure = cause; +} +if (cleanupFailure !== undefined) + throw retainWorkspace( + `Hopper command cleanup failed; retained diagnostic workspace: ${workspace}`, + { + cause: + primaryFailure === undefined + ? cleanupFailure + : new AggregateError( + [primaryFailure, cleanupFailure], + "Hopper verification and command cleanup both failed", + ), + }, + ); +const completedVerifierRun = await completeVerifierRun(verifierRun); +try { + assertVerifierCleanupComplete(completedVerifierRun.process_lineage); +} catch (cause) { + throw retainWorkspace( + `Cleanup unverified; retained diagnostic workspace: ${workspace}`, + { + cause: + primaryFailure === undefined + ? cause + : new AggregateError( + [primaryFailure, cause], + "Hopper verification failed and cleanup was unverified", + ), + }, ); } -if (summary === undefined) - throw new Error("Real-Hopper verification did not produce a summary"); -if (process.platform === "linux") - summary.ownerExit = await verifyLinuxHopperOwnerExit({ - primary: targetA, - secondary: targetB, - serverCommand, - serverArgs, - environment: serverEnvironment, - }); -const completedVerifierRun = await completeVerifierRun(verifierRun); -if ( - completedVerifierRun.process_lineage.status !== "verified" || - completedVerifierRun.process_lineage.descendants.length !== 0 -) - throw new Error( - "The verifier's process lineage did not prove owned child cleanup", +if (primaryFailure !== undefined) + throw retainWorkspace( + `Real-Hopper verification failed; retained workspace: ${workspace}`, + { cause: primaryFailure }, ); +await workspaceFixture.release(); await writeVerificationReport({ verifier_run: completedVerifierRun, ...summary, diff --git a/scripts/verify-real-kuna.mjs b/scripts/verify-real-kuna.mjs index db1a424c81..17791c8aca 100644 --- a/scripts/verify-real-kuna.mjs +++ b/scripts/verify-real-kuna.mjs @@ -8,12 +8,11 @@ // `--compare-ghidra` runs the same functions through a bring-your-own Ghidra // and reports latency, CPU, memory, output size and simple quality signals. import assert from "node:assert/strict"; -import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; import { mkdir, readFile, readdir, stat, writeFile } from "node:fs/promises"; import { join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; -import { parseArgs, promisify } from "node:util"; +import { parseArgs } from "node:util"; import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { inspectKunaInstallation } from "../dist/kuna/KunaInstallation.js"; @@ -44,9 +43,8 @@ import { completeVerifierRun, createVerifierRun, } from "./lib/verifier-run.mjs"; +import { createVerifierCommands } from "./lib/verify-commands.mjs"; import { VerifierFixture } from "./lib/verifier-fixture.mjs"; - -const exec = promisify(execFile); const { values } = parseArgs({ options: { targets: { type: "string" }, @@ -87,10 +85,21 @@ const resourceScope = { }), run: (operation) => inspectionResources.run(operation), }; +// nm, ps, cc and strip share one owner, created before the workspace. If +// allocation fails, the owner is still closed and no root is left behind. +const commands = await createVerifierCommands(); // Kuna children cache under the root; a plain root admits them on every host. const workspaceFixture = await VerifierFixture.open({ prefix: "rea-kuna-lane-", private: false, +}).catch(async (cause) => { + await commands.close().catch((closeCause) => { + throw new AggregateError( + [cause, closeCause], + "Kuna workspace allocation and command cleanup both failed", + ); + }); + throw cause; }); const workspace = workspaceFixture.path; const cacheRoot = join(workspace, "cache"); @@ -135,7 +144,7 @@ int main(int argc, char **argv) { /** Resolve function entry addresses from a symbol-bearing build. */ const symbolAddresses = async (binary, names) => { - const { stdout } = await exec(process.env.REA_NM ?? "nm", [binary]); + const { stdout } = await commands.exec(process.env.REA_NM ?? "nm", [binary]); const found = new Map(); for (const line of stdout.split("\n")) { const [address, type, symbol] = line.trim().split(/\s+/u); @@ -150,7 +159,11 @@ const symbolAddresses = async (binary, names) => { /** Sum the resident memory and CPU time of a process and its descendants. */ const sampleTree = async (root) => { - const { stdout } = await exec("ps", ["-A", "-o", "pid=,ppid=,rss=,time="]); + const { stdout } = await commands.exec("ps", [ + "-A", + "-o", + "pid=,ppid=,rss=,time=", + ]); const rows = stdout .trim() .split("\n") @@ -778,10 +791,10 @@ try { const source = join(fixtureDir, "fixture.c"); await writeFile(source, FIXTURE_SOURCE); const built = join(fixtureDir, "rea-kuna-fixture"); - await exec(process.env.REA_CC ?? "cc", ["-O2", "-o", built, source]); + await commands.exec(process.env.REA_CC ?? "cc", ["-O2", "-o", built, source]); const stripped = `${built}.stripped`; await writeFile(stripped, await readFile(built)); - await exec(process.env.REA_STRIP ?? "strip", [stripped]); + await commands.exec(process.env.REA_STRIP ?? "strip", [stripped]); const fixtureFunctions = await symbolAddresses(built, [ "rea_checksum", "rea_classify", @@ -829,24 +842,43 @@ try { inspectionResourcesClean = false; failures.push(cause); } +let commandCleanupFailure; +try { + await workspaceFixture.closeEarly([() => commands.close()]); +} catch (cause) { + commandCleanupFailure = cause; + failures.push(cause); +} const lifecycle = await completeVerifierRun(run); let cleanupComplete = pendingServers.size === 0 && retainedNative.size === 0 && - inspectionResourcesClean; + inspectionResourcesClean && + commandCleanupFailure === undefined; try { assertVerifierCleanupComplete(lifecycle.process_lineage); } catch (cause) { cleanupComplete = false; failures.push(cause); } +let workspaceReleased = false; +// A verification failure with successful provider, command, and lineage +// cleanup still releases the root. Only an unconfirmed close retains it. +// release() closes cleanup registration before it removes the root, so a +// failed release already keeps the workspace and retain() would throw +// before this report is written. if (cleanupComplete) try { await workspaceFixture.release(); + workspaceReleased = true; } catch (cause) { cleanupComplete = false; failures.push(cause); } +else + workspaceFixture.retain( + `Kuna cleanup was not confirmed; retained diagnostic workspace: ${workspace}`, + ); const report = { lane: "kuna", status: failures.length === 0 ? "passed" : "failed", @@ -867,7 +899,7 @@ const report = { "Response bytes are UTF-8 serialization sizes. Model token usage, tool schema costs and agent task completion were not measured.", ], cleanup: cleanupComplete ? "complete" : "incomplete", - ...(cleanupComplete ? {} : { retained_workspace: workspace }), + ...(workspaceReleased ? {} : { retained_workspace: workspace }), ...(failures.length === 0 ? {} : { failures: failures.map(describeFailure) }), }; if (values.report !== undefined) diff --git a/scripts/verify/apple/asset-catalog.mjs b/scripts/verify/apple/asset-catalog.mjs index 9912072827..97492e6bb0 100644 --- a/scripts/verify/apple/asset-catalog.mjs +++ b/scripts/verify/apple/asset-catalog.mjs @@ -12,6 +12,7 @@ import { withArtifactMcp, withBundleResourcesMcp, } from "../../lib/artifact-e2e.mjs"; +import { requireMcpToolError } from "../../lib/mcp-verifier-results.mjs"; import { VerifierFixture, withVerifierFixture, @@ -106,7 +107,15 @@ await withVerifierFixture(fixture, async () => { name: "inspect_asset_catalog", arguments: { offset: -1 }, }); - assert.equal(invalid.isError, true); + // appleAssetCatalogInputSchema requires an inclusive offset minimum of 0. + // projectInputIssues reports that bound as out_of_range. + assert.partialDeepStrictEqual(requireMcpToolError(invalid), { + code: "invalid_request", + details: { + operation: "inspect_asset_catalog", + issues: [{ path: ["offset"], reason: "out_of_range", minimum: 0 }], + }, + }); }); const { resource_root, target_sha256, ...resourcePayload } = cli; assert.equal(resource_root.executable_sha256, null); diff --git a/scripts/verify/apple/keyed-archive.mjs b/scripts/verify/apple/keyed-archive.mjs index d963dcb04c..028b92e8e2 100644 --- a/scripts/verify/apple/keyed-archive.mjs +++ b/scripts/verify/apple/keyed-archive.mjs @@ -11,6 +11,7 @@ import { artifactMcpCall, withArtifactMcp, } from "../../lib/artifact-e2e.mjs"; +import { requireMcpToolError } from "../../lib/mcp-verifier-results.mjs"; import { assertKeyedArchiveIntegerResult, verifyKeyedArchiveIntegerArchives, @@ -62,7 +63,15 @@ await withVerifierFixture(fixture, async () => { name: "inspect_keyed_archive", arguments: { path: "../other.plist" }, }); - assert.equal(invalid.isError, true); + // Standalone archives reject a path other than the admitted basename. + // KeyedArchiveReader throws AnalysisInputError; classification is invalid_request. + assert.partialDeepStrictEqual(requireMcpToolError(invalid), { + code: "invalid_request", + details: { + operation: "inspect_keyed_archive", + issues: [{ path: ["path"], reason: "invalid_value" }], + }, + }); }); const xml = join(root, "model.xml.plist"); await promisify(execFile)("/usr/bin/plutil", [ diff --git a/scripts/verify/apple/macos-dylib-e2e.mjs b/scripts/verify/apple/macos-dylib-e2e.mjs index 0ea4a674cc..e9e60a2241 100644 --- a/scripts/verify/apple/macos-dylib-e2e.mjs +++ b/scripts/verify/apple/macos-dylib-e2e.mjs @@ -10,6 +10,7 @@ import { artifactMcpCall, withArtifactMcp, } from "../../lib/artifact-e2e.mjs"; +import { requireMcpToolError } from "../../lib/mcp-verifier-results.mjs"; const exec = promisify(execFile); @@ -185,7 +186,15 @@ export async function verifyDylibResolution(app) { name: "trace_dylib_resolution", arguments: { roots: ["../outside"] }, }); - assert.equal(invalid.isError, true); + // dylibResolutionInputSchema rejects a root that is not a normalized + // relative path. projectInputIssues maps that regex failure to invalid_format. + assert.partialDeepStrictEqual(requireMcpToolError(invalid), { + code: "invalid_request", + details: { + operation: "trace_dylib_resolution", + issues: [{ path: ["roots", 0], reason: "invalid_format" }], + }, + }); // Selecting the observed roots removes discovery uncertainty while retaining // complete dependency-resolution coverage for this known fixture. const roots = [...new Set(trace.roots.map(({ image }) => image))]; diff --git a/tests/conformance/providers/hopper/realHopperFixture.test.ts b/tests/conformance/providers/hopper/realHopperFixture.test.ts index 1bee8774d5..b325246af2 100644 --- a/tests/conformance/providers/hopper/realHopperFixture.test.ts +++ b/tests/conformance/providers/hopper/realHopperFixture.test.ts @@ -24,13 +24,23 @@ describe("real Hopper fixture binding", () => { ); }); - it("binds exact source-owned artifacts and their semantic oracle", async () => { + it("binds a caller-selected local manifest and its exact artifact digests", async () => { const manifest = await fixtureManifest(); await expect(loadRealHopperFixtureTargets(manifest)).resolves.toMatchObject( { - primary: { path: join(root ?? "", "c") }, - secondary: { path: join(root ?? "", "version-v2") }, - large: { path: join(root ?? "", "large") }, + manifestPath: manifest, + primary: { + path: join(root ?? "", "c"), + sha256: digest(Buffer.from("primary")), + }, + secondary: { + path: join(root ?? "", "version-v2"), + sha256: digest(Buffer.from("secondary")), + }, + large: { + path: join(root ?? "", "large"), + sha256: digest(Buffer.from("large")), + }, largeOracle: { symbolPrefix: "_rea_inventory_", symbolCount: 1_205,