From b990eb496d10ca7d31b98edd2d22a4f58f41c4f5 Mon Sep 17 00:00:00 2001 From: gensokyo-dev Date: Sat, 14 Nov 2026 10:10:01 +0000 Subject: [PATCH] perf(cli): load only the registrar a one-shot command needs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every rea process imported and registered all 52 command registrars (about 1,900 modules) before running any command. A one-shot command now loads only the registrar that defines its top-level command. - src/cli/cliCommandRegistrations.ts holds every registrar in one table, grouped by tool family, with its module loader and its call. The complete createCli registers the table with static imports, and the types check that each entry matches its module. - src/cli/cliShell.ts owns what does not depend on which commands are registered: identity, globals, argument policy, completions and the registration builtins. The full and lazy paths share it. - scripts/record-cli-commands.mjs records, after tsc, which registrar defines each top-level command, from the same compiled table. The bundle keeps the record as an external dist module. - cli/cliLoader.ts selects the recorded registrar when a known top-level command runs. It loads the complete CLI for help, --schema, --llms, completions, builtins, unknown names ("did you mean"), or a missing record. - Process-owned CLI resources register their closers when their module loads (cli/cliResources.ts), so the shell no longer imports JavaScript-analysis and guidance owners just to close them. Measured on one pinned core (median of 5 warm runs, base main vs this change): - guidance: 1.95 s / 253 MB → 0.42 s / 101 MB - list-cutter-sessions: 1.97 s → 0.49 s - inspect-go-binary: 1.95 s → 0.95 s - setup --dry-run: 2.40 s → 1.88 s - analyze: 2.39 s → 1.78 s - --help: 1.99 s → 2.04 s (complete CLI, as before) Co-Authored-By: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com> --- docs/architecture/architecture.mermaid | 2 +- docs/development/tool-design.md | 13 +- package.json | 4 +- scripts/bundle-runtime.mjs | 11 +- scripts/record-cli-commands.mjs | 67 +++ src/cli.ts | 394 +++++--------- src/cli/cliCommandRegistration.ts | 41 ++ src/cli/cliCommandRegistrations.ts | 485 ++++++++++++++++++ src/cli/cliLoader.ts | 98 ++++ src/cli/cliResources.ts | 45 ++ src/cli/cliShell.ts | 184 +++++++ src/cli/guidanceCommands.ts | 3 + src/cli/javascriptApplicationAnalysis.ts | 5 + src/cli/mcpRegistration.ts | 3 + src/cliOneShot.ts | 7 +- src/recordedCliCommandRegistrations.ts | 8 + .../boundary/cli/cliLazyRegistration.test.ts | 157 ++++++ tests/boundary/cli/cliResourceClose.test.ts | 43 +- tests/boundary/cli/dispatcher.test.ts | 7 +- turbo.json | 1 + 20 files changed, 1282 insertions(+), 296 deletions(-) create mode 100644 scripts/record-cli-commands.mjs create mode 100644 src/cli/cliCommandRegistration.ts create mode 100644 src/cli/cliCommandRegistrations.ts create mode 100644 src/cli/cliLoader.ts create mode 100644 src/cli/cliResources.ts create mode 100644 src/cli/cliShell.ts create mode 100644 src/recordedCliCommandRegistrations.ts create mode 100644 tests/boundary/cli/cliLazyRegistration.test.ts diff --git a/docs/architecture/architecture.mermaid b/docs/architecture/architecture.mermaid index 09621773f2..825b0c9b4f 100644 --- a/docs/architecture/architecture.mermaid +++ b/docs/architecture/architecture.mermaid @@ -2,7 +2,7 @@ graph TD %% Selected layer owners and dependency/wiring paths, not a directory or import inventory. %% Current deep-provider candidates are wired in src/composition/binary.ts. subgraph "Entry Points" - CLI["src/cli.ts + src/cli/
One-shot CLI adapter
setup, diagnostics, analysis, decompilation"] + CLI["src/cli.ts + src/cli/
One-shot CLI adapter
setup, diagnostics, analysis, decompilation
cliLoader: recorded registrar per command, complete CLI otherwise"] MAIN["src/main.ts + src/main/
MCP server adapter: stdio default, streamable HTTP opt-in
config, session, process lifetime"] DISPATCH["scripts/rea.mjs
Executable dispatcher
routes mcp / mcp --http / one-shot"] end diff --git a/docs/development/tool-design.md b/docs/development/tool-design.md index f8677892ee..b4fbb589d9 100644 --- a/docs/development/tool-design.md +++ b/docs/development/tool-design.md @@ -479,7 +479,18 @@ through the producing owner and affected CLI/MCP consumers. ## Implement and evaluate Put provider wire-protocol parsing in adapters and shared CLI/MCP workflows in -the application layer. Reusable format declarations may live in pure domain +the application layer. + +CLI commands register through +[`cliCommandRegistrations.ts`](https://github.com/morluto/rea/blob/main/src/cli/cliCommandRegistrations.ts): +add the registrar to its family there and its module to `src/cli.ts`. The build +records which registrar defines each top-level command, and a one-shot command +loads only that registrar's module graph; help, completions, `--llms` and unknown +names load the complete CLI. A registrar's command definitions must not depend +on other registrars, and `tests/boundary/cli/cliLazyRegistration.test.ts` checks +that each one registers alone with the definitions the complete CLI gives it. +Modules that every registrar imports set the startup cost of every command; load +optional heavy services where a command needs them. Reusable format declarations may live in pure domain owners: [`peOptionalHeader.ts`](https://github.com/morluto/rea/blob/main/src/domain/peOptionalHeader.ts) decodes PE structure, while consumers retain machine admission, directory-address interpretation, and file/mapping bounds. Share Evidence provenance, unknown diff --git a/package.json b/package.json index d1256a70e9..d885dca380 100644 --- a/package.json +++ b/package.json @@ -344,8 +344,8 @@ "verify:solidity:metadata": "npm run build:cached && node scripts/verify-solidity-metadata.mjs", "verify:recorded:crash": "npm run build:cached && node scripts/verify-recorded-crash.mjs", "mcp-catalog:check:built": "node scripts/generate-mcp-tool-catalog.mjs --check", - "compile": "node scripts/clean-build-output.mjs && node scripts/run-typescript.mjs -p tsconfig.build.json && node scripts/record-catalog-digests.mjs && node scripts/record-mcp-catalog.mjs && node scripts/bundle-runtime.mjs", - "compile:termux": "node scripts/clean-build-output.mjs && node --max-old-space-size=3072 node_modules/typescript-termux/lib/tsc.js -p tsconfig.build.json && node scripts/record-catalog-digests.mjs && node scripts/record-mcp-catalog.mjs && node scripts/bundle-runtime.mjs", + "compile": "node scripts/clean-build-output.mjs && node scripts/run-typescript.mjs -p tsconfig.build.json && node scripts/record-catalog-digests.mjs && node scripts/record-mcp-catalog.mjs && node scripts/record-cli-commands.mjs && node scripts/bundle-runtime.mjs", + "compile:termux": "node scripts/clean-build-output.mjs && node --max-old-space-size=3072 node_modules/typescript-termux/lib/tsc.js -p tsconfig.build.json && node scripts/record-catalog-digests.mjs && node scripts/record-mcp-catalog.mjs && node scripts/record-cli-commands.mjs && node scripts/bundle-runtime.mjs", "artifacts:mcp-catalog": "node scripts/generate-mcp-tool-catalog.mjs", "artifacts:skills": "node scripts/generate-skill-metadata.mjs", "artifacts:product-catalog": "node scripts/generate-product-catalog.mjs", diff --git a/scripts/bundle-runtime.mjs b/scripts/bundle-runtime.mjs index c135343c59..1d0499c801 100644 --- a/scripts/bundle-runtime.mjs +++ b/scripts/bundle-runtime.mjs @@ -11,9 +11,14 @@ import { fileURLToPath, pathToFileURL } from "node:url"; export const bundleRuntime = async (root) => { const compiledRoot = resolve(root, "dist"); // CLI, MCP, workers and updater fixtures share one release-identity owner. - const metadataModule = join(compiledRoot, "generatedPackageMetadata.js"); + // The recorded CLI command registrars stay one replaceable build fact too. + const recordedModules = new Set( + ["generatedPackageMetadata.js", "recordedCliCommandRegistrations.js"].map( + (name) => join(compiledRoot, name), + ), + ); const isMetadataModule = (id) => - isAbsolute(id) && resolve(id) === metadataModule; + isAbsolute(id) && recordedModules.has(resolve(id)); const { rolldown, RUNTIME_MODULE_ID } = await import("rolldown"); const { imports = {} } = JSON.parse( await readFile(join(root, "package.json"), "utf8"), @@ -122,7 +127,7 @@ export const bundleRuntime = async (root) => { return `${path.replace(/\.js$/u, "").split(sep).join("__")}-[hash].js`; }, paths: (id) => - isMetadataModule(id) ? "../generatedPackageMetadata.js" : id, + isMetadataModule(id) ? `../${relative(compiledRoot, id)}` : id, minifyInternalExports: directory === "mcp", codeSplitting: true, strictExecutionOrder: true, diff --git a/scripts/record-cli-commands.mjs b/scripts/record-cli-commands.mjs new file mode 100644 index 0000000000..7ebdd59189 --- /dev/null +++ b/scripts/record-cli-commands.mjs @@ -0,0 +1,67 @@ +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { Cli } from "incur"; + +// Record which registrar defines each top-level CLI command. Run after `tsc` +// and before bundling, so the record comes from the same compiled registration +// table that one-shot commands load from. +const compiled = resolve(dirname(fileURLToPath(import.meta.url)), "../dist"); +const read = (name) => import(pathToFileURL(join(compiled, name)).href); +const [ + { CLI_COMMAND_REGISTRATIONS, CLI_REGISTRATION_ORDER }, + { createCliShell }, + { createDirectAnalysis }, +] = await Promise.all([ + read("cli/cliCommandRegistrations.js"), + read("cli/cliShell.js"), + read("composition/directAnalysis.js"), +]); + +// Registration is independent of the host account and user configuration. +const home = await mkdtemp(join(tmpdir(), "rea-cli-commands-")); +const recorded = {}; +try { + const registrations = await Promise.all( + CLI_REGISTRATION_ORDER.map(async (id) => { + const registration = CLI_COMMAND_REGISTRATIONS[id]; + return { + module: await registration.load(), + registration: { + ...registration, + register(module, context) { + const before = new Set(Cli.toCommands.get(context.cli)?.keys()); + registration.register(module, context); + for (const name of Cli.toCommands.get(context.cli)?.keys() ?? []) + if (!before.has(name)) recorded[name] = id; + }, + }, + }; + }), + ); + createCliShell({ + selectedEnvironment: { + PATH: process.env.PATH, + HOME: home, + USERPROFILE: home, + XDG_CONFIG_HOME: join(home, "config"), + XDG_CACHE_HOME: join(home, "cache"), + XDG_STATE_HOME: join(home, "state"), + }, + registrations, + createAnalysis: createDirectAnalysis, + }); +} finally { + await rm(home, { recursive: true, force: true }); +} +if (Object.keys(recorded).length === 0) + throw new Error("No CLI commands were registered"); +const sorted = Object.fromEntries( + Object.entries(recorded).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)), +); +await writeFile( + join(compiled, "recordedCliCommandRegistrations.js"), + `/** Recorded from the compiled CLI registration table by scripts/record-cli-commands.mjs; do not edit. */\nexport const RECORDED_CLI_COMMAND_REGISTRATIONS = Object.freeze(${JSON.stringify(sorted)});\n`, + "utf8", +); diff --git a/src/cli.ts b/src/cli.ts index b288cefc65..7b66580b55 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,112 +1,128 @@ -import { parseClientRegistrationCommand } from "./domain/clientRegistrationCommand.js"; -import { - serveMcpRegistration, - closeCliMcpRegistrationResources, -} from "./cli/mcpRegistration.js"; -import { registerToolFamilies } from "./contracts/toolFamilyIds.js"; -import { registerGpuCommands } from "./cli/gpuCommands.js"; -import { registerNativeFunctionMatchingCommands } from "./cli/nativeFunctionMatchingCommands.js"; -import { registerCaptureCommands } from "./cli/captureCommands.js"; -import { registerSignalCommands } from "./cli/signalCommands.js"; -import { registerWasmCommands } from "./cli/wasmCommands.js"; -import { registerApkSignatureCommands } from "./cli/apkSignatureCommands.js"; -import { snapshotEnvironment } from "./process/snapshotEnvironment.js"; -import { createDirectAnalysis } from "./composition/directAnalysis.js"; -import { registerEvmCommands } from "./cli/evmCommands.js"; -import { registerDesktopApplicationCommands } from "./cli/desktopApplicationCommands.js"; -import { registerReleaseExposureCommands } from "./cli/releaseExposureCommands.js"; -import { registerGoCommands } from "./cli/goCommands.js"; -import { registerModelCommands } from "./cli/modelCommands.js"; -import { registerIl2cppCommands } from "./cli/il2cppCommands.js"; -import { registerRustCommands } from "./cli/rustCommands.js"; -import { registerPythonPycCommands } from "./cli/pythonPycCommands.js"; -import { registerErlangCommands } from "./cli/erlangCommands.js"; -import { registerV8CodeCacheCommands } from "./cli/v8CodeCacheCommands.js"; -import { registerSqliteCommands } from "./cli/sqliteCommands.js"; -import { registerAnalysisViewCommands } from "./cli/analysisViewCommands.js"; -import { registerBinaryDiagnosticsCommands } from "./cli/binaryDiagnosticsCommands.js"; -import { registerFridaCommands } from "./cli/fridaCommands.js"; import type { FridaInstrumentationPort } from "./application/frida/FridaInstrumentationPort.js"; -import { Cli, z } from "incur"; import { - adaptCliOutputArgumentsForIncur, - normalizeCliArguments, -} from "./cliBuiltinArguments.js"; - -import { createLogger } from "./logger.js"; -import { parseLogLevel } from "./config/processStart.js"; -import { PRODUCT_IDENTITY } from "./identity.js"; -import { currentClientRegistrationCommand } from "./application/ClientRegistrationIdentity.js"; -import { registerSetupCommands } from "./cli/setupCommands.js"; -import { registerCoreAnalysisCommands } from "./cli/coreAnalysisCommands.js"; -import { registerUtilityCommands } from "./cli/utilityCommands.js"; -import { registerArtifactCommands } from "./cli/artifactCommands.js"; -import { registerManagedCommands } from "./cli/managedCommands.js"; -import { registerFirmwareCommands } from "./cli/firmwareCommands.js"; -import { registerAndroidCommands } from "./cli/androidCommands.js"; -import { registerApktoolCommands } from "./cli/apktoolCommands.js"; -import { registerPcapCommands } from "./cli/pcapCommands.js"; -import { registerFlutterCommands } from "./cli/flutterCommands.js"; -import { registerJebCommands } from "./cli/jebCommands.js"; -import { registerAdbCommands } from "./cli/adbCommands.js"; -import { registerEvidenceCommands } from "./cliEvidenceCommands.js"; -import { registerEnvironmentCommands } from "./cli/environmentCommands.js"; -import { registerProcessCommands } from "./cli/processCommands.js"; -import { registerDesktopCaptureCommands } from "./cli/desktopCaptureCommands.js"; -import { registerBrowserCommands } from "./cli/browserCommands.js"; -import { registerWebScriptCommands } from "./cli/webScriptCommands.js"; -import { registerWebModuleCommands } from "./cli/webModuleCommands.js"; -import { registerWebSourceLocationCommands } from "./cli/webSourceLocationCommands.js"; -import { registerWebRuntimeCommands } from "./cli/webRuntimeCommands.js"; -import { registerWebNetworkCaptureCommands } from "./cli/webNetworkCaptureCommands.js"; -import { registerJavaScriptRecoveryCommands } from "./cli/javascriptRecoveryCommands.js"; -import { registerAdvancedBrowserCommands } from "./cli/browserAdvancedCommands.js"; -import { registerBrowserScenarioCommands } from "./cliBrowserScenarioCommands.js"; -import { registerElectronCommands } from "./cli/electronCommands.js"; -import { registerJavaScriptRuntimeObservationCommands } from "./cli/javascriptRuntimeCommands.js"; -import { registerApplicationCommands } from "./cli/applicationCommands.js"; -import { presentOmittableDefaults } from "./cli/omittableDefaults.js"; + CLI_COMMAND_REGISTRATIONS, + CLI_REGISTRATION_ORDER, + type CliCommandModules, +} from "./cli/cliCommandRegistrations.js"; +import type { CliResultOutput } from "./cli/cliResultOutput.js"; +import { createCliShell } from "./cli/cliShell.js"; import type { CliInstance } from "./cli/types.js"; -import { - streamCliCommandResults, - type CliResultOutput, -} from "./cli/cliResultOutput.js"; -import { registerReverseEngineeringCommands } from "./cli/reverseEngineeringCommands.js"; -import { registerCutterCommands } from "./cli/cutterCommands.js"; -import { excludeDeclaredSensitiveArguments } from "./cli/explicitSensitiveArguments.js"; -import { validateCliArguments } from "./cli/strictPositionals.js"; -import { - encodeCliArgumentErrors, - presentCliArgumentErrors, -} from "./cli/argumentErrors.js"; -import { completionHelpNote, createCliCompletions } from "./cli/completions.js"; -import { reportUnknownMcpSubcommand } from "./cli/mcpBuiltinCommands.js"; -import { - closeCliGuidanceResources, - registerGuidanceCommands, -} from "./cli/guidanceCommands.js"; -import { closeCliJavaScriptResources } from "./cli/javascriptApplicationAnalysis.js"; -import { registerGuidedWorkflowCommands } from "./cli/guidedWorkflowCommands.js"; +import { createDirectAnalysis } from "./composition/directAnalysis.js"; +import * as setupModule from "./cli/setupCommands.js"; +import * as coreAnalysisModule from "./cli/coreAnalysisCommands.js"; +import * as reverseEngineeringModule from "./cli/reverseEngineeringCommands.js"; +import * as cutterModule from "./cli/cutterCommands.js"; +import * as utilityModule from "./cli/utilityCommands.js"; +import * as binaryDiagnosticsModule from "./cli/binaryDiagnosticsCommands.js"; +import * as wasmModule from "./cli/wasmCommands.js"; +import * as apkSignatureModule from "./cli/apkSignatureCommands.js"; +import * as captureModule from "./cli/captureCommands.js"; +import * as signalModule from "./cli/signalCommands.js"; +import * as evmModule from "./cli/evmCommands.js"; +import * as desktopApplicationModule from "./cli/desktopApplicationCommands.js"; +import * as releaseExposureModule from "./cli/releaseExposureCommands.js"; +import * as goModule from "./cli/goCommands.js"; +import * as il2cppModule from "./cli/il2cppCommands.js"; +import * as modelModule from "./cli/modelCommands.js"; +import * as gpuModule from "./cli/gpuCommands.js"; +import * as rustModule from "./cli/rustCommands.js"; +import * as pythonPycModule from "./cli/pythonPycCommands.js"; +import * as erlangModule from "./cli/erlangCommands.js"; +import * as v8CodeCacheModule from "./cli/v8CodeCacheCommands.js"; +import * as sqliteModule from "./cli/sqliteCommands.js"; +import * as artifactModule from "./cli/artifactCommands.js"; +import * as managedModule from "./cli/managedCommands.js"; +import * as firmwareModule from "./cli/firmwareCommands.js"; +import * as androidModule from "./cli/androidCommands.js"; +import * as pcapModule from "./cli/pcapCommands.js"; +import * as flutterModule from "./cli/flutterCommands.js"; +import * as fridaModule from "./cli/fridaCommands.js"; +import * as apktoolModule from "./cli/apktoolCommands.js"; +import * as jebModule from "./cli/jebCommands.js"; +import * as adbModule from "./cli/adbCommands.js"; +import * as browserModule from "./cli/browserCommands.js"; +import * as webRuntimeModule from "./cli/webRuntimeCommands.js"; +import * as webNetworkCaptureModule from "./cli/webNetworkCaptureCommands.js"; +import * as advancedBrowserModule from "./cli/browserAdvancedCommands.js"; +import * as browserScenarioModule from "./cliBrowserScenarioCommands.js"; +import * as electronModule from "./cli/electronCommands.js"; +import * as javaScriptRuntimeObservationModule from "./cli/javascriptRuntimeCommands.js"; +import * as guidanceModule from "./cli/guidanceCommands.js"; +import * as guidedWorkflowModule from "./cli/guidedWorkflowCommands.js"; +import * as analysisViewModule from "./cli/analysisViewCommands.js"; +import * as evidenceModule from "./cliEvidenceCommands.js"; +import * as environmentModule from "./cli/environmentCommands.js"; +import * as processModule from "./cli/processCommands.js"; +import * as desktopCaptureModule from "./cli/desktopCaptureCommands.js"; +import * as javaScriptRecoveryModule from "./cli/javascriptRecoveryCommands.js"; +import * as webScriptModule from "./cli/webScriptCommands.js"; +import * as webModuleModule from "./cli/webModuleCommands.js"; +import * as webSourceLocationModule from "./cli/webSourceLocationCommands.js"; +import * as applicationModule from "./cli/applicationCommands.js"; +import * as nativeFunctionMatchingModule from "./cli/nativeFunctionMatchingCommands.js"; -/** Close every process-owned resource used by CLI commands. */ -export const closeCliResources = async (): Promise => { - const results = await Promise.allSettled([ - closeCliJavaScriptResources(), - closeCliGuidanceResources(), - closeCliMcpRegistrationResources(), - ]); - const failures = results.flatMap((result) => - result.status === "rejected" ? [result.reason] : [], - ); - if (failures.length === 1) throw failures[0]; - if (failures.length > 1) - throw new AggregateError(failures, "CLI resource cleanup failed"); +export { closeCliResources } from "./cli/cliResources.js"; + +const CLI_COMMAND_MODULES: CliCommandModules = { + setup: setupModule, + coreAnalysis: coreAnalysisModule, + reverseEngineering: reverseEngineeringModule, + cutter: cutterModule, + utility: utilityModule, + binaryDiagnostics: binaryDiagnosticsModule, + wasm: wasmModule, + apkSignature: apkSignatureModule, + capture: captureModule, + signal: signalModule, + evm: evmModule, + desktopApplication: desktopApplicationModule, + releaseExposure: releaseExposureModule, + go: goModule, + il2cpp: il2cppModule, + model: modelModule, + gpu: gpuModule, + rust: rustModule, + pythonPyc: pythonPycModule, + erlang: erlangModule, + v8CodeCache: v8CodeCacheModule, + sqlite: sqliteModule, + artifact: artifactModule, + managed: managedModule, + firmware: firmwareModule, + android: androidModule, + pcap: pcapModule, + flutter: flutterModule, + frida: fridaModule, + apktool: apktoolModule, + jeb: jebModule, + adb: adbModule, + browser: browserModule, + webRuntime: webRuntimeModule, + webNetworkCapture: webNetworkCaptureModule, + advancedBrowser: advancedBrowserModule, + browserScenario: browserScenarioModule, + electron: electronModule, + javaScriptRuntimeObservation: javaScriptRuntimeObservationModule, + guidance: guidanceModule, + guidedWorkflow: guidedWorkflowModule, + analysisView: analysisViewModule, + evidence: evidenceModule, + environment: environmentModule, + process: processModule, + desktopCapture: desktopCaptureModule, + javaScriptRecovery: javaScriptRecoveryModule, + webScript: webScriptModule, + webModule: webModuleModule, + webSourceLocation: webSourceLocationModule, + application: applicationModule, + nativeFunctionMatching: nativeFunctionMatchingModule, }; /** - * Build the one-shot Incur CLI without starting Hopper at import time. + * Build the complete one-shot Incur CLI without starting Hopper at import time. * Analysis commands acquire and close their own sessions; bare `mcp` and * `--mcp` are intercepted by the executable dispatcher before this module loads. + * One-shot commands load only their own registrars; see `cli/cliLoader.ts`. */ export const createCli = ( selectedEnvironment: Readonly< @@ -114,172 +130,14 @@ export const createCli = ( > = process.env, resultOutput?: CliResultOutput, fridaInstrumentation?: FridaInstrumentationPort, -): CliInstance => { - const environment = snapshotEnvironment(selectedEnvironment); - const analysis = createDirectAnalysis(environment); - const defaultRegistration = parseClientRegistrationCommand( - currentClientRegistrationCommand({ environment }), - ); - // The executable reports an invalid level before every command except doctor. - // Doctor keeps going so its readiness report can name REA_LOG_LEVEL. - const level = - environment.REA_LOG_LEVEL === undefined - ? undefined - : parseLogLevel(environment.REA_LOG_LEVEL); - const logger = createLogger( - "cli", - level?.ok === true ? level.value : "silent", - ); - const timedCli = Cli.create(PRODUCT_IDENTITY.cliBinary, { - version: PRODUCT_IDENTITY.packageVersion, - description: "Reverse engineer anything from your terminal or agent.", - globals: z.object({ - timings: z - .boolean() - .default(false) - .describe("Write measured command timing to stderr."), - }), - mcp: { - instructions: - "Ask what software, artifact, protocol, or behavior the user wants to understand, then choose the available investigation capabilities that can produce evidence.", - }, - sync: false, - update: { - // Incur's update:false only suppresses notices. Reject its explicit - // installer so ambiguous flags cannot bypass REA's verified workflow. - install: () => { - throw new Error( - "Use 'rea update' to update the identified installation. The '--update' alias accepts only global output flags.", - ); - }, - }, +): CliInstance => + createCliShell({ + selectedEnvironment, + resultOutput, + fridaInstrumentation, + createAnalysis: createDirectAnalysis, + registrations: CLI_REGISTRATION_ORDER.map((id) => ({ + registration: CLI_COMMAND_REGISTRATIONS[id], + module: CLI_COMMAND_MODULES[id], + })), }); - - timedCli.use(async (context, next) => { - if (!context.globals.timings) return next(); - const started = performance.now(); - try { - await next(); - } finally { - // This interval covers CLI command dispatch, including result delivery. - // It is not a provider-only or analysis-only measurement. - process.stderr.write( - `${JSON.stringify({ rea_timings: { version: 1, command_ms: Math.round(performance.now() - started) } })}\n`, - ); - } - }); - // Command registrars do not inspect global options; preserve their existing - // type while Incur owns parsing and the middleware owns this added flag. - const cli = timedCli as unknown as CliInstance; - - registerSetupCommands(cli, logger, environment); - registerToolFamilies({ - direct: () => - registerCoreAnalysisCommands( - cli, - logger, - analysis.runDirectAnalysis, - environment, - ), - // Enhanced workflows are MCP tools. Their CLI entry points live with core analysis. - enhanced: () => undefined, - native: () => { - registerReverseEngineeringCommands(cli, logger, environment); - registerCutterCommands(cli, logger, environment); - registerUtilityCommands(cli, logger, environment, analysis); - registerBinaryDiagnosticsCommands(cli, logger, environment); - }, - artifact: () => { - registerWasmCommands(cli, logger, environment); - registerApkSignatureCommands(cli, logger, environment); - registerCaptureCommands(cli, logger, environment); - registerSignalCommands(cli, logger, environment); - registerEvmCommands(cli, logger, environment); - registerDesktopApplicationCommands(cli, logger); - registerReleaseExposureCommands(cli, logger, environment); - registerGoCommands(cli, logger); - registerIl2cppCommands(cli, logger); - registerModelCommands(cli, logger); - registerGpuCommands(cli, logger); - registerRustCommands(cli, logger); - registerPythonPycCommands(cli, logger); - registerErlangCommands(cli, logger); - registerV8CodeCacheCommands(cli, logger); - registerSqliteCommands(cli, logger, environment); - registerArtifactCommands(cli, logger, analysis.runProviderAnalysis); - }, - managed: () => - registerManagedCommands(cli, logger, analysis.runProviderAnalysis), - firmware: () => registerFirmwareCommands(cli, logger, environment), - android: () => registerAndroidCommands(cli, logger, environment), - pcap: () => registerPcapCommands(cli, logger, environment), - flutter: () => registerFlutterCommands(cli, logger), - frida: () => registerFridaCommands(cli, logger, fridaInstrumentation), - apktool: () => registerApktoolCommands(cli, logger, environment), - jeb: () => registerJebCommands(cli, logger, environment), - adb: () => registerAdbCommands(cli, logger, environment), - browser: () => { - registerBrowserCommands(cli, logger); - registerWebRuntimeCommands(cli, logger); - registerWebNetworkCaptureCommands(cli, logger, environment); - registerAdvancedBrowserCommands(cli, logger); - registerBrowserScenarioCommands(cli, logger, environment); - }, - electron: () => registerElectronCommands(cli, logger, environment), - "javascript-runtime": () => - registerJavaScriptRuntimeObservationCommands(cli, logger, environment), - application: () => { - registerGuidanceCommands(cli, logger); - registerGuidedWorkflowCommands(cli, logger); - registerAnalysisViewCommands(cli, logger); - registerEvidenceCommands(cli, logger); - registerEnvironmentCommands(cli, logger, environment); - registerProcessCommands(cli, logger, environment); - registerDesktopCaptureCommands(cli, logger, environment); - registerJavaScriptRecoveryCommands(cli, logger, environment); - registerWebScriptCommands(cli, logger); - registerWebModuleCommands(cli, logger, environment); - registerWebSourceLocationCommands(cli, logger, environment); - registerApplicationCommands(cli, logger); - }, - session: () => - registerNativeFunctionMatchingCommands(cli, logger, environment), - }); - presentOmittableDefaults(cli); - if (resultOutput !== undefined) streamCliCommandResults(cli, resultOutput); - const serveStrict = validateCliArguments(cli); - presentCliArgumentErrors(cli); - const serveDeclared = excludeDeclaredSensitiveArguments(cli); - encodeCliArgumentErrors(cli); - const serve = cli.serve.bind(cli); - const complete = createCliCompletions( - cli as unknown as Parameters[0], - environment, - ); - cli.serve = (arguments_ = process.argv.slice(2), options) => { - if (complete(arguments_, options ?? {})) return Promise.resolve(); - const normalized = normalizeCliArguments(arguments_); - const registration = serveMcpRegistration( - normalized, - environment, - defaultRegistration, - options ?? {}, - ); - if (registration !== undefined) return registration; - const result = serveStrict(normalized, options ?? {}, (tokens, selected) => - serveDeclared(tokens, selected, (declared, declaredOptions) => - reportUnknownMcpSubcommand(cli.name, declared, declaredOptions) - ? Promise.resolve() - : serve(adaptCliOutputArgumentsForIncur(declared), declaredOptions), - ), - ); - const note = completionHelpNote(normalized); - if (note === undefined) return result; - return result.then(() => { - const write = - options?.stdout ?? ((text: string) => process.stdout.write(text)); - write(`\n${note}`); - }); - }; - return cli; -}; diff --git a/src/cli/cliCommandRegistration.ts b/src/cli/cliCommandRegistration.ts new file mode 100644 index 0000000000..fc5c25796f --- /dev/null +++ b/src/cli/cliCommandRegistration.ts @@ -0,0 +1,41 @@ +import type { Logger } from "pino"; + +import type { FridaInstrumentationPort } from "../application/frida/FridaInstrumentationPort.js"; +import type { DirectAnalysis } from "../composition/directAnalysis.js"; +import type { CliInstance } from "./types.js"; + +/** What a command registrar receives. */ +export interface CliRegistrationContext { + readonly cli: CliInstance; + readonly logger: Logger; + readonly environment: Readonly>; + readonly fridaInstrumentation: FridaInstrumentationPort | undefined; +} + +/** Context for registrars that run direct or provider analysis. */ +export interface CliAnalysisRegistrationContext extends CliRegistrationContext { + readonly analysis: DirectAnalysis; +} + +/** One command registrar: the module that defines it and how to call it. */ +export type CliCommandRegistration = + | { + readonly analysis: false; + load(): Promise; + register(module: Module, context: CliRegistrationContext): void; + } + | { + readonly analysis: true; + load(): Promise; + register(module: Module, context: CliAnalysisRegistrationContext): void; + }; + +export const plain = ( + load: () => Promise, + register: (module: Module, context: CliRegistrationContext) => void, +) => ({ analysis: false, load, register }) as const; + +export const withAnalysis = ( + load: () => Promise, + register: (module: Module, context: CliAnalysisRegistrationContext) => void, +) => ({ analysis: true, load, register }) as const; diff --git a/src/cli/cliCommandRegistrations.ts b/src/cli/cliCommandRegistrations.ts new file mode 100644 index 0000000000..a8039b2dbe --- /dev/null +++ b/src/cli/cliCommandRegistrations.ts @@ -0,0 +1,485 @@ +import { + TOOL_FAMILY_IDS, + type ToolFamilyId, +} from "../contracts/toolFamilyIds.js"; +import { plain, withAnalysis } from "./cliCommandRegistration.js"; + +/** Every CLI command registrar; see `cliLoader.ts` for one-shot loading. */ +export const CLI_COMMAND_REGISTRATIONS = { + setup: plain( + () => import("./setupCommands.js"), + (module, context) => + module.registerSetupCommands( + context.cli, + context.logger, + context.environment, + ), + ), + coreAnalysis: withAnalysis( + () => import("./coreAnalysisCommands.js"), + (module, context) => + module.registerCoreAnalysisCommands( + context.cli, + context.logger, + context.analysis.runDirectAnalysis, + context.environment, + ), + ), + reverseEngineering: plain( + () => import("./reverseEngineeringCommands.js"), + (module, context) => + module.registerReverseEngineeringCommands( + context.cli, + context.logger, + context.environment, + ), + ), + cutter: plain( + () => import("./cutterCommands.js"), + (module, context) => + module.registerCutterCommands( + context.cli, + context.logger, + context.environment, + ), + ), + utility: withAnalysis( + () => import("./utilityCommands.js"), + (module, context) => + module.registerUtilityCommands( + context.cli, + context.logger, + context.environment, + context.analysis, + ), + ), + binaryDiagnostics: plain( + () => import("./binaryDiagnosticsCommands.js"), + (module, context) => + module.registerBinaryDiagnosticsCommands( + context.cli, + context.logger, + context.environment, + ), + ), + wasm: plain( + () => import("./wasmCommands.js"), + (module, context) => + module.registerWasmCommands( + context.cli, + context.logger, + context.environment, + ), + ), + apkSignature: plain( + () => import("./apkSignatureCommands.js"), + (module, context) => + module.registerApkSignatureCommands( + context.cli, + context.logger, + context.environment, + ), + ), + capture: plain( + () => import("./captureCommands.js"), + (module, context) => + module.registerCaptureCommands( + context.cli, + context.logger, + context.environment, + ), + ), + signal: plain( + () => import("./signalCommands.js"), + (module, context) => + module.registerSignalCommands( + context.cli, + context.logger, + context.environment, + ), + ), + evm: plain( + () => import("./evmCommands.js"), + (module, context) => + module.registerEvmCommands( + context.cli, + context.logger, + context.environment, + ), + ), + desktopApplication: plain( + () => import("./desktopApplicationCommands.js"), + (module, context) => + module.registerDesktopApplicationCommands(context.cli, context.logger), + ), + releaseExposure: plain( + () => import("./releaseExposureCommands.js"), + (module, context) => + module.registerReleaseExposureCommands( + context.cli, + context.logger, + context.environment, + ), + ), + go: plain( + () => import("./goCommands.js"), + (module, context) => module.registerGoCommands(context.cli, context.logger), + ), + il2cpp: plain( + () => import("./il2cppCommands.js"), + (module, context) => + module.registerIl2cppCommands(context.cli, context.logger), + ), + model: plain( + () => import("./modelCommands.js"), + (module, context) => + module.registerModelCommands(context.cli, context.logger), + ), + gpu: plain( + () => import("./gpuCommands.js"), + (module, context) => + module.registerGpuCommands(context.cli, context.logger), + ), + rust: plain( + () => import("./rustCommands.js"), + (module, context) => + module.registerRustCommands(context.cli, context.logger), + ), + pythonPyc: plain( + () => import("./pythonPycCommands.js"), + (module, context) => + module.registerPythonPycCommands(context.cli, context.logger), + ), + erlang: plain( + () => import("./erlangCommands.js"), + (module, context) => + module.registerErlangCommands(context.cli, context.logger), + ), + v8CodeCache: plain( + () => import("./v8CodeCacheCommands.js"), + (module, context) => + module.registerV8CodeCacheCommands(context.cli, context.logger), + ), + sqlite: plain( + () => import("./sqliteCommands.js"), + (module, context) => + module.registerSqliteCommands( + context.cli, + context.logger, + context.environment, + ), + ), + artifact: withAnalysis( + () => import("./artifactCommands.js"), + (module, context) => + module.registerArtifactCommands( + context.cli, + context.logger, + context.analysis.runProviderAnalysis, + ), + ), + managed: withAnalysis( + () => import("./managedCommands.js"), + (module, context) => + module.registerManagedCommands( + context.cli, + context.logger, + context.analysis.runProviderAnalysis, + ), + ), + firmware: plain( + () => import("./firmwareCommands.js"), + (module, context) => + module.registerFirmwareCommands( + context.cli, + context.logger, + context.environment, + ), + ), + android: plain( + () => import("./androidCommands.js"), + (module, context) => + module.registerAndroidCommands( + context.cli, + context.logger, + context.environment, + ), + ), + pcap: plain( + () => import("./pcapCommands.js"), + (module, context) => + module.registerPcapCommands( + context.cli, + context.logger, + context.environment, + ), + ), + flutter: plain( + () => import("./flutterCommands.js"), + (module, context) => + module.registerFlutterCommands(context.cli, context.logger), + ), + frida: plain( + () => import("./fridaCommands.js"), + (module, context) => + module.registerFridaCommands( + context.cli, + context.logger, + context.fridaInstrumentation, + ), + ), + apktool: plain( + () => import("./apktoolCommands.js"), + (module, context) => + module.registerApktoolCommands( + context.cli, + context.logger, + context.environment, + ), + ), + jeb: plain( + () => import("./jebCommands.js"), + (module, context) => + module.registerJebCommands( + context.cli, + context.logger, + context.environment, + ), + ), + adb: plain( + () => import("./adbCommands.js"), + (module, context) => + module.registerAdbCommands( + context.cli, + context.logger, + context.environment, + ), + ), + browser: plain( + () => import("./browserCommands.js"), + (module, context) => + module.registerBrowserCommands(context.cli, context.logger), + ), + webRuntime: plain( + () => import("./webRuntimeCommands.js"), + (module, context) => + module.registerWebRuntimeCommands(context.cli, context.logger), + ), + webNetworkCapture: plain( + () => import("./webNetworkCaptureCommands.js"), + (module, context) => + module.registerWebNetworkCaptureCommands( + context.cli, + context.logger, + context.environment, + ), + ), + advancedBrowser: plain( + () => import("./browserAdvancedCommands.js"), + (module, context) => + module.registerAdvancedBrowserCommands(context.cli, context.logger), + ), + browserScenario: plain( + () => import("../cliBrowserScenarioCommands.js"), + (module, context) => + module.registerBrowserScenarioCommands( + context.cli, + context.logger, + context.environment, + ), + ), + electron: plain( + () => import("./electronCommands.js"), + (module, context) => + module.registerElectronCommands( + context.cli, + context.logger, + context.environment, + ), + ), + javaScriptRuntimeObservation: plain( + () => import("./javascriptRuntimeCommands.js"), + (module, context) => + module.registerJavaScriptRuntimeObservationCommands( + context.cli, + context.logger, + context.environment, + ), + ), + guidance: plain( + () => import("./guidanceCommands.js"), + (module, context) => + module.registerGuidanceCommands(context.cli, context.logger), + ), + guidedWorkflow: plain( + () => import("./guidedWorkflowCommands.js"), + (module, context) => + module.registerGuidedWorkflowCommands(context.cli, context.logger), + ), + analysisView: plain( + () => import("./analysisViewCommands.js"), + (module, context) => + module.registerAnalysisViewCommands(context.cli, context.logger), + ), + evidence: plain( + () => import("../cliEvidenceCommands.js"), + (module, context) => + module.registerEvidenceCommands(context.cli, context.logger), + ), + environment: plain( + () => import("./environmentCommands.js"), + (module, context) => + module.registerEnvironmentCommands( + context.cli, + context.logger, + context.environment, + ), + ), + process: plain( + () => import("./processCommands.js"), + (module, context) => + module.registerProcessCommands( + context.cli, + context.logger, + context.environment, + ), + ), + desktopCapture: plain( + () => import("./desktopCaptureCommands.js"), + (module, context) => + module.registerDesktopCaptureCommands( + context.cli, + context.logger, + context.environment, + ), + ), + javaScriptRecovery: plain( + () => import("./javascriptRecoveryCommands.js"), + (module, context) => + module.registerJavaScriptRecoveryCommands( + context.cli, + context.logger, + context.environment, + ), + ), + webScript: plain( + () => import("./webScriptCommands.js"), + (module, context) => + module.registerWebScriptCommands(context.cli, context.logger), + ), + webModule: plain( + () => import("./webModuleCommands.js"), + (module, context) => + module.registerWebModuleCommands( + context.cli, + context.logger, + context.environment, + ), + ), + webSourceLocation: plain( + () => import("./webSourceLocationCommands.js"), + (module, context) => + module.registerWebSourceLocationCommands( + context.cli, + context.logger, + context.environment, + ), + ), + application: plain( + () => import("./applicationCommands.js"), + (module, context) => + module.registerApplicationCommands(context.cli, context.logger), + ), + nativeFunctionMatching: plain( + () => import("./nativeFunctionMatchingCommands.js"), + (module, context) => + module.registerNativeFunctionMatchingCommands( + context.cli, + context.logger, + context.environment, + ), + ), +}; + +export type CliCommandRegistrationId = keyof typeof CLI_COMMAND_REGISTRATIONS; + +/** Each registrar's module, as `load` resolves it. */ +export type CliCommandModules = { + readonly [Id in CliCommandRegistrationId]: Awaited< + ReturnType<(typeof CLI_COMMAND_REGISTRATIONS)[Id]["load"]> + >; +}; + +/** + * Registrars of setup and of each published tool family. Every family id must + * appear, so a new family decides its CLI commands; enhanced workflows are + * MCP tools whose CLI entry points live with core analysis. + */ +const CLI_FAMILY_REGISTRATIONS: { + readonly [ + Family in "setup" | ToolFamilyId + ]: readonly CliCommandRegistrationId[]; +} = { + setup: ["setup"], + direct: ["coreAnalysis"], + enhanced: [], + native: ["reverseEngineering", "cutter", "utility", "binaryDiagnostics"], + artifact: [ + "wasm", + "apkSignature", + "capture", + "signal", + "evm", + "desktopApplication", + "releaseExposure", + "go", + "il2cpp", + "model", + "gpu", + "rust", + "pythonPyc", + "erlang", + "v8CodeCache", + "sqlite", + "artifact", + ], + managed: ["managed"], + firmware: ["firmware"], + android: ["android"], + pcap: ["pcap"], + flutter: ["flutter"], + frida: ["frida"], + apktool: ["apktool"], + jeb: ["jeb"], + adb: ["adb"], + browser: [ + "browser", + "webRuntime", + "webNetworkCapture", + "advancedBrowser", + "browserScenario", + ], + electron: ["electron"], + "javascript-runtime": ["javaScriptRuntimeObservation"], + application: [ + "guidance", + "guidedWorkflow", + "analysisView", + "evidence", + "environment", + "process", + "desktopCapture", + "javaScriptRecovery", + "webScript", + "webModule", + "webSourceLocation", + "application", + ], + session: ["nativeFunctionMatching"], +}; + +/** Registration order: setup, then each family in catalog order. */ +export const CLI_REGISTRATION_ORDER: readonly CliCommandRegistrationId[] = [ + "setup" as const, + ...TOOL_FAMILY_IDS, +].flatMap((family) => CLI_FAMILY_REGISTRATIONS[family]); diff --git a/src/cli/cliLoader.ts b/src/cli/cliLoader.ts new file mode 100644 index 0000000000..42cc3d09d3 --- /dev/null +++ b/src/cli/cliLoader.ts @@ -0,0 +1,98 @@ +import { + cliCompletionShell, + splitIncurBuiltinArguments, +} from "../cliBuiltinArguments.js"; +import { RECORDED_CLI_COMMAND_REGISTRATIONS } from "../recordedCliCommandRegistrations.js"; +import { + CLI_COMMAND_REGISTRATIONS, + type CliCommandRegistrationId, +} from "./cliCommandRegistrations.js"; +import type { CliResultOutput } from "./cliResultOutput.js"; +import type { CliInstance } from "./types.js"; + +/** Global switches that change only how a command's result is written. */ +const COMMAND_OUTPUT_FLAGS: ReadonlySet = new Set([ + "--json", + "--full-output", + "--token-count", + "--timings", + "--no-timings", +]); + +const isRegistrationId = (id: string): id is CliCommandRegistrationId => + Object.hasOwn(CLI_COMMAND_REGISTRATIONS, id); + +/** + * The registrar that defines the command these arguments run, or undefined + * when the complete CLI must answer. Help, schemas, `--llms`, completions, + * Incur builtins, unknown names ("did you mean"), and a missing or unusable + * recording all describe or search every command. + */ +export const selectCliRegistration = ( + arguments_: readonly string[], + environment: Readonly, + recorded: Readonly> | undefined, +): CliCommandRegistrationId | undefined => { + if (recorded === undefined || cliCompletionShell(environment) !== undefined) + return undefined; + const { command, flags } = splitIncurBuiltinArguments(arguments_); + for (const flag of flags) + if (!COMMAND_OUTPUT_FLAGS.has(flag) && !flag.startsWith("--timings=")) + return undefined; + const name = command[0]; + if (name === undefined || !Object.hasOwn(recorded, name)) return undefined; + const id = recorded[name]!; + return isRegistrationId(id) ? id : undefined; +}; + +/** The command runtime one invocation loaded, before its CLI is built. */ +export interface CliRuntime { + createCli(resultOutput: CliResultOutput | undefined): CliInstance; + /** Closes the process-owned resources of this runtime's commands. */ + closeCliResources(): Promise; +} + +/** + * Load the command runtime for one invocation. A recorded top-level command + * loads only its registrar's module graph; anything else loads the complete + * CLI. + */ +export const loadCliRuntime = async ( + arguments_: readonly string[], + environment: NodeJS.ProcessEnv, +): Promise => { + const id = selectCliRegistration( + arguments_, + environment, + RECORDED_CLI_COMMAND_REGISTRATIONS, + ); + if (id === undefined) { + const { createCli, closeCliResources } = await import("../cli.js"); + return { + createCli: (resultOutput) => createCli(environment, resultOutput), + closeCliResources, + }; + } + const registration = CLI_COMMAND_REGISTRATIONS[id]; + const [module, { createCliShell }, { closeCliResources }, analysis] = + await Promise.all([ + registration.load(), + import("./cliShell.js"), + import("./cliResources.js"), + registration.analysis + ? import("../composition/directAnalysis.js") + : undefined, + ]); + return { + createCli: (resultOutput) => + createCliShell({ + selectedEnvironment: environment, + resultOutput, + registrations: [{ registration, module }], + ...(analysis === undefined + ? {} + : { createAnalysis: analysis.createDirectAnalysis }), + }), + closeCliResources, + }; +}; diff --git a/src/cli/cliResources.ts b/src/cli/cliResources.ts new file mode 100644 index 0000000000..10b1a5f6f0 --- /dev/null +++ b/src/cli/cliResources.ts @@ -0,0 +1,45 @@ +/** Closers of process-owned CLI resources, by owner name. */ +export class CliResourceOwners { + readonly #closers = new Map Promise>(); + + /** Register an owner's closer; a repeated name keeps its first closer. */ + own(name: string, close: () => Promise): void { + if (!this.#closers.has(name)) this.#closers.set(name, close); + } + + names(): readonly string[] { + return [...this.#closers.keys()]; + } + + /** Settle every owner, then report each failure. */ + async close(): Promise { + const results = await Promise.allSettled( + [...this.#closers.values()].map((close) => close()), + ); + const failures = results.flatMap((result) => + result.status === "rejected" ? [result.reason] : [], + ); + if (failures.length === 1) throw failures[0]; + if (failures.length > 1) + throw new AggregateError(failures, "CLI resource cleanup failed"); + } +} + +/** + * Process-owned CLI resources. Each owning module registers its closer when it + * loads, so a CLI that loads only some command families closes exactly the + * owners it loaded without importing the others. + */ +const processOwners = new CliResourceOwners(); + +export const ownCliProcessResource = ( + name: string, + close: () => Promise, +): void => processOwners.own(name, close); + +/** Owners registered in this process, in registration order. */ +export const cliProcessResourceOwners = (): readonly string[] => + processOwners.names(); + +/** Close every process-owned resource used by CLI commands. */ +export const closeCliResources = (): Promise => processOwners.close(); diff --git a/src/cli/cliShell.ts b/src/cli/cliShell.ts new file mode 100644 index 0000000000..60f3351c80 --- /dev/null +++ b/src/cli/cliShell.ts @@ -0,0 +1,184 @@ +import { Cli, z } from "incur"; + +import type { FridaInstrumentationPort } from "../application/frida/FridaInstrumentationPort.js"; +import { currentClientRegistrationCommand } from "../application/ClientRegistrationIdentity.js"; +import { + adaptCliOutputArgumentsForIncur, + normalizeCliArguments, +} from "../cliBuiltinArguments.js"; +import type { DirectAnalysis } from "../composition/directAnalysis.js"; +import { parseLogLevel } from "../config/processStart.js"; +import { parseClientRegistrationCommand } from "../domain/clientRegistrationCommand.js"; +import { PRODUCT_IDENTITY } from "../identity.js"; +import { createLogger } from "../logger.js"; +import { snapshotEnvironment } from "../process/snapshotEnvironment.js"; +import { + encodeCliArgumentErrors, + presentCliArgumentErrors, +} from "./argumentErrors.js"; +import type { + CliCommandRegistration, + CliRegistrationContext, +} from "./cliCommandRegistration.js"; +import { + streamCliCommandResults, + type CliResultOutput, +} from "./cliResultOutput.js"; +import { completionHelpNote, createCliCompletions } from "./completions.js"; +import { excludeDeclaredSensitiveArguments } from "./explicitSensitiveArguments.js"; +import { reportUnknownMcpSubcommand } from "./mcpBuiltinCommands.js"; +import { serveMcpRegistration } from "./mcpRegistration.js"; +import { presentOmittableDefaults } from "./omittableDefaults.js"; +import { validateCliArguments } from "./strictPositionals.js"; +import type { CliInstance } from "./types.js"; + +/** A registrar and the module it was loaded from. */ +export interface LoadedCliCommandRegistration { + readonly registration: CliCommandRegistration; + readonly module: unknown; +} + +export interface CliShellOptions { + readonly selectedEnvironment: Readonly>; + /** Registrars to run, in registration order. */ + readonly registrations: readonly LoadedCliCommandRegistration[]; + /** Required when any registrar declares analysis. */ + readonly createAnalysis?: ( + environment: Readonly>, + ) => DirectAnalysis; + readonly resultOutput?: CliResultOutput | undefined; + readonly fridaInstrumentation?: FridaInstrumentationPort | undefined; +} + +/** Run each registrar, creating analysis once for those that declare it. */ +const registerCommands = ( + context: CliRegistrationContext, + options: CliShellOptions, +): void => { + let analysis: DirectAnalysis | undefined; + for (const { registration, module } of options.registrations) { + if (!registration.analysis) { + registration.register(module, context); + continue; + } + if (options.createAnalysis === undefined) + throw new Error("A CLI registrar needs analysis that was not provided"); + analysis ??= options.createAnalysis(context.environment); + registration.register(module, { ...context, analysis }); + } +}; + +/** + * Build the Incur CLI around the given registrars. Identity, global options, + * argument policy, completions, and registration builtins are the same for + * every set of registrars; they apply to whichever commands were registered. + */ +export const createCliShell = (options: CliShellOptions): CliInstance => { + const environment = snapshotEnvironment(options.selectedEnvironment); + const defaultRegistration = parseClientRegistrationCommand( + currentClientRegistrationCommand({ environment }), + ); + // The executable reports an invalid level before every command except doctor. + // Doctor keeps going so its readiness report can name REA_LOG_LEVEL. + const level = + environment.REA_LOG_LEVEL === undefined + ? undefined + : parseLogLevel(environment.REA_LOG_LEVEL); + const logger = createLogger( + "cli", + level?.ok === true ? level.value : "silent", + ); + const timedCli = Cli.create(PRODUCT_IDENTITY.cliBinary, { + version: PRODUCT_IDENTITY.packageVersion, + description: "Reverse engineer anything from your terminal or agent.", + globals: z.object({ + timings: z + .boolean() + .default(false) + .describe("Write measured command timing to stderr."), + }), + mcp: { + instructions: + "Ask what software, artifact, protocol, or behavior the user wants to understand, then choose the available investigation capabilities that can produce evidence.", + }, + sync: false, + update: { + // Incur's update:false only suppresses notices. Reject its explicit + // installer so ambiguous flags cannot bypass REA's verified workflow. + install: () => { + throw new Error( + "Use 'rea update' to update the identified installation. The '--update' alias accepts only global output flags.", + ); + }, + }, + }); + + timedCli.use(async (context, next) => { + if (!context.globals.timings) return next(); + const started = performance.now(); + try { + await next(); + } finally { + // This interval covers CLI command dispatch, including result delivery. + // It is not a provider-only or analysis-only measurement. + process.stderr.write( + `${JSON.stringify({ rea_timings: { version: 1, command_ms: Math.round(performance.now() - started) } })}\n`, + ); + } + }); + // Command registrars do not inspect global options; preserve their existing + // type while Incur owns parsing and the middleware owns this added flag. + const cli = timedCli as unknown as CliInstance; + + registerCommands( + { + cli, + logger, + environment, + fridaInstrumentation: options.fridaInstrumentation, + }, + options, + ); + + presentOmittableDefaults(cli); + if (options.resultOutput !== undefined) + streamCliCommandResults(cli, options.resultOutput); + const serveStrict = validateCliArguments(cli); + presentCliArgumentErrors(cli); + const serveDeclared = excludeDeclaredSensitiveArguments(cli); + encodeCliArgumentErrors(cli); + const serve = cli.serve.bind(cli); + const complete = createCliCompletions( + cli as unknown as Parameters[0], + environment, + ); + cli.serve = (arguments_ = process.argv.slice(2), serveOptions) => { + if (complete(arguments_, serveOptions ?? {})) return Promise.resolve(); + const normalized = normalizeCliArguments(arguments_); + const registration = serveMcpRegistration( + normalized, + environment, + defaultRegistration, + serveOptions ?? {}, + ); + if (registration !== undefined) return registration; + const result = serveStrict( + normalized, + serveOptions ?? {}, + (tokens, selected) => + serveDeclared(tokens, selected, (declared, declaredOptions) => + reportUnknownMcpSubcommand(cli.name, declared, declaredOptions) + ? Promise.resolve() + : serve(adaptCliOutputArgumentsForIncur(declared), declaredOptions), + ), + ); + const note = completionHelpNote(normalized); + if (note === undefined) return result; + return result.then(() => { + const write = + serveOptions?.stdout ?? ((text: string) => process.stdout.write(text)); + write(`\n${note}`); + }); + }; + return cli; +}; diff --git a/src/cli/guidanceCommands.ts b/src/cli/guidanceCommands.ts index 6758d93bf7..7f0c0c5f12 100644 --- a/src/cli/guidanceCommands.ts +++ b/src/cli/guidanceCommands.ts @@ -15,6 +15,7 @@ import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; import type { Logger } from "pino"; import type { CliInstance } from "./types.js"; import { cliDecimalField } from "./options.js"; +import { ownCliProcessResource } from "./cliResources.js"; /** Guidance descriptors owned by one CLI host. */ export class CliGuidanceResources { @@ -42,6 +43,8 @@ export const closeCliGuidanceResources = ( resources: CliGuidanceResources = processGuidanceResources, ): Promise => resources.close(); +ownCliProcessResource("guidance", () => closeCliGuidanceResources()); + /** Register offline search and retrieval for the installed REA guide bundle. */ export const registerGuidanceCommands = ( cli: CliInstance, diff --git a/src/cli/javascriptApplicationAnalysis.ts b/src/cli/javascriptApplicationAnalysis.ts index 5f444855c6..bc2522a757 100644 --- a/src/cli/javascriptApplicationAnalysis.ts +++ b/src/cli/javascriptApplicationAnalysis.ts @@ -12,6 +12,7 @@ import { ProviderCleanupError } from "../domain/providerCleanupError.js"; import { analysisInputErrorFromIssues } from "../domain/inputIssueProjection.js"; import { javaScriptSourcePathsSchema } from "../domain/javascript/javascriptSourceSelection.js"; import { parseCliJsonInput } from "../cliJsonInput.js"; +import { ownCliProcessResource } from "./cliResources.js"; const OPERATION = "analyze_javascript_application"; @@ -94,6 +95,10 @@ export const closeCliJavaScriptResources = ( host: CliJavaScriptAnalysisHost = processJavaScriptAnalysis, ): Promise => host.close(); +ownCliProcessResource("javascript-analysis", () => + closeCliJavaScriptResources(), +); + /** Execute the shared CLI boundary for static JavaScript analysis. */ export const runCliJavaScriptApplicationAnalysis = async ( input: unknown, diff --git a/src/cli/mcpRegistration.ts b/src/cli/mcpRegistration.ts index f8204508f8..00cf8fe807 100644 --- a/src/cli/mcpRegistration.ts +++ b/src/cli/mcpRegistration.ts @@ -41,6 +41,7 @@ import { PRODUCT_IDENTITY } from "../identity.js"; import { createCliResultOutput } from "./cliResultOutput.js"; import { withCommandCancellation } from "./commandCancellation.js"; import type { CliInstance } from "./types.js"; +import { ownCliProcessResource } from "./cliResources.js"; let retainedResources: ArtifactResourceScope | undefined; @@ -52,6 +53,8 @@ export const closeCliMcpRegistrationResources = async (): Promise => { if (retainedResources === resources) retainedResources = undefined; }; +ownCliProcessResource("mcp-registration", closeCliMcpRegistrationResources); + type ServeOptions = NonNullable[1]>; interface RegistrationRequest { readonly agents: readonly string[]; diff --git a/src/cliOneShot.ts b/src/cliOneShot.ts index 49c6148f9d..41580c7fc9 100644 --- a/src/cliOneShot.ts +++ b/src/cliOneShot.ts @@ -122,8 +122,9 @@ export const runOneShotCli = async ( } // Loaded only now: the documents above answer without the command catalog. - const { closeCliResources, createCli } = await import("./cli.js"); - host.ownCliResources(closeCliResources); + const { loadCliRuntime } = await import("./cli/cliLoader.js"); + const runtime = await loadCliRuntime(cliArguments, host.environment); + host.ownCliResources(runtime.closeCliResources); const { createCliResultOutput } = await import("./cli/cliResultOutput.js"); let wroteOutput = false; const resultOutput = createCliResultOutput( @@ -131,7 +132,7 @@ export const runOneShotCli = async ( host.resultDestination, ); try { - await createCli(host.environment, resultOutput).serve(cliArguments, { + await runtime.createCli(resultOutput).serve(cliArguments, { stdout: (output) => { if (resultOutput?.handled) { if (resultOutput.failed && !host.stdoutClosed()) diff --git a/src/recordedCliCommandRegistrations.ts b/src/recordedCliCommandRegistrations.ts new file mode 100644 index 0000000000..09def76262 --- /dev/null +++ b/src/recordedCliCommandRegistrations.ts @@ -0,0 +1,8 @@ +/** + * The registrar of each top-level CLI command name, recorded from the compiled + * registration table by `scripts/record-cli-commands.mjs` after compilation. + * Source runs leave this undefined, so every command loads the complete CLI. + */ +export const RECORDED_CLI_COMMAND_REGISTRATIONS: + | Readonly> + | undefined = undefined; diff --git a/tests/boundary/cli/cliLazyRegistration.test.ts b/tests/boundary/cli/cliLazyRegistration.test.ts new file mode 100644 index 0000000000..9d46a7f439 --- /dev/null +++ b/tests/boundary/cli/cliLazyRegistration.test.ts @@ -0,0 +1,157 @@ +import { join, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { Cli } from "incur"; +import { describe, expect, it } from "vitest"; + +import { createCli } from "../../../src/cli.js"; +import { + CLI_COMMAND_REGISTRATIONS, + CLI_REGISTRATION_ORDER, + type CliCommandRegistrationId, +} from "../../../src/cli/cliCommandRegistrations.js"; +import { selectCliRegistration } from "../../../src/cli/cliLoader.js"; +import { createCliShell } from "../../../src/cli/cliShell.js"; +import type { CliInstance } from "../../../src/cli/types.js"; +import { visitRegisteredCommands } from "../../../src/cli/visitRegisteredCommands.js"; +import { createDirectAnalysis } from "../../../src/composition/directAnalysis.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { processTest } from "../../support/process/processFixture.js"; +import { serveInProcessCli } from "../../support/cli/serveInProcessCli.js"; + +const ENVIRONMENT = { PATH: process.env.PATH }; + +// Incur types its router key without vars, env, or globals; accept any CLI. +const registeredCommands: WeakMap< + object, + Map +> = Cli.toCommands; + +const topLevelNames = (cli: CliInstance): readonly string[] => [ + ...(registeredCommands.get(cli)?.keys() ?? []), +]; + +const commandPaths = (cli: CliInstance): readonly string[] => { + const paths: string[] = []; + visitRegisteredCommands(cli, (_command, path) => paths.push(path)); + return paths; +}; + +const createRegistrarCli = async ( + id: CliCommandRegistrationId, +): Promise => { + const registration = CLI_COMMAND_REGISTRATIONS[id]; + return createCliShell({ + selectedEnvironment: ENVIRONMENT, + registrations: [{ registration, module: await registration.load() }], + createAnalysis: createDirectAnalysis, + }); +}; + +const help = async (cli: CliInstance, path: string) => + serveInProcessCli(cli, [...path.split(" "), "--help"]); + +const recordedRegistrations = async (): Promise> => { + const module = (await import( + pathToFileURL(resolve("dist/recordedCliCommandRegistrations.js")).href + )) as { RECORDED_CLI_COMMAND_REGISTRATIONS: Record }; + return module.RECORDED_CLI_COMMAND_REGISTRATIONS; +}; + +describe("lazy CLI registration", () => { + it("records every top-level command under the one registrar that defines it", async () => { + const recorded = await recordedRegistrations(); + const complete = createCli(ENVIRONMENT); + expect(Object.keys(recorded).sort()).toEqual( + [...topLevelNames(complete)].sort(), + ); + for (const id of CLI_REGISTRATION_ORDER) { + const alone = await createRegistrarCli(id); + expect( + [...topLevelNames(alone)].sort(), + `${id} registers what the recording assigns it`, + ).toEqual( + Object.entries(recorded) + .filter(([, owner]) => owner === id) + .map(([name]) => name) + .sort(), + ); + } + }); + + it("gives each command the same definition alone as in the complete CLI", async () => { + const complete = createCli(ENVIRONMENT); + const completePaths = new Set(commandPaths(complete)); + const covered = new Set(); + for (const id of CLI_REGISTRATION_ORDER) { + const alone = await createRegistrarCli(id); + for (const path of commandPaths(alone)) { + expect(completePaths.has(path), path).toBe(true); + covered.add(path); + expect(await help(alone, path), path).toEqual( + await help(complete, path), + ); + } + } + expect(covered).toEqual(completePaths); + }); + + it("loads one registrar only for a recorded command that runs", () => { + const recorded = { "inspect-go-binary": "go", setup: "setup" }; + const select = ( + arguments_: readonly string[], + environment: NodeJS.ProcessEnv = {}, + record: Readonly> | null = recorded, + ) => selectCliRegistration(arguments_, environment, record ?? undefined); + + expect(select(["inspect-go-binary", "a.out"])).toBe("go"); + expect(select(["inspect-go-binary", "a.out", "--json"])).toBe("go"); + expect(select(["--format", "json", "inspect-go-binary", "a.out"])).toBe( + "go", + ); + expect(select(["--timings", "setup", "--dry-run"])).toBe("setup"); + // Help, schemas, LLM manifests and builtins describe the whole catalog. + for (const flag of ["--help", "-h", "--schema", "--llms", "--llms-full"]) + expect(select(["inspect-go-binary", flag]), flag).toBeUndefined(); + expect(select(["inspect-go-binary", "--mcp"])).toBeUndefined(); + expect(select([])).toBeUndefined(); + expect(select(["skills", "add"])).toBeUndefined(); + // An unknown name needs every command for "did you mean". + expect(select(["inspect-go-binar"])).toBeUndefined(); + expect(select(["constructor"])).toBeUndefined(); + // Shell completion and a missing or stale recording use the complete CLI. + expect(select(["inspect-go-binary"], { COMPLETE: "bash" })).toBeUndefined(); + expect(select(["inspect-go-binary"], {}, null)).toBeUndefined(); + expect( + select(["inspect-go-binary"], {}, { "inspect-go-binary": "removed" }), + ).toBeUndefined(); + }); + + processTest( + "runs a recorded command without loading the complete CLI", + async ({ processes }) => { + const marker = "REA_TEST_CLI_ENTRY_RESOLVED"; + // Report a resolution of the complete CLI entry without changing it. + const hook = `data:text/javascript,${encodeURIComponent(` +import { registerHooks } from "node:module"; +registerHooks({ resolve(specifier, context, next) { + const resolved = next(specifier, context); + if (resolved.url.endsWith("/dist/cli-runtime/cli.js")) process.stderr.write(${JSON.stringify(`${marker}\n`)}); + return resolved; +} });`)}`; + const home = await createTestTempDirectory("rea-cli-lazy-home-"); + const run = (arguments_: readonly string[]) => + processes.run( + process.execPath, + [`--import=${hook}`, join("scripts", "rea.mjs"), ...arguments_], + { env: { ...ENVIRONMENT, HOME: home } }, + ); + const lazy = await run(["list-cutter-sessions", "--json"]); + expect(lazy.exitCode, lazy.stderr).toBe(0); + expect(JSON.parse(lazy.stdout)).toMatchObject({ sessions: [] }); + expect(lazy.stderr).not.toContain(marker); + const complete = await run(["list-cutter-sessions", "--help"]); + expect(complete.exitCode, complete.stderr).toBe(0); + expect(complete.stderr).toContain(marker); + }, + ); +}); diff --git a/tests/boundary/cli/cliResourceClose.test.ts b/tests/boundary/cli/cliResourceClose.test.ts index 4a1a9005c7..1c35d824e5 100644 --- a/tests/boundary/cli/cliResourceClose.test.ts +++ b/tests/boundary/cli/cliResourceClose.test.ts @@ -1,9 +1,9 @@ -import { afterEach, expect, it, vi } from "vitest"; -import { closeCliResources } from "../../../src/cli.js"; -import * as javascript from "../../../src/cli/javascriptApplicationAnalysis.js"; -import * as guidance from "../../../src/cli/guidanceCommands.js"; - -afterEach(() => vi.restoreAllMocks()); +import { expect, it, vi } from "vitest"; +import "../../../src/cli.js"; +import { + CliResourceOwners, + cliProcessResourceOwners, +} from "../../../src/cli/cliResources.js"; it.each([ [false, false], @@ -15,17 +15,16 @@ it.each([ async (jsFails, guideFails) => { const jsError = new Error("JavaScript resources retained"); const guideError = new Error("Guidance descriptor retained"); - const js = vi - .spyOn(javascript, "closeCliJavaScriptResources") - .mockImplementation(async () => { - if (jsFails) throw jsError; - }); - const guides = vi - .spyOn(guidance, "closeCliGuidanceResources") - .mockImplementation(async () => { - if (guideFails) throw guideError; - }); - const closing = closeCliResources(); + const js = vi.fn(async () => { + if (jsFails) throw jsError; + }); + const guides = vi.fn(async () => { + if (guideFails) throw guideError; + }); + const owners = new CliResourceOwners(); + owners.own("javascript-analysis", js); + owners.own("guidance", guides); + const closing = owners.close(); if (jsFails && guideFails) await expect(closing).rejects.toMatchObject({ errors: [jsError, guideError], @@ -37,3 +36,13 @@ it.each([ expect(guides).toHaveBeenCalledOnce(); }, ); + +it("registers every process-owned CLI resource the complete CLI loads", () => { + expect(cliProcessResourceOwners()).toEqual( + expect.arrayContaining([ + "javascript-analysis", + "guidance", + "mcp-registration", + ]), + ); +}); diff --git a/tests/boundary/cli/dispatcher.test.ts b/tests/boundary/cli/dispatcher.test.ts index 928de7b9f0..ca8e36a1d4 100644 --- a/tests/boundary/cli/dispatcher.test.ts +++ b/tests/boundary/cli/dispatcher.test.ts @@ -17,7 +17,7 @@ const installCliRuntime = async ( await workspace.mkdir("dist"); // The bundle's entrypoints and shared chunks form one emitted runtime. The // one-shot owner and its output admission stay production code; only the - // command runtime and result output are replaced below. + // command runtime, its recorded registrars, and result output are replaced. await cp("dist/cli-runtime", workspace.path("dist/cli-runtime"), { recursive: true, }); @@ -43,6 +43,11 @@ const installCliRuntime = async ( ), workspace.write("dist/cli-runtime/cli.js", cliSource), workspace.write("dist/cli-runtime/cliResultOutput.js", resultOutput), + // Without a recording, every command loads the replaced complete CLI. + workspace.write( + "dist/recordedCliCommandRegistrations.js", + "export const RECORDED_CLI_COMMAND_REGISTRATIONS = undefined;\n", + ), ]); }; diff --git a/turbo.json b/turbo.json index 653a253846..216f441745 100644 --- a/turbo.json +++ b/turbo.json @@ -22,6 +22,7 @@ "scripts/bundle-runtime.mjs", "scripts/record-catalog-digests.mjs", "scripts/record-mcp-catalog.mjs", + "scripts/record-cli-commands.mjs", "scripts/generate-package-metadata.mjs", "src/**/*.{cts,mts,ts,tsx}", "src/**/package.json",