From c1fd496e6a1d3a08b30ee0f2157e827bb1049b58 Mon Sep 17 00:00:00 2001 From: Kaoru0822 Date: Thu, 15 Oct 2026 00:53:26 -0700 Subject: [PATCH] feat(kuna): serve procedures, pseudocode and assembly from Kuna exports (#3854) * feat(kuna): serve procedures, pseudocode and assembly from Kuna exports Register Kuna as a bring-your-own deep-analysis provider. It answers list_documents, list_procedures, search_procedures, procedure_address, address_name, procedure_pseudo_code, procedure_assembly and analyze_function (and batch_decompile through composition) from the verified project export. While a cold export builds in the background, pseudocode comes from a single-function `kuna decompile` run on a private copy and says so; assembly and data names wait for the export. Each answer records its source and file span in raw_result, limitations are scoped per operation, and the session-owned export is stopped on close unless another session still waits for it. The literal/regex procedure search predicate moves from the IDA runner into a shared domain helper. `npm run verify:kuna` adds a real-provider lane with an optional Ghidra comparison. Refs #1957 Co-Authored-By: Claude Opus 5.5 * fix(kuna): refuse a Ghidra comparison without caller-built targets Refs #1957 Co-Authored-By: Claude Opus 5.5 * fix(kuna): retry owned cleanup and keep answers bound to the opened Kuna - Re-verify the Kuna executable's digest before every launch (memoized by file identity), so no answer names one build while another produced it. - Keep the process supervisor of an unproven stop, and retry it on every later close in both the session and the cache; a failed close keeps failing until the stop is proven instead of reporting success. - Report an abandoned export's retained cleanup from session close. - Forget and rebuild an adopted export whose files another REA process evicted, instead of failing every later read. - Close a private copy whose setup failed, and retry failed removals. - Let the session own the shared quick inventory; each caller only stops waiting when it cancels. - Report a failed export as `failed` with its reason. - Lane: release the MCP client when open_binary rejects a target, and accept either valid source for the first answer. Refs #1957 Co-Authored-By: Claude Opus 5.5 * fix(kuna): drain session requests before releasing workspaces Combine caller cancellation with session shutdown and wait for active reads before releasing their process supervisors or private inputs. Retain unproven cleanup for later close attempts. Measure UTF-8 reply bytes without implying model token usage. Co-authored-by: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com> Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> * perf(kuna): keep document metadata reads free of target analysis Listing the selected path no longer initializes the project cache or starts an export that the request never consumes. Co-authored-by: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com> Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> * fix(kuna): reject verification matrices with missing functions Validate requested matrix coverage before building the fixture or opening providers. Empty matrices, unmatched names and partially matched function sets must fail instead of reporting an unperformed comparison as successful. Co-authored-by: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com> Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> * fix(kuna): retain component provenance in function dossiers Record each source facet as one file span and preserve both pseudocode and assembly digests/ranges in combined reports. Keep direct text source fields compatible and verify every combined span through the real MCP lane. Co-authored-by: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com> Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> * fix(kuna): keep retired file cleanup bound to its export Retain the export key with each failed file removal so a cancelled session reports and retries its own stopped resources. Preserve that association across sweeping and process-cleanup recovery without attributing unrelated cache failures to the session. Co-authored-by: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com> Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> * docs(kuna): document the Kuna provider and list it in the catalog (#3855) * docs(kuna): document the Kuna provider and list it in the catalog Add the Kuna provider guide (configuration, answered tools, cold and warm paths, provenance, cache ownership, verification lane), link it from the README and its translations, the CLI, installation and index pages and the site navigation, describe `npm run verify:kuna` in the testing guide, credit Kuna in THIRD_PARTY_NOTICES, teach the agent skill when to select Kuna, and register its identity and operations in the generated product catalog. Refs #1957 Co-Authored-By: Claude Opus 5.5 * docs(kuna): state the Linux cache fallback, raw_result scope and comparison inputs Refs #1957 Co-Authored-By: Claude Opus 5.5 * docs(kuna): note that cache pins hold within one REA process Refs #1957 Co-Authored-By: Claude Opus 5.5 * docs(kuna): describe retired staging and retried cleanup Refs #1957 Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 * fix(kuna): supervise regex and bind reads to one export Share a cancellable ECMAScript matching worker with Hopper while preserving its error contract. Bind C and assembly indexes to immutable project readers, retry whole dossiers after eviction, and enforce the existing buffered-text capacity before allocating cached spans. Document the verified Linux scope and keep complete exports accessible. Co-authored-by: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com> Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: N0zoM1z0 <161784452+N0zoM1z0@users.noreply.github.com> Co-authored-by: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com> --- .../skills/reverse-engineer-anything/SKILL.md | 13 +- README.md | 3 +- THIRD_PARTY_NOTICES.md | 1 + docs/.vitepress/config.ts | 1 + docs/cli.md | 5 +- docs/index.md | 2 +- docs/installation.md | 14 +- docs/kuna-provider.md | 84 ++ docs/readme/README_ar.md | 2 +- docs/readme/README_de.md | 2 +- docs/readme/README_el.md | 2 +- docs/readme/README_es.md | 2 +- docs/readme/README_fa.md | 2 +- docs/readme/README_fr.md | 2 +- docs/readme/README_id.md | 2 +- docs/readme/README_it.md | 2 +- docs/readme/README_ja.md | 2 +- docs/readme/README_ko.md | 2 +- docs/readme/README_pl.md | 2 +- docs/readme/README_pt-BR.md | 2 +- docs/readme/README_ru.md | 2 +- docs/readme/README_th.md | 2 +- docs/readme/README_tr.md | 2 +- docs/readme/README_uk.md | 2 +- docs/readme/README_vi.md | 2 +- docs/readme/README_zh-TW.md | 2 +- docs/readme/README_zh.md | 2 +- docs/testing.md | 28 + package.json | 1 + scripts/lib/catalog-builders.mjs | 7 + scripts/lib/catalog-core.mjs | 1 + scripts/verify-real-kuna.mjs | 458 +++++++ src/composition/binary.ts | 4 +- src/domain/analysisSearch.ts | 33 + src/hopper/HopperProviderCapabilities.ts | 3 +- src/hopper/HopperRegexSearch.ts | 139 +-- src/hopper/HopperRegexWorker.ts | 27 - src/ida/IdaOperationRunner.ts | 34 +- src/kuna/KunaCommand.ts | 267 +++++ src/kuna/KunaConfiguration.ts | 18 + src/kuna/KunaDoctor.ts | 1 + src/kuna/KunaLimits.ts | 2 + src/kuna/KunaProjectCache.ts | 171 ++- src/kuna/KunaProjectExport.ts | 233 +--- src/kuna/KunaProjectFiles.ts | 186 ++- src/kuna/KunaProjectReader.ts | 180 +++ src/kuna/KunaProvider.ts | 224 ++++ src/kuna/KunaProviderCapabilities.ts | 115 ++ src/kuna/KunaSessionClient.ts | 1060 +++++++++++++++++ src/process/RegexMatcher.ts | 176 +++ .../directAnalysisPermissions.test.ts | 2 +- tests/boundary/process/regexMatcher.test.ts | 50 + tests/boundary/providers/kuna/fakeKuna.ts | 20 +- .../providers/kuna/kunaInstallation.test.ts | 10 +- .../providers/kuna/kunaProjectFiles.test.ts | 43 + .../kuna/kunaProjectReaderConcurrency.test.ts | 132 ++ .../providers/kuna/kunaProvider.test.ts | 787 ++++++++++++ .../providers/kuna/kunaRegexSearch.test.ts | 66 + .../providers/kuna/kunaSessionFixture.ts | 78 ++ .../providers/kuna/kunaVerifier.test.ts | 73 ++ 60 files changed, 4297 insertions(+), 493 deletions(-) create mode 100644 docs/kuna-provider.md create mode 100644 scripts/verify-real-kuna.mjs create mode 100644 src/domain/analysisSearch.ts delete mode 100644 src/hopper/HopperRegexWorker.ts create mode 100644 src/kuna/KunaCommand.ts create mode 100644 src/kuna/KunaLimits.ts create mode 100644 src/kuna/KunaProjectReader.ts create mode 100644 src/kuna/KunaProvider.ts create mode 100644 src/kuna/KunaProviderCapabilities.ts create mode 100644 src/kuna/KunaSessionClient.ts create mode 100644 src/process/RegexMatcher.ts create mode 100644 tests/boundary/process/regexMatcher.test.ts create mode 100644 tests/boundary/providers/kuna/kunaProjectReaderConcurrency.test.ts create mode 100644 tests/boundary/providers/kuna/kunaProvider.test.ts create mode 100644 tests/boundary/providers/kuna/kunaRegexSearch.test.ts create mode 100644 tests/boundary/providers/kuna/kunaSessionFixture.ts create mode 100644 tests/boundary/providers/kuna/kunaVerifier.test.ts diff --git a/.agents/skills/reverse-engineer-anything/SKILL.md b/.agents/skills/reverse-engineer-anything/SKILL.md index 426a05ae56..d0f2f640ae 100644 --- a/.agents/skills/reverse-engineer-anything/SKILL.md +++ b/.agents/skills/reverse-engineer-anything/SKILL.md @@ -59,7 +59,7 @@ availability have different remedies; avoid repeating setup for all of them. absent, inspect the client's MCP launch error rather than repeating setup. REA setup never installs or upgrades Node.js, npm, Homebrew, Java, Ghidra, IDA, -JADX, Binwalk, or Unblob. Use existing prerequisites; do not install unrelated +JADX, Kuna, Binwalk, or Unblob. Use existing prerequisites; do not install unrelated software to repair MCP registration. For an unsupported client, use manual stdio registration with a version-pinned `rea-agents` package or continue through the CLI. @@ -79,7 +79,7 @@ is within the authorized scope, use `npx -y rea-agents@latest` in its place. Configuration repair starts with a read-only setup plan; show paths, backups and skill replacement before applying authorized changes. Preserve existing approval and target-execution boundaries. REA setup never installs or upgrades unrelated -software, including Node.js, npm, Homebrew, Java, Ghidra, IDA, JADX, Binwalk or Unblob. +software, including Node.js, npm, Homebrew, Java, Ghidra, IDA, JADX, Kuna, Binwalk or Unblob. ## Route the target first @@ -215,6 +215,15 @@ after opening; IDA does not supply `binary_overview`. Consult session availabili when composing broader workflows. Modern direct callers and unsupported dossier facets remain unknown; live IDA results are not replayed from snapshots. +With `REA_KUNA_COMMAND` configured, `provider_id: "kuna"` answers procedure +lists, lookups, pseudocode, assembly and `analyze_function` from Kuna's cached +whole-program export; it has no cross-references, callers or CFG. A cold first +answer can come from a single-function run (its limitations say so); assembly +waits for the export. For pseudocode and assembly, `raw_result.source` names the +cached `.c`/`.asm` file and byte span, which you may search directly instead of +calling per function. See the +[Kuna provider guide](https://github.com/morluto/rea/blob/main/docs/kuna-provider.md). + If the user says only "my app" without a name, path, or target already selected for this task, ask which app before searching for artifacts. Paths in unrelated saved reports and nearby test fixtures do not select the user's target. Resolve diff --git a/README.md b/README.md index 0cf0c68de2..89a53ff755 100644 --- a/README.md +++ b/README.md @@ -205,7 +205,8 @@ or update either provider. Native formats and host support vary by provider. See [Hopper and Ghidra setup](docs/installation.md#hopper), the -[IDA guide](docs/ida-provider.md) (including an optional native MCP bridge), and +[IDA guide](docs/ida-provider.md) (including an optional native MCP bridge), the +[Kuna decompiler guide](docs/kuna-provider.md), and [experimental Windows Ghidra support](docs/windows-ghidra-p0.md). Ghidra also supports [16-bit DOS analysis](docs/ghidra-dos.md). For large binaries, raise the provider's startup deadline with diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 5fceaaa474..2648af329b 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -58,6 +58,7 @@ pin inspection sources; they are not included in the npm package. | ghidra-nativeaot | [MIT](https://github.com/Washi1337/ghidra-nativeaot/blob/effeb734fc570c32650f88b159608979dc7b423e/LICENSE.md) | Pinned inspection source; derived parser attribution above. | | JADX | [Apache-2.0](https://github.com/skylot/jadx/blob/v1.5.6/LICENSE) | The audited jadx-headless-mcp source uses JADX 1.5.6 for Android decompilation; the engine JAR is separately supplied. | | jadx-headless-mcp | [Apache-2.0](https://github.com/1013503897/jadx-headless-mcp/blob/5844800a486d2248fa949b74c9896285f5b54de2/LICENSE) | v0.7.1; pinned submodule and user-supplied JAR. | +| Kuna | [Apache-2.0](https://github.com/Noelo-Lab/kuna/blob/v1.833/LICENSE), with its [NOTICE](https://github.com/Noelo-Lab/kuna/blob/v1.833/NOTICE) | Bring your own `kuna`; REA verified 1.833 and never downloads, installs or bundles it. REA invokes it as a separate process. | | Apktool | [Apache-2.0](https://github.com/iBotPeaches/Apktool/blob/master/LICENSE.md) | User-supplied Android resource decoder. | | Binwalk | [MIT](https://github.com/ReFirmLabs/binwalk/blob/4fdab3d464d97b68e0af9088df3f9e2e1545b21c/LICENSE) | v3.1.0; pinned submodule and user-supplied firmware engine. | | Unblob | [MIT](https://github.com/onekey-sec/unblob/blob/1fcc7a0a584a70a96c31f5a276c20944d199a089/LICENSE) | 26.6.4; pinned submodule and user-supplied firmware engine. | diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index 440338f987..0c00bb9972 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -60,6 +60,7 @@ export default defineConfig({ collapsed: true, items: [ { text: "IDA", link: "/ida-provider" }, + { text: "Kuna", link: "/kuna-provider" }, { text: "Ghidra on Windows", link: "/windows-ghidra-p0" }, { text: "Ghidra DOS analysis", link: "/ghidra-dos" }, { text: "Ghidra raw binary", link: "/raw-binary" }, diff --git a/docs/cli.md b/docs/cli.md index 0922f584a7..b087b1b902 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -95,8 +95,9 @@ dependency edges, and the section's file range. A missing `.dep-v0` section is null metadata, not proof that the binary has no dependencies. See [Rust binaries](rust-binaries.md). -Configure [Hopper or Ghidra](installation.md#hopper), or the -[IDA adapter](ida-provider.md), before analyzing a native target. Substitute +Configure [Hopper or Ghidra](installation.md#hopper), the +[IDA adapter](ida-provider.md), or [Kuna](kuna-provider.md) before analyzing a +native target. Substitute your target, search text and function name or address in these examples: ```bash diff --git a/docs/index.md b/docs/index.md index 5acee1b55c..fbdc936b1c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -50,7 +50,7 @@ For runtime instrumentation, see the [Frida guide](./frida-instrumentation.md). | Task | Guides | | --------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Inspect native functions, references, or types | [Native binaries](native-investigation.md), [Go metadata](go-binaries.md), [Rust dependencies](rust-binaries.md), [offline ELF layout](binary-diagnostics.md), [recorded crashes](recorded-crashes.md) | -| Select a native provider or import format | [Ghidra semantics](ghidra-provider.md), [Windows](windows-ghidra-p0.md), [DOS](ghidra-dos.md), [raw bytes](raw-binary.md), [MIPS](ghidra-mips.md), [PSP](ghidra-psp.md), [NativeAOT](ghidra-nativeaot.md), [IDA](ida-provider.md), [JEB](jeb-analysis.md) | +| Select a native provider or import format | [Ghidra semantics](ghidra-provider.md), [Windows](windows-ghidra-p0.md), [DOS](ghidra-dos.md), [raw bytes](raw-binary.md), [MIPS](ghidra-mips.md), [PSP](ghidra-psp.md), [NativeAOT](ghidra-nativeaot.md), [IDA](ida-provider.md), [Kuna](kuna-provider.md), [JEB](jeb-analysis.md) | | Analyze JavaScript or Electron artifacts | [Static analysis](javascript-artifact-reconstruction.md), [tracing and comparisons](javascript-application-workflows.md), [application graph](javascript-application-graph.md), [source recovery](javascript-recovery.md) | | Compare static JavaScript with runtime observations | [Runtime attribution](javascript-runtime-observation.md), [reconciliation](javascript-runtime-reconciliation.md), [Electron observation](electron-observation.md), [Electron fuses](electron-fuses.md) | | Inspect assemblies, packages, or firmware | [.NET](managed-code-analysis.md), [Android](android-analysis.md), [Apktool resources](apktool-resource-analysis.md), [ADB acquisition](adb-device-analysis.md), [Apple](apple-application-analysis.md), [Harmony](harmony-analysis.md), [firmware](firmware-analysis.md), [WASM](wasm-artifacts.md), [EVM](evm-bytecode.md) | diff --git a/docs/installation.md b/docs/installation.md index 527014e54f..b057b981e9 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -3,13 +3,13 @@ Run REA, connect your selected agents, and check the tools needed for your task. Setup previews configuration changes and asks for approval before applying them. -| Task | Section | -| --------------------------------- | ------------------------------------------------------------------------- | -| Connect an agent | [Start setup](#start-setup), then [review changes](#review-setup-changes) | -| Diagnose one task | [Scoped readiness checks](#check-readiness-for-your-task) | -| Configure an analysis engine | [Hopper](#hopper), [Ghidra](#ghidra), or the [IDA guide](ida-provider.md) | -| Update or remove REA | [Maintenance](#diagnose-update-and-remove) | -| Resolve a client-specific problem | [Client configuration](client-configuration.md) | +| Task | Section | +| --------------------------------- | --------------------------------------------------------------------------------------------------- | +| Connect an agent | [Start setup](#start-setup), then [review changes](#review-setup-changes) | +| Diagnose one task | [Scoped readiness checks](#check-readiness-for-your-task) | +| Configure an analysis engine | [Hopper](#hopper), [Ghidra](#ghidra), the [IDA guide](ida-provider.md), or [Kuna](kuna-provider.md) | +| Update or remove REA | [Maintenance](#diagnose-update-and-remove) | +| Resolve a client-specific problem | [Client configuration](client-configuration.md) | ## Start setup diff --git a/docs/kuna-provider.md b/docs/kuna-provider.md new file mode 100644 index 0000000000..67cbe8b2eb --- /dev/null +++ b/docs/kuna-provider.md @@ -0,0 +1,84 @@ +# Kuna provider + +[Kuna](https://github.com/Noelo-Lab/kuna) is an Apache-2.0 decompiler distributed as one `kuna` executable. REA uses a `kuna` you supply. It does not download, install, update or bundle Kuna. REA verified this provider with Kuna 1.833; it accepts other releases and reports them as unverified in `rea doctor` and in each analysis profile. + +Kuna is static: it reads the selected image and never executes it. + +## Configure + +```bash +export REA_KUNA_COMMAND=/absolute/path/to/kuna +rea doctor --provider kuna +``` + +Select it explicitly with `--provider kuna`, MCP's `open_binary.provider_id: "kuna"`, or `REA_ANALYSIS_PROVIDER=kuna`. With Kuna and another provider configured for the same target, `auto` reports `ambiguous` instead of choosing one. + +| Setting | Default | Meaning | +| ----------------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `REA_KUNA_COMMAND` | none (required) | Absolute path of the `kuna` executable. | +| `REA_KUNA_CACHE_DIR` | `~/Library/Caches/rea/kuna` (macOS), `$XDG_CACHE_HOME/rea/kuna` (Linux; `~/.cache/rea/kuna` when `XDG_CACHE_HOME` is unset or relative) | Absolute directory REA owns for project exports. Kuna's own spec cache also lives here, not in your Kuna cache. | +| `REA_KUNA_CACHE_MAX_BYTES` | 4 GiB | Size cap; least recently used exports are evicted above it. | +| `REA_KUNA_PROJECT_TIMEOUT_MS` | 30 minutes | Deadline for one project export. | +| `REA_KUNA_TIMEOUT_MS` | 120 s | Deadline for one single-function decompile. | +| `REA_KUNA_JOBS` | 1 | `decompile-project --jobs`. 1 is Kuna's reference answer; more jobs finish sooner but lose callee-first prototypes, and change about 40–60% of function bodies on the measured binaries. | + +Windows is not supported yet. Supported targets are ELF and thin Mach-O executables for x86-64, AArch64 and ARM; PE and universal Mach-O are rejected with a typed reason. + +## What Kuna answers + +| Tool | Source | +| ----------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- | +| `list_procedures`, `search_procedures`, `procedure_address` | Kuna's callable inventory (`kuna functions --json`), including import slots it does not decompile | +| `address_name` | Function entries, then the data labels Kuna's export assigns | +| `procedure_pseudo_code`, `batch_decompile` | The project export, or a single-function run while the export builds | +| `procedure_assembly` | The project export's labeled listing | +| `analyze_function` | Pseudocode and assembly from the export; callers, callees, references, comments, strings and CFG are declared unavailable | + +Cross-references, callers, callees and basic blocks are not available from this provider. Use another provider when you need them. + +## How answers are produced + +The first analysis request in a session looks for this target's export in the cache. `list_documents` reads the selected path without initializing analysis. The export is keyed by the target's SHA-256, the Kuna executable's SHA-256, its version and `--jobs`. + +- **Warm cache:** analysis reads reuse the verified export. A fresh server verifies its file digests; that cost scales with export size. +- **Cold cache:** + - REA starts `kuna decompile-project` in the background on a private copy of the target. + - Meanwhile, pseudocode comes from one `kuna decompile --addr` run, and listings from `kuna functions --json`. + - Assembly and data names wait for the export; progress notifications report the wait every ten seconds. + - A single-function answer says so in its limitations. It lacks the export's whole-program prototypes, so a later request can return different text for the same function. + +Pseudocode, assembly, `analyze_function` and data-name answers record where their text came from in `raw_result.source`: `single-function`, or `project-export` with the cache directory. `procedure_pseudo_code` and `procedure_assembly` answers from the export also name the file, byte span and the file's SHA-256. Combined `analyze_function` answers retain both facet records in `raw_result.source.files`, each with its file, digest and byte span. Both texts come from one export; losing that project retries the complete report. Listings and procedure lookups carry no `raw_result`. Agents can open or search the export's `.c` and `.asm` there directly when that is cheaper than calling a tool per function. Text read that way is target-derived, like any decompiler output, but it bypasses the untrusted-content marker that REA's tool results carry. + +The background export belongs to the session. `close_binary`, or the end of a CLI command, cancels it unless another session still waits for it. A one-shot CLI decompile therefore answers from a single-function run; any request that needs the export, such as `procedure_assembly` or `analyze_function`, keeps it. + +Regex procedure search uses ECMAScript Unicode matching in a cancellable worker with a five-second matching deadline. Literal matching remains the default. Cancellation or a matching deadline leaves the session available. + +Single-function subprocess output and each cached source facet share a 16 MiB buffered-text capacity. Cached spans are checked before allocation; larger text remains in the local export and the error identifies the file and range. `analyze_function` initially returns the complete dossier, which can still be large; later retained views do not remove that first transfer. Prefer the individual text tools for a focused question. + +## Provenance and limitations + +- Addresses and bytes are Kuna's observations of the image. Procedure entries and extents are its derivations: extents are upper bounds clipped at the next entry, and Kuna can split one function at an entry its own analysis infers. Pseudocode is its reconstruction, never original source. +- Pseudocode and assembly are exact Kuna text; text from Kuna and from another provider is not comparable as equal or different. +- Results are re-read rather than replayed from REA's analysis snapshots, because a cold answer can differ from the export's. + +## Cache ownership + +REA writes Kuna's exports only inside `REA_KUNA_CACHE_DIR`: + +- `projects//` holds published, immutable exports with a manifest of file digests. +- `staging/` holds builds whose processes may still be live. +- `trash/` holds retired builds and evicted exports awaiting removal. +- `home/` is Kuna's HOME. + +Builds publish by one rename, so a reader sees a complete export or none. A session keeps the export it uses from eviction only within its own REA process. If another REA process sharing the cache evicts it, the session rebuilds it; meanwhile pseudocode falls back to single-function runs. A damaged or edited export is removed and rebuilt. Staging moves to `trash/` for removal only after REA proves its Kuna process stopped. Staging of a process it cannot prove stopped is never swept; REA keeps it owned, retries the stop on `close_binary`, and reports it until the stop is proven. Failed file removal also stays bound to its export and is retried by `close_binary`; closing does not report success while that cleanup remains. `rea doctor --provider kuna` reports the cache directory without creating it. + +## Verify with a real Kuna + +```bash +REA_KUNA_COMMAND=/absolute/path/to/kuna npm run verify:kuna +# Compare with Ghidra on caller-built binaries +REA_KUNA_COMMAND=/absolute/path/to/kuna GHIDRA_INSTALL_DIR=/absolute/path/to/ghidra \ + npm run verify:kuna -- --targets DIR --functions inflate,crc32 --compare-ghidra +``` + +The lane compiles a small source-owned fixture with the host C compiler and verifies the MCP workflow end to end. `--targets DIR --functions a,b` adds caller-built binaries. `--compare-ghidra` (with `GHIDRA_INSTALL_DIR`) runs those same functions through Ghidra; it requires `--targets` and `--functions` and refuses empty matrices or requested function names missing from every target. See [testing](testing.md#kuna-lane) for coverage. diff --git a/docs/readme/README_ar.md b/docs/readme/README_ar.md index 3e483ea9b5..c207ad00b8 100644 --- a/docs/readme/README_ar.md +++ b/docs/readme/README_ar.md @@ -166,7 +166,7 @@ npx rea-agents@latest setup -تختلف صيغ ملفات الشيفرة الأصلية والأنظمة المضيفة المدعومة باختلاف الموفّر. راجع [إعداد Hopper وGhidra](../installation.md#hopper)، و[دليل IDA](../ida-provider.md)، و[دعم Ghidra التجريبي على Windows](../windows-ghidra-p0.md). يدعم Ghidra أيضًا [تحليل DOS ذي 16 بت](../ghidra-dos.md). لاختيار الموفّر، راجع [دليل CLI](../cli.md#choose-a-provider). تحقّق من [توفر الميزات في الإصدارات](../installation.md#released-package-and-main) للميزات المضافة بعد أحدث إصدار على npm. +تختلف صيغ ملفات الشيفرة الأصلية والأنظمة المضيفة المدعومة باختلاف الموفّر. راجع [إعداد Hopper وGhidra](../installation.md#hopper)، و[دليل IDA](../ida-provider.md)، [Kuna](../kuna-provider.md)، و[دعم Ghidra التجريبي على Windows](../windows-ghidra-p0.md). يدعم Ghidra أيضًا [تحليل DOS ذي 16 بت](../ghidra-dos.md). لاختيار الموفّر، راجع [دليل CLI](../cli.md#choose-a-provider). تحقّق من [توفر الميزات في الإصدارات](../installation.md#released-package-and-main) للميزات المضافة بعد أحدث إصدار على npm. ## دراسات الحالة diff --git a/docs/readme/README_de.md b/docs/readme/README_de.md index 014b9d21b5..d069bd0750 100644 --- a/docs/readme/README_de.md +++ b/docs/readme/README_de.md @@ -166,7 +166,7 @@ Statische JavaScript- und .NET-Untersuchungen lesen die bereitgestellten Dateien -Native Formate und unterstützte Hostsysteme unterscheiden sich je nach Provider. Siehe [Hopper- und Ghidra-Einrichtung](../installation.md#hopper), den [IDA-Leitfaden](../ida-provider.md) und die [experimentelle Ghidra-Unterstützung unter Windows](../windows-ghidra-p0.md). Ghidra unterstützt auch [16-Bit-DOS-Analyse](../ghidra-dos.md). Die Provider-Auswahl beschreibt der [CLI-Leitfaden](../cli.md#choose-a-provider). Prüfe die [Verfügbarkeit in Releases](../installation.md#released-package-and-main) für Funktionen, die seit der neuesten npm-Version hinzugekommen sind. +Native Formate und unterstützte Hostsysteme unterscheiden sich je nach Provider. Siehe [Hopper- und Ghidra-Einrichtung](../installation.md#hopper), den [IDA-Leitfaden](../ida-provider.md), [Kuna](../kuna-provider.md) und die [experimentelle Ghidra-Unterstützung unter Windows](../windows-ghidra-p0.md). Ghidra unterstützt auch [16-Bit-DOS-Analyse](../ghidra-dos.md). Die Provider-Auswahl beschreibt der [CLI-Leitfaden](../cli.md#choose-a-provider). Prüfe die [Verfügbarkeit in Releases](../installation.md#released-package-and-main) für Funktionen, die seit der neuesten npm-Version hinzugekommen sind. ## Fallstudien diff --git a/docs/readme/README_el.md b/docs/readme/README_el.md index 7e7c53487e..83ac1a9628 100644 --- a/docs/readme/README_el.md +++ b/docs/readme/README_el.md @@ -179,7 +179,7 @@ npx rea-agents@latest setup -Οι μορφές κώδικα μηχανής και τα υποστηριζόμενα συστήματα διαφέρουν ανά πάροχο ανάλυσης. Δες τη [ρύθμιση Hopper και Ghidra](../installation.md#hopper), τον [οδηγό IDA](../ida-provider.md) (που περιλαμβάνει προαιρετική γέφυρα MCP για κώδικα μηχανής) και την [πειραματική υποστήριξη Ghidra στα Windows](../windows-ghidra-p0.md). Το Ghidra υποστηρίζει επίσης [ανάλυση DOS 16 bit](../ghidra-dos.md). Για μεγάλα δυαδικά αρχεία, αύξησε το χρονικό όριο εκκίνησης του παρόχου με `REA_GHIDRA_STARTUP_TIMEOUT_MS` ή `REA_HOPPER_STARTUP_TIMEOUT_MS`. Για την επιλογή παρόχου, δες τον [οδηγό CLI](../cli.md#choose-a-provider). Έλεγξε τη [διαθεσιμότητα ανά έκδοση](../installation.md#released-package-and-main) για λειτουργίες που προστέθηκαν μετά την τελευταία έκδοση στο npm. +Οι μορφές κώδικα μηχανής και τα υποστηριζόμενα συστήματα διαφέρουν ανά πάροχο ανάλυσης. Δες τη [ρύθμιση Hopper και Ghidra](../installation.md#hopper), τον [οδηγό IDA](../ida-provider.md) (που περιλαμβάνει προαιρετική γέφυρα MCP για κώδικα μηχανής), [Kuna](../kuna-provider.md) και την [πειραματική υποστήριξη Ghidra στα Windows](../windows-ghidra-p0.md). Το Ghidra υποστηρίζει επίσης [ανάλυση DOS 16 bit](../ghidra-dos.md). Για μεγάλα δυαδικά αρχεία, αύξησε το χρονικό όριο εκκίνησης του παρόχου με `REA_GHIDRA_STARTUP_TIMEOUT_MS` ή `REA_HOPPER_STARTUP_TIMEOUT_MS`. Για την επιλογή παρόχου, δες τον [οδηγό CLI](../cli.md#choose-a-provider). Έλεγξε τη [διαθεσιμότητα ανά έκδοση](../installation.md#released-package-and-main) για λειτουργίες που προστέθηκαν μετά την τελευταία έκδοση στο npm. ## Μελέτες περίπτωσης diff --git a/docs/readme/README_es.md b/docs/readme/README_es.md index b218c4e22e..9ef6032d89 100644 --- a/docs/readme/README_es.md +++ b/docs/readme/README_es.md @@ -166,7 +166,7 @@ La inspección estática de JavaScript y .NET lee los archivos suministrados sin -Los formatos nativos y los sistemas anfitriones compatibles varían según el proveedor. Consulta [configuración de Hopper y Ghidra](../installation.md#hopper), la [guía de IDA](../ida-provider.md) y la [compatibilidad experimental de Ghidra en Windows](../windows-ghidra-p0.md). Ghidra también admite [análisis de DOS de 16 bits](../ghidra-dos.md). Para elegir un proveedor, consulta la [guía de CLI](../cli.md#choose-a-provider). Comprueba la [disponibilidad por versión](../installation.md#released-package-and-main) de las funciones añadidas después de la última versión publicada en npm. +Los formatos nativos y los sistemas anfitriones compatibles varían según el proveedor. Consulta [configuración de Hopper y Ghidra](../installation.md#hopper), la [guía de IDA](../ida-provider.md), [Kuna](../kuna-provider.md) y la [compatibilidad experimental de Ghidra en Windows](../windows-ghidra-p0.md). Ghidra también admite [análisis de DOS de 16 bits](../ghidra-dos.md). Para elegir un proveedor, consulta la [guía de CLI](../cli.md#choose-a-provider). Comprueba la [disponibilidad por versión](../installation.md#released-package-and-main) de las funciones añadidas después de la última versión publicada en npm. ## Casos prácticos diff --git a/docs/readme/README_fa.md b/docs/readme/README_fa.md index c7f3d2d7da..fe9e68fdc3 100644 --- a/docs/readme/README_fa.md +++ b/docs/readme/README_fa.md @@ -158,7 +158,7 @@ REA به Node.js 22.x (>=22.19)، 24.x (>=24.11) یا 26+، به‌همراه np -فرمت‌های بومی و پشتیبانی سیستم میزبان بر اساس ارائه‌دهنده متفاوت‌اند. [راه‌اندازی Hopper و Ghidra](../installation.md#hopper)، [راهنمای IDA](../ida-provider.md) و [پشتیبانی آزمایشی Ghidra در ویندوز](../windows-ghidra-p0.md) را ببینید. Ghidra همچنین از [تحلیل DOS شانزده‌بیتی](../ghidra-dos.md) پشتیبانی می‌کند. برای انتخاب ارائه‌دهنده، به [راهنمای CLI](../cli.md#choose-a-provider) مراجعه کنید. برای قابلیت‌هایی که پس از آخرین انتشار npm اضافه شده‌اند، [وضعیت انتشار](../installation.md#released-package-and-main) را بررسی کنید. +فرمت‌های بومی و پشتیبانی سیستم میزبان بر اساس ارائه‌دهنده متفاوت‌اند. [راه‌اندازی Hopper و Ghidra](../installation.md#hopper)، [راهنمای IDA](../ida-provider.md)، [Kuna](../kuna-provider.md) و [پشتیبانی آزمایشی Ghidra در ویندوز](../windows-ghidra-p0.md) را ببینید. Ghidra همچنین از [تحلیل DOS شانزده‌بیتی](../ghidra-dos.md) پشتیبانی می‌کند. برای انتخاب ارائه‌دهنده، به [راهنمای CLI](../cli.md#choose-a-provider) مراجعه کنید. برای قابلیت‌هایی که پس از آخرین انتشار npm اضافه شده‌اند، [وضعیت انتشار](../installation.md#released-package-and-main) را بررسی کنید. ## نمونه‌های عملی diff --git a/docs/readme/README_fr.md b/docs/readme/README_fr.md index a5fbe8eb95..bde58fa38b 100644 --- a/docs/readme/README_fr.md +++ b/docs/readme/README_fr.md @@ -170,7 +170,7 @@ L'inspection statique JavaScript et .NET lit les fichiers fournis sans exécuter -Les formats natifs et les hôtes pris en charge varient selon le fournisseur. Consultez [la configuration de Hopper et Ghidra](../installation.md#hopper), le [guide IDA](../ida-provider.md) et [la prise en charge expérimentale de Ghidra sous Windows](../windows-ghidra-p0.md). Ghidra prend aussi en charge [l'analyse DOS 16 bits](../ghidra-dos.md). Pour les gros binaires, augmentez son délai de démarrage avec `REA_GHIDRA_STARTUP_TIMEOUT_MS`. Pour le choix du fournisseur, consultez le [guide CLI](../cli.md#choose-a-provider). Vérifiez la [disponibilité dans les versions publiées](../installation.md#released-package-and-main) pour les fonctionnalités ajoutées depuis la dernière version npm. +Les formats natifs et les hôtes pris en charge varient selon le fournisseur. Consultez [la configuration de Hopper et Ghidra](../installation.md#hopper), le [guide IDA](../ida-provider.md), [Kuna](../kuna-provider.md) et [la prise en charge expérimentale de Ghidra sous Windows](../windows-ghidra-p0.md). Ghidra prend aussi en charge [l'analyse DOS 16 bits](../ghidra-dos.md). Pour les gros binaires, augmentez son délai de démarrage avec `REA_GHIDRA_STARTUP_TIMEOUT_MS`. Pour le choix du fournisseur, consultez le [guide CLI](../cli.md#choose-a-provider). Vérifiez la [disponibilité dans les versions publiées](../installation.md#released-package-and-main) pour les fonctionnalités ajoutées depuis la dernière version npm. ## Études de cas diff --git a/docs/readme/README_id.md b/docs/readme/README_id.md index 2fbc3ba714..6f1eba103e 100644 --- a/docs/readme/README_id.md +++ b/docs/readme/README_id.md @@ -173,7 +173,7 @@ Pemeriksaan JavaScript dan .NET statis membaca berkas yang disediakan tanpa menj -Format native dan dukungan sistem host berbeda-beda menurut penyedia. Lihat [penyiapan Hopper dan Ghidra](../installation.md#hopper), [panduan IDA](../ida-provider.md), dan [dukungan eksperimental Ghidra di Windows](../windows-ghidra-p0.md). +Format native dan dukungan sistem host berbeda-beda menurut penyedia. Lihat [penyiapan Hopper dan Ghidra](../installation.md#hopper), [panduan IDA](../ida-provider.md), [Kuna](../kuna-provider.md), dan [dukungan eksperimental Ghidra di Windows](../windows-ghidra-p0.md). Ghidra juga mendukung [analisis DOS 16-bit](../ghidra-dos.md). diff --git a/docs/readme/README_it.md b/docs/readme/README_it.md index ae163e83a5..5513321715 100644 --- a/docs/readme/README_it.md +++ b/docs/readme/README_it.md @@ -170,7 +170,7 @@ L'ispezione statica di JavaScript e .NET legge i file forniti senza eseguire l'a -I formati nativi e il supporto host variano in base al provider. Vedi la [configurazione di Hopper e Ghidra](../installation.md#hopper), la [guida IDA](../ida-provider.md) e il [supporto sperimentale Windows Ghidra](../windows-ghidra-p0.md). Ghidra supporta anche l'[analisi DOS a 16 bit](../ghidra-dos.md). Per binari di grandi dimensioni, aumenta il timeout di avvio con `REA_GHIDRA_STARTUP_TIMEOUT_MS`. Per la selezione del provider, vedi la [guida CLI](../cli.md#choose-a-provider). Controlla la [disponibilità delle release](../installation.md#released-package-and-main) per le funzionalità aggiunte dall'ultima release npm. +I formati nativi e il supporto host variano in base al provider. Vedi la [configurazione di Hopper e Ghidra](../installation.md#hopper), la [guida IDA](../ida-provider.md), [Kuna](../kuna-provider.md) e il [supporto sperimentale Windows Ghidra](../windows-ghidra-p0.md). Ghidra supporta anche l'[analisi DOS a 16 bit](../ghidra-dos.md). Per binari di grandi dimensioni, aumenta il timeout di avvio con `REA_GHIDRA_STARTUP_TIMEOUT_MS`. Per la selezione del provider, vedi la [guida CLI](../cli.md#choose-a-provider). Controlla la [disponibilità delle release](../installation.md#released-package-and-main) per le funzionalità aggiunte dall'ultima release npm. ## Showcase diff --git a/docs/readme/README_ja.md b/docs/readme/README_ja.md index 1d2aec01ce..53745cadc3 100644 --- a/docs/readme/README_ja.md +++ b/docs/readme/README_ja.md @@ -166,7 +166,7 @@ REA には Node.js 22.x（>=22.19）、24.x（>=24.11）、または 26+ と npm -対応するネイティブ形式とホストは、プロバイダーによって異なります。[Hopper と Ghidra のセットアップ](../installation.md#hopper)、[IDA ガイド](../ida-provider.md)、[Windows 向け Ghidra の実験的対応](../windows-ghidra-p0.md)を参照してください。Ghidra は [16 ビット DOS 分析](../ghidra-dos.md)にも対応しています。プロバイダーの選択は [CLI ガイド](../cli.md#choose-a-provider)を参照してください。最新の npm リリース以降に追加された機能については、[リリースでの利用可否](../installation.md#released-package-and-main)を確認してください。 +対応するネイティブ形式とホストは、プロバイダーによって異なります。[Hopper と Ghidra のセットアップ](../installation.md#hopper)、[IDA ガイド](../ida-provider.md)、[Kuna](../kuna-provider.md)、[Windows 向け Ghidra の実験的対応](../windows-ghidra-p0.md)を参照してください。Ghidra は [16 ビット DOS 分析](../ghidra-dos.md)にも対応しています。プロバイダーの選択は [CLI ガイド](../cli.md#choose-a-provider)を参照してください。最新の npm リリース以降に追加された機能については、[リリースでの利用可否](../installation.md#released-package-and-main)を確認してください。 ## 事例 diff --git a/docs/readme/README_ko.md b/docs/readme/README_ko.md index c6c499056e..1411688c18 100644 --- a/docs/readme/README_ko.md +++ b/docs/readme/README_ko.md @@ -175,7 +175,7 @@ REA에는 Node.js 22.x(>=22.19), 24.x(>=24.11) 또는 26+와 npm이 필요합니 -지원하는 네이티브 형식과 호스트는 제공자마다 다릅니다. [Hopper 및 Ghidra 설정](../installation.md#hopper), [IDA 가이드](../ida-provider.md)(선택 사항인 네이티브 MCP 브리지 포함), [실험적 Windows Ghidra 지원](../windows-ghidra-p0.md)을 참고하세요. Ghidra는 [16비트 DOS 분석](../ghidra-dos.md)도 지원합니다. 큰 바이너리를 분석할 때는 `REA_GHIDRA_STARTUP_TIMEOUT_MS` 또는 `REA_HOPPER_STARTUP_TIMEOUT_MS`로 제공자의 시작 제한 시간을 늘리세요. 제공자 선택은 [CLI 가이드](../cli.md#choose-a-provider)를 참고하세요. 최신 npm 릴리스 이후 추가된 기능은 [릴리스에서의 사용 가능 여부](../installation.md#released-package-and-main)를 확인하세요. +지원하는 네이티브 형식과 호스트는 제공자마다 다릅니다. [Hopper 및 Ghidra 설정](../installation.md#hopper), [IDA 가이드](../ida-provider.md)(선택 사항인 네이티브 MCP 브리지 포함), [Kuna](../kuna-provider.md), [실험적 Windows Ghidra 지원](../windows-ghidra-p0.md)을 참고하세요. Ghidra는 [16비트 DOS 분석](../ghidra-dos.md)도 지원합니다. 큰 바이너리를 분석할 때는 `REA_GHIDRA_STARTUP_TIMEOUT_MS` 또는 `REA_HOPPER_STARTUP_TIMEOUT_MS`로 제공자의 시작 제한 시간을 늘리세요. 제공자 선택은 [CLI 가이드](../cli.md#choose-a-provider)를 참고하세요. 최신 npm 릴리스 이후 추가된 기능은 [릴리스에서의 사용 가능 여부](../installation.md#released-package-and-main)를 확인하세요. ## 사례 diff --git a/docs/readme/README_pl.md b/docs/readme/README_pl.md index bb32355f9e..3102f2784a 100644 --- a/docs/readme/README_pl.md +++ b/docs/readme/README_pl.md @@ -166,7 +166,7 @@ Statyczna analiza JavaScript i .NET odczytuje dostarczone pliki bez uruchamiania -Obsługiwane formaty natywne i platformy zależą od dostawcy analizy. Zobacz [konfigurację Hoppera i Ghidry](../installation.md#hopper), [przewodnik IDA](../ida-provider.md) oraz [eksperymentalną obsługę Ghidry w systemie Windows](../windows-ghidra-p0.md). Ghidra obsługuje też [analizę 16-bitowego DOS-a](../ghidra-dos.md). Wybór dostawcy opisuje [przewodnik CLI](../cli.md#choose-a-provider). Sprawdź [dostępność w wydaniach](../installation.md#released-package-and-main) dla funkcji dodanych po najnowszym wydaniu npm. +Obsługiwane formaty natywne i platformy zależą od dostawcy analizy. Zobacz [konfigurację Hoppera i Ghidry](../installation.md#hopper), [przewodnik IDA](../ida-provider.md), [Kuna](../kuna-provider.md) oraz [eksperymentalną obsługę Ghidry w systemie Windows](../windows-ghidra-p0.md). Ghidra obsługuje też [analizę 16-bitowego DOS-a](../ghidra-dos.md). Wybór dostawcy opisuje [przewodnik CLI](../cli.md#choose-a-provider). Sprawdź [dostępność w wydaniach](../installation.md#released-package-and-main) dla funkcji dodanych po najnowszym wydaniu npm. ## Przykłady zastosowań diff --git a/docs/readme/README_pt-BR.md b/docs/readme/README_pt-BR.md index 9802205ed5..c52429eff2 100644 --- a/docs/readme/README_pt-BR.md +++ b/docs/readme/README_pt-BR.md @@ -166,7 +166,7 @@ A inspeção estática de JavaScript e .NET lê os arquivos fornecidos sem execu -Os formatos nativos e os sistemas anfitriões compatíveis variam por provedor. Consulte [configuração do Hopper e Ghidra](../installation.md#hopper), o [guia do IDA](../ida-provider.md) e o [suporte experimental ao Ghidra no Windows](../windows-ghidra-p0.md). O Ghidra também oferece [análise de DOS de 16 bits](../ghidra-dos.md). Para selecionar um provedor, consulte o [guia da CLI](../cli.md#choose-a-provider). Verifique a [disponibilidade nas versões](../installation.md#released-package-and-main) das funcionalidades adicionadas após a versão mais recente no npm. +Os formatos nativos e os sistemas anfitriões compatíveis variam por provedor. Consulte [configuração do Hopper e Ghidra](../installation.md#hopper), o [guia do IDA](../ida-provider.md), [Kuna](../kuna-provider.md) e o [suporte experimental ao Ghidra no Windows](../windows-ghidra-p0.md). O Ghidra também oferece [análise de DOS de 16 bits](../ghidra-dos.md). Para selecionar um provedor, consulte o [guia da CLI](../cli.md#choose-a-provider). Verifique a [disponibilidade nas versões](../installation.md#released-package-and-main) das funcionalidades adicionadas após a versão mais recente no npm. ## Casos de uso diff --git a/docs/readme/README_ru.md b/docs/readme/README_ru.md index 502583328b..96a6d14d67 100644 --- a/docs/readme/README_ru.md +++ b/docs/readme/README_ru.md @@ -166,7 +166,7 @@ REA требует Node.js 22.x (>=22.19), 24.x (>=24.11) или 26+, а так -Нативные форматы и поддерживаемые системы различаются у разных провайдеров. См. [настройку Hopper и Ghidra](../installation.md#hopper), [руководство по IDA](../ida-provider.md) и [экспериментальную поддержку Ghidra в Windows](../windows-ghidra-p0.md). Ghidra также поддерживает [анализ 16-битного DOS](../ghidra-dos.md). Выбор провайдера описан в [руководстве по CLI](../cli.md#choose-a-provider). Для функций, добавленных после последней версии в npm, проверьте [доступность в релизах](../installation.md#released-package-and-main). +Нативные форматы и поддерживаемые системы различаются у разных провайдеров. См. [настройку Hopper и Ghidra](../installation.md#hopper), [руководство по IDA](../ida-provider.md), [Kuna](../kuna-provider.md) и [экспериментальную поддержку Ghidra в Windows](../windows-ghidra-p0.md). Ghidra также поддерживает [анализ 16-битного DOS](../ghidra-dos.md). Выбор провайдера описан в [руководстве по CLI](../cli.md#choose-a-provider). Для функций, добавленных после последней версии в npm, проверьте [доступность в релизах](../installation.md#released-package-and-main). ## Примеры diff --git a/docs/readme/README_th.md b/docs/readme/README_th.md index 747a4c1adb..4ee882aedd 100644 --- a/docs/readme/README_th.md +++ b/docs/readme/README_th.md @@ -166,7 +166,7 @@ REA ต้องใช้ Node.js 22.x (>=22.19), 24.x (>=24.11) หรือ 26 -รูปแบบไบนารีเนทีฟและแพลตฟอร์มที่รองรับแตกต่างกันตามผู้ให้บริการวิเคราะห์ ดู[การตั้งค่า Hopper และ Ghidra](../installation.md#hopper), [คู่มือ IDA](../ida-provider.md) และ[การรองรับ Ghidra บน Windows แบบทดลอง](../windows-ghidra-p0.md) Ghidra ยังรองรับ[การวิเคราะห์ DOS แบบ 16 บิต](../ghidra-dos.md) สำหรับการเลือกผู้ให้บริการ ดู[คู่มือ CLI](../cli.md#choose-a-provider) ตรวจสอบ[ความพร้อมใช้งานในแต่ละรุ่น](../installation.md#released-package-and-main) สำหรับฟีเจอร์ที่เพิ่มหลังรุ่นล่าสุดบน npm +รูปแบบไบนารีเนทีฟและแพลตฟอร์มที่รองรับแตกต่างกันตามผู้ให้บริการวิเคราะห์ ดู[การตั้งค่า Hopper และ Ghidra](../installation.md#hopper), [คู่มือ IDA](../ida-provider.md) [Kuna](../kuna-provider.md) และ[การรองรับ Ghidra บน Windows แบบทดลอง](../windows-ghidra-p0.md) Ghidra ยังรองรับ[การวิเคราะห์ DOS แบบ 16 บิต](../ghidra-dos.md) สำหรับการเลือกผู้ให้บริการ ดู[คู่มือ CLI](../cli.md#choose-a-provider) ตรวจสอบ[ความพร้อมใช้งานในแต่ละรุ่น](../installation.md#released-package-and-main) สำหรับฟีเจอร์ที่เพิ่มหลังรุ่นล่าสุดบน npm ## ตัวอย่างการใช้งาน diff --git a/docs/readme/README_tr.md b/docs/readme/README_tr.md index 6562b4796e..5ba33b34f5 100644 --- a/docs/readme/README_tr.md +++ b/docs/readme/README_tr.md @@ -171,7 +171,7 @@ Statik JavaScript ve .NET incelemesi, uygulamayı çalıştırmadan sağlanan do -Yerel makine kodu içeren ikili dosya biçimleri ve ana bilgisayar desteği sağlayıcıya göre değişir. [Hopper ve Ghidra kurulumu](../installation.md#hopper), isteğe bağlı native MCP bridge'i de açıklayan [IDA rehberi](../ida-provider.md) ve [Windows için deneysel Ghidra desteği](../windows-ghidra-p0.md) belgelerine bakın. Ghidra ayrıca [16 bit DOS analizini](../ghidra-dos.md) destekler. Büyük ikili dosyalar için sağlayıcının başlatma zaman aşımını `REA_GHIDRA_STARTUP_TIMEOUT_MS` veya `REA_HOPPER_STARTUP_TIMEOUT_MS` ile artırın. Sağlayıcı seçimi için [CLI rehberine](../cli.md#choose-a-provider) bakın. En son npm sürümünden sonra eklenen özellikler için [sürümlerde kullanılabilirliği](../installation.md#released-package-and-main) kontrol edin. +Yerel makine kodu içeren ikili dosya biçimleri ve ana bilgisayar desteği sağlayıcıya göre değişir. [Hopper ve Ghidra kurulumu](../installation.md#hopper), isteğe bağlı native MCP bridge'i de açıklayan [IDA rehberi](../ida-provider.md), [Kuna](../kuna-provider.md) ve [Windows için deneysel Ghidra desteği](../windows-ghidra-p0.md) belgelerine bakın. Ghidra ayrıca [16 bit DOS analizini](../ghidra-dos.md) destekler. Büyük ikili dosyalar için sağlayıcının başlatma zaman aşımını `REA_GHIDRA_STARTUP_TIMEOUT_MS` veya `REA_HOPPER_STARTUP_TIMEOUT_MS` ile artırın. Sağlayıcı seçimi için [CLI rehberine](../cli.md#choose-a-provider) bakın. En son npm sürümünden sonra eklenen özellikler için [sürümlerde kullanılabilirliği](../installation.md#released-package-and-main) kontrol edin. ## Örnek çalışmalar diff --git a/docs/readme/README_uk.md b/docs/readme/README_uk.md index cd28f22d00..232ced41fe 100644 --- a/docs/readme/README_uk.md +++ b/docs/readme/README_uk.md @@ -166,7 +166,7 @@ REA потребує Node.js 22.x (>=22.19), 24.x (>=24.11) або 26+, а та -Нативні формати й підтримка систем відрізняються залежно від провайдера. Дивіться [налаштування Hopper і Ghidra](../installation.md#hopper), [посібник IDA](../ida-provider.md) та [експериментальну підтримку Ghidra у Windows](../windows-ghidra-p0.md). Ghidra також підтримує [аналіз 16-бітного DOS](../ghidra-dos.md). Для вибору провайдера дивіться [посібник CLI](../cli.md#choose-a-provider). Для функцій, доданих після найновішої версії в npm, перевірте [доступність у випусках](../installation.md#released-package-and-main). +Нативні формати й підтримка систем відрізняються залежно від провайдера. Дивіться [налаштування Hopper і Ghidra](../installation.md#hopper), [посібник IDA](../ida-provider.md), [Kuna](../kuna-provider.md) та [експериментальну підтримку Ghidra у Windows](../windows-ghidra-p0.md). Ghidra також підтримує [аналіз 16-бітного DOS](../ghidra-dos.md). Для вибору провайдера дивіться [посібник CLI](../cli.md#choose-a-provider). Для функцій, доданих після найновішої версії в npm, перевірте [доступність у випусках](../installation.md#released-package-and-main). ## Приклади diff --git a/docs/readme/README_vi.md b/docs/readme/README_vi.md index 829d9395a7..8cadf6f2fc 100644 --- a/docs/readme/README_vi.md +++ b/docs/readme/README_vi.md @@ -166,7 +166,7 @@ Kiểm tra tĩnh JavaScript và .NET đọc các tệp được cung cấp mà k -Định dạng mã máy và nền tảng chạy REA được hỗ trợ khác nhau tùy trình phân tích. Xem [thiết lập Hopper và Ghidra](../installation.md#hopper), [hướng dẫn IDA](../ida-provider.md) và [hỗ trợ Ghidra thử nghiệm trên Windows](../windows-ghidra-p0.md). Ghidra cũng hỗ trợ [phân tích DOS 16 bit](../ghidra-dos.md). Để chọn trình phân tích, xem [hướng dẫn CLI](../cli.md#choose-a-provider). Kiểm tra [khả năng sử dụng theo bản phát hành](../installation.md#released-package-and-main) đối với tính năng được bổ sung sau phiên bản npm mới nhất. +Định dạng mã máy và nền tảng chạy REA được hỗ trợ khác nhau tùy trình phân tích. Xem [thiết lập Hopper và Ghidra](../installation.md#hopper), [hướng dẫn IDA](../ida-provider.md), [Kuna](../kuna-provider.md) và [hỗ trợ Ghidra thử nghiệm trên Windows](../windows-ghidra-p0.md). Ghidra cũng hỗ trợ [phân tích DOS 16 bit](../ghidra-dos.md). Để chọn trình phân tích, xem [hướng dẫn CLI](../cli.md#choose-a-provider). Kiểm tra [khả năng sử dụng theo bản phát hành](../installation.md#released-package-and-main) đối với tính năng được bổ sung sau phiên bản npm mới nhất. ## Các nghiên cứu điển hình diff --git a/docs/readme/README_zh-TW.md b/docs/readme/README_zh-TW.md index e18576f569..173e9e0780 100644 --- a/docs/readme/README_zh-TW.md +++ b/docs/readme/README_zh-TW.md @@ -170,7 +170,7 @@ REA 需要 Node.js 22.x（>=22.19）、24.x（>=24.11）或 26+，以及 npm。 -不同提供者支援的原生格式和主機系統有所不同。請參閱 [Hopper 與 Ghidra 設定](../installation.md#hopper)、[IDA 指南](../ida-provider.md)和[實驗性 Windows Ghidra 支援](../windows-ghidra-p0.md)。Ghidra 也支援 [16 位 DOS 分析](../ghidra-dos.md)。提供者選擇見 [CLI 指南](../cli.md#choose-a-provider)。對於最新 npm 版本發佈後新增的功能，請查看[發佈可用性](../installation.md#released-package-and-main)。 +不同提供者支援的原生格式和主機系統有所不同。請參閱 [Hopper 與 Ghidra 設定](../installation.md#hopper)、[IDA 指南](../ida-provider.md)、[Kuna](../kuna-provider.md)和[實驗性 Windows Ghidra 支援](../windows-ghidra-p0.md)。Ghidra 也支援 [16 位 DOS 分析](../ghidra-dos.md)。提供者選擇見 [CLI 指南](../cli.md#choose-a-provider)。對於最新 npm 版本發佈後新增的功能，請查看[發佈可用性](../installation.md#released-package-and-main)。 ## 案例 diff --git a/docs/readme/README_zh.md b/docs/readme/README_zh.md index 06db6238f5..bf1257ead7 100644 --- a/docs/readme/README_zh.md +++ b/docs/readme/README_zh.md @@ -170,7 +170,7 @@ REA 需要 Node.js 22.x（>=22.19）、24.x（>=24.11）或 26+，以及 npm。 -不同提供方支持的原生格式和主机系统有所不同。请参阅 [Hopper 与 Ghidra 设置](../installation.md#hopper)、[IDA 指南](../ida-provider.md)和[实验性 Windows Ghidra 支持](../windows-ghidra-p0.md)。Ghidra 也支持 [16 位 DOS 分析](../ghidra-dos.md)。提供方选择见 [CLI 指南](../cli.md#choose-a-provider)。对于最新 npm 版本发布后新增的功能，请查看[发布可用性](../installation.md#released-package-and-main)。 +不同提供方支持的原生格式和主机系统有所不同。请参阅 [Hopper 与 Ghidra 设置](../installation.md#hopper)、[IDA 指南](../ida-provider.md)、[Kuna](../kuna-provider.md)和[实验性 Windows Ghidra 支持](../windows-ghidra-p0.md)。Ghidra 也支持 [16 位 DOS 分析](../ghidra-dos.md)。提供方选择见 [CLI 指南](../cli.md#choose-a-provider)。对于最新 npm 版本发布后新增的功能，请查看[发布可用性](../installation.md#released-package-and-main)。 ## 案例 diff --git a/docs/testing.md b/docs/testing.md index 93a65ffcc9..daf5c80b71 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -817,6 +817,34 @@ Native IDA MCP verification uses `"protocol": "native"` with an already-running The initial Linux x64 IDA Free 9.4 run used native MCP 1.1.2 with a local navigation correction. Windows x64 verification uses **IDA Pro 9.3**, the two-function public fixture, and the shipped Windows `whoami.exe` application. The native plugin was built from upstream `c9d15d98e8a143fc74f6e14037c48f63889b6d6c`, including the merged navigation fix, with a local loopback-listener change. Production CLI/MCP parity, switching, identity and cleanup passed. Correctness and performance measurements are recorded in the [Windows native verification notes](https://github.com/morluto/rea/blob/main/experiments/analysis-efficiency/ida-native-windows.md). Stock 1.1.2 switching, native macOS, and Windows IDA Free remain unverified. See the [native bridge prerequisites](ida-provider.md#optional-native-ida-mcp-bridge). +## Kuna lane + +`npm run verify:kuna` requires a caller-supplied `kuna` in `REA_KUNA_COMMAND` +and a host C compiler (`REA_CC`, default `cc`) with `strip` and `nm`. It builds a +source-owned fixture with symbols and stripped, then verifies through MCP that +the first cold pseudocode answer comes from a single-function run or an export +that already finished, assembly waits for and serves from that export, and +pseudocode and both dossier facets match their recorded file spans and digests, +a fresh server reuses the export without rebuilding it, staging is empty after +a completed export, and no owned process survives. Pass +`--targets DIR --functions a,b` to add caller-built binaries laid out as +`DIR//` beside `.stripped`; addresses come from the +symbol-bearing build. `--compare-ghidra` with `GHIDRA_INSTALL_DIR` runs those +caller-built functions through Ghidra (it refuses to run without `--targets` and +`--functions`, an empty matrix, or names missing from every supplied target) and +reports latency, sampled process-tree CPU and peak memory, UTF-8 response size, +and goto and warning-marker counts. Those counts are not a correctness judgement, +and model tokens are not measured. `--report PATH` writes the JSON report. + +The first matrix run used Kuna 1.833 and Ghidra 12.1.4 with JDK 21 on macOS +arm64. It covered zlib 1.3.2 `minigzip`, coreutils 9.9 `sort` and the sqlite +3.54.0 shell, built at `-O2` for ELF x86-64, ELF AArch64, Mach-O arm64 and +Mach-O x86-64, each with symbols and stripped. A separate Linux x86-64 run verified +the fixture and system zlib 1.2.13 / SQLite 3.40.1 libraries through MCP, including +cached byte spans and a Ghidra comparison. See the [Linux report](https://github.com/morluto/rea/pull/3854#issuecomment-6185870391) +for measured scope and limitations. Windows, PE and universal Mach-O remain +unverified by this lane. + ## DOS Ghidra analysis `npm run verify:ghidra:dos` requires the supported Ghidra and JDK installation diff --git a/package.json b/package.json index 305c0bd4fd..330d0d91b7 100644 --- a/package.json +++ b/package.json @@ -194,6 +194,7 @@ "verify:ghidra:swift": "npm run build:cached && node scripts/verify-real-ghidra-swift.mjs", "verify:ida": "npm run build:cached && node scripts/verify-real-ida.mjs", "verify:ghidra:switch": "npm run build:cached && node scripts/verify-real-ghidra-switch.mjs", + "verify:kuna": "npm run build:cached && node scripts/verify-real-kuna.mjs", "verify:ghidra:dos": "npm run build:cached && node scripts/verify-real-ghidra-dos.mjs", "verify:ghidra:com": "npm run build:cached && node scripts/verify-real-ghidra-com.mjs", "verify:ghidra:raw": "npm run build:cached && node scripts/verify-real-ghidra-raw.mjs", diff --git a/scripts/lib/catalog-builders.mjs b/scripts/lib/catalog-builders.mjs index 64296b3ceb..4e8e10f1b9 100644 --- a/scripts/lib/catalog-builders.mjs +++ b/scripts/lib/catalog-builders.mjs @@ -218,6 +218,13 @@ export const providerCatalog = (sources) => { identity: sources.idaProvider.IDA_PROVIDER_IDENTITY, contracts: declaredContracts("ida", sources.idaProvider.IDA_OPERATIONS), }, + { + identity: sources.kunaProvider.KUNA_PROVIDER_IDENTITY, + contracts: declaredContracts( + "kuna", + sources.kunaProvider.KUNA_OPERATIONS, + ), + }, { identity: sources.nativeProvider.NATIVE_MACOS_PROVIDER_IDENTITY, contracts: sources.nativeContracts.NATIVE_TOOL_CONTRACTS, diff --git a/scripts/lib/catalog-core.mjs b/scripts/lib/catalog-core.mjs index 2cdfb64d20..afa8bbcb55 100644 --- a/scripts/lib/catalog-core.mjs +++ b/scripts/lib/catalog-core.mjs @@ -109,6 +109,7 @@ const SOURCE_PATHS = { hopperProvider: "dist/hopper/HopperProviderCapabilities.js", ghidraProvider: "dist/ghidra/GhidraProviderCapabilities.js", idaProvider: "dist/ida/IdaProviderCapabilities.js", + kunaProvider: "dist/kuna/KunaProviderCapabilities.js", nativeProvider: "dist/native/NativeMacOSProviderMetadata.js", artifactProviders: "dist/application/InvestigationProviders.js", browserIdentities: "dist/browser/providerIdentities.js", diff --git a/scripts/verify-real-kuna.mjs b/scripts/verify-real-kuna.mjs new file mode 100644 index 0000000000..a59b6858dd --- /dev/null +++ b/scripts/verify-real-kuna.mjs @@ -0,0 +1,458 @@ +#!/usr/bin/env node +// Real Kuna provider lane. Requires a caller-supplied kuna (REA_KUNA_COMMAND) +// and a host C compiler; REA installs neither. The default run builds a +// source-owned fixture, with symbols and stripped, and verifies the public MCP +// workflow: cold single-function pseudocode, the project export, warm reuse +// from a fresh server, and cleanup. `--targets DIR --functions a,b` adds +// caller-built binaries (DIR// plus .stripped). +// `--compare-ghidra` runs the same functions through a bring-your-own Ghidra +// and reports latency, CPU, memory, output size and simple quality signals. +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { + mkdir, + mkdtemp, + readFile, + readdir, + rm, + stat, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { parseArgs, promisify } from "node:util"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs"; +import { + assertVerifierCleanupComplete, + completeVerifierRun, + createVerifierRun, +} from "./lib/verifier-run.mjs"; + +const exec = promisify(execFile); +const { values } = parseArgs({ + options: { + targets: { type: "string" }, + functions: { type: "string" }, + "compare-ghidra": { type: "boolean", default: false }, + report: { type: "string" }, + entrypoint: { type: "string" }, + }, +}); +const entrypoint = + values.entrypoint === undefined + ? fileURLToPath(new URL("./rea.mjs", import.meta.url)) + : resolve(values.entrypoint); +const kuna = process.env.REA_KUNA_COMMAND; +assert.ok( + kuna !== undefined && kuna.startsWith("/"), + "Kuna lane prerequisite missing: REA_KUNA_COMMAND must name a caller-supplied kuna by absolute path (bring your own)", +); +if (values["compare-ghidra"]) { + // The comparison runs on caller-built targets only; the fixture has none. + assert.ok( + values.targets !== undefined && values.functions !== undefined, + "--compare-ghidra compares caller-built targets: pass --targets DIR --functions a,b as well", + ); + assert.ok( + process.env.GHIDRA_INSTALL_DIR, + "Ghidra comparison prerequisite missing: GHIDRA_INSTALL_DIR (bring your own Ghidra 12.1.4 and JDK 21)", + ); +} +const kunaVersion = (await exec(kuna, ["--version"])).stdout.trim(); +const run = createVerifierRun(); +const workspace = await mkdtemp(join(tmpdir(), "rea-kuna-lane-")); +const cacheRoot = join(workspace, "cache"); + +const FIXTURE_SOURCE = String.raw` +#include +#include +#include +unsigned rea_checksum(const unsigned char *p, size_t n) { + unsigned h = 2166136261u; + for (size_t i = 0; i < n; i++) h = (h ^ p[i]) * 16777619u; + return h; +} +int rea_classify(int c) { + switch (c) { + case 'a': return 1; case 'e': return 2; case 'i': return 3; + case 'o': return 4; case 'u': return 5; default: return c > 'z' ? -1 : 0; + } +} +int main(int argc, char **argv) { + if (argc < 2) return 2; + printf("%u %d\n", rea_checksum((const unsigned char *)argv[1], strlen(argv[1])), + rea_classify(argv[1][0])); + return 0; +} +`; + +/** Resolve function entry addresses from a symbol-bearing build. */ +const symbolAddresses = async (binary, names) => { + const { stdout } = await exec(process.env.REA_NM ?? "nm", [binary]); + const found = new Map(); + for (const line of stdout.split("\n")) { + const [address, type, symbol] = line.trim().split(/\s+/u); + if (symbol === undefined || !/^[tT]$/u.test(type ?? "")) continue; + const plain = symbol.startsWith("_") ? symbol.slice(1) : symbol; + for (const name of names) + if ((symbol === name || plain === name) && !found.has(name)) + found.set(name, `0x${BigInt(`0x${address}`).toString(16)}`); + } + return found; +}; + +/** Sum the resident memory and CPU time of a process and its descendants. */ +const sampleTree = async (root) => { + const { stdout } = await exec("ps", ["-A", "-o", "pid=,ppid=,rss=,time="]); + const rows = stdout + .trim() + .split("\n") + .map((line) => line.trim().split(/\s+/u)) + .map(([pid, ppid, rss, time]) => ({ + pid: Number(pid), + ppid: Number(ppid), + rssKiB: Number(rss), + cpuSeconds: (time ?? "0:0") + .split(/[:-]/u) + .map(Number) + .reduce((total, part) => total * 60 + part, 0), + })); + const members = new Set([root]); + for (let grew = true; grew;) { + grew = false; + for (const row of rows) + if (!members.has(row.pid) && members.has(row.ppid)) { + members.add(row.pid); + grew = true; + } + } + const tree = rows.filter(({ pid }) => members.has(pid)); + return { + rssKiB: tree.reduce((total, row) => total + row.rssKiB, 0), + cpu: new Map(tree.map((row) => [row.pid, row.cpuSeconds])), + }; +}; + +/** One MCP server with a selected provider, sampled while it runs. */ +const openServer = async (provider) => { + const transport = new StdioClientTransport({ + command: process.execPath, + args: [entrypoint, "--mcp"], + env: { + PATH: process.env.PATH ?? "/usr/bin:/bin", + HOME: process.env.HOME ?? workspace, + REA_LOG_LEVEL: "silent", + REA_PROCESS_RUN_ID: run.run_id, + REA_KUNA_COMMAND: kuna, + REA_KUNA_CACHE_DIR: cacheRoot, + ...(process.env.GHIDRA_INSTALL_DIR === undefined + ? {} + : { GHIDRA_INSTALL_DIR: process.env.GHIDRA_INSTALL_DIR }), + ...(process.env.JAVA_HOME === undefined + ? {} + : { JAVA_HOME: process.env.JAVA_HOME }), + }, + stderr: "ignore", + }); + const client = new Client({ name: "rea-real-kuna", version: "1" }); + await client.connect(transport); + const pid = transport.pid; + let peakKiB = 0; + const cpu = new Map(); + const sample = async () => { + if (pid === null) return; + const observed = await sampleTree(pid).catch(() => undefined); + if (observed === undefined) return; + peakKiB = Math.max(peakKiB, observed.rssKiB); + for (const [member, seconds] of observed.cpu) + cpu.set(member, Math.max(cpu.get(member) ?? 0, seconds)); + }; + const timer = setInterval(() => void sample(), 200); + const call = async (name, arguments_ = {}) => { + const started = performance.now(); + const result = await client.callTool( + { name, arguments: arguments_ }, + { timeout: 3_600_000 }, + ); + const ms = Math.round(performance.now() - started); + const bytes = Buffer.byteLength(JSON.stringify(result)); + return { + ms, + bytes, + value: requireMcpOperationResult(result, name), + structured: result.structuredContent, + }; + }; + const close = async () => { + await sample(); + clearInterval(timer); + try { + await call("close_binary"); + } finally { + await client.close(); + await transport.close(); + } + return { + peak_rss_mib: Math.round(peakKiB / 1024), + cpu_seconds: [...cpu.values()].reduce((total, value) => total + value, 0), + }; + }; + try { + await call("open_binary", { + path: provider.path, + provider_id: provider.id, + }); + } catch (cause) { + // A rejected target still owns the client, transport and sampler. + clearInterval(timer); + await client.close().catch(() => undefined); + await transport.close().catch(() => undefined); + throw cause; + } + return { call, close }; +}; + +/** Count structuring and diagnostic markers in provider pseudocode. */ +const quality = (text) => ({ + bytes: text === null ? 0 : Buffer.byteLength(text), + gotos: text === null ? 0 : (text.match(/\bgoto\b/gu) ?? []).length, + warnings: + text === null ? 0 : (text.match(/WARNING|\/\/ warn:/gu) ?? []).length, +}); + +/** Verify Kuna on one binary and measure its cold and warm answers. */ +const verifyKuna = async (path, functions) => { + const sha256 = createHash("sha256") + .update(await readFile(path)) + .digest("hex"); + const cold = await openServer({ path, id: "kuna" }); + const answers = []; + try { + const listed = await cold.call("list_procedures"); + for (const [name, address] of functions) { + assert.ok( + listed.value.some((entry) => entry.address === address), + `${name} at ${address} is missing from Kuna's procedure list for ${path}`, + ); + const pseudocode = await cold.call("procedure_pseudo_code", { + procedure: address, + }); + assert.equal(pseudocode.structured.subject.digest.sha256, sha256); + assert.equal(pseudocode.structured.provider.id, "kuna"); + const source = pseudocode.structured.raw_result?.source?.kind; + // A fast export can finish before the first request reaches the + // decompile path; both sources are valid answers. The single-function + // fallback itself is covered by the provider's boundary tests. + assert.ok( + source === "single-function" || source === "project-export", + `Unexpected pseudocode source ${String(source)}`, + ); + const assembly = await cold.call("procedure_assembly", { + procedure: address, + }); + assert.equal( + assembly.structured.raw_result.source.kind, + "project-export", + ); + assert.match(assembly.value, new RegExp(`: ; ${address}\\n`, "u")); + const exported = await cold.call("procedure_pseudo_code", { + procedure: address, + }); + const recorded = exported.structured.raw_result.source; + assert.equal(recorded.kind, "project-export"); + const block = (await readFile(join(recorded.directory, recorded.file))) + .subarray( + recorded.byte_offset, + recorded.byte_offset + recorded.byte_length, + ) + .toString("utf8"); + assert.ok( + exported.value === null || block.includes(exported.value), + "Served pseudocode differs from the recorded export span", + ); + const dossier = await cold.call("analyze_function", { + procedure: address, + }); + const combined = dossier.structured.raw_result.source; + assert.equal(combined.kind, "project-export"); + assert.deepEqual( + combined.files.map(({ facet }) => facet), + ["pseudocode", "assembly"], + ); + for (const file of combined.files) { + const bytes = await readFile(join(combined.directory, file.file)); + assert.equal( + createHash("sha256").update(bytes).digest("hex"), + file.file_sha256, + ); + const span = bytes + .subarray(file.byte_offset, file.byte_offset + file.byte_length) + .toString("utf8"); + const facet = dossier.value[file.facet]; + assert.ok( + span.includes(Array.isArray(facet) ? facet.join("\n") : facet), + ); + } + answers.push({ + name, + address, + cold_source: source, + cold_ms: pseudocode.ms, + export_wait_ms: assembly.ms, + warm_same_session_ms: exported.ms, + response_bytes: exported.bytes, + dossier_response_bytes: dossier.bytes, + ...quality(exported.value), + }); + } + } finally { + answers.push({ cold_server: await cold.close() }); + } + const staging = await readdir(join(cacheRoot, "staging")); + assert.deepEqual(staging, [], "Kuna staging survived a completed export"); + // A fresh server reuses the export immediately. + const warm = await openServer({ path, id: "kuna" }); + try { + for (const answer of answers) + if (answer.address !== undefined) { + const reread = await warm.call("procedure_pseudo_code", { + procedure: answer.address, + }); + assert.equal( + reread.structured.raw_result.source.kind, + "project-export", + ); + answer.warm_fresh_server_ms = reread.ms; + } + } finally { + answers.push({ warm_server: await warm.close() }); + } + return answers; +}; + +/** Measure Ghidra on the same functions through the same public workflow. */ +const measureGhidra = async (path, functions) => { + const server = await openServer({ path, id: "ghidra" }); + const answers = []; + try { + for (const [name, address] of functions) { + const cold = await server.call("procedure_pseudo_code", { + procedure: address, + }); + const warm = await server.call("procedure_pseudo_code", { + procedure: address, + }); + answers.push({ + name, + address, + first_call_ms: cold.ms, + repeat_ms: warm.ms, + response_bytes: cold.bytes, + ...quality(cold.value), + }); + } + } finally { + answers.push({ server: await server.close() }); + } + return answers; +}; + +/** Symbol-bearing caller binaries with the requested functions, beside their stripped copies. */ +const targetPairs = async (root, names) => { + const pairs = []; + for (const flavor of (await readdir(root)).sort()) { + const directory = join(root, flavor); + if (!(await stat(directory)).isDirectory()) continue; + for (const file of (await readdir(directory)).sort()) { + if (file.endsWith(".stripped")) continue; + const symbols = await symbolAddresses(join(directory, file), names); + if (symbols.size > 0) pairs.push({ flavor, directory, file, symbols }); + } + } + return pairs; +}; + +/** Verify Kuna, and optionally measure Ghidra, on one binary of a pair. */ +const verifyVariant = async ({ flavor, directory, symbols }, variant) => { + const path = join(directory, variant); + const entry = { binary: `${flavor}/${variant}` }; + entry.kuna = await verifyKuna(path, [...symbols]); + if (values["compare-ghidra"]) + entry.ghidra = await measureGhidra(path, [...symbols]); + process.stderr.write(`verified ${entry.binary}\n`); + return entry; +}; + +const results = []; +try { + let callerPairs = []; + if (values.targets !== undefined) { + const names = (values.functions ?? "").split(",").filter(Boolean); + assert.ok(names.length > 0, "--targets requires --functions"); + callerPairs = await targetPairs(values.targets, names); + assert.ok( + callerPairs.length > 0, + `No caller-built targets contain the requested functions in ${values.targets}`, + ); + const found = new Set( + callerPairs.flatMap(({ symbols }) => [...symbols.keys()]), + ); + const missing = names.filter((name) => !found.has(name)); + assert.deepEqual( + missing, + [], + "Requested functions were not found in any caller-built target", + ); + } + + // Source-owned fixture: always built and verified. + const fixtureDir = join(workspace, "fixture"); + await mkdir(fixtureDir); + const source = join(fixtureDir, "fixture.c"); + await writeFile(source, FIXTURE_SOURCE); + const built = join(fixtureDir, "rea-kuna-fixture"); + await exec(process.env.REA_CC ?? "cc", ["-O2", "-o", built, source]); + const stripped = `${built}.stripped`; + await writeFile(stripped, await readFile(built)); + await exec(process.env.REA_STRIP ?? "strip", [stripped]); + const fixtureFunctions = await symbolAddresses(built, [ + "rea_checksum", + "rea_classify", + ]); + assert.equal(fixtureFunctions.size, 2, "Fixture symbols were not found"); + for (const binary of [built, stripped]) + results.push({ + binary: binary.slice(workspace.length + 1), + kuna: await verifyKuna(binary, [...fixtureFunctions]), + }); + + // Caller-built matrix. + for (const pair of callerPairs) + for (const variant of [pair.file, `${pair.file}.stripped`]) + results.push(await verifyVariant(pair, variant)); +} finally { + await rm(workspace, { recursive: true, force: true }); +} +const lifecycle = await completeVerifierRun(run); +assertVerifierCleanupComplete(lifecycle.process_lineage); +const report = { + lane: "kuna", + kuna_version: kunaVersion, + platform: `${process.platform}/${process.arch}`, + verifier_run: lifecycle, + results, + limitations: [ + "Targets are analyzed statically; they are never executed.", + "Memory is the sampled peak resident set of the MCP server process tree; CPU is its sampled cumulative time. A short-lived peak between 200 ms samples is not observed.", + "Quality signals count gotos and warning markers only; they are not a correctness judgement and not a DecBench reproduction.", + "Response bytes are UTF-8 serialization sizes. Model token usage, tool schema costs and agent task completion were not measured.", + ], + cleanup: "complete", +}; +if (values.report !== undefined) + await writeFile(values.report, `${JSON.stringify(report, null, 2)}\n`); +process.stdout.write(`${JSON.stringify(report)}\n`); diff --git a/src/composition/binary.ts b/src/composition/binary.ts index 0cb438cb36..2986d93439 100644 --- a/src/composition/binary.ts +++ b/src/composition/binary.ts @@ -4,6 +4,7 @@ import type { BinarySession } from "../application/binary/BinarySession.js"; import { HopperProvider } from "../hopper/HopperProvider.js"; import { GhidraProvider } from "../ghidra/GhidraProvider.js"; import { IdaProvider } from "../ida/IdaProvider.js"; +import { KunaProvider } from "../kuna/KunaProvider.js"; import type { Logger } from "pino"; import { auxiliaryAnalysisProviderDeclarations } from "./auxiliaryAnalysisProviders.js"; import { AnalysisProviderRegistry } from "../application/binary/AnalysisProviderRegistry.js"; @@ -25,9 +26,10 @@ export const createBinarySession = ( const hopper = new HopperProvider(config, logger, environment); const ghidra = new GhidraProvider(config, logger, environment); const ida = new IdaProvider(config, environment); + const kuna = new KunaProvider(environment); return composeBinarySession( new AnalysisProviderRegistry( - [hopper, ghidra, ida], + [hopper, ghidra, ida, kuna], config.analysisProvider, ), auxiliaryAnalysisProviderDeclarations(environment).map( diff --git a/src/domain/analysisSearch.ts b/src/domain/analysisSearch.ts new file mode 100644 index 0000000000..4dd2d9193a --- /dev/null +++ b/src/domain/analysisSearch.ts @@ -0,0 +1,33 @@ +import { z } from "zod"; + +import { analysisSearchInput } from "../contracts/analysisSearchContract.js"; +import { AnalysisInputError } from "./analysisErrorCore.js"; + +/** Parsed `search_procedures`/`search_strings` query shared by providers that match in REA. */ +export const analysisSearchQuerySchema = z.object(analysisSearchInput); + +export type AnalysisSearchQuery = z.infer; + +/** + * Match names the way the search contract describes: a literal substring by + * default, or a Unicode regular expression, case-insensitive unless requested. + * An invalid pattern is the caller's input error. + */ +export const analysisSearchPredicate = ( + query: AnalysisSearchQuery, +): ((value: string) => boolean) => { + if (query.mode === "regex") { + let regex: RegExp; + try { + regex = new RegExp(query.pattern, query.case_sensitive ? "u" : "iu"); + } catch (cause: unknown) { + throw new AnalysisInputError("search", { cause }); + } + return (value) => regex.test(value); + } + const needle = query.case_sensitive + ? query.pattern + : query.pattern.toLowerCase(); + return (value) => + (query.case_sensitive ? value : value.toLowerCase()).includes(needle); +}; diff --git a/src/hopper/HopperProviderCapabilities.ts b/src/hopper/HopperProviderCapabilities.ts index 2dbac46590..ac514adfd3 100644 --- a/src/hopper/HopperProviderCapabilities.ts +++ b/src/hopper/HopperProviderCapabilities.ts @@ -1,3 +1,4 @@ +import { REGEX_MATCHING_LIMITATION } from "../process/RegexMatcher.js"; import type { CapabilityDescriptor, ProviderIdentity, @@ -90,7 +91,7 @@ export const CAPABILITIES: readonly CapabilityDescriptor[] = Object.freeze( : []), ...(operation === "search_strings" || operation === "search_procedures" ? [ - "Regex mode uses ECMAScript Unicode semantics in a supervised worker with a five-second matching deadline. Cancellation or deadline failure stops matching without blocking Hopper; literal mode retains native casefold semantics.", + `${REGEX_MATCHING_LIMITATION} Literal mode retains native casefold semantics.`, ] : []), ...(operation === "list_strings" || diff --git a/src/hopper/HopperRegexSearch.ts b/src/hopper/HopperRegexSearch.ts index c1a0db1093..39ddd0e24a 100644 --- a/src/hopper/HopperRegexSearch.ts +++ b/src/hopper/HopperRegexSearch.ts @@ -1,4 +1,3 @@ -import { Worker } from "node:worker_threads"; import { z } from "zod"; import type { ExecutionOptions } from "../application/AnalysisProvider.js"; @@ -6,7 +5,6 @@ import type { AnalysisError } from "../domain/analysisErrorBase.js"; import { AnalysisCancelledError, AnalysisOutputError, - AnalysisResourceConstraintError, } from "../domain/analysisErrorCore.js"; import { analysisStringSchema } from "../domain/hopperValues.js"; import { HopperRemoteError } from "../domain/hopperErrors.js"; @@ -14,20 +12,12 @@ import type { JsonValue } from "../domain/jsonValue.js"; import { err, ok, type Result } from "../domain/result.js"; import { analysisSearchInput } from "../contracts/analysisSearchContract.js"; import type { HopperClient } from "./HopperClient.js"; +import { matchRegexValues } from "../process/RegexMatcher.js"; -const MATCHING_DEADLINE_MS = 5_000; const inputSchema = z.strictObject({ ...analysisSearchInput, mode: z.literal("regex"), }); -const replySchema = z.discriminatedUnion("status", [ - z.strictObject({ - status: z.literal("matched"), - indices: z.array(z.number().int().nonnegative()), - }), - z.strictObject({ status: z.literal("invalid_pattern"), reason: z.string() }), -]); - /** Match collected native facts off Hopper's API thread, with cancellable lifetime. */ export class HopperRegexSearch { readonly #active = new Map< @@ -35,7 +25,7 @@ export class HopperRegexSearch { Promise> >(); - constructor(private readonly client: HopperClient) {} + constructor(private readonly client: Pick) {} execute( operation: "search_strings" | "search_procedures", @@ -125,117 +115,20 @@ export class HopperRegexSearch { .catch(() => undefined); if (options.signal?.aborted) return err(new AnalysisCancelledError(operation)); - let worker: Worker; - try { - worker = new Worker(new URL("./HopperRegexWorker.js", import.meta.url), { - workerData: { - pattern: input.data.pattern, - caseSensitive: input.data.case_sensitive, - values: parsed.data.map((item) => item.value), - }, - }); - } catch (cause: unknown) { - return err( - new AnalysisOutputError( + const matched = await matchRegexValues({ + operation, + pattern: input.data.pattern, + caseSensitive: input.data.case_sensitive, + values: parsed.data.map((item) => item.value), + signal: options.signal, + invalidPattern: (reason) => + new HopperRemoteError(-32000, `Invalid regex pattern: ${reason}`, { + diagnosticType: "invalid_request", operation, - `Cannot start regex worker: ${cause instanceof Error ? cause.message : String(cause)}`, - { cause }, - ), - ); - } - try { - return await new Promise>((resolve) => { - const finish = (result: Result): void => { - clearTimeout(timer); - options.signal?.removeEventListener("abort", onAbort); - resolve(result); - }; - const onAbort = (): void => - finish(err(new AnalysisCancelledError(operation))); - const timer = setTimeout( - () => - finish( - err( - new AnalysisResourceConstraintError( - operation, - "cpu", - "Regex matching exceeded its supervised deadline; the Hopper API thread remains available.", - { matching_deadline_ms: MATCHING_DEADLINE_MS }, - { - remediationAction: - "Simplify the ECMAScript Unicode regex or use literal mode, then retry in the same session.", - }, - ), - ), - ), - MATCHING_DEADLINE_MS, - ); - options.signal?.addEventListener("abort", onAbort, { once: true }); - if (options.signal?.aborted) onAbort(); - worker.once("message", (message: unknown) => { - const reply = replySchema.safeParse(message); - if (!reply.success) - return finish( - err( - new AnalysisOutputError( - operation, - "Regex worker reply is malformed", - ), - ), - ); - if (reply.data.status === "invalid_pattern") - return finish( - err( - new HopperRemoteError( - -32000, - `Invalid regex pattern: ${reply.data.reason}`, - { diagnosticType: "invalid_request", operation }, - ), - ), - ); - const indices = reply.data.indices; - if ( - indices.some( - (index, position) => - index >= parsed.data.length || - (position > 0 && index <= (indices[position - 1] ?? -1)), - ) - ) - return finish( - err( - new AnalysisOutputError( - operation, - "Regex worker returned invalid inventory indices", - ), - ), - ); - const selected = new Set(indices); - finish(ok(parsed.data.filter((_, index) => selected.has(index)))); - }); - worker.once("error", (cause: Error) => - finish( - err( - new AnalysisOutputError( - operation, - `Regex worker failed: ${cause.message}`, - { cause }, - ), - ), - ), - ); - worker.once("exit", (code) => - finish( - err( - new AnalysisOutputError( - operation, - `Regex worker exited without a result (${code})`, - ), - ), - ), - ); - }); - } finally { - await worker.terminate(); - } + }), + }); + if (!matched.ok) return matched; + const selected = new Set(matched.value); + return ok(parsed.data.filter((_, index) => selected.has(index))); } } diff --git a/src/hopper/HopperRegexWorker.ts b/src/hopper/HopperRegexWorker.ts deleted file mode 100644 index 71fd79b509..0000000000 --- a/src/hopper/HopperRegexWorker.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { parentPort, workerData } from "node:worker_threads"; -import { z } from "zod"; - -const input = z - .strictObject({ - pattern: z.string().min(1), - caseSensitive: z.boolean(), - values: z.array(z.string()), - }) - .parse(workerData); - -let expression: RegExp | undefined; -try { - expression = new RegExp(input.pattern, input.caseSensitive ? "u" : "iu"); -} catch (cause: unknown) { - parentPort?.postMessage({ - status: "invalid_pattern", - reason: cause instanceof Error ? cause.message : String(cause), - }); -} -if (expression !== undefined) { - const indices = []; - for (const [index, value] of input.values.entries()) { - if (expression.test(value)) indices.push(index); - } - parentPort?.postMessage({ status: "matched", indices }); -} diff --git a/src/ida/IdaOperationRunner.ts b/src/ida/IdaOperationRunner.ts index 252bdbe3a1..f7cf5ae2b4 100644 --- a/src/ida/IdaOperationRunner.ts +++ b/src/ida/IdaOperationRunner.ts @@ -7,7 +7,10 @@ import { nativeLiteralSearchPattern, nativeStringsSchema, } from "./IdaNativeProtocol.js"; -import { analysisSearchInput } from "../contracts/analysisSearchContract.js"; +import { + analysisSearchPredicate, + analysisSearchQuerySchema, +} from "../domain/analysisSearch.js"; import { AnalysisInputError, AnalysisCancelledError, @@ -82,7 +85,6 @@ const MISSING_PROCEDURE_ANSWERS: ReadonlySet = new Set([ ]); const lines = (asm: IdaDisassembly) => asm.lines.map(({ address, instruction }) => `${address}: ${instruction}`); -const searchQuerySchema = z.object(analysisSearchInput); /** Translate explicit analyst questions into a closed set of read-only upstream requests. */ export class IdaOperationRunner { @@ -581,10 +583,10 @@ export class IdaOperationRunner { ): Promise { const query = operation === "search_procedures" || operation === "search_strings" - ? searchQuerySchema.parse(parameters) + ? analysisSearchQuerySchema.parse(parameters) : undefined; const matches = - query === undefined ? undefined : this.searchPredicate(query); + query === undefined ? undefined : analysisSearchPredicate(query); if (operation === "list_procedures" || operation === "search_procedures") { const items = (await this.functions()).map((fn) => ({ address: fn.address, @@ -660,30 +662,6 @@ export class IdaOperationRunner { } } - private searchPredicate( - query: z.infer, - ): (value: string) => boolean { - let matches: (value: string) => boolean; - if (query.mode === "regex") { - try { - const regex = new RegExp( - query.pattern, - query.case_sensitive ? "u" : "iu", - ); - matches = (value) => regex.test(value); - } catch (cause: unknown) { - throw new AnalysisInputError("search", { cause }); - } - } else { - const needle = query.case_sensitive - ? query.pattern - : query.pattern.toLowerCase(); - matches = (value) => - (query.case_sensitive ? value : value.toLowerCase()).includes(needle); - } - return matches; - } - private async dossier(fn: IdaFunction): Promise { const asm = await this.assembly(fn); const pseudocode = await this.pseudocode(fn); diff --git a/src/kuna/KunaCommand.ts b/src/kuna/KunaCommand.ts new file mode 100644 index 0000000000..c8e4b7e76b --- /dev/null +++ b/src/kuna/KunaCommand.ts @@ -0,0 +1,267 @@ +import { createHash, randomUUID } from "node:crypto"; +import { createReadStream, createWriteStream } from "node:fs"; +import { stat } from "node:fs/promises"; +import { Transform } from "node:stream"; +import { pipeline } from "node:stream/promises"; + +import type { AnalysisError } from "../domain/analysisErrorBase.js"; +import { + AnalysisCancelledError, + AnalysisResourceConstraintError, + AnalysisTimeoutError, +} from "../domain/analysisErrorCore.js"; +import { ProviderAdapterError } from "../domain/providerAdapterError.js"; +import { ProviderCleanupError } from "../domain/providerCleanupError.js"; +import { + OwnedCommandFailure, + runOwnedCommand, +} from "../process/OwnedCommand.js"; +import { hashStableFile } from "../filesystem/StableFileHash.js"; +import type { + OwnedProviderProcessSpawnOptions, + ProviderProcessSnapshot, + ProviderProcessSupervisor, + SpawnedOwnedProviderProcess, +} from "../process/ProviderProcess.js"; +import { + kunaChildEnvironment, + type KunaInstallation, +} from "./KunaInstallation.js"; +import { KUNA_PROVIDER_ID } from "./KunaRelease.js"; + +/** + * `kuna functions --json` costs ~210 bytes per function; this admits about + * 600,000 functions, above the 392,814 of the largest image Kuna documents. + */ +export const KUNA_INVENTORY_OUTPUT_BYTES = 128 * 1024 * 1024; + +/** Test seam for the owned process launcher; production uses the default. */ +export type KunaProcessLauncher = ( + options: OwnedProviderProcessSpawnOptions, +) => Promise; + +/** One bounded Kuna run inside a private workspace. */ +export interface KunaCommandRequest { + readonly installation: KunaInstallation; + readonly arguments: readonly string[]; + /** Private directory used as the working and temporary directory. */ + readonly workspace: string; + /** Operation named by every failure this run reports. */ + readonly operation: string; + readonly timeoutMs: number; + /** The deadline callers configured, reported when the run times out. */ + readonly reportedTimeoutMs: number; + readonly outputBytes: number; + readonly signal: AbortSignal; + readonly launcher?: KunaProcessLauncher | undefined; +} + +/** + * How a Kuna run ended. `retained` means the process tree could not be proven + * stopped, so it may still write into the workspace, which therefore stays + * with the caller's cleanup owner. + */ +export type KunaCommandOutcome = + | { readonly kind: "ok"; readonly snapshot: ProviderProcessSnapshot } + | { readonly kind: "failed"; readonly error: AnalysisError } + | { + readonly kind: "retained"; + readonly error: ProviderCleanupError; + /** The supervisor that can retry stopping the process tree, when one exists. */ + readonly owner: ProviderProcessSupervisor | undefined; + }; + +const message = (cause: unknown): string => + cause instanceof Error ? cause.message : String(cause); + +/** File metadata that, unchanged, lets a verified digest stand without re-hashing. */ +const verifiedExecutables = new Map< + string, + { readonly identity: string; readonly sha256: string } +>(); + +/** + * Confirm the executable still has the digest its analysis profile committed, + * so no answer names one Kuna build while another produced it. + */ +const verifyExecutable = async ( + installation: KunaInstallation, + signal: AbortSignal, +): Promise => { + const status = await stat(installation.executable, { bigint: true }); + const identity = [ + status.dev, + status.ino, + status.size, + status.mtimeNs, + status.ctimeNs, + ].join(":"); + const known = verifiedExecutables.get(installation.executable); + let sha256 = known?.identity === identity ? known.sha256 : undefined; + if (sha256 === undefined) { + sha256 = (await hashStableFile(installation.executable, signal)).sha256; + verifiedExecutables.set(installation.executable, { identity, sha256 }); + } + return sha256 === installation.sha256 ? undefined : sha256; +}; + +/** Run Kuna under an owned process group with REA's environment and limits. */ +export const runKunaCommand = async ( + request: KunaCommandRequest, +): Promise => { + const { installation } = request; + try { + const changed = await verifyExecutable(installation, request.signal); + if (changed !== undefined) + return { + kind: "failed", + error: new ProviderAdapterError(KUNA_PROVIDER_ID, request.operation, { + userMessage: `${installation.executable} changed after this target was opened; reopen the target so REA probes the current Kuna executable.`, + diagnostics: { + expected_sha256: installation.sha256, + observed_sha256: changed, + }, + }), + }; + } catch (cause: unknown) { + return { + kind: "failed", + error: request.signal.aborted + ? new AnalysisCancelledError(request.operation, { cause }) + : new ProviderAdapterError(KUNA_PROVIDER_ID, request.operation, { + cause, + userMessage: `Cannot verify ${installation.executable}: ${message(cause)}`, + }), + }; + } + try { + const snapshot = await runOwnedCommand( + { + command: installation.executable, + arguments: request.arguments, + runId: `rea-kuna-${randomUUID()}`, + // A script's process reports its interpreter, not the configured path. + ...(installation.script ? { expectedCommand: null } : {}), + cwd: request.workspace, + hostEnvironment: kunaChildEnvironment(installation.configuration), + env: { TMPDIR: request.workspace }, + }, + { + timeoutMs: Math.max(1, request.timeoutMs), + diagnosticBytes: request.outputBytes, + }, + { + signal: request.signal, + ...(request.launcher === undefined + ? {} + : { launcher: request.launcher }), + }, + ); + return { kind: "ok", snapshot }; + } catch (cause: unknown) { + return commandFailure(cause, request); + } +}; + +/** Copy a file and return the SHA-256 of exactly the bytes copied. */ +export const copyAndHash = async ( + source: string, + destination: string, + signal: AbortSignal, +): Promise => { + const hash = createHash("sha256"); + await pipeline( + createReadStream(source), + new Transform({ + transform(chunk: Buffer, _encoding, callback) { + hash.update(chunk); + callback(null, chunk); + }, + }), + createWriteStream(destination, { flags: "wx", mode: 0o600 }), + { signal }, + ); + return hash.digest("hex"); +}; + +/** Map an owned-command failure to its typed error, keeping unproven cleanup owned. */ +const commandFailure = ( + cause: unknown, + request: KunaCommandRequest, +): KunaCommandOutcome => { + const { operation } = request; + const subcommand = request.arguments[0] ?? ""; + if (!(cause instanceof OwnedCommandFailure)) + return { + kind: "failed", + error: request.signal.aborted + ? new AnalysisCancelledError(operation, { cause }) + : new ProviderAdapterError(KUNA_PROVIDER_ID, operation, { + cause, + userMessage: `Cannot run ${request.installation.executable}: ${message(cause)}`, + }), + }; + const snapshot = cause.snapshot; + const capturedOutput = + snapshot === null + ? undefined + : { + // Output can be evidence-sized, not diagnostic-sized; keep only its tail. + stdout: snapshot.stdout.text.slice(-4096), + stderr: snapshot.stderr.text, + truncated: + snapshot.diagnosticTruncated === true || + snapshot.outputIncomplete === true || + snapshot.stdout.text.length > 4096, + exit_code: snapshot.exitCode ?? null, + signal: snapshot.signal ?? null, + }; + const details = { + cause, + ...(capturedOutput === undefined ? {} : { capturedOutput }), + }; + if (cause.cleanupFailure !== null) + return { + kind: "retained", + error: new ProviderCleanupError( + KUNA_PROVIDER_ID, + [...cause.resources, request.workspace], + { reason: cause.cleanupFailure }, + { operation, cause }, + ), + owner: cause.cleanupOwner, + }; + if (cause.reason === "cancelled") + return { + kind: "failed", + error: new AnalysisCancelledError(operation, details), + }; + if (cause.reason === "timeout") + return { + kind: "failed", + error: new AnalysisTimeoutError( + operation, + request.reportedTimeoutMs, + details, + ), + }; + if (cause.reason === "output-limit") + return { + kind: "failed", + error: new AnalysisResourceConstraintError( + operation, + "capacity", + `kuna ${subcommand} wrote more than ${String(request.outputBytes)} bytes of output.`, + { output_limit_bytes: request.outputBytes }, + details, + ), + }; + const exitCode = capturedOutput?.exit_code; + return { + kind: "failed", + error: new ProviderAdapterError(KUNA_PROVIDER_ID, operation, { + ...details, + userMessage: `kuna ${subcommand} failed${exitCode === undefined || exitCode === null ? "" : ` with exit code ${String(exitCode)}`}: ${capturedOutput?.stderr.trim().split("\n").at(-1) ?? cause.message}`, + }), + }; +}; diff --git a/src/kuna/KunaConfiguration.ts b/src/kuna/KunaConfiguration.ts index 48fe418645..7de37b635d 100644 --- a/src/kuna/KunaConfiguration.ts +++ b/src/kuna/KunaConfiguration.ts @@ -18,6 +18,9 @@ const MAX_KUNA_JOBS = 32; */ const DEFAULT_PROJECT_TIMEOUT_MS = 30 * 60 * 1000; +/** One function decompiles in well under a second; sqlite3VdbeExec (42 KB) took 15-25 s. */ +const DEFAULT_FUNCTION_TIMEOUT_MS = 120_000; + /** * A serial (`--jobs 1`) export is Kuna's reference answer: it decompiles * callees before callers, which recovers prototypes that parallel workers @@ -39,6 +42,8 @@ export interface KunaConfiguration { readonly cacheRoot: string; readonly cacheMaxBytes: number; readonly projectTimeoutMs: number; + /** Deadline for one single-function `kuna decompile` run. */ + readonly functionTimeoutMs: number; readonly jobs: number; } @@ -125,6 +130,18 @@ export const readKunaConfiguration = ( "REA_KUNA_PROJECT_TIMEOUT_MS", ); + const rawFunctionTimeout = environment.REA_KUNA_TIMEOUT_MS; + const functionTimeout = + rawFunctionTimeout === undefined + ? undefined + : providerStartupTimeoutSchema.safeParse(rawFunctionTimeout); + if (functionTimeout !== undefined && !functionTimeout.success) + return failure( + "open_options_invalid", + `REA_KUNA_TIMEOUT_MS must be a decimal millisecond count from 1 to 2147483647; received ${String(rawFunctionTimeout)}.`, + "REA_KUNA_TIMEOUT_MS", + ); + const rawJobs = environment.REA_KUNA_JOBS; const jobs = rawJobs === undefined ? DEFAULT_JOBS : positiveInteger(rawJobs); if (jobs === undefined || jobs > MAX_KUNA_JOBS) @@ -139,6 +156,7 @@ export const readKunaConfiguration = ( cacheRoot: cacheRoot ?? defaultCacheRoot(environment, platform), cacheMaxBytes, projectTimeoutMs: timeout?.data ?? DEFAULT_PROJECT_TIMEOUT_MS, + functionTimeoutMs: functionTimeout?.data ?? DEFAULT_FUNCTION_TIMEOUT_MS, jobs, }); }; diff --git a/src/kuna/KunaDoctor.ts b/src/kuna/KunaDoctor.ts index 4c3488dfb2..e8166b2163 100644 --- a/src/kuna/KunaDoctor.ts +++ b/src/kuna/KunaDoctor.ts @@ -25,6 +25,7 @@ const REGISTRATION_SETTINGS = [ "REA_KUNA_CACHE_DIR", "REA_KUNA_CACHE_MAX_BYTES", "REA_KUNA_PROJECT_TIMEOUT_MS", + "REA_KUNA_TIMEOUT_MS", "REA_KUNA_JOBS", ] as const; diff --git a/src/kuna/KunaLimits.ts b/src/kuna/KunaLimits.ts new file mode 100644 index 0000000000..ae2ceb48cc --- /dev/null +++ b/src/kuna/KunaLimits.ts @@ -0,0 +1,2 @@ +/** Shared buffered-text capacity for a subprocess answer or one cached source facet. */ +export const KUNA_FUNCTION_TEXT_BYTES = 16 * 1024 * 1024; diff --git a/src/kuna/KunaProjectCache.ts b/src/kuna/KunaProjectCache.ts index 9bdcb406f4..29a0f1e959 100644 --- a/src/kuna/KunaProjectCache.ts +++ b/src/kuna/KunaProjectCache.ts @@ -31,8 +31,9 @@ import { readKunaExport, type KunaExport, type KunaExportStatus, - type KunaProcessLauncher, } from "./KunaProjectExport.js"; +import type { ProviderProcessSupervisor } from "../process/ProviderProcess.js"; +import type { KunaProcessLauncher } from "./KunaCommand.js"; import type { KunaInventoryFunction, KunaProjectFunction, @@ -129,6 +130,15 @@ export interface KunaProjectRequest { readonly onStatus?: (status: KunaExportStatus) => void; } +/** One build whose process tree REA could not prove stopped. */ +interface RetainedProcess { + readonly key: string; + readonly error: ProviderCleanupError; + /** Retries stopping the process tree, when the launcher left one. */ + readonly owner: ProviderProcessSupervisor | undefined; + readonly staging: string; +} + interface StagedResult { readonly result: Result; /** Set when an owned process may still write into staging. */ @@ -205,9 +215,13 @@ export class KunaProjectCache { #capacity: "within_limit" | "pinned" | "building" = "within_limit"; #maintenance: Promise | undefined; #maintenanceRequested = false; - readonly #retainedProcesses: ProviderCleanupError[] = []; + /** Builds whose process tree could not be proven stopped; their staging stays. */ + readonly #retainedProcesses: RetainedProcess[] = []; /** Only resources whose process cleanup is proven may be retried as files. */ - readonly #retainedFiles = new Map(); + readonly #retainedFiles = new Map< + string, + { readonly key: string | null; readonly error: ProviderCleanupError } + >(); constructor( root: string, @@ -341,33 +355,81 @@ export class KunaProjectCache { return this.#await(created, request); } + /** + * Wait until a build that every waiter abandoned has stopped its Kuna + * process and removed its staging. A build another request still waits for + * keeps running and is not awaited. + */ + async settled(key: string): Promise> { + await Promise.allSettled( + [...this.#unfinished] + .filter((entry) => entry.key === key && entry.controller.signal.aborted) + .map((entry) => entry.promise), + ); + await this.#retryProcesses((retained) => retained.key === key); + await this.#retryFiles(key); + return cleanupResult( + [...this.#retainedProcesses, ...this.#retainedFiles.values()] + .filter((retained) => retained.key === key) + .map(({ error }) => error), + ); + } + /** Wait for builds, retry safe file removal, and report unresolved cleanup. */ async close(): Promise> { await Promise.allSettled( [...this.#unfinished].map((entry) => entry.promise), ); while (this.#maintenance !== undefined) await this.#maintenance; - for (const path of [...this.#retainedFiles.keys()]) - await this.#removeRetired(path).catch(() => undefined); - const retained = [ - ...this.#retainedProcesses, - ...this.#retainedFiles.values(), - ]; - if (retained.length === 0) return ok(null); - return err( - new ProviderCleanupError( - KUNA_PROVIDER_ID, - retained.flatMap((entry) => entry.cleanupResources), - { - reason: retained - .map((entry) => - String(entry.diagnostics?.["reason"] ?? entry.message), - ) - .join("; "), - }, - { operation: KUNA_EXPORT_OPERATION }, - ), - ); + await this.#retryProcesses(() => true); + await this.#retryFiles(); + return cleanupResult([ + ...this.#retainedProcesses.map(({ error }) => error), + ...[...this.#retainedFiles.values()].map(({ error }) => error), + ]); + } + + /** Retry stopping retained process trees; retire the staging of each one proven stopped. */ + async #retryProcesses( + selected: (retained: RetainedProcess) => boolean, + ): Promise { + for (const retained of this.#retainedProcesses.filter(selected)) { + const stopped = await retained.owner?.stop(); + if (stopped === undefined || stopped.status === "incomplete") continue; + this.#retainedProcesses.splice( + this.#retainedProcesses.indexOf(retained), + 1, + ); + await this.#retireStaging(retained.staging, retained.key); + } + } + + /** Retry only stopped file resources belonging to the closing task, or all. */ + async #retryFiles(key?: string): Promise { + for (const [path, retained] of [...this.#retainedFiles]) + if (key === undefined || retained.key === key) + await this.#removeRetired(path).catch(() => undefined); + } + + /** + * Move proven-stopped staging into trash and remove it. A failed removal + * stays owned as a file cleanup that close retries. + */ + async #retireStaging( + staging: string, + key: string, + ): Promise { + let cleanupPath = staging; + try { + const retired = join(this.#root, "trash", `build-${randomUUID()}`); + await rename(staging, retired); + cleanupPath = retired; + await this.#removeRetired(retired, key); + return undefined; + } catch (cause: unknown) { + if (cause instanceof ProviderCleanupError) return cause; + return this.#retainFileCleanup(cleanupPath, cause, key); + } } #await( @@ -437,18 +499,10 @@ export class KunaProjectCache { retainStaging: false, })); if (built.retainStaging) return built.result; - let cleanupPath = staging; - try { - // Only a proven-stopped build may enter trash. A failed write, crash or - // lost cleanup identity therefore never makes live staging reclaimable. - const retired = join(this.#root, "trash", `build-${randomUUID()}`); - await rename(staging, retired); - cleanupPath = retired; - await this.#removeRetired(retired); - } catch (cause: unknown) { - if (cause instanceof ProviderCleanupError) return err(cause); - return err(this.#retainFileCleanup(cleanupPath, cause)); - } + // Only a proven-stopped build may enter trash. A failed write, crash or + // lost cleanup identity therefore never makes live staging reclaimable. + const retired = await this.#retireStaging(staging, key); + if (retired !== undefined) return err(retired); if (built.result.ok) // The entry is published either way; a failed eviction retries on the next publish. await this.#evict(key).catch(() => undefined); @@ -473,7 +527,12 @@ export class KunaProjectCache { launcher: this.#launcher, }); if (outcome.kind === "retained") { - this.#retainedProcesses.push(outcome.error); + this.#retainedProcesses.push({ + key, + error: outcome.error, + owner: outcome.owner, + staging, + }); return { result: err(outcome.error), retainStaging: true }; } return { @@ -643,16 +702,16 @@ export class KunaProjectCache { if (isMissing(cause)) return; throw cause; } - await this.#removeRetired(trash); + await this.#removeRetired(trash, key); } /** Keep each failed removal owned until that path is successfully removed. */ - async #removeRetired(path: string): Promise { + async #removeRetired(path: string, key?: string): Promise { try { await rm(path, { recursive: true, force: true }); this.#retainedFiles.delete(path); } catch (cause: unknown) { - throw this.#retainFileCleanup(path, cause); + throw this.#retainFileCleanup(path, cause, key); } } @@ -677,14 +736,42 @@ export class KunaProjectCache { return readFile(path, "utf8"); } - #retainFileCleanup(path: string, cause: unknown): ProviderCleanupError { + #retainFileCleanup( + path: string, + cause: unknown, + key?: string, + ): ProviderCleanupError { const error = new ProviderCleanupError( KUNA_PROVIDER_ID, [path], { reason: cause instanceof Error ? cause.message : String(cause) }, { operation: KUNA_EXPORT_OPERATION, cause }, ); - this.#retainedFiles.set(path, error); + this.#retainedFiles.set(path, { + key: key ?? this.#retainedFiles.get(path)?.key ?? null, + error, + }); return error; } } + +/** One cleanup error aggregating what remains owned, or success when nothing does. */ +const cleanupResult = ( + remaining: readonly ProviderCleanupError[], +): Result => { + if (remaining.length === 0) return ok(null); + return err( + new ProviderCleanupError( + KUNA_PROVIDER_ID, + remaining.flatMap((entry) => entry.cleanupResources), + { + reason: remaining + .map((entry) => + String(entry.diagnostics?.["reason"] ?? entry.message), + ) + .join("; "), + }, + { operation: KUNA_EXPORT_OPERATION }, + ), + ); +}; diff --git a/src/kuna/KunaProjectExport.ts b/src/kuna/KunaProjectExport.ts index 7b45f1c871..a782cdeccb 100644 --- a/src/kuna/KunaProjectExport.ts +++ b/src/kuna/KunaProjectExport.ts @@ -1,35 +1,25 @@ -import { createHash, randomUUID } from "node:crypto"; -import { createReadStream, createWriteStream } from "node:fs"; +import { createReadStream } from "node:fs"; import { lstat, mkdir, readFile, writeFile } from "node:fs/promises"; import { basename, join } from "node:path"; -import { Transform } from "node:stream"; -import { pipeline } from "node:stream/promises"; import { AnalysisError } from "../domain/analysisErrorBase.js"; import { AnalysisArtifactChangedError, AnalysisCancelledError, - AnalysisResourceConstraintError, - AnalysisTimeoutError, } from "../domain/analysisErrorCore.js"; import { ProviderAdapterError } from "../domain/providerAdapterError.js"; -import { ProviderCleanupError } from "../domain/providerCleanupError.js"; +import type { ProviderCleanupError } from "../domain/providerCleanupError.js"; import { err, ok, type Result } from "../domain/result.js"; import { hashStableFile } from "../filesystem/StableFileHash.js"; -import { - OwnedCommandFailure, - runOwnedCommand, -} from "../process/OwnedCommand.js"; import { prepareProcessOwnershipInspection } from "../process/ProcessOwnershipObservation.js"; -import type { - OwnedProviderProcessSpawnOptions, - ProviderProcessSnapshot, - SpawnedOwnedProviderProcess, -} from "../process/ProviderProcess.js"; +import type { ProviderProcessSupervisor } from "../process/ProviderProcess.js"; import { - kunaChildEnvironment, - type KunaInstallation, -} from "./KunaInstallation.js"; + copyAndHash, + KUNA_INVENTORY_OUTPUT_BYTES, + runKunaCommand, + type KunaProcessLauncher, +} from "./KunaCommand.js"; +import type { KunaInstallation } from "./KunaInstallation.js"; import { parseKunaExportSummary, parseKunaInventory, @@ -46,12 +36,6 @@ export const KUNA_EXPORT_OPERATION = "kuna:decompile-project"; /** Kuna prints one summary line per export; notes on a large image stay far below this. */ const DIAGNOSTIC_BYTES = 8 * 1024 * 1024; -/** - * `kuna functions --json` costs ~210 bytes per function; this admits about - * 600,000 functions, above the 392,814 of the largest image Kuna documents. - */ -const INVENTORY_BYTES = 128 * 1024 * 1024; - const INVENTORY_FILE = "functions.json"; /** Identity of one file an export consists of. */ @@ -87,11 +71,6 @@ export interface KunaExportStatus { readonly inventoryFunctions: number | null; } -/** Test seam for the owned process launcher; production uses the default. */ -export type KunaProcessLauncher = ( - options: OwnedProviderProcessSpawnOptions, -) => Promise; - /** * How an export ended. `retained` means an owned Kuna process could not be * proven stopped, so it may still write into the workspace, which therefore @@ -102,7 +81,11 @@ export type KunaExportOutcome = readonly kind: "done"; readonly result: Result; } - | { readonly kind: "retained"; readonly error: ProviderCleanupError }; + | { + readonly kind: "retained"; + readonly error: ProviderCleanupError; + readonly owner: ProviderProcessSupervisor | undefined; + }; /** One export of a target into a private workspace. */ export interface KunaExportRequest { @@ -197,11 +180,38 @@ export const exportKunaProject = async ( ), }; await prepareProcessOwnershipInspection(signal); - const run = (arguments_: readonly string[], diagnosticBytes: number) => - runKuna(request, arguments_, { deadline, diagnosticBytes }); + const run = async (arguments_: readonly string[], outputBytes: number) => { + const outcome = await runKunaCommand({ + installation, + arguments: arguments_, + workspace: request.workspace, + operation: KUNA_EXPORT_OPERATION, + timeoutMs: deadline - Date.now(), + reportedTimeoutMs: configuration.projectTimeoutMs, + outputBytes, + signal, + launcher: request.launcher, + }); + if (outcome.kind === "ok") + return { ok: true as const, snapshot: outcome.snapshot }; + return { + ok: false as const, + outcome: + outcome.kind === "retained" + ? ({ + kind: "retained", + error: outcome.error, + owner: outcome.owner, + } as const) + : ({ kind: "done", result: err(outcome.error) } as const), + }; + }; request.onStatus({ phase: "inventory", inventoryFunctions: null }); - const listed = await run(["functions", input, "--json"], INVENTORY_BYTES); + const listed = await run( + ["functions", input, "--json"], + KUNA_INVENTORY_OUTPUT_BYTES, + ); if (!listed.ok) return listed.outcome; const inventory = parseKunaInventory(listed.snapshot.stdout.text); if (!inventory.ok) return unusable(inventory.error); @@ -309,158 +319,3 @@ export const readKunaExport = async ( if (!inventory.ok) return inventory; return ok({ functions: functions.value, inventory: inventory.value }); }; - -const runKuna = async ( - request: KunaExportRequest, - arguments_: readonly string[], - limits: { readonly deadline: number; readonly diagnosticBytes: number }, -): Promise< - | { readonly ok: true; readonly snapshot: ProviderProcessSnapshot } - | { readonly ok: false; readonly outcome: KunaExportOutcome } -> => { - const { installation, signal } = request; - try { - const identity = await verifyKunaExecutable(installation, signal); - if (!identity.ok) - return { ok: false, outcome: { kind: "done", result: identity } }; - const snapshot = await runOwnedCommand( - { - command: installation.executable, - arguments: arguments_, - runId: `rea-kuna-${randomUUID()}`, - // A script's process reports its interpreter, not the configured path. - ...(installation.script ? { expectedCommand: null } : {}), - cwd: request.workspace, - hostEnvironment: kunaChildEnvironment(installation.configuration), - env: { TMPDIR: request.workspace }, - }, - { - timeoutMs: Math.max(1, limits.deadline - Date.now()), - diagnosticBytes: limits.diagnosticBytes, - }, - { - signal, - ...(request.launcher === undefined - ? {} - : { launcher: request.launcher }), - }, - ); - return { ok: true, snapshot }; - } catch (cause: unknown) { - return { - ok: false, - outcome: commandFailure(cause, request, arguments_[0] ?? "", limits), - }; - } -}; - -/** Copy the target into the workspace and return the SHA-256 of exactly the bytes copied. */ -const copyAndHash = async ( - source: string, - destination: string, - signal: AbortSignal, -): Promise => { - const hash = createHash("sha256"); - await pipeline( - createReadStream(source), - new Transform({ - transform(chunk: Buffer, _encoding, callback) { - hash.update(chunk); - callback(null, chunk); - }, - }), - createWriteStream(destination, { flags: "wx", mode: 0o600 }), - { signal }, - ); - return hash.digest("hex"); -}; - -/** Map an owned-command failure to its typed error, keeping unproven cleanup owned. */ -const commandFailure = ( - cause: unknown, - request: KunaExportRequest, - subcommand: string, - limits: { readonly diagnosticBytes: number }, -): KunaExportOutcome => { - const { installation, signal } = request; - if (!(cause instanceof OwnedCommandFailure)) - return { - kind: "done", - result: err( - signal.aborted - ? new AnalysisCancelledError(KUNA_EXPORT_OPERATION, { cause }) - : new ProviderAdapterError(KUNA_PROVIDER_ID, KUNA_EXPORT_OPERATION, { - cause, - userMessage: `Cannot run ${installation.executable}: ${message(cause)}`, - }), - ), - }; - const snapshot = cause.snapshot; - const capturedOutput = - snapshot === null - ? undefined - : { - // An inventory is evidence-sized, not diagnostic-sized; keep only its tail. - stdout: snapshot.stdout.text.slice(-4096), - stderr: snapshot.stderr.text, - truncated: - snapshot.diagnosticTruncated === true || - snapshot.outputIncomplete === true || - snapshot.stdout.text.length > 4096, - exit_code: snapshot.exitCode ?? null, - signal: snapshot.signal ?? null, - }; - const details = { - cause, - ...(capturedOutput === undefined ? {} : { capturedOutput }), - }; - if (cause.cleanupFailure !== null) - return { - kind: "retained", - error: new ProviderCleanupError( - KUNA_PROVIDER_ID, - [...cause.resources, request.workspace], - { reason: cause.cleanupFailure }, - { operation: KUNA_EXPORT_OPERATION, cause }, - ), - }; - if (cause.reason === "cancelled") - return { - kind: "done", - result: err(new AnalysisCancelledError(KUNA_EXPORT_OPERATION, details)), - }; - if (cause.reason === "timeout") - return { - kind: "done", - result: err( - new AnalysisTimeoutError( - KUNA_EXPORT_OPERATION, - installation.configuration.projectTimeoutMs, - details, - ), - ), - }; - if (cause.reason === "output-limit") - return { - kind: "done", - result: err( - new AnalysisResourceConstraintError( - KUNA_EXPORT_OPERATION, - "capacity", - `kuna ${subcommand} wrote more than ${String(limits.diagnosticBytes)} bytes of output; the export was not retained.`, - { output_limit_bytes: limits.diagnosticBytes }, - details, - ), - ), - }; - const exitCode = capturedOutput?.exit_code; - return { - kind: "done", - result: err( - new ProviderAdapterError(KUNA_PROVIDER_ID, KUNA_EXPORT_OPERATION, { - ...details, - userMessage: `kuna ${subcommand} failed${exitCode === undefined || exitCode === null ? "" : ` with exit code ${String(exitCode)}`}: ${capturedOutput?.stderr.trim().split("\n").at(-1) ?? cause.message}`, - }), - ), - }; -}; diff --git a/src/kuna/KunaProjectFiles.ts b/src/kuna/KunaProjectFiles.ts index 9f0714f58b..5d8974c0ad 100644 --- a/src/kuna/KunaProjectFiles.ts +++ b/src/kuna/KunaProjectFiles.ts @@ -41,50 +41,37 @@ export const parseKunaExportSummary = ( return { decompiled: Number(match[1]), failed: Number(match[2]) }; }; -/** A header line longer than this is not one Kuna wrote. */ -const MAX_HEADER_BYTES = 64 * 1024; -const PREFIX_BYTES = Buffer.from(HEADER_PREFIX); +/** A structural line longer than this is not one Kuna wrote. */ +const MAX_KEPT_LINE_BYTES = 64 * 1024; -interface Header { - readonly name: string; - readonly address: string; - readonly error: string | null; +/** One line a scan kept: where it starts in the file and its text. */ +interface KeptLine { readonly offset: number; + readonly text: string; } /** - * Split a finished export's `.c` into its function blocks, reading it as a - * stream so a very large export is never held in memory. - * - * The `#include` preamble precedes the first header. The header count must - * equal the functions Kuna reported, and every address must be distinct; an - * export that fails either check is not a usable function inventory. + * Stream a file's lines, keeping the text of only the lines whose first + * `headBytes` bytes `keep` accepts. Every other line is skipped without being + * buffered, so a large export is never held in memory. */ -export const scanKunaProjectSource = async ( +const scanLines = async ( chunks: AsyncIterable, - summary: KunaExportSummary, -): Promise> => { - const headers: Header[] = []; + headBytes: number, + keep: (head: Buffer) => boolean, +): Promise< + Result< + { readonly lines: readonly KeptLine[]; readonly bytes: number }, + string + > +> => { + const lines: KeptLine[] = []; let offset = 0; let lineStart = 0; - // Bytes of the current line, kept only while it may still be a header. + // The current line's bytes while it is undecided or kept; undefined once skipped. let line: Buffer[] | undefined = []; let lineBytes = 0; - const endLine = (): string | undefined => { - if (line === undefined) return undefined; - const text = Buffer.concat(line).toString("utf8"); - if (!text.startsWith(HEADER_PREFIX)) return undefined; - const match = HEADER.exec(text); - if (match?.[1] === undefined || match[2] === undefined) - return `unrecognized function header at byte ${String(lineStart)}`; - headers.push({ - name: match[1], - address: match[2], - error: match[3] ?? null, - offset: lineStart, - }); - return undefined; - }; + let decided = false; for await (const chunk of chunks) { let position = 0; while (position < chunk.length) { @@ -94,27 +81,71 @@ export const scanKunaProjectSource = async ( const segment = chunk.subarray(position, end); line.push(segment); lineBytes += segment.length; - const seen = Buffer.concat(line).subarray(0, PREFIX_BYTES.length); - if (!PREFIX_BYTES.subarray(0, seen.length).equals(seen)) - line = undefined; - else if (lineBytes > MAX_HEADER_BYTES) - return err( - `function header at byte ${String(lineStart)} is too long`, - ); + if (!decided && (lineBytes >= headBytes || newline !== -1)) { + decided = true; + if (!keep(Buffer.concat(line).subarray(0, headBytes))) + line = undefined; + } + if (line !== undefined && lineBytes > MAX_KEPT_LINE_BYTES) + return err(`the line at byte ${String(lineStart)} is too long`); } if (newline === -1) break; - const failure = endLine(); - if (failure !== undefined) return err(failure); + if (line !== undefined) + lines.push({ + offset: lineStart, + text: Buffer.concat(line).toString("utf8"), + }); position = newline + 1; lineStart = offset + position; line = []; lineBytes = 0; + decided = false; } offset += chunk.length; } - const failure = endLine(); - if (failure !== undefined) return err(failure); - const total = offset; + if (line !== undefined && lineBytes > 0) { + const head = Buffer.concat(line); + if (decided || keep(head.subarray(0, headBytes))) + lines.push({ offset: lineStart, text: head.toString("utf8") }); + } + return ok({ lines, bytes: offset }); +}; + +const HEADER_PREFIX_BYTES = Buffer.from(HEADER_PREFIX); + +/** + * Split a finished export's `.c` into its function blocks. + * + * The `#include` preamble precedes the first header. The header count must + * equal the functions Kuna reported, and every address must be distinct; an + * export that fails either check is not a usable function inventory. + */ +export const scanKunaProjectSource = async ( + chunks: AsyncIterable, + summary: KunaExportSummary, +): Promise> => { + const scanned = await scanLines(chunks, HEADER_PREFIX_BYTES.length, (head) => + head.equals(HEADER_PREFIX_BYTES), + ); + if (!scanned.ok) return scanned; + const headers: { + readonly name: string; + readonly address: string; + readonly error: string | null; + readonly offset: number; + }[] = []; + for (const { offset, text } of scanned.value.lines) { + const match = HEADER.exec(text); + if (match?.[1] === undefined || match[2] === undefined) + return err(`unrecognized function header at byte ${String(offset)}`); + headers.push({ + name: match[1], + address: match[2], + error: match[3] ?? null, + offset, + }); + } + const total = scanned.value.bytes; const expected = summary.decompiled + summary.failed; if (headers.length !== expected) return err( @@ -187,3 +218,68 @@ export const parseKunaInventory = ( })), ); }; + +/** One labeled span of an export's `.asm`. */ +export interface KunaAssemblySpan { + readonly name: string; + readonly address: string; + readonly offset: number; + readonly length: number; +} + +/** Function listings and data labels found in an export's `.asm`. */ +export interface KunaAssemblyIndex { + /** Code labels, which name the same functions as the `.c`, by address. */ + readonly code: ReadonlyMap; + /** Labels in the `; --- data ---` tail: named globals and Kuna's `dat_`/`s_` names. */ + readonly data: ReadonlyMap; +} + +// Rendered by kuna-console: `{name}: ; 0x{addr:x}` at column 0, optionally +// followed by ` = dat_{addr:x}` in the data tail. A code label's listing runs +// to the next label, the next `; --- ` section marker, or the end of the file. +const LABEL = /^(.+): ; (0x[0-9a-f]+)(?: = dat_[0-9a-f]+)?$/u; +const SECTION_MARKER = "; --- "; +const DATA_MARKER = "; --- data ---"; +const SPACE = 0x20; +const TAB = 0x09; + +/** Index the labels of an export's `.asm` without holding the file in memory. */ +export const scanKunaAssembly = async ( + chunks: AsyncIterable, +): Promise> => { + const scanned = await scanLines( + chunks, + 1, + (head) => head[0] !== SPACE && head[0] !== TAB && head.length > 0, + ); + if (!scanned.ok) return scanned; + const code = new Map(); + const data = new Map(); + let inData = false; + let open: { name: string; address: string; offset: number } | undefined; + const close = (end: number) => { + if (open === undefined) return; + code.set(open.address, { ...open, length: end - open.offset }); + open = undefined; + }; + for (const { offset, text } of scanned.value.lines) { + if (text.startsWith(SECTION_MARKER)) { + close(offset); + if (text.startsWith(DATA_MARKER)) inData = true; + continue; + } + const match = LABEL.exec(text); + if (match?.[1] === undefined || match[2] === undefined) continue; + if (inData) { + if (!data.has(match[2])) data.set(match[2], match[1]); + continue; + } + close(offset); + if (code.has(match[2])) + return err(`the .asm repeats code address ${match[2]}`); + open = { name: match[1], address: match[2], offset }; + } + close(scanned.value.bytes); + return ok({ code, data }); +}; diff --git a/src/kuna/KunaProjectReader.ts b/src/kuna/KunaProjectReader.ts new file mode 100644 index 0000000000..ba5f983a27 --- /dev/null +++ b/src/kuna/KunaProjectReader.ts @@ -0,0 +1,180 @@ +import { createReadStream } from "node:fs"; +import { open } from "node:fs/promises"; +import { join } from "node:path"; + +import type { AnalysisError } from "../domain/analysisErrorBase.js"; +import { + AnalysisCapabilityUnavailableError, + AnalysisResourceConstraintError, +} from "../domain/analysisErrorCore.js"; +import { ProviderAdapterError } from "../domain/providerAdapterError.js"; +import { err, ok, type Result } from "../domain/result.js"; +import type { KunaProject } from "./KunaProjectCache.js"; +import { + scanKunaAssembly, + type KunaAssemblyIndex, + type KunaProjectFunction, +} from "./KunaProjectFiles.js"; +import { KUNA_PROVIDER_ID } from "./KunaRelease.js"; +import { KUNA_FUNCTION_TEXT_BYTES } from "./KunaLimits.js"; + +/** The exact producing range for one facet, including a failed C block. */ +export interface KunaProjectFileSpan { + readonly facet: "pseudocode" | "assembly"; + readonly path: string; + readonly byteOffset: number; + readonly byteLength: number; +} + +export interface KunaProjectText { + readonly result: string | null; + readonly files?: readonly KunaProjectFileSpan[]; + readonly limitations: readonly string[]; +} + +const readSpan = async ( + operation: string, + path: string, + offset: number, + length: number, +): Promise> => { + if (length > KUNA_FUNCTION_TEXT_BYTES) + return err( + new AnalysisResourceConstraintError( + operation, + "capacity", + "Kuna's function text exceeds the buffered source-facet capacity; the complete export remains accessible locally.", + { + limit_bytes: KUNA_FUNCTION_TEXT_BYTES, + requested_bytes: length, + file: path, + byte_offset: offset, + }, + ), + ); + const handle = await open(path, "r"); + try { + const buffer = Buffer.alloc(length); + const { bytesRead } = await handle.read(buffer, 0, length, offset); + if (bytesRead !== length) + throw new Error(`${path} is shorter than its recorded span`); + return ok(buffer.toString("utf8")); + } finally { + await handle.close(); + } +}; + +/** One immutable export owns both its text ranges and its parsed label indexes. */ +export class KunaProjectReader { + #blocks: ReadonlyMap | undefined; + #assembly: Promise> | undefined; + + constructor(readonly project: KunaProject) {} + + async pseudocode( + address: string, + operation: string, + ): Promise> { + this.#blocks ??= new Map( + this.project.functions.map((block) => [block.address, block]), + ); + const block = this.#blocks.get(address); + if (block === undefined) + return ok({ + result: null, + limitations: [ + "Kuna's export decompiles code entries only; this inventory entry, such as an import slot, has no body.", + ], + }); + const files = [ + { + facet: "pseudocode" as const, + path: this.project.manifest.files.c.path, + byteOffset: block.cOffset, + byteLength: block.cLength, + }, + ]; + if (block.error !== null) + return ok({ + result: null, + files, + limitations: [`Kuna could not decompile this function: ${block.error}`], + }); + const text = await readSpan( + operation, + join(this.project.directory, this.project.manifest.files.c.path), + block.cOffset, + block.cLength, + ); + if (!text.ok) return text; + return ok({ + result: text.value + .slice(text.value.indexOf("\n") + 1) + .replace(/\n+$/u, ""), + files, + limitations: [], + }); + } + + async assembly( + address: string, + operation: string, + ): Promise> { + const index = await this.#index(operation); + if (!index.ok) return index; + const span = index.value.code.get(address); + if (span === undefined) + return err( + new AnalysisCapabilityUnavailableError( + KUNA_PROVIDER_ID, + operation, + `Kuna's export lists no code at ${address}; import slots and data entries have no listing.`, + ), + ); + const text = await readSpan( + operation, + join(this.project.directory, this.project.manifest.files.asm.path), + span.offset, + span.length, + ); + if (!text.ok) return text; + return ok({ + result: text.value.replace(/\n+$/u, ""), + files: [ + { + facet: "assembly", + path: this.project.manifest.files.asm.path, + byteOffset: span.offset, + byteLength: span.length, + }, + ], + limitations: [], + }); + } + + async dataName( + address: string, + operation: string, + ): Promise> { + const index = await this.#index(operation); + return index.ok ? ok(index.value.data.get(address) ?? null) : index; + } + + async #index( + operation: string, + ): Promise> { + this.#assembly ??= scanKunaAssembly( + createReadStream( + join(this.project.directory, this.project.manifest.files.asm.path), + ), + ); + const index = await this.#assembly; + return index.ok + ? index + : err( + new ProviderAdapterError(KUNA_PROVIDER_ID, operation, { + userMessage: `Kuna's .asm cannot be indexed: ${index.error}`, + }), + ); + } +} diff --git a/src/kuna/KunaProvider.ts b/src/kuna/KunaProvider.ts new file mode 100644 index 0000000000..4f79acab2a --- /dev/null +++ b/src/kuna/KunaProvider.ts @@ -0,0 +1,224 @@ +import { accessSync, constants, statSync } from "node:fs"; + +import type { + AnalysisClient, + AnalysisProfileResolutionOptions, + AnalysisProviderCandidate, + ProviderAvailability, + ProviderTargetSupport, +} from "../application/AnalysisProvider.js"; +import { + AnalysisCancelledError, + AnalysisCapabilityUnavailableError, +} from "../domain/analysisErrorCore.js"; +import { + createAnalysisProfile, + type AnalysisProfileCommitment, +} from "../domain/analysisProfile.js"; +import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; +import { err, ok } from "../domain/result.js"; +import { snapshotEnvironment } from "../process/snapshotEnvironment.js"; +import type { KunaProcessLauncher } from "./KunaCommand.js"; +import { + readKunaConfiguration, + type KunaConfiguration, +} from "./KunaConfiguration.js"; +import { inspectKunaInstallation } from "./KunaInstallation.js"; +import { KunaProjectCache } from "./KunaProjectCache.js"; +import { + KUNA_PROVIDER_IDENTITY, + kunaCapabilities, +} from "./KunaProviderCapabilities.js"; +import { KUNA_COMMAND_REMEDIATION, KUNA_PROVIDER_ID } from "./KunaRelease.js"; +import { KunaSessionClient } from "./KunaSessionClient.js"; + +/** Architectures REA verified Kuna on: x86-64 and AArch64 ELF and Mach-O, plus ARM Thumb. */ +const VERIFIED_ARCHITECTURES: ReadonlySet = new Set([ + "x86_64", + "arm64", + "arm", +]); + +/** + * Bring-your-own Kuna candidate. Discovery reads configuration and the + * executable's metadata only; the version probe runs when a target is opened. + */ +export class KunaProvider implements AnalysisProviderCandidate { + readonly #environment: Readonly; + readonly #platform: NodeJS.Platform; + readonly #launcher: KunaProcessLauncher | undefined; + #cache: KunaProjectCache | undefined; + + constructor( + environment: Readonly, + options: { + readonly platform?: NodeJS.Platform; + readonly launcher?: KunaProcessLauncher; + } = {}, + ) { + this.#platform = options.platform ?? process.platform; + this.#environment = snapshotEnvironment(environment, this.#platform); + this.#launcher = options.launcher; + } + + identity() { + return KUNA_PROVIDER_IDENTITY; + } + + capabilities() { + const availability = this.inspectAvailability(); + return kunaCapabilities( + availability.status === "available" + ? { available: true } + : { + available: false, + reason: availability.reason, + code: availability.code, + }, + ); + } + + inspectAvailability(): ProviderAvailability { + const configuration = readKunaConfiguration( + this.#environment, + this.#platform, + ); + if (!configuration.ok) + return { + status: "unavailable", + code: configuration.error.code, + reason: configuration.error.reason, + diagnostics: { + configured: configuration.error.code !== "not_configured", + version_probed: false, + }, + }; + const { command } = configuration.value; + try { + accessSync(command, constants.X_OK); + if (!statSync(command).isFile()) throw new Error("not a regular file"); + } catch (cause: unknown) { + return { + status: "unavailable", + code: "executable_missing", + reason: `Cannot execute REA_KUNA_COMMAND ${command}: ${cause instanceof Error ? cause.message : String(cause)}. ${KUNA_COMMAND_REMEDIATION}`, + diagnostics: { configured: true, command, version_probed: false }, + }; + } + return { + status: "available", + code: null, + reason: null, + diagnostics: { + configured: true, + command, + cache_root: configuration.value.cacheRoot, + jobs: configuration.value.jobs, + version_probed: false, + }, + }; + } + + inspectTargetSupport(target: BinaryTarget): ProviderTargetSupport { + const diagnostics = { + target_kind: target.kind, + target_format: target.format, + }; + if (target.kind !== "executable") + return { + status: "unsupported", + code: "target_kind_unsupported", + reason: "Kuna decompiles executable images, not databases or archives.", + diagnostics, + }; + if (target.format !== "elf" && target.format !== "mach-o") + return { + status: "unsupported", + code: "target_format_unsupported", + reason: + target.format === "mach-o-universal" + ? "REA's Kuna provider has no verified universal Mach-O slice selection yet; open a thin slice or use another provider." + : `REA verified the Kuna provider on ELF and thin Mach-O images; ${target.format} is not admitted yet.`, + diagnostics, + }; + if (!VERIFIED_ARCHITECTURES.has(target.architecture)) + return { + status: "unsupported", + code: "architecture_unsupported", + reason: `REA verified the Kuna provider on x86-64, AArch64 and ARM; ${target.architecture} is not admitted yet.`, + diagnostics: { ...diagnostics, architecture: target.architecture }, + }; + return { status: "supported", code: null, reason: null, diagnostics }; + } + + async resolveAnalysisProfile( + _target: BinaryTarget, + options?: AnalysisProfileResolutionOptions, + ) { + if (options?.signal?.aborted === true) + return err(new AnalysisCancelledError("kuna:profile")); + const installation = await inspectKunaInstallation(this.#environment, { + platform: this.#platform, + ...(options?.signal === undefined ? {} : { signal: options.signal }), + }); + if (!installation.ok) + return err( + new AnalysisCapabilityUnavailableError( + KUNA_PROVIDER_ID, + "open_binary", + installation.error.reason, + { userMessage: installation.error.reason }, + ), + ); + const { value } = installation; + return ok({ + profile: createAnalysisProfile( + { ...KUNA_PROVIDER_IDENTITY, version: value.version }, + { + kuna_version: value.version, + kuna_sha256: value.sha256, + verified_release: value.verified, + jobs: value.configuration.jobs, + export: "decompile-project", + cache_policy: "live", + }, + ), + }); + } + + createClient( + target: BinaryTarget, + profile?: AnalysisProfileCommitment, + ): AnalysisClient { + const configuration = readKunaConfiguration( + this.#environment, + this.#platform, + ); + if (!configuration.ok) + return { + execute: async (operation) => + err( + new AnalysisCapabilityUnavailableError( + KUNA_PROVIDER_ID, + operation, + configuration.error.reason, + ), + ), + close: async () => ok(null), + }; + return new KunaSessionClient(target, profile, { + cache: this.#projectCache(configuration.value), + environment: this.#environment, + ...(this.#launcher === undefined ? {} : { launcher: this.#launcher }), + }); + } + + /** One cache per provider, so sessions of one server share exports in flight. */ + #projectCache(configuration: KunaConfiguration): KunaProjectCache { + this.#cache ??= new KunaProjectCache(configuration.cacheRoot, { + maxBytes: configuration.cacheMaxBytes, + ...(this.#launcher === undefined ? {} : { launcher: this.#launcher }), + }); + return this.#cache; + } +} diff --git a/src/kuna/KunaProviderCapabilities.ts b/src/kuna/KunaProviderCapabilities.ts new file mode 100644 index 0000000000..00baf6113f --- /dev/null +++ b/src/kuna/KunaProviderCapabilities.ts @@ -0,0 +1,115 @@ +import type { + AnalysisOperation, + CapabilityDescriptor, +} from "../application/AnalysisProvider.js"; +import type { ProviderRejectionCode } from "../contracts/providerSelection.js"; +import { KUNA_PROVIDER_ID } from "./KunaRelease.js"; +import { REGEX_MATCHING_LIMITATION } from "../process/RegexMatcher.js"; +import { KUNA_FUNCTION_TEXT_BYTES } from "./KunaLimits.js"; + +/** Static identity; the observed Kuna release is reported in each analysis profile. */ +export const KUNA_PROVIDER_IDENTITY = Object.freeze({ + id: KUNA_PROVIDER_ID, + name: "Kuna", + version: null, +}); + +/** Analyst outcomes Kuna's inventory and project export can answer. */ +export const KUNA_OPERATIONS = Object.freeze([ + "list_documents", + "list_procedures", + "search_procedures", + "procedure_address", + "address_name", + "procedure_pseudo_code", + "procedure_assembly", + "analyze_function", +] as const satisfies readonly Exclude[]); +export type KunaOperation = (typeof KUNA_OPERATIONS)[number]; + +/** Whether an operation is one Kuna answers. */ +export const isKunaOperation = ( + operation: AnalysisOperation, +): operation is KunaOperation => + (KUNA_OPERATIONS as readonly string[]).includes(operation); + +/** How to read every Kuna answer. */ +const KUNA_PROVENANCE = + "Kuna static analysis: addresses and bytes are its observations, procedure entries and extents its derivations, and pseudocode its reconstruction, never original source."; + +/** Entries come from Kuna's own discovery. */ +const KUNA_ENTRY_LIMITATION = + "Kuna may split one function at an entry its own analysis infers; entry extents are upper bounds."; + +/** Provider text is not comparable across providers. */ +const KUNA_TEXT_LIMITATION = + "Pseudocode and assembly are exact Kuna text, not comparable as equal or different with another provider's text."; +const KUNA_TEXT_CAPACITY_LIMITATION = `Each buffered source facet admits at most ${String(KUNA_FUNCTION_TEXT_BYTES)} bytes before allocation; larger text stays accessible in the local project export.`; + +/** The limitations that apply to one operation's answer, and only those. */ +export const kunaLimitations = ( + operation: KunaOperation, +): readonly string[] => { + switch (operation) { + case "list_documents": + return []; + case "list_procedures": + case "procedure_address": + case "address_name": + return [KUNA_PROVENANCE, KUNA_ENTRY_LIMITATION]; + case "search_procedures": + return [ + KUNA_PROVENANCE, + KUNA_ENTRY_LIMITATION, + REGEX_MATCHING_LIMITATION, + ]; + case "procedure_pseudo_code": + case "procedure_assembly": + case "analyze_function": + return [ + KUNA_PROVENANCE, + KUNA_TEXT_LIMITATION, + KUNA_TEXT_CAPACITY_LIMITATION, + ]; + } +}; + +/** Pseudocode answered before the project export finishes differs from the export's. */ +export const KUNA_SINGLE_FUNCTION_LIMITATION = + "This pseudocode came from a single-function `kuna decompile` run while the project export was still building. It lacks the export's whole-program prototypes, and a later request for the same procedure can return the export's text."; + +/** Advertise Kuna's static, process-launching operations truthfully. */ +export const kunaCapabilities = ( + availability: + | { readonly available: true } + | { + readonly available: false; + readonly reason: string; + readonly code: ProviderRejectionCode; + }, +): readonly CapabilityDescriptor[] => + KUNA_OPERATIONS.map((operation) => ({ + provider: KUNA_PROVIDER_IDENTITY, + operation, + ...(availability.available + ? ({ available: true, reason: null } as const) + : ({ + available: false, + reason: availability.reason, + availabilityCode: availability.code, + } as const)), + // A cold answer can come from the single-function path and differ from the + // export's later answer, so results are re-read rather than replayed. The + // project cache already serves warm reads in milliseconds. + cachePolicy: "live", + effects: { + mutatesArtifact: false, + launchesProcess: true, + mayShowUi: false, + mayAccessNetwork: false, + mayWriteFilesystem: true, + changesPermissions: false, + requiresRoot: false, + }, + limitations: kunaLimitations(operation), + })); diff --git a/src/kuna/KunaSessionClient.ts b/src/kuna/KunaSessionClient.ts new file mode 100644 index 0000000000..233873e082 --- /dev/null +++ b/src/kuna/KunaSessionClient.ts @@ -0,0 +1,1060 @@ +import { basename, join } from "node:path"; + +import { z } from "zod"; + +import { + createAnalysisExecution, + type AnalysisClient, + type AnalysisExecution, + type AnalysisOperation, + type ExecutionOptions, +} from "../application/AnalysisProvider.js"; +import { AnalysisError } from "../domain/analysisErrorBase.js"; +import { + AnalysisArtifactChangedError, + AnalysisCancelledError, + AnalysisCapabilityUnavailableError, + AnalysisInputError, +} from "../domain/analysisErrorCore.js"; +import type { AnalysisProfileCommitment } from "../domain/analysisProfile.js"; +import { + analysisSearchPredicate, + analysisSearchQuerySchema, +} from "../domain/analysisSearch.js"; +import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; +import type { EvidenceLocation } from "../domain/evidence.js"; +import { functionDossierSchema } from "../domain/hopperValues.js"; +import type { JsonValue } from "../domain/jsonValue.js"; +import { ProviderAdapterError } from "../domain/providerAdapterError.js"; +import { ProviderCleanupError } from "../domain/providerCleanupError.js"; +import { err, ok, type Result } from "../domain/result.js"; +import { PrivateRuntimeRoot } from "../process/PrivateRuntimeRoot.js"; +import { matchRegexValues } from "../process/RegexMatcher.js"; +import { prepareProcessOwnershipInspection } from "../process/ProcessOwnershipObservation.js"; +import type { ProviderProcessSupervisor } from "../process/ProviderProcess.js"; +import { + copyAndHash, + KUNA_INVENTORY_OUTPUT_BYTES, + runKunaCommand, + type KunaProcessLauncher, +} from "./KunaCommand.js"; +import { + inspectKunaInstallation, + type KunaInstallation, +} from "./KunaInstallation.js"; +import { + kunaProjectKey, + type KunaProject, + type KunaProjectCache, +} from "./KunaProjectCache.js"; +import type { KunaExportStatus } from "./KunaProjectExport.js"; +import { + parseKunaInventory, + type KunaInventoryFunction, +} from "./KunaProjectFiles.js"; +import { + isKunaOperation, + KUNA_SINGLE_FUNCTION_LIMITATION, + kunaLimitations, + type KunaOperation, +} from "./KunaProviderCapabilities.js"; +import { KUNA_PROVIDER_ID } from "./KunaRelease.js"; +import { KUNA_FUNCTION_TEXT_BYTES } from "./KunaLimits.js"; +import { + KunaProjectReader, + type KunaProjectFileSpan, +} from "./KunaProjectReader.js"; + +/** How often a request waiting on the project export reports that it still waits. */ +const WAIT_PROGRESS_INTERVAL_MS = 10_000; + +const HEX_ADDRESS = /^0x[0-9a-f]+$/iu; + +/** Where one answer's text came from, recorded in its raw result. */ +type AnswerSource = + | { readonly kind: "project-export"; readonly project: KunaProject } + | { readonly kind: "single-function" }; + +interface InventoryIndex { + readonly inventory: readonly KunaInventoryFunction[]; + readonly byAddress: ReadonlyMap; + readonly byName: ReadonlyMap; +} + +interface Answer { + readonly result: JsonValue; + readonly source: AnswerSource | null; + readonly files?: readonly KunaProjectFileSpan[]; + readonly locations: readonly EvidenceLocation[]; + readonly limitations: readonly string[]; +} + +/** + * Wait for work this session owns, stopping only this caller's wait when its + * signal aborts; the work itself continues for other callers. + */ +const waitFor = ( + work: Promise>, + signal: AbortSignal | undefined, + operation: string, +): Promise> => { + if (signal === undefined) return work; + if (signal.aborted) + return Promise.resolve(err(new AnalysisCancelledError(operation))); + return new Promise((resolve) => { + const onAbort = () => resolve(err(new AnalysisCancelledError(operation))); + signal.addEventListener("abort", onAbort, { once: true }); + void work.then((result) => { + signal.removeEventListener("abort", onAbort); + resolve(result); + }); + }); +}; + +/** A file the adopted export named is gone, e.g. evicted by another REA process. */ +const isMissing = (cause: unknown): boolean => + cause instanceof Error && + "code" in cause && + (cause.code === "ENOENT" || cause.code === "ENOTDIR"); + +/** Read through a call so a check before an await does not narrow later ones. */ +const cancelled = (signal: AbortSignal | undefined): boolean => + signal?.aborted === true; + +const message = (cause: unknown): string => + cause instanceof Error ? cause.message : String(cause); + +const failure = (operation: string, cause: unknown): AnalysisError => + cause instanceof AnalysisError + ? cause + : new ProviderAdapterError(KUNA_PROVIDER_ID, operation, { + cause, + userMessage: `Kuna ${operation} failed: ${message(cause)}`, + }); + +/** Kuna names a procedure by its inventory entry or by an exact entry address. */ +const procedureSelectorSchema = z.string().min(1); + +/** + * One Kuna analysis session for one target. + * + * Reads come from the verified project export in REA's cache. Until that + * export exists, listings use a quick `kuna functions` inventory and + * pseudocode a single-function `kuna decompile`, while the export builds in + * the background; assembly waits for it. Kuna always runs on a private copy + * whose digest matches the selected target. + */ +export class KunaSessionClient implements AnalysisClient { + readonly #target: BinaryTarget; + readonly #profile: AnalysisProfileCommitment | undefined; + readonly #cache: KunaProjectCache; + readonly #environment: Readonly; + readonly #launcher: KunaProcessLauncher | undefined; + /** Owns background work: aborting it cancels this session's export interest. */ + readonly #session = new AbortController(); + /** Requests keep their workspaces and process cleanup owned until they settle. */ + readonly #requests = new Set< + Promise> + >(); + /** Single-function runs that could not be proven stopped, with their retry owners. */ + readonly #retained: { + readonly error: ProviderCleanupError; + owner: ProviderProcessSupervisor | undefined; + }[] = []; + /** Private roots whose removal failed during setup or close. */ + readonly #leakedRoots: PrivateRuntimeRoot[] = []; + #installation: Promise> | undefined; + #workspace: + | Promise< + Result< + { readonly root: PrivateRuntimeRoot; readonly input: string }, + AnalysisError + > + > + | undefined; + #inventory: + | Promise> + | undefined; + #project: Promise> | undefined; + #reader: KunaProjectReader | undefined; + #inventoryIndex: InventoryIndex | undefined; + #status: KunaExportStatus | undefined; + #key: string | undefined; + #exportFailure: string | undefined; + #releasePin: (() => void) | undefined; + /** Set once close starts; new requests are refused from then on. */ + #closing = false; + /** Set only when close has released every owned resource. */ + #closed = false; + + constructor( + target: BinaryTarget, + profile: AnalysisProfileCommitment | undefined, + dependencies: { + readonly cache: KunaProjectCache; + readonly environment: Readonly; + readonly launcher?: KunaProcessLauncher; + }, + ) { + this.#target = target; + this.#profile = profile; + this.#cache = dependencies.cache; + this.#environment = dependencies.environment; + this.#launcher = dependencies.launcher; + } + + async execute( + operation: AnalysisOperation, + parameters: Readonly>, + options: ExecutionOptions = {}, + ): Promise> { + if (this.#closing) + return err( + new AnalysisCapabilityUnavailableError( + KUNA_PROVIDER_ID, + operation, + "This Kuna session is closed.", + ), + ); + const request = this.#execute(operation, parameters, { + ...options, + signal: + options.signal === undefined + ? this.#session.signal + : AbortSignal.any([this.#session.signal, options.signal]), + }); + this.#requests.add(request); + try { + return await request; + } finally { + this.#requests.delete(request); + } + } + + async #execute( + operation: AnalysisOperation, + parameters: Readonly>, + options: ExecutionOptions, + ): Promise> { + if (cancelled(options.signal)) + return err(new AnalysisCancelledError(operation)); + if (operation !== "health" && !isKunaOperation(operation)) + return err( + new AnalysisCapabilityUnavailableError( + KUNA_PROVIDER_ID, + operation, + "Kuna answers procedure listings, lookups, pseudocode and assembly; this operation needs another provider.", + ), + ); + const document = parameters.document; + if (document !== undefined && document !== this.#target.path) + return err( + new AnalysisInputError( + operation, + { + userMessage: + "A Kuna session analyzes one program; omit document or pass the selected target path.", + }, + [ + { + path: ["document"], + reason: "invalid_value", + expected: this.#target.path, + }, + ], + ), + ); + const installation = await this.#ensureInstallation(); + if (!installation.ok) return installation; + try { + if (operation === "health") + return ok( + this.#execution([], installation.value, { + result: { + ready: true, + kuna: { + version: installation.value.version, + verified: installation.value.verified, + sha256: installation.value.sha256, + }, + }, + source: null, + locations: [], + limitations: [], + }), + ); + if (operation !== "list_documents") + await this.#startProject(installation.value); + const answer = await this.#answer( + operation, + parameters, + installation.value, + options, + ); + if (!answer.ok) return answer; + if (cancelled(options.signal)) + return err(new AnalysisCancelledError(operation)); + return ok( + this.#execution( + kunaLimitations(operation), + installation.value, + answer.value, + ), + ); + } catch (cause: unknown) { + if (cancelled(options.signal)) + return err(new AnalysisCancelledError(operation, { cause })); + return err(failure(operation, cause)); + } + } + + /** + * Release everything this session owns. A failed close keeps reporting its + * failure, and each later call retries the cleanup that remains. + */ + async close(): Promise> { + if (this.#closed) return ok(null); + if (!this.#closing) { + this.#closing = true; + this.#session.abort(); + } + // Cancellation starts process shutdown; its completion proves whether it + // is safe to remove private copies and exposes any retained cleanup owner. + await Promise.allSettled(this.#requests); + await this.#project; + await this.#inventory; + const failures: ProviderCleanupError[] = []; + // Kuna runs in its own process group; an export no other session wants + // must stop before this session reports itself closed. + if (this.#key !== undefined) { + const settled = await this.#cache.settled(this.#key); + if (!settled.ok) failures.push(settled.error); + } + for (const retained of [...this.#retained]) { + const stopped = await retained.owner?.stop(); + if (stopped !== undefined && stopped.status !== "incomplete") + this.#retained.splice(this.#retained.indexOf(retained), 1); + } + failures.push(...this.#retained.map(({ error }) => error)); + const workspace = await this.#workspace; + // A single-function run that may still be running keeps its copy. + if (this.#retained.length === 0) { + if (workspace?.ok === true) { + this.#leakedRoots.push(workspace.value.root); + this.#workspace = undefined; + } + for (const root of [...this.#leakedRoots]) + try { + await root.close(); + this.#leakedRoots.splice(this.#leakedRoots.indexOf(root), 1); + } catch (cause: unknown) { + failures.push( + new ProviderCleanupError( + KUNA_PROVIDER_ID, + [root.path], + { reason: message(cause) }, + { cause }, + ), + ); + } + } + if (failures.length > 0) + return err( + new ProviderCleanupError( + KUNA_PROVIDER_ID, + failures.flatMap((failed) => failed.cleanupResources), + { + reason: failures + .map((failed) => + String(failed.diagnostics?.["reason"] ?? failed.message), + ) + .join("; "), + }, + ), + ); + this.#releasePin?.(); + this.#releasePin = undefined; + this.#closed = true; + return ok(null); + } + + #execution( + operationLimitations: readonly string[], + installation: KunaInstallation, + answer: Answer, + ): AnalysisExecution { + const source = answer.source; + const files = + source?.kind !== "project-export" + ? [] + : (answer.files ?? []).map((file) => ({ + facet: file.facet, + file: file.path, + file_sha256: + Object.values(source.project.manifest.files).find( + ({ path }) => path === file.path, + )?.sha256 ?? null, + byte_offset: file.byteOffset, + byte_length: file.byteLength, + })); + // Preserve the direct text source shape; combined answers retain every + // component once, including its producing file digest and byte range. + const recordedFiles = + files.length === 1 ? files[0] : files.length === 0 ? {} : { files }; + return createAnalysisExecution( + answer.result, + { + id: KUNA_PROVIDER_ID, + name: "Kuna", + version: installation.version, + }, + { + // Kuna's identity is in the analysis profile; the raw result records + // only where file-backed text came from. + ...(source === null + ? {} + : { + rawResult: { + source: + source.kind === "single-function" + ? { kind: "single-function" } + : { + kind: "project-export", + key: source.project.key, + directory: source.project.directory, + ...recordedFiles, + }, + project_export: this.#exportState(), + }, + }), + limitations: [...operationLimitations, ...answer.limitations], + locations: [ + { kind: "artifact-path", path: this.#target.path }, + ...answer.locations, + ], + ...(this.#profile === undefined + ? {} + : { analysisProfile: this.#profile }), + }, + ); + } + + /** Where the background export stands, for callers deciding whether to wait. */ + #exportState(): JsonValue { + if (this.#reader !== undefined) + return { state: "ready", key: this.#reader.project.key }; + if (this.#project === undefined) return { state: "not-started" }; + if (this.#exportFailure !== undefined) + return { state: "failed", reason: this.#exportFailure }; + return { + state: "building", + phase: this.#status?.phase ?? null, + inventory_functions: this.#status?.inventoryFunctions ?? null, + }; + } + + async #answer( + operation: KunaOperation, + parameters: Readonly>, + installation: KunaInstallation, + options: ExecutionOptions, + ): Promise> { + if (operation === "list_documents") + return ok({ + result: [this.#target.path], + source: null, + locations: [], + limitations: [], + }); + const inventory = await this.#ensureInventory(installation, options); + if (!inventory.ok) return inventory; + if (operation === "list_procedures" || operation === "search_procedures") { + const items = inventory.value.map(({ address, name }) => ({ + address, + value: name, + })); + let selected = items; + if (operation === "search_procedures") { + const query = analysisSearchQuerySchema.parse(parameters); + if (query.mode === "regex") { + const matches = await matchRegexValues({ + operation, + pattern: query.pattern, + caseSensitive: query.case_sensitive, + values: items.map(({ value }) => value), + signal: options.signal, + }); + if (!matches.ok) return matches; + const indices = new Set(matches.value); + selected = items.filter((_, index) => indices.has(index)); + } else { + const matches = analysisSearchPredicate(query); + selected = items.filter(({ value }) => matches(value)); + } + } + return ok({ + result: selected, + source: null, + locations: [], + limitations: [], + }); + } + if (operation === "address_name") + return this.#addressName( + parameters, + inventory.value, + installation, + options, + ); + const entry = this.#resolve(operation, parameters, inventory.value); + if (!entry.ok) return entry; + const locations = [ + { kind: "address", address: entry.value.address } as const, + ]; + switch (operation) { + case "procedure_address": + return ok({ + result: entry.value.address, + source: null, + locations, + limitations: [], + }); + case "procedure_pseudo_code": { + const pseudocode = await this.#pseudocode( + operation, + entry.value, + installation, + options, + ); + if (!pseudocode.ok) return pseudocode; + return ok({ ...pseudocode.value, locations }); + } + case "procedure_assembly": { + const assembly = await this.#assemblyText( + operation, + entry.value, + installation, + options, + ); + if (!assembly.ok) return assembly; + return ok({ ...assembly.value, locations }); + } + case "analyze_function": + return this.#withProject(operation, installation, options, (reader) => + this.#dossier(operation, entry.value, reader), + ); + } + } + + /** Resolve an exact inventory name, alias or entry address. */ + #resolve( + operation: string, + parameters: Readonly>, + inventory: readonly KunaInventoryFunction[], + ): Result { + const parsed = procedureSelectorSchema.safeParse(parameters.procedure); + if (!parsed.success) + return err( + new AnalysisInputError( + operation, + { userMessage: "procedure is required." }, + [{ path: ["procedure"], reason: "missing_argument" }], + ), + ); + const selector = parsed.data; + if (HEX_ADDRESS.test(selector)) { + const address = `0x${BigInt(selector).toString(16)}`; + const entry = this.#index(inventory).byAddress.get(address); + return entry === undefined + ? err( + new AnalysisInputError(operation, { + userMessage: `${address} is not a procedure entry Kuna found; list_procedures shows the entries it found.`, + }), + ) + : ok(entry); + } + const matches = this.#index(inventory).byName.get(selector) ?? []; + if (matches.length === 1 && matches[0] !== undefined) return ok(matches[0]); + return err( + new AnalysisInputError(operation, { + userMessage: + matches.length === 0 + ? `Kuna found no procedure named ${selector}; search_procedures finds similar names.` + : `${selector} names ${String(matches.length)} Kuna entries (${matches.map(({ address }) => address).join(", ")}); select one by address.`, + }), + ); + } + + async #addressName( + parameters: Readonly>, + inventory: readonly KunaInventoryFunction[], + installation: KunaInstallation, + options: ExecutionOptions, + ): Promise> { + const raw = parameters.address; + if (typeof raw !== "string" || !HEX_ADDRESS.test(raw)) + return err( + new AnalysisInputError( + "address_name", + { + userMessage: + "Kuna has no cursor; pass an explicit 0x-prefixed address.", + }, + [{ path: ["address"], reason: "invalid_format" }], + ), + ); + const address = `0x${BigInt(raw).toString(16)}`; + const locations = [{ kind: "address", address } as const]; + const entry = this.#index(inventory).byAddress.get(address); + if (entry !== undefined) + return ok({ + result: entry.name, + source: null, + locations, + limitations: [], + }); + return this.#withProject( + "address_name", + installation, + options, + async (reader) => { + const name = await reader.dataName(address, "address_name"); + if (!name.ok) return name; + return ok({ + result: name.value, + source: { kind: "project-export", project: reader.project }, + locations, + limitations: [ + "Kuna names function entries and the data its export references; null means Kuna assigned no name at this address, not that no symbol exists.", + ], + }); + }, + ); + } + + async #pseudocode( + operation: string, + entry: KunaInventoryFunction, + installation: KunaInstallation, + options: ExecutionOptions, + ): Promise, AnalysisError>> { + for (let attempt = 0; this.#reader !== undefined; attempt += 1) { + const reader = this.#reader; + try { + const text = await reader.pseudocode(entry.address, operation); + if (!text.ok) return text; + return ok({ + ...text.value, + source: { kind: "project-export", project: reader.project }, + }); + } catch (cause: unknown) { + if (!isMissing(cause) || attempt !== 0) throw cause; + this.#invalidateProject(reader); + await this.#startProject(installation); + } + } + const workspace = await this.#ensureWorkspace(); + if (!workspace.ok) return workspace; + const run = await runKunaCommand({ + installation, + arguments: ["decompile", workspace.value.input, entry.address, "--addr"], + workspace: workspace.value.root.path, + operation, + timeoutMs: installation.configuration.functionTimeoutMs, + reportedTimeoutMs: installation.configuration.functionTimeoutMs, + outputBytes: KUNA_FUNCTION_TEXT_BYTES, + signal: options.signal ?? this.#session.signal, + launcher: this.#launcher, + }); + if (run.kind === "retained") { + this.#retained.push({ error: run.error, owner: run.owner }); + return err(run.error); + } + if (run.kind === "failed") return err(run.error); + const exportFailure = await this.#settledExportFailure(); + return ok({ + result: run.snapshot.stdout.text.replace(/\n+$/u, ""), + source: { kind: "single-function" }, + limitations: [ + KUNA_SINGLE_FUNCTION_LIMITATION, + ...(exportFailure === undefined + ? [] + : [ + `The project export failed, so later requests also use single-function runs: ${exportFailure}`, + ]), + ], + }); + } + + async #assemblyText( + operation: string, + entry: KunaInventoryFunction, + installation: KunaInstallation, + options: ExecutionOptions, + ): Promise, AnalysisError>> { + return this.#withProject( + operation, + installation, + options, + async (reader) => { + const text = await reader.assembly(entry.address, operation); + return text.ok + ? ok({ + ...text.value, + source: { kind: "project-export", project: reader.project }, + }) + : text; + }, + ); + } + + async #dossier( + operation: string, + entry: KunaInventoryFunction, + reader: KunaProjectReader, + ): Promise> { + const assembly = await reader.assembly(entry.address, operation); + if (!assembly.ok) return assembly; + const pseudocode = await reader.pseudocode(entry.address, operation); + if (!pseudocode.ok) return pseudocode; + const text = + typeof pseudocode.value.result === "string" + ? pseudocode.value.result + : null; + const unavailable = (reason: string) => + ({ available: false, reason }) as const; + const dossier = functionDossierSchema.parse({ + procedure: { + address: entry.address, + name: entry.name, + classification: null, + body: unavailable( + "Kuna reports an inventory extent that is an upper bound, not observed body ranges.", + ), + signature: null, + locals: [], + }, + pseudocode: text ?? "", + assembly: + typeof assembly.value.result === "string" + ? assembly.value.result.split("\n") + : [], + comments: [], + callers: [], + callees: [], + incoming_references: [], + outgoing_references: [], + referenced_strings: [], + referenced_names: [], + basic_blocks: [], + native_api: null, + native_value_flow: null, + facet_availability: { + pseudocode: + text === null + ? unavailable( + pseudocode.value.limitations.at(-1) ?? + "Kuna did not supply pseudocode.", + ) + : { available: true }, + assembly: { available: true }, + comments: unavailable("Kuna exports no analyst comments."), + callers: unavailable( + "This provider does not report Kuna cross-references.", + ), + callees: unavailable( + "This provider does not report Kuna cross-references.", + ), + references: unavailable( + "This provider does not report typed references.", + ), + strings_names: unavailable( + "This provider does not report function-referenced strings and names.", + ), + cfg: unavailable("Kuna exports no basic blocks."), + }, + limitations: [ + ...kunaLimitations("analyze_function"), + ...pseudocode.value.limitations, + ], + }); + return ok({ + result: dossier, + source: { kind: "project-export", project: reader.project }, + files: [ + ...(pseudocode.value.files ?? []), + ...(assembly.value.files ?? []), + ], + locations: [{ kind: "address", address: entry.address }], + limitations: pseudocode.value.limitations, + }); + } + + /** Address and name lookups over one inventory, built once. */ + #index(inventory: readonly KunaInventoryFunction[]): InventoryIndex { + if (this.#inventoryIndex?.inventory === inventory) + return this.#inventoryIndex; + const byAddress = new Map(); + const byName = new Map(); + for (const entry of inventory) { + byAddress.set(entry.address, entry); + for (const name of new Set([entry.name, ...entry.aliases])) { + const named = byName.get(name); + if (named === undefined) byName.set(name, [entry]); + else named.push(entry); + } + } + this.#inventoryIndex = { inventory, byAddress, byName }; + return this.#inventoryIndex; + } + + async #ensureInstallation(): Promise< + Result + > { + this.#installation ??= inspectKunaInstallation(this.#environment).then( + (inspected): Result => { + if (!inspected.ok) + return err( + new AnalysisCapabilityUnavailableError( + KUNA_PROVIDER_ID, + "health", + inspected.error.reason, + { userMessage: inspected.error.reason }, + ), + ); + const committed = this.#profile?.parameters["kuna_sha256"]; + if (committed !== undefined && committed !== inspected.value.sha256) + return err( + new ProviderAdapterError(KUNA_PROVIDER_ID, "health", { + userMessage: + "The Kuna executable changed after this target was opened; reopen the target to analyze it with the current executable.", + diagnostics: { + committed_sha256: committed, + observed_sha256: inspected.value.sha256, + }, + }), + ); + return ok(inspected.value); + }, + ); + return this.#installation; + } + + /** Reuse a verified export, or start building one in the background. */ + async #startProject(installation: KunaInstallation): Promise { + if (this.#project !== undefined) return; + const key = kunaProjectKey({ + targetSha256: this.#target.sha256, + kunaSha256: installation.sha256, + kunaVersion: installation.version, + jobs: installation.configuration.jobs, + }); + this.#key = key; + const cached = await this.#cache.lookup(key, this.#session.signal); + if (cached !== undefined) { + this.#adopt(cached); + this.#project = Promise.resolve(ok(cached)); + return; + } + this.#project = this.#cache + .build({ + installation, + targetPath: this.#target.path, + targetSha256: this.#target.sha256, + signal: this.#session.signal, + onStatus: (status) => { + this.#status = status; + }, + }) + .then((result) => { + if (result.ok && !this.#closing) this.#adopt(result.value); + else if (!result.ok && !this.#closing) + this.#exportFailure = + result.error.userMessage ?? result.error.message; + return result; + }); + } + + /** Retry the complete read when its immutable project disappears, never just one facet. */ + async #withProject( + operation: string, + installation: KunaInstallation, + options: ExecutionOptions, + read: (reader: KunaProjectReader) => Promise>, + ): Promise> { + for (let attempt = 0; ; attempt += 1) { + const reader = await this.#awaitProject(operation, options); + if (!reader.ok) return reader; + try { + return await read(reader.value); + } catch (cause: unknown) { + if (!isMissing(cause) || attempt !== 0) throw cause; + this.#invalidateProject(reader.value); + await this.#startProject(installation); + } + } + } + + #invalidateProject(reader: KunaProjectReader): void { + // Another request may already have replaced this reader and its build. + if (this.#reader !== reader) return; + this.#releasePin?.(); + this.#releasePin = undefined; + this.#reader = undefined; + this.#project = undefined; + this.#status = undefined; + this.#exportFailure = undefined; + } + + #adopt(project: KunaProject): void { + this.#reader = this.#readerFor(project); + this.#releasePin ??= this.#cache.pin(project.key); + } + + #readerFor(project: KunaProject): KunaProjectReader { + return this.#reader?.project === project + ? this.#reader + : new KunaProjectReader(project); + } + + /** The export's failure once it has settled; undefined while it builds or after success. */ + async #settledExportFailure(): Promise { + const settled = await Promise.race([ + this.#project, + Promise.resolve(undefined), + ]); + if (settled === undefined || settled.ok) return undefined; + return settled.error.userMessage ?? settled.error.message; + } + + /** Wait for the export, reporting progress, without cancelling it for other callers. */ + async #awaitProject( + operation: string, + options: ExecutionOptions, + ): Promise> { + if (this.#reader !== undefined) return ok(this.#reader); + const project = this.#project; + if (project === undefined) + return err( + new ProviderAdapterError(KUNA_PROVIDER_ID, operation, { + userMessage: "The Kuna project export was never started.", + }), + ); + const started = Date.now(); + const progress = options.progress; + // Report waiting only; Kuna exposes no measure of completed export work. + const timer = setInterval(() => { + void progress + ?.report({ + phase: "kuna_export", + completed: 0, + total: null, + message: `Waiting for Kuna's project export (${this.#status?.phase ?? "starting"}, ${String(Math.floor((Date.now() - started) / 1000))}s elapsed); it serves this request and every later one`, + }) + .catch(() => undefined); + }, WAIT_PROGRESS_INTERVAL_MS); + try { + const built = await waitFor(project, options.signal, operation); + if (!built.ok) return built; + return ok(this.#readerFor(built.value)); + } finally { + clearInterval(timer); + } + } + + /** The export's inventory when ready, otherwise one quick `kuna functions` run. */ + async #ensureInventory( + installation: KunaInstallation, + options: ExecutionOptions, + ): Promise> { + if (this.#reader !== undefined) return ok(this.#reader.project.inventory); + if (this.#inventory === undefined) { + // The session owns the shared run; each caller only stops waiting. + const run = this.#listInventory(installation); + this.#inventory = run; + void run.then((listed) => { + // A failed quick listing is retried by the next request. + if (!listed.ok && this.#inventory === run) this.#inventory = undefined; + }); + } + return waitFor(this.#inventory, options.signal, "list_procedures"); + } + + async #listInventory( + installation: KunaInstallation, + ): Promise> { + const workspace = await this.#ensureWorkspace(); + if (!workspace.ok) return workspace; + const run = await runKunaCommand({ + installation, + arguments: ["functions", workspace.value.input, "--json"], + workspace: workspace.value.root.path, + operation: "list_procedures", + timeoutMs: installation.configuration.functionTimeoutMs, + reportedTimeoutMs: installation.configuration.functionTimeoutMs, + outputBytes: KUNA_INVENTORY_OUTPUT_BYTES, + signal: this.#session.signal, + launcher: this.#launcher, + }); + if (run.kind === "retained") { + this.#retained.push({ error: run.error, owner: run.owner }); + return err(run.error); + } + if (run.kind === "failed") return err(run.error); + const inventory = parseKunaInventory(run.snapshot.stdout.text); + return inventory.ok + ? ok(inventory.value) + : err( + new ProviderAdapterError(KUNA_PROVIDER_ID, "list_procedures", { + userMessage: `Kuna's function inventory is not usable: ${inventory.error}`, + }), + ); + } + + /** A private, digest-verified copy of the target for single-function runs. */ + async #ensureWorkspace(): Promise< + Result< + { readonly root: PrivateRuntimeRoot; readonly input: string }, + AnalysisError + > + > { + this.#workspace ??= (async () => { + const root = await PrivateRuntimeRoot.create({ prefix: "rea-kuna-" }); + try { + const input = join(root.path, basename(this.#target.path)); + const digest = await copyAndHash( + this.#target.path, + input, + this.#session.signal, + ); + if (digest !== this.#target.sha256) { + await this.#release(root); + return err( + new AnalysisArtifactChangedError( + "kuna:workspace", + this.#target.path, + "the target no longer matches the digest selected for analysis", + ), + ); + } + await prepareProcessOwnershipInspection(this.#session.signal); + return ok({ root, input }); + } catch (cause: unknown) { + await this.#release(root); + throw cause; + } + })().catch((cause: unknown) => err(failure("kuna:workspace", cause))); + const workspace = await this.#workspace; + if (!workspace.ok) this.#workspace = undefined; + return workspace; + } + + /** Remove a private root, keeping it for close to retry when removal fails. */ + async #release(root: PrivateRuntimeRoot): Promise { + try { + await root.close(); + } catch { + this.#leakedRoots.push(root); + } + } +} diff --git a/src/process/RegexMatcher.ts b/src/process/RegexMatcher.ts new file mode 100644 index 0000000000..215c4bdfbd --- /dev/null +++ b/src/process/RegexMatcher.ts @@ -0,0 +1,176 @@ +import { Worker } from "node:worker_threads"; + +import { z } from "zod"; + +import type { AnalysisError } from "../domain/analysisErrorBase.js"; +import { + AnalysisCancelledError, + AnalysisInputError, + AnalysisOutputError, + AnalysisResourceConstraintError, +} from "../domain/analysisErrorCore.js"; +import { err, ok, type Result } from "../domain/result.js"; + +const MATCHING_DEADLINE_MS = 5_000; +const WORKER_HEAP_MIB = 128; +/** Shared capability text for the supervised ECMAScript matching boundary. */ +export const REGEX_MATCHING_LIMITATION = `Regex mode uses ECMAScript Unicode semantics in a cancellable worker with a ${String(MATCHING_DEADLINE_MS / 1000)}-second matching deadline; matching failure leaves the service available.`; +const replySchema = z.discriminatedUnion("status", [ + z.strictObject({ + status: z.literal("matched"), + indices: z.array(z.number().int().nonnegative()), + }), + z.strictObject({ status: z.literal("invalid_pattern"), reason: z.string() }), +]); + +// Code is constant; caller patterns and target strings cross only workerData. +const WORKER_SOURCE = String.raw` +(async () => { +const { parentPort, workerData } = await import("node:worker_threads"); +let expression; +try { + expression = new RegExp(workerData.pattern, workerData.caseSensitive ? "u" : "iu"); +} catch (cause) { + parentPort.postMessage({ status: "invalid_pattern", reason: cause instanceof Error ? cause.message : String(cause) }); +} +if (expression !== undefined) { + const indices = []; + for (const [index, value] of workerData.values.entries()) + if (expression.test(value)) indices.push(index); + parentPort.postMessage({ status: "matched", indices }); +} +})(); +`; + +/** Match an inventory off the service thread, and prove worker exit before returning. */ +export const matchRegexValues = async (input: { + readonly operation: string; + readonly pattern: string; + readonly caseSensitive: boolean; + readonly values: readonly string[]; + readonly signal?: AbortSignal | undefined; + readonly invalidPattern?: (reason: string) => AnalysisError; +}): Promise> => { + if (input.signal?.aborted) + return err(new AnalysisCancelledError(input.operation)); + let worker: Worker; + try { + worker = new Worker(WORKER_SOURCE, { + eval: true, + workerData: { + pattern: input.pattern, + caseSensitive: input.caseSensitive, + values: input.values, + }, + resourceLimits: { maxOldGenerationSizeMb: WORKER_HEAP_MIB }, + }); + } catch (cause: unknown) { + return err( + new AnalysisOutputError(input.operation, "Cannot start regex worker", { + cause, + }), + ); + } + try { + return await new Promise((resolve) => { + let settled = false; + const finish = (result: Result) => { + if (settled) return; + settled = true; + clearTimeout(timer); + input.signal?.removeEventListener("abort", onAbort); + resolve(result); + }; + const onAbort = () => + finish(err(new AnalysisCancelledError(input.operation))); + const timer = setTimeout( + () => + finish( + err( + new AnalysisResourceConstraintError( + input.operation, + "cpu", + "Regex matching exceeded its supervised deadline; the service remains available.", + { matching_deadline_ms: MATCHING_DEADLINE_MS }, + { + remediationAction: + "Simplify the ECMAScript Unicode regex or use literal mode, then retry in the same session.", + }, + ), + ), + ), + MATCHING_DEADLINE_MS, + ); + input.signal?.addEventListener("abort", onAbort, { once: true }); + if (input.signal?.aborted) onAbort(); + worker.once("message", (message: unknown) => { + const reply = replySchema.safeParse(message); + if (!reply.success) + return finish( + err( + new AnalysisOutputError( + input.operation, + "Regex worker reply is malformed", + ), + ), + ); + if (reply.data.status === "invalid_pattern") + return finish( + err( + input.invalidPattern?.(reply.data.reason) ?? + new AnalysisInputError(input.operation, { + userMessage: `Invalid regex pattern: ${reply.data.reason}`, + }), + ), + ); + if ( + reply.data.indices.some( + (index, position, indices) => + index >= input.values.length || + (position > 0 && index <= (indices[position - 1] ?? -1)), + ) + ) + return finish( + err( + new AnalysisOutputError( + input.operation, + "Regex worker returned invalid inventory indices", + ), + ), + ); + finish(ok(reply.data.indices)); + }); + worker.once("error", (cause: Error & { code?: string }) => + finish( + err( + cause.code === "ERR_WORKER_OUT_OF_MEMORY" + ? new AnalysisResourceConstraintError( + input.operation, + "memory", + "Regex worker exhausted its heap budget", + { worker_heap_mib: WORKER_HEAP_MIB }, + { cause }, + ) + : new AnalysisOutputError( + input.operation, + `Regex worker failed: ${cause.message}`, + { cause }, + ), + ), + ), + ); + worker.once("exit", (code) => + finish( + err( + new AnalysisOutputError( + input.operation, + `Regex worker exited without a result (${String(code)})`, + ), + ), + ), + ); + }); + } finally { + await worker.terminate(); + } +}; diff --git a/tests/boundary/filesystem/directAnalysisPermissions.test.ts b/tests/boundary/filesystem/directAnalysisPermissions.test.ts index 26e308c44a..e9e1838288 100644 --- a/tests/boundary/filesystem/directAnalysisPermissions.test.ts +++ b/tests/boundary/filesystem/directAnalysisPermissions.test.ts @@ -50,7 +50,7 @@ describe("direct analysis snapshot files", () => { details: { selection_reason: "unknown_provider", requested_provider_id: "request-provider", - candidate_ids: ["ghidra", "hopper", "ida"], + candidate_ids: ["ghidra", "hopper", "ida", "kuna"], }, }); }); diff --git a/tests/boundary/process/regexMatcher.test.ts b/tests/boundary/process/regexMatcher.test.ts new file mode 100644 index 0000000000..0666faf154 --- /dev/null +++ b/tests/boundary/process/regexMatcher.test.ts @@ -0,0 +1,50 @@ +import { expect, it } from "vitest"; + +import { ok } from "../../../src/domain/result.js"; +import { HopperRegexSearch } from "../../../src/hopper/HopperRegexSearch.js"; +import { matchRegexValues } from "../../../src/process/RegexMatcher.js"; + +it("preserves Unicode matching order and refuses a pre-cancelled match", async () => { + expect( + await matchRegexValues({ + operation: "search_procedures", + pattern: "^δ", + caseSensitive: false, + values: ["Δelta", "delta", "δίκη"], + }), + ).toEqual(ok([0, 2])); + expect( + await matchRegexValues({ + operation: "search_procedures", + pattern: ".", + caseSensitive: true, + values: ["value"], + signal: AbortSignal.abort(), + }), + ).toMatchObject({ ok: false, error: { _tag: "AnalysisCancelledError" } }); +}); + +it("keeps Hopper's result projection and invalid-pattern contract with the shared worker", async () => { + const search = new HopperRegexSearch({ + callTool: async () => + ok([ + { address: "0x10", value: "Δelta" }, + { address: "0x20", value: "other" }, + ]), + }); + expect( + await search.execute("search_strings", { pattern: "^δ", mode: "regex" }), + ).toEqual(ok([{ address: "0x10", value: "Δelta" }])); + expect( + await search.execute("search_strings", { pattern: "[", mode: "regex" }), + ).toMatchObject({ + ok: false, + error: { + _tag: "HopperRemoteError", + code: -32000, + diagnosticType: "invalid_request", + operation: "search_strings", + }, + }); + await search.close(); +}); diff --git a/tests/boundary/providers/kuna/fakeKuna.ts b/tests/boundary/providers/kuna/fakeKuna.ts index 24c04b1a95..0195cb9d36 100644 --- a/tests/boundary/providers/kuna/fakeKuna.ts +++ b/tests/boundary/providers/kuna/fakeKuna.ts @@ -26,6 +26,11 @@ export interface FakeKunaBehavior { readonly functions?: readonly FakeKunaFunction[]; /** Extra `.asm` bytes, for capacity tests. */ readonly asmPadding?: number; + /** Extra C comment bytes inside each exported body, for buffered-read capacity. */ + readonly cPadding?: number; + readonly decompile?: "ok" | "hang"; + /** Milliseconds `functions --json` takes before printing. */ + readonly inventoryDelayMs?: number; /** Milliseconds the export takes before writing its files. */ readonly delayMs?: number; } @@ -55,7 +60,7 @@ if (args[0] === "--version") { else if (mode === "not-json") process.stdout.write("functions: 2\n"); else { const listed = [...functions, { name: "puts", addr: "0x2000" }].map((fn) => ({ name: fn.name, address: parseInt(fn.addr, 16), address_hex: fn.addr, aliases: [], object_location: null, size: fn.name === "puts" ? 0 : 16 })); - process.stdout.write(JSON.stringify({ binary: args[1], count: listed.length, total: listed.length, error: null, functions: listed }, null, 2) + "\n"); + setTimeout(() => process.stdout.write(JSON.stringify({ binary: args[1], count: listed.length, total: listed.length, error: null, functions: listed }, null, 2) + "\n"), behavior.inventoryDelayMs ?? 0); } } else if (args[0] === "decompile-project") { const input = args[1]; @@ -69,12 +74,15 @@ if (args[0] === "--version") { let c = '#include "' + stem + '.h"\n\n'; functions.forEach((fn, index) => { c += fn.error === undefined - ? "// Function: " + fn.name + " @ " + fn.addr + "\nint " + fn.name + "(void)\n{\n return " + index + ";\n}\n\n" + ? "// Function: " + fn.name + " @ " + fn.addr + "\nint " + fn.name + "(void)\n{\n return " + index + ";\n" + (behavior.cPadding ? " /*" + "x".repeat(behavior.cPadding) + "*/\n" : "") + "}\n\n" : "// Function: " + fn.name + " @ " + fn.addr + " (error: " + fn.error + ")\n\n"; }); fs.writeFileSync(path.join(output, stem + ".c"), c); fs.writeFileSync(path.join(output, stem + ".h"), "#pragma once\n"); - fs.writeFileSync(path.join(output, stem + ".asm"), "; asm\n" + "x".repeat(behavior.asmPadding ?? 0)); + let asm = "; kuna decompile-project — " + stem + "\n\n; --- code section 0x1000..0x2000 ---\n\n"; + functions.forEach((fn) => { asm += fn.name + ": ; " + fn.addr + "\n " + fn.addr.slice(2).padStart(8, "0") + ": c3 RET\n\n"; }); + asm += "; --- data ---\n\ndat_3000: ; 0x3000 = dat_3000\n 00003000: 00\n\ng_table: ; 0x3010\n 00003010: 01\n" + "x".repeat(behavior.asmPadding ?? 0); + fs.writeFileSync(path.join(output, stem + ".asm"), asm); fs.writeFileSync(path.join(output, "README.md"), "# " + stem + "\n"); // Occupy the manifest path so REA's publication write fails. if (mode === "block-manifest") fs.mkdirSync(path.join(output, "..", "manifest.json")); @@ -82,6 +90,12 @@ if (args[0] === "--version") { const reported = mode === "miscounted" ? functions.length + 1 : functions.length - failed; if (mode !== "silent") process.stdout.write("wrote " + output + ": " + stem + ".c (" + c.length + " bytes); functions: " + reported + " ok, " + failed + " failed\n"); }, behavior.delayMs ?? 0); +} else if (args[0] === "decompile") { + // decompile <0xaddr> --addr: the single-function path. + const fn = functions.find((candidate) => candidate.addr === args[2]); + if (behavior.decompile === "hang") hang(); + else if (fn === undefined) { process.stderr.write("error: no function at " + args[2] + "\n"); process.exit(1); } + else process.stdout.write("int " + fn.name + "(void) // single-function\n{\n return 0;\n}\n"); } else { process.stderr.write("unexpected command\n"); process.exit(2); diff --git a/tests/boundary/providers/kuna/kunaInstallation.test.ts b/tests/boundary/providers/kuna/kunaInstallation.test.ts index 13b98bcba9..51bc524504 100644 --- a/tests/boundary/providers/kuna/kunaInstallation.test.ts +++ b/tests/boundary/providers/kuna/kunaInstallation.test.ts @@ -145,6 +145,7 @@ it("parses cache, timeout and job settings at the configuration boundary", () => cacheRoot: "/home/analyst/.cache/rea/kuna", cacheMaxBytes: 4 * 1024 ** 3, projectTimeoutMs: 1_800_000, + functionTimeoutMs: 120_000, // Kuna's reference answer is a serial export; more jobs change results. jobs: 1, }, @@ -164,18 +165,25 @@ it("parses cache, timeout and job settings at the configuration boundary", () => ...environment, REA_KUNA_CACHE_MAX_BYTES: "2097152", REA_KUNA_PROJECT_TIMEOUT_MS: "60000", + REA_KUNA_TIMEOUT_MS: "5000", REA_KUNA_JOBS: "3", }, "linux", ), ).toMatchObject({ - value: { cacheMaxBytes: 2097152, projectTimeoutMs: 60000, jobs: 3 }, + value: { + cacheMaxBytes: 2097152, + projectTimeoutMs: 60000, + functionTimeoutMs: 5000, + jobs: 3, + }, }); for (const [setting, value] of [ ["REA_KUNA_CACHE_DIR", "relative/cache"], ["REA_KUNA_CACHE_MAX_BYTES", "1024"], ["REA_KUNA_CACHE_MAX_BYTES", "4GiB"], ["REA_KUNA_PROJECT_TIMEOUT_MS", "0"], + ["REA_KUNA_TIMEOUT_MS", "-1"], ["REA_KUNA_JOBS", "33"], ["REA_KUNA_JOBS", "0"], ] as const) diff --git a/tests/boundary/providers/kuna/kunaProjectFiles.test.ts b/tests/boundary/providers/kuna/kunaProjectFiles.test.ts index b39dbb5749..9c173b9acd 100644 --- a/tests/boundary/providers/kuna/kunaProjectFiles.test.ts +++ b/tests/boundary/providers/kuna/kunaProjectFiles.test.ts @@ -3,6 +3,7 @@ import { expect, it } from "vitest"; import { parseKunaExportSummary, parseKunaInventory, + scanKunaAssembly, scanKunaProjectSource, } from "../../../../src/kuna/KunaProjectFiles.js"; @@ -162,3 +163,45 @@ it("reads the callable inventory, ignoring fields a later release adds", () => { parseKunaInventory(JSON.stringify({ functions: [{ name: "x" }] })), ).toMatchObject({ ok: false }); }); + +it("indexes code listings and data labels in Kuna's .asm", async () => { + const asm = [ + "; kuna decompile-project — a.out", + "; arch: x86:LE:64:default:gcc", + "", + "; --- code section 0x1000..0x1020 ---", + "", + "main: ; 0x1000", + "; arg: param_1 (int)", + " 00001000: c3 RET", + "", + "operator new(unsigned long): ; 0x1010", + " 00001010: c3 RET", + "", + "; --- data ---", + "", + "dat_3000: ; 0x3000 = dat_3000", + " 00003000: 00", + "", + "", + ].join("\n"); + const scanned = await scanKunaAssembly(chunked(asm, 5)); + if (!scanned.ok) throw new Error(scanned.error); + const main = scanned.value.code.get("0x1000"); + // Offsets are bytes; the header's em dash is three of them. + expect( + Buffer.from(asm) + .subarray(main?.offset, (main?.offset ?? 0) + (main?.length ?? 0)) + .toString(), + ).toBe( + "main: ; 0x1000\n; arg: param_1 (int)\n 00001000: c3 RET\n\n", + ); + expect(scanned.value.code.get("0x1010")?.name).toBe( + "operator new(unsigned long)", + ); + expect([...scanned.value.code.keys()]).toEqual(["0x1000", "0x1010"]); + expect([...scanned.value.data]).toEqual([["0x3000", "dat_3000"]]); + expect( + await scanKunaAssembly(chunked("a: ; 0x10\n\nb: ; 0x10\n", 64)), + ).toEqual({ ok: false, error: "the .asm repeats code address 0x10" }); +}); diff --git a/tests/boundary/providers/kuna/kunaProjectReaderConcurrency.test.ts b/tests/boundary/providers/kuna/kunaProjectReaderConcurrency.test.ts new file mode 100644 index 0000000000..f7743c15c5 --- /dev/null +++ b/tests/boundary/providers/kuna/kunaProjectReaderConcurrency.test.ts @@ -0,0 +1,132 @@ +import { readdir, rename } from "node:fs/promises"; +import { join } from "node:path"; + +import { afterEach, expect, it as test, vi } from "vitest"; + +import { spawnOwnedProviderProcess } from "../../../../src/process/ProviderProcess.js"; +import { run, session } from "./kunaSessionFixture.js"; + +const reading = vi.hoisted(() => ({ + asm: "", + c: "", + held: false, + gate: undefined as Promise | undefined, + missing: 0, + onMissing: undefined as (() => void) | undefined, +})); + +vi.mock("node:fs/promises", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + open: async (...args: Parameters) => { + try { + const handle = await actual.open(...args); + if (String(args[0]) !== reading.asm || reading.held) return handle; + return new Proxy(handle, { + get(target, key) { + if (key === "read") + return async (...readArgs: unknown[]) => { + const result: unknown = await Reflect.apply( + target.read, + target, + readArgs, + ); + reading.held = true; + await reading.gate; + return result; + }; + const value: unknown = Reflect.get(target, key, target); + return typeof value === "function" ? value.bind(target) : value; + }, + }); + } catch (cause: unknown) { + if (String(args[0]) === reading.c) { + reading.missing += 1; + if (reading.missing === 2) reading.onMissing?.(); + } + throw cause; + } + }, + }; +}); + +afterEach(() => { + reading.asm = ""; + reading.c = ""; + reading.held = false; + reading.gate = undefined; + reading.missing = 0; + reading.onMissing = undefined; +}); + +const barrier = () => { + let release = () => {}; + const promise = new Promise((resolve) => { + release = resolve; + }); + return { promise, release }; +}; + +test.skipIf(process.platform === "win32")( + "retries the whole dossier when another read loses its project", + async () => { + const assembly = barrier(); + const rebuild = barrier(); + const missing = barrier(); + let exports = 0; + const { cacheRoot, open } = await session( + "rea-kuna-reader-eviction-", + {}, + {}, + async (options) => { + if (options.arguments[0] === "decompile-project" && ++exports === 2) + await rebuild.promise; + return spawnOwnedProviderProcess(options); + }, + ); + const client = open(); + let dossier: ReturnType | undefined; + try { + await run(client, "procedure_assembly", { procedure: "main" }); + const [key] = await readdir(join(cacheRoot, "projects")); + const project = join(cacheRoot, "projects", key ?? ""); + reading.asm = join(project, "project", "program.asm"); + reading.c = join(project, "project", "program.c"); + reading.gate = assembly.promise; + reading.onMissing = missing.release; + dossier = run(client, "analyze_function", { procedure: "main" }); + await expect.poll(() => reading.held).toBe(true); + await rename(project, join(cacheRoot, "trash", "evicted-elsewhere")); + + const fallback = await run(client, "procedure_pseudo_code", { + procedure: "parse", + }); + expect(fallback.rawResult).toMatchObject({ + source: { kind: "single-function" }, + }); + await expect.poll(() => exports).toBe(2); + assembly.release(); + // The fixed reader attempts the missing C before retrying the entire + // report. The old reader instead completes with a single-function source. + await Promise.race([dossier, missing.promise]); + rebuild.release(); + const result = await dossier; + expect(result.rawResult).toMatchObject({ + source: { + kind: "project-export", + files: [{ facet: "pseudocode" }, { facet: "assembly" }], + }, + }); + expect(result.result).toMatchObject({ + pseudocode: "int main(void)\n{\n return 0;\n}", + assembly: ["main: ; 0x1000", " 00001000: c3 RET"], + }); + } finally { + assembly.release(); + rebuild.release(); + await dossier?.catch(() => undefined); + expect(await client.close()).toEqual({ ok: true, value: null }); + } + }, +); diff --git a/tests/boundary/providers/kuna/kunaProvider.test.ts b/tests/boundary/providers/kuna/kunaProvider.test.ts new file mode 100644 index 0000000000..9c27530c24 --- /dev/null +++ b/tests/boundary/providers/kuna/kunaProvider.test.ts @@ -0,0 +1,787 @@ +import { createHash } from "node:crypto"; +import { + appendFile, + chmod, + mkdir, + readdir, + readFile, + rename, + writeFile, +} from "node:fs/promises"; +import { join } from "node:path"; + +import { expect, it as test } from "vitest"; + +import type { BinaryTarget } from "../../../../src/domain/binaryTargetTypes.js"; +import type { JsonValue } from "../../../../src/domain/jsonValue.js"; +import type { KunaProcessLauncher } from "../../../../src/kuna/KunaCommand.js"; +import { KunaProvider } from "../../../../src/kuna/KunaProvider.js"; +import { KUNA_FUNCTION_TEXT_BYTES } from "../../../../src/kuna/KunaLimits.js"; +import { cleanupOwnedProcessGroup } from "../../../../src/process/ProcessOwnership.js"; +import { spawnOwnedProviderProcess } from "../../../../src/process/ProviderProcess.js"; +import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; +import { + fakeKunaCalls, + setFakeKunaBehavior, + writeFakeKuna, +} from "./fakeKuna.js"; + +import { session, run, exportsOf, FUNCTIONS } from "./kunaSessionFixture.js"; + +const it = test.skipIf(process.platform === "win32"); + +it("discovers configuration without probing and admits only verified targets", async () => { + const root = await createTestTempDirectory("rea-kuna-discovery-"); + const kuna = await writeFakeKuna(root); + const configured = new KunaProvider({ REA_KUNA_COMMAND: kuna }); + + expect(new KunaProvider({}).inspectAvailability()).toMatchObject({ + status: "unavailable", + code: "not_configured", + }); + expect( + new KunaProvider({ + REA_KUNA_COMMAND: join(root, "absent"), + }).inspectAvailability(), + ).toMatchObject({ status: "unavailable", code: "executable_missing" }); + expect(configured.inspectAvailability()).toMatchObject({ + status: "available", + diagnostics: { version_probed: false }, + }); + expect(await fakeKunaCalls(kuna)).toEqual([]); + expect( + configured + .capabilities() + .every( + ({ available, cachePolicy }) => available && cachePolicy === "live", + ), + ).toBe(true); + + const executable = ( + format: "elf" | "mach-o" | "pe" | "mach-o-universal", + architecture: "x86_64" | "arm64" | "arm" | "x86", + ) => + ({ + path: "/tmp/fixture", + sha256: "a".repeat(64), + kind: "executable", + format, + architecture, + availableArchitectures: [architecture], + }) as unknown as BinaryTarget; + expect( + configured.inspectTargetSupport(executable("elf", "x86_64")).status, + ).toBe("supported"); + expect( + configured.inspectTargetSupport(executable("mach-o", "arm64")).status, + ).toBe("supported"); + expect( + configured.inspectTargetSupport(executable("pe", "x86_64")), + ).toMatchObject({ status: "unsupported", code: "target_format_unsupported" }); + expect( + configured.inspectTargetSupport(executable("mach-o-universal", "arm64")), + ).toMatchObject({ code: "target_format_unsupported" }); + expect( + configured.inspectTargetSupport(executable("elf", "x86")), + ).toMatchObject({ + code: "architecture_unsupported", + }); +}); + +it("commits the observed Kuna release and digest to the analysis profile", async () => { + const { profile } = await session("rea-kuna-profile-"); + + expect(profile).toMatchObject({ + provider: { id: "kuna", name: "Kuna", version: "1.833" }, + parameters: { + kuna_version: "1.833", + kuna_sha256: expect.stringMatching(/^[0-9a-f]{64}$/u), + verified_release: true, + jobs: 1, + cache_policy: "live", + }, + }); +}); + +it("lists the selected document without starting target analysis", async () => { + const { kuna, target, cacheRoot, open } = await session( + "rea-kuna-document-metadata-", + { project: "hang" }, + ); + const client = open(); + try { + expect((await run(client, "list_documents")).result).toEqual([target.path]); + expect( + (await fakeKunaCalls(kuna)).every(({ argv }) => argv[0] === "--version"), + ).toBe(true); + await expect(readdir(cacheRoot)).rejects.toMatchObject({ code: "ENOENT" }); + } finally { + await client.close(); + } +}); + +it("answers a cold request from one function, then serves the finished export", async () => { + const { kuna, cacheRoot, open } = await session("rea-kuna-cold-", { + delayMs: 300, + }); + const client = open(); + + const cold = await run(client, "procedure_pseudo_code", { + procedure: "main", + }); + expect(cold.result).toBe( + "int main(void) // single-function\n{\n return 0;\n}", + ); + expect(cold.rawResult).toMatchObject({ + source: { kind: "single-function" }, + project_export: { state: "building" }, + }); + expect(cold.limitations.at(-1)).toContain("single-function `kuna decompile`"); + + // Assembly waits for the export, which then serves every later request. + const assembly = await run(client, "procedure_assembly", { + procedure: "0x1000", + }); + expect(assembly.result).toBe("main: ; 0x1000\n 00001000: c3 RET"); + const warm = await run(client, "procedure_pseudo_code", { + procedure: "main", + }); + expect(warm.result).toBe("int main(void)\n{\n return 0;\n}"); + expect(warm.rawResult).toMatchObject({ + source: { + kind: "project-export", + directory: expect.stringContaining(join(cacheRoot, "projects")), + file: "program.c", + file_sha256: expect.stringMatching(/^[0-9a-f]{64}$/u), + }, + project_export: { state: "ready" }, + }); + // The recorded span is exactly the function's block in the cached .c. + const source = warm.rawResult as { + source: { directory: string; byte_offset: number; byte_length: number }; + }; + const c = await readFile(join(source.source.directory, "program.c")); + expect( + c + .subarray( + source.source.byte_offset, + source.source.byte_offset + source.source.byte_length, + ) + .toString(), + ).toBe("// Function: main @ 0x1000\nint main(void)\n{\n return 0;\n}\n\n"); + expect(await exportsOf(kuna)).toHaveLength(1); + expect(await client.close()).toEqual({ ok: true, value: null }); + + // A new session over a warm cache never takes the single-function path. + const again = open(); + expect( + await run(again, "procedure_pseudo_code", { procedure: "parse" }), + ).toMatchObject({ + result: "int parse(void)\n{\n return 1;\n}", + rawResult: { source: { kind: "project-export" } }, + }); + expect( + (await fakeKunaCalls(kuna)).filter(({ argv }) => argv[0] === "decompile"), + ).toHaveLength(1); + await again.close(); +}); + +it("lists, searches and resolves procedures from Kuna's inventory", async () => { + const { open } = await session("rea-kuna-inventory-"); + const client = open(); + + expect((await run(client, "list_procedures")).result).toEqual([ + { address: "0x1000", value: "main" }, + { address: "0x1040", value: "parse" }, + { address: "0x1080", value: "broken" }, + { address: "0x2000", value: "puts" }, + ]); + expect( + (await run(client, "search_procedures", { pattern: "^p", mode: "regex" })) + .result, + ).toEqual([ + { address: "0x1040", value: "parse" }, + { address: "0x2000", value: "puts" }, + ]); + expect( + (await run(client, "procedure_address", { procedure: "0X1040" })).result, + ).toBe("0x1040"); + for (const procedure of ["missing", "0x1044"]) + expect( + await client.execute("procedure_address", { procedure }), + ).toMatchObject({ ok: false, error: { _tag: "AnalysisInputError" } }); + await client.close(); +}); + +it("rejects a name that identifies several entries", async () => { + const { open } = await session("rea-kuna-ambiguous-", { + functions: [ + { name: "init", addr: "0x1000" }, + { name: "init", addr: "0x1040" }, + ], + }); + const client = open(); + + expect( + await client.execute("procedure_address", { procedure: "init" }), + ).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisInputError", + userMessage: expect.stringContaining("0x1000, 0x1040"), + }, + }); + await client.close(); +}); + +it("names function entries and the data Kuna's export labels", async () => { + const { open } = await session("rea-kuna-names-"); + const client = open(); + + expect( + (await run(client, "address_name", { address: "0x1040" })).result, + ).toBe("parse"); + expect( + (await run(client, "address_name", { address: "0x3000" })).result, + ).toBe("dat_3000"); + expect( + (await run(client, "address_name", { address: "0x3010" })).result, + ).toBe("g_table"); + const unnamed = await run(client, "address_name", { address: "0x9999" }); + expect(unnamed.result).toBeNull(); + expect(unnamed.limitations.at(-1)).toContain("not that no symbol exists"); + await client.close(); +}); + +it("reports a failed function and an import slot as unavailable text, not errors", async () => { + const { open } = await session("rea-kuna-unavailable-"); + const client = open(); + await run(client, "procedure_assembly", { procedure: "main" }); + + const broken = await run(client, "procedure_pseudo_code", { + procedure: "broken", + }); + expect(broken.result).toBeNull(); + expect(broken.limitations.at(-1)).toBe( + "Kuna could not decompile this function: Flow exceeded maximum instructions", + ); + const imported = await run(client, "procedure_pseudo_code", { + procedure: "puts", + }); + expect(imported.result).toBeNull(); + expect( + await client.execute("procedure_assembly", { procedure: "puts" }), + ).toMatchObject({ + ok: false, + error: { _tag: "AnalysisCapabilityUnavailableError" }, + }); + await client.close(); +}); + +it("builds a dossier with pseudocode and assembly, and the rest explicitly unavailable", async () => { + const { open } = await session("rea-kuna-dossier-"); + const client = open(); + + const dossier = await run(client, "analyze_function", { procedure: "main" }); + + expect(dossier.rawResult).toMatchObject({ + source: { + kind: "project-export", + files: [ + { + facet: "pseudocode", + file_sha256: expect.stringMatching(/^[0-9a-f]{64}$/u), + }, + { + facet: "assembly", + file_sha256: expect.stringMatching(/^[0-9a-f]{64}$/u), + }, + ], + }, + }); + const { source } = dossier.rawResult as { + source: { + directory: string; + files: { + facet: "pseudocode" | "assembly"; + file: string; + file_sha256: string; + byte_offset: number; + byte_length: number; + }[]; + }; + }; + for (const file of source.files) { + const bytes = await readFile(join(source.directory, file.file)); + expect(createHash("sha256").update(bytes).digest("hex")).toBe( + file.file_sha256, + ); + const text = bytes + .subarray(file.byte_offset, file.byte_offset + file.byte_length) + .toString("utf8"); + const facet = (dossier.result as Record)[file.facet]; + expect(text).toContain(Array.isArray(facet) ? facet.join("\n") : facet); + } + + expect(dossier.result).toMatchObject({ + procedure: { + address: "0x1000", + name: "main", + body: { available: false }, + }, + pseudocode: "int main(void)\n{\n return 0;\n}", + assembly: ["main: ; 0x1000", " 00001000: c3 RET"], + facet_availability: { + pseudocode: { available: true }, + assembly: { available: true }, + callers: { available: false }, + callees: { available: false }, + references: { available: false }, + cfg: { available: false }, + }, + }); + await client.close(); +}); + +it("rejects oversized cached text before buffering while retaining its full export", async () => { + const { cacheRoot, open } = await session("rea-kuna-text-capacity-", { + functions: [{ name: "main", addr: "0x1000" }], + cPadding: KUNA_FUNCTION_TEXT_BYTES, + }); + const client = open(); + try { + await run(client, "procedure_assembly", { procedure: "main" }); + for (const operation of [ + "procedure_pseudo_code", + "analyze_function", + ] as const) { + const result = await client.execute(operation, { procedure: "main" }); + expect(result.ok).toBe(false); + if (!result.ok) + expect(result.error).toMatchObject({ + _tag: "AnalysisResourceConstraintError", + resource: "capacity", + reportedLimits: { limit_bytes: KUNA_FUNCTION_TEXT_BYTES }, + }); + } + const [key] = await readdir(join(cacheRoot, "projects")); + const bytes = await readFile( + join(cacheRoot, "projects", key ?? "", "project", "program.c"), + ); + expect(bytes.length).toBeGreaterThan(KUNA_FUNCTION_TEXT_BYTES); + } finally { + await client.close(); + } +}); + +it("keeps answering pseudocode one function at a time when the export fails", async () => { + const { open } = await session("rea-kuna-export-failed-", { + project: "exit-failure", + }); + const client = open(); + + expect( + await client.execute("procedure_assembly", { procedure: "main" }), + ).toMatchObject({ + ok: false, + error: { + userMessage: expect.stringContaining( + "not in recognized object file format", + ), + }, + }); + const pseudocode = await run(client, "procedure_pseudo_code", { + procedure: "main", + }); + expect(pseudocode.rawResult).toMatchObject({ + source: { kind: "single-function" }, + }); + expect(pseudocode.limitations.at(-1)).toContain("The project export failed"); + await client.close(); +}); + +it("stops waiting for the export when the caller cancels, without abandoning it", async () => { + const { open } = await session("rea-kuna-wait-cancel-", { delayMs: 1500 }); + const client = open(); + const caller = new AbortController(); + + const waiting = client.execute( + "procedure_assembly", + { procedure: "main" }, + { signal: caller.signal }, + ); + setTimeout(() => caller.abort(), 200); + expect(await waiting).toMatchObject({ + ok: false, + error: { _tag: "AnalysisCancelledError" }, + }); + // The session's export continues and serves the next request. + expect( + (await run(client, "procedure_assembly", { procedure: "main" })).rawResult, + ).toMatchObject({ source: { kind: "project-export" } }); + await client.close(); +}); + +it("stops the export it alone wanted before reporting itself closed", async () => { + const { kuna, cacheRoot, open } = await session("rea-kuna-close-", { + project: "hang", + }); + const client = open(); + await run(client, "procedure_pseudo_code", { procedure: "main" }); + await expect.poll(() => exportsOf(kuna)).toHaveLength(1); + const pid = (await exportsOf(kuna))[0]?.pid ?? 0; + + expect(await client.close()).toEqual({ ok: true, value: null }); + + expect(() => process.kill(pid, 0)).toThrow(); + expect(await readdir(join(cacheRoot, "staging"))).toEqual([]); + expect(await client.close()).toEqual({ ok: true, value: null }); + expect( + await client.execute("procedure_pseudo_code", { procedure: "main" }), + ).toMatchObject({ + ok: false, + error: { _tag: "AnalysisCapabilityUnavailableError" }, + }); +}); + +it.each([false, true])( + "drains an active single-function request before close (caller signal: %s)", + async (withCallerSignal) => { + const { kuna, open } = await session("rea-kuna-close-active-", { + project: "hang", + decompile: "hang", + }); + const client = open(); + const caller = new AbortController(); + const request = client.execute( + "procedure_pseudo_code", + { procedure: "main" }, + withCallerSignal ? { signal: caller.signal } : {}, + ); + try { + await expect + .poll(async () => + (await fakeKunaCalls(kuna)).filter( + ({ argv }) => argv[0] === "decompile", + ), + ) + .toHaveLength(1); + const call = (await fakeKunaCalls(kuna)).find( + ({ argv }) => argv[0] === "decompile", + ); + expect(await client.close()).toEqual({ ok: true, value: null }); + expect(() => process.kill(call?.pid ?? 0, 0)).toThrow(); + expect(await request).toMatchObject({ + ok: false, + error: { _tag: "AnalysisCancelledError" }, + }); + } finally { + caller.abort(); + await request; + await client.close(); + } + }, +); + +it("refuses another document, an unsupported operation and a replaced executable", async () => { + const { kuna, open } = await session("rea-kuna-refusals-"); + const client = open(); + + expect( + await client.execute("procedure_pseudo_code", { + procedure: "main", + document: "/other/program", + }), + ).toMatchObject({ ok: false, error: { _tag: "AnalysisInputError" } }); + expect(await client.execute("xrefs", { address: "0x1000" })).toMatchObject({ + ok: false, + error: { _tag: "AnalysisCapabilityUnavailableError" }, + }); + await client.close(); + + await appendFile(kuna, "\n// replaced\n"); + await setFakeKunaBehavior(kuna, { functions: FUNCTIONS }); + const replaced = open(); + expect( + await replaced.execute("procedure_pseudo_code", { procedure: "main" }), + ).toMatchObject({ + ok: false, + error: { + _tag: "ProviderAdapterError", + userMessage: expect.stringContaining( + "changed after this target was opened", + ), + }, + }); + await replaced.close(); +}); + +/** + * Launch for real, but have cleanup of the selected subcommand report an + * unproven stop until `release` is called, as a lost ownership proof would. + */ +const unprovenCleanupLauncher = (subcommand: string) => { + let proven = false; + const launcher: KunaProcessLauncher = async (options) => { + const spawned = await spawnOwnedProviderProcess(options); + if (options.arguments[0] !== subcommand) return spawned; + return { + ...spawned, + cleanup: async () => { + const cleaned = await cleanupOwnedProcessGroup(spawned.ownership); + return proven + ? cleaned + : { cleaned: false, reason: `${subcommand} stop unproven` }; + }, + }; + }; + return { launcher, release: () => (proven = true) }; +}; + +it("retains an active request's workspace when close cannot prove its stop", async () => { + const unproven = unprovenCleanupLauncher("decompile"); + const { kuna, open } = await session( + "rea-kuna-close-active-unproven-", + { project: "hang", decompile: "hang" }, + {}, + unproven.launcher, + ); + const client = open(); + const caller = new AbortController(); + const request = client.execute( + "procedure_pseudo_code", + { procedure: "main" }, + { signal: caller.signal }, + ); + try { + await expect + .poll(async () => + (await fakeKunaCalls(kuna)).filter( + ({ argv }) => argv[0] === "decompile", + ), + ) + .toHaveLength(1); + const closed = await client.close(); + expect(closed).toMatchObject({ + ok: false, + error: { cleanupIncomplete: true }, + }); + expect(await request).toMatchObject({ + ok: false, + error: { cleanupIncomplete: true }, + }); + const input = (await fakeKunaCalls(kuna)).find( + ({ argv }) => argv[0] === "decompile", + )?.argv[1]; + expect(await readFile(input ?? "", "utf8")).toBe("program bytes"); + } finally { + caller.abort(); + await request; + unproven.release(); + expect(await client.close()).toEqual({ ok: true, value: null }); + } +}); + +it("keeps reporting a failed close and succeeds once a retry proves the stop", async () => { + const unproven = unprovenCleanupLauncher("decompile"); + const { open } = await session( + "rea-kuna-close-retry-", + { project: "hang", decompile: "hang" }, + { REA_KUNA_TIMEOUT_MS: "300" }, + unproven.launcher, + ); + const client = open(); + + expect( + await client.execute("procedure_pseudo_code", { procedure: "main" }), + ).toMatchObject({ ok: false, error: { cleanupIncomplete: true } }); + const first = await client.close(); + expect(first).toMatchObject({ + ok: false, + error: { diagnostics: { reason: "decompile stop unproven" } }, + }); + // A retry that still cannot prove the stop keeps failing. + expect((await client.close()).ok).toBe(false); + + unproven.release(); + expect(await client.close()).toEqual({ ok: true, value: null }); + expect(await client.close()).toEqual({ ok: true, value: null }); +}); + +it("reports an abandoned export's unproven cleanup from close", async () => { + const unproven = unprovenCleanupLauncher("decompile-project"); + const { kuna, open } = await session( + "rea-kuna-close-export-", + { project: "hang" }, + {}, + unproven.launcher, + ); + const client = open(); + await run(client, "procedure_pseudo_code", { procedure: "main" }); + await expect.poll(() => exportsOf(kuna)).toHaveLength(1); + + expect(await client.close()).toMatchObject({ + ok: false, + error: { diagnostics: { reason: "decompile-project stop unproven" } }, + }); + unproven.release(); + expect(await client.close()).toEqual({ ok: true, value: null }); +}); + +it("retries the abandoned export's file cleanup before reporting close success", async () => { + const launcher: KunaProcessLauncher = async (options) => { + const spawned = await spawnOwnedProviderProcess(options); + if (options.arguments[0] !== "decompile-project") return spawned; + return { + ...spawned, + cleanup: async () => { + const stopped = await cleanupOwnedProcessGroup(spawned.ownership); + // The child is stopped, but its private input cannot yet be removed. + await chmod(options.cwd ?? "", 0o500); + return stopped; + }, + }; + }; + const { kuna, cacheRoot, open } = await session( + "rea-kuna-close-export-files-", + { project: "hang" }, + {}, + launcher, + ); + const client = open(); + try { + await run(client, "procedure_pseudo_code", { procedure: "main" }); + await expect.poll(() => exportsOf(kuna)).toHaveLength(1); + expect(await client.close()).toMatchObject({ + ok: false, + error: { cleanupIncomplete: true }, + }); + expect((await client.close()).ok).toBe(false); + } finally { + const restore = async (directory: string): Promise => { + await chmod(directory, 0o700); + for (const entry of await readdir(directory, { withFileTypes: true })) + if (entry.isDirectory()) await restore(join(directory, entry.name)); + }; + await restore(cacheRoot); + expect(await client.close()).toEqual({ ok: true, value: null }); + } + expect(await readdir(join(cacheRoot, "staging"))).toEqual([]); + expect(await readdir(join(cacheRoot, "trash"))).toEqual([]); +}); + +it("refuses to launch an executable replaced after the target was opened", async () => { + const { kuna, open } = await session("rea-kuna-swapped-", { + project: "exit-failure", + }); + const client = open(); + await run(client, "procedure_pseudo_code", { procedure: "main" }); + + await appendFile(kuna, "\n// replaced mid-session\n"); + expect( + await client.execute("procedure_pseudo_code", { procedure: "parse" }), + ).toMatchObject({ + ok: false, + error: { + _tag: "ProviderAdapterError", + userMessage: expect.stringContaining( + "changed after this target was opened", + ), + }, + }); + await client.close(); +}); + +it("recovers when another server evicts the export this session adopted", async () => { + const { cacheRoot, open } = await session("rea-kuna-evicted-"); + const client = open(); + await run(client, "procedure_assembly", { procedure: "main" }); + const [key] = await readdir(join(cacheRoot, "projects")); + await mkdir(join(cacheRoot, "trash"), { recursive: true }); + await rename( + join(cacheRoot, "projects", key ?? ""), + join(cacheRoot, "trash", "elsewhere"), + ); + + // Pseudocode falls back while the export rebuilds; assembly waits for it. + const pseudocode = await run(client, "procedure_pseudo_code", { + procedure: "parse", + }); + expect(pseudocode.result).toContain("parse"); + const assembly = await run(client, "procedure_assembly", { + procedure: "parse", + }); + expect(assembly.rawResult).toMatchObject({ + source: { kind: "project-export" }, + }); + expect(await readdir(join(cacheRoot, "projects"))).toEqual([key]); + await client.close(); +}); + +it("reports a failed export as failed rather than building", async () => { + const { open } = await session("rea-kuna-failed-state-", { + project: "exit-failure", + }); + const client = open(); + await client.execute("procedure_assembly", { procedure: "main" }); + + expect( + (await run(client, "procedure_pseudo_code", { procedure: "main" })) + .rawResult, + ).toMatchObject({ + project_export: { + state: "failed", + reason: expect.stringContaining("not in recognized object file format"), + }, + }); + await client.close(); +}); + +it("lets one caller stop waiting for the shared inventory without failing another", async () => { + const { open } = await session("rea-kuna-shared-inventory-", { + inventoryDelayMs: 700, + }); + const client = open(); + const first = new AbortController(); + + const cancelled = client.execute( + "list_procedures", + {}, + { signal: first.signal }, + ); + const waiting = client.execute("search_procedures", { pattern: "main" }); + setTimeout(() => first.abort(), 100); + + expect(await cancelled).toMatchObject({ + ok: false, + error: { _tag: "AnalysisCancelledError" }, + }); + expect(await waiting).toMatchObject({ + ok: true, + value: { result: [{ address: "0x1000", value: "main" }] }, + }); + await client.close(); +}); + +it("leaves no private copy behind when workspace setup fails", async () => { + const { target, open } = await session("rea-kuna-workspace-leak-", { + project: "hang", + }); + const runtime = await createTestTempDirectory("rea-kuna-runtime-"); + const previous = process.env.TMPDIR; + process.env.TMPDIR = runtime; + try { + const client = open(); + // A vanished target makes the copy itself throw mid-setup. + await rename(target.path, `${target.path}.moved`); + + expect( + await client.execute("procedure_pseudo_code", { procedure: "main" }), + ).toMatchObject({ ok: false }); + expect(await readdir(runtime)).toEqual([]); + expect(await client.close()).toEqual({ ok: true, value: null }); + } finally { + if (previous === undefined) delete process.env.TMPDIR; + else process.env.TMPDIR = previous; + } +}); diff --git a/tests/boundary/providers/kuna/kunaRegexSearch.test.ts b/tests/boundary/providers/kuna/kunaRegexSearch.test.ts new file mode 100644 index 0000000000..f62f96ef0d --- /dev/null +++ b/tests/boundary/providers/kuna/kunaRegexSearch.test.ts @@ -0,0 +1,66 @@ +import { spawnSync } from "node:child_process"; +import { resolve } from "node:path"; +import { pathToFileURL } from "node:url"; + +import { expect, it as test } from "vitest"; + +import { session } from "./kunaSessionFixture.js"; + +const it = test.skipIf(process.platform === "win32"); + +it.each(["cancel", "deadline"])( + "keeps regex %s bounded and the same session usable", + async (mode) => { + const { kuna, target, cacheRoot, profile } = await session( + "rea-kuna-regex-", + { + functions: [ + { name: `${"a".repeat(40)}!`, addr: "0x1000" }, + { name: "Δelta", addr: "0x1040" }, + ], + }, + ); + // A parent watchdog bounds this regression even if matching freezes the + // child's event loop. All Kuna commands finish before adversarial matching. + const script = ` +import assert from "node:assert/strict"; +import { KunaProvider } from ${JSON.stringify(pathToFileURL(resolve("dist/kuna/KunaProvider.js")).href)}; +const provider = new KunaProvider(${JSON.stringify({ REA_KUNA_COMMAND: kuna, REA_KUNA_CACHE_DIR: cacheRoot })}); +const client = provider.createClient(${JSON.stringify(target)}, ${JSON.stringify(profile)}); +try { + assert.equal((await client.execute("procedure_assembly", {procedure:"0x1000"})).ok, true); + const options = ${mode === "cancel" ? "{signal:AbortSignal.timeout(100)}" : "{}"}; + const matched = await client.execute("search_procedures", {pattern:"^(a+)+$",mode:"regex"}, options); + assert.equal(matched.ok, false); + assert.equal(matched.error._tag, ${JSON.stringify(mode === "cancel" ? "AnalysisCancelledError" : "AnalysisResourceConstraintError")}); + if (${JSON.stringify(mode)} === "deadline") assert.equal(matched.error.reportedLimits.matching_deadline_ms, 5000); + assert.equal((await client.execute("health", {})).ok, true); + const unicode = await client.execute("search_procedures", {pattern:"^δ",mode:"regex"}); + assert.equal(unicode.ok, true); + assert.deepEqual(unicode.value.result, [{address:"0x1040",value:"Δelta"}]); + const invalid = await client.execute("search_procedures", {pattern:"[",mode:"regex"}); + assert.equal(invalid.ok, false); + assert.equal(invalid.error._tag, "AnalysisInputError"); + const literal = await client.execute("search_procedures", {pattern:"^(a+)+$"}); + assert.equal(literal.ok, true); + assert.deepEqual(literal.value.result, []); +} finally { + assert.equal((await client.close()).ok, true); +} +console.log("bounded and healthy"); +`; + const result = spawnSync( + process.execPath, + ["--input-type=module", "-e", script], + { + encoding: "utf8", + timeout: 25_000, + killSignal: "SIGKILL", + }, + ); + expect(result.error).toBeUndefined(); + expect(result.stderr).toBe(""); + expect(result.status).toBe(0); + expect(result.stdout.trim()).toBe("bounded and healthy"); + }, +); diff --git a/tests/boundary/providers/kuna/kunaSessionFixture.ts b/tests/boundary/providers/kuna/kunaSessionFixture.ts new file mode 100644 index 0000000000..36f904c112 --- /dev/null +++ b/tests/boundary/providers/kuna/kunaSessionFixture.ts @@ -0,0 +1,78 @@ +import { createHash } from "node:crypto"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import type { AnalysisClient } from "../../../../src/application/AnalysisProvider.js"; +import type { BinaryTarget } from "../../../../src/domain/binaryTargetTypes.js"; +import type { JsonValue } from "../../../../src/domain/jsonValue.js"; +import type { KunaProcessLauncher } from "../../../../src/kuna/KunaCommand.js"; +import { KunaProvider } from "../../../../src/kuna/KunaProvider.js"; +import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; +import { + fakeKunaCalls, + writeFakeKuna, + type FakeKunaBehavior, +} from "./fakeKuna.js"; + +export const FUNCTIONS = [ + { name: "main", addr: "0x1000" }, + { name: "parse", addr: "0x1040" }, + { + name: "broken", + addr: "0x1080", + error: "Flow exceeded maximum instructions", + }, +] as const; + +export const session = async ( + prefix: string, + behavior: FakeKunaBehavior = {}, + environment: Readonly> = {}, + launcher?: KunaProcessLauncher, +) => { + const root = await createTestTempDirectory(prefix); + const kuna = await writeFakeKuna(root, { functions: FUNCTIONS, ...behavior }); + const targetPath = join(root, "program"); + await writeFile(targetPath, "program bytes"); + const target: BinaryTarget = { + path: targetPath, + sha256: createHash("sha256").update("program bytes").digest("hex"), + kind: "executable", + format: "elf", + architecture: "x86_64", + availableArchitectures: ["x86_64"], + }; + const cacheRoot = join(root, "cache"); + const provider = new KunaProvider( + { REA_KUNA_COMMAND: kuna, REA_KUNA_CACHE_DIR: cacheRoot, ...environment }, + { + platform: process.platform, + ...(launcher === undefined ? {} : { launcher }), + }, + ); + const profile = await provider.resolveAnalysisProfile(target); + if (!profile.ok || profile.value.profile === null) + throw new Error("profile did not resolve"); + const resolved = profile.value.profile; + const open = () => provider.createClient(target, resolved); + return { root, kuna, target, cacheRoot, provider, profile: resolved, open }; +}; + +export const run = async ( + client: AnalysisClient, + operation: Parameters[0], + parameters: Readonly> = {}, + signal?: AbortSignal, +) => { + const result = await client.execute( + operation, + parameters, + signal === undefined ? {} : { signal }, + ); + if (!result.ok) throw result.error; + return result.value; +}; + +export const exportsOf = async (kuna: string) => + (await fakeKunaCalls(kuna)).filter( + ({ argv }) => argv[0] === "decompile-project", + ); diff --git a/tests/boundary/providers/kuna/kunaVerifier.test.ts b/tests/boundary/providers/kuna/kunaVerifier.test.ts new file mode 100644 index 0000000000..a83a327405 --- /dev/null +++ b/tests/boundary/providers/kuna/kunaVerifier.test.ts @@ -0,0 +1,73 @@ +import { spawnSync } from "node:child_process"; +import { mkdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect, it as test } from "vitest"; + +import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; +import { fakeKunaCalls, writeFakeKuna } from "./fakeKuna.js"; + +const it = test.skipIf(process.platform === "win32"); + +it.each([ + { populated: false, names: "main", reason: "No caller-built targets" }, + { populated: true, names: "typo", reason: "No caller-built targets" }, + { + populated: true, + names: "main,typo", + reason: "Requested functions were not found", + }, +])( + "rejects an unverified caller matrix: $names, populated=$populated", + async ({ populated, names, reason }) => { + const root = await createTestTempDirectory("rea-kuna-verifier-selection-"); + const kuna = await writeFakeKuna(root); + const targets = join(root, "targets"); + await mkdir(targets); + if (populated) { + const flavor = join(targets, "linux"); + await mkdir(flavor); + await writeFile(join(flavor, "program"), "not analyzed"); + await writeFile(join(flavor, "program.stripped"), "not analyzed"); + } + // Only symbol discovery is simulated; no provider workflow should start. + const nm = join(root, "nm"); + await writeFile( + nm, + `#!${process.execPath}\nprocess.stdout.write("00001000 T main\\n");\n`, + { + mode: 0o755, + }, + ); + const result = spawnSync( + process.execPath, + [ + "scripts/verify-real-kuna.mjs", + "--targets", + targets, + "--functions", + names, + "--compare-ghidra", + ], + { + encoding: "utf8", + timeout: 10_000, + env: { + PATH: process.env.PATH, + HOME: root, + REA_KUNA_COMMAND: kuna, + REA_NM: nm, + REA_CC: join(root, "compiler-must-not-run"), + GHIDRA_INSTALL_DIR: join(root, "ghidra-must-not-run"), + }, + }, + ); + expect(result.error).toBeUndefined(); + expect(result.status).toBe(1); + expect(result.stderr).toContain(reason); + expect(result.stderr).not.toContain("compiler-must-not-run"); + expect((await fakeKunaCalls(kuna)).map(({ argv }) => argv)).toEqual([ + ["--version"], + ]); + }, +);