From 2995ec6910ead327829378f5ab1b9172ecce33fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=AE=B0=E5=BE=97=E5=90=83=E4=B8=AD=E9=A5=AD?= <17373302683@163.com> Date: Tue, 13 Oct 2026 18:23:17 +0800 Subject: [PATCH] fix(ghidra): resolve file offsets in decompile (#2101) * fix(native): expose loaded image bases in overviews * test(ghidra): retain resources when verifier cleanup fails * docs(ghidra): spell out file-offset address mapping * fix(ci): separate and retry NativeAOT dependency restoration * fix(test): resolve CI script file URLs on Windows * fix(native): retain conflict observations and actionable CLI guidance * fix(fixtures): keep NativeAOT retry logs credential-safe * fix(fixtures): redact authenticated URLs at diagnostic creation * fix(ghidra): restrict source-coordinate observations to decompilation * fix(cli): identify conflicting file-offset selectors accurately * fix(ghidra): bind file offsets to admitted target sources * fix(ghidra): preserve resolved file-offset failure context * test(ghidra): verify ELF address and file-offset round trips * test(nativeaot): align diagnostic regressions with fixture host support --- .../references/native-and-artifacts.md | 2 + .github/workflows/ci.yml | 2 + .github/workflows/nativeaot-fixtures.yml | 3 +- bridge/ghidra/ReaGhidraBridge.java | 127 ++++++- docs/ghidra-provider.md | 74 +++++ docs/testing.md | 38 +++ package.json | 1 + scripts/ci/scopes.mjs | 1 + scripts/generate-nativeaot-fixtures.mjs | 29 +- scripts/lib/nativeaot-build.d.mts | 25 ++ scripts/lib/nativeaot-build.mjs | 60 ++++ scripts/lib/real-ghidra-file-offsets.mjs | 264 +++++++++++++++ .../verify-real-ghidra-address-context.mjs | 313 ++++++++++++++++++ scripts/verify-real-ghidra-windows.mjs | 6 + src/application/EnhancedTools.ts | 24 +- src/application/InvestigationProviders.ts | 10 +- src/cli/coreBinaryCommands.ts | 20 +- src/contracts/officialToolContracts.ts | 2 +- src/contracts/toolOutputSchemaGroups.ts | 4 +- src/domain/hopperValues.test.ts | 14 - src/domain/hopperValues.ts | 26 +- src/ghidra/GhidraAnalysisProfile.ts | 4 + src/ghidra/GhidraDoctor.test.ts | 13 +- src/ghidra/GhidraFunctionValues.test.ts | 24 ++ src/ghidra/GhidraFunctionValues.ts | 38 ++- src/ghidra/GhidraProviderClient.ts | 6 + .../boundary/cli/analysisSelectorCli.test.ts | 32 ++ .../boundary/filesystem/ciChangeScope.test.ts | 8 +- ...4-uses-complete-inventories-inline.test.ts | 18 +- .../mcp/nativeAddressContextMcp.test.ts | 121 +++++++ tests/conformance/c/pie-address-context.c | 7 + .../ghidra/ReaFileOffsetSourceProbe.java | 118 +++++++ .../evaluation/package/nativeAotBuild.test.ts | 90 +++++ .../package/nativeAotDiagnostics.test.ts | 150 +++++++++ 34 files changed, 1588 insertions(+), 86 deletions(-) create mode 100644 scripts/lib/nativeaot-build.d.mts create mode 100644 scripts/lib/nativeaot-build.mjs create mode 100644 scripts/lib/real-ghidra-file-offsets.mjs create mode 100644 scripts/verify-real-ghidra-address-context.mjs create mode 100644 tests/boundary/mcp/nativeAddressContextMcp.test.ts create mode 100644 tests/conformance/c/pie-address-context.c create mode 100644 tests/conformance/ghidra/ReaFileOffsetSourceProbe.java create mode 100644 tests/evaluation/package/nativeAotBuild.test.ts create mode 100644 tests/evaluation/package/nativeAotDiagnostics.test.ts diff --git a/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md b/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md index 71fe3e9508..6a942c73cf 100644 --- a/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md +++ b/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md @@ -119,6 +119,8 @@ loaded bytes, file mappings, relocations and the entry against the immutable snapshot. Keep `mismatch` and `unsupported` explicit. Verified import covers the reported static image, not runtime DOS or PC-98 hardware behavior. +For Ghidra decompilation, `procedure_pseudo_code` also accepts `procedure: "file-offset:0x..."`; CLI `rea decompile --file-offset=0x... --provider ghidra` resolves the source byte offset internally. An unresolved 0x-prefixed numeric procedure selector also falls back to a unique file mapping. Existing symbols and analysis addresses win; multiple mappings fail as ambiguous. Preserve the returned file-offset and address locations and raw selection observations. ELF virtual addresses are distinct from file offsets; never assume an arbitrary offset plus image base is an analysis address. + Use `read_bytes` for initialized analysis memory and `address_to_file_offset` to anchor one observed address to original file bytes. Loader fixups can change a word; a source offset does not imply byte equality. Complete function body ranges diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7faf9e4ee0..6a9977f0d6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -436,6 +436,8 @@ jobs: name: windows-native-controls path: native/windows/build/ - run: npm run verify:windows-native:artifact + - name: Verify NativeAOT diagnostic applicability on this host + run: npm run test:focused -- tests/evaluation/package/nativeAotDiagnostics.test.ts - name: Resolve npm cache before verifier HOME isolation id: npm-cache run: | diff --git a/.github/workflows/nativeaot-fixtures.yml b/.github/workflows/nativeaot-fixtures.yml index e819ca5fbc..ce7dc1ea18 100644 --- a/.github/workflows/nativeaot-fixtures.yml +++ b/.github/workflows/nativeaot-fixtures.yml @@ -12,7 +12,8 @@ jobs: name: Build Windows x64 NativeAOT fixture # .NET 8.0.22's bundled linker discovery supports VS 2019/2022. runs-on: windows-2022 - timeout-minutes: 15 + # Two bounded restore attempts and one compile can each take five minutes. + timeout-minutes: 20 env: DOTNET_CLI_TELEMETRY_OPTOUT: "1" DOTNET_PROCESSOR_COUNT: "1" diff --git a/bridge/ghidra/ReaGhidraBridge.java b/bridge/ghidra/ReaGhidraBridge.java index e45f971723..02fc0f148a 100644 --- a/bridge/ghidra/ReaGhidraBridge.java +++ b/bridge/ghidra/ReaGhidraBridge.java @@ -106,6 +106,8 @@ public final class ReaGhidraBridge extends HeadlessScript { private static final int MAX_HIGH_PCODE_INPUTS_PER_OP = 64; private static final int MAX_HIGH_PCODE_DEF_USE_EDGES = 12000; private static final Gson GSON = new GsonBuilder().serializeNulls().create(); + private final Map targetSourceIdentity = new IdentityHashMap<>(); + private String admittedTargetSha256; private static final Set DESCRIPTOR_KEYS = Set.of( "transport", "endpoint_path", @@ -207,6 +209,7 @@ public final class ReaGhidraBridge extends HeadlessScript { "Ghidra imported-byte digest does not match the admitted target" ); } + admittedTargetSha256 = descriptor.targetSha256; // GhidraScript wraps run() in a transaction. End it before serving so // each edit owns a real outer transaction and can roll back immediately. end(true); @@ -633,7 +636,7 @@ public final class ReaGhidraBridge extends HeadlessScript { return result; } - private JsonObject addressToFileOffset(JsonObject params) { + private JsonObject addressToFileOffset(JsonObject params) throws Exception { requireKeys(params, Set.of("document", "address")); requireDocument(params); Address address = requireAddress(params, "address"); @@ -643,14 +646,15 @@ public final class ReaGhidraBridge extends HeadlessScript { Long fileOffset = null; int mappings = 0; for (MemoryBlockSourceInfo source : block.getSourceInfos()) { - if (!source.contains(address) || source.getFileBytes().isEmpty()) continue; + if (!source.contains(address) || source.getFileBytes().isEmpty() || + !isTargetSource(source.getFileBytes().get())) continue; long offset = source.getFileBytesOffset(address); if (offset < 0) continue; fileOffset = offset; mappings++; } if (mappings != 1) - throw new RequestFailure("invalid_request", "Address has " + mappings + " authoritative source mappings: " + canonicalAddress(address)); + throw new RequestFailure("invalid_request", "Address has " + mappings + " authoritative target-file mappings: " + canonicalAddress(address)); JsonObject result = new JsonObject(); result.addProperty("address", canonicalAddress(address)); result.addProperty("file_offset", fileOffset); @@ -673,14 +677,96 @@ public final class ReaGhidraBridge extends HeadlessScript { private JsonElement procedurePseudocode(JsonObject params) throws Exception { requireKeys(params, Set.of("document", "procedure")); requireDocument(params); - Function function = resolveProcedure(requireString(params, "procedure")); - String value = decompiledText(decompile(function)); - return functionObservation( - value == null ? JsonNull.INSTANCE : GSON.toJsonTree(value), - noReturnCallLimitations( - currentProgram.getListing().getInstructions(function.getBody(), true) - ) - ); + String selector = requireString(params, "procedure"); + JsonObject selection = new JsonObject(); + Function function = resolveDecompileProcedure(selector, selection); + try { + String value = decompiledText(decompile(function)); + JsonObject result = functionObservation( + value == null ? JsonNull.INSTANCE : GSON.toJsonTree(value), + noReturnCallLimitations( + currentProgram.getListing().getInstructions(function.getBody(), true) + ) + ); + if (selection.size() > 0) result.add("selection", selection); + return result; + } catch (RequestFailure failure) { + if (selection.size() == 0) throw failure; + // Preserve completed resolution facts without changing the backend's + // failure code or suggesting that the successful mapping was invalid. + throw new RequestFailure(failure.code, "Selector " + selector + + " mapped to analysis address " + selection.get("mapped_address").getAsString() + + " and procedure " + canonicalAddress(function.getEntryPoint()) + + "; " + failure.getMessage()); + } + } + + private Function resolveDecompileProcedure(String selector, JsonObject selection) throws Exception { + String prefix = "file-offset:"; + boolean explicitOffset = selector.startsWith(prefix); + if (!explicitOffset) { + try { + return resolveProcedure(selector); + } catch (RequestFailure failure) { + // Existing names/addresses win. Only an unresolved numeric input + // can fall back to source coordinates; ambiguity never falls back. + if (!failure.code.equals("not_found") || !selector.matches("0x[0-9a-fA-F]+")) + throw failure; + } + } + String coordinate = explicitOffset ? selector.substring(prefix.length()) : selector; + if (!coordinate.matches("0x[0-9a-fA-F]+")) + throw new RequestFailure("invalid_request", "File offsets require hexadecimal file-offset:0x... syntax"); + BigInteger parsed = new BigInteger(coordinate.substring(2), 16); + if (parsed.compareTo(BigInteger.valueOf(9007199254740991L)) > 0) + throw new RequestFailure("invalid_request", "File offset exceeds the exact JSON integer range"); + long offset = parsed.longValueExact(); + monitor.checkCancelled(); + // Ghidra's convenience lookup spans every FileBytes and uses coordinates + // local to each object. Only a complete, digest-matched target source + // establishes that the caller's offset belongs to the admitted file. + Set
mappedAddresses = new java.util.LinkedHashSet<>(); + for (MemoryBlock block : currentProgram.getMemory().getBlocks()) { + for (MemoryBlockSourceInfo source : block.getSourceInfos()) { + monitor.checkCancelled(); + FileBytes backing = source.getFileBytes().orElse(null); + if (backing == null || !isTargetSource(backing)) continue; + Address mapped = source.locateAddressForFileOffset(offset); + if (mapped != null) mappedAddresses.add(mapped); + } + } + List
candidates = new ArrayList<>(mappedAddresses); + if (candidates.size() > 1) + throw new RequestFailure("ambiguous", "File offset " + coordinate + + " has multiple loaded mappings: " + candidates.stream().map(ReaGhidraBridge::canonicalAddress).toList() + + "; select an explicit analysis address"); + if (candidates.isEmpty()) + throw new RequestFailure("not_found", "File offset " + coordinate + + " has no loaded source mapping for the admitted target file; Ghidra image base: " + canonicalAddress(currentProgram.getImageBase()) + + ". Inspect mappings with rea inspect-native-load-image --provider ghidra --json or MCP inspect_native_load_image."); + Address mapped = candidates.get(0); + Function function = functionAtOrContaining(mapped); + if (function == null) + throw new RequestFailure("not_found", "File offset " + coordinate + + " maps to analysis address " + canonicalAddress(mapped) + + ", but no procedure starts at or contains that address in the current Ghidra analysis. " + + "Choose an observed procedure with CLI rea search . --kind procedures --mode regex --provider ghidra --json or MCP list_procedures."); + selection.addProperty("file_offset", offset); + selection.addProperty("mapped_address", canonicalAddress(mapped)); + selection.addProperty("procedure_address", canonicalAddress(function.getEntryPoint())); + return function; + } + + private boolean isTargetSource(FileBytes source) throws Exception { + Boolean observed = targetSourceIdentity.get(source); + if (observed != null) return observed; + // Original FileBytes are immutable even when analysis memory is edited. + // Partial/auxiliary sources remain inspectable but cannot establish an + // original-target offset from their object-local coordinates alone. + boolean matched = source.getFileOffset() == 0 && + constantTimeEquals(hashFileBytes(source, false), admittedTargetSha256); + targetSourceIdentity.put(source, matched); + return matched; } private JsonElement procedureAssembly(JsonObject params) throws Exception { @@ -1833,14 +1919,25 @@ public final class ReaGhidraBridge extends HeadlessScript { Address address = explicitAddress ? parseReaAddress(value) : tryParseAddress(value); if (address == null) throw new RequestFailure("not_found", "Unknown Ghidra procedure name or address: " + value); - Function function = currentProgram.getFunctionManager().getFunctionAt(address); - if (function == null && address.isMemoryAddress()) - function = currentProgram.getFunctionManager().getFunctionContaining(address); + Function function = functionAtOrContaining(address); if (function == null) - throw new RequestFailure("not_found", "No procedure exists at the requested address: " + value); + // Other procedure operations retain their analysis-address semantics. + throw new RequestFailure("not_found", "No procedure exists at the requested address: " + value + + "; Ghidra image base: " + canonicalAddress(currentProgram.getImageBase()) + + ". Choose an observed analysis address. CLI: rea search . --kind procedures --mode regex --provider ghidra --json; " + + "MCP: list_procedures. Inspect the loaded base and source mappings with CLI rea analyze --provider ghidra --json " + + "and rea inspect-native-load-image --provider ghidra --json, or MCP binary_overview and inspect_native_load_image. " + + "Repeat the original target options. For decompile, select source coordinates explicitly with --file-offset=0x... or MCP procedure file-offset:0x...."); return function; } + private Function functionAtOrContaining(Address address) { + Function function = currentProgram.getFunctionManager().getFunctionAt(address); + return function == null && address.isMemoryAddress() + ? currentProgram.getFunctionManager().getFunctionContaining(address) + : function; + } + private JsonObject nativeApiBoundary( Function function, DecompileResults decompilation diff --git a/docs/ghidra-provider.md b/docs/ghidra-provider.md index a79afecbd9..32edbd3207 100644 --- a/docs/ghidra-provider.md +++ b/docs/ghidra-provider.md @@ -41,6 +41,80 @@ does not prove text is absent. Both list and search Evidence carry this limitation. When an address is known, use `read_bytes` and `xrefs` to inspect bytes and references. +## Loaded addresses and file offsets + +`rea analyze --provider ghidra --json` and MCP `binary_overview` report +the observed loaded `image_base` in `normalized_result`. They retain each +segment's reported `image_base` and `address_space`. A provider that does not +report a base, or whose segments disagree, omits the overview field, following +the existing contract on main. REA does not substitute zero or the first +segment's address. All reported bases and address spaces remain visible in the +complete segment inventory, including conflicting observations. + +`function` and most MCP procedure selectors use procedure names or +provider analysis addresses. Ghidra `decompile` / MCP `procedure_pseudo_code` +also accept source-file byte offsets. Existing symbols and analysis addresses +take precedence; a `0x`-prefixed number that resolves to no procedure falls back +to its observed source mapping. Use `--file-offset=0x...` on the CLI or +`procedure: "file-offset:0x..."` through MCP to select file coordinates explicitly. +Multiple mapping candidates fail as ambiguous rather than selecting a bank or +address space. A successful conversion retains the file offset, mapped address +and procedure entry in raw Evidence and `locations`. + +If an offset maps to loaded target bytes but no procedure starts at or contains +the mapped address, the failure preserves both the original file offset and +analysis address. This differs from an unmapped offset and does not recommend +retrying the same offset syntax. +If the function resolves but the backend fails or cancels decompilation, the +failure retains the selector, mapped address and procedure entry without +changing the backend failure code. + +Both mapping directions require a complete original `FileBytes` source whose +SHA-256 matches the admitted target. Auxiliary files and partial source objects +remain visible in the load-image inventory but do not establish offsets in the +original target. Ghidra's object-local offsets are not silently treated as +original-file coordinates. Original-source identity is checked once per source +object per session; analysis-memory edits do not change the original bytes. + +ELF virtual addresses and file offsets are +different coordinates. Ghidra can rebase a PIE ELF during import: in the +source-owned verification fixture, ELF entry `0x1013` becomes Ghidra procedure +`0x101013` with image base `0x100000`. That example is not a general conversion +rule. + +Use `list_procedures` to select an observed procedure address. To relate file +bytes to loaded memory, call MCP `inspect_native_load_image` or: + +```sh +rea search . --kind procedures --mode regex --provider ghidra --json +rea inspect-native-load-image --provider ghidra --json +rea decompile --provider ghidra --json +rea decompile --file-offset=0x1013 --provider ghidra --json +rea address-to-file-offset --provider ghidra --json +``` + +The load-image result preserves source-file indexes, file offsets, address +spaces, and inclusive mapping ranges. Follow the mapping for the selected +source file and region; adding `image_base` to an arbitrary file offset is not +valid. For a linear byte-addressed, file-backed mapping, a file offset `F` in +`[mapping.file_offset, mapping.file_offset + mapping.length)` maps to +`mapping.start + (F - mapping.file_offset)` in the reported `address_space`. +If several mappings cover the same file offset, keep those candidates distinct +and select the procedure from the observed inventory. Uninitialized regions may +have no file backing. ELF mappings are observed +without independent format-specific load-image verification, as the result's +status and limitations state. + +An unsuccessful numeric procedure lookup reports the current Ghidra image base +and gives the CLI commands above alongside the corresponding MCP operations. +Repeat any target-format, architecture, and entry-point options used to open the +target. Other procedure operations keep their analysis-address semantics; +file-offset fallback applies only to decompilation. + +Overview workflow revision 3 binds this output to snapshot identity. Older +serialized overview Evidence remains readable; it is not replayed as a current +overview merely because its source artifact is unchanged. + ## Admitted function-analysis semantics | Concern | Ghidra contract | diff --git a/docs/testing.md b/docs/testing.md index 4e79c3fe7e..5c43a5c78a 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -721,6 +721,44 @@ specifications, independent mappings, unchanged bytes, and decompilation of a known return value. AArch64 remains covered by deterministic tests rather than this real-provider lane. See [raw binary analysis](raw-binary.md). +`npm run verify:ghidra:address-context` runs on Linux with a caller-installed +Ghidra, full JDK, and an x86-64 ELF GCC toolchain (`CC` selects another compatible +compiler). It builds +`tests/conformance/c/pie-address-context.c` as a PIE ELF without executing it. +The compiled CLI and real stdio MCP workflow must expose the measured image +base, retain segment address spaces, agree on the overview, resolve explicit and +numeric file offsets, and reject unmapped analysis addresses with actionable +guidance. It checks the reverse `address-to-file-offset` mapping, file-coordinate +Evidence locations, and the suggested CLI search and load-image commands. +The verifier compares mapped bytes with the source file and checks source +identity, session preservation after failed lookup, owned runtime cleanup, and +verified empty process lineage. This lane does not establish Windows or macOS +real-provider coverage. + +For source-identity boundaries that a normal single-file import cannot create, +`tests/conformance/ghidra/ReaFileOffsetSourceProbe.java` exercises the production +bridge against real Ghidra `FileBytes`. It rejects auxiliary and partial sources, +checks the original target's round trip, and retains ambiguity when duplicate +target bytes have distinct loaded mappings. Injected decompiler failures and +cancellation verify that completed source/function resolution facts and backend +failure codes survive; these injections do not establish natural engine failures. +On Windows, after creating the +source-owned PE fixture, run from the repository root: + +```powershell +node scripts/create-ghidra-windows-fixture.mjs +$proofRoot = Join-Path $env:TEMP 'rea-file-offset-source-proof' +New-Item -ItemType Directory -Path $proofRoot -Force | Out-Null +Copy-Item bridge/ghidra/ReaGhidraBridge.java, bridge/ghidra/ReaGhidraExtensions.java, tests/conformance/ghidra/ReaFileOffsetSourceProbe.java -Destination $proofRoot +& "$env:GHIDRA_INSTALL_DIR/support/analyzeHeadless.bat" $proofRoot source-proof -import "$PWD/build/fixtures/rea-ghidra-windows.exe" -scriptPath $proofRoot -postScript ReaFileOffsetSourceProbe.java -deleteProject +``` + +Require the `REA_FILE_OFFSET_SOURCE_OK` marker and no script errors; Ghidra may +exit successfully even when a post-script fails. The probe edits only its +temporary analysis database and does not execute the fixture. This producer +regression supplements public CLI/MCP coverage and does not establish Linux +host behavior or native process-supervision controls. + ## Apple Interface Builder archives `npm run verify:interface-builder` compiles the source-owned AppKit XIB into a diff --git a/package.json b/package.json index cbb173b4fb..b2e2a51d3e 100644 --- a/package.json +++ b/package.json @@ -182,6 +182,7 @@ "verify:ghidra:dos": "npm run build:cached && node scripts/verify-real-ghidra-dos.mjs", "verify:ghidra:com": "npm run build:cached && node scripts/verify-real-ghidra-com.mjs", "verify:ghidra:raw": "npm run build:cached && node scripts/verify-real-ghidra-raw.mjs", + "verify:ghidra:address-context": "npm run build:cached && node scripts/verify-real-ghidra-address-context.mjs", "verify:ghidra:aarch64-jump-table": "npm run build:cached && node scripts/verify-real-ghidra.mjs --aarch64-jump-table", "verify:interface-builder": "npm run build:cached && node scripts/verify/apple/interface-builder.mjs", "verify:ghidra:cross-format": "npm run build:cached && node scripts/verify-real-ghidra.mjs --cross-format", diff --git a/scripts/ci/scopes.mjs b/scripts/ci/scopes.mjs index 56391365ca..a8679726c6 100644 --- a/scripts/ci/scopes.mjs +++ b/scripts/ci/scopes.mjs @@ -349,6 +349,7 @@ export const SCOPES = { nativeaot: [ "bridge/nativeaot/**", "scripts/generate-nativeaot-fixtures.mjs", + "scripts/lib/nativeaot-build.*", "tests/fixtures/nativeaot/**", ".github/workflows/nativeaot-fixtures.yml", ], diff --git a/scripts/generate-nativeaot-fixtures.mjs b/scripts/generate-nativeaot-fixtures.mjs index 84a8ac5336..fa9127cf37 100644 --- a/scripts/generate-nativeaot-fixtures.mjs +++ b/scripts/generate-nativeaot-fixtures.mjs @@ -4,9 +4,14 @@ import { createHash } from "node:crypto"; import { copyFile, mkdir, readFile, writeFile } from "node:fs/promises"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { + buildNativeAotFixture, + nativeAotCommandError, + supportsNativeAotFixtureHost, +} from "./lib/nativeaot-build.mjs"; const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); -if (process.arch !== "x64" || !["linux", "win32"].includes(process.platform)) +if (!supportsNativeAotFixtureHost(process.platform, process.arch)) throw new Error( "NativeAOT fixture lane requires Linux or Windows x64; it does not cross-compile NativeAOT.", ); @@ -40,9 +45,7 @@ const run = (command, args) => { maxBuffer: 8 * 1024 * 1024, }); if (result.error || result.status !== 0) - throw new Error( - `NativeAOT fixture lane requires/action failed ${command}: ${result.error?.message ?? ""}\n${result.stdout ?? ""}${result.stderr ?? ""}`, - ); + throw nativeAotCommandError(command, args[0], result); return result.stdout; }; const dotnet = process.env.REA_NATIVEAOT_DOTNET ?? "dotnet"; @@ -57,20 +60,12 @@ if (process.platform === "linux") { run("clang", ["--version"]); } const map = join(output, "NativeAotFixture.map"); -run(dotnet, [ - "publish", - join(root, "tests/fixtures/nativeaot/NativeAotFixture.csproj"), - "-r", +await buildNativeAotFixture(run, { + dotnet, + project: join(root, "tests/fixtures/nativeaot/NativeAotFixture.csproj"), runtime, - "-c", - "Release", - "-o", - join(output, "symbols"), - "--maxcpucount:1", - `-p:BaseIntermediateOutputPath=${join(output, "obj")}/`, - `-p:BaseOutputPath=${join(output, "bin")}/`, - `-p:NativeAotMapPath=${map}`, -]); + output, +}); const name = process.platform === "win32" ? "NativeAotFixture.exe" : "NativeAotFixture"; const binary = join(output, "symbols", name); diff --git a/scripts/lib/nativeaot-build.d.mts b/scripts/lib/nativeaot-build.d.mts new file mode 100644 index 0000000000..418111852f --- /dev/null +++ b/scripts/lib/nativeaot-build.d.mts @@ -0,0 +1,25 @@ +export function supportsNativeAotFixtureHost( + platform: string, + architecture: string, +): boolean; + +export function buildNativeAotFixture( + run: (command: string, args: string[]) => unknown, + options: { + dotnet: string; + project: string; + runtime: string; + output: string; + }, +): Promise; + +export function nativeAotCommandError( + command: string, + operation: string, + result: { + error?: { message: string }; + status: number | null; + stdout?: string | null; + stderr?: string | null; + }, +): Error; diff --git a/scripts/lib/nativeaot-build.mjs b/scripts/lib/nativeaot-build.mjs new file mode 100644 index 0000000000..3b49ce8398 --- /dev/null +++ b/scripts/lib/nativeaot-build.mjs @@ -0,0 +1,60 @@ +import { join } from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; + +/** NativeAOT fixtures use native toolchains, not cross-compilation. */ +export function supportsNativeAotFixtureHost(platform, architecture) { + return architecture === "x64" && ["linux", "win32"].includes(platform); +} + +/** Keep network recovery separate from compilation and preserve restore failures. */ +export async function buildNativeAotFixture(run, options) { + const { dotnet, project, runtime, output } = options; + const common = [ + project, + "-r", + runtime, + "--maxcpucount:1", + "-p:Configuration=Release", + `-p:BaseIntermediateOutputPath=${join(output, "obj")}/`, + `-p:BaseOutputPath=${join(output, "bin")}/`, + `-p:NativeAotMapPath=${join(output, "NativeAotFixture.map")}`, + ]; + for (let attempt = 1; attempt <= 2; attempt++) { + console.error(`NativeAOT dependency restore: attempt ${attempt}/2`); + try { + run(dotnet, ["restore", ...common, "--verbosity", "normal"]); + break; + } catch (cause) { + if (attempt === 2) + throw new Error( + "NativeAOT dependency restore failed after two attempts", + { + cause, + }, + ); + // Child diagnostics can contain authenticated feed URLs; keep retry logs static. + console.error( + "NativeAOT dependency restore failed; retrying in 1 second", + ); + await delay(1_000); + } + } + console.error("NativeAOT compilation (dependencies already restored)"); + run(dotnet, [ + "publish", + ...common, + "--no-restore", + "-o", + join(output, "symbols"), + ]); +} + +/** Sanitize child diagnostics before any retry, cause chain, or uncaught output. */ +export function nativeAotCommandError(command, operation, result) { + const diagnostic = `NativeAOT fixture lane requires/action failed ${command} ${operation}: ${result.error?.message ?? `exit ${result.status}`}\n${result.stdout ?? ""}${result.stderr ?? ""}`; + // Only URL authority userinfo is authentication; preserve host, path, query, + // and unrelated diagnostic values. Never retain the raw result as a cause. + return new Error( + diagnostic.replace(/\b(https?:\/\/)[^\\\s/?#]*@/giu, "$1[redacted]@"), + ); +} diff --git a/scripts/lib/real-ghidra-file-offsets.mjs b/scripts/lib/real-ghidra-file-offsets.mjs new file mode 100644 index 0000000000..ea881dd04c --- /dev/null +++ b/scripts/lib/real-ghidra-file-offsets.mjs @@ -0,0 +1,264 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { + mkdir, + mkdtemp, + readFile, + readdir, + rm, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { parseEvidence } from "../../dist/domain/evidence.js"; +import { + requireMcpOperationResult, + requireMcpToolError, + mcpTextValue, +} from "./mcp-verifier-results.mjs"; + +/** Exercise file coordinates through public CLI/MCP using the source-owned PE fixture. */ +export async function verifyGhidraFileOffsets( + targetPath, + entrypoint, + procedure, +) { + const original = await readFile(targetPath); + const workspace = await mkdtemp(join(tmpdir(), "rea-file-offsets-")); + const runtime = join(workspace, "runtime"); + await mkdir(runtime); + const env = { + ...process.env, + TEMP: runtime, + TMP: runtime, + TMPDIR: runtime, + REA_ANALYSIS_PROVIDER: "ghidra", + REA_LOG_LEVEL: "silent", + }; + const client = new Client({ name: "rea-file-offsets", version: "1" }); + const transport = new StdioClientTransport({ + command: process.execPath, + args: [entrypoint, "mcp"], + env, + stderr: "pipe", + }); + let connected = false; + let complete = false; + let primaryFailure; + const call = async (name, arguments_ = {}) => { + const result = await client.callTool( + { name, arguments: arguments_ }, + { timeout: 240_000 }, + ); + const projected = requireMcpOperationResult(result, name); + return ["open_binary", "binary_session", "close_binary"].includes(name) + ? projected + : (result.structuredContent ?? JSON.parse(mcpTextValue(result))); + }; + try { + await client.connect(transport); + connected = true; + await call("open_binary", { path: targetPath, provider_id: "ghidra" }); + const session = await call("binary_session"); + const mapping = parseEvidence( + await call("address_to_file_offset", { address: procedure }), + ); + const offset = mapping.normalized_result.file_offset; + const coordinate = `0x${offset.toString(16)}`; + const expected = parseEvidence( + await call("procedure_pseudo_code", { procedure }), + ); + const explicit = parseEvidence( + await call("procedure_pseudo_code", { + procedure: `file-offset:${coordinate}`, + }), + ); + const automatic = parseEvidence( + await call("procedure_pseudo_code", { procedure: coordinate }), + ); + for (const actual of [explicit, automatic]) { + assert.equal(actual.normalized_result, expected.normalized_result); + assert.equal(actual.raw_result.selection.file_offset, offset); + assert.equal(actual.raw_result.selection.mapped_address, procedure); + assert.ok( + actual.locations.some( + (location) => + location.kind === "file-offset" && location.offset === offset, + ), + ); + } + const invalid = await client.callTool({ + name: "procedure_pseudo_code", + arguments: { procedure: "file-offset:0x20000000000000" }, + }); + assert.equal(requireMcpToolError(invalid).code, "invalid_request"); + const unmapped = await client.callTool({ + name: "procedure_pseudo_code", + arguments: { procedure: "file-offset:0x100000" }, + }); + assert.ok( + JSON.stringify(requireMcpToolError(unmapped)).includes( + "no loaded source mapping", + ), + ); + assert.equal( + (await call("binary_session")).analysis_run.run_id, + session.analysis_run.run_id, + ); + const image = parseEvidence(await call("inspect_native_load_image")); + const headerMapping = image.normalized_result.observations.mappings.find( + (item) => item.initialized && item.file_offset === 0, + ); + assert.ok( + headerMapping, + "Fixture header must have an observed source mapping", + ); + const headerOffset = parseEvidence( + await call("address_to_file_offset", { address: headerMapping.start }), + ); + assert.equal(headerOffset.normalized_result.file_offset, 0); + for (const selector of ["file-offset:0x0", "0x0"]) { + const missing = requireMcpToolError( + await client.callTool({ + name: "procedure_pseudo_code", + arguments: { procedure: selector }, + }), + ); + assertMissingMappedProcedure(missing, headerMapping.start); + } + assert.equal( + (await call("binary_session")).analysis_run.run_id, + session.analysis_run.run_id, + ); + await call("close_binary"); + const cli = JSON.parse( + ( + await promisify(execFile)( + process.execPath, + [ + entrypoint, + "decompile", + targetPath, + `--file-offset=${coordinate}`, + "--provider", + "ghidra", + "--json", + ], + { env, timeout: 240_000, maxBuffer: 16 * 1024 * 1024 }, + ) + ).stdout, + ); + assert.equal(cli.normalized_result, explicit.normalized_result); + assert.deepEqual(cli.locations, explicit.locations); + const missingCli = await promisify(execFile)( + process.execPath, + [ + entrypoint, + "decompile", + targetPath, + "--file-offset=0x0", + "--provider", + "ghidra", + "--json", + ], + { env, timeout: 240_000, maxBuffer: 16 * 1024 * 1024 }, + ).then( + () => { + throw new Error( + "Mapped header bytes must not decompile as a procedure", + ); + }, + (cause) => JSON.parse(cause.stdout), + ); + assertMissingMappedProcedure(missingCli, headerMapping.start); + const alias = Buffer.from(original); + const pe = alias.readUInt32LE(0x3c); + if (alias.readUInt16LE(pe + 6) >= 2) { + const secondSection = pe + 24 + alias.readUInt16LE(pe + 20) + 40; + // Both sections now carry the same original bytes at distinct virtual addresses. + alias.writeUInt32LE(0x200, secondSection + 20); + const ambiguousPath = join(workspace, "ambiguous.exe"); + await writeFile(ambiguousPath, alias); + await call("open_binary", { path: ambiguousPath, provider_id: "ghidra" }); + const ambiguous = await client.callTool( + { + name: "procedure_pseudo_code", + arguments: { procedure: `file-offset:${coordinate}` }, + }, + { timeout: 240_000 }, + ); + assert.ok( + JSON.stringify(requireMcpToolError(ambiguous)).includes( + "multiple loaded mappings", + ), + ); + await call("close_binary"); + } + assert.deepEqual(await readFile(targetPath), original); + await client.close(); + connected = false; + assert.deepEqual(await readdir(runtime), []); + complete = true; + return { + cli_mcp_parity: true, + explicit_and_numeric_offsets: true, + invalid_and_unmapped_offsets: true, + mapped_offset_without_procedure: true, + ambiguous_mappings: + alias.readUInt16LE(pe + 6) >= 2 + ? "verified" + : "unavailable-single-section", + source_unchanged: true, + cleanup: "complete", + }; + } catch (cause) { + primaryFailure = cause; + throw cause; + } finally { + const cleanupFailures = []; + if (connected) { + try { + await call("close_binary"); + } catch (cause) { + cleanupFailures.push(cause); + } + } + try { + await client.close(); + } catch (cause) { + cleanupFailures.push(cause); + } + if (cleanupFailures.length > 0) { + process.stderr.write( + `Retained file-offset verifier workspace: ${workspace}\n`, + ); + throw new AggregateError( + primaryFailure === undefined + ? cleanupFailures + : [primaryFailure, ...cleanupFailures], + "File-offset verifier cleanup failed", + ); + } + if (complete) await rm(workspace, { recursive: true, force: true }); + else + process.stderr.write( + `Retained file-offset verifier workspace: ${workspace}\n`, + ); + } +} + +function assertMissingMappedProcedure(error, mappedAddress) { + assert.equal(error.code, "invalid_request"); + const message = error.details?.issues?.[0]?.message; + assert.equal(typeof message, "string"); + assert.ok( + message.includes("File offset 0x0 maps to analysis address"), + message, + ); + assert.ok(message.includes(mappedAddress), message); + assert.ok(message.includes("no procedure starts at or contains"), message); + assert.ok(!message.includes("--file-offset"), message); +} diff --git a/scripts/verify-real-ghidra-address-context.mjs b/scripts/verify-real-ghidra-address-context.mjs new file mode 100644 index 0000000000..2b650df933 --- /dev/null +++ b/scripts/verify-real-ghidra-address-context.mjs @@ -0,0 +1,313 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, readFile, readdir, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { parseEvidence } from "../dist/domain/evidence.js"; +import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs"; +import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; + +if (process.argv.length !== 2 || process.platform !== "linux") + throw new Error("Run this ELF verifier on Linux without arguments."); +if (!process.env.GHIDRA_INSTALL_DIR) + throw new Error("Set GHIDRA_INSTALL_DIR to a caller-installed Ghidra."); +const exec = promisify(execFile); +const run = createVerifierRun(); +const root = fileURLToPath(new URL("..", import.meta.url)); +const workspace = await mkdtemp(join(tmpdir(), "rea-address-context-")); +const runtime = join(workspace, "runtime"); +await mkdir(runtime); +const path = join(workspace, "pie.elf"); +const entrypoint = join(root, "scripts/rea.mjs"); +const env = { + ...process.env, + TMPDIR: runtime, + REA_ANALYSIS_PROVIDER: "ghidra", + REA_LOG_LEVEL: "silent", +}; +const transport = new StdioClientTransport({ + command: process.execPath, + args: [entrypoint, "mcp"], + env, + stderr: "pipe", +}); +const client = new Client({ name: "rea-real-address-context", version: "1" }); +let stderr = ""; +transport.stderr?.on("data", (chunk) => { + stderr = (stderr + chunk.toString()).slice(-65536); +}); +let report; +let connected = false; +let primaryFailure; +let verifierRun; +try { + const compiler = process.env.CC ?? "gcc"; + const version = await exec(compiler, ["--version"], { timeout: 10000 }); + await exec( + compiler, + [ + "-O0", + "-g", + "-fPIE", + "-pie", + "-nostdlib", + "-Wl,-e,rea_pie_entry", + "-Wl,--build-id=none", + join(root, "tests/conformance/c/pie-address-context.c"), + "-o", + path, + ], + { timeout: 30000 }, + ); + const bytes = await readFile(path); + assert.equal(bytes.subarray(0, 6).toString("hex"), "7f454c460201"); + assert.equal(bytes.readUInt16LE(16), 3); // ET_DYN: source-owned x86-64 PIE. + assert.equal(bytes.readUInt16LE(18), 62); + const declaredEntry = `0x${bytes.readBigUInt64LE(24).toString(16)}`; + const sha256 = createHash("sha256").update(bytes).digest("hex"); + await client.connect(transport); + connected = true; + await call("open_binary", { path, provider_id: "ghidra" }); + const overview = parseEvidence(await call("binary_overview")); + assert.equal(overview.subject.digest.sha256, sha256); + const base = overview.normalized_result.image_base; + assert.match(base, /^0x[0-9a-f]+$/u); + assert.notEqual(base, "0x0"); + const active = await call("binary_session"); + const image = parseEvidence(await call("inspect_native_load_image")); + assert.equal(base, image.normalized_result.observations.image_base); + assert.ok(overview.normalized_result.segments.length > 0); + assert.ok( + overview.normalized_result.segments.every( + (segment) => + segment.image_base === base && + typeof segment.address_space === "string", + ), + ); + const procedures = parseEvidence(await call("list_procedures")); + const entry = procedures.normalized_result.find( + (item) => item.value === "rea_pie_entry", + ); + const addition = procedures.normalized_result.find( + (item) => item.value === "rea_pie_add", + ); + assert.ok(entry && addition); + assert.notEqual(entry.address, declaredEntry); + assert.ok( + image.normalized_result.observations.entry_points.includes(entry.address), + ); + const mapping = image.normalized_result.observations.mappings.find( + (item) => + item.source_file_index === 0 && + item.file_offset !== null && + item.address_space === "ram" && + BigInt(item.start) <= BigInt(entry.address) && + BigInt(item.end) >= BigInt(entry.address), + ); + assert.ok(mapping); + const offset = + mapping.file_offset + Number(BigInt(entry.address) - BigInt(mapping.start)); + const reverseMapping = parseEvidence( + await call("address_to_file_offset", { address: entry.address }), + ); + assert.equal(reverseMapping.normalized_result.file_offset, offset); + const memory = parseEvidence( + await call("read_bytes", { address: entry.address, length: 8 }), + ); + assert.equal( + memory.normalized_result.bytes_hex, + bytes.subarray(offset, offset + 8).toString("hex"), + ); + const fromOffset = parseEvidence( + await call("procedure_pseudo_code", { + procedure: `file-offset:0x${offset.toString(16)}`, + }), + ); + assert.ok( + fromOffset.locations.some( + (location) => + location.kind === "file-offset" && location.offset === offset, + ), + ); + assert.equal(fromOffset.raw_result.selection.mapped_address, entry.address); + const fromNumericOffset = parseEvidence( + await call("procedure_pseudo_code", { + procedure: `0x${offset.toString(16)}`, + }), + ); + assert.equal( + fromNumericOffset.normalized_result, + fromOffset.normalized_result, + ); + const missingAddress = "ram:0xffffffff"; + const rejected = await client.callTool( + { name: "procedure_pseudo_code", arguments: { procedure: missingAddress } }, + { timeout: 240000 }, + ); + assert.equal(rejected.isError, true); + assert.ok( + JSON.stringify(rejected.content).includes(`Ghidra image base: ${base}`), + ); + assert.ok(JSON.stringify(rejected.content).includes("list_procedures")); + assert.ok( + JSON.stringify(rejected.content).includes("inspect_native_load_image"), + ); + const pseudo = parseEvidence( + await call("procedure_pseudo_code", { procedure: addition.address }), + ); + assert.match(pseudo.normalized_result, /\+\s*(?:7|0x7)\b/u); + const retained = await call("binary_session"); + assert.equal(retained.analysis_run.run_id, active.analysis_run.run_id); + await call("close_binary"); + const cli = async (args) => + exec( + process.execPath, + [entrypoint, ...args, "--provider", "ghidra", "--json"], + { env, timeout: 240000, maxBuffer: 16 * 1024 * 1024 }, + ); + const cliOverview = parseEvidence( + JSON.parse((await cli(["analyze", path])).stdout), + ); + assert.deepEqual(cliOverview.normalized_result, overview.normalized_result); + let cliFailure; + try { + await cli(["decompile", path, missingAddress]); + } catch (cause) { + cliFailure = cause; + } + assert.ok(cliFailure && typeof cliFailure.stdout === "string"); + const error = JSON.parse(cliFailure.stdout); + assert.equal(error.code, "invalid_request"); + assert.ok(JSON.stringify(error).includes(`Ghidra image base: ${base}`)); + for (const command of [ + "rea search . --kind procedures --mode regex", + "rea analyze ", + "rea inspect-native-load-image ", + ]) + assert.ok(JSON.stringify(error).includes(command)); + const cliProcedures = parseEvidence( + JSON.parse( + ( + await cli([ + "search", + path, + ".", + "--kind", + "procedures", + "--mode", + "regex", + ]) + ).stdout, + ), + ); + const cliEntry = cliProcedures.normalized_result.find( + (item) => item.value === "rea_pie_entry", + ); + assert.ok(cliEntry); + assert.equal(cliEntry.address, entry.address); + const cliImage = parseEvidence( + JSON.parse((await cli(["inspect-native-load-image", path])).stdout), + ); + assert.equal(cliImage.normalized_result.observations.image_base, base); + const cliPseudo = parseEvidence( + JSON.parse((await cli(["decompile", path, cliEntry.address])).stdout), + ); + const cliOffset = parseEvidence( + JSON.parse( + (await cli(["decompile", path, `--file-offset=0x${offset.toString(16)}`])) + .stdout, + ), + ); + assert.equal(cliOffset.normalized_result, fromOffset.normalized_result); + assert.deepEqual(cliOffset.locations, fromOffset.locations); + assert.match(cliPseudo.normalized_result, /rea_pie_add/u); + assert.deepEqual(await readFile(path), bytes); + assert.deepEqual(await readdir(runtime), []); + report = { + ok: true, + mocked: false, + provider: image.provider, + compiler: version.stdout.split("\n")[0], + sha256, + declared_elf_entry: declaredEntry, + loaded_entry: entry.address, + observed_image_base: base, + cli_mcp_overview_parity: true, + failed_lookup_guidance: true, + cli_recovery_commands: true, + loaded_decompilation: true, + file_offset_decompilation: true, + address_file_offset_round_trip: true, + source_unchanged: true, + runtime_cleanup: "complete", + }; +} catch (cause) { + primaryFailure = { cause }; + if (stderr) process.stderr.write(stderr); +} + +const cleanupFailures = []; +// Await provider-owned cleanup before the SDK can terminate its MCP parent. +if (connected) { + try { + await call("close_binary"); + } catch (cause) { + cleanupFailures.push(cause); + } +} +try { + await client.close(); +} catch (cause) { + cleanupFailures.push(cause); +} +try { + await transport.close(); +} catch (cause) { + cleanupFailures.push(cause); +} +try { + assert.deepEqual(await readdir(runtime), []); + verifierRun = await completeVerifierRun(run); + assert.equal(verifierRun.process_lineage.status, "verified"); + assert.deepEqual(verifierRun.process_lineage.descendants, []); +} catch (cause) { + cleanupFailures.push(cause); +} +if (cleanupFailures.length === 0) { + try { + await rm(workspace, { recursive: true, force: true }); + } catch (cause) { + cleanupFailures.push(cause); + } +} +if (cleanupFailures.length > 0) { + process.stderr.write( + `Retained verifier workspace after cleanup failure: ${workspace}\n`, + ); + throw new AggregateError( + primaryFailure === undefined + ? cleanupFailures + : [primaryFailure.cause, ...cleanupFailures], + "Ghidra address-context verifier cleanup failed", + ); +} +if (primaryFailure !== undefined) throw primaryFailure.cause; +process.stdout.write( + JSON.stringify({ verifier_run: verifierRun, ...report }) + "\n", +); + +async function call(name, args = {}) { + const result = await client.callTool( + { name, arguments: args }, + { timeout: 240000 }, + ); + assert.notEqual(result.isError, true, JSON.stringify(result)); + return ["open_binary", "binary_session", "close_binary"].includes(name) + ? requireMcpOperationResult(result, name) + : result.structuredContent; +} diff --git a/scripts/verify-real-ghidra-windows.mjs b/scripts/verify-real-ghidra-windows.mjs index 5adf96009d..bc822211f1 100644 --- a/scripts/verify-real-ghidra-windows.mjs +++ b/scripts/verify-real-ghidra-windows.mjs @@ -28,6 +28,7 @@ import { import { parseBinaryTarget } from "../dist/application/BinaryTargetResolver.js"; import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs"; import { verifyGhidraCompactReferences } from "./lib/real-ghidra-compact-references.mjs"; +import { verifyGhidraFileOffsets } from "./lib/real-ghidra-file-offsets.mjs"; const verifierRun = createVerifierRun(); const SESSION_CAPABILITIES = GHIDRA_SESSION_CAPABILITIES.filter( @@ -300,6 +301,11 @@ try { } if (report === undefined) throw new Error("Windows Ghidra report was not produced"); +report.file_offsets = await verifyGhidraFileOffsets( + targetPath, + resolve(root, "scripts", "rea.mjs"), + architecture === "x86" ? "0x401010" : "0x140001010", +); process.stdout.write( `${JSON.stringify({ verifier_run: await completeVerifierRun(verifierRun), ...report })}\n`, ); diff --git a/src/application/EnhancedTools.ts b/src/application/EnhancedTools.ts index c8b80519fe..87d27fc5f1 100644 --- a/src/application/EnhancedTools.ts +++ b/src/application/EnhancedTools.ts @@ -21,8 +21,8 @@ import { parseFunctionDossier, parseListCount, parseRelatedAddresses, - parseSegmentImageBase, parseSegments, + parseSegmentImageBase, } from "../domain/hopperValues.js"; import { err, ok, type Result } from "../domain/result.js"; import { @@ -46,6 +46,7 @@ import { traceCallPath } from "./CallPathTracing.js"; import { resolveProcedureAddress } from "./ProcedureAddressResolution.js"; import { traceLiteralFeature } from "./EnhancedLiteralTracing.js"; import { projectNativeApiInspection } from "./native/NativeApiInspection.js"; + /** * Composes direct provider operations into bounded reverse-engineering tools. * Direct callers use `execute`; adapters that own validation use @@ -528,24 +529,29 @@ export class EnhancedTools { const segments = parseSegments(segmentsResult.value); if (!segments.ok) return segments; - const imageBase = parseSegmentImageBase(segmentsResult.value); const procedureCount = parseListCount(proceduresResult.value, "procedures"); if (!procedureCount.ok) return procedureCount; const stringCount = parseListCount(stringsResult.value, "strings"); if (!stringCount.ok) return stringCount; + const imageBase = parseSegmentImageBase(segments.value); + return ok({ document: document.value, - segments: segments.value.map(({ name, start, end }) => ({ - name, - start, - end, - length: addressDistance(start, end), - })), + ...(imageBase === undefined ? {} : { image_base: imageBase }), + segments: segments.value.map( + ({ name, start, end, image_base, address_space }) => ({ + name, + start, + end, + length: addressDistance(start, end), + ...(image_base === undefined ? {} : { image_base }), + ...(address_space === undefined ? {} : { address_space }), + }), + ), segment_count: segments.value.length, procedure_count: procedureCount.value, string_count: stringCount.value, - ...(imageBase === undefined ? {} : { image_base: imageBase }), }); } diff --git a/src/application/InvestigationProviders.ts b/src/application/InvestigationProviders.ts index 883d3878c2..5dcb181504 100644 --- a/src/application/InvestigationProviders.ts +++ b/src/application/InvestigationProviders.ts @@ -18,11 +18,11 @@ export const workflowAnalysisProfile = ( ): AnalysisProfileCommitment => createAnalysisProfile(REA_WORKFLOW_PROVIDER, { upstream_analysis_profile: upstream, - ...(operation === "binary_overview" || - operation === "batch_decompile" || - operation === "get_call_graph" - ? { workflow_revision: 2 } - : {}), + ...(operation === "binary_overview" + ? { workflow_revision: 3 } + : operation === "batch_decompile" || operation === "get_call_graph" + ? { workflow_revision: 2 } + : {}), }); /** Provider identity for deterministic artifact inventories. */ diff --git a/src/cli/coreBinaryCommands.ts b/src/cli/coreBinaryCommands.ts index 70405302ea..710b391817 100644 --- a/src/cli/coreBinaryCommands.ts +++ b/src/cli/coreBinaryCommands.ts @@ -137,6 +137,13 @@ const registerDecompileCommand = ( .describe( "Procedure name or address; use --procedure= for leading dashes", ), + "file-offset": z + .string() + .regex(/^0x[0-9a-fA-F]+$/u) + .optional() + .describe( + "Ghidra source-file byte offset; mapped internally to an analysis address", + ), snapshot: z .string() .min(1) @@ -152,8 +159,17 @@ const registerDecompileCommand = ( { procedure: resolveCliAnalysisSelector( args.address, - options.procedure, - "procedure", + options["file-offset"] === undefined + ? options.procedure + : resolveCliAnalysisSelector( + options.procedure, + `file-offset:${options["file-offset"]}`, + "file-offset", + "procedure_pseudo_code", + ), + options["file-offset"] === undefined + ? "procedure" + : "file-offset", "procedure_pseudo_code", ), }, diff --git a/src/contracts/officialToolContracts.ts b/src/contracts/officialToolContracts.ts index f9c024fe51..82f1dc83d6 100644 --- a/src/contracts/officialToolContracts.ts +++ b/src/contracts/officialToolContracts.ts @@ -181,7 +181,7 @@ export const OFFICIAL_TOOL_CONTRACTS = [ ), official( "procedure_pseudo_code", - "Decompile one analyzed procedure by symbol name or provider-normalized address. Returns provider-specific pseudocode, never original source or cross-provider text equivalence, and may return null; request procedure_assembly when instruction precision matters. Ghidra no-return assumptions may cut the analyzed body; Evidence limitations report observed terminal calls and available callee identities.", + "Decompile one analyzed procedure by symbol name or provider-normalized address. Ghidra also accepts file-offset:0x... selectors and falls back to a unique source-file mapping when a 0x-prefixed number resolves to no procedure; existing analysis addresses take precedence and ambiguous mappings fail explicitly. Returns provider-specific pseudocode, never original source or cross-provider text equivalence, and may return null; request procedure_assembly when instruction precision matters. Ghidra no-return assumptions may cut the analyzed body; Evidence limitations report observed terminal calls and available callee identities.", z.object({ procedure, document }), ), official( diff --git a/src/contracts/toolOutputSchemaGroups.ts b/src/contracts/toolOutputSchemaGroups.ts index 174339bdf0..dcf7ad0330 100644 --- a/src/contracts/toolOutputSchemaGroups.ts +++ b/src/contracts/toolOutputSchemaGroups.ts @@ -357,18 +357,20 @@ export const enhancedOutputSchemas: Readonly> = { binary_overview: evidenceResultOf( z.object({ document: z.string(), + image_base: z.string().min(1).optional(), segments: z.array( z.object({ name: z.string(), start: z.string(), end: z.string(), length: z.number().min(0), + image_base: z.string().exactOptional(), + address_space: z.string().exactOptional(), }), ), segment_count: z.number().int().min(0), procedure_count: z.number().int().min(0), string_count: z.number().int().min(0), - image_base: z.string().min(1).optional(), }), ), analyze_function: functionDossierOutput, diff --git a/src/domain/hopperValues.test.ts b/src/domain/hopperValues.test.ts index 7af5df704d..c5931cd35c 100644 --- a/src/domain/hopperValues.test.ts +++ b/src/domain/hopperValues.test.ts @@ -8,7 +8,6 @@ import { parseNames, parseProcedures, parseRelatedAddresses, - parseSegmentImageBase, parseSegments, } from "./hopperValues.js"; @@ -66,19 +65,6 @@ describe("Hopper boundary values", () => { ok: true, value: ["fixture"], }); - expect( - parseSegmentImageBase([ - { image_base: "0x100000" }, - { image_base: "0x100000" }, - ]), - ).toBe("0x100000"); - expect( - parseSegmentImageBase([ - { image_base: "0x100000" }, - { image_base: "0x200000" }, - ]), - ).toBeUndefined(); - expect(parseSegmentImageBase([{ name: "__TEXT" }])).toBeUndefined(); }); it.each([ diff --git a/src/domain/hopperValues.ts b/src/domain/hopperValues.ts index 5e5a3987b6..769d0f040c 100644 --- a/src/domain/hopperValues.ts +++ b/src/domain/hopperValues.ts @@ -20,6 +20,8 @@ export interface SegmentSummary { readonly readable: boolean | null; readonly writable: boolean | null; readonly executable: boolean | null; + readonly image_base?: string; + readonly address_space?: string; } /** Return a bounded byte distance between canonical addresses in one space. */ @@ -58,10 +60,9 @@ const segmentSchema = z.object({ readable: z.boolean().nullable(), writable: z.boolean().nullable(), executable: z.boolean().nullable(), + image_base: z.string().min(1).exactOptional(), + address_space: z.string().min(1).exactOptional(), }); -const segmentImageBaseSchema = z - .object({ image_base: z.string().min(1) }) - .passthrough(); const unavailableAnalysisFactSchema = z .object({ available: z.literal(false), reason: z.string() }) .strict(); @@ -415,7 +416,7 @@ export const parseRelatedAddresses = ( : invalid(`${relation} list`, parsed.error); }; -/** Parse direct or wrapped Hopper segment records. */ +/** Preserve reported address context in direct or wrapped provider segments. */ export const parseSegments = ( value: JsonValue, ): Result => { @@ -427,14 +428,15 @@ export const parseSegments = ( : invalid("segment list", parsed.error); }; -/** Return a provider-reported image base only when every segment agrees on it. */ -export const parseSegmentImageBase = (value: JsonValue): string | undefined => { - const parsed = z - .array(segmentImageBaseSchema) - .safeParse(unwrapProperty(value, "segments")); - if (!parsed.success) return undefined; - const bases = new Set(parsed.data.map(({ image_base }) => image_base)); - return bases.size === 1 ? parsed.data[0]?.image_base : undefined; +/** Return a reported overview base only when every parsed segment agrees. */ +export const parseSegmentImageBase = ( + segments: readonly SegmentSummary[], +): string | undefined => { + const base = segments[0]?.image_base; + return base !== undefined && + segments.every((segment) => segment.image_base === base) + ? base + : undefined; }; /** Parse direct or wrapped Hopper document names. */ diff --git a/src/ghidra/GhidraAnalysisProfile.ts b/src/ghidra/GhidraAnalysisProfile.ts index 60edb0971b..fd7cdff117 100644 --- a/src/ghidra/GhidraAnalysisProfile.ts +++ b/src/ghidra/GhidraAnalysisProfile.ts @@ -123,6 +123,8 @@ export const resolveGhidraAnalysisProfile = async ( call_relationship_coverage: "resolved-call-reference-diagnostics-v1", string_inventory_evidence: "defined-data-coverage-v1", location_resolution: "explicit-address-exact-entry-symbol-first-v3", + procedure_file_offset_resolution: + "address-first-unique-target-source-mapping-v2", instruction_flow_evidence: "decoded-return-pcode-v1", instruction_references: "exact-address-incoming-v1", function_boundary_observations: "ghidra-terminal-call-limitations-v1", @@ -212,6 +214,8 @@ export const resolveGhidraAnalysisProfile = async ( call_relationship_coverage: "resolved-call-reference-diagnostics-v1", string_inventory_evidence: "defined-data-coverage-v1", location_resolution: "explicit-address-exact-entry-symbol-first-v3", + procedure_file_offset_resolution: + "address-first-unique-target-source-mapping-v2", instruction_flow_evidence: "decoded-return-pcode-v1", instruction_references: "exact-address-incoming-v1", function_boundary_observations: "ghidra-terminal-call-limitations-v1", diff --git a/src/ghidra/GhidraDoctor.test.ts b/src/ghidra/GhidraDoctor.test.ts index e5d20f4c85..69b9acb623 100644 --- a/src/ghidra/GhidraDoctor.test.ts +++ b/src/ghidra/GhidraDoctor.test.ts @@ -1,9 +1,10 @@ -import { describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { runDoctor, type DoctorHost } from "../application/Doctor.js"; import { createDoctorHostFixture } from "../application/Doctor.fixture.js"; import { providerRegistrationEnvironment } from "../application/SetupRegistrationEnvironment.js"; import { WINDOWS_NATIVE_AUTHORITY_UNAVAILABLE_REASON } from "../process/WindowsAuthority.js"; +import * as windowsAuthority from "../process/WindowsAuthority.js"; import { inspectGhidraInstallation, type GhidraInstallationHost, @@ -173,6 +174,16 @@ describe("Ghidra doctor integration", () => { }); describe("Windows Ghidra doctor authority", () => { + beforeEach(() => { + vi.spyOn(windowsAuthority, "hasWindowsNativeAuthority").mockReturnValue( + false, + ); + vi.spyOn( + windowsAuthority, + "windowsNativeAuthorityUnavailableReason", + ).mockReturnValue(WINDOWS_NATIVE_AUTHORITY_UNAVAILABLE_REASON); + }); + afterEach(() => vi.restoreAllMocks()); it("distinguishes a valid Windows installation from unavailable native authority", async () => { const ghidra = inspection({ installDir: "C:\\tools\\ghidra_12.1.4_PUBLIC", diff --git a/src/ghidra/GhidraFunctionValues.test.ts b/src/ghidra/GhidraFunctionValues.test.ts index 662db0a263..75d59c97fa 100644 --- a/src/ghidra/GhidraFunctionValues.test.ts +++ b/src/ghidra/GhidraFunctionValues.test.ts @@ -42,6 +42,30 @@ const parseFunctionValueFixture = ( }; describe("Ghidra function-analysis result values", () => { + it("preserves file-offset resolution in Evidence locations and rejects inexact offsets", () => { + const selection = { + file_offset: 0x210, + mapped_address: "0x140001010", + procedure_address: "0x140001010", + }; + const wire = { value: "return 42;", limitations: [], selection }; + expect(parseGhidraFunctionResult("procedure_assembly", wire)).toMatchObject( + { ok: false, error: { _tag: "AnalysisOutputError" } }, + ); + const parsed = parseGhidraFunctionResult("procedure_pseudo_code", wire); + if (!parsed.ok) throw parsed.error; + expect(parsed.value.locations).toEqual([ + { kind: "file-offset", offset: 0x210 }, + { kind: "address", address: "0x140001010" }, + { kind: "address", address: "0x140001010" }, + ]); + expect( + parseGhidraFunctionResult("procedure_pseudo_code", { + ...wire, + selection: { ...selection, file_offset: Number.MAX_SAFE_INTEGER + 1 }, + }), + ).toMatchObject({ ok: false, error: { _tag: "AnalysisOutputError" } }); + }); it.each([...observationOperations])( "requires the function-observation limitations envelope for %s", (operation) => { diff --git a/src/ghidra/GhidraFunctionValues.ts b/src/ghidra/GhidraFunctionValues.ts index 8876e31aed..d45e41ba93 100644 --- a/src/ghidra/GhidraFunctionValues.ts +++ b/src/ghidra/GhidraFunctionValues.ts @@ -26,6 +26,7 @@ import { } from "../domain/jsonValue.js"; import { nativeApiBoundarySchema } from "../domain/native/nativeApiBoundary.js"; import { nativeValueFlowSchema } from "../domain/native/nativeValueFlow.js"; +import type { EvidenceLocation } from "../domain/evidence.js"; import { err, ok, type Result } from "../domain/result.js"; import { analysisInputErrorFromIssues } from "../domain/inputIssueProjection.js"; import { @@ -399,11 +400,19 @@ const functionBoundaryObservationOperations: ReadonlySet["selection"]; }; if (functionBoundaryObservationOperations.has(operation)) { const envelope = functionObservationEnvelope.safeParse(value); - if (!envelope.success) + if ( + !envelope.success || + (operation !== "procedure_pseudo_code" && + envelope.data.selection !== undefined) + ) return err( new AnalysisOutputError( operation, "Ghidra bridge omitted the function observation envelope", - { cause: envelope.error }, + envelope.success ? undefined : { cause: envelope.error }, ), ); observation = envelope.data; @@ -434,6 +450,24 @@ export const parseGhidraFunctionResult = ( ? ok({ value: jsonValueSchema.parse(parsed.data), limitations: observation.limitations, + ...(observation.selection === undefined + ? {} + : { + locations: [ + { + kind: "file-offset" as const, + offset: observation.selection.file_offset, + }, + { + kind: "address" as const, + address: observation.selection.mapped_address, + }, + { + kind: "address" as const, + address: observation.selection.procedure_address, + }, + ], + }), }) : err( new AnalysisOutputError( diff --git a/src/ghidra/GhidraProviderClient.ts b/src/ghidra/GhidraProviderClient.ts index 5a4f4eafd2..5911985dfc 100644 --- a/src/ghidra/GhidraProviderClient.ts +++ b/src/ghidra/GhidraProviderClient.ts @@ -32,6 +32,7 @@ import { AnalysisTimeoutError, } from "../domain/analysisErrorCore.js"; import type { AnalysisError } from "../domain/analysisErrorBase.js"; +import type { EvidenceLocation } from "../domain/evidence.js"; import { jsonValueSchema, type JsonValue } from "../domain/jsonValue.js"; import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import { err, ok, type Result } from "../domain/result.js"; @@ -678,10 +679,12 @@ export const createGhidraProviderClient = (input: { normalized: JsonValue, rawResult: JsonValue, observationLimitations: readonly string[] = [], + locations: readonly EvidenceLocation[] = [], ) => { const relationshipCoverage = relationshipCoverageFor(operation); return createAnalysisExecution(normalized, committedProfile.provider, { rawResult, + locations, analysisProfile: committedProfile, ...(relationshipCoverage === undefined ? {} : { relationshipCoverage }), limitations: [ @@ -776,11 +779,13 @@ export const createGhidraProviderClient = (input: { ); let normalized: JsonValue; let observationLimitations: readonly string[] = []; + let observationLocations: readonly EvidenceLocation[] = []; if (isGhidraFunctionOperation(operation)) { const result = parseGhidraFunctionResult(operation, called.value); if (!result.ok) return result; normalized = result.value.value; observationLimitations = result.value.limitations; + observationLocations = result.value.locations ?? []; } else { const result = parseGhidraInventoryResult(operation, called.value); if (!result.ok) return result; @@ -875,6 +880,7 @@ export const createGhidraProviderClient = (input: { normalized, called.value, observationLimitations, + observationLocations, ), ); }, diff --git a/tests/boundary/cli/analysisSelectorCli.test.ts b/tests/boundary/cli/analysisSelectorCli.test.ts index 404cad6a2c..5e89daa2b8 100644 --- a/tests/boundary/cli/analysisSelectorCli.test.ts +++ b/tests/boundary/cli/analysisSelectorCli.test.ts @@ -12,6 +12,38 @@ const selectors = [ ] as const; describe("CLI analysis selector boundaries", () => { + cliTest( + "accepts file offsets and rejects mixed coordinate selectors before opening a target", + async ({ cli }) => { + const selected = await cli.run({ + arguments: [ + "decompile", + "does-not-exist", + "--file-offset=0x210", + "--json", + ], + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect(selected.json).toMatchObject({ code: "target_unavailable" }); + const conflicting = await cli.run({ + arguments: [ + "decompile", + "does-not-exist", + "0x140001010", + "--file-offset=0x210", + "--json", + ], + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect(conflicting.json).toMatchObject({ + code: "invalid_request", + details: { + operation: "procedure_pseudo_code", + issues: [{ path: ["file-offset"], reason: "invalid_value" }], + }, + }); + }, + ); cliTest( "rejects missing and conflicting selectors before opening a target", async ({ cli }) => { diff --git a/tests/boundary/filesystem/ciChangeScope.test.ts b/tests/boundary/filesystem/ciChangeScope.test.ts index 9a3f7515b8..8ed397e845 100644 --- a/tests/boundary/filesystem/ciChangeScope.test.ts +++ b/tests/boundary/filesystem/ciChangeScope.test.ts @@ -8,6 +8,7 @@ import { writeFile, } from "node:fs/promises"; import { dirname, join, matchesGlob } from "node:path"; +import { fileURLToPath } from "node:url"; import { afterAll, beforeAll, expect, it } from "vitest"; import { parse } from "yaml"; @@ -85,7 +86,7 @@ async function classify( await writeFile(output, ""); const result = await execFileAsync( process.execPath, - [new URL("../../../scripts/ci/plan.mjs", import.meta.url).pathname], + [fileURLToPath(new URL("../../../scripts/ci/plan.mjs", import.meta.url))], { cwd: directory, env: { @@ -658,8 +659,9 @@ it.each([ ); it("rejects skipped selected work while allowing unselected lanes to skip", async () => { - const command = new URL("../../../scripts/ci/required.mjs", import.meta.url) - .pathname; + const command = fileURLToPath( + new URL("../../../scripts/ci/required.mjs", import.meta.url), + ); const jobs = { changes: { result: "success", outputs: { evm: "true" } }, static: { result: "success" }, diff --git a/tests/boundary/mcp/enhancedTools.04-uses-complete-inventories-inline.test.ts b/tests/boundary/mcp/enhancedTools.04-uses-complete-inventories-inline.test.ts index 8e3a399691..e452d70c36 100644 --- a/tests/boundary/mcp/enhancedTools.04-uses-complete-inventories-inline.test.ts +++ b/tests/boundary/mcp/enhancedTools.04-uses-complete-inventories-inline.test.ts @@ -144,14 +144,26 @@ describe("enhanced MCP tools", () => { ); expect(result).toEqual({ document: "fixture", + image_base: "0x1000", segments: [ - { name: "__TEXT", start: "0x1000", end: "0x1800", length: 2048 }, - { name: "__DATA", start: "0x1800", end: "0x2000", length: 2048 }, + { + name: "__TEXT", + start: "0x1000", + end: "0x1800", + length: 2048, + image_base: "0x1000", + }, + { + name: "__DATA", + start: "0x1800", + end: "0x2000", + length: 2048, + image_base: "0x1000", + }, ], segment_count: 2, procedure_count: 2, string_count: 700, - image_base: "0x1000", }); }); diff --git a/tests/boundary/mcp/nativeAddressContextMcp.test.ts b/tests/boundary/mcp/nativeAddressContextMcp.test.ts new file mode 100644 index 0000000000..5b855fc26a --- /dev/null +++ b/tests/boundary/mcp/nativeAddressContextMcp.test.ts @@ -0,0 +1,121 @@ +import { afterEach, expect, it } from "vitest"; +import { Ajv2020 } from "ajv/dist/2020.js"; +import type { AnySchemaObject } from "ajv"; +import { + closeEnhancedToolResources, + connect, + jsonResult, +} from "./enhancedToolsHarness.js"; +import { observed as ok } from "../../fixtures/analysisExecution.js"; +import type { SegmentSummary } from "../../../src/domain/hopperValues.js"; + +afterEach(closeEnhancedToolResources); + +const segment = { + name: ".text", + start: "0x101000", + end: "0x101100", + readable: true, + writable: false, + executable: true, + image_base: "0x100000", + address_space: "ram", +}; +const overviewClient = (segments: SegmentSummary[]) => + connect({ + execute: (name) => { + switch (name) { + case "current_document": + return Promise.resolve(ok("fixture")); + case "list_documents": + return Promise.resolve(ok(["fixture"])); + case "list_segments": + return Promise.resolve(ok(segments)); + case "list_procedures": + return Promise.resolve(ok([{ address: "0x101013", value: "entry" }])); + case "list_strings": + return Promise.resolve(ok([])); + default: + throw new Error(`Unexpected operation: ${name}`); + } + }, + }); + +it("preserves observed coordinates and main's optional-string output contract after SDK conversion", async () => { + const client = await overviewClient([segment]); + const schema = (await client.listTools()).tools.find( + ({ name }) => name === "binary_overview", + )?.outputSchema; + if (schema === undefined) throw new Error("Missing overview output schema"); + const validate = new Ajv2020({ strict: false }).compile( + schema as AnySchemaObject, + ); + const response = await client.callTool({ + name: "binary_overview", + arguments: {}, + }); + expect(validate(response.structuredContent)).toBe(true); + expect(jsonResult(response)).toMatchObject({ + image_base: "0x100000", + segments: [ + { start: "0x101000", image_base: "0x100000", address_space: "ram" }, + ], + }); + const absent = JSON.parse(JSON.stringify(response.structuredContent)); + delete absent.normalized_result.image_base; + expect(validate(absent)).toBe(true); + absent.normalized_result.image_base = null; + expect(validate(absent)).toBe(false); +}); + +it("omits unreported bases without inventing zero and preserves explicit zero", async () => { + const { image_base: _base, ...unreported } = segment; + const client = await overviewClient([unreported]); + expect( + jsonResult( + await client.callTool({ name: "binary_overview", arguments: {} }), + ), + ).not.toHaveProperty("image_base"); + const zero = await overviewClient([{ ...segment, image_base: "0x0" }]); + expect( + jsonResult(await zero.callTool({ name: "binary_overview", arguments: {} })), + ).toMatchObject({ image_base: "0x0" }); +}); + +it("keeps a partially reported base unknown while retaining the reported segment observation", async () => { + const { image_base: _base, ...unreported } = segment; + const client = await overviewClient([ + segment, + { ...unreported, name: ".data" }, + ]); + const result = jsonResult( + await client.callTool({ name: "binary_overview", arguments: {} }), + ); + expect(result).not.toHaveProperty("image_base"); + expect(result).toMatchObject({ + segments: [{ image_base: "0x100000" }, { name: ".data" }], + }); +}); + +it("retains all conflicting observations after repeated early bases without selecting one", async () => { + const { address_space: _space, ...unreported } = segment; + const client = await overviewClient([ + ...Array.from({ length: 40 }, () => unreported), + { + ...segment, + name: ".data", + start: "other%20space:0x201000", + end: "other%20space:0x201100", + image_base: "0x200000", + address_space: "other space", + }, + ]); + const result = jsonResult( + await client.callTool({ name: "binary_overview", arguments: {} }), + ); + expect(result).not.toHaveProperty("image_base"); + expect(result).toHaveProperty("segments.length", 41); + expect(result).toHaveProperty("segments.40.image_base", "0x200000"); + expect(result).toHaveProperty("segments.40.address_space", "other space"); + expect(result).not.toHaveProperty("segments.0.address_space"); +}); diff --git a/tests/conformance/c/pie-address-context.c b/tests/conformance/c/pie-address-context.c new file mode 100644 index 0000000000..970d2fc6ed --- /dev/null +++ b/tests/conformance/c/pie-address-context.c @@ -0,0 +1,7 @@ +__attribute__((noinline)) int rea_pie_add(int value) { + return value + 7; +} + +int rea_pie_entry(void) { + return rea_pie_add(35); +} diff --git a/tests/conformance/ghidra/ReaFileOffsetSourceProbe.java b/tests/conformance/ghidra/ReaFileOffsetSourceProbe.java new file mode 100644 index 0000000000..57da9d28ef --- /dev/null +++ b/tests/conformance/ghidra/ReaFileOffsetSourceProbe.java @@ -0,0 +1,118 @@ +//@category REA Verification + +import java.io.ByteArrayInputStream; +import java.io.PrintWriter; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.Locale; + +import com.google.gson.JsonNull; +import com.google.gson.JsonObject; +import ghidra.app.script.GhidraScriptUtil; +import ghidra.app.util.headless.HeadlessScript; +import ghidra.program.model.address.AddressSet; +import ghidra.program.model.listing.Function; +import ghidra.program.model.symbol.SourceType; +import ghidra.app.decompiler.DecompInterface; +import ghidra.app.decompiler.DecompileResults; +import ghidra.app.decompiler.DecompileProcess.DisposeState; +import ghidra.util.task.TaskMonitor; + +/** Real source-identity cases unavailable through ordinary single-file CLI imports. */ +public final class ReaFileOffsetSourceProbe extends HeadlessScript { + @Override + public void run() throws Exception { + var file = GhidraScriptUtil.findScriptByName("ReaGhidraBridge.java"); + var bridge = GhidraScriptUtil.getProvider(file).getScriptInstance(file, new PrintWriter(System.out, true)); + bridge.set(state, monitor, new PrintWriter(System.out, true)); + var space = bridge.getClass().getDeclaredField("sessionDefaultAddressSpace"); + space.setAccessible(true); + space.set(null, currentProgram.getAddressFactory().getDefaultAddressSpace()); + var identity = bridge.getClass().getDeclaredField("admittedTargetSha256"); + identity.setAccessible(true); + identity.set(bridge, currentProgram.getExecutableSHA256().toLowerCase(Locale.ROOT)); + var resolve = bridge.getClass().getDeclaredMethod("resolveDecompileProcedure", String.class, JsonObject.class); + resolve.setAccessible(true); + var reverse = bridge.getClass().getDeclaredMethod("addressToFileOffset", JsonObject.class); + reverse.setAccessible(true); + + var memory = currentProgram.getMemory(); + var original = memory.getAllFileBytes().get(0); + byte[] bytes = new byte[Math.toIntExact(original.getSize())]; + require(original.getOriginalBytes(0, bytes) == bytes.length, "complete fixture source"); + byte[] auxiliary = new byte[0x1000]; + auxiliary[0x910] = (byte)0xc3; + var foreign = memory.createFileBytes("auxiliary.bin", 0, auxiliary.length, + new ByteArrayInputStream(auxiliary), monitor); + var address = toAddr(0x140100000L); + memory.createInitializedBlock("auxiliary", address, foreign, 0, auxiliary.length, false); + var entry = address.add(0x910); + currentProgram.getFunctionManager().createFunction("foreign_only", entry, + new AddressSet(entry), SourceType.USER_DEFINED); + rejected(resolve, bridge, "no loaded source mapping", "file-offset:0x910", new JsonObject()); + rejected(reverse, bridge, "0 authoritative target-file mappings", reverseInput("0x140100910")); + + // A source object's declared original-file start does not make its local + // offsets a complete, digest-matched target representation. + var partial = memory.createFileBytes("partial.bin", 0x200, 0x200, + new ByteArrayInputStream(bytes, 0x200, 0x200), monitor); + memory.createInitializedBlock("partial", toAddr(0x140300000L), partial, 0, 0x200, false); + rejected(reverse, bridge, "0 authoritative target-file mappings", reverseInput("0x140300010")); + var selected = (Function)resolve.invoke(bridge, "file-offset:0x210", new JsonObject()); + require(selected.getEntryPoint().equals(toAddr(0x140001010L)), "original target selected despite other sources"); + var mapped = (JsonObject)reverse.invoke(bridge, reverseInput("0x140001010")); + require(mapped.get("file_offset").getAsLong() == 0x210, "original target round trip"); + + // Inject backend failures after real source/function resolution. This + // verifies context propagation, not a naturally failing decompiler. + var pseudocode = bridge.getClass().getDeclaredMethod("procedurePseudocode", JsonObject.class); + pseudocode.setAccessible(true); + var decompiler = bridge.getClass().getDeclaredField("decompiler"); + decompiler.setAccessible(true); + JsonObject request = new JsonObject(); + request.add("document", JsonNull.INSTANCE); + request.addProperty("procedure", "file-offset:0x210"); + for (DisposeState disposition : new DisposeState[] { DisposeState.NOT_DISPOSED, DisposeState.DISPOSED_ON_CANCEL }) { + decompiler.set(bridge, new DecompInterface() { + @Override + public DecompileResults decompileFunction(Function function, int timeout, TaskMonitor taskMonitor) { + return new DecompileResults(function, currentProgram.getLanguage(), + currentProgram.getCompilerSpec(), null, "Injected backend failure", null, disposition); + } + }); + Throwable failure = rejected(pseudocode, bridge, + "Selector file-offset:0x210 mapped to analysis address 0x140001010 and procedure 0x140001010", request); + var code = failure.getClass().getDeclaredField("code"); + code.setAccessible(true); + require(code.get(failure).equals(disposition == DisposeState.DISPOSED_ON_CANCEL + ? "decompile_cancelled" : "decompile_failed"), "backend failure code retained"); + } + + var duplicate = memory.createFileBytes("duplicate.bin", 0, bytes.length, + new ByteArrayInputStream(bytes), monitor); + memory.createInitializedBlock("duplicate", toAddr(0x140201000L), duplicate, 0x200, 0x200, false); + rejected(resolve, bridge, "multiple loaded mappings", "file-offset:0x210", new JsonObject()); + println("REA_FILE_OFFSET_SOURCE_OK foreign/partial rejected; original round trip; duplicate ambiguity; injected backend failure context retained"); + } + + private JsonObject reverseInput(String address) { + JsonObject params = new JsonObject(); + params.add("document", JsonNull.INSTANCE); + params.addProperty("address", address); + return params; + } + + private Throwable rejected(Method method, Object bridge, String reason, Object... arguments) throws Exception { + try { + method.invoke(bridge, arguments); + } catch (InvocationTargetException rejected) { + require(rejected.getCause().getMessage().contains(reason), "expected mapping failure: " + reason); + return rejected.getCause(); + } + throw new AssertionError("Expected mapping failure: " + reason); + } + + private void require(boolean condition, String message) { + if (!condition) throw new AssertionError(message); + } +} diff --git a/tests/evaluation/package/nativeAotBuild.test.ts b/tests/evaluation/package/nativeAotBuild.test.ts new file mode 100644 index 0000000000..6e71db42a7 --- /dev/null +++ b/tests/evaluation/package/nativeAotBuild.test.ts @@ -0,0 +1,90 @@ +import { join } from "node:path"; + +import { expect, it, vi } from "vitest"; + +import { buildNativeAotFixture } from "../../../scripts/lib/nativeaot-build.mjs"; + +const options = { + dotnet: "dotnet", + project: "fixture.csproj", + runtime: "win-x64", + output: "fixture-output", +}; + +it("restores the exact publish target before compiling without network restore", async () => { + const run = vi.fn<(command: string, args: string[]) => void>(); + await buildNativeAotFixture(run, options); + expect(run).toHaveBeenCalledTimes(2); + const restore = run.mock.calls[0]?.[1] ?? []; + const publish = run.mock.calls[1]?.[1] ?? []; + expect(restore[0]).toBe("restore"); + expect(publish[0]).toBe("publish"); + expect(publish).toContain("--no-restore"); + for (const argument of [ + options.project, + options.runtime, + "-p:Configuration=Release", + `-p:BaseIntermediateOutputPath=${join(options.output, "obj")}/`, + ]) { + expect(restore).toContain(argument); + expect(publish).toContain(argument); + } +}); + +it("retries without logging credential-bearing restore diagnostics", async () => { + const logged = vi.spyOn(console, "error").mockImplementation(() => {}); + const timeout = Object.assign( + new Error( + "dotnet restore ETIMEDOUT https://fixture-user:fixture-password@feed.invalid/v3/index.json", + ), + { + code: "ETIMEDOUT", + }, + ); + const run = vi + .fn<(command: string, args: string[]) => void>() + .mockImplementationOnce(() => { + throw timeout; + }); + try { + await buildNativeAotFixture(run, options); + expect( + logged.mock.calls.flat().every((value) => typeof value === "string"), + ).toBe(true); + expect(logged.mock.calls.flat().join("\n")).not.toContain( + "fixture-password", + ); + } finally { + logged.mockRestore(); + } + expect(run.mock.calls.map((call) => call[1][0])).toEqual([ + "restore", + "restore", + "publish", + ]); +}); + +it("stops after two restore failures and preserves the final diagnostic", async () => { + const failure = new Error("NuGet source unavailable"); + const run = vi.fn<(command: string, args: string[]) => void>(() => { + throw failure; + }); + await expect(buildNativeAotFixture(run, options)).rejects.toMatchObject({ + message: "NativeAOT dependency restore failed after two attempts", + cause: failure, + }); + expect(run).toHaveBeenCalledTimes(2); + expect(run.mock.calls.every((call) => call[1][0] === "restore")).toBe(true); +}); + +it("does not retry compilation failures", async () => { + const failure = new Error("Native compiler rejected the fixture"); + const run = vi + .fn<(command: string, args: string[]) => void>() + .mockImplementationOnce(() => {}) + .mockImplementationOnce(() => { + throw failure; + }); + await expect(buildNativeAotFixture(run, options)).rejects.toBe(failure); + expect(run).toHaveBeenCalledTimes(2); +}); diff --git a/tests/evaluation/package/nativeAotDiagnostics.test.ts b/tests/evaluation/package/nativeAotDiagnostics.test.ts new file mode 100644 index 0000000000..f5b42a5ce7 --- /dev/null +++ b/tests/evaluation/package/nativeAotDiagnostics.test.ts @@ -0,0 +1,150 @@ +import { execFile } from "node:child_process"; +import { readdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + +import { expect, it } from "vitest"; + +import { + nativeAotCommandError, + supportsNativeAotFixtureHost, +} from "../../../scripts/lib/nativeaot-build.mjs"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const generator = fileURLToPath( + new URL("../../../scripts/generate-nativeaot-fixtures.mjs", import.meta.url), +); + +const supportedHost = supportsNativeAotFixtureHost( + process.platform, + process.arch, +); + +it.skipIf(!supportedHost).each(["restore", "publish"])( + "redacts authenticated feed URLs from the uncaught %s diagnostic", + async (operation) => { + const directory = await createTestTempDirectory( + "rea-nativeaot-diagnostic-", + ); + const hook = join(directory, "child-output.cjs"); + await writeFile( + hook, + ` +const child = require("node:child_process"); +child.spawnSync = (_command, args) => { + if (args[0] === "--version") return { status: 0, stdout: "8.0.416\\n", stderr: "" }; + if (args[0] === ${JSON.stringify(operation)}) return { + status: 1, + error: new Error("failed HTTPS://error-user:error-secret@feed.invalid/v3/index.json"), + stdout: "NU1301 https://stdout-user:stdout-secret@feed.invalid/v3/index.json\\n", + stderr: "source https://stderr-user:stderr-secret%40suffix@feed.invalid/v3/index.json\\nkeep https://feed.invalid/path/user@name?label=token-name\\n" + }; + return { status: 0, stdout: "", stderr: "" }; +}; +require("node:module").syncBuiltinESMExports(); +`, + "utf8", + ); + const failure = await promisify(execFile)( + process.execPath, + ["--require", hook, generator], + { + env: { + ...process.env, + REA_NATIVEAOT_FIXTURE_ROOT: join(directory, "output"), + }, + timeout: 15_000, + }, + ).then( + () => { + throw new Error("Expected fixture generation to fail"); + }, + (error: { stderr: string; code: number }) => error, + ); + expect(failure.code).toBe(1); + for (const secret of [ + "error-user", + "error-secret", + "stdout-user", + "stdout-secret", + "stderr-user", + "stderr-secret", + ]) + expect(failure.stderr).not.toContain(secret); + expect(failure.stderr).toContain("NU1301"); + expect(failure.stderr).toContain("feed.invalid/v3/index.json"); + expect(failure.stderr).toContain( + "https://feed.invalid/path/user@name?label=token-name", + ); + if (operation === "restore") { + expect(failure.stderr).toContain("attempt 2/2"); + expect(failure.stderr).toContain("failed after two attempts"); + expect(failure.stderr).toContain("[cause]"); + } + }, +); + +it.skipIf(supportedHost)( + "refuses unsupported fixture hosts before tooling or output creation", + async () => { + const directory = await createTestTempDirectory("rea-nativeaot-host-"); + const hook = join(directory, "unexpected-child.cjs"); + await writeFile( + hook, + `const child = require("node:child_process"); +child.spawnSync = () => { throw new Error("Unexpected fixture tooling invocation"); }; +require("node:module").syncBuiltinESMExports(); +`, + "utf8", + ); + const failure = await promisify(execFile)( + process.execPath, + ["--require", hook, generator], + { + env: { + ...process.env, + REA_NATIVEAOT_FIXTURE_ROOT: join(directory, "output"), + }, + timeout: 15_000, + }, + ).then( + () => { + throw new Error("Expected unsupported fixture host refusal"); + }, + (error: { stderr: string; code: number }) => error, + ); + expect(failure.code).toBe(1); + expect(failure.stderr).toContain("requires Linux or Windows x64"); + expect(failure.stderr).not.toContain( + "Unexpected fixture tooling invocation", + ); + expect(await readdir(directory)).toEqual(["unexpected-child.cjs"]); + }, +); + +it("preserves diagnostics without transport authentication exactly", () => { + const error = nativeAotCommandError( + "C:/token projects/dotnet.exe", + "restore", + { + status: 1, + stdout: "NU1301 https://feed.invalid/user@name?label=password\n", + stderr: "project C:/token projects/fixture.csproj unavailable", + }, + ); + expect(error.message).toBe( + "NativeAOT fixture lane requires/action failed C:/token projects/dotnet.exe restore: exit 1\nNU1301 https://feed.invalid/user@name?label=password\nproject C:/token projects/fixture.csproj unavailable", + ); + expect(error.cause).toBeUndefined(); +}); + +it("treats HTTP backslashes as path separators rather than authentication", () => { + const output = String.raw`https://feed.invalid\path\user@name`; + expect(new URL(output).username).toBe(""); + const error = nativeAotCommandError("dotnet", "restore", { + status: 1, + stdout: output, + }); + expect(error.message).toContain(output); +});