From a84e0cfcfe0035094845946088d470c5bdd8cb3d Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Mon, 26 Oct 2026 23:12:14 +0800 Subject: [PATCH 01/25] refactor(verifiers): consolidate capture result admission and parity --- scripts/lib/mcp-verifier-results.mjs | 19 +++---- scripts/verify-http3-network-captures.mjs | 7 +-- scripts/verify-packet-capture.mjs | 41 +++++++------- scripts/verify-qlog-captures.mjs | 22 +++++--- scripts/verify-signal-recording.mjs | 29 ++++++---- scripts/verify-web-network-captures.mjs | 28 +++++----- .../process/mcpVerifierResults.test.ts | 56 +++++++++++++++++++ 7 files changed, 134 insertions(+), 68 deletions(-) diff --git a/scripts/lib/mcp-verifier-results.mjs b/scripts/lib/mcp-verifier-results.mjs index b179ac01cc..a9d10876a9 100644 --- a/scripts/lib/mcp-verifier-results.mjs +++ b/scripts/lib/mcp-verifier-results.mjs @@ -38,7 +38,15 @@ export const requireMcpOutput = ( throw new Error(`${operation} failed: ${mcpTextValue(result)}`); if (contract.name !== operation) throw new Error(`${operation} received the ${contract.name} contract`); - return contract.outputSchema.parse(jsonValue(result)); + const text = jsonValue(result); + const output = contract.outputSchema.parse(text); + if (result.structuredContent !== undefined) { + // The producer emits both representations from the same JSON value. + // Compare before schema projection can discard disagreeing source fields. + if (!isDeepStrictEqual(text, result.structuredContent)) + throw new Error(`${operation} text and structured results disagree`); + } + return output; }; const operationResultKey = (contract) => { @@ -124,15 +132,6 @@ export const requireMcpEvidenceBundle = (result) => { throw new Error( "get_evidence_bundle returned summary metadata instead of complete Evidence", ); - if (result.structuredContent !== undefined) { - const structured = contract.outputSchema.parse( - result.structuredContent, - ).result; - if (!isDeepStrictEqual(bundle, structured)) - throw new Error( - "get_evidence_bundle text and structured results disagree", - ); - } return authenticateParsedEvidenceBundle(bundle); }; diff --git a/scripts/verify-http3-network-captures.mjs b/scripts/verify-http3-network-captures.mjs index c601cac8bc..55997a95dc 100644 --- a/scripts/verify-http3-network-captures.mjs +++ b/scripts/verify-http3-network-captures.mjs @@ -26,7 +26,7 @@ import { } from "./lib/http3-verifier-lifecycle.mjs"; import { requireMcpEvidenceBundle, - mcpTextValue, + requireMcpOutput, } from "./lib/mcp-verifier-results.mjs"; const root = process.argv[2]; @@ -137,14 +137,11 @@ async function mcp(input, name) { join(directory, `${name}.mcp.reply.json`), JSON.stringify(reply, null, 2), ); - const value = JSON.parse(mcpTextValue(reply)); - assert.notEqual(reply.isError, true, mcpTextValue(reply)); + const evidence = requireMcpOutput(reply, contract.name, contract); assert.ok( validateOutput(reply.structuredContent), JSON.stringify(validateOutput.errors), ); - const evidence = contract.outputSchema.parse(value); - assert.deepEqual(reply.structuredContent, value); retainedEvidence.push(evidence); return evidence; } diff --git a/scripts/verify-packet-capture.mjs b/scripts/verify-packet-capture.mjs index 91ad2a8829..9e80b4ea8b 100644 --- a/scripts/verify-packet-capture.mjs +++ b/scripts/verify-packet-capture.mjs @@ -1,6 +1,10 @@ #!/usr/bin/env node import assert from "node:assert/strict"; -import { requireMcpEvidenceBundle } from "./lib/mcp-verifier-results.mjs"; +import { + requireMcpEvidenceBundle, + requireMcpOutput, + requireMcpToolError, +} from "./lib/mcp-verifier-results.mjs"; import { estimateTokenCount } from "tokenx"; import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; @@ -124,12 +128,13 @@ await withVerifierCleanup(async () => { name: tool.name, arguments: input, }); - assert.notEqual(response.isError, true, JSON.stringify(response)); assert.ok( ajv.validate(tool.outputSchema, response.structuredContent), JSON.stringify(ajv.errors), ); - const evidence = parseEvidence(response.structuredContent); + const evidence = parseEvidence( + requireMcpOutput(response, "inspect_packet_capture"), + ); const report = evidence.normalized_result; assert.equal( report.artifact.sha256, @@ -305,12 +310,13 @@ await withVerifierCleanup(async () => { name: pcapTool.name, arguments: { path: capturePath, detail: "summary" }, }); - assert.notEqual(response.isError, true, JSON.stringify(response)); assert.ok( pcapAjv.validate(pcapTool.outputSchema, response.structuredContent), pcapAjv.errorsText(), ); - const summary = parseEvidence(response.structuredContent).normalized_result; + const summary = parseEvidence( + requireMcpOutput(response, "inspect_pcap_capture"), + ).normalized_result; assert.equal(summary.summary.capture_start, expectedCaptureStart); assert.equal(summary.summary.capture_end, expectedCaptureEnd); assert.equal( @@ -343,10 +349,9 @@ await withVerifierCleanup(async () => { view: { kind: "pcap", facet: "protocols", limit: 10 }, }, }); - assert.notEqual(page.isError, true, JSON.stringify(page)); assert.equal( - parseEvidence(page.structuredContent).normalized_result - .parent_evidence_id, + parseEvidence(requireMcpOutput(page, "inspect_analysis_view")) + .normalized_result.parent_evidence_id, source.evidence_id, ); cases++; @@ -368,19 +373,16 @@ await withVerifierCleanup(async () => { name: tool.name, arguments: input, }); - assert.equal(response.isError, true); + requireMcpToolError(response); await client.ping(); cases++; } await writeFile(path, "not a capture"); - assert.equal( - ( - await client.callTool({ - name: tool.name, - arguments: { path, fields: ["frame.number"] }, - }) - ).isError, - true, + requireMcpToolError( + await client.callTool({ + name: tool.name, + arguments: { path, fields: ["frame.number"] }, + }), ); cases++; const discovery = (await client.listTools()).tools.find( @@ -414,12 +416,13 @@ await withVerifierCleanup(async () => { name: discovery.name, arguments: input, }); - assert.notEqual(response.isError, true, JSON.stringify(response)); assert.ok( ajv.validate(discovery.outputSchema, response.structuredContent), JSON.stringify(ajv.errors), ); - const evidence = parseEvidence(response.structuredContent); + const evidence = parseEvidence( + requireMcpOutput(response, "inspect_packet_fields"), + ); const report = evidence.normalized_result; const kind = input.kind ?? "protocols"; const query = (input.query ?? "").toLowerCase(); diff --git a/scripts/verify-qlog-captures.mjs b/scripts/verify-qlog-captures.mjs index 8d1a16f6cd..3e82414b8d 100644 --- a/scripts/verify-qlog-captures.mjs +++ b/scripts/verify-qlog-captures.mjs @@ -10,6 +10,11 @@ import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { Ajv2020 } from "ajv/dist/2020.js"; import { parseEvidence } from "../dist/domain/evidence.js"; +import { toolContract } from "../dist/contracts/toolContracts.js"; +import { + requireMcpOutput, + requireMcpToolError, +} from "./lib/mcp-verifier-results.mjs"; // Caller-supplied producer evidence stays local; no fixture capture or secrets are bundled. const [controlPath, lossPath, reportPath] = process.argv.slice(2); @@ -40,6 +45,7 @@ transport.stderr?.on("data", (bytes) => { }); const ajv = new Ajv2020({ strict: false, validateFormats: false }); const measurements = []; +const contract = toolContract("inspect_web_network_capture"); try { try { await client.connect(transport); @@ -68,12 +74,13 @@ try { name: tool.name, arguments: input, }); - assert.notEqual(response.isError, true); + const mcp = parseEvidence( + requireMcpOutput(response, contract.name, contract), + ); assert.ok( validateOutput(response.structuredContent), JSON.stringify(validateOutput.errors), ); - const mcp = parseEvidence(response.structuredContent); const completed = await promisify(execFile)( process.execPath, [entrypoint, "inspect-web-network-capture", path, "qlog", "--json"], @@ -153,17 +160,14 @@ try { view: { kind: "api-surface" }, }, }); - assert.equal(retained.isError, true); - assert.equal( - JSON.parse(retained.content.find((item) => item.type === "text").text) - .error.code, - "unsupported_target", - ); + assert.equal(requireMcpToolError(retained).code, "unsupported_target"); const selected = await client.callTool({ name: tool.name, arguments: { ...input, record_ordinals: [capture.total_records - 1, 0] }, }); - const subset = parseEvidence(selected.structuredContent).normalized_result; + const subset = parseEvidence( + requireMcpOutput(selected, contract.name, contract), + ).normalized_result; assert.deepEqual( subset.records.map((record) => record.ordinal), [capture.total_records - 1, 0], diff --git a/scripts/verify-signal-recording.mjs b/scripts/verify-signal-recording.mjs index 1cafab175c..61d3afd831 100644 --- a/scripts/verify-signal-recording.mjs +++ b/scripts/verify-signal-recording.mjs @@ -1,6 +1,10 @@ #!/usr/bin/env node import assert from "node:assert/strict"; -import { requireMcpEvidenceBundle } from "./lib/mcp-verifier-results.mjs"; +import { + requireMcpEvidenceBundle, + requireMcpOutput, + requireMcpToolError, +} from "./lib/mcp-verifier-results.mjs"; import { estimateTokenCount } from "tokenx"; import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; @@ -47,9 +51,7 @@ const assertRejectedSelector = ( field, expectedExitCode = 1, ) => { - assert.equal(response.isError, true); - const text = response.content.find((item) => item.type === "text")?.text; - const { error } = JSON.parse(text ?? "{}"); + const error = requireMcpToolError(response); assert.equal(error.code, "invalid_request"); assert.equal(error.details.operation, operation); assert.deepEqual( @@ -123,12 +125,13 @@ await withVerifierCleanup(async () => { name: advertised.name, arguments: input, }); - assert.notEqual(response.isError, true, JSON.stringify(response)); assert.ok( ajv.validate(advertised.outputSchema, response.structuredContent), JSON.stringify(ajv.errors), ); - const evidence = parseEvidence(response.structuredContent); + const evidence = parseEvidence( + requireMcpOutput(response, "decode_signal_recording"), + ); const report = evidence.normalized_result; for (const byte of ["52", "45", "41"]) assert.match( @@ -238,12 +241,13 @@ await withVerifierCleanup(async () => { name: advertised.name, arguments: input, }); - assert.notEqual(response.isError, true, JSON.stringify(response)); assert.ok( ajv.validate(advertised.outputSchema, response.structuredContent), JSON.stringify(ajv.errors), ); - const evidence = parseEvidence(response.structuredContent); + const evidence = parseEvidence( + requireMcpOutput(response, "decode_signal_recording"), + ); const report = evidence.normalized_result; assert.deepEqual(report.output_window, { total_lines: 120, @@ -298,9 +302,9 @@ await withVerifierCleanup(async () => { name: advertised.name, arguments: input, }); - assert.equal(response.isError, true); if (rejectedField !== undefined) assertRejectedSelector(response, advertised.name, rejectedField); + else requireMcpToolError(response); await client.ping(); cases++; } @@ -309,7 +313,7 @@ await withVerifierCleanup(async () => { name: advertised.name, arguments: { path, decoders: "uart" }, }); - assert.equal(invalid.isError, true); + requireMcpToolError(invalid); cases++; const discoveryTool = (await client.listTools()).tools.find( ({ name }) => name === "inspect_signal_decoders", @@ -367,12 +371,13 @@ await withVerifierCleanup(async () => { name: discoveryTool.name, arguments: input, }); - assert.notEqual(response.isError, true, JSON.stringify(response)); assert.ok( ajv.validate(discoveryTool.outputSchema, response.structuredContent), JSON.stringify(ajv.errors), ); - const evidence = parseEvidence(response.structuredContent); + const evidence = parseEvidence( + requireMcpOutput(response, "inspect_signal_decoders"), + ); const report = evidence.normalized_result; discoveryMeasurements.push({ input, diff --git a/scripts/verify-web-network-captures.mjs b/scripts/verify-web-network-captures.mjs index bd64b1ea80..3c8d813f30 100644 --- a/scripts/verify-web-network-captures.mjs +++ b/scripts/verify-web-network-captures.mjs @@ -18,7 +18,11 @@ import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { PrivateRuntimeRoot } from "../dist/process/PrivateRuntimeRoot.js"; import { runOwnedCommand } from "../dist/process/OwnedCommand.js"; import { WEB_NETWORK_CAPTURE_LIMITS } from "../dist/domain/webNetworkCapture.js"; -import { mcpTextValue } from "./lib/mcp-verifier-results.mjs"; +import { + mcpTextValue, + requireMcpOutput, + requireMcpToolError, +} from "./lib/mcp-verifier-results.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; const executable = process.env.REA_MITMDUMP_COMMAND; @@ -903,15 +907,15 @@ async function inspect(mode, input, errorCategory) { name: "inspect_web_network_capture", arguments: input, }); - const value = JSON.parse(mcpTextValue(result)); if (errorCategory !== undefined) { - assert.equal(result.isError, true); - assert.equal(value.error.category, errorCategory); + const error = requireMcpToolError(result); + assert.equal(error.category, errorCategory); + const rawError = JSON.parse(mcpTextValue(result)).error; for (const literal of input.sensitive_values ?? []) - assert.ok(!JSON.stringify(value.error).includes(literal)); - return value.error; + assert.ok(!JSON.stringify(rawError).includes(literal)); + return error; } - assert.notEqual(result.isError, true, mcpTextValue(result)); + const value = requireMcpOutput(result, "inspect_web_network_capture"); assertSensitiveLimitations(value, input.sensitive_values ?? []); if (input.format === "har") { const surface = await client.callTool({ @@ -923,13 +927,12 @@ async function inspect(mode, input, errorCategory) { }); if ((input.sensitive_values ?? []).length > 0) { // Derived view text cannot be checked against undisclosed declarations. - assert.equal(surface.isError, true); + requireMcpToolError(surface); const refused = JSON.parse(mcpTextValue(surface)); for (const literal of input.sensitive_values) assert.ok(!JSON.stringify(refused).includes(literal)); } else { - assert.notEqual(surface.isError, true, mcpTextValue(surface)); - const projected = JSON.parse(mcpTextValue(surface)); + const projected = requireMcpOutput(surface, "inspect_analysis_view"); assert.equal(projected.normalized_result.kind, "api-surface"); assert.equal( projected.normalized_result.records.examined, @@ -944,13 +947,12 @@ async function inspect(mode, input, errorCategory) { }, }); if ((input.sensitive_values ?? []).length > 0) { - assert.equal(manifest.isError, true); + requireMcpToolError(manifest); const refusedManifest = JSON.parse(mcpTextValue(manifest)); for (const literal of input.sensitive_values) assert.ok(!JSON.stringify(refusedManifest).includes(literal)); } else { - assert.notEqual(manifest.isError, true, mcpTextValue(manifest)); - const projected = JSON.parse(mcpTextValue(manifest)); + const projected = requireMcpOutput(manifest, "inspect_analysis_view"); assert.equal(projected.normalized_result.kind, "environment-manifest"); assert.equal(projected.normalized_result.unknowns.length, 3); for (const record of projected.normalized_result.records) diff --git a/tests/boundary/process/mcpVerifierResults.test.ts b/tests/boundary/process/mcpVerifierResults.test.ts index 9cda004246..1f0e4d5619 100644 --- a/tests/boundary/process/mcpVerifierResults.test.ts +++ b/tests/boundary/process/mcpVerifierResults.test.ts @@ -219,6 +219,62 @@ it("returns complete named Evidence with its subject, provider and analysis prov ); }); +it("admits matching complete text and structured Evidence and rejects either malformed shape", async () => { + const { requireMcpOutput } = await verifierResults(); + const evidence = namedEvidence(); + expect( + requireMcpOutput( + { ...textReply(evidence), structuredContent: evidence }, + "address_name", + ), + ).toEqual(evidence); + const malformed = { ...evidence, normalized_result: { name: "entry" } }; + for (const [text, structuredContent] of [ + [evidence, malformed], + [malformed, evidence], + ]) + expect(() => + requireMcpOutput( + { ...textReply(text), structuredContent }, + "address_name", + ), + ).toThrow(); +}); + +it("rejects disagreement in lifecycle envelope fields that the named schema would otherwise discard", async () => { + const { requireMcpOutput } = await verifierResults(); + const result = retainedBundle(); + const text = { result, vendor_note: "text observation" }; + const structuredContent = { result, vendor_note: "different observation" }; + const contract = toolContract("get_evidence_bundle"); + expect(contract.outputSchema.parse(text)).toEqual( + contract.outputSchema.parse(structuredContent), + ); + expect(() => + requireMcpOutput( + { ...textReply(text), structuredContent }, + "get_evidence_bundle", + ), + ).toThrow("text and structured results disagree"); +}); + +it("rejects disagreement in complete Evidence provenance before downstream extraction", async () => { + const { requireMcpOutput } = await verifierResults(); + const evidence = namedEvidence(); + expect(() => + requireMcpOutput( + { + ...textReply(evidence), + structuredContent: { + ...evidence, + raw_result: { engine_name: "different" }, + }, + }, + "address_name", + ), + ).toThrow("text and structured results disagree"); +}); + it("rejects mismatched named contracts, error-marked success and malformed normalized results", async () => { const { requireMcpOutput } = await verifierResults(); const evidence = namedEvidence(); From 625e32744ecf084a39c72daa042ca8abf90a15ba Mon Sep 17 00:00:00 2001 From: Wenyu Chiou <162016108+WenyuChiou@users.noreply.github.com> Date: Mon, 26 Oct 2026 11:16:28 -0400 Subject: [PATCH 02/25] fix(javascript): ignore async exec stdio options (#7087) Respect Node asynchronous exec/execFile method-default stdio instead of interpreting an ignored option. Remove the superseded callback-shape query while preserving caller input, Evidence, uncertainty, argv and fork controls. Independently validated on current main through actual Node pipe capture, matching real-app CLI/MCP facts, 81 targeted tests and static checking. Closes #7217. --- .../javascriptSemanticChildProcesses.ts | 9 +-- ...javascriptSemanticExpressionShapes.test.ts | 73 ++++++------------- .../javascriptSemanticExpressionShapes.ts | 20 +---- .../javascriptChildProcessStdioCli.test.ts | 63 ++++++++++++---- 4 files changed, 76 insertions(+), 89 deletions(-) diff --git a/src/domain/javascript/javascriptSemanticChildProcesses.ts b/src/domain/javascript/javascriptSemanticChildProcesses.ts index 61feb46e24..10722d67e5 100644 --- a/src/domain/javascript/javascriptSemanticChildProcesses.ts +++ b/src/domain/javascript/javascriptSemanticChildProcesses.ts @@ -416,14 +416,11 @@ const stdioMode = ( method: JavaScriptSemanticChildProcessSpawn["method"], context: ChildSpawnContext, ): string => { + // Async exec/execFile do not forward options.stdio to their underlying spawn. + if (method === "exec" || method === "execFile") return "default"; if (options === undefined) return "default"; const expression = unwrapJavaScriptExpression(options).node; - if ( - context.isNullish(expression) || - ((method === "exec" || method === "execFile") && - context.isCallback(expression)) - ) - return "default"; + if (context.isNullish(expression)) return "default"; const lookup = staticObjectProperty(expression, "stdio", context.isNullish); if (lookup.status === "missing") return unsetStdioMode(expression, method, context); diff --git a/src/domain/javascript/javascriptSemanticExpressionShapes.test.ts b/src/domain/javascript/javascriptSemanticExpressionShapes.test.ts index 45a1367a3d..e8e37d3079 100644 --- a/src/domain/javascript/javascriptSemanticExpressionShapes.test.ts +++ b/src/domain/javascript/javascriptSemanticExpressionShapes.test.ts @@ -13,9 +13,7 @@ import { } from "./javascriptSemanticTraversal.js"; it.each([ - ["function f(){read(cb);function cb(){}}", ["callback"]], - ["function f(){const cb=()=>{};const alias=cb;read(alias)}", ["callback"]], - ["function f(cb){cb=()=>{};arguments[0]={};read(cb)}", ["unknown"]], + ["function f(a){a=[];arguments[0]={};read(a)}", ["unknown"]], ["declare let erased:any;function f(){const a=[];read(a)}", ["unknown"]], ["namespace X{};function f(){const a=[];read(a)}", ["unknown"]], ["var a=[];(()=>{}).constructor('a={}')();read(a);", ["unknown"]], @@ -60,11 +58,9 @@ it.each([ shapes.push( resolver.isArray(arg) ? "array" - : resolver.isCallback(arg) - ? "callback" - : resolver.isNullish(arg) - ? "nullish" - : "unknown", + : resolver.isNullish(arg) + ? "nullish" + : "unknown", ); }, }); @@ -72,10 +68,10 @@ it.each([ }, ); -it("bounds long callback aliases without invoking the shared callable resolver", () => { - const source = `export function run(){const cb=()=>{}; - ${Array.from({ length: 4096 }, (_, i) => `const a${i}=${i === 0 ? "cb" : `a${i - 1}`};`).join("\n")} - read(a4095);read(a10);read(cb);}`; +it("bounds long array aliases without invoking the shared callable resolver", () => { + const source = `export function run(){const argv=[]; + ${Array.from({ length: 4096 }, (_, i) => `const a${i}=${i === 0 ? "argv" : `a${i - 1}`};`).join("\n")} + read(a4095);read(a10);read(argv);}`; const file = parseJavaScriptSource(source, "deep.mjs"); if (file === null) throw new Error("Expected valid deep source"); const state = completeSemanticSteps( @@ -89,7 +85,7 @@ it("bounds long callback aliases without invoking the shared callable resolver", t.isCallExpression(node) && t.isIdentifier(node.callee, { name: "read" }) ) - results.push(resolver.isCallback(node.arguments[0]!)); + results.push(resolver.isArray(node.arguments[0]!)); }, }); expect(results).toEqual([false, true, true]); @@ -97,55 +93,30 @@ it("bounds long callback aliases without invoking the shared callable resolver", }); it.each([ - ["fixture.cjs", "const argv=[];const done=()=>{};", ["inherit", "default"]], - ["fixture.cts", "const argv=[];const done=()=>{};", ["inherit", "default"]], + ["fixture.cjs", "const argv=[];", ["inherit"]], + ["fixture.cts", "const argv=[];", ["inherit"]], + ["fixture.cjs", "var argv=[];{function argv(){}}", ["dynamic"]], + ["fixture.mjs", "var argv=[];{function argv(){}}", ["inherit"]], + ["fixture.js", "const argv=[];", ["dynamic"]], + ["fixture.cjs", "let argv=[];argv=input;", ["dynamic"]], + ["fixture.cts", "const argv=[];eval(input);", ["inherit"]], + ["fixture.cts", "namespace N{};const argv=[];", ["dynamic"]], [ "fixture.cjs", - "var argv=[];{function argv(){}}const done=()=>{};", - ["dynamic", "default"], - ], - [ - "fixture.mjs", - "var argv=[];{function argv(){}}const done=()=>{};", - ["inherit", "default"], - ], - ["fixture.js", "const argv=[];const done=()=>{};", ["dynamic", "dynamic"]], - [ - "fixture.cjs", - "let argv=[];argv=input;const done=()=>{};", - ["dynamic", "default"], - ], - [ - "fixture.cts", - "const argv=[];const done=()=>{};eval(input);", - ["inherit", "default"], - ], - [ - "fixture.cts", - "namespace N{};const argv=[];const done=()=>{};", - ["dynamic", "dynamic"], - ], - [ - "fixture.cjs", - "var module=[];const argv=module;const done=()=>{};const args=arguments;args[2]={};", - ["dynamic", "default"], - ], - [ - "fixture.cts", - "const argv=[];function exports(){};const args=arguments;args[0]={};const done=exports;", - ["inherit", "dynamic"], + "var module=[];const argv=module;const args=arguments;args[2]={};", + ["dynamic"], ], ] as const)( "distinguishes wrapper-local child identities and uncertain bindings: %s (%s)", (path, declarations, expected) => { const loader = path.endsWith(".mjs") - ? 'import {spawn,exec} from "node:child_process";' - : 'const {spawn,exec}=require("node:child_process");'; + ? 'import {spawn} from "node:child_process";' + : 'const {spawn}=require("node:child_process");'; const file = parseJavaScriptSource( ` ${loader} ${declarations} - spawn("argv",argv,{stdio:"inherit"}); exec("callback",done); + spawn("argv",argv,{stdio:"inherit"}); `, path, ); diff --git a/src/domain/javascript/javascriptSemanticExpressionShapes.ts b/src/domain/javascript/javascriptSemanticExpressionShapes.ts index db2ce9d067..f31ec62430 100644 --- a/src/domain/javascript/javascriptSemanticExpressionShapes.ts +++ b/src/domain/javascript/javascriptSemanticExpressionShapes.ts @@ -23,7 +23,7 @@ import { import { traverseJavaScriptAst } from "./javascriptSemanticTraversal.js"; import { javaScriptRuntimeContext } from "./javascriptSourceParser.js"; -type Shape = "array" | "callback" | "unknown"; +type Shape = "array" | "unknown"; /** Bounded argument-shape queries shared by call-signature owners. */ export type JavaScriptExpressionShapeResolver = ReturnType< @@ -41,15 +41,6 @@ const commonJsWrapperParameterNames = new Set([ export const createSemanticExpressionShapeResolver = ( state: JavaScriptSemanticAnalysisState, ) => { - const hoistedFunction = (node: t.Node): boolean => { - const parent = state.parentsByNode.get(node); - return ( - t.isFunctionDeclaration(node) && - (t.isProgram(parent) || - (t.isBlockStatement(parent) && - t.isFunction(state.parentsByNode.get(parent)))) - ); - }; // Most modules have no named child arguments. Collect source-wide blockers // only when a query actually needs a named binding or ambient undefined. let blocked: boolean | undefined; @@ -80,8 +71,6 @@ export const createSemanticExpressionShapeResolver = ( if (seen.has(node)) return "unknown"; seen.add(node); if (t.isArrayExpression(node)) return "array"; - if (t.isFunction(node) && !("declare" in node && node.declare)) - return "callback"; if (!t.isIdentifier(node)) return "unknown"; const binding = resolveSemanticBindingState(state, node, node.name); if (binding === undefined || binding.referenceInitializers.length > 0) @@ -106,11 +95,7 @@ export const createSemanticExpressionShapeResolver = ( if (initializer === undefined || initializer.projection.length > 0) return "unknown"; const read = semanticBindingAtRead(binding, node, state.parentsByNode); - // Hoisted function declarations are the only accepted unordered source. - if ( - !hoistedFunction(initializer.node) && - (read.status !== "ordered" || !read.hasDefiniteInitializer) - ) + if (read.status !== "ordered" || !read.hasDefiniteInitializer) return "unknown"; node = initializer.node; depth++; @@ -118,7 +103,6 @@ export const createSemanticExpressionShapeResolver = ( }; return { isArray: (node: t.Node) => shape(node) === "array", - isCallback: (node: t.Node) => shape(node) === "callback", isNullish: (input: t.Node) => { const { node } = unwrapJavaScriptExpression(input); return ( diff --git a/tests/acceptance/applications/javascriptChildProcessStdioCli.test.ts b/tests/acceptance/applications/javascriptChildProcessStdioCli.test.ts index 27c722df49..3099f0f209 100644 --- a/tests/acceptance/applications/javascriptChildProcessStdioCli.test.ts +++ b/tests/acceptance/applications/javascriptChildProcessStdioCli.test.ts @@ -102,6 +102,33 @@ const cases = [ 0, ], ['spawn("missing-stdio", [])', "missing-stdio", "default", 0], + // Async exec/execFile ignore stdio even when the supplied value is dynamic. + ['exec("exec-inherit", {stdio:"inherit"})', "exec-inherit", "default", null], + [ + 'execFile("execfile-ignore", {stdio:"ignore"})', + "execfile-ignore", + "default", + null, + ], + [ + 'execFile("execfile-array", [], {stdio:["inherit","inherit","inherit"]})', + "execfile-array", + "default", + 0, + ], + [ + 'execFile("execfile-dynamic", ["one"], {stdio:input,env:{A:"1"}})', + "execfile-dynamic", + "default", + 1, + ], + ['fork("fork-silent", [], {silent:true})', "fork-silent", "pipe", 0], + [ + 'fork("fork-explicit", [], {silent:true,stdio:"inherit"})', + "fork-explicit", + "inherit", + 0, + ], ['exec("null-stdio", {stdio:null})', "null-stdio", "default", null], [ 'execFile("undefined-options", [], undefined)', @@ -162,11 +189,11 @@ const cases = [ ['spawn(...input, [], {stdio:"inherit"})', null, "dynamic", null], ['exec("named-callback", done)', "named-callback", "default", null], ['execFile("callback-alias", [], callback)', "callback-alias", "default", 0], - ['exec("imported-callback", imported)', "imported-callback", "dynamic", null], + ['exec("imported-callback", imported)', "imported-callback", "default", null], [ 'exec("reassigned-callback", changed)', "reassigned-callback", - "dynamic", + "default", null, ], ['spawn("named-nullish", [], {stdio:absent})', "named-nullish", "dynamic", 0], @@ -179,15 +206,15 @@ const cases = [ ] as const; const annexBSource = ` - const {execFile}=require("node:child_process"); + const {spawn,execFile}=require("node:child_process"); var argv=[]; {function argv(){}} - execFile("annex-b-argv",argv,{stdio:"inherit"}); + spawn("annex-b-argv",argv,{stdio:"inherit"}); function run(){ var args=[]; switch(input){case 0:function args(){}} - execFile("annex-b-local",args,{stdio:"inherit"}); + spawn("annex-b-local",args,{stdio:"inherit"}); } var stable=[]; function done(){} - execFile("cjs-var-argv",stable,{stdio:"inherit"}); + spawn("cjs-var-argv",stable,{stdio:"inherit"}); execFile("cjs-hoisted-callback",done); `; @@ -199,13 +226,21 @@ const importEqualsSource = ` `; const exportAssignmentSource = ` - const {spawn,execFile}=require("node:child_process"); + const {spawn}=require("node:child_process"); const argv=[]; spawn("export-assignment-argv",argv,{stdio:"inherit"}); var args=[]; {function args(){}} - execFile("export-assignment-annex",args,{stdio:"inherit"}); + spawn("export-assignment-annex",args,{stdio:"inherit"}); export = argv; `; +// Callback/options ambiguity cannot change exec's effective stdio. +const opaqueExecCallbackCommands = [ + "eval-callback", + "constructed-callback", + "mapped-callback", + "cjs-wrapper-callback", +]; + cliTest.for(["CLI", "stdio MCP"] as const)( "preserves effective stdio, overloads and explicit unknowns through %s", async (surface, { cli }) => { @@ -277,17 +312,15 @@ cliTest.for(["CLI", "stdio MCP"] as const)( expected.set("import-equals-argv", ["inherit", null]); expected.set("import-equals-callback", ["default", null]); expected.set("export-assignment-argv", ["inherit", null]); + for (const command of opaqueExecCallbackCommands) + expected.set(command, ["default", null]); for (const [command, count] of [ ["Function-global", 0], ["frames-global", 0], ["mapped-arguments", 0], - ["eval-callback", null], ["shadowed-undefined", 0], ["constructed-argv", null], - ["constructed-callback", null], - ["mapped-callback", null], ["cjs-wrapper-argv", null], - ["cjs-wrapper-callback", null], ["annex-b-argv", null], ["annex-b-local", null], ["export-assignment-annex", null], @@ -309,7 +342,7 @@ cliTest.for(["CLI", "stdio MCP"] as const)( expect(child.properties.stdio_mode, String(command)).toBe(mode); expect(child.properties.argv_count, String(command)).toBe(count); expect(child.properties.environment_supplied, String(command)).toBe( - command === "named-argv", + ["named-argv", "execfile-dynamic"].includes(String(command)), ); expect(child.identity.source_range).not.toBeNull(); const stdio = graph.relations.filter( @@ -342,7 +375,9 @@ cliTest.for(["CLI", "stdio MCP"] as const)( relation.relation === "supplies-env" && relation.target_node_id === child.node_id, ), - ).toHaveLength(command === "named-argv" ? 1 : 0); + ).toHaveLength( + ["named-argv", "execfile-dynamic"].includes(String(command)) ? 1 : 0, + ); expect( graph.relations.filter( (relation) => From a7aad91320213f136062668110bf5c1e63606d0d Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Mon, 26 Oct 2026 23:21:36 +0800 Subject: [PATCH 03/25] fix(firmware): preserve report metadata before semantic validation --- src/firmware/FirmwareReports.ts | 19 +++- .../firmware/firmwareReportFiles.test.ts | 101 +++++++++++++++++- 2 files changed, 116 insertions(+), 4 deletions(-) diff --git a/src/firmware/FirmwareReports.ts b/src/firmware/FirmwareReports.ts index f7dcb3c93b..7d9f51adce 100644 --- a/src/firmware/FirmwareReports.ts +++ b/src/firmware/FirmwareReports.ts @@ -1,6 +1,10 @@ import { z } from "zod"; import { AnalysisOutputError } from "../domain/analysisErrorCore.js"; -import { jsonValueSchema, type JsonValue } from "../domain/jsonValue.js"; +import { + jsonObjectSchema, + jsonValueSchema, + type JsonValue, +} from "../domain/jsonValue.js"; import { digestSchema } from "../domain/digests.js"; import { uniqueSorted } from "../domain/canonicalOrdering.js"; import { firmwareLogicalPath } from "./FirmwareFiles.js"; @@ -21,11 +25,20 @@ const scan = z.array( Analysis: z.object({ file_path: z.string(), file_map: z.array(signature), - extractions: z.record(z.string(), jsonValueSchema), + extractions: jsonObjectSchema, }), }), ); -const report = z.object({ __typename__: z.string() }).catchall(jsonValueSchema); +const report = jsonObjectSchema.pipe( + z.custom & { __typename__: string }>( + (value) => + typeof value === "object" && + value !== null && + "__typename__" in value && + typeof value.__typename__ === "string", + "Expected report __typename__ string", + ), +); const task = z.object({ path: z.string(), depth: integer, diff --git a/tests/boundary/firmware/firmwareReportFiles.test.ts b/tests/boundary/firmware/firmwareReportFiles.test.ts index d5c2e13357..c3067b89d9 100644 --- a/tests/boundary/firmware/firmwareReportFiles.test.ts +++ b/tests/boundary/firmware/firmwareReportFiles.test.ts @@ -11,7 +11,14 @@ import { FIRMWARE_LIMITS } from "../../../src/firmware/FirmwareRelease.js"; import { AnalysisError } from "../../../src/domain/analysisErrorBase.js"; import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; import { SafeOutputTree } from "../../../src/artifacts/SafeOutputTree.js"; -import { firmwareRequestSchema } from "../../../src/domain/firmware/firmwareAnalysis.js"; +import { + normalizeBinwalkReport, + normalizeUnblobReport, +} from "../../../src/firmware/FirmwareReports.js"; +import { + firmwareResultSchemas, + firmwareRequestSchema, +} from "../../../src/domain/firmware/firmwareAnalysis.js"; import { createHash } from "node:crypto"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; @@ -308,3 +315,95 @@ it.skipIf(process.platform === "win32" || process.getuid?.() === 0)( } }, ); + +it("retains unfamiliar Unblob report members from JSON files through public diagnostics", async () => { + const root = await createTestTempDirectory("rea-firmware-report-metadata-"); + const inputPath = join(root, "input.bin"); + const reportPath = join(root, "report.json"); + const sha256 = "a".repeat(64); + const unfamiliar: unknown = JSON.parse( + '{"__typename__":"FutureReport","__proto__":{"source":"local","empty":null},"constructor":false,"prototype":0,"nested":{"__proto__":"retained"}}', + ); + const record = { + task: { path: inputPath, depth: 0, blob_id: "", is_multi_file: false }, + reports: [ + { __typename__: "HashReport", sha256 }, + { + __typename__: "StatReport", + path: inputPath, + size: 4, + is_dir: false, + is_file: true, + is_link: false, + link_target: null, + }, + unfamiliar, + ], + subtasks: [], + }; + await writeFile(reportPath, JSON.stringify([record])); + const files = new FirmwareFileAccess(); + try { + const raw = await files.report(reportPath, "extract_firmware"); + const normalized = normalizeUnblobReport(raw, { + inputPath, + outputRoot: join(root, "output"), + sha256, + length: 4, + offset: 0, + maxDepth: 3, + }); + const diagnostics = + firmwareResultSchemas.extract_firmware.shape.diagnostics.parse( + normalized.diagnostics, + ); + expect(JSON.stringify(diagnostics)).toBe( + JSON.stringify([{ input_path: "$input", report: unfamiliar }]), + ); + expect(normalized.coverage).toBe("partial"); + for (const malformed of [{}, { __typename__: null }, { __typename__: 1 }]) { + expect(() => + normalizeUnblobReport( + [{ ...record, reports: [...record.reports.slice(0, 2), malformed] }], + { + inputPath, + outputRoot: join(root, "output"), + sha256, + length: 4, + offset: 0, + maxDepth: 3, + }, + ), + ).toThrow(); + } + } finally { + await files.close(); + } +}); + +it("rejects Binwalk extraction activity under every own JSON member", async () => { + const root = await createTestTempDirectory( + "rea-firmware-report-extractions-", + ); + const reportPath = join(root, "report.json"); + const inputPath = join(root, "input.bin"); + const files = new FirmwareFileAccess(); + try { + for (const extractions of [ + "{}", + '{"__proto__":null}', + '{"ordinary":false}', + ]) { + await writeFile( + reportPath, + `[{"Analysis":{"file_path":${JSON.stringify(inputPath)},"file_map":[],"extractions":${extractions}}}]`, + ); + const raw = await files.report(reportPath, "inspect_firmware_regions"); + const normalize = () => normalizeBinwalkReport(raw, inputPath, 4); + if (extractions === "{}") expect(normalize()).toEqual([]); + else expect(normalize).toThrow("Expected one inspection-only report"); + } + } finally { + await files.close(); + } +}); From 65819e8cc56cdc07a7d27456084596b838c89fd7 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Mon, 26 Oct 2026 23:21:37 +0800 Subject: [PATCH 04/25] fix(pcap): capture provider environment once --- src/pcap/PcapProvider.ts | 3 +- .../pcap/pcapProviderCrossPlatform.test.ts | 63 +++++++++++++++++++ 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/src/pcap/PcapProvider.ts b/src/pcap/PcapProvider.ts index 7f83311579..ab316ac025 100644 --- a/src/pcap/PcapProvider.ts +++ b/src/pcap/PcapProvider.ts @@ -54,6 +54,7 @@ import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; import { artifactFileCancellationCause } from "../artifacts/ArtifactFileOperation.js"; import { ArtifactAnalysisResources } from "../process/ArtifactAnalysisResources.js"; import { readLinuxResourceLimits } from "../process/LinuxResourceLimits.js"; +import { snapshotEnvironment } from "../process/snapshotEnvironment.js"; import { assertArtifactCommandPath, selectArtifactCommandPath, @@ -225,7 +226,7 @@ export class PcapProvider { >(); constructor(options: PcapProviderOptions = {}) { - this.environment = options.environment ?? process.env; + this.environment = snapshotEnvironment(options.environment ?? process.env); this.launcher = options.launcher; this.#resources = new ArtifactAnalysisResources( "pcap", diff --git a/tests/boundary/pcap/pcapProviderCrossPlatform.test.ts b/tests/boundary/pcap/pcapProviderCrossPlatform.test.ts index 497e87148c..7bc56e900d 100644 --- a/tests/boundary/pcap/pcapProviderCrossPlatform.test.ts +++ b/tests/boundary/pcap/pcapProviderCrossPlatform.test.ts @@ -24,6 +24,7 @@ import { expect, it, onTestFinished, vi } from "vitest"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { PcapProvider } from "../../../src/pcap/PcapProvider.js"; +import { PcapAnalysisService } from "../../../src/application/pcap/PcapAnalysisService.js"; import { CAPINFOS_COMMAND_ENV, TSHARK_COMMAND_ENV, @@ -58,6 +59,68 @@ const providerOver = (): PcapProvider => { return provider; }; +it.skipIf(isWindows).each(["path", "explicit"] as const)( + "retains the selected %s environment after a successful Wireshark probe", + async (selection) => { + const root = await createTestTempDirectory("rea-pcap-environment-"); + for (const [name, banner] of [ + ["tshark", "TShark"], + ["capinfos", "Capinfos"], + ] as const) { + await writeFile( + join(root, name), + `#!/bin/sh +printf '${banner} (Wireshark) 4.6.6-%s\\n' "$PCAP_TEST_PROFILE" +printf '%s|%s|%s\\n' "$PCAP_TEST_PROFILE" "$PCAP_TEST_EMPTY" "\${PCAP_TEST_ABSENT-unset}" >> "$0.calls" +`, + { mode: 0o755 }, + ); + } + const environment: NodeJS.ProcessEnv = { + ...process.env, + PATH: root, + PCAP_TEST_PROFILE: "selected", + PCAP_TEST_EMPTY: "", + }; + delete environment.PCAP_TEST_ABSENT; + delete environment.REA_TSHARK_COMMAND; + delete environment.REA_CAPINFOS_COMMAND; + if (selection === "explicit") { + environment.REA_TSHARK_COMMAND = join(root, "tshark"); + environment.REA_CAPINFOS_COMMAND = join(root, "capinfos"); + } + const provider = new PcapProvider({ environment }); + onTestFinished(() => provider.close()); + const service = new PcapAnalysisService(provider); + const first = await service.execute("inspect_pcap_client", {}); + if (!first.ok) throw first.error; + expect(first.value.normalized_result).toMatchObject({ + client: { + tshark_version: "TShark (Wireshark) 4.6.6-selected", + capinfos_version: "Capinfos (Wireshark) 4.6.6-selected", + }, + }); + environment.PATH = join(root, "missing-bin"); + environment.PCAP_TEST_PROFILE = "later"; + environment.PCAP_TEST_EMPTY = "later"; + environment.PCAP_TEST_ABSENT = "later"; + if (selection === "explicit") { + environment.REA_TSHARK_COMMAND = join(root, "missing-tshark"); + environment.REA_CAPINFOS_COMMAND = join(root, "missing-capinfos"); + } + const repeated = await service.execute("inspect_pcap_client", {}); + if (!repeated.ok) throw repeated.error; + expect(repeated.value.normalized_result).toEqual( + first.value.normalized_result, + ); + expect(JSON.stringify(repeated.value)).not.toContain("PCAP_TEST_PROFILE"); + for (const name of ["tshark", "capinfos"]) + expect(await readFile(join(root, `${name}.calls`), "utf8")).toBe( + "selected||unset\nselected||unset\n", + ); + }, +); + it.skipIf(isWindows)( "preserves selected Wireshark wrappers and re-admits changed symlink targets after success", async () => { From b45a83368c11d02b9ed2acfeb04a19cac747b611 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Mon, 26 Oct 2026 23:21:38 +0800 Subject: [PATCH 05/25] fix(ci): preserve lint diagnostics without source locations --- scripts/ci/artifacts.mjs | 15 ++-- scripts/ci/static-check-report.mjs | 8 +- .../filesystem/ciFailureReport.test.ts | 17 ++++ .../filesystem/ciStaticArtifacts.test.ts | 85 ++++++++++++++++++- .../filesystem/staticCheckReports.test.ts | 33 ++++++- 5 files changed, 147 insertions(+), 11 deletions(-) diff --git a/scripts/ci/artifacts.mjs b/scripts/ci/artifacts.mjs index dbd77c5d60..ee12e2013f 100644 --- a/scripts/ci/artifacts.mjs +++ b/scripts/ci/artifacts.mjs @@ -108,7 +108,11 @@ export function classifyFailure({ } export function diagnosticPaths(item) { - return item.kind === "jscpd" ? item.paths : [item.path]; + return item.kind === "jscpd" + ? item.paths + : item.path === undefined + ? [] + : [item.path]; } export function isFailedConclusion(value) { return value === "failure" || value === "timed_out"; @@ -269,16 +273,15 @@ function normalizeLint(report) { .filter((item) => item.severity === "error") .map((item) => { if ( - typeof item.filename !== "string" || - !repoPath(item.filename) || - typeof item.code !== "string" || + (item.filename !== undefined && !repoPath(item.filename)) || + (item.code !== undefined && typeof item.code !== "string") || typeof item.message !== "string" ) throw new Error("oxlint error diagnostic is malformed"); return { kind: "lint", - path: item.filename, - code: item.code, + ...(item.filename === undefined ? {} : { path: item.filename }), + ...(item.code === undefined ? {} : { code: item.code }), message: item.message, severity: "error", }; diff --git a/scripts/ci/static-check-report.mjs b/scripts/ci/static-check-report.mjs index 6654f9a60b..32a27ab020 100644 --- a/scripts/ci/static-check-report.mjs +++ b/scripts/ci/static-check-report.mjs @@ -15,7 +15,7 @@ const lintReport = z.object({ message: z.string(), code: z.string().optional(), severity: z.enum(["error", "warning", "advice"]), - filename: z.string(), + filename: z.string().optional(), labels: z.array( z.object({ span: z.object({ line: z.number(), column: z.number() }), @@ -115,8 +115,12 @@ try { ); for (const diagnostic of errors) { const location = diagnostic.labels[0]?.span; + const source = + diagnostic.filename === undefined + ? "" + : `${diagnostic.filename}${location === undefined ? "" : `:${location.line}:${location.column}`}: `; console.error( - `${diagnostic.filename}:${location?.line ?? 1}:${location?.column ?? 1}: ${diagnostic.code ?? "lint"}: ${diagnostic.message}`, + `${source}${diagnostic.code ?? "lint"}: ${diagnostic.message}`, ); } const counts = new Map(); diff --git a/tests/boundary/filesystem/ciFailureReport.test.ts b/tests/boundary/filesystem/ciFailureReport.test.ts index 3e77d8e8c2..0811d89f25 100644 --- a/tests/boundary/filesystem/ciFailureReport.test.ts +++ b/tests/boundary/filesystem/ciFailureReport.test.ts @@ -120,6 +120,23 @@ it("attributes only an identical structured lint failure with matching task, con }); }); +it.each(["path", "code"] as const)( + "keeps applicability unknown when the lint %s is unavailable on both runs", + (missing) => { + const { [missing]: _omitted, ...retained } = diagnostic; + expect( + classifyFailure({ + ...comparison(), + diagnostics: [retained], + mainDiagnostics: [retained], + }), + ).toEqual({ + classification: "unknown", + reason: "diagnostic_shape_not_supported", + }); + }, +); + it.each([ ["task hash", { mainContext: context(sha("c"), "b".repeat(16)) }], ["dirty source", { context: { ...context(sha("a")), sourceClean: false } }], diff --git a/tests/boundary/filesystem/ciStaticArtifacts.test.ts b/tests/boundary/filesystem/ciStaticArtifacts.test.ts index 45915f4504..a499fef684 100644 --- a/tests/boundary/filesystem/ciStaticArtifacts.test.ts +++ b/tests/boundary/filesystem/ciStaticArtifacts.test.ts @@ -1,11 +1,14 @@ import { execFile } from "node:child_process"; -import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { chmod, mkdir, readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; import { expect, it } from "vitest"; -import { parseStaticArchive } from "../../../scripts/ci/artifacts.mjs"; +import { + diagnosticPaths, + parseStaticArchive, +} from "../../../scripts/ci/artifacts.mjs"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; const execute = promisify(execFile); @@ -13,6 +16,84 @@ const producer = fileURLToPath( new URL("../../../scripts/ci/static-check-report.mjs", import.meta.url), ); +it.skipIf(process.platform === "win32" || process.getuid?.() === 0)( + "archives real lint I/O errors with unavailable source and rule identity", + async () => { + const root = await createTestTempDirectory("rea-ci-static-io-"); + const source = join(root, "unreadable.js"); + await writeFile(source, "debugger;\n"); + await chmod(source, 0); + try { + await expect( + execute(process.execPath, [producer, "lint", source], { cwd: root }), + ).rejects.toMatchObject({ code: 1 }); + const reports = join(root, ".cache/static-checks"); + const report = JSON.parse( + await readFile(join(reports, "oxlint.json"), "utf8"), + ); + expect(report.diagnostics).toEqual([ + { + severity: "error", + labels: [], + message: expect.stringContaining("Permission denied"), + }, + ]); + await writeFile( + join(reports, "static-task-context.json"), + JSON.stringify({ + version: 1, + sourceSha: "a".repeat(40), + sourceClean: true, + nodeVersion: process.version, + npmVersion: "11.16.0", + turboVersion: "2.11.7", + platform: process.platform, + architecture: process.arch, + taskHashes: { "lint:built": "a".repeat(16) }, + }), + ); + const archive = join(root, "reports.zip"); + const pack = () => + execute( + "zip", + ["-q", archive, "oxlint.json", "static-task-context.json"], + { cwd: reports }, + ); + await pack(); + const evidence = await parseStaticArchive(archive); + expect(evidence.diagnostics).toEqual([ + { + kind: "lint", + message: report.diagnostics[0].message, + severity: "error", + }, + ]); + expect(diagnosticPaths(evidence.diagnostics[0]!)).toEqual([]); + for (const malformed of [ + { filename: null }, + { filename: "../outside.js" }, + { filename: source }, + { code: null }, + { code: 1 }, + ]) { + await writeFile( + join(reports, "oxlint.json"), + JSON.stringify({ + ...report, + diagnostics: [{ ...report.diagnostics[0], ...malformed }], + }), + ); + await pack(); + await expect(parseStaticArchive(archive)).rejects.toThrow( + "oxlint error diagnostic is malformed", + ); + } + } finally { + await chmod(source, 0o600); + } + }, +); + it.each([3.5, 100, 0])( "retains real lint failures when the duplication threshold is %s", async (threshold) => { diff --git a/tests/boundary/filesystem/staticCheckReports.test.ts b/tests/boundary/filesystem/staticCheckReports.test.ts index b433dc9f55..dd3a146f6b 100644 --- a/tests/boundary/filesystem/staticCheckReports.test.ts +++ b/tests/boundary/filesystem/staticCheckReports.test.ts @@ -1,5 +1,5 @@ import { spawnSync } from "node:child_process"; -import { access, mkdir, readFile, writeFile } from "node:fs/promises"; +import { access, chmod, mkdir, readFile, writeFile } from "node:fs/promises"; import { dirname, join, sep } from "node:path"; import { fileURLToPath } from "node:url"; import { expect, it } from "vitest"; @@ -193,6 +193,37 @@ it("keeps malformed source failing with its parser diagnostic", async () => { ); }); +it.skipIf(process.platform === "win32" || process.getuid?.() === 0)( + "retains location-free lint I/O errors without inventing a source location", + async () => { + const root = await fixture(); + const source = join(root, "src/unreadable.js"); + await writeFile(source, "debugger;\n"); + await chmod(source, 0); + try { + const result = invoke(root, "lint", source); + expect(result.error).toBeUndefined(); + expect(result.status).toBe(1); + expect(result.stdout).toContain("1 files, 1 errors, 0 warnings"); + expect(result.stdout).toContain("Full diagnostics:"); + const report = JSON.parse( + await readFile(join(root, ".cache/static-checks/oxlint.json"), "utf8"), + ); + expect(report.diagnostics).toEqual([ + { + message: expect.stringContaining("Permission denied"), + severity: "error", + labels: [], + }, + ]); + expect(report.diagnostics[0].message).toContain(source); + expect(result.stderr).toBe(`lint: ${report.diagnostics[0].message}\n`); + } finally { + await chmod(source, 0o600); + } + }, +); + it("preserves the duplication threshold and complete clone locations", async () => { const root = await fixture(); const source = [ From d9908fd7a42088992ddc74c75f3a3652e4839dcb Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Mon, 26 Oct 2026 23:27:36 +0800 Subject: [PATCH 06/25] refactor(ghidra): consolidate function and inventory admission --- src/contracts/hopperMemoryContracts.ts | 3 +- src/domain/hopperValues.ts | 209 +++++++++--------- src/domain/native/bytePattern.ts | 5 +- src/domain/native/nativeApiBoundary.ts | 4 +- src/domain/native/nativeValueFlow.ts | 64 +++--- src/ghidra/GhidraFunctionValues.test.ts | 205 ++++++++++++++++++ src/ghidra/GhidraFunctionValues.ts | 258 +++++++---------------- src/ghidra/GhidraInventoryValues.test.ts | 111 ++++++++++ src/ghidra/GhidraInventoryValues.ts | 98 ++++----- src/ghidra/GhidraNativeAnalysisValues.ts | 87 ++++++++ 10 files changed, 678 insertions(+), 366 deletions(-) create mode 100644 src/ghidra/GhidraNativeAnalysisValues.ts diff --git a/src/contracts/hopperMemoryContracts.ts b/src/contracts/hopperMemoryContracts.ts index 8a1782e0e4..4ca870e1ef 100644 --- a/src/contracts/hopperMemoryContracts.ts +++ b/src/contracts/hopperMemoryContracts.ts @@ -1,6 +1,7 @@ import { z } from "zod"; import { address, document } from "./toolContractHelpers.js"; import { + MAX_BYTE_SEARCH_MATCHES, bytePatternFields, validateBytePattern, } from "../domain/native/bytePattern.js"; @@ -17,7 +18,7 @@ export const HOPPER_MEMORY_TOOL_DEFINITIONS = [ document, start: address.optional(), end: address.optional(), - limit: z.int().min(1).max(1000).default(100), + limit: z.int().min(1).max(MAX_BYTE_SEARCH_MATCHES).default(100), cursor: z.string().min(1).optional(), }) .superRefine(validateBytePattern), diff --git a/src/domain/hopperValues.ts b/src/domain/hopperValues.ts index 9d5c4eb520..02e05e0110 100644 --- a/src/domain/hopperValues.ts +++ b/src/domain/hopperValues.ts @@ -88,87 +88,88 @@ export const analysisBookmarkSchema = z.strictObject({ name: z.string().nullable(), }); /** Complete observed function-body ranges; every range endpoint is inclusive. */ -export const functionBodySchema = z.discriminatedUnion("available", [ - unavailableAnalysisFactSchema, - z - .strictObject({ - available: z.literal(true), - provenance: z.string().min(1), - ranges: z.array( - z.strictObject({ - start: z.string(), - end: z - .string() - .describe("Inclusive last address of this observed body range."), - }), - ), - total_bytes: z.int().min(0).max(Number.MAX_SAFE_INTEGER), - span_bytes: z.int().min(0).max(Number.MAX_SAFE_INTEGER).nullable(), - non_contiguous: z.boolean(), - contains_entry: z.boolean(), - }) - .superRefine((body, context) => { - let total = 0n; - const bounds = new Map(); - const closedSpaces = new Set(); - let previousSpace: string | null | undefined; - for (const range of body.ranges) { - const start = addressCoordinate(range.start); - const end = addressCoordinate(range.end); - if ( - start === null || - end === null || - start.space !== end.space || - end.offset < start.offset - ) { - context.addIssue({ - code: "custom", - message: "Function body range has invalid inclusive endpoints", - }); - return; - } - if (previousSpace !== undefined && previousSpace !== start.space) - closedSpaces.add(previousSpace); - const previous = bounds.get(start.space); - if ( - closedSpaces.has(start.space) || - (previous !== undefined && start.offset <= previous.end + 1n) - ) { - context.addIssue({ - code: "custom", - message: - "Function body ranges must be ordered, disjoint and maximal within each address space", - }); - return; - } - bounds.set(start.space, { - start: previous?.start ?? start.offset, - end: end.offset, - }); - previousSpace = start.space; - total += end.offset - start.offset + 1n; - } - const onlyBounds = - bounds.size === 1 ? [...bounds.values()][0] : undefined; - const span = - bounds.size === 0 - ? 0n - : onlyBounds === undefined - ? null - : onlyBounds.end - onlyBounds.start + 1n; - const reportedSpan = - body.span_bytes === null ? null : BigInt(body.span_bytes); +export const availableFunctionBodySchema = z + .strictObject({ + available: z.literal(true), + provenance: z.string().min(1), + ranges: z.array( + z.strictObject({ + start: z.string(), + end: z + .string() + .describe("Inclusive last address of this observed body range."), + }), + ), + total_bytes: z.int().min(0).max(Number.MAX_SAFE_INTEGER), + span_bytes: z.int().min(0).max(Number.MAX_SAFE_INTEGER).nullable(), + non_contiguous: z.boolean(), + contains_entry: z.boolean(), + }) + .superRefine((body, context) => { + let total = 0n; + const bounds = new Map(); + const closedSpaces = new Set(); + let previousSpace: string | null | undefined; + for (const range of body.ranges) { + const start = addressCoordinate(range.start); + const end = addressCoordinate(range.end); if ( - total !== BigInt(body.total_bytes) || - span !== reportedSpan || - body.non_contiguous !== body.ranges.length > 1 - ) + start === null || + end === null || + start.space !== end.space || + end.offset < start.offset + ) { + context.addIssue({ + code: "custom", + message: "Function body range has invalid inclusive endpoints", + }); + return; + } + if (previousSpace !== undefined && previousSpace !== start.space) + closedSpaces.add(previousSpace); + const previous = bounds.get(start.space); + if ( + closedSpaces.has(start.space) || + (previous !== undefined && start.offset <= previous.end + 1n) + ) { context.addIssue({ code: "custom", message: - "Function body counts, enclosing span or contiguity disagree with inclusive ranges", + "Function body ranges must be ordered, disjoint and maximal within each address space", }); - }), + return; + } + bounds.set(start.space, { + start: previous?.start ?? start.offset, + end: end.offset, + }); + previousSpace = start.space; + total += end.offset - start.offset + 1n; + } + const onlyBounds = bounds.size === 1 ? [...bounds.values()][0] : undefined; + const span = + bounds.size === 0 + ? 0n + : onlyBounds === undefined + ? null + : onlyBounds.end - onlyBounds.start + 1n; + const reportedSpan = + body.span_bytes === null ? null : BigInt(body.span_bytes); + if ( + total !== BigInt(body.total_bytes) || + span !== reportedSpan || + body.non_contiguous !== body.ranges.length > 1 + ) + context.addIssue({ + code: "custom", + message: + "Function body counts, enclosing span or contiguity disagree with inclusive ranges", + }); + }); + +export const functionBodySchema = z.discriminatedUnion("available", [ + unavailableAnalysisFactSchema, + availableFunctionBodySchema, ]); /** Check the declared entry membership against complete, inclusive body ranges. */ @@ -325,23 +326,25 @@ const swiftStringLiteralFields = { }; /** A Swift String literal decoded from the two words of its ABI value, not a data reference. */ +export const swiftSmallStringLiteralSchema = z.strictObject({ + ...swiftStringLiteralFields, + representation: z + .literal("small") + .describe("The UTF-8 bytes are inside the two words."), + address: z.null().describe("A small literal has no storage of its own."), +}); +export const swiftLargeStringLiteralSchema = z.strictObject({ + ...swiftStringLiteralFields, + representation: z + .literal("large") + .describe( + "An immortal literal whose words hold its count and a pointer to its bytes.", + ), + address: z.string().describe("Address of a large literal's bytes."), +}); export const swiftStringLiteralSchema = z.discriminatedUnion("representation", [ - z.strictObject({ - ...swiftStringLiteralFields, - representation: z - .literal("small") - .describe("The UTF-8 bytes are inside the two words."), - address: z.null().describe("A small literal has no storage of its own."), - }), - z.strictObject({ - ...swiftStringLiteralFields, - representation: z - .literal("large") - .describe( - "An immortal literal whose words hold its count and a pointer to its bytes.", - ), - address: z.string().describe("Address of a large literal's bytes."), - }), + swiftSmallStringLiteralSchema, + swiftLargeStringLiteralSchema, ]); const swiftStringLiteralOmissionFields = { @@ -357,19 +360,21 @@ const swiftStringLiteralOmissionFields = { }; /** A Swift literal candidate omitted because decoding capacity or control flow left it unknown. */ +export const swiftSmallStringLiteralOmissionSchema = z.strictObject({ + ...swiftStringLiteralOmissionFields, + representation: z.literal("small"), + address: z.null(), +}); +export const swiftLargeStringLiteralOmissionSchema = z.strictObject({ + ...swiftStringLiteralOmissionFields, + representation: z.literal("large"), + address: z.string().describe("Address of the literal's bytes."), +}); export const swiftStringLiteralOmissionSchema = z.discriminatedUnion( "representation", [ - z.strictObject({ - ...swiftStringLiteralOmissionFields, - representation: z.literal("small"), - address: z.null(), - }), - z.strictObject({ - ...swiftStringLiteralOmissionFields, - representation: z.literal("large"), - address: z.string().describe("Address of the literal's bytes."), - }), + swiftSmallStringLiteralOmissionSchema, + swiftLargeStringLiteralOmissionSchema, ], ); diff --git a/src/domain/native/bytePattern.ts b/src/domain/native/bytePattern.ts index 3052a509a4..5f821493c9 100644 --- a/src/domain/native/bytePattern.ts +++ b/src/domain/native/bytePattern.ts @@ -3,6 +3,9 @@ import { addressCoordinate } from "./addressCoordinate.js"; import { AnalysisInputError } from "../analysisErrorCore.js"; import { err, ok, type Result } from "../result.js"; +/** Maximum matches delivered by one byte-search page. */ +export const MAX_BYTE_SEARCH_MATCHES = 1000; + /** Exact grammar belongs to the object refinement, not a repeating advertised regex. */ export const bytePatternFields = { pattern: z @@ -152,7 +155,7 @@ export const byteSearchResultSchema = z .nullable(), }), ) - .max(1000), + .max(MAX_BYTE_SEARCH_MATCHES), searched: z.strictObject({ blocks: z.array(coverageBlock), bytes: z.int().nonnegative().max(Number.MAX_SAFE_INTEGER), diff --git a/src/domain/native/nativeApiBoundary.ts b/src/domain/native/nativeApiBoundary.ts index 0b11ba3be5..09b021c118 100644 --- a/src/domain/native/nativeApiBoundary.ts +++ b/src/domain/native/nativeApiBoundary.ts @@ -87,7 +87,7 @@ const inferredJumpTableSchema = z }) .strict(); -const availableNativeApiBoundarySchema = z +export const availableNativeApiBoundarySchema = z .object({ available: z.literal(true), provenance: z.string().min(1), @@ -115,7 +115,7 @@ const availableNativeApiBoundarySchema = z }) .strict(); -const unavailableNativeApiBoundarySchema = z +export const unavailableNativeApiBoundarySchema = z .object({ available: z.literal(false), reason: z.string().min(1), diff --git a/src/domain/native/nativeValueFlow.ts b/src/domain/native/nativeValueFlow.ts index 9e65fa81d0..1289b0362a 100644 --- a/src/domain/native/nativeValueFlow.ts +++ b/src/domain/native/nativeValueFlow.ts @@ -46,36 +46,38 @@ export const nativePcodeEffectSchema = z.strictObject({ }); /** Bounded data-flow observations; STORE is not necessarily a persistent/global write. */ +export const availableNativeValueFlowSchema = z.strictObject({ + available: z.literal(true), + provenance: z.literal("ghidra-high-pcode"), + operations: z.array(nativePcodeOperationSchema), + def_use: z.array(nativePcodeDefUseSchema), + effects: z.array(nativePcodeEffectSchema), + parameters: z.array( + z.strictObject({ + ordinal: z.int().nonnegative(), + name: z.string(), + data_type: z.string(), + }), + ), + parameter_uses: z.array( + z.strictObject({ + ordinal: z.int().nonnegative(), + use: z.string(), + input_index: z.int().nonnegative(), + }), + ), + truncated: z.boolean(), + omitted_operations_lower_bound: z.int().nonnegative(), + known_omitted_inputs: z.int().nonnegative(), + known_omitted_edges: z.int().nonnegative(), + limitations: z.array(z.string().min(1)), +}); +export const unavailableNativeValueFlowSchema = z.strictObject({ + available: z.literal(false), + reason: z.string().min(1), + limitations: z.array(z.string().min(1)), +}); export const nativeValueFlowSchema = z.discriminatedUnion("available", [ - z.strictObject({ - available: z.literal(true), - provenance: z.literal("ghidra-high-pcode"), - operations: z.array(nativePcodeOperationSchema), - def_use: z.array(nativePcodeDefUseSchema), - effects: z.array(nativePcodeEffectSchema), - parameters: z.array( - z.strictObject({ - ordinal: z.int().nonnegative(), - name: z.string(), - data_type: z.string(), - }), - ), - parameter_uses: z.array( - z.strictObject({ - ordinal: z.int().nonnegative(), - use: z.string(), - input_index: z.int().nonnegative(), - }), - ), - truncated: z.boolean(), - omitted_operations_lower_bound: z.int().nonnegative(), - known_omitted_inputs: z.int().nonnegative(), - known_omitted_edges: z.int().nonnegative(), - limitations: z.array(z.string().min(1)), - }), - z.strictObject({ - available: z.literal(false), - reason: z.string().min(1), - limitations: z.array(z.string().min(1)), - }), + availableNativeValueFlowSchema, + unavailableNativeValueFlowSchema, ]); diff --git a/src/ghidra/GhidraFunctionValues.test.ts b/src/ghidra/GhidraFunctionValues.test.ts index 2880b6529e..bde8f319e8 100644 --- a/src/ghidra/GhidraFunctionValues.test.ts +++ b/src/ghidra/GhidraFunctionValues.test.ts @@ -1121,3 +1121,208 @@ describe("Ghidra bridge script processor context contract", () => { expect(content).toContain('"ghidra-program-context"'); }); }); + +describe("Ghidra dossier nested boundary composition", () => { + const identityLocations = [ + "caller", + "callee", + "incoming source", + "incoming target", + "outgoing source", + "outgoing target", + ] as const; + const withIdentity = ( + location: (typeof identityLocations)[number], + identity: JsonValue, + ): JsonValue => { + const dossier = functionDossierSchema.parse(ghidraFunctionDossier()); + if (location === "caller") return { ...dossier, callers: [identity] }; + if (location === "callee") return { ...dossier, callees: [identity] }; + const collection = location.startsWith("incoming") + ? "incoming_references" + : "outgoing_references"; + const field = location.endsWith("source") + ? "source_procedure" + : "target_procedure"; + return { + ...dossier, + [collection]: [{ ...ghidraReferenceEdge(), [field]: identity }], + }; + }; + + it.each(identityLocations)( + "retains the dossier's optional symbol and empty name on a nested %s identity", + (location) => { + const { symbol: omitted, ...identity } = ghidraFunctionIdentity(); + expect(omitted).toBeDefined(); + const value = withIdentity(location, { ...identity, name: "" }); + expect( + parseFunctionValueFixture("analyze_function", value), + ).toMatchObject({ + ok: true, + value, + }); + }, + ); + + it.each(identityLocations)( + "rejects noncanonical address and foreign body/classification provenance on a nested %s identity", + (location) => { + const identity = ghidraFunctionIdentity(); + for (const change of [ + { address: "401000" }, + { body: { ...identity.body, provenance: "foreign-provider" } }, + { body: { ...identity.body, contains_entry: false } }, + { + classification: { + ...identity.classification, + provenance: "foreign-provider", + }, + }, + ]) + expect( + parseFunctionValueFixture( + "analyze_function", + withIdentity(location, { ...identity, ...change }), + ), + ).toMatchObject({ ok: false, error: { _tag: "AnalysisOutputError" } }); + }, + ); + + it.each(["incoming_references", "outgoing_references"] as const)( + "requires Ghidra reference-manager provenance inside %s", + (collection) => { + const edge = ghidraReferenceEdge(); + expect( + parseFunctionValueFixture("analyze_function", { + ...functionDossierSchema.parse(ghidraFunctionDossier()), + [collection]: [ + { + ...edge, + kind: { ...edge.kind, provenance: "hopper-public-python-api" }, + }, + ], + }), + ).toMatchObject({ ok: false, error: { _tag: "AnalysisOutputError" } }); + }, + ); + + it.each([ + { + name: "comment address", + fixture: (address: string) => ({ + comments: [{ address, kind: "comment", text: "observed comment" }], + }), + }, + { + name: "basic-block start", + fixture: (address: string) => ({ + basic_blocks: [{ start: address, end: "0x401006", successors: [] }], + }), + }, + { + name: "basic-block successor", + fixture: (address: string) => ({ + basic_blocks: [ + { start: "0x401000", end: "0x401006", successors: [address] }, + ], + }), + }, + { + name: "referenced name source", + fixture: (address: string) => ({ + referenced_names: [ + { address: "0x402000", source_address: address, value: "observed" }, + ], + }), + }, + { + name: "referenced string source", + fixture: (address: string) => ({ + referenced_strings: [ + { address: "0x402000", source_address: address, value: "observed" }, + ], + }), + }, + ])("requires a canonical $name", ({ fixture }) => { + const dossier = functionDossierSchema.parse(ghidraFunctionDossier()); + expect( + parseFunctionValueFixture("analyze_function", { + ...dossier, + ...fixture("0x401000"), + }).ok, + ).toBe(true); + expect( + parseFunctionValueFixture("analyze_function", { + ...dossier, + ...fixture("401000"), + }), + ).toMatchObject({ ok: false, error: { _tag: "AnalysisOutputError" } }); + }); +}); + +describe("Ghidra native facet coordinate admission", () => { + it.each(["dispatch", "data source"] as const)( + "requires canonical jump-table %s coordinates", + (location) => { + const dossier = functionDossierSchema.parse(ghidraFunctionDossier()); + const boundary = dossier.native_api; + if (boundary?.available !== true) + throw new Error("Missing native API fixture"); + const table = boundary.jump_tables[0]; + const source = table?.data_sources[0]; + if (table === undefined || source === undefined) + throw new Error("Missing jump-table fixture"); + expect(parseFunctionValueFixture("analyze_function", dossier).ok).toBe( + true, + ); + expect( + parseFunctionValueFixture("analyze_function", { + ...dossier, + native_api: { + ...boundary, + jump_tables: [ + { + ...table, + ...(location === "dispatch" + ? { dispatch_address: "401010" } + : { data_sources: [{ ...source, address: "403000" }] }), + }, + ], + }, + }), + ).toMatchObject({ ok: false, error: { _tag: "AnalysisOutputError" } }); + }, + ); + + it.each(["operation", "input", "output"] as const)( + "requires canonical p-code %s coordinates without inventing null locations", + (location) => { + const dossier = functionDossierSchema.parse(ghidraFunctionDossier()); + const flow = dossier.native_value_flow; + if (flow?.available !== true) throw new Error("Missing p-code fixture"); + const operation = flow.operations[0]; + const input = operation?.inputs[0]; + const output = operation?.output; + if (operation === undefined || input === undefined || output == null) + throw new Error("Missing p-code operation fixture"); + for (const address of ["register:0x0", null, "register:0"] as const) { + if (location === "operation" && address === null) continue; + const changed = { + ...operation, + ...(location === "operation" + ? { address } + : location === "input" + ? { inputs: [{ ...input, location: address }] } + : { output: { ...output, location: address } }), + }; + expect( + parseFunctionValueFixture("analyze_function", { + ...dossier, + native_value_flow: { ...flow, operations: [changed] }, + }).ok, + ).toBe(address !== "register:0"); + } + }, + ); +}); diff --git a/src/ghidra/GhidraFunctionValues.ts b/src/ghidra/GhidraFunctionValues.ts index 7c2fd71db5..8f5ec281ce 100644 --- a/src/ghidra/GhidraFunctionValues.ts +++ b/src/ghidra/GhidraFunctionValues.ts @@ -1,6 +1,10 @@ import { nativeAddressObservationSchema } from "../domain/native/nativeAddress.js"; import { procedureInfoSchema } from "../domain/native/procedureInfo.js"; import { z } from "zod"; +import { + ghidraNativeApiBoundary, + ghidraNativeValueFlow, +} from "./GhidraNativeAnalysisValues.js"; import { nativeFunctionAnnotationsInputSchema, nativeFunctionAnnotationsSchema, @@ -20,15 +24,17 @@ import { functionInstructionWindowSchema, functionDossierSchema, availableReferenceKindSchema, - type FunctionDossier, + unresolvedCallSchema, + swiftSmallStringLiteralSchema, + swiftLargeStringLiteralSchema, + swiftSmallStringLiteralOmissionSchema, + swiftLargeStringLiteralOmissionSchema, } from "../domain/hopperValues.js"; import { jsonObjectSchema, jsonValueSchema, type JsonValue, } from "../domain/jsonValue.js"; -import { nativeApiBoundarySchema } from "../domain/native/nativeApiBoundary.js"; -import { nativeValueFlowSchema } from "../domain/native/nativeValueFlow.js"; import type { EvidenceLocation } from "../domain/evidence.js"; import { relationshipCoverageSchema, @@ -42,6 +48,7 @@ import { ghidraFunctionClassificationSchema as classification, ghidraFunctionBodySchema as ghidraFunctionBody, ghidraProcedureIdentitySchema as procedureIdentity, + ghidraFunctionIdentitySchema as dossierProcedureIdentity, } from "./GhidraInventoryValues.js"; /** Function analysis and annotation operations admitted by the Ghidra adapter. */ @@ -190,92 +197,82 @@ const procedureInfo = procedureInfoSchema }); }); -const ghidraNativeApiBoundary = nativeApiBoundarySchema.superRefine( - (value, context) => { - if (!value.available) return; - if (value.provenance !== "ghidra-high-function") - context.addIssue({ - code: "custom", - message: - "Ghidra native API observations require HighFunction provenance", - }); - const addresses = value.jump_tables.flatMap((table) => [ - table.dispatch_address, - ...table.data_sources.map(({ address }) => address), - ...table.mappings.map(({ target_address }) => target_address), - ...table.default_targets.map(({ target_address }) => target_address), - ]); - if ( - addresses.some( - (address) => !nativeAddressObservationSchema.safeParse(address).success, - ) - ) - context.addIssue({ - code: "custom", - message: - "Ghidra native API observation contains a non-canonical address", - }); - }, -); - -const ghidraNativeValueFlow = nativeValueFlowSchema.superRefine( - (value, context) => { - if (!value.available) return; - const operations = value.operations; - const addresses = operations.flatMap((operation) => [ - operation.address, - ...operation.inputs.flatMap(({ location }) => - location === null ? [] : [location], - ), - ...(operation.output?.location == null - ? [] - : [operation.output.location]), - ]); - if ( - addresses.some( - (address) => !nativeAddressObservationSchema.safeParse(address).success, - ) - ) - context.addIssue({ - code: "custom", - message: "Ghidra p-code flow contains a non-canonical address", - }); - const operationIds = new Set(operations.map(({ id }) => id)); - if (operationIds.size !== operations.length) - context.addIssue({ - code: "custom", - message: "Ghidra p-code operation identifiers are not unique", - }); - if ( - value.def_use.some( - ({ definition, use }) => - !operationIds.has(definition) || !operationIds.has(use), - ) || - value.effects.some(({ operation }) => !operationIds.has(operation)) - ) - context.addIssue({ - code: "custom", - message: "Ghidra p-code relationship references a missing operation", - }); - if ( - value.truncated !== - (value.omitted_operations_lower_bound > 0 || - value.known_omitted_inputs > 0 || - value.known_omitted_edges > 0) - ) - context.addIssue({ - code: "custom", - message: "Ghidra p-code truncation flag contradicts omitted counts", - }); - }, -); +const dossierReferenceEdge = + functionDossierSchema.shape.incoming_references.element.extend({ + source_address: nativeAddressObservationSchema, + target_address: nativeAddressObservationSchema, + source_procedure: dossierProcedureIdentity.nullable(), + target_procedure: dossierProcedureIdentity.nullable(), + kind: referenceKind, + }); const ghidraFunctionDossier = functionDossierSchema .extend({ - procedure: functionDossierSchema.shape.procedure.safeExtend({ - classification, - body: ghidraFunctionBody, + procedure: dossierProcedureIdentity.safeExtend({ + signature: functionDossierSchema.shape.procedure.shape.signature, + locals: z.array(localVariable), }), + comments: z.array( + functionDossierSchema.shape.comments.element.extend({ + address: nativeAddressObservationSchema, + }), + ), + callers: z.array(dossierProcedureIdentity), + callees: z.array(dossierProcedureIdentity), + incoming_references: z.array(dossierReferenceEdge), + outgoing_references: z.array(dossierReferenceEdge), + unresolved_calls: z + .array( + unresolvedCallSchema.extend({ + address: nativeAddressObservationSchema, + }), + ) + .exactOptional(), + referenced_strings: z.array( + functionDossierSchema.shape.referenced_strings.element.extend({ + address: nativeAddressObservationSchema, + source_address: nativeAddressObservationSchema, + }), + ), + referenced_names: z.array( + functionDossierSchema.shape.referenced_names.element.extend({ + address: nativeAddressObservationSchema, + source_address: nativeAddressObservationSchema, + }), + ), + swift_string_literals: z + .array( + z.discriminatedUnion("representation", [ + swiftSmallStringLiteralSchema.extend({ + source_address: nativeAddressObservationSchema, + }), + swiftLargeStringLiteralSchema.extend({ + source_address: nativeAddressObservationSchema, + address: nativeAddressObservationSchema, + }), + ]), + ) + .exactOptional(), + swift_string_literal_omissions: z + .array( + z.discriminatedUnion("representation", [ + swiftSmallStringLiteralOmissionSchema.extend({ + source_address: nativeAddressObservationSchema, + }), + swiftLargeStringLiteralOmissionSchema.extend({ + source_address: nativeAddressObservationSchema, + address: nativeAddressObservationSchema, + }), + ]), + ) + .exactOptional(), + basic_blocks: z.array( + functionDossierSchema.shape.basic_blocks.element.extend({ + start: nativeAddressObservationSchema, + end: nativeAddressObservationSchema, + successors: z.array(nativeAddressObservationSchema), + }), + ), native_api: ghidraNativeApiBoundary, native_value_flow: ghidraNativeValueFlow, }) @@ -290,79 +287,6 @@ const ghidraFunctionDossier = functionDossierSchema message: "Ghidra dossier must record Swift literals and their omissions together", }); - const addresses = [ - value.procedure.address, - ...value.comments.map(({ address }) => address), - ...value.callers.map(({ address }) => address), - ...value.callees.map(({ address }) => address), - ...(value.unresolved_calls ?? []).map(({ address }) => address), - ...value.incoming_references.flatMap(referenceAddresses), - ...value.outgoing_references.flatMap(referenceAddresses), - ...value.referenced_strings.flatMap(({ address, source_address }) => [ - address, - source_address, - ]), - ...value.referenced_names.flatMap(({ address, source_address }) => [ - address, - source_address, - ]), - ...(value.swift_string_literals ?? []).flatMap( - ({ address, source_address }) => - address === null ? [source_address] : [address, source_address], - ), - ...(value.swift_string_literal_omissions ?? []).flatMap( - ({ address, source_address }) => - address === null ? [source_address] : [address, source_address], - ), - ...value.basic_blocks.flatMap(({ start, end, successors }) => [ - start, - end, - ...successors, - ]), - ]; - if ( - addresses.some( - (address) => !nativeAddressObservationSchema.safeParse(address).success, - ) - ) - context.addIssue({ - code: "custom", - message: "Ghidra dossier contains a non-canonical address", - }); - const identities = [ - value.procedure, - ...value.callers, - ...value.callees, - ...value.incoming_references.flatMap(referenceProcedures), - ...value.outgoing_references.flatMap(referenceProcedures), - ]; - if ( - identities.some( - (identity) => - !classification.safeParse(identity.classification).success || - !ghidraFunctionBody.safeParse(identity.body).success || - !functionBodyEntryAgrees(identity.body, identity.address) || - (identity.body.available && - identity.classification?.external === false && - !identity.body.contains_entry), - ) - ) - context.addIssue({ - code: "custom", - message: "Ghidra dossier omitted function classification", - }); - if ( - value.procedure.locals.some( - (variable) => !localVariable.safeParse(variable).success, - ) || - [...value.incoming_references, ...value.outgoing_references].some( - ({ kind }) => !referenceKind.safeParse(kind).success, - ) - ) - context.addIssue({ - code: "custom", - message: "Ghidra dossier contains invalid provenance", - }); if ( !value.limitations.some((item) => /indirect|computed/u.test(item)) || !value.limitations.some((item) => /provider|Ghidra|Hopper/u.test(item)) @@ -507,21 +431,3 @@ export const parseGhidraFunctionResult = ( ), ); }; - -const referenceAddresses = ( - edge: FunctionDossier["incoming_references"][number], -): readonly string[] => [ - edge.source_address, - edge.target_address, - ...(edge.source_procedure === null ? [] : [edge.source_procedure.address]), - ...(edge.target_procedure === null ? [] : [edge.target_procedure.address]), -]; - -const referenceProcedures = ( - edge: FunctionDossier["incoming_references"][number], -): readonly NonNullable< - FunctionDossier["incoming_references"][number]["source_procedure"] ->[] => [ - ...(edge.source_procedure === null ? [] : [edge.source_procedure]), - ...(edge.target_procedure === null ? [] : [edge.target_procedure]), -]; diff --git a/src/ghidra/GhidraInventoryValues.test.ts b/src/ghidra/GhidraInventoryValues.test.ts index 0810d43659..0831b29dd4 100644 --- a/src/ghidra/GhidraInventoryValues.test.ts +++ b/src/ghidra/GhidraInventoryValues.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from "vitest"; +import { z } from "zod"; import { parseGhidraInventoryInput, @@ -256,3 +257,113 @@ describe("containing-procedure complete body evidence", () => { ).toBe(false); }); }); + +const byteSearch = (space = "") => ({ + matches: [ + { + address: `${space}0x000a`, + segment: "text", + file_offset: 10, + procedure: { address: `${space}0x0008`, name: "entry" }, + }, + ], + searched: { + blocks: [ + { + segment: "text", + start: `${space}0x0000`, + end: `${space}0x000f`, + bytes: 16, + }, + ], + bytes: 16, + skipped_uninitialized: [ + { + segment: "bss", + start: `${space}0x0020`, + end: `${space}0x002f`, + bytes: 16, + }, + ], + skipped_not_loaded: [ + { + segment: "unloaded", + start: `${space}0x0030`, + end: `${space}0x003f`, + bytes: 16, + }, + ], + }, + complete: true, + next_cursor: null as string | null, +}); + +it.each(["", "ram%3Aoverlay:"])( + "preserves padded byte-search coordinates in space %j", + (space) => { + const value = byteSearch(space); + expect(parseGhidraInventoryResult("search_bytes", value)).toEqual({ + ok: true, + value: { value }, + }); + }, +); + +it.each([ + ["match", ["matches", 0, "address"]], + ["procedure", ["matches", 0, "procedure", "address"]], + ["block start", ["searched", "blocks", 0, "start"]], + ["block end", ["searched", "blocks", 0, "end"]], + ["uninitialized start", ["searched", "skipped_uninitialized", 0, "start"]], + ["uninitialized end", ["searched", "skipped_uninitialized", 0, "end"]], + ["unloaded start", ["searched", "skipped_not_loaded", 0, "start"]], + ["unloaded end", ["searched", "skipped_not_loaded", 0, "end"]], +] as const)( + "rejects noncanonical %s at its owning address field", + (_label, path) => { + const value = byteSearch("ram%3Aoverlay:"); + let object: unknown = value; + for (const part of path.slice(0, -1)) + object = Reflect.get(object as object, part); + const field = path.at(-1)!; + const address: unknown = Reflect.get(object as object, field); + if (typeof address !== "string") + throw new Error("Expected an address fixture"); + Reflect.set(object as object, field, address.replace("%3A", "%3a")); + const result = parseGhidraInventoryResult("search_bytes", value); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected canonical-address refusal"); + expect(result.error.cause).toBeInstanceOf(z.ZodError); + if (!(result.error.cause instanceof z.ZodError)) + throw new Error("Expected field schema issues"); + expect(result.error.cause.issues).toContainEqual( + expect.objectContaining({ path: [...path] }), + ); + }, +); + +it("retains byte-search match capacity and coverage/continuation rules", () => { + const overCapacity = byteSearch(); + overCapacity.matches = Array.from( + { length: 1001 }, + () => overCapacity.matches[0]!, + ); + expect(parseGhidraInventoryResult("search_bytes", overCapacity).ok).toBe( + false, + ); + overCapacity.matches.pop(); + expect(parseGhidraInventoryResult("search_bytes", overCapacity).ok).toBe( + true, + ); + const continuation = byteSearch(); + continuation.next_cursor = "next"; + expect(parseGhidraInventoryResult("search_bytes", continuation).ok).toBe( + false, + ); + const outside = byteSearch(); + outside.matches[0]!.address = "0x0010"; + expect(parseGhidraInventoryResult("search_bytes", outside).ok).toBe(false); + const range = byteSearch(); + range.searched.skipped_not_loaded[0]!.bytes = 15; + expect(parseGhidraInventoryResult("search_bytes", range).ok).toBe(false); +}); diff --git a/src/ghidra/GhidraInventoryValues.ts b/src/ghidra/GhidraInventoryValues.ts index 7fbe4f4a28..1b73e7b964 100644 --- a/src/ghidra/GhidraInventoryValues.ts +++ b/src/ghidra/GhidraInventoryValues.ts @@ -4,6 +4,7 @@ import { nativeByteReadSchema } from "../domain/native/nativeByteRead.js"; import { z } from "zod"; import { jsonRecordSchema } from "../domain/jsonRecord.js"; import { + MAX_BYTE_SEARCH_MATCHES, bytePatternFields, validateBytePattern, byteSearchResultSchema, @@ -18,7 +19,8 @@ import { AnalysisOutputError, } from "../domain/analysisErrorCore.js"; import { - functionBodySchema, + availableFunctionBodySchema, + procedureIdentitySchema, functionBodyEntryAgrees, projectBinaryOverview, } from "../domain/hopperValues.js"; @@ -94,7 +96,7 @@ const inputSchemas = { document, start: explicitAddress.optional(), end: explicitAddress.optional(), - limit: z.int().min(1).max(1000).default(100), + limit: z.int().min(1).max(MAX_BYTE_SEARCH_MATCHES).default(100), cursor: z.string().min(1).optional(), }) .superRefine(validateBytePattern), @@ -171,48 +173,38 @@ export const ghidraFunctionClassificationSchema = procedureFacts .extend({ provenance: z.literal("ghidra-function-manager") }) .strict(); /** Require complete canonical AddressSet evidence on every Ghidra function identity. */ -export const ghidraFunctionBodySchema = functionBodySchema.superRefine( - (body, context) => { - if ( - !body.available || - body.provenance !== "ghidra-function-body-address-set" || - body.ranges.some( - ({ start, end }) => - !nativeAddressObservationSchema.safeParse(start).success || - !nativeAddressObservationSchema.safeParse(end).success, - ) - ) - context.addIssue({ - code: "custom", - message: - "Ghidra omitted complete canonical function-body AddressSet evidence", - }); - }, -); +export const ghidraFunctionBodySchema = availableFunctionBodySchema.safeExtend({ + provenance: z.literal("ghidra-function-body-address-set"), + ranges: z.array( + availableFunctionBodySchema.shape.ranges.element.extend({ + start: nativeAddressObservationSchema, + end: nativeAddressObservationSchema, + }), + ), +}); -/** Strict provider identity shared by inventory and function-analysis results. */ -export const ghidraProcedureIdentitySchema = z - .object({ +/** Canonical Ghidra identity, retaining the generic dossier's name and symbol admission. */ +export const ghidraFunctionIdentitySchema = procedureIdentitySchema + .safeExtend({ address: nativeAddressObservationSchema, - name: z.string().min(1), - symbol: nativeSymbolMetadataSchema.nullable(), classification: ghidraFunctionClassificationSchema, body: ghidraFunctionBodySchema, }) - .strict() .superRefine((value, context) => { - if ( - !functionBodyEntryAgrees(value.body, value.address) || - (value.body.available && - !value.classification.external && - !value.body.contains_entry) - ) + if (!value.classification.external && !value.body.contains_entry) context.addIssue({ code: "custom", message: "Ghidra function body disagrees with its entry point", }); }); +/** Inventory identities require a nonempty name and an explicit symbol observation. */ +export const ghidraProcedureIdentitySchema = + ghidraFunctionIdentitySchema.safeExtend({ + name: z.string().min(1), + symbol: nativeSymbolMetadataSchema.nullable(), + }); + const stringFacts = z .object({ encoding: z.string().min(1), @@ -293,6 +285,20 @@ const containingProcedure = z.discriminatedUnion("found", [ .strict(), ]); +const byteMatch = byteSearchResultSchema.shape.matches.element; +const canonicalByteMatch = byteMatch.safeExtend({ + address: nativeAddressObservationSchema, + procedure: byteMatch.shape.procedure + .unwrap() + .safeExtend({ address: nativeAddressObservationSchema }) + .nullable(), +}); +const byteCoverage = byteSearchResultSchema.shape.searched.shape; +const canonicalByteCoverage = byteCoverage.blocks.element.safeExtend({ + start: nativeAddressObservationSchema, + end: nativeAddressObservationSchema, +}); + const resultSchemas = { read_binary_inventory_summary: z .strictObject({ @@ -302,27 +308,13 @@ const resultSchemas = { string_count: z.int().min(0).max(Number.MAX_SAFE_INTEGER), }) .transform(projectBinaryOverview), - search_bytes: byteSearchResultSchema.superRefine((value, context) => { - const addresses = [ - ...value.matches.flatMap((match) => [ - match.address, - ...(match.procedure === null ? [] : [match.procedure.address]), - ]), - ...[ - ...value.searched.blocks, - ...value.searched.skipped_uninitialized, - ...value.searched.skipped_not_loaded, - ].flatMap((block) => [block.start, block.end]), - ]; - if ( - addresses.some( - (value) => !nativeAddressObservationSchema.safeParse(value).success, - ) - ) - context.addIssue({ - code: "custom", - message: "Byte search addresses must be canonical", - }); + search_bytes: byteSearchResultSchema.safeExtend({ + matches: z.array(canonicalByteMatch).max(MAX_BYTE_SEARCH_MATCHES), + searched: byteSearchResultSchema.shape.searched.safeExtend({ + blocks: z.array(canonicalByteCoverage), + skipped_uninitialized: z.array(canonicalByteCoverage), + skipped_not_loaded: z.array(canonicalByteCoverage), + }), }), inspect_native_load_image: nativeLoadImageObservationSchema, read_bytes: nativeByteReadSchema diff --git a/src/ghidra/GhidraNativeAnalysisValues.ts b/src/ghidra/GhidraNativeAnalysisValues.ts new file mode 100644 index 0000000000..7a0106c61a --- /dev/null +++ b/src/ghidra/GhidraNativeAnalysisValues.ts @@ -0,0 +1,87 @@ +import { z } from "zod"; +import { nativeAddressObservationSchema } from "../domain/native/nativeAddress.js"; +import { + availableNativeApiBoundarySchema, + unavailableNativeApiBoundarySchema, +} from "../domain/native/nativeApiBoundary.js"; +import { + availableNativeValueFlowSchema, + unavailableNativeValueFlowSchema, + nativePcodeOperationSchema, + nativeVarnodeSchema, +} from "../domain/native/nativeValueFlow.js"; + +const jumpTable = availableNativeApiBoundarySchema.shape.jump_tables.element; +const ghidraJumpTable = jumpTable.extend({ + dispatch_address: nativeAddressObservationSchema, + data_sources: z.array( + jumpTable.shape.data_sources.element.extend({ + address: nativeAddressObservationSchema, + }), + ), + mappings: z.array( + jumpTable.shape.mappings.element.extend({ + target_address: nativeAddressObservationSchema, + }), + ), + default_targets: z.array( + jumpTable.shape.default_targets.element.extend({ + target_address: nativeAddressObservationSchema, + }), + ), +}); + +/** Ghidra API facts carry canonical coordinates and HighFunction provenance. */ +export const ghidraNativeApiBoundary = z.discriminatedUnion("available", [ + unavailableNativeApiBoundarySchema, + availableNativeApiBoundarySchema.extend({ + provenance: z.literal("ghidra-high-function"), + jump_tables: z.array(ghidraJumpTable), + }), +]); + +const ghidraVarnode = nativeVarnodeSchema.extend({ + location: nativeAddressObservationSchema.nullable(), +}); +const ghidraPcodeOperation = nativePcodeOperationSchema.extend({ + address: nativeAddressObservationSchema, + inputs: z.array(ghidraVarnode), + output: ghidraVarnode.nullable(), +}); + +/** Validate relationships only after each p-code observation has been admitted. */ +export const ghidraNativeValueFlow = z.discriminatedUnion("available", [ + unavailableNativeValueFlowSchema, + availableNativeValueFlowSchema + .extend({ operations: z.array(ghidraPcodeOperation) }) + .superRefine((value, context) => { + const { operations } = value; + const operationIds = new Set(operations.map(({ id }) => id)); + if (operationIds.size !== operations.length) + context.addIssue({ + code: "custom", + message: "Ghidra p-code operation identifiers are not unique", + }); + if ( + value.def_use.some( + ({ definition, use }) => + !operationIds.has(definition) || !operationIds.has(use), + ) || + value.effects.some(({ operation }) => !operationIds.has(operation)) + ) + context.addIssue({ + code: "custom", + message: "Ghidra p-code relationship references a missing operation", + }); + if ( + value.truncated !== + (value.omitted_operations_lower_bound > 0 || + value.known_omitted_inputs > 0 || + value.known_omitted_edges > 0) + ) + context.addIssue({ + code: "custom", + message: "Ghidra p-code truncation flag contradicts omitted counts", + }); + }), +]); From 81add3d23d66fc0cfc94e4eeccfde2f7b9630751 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Mon, 26 Oct 2026 23:33:54 +0800 Subject: [PATCH 07/25] refactor(pyinstaller): centralize PYZ header and version facts --- .../pyinstaller/PyInstallerMetadata.ts | 14 +- src/artifacts/pyinstaller/PyzArchive.test.ts | 91 +++++++++--- src/artifacts/pyinstaller/PyzArchive.ts | 56 +++++--- .../mcp/pyinstallerArtifactMcp.test.ts | 136 ++++++++++++++++++ 4 files changed, 248 insertions(+), 49 deletions(-) diff --git a/src/artifacts/pyinstaller/PyInstallerMetadata.ts b/src/artifacts/pyinstaller/PyInstallerMetadata.ts index 8319721064..a5848641ad 100644 --- a/src/artifacts/pyinstaller/PyInstallerMetadata.ts +++ b/src/artifacts/pyinstaller/PyInstallerMetadata.ts @@ -4,9 +4,8 @@ import type { } from "../../domain/artifactFormatMetadata.js"; import type { CArchiveEntry, PyInstallerCookie } from "./CArchiveLayout.js"; import type { PyzArchive, PyzEntry } from "./PyzArchive.js"; -import { pythonVersionForMagic } from "./PythonBytecode.js"; -const pythonVersion = (value: number) => +const cookiePythonVersion = (value: number) => value >= 100 ? { major: Math.floor(value / 100), minor: value % 100 } : value >= 20 @@ -18,7 +17,7 @@ export const cookieMetadata = ( entries: readonly CArchiveEntry[], encryptionMarker: boolean, ): PyInstallerMetadata => { - const version = pythonVersion(cookie.pythonVersion); + const version = cookiePythonVersion(cookie.pythonVersion); return { kind: "pyinstaller", layout_version: cookie.layout, @@ -45,13 +44,13 @@ export const scriptPythonMagic = ( cookie: PyInstallerCookie, archives: Iterable, ): Buffer | undefined => { - const version = pythonVersion(cookie.pythonVersion); + const version = cookiePythonVersion(cookie.pythonVersion); if (version === undefined) return undefined; let selected: Buffer | undefined; for (const archive of archives) { const magic = archive.pythonMagic; if ( - pythonVersionForMagic(magic) !== + archive.pythonVersion !== `${String(version.major)}.${String(version.minor)}` || (selected !== undefined && !selected.equals(magic)) ) @@ -123,10 +122,7 @@ export const cArchiveEntryMetadata = ( directory_record_size: entry.tocRecordLength, pyz_sha256: archive?.sha256 ?? null, pyz_python_magic: archive?.pythonMagic.toString("hex") ?? null, - pyz_python_version: - archive === undefined - ? null - : (pythonVersionForMagic(archive.pythonMagic) ?? null), + pyz_python_version: archive?.pythonVersion ?? null, pyz_entry_count: archive?.kind === "entries" ? archive.entries.length : null, encryption_source: archive !== undefined && encryptionMarker ? "carchive-marker" : null, diff --git a/src/artifacts/pyinstaller/PyzArchive.test.ts b/src/artifacts/pyinstaller/PyzArchive.test.ts index e553589d5b..a70efae0f4 100644 --- a/src/artifacts/pyinstaller/PyzArchive.test.ts +++ b/src/artifacts/pyinstaller/PyzArchive.test.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import { open, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { expect, it } from "vitest"; @@ -31,6 +32,7 @@ it("lists modules from a supported PYZ and leaves an unknown magic opaque", asyn if (!opaque.ok || opaque.value.kind !== "opaque") return; expect(Buffer.compare(opaque.value.pythonMagic, Buffer.alloc(4))).toBe(0); expect(opaque.value.limitation).toContain("left opaque"); + expect(opaque.value.pythonVersion).toBeUndefined(); }); it("leaves a PYZ opaque when the table uses an unsupported marshal type", async () => { @@ -44,6 +46,10 @@ it("leaves a PYZ opaque when the table uses an unsupported marshal type", async if (!parsed.ok || parsed.value.kind !== "opaque") return; expect(parsed.value.limitation).toContain("left opaque"); expect(parsed.value.pythonMagic).toEqual(PYTHON_39_MAGIC); + expect(parsed.value.pythonVersion).toBe("3.9"); + expect(parsed.value.sha256).toBe( + createHash("sha256").update(image).digest("hex"), + ); }); it("reports an entry that extends outside the PYZ as an integrity error", async () => { @@ -58,26 +64,29 @@ it("reports an entry that extends outside the PYZ as an integrity error", async expect(parsed.error.reason).toBe("integrity"); }); -it("rejects an oversized sparse PYZ table before allocating its bytes", async () => { - const root = await createTestTempDirectory("rea-pyz-table-budget-"); - const path = join(root, "image.pyz"); - const header = Buffer.alloc(12); - header.write("PYZ\0"); - PYTHON_39_MAGIC.copy(header, 4); - header.writeUInt32BE(12, 8); - await writeFile(path, header); - const handle = await open(path, "r+"); - try { - const size = 12 + MAX_PYZ_TABLE_BYTES + 1; - await handle.truncate(size); - expect(await readPyzArchive(handle, 0, size)).toMatchObject({ - ok: false, - error: { reason: "limit" }, - }); - } finally { - await handle.close(); - } -}); +it.each(["PYZ\0", "BAD\0"])( + "rejects an oversized sparse table before deferred signature rejection (%s)", + async (signature) => { + const root = await createTestTempDirectory("rea-pyz-table-budget-"); + const path = join(root, "image.pyz"); + const header = Buffer.alloc(12); + header.write(signature); + PYTHON_39_MAGIC.copy(header, 4); + header.writeUInt32BE(12, 8); + await writeFile(path, header); + const handle = await open(path, "r+"); + try { + const size = 12 + MAX_PYZ_TABLE_BYTES + 1; + await handle.truncate(size); + expect(await readPyzArchive(handle, 0, size)).toMatchObject({ + ok: false, + error: { reason: "limit" }, + }); + } finally { + await handle.close(); + } + }, +); it("rejects a PYZ table whose back-references repeat one long row", async () => { // One namespace row with a 16 KiB name, flagged for reference, then 999 @@ -124,14 +133,52 @@ const int32 = (value: number): Buffer => { return bytes; }; -const readImage = async (bytes: Buffer) => { +const readImage = async (bytes: Buffer, declaredSize = bytes.length) => { const root = await createTestTempDirectory("rea-pyz-file-"); const path = join(root, "image.pyz"); await writeFile(path, bytes); const handle = await open(path, "r"); try { - return await readPyzArchive(handle, 0, bytes.length); + return await readPyzArchive(handle, 0, declaredSize); } finally { await handle.close(); } }; + +it.each([0, 0xffffffff])( + "retains an unknown Python magic and full-image digest with invalid TOC offset %s", + async (offset) => { + const image = pyzArchive({ magic: Buffer.from([1, 2, 3, 4]), modules: [] }); + image.writeUInt32BE(offset, 8); + expect(await readImage(image)).toMatchObject({ + ok: true, + value: { + kind: "opaque", + pythonMagic: Buffer.from([1, 2, 3, 4]), + pythonVersion: undefined, + sha256: createHash("sha256").update(image).digest("hex"), + limitation: expect.stringContaining("not in the supported set"), + }, + }); + }, +); + +it("reports a truncated table before deferred signature rejection", async () => { + const header = Buffer.alloc(12); + header.write("BAD\0"); + PYTHON_39_MAGIC.copy(header, 4); + header.writeUInt32BE(12, 8); + expect(await readImage(header, 16)).toMatchObject({ + ok: false, + error: { reason: "format", message: "PYZ table is truncated" }, + }); +}); + +it("reports the signature after completing a readable table and image hash", async () => { + const image = pyzArchive({ magic: PYTHON_39_MAGIC, modules: [] }); + image.write("BAD\0"); + expect(await readImage(image)).toMatchObject({ + ok: false, + error: { reason: "format", message: "PYZ archive magic does not match" }, + }); +}); diff --git a/src/artifacts/pyinstaller/PyzArchive.ts b/src/artifacts/pyinstaller/PyzArchive.ts index c398cfae0f..7dcbccbec2 100644 --- a/src/artifacts/pyinstaller/PyzArchive.ts +++ b/src/artifacts/pyinstaller/PyzArchive.ts @@ -38,6 +38,7 @@ export type PyzArchive = | { readonly kind: "opaque"; readonly pythonMagic: Buffer; + readonly pythonVersion: string | undefined; readonly sha256: string; readonly limitation: string; }; @@ -45,6 +46,31 @@ export type PyzArchive = /** Bound the transient marshal table allocation independently of image expansion. */ export const MAX_PYZ_TABLE_BYTES = 16 * 1024 * 1024; +interface PyzHeaderFacts { + readonly signatureMatches: boolean; + readonly pythonMagic: Buffer; + readonly pythonVersion: string | undefined; + readonly tocOffset: number; + /** Null means the declaration lies outside this image, not an empty table. */ + readonly tableBytes: number | null; +} + +/** Called only after the exact fixed-header read has succeeded. */ +const readPyzHeaderFacts = (header: Buffer, size: number): PyzHeaderFacts => { + const pythonMagic = Buffer.from(header.subarray(4, 8)); + const tocOffset = header.readUInt32BE(8); + return { + signatureMatches: header.subarray(0, 4).equals(PYZ_MAGIC), + pythonMagic, + pythonVersion: pythonVersionForMagic(pythonMagic), + tocOffset, + tableBytes: + tocOffset >= PYZ_HEADER_BYTES && tocOffset <= size + ? size - tocOffset + : null, + }; +}; + /** Parse only the header and table in memory; hash the complete file range in chunks. */ export const readPyzArchive = async ( handle: FileHandle, @@ -63,12 +89,9 @@ export const readPyzArchive = async ( signal, }); if (!headerRead.ok) return headerRead; - const header = headerRead.value; - const tocOffset = - header.length === PYZ_HEADER_BYTES ? header.readUInt32BE(8) : size; - const validOffset = tocOffset >= PYZ_HEADER_BYTES && tocOffset <= size; - const knownMagic = pythonVersionForMagic(header.subarray(4, 8)) !== undefined; - const tableSize = validOffset && knownMagic ? size - tocOffset : 0; + const header = readPyzHeaderFacts(headerRead.value, size); + const tableSize = + header.pythonVersion === undefined ? 0 : (header.tableBytes ?? 0); if (tableSize > MAX_PYZ_TABLE_BYTES) return err( pyInstallerFailure( @@ -77,7 +100,7 @@ export const readPyzArchive = async ( ), ); const read = await readPyInstallerBytes(handle, { - position: start + tocOffset, + position: start + header.tocOffset, length: tableSize, label: "PYZ table", signal, @@ -96,33 +119,28 @@ export const readPyzArchive = async ( return err( pyInstallerFailure("format", "PYZ image extends outside the file"), ); - return parsePyzParts(header, table, size, digest.sha256); + return parsePyzParts(header, table, digest.sha256); }; const parsePyzParts = ( - header: Buffer, + header: PyzHeaderFacts, table: Buffer, - size: number, sha256: string, ): Result => { - if ( - header.length < PYZ_HEADER_BYTES || - !header.subarray(0, 4).equals(PYZ_MAGIC) - ) + if (!header.signatureMatches) return err( pyInstallerFailure("format", "PYZ archive magic does not match"), ); - const pythonMagic = Buffer.from(header.subarray(4, 8)); - const pythonVersion = pythonVersionForMagic(pythonMagic); + const { pythonMagic, pythonVersion, tocOffset } = header; if (pythonVersion === undefined) return ok({ kind: "opaque", pythonMagic, + pythonVersion, sha256, limitation: `PYZ Python magic ${pythonMagic.toString("hex")} is not in the supported set; the member was left opaque.`, }); - const tocOffset = header.readUInt32BE(8); - if (tocOffset < PYZ_HEADER_BYTES || tocOffset > size) + if (header.tableBytes === null) return err( pyInstallerFailure( "format", @@ -135,6 +153,7 @@ const parsePyzParts = ( return ok({ kind: "opaque", pythonMagic, + pythonVersion, sha256, limitation: `PYZ table of contents could not be parsed (${marshalled.error.message}); the member was left opaque.`, }); @@ -145,6 +164,7 @@ const parsePyzParts = ( ? ok({ kind: "opaque", pythonMagic, + pythonVersion, sha256, limitation: `${entries.error.message}; the member was left opaque.`, }) diff --git a/tests/boundary/mcp/pyinstallerArtifactMcp.test.ts b/tests/boundary/mcp/pyinstallerArtifactMcp.test.ts index 4401226c77..cb210b2235 100644 --- a/tests/boundary/mcp/pyinstallerArtifactMcp.test.ts +++ b/tests/boundary/mcp/pyinstallerArtifactMcp.test.ts @@ -436,3 +436,139 @@ it("preserves surrogatepass PYZ module names through inspect_artifact", async () await session.close(); } }); + +it.each([ + { + name: "recognized opaque magic agrees with cookie", + cookie: 309, + magic: PYTHON_39_MAGIC, + version: "3.9", + conflict: false, + derived: true, + }, + { + name: "recognized opaque magic disagrees with cookie", + cookie: 312, + magic: PYTHON_39_MAGIC, + version: "3.9", + conflict: false, + derived: false, + }, + { + name: "unknown opaque magic with invalid TOC", + cookie: 309, + magic: Buffer.alloc(4), + version: null, + conflict: false, + derived: false, + }, + { + name: "recognized opaque siblings disagree", + cookie: 309, + magic: PYTHON_39_MAGIC, + version: "3.9", + conflict: true, + derived: false, + }, +])( + "retains PYZ header evidence through inspect_artifact: $name", + async ({ cookie, magic, version, conflict, derived }) => { + const payload = pyzArchive({ + magic, + modules: [], + toc: Buffer.from([0x4e]), + }); + if (version === null) payload.writeUInt32BE(0xffffffff, 8); + const directory = await createTestTempDirectory("rea-opaque-pyz-metadata-"); + const path = join(directory, "opaque-header"); + await writeFile( + path, + cArchiveExecutable({ + layout: "2.1", + pythonVersion: cookie, + members: [ + { + name: "hello", + typeCode: "s", + payload: Buffer.from("entry script"), + compress: false, + }, + { name: "PYZ.pyz", typeCode: "z", payload, compress: true }, + ...(conflict + ? [ + { + name: "other.pyz", + typeCode: "z", + payload: pyzArchive({ + magic: Buffer.from("f30d0d0a", "hex"), + modules: [], + toc: Buffer.from([0x4e]), + }), + compress: false, + }, + ] + : []), + ], + }), + ); + const session = await createArtifactBinarySession(process.env); + const server = createServer({ kind: "session", session }); + const client = new Client({ + name: "opaque-pyz-metadata-test", + version: "1", + }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + try { + await server.connect(serverTransport); + await client.connect(clientTransport); + expect( + (await client.callTool({ name: "open_binary", arguments: { path } })) + .isError, + ).not.toBe(true); + const reply = await client.callTool({ + name: "inspect_artifact", + arguments: {}, + }); + expect(reply.isError).not.toBe(true); + const inspection = artifactInspectionResultSchema.parse( + parseEvidence(reply.structuredContent).normalized_result, + ); + const inventory = inspection.substeps[0]?.evidence.normalized_result; + expect(inventory?.coverage.status).toBe("unknown"); + expect( + inventory?.occurrences.find((row) => row.logical_path === "PYZ.pyz"), + ).toMatchObject({ + entry_kind: "file", + hash_status: "verified", + format_metadata: { + pyz_status: "opaque", + pyz_sha256: createHash("sha256").update(payload).digest("hex"), + pyz_python_magic: magic.toString("hex"), + pyz_python_version: version, + pyz_entry_count: null, + }, + }); + expect( + inventory?.occurrences.find((row) => row.logical_path === "hello") + ?.format_metadata, + ).toMatchObject({ + bytecode_header: derived ? "derived" : "unknown", + bytecode_magic: derived ? magic.toString("hex") : null, + }); + if (conflict) + expect( + inventory?.occurrences.find((row) => row.logical_path === "other.pyz") + ?.format_metadata, + ).toMatchObject({ + pyz_status: "opaque", + pyz_python_version: "3.13", + pyz_python_magic: "f30d0d0a", + }); + } finally { + await client.close(); + await server.close(); + await session.close(); + } + }, +); From cadd5b75b3605bdac72d4661bdf45f1803c39c67 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Mon, 26 Oct 2026 23:40:04 +0800 Subject: [PATCH 08/25] refactor(elf): consolidate header and section-name interpretation --- src/domain/elfFixedHeader.test.ts | 65 ++++++++++++++++++- src/domain/elfFixedHeader.ts | 57 ++++++++++++---- src/domain/elfSectionName.ts | 50 ++++++++++++++ src/native/elf/ElfNoteParser.test.ts | 49 ++++++++++++++ src/native/elf/ElfNoteParser.ts | 32 ++++----- src/native/metadata/NativeSectionDirectory.ts | 58 +++++++++-------- .../RecordedCrashStructureBindings.test.ts | 43 ++++++++++++ .../RecordedCrashStructureBindings.ts | 20 +++--- 8 files changed, 308 insertions(+), 66 deletions(-) create mode 100644 src/domain/elfSectionName.ts diff --git a/src/domain/elfFixedHeader.test.ts b/src/domain/elfFixedHeader.test.ts index 154f62ecce..00572006d5 100644 --- a/src/domain/elfFixedHeader.test.ts +++ b/src/domain/elfFixedHeader.test.ts @@ -1,5 +1,5 @@ import { expect, it } from "vitest"; -import { readElfFixedHeader } from "./elfFixedHeader.js"; +import { readElfFixedHeader, readElfHeaderFacts } from "./elfFixedHeader.js"; import { elf } from "./binaryTarget.fixture.js"; it.each([ @@ -92,3 +92,66 @@ it("accepts NVIDIA's vendor ELF version only for the CUDA machine and OS ABI", ( bytes.writeUInt16LE(62, 18); expect(readElfFixedHeader(bytes).ok).toBe(false); }); + +it.each([ + { cls: 1, data: 1 }, + { cls: 1, data: 2 }, + { cls: 2, data: 1 }, + { cls: 2, data: 2 }, +] as const)( + "preserves raw version, ABI and declared size facts ($cls/$data)", + ({ cls, data }) => { + const original = elf(cls, data, 0xfffe); + const bytes = Buffer.concat([Buffer.alloc(7), original]).subarray(7); + bytes[8] = 0xff; + if (data === 1) { + bytes.writeUInt32LE(0x89abcdef, 20); + bytes.writeUInt16LE(0, cls === 1 ? 40 : 52); + } else { + bytes.writeUInt32BE(0x89abcdef, 20); + bytes.writeUInt16BE(0, cls === 1 ? 40 : 52); + } + expect(readElfHeaderFacts(bytes)).toMatchObject({ + ok: true, + value: { + elfClass: cls === 1 ? 32 : 64, + byteOrder: data === 1 ? "little" : "big", + machine: 0xfffe, + version: 0x89abcdef, + abiVersion: 0xff, + headerSize: 0, + }, + }); + expect(readElfFixedHeader(bytes)).toEqual({ + ok: false, + error: "unsupported ELF header version", + }); + }, +); + +it("keeps framing failures ahead of version and declared-size admission", () => { + const bytes = elf(2, 1, 62); + bytes[6] = 0; + bytes.writeUInt32LE(0, 20); + bytes.writeUInt16LE(0, 52); + expect(readElfFixedHeader(bytes.subarray(0, 16), 16)).toEqual({ + ok: false, + error: "unsupported ELF identification version", + }); + bytes[6] = 1; + const truncated = readElfFixedHeader(bytes, 63); + expect(truncated).toEqual(readElfHeaderFacts(bytes, 63)); + expect(truncated).toMatchObject({ + ok: false, + error: expect.stringContaining("truncated ELF header"), + }); + expect(readElfFixedHeader(bytes)).toEqual({ + ok: false, + error: "unsupported ELF header version", + }); + bytes.writeUInt32LE(1, 20); + expect(readElfFixedHeader(bytes)).toEqual({ + ok: false, + error: "invalid ELF header size", + }); +}); diff --git a/src/domain/elfFixedHeader.ts b/src/domain/elfFixedHeader.ts index 1504fb93ce..df88f8fe64 100644 --- a/src/domain/elfFixedHeader.ts +++ b/src/domain/elfFixedHeader.ts @@ -10,7 +10,8 @@ type ElfArchitecture = Exclude< RawOnlyArchitecture | "arm64_32" >; -export interface ElfFixedHeader { +/** Fixed declarations after common ELF framing, before consumer admission. */ +export interface ElfHeaderFacts { readonly elfClass: 32 | 64; readonly byteOrder: "little" | "big"; readonly headerSize: number; @@ -18,20 +19,28 @@ export interface ElfFixedHeader { readonly type: number; readonly flags: number; readonly osAbi: number; + readonly version: number; + readonly abiVersion: number; +} + +export interface ElfFixedHeader extends Omit< + ElfHeaderFacts, + "version" | "abiVersion" +> { readonly architecture: ElfArchitecture | null; } /** - * Validate the common, fixed ELF header commitments. + * Decode common ELF framing and fixed fields; version and declared-size admission belong to consumers. * - * `bytes` may be a bounded prefix. `totalBytes` is the containing file size, - * so the declared header extent is checked against the file without requiring - * the whole declared extent to be present in this prefix. + * `bytes` may be a bounded prefix. `totalBytes` is the containing file size; + * common framing requires a complete fixed header in both. The declared + * headerSize is retained without requiring or admitting that entire extent. */ -export const readElfFixedHeader = ( +export const readElfHeaderFacts = ( bytes: Buffer, totalBytes = bytes.length, -): Result => { +): Result => { if (bytes.length >= 4 && bytes.toString("hex", 0, 4) !== "7f454c46") return err("not an ELF image"); if (bytes.length < 16) return err("ELF identification is truncated"); @@ -59,12 +68,7 @@ export const readElfFixedHeader = ( const readU32 = (offset: number): number => littleEndian ? bytes.readUInt32LE(offset) : bytes.readUInt32BE(offset); const machine = readU16(18); - // NVIDIA CUBIN uses ELFOSABI_CUDA and a vendor ABI version in e_version. - if (readU32(20) !== 1 && !(machine === 190 && bytes[7] === 0x33)) - return err("unsupported ELF header version"); const headerSize = readU16(elfClass === 32 ? 40 : 52); - if (headerSize < minimumSize || headerSize > totalBytes) - return err("invalid ELF header size"); return ok({ elfClass, byteOrder: littleEndian ? "little" : "big", @@ -73,7 +77,34 @@ export const readElfFixedHeader = ( type: readU16(16), flags: readU32(elfClass === 32 ? 36 : 48), osAbi: bytes[7]!, - architecture: elfArchitecture(machine) ?? null, + version: readU32(20), + abiVersion: bytes[8]!, + }); +}; + +/** Admit the ELF version and declared header extent for general analysis. */ +export const readElfFixedHeader = ( + bytes: Buffer, + totalBytes = bytes.length, +): Result => { + const parsed = readElfHeaderFacts(bytes, totalBytes); + if (!parsed.ok) return parsed; + const facts = parsed.value; + // NVIDIA CUBIN uses ELFOSABI_CUDA and a vendor ABI version in e_version. + if (facts.version !== 1 && !(facts.machine === 190 && facts.osAbi === 0x33)) + return err("unsupported ELF header version"); + const minimumSize = facts.elfClass === 32 ? 52 : 64; + if (facts.headerSize < minimumSize || facts.headerSize > totalBytes) + return err("invalid ELF header size"); + return ok({ + elfClass: facts.elfClass, + byteOrder: facts.byteOrder, + headerSize: facts.headerSize, + machine: facts.machine, + type: facts.type, + flags: facts.flags, + osAbi: facts.osAbi, + architecture: elfArchitecture(facts.machine) ?? null, }); }; diff --git a/src/domain/elfSectionName.ts b/src/domain/elfSectionName.ts new file mode 100644 index 0000000000..d995c3f238 --- /dev/null +++ b/src/domain/elfSectionName.ts @@ -0,0 +1,50 @@ +/** Bounded source observation; consumers own scan limits, charging and caching. */ +export type ElfSectionNameObservation = + | { + readonly state: "outside"; + readonly bytes: null; + readonly examinedBytes: 0; + readonly contentBytes: 0; + } + | { + readonly state: "terminated"; + readonly bytes: Buffer; + readonly examinedBytes: number; + readonly contentBytes: number; + } + | { + readonly state: "truncated"; + readonly bytes: null; + readonly examinedBytes: number; + readonly contentBytes: number; + }; + +/** Observe an ELF string-table name without treating a bounded missing NUL as absence. */ +export const observeElfSectionName = ( + table: Buffer, + offset: number, + maximumBytes: number, +): ElfSectionNameObservation => { + if (offset < 0 || offset >= table.length) + return { state: "outside", bytes: null, examinedBytes: 0, contentBytes: 0 }; + const window = table.subarray(offset, offset + maximumBytes); + const end = window.indexOf(0); + if (end < 0) + return { + state: "truncated", + bytes: null, + examinedBytes: window.length, + contentBytes: window.length, + }; + return { + state: "terminated", + bytes: window.subarray(0, end), + examinedBytes: end + 1, + contentBytes: end, + }; +}; + +/** Project admitted source bytes without inferring an original text encoding. */ +export const elfSectionNameText = ( + observed: ElfSectionNameObservation, +): string | null => observed.bytes?.toString("latin1") ?? null; diff --git a/src/native/elf/ElfNoteParser.test.ts b/src/native/elf/ElfNoteParser.test.ts index 0db69cc8d7..5ce0c87a10 100644 --- a/src/native/elf/ElfNoteParser.test.ts +++ b/src/native/elf/ElfNoteParser.test.ts @@ -5,6 +5,7 @@ import { inspectElfNotesInputSchema } from "../../domain/native/elfNotes.js"; import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js"; import { BinaryFormatFailure } from "../metadata/BinaryContainer.js"; import { decodeElfSectionNameTable } from "../metadata/ElfSectionNameTable.js"; +import { readElfDirectoryFromBuffer } from "../metadata/NativeSectionDirectory.js"; import { compressedElfNoteFixture, compressedElfNoteSectionFixture, @@ -474,6 +475,54 @@ const unterminatedNoteSections = ( return image; }; +it.each([ + { offset: 2, name: "é", state: "known" }, + { offset: 3, name: "", state: "known" }, + { offset: 4, name: null, state: "unavailable" }, +])( + "retains ELF section-name offset $offset consistently across consumers", + ({ offset, name, state }) => { + const bytes = unterminatedNoteSections(1, 4); + bytes.set([0x41, 0x42, 0xe9, 0], bytes.length - 4); + bytes.writeUInt32LE(offset, 64 + 2 * 64); + const notes = parse(bytes); + const directory = readElfDirectoryFromBuffer(bytes); + expect(notes.tables[0]?.name).toBe(name); + expect(directory.sections[2]).toMatchObject({ name, name_state: state }); + if (name === null) { + expect(notes.coverage.status).toBe("complete"); + expect(notes.limitations).toContain( + "ELF section names are incomplete because a name offset is outside its string table.", + ); + expect(directory.section_names_limitation).toBe( + "ELF section names are incomplete because a name offset is outside its string table", + ); + } + }, +); + +it.each([65535, 65536])( + "preserves consumer-specific section-name windows for NUL at byte %i", + (terminatorOffset) => { + const bytes = unterminatedNoteSections(1, terminatorOffset + 1); + bytes[bytes.length - 1] = 0; + const name = "A".repeat(terminatorOffset); + const notes = parse(bytes); + const directory = readElfDirectoryFromBuffer(bytes); + expect(notes.tables[0]?.name).toBe(name); + expect(notes.coverage.status).toBe("complete"); + expect(directory.sections[2]).toMatchObject( + terminatorOffset === 65535 + ? { name, name_state: "known" } + : { name: null, name_state: "truncated" }, + ); + if (terminatorOffset === 65536) + expect(directory.section_names_limitation).toBe( + "ELF section names are unavailable because a name is not NUL-terminated", + ); + }, +); + it("bounds repeated scans of an unterminated section name", () => { const report = parse(unterminatedNoteSections(2, 64)); expect(report.tables.map(({ name }) => name)).toEqual([null, null]); diff --git a/src/native/elf/ElfNoteParser.ts b/src/native/elf/ElfNoteParser.ts index 103ec300fd..3b0bb17758 100644 --- a/src/native/elf/ElfNoteParser.ts +++ b/src/native/elf/ElfNoteParser.ts @@ -1,3 +1,7 @@ +import { + observeElfSectionName, + elfSectionNameText, +} from "../../domain/elfSectionName.js"; import { readElfSectionHeader } from "../../domain/elfSectionHeader.js"; import { readElfProgramHeader } from "../../domain/elfProgramHeader.js"; import { readElfRawTableDeclarations } from "../../domain/elfRawTableDeclarations.js"; @@ -124,28 +128,24 @@ class ElfReader { offset: number, text: TextBudget, ): { name: string | null; limitation: string | null } { - if (offset < 0 || offset >= table.length) + const observed = observeElfSectionName( + table, + offset, + text.limit - text.used + 1, + ); + if (observed.state === "outside") return { name: null, limitation: "ELF section names are incomplete because a name offset is outside its string table.", }; - const affordable = text.limit - text.used; - const available = table.length - offset; - const scanned = Math.min(available, affordable + 1); - const end = table.subarray(offset, offset + scanned).indexOf(0); - if (end < 0) { - chargeText(text, scanned); - return { - name: null, - limitation: - "ELF section names are incomplete because a name is not NUL-terminated.", - }; - } - chargeText(text, end); + chargeText(text, observed.contentBytes); return { - name: table.subarray(offset, offset + end).toString("latin1"), - limitation: null, + name: elfSectionNameText(observed), + limitation: + observed.state === "truncated" + ? "ELF section names are incomplete because a name is not NUL-terminated." + : null, }; } } diff --git a/src/native/metadata/NativeSectionDirectory.ts b/src/native/metadata/NativeSectionDirectory.ts index 13a1e2f409..69ed2040af 100644 --- a/src/native/metadata/NativeSectionDirectory.ts +++ b/src/native/metadata/NativeSectionDirectory.ts @@ -1,3 +1,7 @@ +import { + observeElfSectionName, + elfSectionNameText, +} from "../../domain/elfSectionName.js"; import { observeMzWindowsHeader } from "../../domain/dosMz.js"; import { readElfSectionHeader } from "../../domain/elfSectionHeader.js"; import { readElfProgramHeader } from "../../domain/elfProgramHeader.js"; @@ -215,36 +219,36 @@ export function* readElfSectionDirectory( let name: string | null = null; let name_state: NativeSection["name_state"] = "unavailable"; if (names !== null) { - if (nameOffset >= names.length) { - section_names_limitation ??= - "ELF section names are incomplete because a name offset is outside its string table"; - } else { - let decoded = decodedNames.get(nameOffset); - if (decoded === undefined) { - const window = names.subarray( - nameOffset, - Math.min(nameOffset + 65536, names.length), + let decoded = decodedNames.get(nameOffset); + if (decoded === undefined) { + const observed = observeElfSectionName(names, nameOffset, 65536); + nameBytes += observed.examinedBytes; + if (nameBytes > STRUCTURE_BYTES) + throw new BinaryResourceFailure( + "structure", + STRUCTURE_BYTES, + "Decoded ELF section names exceed the 16 MiB structural budget", ); - const end = window.indexOf(0); - nameBytes += end < 0 ? window.length : end + 1; - if (nameBytes > STRUCTURE_BYTES) - throw new BinaryResourceFailure( - "structure", - STRUCTURE_BYTES, - "Decoded ELF section names exceed the 16 MiB structural budget", - ); - decoded = { - name: end < 0 ? null : window.toString("latin1", 0, end), - state: end < 0 ? "truncated" : "known", - }; - if (end < 0) - section_names_limitation ??= - "ELF section names are unavailable because a name is not NUL-terminated"; + decoded = { + name: elfSectionNameText(observed), + state: + observed.state === "terminated" + ? "known" + : observed.state === "outside" + ? "unavailable" + : "truncated", + }; + if (observed.state === "outside") + section_names_limitation ??= + "ELF section names are incomplete because a name offset is outside its string table"; + else if (observed.state === "truncated") + section_names_limitation ??= + "ELF section names are unavailable because a name is not NUL-terminated"; + if (observed.state !== "outside") decodedNames.set(nameOffset, decoded); - } - name_state = decoded.state; - name = decoded.name; } + name_state = decoded.state; + name = decoded.name; } const { type, offset, size } = section; sections.push({ diff --git a/src/native/pwntools/RecordedCrashStructureBindings.test.ts b/src/native/pwntools/RecordedCrashStructureBindings.test.ts index 9f576a20c8..4f603028be 100644 --- a/src/native/pwntools/RecordedCrashStructureBindings.test.ts +++ b/src/native/pwntools/RecordedCrashStructureBindings.test.ts @@ -80,3 +80,46 @@ it("binds PN_XNUM through raw section-zero info without full ELF numbering admis validateRecordedCrashStructure(report, bytes, report.decoder_diagnostics), ).toThrow(/original ELF\/note headers/); }); + +it.each([ + ["magic", 0, 0], + ["class", 4, 1], + ["byte order", 5, 2], + ["identification version", 6, 0], + ["OS ABI", 7, 0xff], + ["ABI version", 8, 1], + ["object type", 16, 2], + ["machine", 18, 0], + ["header version", 20, 0], +] as const)( + "rejects mismatched core %s while retaining decoder diagnostics", + (_name, offset, value) => { + const report = recordedCrashSchema.parse(recordedCrashFixture()); + const bytes = recordedCrashFixtureBytes(report); + bytes[offset] = value; + let failure: unknown; + try { + validateRecordedCrashStructure(report, bytes, report.decoder_diagnostics); + } catch (cause: unknown) { + failure = cause; + } + expect(failure).toMatchObject({ + _tag: "AnalysisOutputError", + operation: "inspect_recorded_crash", + capturedOutput: report.decoder_diagnostics, + }); + }, +); + +it.each([0, 3])( + "retains supported core OS ABI %i independently of declared header size", + (osAbi) => { + const report = recordedCrashSchema.parse(recordedCrashFixture()); + const bytes = recordedCrashFixtureBytes(report); + bytes[7] = osAbi; + bytes.writeUInt16LE(0xffff, 52); + expect(() => + validateRecordedCrashStructure(report, bytes, report.decoder_diagnostics), + ).not.toThrow(); + }, +); diff --git a/src/native/pwntools/RecordedCrashStructureBindings.ts b/src/native/pwntools/RecordedCrashStructureBindings.ts index eb2f9fedcc..6ec7177b04 100644 --- a/src/native/pwntools/RecordedCrashStructureBindings.ts +++ b/src/native/pwntools/RecordedCrashStructureBindings.ts @@ -1,3 +1,4 @@ +import { readElfHeaderFacts } from "../../domain/elfFixedHeader.js"; import { readElfNoteHeader } from "../../domain/native/elfNoteLayout.js"; import { readElfSectionHeader } from "../../domain/elfSectionHeader.js"; import { readElfProgramHeader } from "../../domain/elfProgramHeader.js"; @@ -96,16 +97,17 @@ export function validateRecordedCrashStructure( const header = bytesAt(0n, 64); const expectedMachine = RECORDED_LINUX_CORE_PROFILES[report.architecture].machine; + const parsedHeader = readElfHeaderFacts(header); + if (!parsedHeader.ok) return fail(); + const facts = parsedHeader.value; if ( - !header - .subarray(0, 6) - .equals(Buffer.from([0x7f, 0x45, 0x4c, 0x46, 2, 1])) || - header[6] !== 1 || - header.readUInt32LE(20) !== 1 || - ![0, 3].includes(header[7] ?? -1) || - header[8] !== 0 || - header.readUInt16LE(16) !== 4 || - header.readUInt16LE(18) !== expectedMachine + facts.elfClass !== 64 || + facts.byteOrder !== "little" || + facts.version !== 1 || + ![0, 3].includes(facts.osAbi) || + facts.abiVersion !== 0 || + facts.type !== 4 || + facts.machine !== expectedMachine ) fail(); const rawTables = readElfRawTableDeclarations(header, { From a5ad1642831b386be04b97ab9de9fb8038f74d84 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Mon, 26 Oct 2026 23:44:08 +0800 Subject: [PATCH 09/25] fix(browser): unify worker target admission across capture workflows --- src/browser/CdpElectronWorkers.ts | 49 ++++-------- src/browser/CdpPageCaptureWorkers.test.ts | 75 +++++++++++++++++- src/browser/CdpPageCaptureWorkers.ts | 49 ++++-------- src/browser/CdpWorkerTarget.ts | 70 ++++++++++++++++ src/browser/CdpWorkerTargetOwnership.ts | 38 --------- .../browser/cdpElectronProvider.test.ts | 79 +++++++++++++++++++ 6 files changed, 253 insertions(+), 107 deletions(-) create mode 100644 src/browser/CdpWorkerTarget.ts delete mode 100644 src/browser/CdpWorkerTargetOwnership.ts diff --git a/src/browser/CdpElectronWorkers.ts b/src/browser/CdpElectronWorkers.ts index 51a6b1e9e2..18f1bd3304 100644 --- a/src/browser/CdpElectronWorkers.ts +++ b/src/browser/CdpElectronWorkers.ts @@ -5,13 +5,16 @@ import type { } from "../domain/javascript/electronObservation.js"; import type { CdpConnection } from "./CdpConnection.js"; import { CdpCaptureCompleteness } from "./CdpCaptureCompleteness.js"; -import { recordValue, cdpStringValue } from "./CdpCaptureValues.js"; +import { recordValue } from "./CdpCaptureValues.js"; import type { CdpEndpointTarget } from "./CdpEndpoint.js"; import { optionalCdpCommand } from "./CdpOptionalCommand.js"; import { authorizedElectronFile } from "./ElectronFileScope.js"; import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; import { AnalysisCancelledError } from "../domain/analysisErrorCore.js"; -import { workerTargetOwnership } from "./CdpWorkerTargetOwnership.js"; +import { + parseCdpWorkerTarget, + workerTargetOwnership, +} from "./CdpWorkerTarget.js"; /** Inventory local worker targets without attaching to them. */ export const captureElectronWorkers = async (input: { @@ -47,33 +50,27 @@ export const captureElectronWorkers = async (input: { } for (const entry of targetInfos) { throwIfElectronWorkerCaptureCancelled(input.signal); - const target = recordValue(entry); - if (target === undefined) { + const parsed = parseCdpWorkerTarget(entry); + if (!parsed.ok) { input.completeness.exclude("workers", "invalid_protocol_value"); continue; } - const type = cdpStringValue(target.type) ?? ""; - if (!type.includes("worker")) continue; + const target = parsed.value; + if (target.kind === "non_worker") continue; const ownership = workerTargetOwnership( target, input.target.id, input.frameIds, ); - if (!ownership.ok) { - input.completeness.exclude("workers", "invalid_protocol_value"); - continue; - } - if (ownership.value.scope !== "selected") { + if (ownership !== "selected") { input.completeness.exclude( "workers", - ownership.value.scope === "unknown" - ? "unattributed_target" - : "out_of_target_scope", + ownership === "unknown" ? "unattributed_target" : "out_of_target_scope", ); continue; } const path = await authorizedElectronFile( - cdpStringValue(target.url) ?? "", + target.url, input.signal, "inspect_electron_page", input.asarHeaderAdmission, @@ -82,22 +79,13 @@ export const captureElectronWorkers = async (input: { input.completeness.exclude("workers", "out_of_target_scope"); continue; } - const targetId = boundedTargetField(target.targetId); - if (targetId === null) { - input.completeness.exclude("workers", "invalid_protocol_value"); - continue; - } - if (typeof target.attached !== "boolean") { - input.completeness.exclude("workers", "invalid_protocol_value"); - continue; - } workers.push({ - target_id: targetId, - type, + target_id: target.targetId, + type: target.type, file_path: path, attached: target.attached, - opener_target_id: ownership.value.openerTargetId, - parent_frame_id: ownership.value.parentFrameId, + opener_target_id: target.openerTargetId, + parent_frame_id: target.parentFrameId, }); } return workers.sort((left, right) => @@ -111,8 +99,3 @@ const throwIfElectronWorkerCaptureCancelled = (signal?: AbortSignal): void => { cause: signal.reason, }); }; - -const boundedTargetField = (value: unknown): string | null => { - const field = cdpStringValue(value); - return field === undefined || field === "" ? null : field; -}; diff --git a/src/browser/CdpPageCaptureWorkers.test.ts b/src/browser/CdpPageCaptureWorkers.test.ts index 8a3f9e5e04..cc9ec57d91 100644 --- a/src/browser/CdpPageCaptureWorkers.test.ts +++ b/src/browser/CdpPageCaptureWorkers.test.ts @@ -9,7 +9,7 @@ import type { CaptureContext } from "./CdpPageCapture.js"; const origin = "http://127.0.0.1:43127"; -const capture = async (inventory: unknown) => { +const capture = async (inventory: unknown, frameIds = new Set()) => { const input = inspectWebPageInputSchema.parse({ cdp_endpoint: origin, allowed_origins: [origin], @@ -38,7 +38,7 @@ const capture = async (inventory: unknown) => { limitations: [], events, }, - new Set(), + frameIds, ); return { workers, completeness: events.completeness.snapshot() }; }; @@ -199,3 +199,74 @@ it.each([{ openerId: "" }, { parentFrameId: null }])( }); }, ); + +it.each([ + { type: undefined }, + { type: 7 }, + { type: "" }, + { targetId: "" }, + { url: undefined }, + { attached: null }, + { openerId: null }, + { parentFrameId: "" }, +])( + "reports malformed worker facts before ownership and URL policy: %j", + async (override) => { + const result = await capture({ + targetInfos: [ + { + targetId: "worker", + type: "worker", + url: "https://outside.example/worker.js", + attached: false, + openerId: "foreign-page", + ...override, + }, + ], + }); + expect(result.workers).toEqual([]); + expect(result.completeness.excluded).toEqual([ + { section: "workers", reason: "invalid_protocol_value", count: 1 }, + ]); + }, +); + +it("uses parsed frame relationships without treating valid non-workers as malformed", async () => { + const result = await capture( + { + targetInfos: [ + { type: "page" }, + { + targetId: "worker", + type: "shared_worker", + url: `${origin}/worker.js`, + attached: false, + openerId: "foreign-page", + parentFrameId: "selected-frame", + }, + { + targetId: "foreign", + type: "worker", + url: `${origin}/foreign.js`, + attached: true, + parentFrameId: "foreign-frame", + }, + ], + }, + new Set(["selected-frame"]), + ); + expect(result.workers).toEqual([ + { + target_id: "worker", + type: "shared_worker", + url: `${origin}/worker.js`, + origin, + attached: false, + opener_target_id: "foreign-page", + parent_frame_id: "selected-frame", + }, + ]); + expect(result.completeness.excluded).toEqual([ + { section: "workers", reason: "out_of_target_scope", count: 1 }, + ]); +}); diff --git a/src/browser/CdpPageCaptureWorkers.ts b/src/browser/CdpPageCaptureWorkers.ts index 7415feef19..28f066f8e1 100644 --- a/src/browser/CdpPageCaptureWorkers.ts +++ b/src/browser/CdpPageCaptureWorkers.ts @@ -4,11 +4,13 @@ import { admitRecords, allowedSanitizedUrl, recordValue, - cdpStringValue, } from "./CdpCaptureValues.js"; import type { CaptureContext } from "./CdpPageCapture.js"; import { optionalCdpCommand } from "./CdpOptionalCommand.js"; -import { workerTargetOwnership } from "./CdpWorkerTargetOwnership.js"; +import { + parseCdpWorkerTarget, + workerTargetOwnership, +} from "./CdpWorkerTarget.js"; import { exclusionReasonForUrl } from "./CdpCaptureEventHelpers.js"; interface CaptureWorkersInput { @@ -42,59 +44,38 @@ export const captureWorkers = async ( } let malformedEntries = 0; for (const entry of targets ?? []) { - const target = recordValue(entry); - if (target === undefined) { - malformedEntries += 1; - continue; - } - const type = cdpStringValue(target.type); - if (type === undefined || type.length === 0) { - malformedEntries += 1; - continue; - } - if (!type.includes("worker")) continue; - const targetId = cdpStringValue(target.targetId); - if ( - targetId === undefined || - targetId.length === 0 || - typeof target.attached !== "boolean" || - typeof target.url !== "string" - ) { + const parsed = parseCdpWorkerTarget(entry); + if (!parsed.ok) { malformedEntries += 1; continue; } + const target = parsed.value; + if (target.kind === "non_worker") continue; const ownership = workerTargetOwnership( target, context.target.id, frameIds, ); - if (!ownership.ok) { - malformedEntries += 1; - continue; - } - if (ownership.value.scope !== "selected") { + if (ownership !== "selected") { completeness.exclude( "workers", - ownership.value.scope === "unknown" - ? "unattributed_target" - : "out_of_target_scope", + ownership === "unknown" ? "unattributed_target" : "out_of_target_scope", ); continue; } const url = allowedSanitizedUrl(target.url, allowedOrigins); if (url === undefined) { - const rawUrl = cdpStringValue(target.url); - completeness.exclude("workers", exclusionReasonForUrl(rawUrl)); + completeness.exclude("workers", exclusionReasonForUrl(target.url)); continue; } items.push({ - target_id: targetId, - type, + target_id: target.targetId, + type: target.type, url: url.url, origin: url.origin, attached: target.attached, - opener_target_id: ownership.value.openerTargetId, - parent_frame_id: ownership.value.parentFrameId, + opener_target_id: target.openerTargetId, + parent_frame_id: target.parentFrameId, }); } if (malformedEntries > 0) diff --git a/src/browser/CdpWorkerTarget.ts b/src/browser/CdpWorkerTarget.ts new file mode 100644 index 0000000000..a90e6a5d55 --- /dev/null +++ b/src/browser/CdpWorkerTarget.ts @@ -0,0 +1,70 @@ +import { err, ok, type Result } from "../domain/result.js"; +import { cdpStringValue, recordValue } from "./CdpCaptureValues.js"; + +export interface CdpWorkerRelationships { + readonly openerTargetId: string | null; + readonly parentFrameId: string | null; +} + +export interface CdpWorkerTarget extends CdpWorkerRelationships { + readonly kind: "worker"; + readonly targetId: string; + readonly type: string; + readonly url: string; + readonly attached: boolean; +} + +/** Admit protocol facts before either consumer applies target or URL policy. */ +export const parseCdpWorkerTarget = ( + input: unknown, +): Result< + CdpWorkerTarget | { readonly kind: "non_worker" }, + { readonly _tag: "InvalidWorkerTarget" } +> => { + const target = recordValue(input); + const type = cdpStringValue(target?.type); + if (target === undefined || type === undefined || type === "") + return err({ _tag: "InvalidWorkerTarget" }); + if (!type.includes("worker")) return ok({ kind: "non_worker" }); + const targetId = cdpStringValue(target.targetId); + const url = cdpStringValue(target.url); + const openerTargetId = cdpStringValue(target.openerId); + const parentFrameId = cdpStringValue(target.parentFrameId); + if ( + targetId === undefined || + targetId === "" || + url === undefined || + typeof target.attached !== "boolean" || + (target.openerId !== undefined && + (openerTargetId === undefined || openerTargetId === "")) || + (target.parentFrameId !== undefined && + (parentFrameId === undefined || parentFrameId === "")) + ) + return err({ _tag: "InvalidWorkerTarget" }); + return ok({ + kind: "worker", + targetId, + type, + url, + attached: target.attached, + openerTargetId: openerTargetId ?? null, + parentFrameId: parentFrameId ?? null, + }); +}; + +/** Optional CDP relationships establish ownership only when supplied. */ +export const workerTargetOwnership = ( + relationships: CdpWorkerRelationships, + selectedTargetId: string, + frameIds: ReadonlySet, +): "selected" | "foreign" | "unknown" => { + const { openerTargetId, parentFrameId } = relationships; + if ( + openerTargetId === selectedTargetId || + (parentFrameId !== null && frameIds.has(parentFrameId)) + ) + return "selected"; + return openerTargetId === null && parentFrameId === null + ? "unknown" + : "foreign"; +}; diff --git a/src/browser/CdpWorkerTargetOwnership.ts b/src/browser/CdpWorkerTargetOwnership.ts deleted file mode 100644 index 27611119c0..0000000000 --- a/src/browser/CdpWorkerTargetOwnership.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { err, ok, type Result } from "../domain/result.js"; -import { cdpStringValue } from "./CdpCaptureValues.js"; - -/** Optional CDP relationships establish ownership only when supplied. */ -export const workerTargetOwnership = ( - target: Readonly>, - selectedTargetId: string, - frameIds: ReadonlySet, -): Result< - { - readonly scope: "selected" | "foreign" | "unknown"; - readonly openerTargetId: string | null; - readonly parentFrameId: string | null; - }, - { readonly _tag: "InvalidWorkerRelationship" } -> => { - const openerTargetId = cdpStringValue(target.openerId); - const parentFrameId = cdpStringValue(target.parentFrameId); - if ( - (target.openerId !== undefined && - (openerTargetId === undefined || openerTargetId === "")) || - (target.parentFrameId !== undefined && - (parentFrameId === undefined || parentFrameId === "")) - ) - return err({ _tag: "InvalidWorkerRelationship" }); - const selected = - openerTargetId === selectedTargetId || - (parentFrameId !== undefined && frameIds.has(parentFrameId)); - return ok({ - scope: selected - ? "selected" - : openerTargetId === undefined && parentFrameId === undefined - ? "unknown" - : "foreign", - openerTargetId: openerTargetId ?? null, - parentFrameId: parentFrameId ?? null, - }); -}; diff --git a/tests/boundary/browser/cdpElectronProvider.test.ts b/tests/boundary/browser/cdpElectronProvider.test.ts index e4ca0f6886..f18b45ec17 100644 --- a/tests/boundary/browser/cdpElectronProvider.test.ts +++ b/tests/boundary/browser/cdpElectronProvider.test.ts @@ -849,6 +849,85 @@ describe("CdpElectronProvider malformed inventory replies", () => { ); }); + it.each([ + ["missing type", { type: undefined }], + ["non-string type", { type: 7 }], + ["empty type", { type: "" }], + ["missing target identity", { targetId: undefined }], + ["non-string target identity", { targetId: 7 }], + ["empty target identity", { targetId: "" }], + ["missing URL", { url: undefined }], + ["non-string URL", { url: 7 }], + ["missing attached state", { attached: undefined }], + ["non-boolean attached state", { attached: "false" }], + ["non-string opener relationship", { openerId: 7 }], + ["empty opener relationship", { openerId: "" }], + ["non-string frame relationship", { parentFrameId: 7 }], + ["empty frame relationship", { parentFrameId: "" }], + ] as const)( + "retains valid workers while reporting %s before target policy", + async (_kind, malformed) => { + const inspected = await inspectWith("Target.getTargets", (fileUrl) => ({ + targetInfos: [ + { + targetId: "selected-worker", + type: "worker", + url: fileUrl, + attached: false, + openerId: "electron-page", + }, + { + targetId: "malformed-worker", + type: "worker", + url: fileUrl, + attached: false, + openerId: "foreign-page", + ...malformed, + }, + { + targetId: "foreign-worker", + type: "shared_worker", + url: fileUrl, + attached: true, + openerId: "foreign-page", + }, + { + targetId: "unattributed-worker", + type: "service_worker", + url: fileUrl, + attached: false, + }, + // Non-worker rows do not need worker identity or relationship fields. + { type: "page", targetId: 7, openerId: null }, + ], + })); + expect(inspected.workers).toEqual([ + expect.objectContaining({ + target_id: "selected-worker", + type: "worker", + attached: false, + opener_target_id: "electron-page", + parent_frame_id: null, + }), + ]); + const workerExclusions = inspected.completeness.excluded.filter( + ({ section }) => section === "workers", + ); + expect(workerExclusions).toHaveLength(3); + expect(workerExclusions).toEqual( + expect.arrayContaining([ + { section: "workers", reason: "invalid_protocol_value", count: 1 }, + { section: "workers", reason: "out_of_target_scope", count: 1 }, + { section: "workers", reason: "unattributed_target", count: 1 }, + ]), + ); + expect(inspected.completeness.unavailable_sections).toContain("workers"); + expect(inspected.completeness.policy_filtered_sections).toContain( + "workers", + ); + }, + ); + it.each([ ["dom", "DOMSnapshot.captureSnapshot", () => ({ strings: [] })], [ From 4da7e46e9ce90d8a59dfc50dd7a9e400b657461a Mon Sep 17 00:00:00 2001 From: YingNing <59612604+dxqlovestudy@users.noreply.github.com> Date: Mon, 26 Oct 2026 23:49:11 +0800 Subject: [PATCH 10/25] fix(provider-selection): name whether a rejection came from availability or target support (#7220) Distinguish local provider readiness failures from target-support refusals in shared CLI/MCP diagnostics, preserving original codes and evidence. Avoid claiming configured providers are unconfigured. 120 focused tests, static/docs checks and real ELF CLI/MCP verification pass. Hosted CI has the same four ADB failures as exact main d9908fd7a; three canceled shards remain unverified. Merge uses the user's explicit authorization to bypass proven main failures. Fresh Codex and Greptile reviews are clear. Fixes #7214 Co-authored-by: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com> --- docs/cli.md | 8 +- .../binary/AnalysisProviderRegistry.test.ts | 125 ++++++++++++++++++ .../binary/AnalysisProviderRegistry.ts | 53 +++++--- src/domain/analysisErrorProjection.ts | 1 + src/domain/providerSelectionError.ts | 16 ++- src/native/pwntools/PwntoolsFailures.ts | 1 + 6 files changed, 187 insertions(+), 17 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index 367731119a..eabae901c8 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -443,7 +443,13 @@ The selected provider remains bound to the session until an explicit switch or close. Provider failures are returned with their original reason. For an `ambiguous` selection error, choose from `details.candidate_ids`; for `provider_unavailable`, follow `message`, `remediation.action`, and the original -reasons in `details.rejections`. For Hopper, Ghidra, IDA, JADX, and Kuna, run +reasons in `details.rejections`. Each rejection names its `origin`: `availability` +entries describe local readiness failures, including missing configuration or an +invalid existing registration; they do not establish target support. `target` +entries record a refused kind, format, architecture, or role. An availability-only +message describes the recorded rejections without claiming that every provider +is unconfigured or that target support is known. +For Hopper, Ghidra, IDA, JADX, and Kuna, run `rea doctor --provider ID --json` to diagnose the selected engine. Other adapters, such as `pwntools-elf`, report their configuration requirements directly and are not accepted by `doctor --provider`. See diff --git a/src/application/binary/AnalysisProviderRegistry.test.ts b/src/application/binary/AnalysisProviderRegistry.test.ts index c464390a76..10c440daa0 100644 --- a/src/application/binary/AnalysisProviderRegistry.test.ts +++ b/src/application/binary/AnalysisProviderRegistry.test.ts @@ -433,6 +433,131 @@ describe("analysis provider registry: selection failures", () => { }); }); +describe("analysis provider registry: rejection origins", () => { + it("distinguishes recorded readiness failures from an unobserved target refusal", async () => { + const registry = new AnalysisProviderRegistry([ + candidate("alpha", { available: false, unconfigured: true }).provider, + candidate("gamma", { available: false, unconfigured: true }).provider, + ]); + const selection = await registry.select(DATABASE_TARGET); + expect(selection.ok).toBe(true); + if (!selection.ok) return; + const projected = projectAnalysisError( + registry.unboundOperationError("binary_overview", selection.value), + ); + expect(projected.message).toContain( + "Provider selection failed local readiness checks. No target-support refusal was reported.", + ); + }); + + it("reports a configured provider's readiness failure without hiding a ready alternative", async () => { + const invalid = candidate("alpha", { available: false }); + const ready = candidate("beta"); + const registry = new AnalysisProviderRegistry([ + { + ...invalid.provider, + inspectAvailability: () => ({ + status: "unavailable", + code: "open_options_invalid", + reason: "Configured registration is invalid", + diagnostics: { registration_path: "/configured/alpha.json" }, + }), + }, + ready.provider, + ]); + const selected = await registry.select(DATABASE_TARGET, "alpha"); + if (selected.ok) + throw new Error("Expected the selected provider to refuse"); + const projected = projectAnalysisError(selected.error); + expect(projected.message).toContain("local readiness"); + expect(projected.message).not.toContain( + "No analysis provider is configured", + ); + expect(projected.details).toMatchObject({ + requested_provider_id: "alpha", + candidate_ids: ["alpha", "beta"], + rejections: [ + { + provider_id: "alpha", + origin: "availability", + code: "open_options_invalid", + reason: "Configured registration is invalid", + diagnostics: { registration_path: "/configured/alpha.json" }, + }, + ], + }); + expect((await registry.select(DATABASE_TARGET, "beta")).ok).toBe(true); + expect(invalid.created).toEqual([]); + }); + + it("names whether a rejection came from provider availability or target support", async () => { + const availabilityOnly = new AnalysisProviderRegistry([ + candidate("alpha", { available: false, unconfigured: true }).provider, + candidate("gamma", { available: false, unconfigured: true }).provider, + ]); + const availabilitySelection = + await availabilityOnly.select(DATABASE_TARGET); + expect(availabilitySelection.ok).toBe(true); + if (availabilitySelection.ok) { + const { details } = projectAnalysisError( + availabilityOnly.unboundOperationError( + "binary_overview", + availabilitySelection.value, + ), + ); + const rejections = details?.rejections as unknown as readonly { + origin: string; + }[]; + expect(rejections.map(({ origin }) => origin)).toEqual([ + "availability", + "availability", + ]); + } + + const unconfiguredButAdmitting: AnalysisProviderCandidate = { + ...candidate("alpha", { available: false, unconfigured: true }).provider, + inspectTargetSupport: () => ({ + status: "supported", + code: null, + reason: null, + diagnostics: { target_kind: "archive" }, + }), + }; + const refusingTarget: AnalysisProviderCandidate = { + ...candidate("beta").provider, + inspectTargetSupport: () => ({ + status: "unsupported", + code: "target_format_unsupported", + reason: "beta refuses archives", + diagnostics: { target_kind: "archive" }, + }), + }; + const mixed = new AnalysisProviderRegistry([ + unconfiguredButAdmitting, + refusingTarget, + ]); + const mixedSelection = await mixed.select(ARTIFACT_TARGET); + expect(mixedSelection.ok).toBe(true); + if (mixedSelection.ok) { + const projected = projectAnalysisError( + mixed.unboundOperationError("binary_overview", mixedSelection.value), + ); + expect(projected.message).toContain( + "The selected analysis provider is unavailable", + ); + const { details } = projected; + const rejections = details?.rejections as unknown as readonly { + origin: string; + code: string; + }[]; + expect(rejections).toMatchObject([ + { origin: "availability", code: "not_configured" }, + { origin: "target", code: "target_format_unsupported" }, + ]); + } + }); +}); + describe("analysis provider registry: profile and cleanup failures", () => { it("does not bind a profile with unresolved version or adapter failure", async () => { const unresolved = candidate("unresolved", { profile: "missing" }); diff --git a/src/application/binary/AnalysisProviderRegistry.ts b/src/application/binary/AnalysisProviderRegistry.ts index 752f992116..e07ce2820e 100644 --- a/src/application/binary/AnalysisProviderRegistry.ts +++ b/src/application/binary/AnalysisProviderRegistry.ts @@ -21,6 +21,7 @@ import { import { AnalysisError } from "../../domain/analysisErrorBase.js"; import { ProviderSelectionError, + type ProviderRejectionOrigin, type ProviderSelectionErrorOptions, type ProviderSelectionFailureReason, type ProviderSelectionRejection, @@ -880,21 +881,43 @@ const selectionError = ( const candidateRejections = ( candidates: readonly AnalysisProviderCandidateStatus[], ): readonly ProviderSelectionRejection[] => - candidates.flatMap((candidate) => - [ - ...(candidate.availability.status === "unavailable" - ? [candidate.availability] - : []), - ...(candidate.targetSupport.status === "unsupported" - ? [candidate.targetSupport] - : []), - ].map((rejection) => ({ - providerId: candidate.provider.id, - code: rejection.code, - reason: rejection.reason, - diagnostics: structuredClone(rejection.diagnostics), - })), - ); + candidates.flatMap((candidate) => { + const rejections: ProviderSelectionRejection[] = []; + if (candidate.availability.status === "unavailable") + rejections.push( + selectedRejection(candidate, "availability", candidate.availability), + ); + if (candidate.targetSupport.status === "unsupported") + rejections.push( + selectedRejection(candidate, "target", candidate.targetSupport), + ); + return rejections; + }); + +/** One availability or target-support observation that rejected a candidate. */ +type RejectionObservation = + | Extract< + AnalysisProviderCandidateStatus["availability"], + { status: "unavailable" } + > + | Extract< + AnalysisProviderCandidateStatus["targetSupport"], + { status: "unsupported" } + >; + +/** Name the observation a rejection came from so availability cannot read as a refusal. */ +const selectedRejection = ( + candidate: AnalysisProviderCandidateStatus, + origin: ProviderRejectionOrigin, + rejection: RejectionObservation, +): ProviderSelectionRejection => ({ + providerId: candidate.provider.id, + origin, + code: rejection.code, + reason: rejection.reason, + // ProviderSelectionError owns the single defensive diagnostic copy. + diagnostics: rejection.diagnostics, +}); const cloneAvailability = ( availability: AnalysisProviderCandidateStatus["availability"], diff --git a/src/domain/analysisErrorProjection.ts b/src/domain/analysisErrorProjection.ts index a852e2cd66..369318b374 100644 --- a/src/domain/analysisErrorProjection.ts +++ b/src/domain/analysisErrorProjection.ts @@ -448,6 +448,7 @@ const providerErrorDetails = ( candidate_ids: [...error.candidateIds], rejections: error.rejections.map((rejection) => ({ provider_id: rejection.providerId, + origin: rejection.origin, code: rejection.code, reason: rejection.reason, diagnostics: rejection.diagnostics, diff --git a/src/domain/providerSelectionError.ts b/src/domain/providerSelectionError.ts index e53f6ae685..1ce240946b 100644 --- a/src/domain/providerSelectionError.ts +++ b/src/domain/providerSelectionError.ts @@ -13,9 +13,13 @@ export type ProviderSelectionFailureReason = | "ambiguous" | "invalid_options"; +/** Which observation produced a rejection; availability says nothing about the target. */ +export type ProviderRejectionOrigin = "availability" | "target"; + /** One actionable candidate rejection retained in caller diagnostics. */ export interface ProviderSelectionRejection { readonly providerId: string; + readonly origin: ProviderRejectionOrigin; readonly code: string; readonly reason: string; readonly diagnostics: Readonly>; @@ -80,6 +84,14 @@ const providerSelectionDiagnostic = ( return `Analysis provider selection failed (${options.reason}) for ${options.requestedProviderId}; candidates: ${candidates}`; }; +/** True when every recorded rejection describes availability, not target support. */ +const providerAvailabilityOnly = ( + rejections: readonly ProviderSelectionRejection[] | undefined, +): boolean => + rejections !== undefined && + rejections.length > 0 && + rejections.every(({ origin }) => origin === "availability"); + const providerSelectionUserMessage = ( options: ProviderSelectionErrorOptions, ): string => { @@ -89,7 +101,9 @@ const providerSelectionUserMessage = ( case "unknown_provider": return `The requested analysis provider is unknown. Choose one of: ${options.candidateIds.join(", ") || "no configured providers"}.`; case "provider_unavailable": - return "The selected analysis provider is unavailable. Review the reported local diagnostics or run `rea doctor`, then retry."; + return providerAvailabilityOnly(options.rejections) + ? "Provider selection failed local readiness checks. No target-support refusal was reported. Review the local provider diagnostics or run `rea doctor`, then retry." + : "The selected analysis provider is unavailable. Review the reported local diagnostics or run `rea doctor`, then retry."; case "target_unsupported": return "The selected analysis provider does not support this target. Choose another provider or open a supported extracted binary."; case "invalid_options": diff --git a/src/native/pwntools/PwntoolsFailures.ts b/src/native/pwntools/PwntoolsFailures.ts index 82cb960c7f..1197d1d913 100644 --- a/src/native/pwntools/PwntoolsFailures.ts +++ b/src/native/pwntools/PwntoolsFailures.ts @@ -343,6 +343,7 @@ export const pwntoolsUnavailable = ( rejections: [ { providerId: context.providerId, + origin: "availability", code: "provider_unavailable", reason, diagnostics: { From ab44487ff86d064e924fd2c44b02f5a5430f57f3 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Mon, 26 Oct 2026 23:49:46 +0800 Subject: [PATCH 11/25] fix(adb): preserve process names and pin fixture interpreters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Consume Toybox’s single NAME separator without stripping source whitespace. Keep MCP subprocess fixtures independent of ambient PATH and validate calls through the existing named-schema helper. Validation: focused ADB provider/parser and MCP tests passed; no real Android device was used. --- src/adb/AdbProvider.test.ts | 18 ++++++++++++++++++ src/adb/AdbShellOutput.ts | 15 ++++++++------- tests/boundary/mcp/adbStartMcp.test.ts | 25 +++++++++++++------------ 3 files changed, 39 insertions(+), 19 deletions(-) diff --git a/src/adb/AdbProvider.test.ts b/src/adb/AdbProvider.test.ts index 7542dc8357..1bccde00bc 100644 --- a/src/adb/AdbProvider.test.ts +++ b/src/adb/AdbProvider.test.ts @@ -1505,6 +1505,24 @@ exit 1`, } }); +it("preserves process names printed by Toybox without treating their whitespace as column padding", async () => { + // Toybox's final NAME column is left justified without padding; only the + // single separator after S belongs to the table (toys/posix/ps.c show_ps). + const name = " native tool "; + const { binary } = await writeAdbStub( + shellStub({ + ps: `USER PID PPID VSZ RSS WCHAN ADDR S NAME\nroot 1234 567 123456 7890 0 0 S ${name}\r\n`, + }), + ); + const result = await evidenceResult(providerOver(binary), { + operation: "list_adb_processes", + input: { serial: "emulator-5554" }, + }); + expect(result.processes).toEqual([ + { user: "root", pid: 1234, ppid: 567, rss: 7890, state: "S", name }, + ]); +}); + it("lists processes, features, and services through fixed commands", async () => { const { binary } = await writeAdbStub( shellStub({ diff --git a/src/adb/AdbShellOutput.ts b/src/adb/AdbShellOutput.ts index d0112a2ca5..ed35d975a9 100644 --- a/src/adb/AdbShellOutput.ts +++ b/src/adb/AdbShellOutput.ts @@ -27,8 +27,10 @@ export interface AdbProcessEntry { readonly name: string; } +// Toybox emits one separator before the unpadded final NAME column. +// Remaining whitespace belongs to argv[0], not the table layout. const PROCESS_LINE = - /^(\S+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(.+)$/u; + /^(\S+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(\S+)\s+(\S+) (.+)$/u; const safeInteger = (text: string): number | null => { const value = Number(text); @@ -46,22 +48,21 @@ export const parseProcessListOutput = ( const processes: AdbProcessEntry[] = []; const unparsed = collectUnparsedLines(); let columns: string | null = null; - for (const rawLine of stdout.split(/\r?\n/u)) { - const line = rawLine.trimEnd(); - if (line === "") continue; + for (const line of stdout.split(/\r?\n/u)) { + if (line.trim() === "") continue; if (/^USER\s+PID\s+PPID\b/u.test(line)) { columns = line.trim(); continue; } const match = PROCESS_LINE.exec(line); if (match === null) { - unparsed.add(rawLine); + unparsed.add(line); continue; } const pid = safeInteger(match[2]!); const ppid = safeInteger(match[3]!); const rss = safeInteger(match[5]!); - if (pid === null || ppid === null || rss === null) unparsed.add(rawLine); + if (pid === null || ppid === null || rss === null) unparsed.add(line); if (pid === null) continue; processes.push({ user: match[1]!, @@ -69,7 +70,7 @@ export const parseProcessListOutput = ( ppid, rss, state: match[8]!, - name: match[9]!.trim(), + name: match[9]!, }); } return { columns, processes, unparsedLines: unparsed.facts() }; diff --git a/tests/boundary/mcp/adbStartMcp.test.ts b/tests/boundary/mcp/adbStartMcp.test.ts index 2f735ded6f..063e33726f 100644 --- a/tests/boundary/mcp/adbStartMcp.test.ts +++ b/tests/boundary/mcp/adbStartMcp.test.ts @@ -1,3 +1,4 @@ +import { artifactMcpCall } from "../../../scripts/lib/artifact-e2e.mjs"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { Ajv2020 } from "ajv/dist/2020.js"; @@ -28,7 +29,7 @@ it.skipIf(process.platform === "win32")( await chmod(localPath, 0o755); await writeFile( adbPath, - `#!/usr/bin/env node + `#!${process.execPath} const fs = require("node:fs"); const args = process.argv.slice(2); if (args[0] === "version") { process.stdout.write("Android Debug Bridge version 1.0.41\\nVersion 34.0.5-stub\\n"); } @@ -63,7 +64,7 @@ else if (args[0] === "-s" && args[2] === "shell" && args[3] === "test") { const client = new Client({ name: "adb-push-boundary", version: "0.0.0" }); onTestFinished(() => client.close()); await client.connect(clientTransport); - const response = await client.callTool({ + const response = await artifactMcpCall(client, { name: "push_adb_file", arguments: { serial: "emulator-5554", @@ -104,7 +105,7 @@ const createFakeAdb = async () => { const stderrFile = join(directory, "stderr"); const callsFile = join(directory, "calls"); const exitCodeFile = join(directory, "exit-code"); - const fakeAdbSource = `#!/usr/bin/env node + const fakeAdbSource = `#!${process.execPath} const fs = require("node:fs"); const [stdoutPath, stderrPath, callsPath, exitCodePath] = ${JSON.stringify([stdoutFile, stderrFile, callsFile, exitCodeFile])}; const args = process.argv.slice(2); @@ -158,7 +159,7 @@ const assertAdvertisedActivityExtras = async ( ).toBe(true); // Canonical validation rejects malformed extras before starting ADB. const callsBeforeInvalidExtra = await calls(); - const invalidExtra = await client.callTool({ + const invalidExtra = await artifactMcpCall(client, { name: "start_adb_activity", arguments: { ...input, @@ -220,7 +221,7 @@ it { key: "scale", type: "float" as const, value: "1e-3" }, ], }; - const success = await client.callTool({ + const success = await artifactMcpCall(client, { name: operation, arguments: input, }); @@ -241,7 +242,7 @@ it stdout = `${starting}Status: timeout\nActivity: com.example/.Main\nWaitTime: 10004\nComplete\n`; await fakeAdb.setReport(stdout, stderr); - const timeout = await client.callTool({ + const timeout = await artifactMcpCall(client, { name: operation, arguments: input, }); @@ -272,7 +273,7 @@ it stdout = `${starting}Status: ok\nActivity: com.example/.Main\nComplete\n`; stderr = "Exception: Activity class does not exist\n"; await fakeAdb.setReport(stdout, stderr); - const hardError = await client.callTool({ + const hardError = await artifactMcpCall(client, { name: operation, arguments: input, }); @@ -362,7 +363,7 @@ it.skipIf(process.platform === "win32")( validateFormats: false, }).compile(advertised.inputSchema); expect(validateInput(input)).toBe(true); - const refused = await client.callTool({ + const refused = await artifactMcpCall(client, { name: "start_adb_activity", arguments: input, }); @@ -393,7 +394,7 @@ it.skipIf(process.platform === "win32")( "Starting: Intent { act=android.intent.action.MAIN cmp=com.example/.Outer$InnerActivity }\n"; const transportStderr = "error: device offline\n"; await fakeAdb.setReport(transportStdout, transportStderr, 1); - const transport = await client.callTool({ + const transport = await artifactMcpCall(client, { name: "start_adb_activity", arguments: input, }); @@ -411,7 +412,7 @@ it.skipIf(process.platform === "win32")( "Starting: Intent { act=android.intent.action.MAIN cmp=com.example/.Outer$InnerActivity }\n"; const incompleteStderr = "transport disconnected by peer\n"; await fakeAdb.setReport(incompleteStdout, incompleteStderr, 1); - const incomplete = await client.callTool({ + const incomplete = await artifactMcpCall(client, { name: "start_adb_activity", arguments: input, }); @@ -435,7 +436,7 @@ it.skipIf(process.platform === "win32")( const stderrOnly = "Error: Activity class {com.example/com.example.Outer$InnerActivity} does not exist.\n"; await fakeAdb.setReport("", stderrOnly, 1); - const stderrOnlyFailure = await client.callTool({ + const stderrOnlyFailure = await artifactMcpCall(client, { name: "start_adb_activity", arguments: input, }); @@ -461,7 +462,7 @@ it.skipIf(process.platform === "win32")( "Error: " + "x".repeat(70 * 1024); await fakeAdb.setReport(oversizedReport, "", 1); - const truncated = await client.callTool({ + const truncated = await artifactMcpCall(client, { name: "start_adb_activity", arguments: input, }); From d27e3f8e89c2a5b37bacb784e2275256441e6292 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Mon, 26 Oct 2026 23:50:00 +0800 Subject: [PATCH 12/25] fix(native): preserve import names across dyld text formats Parse literal import framing and retain ambiguous boundaries. Consult undefined symbol-table names only when needed to distinguish producer spacing, with import and export captures owned separately. Validation: 59 native tests passed and real Apple tools on inert arm64 fixtures reproduced the old loss and verified the fix. Fresh build, typecheck and lint passed. Small warm-task median elapsed was unchanged; cold costs and agent tokens were not measured. --- src/native/NativeMachoInspection.ts | 66 +++++--- src/native/parsers/dyldInfo.ts | 107 ++++++++++--- .../native/nativeInventory.golden.test.ts | 151 +++++++++++++++++- 3 files changed, 269 insertions(+), 55 deletions(-) diff --git a/src/native/NativeMachoInspection.ts b/src/native/NativeMachoInspection.ts index cbb7650a93..d9388393f2 100644 --- a/src/native/NativeMachoInspection.ts +++ b/src/native/NativeMachoInspection.ts @@ -59,6 +59,7 @@ const DISCOVERY_COMMANDS = [["lipo", ["-detailed_info"]]] as const; // from dyld metadata. It takes seconds on large apps (Keynote arm64: 5.8 s // versus 0.01 s without it), and under -gU it can only add export-trie names. const SYMBOL_TABLE_EXPORTS = ["nm", ["-gjU", "--no-dyldinfo"]] as const; +const SYMBOL_TABLE_IMPORTS = ["nm", ["-gju", "--no-dyldinfo"]] as const; const DYLD_INFO_EXPORTS = ["nm", ["-gjU"]] as const; const REQUIRED_SLICE_COMMANDS = [ @@ -232,6 +233,32 @@ export const inspectNativeMacho = async ( if (!withDyldInfo.ok) return err(withDyldInfo.error); captures[nmIndex] = withDyldInfo.value; } + const dyldImports = captures.find( + ({ tool, arguments: arguments_ }) => + tool === "dyld_info" && arguments_[0] === "-imports", + ); + if (dyldImports === undefined) + throw new TypeError("Missing dyld_info import capture"); + const hasChainedFixups = load.commands.some( + ({ kind }) => kind === "LC_DYLD_CHAINED_FIXUPS", + ); + let parsedImports = parseDyldImports( + sliceOutput(dyldImports), + hasChainedFixups, + ); + if ( + !isMissingUuidDyldFailure(dyldImports) && + parsedImports.needsNameInventory + ) { + const importNames = await runSliceCommand(SYMBOL_TABLE_IMPORTS); + if (!importNames.ok) return err(importNames.error); + captures.push(importNames.value); + parsedImports = parseDyldImports( + sliceOutput(dyldImports), + hasChainedFixups, + new Set(parseNmSymbolNames(importNames.value.stdout)), + ); + } const selectedArchitectureName = selectedArchitecture ?? (architectures.length === 1 ? (architectures[0]?.name ?? null) : null); @@ -248,6 +275,8 @@ export const inspectNativeMacho = async ( context.invocation, limitations, load, + parsedImports.items, + captures[nmIndex]!, architectures, context.target.path, { @@ -280,6 +309,8 @@ const normalizeMacho = ( toInvocation: (capture: NativeCommandCapture) => NativeCommandInvocation, additionalLimitations: readonly string[], load: ReturnType, + parsedImports: ReturnType["items"], + exportNames: NativeCommandCapture, architectures: readonly LipoArchitecture[], artifactPath: string, selectedArchitecture: { @@ -292,19 +323,12 @@ const normalizeMacho = ( const dyldExportsCapture = byTool("dyld_info", 1); const missingImportsUuid = isMissingUuidDyldFailure(dyldImportsCapture); const missingExportsUuid = isMissingUuidDyldFailure(dyldExportsCapture); - const hasChainedFixups = load.commands.some( - ({ kind }) => kind === "LC_DYLD_CHAINED_FIXUPS", - ); - const imports = missingImportsUuid - ? [] - : parseDyldImports(sliceOutput(dyldImportsCapture), hasChainedFixups).map( - withRuntimeName, - ); + const imports = missingImportsUuid ? [] : parsedImports.map(withRuntimeName); const imageBase = load.segments.find( (segment) => segment.file_offset === 0 && (segment.file_size ?? 0) > 0, )?.vm_address ?? null; - const nmExports = parseNmExports(byTool("nm").stdout); + const nmExports = parseNmExports(exportNames.stdout); const dyldExports = missingExportsUuid ? [] : parseDyldExports( @@ -386,7 +410,7 @@ const normalizeMacho = ( ...(missingImportsUuid ? [missingUuidLimitation("imports")] : []), ...(imports.some(({ name_status }) => name_status === "ambiguous") ? [ - "Chained import text contains symbol/addend expressions whose split is ambiguous.", + "Import text contains name, flags, library, or addend boundaries whose split is ambiguous.", ] : []), ]), @@ -434,7 +458,7 @@ const namesEveryExport = ( nmOutput: string, dyldExports: NativeCommandCapture, ): boolean => { - const names = new Set(parseNmExports(nmOutput).map(({ name }) => name)); + const names = new Set(parseNmSymbolNames(nmOutput)); return sliceOutput(dyldExports) .split("\n") .every((rawLine, index) => { @@ -456,17 +480,17 @@ const namesEveryExport = ( }; // nm -j ends each name with LF; a carriage return before it is part of the name. +const parseNmSymbolNames = (output: string): string[] => + output.split("\n").filter((name) => name.length > 0); + const parseNmExports = (output: string) => - output - .split("\n") - .filter((name) => name.length > 0) - .map((name) => ({ - name, - address: null, - weak: null, - reexport: null, - source: "nm", - })); + parseNmSymbolNames(output).map((name) => ({ + name, + address: null, + weak: null, + reexport: null, + source: "nm", + })); const dependencyPayloadLimitations = ( commands: readonly string[], diff --git a/src/native/parsers/dyldInfo.ts b/src/native/parsers/dyldInfo.ts index 777725f485..8ba8909939 100644 --- a/src/native/parsers/dyldInfo.ts +++ b/src/native/parsers/dyldInfo.ts @@ -7,7 +7,7 @@ interface ParsedDyldSymbolFields { interface ParsedDyldImport extends ParsedDyldSymbolFields { readonly name: string | null; - /** Full producer text when the symbol/addend split cannot be established. */ + /** Producer text when name, flags, library, or addend boundaries are unknown. */ readonly raw_name: string | null; readonly name_status: "exact" | "ambiguous"; readonly addend: string | null; @@ -19,34 +19,89 @@ interface ParsedDyldExport extends ParsedDyldSymbolFields { readonly sourceSymbol?: string; } -/** Decode Apple dyld import rows, retaining chained suffix ambiguity. */ +/** Decode Apple's literal import framing; request nm facts for competing names. */ export const parseDyldImports = ( output: string, hasChainedFixups: boolean, -): ParsedDyldImport[] => - output.split("\n").flatMap((rawLine) => { - const line = rawLine.trim(); - const imported = - /^(?:0x[\da-f]+\s+)?(.+?)(?:\s+\[([^\]]+)\])?\s+\(from (.+)\)$/iu.exec( - line, - ); - if (imported?.[1] === undefined) return []; - const ambiguousSuffix = hasChainedFixups - ? /^(.*)\+0x[\da-f]+$/iu.test(imported[1]) - : false; - return [ - { - name: ambiguousSuffix ? null : imported[1], - raw_name: ambiguousSuffix ? imported[1] : null, - name_status: ambiguousSuffix ? "ambiguous" : "exact", - addend: null, - address: null, - weak: /\bweak-import\b/u.test(imported[2] ?? "") ? true : null, - reexport: null, - source: imported[3] ?? null, - }, - ]; - }); + knownImportNames?: ReadonlySet, +): { + readonly items: ParsedDyldImport[]; + readonly needsNameInventory: boolean; +} => { + let needsNameInventory = false; + const items = output + .split("\n") + .flatMap((rawLine, index) => { + const framed = rawLine.startsWith(" "); + const line = framed ? rawLine.slice(6) : rawLine; + if ( + line.length === 0 || + line.trim() === "-imports:" || + (index === 0 && line.endsWith(":")) + ) + return []; + // Apple prints "%s %s (from %s)": even an empty weak flag has its + // own separating spaces. Whitespace inside the name remains payload. + // Chained rows have one space after the ordinal in current Apple source, + // but two in installed-tool captures. nm names establish that boundary. + const ordinal = hasChainedFixups ? /^0x[\da-f]+ /iu.exec(line) : null; + const row = ordinal === null ? line : line.slice(ordinal[0].length); + const starts = + ordinal !== null && row.startsWith(" ") ? [row, row.slice(1)] : [row]; + const readings: { name: string; weak: boolean | null; source: string }[] = + []; + for (const start of starts) + for (const separator of start.matchAll( + / (?:\[weak-import\])? \(from /gu, + )) { + const source = start.slice(separator.index + separator[0].length, -1); + if (!start.endsWith(")") || source.length === 0) continue; + const name = start.slice(0, separator.index); + if (name.length === 0) continue; + readings.push({ + name, + weak: separator[0].includes("[weak-import]") ? true : null, + source, + }); + } + if (readings.length > 1 && knownImportNames === undefined) + needsNameInventory = true; + const matching = + readings.length > 1 && knownImportNames !== undefined + ? readings.filter(({ name }) => knownImportNames.has(name)) + : readings; + const reading = matching.length === 1 ? matching[0] : undefined; + const ambiguousSuffix = + reading !== undefined && + hasChainedFixups && + /\+0x[\da-f]+$/iu.test(reading.name); + const unknownName = + !framed || + (hasChainedFixups && ordinal === null) || + reading === undefined; + // Prefix uncertainty need not erase a separately established flag/library. + const common = readings.every( + (candidate) => + candidate.weak === readings[0]?.weak && + candidate.source === readings[0]?.source, + ) + ? readings[0] + : undefined; + return [ + { + name: unknownName || ambiguousSuffix ? null : reading.name, + raw_name: unknownName ? line : ambiguousSuffix ? reading.name : null, + name_status: unknownName || ambiguousSuffix ? "ambiguous" : "exact", + addend: null, + address: null, + weak: (reading ?? common)?.weak ?? null, + reexport: null, + source: (reading ?? common)?.source ?? null, + }, + ]; + }); + return { items, needsNameInventory }; +}; /** Decode Apple dyld export rows, projecting export offsets to VM addresses. */ export const parseDyldExports = ( diff --git a/tests/boundary/providers/native/nativeInventory.golden.test.ts b/tests/boundary/providers/native/nativeInventory.golden.test.ts index 89ddd81b8e..86eccf6858 100644 --- a/tests/boundary/providers/native/nativeInventory.golden.test.ts +++ b/tests/boundary/providers/native/nativeInventory.golden.test.ts @@ -262,6 +262,7 @@ it("retains native inventory facts from captured Apple tool output", async () => for (const tool of ["lipo", "otool", "nm", "dwarfdump", "vtool"]) outputs[tool] = await fixture(`dyld-inventory/${tool}.txt`); outputs.otool = withOtoolMachHeader(outputs.otool ?? ""); + outputs["nm:-gju"] = "_puts\n_fixture_weak\n"; outputs["dyld_info:-imports"] = await fixture("dyld-inventory/imports.txt"); const exportsOutput = await fixture("dyld-inventory/exports.txt"); outputs["dyld_info:-exports"] = exportsOutput; @@ -278,12 +279,14 @@ it("retains native inventory facts from captured Apple tool output", async () => { name: "_puts", runtime_name: "puts", + name_status: "exact", address: null, source: "libSystem", }, { name: "_fixture_weak", runtime_name: "fixture_weak", + name_status: "exact", address: null, source: "", }, @@ -308,7 +311,9 @@ it("retains native inventory facts from captured Apple tool output", async () => ]), }, }); - expect(parseDyldImports(await fixture("dyld-imports.txt"), false)).toEqual([ + expect( + parseDyldImports(await fixture("dyld-imports.txt"), false).items, + ).toEqual([ { name: "__Block_copy", raw_name: null, @@ -354,6 +359,135 @@ it("retains native inventory facts from captured Apple tool output", async () => }); }); +it("preserves literal import suffixes and whitespace through native inspection", async () => { + // Actual Apple dyld_info output for .quad references in an inert arm64 + // executable, linked with -no_fixup_chains and -undefined dynamic_lookup. + const imports = [ + " -imports:", + " _leading (from )", + " _literal [weak-import] (from )", + " _trailing (from )", + " _weak [weak-import] (from )", + "", + ].join("\n"); + const client = new NativeMacOSProvider( + {}, + new FixtureRunner({ "dyld_info:-imports": imports }), + "darwin", + ).createClient(machoTarget("/private/fixture")); + const execution = await client.execute("inspect_macho", {}); + if (!execution.ok) throw execution.error; + expect(execution.value.result).toMatchObject({ + imports: { + exhaustive: false, + items: [ + { + name: " _leading", + runtime_name: " _leading", + name_status: "exact", + }, + { + name: "_literal [weak-import]", + runtime_name: "literal [weak-import]", + weak: null, + name_status: "exact", + }, + { + name: "_trailing ", + runtime_name: "trailing ", + weak: null, + name_status: "exact", + }, + { + name: "_weak", + runtime_name: "weak", + weak: true, + name_status: "exact", + }, + ], + }, + }); +}); + +it("retains unfamiliar and competing import boundaries as unknown", () => { + for (const row of [ + "_future [new-flag] (from library)", + "_literal (from name) (from library)", + ]) + expect(parseDyldImports(` ${row}`, false).items).toEqual([ + { + name: null, + raw_name: row, + name_status: "ambiguous", + addend: null, + address: null, + weak: null, + reexport: null, + source: null, + }, + ]); +}); + +it("uses independent undefined names to resolve chained producer spacing", async () => { + const outputs: Record = {}; + for (const tool of ["lipo", "otool", "nm", "dwarfdump", "vtool"]) + outputs[tool] = await fixture(`dyld-inventory/${tool}.txt`); + outputs.otool = withOtoolMachHeader(outputs.otool ?? ""); + outputs["nm:-gju"] = + " _leading\n_literal [weak-import]\n_trailing \n_weak\n"; + // Actual forced-chained output from the same inert assembler references. + outputs["dyld_info:-imports"] = [ + " -imports:", + " 0x0000 _weak [weak-import] (from )", + " 0x0001 _trailing (from )", + " 0x0002 _literal [weak-import] (from )", + " 0x0003 _leading (from )", + ].join("\n"); + const client = new NativeMacOSProvider( + {}, + new FixtureRunner(outputs), + "darwin", + ).createClient(machoTarget("/private/fixture")); + const execution = await client.execute("inspect_macho", {}); + if (!execution.ok) throw execution.error; + expect(execution.value.result).toMatchObject({ + imports: { + items: [ + { name: "_weak", weak: true, name_status: "exact" }, + { name: "_trailing ", weak: null, name_status: "exact" }, + { name: "_literal [weak-import]", weak: null, name_status: "exact" }, + { name: " _leading", weak: null, name_status: "exact" }, + ], + }, + exports: { + items: expect.arrayContaining([ + expect.objectContaining({ name: "_main", runtime_name: "main" }), + ]), + }, + }); +}); + +it("keeps missing and competing undefined-name matches unknown", () => { + const output = " 0x0000 _literal (from library)"; + expect(parseDyldImports(output, true).needsNameInventory).toBe(true); + for (const names of [new Set(), new Set(["_literal", " _literal"])]) + expect(parseDyldImports(output, true, names)).toEqual({ + needsNameInventory: false, + items: [ + { + name: null, + raw_name: "0x0000 _literal (from library)", + name_status: "ambiguous", + addend: null, + address: null, + weak: null, + reexport: null, + source: "library", + }, + ], + }); +}); + it("reports a chained bind addend separately in native inspection", async () => { const outputs: Record = {}; for (const tool of ["lipo", "otool", "nm", "dwarfdump", "vtool"]) @@ -367,7 +501,7 @@ it("reports a chained bind addend separately in native inspection", async () => `, ); outputs["dyld_info:-imports"] = - "-imports:\n 0x0000 _fixture_data (from libFixture)\n 0x0001 _fixture_data+0x4 (from libFixture)\n"; + "-imports:\n 0x0000 _fixture_data (from libFixture)\n 0x0001 _fixture_data+0x4 (from libFixture)\n"; outputs["dyld_info:-exports"] = await fixture("dyld-inventory/exports.txt"); const client = new NativeMacOSProvider( {}, @@ -380,7 +514,7 @@ it("reports a chained bind addend separately in native inspection", async () => imports: { exhaustive: false, limitations: expect.arrayContaining([ - "Chained import text contains symbol/addend expressions whose split is ambiguous.", + "Import text contains name, flags, library, or addend boundaries whose split is ambiguous.", ]), items: [ { @@ -484,7 +618,7 @@ it("splits re-export rows at the source library", () => { it("retains symbol names containing spaces in dyld inventory rows", () => { expect( - parseDyldImports(" 0x0000 _entry with space (from fixture)", false), + parseDyldImports(" _entry with space (from fixture)", false).items, ).toEqual([ { name: "_entry with space", @@ -710,9 +844,9 @@ it("keeps true weak annotations when nm establishes the unannotated name", () => it("retains chained import suffixes as ambiguous raw text", () => { expect( parseDyldImports( - "0x0000 _fixture_data (from libFixture)\n0x0001 _fixture_data+0x4 (from libFixture)\n0x0002 _fixture_weak+0x8 [weak-import] (from libFixture)", + " 0x0000 _fixture_data (from libFixture)\n 0x0001 _fixture_data+0x4 (from libFixture)\n 0x0002 _fixture_weak+0x8 [weak-import] (from libFixture)", true, - ), + ).items, ).toEqual([ { name: "_fixture_data", @@ -746,7 +880,8 @@ it("retains chained import suffixes as ambiguous raw text", () => { }, ]); expect( - parseDyldImports("0x0001 _literal+0x4 (from libFixture)", true), + parseDyldImports(" 0x0001 _literal+0x4 (from libFixture)", true) + .items, ).toEqual([ { name: null, @@ -760,7 +895,7 @@ it("retains chained import suffixes as ambiguous raw text", () => { }, ]); expect( - parseDyldImports("0x0001 _literal+0x4 (from libFixture)", false), + parseDyldImports(" _literal+0x4 (from libFixture)", false).items, ).toMatchObject([{ name: "_literal+0x4", name_status: "exact" }]); }); From 13fba271e6e14bf6235379564eb9d32aee7049d5 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Mon, 26 Oct 2026 23:50:16 +0800 Subject: [PATCH 13/25] refactor(plist): centralize binary layout and offset ownership --- src/artifacts/apple/BinaryPlistLayout.ts | 66 +++++++++++++++ .../apple/BinaryPlistMaterializer.test.ts | 67 +++++++++++++++ .../apple/BinaryPlistMaterializer.ts | 15 ++-- src/artifacts/apple/PlistScalars.test.ts | 27 ++++++- src/artifacts/apple/PlistScalars.ts | 81 ++++++++----------- .../apple/PropertyListDecodeBudget.ts | 39 ++++----- 6 files changed, 223 insertions(+), 72 deletions(-) create mode 100644 src/artifacts/apple/BinaryPlistLayout.ts diff --git a/src/artifacts/apple/BinaryPlistLayout.ts b/src/artifacts/apple/BinaryPlistLayout.ts new file mode 100644 index 0000000000..1c9bf330a9 --- /dev/null +++ b/src/artifacts/apple/BinaryPlistLayout.ts @@ -0,0 +1,66 @@ +import { err, ok, type Result } from "../../domain/result.js"; + +/** + * Preserve REA's source integer interpretation, including uint64 + * wrapping for unusual widths. Callers own byte-range and safe-number admission. + */ +export const readBinaryPlistUnsigned64 = ( + bytes: Uint8Array, + offset: number, + width: number, +): bigint => { + let value = 0n; + for (let index = 0; index < width; index += 1) + value = BigInt.asUintN( + 64, + (value << 8n) | BigInt(bytes[offset + index] ?? 0), + ); + return value; +}; + +/** Raw declarations tied to one source or decoder-copy byte representation. */ +export interface BinaryPlistLayout { + readonly trailerOffset: number; + readonly offsetWidth: number; + readonly referenceWidth: number; + readonly objectCount: bigint; + readonly topObject: bigint; + readonly offsetTable: bigint; + readonly objectOffset: (object: number) => bigint; + readonly objectOffsets: () => Generator; +} + +/** Observe declarations only; signature, bounds, capacities and reachability remain caller policy. */ +export const readBinaryPlistLayout = ( + bytes: Uint8Array, +): Result< + BinaryPlistLayout, + { readonly _tag: "InvalidBinaryPlistLayout"; readonly reason: "truncated" } +> => { + if (bytes.byteLength < 32) + return err({ _tag: "InvalidBinaryPlistLayout", reason: "truncated" }); + const trailerOffset = bytes.byteLength - 32; + const offsetWidth = bytes[trailerOffset + 6]!; + const referenceWidth = bytes[trailerOffset + 7]!; + const objectCount = readBinaryPlistUnsigned64(bytes, trailerOffset + 8, 8); + const offsetTable = readBinaryPlistUnsigned64(bytes, trailerOffset + 24, 8); + const objectOffset = (object: number): bigint => + readBinaryPlistUnsigned64( + bytes, + Number(offsetTable) + object * offsetWidth, + offsetWidth, + ); + return ok({ + trailerOffset, + offsetWidth, + referenceWidth, + objectCount, + topObject: readBinaryPlistUnsigned64(bytes, trailerOffset + 16, 8), + offsetTable, + objectOffset, + *objectOffsets() { + for (let object = 0; object < Number(objectCount); object += 1) + yield objectOffset(object); + }, + }); +}; diff --git a/src/artifacts/apple/BinaryPlistMaterializer.test.ts b/src/artifacts/apple/BinaryPlistMaterializer.test.ts index 07266303cd..05c8277647 100644 --- a/src/artifacts/apple/BinaryPlistMaterializer.test.ts +++ b/src/artifacts/apple/BinaryPlistMaterializer.test.ts @@ -1,5 +1,8 @@ import { expect, it } from "vitest"; import { binaryPlistObjectTable } from "../../../tests/fixtures/binaryPlistObjectTable.js"; +import { createPlistNumberProjection } from "../../domain/apple/plistNumbers.js"; +import { projectPlistValue } from "../../domain/apple/plistValue.js"; +import { scanBinaryPlistScalars } from "./PlistScalars.js"; import { plistUidValue } from "../../domain/apple/plistUid.js"; import { decodeBinaryPlistValue } from "./BinaryPlistMaterializer.js"; import { estimateBinaryPlistExpansion } from "./PropertyListDecodeBudget.js"; @@ -80,3 +83,67 @@ it("keeps a source UID distinct from an ordinary UID-shaped dictionary", () => { expect(plistUidValue(value[0])).toBe(7); expect(plistUidValue(value[1])).toBeUndefined(); }); + +it("materializes normalized three-byte tables from a nonzero root without losing source numbers", () => { + const tableOffset = 36; + const bytes = Buffer.alloc(tableOffset + 9 + 32); + bytes.write("bplist00"); + bytes[8] = 0x14; + bytes.writeBigUInt64BE(1n, 9); + bytes.writeBigUInt64BE(2n, 17); + bytes[25] = 0xa2; + bytes.writeUIntBE(0, 26, 3); + bytes.writeUIntBE(2, 29, 3); + bytes[32] = 0x82; + bytes.writeUIntBE(0xabcdef, 33, 3); + for (const [object, offset] of [8, 25, 32].entries()) + bytes.writeUIntBE(offset, tableOffset + object * 3, 3); + const trailer = bytes.length - 32; + bytes[trailer + 6] = 3; + bytes[trailer + 7] = 3; + bytes.writeBigUInt64BE(3n, trailer + 8); + bytes.writeBigUInt64BE(1n, trailer + 16); + bytes.writeBigUInt64BE(BigInt(tableOffset), trailer + 24); + const original = Buffer.from(bytes); + const plan = estimateBinaryPlistExpansion(bytes, 1024 * 1024, { + numberProjection: true, + uidProjection: true, + }); + const metadata = scanBinaryPlistScalars(bytes, plan); + expect(metadata.wideIntegers).toEqual([ + { offset: 8, value: 18446744073709551618n }, + ]); + expect(metadata.literals.incomplete).toBe(false); + const decoded = decodeBinaryPlistValue(bytes, plan, metadata) as unknown[]; + expect( + createPlistNumberProjection(metadata.literals).project( + projectPlistValue(decoded[0]).value, + ), + ).toEqual({ + $plist_type: "integer", + decimal: "18446744073709551618", + }); + expect(plistUidValue(decoded[1])).toBe(0xabcdef); + expect(decoded).toHaveLength(2); + expect(bytes.equals(original)).toBe(true); +}); + +it("keeps uint64 wrapping for nine-byte offset and reference declarations", () => { + const bytes = Buffer.alloc(70); + bytes.write("bplist00"); + bytes.set([0x10, 7, 0xa1], 8); + bytes[11] = 0xff; + bytes[20] = 0xff; + bytes.writeBigUInt64BE(8n, 21); + bytes[29] = 0xff; + bytes.writeBigUInt64BE(10n, 30); + const trailer = 38; + bytes[trailer + 6] = 9; + bytes[trailer + 7] = 9; + bytes.writeBigUInt64BE(2n, trailer + 8); + bytes.writeBigUInt64BE(1n, trailer + 16); + bytes.writeBigUInt64BE(20n, trailer + 24); + const original = Buffer.from(bytes); + expect(decode(bytes)).toEqual([7]); + expect(bytes.equals(original)).toBe(true); +}); diff --git a/src/artifacts/apple/BinaryPlistMaterializer.ts b/src/artifacts/apple/BinaryPlistMaterializer.ts index 2d0086a315..d91f86e812 100644 --- a/src/artifacts/apple/BinaryPlistMaterializer.ts +++ b/src/artifacts/apple/BinaryPlistMaterializer.ts @@ -1,7 +1,7 @@ +import { readBinaryPlistLayout } from "./BinaryPlistLayout.js"; import { parseBinary } from "plist"; import { markPlistDictionary } from "../../domain/apple/plistUid.js"; import { - binaryPlistObjectOffsets, decodeBinaryPlistScalars, scanBinaryPlistScalars, type BinaryPlistScalarMetadata, @@ -13,9 +13,14 @@ const scalarArrayInput = ( bytes: Buffer, objects: readonly number[], ): Buffer => { - const trailer = bytes.length - 32; - const referenceWidth = bytes[trailer + 7]!; - const objectCount = Number(bytes.readBigUInt64BE(trailer + 8)); + // Normalization may relocate objects and change both widths. These facts + // belong to the compatible decoder copy, not the original source plan. + const compatibleLayoutResult = readBinaryPlistLayout(bytes); + if (!compatibleLayoutResult.ok) + throw new RangeError("binary plist trailer is truncated"); + const compatibleLayout = compatibleLayoutResult.value; + const referenceWidth = compatibleLayout.referenceWidth; + const objectCount = Number(compatibleLayout.objectCount); const rootOffset = bytes.length; const rootLength = 10 + objects.length * referenceWidth; const tableOffset = rootOffset + rootLength; @@ -33,7 +38,7 @@ const scalarArrayInput = ( } } let object = 0; - for (const offset of binaryPlistObjectOffsets(bytes)) { + for (const offset of compatibleLayout.objectOffsets()) { output.writeBigUInt64BE(offset, tableOffset + object * 8); object += 1; } diff --git a/src/artifacts/apple/PlistScalars.test.ts b/src/artifacts/apple/PlistScalars.test.ts index afee17e250..4a12750862 100644 --- a/src/artifacts/apple/PlistScalars.test.ts +++ b/src/artifacts/apple/PlistScalars.test.ts @@ -488,7 +488,15 @@ describe("binary plist shared object-table entries", () => { const table = Number(bytes.readBigUInt64BE(bytes.length - 8)); bytes[table + 2] = 7; expect(bytes.subarray(0, 8).toString("ascii")).toBe("bplist00"); - expect(scanBinaryPlistScalars(bytes).literals.incomplete).toBe(true); + const metadata = scanBinaryPlistScalars(bytes); + expect(metadata.literals.incomplete).toBe(true); + expect(metadata.literals.integers).toEqual( + new Map([ + [9007199254740992, new Set(["9007199254740993"])], + [Number(1152921504606846979n), new Set(["1152921504606846979"])], + ]), + ); + expect(metadata.literals.unsafeIntegerLiterals).toBe(2); }); }); @@ -586,3 +594,20 @@ describe("binary plist scalar clone budgeting", () => { expect(bytes.readDoubleBE(14)).toBe(0.1005); }); }); + +it.each([4n, 0xffffffffffffffffn])( + "scans source scalars independently of invalid top object %s", + (topObject) => { + const bytes = sharedScalarEntries(int64(9007199254740993n)); + const expected = scanBinaryPlistScalars(bytes); + bytes.writeBigUInt64BE(topObject, bytes.length - 16); + expect(scanBinaryPlistScalars(bytes)).toEqual(expected); + expect(expected.literals.incomplete).toBe(false); + expect(expected.literals.unsafeIntegerLiterals).toBe(2); + expect(() => + estimateBinaryPlistExpansion(bytes, 1024 * 1024, { + numberProjection: true, + }), + ).toThrow("binary plist trailer or object table is invalid"); + }, +); diff --git a/src/artifacts/apple/PlistScalars.ts b/src/artifacts/apple/PlistScalars.ts index 075b14bbf5..0ca721dae4 100644 --- a/src/artifacts/apple/PlistScalars.ts +++ b/src/artifacts/apple/PlistScalars.ts @@ -1,3 +1,8 @@ +import { + readBinaryPlistLayout, + readBinaryPlistUnsigned64, + type BinaryPlistLayout, +} from "./BinaryPlistLayout.js"; import { createPlistNumberCollector, type PlistNumberLiterals, @@ -196,10 +201,7 @@ export const binaryPlistObjectEnd = ( const width = 1 << (extendedMarker & 0x0f); if (width > 8 || cursor + 1 + width > objectEnd) return undefined; cursor += 1; - let extendedLength = 0n; - for (let index = 0; index < width; index += 1) - extendedLength = - (extendedLength << 8n) | BigInt(bytes[cursor + index] ?? 0); + const extendedLength = readBinaryPlistUnsigned64(bytes, cursor, width); if (extendedLength > BigInt(Number.MAX_SAFE_INTEGER)) return undefined; length = Number(extendedLength); cursor += width; @@ -247,11 +249,17 @@ export const scanBinaryPlistScalars = ( }); if (bytes.length < 40 || !hasBinaryPlistSignature(bytes)) return result(true); const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); - const trailer = bytes.length - 32; - const offsetWidth = bytes[trailer + 6] ?? 0; + const sourceLayoutResult = readBinaryPlistLayout(bytes); + if (!sourceLayoutResult.ok) return result(true); + const sourceLayout = sourceLayoutResult.value; + const { + trailerOffset: trailer, + offsetWidth, + referenceWidth, + objectCount: countRaw, + offsetTable: tableRaw, + } = sourceLayout; if (offsetWidth < 1) return result(true); - const countRaw = view.getBigUint64(trailer + 8); - const tableRaw = view.getBigUint64(trailer + 24); const safeMaximum = BigInt(Number.MAX_SAFE_INTEGER); if (countRaw === 0n || countRaw > safeMaximum || tableRaw > safeMaximum) return result(true); @@ -260,23 +268,12 @@ export const scanBinaryPlistScalars = ( const capacity = Math.floor((trailer - offsetTable) / offsetWidth); if (countRaw > BigInt(capacity)) return result(true); const count = Number(countRaw); - const referenceWidth = bytes[trailer + 7] ?? 0; if (referenceWidth < 1) return result(true); - const unsigned = (offset: number): number | undefined => { - if (offset < offsetTable || offset + offsetWidth > trailer) - return undefined; - let value = 0n; - for (let index = 0; index < offsetWidth; index += 1) - value = BigInt.asUintN( - 64, - (value << 8n) | BigInt(bytes[offset + index] ?? 0), - ); - return value > safeMaximum ? undefined : Number(value); - }; const objectOffsets: number[] = []; let incomplete = false; for (let index = 0; index < count; index += 1) { - const offset = unsigned(offsetTable + index * offsetWidth); + const rawOffset = sourceLayout.objectOffset(index); + const offset = rawOffset > safeMaximum ? undefined : Number(rawOffset); if (offset === undefined || offset < 8 || offset >= offsetTable) { incomplete = true; continue; @@ -405,8 +402,11 @@ export const decodeBinaryPlistScalars = ( replacement.writeDoubleBE(Number(value), 1); replacements.set(offset, replacement); } - const trailerOffset = bytes.length - 32; - const sourceReferenceSize = bytes[trailerOffset + 7] ?? 0; + const sourceLayoutResult = readBinaryPlistLayout(bytes); + if (!sourceLayoutResult.ok) + throw new RangeError("binary plist trailer is truncated"); + const sourceLayout = sourceLayoutResult.value; + const sourceReferenceSize = sourceLayout.referenceWidth; const normalizedReferenceSize = scalarPlan.normalizedReferenceSize ?? sourceReferenceSize; for (const container of scalarPlan.containersToNormalize ?? []) @@ -415,8 +415,8 @@ export const decodeBinaryPlistScalars = ( binaryPlistContainerRecord(bytes, container, normalizedReferenceSize), ); const needsTableNormalization = - (scalarPlan.normalizedOffsetSize ?? bytes[trailerOffset + 6] ?? 0) !== - (bytes[trailerOffset + 6] ?? 0) || + (scalarPlan.normalizedOffsetSize ?? sourceLayout.offsetWidth) !== + sourceLayout.offsetWidth || normalizedReferenceSize !== sourceReferenceSize; if ( reachableWideIntegers.length === 0 && @@ -430,8 +430,9 @@ export const decodeBinaryPlistScalars = ( ? Buffer.from(bytes) : cloneBinaryPlistObjects( bytes, + sourceLayout, replacements, - scalarPlan.normalizedOffsetSize ?? bytes[trailerOffset + 6] ?? 0, + scalarPlan.normalizedOffsetSize ?? sourceLayout.offsetWidth, normalizedReferenceSize, ); for (const { offset, value } of reachableWideIntegers) { @@ -450,34 +451,18 @@ export const decodeBinaryPlistScalars = ( return compatible; }; -/** Iterate source offset declarations using the same uint64 interpretation as normalization. */ -export function* binaryPlistObjectOffsets(bytes: Buffer): Generator { - const trailer = bytes.length - 32; - const width = bytes[trailer + 6] ?? 0; - const count = Number(bytes.readBigUInt64BE(trailer + 8)); - const table = Number(bytes.readBigUInt64BE(trailer + 24)); - for (let object = 0; object < count; object += 1) { - let offset = 0n; - for (let byte = 0; byte < width; byte += 1) - offset = BigInt.asUintN( - 64, - (offset << 8n) | BigInt(bytes[table + object * width + byte] ?? 0), - ); - yield offset; - } -} - const cloneBinaryPlistObjects = ( bytes: Buffer, + sourceLayout: BinaryPlistLayout, replacements: ReadonlyMap, normalizedOffsetSize?: number, normalizedReferenceSize?: number, ): Buffer => { - const trailerOffset = bytes.length - 32; - const offsetSize = bytes[trailerOffset + 6] ?? 0; - const objectCount = Number(bytes.readBigUInt64BE(trailerOffset + 8)); - const oldTableOffset = Number(bytes.readBigUInt64BE(trailerOffset + 24)); - const offsets = [...binaryPlistObjectOffsets(bytes)]; + const trailerOffset = sourceLayout.trailerOffset; + const offsetSize = sourceLayout.offsetWidth; + const objectCount = Number(sourceLayout.objectCount); + const oldTableOffset = Number(sourceLayout.offsetTable); + const offsets = [...sourceLayout.objectOffsets()]; const cloneEntries = [...replacements.entries()]; const replacementLengths = cloneEntries.map(([, record]) => record.length); const newOffsetSize = diff --git a/src/artifacts/apple/PropertyListDecodeBudget.ts b/src/artifacts/apple/PropertyListDecodeBudget.ts index 2ae8ce22a9..5c426a5b9c 100644 --- a/src/artifacts/apple/PropertyListDecodeBudget.ts +++ b/src/artifacts/apple/PropertyListDecodeBudget.ts @@ -1,3 +1,7 @@ +import { + readBinaryPlistLayout, + readBinaryPlistUnsigned64, +} from "./BinaryPlistLayout.js"; import { decodeXmlPlistText } from "../../domain/propertyListXmlText.js"; import { hasBinaryPlistSignature } from "../../domain/apple/appleArchiveFormat.js"; import { @@ -265,24 +269,26 @@ export const estimateBinaryPlistExpansion = ( ): PlistDecodePreflight => { if (bytes.length < 40) throw new TypeError("binary plist trailer is truncated"); - const trailer = bytes.length - 32; - const offsetSize = bytes[trailer + 6] ?? 0; - const referenceSize = bytes[trailer + 7] ?? 0; + const sourceLayoutResult = readBinaryPlistLayout(bytes); + if (!sourceLayoutResult.ok) + throw new TypeError("binary plist trailer is truncated"); + const sourceLayout = sourceLayoutResult.value; + const { + trailerOffset: trailer, + offsetWidth: offsetSize, + referenceWidth: referenceSize, + } = sourceLayout; if (offsetSize < 1 || referenceSize < 1) throw new TypeError("binary plist trailer integer sizes are invalid"); + const safeDeclaration = (value: bigint): number => + value <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(value) : -1; const readInteger = (offset: number, size: number): number => { if (offset < 0 || size < 1 || offset + size > bytes.length) return -1; - let value = 0n; - for (let index = 0; index < size; index += 1) - value = BigInt.asUintN( - 64, - (value << 8n) | BigInt(bytes[offset + index] ?? 0), - ); - return value <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(value) : -1; + return safeDeclaration(readBinaryPlistUnsigned64(bytes, offset, size)); }; - const objectCount = readInteger(trailer + 8, 8); - const topObject = readInteger(trailer + 16, 8); - const offsetTable = readInteger(trailer + 24, 8); + const objectCount = safeDeclaration(sourceLayout.objectCount); + const topObject = safeDeclaration(sourceLayout.topObject); + const offsetTable = safeDeclaration(sourceLayout.offsetTable); const tableEnd = bounds === "strict" ? trailer : bytes.length; const objectsEnd = bounds === "strict" ? offsetTable : bytes.length; if ( @@ -341,7 +347,7 @@ export const estimateBinaryPlistExpansion = ( const readDictionaryKey = (object: number): string => { const prior = dictionaryKeys.get(object); if (prior !== undefined) return prior; - const offset = readInteger(offsetTable + object * offsetSize, offsetSize); + const offset = safeDeclaration(sourceLayout.objectOffset(object)); if (offset < 8 || offset >= objectsEnd) throw new TypeError("binary plist object offset is invalid"); const marker = bytes[offset] ?? 0; @@ -403,10 +409,7 @@ export const estimateBinaryPlistExpansion = ( "expansion", `binary plist reference expansion exceeds ${budget}`, ); - const offset = readInteger( - offsetTable + entry.object * offsetSize, - offsetSize, - ); + const offset = safeDeclaration(sourceLayout.objectOffset(entry.object)); if (offset < 8 || offset >= objectsEnd) throw new TypeError("binary plist object offset is invalid"); if (numberProjection) reachableObjectOffsets.add(offset); From 5508b2affd8bdd0788aed9d2246c373fd1b01bb7 Mon Sep 17 00:00:00 2001 From: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com> Date: Mon, 26 Oct 2026 23:56:03 +0800 Subject: [PATCH 14/25] fix(analysis): retain source evidence through directory exports (#7250) Retain authenticated provider source records independently of snapshot replay eligibility and return them explicitly to plain application callers, including late cancellation. CLI/MCP commit source records before delivering links; function dossiers share the canonical Evidence constructor. Keep directory replies compact and preserve successful retry verification. Real Ghidra CLI/MCP investigations verify 652-function ls, historical report provenance, cross-session resume, unchanged source/project content and complete offline Evidence. Final full 14,338-function libcrypto comparison matches all metadata/pseudocode facts and independent fresh-import observations: 14,194 written, 143 external, one recorded deadline; 494.03 s, 445.63 CPU s, 1.584 GiB sampled family peak under 2-CPU/8-GiB guards, with no OOM or cleanup failure. 144 focused tests and static/docs/site checks pass; latest-main integration passes 289 cases plus typecheck/lint and actual CLI snapshot source retrieval. Final CI source/static/build/docs/quality/package checks pass. Four ADB shard failures also occur in main CI 40349559391; cancelled coverage remains unverified and author declarations stay unchecked under the user's explicit bypass authorization. No human-review attestation is made. --- .../native-decompile-directory-2026-10-23.md | 4 + .../decompile-directory.md | 149 ++++++++ .../decompile-directory.mjs | 328 ++++++++++++++++++ src/application/DirectAnalysis.ts | 15 +- .../FunctionAnalysisEvidence.test.ts | 15 +- src/application/FunctionAnalysisEvidence.ts | 58 ++-- src/application/WorkflowEvidence.test.ts | 42 +++ src/application/WorkflowEvidence.ts | 66 ++-- src/application/binary/BinarySession.ts | 14 +- .../investigation/EvidenceLedger.ts | 5 + .../recordEvidenceSources.test.ts | 32 ++ .../investigation/recordEvidenceSources.ts | 44 ++- src/server/registerEnhancedTools.ts | 15 +- .../mcp/analyzeFunctionDetail.test.ts | 83 ++++- .../analysis-sessions/binarySession.test.ts | 83 +++++ 15 files changed, 887 insertions(+), 66 deletions(-) create mode 100644 experiments/analysis-efficiency/decompile-directory.md create mode 100644 experiments/analysis-efficiency/decompile-directory.mjs create mode 100644 src/application/investigation/recordEvidenceSources.test.ts diff --git a/docs/verification/native-decompile-directory-2026-10-23.md b/docs/verification/native-decompile-directory-2026-10-23.md index 9f9ff5d7fe..75f555e6d4 100644 --- a/docs/verification/native-decompile-directory-2026-10-23.md +++ b/docs/verification/native-decompile-directory-2026-10-23.md @@ -1,5 +1,9 @@ # Native directory decompilation verification — 2026-10-23 +The [2026-10-26 follow-up](https://github.com/morluto/rea/blob/72454004772f826440c5e21002f4dd52b4ad43c0/experiments/analysis-efficiency/decompile-directory.md) +adds a 14,338-function real-provider capacity run, cross-session project reuse, +and a correction preserving historical source Evidence through shared workflows. + The completed analyst task is to export the bound provider's function inventory as one local pseudocode file per non-external function, with names, body ranges, provider/profile identity and the limitations of each used observation. The diff --git a/experiments/analysis-efficiency/decompile-directory.md b/experiments/analysis-efficiency/decompile-directory.md new file mode 100644 index 0000000000..300cc94b7a --- /dev/null +++ b/experiments/analysis-efficiency/decompile-directory.md @@ -0,0 +1,149 @@ +# Directory decompilation: capacity, resume and source Evidence + +The directory-export feature requested in #6090 already shipped in #6170. +This follow-up verifies the current public workflow and fixes a source-provenance +gap found while combining it with existing Ghidra projects (#1048). + +## Completed task and correctness + +The analyst asks for all functions as local pseudocode files, without putting +their bodies in agent context. The committed `decompile-directory.mjs` driver +uses the production stdio MCP entrypoint and installed SDK. It discovers the +catalog, exports an initial chunk, closes/reopens the provider, resumes all +remaining work, and resumes again to verify settled files. It authenticates an +explicitly exported Evidence bundle and checks the original target/project. + +Every written file is checked against its index SHA-256 and address/profile +header. Existing file hashes and modification times must survive both resumes. +Comparison facts include every address, name, status, body range and pseudocode +digest. An independent fresh-import investigation from #1048 additionally checks +the complete inventory and two nonempty pseudocode observations. This establishes +export preservation of provider observations; it does not establish discovery of +every actual function or equivalence to original source. + +| Real artifact | Provider inventory | Written | External | Failed | +| ----------------------- | -----------------: | ------: | -------: | -----: | +| `/usr/bin/ls` | 652 | 541 | 111 | 0 | +| system `libcrypto.so.3` | 14,338 | 14,194 | 143 | 1 | + +Target SHA-256 values are respectively +`cb30d69b24245bf2ecdc9e7f53bbad19159999970b6d82c0c00c7d32d9e37aa4` and +`aa01e614af634bc7c8b678c9250b5c4f613c61989bacf0a5d6aed6570e7dcabd`. +These installed ELF x86-64 binaries were analyzed statically, never executed. +The large run explicitly selected a five-second per-function deadline; function +`0x387f00` exceeded it, remained a recorded failure, and was retried on resume. +Later functions were still exported. No disappearance of that failure is claimed. +The resulting large tree occupies about 107 MiB on this filesystem. + +## Source-provenance correction + +Before this change, `BinarySession` retained provider `sourceEvidence` only inside +its snapshot-eligible branch. Ghidra procedure reads without a document selector +are not snapshot eligible. A composed directory export therefore lost saved +project reports even though the provider supplied them. The workflow also ignored +source-record IDs, and the function-dossier adapter copied observation fields +manually without those links. + +The session now authenticates/retains source records independently of replay +eligibility. Plain application helpers also return authenticated immutable source +records with their outcome, including late cancellation. Both adapters explicitly +commit them before delivering links; an adapter without a retention owner refuses +the response. Source collection borrows ledger records without constructing or +hashing an intermediate bundle. The workflow links distinct source IDs without copying their payloads. +The function adapter uses the common observation-to-Evidence constructor, +preserving sources through normal and late-cancelled completion. Queries with no +source records avoid the extra recording path. Complete provider bodies remain +in the exported function files; this does not add per-function payload retention. + +A real test project contains historical seed and disabled-analyzer reports saved +by REA's actual Ghidra pre-scripts. The unmodified runtime completed the export +but failed the provenance oracle: its bundle had two workflow records and no +historical source. The fixed runtime retains exactly one historical source and +links the workflow to it. All 652 inventory entries and the sampled pseudocode +still match fresh import; source project content stays unchanged. The report is +historical, with remaining original settings/completeness explicitly unknown. + +## Measured costs + +Baseline runtime: `19447516a4bdbc9437d112ebd3f12a5fd062db83`. +Initial fixed runtime: `f50763fd21527bdf3bab944633617b9df61dded4`, rebased on +main `f02df45a4`. Both use Node 24.22.1, locked dependencies, actual Ghidra 12.1.4 +and JDK 21.0.12.1+1. The fixed driver later delegates transcript publication to +the existing cleanup/receipt owner; the protocol and task are unchanged. + +| Completed task | Wall s | Family CPU s | Peak family GiB | Full-wire token estimate | +| --------------------------------- | ------: | -----------: | --------------: | -----------------------: | +| ls, before | 51.411 | 78.984 | 0.943 | 57,082 | +| ls, initial fix | 50.937 | 79.891 | 1.201 | 57,072 | +| ls with historical reports, fixed | 52.827 | 80.236 | 1.097 | 57,207 | +| libcrypto, baseline capacity run | 506.387 | 459.818 | 1.502 | 57,322 | + +The plain ls pair has identical complete comparison facts. Runtime is similar; +the higher sampled memory in the fixed run is reported rather than treated as an +improvement. These sequential shared-host samples establish no generic speed or +memory ratio. [PR #7250](https://github.com/morluto/rea/pull/7250) records the final +large-run comparison and CI/review results. +The failed historical baseline is excluded from completed-task comparisons. + +The complete MCP replies stayed around 11–12 KiB, including repeated textual and +structured representations, profile and caveats. The normalized result is only +the manifest. For example, the large continuation costs about 3,159 estimated +wire tokens; its complete task is 57,322 because discovery alone costs 47,049. +Both numbers include actual protocol content, rather than only pseudocode bytes. + +Estimates use `cl100k_base` on the actual JSON-RPC messages, including discovery, +requests, responses and notifications, excluding only transport framing and the +driver's local transcript wrapper. They are not actual model billing. Host +projection, prompts/reasoning and later agent file reads remain unmeasured. +The historical source increases response token cost slightly while preserving +provenance. No token-reduction claim is made for this correctness fix. + +CPU and memory cover the driver, REA and provider process family. Each run uses +CPU affinity 0,1 and an 8-GiB sampled family-RSS guard; Java heaps are 1536M for ls +and 3072M for crypto. RSS is sampled every 100 ms and summed: shared pages may be +counted repeatedly and short peaks may be missed. CPU is waited child usage. +Reported completed runs had no resource stop, OOM or cleanup failure. +Project preparation is excluded and documented separately in +[project reuse](ghidra-project-reuse.md). Provider starts are process-cold, with +host filesystem caches unflushed; verified resume occurs in the second session. + +The index and inventory still scale with function count. This 14,338-item run +does not establish a larger capacity guarantee. Real Windows/macOS filesystems, +other decompilers and native DWARF directory layout remain unverified. The current +workflow reports unavailable DWARF layout explicitly instead of inventing it. + +Actual CLI `decompile-all` calls additionally selected only `0x10a420`, reported +651 excluded entries and `debug_layout_unavailable` for requested dwarf layout, +then resumed in a new process. The existing file's hash and modification time +stayed unchanged; the returned historical source ID stayed stable. This CLI check +verifies the returned manifest/link and local function file, not offline retrieval +of the separately retained source record after the CLI process exits. + +## Reproduce and inspect + +Prepare an owned project using the documented +[existing-project route](../../docs/installation.md#reuse-an-existing-ghidra-project). +Use fresh receipt/output paths; the driver refuses existing output directories. + +```sh +node experiments/analysis-efficiency/decompile-directory.mjs \ + --runtime /absolute/rea --output /absolute/new-receipt \ + --target /usr/bin/ls --project /absolute/ls-analysis.gpr --program ls \ + --ghidra "$GHIDRA_INSTALL_DIR" --java "$JAVA_HOME" +``` + +Use `--chunk 512 --heap 3072M --decompile-seconds 5` for the crypto case. +`--control /absolute/previous-receipt` compares all exported facts; +`--golden /absolute/fresh-import-receipt` checks the #1048 driver's inventory and +pseudocode oracle. `--history required` additionally requires a linked historical +source. The driver allows failed functions to succeed on a later retry, verifies only +previously written files remain unchanged, and reloads the final index before +comparison. Complete local receipts include the transcript, resource report, index, +function files, comparison facts and authenticated Evidence export. The resource +report is supplied by the external process-family sampler, not the driver. + +The current nine-file suite passed 144 tests. Against unmodified main source, +the three source-retention/constructor suites reproduce seven failures and 70 +passes; the fixed full suite passes. This includes invalid source commitments, +live and cursor-dependent reads, shared source deduplication, selected/null/provider +subjects, and source links on late-cancelled completion. diff --git a/experiments/analysis-efficiency/decompile-directory.mjs b/experiments/analysis-efficiency/decompile-directory.mjs new file mode 100644 index 0000000000..736ce4588c --- /dev/null +++ b/experiments/analysis-efficiency/decompile-directory.mjs @@ -0,0 +1,328 @@ +/** Real-provider directory export, cross-session resume and retained provenance. */ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { execFile } from "node:child_process"; +import { createWriteStream } from "node:fs"; +import { mkdir, mkdtemp, readFile, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { finished } from "node:stream/promises"; +import { pathToFileURL } from "node:url"; +import { parseArgs, promisify } from "node:util"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { finalizeRetainedQueryReceipt } from "./retained-query-finalization.mjs"; + +const { values } = parseArgs({ + options: Object.fromEntries( + [ + "runtime", + "output", + "target", + "project", + "program", + "ghidra", + "java", + "control", + "heap", + "chunk", + "decompile-seconds", + "history", + "golden", + ].map((key) => [key, { type: "string" }]), + ), +}); +for (const key of [ + "runtime", + "output", + "target", + "project", + "ghidra", + "java", +]) { + assert.ok(values[key], `${key} is required`); + values[key] = resolve(values[key]); +} +const chunk = Number(values.chunk ?? 32); +assert.ok(Number.isSafeInteger(chunk) && chunk > 0); +await mkdir(values.output); +const digest = (bytes) => createHash("sha256").update(bytes).digest("hex"); +const report = { + calls: [], + runtime: values.runtime, + target: values.target, + project: values.project, +}; +let client, transport, account, serverLog, serverLogFinished; +// This workflow has a fixed number of public calls; pseudocode stays on disk. +const transcript = []; +let phase = "initialize"; +const started = performance.now(); +const tree = join(values.output, "tree"); +const readIndex = async () => + JSON.parse(await readFile(join(tree, "decompile-index.json"), "utf8")); +const fileStates = async (index) => { + const states = {}; + for (const row of index.functions.filter((row) => row.status === "written")) { + const path = join(tree, row.relative_path); + const metadata = await stat(path); + states[row.address] = { + sha256: digest(await readFile(path)), + mtime_ms: metadata.mtimeMs, + }; + assert.equal(states[row.address].sha256, row.sha256); + } + return states; +}; +try { + report.revision = ( + await promisify(execFile)("git", [ + "-C", + values.runtime, + "rev-parse", + "HEAD", + ]) + ).stdout.trim(); + report.target_sha256 = digest(await readFile(values.target)); + const { resolveGhidraProject } = await import( + pathToFileURL(join(values.runtime, "dist/ghidra/GhidraProjectSnapshot.js")) + .href + ); + const { ArtifactResourceScope } = await import( + pathToFileURL( + join(values.runtime, "dist/artifacts/ArtifactResourceScope.js"), + ).href + ); + const fingerprint = async () => { + const resources = new ArtifactResourceScope(); + try { + return await resolveGhidraProject( + { + path: values.project, + ...(values.program ? { programName: values.program } : {}), + }, + { path: values.target }, + resources, + ); + } finally { + await resources.close(); + } + }; + const original = await fingerprint(); + report.project_sha256 = original.sha256; + account = await mkdtemp(join(tmpdir(), "rea-directory-investigation-")); + transport = new StdioClientTransport({ + command: process.execPath, + args: [join(values.runtime, "scripts/rea.mjs"), "mcp"], + cwd: values.runtime, + maxBufferSize: 10 * 1024 * 1024, + env: { + PATH: process.env.PATH, + HOME: account, + USERPROFILE: account, + JAVA_HOME: values.java, + GHIDRA_INSTALL_DIR: values.ghidra, + GHIDRA_HEADLESS_MAXMEM: values.heap ?? "1536M", + JAVA_TOOL_OPTIONS: "-XX:ActiveProcessorCount=2", + NODE_OPTIONS: process.env.NODE_OPTIONS ?? "--max-old-space-size=3072", + REA_GHIDRA_PROJECT_PATH: values.project, + ...(values.program ? { REA_GHIDRA_PROGRAM_NAME: values.program } : {}), + REA_GHIDRA_STARTUP_TIMEOUT_MS: "600000", + ...(values["decompile-seconds"] + ? { REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS: values["decompile-seconds"] } + : {}), + REA_MCP_TOOL_FAMILIES: "direct,enhanced,application,session", + REA_MCP_INPUT_SCHEMA_PROFILE: "compact", + REA_MCP_OUTPUT_SCHEMA_PROFILE: "compact", + }, + stderr: "pipe", + }); + const start = transport.start.bind(transport); + transport.start = async () => { + const receive = transport.onmessage; + transport.onmessage = (message) => { + transcript.push({ phase, direction: "response", message }); + receive?.(message); + }; + await start(); + }; + const send = transport.send.bind(transport); + transport.send = async (message) => { + transcript.push({ phase, direction: "request", message }); + await send(message); + }; + serverLog = createWriteStream(join(values.output, "server.log")); + serverLogFinished = finished(serverLog); + serverLogFinished.catch(() => {}); + transport.stderr.pipe(serverLog); + client = new Client({ name: "rea-directory-investigation", version: "1" }); + await client.connect(transport); + phase = "discovery"; + const catalog = await client.listTools({}, { timeout: 15000 }); + await writeFile(join(values.output, "catalog.json"), JSON.stringify(catalog)); + report.tools = catalog.tools.length; + const definitions = new Map(catalog.tools.map((tool) => [tool.name, tool])); + const call = async (name, args, label) => { + phase = label; + const at = performance.now(); + const result = await client.callTool( + { name, arguments: args }, + { timeout: 900000, toolDefinition: definitions.get(name) }, + ); + report.calls.push({ + name, + phase, + elapsed_ms: performance.now() - at, + response_bytes: Buffer.byteLength(JSON.stringify(result)), + is_error: result.isError ?? false, + }); + assert.notEqual( + result.isError, + true, + `${name}: ${JSON.stringify(result).slice(0, 800)}`, + ); + return result.structuredContent; + }; + const open = () => + call("open_binary", { path: values.target, provider_id: "ghidra" }, "open"); + await open(); + const first = await call( + "decompile_all", + { output_directory: tree, limit: chunk }, + "first-chunk", + ); + report.first = first.normalized_result; + const firstFiles = await fileStates(await readIndex()); + await call("close_binary", {}, "close-first-session"); + await open(); + const remaining = await call( + "decompile_all", + { output_directory: tree, resume: true }, + "cross-session-resume", + ); + report.resume = remaining.normalized_result; + report.resume_evidence_links = remaining.evidence_links?.length ?? 0; + let index = await readIndex(); + const allFiles = await fileStates(index); + for (const [address, state] of Object.entries(firstFiles)) + assert.deepEqual(allFiles[address], state); + report.first_files_unchanged = true; + const settled = await call( + "decompile_all", + { output_directory: tree, resume: true }, + "verified-resume", + ); + report.verified_resume = settled.normalized_result; + if (report.resume.counts.failed === 0) + assert.equal(report.verified_resume.counts.written, 0); + assert.equal(report.verified_resume.counts.remaining, 0); + index = await readIndex(); + const settledFiles = await fileStates(index); + for (const [address, state] of Object.entries(allFiles)) + assert.deepEqual(settledFiles[address], state); + assert.equal(index.functions.length, report.resume.counts.selected); + const facts = []; + for (const row of index.functions) { + const fact = { + address: row.address, + name: row.name, + status: row.status, + body: row.body, + }; + if (row.status === "written") { + const text = await readFile(join(tree, row.relative_path), "utf8"); + assert.ok(text.includes(`// address: ${row.address}\n`)); + assert.ok( + text.includes( + `// analysis_profile: ${index.analysis_profile_digest}\n`, + ), + ); + fact.pseudocode_sha256 = digest(text.slice(text.indexOf("\n\n") + 2)); + } + facts.push(fact); + } + report.function_count = facts.length; + report.written = facts.filter((row) => row.status === "written").length; + report.failed = facts.filter((row) => row.status === "failed").length; + report.external = facts.filter((row) => row.status === "external").length; + report.facts_sha256 = digest(JSON.stringify(facts)); + await writeFile(join(values.output, "facts.json"), JSON.stringify(facts)); + const evidencePath = join(values.output, "complete-evidence.json"); + await call( + "export_evidence_bundle", + { path: evidencePath }, + "complete-evidence-export", + ); + await call("close_binary", {}, "close"); + await client.close(); + await transport.close(); + const { parseEvidenceBundle } = await import( + pathToFileURL(join(values.runtime, "dist/domain/evidenceBundle.js")).href + ); + const bundle = parseEvidenceBundle( + JSON.parse(await readFile(evidencePath, "utf8")), + ); + report.evidence_records = bundle.records.length; + const retained = bundle.records.find( + (record) => record.evidence_id === remaining.evidence_id, + ); + assert.ok(retained); + report.retained_links = retained.evidence_links.length; + const byId = new Set(bundle.records.map((record) => record.evidence_id)); + for (const id of retained.evidence_links) assert.ok(byId.has(id)); + report.all_source_links_resolve = true; + if (values.history === "required") { + const historical = bundle.records.filter( + (record) => + record.operation === "health" && + record.parameters.facet === "historical-project-analysis-reports", + ); + assert.equal(historical.length, 1); + assert.equal(historical[0].normalized_result.origin, "caller-project"); + assert.ok(retained.evidence_links.includes(historical[0].evidence_id)); + report.historical_source_retained_and_linked = true; + } + if (values.golden) { + const golden = JSON.parse( + await readFile(join(values.golden, "facts.json"), "utf8"), + ); + const labels = (rows) => + rows + .map((row) => [row.address, row.name ?? row.value]) + .sort((a, b) => a[0].localeCompare(b[0], "en")); + assert.deepEqual(labels(facts), labels(golden.procedures)); + for (const item of golden.pseudocode) { + const fact = facts.find((row) => row.address === item.procedure); + assert.equal(fact?.pseudocode_sha256, digest(item.result)); + } + report.fresh_import_inventory_and_sampled_pseudocode_match = true; + } + if (values.control) { + const control = JSON.parse( + await readFile(join(values.control, "facts.json"), "utf8"), + ); + assert.deepEqual(facts, control); + report.control_facts_match = true; + } + assert.deepEqual(await fingerprint(), original); + assert.equal(digest(await readFile(values.target)), report.target_sha256); + report.source_unchanged = true; + report.outcome = "passed"; +} catch (cause) { + report.outcome = "failed"; + report.failure = String(cause); + process.exitCode = 1; +} finally { + report.elapsed_ms = performance.now() - started; + const published = await finalizeRetainedQueryReceipt({ + report, + client, + transport, + account, + serverLog, + serverLogFinished, + output: values.output, + transcript, + }); + if (!published) process.exitCode = 1; +} diff --git a/src/application/DirectAnalysis.ts b/src/application/DirectAnalysis.ts index 0c16d806ad..c008b5ac61 100644 --- a/src/application/DirectAnalysis.ts +++ b/src/application/DirectAnalysis.ts @@ -13,6 +13,7 @@ import { silentLogger } from "../logger.js"; import type { Logger } from "pino"; import { withEvidenceCallerInputPath } from "../domain/evidence.js"; import { createImmutableEvidenceBundle } from "../domain/evidenceBundle.js"; +import { retainAnalysisResultSources } from "./investigation/recordEvidenceSources.js"; import type { Evidence } from "../domain/evidence.js"; import type { NativeToolName } from "../contracts/native/nativeToolContracts.js"; import { @@ -570,12 +571,17 @@ const executeAnalysisTool = async (input: { if (tool === "analyze_function") { const { executeFunctionAnalysisEvidence } = await import("./FunctionAnalysisEvidence.js"); - const result = await executeFunctionAnalysisEvidence( + let result = await executeFunctionAnalysisEvidence( session, input.arguments, input.openedTarget, { signal }, ); + result = retainAnalysisResultSources( + tool, + (source) => session.recordEvidence(source), + result, + ); return result.ok ? ok({ evidence: result.value }) : result; } if ( @@ -588,12 +594,17 @@ const executeAnalysisTool = async (input: { tool === "decompile_all" ) { const { EnhancedTools } = await import("./EnhancedTools.js"); - const result = await executeComposedWorkflow( + let result = await executeComposedWorkflow( session, (observed) => new EnhancedTools(observed).execute(tool, input.arguments, signal), composedLimitationBudget(tool), ); + result = retainAnalysisResultSources( + tool, + (source) => session.recordEvidence(source), + result, + ); if (!result.ok) return result; const evidence = createWorkflowEvidence({ target: input.openedTarget, diff --git a/src/application/FunctionAnalysisEvidence.test.ts b/src/application/FunctionAnalysisEvidence.test.ts index d38b0938ec..9d8cc3bbaf 100644 --- a/src/application/FunctionAnalysisEvidence.test.ts +++ b/src/application/FunctionAnalysisEvidence.test.ts @@ -2,7 +2,7 @@ import { expect, it } from "vitest"; import { createAnalysisExecution } from "./AnalysisProvider.js"; import { executeFunctionAnalysisEvidence } from "./FunctionAnalysisEvidence.js"; import { AnalysisOutputError } from "../domain/analysisErrorCore.js"; -import { parseEvidence } from "../domain/evidence.js"; +import { createEvidence, parseEvidence } from "../domain/evidence.js"; import { createAnalysisProfile } from "../domain/analysisProfile.js"; import { ghidraFunctionDossier } from "../domain/ghidraValues.fixture.js"; import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; @@ -62,6 +62,16 @@ it.each(["selected-target", "no-subject", "provider-subject"] as const)( path: "/observed/provider.elf", sha256: "b".repeat(64), }; + const source = createEvidence( + undefined, + provider, + { + operation: "health", + parameters: { facet: "historical-report" }, + result: { original: "unknown" }, + }, + { immutable: true }, + ); for (const aborted of [false, true]) { const controller = new AbortController(); const execution = createAnalysisExecution(dossier, provider, { @@ -73,6 +83,7 @@ it.each(["selected-target", "no-subject", "provider-subject"] as const)( rawResult: { provider_text: "preserved" }, limitations: ["Synthetic provider observation"], locations: [{ kind: "address", address: "0x401000" }], + sourceEvidence: [source], }); const result = await executeFunctionAnalysisEvidence( { @@ -85,6 +96,7 @@ it.each(["selected-target", "no-subject", "provider-subject"] as const)( target, { signal: controller.signal }, ); + expect(result.sourceEvidence).toEqual([source]); if (aborted) { if (result.ok) throw new Error("Expected late cancellation"); expect(result.error).toMatchObject({ _tag: "AnalysisCancelledError" }); @@ -100,6 +112,7 @@ it.each(["selected-target", "no-subject", "provider-subject"] as const)( expect(evidence.provider).toEqual(provider); expect(evidence.analysis_profile).toEqual(analysisProfile); expect(evidence.relationship_coverage).toEqual(relationshipCoverage); + expect(evidence.evidence_links).toEqual([source.evidence_id]); expect(evidence.limitations).toContain("Synthetic provider observation"); expect(evidence.locations).toContainEqual({ kind: "address", diff --git a/src/application/FunctionAnalysisEvidence.ts b/src/application/FunctionAnalysisEvidence.ts index b97fa398ca..d8f6463ea2 100644 --- a/src/application/FunctionAnalysisEvidence.ts +++ b/src/application/FunctionAnalysisEvidence.ts @@ -5,10 +5,17 @@ import type { } from "./AnalysisProvider.js"; import { nativeProcedureInputSchema } from "../domain/native/nativeProcedureInput.js"; import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; -import { createEvidence } from "../domain/evidence.js"; +import { createAnalysisExecutionEvidence } from "../domain/analysisObservation.js"; import { parseFunctionDossier } from "../domain/hopperValues.js"; import type { JsonValue } from "../domain/jsonValue.js"; import { completeProviderEvidence } from "./ProviderOperationExecution.js"; +import type { Evidence } from "../domain/evidence.js"; +import { EvidenceLedger } from "./investigation/EvidenceLedger.js"; +import { + recordEvidenceSources, + withEvidenceSources, + type AnalysisResultWithSources, +} from "./investigation/recordEvidenceSources.js"; /** * Preserve the provider's observations when presenting a validated function @@ -20,7 +27,7 @@ export const executeFunctionAnalysisEvidence = async ( parameters: Readonly>, target: BinaryTarget | undefined, options?: ExecutionOptions, -) => { +): Promise> => { const input = parseAnalysisInput( "analyze_function", parameters, @@ -35,30 +42,33 @@ export const executeFunctionAnalysisEvidence = async ( options, ); if (!execution.ok) return execution; + const sourceLedger = new EvidenceLedger(); + const recorded = recordEvidenceSources( + (source) => sourceLedger.record(source), + execution.value.sourceEvidence ?? [], + ); + const sources = sourceLedger.recordsForAnalysis(); + if (!recorded.ok) return withEvidenceSources(recorded, sources); const dossier = parseFunctionDossier(execution.value.result); - if (!dossier.ok) return dossier; + if (!dossier.ok) return withEvidenceSources(dossier, sources); const observation = execution.value; - return completeProviderEvidence( - createEvidence( - observation.subject === undefined - ? target - : (observation.subject ?? undefined), - observation.provider, - { - operation: "analyze_function", - parameters: arguments_, - result: dossier.value, - rawResult: observation.rawResult, - limitations: observation.limitations, - locations: observation.locations, - ...(observation.relationshipCoverage === undefined - ? {} - : { relationshipCoverage: observation.relationshipCoverage }), - ...(observation.analysisProfile === undefined - ? {} - : { analysisProfile: observation.analysisProfile }), - }, + return withEvidenceSources( + completeProviderEvidence( + createAnalysisExecutionEvidence( + "analyze_function", + arguments_, + { + ...observation, + sourceEvidence: sources, + result: dossier.value, + ...(observation.subject === undefined && target !== undefined + ? { subject: target } + : {}), + }, + "observed", + ), + options?.signal, ), - options?.signal, + sources, ); }; diff --git a/src/application/WorkflowEvidence.test.ts b/src/application/WorkflowEvidence.test.ts index ee3629d433..88596f3653 100644 --- a/src/application/WorkflowEvidence.test.ts +++ b/src/application/WorkflowEvidence.test.ts @@ -19,6 +19,48 @@ import { projectNativeApiInspection } from "./native/NativeApiInspection.js"; import { functionDossierSchema } from "../domain/hopperValues.js"; import { ghidraFunctionDossier } from "../domain/ghidraValues.fixture.js"; import { ANALYSIS_SNAPSHOT_PROFILE } from "../domain/analysisSnapshot.fixture.js"; +import { createEvidence } from "../domain/evidence.js"; + +it("links shared provider source Evidence once without copying its payload into the workflow", async () => { + const source = createEvidence( + undefined, + { id: "fixture", name: "Fixture", version: "1" }, + { + operation: "health", + parameters: { facet: "saved-analysis-report" }, + result: { report: "historical content ".repeat(1_000) }, + }, + { immutable: true }, + ); + const result = await executeComposedWorkflow( + { + execute: async () => + ok( + createAnalysisExecution( + "observed", + { id: "fixture", name: "Fixture", version: "1" }, + { sourceEvidence: [source] }, + ), + ), + }, + async (observed) => { + await observed.execute("list_procedures", {}); + await observed.execute("procedure_pseudo_code", { procedure: "0x1000" }); + return ok({ count: 1 }); + }, + ); + expect(result).toEqual({ + ...ok({ + result: { count: 1 }, + limitations: [], + evidenceLinks: [source.evidence_id], + }), + sourceEvidence: [source], + }); + expect(result.ok && JSON.stringify(result.value)).not.toContain( + "historical content", + ); +}); it.each([false, true])( "propagates transient=%s through composed snapshot decisions while retaining Evidence", diff --git a/src/application/WorkflowEvidence.ts b/src/application/WorkflowEvidence.ts index 9a8d6528a7..a68f8ba9fa 100644 --- a/src/application/WorkflowEvidence.ts +++ b/src/application/WorkflowEvidence.ts @@ -23,6 +23,12 @@ import { import type { JsonValue } from "../domain/jsonValue.js"; import type { UnknownRegistryPort } from "./investigation/InvestigationRecordPort.js"; import { UnknownRegistryError } from "../domain/unknownRegistryError.js"; +import { EvidenceLedger } from "./investigation/EvidenceLedger.js"; +import { + recordEvidenceSources, + withEvidenceSources, + type AnalysisResultWithSources, +} from "./investigation/recordEvidenceSources.js"; import { workflowUnknownInput } from "./investigation/WorkflowUnknown.js"; import { renderSourceLimitations, @@ -71,18 +77,16 @@ export const executeComposedWorkflow = async ( ) => Promise>, budget?: ComposedLimitationBudget, ): Promise< - Result< - { - readonly result: JsonValue; - readonly limitations: readonly string[]; - readonly cachePolicy?: "live"; - readonly evidenceLinks?: readonly string[]; - }, - AnalysisError - > + AnalysisResultWithSources<{ + readonly result: JsonValue; + readonly limitations: readonly string[]; + readonly cachePolicy?: "live"; + readonly evidenceLinks?: readonly string[]; + }> > => { const limitations = new Map(); const evidenceLinks = new Set(); + const sourceLedger = new EvidenceLedger(); const perItemGroups = new Set(); let perItemBytes = 0; let omittedObservations = 0; @@ -93,6 +97,13 @@ export const executeComposedWorkflow = async ( const execution = await analysis.execute(operation, parameters, options); if (execution.ok && execution.value.evidenceId !== undefined) evidenceLinks.add(execution.value.evidenceId); + if (execution.ok && execution.value.sourceEvidence !== undefined) { + const recorded = recordEvidenceSources( + (source) => sourceLedger.record(source), + execution.value.sourceEvidence, + ); + if (!recorded.ok) return recorded; + } if (execution.ok && execution.value.cachePolicy === "live") cachePolicy = "live"; if (execution.ok) @@ -129,22 +140,27 @@ export const executeComposedWorkflow = async ( return execution; }, }); - if (!result.ok) return result; - return ok({ - result: result.value, - limitations: [ - ...renderSourceLimitations([...limitations.values()]), - ...(omittedObservations === 0 - ? [] - : [ - `${omittedObservations} per-item provider limitation observation${omittedObservations === 1 ? " was" : "s were"} omitted from this answer; ${budget?.retainedIn}.`, - ]), - ], - ...(cachePolicy === undefined ? {} : { cachePolicy }), - ...(evidenceLinks.size === 0 - ? {} - : { evidenceLinks: [...evidenceLinks].sort(compareUnicodeCodePoints) }), - }); + const sources = sourceLedger.recordsForAnalysis(); + if (!result.ok) return withEvidenceSources(result, sources); + for (const source of sources) evidenceLinks.add(source.evidence_id); + return withEvidenceSources( + ok({ + result: result.value, + limitations: [ + ...renderSourceLimitations([...limitations.values()]), + ...(omittedObservations === 0 + ? [] + : [ + `${omittedObservations} per-item provider limitation observation${omittedObservations === 1 ? " was" : "s were"} omitted from this answer; ${budget?.retainedIn}.`, + ]), + ], + ...(cachePolicy === undefined ? {} : { cachePolicy }), + ...(evidenceLinks.size === 0 + ? {} + : { evidenceLinks: [...evidenceLinks].sort(compareUnicodeCodePoints) }), + }), + sources, + ); }; /** Build the shared Evidence record for a successful composed binary workflow. */ diff --git a/src/application/binary/BinarySession.ts b/src/application/binary/BinarySession.ts index d0f80297d4..f4cb2f5feb 100644 --- a/src/application/binary/BinarySession.ts +++ b/src/application/binary/BinarySession.ts @@ -1254,6 +1254,15 @@ export class BinarySession ) this.#active.nativeSourceAdmitted = true; this.#observeRuntimeAvailability(name, execution); + // Source facts belong to the observation, even when this query cannot be + // replayed (for example a Ghidra procedure read without a document selector). + if (execution.ok && execution.value.sourceEvidence !== undefined) { + const sourcesRecorded = recordEvidenceSources( + (source) => this.recordEvidence(source), + execution.value.sourceEvidence, + ); + if (!sourcesRecorded.ok) return sourcesRecorded; + } if ( execution.ok && cacheable && @@ -1276,11 +1285,6 @@ export class BinarySession }, "observed", ); - const sourcesRecorded = recordEvidenceSources( - (source) => this.recordEvidence(source), - execution.value.sourceEvidence ?? [], - ); - if (!sourcesRecorded.ok) return sourcesRecorded; const recorded = this.recordEvidence(evidence); if (!recorded.ok) return recorded; const retained = { ...execution.value, evidenceId: evidence.evidence_id }; diff --git a/src/application/investigation/EvidenceLedger.ts b/src/application/investigation/EvidenceLedger.ts index 76f3a0484d..3b3bb4cfc7 100644 --- a/src/application/investigation/EvidenceLedger.ts +++ b/src/application/investigation/EvidenceLedger.ts @@ -198,6 +198,11 @@ export class EvidenceLedger { ); } + /** Borrow immutable source records without constructing or hashing a bundle. */ + recordsForAnalysis(): readonly Evidence[] { + return [...this.#records.values()]; + } + /** Check whether one immutable Evidence ID is present in this ledger. */ has(evidenceId: string): boolean { return this.#records.has(evidenceId); diff --git a/src/application/investigation/recordEvidenceSources.test.ts b/src/application/investigation/recordEvidenceSources.test.ts new file mode 100644 index 0000000000..5c5bb3252f --- /dev/null +++ b/src/application/investigation/recordEvidenceSources.test.ts @@ -0,0 +1,32 @@ +import { expect, it } from "vitest"; +import { createEvidence } from "../../domain/evidence.js"; +import { AnalysisCancelledError } from "../../domain/analysisErrorCore.js"; +import { EvidenceIntegrityError } from "../../domain/evidenceErrors.js"; +import { err, ok } from "../../domain/result.js"; +import { + retainAnalysisResultSources, + withEvidenceSources, +} from "./recordEvidenceSources.js"; + +it.each([false, true])( + "reports source recording failure without replacing earlier cancellation (cancelled=%s)", + (cancelled) => { + const source = createEvidence( + undefined, + { id: "fixture", name: "Fixture", version: "1" }, + { operation: "health", parameters: {}, result: "historical" }, + ); + const primary = new AnalysisCancelledError("analyze_function", { + cause: "caller", + }); + const failure = new EvidenceIntegrityError("Recorder failed"); + const result = retainAnalysisResultSources( + "analyze_function", + () => err(failure), + withEvidenceSources(cancelled ? err(primary) : ok("complete"), [source]), + ); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected recording failure"); + expect(result.error).toBe(cancelled ? primary : failure); + }, +); diff --git a/src/application/investigation/recordEvidenceSources.ts b/src/application/investigation/recordEvidenceSources.ts index 05e7f2ff54..0b25b761b5 100644 --- a/src/application/investigation/recordEvidenceSources.ts +++ b/src/application/investigation/recordEvidenceSources.ts @@ -1,7 +1,49 @@ import type { EvidenceWriter } from "./InvestigationRecordPort.js"; import { EvidenceIntegrityError } from "../../domain/evidenceErrors.js"; import type { Evidence } from "../../domain/evidence.js"; -import { ok, type Result } from "../../domain/result.js"; +import { err, ok, type Result } from "../../domain/result.js"; +import type { AnalysisError } from "../../domain/analysisErrorBase.js"; +import { AnalysisCapabilityUnavailableError } from "../../domain/analysisErrorCore.js"; + +/** Source facts remain available even when a completed observation is cancelled. */ +export type AnalysisResultWithSources = Result & { + readonly sourceEvidence?: readonly Evidence[]; +}; + +export const withEvidenceSources = < + Outcome extends Result, +>( + result: Outcome, + sources: readonly Evidence[], +): Outcome & { readonly sourceEvidence?: readonly Evidence[] } => + sources.length === 0 ? result : { ...result, sourceEvidence: sources }; + +/** Never deliver links to records a plain-port adapter cannot retain. */ +export const retainAnalysisResultSources = ( + operation: string, + recordEvidence: EvidenceWriter["recordEvidence"] | undefined, + result: AnalysisResultWithSources, +): Result => { + const sources = result.sourceEvidence ?? []; + const retained = + sources.length > 0 && recordEvidence === undefined + ? err( + new AnalysisCapabilityUnavailableError( + "rea", + operation, + "Provider source Evidence requires an explicit retention owner.", + { + userMessage: + "Use an analysis session that retains provider source Evidence, or supply an Evidence recorder when registering this adapter.", + }, + ), + ) + : recordEvidenceSources(recordEvidence, sources); + if (retained.ok) return result; + if (result.ok) return retained; + result.error.retainSecondaryFailure(retained.error); + return result; +}; /** Record source evidence in order, stopping at the first session rejection. */ export const recordEvidenceSources = ( diff --git a/src/server/registerEnhancedTools.ts b/src/server/registerEnhancedTools.ts index 60e54f610e..57b23bdfb1 100644 --- a/src/server/registerEnhancedTools.ts +++ b/src/server/registerEnhancedTools.ts @@ -34,6 +34,7 @@ import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; import { executeFunctionAnalysisEvidence } from "../application/FunctionAnalysisEvidence.js"; import type { WithAdmittedAnalysis } from "./analysisAdmission.js"; import type { EvidenceLookup } from "../application/EvidenceInputResolver.js"; +import { retainAnalysisResultSources } from "../application/investigation/recordEvidenceSources.js"; import { requireAnalysisViewRetention, summarizeRetainedAnalysis, @@ -266,7 +267,7 @@ const executeEnhancedTool = async ( progress, }), }; - const result = await logToolExecution(registration.logger, name, () => + let result = await logToolExecution(registration.logger, name, () => executeComposedWorkflow( analysis, (observed) => @@ -277,6 +278,11 @@ const executeEnhancedTool = async ( composedLimitationBudget(name), ), ); + result = retainAnalysisResultSources( + name, + registration.recordEvidence, + result, + ); await reportOperationEnd(progress, name, result.ok); if (result.ok) { const evidence = createWorkflowEvidence({ @@ -399,7 +405,7 @@ const deliverFunctionAnalysis = async ( }, ): Promise => { const { detail, parameters, context, progress } = request; - const result = await logToolExecution( + let result = await logToolExecution( registration.logger, "analyze_function", () => @@ -413,6 +419,11 @@ const deliverFunctionAnalysis = async ( }, ), ); + result = retainAnalysisResultSources( + "analyze_function", + registration.recordEvidence, + result, + ); await reportOperationEnd(progress, "analyze_function", result.ok); if (!result.ok) return registration.delivery.toCallToolResult(result, contract); diff --git a/tests/boundary/mcp/analyzeFunctionDetail.test.ts b/tests/boundary/mcp/analyzeFunctionDetail.test.ts index af674bae64..1a92deffcc 100644 --- a/tests/boundary/mcp/analyzeFunctionDetail.test.ts +++ b/tests/boundary/mcp/analyzeFunctionDetail.test.ts @@ -7,7 +7,13 @@ import { registerEnhancedTools } from "../../../src/server/registerEnhancedTools import { createToolResultDelivery } from "../../../src/server/toolResult.js"; import { withAdmittedAnalysis } from "../../../src/server/analysisAdmission.js"; import type { AnalysisOperationPort } from "../../../src/application/AnalysisProvider.js"; -import { parseEvidence, type Evidence } from "../../../src/domain/evidence.js"; +import { + createEvidence, + parseEvidence, + type Evidence, +} from "../../../src/domain/evidence.js"; +import { join } from "node:path"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import type { JsonValue } from "../../../src/domain/jsonValue.js"; import { ok } from "../../../src/domain/result.js"; import { functionDossierSchema } from "../../../src/domain/hopperValues.js"; @@ -29,22 +35,31 @@ const connect = async ( retain = true, result = dossier, path = () => "/fixtures/native.exe", + sources: readonly Evidence[] = [], ) => { const ledger = new EvidenceLedger(); const record = (evidence: Evidence) => ledger.record(evidence); const lookup = (id: string) => ledger.forAnalysis(id); const execute = vi.fn( async (operation: string, parameters: Record) => { - if (operation !== "analyze_function") - throw new Error(`Unexpected operation: ${operation}`); - const input = parseGhidraFunctionInput(operation, parameters); - if (!input.ok) return input; + let observed: JsonValue = result; + if (operation === "analyze_function") { + const input = parseGhidraFunctionInput(operation, parameters); + if (!input.ok) return input; + } else if (operation === "list_procedures") { + observed = [{ address: "0x401000", value: "main" }]; + } else if (operation === "procedure_info") { + observed = null; + } else if (operation === "procedure_pseudo_code") { + observed = "int main(void) { return 0; }"; + } else throw new Error(`Unexpected operation: ${operation}`); return ok({ - result, + result: observed, rawResult: null, provider: { id: "ghidra", name: "Ghidra", version: "12.1.4" }, limitations: [], locations: [], + sourceEvidence: sources, subject: { path: path(), format: "pe" as const, @@ -91,6 +106,62 @@ const connect = async ( return { client, ledger, execute }; }; +it.each(["analyze_function", "decompile_all"] as const)( + "retains plain-port source records before delivering %s links", + async (operation) => { + const source = createEvidence( + undefined, + { id: "ghidra", name: "Ghidra", version: "12.1.4" }, + { + operation: "health", + parameters: { facet: "historical-report" }, + result: { origin: "caller-project" }, + }, + { immutable: true }, + ); + const { client, ledger } = await connect(true, dossier, undefined, [ + source, + ]); + const parent = await createTestTempDirectory("rea-plain-source-"); + const reply = await client.callTool({ + name: operation, + arguments: + operation === "analyze_function" + ? { procedure: "main", detail: "complete" } + : { output_directory: join(parent, "out") }, + }); + expect(reply.isError).not.toBe(true); + const evidence = parseEvidence(reply.structuredContent); + expect(evidence.evidence_links).toContain(source.evidence_id); + expect(JSON.stringify(reply)).not.toContain("caller-project"); + expect(ledger.forAnalysis(source.evidence_id)).toEqual(source); + expect( + ledger + .immutableBundle() + .records.filter((record) => record.evidence_id === source.evidence_id), + ).toHaveLength(1); + }, +); + +it("refuses source links when a plain-port adapter has no retention owner", async () => { + const source = createEvidence( + undefined, + { id: "fixture", name: "Fixture", version: "1" }, + { + operation: "health", + parameters: {}, + result: "historical", + }, + ); + const { client } = await connect(false, dossier, undefined, [source]); + const reply = await client.callTool({ + name: "analyze_function", + arguments: { procedure: "main", detail: "complete" }, + }); + expect(reply.isError).toBe(true); + expect(JSON.stringify(reply)).toContain("retains provider source Evidence"); +}); + it("keeps complete and summary delivery on the current observed path despite retained same-ID history", async () => { let selected = "/old/native.exe"; const { client, ledger } = await connect(true, dossier, () => selected); diff --git a/tests/composition/analysis-sessions/binarySession.test.ts b/tests/composition/analysis-sessions/binarySession.test.ts index 8610248afc..b3442bd31e 100644 --- a/tests/composition/analysis-sessions/binarySession.test.ts +++ b/tests/composition/analysis-sessions/binarySession.test.ts @@ -968,6 +968,89 @@ describe("snapshot switch rollback", () => { }); describe("snapshot cache eligibility", () => { + it.each([false, true])( + "retains source Evidence independently of snapshot eligibility (live=%s)", + async (live) => { + const [target] = await createBinarySessionTargets(); + const provider = createCacheProvider([]); + const capabilities = provider.capabilities(); + provider.capabilities = () => + capabilities.map((capability) => ({ + ...capability, + ...(live ? { cachePolicy: "live" as const } : {}), + })); + const source = createEvidence( + undefined, + provider.identity(), + { + operation: "health", + parameters: { facet: "saved-analysis-report" }, + result: { original_analysis: "unknown" }, + }, + { immutable: true }, + ); + provider.createClient = () => ({ + execute: async () => + resultOk( + createAnalysisExecution("name", provider.identity(), { + sourceEvidence: [source], + }), + ), + close: async () => resultOk(null), + }); + const session = createTestBinarySession(provider); + try { + expect((await session.open(target)).ok).toBe(true); + const parameters: Readonly> = live + ? { address: "0x1000", document: "main" } + : {}; + for (let attempt = 0; attempt < 2; attempt += 1) { + const result = await session.execute("address_name", parameters); + expect(result).toMatchObject({ ok: true }); + if (result.ok) expect(result.value.evidenceId).toBeUndefined(); + } + expect(session.exportEvidenceBundle().records).toEqual([source]); + expect(session.exportAnalysisSnapshot()).toMatchObject({ + ok: true, + value: { entries: [] }, + }); + } finally { + await session.close(); + } + }, + ); + + it("rejects invalid source Evidence on a non-replayable observation", async () => { + const [target] = await createBinarySessionTargets(); + const provider = createCacheProvider([]); + const source = createEvidence(undefined, provider.identity(), { + operation: "health", + parameters: {}, + result: "original", + }); + provider.createClient = () => ({ + execute: async () => + resultOk( + createAnalysisExecution("name", provider.identity(), { + sourceEvidence: [{ ...source, normalized_result: "tampered" }], + }), + ), + close: async () => resultOk(null), + }); + const session = createTestBinarySession(provider); + try { + expect((await session.open(target)).ok).toBe(true); + const result = await session.execute("address_name", {}); + expect(result).toMatchObject({ + ok: false, + error: { _tag: "EvidenceIntegrityError" }, + }); + expect(session.exportEvidenceBundle().records).toEqual([]); + } finally { + await session.close(); + } + }); + it("does not snapshot reads that depend on the provider cursor", async () => { const [first] = await createBinarySessionTargets(); const calls: string[] = []; From 8423c5c6d08512568c2a75753b6b7c3a9b6bbd4e Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Tue, 27 Oct 2026 00:13:35 +0800 Subject: [PATCH 15/25] refactor(macho): consolidate prefix interpretation across readers --- src/artifacts/DesktopFileInspection.ts | 2 +- src/artifacts/NodeSeaArtifactReader.test.ts | 17 +++++ src/artifacts/apple/DylibFileAccess.ts | 3 +- .../apple/MachoLoadCommandReader.test.ts | 2 +- src/artifacts/apple/MachoLoadCommandReader.ts | 47 ++++--------- src/artifacts/apple/machoContainer.test.ts | 8 ++- .../inventory/ArtifactGraphConstruction.ts | 5 +- src/artifacts/nodeSea/NodeSeaLocation.ts | 23 ++++--- src/domain/apple/machoContainer.ts | 68 +++++++++++++------ src/domain/apple/machoLoadCommands.test.ts | 50 ++++++++++++++ src/domain/apple/machoLoadCommands.ts | 11 +-- src/domain/binaryTarget.ts | 7 +- src/electron/ElectronFuseReader.ts | 18 +++-- src/hopper/HopperAnalysisProfile.test.ts | 40 +++++++++++ src/hopper/HopperAnalysisProfile.ts | 9 ++- src/hopper/HopperMachOImage.ts | 7 +- src/native/metadata/BinaryContainer.test.ts | 39 +++++++++++ src/native/metadata/readBinaryContainer.ts | 8 ++- 18 files changed, 258 insertions(+), 106 deletions(-) diff --git a/src/artifacts/DesktopFileInspection.ts b/src/artifacts/DesktopFileInspection.ts index 577fd8e87b..067f14da06 100644 --- a/src/artifacts/DesktopFileInspection.ts +++ b/src/artifacts/DesktopFileInspection.ts @@ -21,7 +21,7 @@ import { readAssemblyReferenceName } from "../dotnet/ManagedMetadataInventoryRow import { ManagedReaderFailure } from "../dotnet/ManagedReaderFailure.js"; import { ArtifactReaderFailure } from "./ArtifactReader.js"; import { abortIfNeeded } from "./ArtifactHash.js"; -import { hasMachoMagic } from "./apple/MachoLoadCommandReader.js"; +import { hasMachoMagic } from "../domain/apple/machoContainer.js"; export interface DesktopReadBudget { bytes: number; diff --git a/src/artifacts/NodeSeaArtifactReader.test.ts b/src/artifacts/NodeSeaArtifactReader.test.ts index fe8a455baa..58b7f2b449 100644 --- a/src/artifacts/NodeSeaArtifactReader.test.ts +++ b/src/artifacts/NodeSeaArtifactReader.test.ts @@ -728,3 +728,20 @@ it("retains SEA admission for a complete segment with a nonaligned command size" await handle.close(); } }); + +it.each([0xfeedface, 0xfeedfacf, 0xcefaedfe, 0xcffaedfe])( + "classifies four-byte thin magic %i without admitting a truncated SEA carrier", + async (magic) => { + const bytes = Buffer.alloc(4); + bytes.writeUInt32BE(magic); + const path = await writeCarrier("thin-prefix", bytes); + const inventory = await scanCanonicalArtifactInventory(path); + expect(inventory.manifest.root_format).toBe("mach-o"); + const handle = await open(path, "r"); + try { + expect(await locateNodeSea(handle, bytes.length)).toBeUndefined(); + } finally { + await handle.close(); + } + }, +); diff --git a/src/artifacts/apple/DylibFileAccess.ts b/src/artifacts/apple/DylibFileAccess.ts index 8fcff4fe11..0b58fd595c 100644 --- a/src/artifacts/apple/DylibFileAccess.ts +++ b/src/artifacts/apple/DylibFileAccess.ts @@ -20,7 +20,8 @@ import { } from "../ArtifactFileOperation.js"; import type { ArtifactResourceScope } from "../ArtifactResourceScope.js"; import { readArtifactFileChunks } from "../readArtifactFileChunks.js"; -import { hasMachoMagic, readMachoImage } from "./MachoLoadCommandReader.js"; +import { hasMachoMagic } from "../../domain/apple/machoContainer.js"; +import { readMachoImage } from "./MachoLoadCommandReader.js"; /** One owner for dyld descriptor admission, physical identity and finalization. */ export class DylibFileAccess { diff --git a/src/artifacts/apple/MachoLoadCommandReader.test.ts b/src/artifacts/apple/MachoLoadCommandReader.test.ts index e9f0980be4..6683099533 100644 --- a/src/artifacts/apple/MachoLoadCommandReader.test.ts +++ b/src/artifacts/apple/MachoLoadCommandReader.test.ts @@ -27,9 +27,9 @@ import { MAX_LAZY_INFO_DEPENDENCIES, MAX_LAZY_INFO_SYMBOL_REFERENCES, MAX_LOAD_COMMAND_BYTES, - hasMachoMagic, readMachoImage, } from "./MachoLoadCommandReader.js"; +import { hasMachoMagic } from "../../domain/apple/machoContainer.js"; import { traceDylibResolution } from "./DylibResolutionReader.js"; import { DirectoryArtifactReader } from "../DirectoryArtifactReader.js"; import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; diff --git a/src/artifacts/apple/MachoLoadCommandReader.ts b/src/artifacts/apple/MachoLoadCommandReader.ts index 5b2d935e89..5bd1429a3b 100644 --- a/src/artifacts/apple/MachoLoadCommandReader.ts +++ b/src/artifacts/apple/MachoLoadCommandReader.ts @@ -22,8 +22,11 @@ import { } from "../../domain/apple/machoPlatform.js"; import { classifyFatContainerIdentity, + hasMachoMagic, readFatContainerHeader, + readFatHeaderPrefix, readFatSliceDeclarations, + readMachoHeaderPrefixFacts, validateFatSlice, validateFatSliceIdentity, type FatSliceDeclaration, @@ -46,14 +49,6 @@ export const MAX_LAZY_INFO_DEPENDENCIES = 1024; * offset, so stop before that map and those decodes grow. */ export const MAX_LAZY_INFO_SYMBOL_REFERENCES = 64 * 1024; -const MH_MAGIC = 0xfeedface; -const MH_MAGIC_64 = 0xfeedfacf; -const MH_CIGAM = 0xcefaedfe; -const MH_CIGAM_64 = 0xcffaedfe; -const FAT_MAGIC = 0xcafebabe; -const FAT_MAGIC_64 = 0xcafebabf; -const FAT_CIGAM = 0xbebafeca; -const FAT_CIGAM_64 = 0xbfbafeca; const LC_SEGMENT = 0x1; const LC_SEGMENT_64 = 0x19; @@ -114,26 +109,6 @@ class MachoFormatIssue extends Error { } } -/** Report whether the first bytes carry a Mach-O or FAT magic number. */ -export const hasMachoMagic = (head: Uint8Array): boolean => { - if (head.byteLength < 4) return false; - const magic = new DataView( - head.buffer, - head.byteOffset, - head.byteLength, - ).getUint32(0, false); - return [ - MH_MAGIC, - MH_MAGIC_64, - MH_CIGAM, - MH_CIGAM_64, - FAT_MAGIC, - FAT_MAGIC_64, - FAT_CIGAM, - FAT_CIGAM_64, - ].includes(magic); -}; - /** Decode every slice's dylib-loading load commands without reading code. */ export const readMachoImage = async ( readAt: ReadAt, @@ -146,10 +121,13 @@ export const readMachoImage = async ( return hasMachoMagic(head) ? { status: "malformed", reason: "Mach-O header is truncated" } : { status: "not-mach-o" }; - const view = viewOf(head); - const magic = view.getUint32(0, false); + const fatPrefix = readFatHeaderPrefix(head); + const mayBeJvmClass = + fatPrefix.state === "parsed" && + !fatPrefix.wide && + !fatPrefix.littleEndian; let header: ReturnType; - if (magic === FAT_MAGIC && size <= MAX_JVM_CLASS_IDENTITY_BYTES) { + if (mayBeJvmClass && size <= MAX_JVM_CLASS_IDENTITY_BYTES) { const complete = await readAt(0, size); header = complete.byteLength === size @@ -187,7 +165,7 @@ export const readMachoImage = async ( // A large CAFEBABE input whose bounded prefix resembles an invalid // known-CPU FAT table remains unknown until its JVM identity can be // checked; do not turn the resource limit into a malformed claim. - if (magic === FAT_MAGIC && size > MAX_JVM_CLASS_IDENTITY_BYTES) + if (mayBeJvmClass && size > MAX_JVM_CLASS_IDENTITY_BYTES) throw new MachoFormatIssue( "unsupported", JVM_CLASS_IDENTITY_LIMIT_REASON, @@ -203,9 +181,10 @@ export const readMachoImage = async ( ...(hiddenSlice === null ? {} : { hiddenSlices: [hiddenSlice] }), }; } - if (magic === MH_CIGAM || magic === MH_CIGAM_64) + const thin = readMachoHeaderPrefixFacts(head).format; + if (thin?.littleEndian === true) return { status: "parsed", slices: [await readSlice(readAt, 0, size)] }; - if (magic === MH_MAGIC || magic === MH_MAGIC_64) + if (thin !== null) throw new MachoFormatIssue( "unsupported", "big-endian Mach-O images are not supported", diff --git a/src/artifacts/apple/machoContainer.test.ts b/src/artifacts/apple/machoContainer.test.ts index 45b51f32ce..40403e397c 100644 --- a/src/artifacts/apple/machoContainer.test.ts +++ b/src/artifacts/apple/machoContainer.test.ts @@ -129,7 +129,9 @@ it("preserves explicit unknown slice evidence instead of falling back to retaine const header = readFatContainerHeader(bytes, bytes.length); if (header.state !== "parsed") throw new Error("Expected FAT header"); const offset = header.slices[0]!.offset; - const unknown = new Map([[offset, { magic: null, identity: null }]]); + const unknown = new Map([ + [offset, readMachoHeaderPrefixFacts(Buffer.alloc(0))], + ]); expect(classifyFatContainerEvidence(header, bytes, new Map())).toEqual({ state: "mach-o", }); @@ -258,6 +260,7 @@ describe("FAT Mach-O identity", () => { bigEndian.writeUInt32BE(CPU.x86_64.subtype, 8); expect(readMachoHeaderPrefixFacts(littleEndian.subarray(0, 12))).toEqual({ magic: 0xcffaedfe, + format: { wide: true, littleEndian: true, headerSize: 32 }, identity: { cpuType: CPU.arm64.type, cpuSubtype: CPU.arm64.subtype, @@ -265,6 +268,7 @@ describe("FAT Mach-O identity", () => { }); expect(readMachoHeaderPrefixFacts(bigEndian)).toEqual({ magic: 0xfeedfacf, + format: { wide: true, littleEndian: false, headerSize: 32 }, identity: { cpuType: CPU.x86_64.type, cpuSubtype: CPU.x86_64.subtype, @@ -272,10 +276,12 @@ describe("FAT Mach-O identity", () => { }); expect(readMachoHeaderPrefixFacts(littleEndian.subarray(0, 11))).toEqual({ magic: 0xcffaedfe, + format: { wide: true, littleEndian: true, headerSize: 32 }, identity: null, }); expect(readMachoHeaderPrefixFacts(littleEndian.subarray(0, 3))).toEqual({ magic: null, + format: null, identity: null, }); }); diff --git a/src/artifacts/inventory/ArtifactGraphConstruction.ts b/src/artifacts/inventory/ArtifactGraphConstruction.ts index 65f2a2250a..b85f400aa7 100644 --- a/src/artifacts/inventory/ArtifactGraphConstruction.ts +++ b/src/artifacts/inventory/ArtifactGraphConstruction.ts @@ -32,6 +32,7 @@ import { parseMzHeaderEvidence } from "../../domain/dosMz.js"; import { classifyFatContainerEvidence, readFatContainerHeader, + readMachoHeaderPrefixFacts, } from "../../domain/apple/machoContainer.js"; /** Mutable internal occurrence used until root-bound IDs are known. */ @@ -581,9 +582,7 @@ export const classifyArtifactBytes = ( const fatFormat = classifyFatArtifact(evidence, fileSize); if (fatFormat !== null) return fatFormat; if (prefix.length >= 4) { - const magic = prefix.readUInt32BE(0); - if ([0xfeedface, 0xfeedfacf, 0xcefaedfe, 0xcffaedfe].includes(magic)) - return "mach-o"; + if (readMachoHeaderPrefixFacts(prefix).format !== null) return "mach-o"; if (prefix.subarray(0, 4).equals(Buffer.from([0x7f, 0x45, 0x4c, 0x46]))) return "elf"; } diff --git a/src/artifacts/nodeSea/NodeSeaLocation.ts b/src/artifacts/nodeSea/NodeSeaLocation.ts index 661cdd373f..cfa25719c9 100644 --- a/src/artifacts/nodeSea/NodeSeaLocation.ts +++ b/src/artifacts/nodeSea/NodeSeaLocation.ts @@ -17,7 +17,11 @@ import type { FileHandle } from "node:fs/promises"; import { abortIfNeeded } from "../ArtifactHash.js"; import { ArtifactReaderFailure } from "../ArtifactReader.js"; -import { readFatContainerHeader } from "../../domain/apple/machoContainer.js"; +import { + readFatContainerHeader, + readFatHeaderPrefix, + readMachoHeaderPrefixFacts, +} from "../../domain/apple/machoContainer.js"; import { readElfFixedHeader } from "../../domain/elfFixedHeader.js"; import { readMachoLoadCommandTableHeader } from "../../domain/apple/machoLoadCommands.js"; import { @@ -52,9 +56,6 @@ const FUSE_LENGTH = "NODE_SEA_FUSE_".length + 32 + 2; // PE header structures must fit this prefix for the bounded name probe. const PE_HEADER_PROBE_BYTES = 64 * 1024; -const THIN_MAGICS = new Set([0xfeedface, 0xfeedfacf, 0xcefaedfe, 0xcffaedfe]); -const FAT_MAGICS = new Set([0xcafebabe, 0xcafebabf, 0xbebafeca, 0xbfbafeca]); - /** One injected blob inside a native carrier. Offsets are carrier file offsets. */ export interface NodeSeaLocation { readonly carrier: "mach-o" | "elf" | "pe"; @@ -96,11 +97,13 @@ export const locateNodeSea = async ( const carrier: Carrier = { handle, fileSize, signal }; const prefix = await readExact(carrier, 0, Math.min(fileSize, 64)); if (prefix === undefined || prefix.length < 4) return undefined; - const magic = prefix.readUInt32BE(0); - if (THIN_MAGICS.has(magic) || FAT_MAGICS.has(magic)) { - const ranges = THIN_MAGICS.has(magic) - ? await locateThinMachO(carrier, 0, fileSize) - : await locateFatMachO(carrier, prefix); + const thin = readMachoHeaderPrefixFacts(prefix).format; + const fat = readFatHeaderPrefix(prefix); + if (thin !== null || fat.state === "incomplete" || fat.state === "parsed") { + const ranges = + thin !== null + ? await locateThinMachO(carrier, 0, fileSize) + : await locateFatMachO(carrier, prefix); return collapseRanges(carrier, "mach-o", ranges); } if (prefix.subarray(0, 4).equals(Buffer.from([0x7f, 0x45, 0x4c, 0x46]))) @@ -216,7 +219,7 @@ const locateFatMachO = async ( header = readFatContainerHeader(bytes, carrier.fileSize); } if (header.state !== "parsed") { - if (bytes.length >= 4 && THIN_MAGICS.has(bytes.readUInt32BE(0))) + if (readMachoHeaderPrefixFacts(bytes).format !== null) return locateThinMachO(carrier, 0, carrier.fileSize); return []; } diff --git a/src/domain/apple/machoContainer.ts b/src/domain/apple/machoContainer.ts index 71a506ab6b..ae771b90a1 100644 --- a/src/domain/apple/machoContainer.ts +++ b/src/domain/apple/machoContainer.ts @@ -46,6 +46,12 @@ export interface FatSliceHeaderEvidence { export interface MachoHeaderPrefixFacts { /** Raw format evidence; null means fewer than four bytes were observed. */ readonly magic: number | null; + /** Present for a recognized four-byte thin magic, before header admission. */ + readonly format: { + readonly wide: boolean; + readonly littleEndian: boolean; + readonly headerSize: 28 | 32; + } | null; /** Present only for a recognized Mach-O magic and a complete 12-byte prefix. */ readonly identity: { readonly cpuType: number; @@ -57,18 +63,22 @@ export interface MachoHeaderPrefixFacts { export const readMachoHeaderPrefixFacts = ( bytes: Uint8Array, ): MachoHeaderPrefixFacts => { - if (bytes.byteLength < 4) return { magic: null, identity: null }; + if (bytes.byteLength < 4) + return { magic: null, format: null, identity: null }; const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); const magic = view.getUint32(0, false); - if (bytes.byteLength < 12 || !MACHO_HEADER_MAGICS.includes(magic)) - return { magic, identity: null }; - const littleEndian = magic === 0xcefaedfe || magic === 0xcffaedfe; + const format = thinMachoFormat(magic); + if (format === null) return { magic, format: null, identity: null }; return { magic, - identity: { - cpuType: view.getUint32(4, littleEndian), - cpuSubtype: view.getUint32(8, littleEndian), - }, + format, + identity: + bytes.byteLength < 12 + ? null + : { + cpuType: view.getUint32(4, format.littleEndian), + cpuSubtype: view.getUint32(8, format.littleEndian), + }, }; }; @@ -108,9 +118,23 @@ const MAX_FAT_ARCHITECTURES = 128; const KNOWN_MACHO_CPU_TYPES: readonly number[] = [ 7, 0x01000007, 12, 0x0100000c, 0x0200000c, 18, 0x01000012, ]; -const MACHO_HEADER_MAGICS: readonly number[] = [ - 0xfeedface, 0xfeedfacf, 0xcefaedfe, 0xcffaedfe, -]; +const thinMachoFormat = (magic: number): MachoHeaderPrefixFacts["format"] => { + if (![0xfeedface, 0xfeedfacf, 0xcefaedfe, 0xcffaedfe].includes(magic)) + return null; + const wide = magic === 0xfeedfacf || magic === 0xcffaedfe; + return { + wide, + littleEndian: magic === 0xcefaedfe || magic === 0xcffaedfe, + headerSize: wide ? 32 : 28, + }; +}; + +/** Raw thin/FAT signature recognition; FAT magic alone does not resolve JVM ambiguity. */ +export const hasMachoMagic = (bytes: Uint8Array): boolean => { + if (readMachoHeaderPrefixFacts(bytes).format !== null) return true; + const fat = readFatHeaderPrefix(bytes); + return fat.state !== "unknown" && fat.state !== "not-fat"; +}; interface FatHeaderFormat { readonly magic: number; @@ -131,18 +155,24 @@ export type FatHeaderPrefix = } )); +/** Interpret a raw FAT magic without asserting container identity or header readiness. */ +export const readFatMagicFormat = (magic: number): FatHeaderFormat | null => { + if (![FAT_MAGIC, FAT_MAGIC_64, FAT_CIGAM, FAT_CIGAM_64].includes(magic)) + return null; + return { + magic, + wide: magic === FAT_MAGIC_64 || magic === FAT_CIGAM_64, + littleEndian: magic === FAT_CIGAM || magic === FAT_CIGAM_64, + }; +}; + /** Observe the fixed FAT header without table, count-limit or container-identity admission. */ export const readFatHeaderPrefix = (bytes: Uint8Array): FatHeaderPrefix => { if (bytes.byteLength < 4) return { state: "unknown", requiredBytes: 4 }; const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); const magic = view.getUint32(0, false); - if (![FAT_MAGIC, FAT_MAGIC_64, FAT_CIGAM, FAT_CIGAM_64].includes(magic)) - return { state: "not-fat", magic }; - const format = { - magic, - wide: magic === FAT_MAGIC_64 || magic === FAT_CIGAM_64, - littleEndian: magic === FAT_CIGAM || magic === FAT_CIGAM_64, - }; + const format = readFatMagicFormat(magic); + if (format === null) return { state: "not-fat", magic }; if (bytes.byteLength < 8) return { state: "incomplete", ...format, requiredBytes: 8 }; const count = view.getUint32(4, format.littleEndian); @@ -255,7 +285,7 @@ export const classifyFatContainerIdentity = ( const magic = sliceHeaderAt(slice.offset); if (magic === undefined || magic === null) missingHeaders.push(slice.offset); - else if (MACHO_HEADER_MAGICS.includes(magic)) observedMachO = true; + else if (thinMachoFormat(magic) !== null) observedMachO = true; else observedNonMachO = true; } if (observedNonMachO) return { state: "not-mach-o" }; diff --git a/src/domain/apple/machoLoadCommands.test.ts b/src/domain/apple/machoLoadCommands.test.ts index 3ef8830340..360e4a2d31 100644 --- a/src/domain/apple/machoLoadCommands.test.ts +++ b/src/domain/apple/machoLoadCommands.test.ts @@ -1,4 +1,9 @@ import { expect, it } from "vitest"; +import { hasMachoMagic, readMachoHeaderPrefixFacts } from "./machoContainer.js"; +import { + claimsExecutableHeader, + parseExecutableHeader, +} from "../binaryTarget.js"; import { readMachoHeaderDeclarations, readMachoLoadCommandDeclarations, @@ -31,6 +36,29 @@ it.each([ fileType: 0xffffffff, identity: { cpuType: 0xfedcba98, cpuSubtype: 0x89abcdef }, }; + const format = { wide, littleEndian, headerSize: wide ? 32 : 28 }; + for (const length of [0, 3, 4, 11, 12]) { + const prefix = bytes.subarray(0, length); + expect(hasMachoMagic(prefix)).toBe(length >= 4); + expect(readMachoHeaderPrefixFacts(prefix)).toEqual({ + magic: length < 4 ? null : magic, + format: length < 4 ? null : format, + identity: length < 12 ? null : expected.identity, + }); + expect(readMachoHeaderDeclarations(prefix, 32)).toEqual( + length < 4 + ? { ok: true, value: null } + : { + ok: false, + error: `truncated Mach-O header: a ${wide ? 64 : 32}-bit header needs ${format.headerSize} bytes; the file has 32`, + }, + ); + } + expect(claimsExecutableHeader(bytes.subarray(0, 4))).toBe(true); + expect(parseExecutableHeader(bytes.subarray(0, 4), "arm64")).toEqual({ + ok: false, + error: "truncated or unsupported binary header", + }); expect(readMachoHeaderDeclarations(bytes, 32)).toEqual({ ok: true, value: expected, @@ -57,3 +85,25 @@ it.each([ ).toMatchObject({ ok: false }); }, ); + +it.each([ + 0xcafebabe, 0xcafebabf, 0xbebafeca, 0xbfbafeca, 0x7f454c46, 0xffffffff, +])( + "retains raw non-thin magic %s without thin format or CPU claims", + (magic) => { + const bytes = Buffer.alloc(12, 0xff); + bytes.writeUInt32BE(magic, 0); + expect(hasMachoMagic(bytes.subarray(0, 4))).toBe( + magic !== 0x7f454c46 && magic !== 0xffffffff, + ); + expect(readMachoHeaderPrefixFacts(bytes)).toEqual({ + magic, + format: null, + identity: null, + }); + expect(readMachoHeaderDeclarations(bytes, bytes.length)).toEqual({ + ok: true, + value: null, + }); + }, +); diff --git a/src/domain/apple/machoLoadCommands.ts b/src/domain/apple/machoLoadCommands.ts index f7232c2bbb..7c9ca95ff1 100644 --- a/src/domain/apple/machoLoadCommands.ts +++ b/src/domain/apple/machoLoadCommands.ts @@ -24,19 +24,12 @@ export const readMachoHeaderDeclarations = ( fileSize: number, ): Result => { const prefix = readMachoHeaderPrefixFacts(bytes); - const magic = prefix.magic; - if ( - magic === null || - ![0xfeedface, 0xfeedfacf, 0xcefaedfe, 0xcffaedfe].includes(magic) - ) - return ok(null); - const wide = magic === 0xfeedfacf || magic === 0xcffaedfe; - const headerSize = wide ? 32 : 28; + if (prefix.format === null) return ok(null); + const { wide, headerSize, littleEndian } = prefix.format; if (bytes.byteLength < 28 || prefix.identity === null) return err( `truncated Mach-O header: a ${wide ? 64 : 32}-bit header needs ${headerSize} bytes; the file has ${fileSize}`, ); - const littleEndian = magic === 0xcefaedfe || magic === 0xcffaedfe; const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); const commandBytes = view.getUint32(20, littleEndian); return ok({ diff --git a/src/domain/binaryTarget.ts b/src/domain/binaryTarget.ts index c5d0aaeb5a..35bd9faf72 100644 --- a/src/domain/binaryTarget.ts +++ b/src/domain/binaryTarget.ts @@ -103,9 +103,7 @@ export const claimsExecutableHeader = ( ): boolean => (bytes.length >= 4 && (bytes.subarray(0, 4).equals(Buffer.from([0x7f, 0x45, 0x4c, 0x46])) || - [0xfeedface, 0xfeedfacf, 0xcefaedfe, 0xcffaedfe].includes( - bytes.readUInt32BE(0), - ))) || + readMachoHeaderPrefixFacts(bytes).format !== null)) || (bytes.length >= 2 && bytes[0] === 0x4d && bytes[1] === 0x5a) || readFatContainerHeader(bytes, fileSize).state !== "not-fat"; @@ -132,8 +130,7 @@ export const parseExecutableHeader = ( parseMzHeaderEvidence(bytes, fileSize, mzWindowsHeaderSignature), ); if (bytes.length < 8) return err("truncated or unsupported binary header"); - const magic = bytes.readUInt32BE(0); - if ([0xfeedface, 0xfeedfacf, 0xcefaedfe, 0xcffaedfe].includes(magic)) + if (readMachoHeaderPrefixFacts(bytes).format !== null) return parseThinMachO(bytes, fileSize); const fatHeader = fatEvidence?.header ?? readFatContainerHeader(bytes, fileSize); diff --git a/src/electron/ElectronFuseReader.ts b/src/electron/ElectronFuseReader.ts index b060b0ac2a..6faa627a93 100644 --- a/src/electron/ElectronFuseReader.ts +++ b/src/electron/ElectronFuseReader.ts @@ -6,10 +6,11 @@ import type { ResolvedElectronFuseFile } from "./ElectronFusePath.js"; import { createHash } from "node:crypto"; import { lstat, type FileHandle } from "node:fs/promises"; +import { readMachoImage } from "../artifacts/apple/MachoLoadCommandReader.js"; import { hasMachoMagic, - readMachoImage, -} from "../artifacts/apple/MachoLoadCommandReader.js"; + readFatHeaderPrefix, +} from "../domain/apple/machoContainer.js"; import type { ReadAt } from "../artifacts/apple/MachoLoadCommandReader.js"; import { electronFuseName, @@ -38,7 +39,6 @@ export const ELECTRON_FUSE_LIMITS = { } as const; const SENTINEL = Buffer.from(ELECTRON_FUSE_SENTINEL, "ascii"); -const FAT_MAGICS = new Set([0xcafebabe, 0xcafebabf, 0xbebafeca, 0xbfbafeca]); /** Container identity and per-slice fuse wires. Offsets are file-absolute. */ export interface ElectronFuseImage { @@ -262,7 +262,11 @@ const identifyMacho = async ( reason, ); } - const format = isFat(head) ? "mach-o-universal" : "mach-o"; + const fat = readFatHeaderPrefix(head); + const format = + fat.state === "parsed" || fat.state === "incomplete" + ? "mach-o-universal" + : "mach-o"; const slices = machoImageSlices(facts).map((slice, index) => ({ index, fileOffset: slice.slice_offset, @@ -471,12 +475,6 @@ const isElf = (head: Uint8Array): boolean => head[2] === 0x4c && head[3] === 0x46; -const isFat = (head: Uint8Array): boolean => { - if (head.length < 4) return false; - const magic = Buffer.from(head.buffer, head.byteOffset, 4).readUInt32BE(0); - return FAT_MAGICS.has(magic); -}; - const identifyPe = async ( readAt: ReadAt, size: number, diff --git a/src/hopper/HopperAnalysisProfile.test.ts b/src/hopper/HopperAnalysisProfile.test.ts index f48f5d6b35..4dbc491000 100644 --- a/src/hopper/HopperAnalysisProfile.test.ts +++ b/src/hopper/HopperAnalysisProfile.test.ts @@ -595,3 +595,43 @@ it.each(["header", "launcher"] as const)( }); }, ); + +it.each([ + [0xcafebabe, "fat32"], + [0xbebafeca, "fat32"], + [0xcafebabf, "fat64"], + [0xbfbafeca, "fat64"], +] as const)( + "retains FAT width from a four-byte profile prefix %i", + async (magic, container) => { + const directory = await createTestTempDirectory( + "rea-hopper-prefix-profile-", + ); + const bytes = Buffer.alloc(4); + bytes.writeUInt32BE(magic); + const path = join(directory, "container.macho"); + await writeFile(path, bytes); + const result = await resolveHopperAnalysisProfile( + { + path, + sha256: createHash("sha256").update(bytes).digest("hex"), + kind: "executable", + format: "mach-o-universal", + architecture: "arm64", + availableArchitectures: ["arm64"], + universalSlice: { offset: 4096, size: 64 }, + }, + { + launcherPath: path, + loaderArgsOverride: ["-l", "Mach-O"], + provider: { id: "hopper", name: "Hopper Disassembler", version: null }, + resourceScope: inspectionResources.owner, + }, + ); + if (!result.ok) throw new Error(result.error.message); + expect(result.value.profile?.parameters).toMatchObject({ + macho_container: container, + loader: { source: "configured_override", arguments: ["-l", "Mach-O"] }, + }); + }, +); diff --git a/src/hopper/HopperAnalysisProfile.ts b/src/hopper/HopperAnalysisProfile.ts index 159ac5b122..1f54825f0e 100644 --- a/src/hopper/HopperAnalysisProfile.ts +++ b/src/hopper/HopperAnalysisProfile.ts @@ -43,6 +43,7 @@ import { type MachoTargetSelection, } from "../domain/apple/machoTargetSelection.js"; import { AnalysisError } from "../domain/analysisErrorBase.js"; +import { readFatMagicFormat } from "../domain/apple/machoContainer.js"; import { err, ok, type Result } from "../domain/result.js"; import { @@ -100,11 +101,9 @@ const resolveHopperAnalysisProfileInScope = async ( options.signal, ); if (!header.ok) return header; - machoContainer = [0xcafebabf, 0xbfbafeca].includes(header.value) - ? "fat64" - : [0xcafebabe, 0xbebafeca].includes(header.value) - ? "fat32" - : "thin"; + const fatFormat = readFatMagicFormat(header.value); + machoContainer = + fatFormat === null ? "thin" : fatFormat.wide ? "fat64" : "fat32"; } let preparedImage: HopperMachOImage | undefined; if ( diff --git a/src/hopper/HopperMachOImage.ts b/src/hopper/HopperMachOImage.ts index d8bcaefb4c..7d0097003b 100644 --- a/src/hopper/HopperMachOImage.ts +++ b/src/hopper/HopperMachOImage.ts @@ -287,9 +287,8 @@ const inspectHopperMachOImage = async ( }; const header = await read(0, 8); const fatPrefix = readFatHeaderPrefix(header); - const magic = - header.length === 8 && fatPrefix.state !== "unknown" ? fatPrefix.magic : 0; - if ([0xfeedface, 0xfeedfacf, 0xcefaedfe, 0xcffaedfe].includes(magic)) { + const thinPrefix = readMachoHeaderPrefixFacts(header); + if (header.length === 8 && thinPrefix.format !== null) { if (target.format !== "mach-o") return invalid("Thin Mach-O source disagrees with its admitted format"); const prefix = Buffer.alloc(12); @@ -330,7 +329,7 @@ const inspectHopperMachOImage = async ( `Thin Mach-O source changed after target admission: expected SHA-256 ${target.sha256}, observed ${observedSha256}`, ); const facts = readMachoHeaderPrefixFacts(prefix); - if (facts.magic !== magic || facts.identity === null) + if (facts.magic !== thinPrefix.magic || facts.identity === null) return invalid("Thin Mach-O header changed during profile resolution"); return ok({ method: "thin_verified_copy", diff --git a/src/native/metadata/BinaryContainer.test.ts b/src/native/metadata/BinaryContainer.test.ts index 65cf2aaa1a..8c32250893 100644 --- a/src/native/metadata/BinaryContainer.test.ts +++ b/src/native/metadata/BinaryContainer.test.ts @@ -1,3 +1,4 @@ +import { readBinaryContainer } from "./readBinaryContainer.js"; import { expect, it } from "vitest"; import { BinaryFormatFailure, @@ -172,3 +173,41 @@ it.each([false, true])( ); }, ); + +it.each([ + { magic: 0xfeedface, bits: 32, headerSize: 28 }, + { magic: 0xfeedfacf, bits: 64, headerSize: 32 }, + { magic: 0xcefaedfe, bits: 32, headerSize: 28 }, + { magic: 0xcffaedfe, bits: 64, headerSize: 32 }, +])( + "retains thin-header truncation after recognizing magic $magic", + ({ magic, bits, headerSize }) => { + const bytes = Buffer.alloc(4); + bytes.writeUInt32BE(magic); + expect(() => readBinaryContainer(bytes)).toThrow( + new BinaryFormatFailure( + "malformed", + `truncated Mach-O header: a ${bits}-bit header needs ${headerSize} bytes; the file has 4`, + ), + ); + }, +); + +it("keeps a JVM-sized FAT count unconfirmed while rejecting malformed zero-count FAT", () => { + const bytes = Buffer.alloc(8); + bytes.writeUInt32BE(0xcafebabe); + bytes.writeUInt32BE(0x0000003d, 4); // JVM major version 61; no FAT table follows. + expect(() => readBinaryContainer(bytes)).toThrow( + new BinaryFormatFailure( + "unsupported", + "Only ELF, PE, and thin Mach-O native images are supported", + ), + ); + bytes.writeUInt32BE(0, 4); + expect(() => readBinaryContainer(bytes)).toThrow( + new BinaryFormatFailure( + "malformed", + "FAT header declares no architectures", + ), + ); +}); diff --git a/src/native/metadata/readBinaryContainer.ts b/src/native/metadata/readBinaryContainer.ts index 18bc93ceae..860c0812c4 100644 --- a/src/native/metadata/readBinaryContainer.ts +++ b/src/native/metadata/readBinaryContainer.ts @@ -5,6 +5,8 @@ import { import { classifyFatContainerIdentity, readFatContainerHeader, + readFatHeaderPrefix, + readMachoHeaderPrefixFacts, } from "../../domain/apple/machoContainer.js"; import { readElfImage } from "./ElfImage.js"; import { readPeImage } from "./PeImage.js"; @@ -16,8 +18,8 @@ export const readBinaryContainer = (bytes: Buffer): BinaryContainer => { return readElfImage(bytes); if (bytes.subarray(0, 2).equals(Buffer.from("MZ"))) return readPeImage(bytes); if (bytes.length >= 4) { - const magic = bytes.readUInt32BE(); - if ([0xcafebabe, 0xcafebabf, 0xbebafeca, 0xbfbafeca].includes(magic)) { + const fatPrefix = readFatHeaderPrefix(bytes); + if (fatPrefix.state === "incomplete" || fatPrefix.state === "parsed") { const fatHeader = readFatContainerHeader(bytes, bytes.length); if (fatHeader.state === "malformed") throw new BinaryFormatFailure("malformed", fatHeader.reason); @@ -38,7 +40,7 @@ export const readBinaryContainer = (bytes: Buffer): BinaryContainer => { "Universal/fat Mach-O containers require selecting a thin slice", ); } - if ([0xfeedface, 0xfeedfacf, 0xcefaedfe, 0xcffaedfe].includes(magic)) + if (readMachoHeaderPrefixFacts(bytes).format !== null) return readMachoImage(bytes); } throw new BinaryFormatFailure( From 2cdc8cb92ef3f800e32fe73ef80a334396461ceb Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Tue, 27 Oct 2026 00:21:00 +0800 Subject: [PATCH 16/25] refactor(managed): centralize TypeDef row facts across consumers --- src/dotnet/ManagedMemberInspectorCore.ts | 43 +++++----- src/dotnet/ManagedMemberTypeInventory.test.ts | 81 +++++++++++++++++++ src/dotnet/ManagedMetadataInventoryRows.ts | 9 ++- src/dotnet/ManagedTypeDefinitionRows.ts | 46 +++++++++++ src/dotnet/ManagedTypeMemberLists.ts | 38 +++++---- 5 files changed, 173 insertions(+), 44 deletions(-) create mode 100644 src/dotnet/ManagedTypeDefinitionRows.ts diff --git a/src/dotnet/ManagedMemberInspectorCore.ts b/src/dotnet/ManagedMemberInspectorCore.ts index 1027f793b3..3df701b7f2 100644 --- a/src/dotnet/ManagedMemberInspectorCore.ts +++ b/src/dotnet/ManagedMemberInspectorCore.ts @@ -23,7 +23,6 @@ import type { MetadataTableLayout, } from "./ManagedMetadataLayout.js"; import { - metadataRowCursor, readManagedCompressedUnsigned, decodeMetadataCodedIndex, metadataCodedIndexCoordinate, @@ -34,7 +33,11 @@ import { sha256Bytes, } from "./ManagedMetadataHeaps.js"; import { memberListTable } from "./ManagedMemberPointers.js"; -import type { TypeMemberListFacts } from "./ManagedTypeMemberLists.js"; +import type { + TypeMemberListEntry, + TypeMemberListFacts, +} from "./ManagedTypeMemberLists.js"; +import type { TypeDefinitionRowFacts } from "./ManagedTypeDefinitionRows.js"; export type ManagedType = ManagedMemberInspection["types"][number]; /** A renderable TypeDef projection paired with its original source row. */ @@ -72,6 +75,7 @@ export interface TypeRange { /** One TypeDef row's names and coordinates, including unreadable declarations. */ interface TypeDefinition extends TypeRange { + readonly source: TypeDefinitionRowFacts; readonly row: number; readonly name: string | undefined; readonly namespace: string | undefined; @@ -173,12 +177,10 @@ const readTypeDefinition = ( layout: ManagedMetadataLayout, row: number, issues: ManagedParseIssue[], - memberLists: TypeMemberListFacts, + current: TypeMemberListEntry, ): TypeDefinition => { - const cursor = metadataRowCursor(bytes, layout, 2, row); - cursor.readUInt32(); - const nameIndex = cursor.readIndex(layout.stringIndexSize); - const namespaceIndex = cursor.readIndex(layout.stringIndexSize); + const { source } = current; + const { nameIndex, namespaceIndex } = source; // Keep both independent name failures, but decode each heap coordinate once. const namespace = readManagedValue( () => @@ -189,9 +191,8 @@ const readTypeDefinition = ( () => readMetadataString(bytes, layout, nameIndex, layout.strings.size), issues, ); - cursor.readIndex(layout.codedIndexSize("TypeDefOrRef")); - const current = memberLists.entries[row - 1]; return { + source, row, name, namespace, @@ -200,10 +201,10 @@ const readTypeDefinition = ( namespace === undefined || name === undefined ? null : fullName(namespace, name), - fieldStart: current?.field.start.value ?? null, - fieldEnd: current?.field.end ?? null, - methodStart: current?.method.start.value ?? null, - methodEnd: current?.method.end ?? null, + fieldStart: current.field.start.value, + fieldEnd: current.field.end, + methodStart: current.method.start.value, + methodEnd: current.method.end, }; }; @@ -214,11 +215,10 @@ export const readTypeDefinitions = ( memberLists: TypeMemberListFacts, ): readonly TypeDefinition[] => { appendManagedParseIssues(issues, memberLists.issues); - const typeTable = layout.table(2); const definitions: TypeDefinition[] = []; - for (let row = 1; row <= (typeTable?.rowCount ?? 0); row += 1) + for (const [index, entry] of memberLists.entries.entries()) definitions.push( - readTypeDefinition(bytes, layout, row, issues, memberLists), + readTypeDefinition(bytes, layout, index + 1, issues, entry), ); return definitions; }; @@ -249,12 +249,7 @@ export const parseTypes = ( definition.fullName === null ) continue; - const cursor = metadataRowCursor(bytes, layout, 2, row); - const flags = cursor.readUInt32(); - cursor.readIndex(layout.stringIndexSize); - cursor.readIndex(layout.stringIndexSize); - const extendsOffset = cursor.offset; - const extendsRaw = cursor.readIndex(layout.codedIndexSize("TypeDefOrRef")); + const { flags, extendsRaw, extendsOffset, rowOffset } = definition.source; const fieldRange = rowRange( fields, definition.fieldStart, @@ -279,12 +274,12 @@ export const parseTypes = ( detail: `TypeDef Extends coded index 0x${extendsRaw.toString(16)} is invalid: ${extendsIndex.reason}`, }); const enclosingRow = nesting.enclosing.get(row); - managedDecodeBudget(layout).reserve(128, cursor.start); + managedDecodeBudget(layout).reserve(128, rowOffset); items.push({ row, type: { token, - row_offset: cursor.start, + row_offset: rowOffset, namespace, name, full_name: definition.fullName, diff --git a/src/dotnet/ManagedMemberTypeInventory.test.ts b/src/dotnet/ManagedMemberTypeInventory.test.ts index 4fd7278c73..6d12a77f00 100644 --- a/src/dotnet/ManagedMemberTypeInventory.test.ts +++ b/src/dotnet/ManagedMemberTypeInventory.test.ts @@ -295,3 +295,84 @@ it("keeps later type identity, member ownership and constructor names at their o }, ]); }); + +it("keeps independent name failures ahead of conditional Extends diagnostics", () => { + const bytes = buildManagedPeFixture({ extendsRaw: 12 }); + const readable = inspectManagedMembersBytes( + bytes, + managedPeFixtureTarget(bytes), + ); + const row = readable.types[0]?.row_offset; + if (row === undefined) throw new Error("Expected TypeDef fixture"); + expect(readable.coverage.issues).toContainEqual( + expect.objectContaining({ + code: "invalid-row", + scope: "metadata.TypeDef:0x02000001", + offset: row + 8, + detail: expect.stringContaining("Extends"), + }), + ); + bytes.writeUInt16LE(0xffff, row + 4); + bytes.writeUInt16LE(0xfffe, row + 6); + const partial = inspectManagedMembersBytes( + bytes, + managedPeFixtureTarget(bytes), + ); + expect(partial.types).toEqual([]); + expect(partial.fields).toHaveLength(1); + expect(partial.methods).toHaveLength(1); + for (const member of [...partial.fields, ...partial.methods]) + expect(member).toMatchObject({ + declaring_type_token: "0x02000001", + declaring_type: null, + }); + expect(partial.module).toEqual(readable.module); + expect(partial.coverage).toMatchObject({ + state: "partial", + issues: [ + expect.objectContaining({ + code: "invalid-heap-index", + scope: "metadata.#Strings", + }), + expect.objectContaining({ + code: "invalid-heap-index", + scope: "metadata.#Strings", + }), + ], + }); + expect( + new Set(partial.coverage.issues.map(({ offset }) => offset)).size, + ).toBe(2); +}); + +it("reads inventory names on demand without admitting TypeDef member lists or Extends", () => { + const bytes = buildManagedPeFixture({ + additionalEmptyTypes: 1, + typeName: "Lookup", + }); + const pe = readManagedPeLayout(bytes); + if (pe.cli === null) throw new Error("Expected CLI fixture"); + const layout = readManagedMetadataLayout( + bytes, + pe.rvaToOffset(pe.cli.metadata.rva, pe.cli.metadata.size, "cli.metadata"), + pe.cli.metadata.size, + ); + expect(layout.stringIndexSize).toBe(2); + const earlier = metadataRowOffset(layout, 2, 1); + bytes.writeUInt16LE(0xffff, earlier + 4); + bytes.writeUInt16LE(0xfffe, earlier + 6); + const selected = metadataRowOffset(layout, 2, 2); + bytes.writeUInt16LE(0xffff, selected + 8); + bytes.writeUInt16LE(0, selected + 10); + bytes.writeUInt16LE(0, selected + 12); + const issues: ManagedParseIssue[] = []; + const names = managedMetadataTypeNames(bytes, layout, issues); + expect(names(2, 2)).toBe("Fixture.Lookup"); + expect(issues).toEqual([]); + expect(readTypeMemberListFacts(bytes, layout).issues).toEqual( + expect.arrayContaining([ + expect.objectContaining({ code: "invalid-row", offset: selected + 10 }), + expect.objectContaining({ code: "invalid-row", offset: selected + 12 }), + ]), + ); +}); diff --git a/src/dotnet/ManagedMetadataInventoryRows.ts b/src/dotnet/ManagedMetadataInventoryRows.ts index 3638aa89c2..9c17b9f2ae 100644 --- a/src/dotnet/ManagedMetadataInventoryRows.ts +++ b/src/dotnet/ManagedMetadataInventoryRows.ts @@ -1,3 +1,4 @@ +import { readTypeDefinitionRowFacts } from "./ManagedTypeDefinitionRows.js"; import { managedDecodeBudget } from "./ManagedDecodeBudget.js"; import { managedTypeReferences } from "./ManagedTypeReferences.js"; import { MAX_TYPE_NESTING_DEPTH } from "./ManagedTypeNameDigests.js"; @@ -315,18 +316,18 @@ export const managedMetadataTypeNames = ( namespace = parsed.namespace; flags = parsed.flags; } else { - const cursor = metadataRowCursor(bytes, layout, table, row); - flags = cursor.readUInt32(); + const source = readTypeDefinitionRowFacts(bytes, layout, row); + flags = source.flags; name = readMetadataString( bytes, layout, - cursor.readIndex(layout.stringIndexSize), + source.nameIndex, layout.strings.size, ); namespace = readMetadataString( bytes, layout, - cursor.readIndex(layout.stringIndexSize), + source.namespaceIndex, layout.strings.size, ); } diff --git a/src/dotnet/ManagedTypeDefinitionRows.ts b/src/dotnet/ManagedTypeDefinitionRows.ts new file mode 100644 index 0000000000..33dc9aff17 --- /dev/null +++ b/src/dotnet/ManagedTypeDefinitionRows.ts @@ -0,0 +1,46 @@ +import type { ManagedMetadataLayout } from "./ManagedMetadataLayout.js"; +import { metadataRowCursor } from "./ManagedMetadataHeaps.js"; +import { memberListIndexSize } from "./ManagedMemberPointers.js"; + +/** Scalar TypeDef source facts; consumers own heap and coded-index admission. */ +export interface TypeDefinitionRowFacts { + readonly rowOffset: number; + readonly flags: number; + readonly nameIndex: number; + readonly namespaceIndex: number; + readonly extendsRaw: number; + readonly extendsOffset: number; + readonly fieldListRaw: number; + readonly fieldListOffset: number; + readonly methodListRaw: number; + readonly methodListOffset: number; +} + +export const readTypeDefinitionRowFacts = ( + bytes: Buffer, + layout: ManagedMetadataLayout, + row: number, +): TypeDefinitionRowFacts => { + const cursor = metadataRowCursor(bytes, layout, 2, row); + const flags = cursor.readUInt32(); + const nameIndex = cursor.readIndex(layout.stringIndexSize); + const namespaceIndex = cursor.readIndex(layout.stringIndexSize); + const extendsOffset = cursor.offset; + const extendsRaw = cursor.readIndex(layout.codedIndexSize("TypeDefOrRef")); + const fieldListOffset = cursor.offset; + const fieldListRaw = cursor.readIndex(memberListIndexSize(layout, 4)); + const methodListOffset = cursor.offset; + const methodListRaw = cursor.readIndex(memberListIndexSize(layout, 6)); + return { + rowOffset: cursor.start, + flags, + nameIndex, + namespaceIndex, + extendsRaw, + extendsOffset, + fieldListRaw, + fieldListOffset, + methodListRaw, + methodListOffset, + }; +}; diff --git a/src/dotnet/ManagedTypeMemberLists.ts b/src/dotnet/ManagedTypeMemberLists.ts index 1db5674c64..e66ea37a2f 100644 --- a/src/dotnet/ManagedTypeMemberLists.ts +++ b/src/dotnet/ManagedTypeMemberLists.ts @@ -1,13 +1,17 @@ import type { ManagedParseIssue } from "../domain/managed/managedArtifact.js"; import { managedDecodeBudget } from "./ManagedDecodeBudget.js"; import type { ManagedMetadataLayout } from "./ManagedMetadataLayout.js"; -import { metadataRowCursor, metadataToken } from "./ManagedMetadataHeaps.js"; +import { metadataToken } from "./ManagedMetadataHeaps.js"; import { - memberListIndexSize, memberListTable, validatedMemberListStart, } from "./ManagedMemberPointers.js"; +import { + readTypeDefinitionRowFacts, + type TypeDefinitionRowFacts, +} from "./ManagedTypeDefinitionRows.js"; + export type MemberDefinitionTable = 4 | 6; export interface TypeMemberListStart { @@ -23,6 +27,7 @@ export interface TypeMemberListRange { } export interface TypeMemberListEntry { + readonly source: TypeDefinitionRowFacts; readonly field: TypeMemberListRange; readonly method: TypeMemberListRange; } @@ -39,33 +44,33 @@ export const readTypeMemberListFacts = ( ): TypeMemberListFacts => { const fields = memberListTable(layout, 4); const methods = memberListTable(layout, 6); - const rows: { field: TypeMemberListStart; method: TypeMemberListStart }[] = - []; + const rows: { + source: TypeDefinitionRowFacts; + field: TypeMemberListStart; + method: TypeMemberListStart; + }[] = []; const issues: ManagedParseIssue[] = []; const types = layout.table(2); const typeCount = types?.rowCount ?? 0; - // Peak retained shape includes both the parsed-start and range arrays, their - // nested records, plus each eventual JSON object/key projection. Malformed + // Peak retained shape includes shared scalar source facts, the parsed-start + // and range arrays, plus each eventual JSON object/key projection. Malformed // row diagnostics are admitted separately before they are allocated. managedDecodeBudget(layout).reserve( - typeCount * 512 + 256, + typeCount * 640 + 256, types?.offset ?? null, ); for (let row = 1; row <= typeCount; row += 1) { - const cursor = metadataRowCursor(bytes, layout, 2, row); - cursor.readUInt32(); - cursor.readIndex(layout.stringIndexSize); - cursor.readIndex(layout.stringIndexSize); - cursor.readIndex(layout.codedIndexSize("TypeDefOrRef")); + const source = readTypeDefinitionRowFacts(bytes, layout, row); const field = { - offset: cursor.offset, - raw: cursor.readIndex(memberListIndexSize(layout, 4)), + offset: source.fieldListOffset, + raw: source.fieldListRaw, }; const method = { - offset: cursor.offset, - raw: cursor.readIndex(memberListIndexSize(layout, 6)), + offset: source.methodListOffset, + raw: source.methodListRaw, }; const entry = { + source, field: { ...field, value: validatedMemberListStart(field.raw, fields) }, method: { ...method, @@ -116,6 +121,7 @@ export const readTypeMemberListFacts = ( const entries: TypeMemberListEntry[] = rows.map((entry, index) => { const next = rows[index + 1]; return { + source: entry.source, field: { start: { ...entry.field, From 1d32eb3aa4ac75ff8b240062a2036c9da5e5ccfa Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Tue, 27 Oct 2026 00:25:22 +0800 Subject: [PATCH 17/25] fix(providers): keep operation environments consistent through launch Capture one environment snapshot per admitted WABT, JADX, firmware and Wakaru operation. Use it for tool selection, session identity and child launches while preserving configuration updates between operations. Regressions reproduce changed JVM options under an old session key, mixed engine/limiter selection and environment drift between subprocesses. Keep selected-wrapper and cleanup coverage in a shared fixture with separate scenarios. Validation: 115 distinct focused tests passed across the final five-file selection, including seven selected-command cases after refactoring. Fresh build, typecheck, lint and formatting passed. Tests use real subprocess fixtures; no real JADX/WABT/Binwalk/Wakaru engine or native Windows verification is claimed. --- src/android/JadxProvider.ts | 23 +- src/firmware/FirmwareProvider.ts | 7 +- src/javascript/recovery/WakaruProvider.ts | 7 +- src/wasm/WabtArtifactProvider.ts | 11 +- .../filesystem/wabtEnvironment.test.ts | 87 +++++ .../process/artifactSelectedCommand.test.ts | 346 ++++++++++++------ .../boundary/process/jadxIntegration.test.ts | 32 ++ 7 files changed, 394 insertions(+), 119 deletions(-) create mode 100644 tests/boundary/filesystem/wabtEnvironment.test.ts diff --git a/src/android/JadxProvider.ts b/src/android/JadxProvider.ts index 0a07217540..b1f2578f7b 100644 --- a/src/android/JadxProvider.ts +++ b/src/android/JadxProvider.ts @@ -1,3 +1,4 @@ +import { snapshotEnvironment } from "../process/snapshotEnvironment.js"; import type { AnalysisSettingsReader } from "../domain/analysisSettings.js"; import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; import { AnalysisCleanupLease } from "../artifacts/AnalysisCleanupLease.js"; @@ -295,7 +296,8 @@ export class JadxProvider implements AndroidAnalysisPort { /** Report actual local prerequisites without starting JADX or opening a target. */ inspectAvailability(signal?: AbortSignal) { - const pending = inspectJadxAvailability(this.environment, { + const environment = snapshotEnvironment(this.environment); + const pending = inspectJadxAvailability(environment, { operationTimeoutMs: this.settings?.().overrides["jadx.operation_timeout_ms"], resourceOwner: this.#prerequisiteOwner, @@ -442,10 +444,11 @@ export class JadxProvider implements AndroidAnalysisPort { request: AndroidRequest, options?: ExecutionOptions, ): Promise { + const environment = snapshotEnvironment(this.environment); // Resolve the accepted operation's override before acquiring any resource. // An unusable launch value is refused without starting the engine. const timeoutMs = resolveJadxOperationTimeout( - this.environment, + environment, request.operation, this.settings?.().overrides["jadx.operation_timeout_ms"], ); @@ -471,7 +474,7 @@ export class JadxProvider implements AndroidAnalysisPort { `Target ${target.path} is ${target.format}; provide one standalone APK.`, ); const configuration = await resolveJadxConfigurationInputs( - this.environment, + environment, request.operation, signal, ); @@ -513,11 +516,11 @@ export class JadxProvider implements AndroidAnalysisPort { java: configuration.java, arguments: configuration.jvmArguments, options: [ - this.environment.JAVA_TOOL_OPTIONS, - this.environment._JAVA_OPTIONS, - this.environment.JDK_JAVA_OPTIONS, - this.environment.PATH, - this.environment.JAVA_HOME, + environment.JAVA_TOOL_OPTIONS, + environment._JAVA_OPTIONS, + environment.JDK_JAVA_OPTIONS, + environment.PATH, + environment.JAVA_HOME, ], }); if (this.#retained !== undefined && this.#retained.key !== key) { @@ -528,7 +531,7 @@ export class JadxProvider implements AndroidAnalysisPort { if (retained === undefined) { await validateJadxConfiguration( configuration, - this.environment, + environment, request.operation, { resourceOwner: this.#prerequisiteOwner, @@ -589,7 +592,7 @@ export class JadxProvider implements AndroidAnalysisPort { bridge, ], cwd: root.path, - hostEnvironment: this.environment, + hostEnvironment: environment, signal: workspaceSignal, }, this.launcher, diff --git a/src/firmware/FirmwareProvider.ts b/src/firmware/FirmwareProvider.ts index 39b2b1782d..6fbc7cff96 100644 --- a/src/firmware/FirmwareProvider.ts +++ b/src/firmware/FirmwareProvider.ts @@ -1,3 +1,4 @@ +import { snapshotEnvironment } from "../process/snapshotEnvironment.js"; import { assertArtifactCommandPath } from "../process/ArtifactCommandPath.js"; import { join } from "node:path"; import type { FirmwareAnalysisPort } from "../application/firmware/FirmwareAnalysisPort.js"; @@ -327,6 +328,8 @@ export class FirmwareProvider implements FirmwareAnalysisPort { request: FirmwareRequest, options?: ExecutionOptions, ): Promise { + // Capture configuration after queue admission; later operations may select updates. + const environment = snapshotEnvironment(this.environment); let root: PrivateRuntimeRoot | undefined; let rootLease: AnalysisCleanupLease | undefined; const engineName = @@ -358,7 +361,7 @@ export class FirmwareProvider implements FirmwareAnalysisPort { ); }); const command = await resolveFirmwareCommand( - this.environment, + environment, engineName, request.operation, this.#files, @@ -385,7 +388,7 @@ export class FirmwareProvider implements FirmwareAnalysisPort { limiter: command.limiter.invocationPath, args, cwd: root?.path ?? "", - environment: this.environment, + environment, signal, deadline, retainCleanup: (supervisor) => { diff --git a/src/javascript/recovery/WakaruProvider.ts b/src/javascript/recovery/WakaruProvider.ts index bd050349e0..adac2b6f53 100644 --- a/src/javascript/recovery/WakaruProvider.ts +++ b/src/javascript/recovery/WakaruProvider.ts @@ -1,3 +1,4 @@ +import { snapshotEnvironment } from "../../process/snapshotEnvironment.js"; import { mkdir, mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -109,6 +110,8 @@ export class WakaruProvider implements JavaScriptRecoveryPort { ): Promise> { const pending = await this.#retire(); if (pending !== undefined) return err(pending); + // Capture configuration after queue/cleanup admission; later requests may select updates. + const environment = snapshotEnvironment(this.environment); let root: string | undefined; let tree: SafeOutputTree | undefined; let failed: { readonly cause: unknown } | undefined; @@ -117,7 +120,7 @@ export class WakaruProvider implements JavaScriptRecoveryPort { try { checkRecoveryCancellation(options?.signal); const engine = await resolveWakaruCommand( - this.environment, + environment, this.#resources, options?.signal, ); @@ -149,7 +152,7 @@ export class WakaruProvider implements JavaScriptRecoveryPort { input, sources, engine, - environment: this.environment, + environment, launcher: this.launcher, options, deadline, diff --git a/src/wasm/WabtArtifactProvider.ts b/src/wasm/WabtArtifactProvider.ts index 9c6b61ff92..420d902b98 100644 --- a/src/wasm/WabtArtifactProvider.ts +++ b/src/wasm/WabtArtifactProvider.ts @@ -45,6 +45,7 @@ import { } from "../domain/wasm/wasmArtifact.js"; import { err, ok, type Result } from "../domain/result.js"; import { runOwnedCommand } from "../process/OwnedCommand.js"; +import { snapshotEnvironment } from "../process/snapshotEnvironment.js"; import { PrivateRuntimeRoot } from "../process/PrivateRuntimeRoot.js"; import { artifactCommandOutput } from "../process/ArtifactAnalysisResources.js"; import { @@ -105,8 +106,11 @@ export class WabtArtifactProvider implements WasmArtifactPort { input: InspectWasmArtifactInput, options?: WasmArtifactExecutionOptions, ): Promise> { + const environment = snapshotEnvironment(this.environment); try { - return await this.#resources.run(() => this.#inspect(input, options)); + return await this.#resources.run(() => + this.#inspect(input, environment, options), + ); } catch (cause: unknown) { if (cause instanceof ArtifactResourceRetryFailure) return err(resourceCleanupFailure(cause)); @@ -130,6 +134,7 @@ export class WabtArtifactProvider implements WasmArtifactPort { async #inspect( input: InspectWasmArtifactInput, + environment: Readonly, options?: WasmArtifactExecutionOptions, ): Promise> { let runtime: WabtRuntime | undefined; @@ -208,7 +213,7 @@ export class WabtArtifactProvider implements WasmArtifactPort { throw new AnalysisCancelledError(OPERATION, { cause: options.signal.reason, }); - const directory = this.environment.REA_WABT_BIN_DIRECTORY ?? ""; + const directory = environment.REA_WABT_BIN_DIRECTORY ?? ""; if (!isAbsolute(directory)) throw new Error( "REA_WABT_BIN_DIRECTORY must be an absolute tool directory.", @@ -296,7 +301,7 @@ export class WabtArtifactProvider implements WasmArtifactPort { // and process-group identity own cleanup after their exec handoff. expectedCommand: null, runId: `rea-wabt-${randomUUID()}`, - hostEnvironment: this.environment, + hostEnvironment: environment, }, { timeoutMs: commandTimeoutMs, diff --git a/tests/boundary/filesystem/wabtEnvironment.test.ts b/tests/boundary/filesystem/wabtEnvironment.test.ts new file mode 100644 index 0000000000..e6b1d9a90a --- /dev/null +++ b/tests/boundary/filesystem/wabtEnvironment.test.ts @@ -0,0 +1,87 @@ +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect, it, onTestFinished } from "vitest"; +import { WabtArtifactProvider } from "../../../src/wasm/WabtArtifactProvider.js"; +import { PrivateRuntimeRoot } from "../../../src/process/PrivateRuntimeRoot.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +// Real subprocesses with WABT-shaped replies establish environment ownership, +// not the real WABT release's interpretation of these bytes. +it.skipIf(process.platform === "win32")( + "keeps one WABT inspection coherent while accepting environment changes on the next call", + async () => { + const directory = await createTestTempDirectory("rea-wabt-environment-"); + const path = join(directory, "selected.wasm"); + const log = join(directory, "children.jsonl"); + await writeFile(path, Buffer.alloc(76, 1)); + const dump = await readFile( + new URL("../../fixtures/wasm/objdump.txt", import.meta.url), + "utf8", + ); + for (const tool of ["wasm-validate", "wasm-objdump", "wasm2wat"]) { + const output = + tool === "wasm-objdump" + ? dump + : tool === "wasm2wat" + ? '(module (@custom "test" "payload"))\n' + : ""; + await writeFile( + join(directory, tool), + `#!${process.execPath} +const fs = require("node:fs"); +fs.appendFileSync(${JSON.stringify(log)}, JSON.stringify({ + marker: process.env.REA_TEST_MARKER, + empty: process.env.REA_TEST_EMPTY, + absent: Object.hasOwn(process.env, "REA_TEST_ABSENT") +}) + "\\n"); +process.stdout.write(process.argv[2] === "--version" ? "1.0.42\\n" : ${JSON.stringify(output)}); +`, + { mode: 0o700 }, + ); + } + const environment: NodeJS.ProcessEnv = { + REA_WABT_BIN_DIRECTORY: directory, + REA_TEST_MARKER: "selected-environment", + REA_TEST_EMPTY: "", + }; + let runtimeCreations = 0; + const provider = new WabtArtifactProvider(environment, undefined, () => { + runtimeCreations += 1; + // Selection already succeeded, but version probes have not launched. + if (runtimeCreations === 2) { + environment.REA_WABT_BIN_DIRECTORY = join(directory, "unselected"); + environment.REA_TEST_MARKER = "changed-environment"; + environment.REA_TEST_EMPTY = "changed"; + environment.REA_TEST_ABSENT = "added"; + } + return PrivateRuntimeRoot.create({ prefix: "rea-wabt-environment-" }); + }); + onTestFinished(() => provider.close()); + const input = { path, glue_paths: [], candidate_paths: [] }; + const first = await provider.inspect(input); + if (!first.ok) throw first.error; + + const second = await provider.inspect(input); + if (!second.ok) throw second.error; + expect(second.value.result).toEqual(first.value.result); + const children = (await readFile(log, "utf8")) + .trimEnd() + .split("\n") + .map((line) => JSON.parse(line) as unknown); + expect(children).toEqual( + Array.from({ length: 12 }, () => ({ + marker: "selected-environment", + empty: "", + absent: false, + })), + ); + expect(JSON.stringify(second.value)).not.toContain("selected-environment"); + // Later calls intentionally observe reconfiguration, not a lifetime freeze. + const third = await provider.inspect(input); + expect(third).toMatchObject({ + ok: false, + error: { _tag: "AnalysisCapabilityUnavailableError" }, + }); + expect(runtimeCreations).toBe(2); + }, +); diff --git a/tests/boundary/process/artifactSelectedCommand.test.ts b/tests/boundary/process/artifactSelectedCommand.test.ts index c2571a00c0..429997bf08 100644 --- a/tests/boundary/process/artifactSelectedCommand.test.ts +++ b/tests/boundary/process/artifactSelectedCommand.test.ts @@ -153,103 +153,132 @@ const selectedEngineLauncher = (kind: "firmware" | "wakaru"): string => { return `#!/bin/sh\nIFS= read -r role < "$0.role" || exit 19\nif [ -f "$0.retarget" ]; then\n IFS= read -r target < "$0.retarget"\n /bin/ln -sf "$target" "$0"\nfi\nif [ "$role" = limiter ]; then\n while [ "$1" != -- ]; do shift; done\n shift\n exec "$@"\nfi\nexec ${invocation} "$@"\n`; }; -// Exercise shared CLI/MCP workflows with real children and filesystem. Only -// Linux/x64 admission and /proc limits are simulated; the launcher restores the -// actual host before entering real process supervision. +// Own real wrapper processes and their workspaces. Linux/x64 admission and +// /proc limits are simulated; supervision always runs on the actual host. +const createSelectedEngineFixture = async ( + kind: "firmware" | "wakaru", + afterLaunch?: () => void, +) => { + const directory = await createTestTempDirectory("rea-selected-command-"); + const bin = join(directory, "bin"), + libexec = join(directory, "libexec"); + await mkdir(bin); + await mkdir(libexec); + const command = join(bin, "engine"), + limiter = join(bin, "prlimit"); + const commandSource = join(libexec, "engine"), + limiterSource = join(libexec, "limiter"); + const replacement = join(libexec, "replacement"); + const script = selectedEngineLauncher(kind); + for (const path of [commandSource, limiterSource, replacement]) + await writeFile(path, script, { mode: 0o755 }); + await symlink(commandSource, command); + await symlink(limiterSource, limiter); + await writeFile(`${command}.role`, `${kind}\n`); + await writeFile(`${limiter}.role`, "limiter\n"); + const input = join(directory, kind === "firmware" ? "input.bin" : "input.js"); + await writeFile( + input, + kind === "firmware" ? "fixture-input-data" : "module.exports = 42;", + ); + const platform = Object.getOwnPropertyDescriptor(process, "platform"); + const arch = Object.getOwnPropertyDescriptor(process, "arch"); + if (platform === undefined || arch === undefined) + throw new Error("Missing host descriptors"); + const restoreHost = () => { + Object.defineProperty(process, "platform", platform); + Object.defineProperty(process, "arch", arch); + }; + const launches: { + cwd?: string; + pid?: number; + command: string; + environmentMarker: string | undefined; + }[] = []; + const environment = { + ...process.env, + REA_BINWALK_COMMAND: command, + REA_FIRMWARE_PRLIMIT_COMMAND: limiter, + REA_WAKARU_COMMAND: command, + REA_JAVASCRIPT_PRLIMIT_COMMAND: limiter, + REA_SELECTED_ENVIRONMENT: "initial", + }; + const launcher: typeof spawnOwnedProviderProcess = async (options) => { + restoreHost(); + const child = await spawnOwnedProviderProcess(options); + launches.push({ + command: options.command, + environmentMarker: options.hostEnvironment?.REA_SELECTED_ENVIRONMENT, + ...(options.cwd === undefined ? {} : { cwd: options.cwd }), + ...(child.process.pid === undefined ? {} : { pid: child.process.pid }), + }); + afterLaunch?.(); + return child; + }; + const fileSizeCeiling = vi + .spyOn(FirmwareFileAccess.prototype, "fileSizeCeiling") + .mockResolvedValue(Number.MAX_SAFE_INTEGER); + onTestFinished(() => fileSizeCeiling.mockRestore()); + const service = + kind === "firmware" + ? new FirmwareAnalysisService(new FirmwareProvider(environment, launcher)) + : new JavaScriptRecoveryService( + new WakaruProvider(environment, launcher), + ); + onTestFinished(async () => { + restoreHost(); + await service.close(); + for (const launch of launches) { + if (launch.pid !== undefined) + expect(await observeProcessStartIdentity(launch.pid)).toBeUndefined(); + if (launch.cwd !== undefined) + await expect(readdir(launch.cwd)).rejects.toMatchObject({ + code: "ENOENT", + }); + } + }); + let sequence = 0; + const execute = async () => { + Object.defineProperty(process, "platform", { + ...platform, + value: "linux", + }); + Object.defineProperty(process, "arch", { ...arch, value: "x64" }); + try { + return service instanceof FirmwareAnalysisService + ? await service.execute("inspect_firmware_regions", { path: input }) + : await service.recover({ + path: input, + output_directory: join(directory, `output-${sequence++}`), + }); + } finally { + restoreHost(); + } + }; + return { + bin, + command, + limiter, + commandSource, + limiterSource, + replacement, + environment, + launches, + execute, + }; +}; + it.skipIf(process.platform === "win32").each(["firmware", "wakaru"] as const)( "executes selected %s wrappers and retains completed output when either alias changes", async (kind) => { - const directory = await createTestTempDirectory("rea-selected-command-"); - const bin = join(directory, "bin"), - libexec = join(directory, "libexec"); - await mkdir(bin); - await mkdir(libexec); - const command = join(bin, "engine"), - limiter = join(bin, "prlimit"); - const commandSource = join(libexec, "engine"), - limiterSource = join(libexec, "limiter"); - const replacement = join(libexec, "replacement"); - const script = selectedEngineLauncher(kind); - for (const path of [commandSource, limiterSource, replacement]) - await writeFile(path, script, { mode: 0o755 }); - await symlink(commandSource, command); - await symlink(limiterSource, limiter); - await writeFile(`${command}.role`, `${kind}\n`); - await writeFile(`${limiter}.role`, "limiter\n"); - const input = join( - directory, - kind === "firmware" ? "input.bin" : "input.js", - ); - await writeFile( - input, - kind === "firmware" ? "fixture-input-data" : "module.exports = 42;", - ); - const platform = Object.getOwnPropertyDescriptor(process, "platform"); - const arch = Object.getOwnPropertyDescriptor(process, "arch"); - if (platform === undefined || arch === undefined) - throw new Error("Missing host descriptors"); - const restoreHost = () => { - Object.defineProperty(process, "platform", platform); - Object.defineProperty(process, "arch", arch); - }; - const launches: { cwd?: string; pid?: number }[] = []; - const launcher: typeof spawnOwnedProviderProcess = async (options) => { - restoreHost(); - const child = await spawnOwnedProviderProcess(options); - launches.push({ - ...(options.cwd === undefined ? {} : { cwd: options.cwd }), - ...(child.process.pid === undefined ? {} : { pid: child.process.pid }), - }); - return child; - }; - const fileSizeCeiling = vi - .spyOn(FirmwareFileAccess.prototype, "fileSizeCeiling") - .mockResolvedValue(Number.MAX_SAFE_INTEGER); - onTestFinished(() => fileSizeCeiling.mockRestore()); - const service = - kind === "firmware" - ? new FirmwareAnalysisService( - new FirmwareProvider( - { - ...process.env, - REA_BINWALK_COMMAND: command, - REA_FIRMWARE_PRLIMIT_COMMAND: limiter, - }, - launcher, - ), - ) - : new JavaScriptRecoveryService( - new WakaruProvider( - { - ...process.env, - REA_WAKARU_COMMAND: command, - REA_JAVASCRIPT_PRLIMIT_COMMAND: limiter, - }, - launcher, - ), - ); - onTestFinished(async () => { - restoreHost(); - await service.close(); - }); - let sequence = 0; - const execute = async () => { - Object.defineProperty(process, "platform", { - ...platform, - value: "linux", - }); - Object.defineProperty(process, "arch", { ...arch, value: "x64" }); - try { - return service instanceof FirmwareAnalysisService - ? await service.execute("inspect_firmware_regions", { path: input }) - : await service.recover({ - path: input, - output_directory: join(directory, `output-${sequence++}`), - }); - } finally { - restoreHost(); - } - }; + const { + command, + limiter, + commandSource, + limiterSource, + replacement, + execute, + } = await createSelectedEngineFixture(kind); const first = await execute(); if (!first.ok) throw first.error; expect(first.value.normalized_result).toMatchObject({ @@ -287,14 +316,127 @@ it.skipIf(process.platform === "win32").each(["firmware", "wakaru"] as const)( selectedPath, ); } - for (const launch of launches) { - if (launch.pid !== undefined) - expect(await observeProcessStartIdentity(launch.pid)).toBeUndefined(); - if (launch.cwd !== undefined) - await expect(readdir(launch.cwd)).rejects.toMatchObject({ - code: "ENOENT", - }); - } + }, +); + +it.skipIf(process.platform === "win32")( + "keeps firmware configuration consistent within an operation and accepts later updates", + async () => { + const { + bin, + command, + limiter, + commandSource, + limiterSource, + environment, + launches, + execute, + } = await createSelectedEngineFixture("firmware"); + const nextCommand = join(bin, "next-engine"); + const nextLimiter = join(bin, "next-prlimit"); + await symlink(commandSource, nextCommand); + await symlink(limiterSource, nextLimiter); + await writeFile(`${nextCommand}.role`, "firmware\n"); + await writeFile(`${nextLimiter}.role`, "limiter\n"); + const originalHash = FirmwareFileAccess.prototype.hash; + const hash = vi.spyOn(FirmwareFileAccess.prototype, "hash"); + onTestFinished(() => hash.mockRestore()); + hash.mockImplementationOnce(async function ( + this: FirmwareFileAccess, + path, + signal, + ) { + const digest = await originalHash.call(this, path, signal); + environment.REA_BINWALK_COMMAND = nextCommand; + environment.REA_FIRMWARE_PRLIMIT_COMMAND = nextLimiter; + environment.REA_SELECTED_ENVIRONMENT = "updated"; + return digest; + }); + const first = await execute(); + if (!first.ok) throw first.error; + expect(first.value.raw_result).toMatchObject({ + version_execution: { command, resource_limiter: limiter }, + }); + expect(launches).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + command: limiter, + environmentMarker: "initial", + }), + ]), + ); + expect( + launches.every((launch) => launch.environmentMarker === "initial"), + ).toBe(true); + const next = await execute(); + if (!next.ok) throw next.error; + expect(next.value.raw_result).toMatchObject({ + version_execution: { + command: nextCommand, + resource_limiter: nextLimiter, + }, + }); + expect(launches.at(-1)).toMatchObject({ + command: nextLimiter, + environmentMarker: "updated", + }); + }, +); + +it.skipIf(process.platform === "win32")( + "keeps Wakaru configuration consistent across version and recovery launches", + async () => { + let mutateOnce = true; + const fixture = await createSelectedEngineFixture("wakaru", () => { + if (!mutateOnce) return; + mutateOnce = false; + fixture.environment.REA_WAKARU_COMMAND = nextCommand; + fixture.environment.REA_JAVASCRIPT_PRLIMIT_COMMAND = nextLimiter; + fixture.environment.REA_SELECTED_ENVIRONMENT = "updated"; + }); + const { + bin, + command, + limiter, + commandSource, + limiterSource, + launches, + execute, + } = fixture; + const nextCommand = join(bin, "next-engine"); + const nextLimiter = join(bin, "next-prlimit"); + await symlink(commandSource, nextCommand); + await symlink(limiterSource, nextLimiter); + await writeFile(`${nextCommand}.role`, "wakaru\n"); + await writeFile(`${nextLimiter}.role`, "limiter\n"); + const first = await execute(); + if (!first.ok) throw first.error; + expect(first.value.raw_result).toMatchObject({ + version: { command, resource_limiter: limiter }, + }); + expect( + launches.map(({ command, environmentMarker }) => ({ + command, + environmentMarker, + })), + ).toEqual([ + { command: limiter, environmentMarker: "initial" }, + { command: limiter, environmentMarker: "initial" }, + ]); + const next = await execute(); + if (!next.ok) throw next.error; + expect(next.value.raw_result).toMatchObject({ + version: { command: nextCommand, resource_limiter: nextLimiter }, + }); + expect( + launches.slice(2).map(({ command, environmentMarker }) => ({ + command, + environmentMarker, + })), + ).toEqual([ + { command: nextLimiter, environmentMarker: "updated" }, + { command: nextLimiter, environmentMarker: "updated" }, + ]); }, ); diff --git a/tests/boundary/process/jadxIntegration.test.ts b/tests/boundary/process/jadxIntegration.test.ts index 0fffb195bc..db13a0fbcf 100644 --- a/tests/boundary/process/jadxIntegration.test.ts +++ b/tests/boundary/process/jadxIntegration.test.ts @@ -459,6 +459,38 @@ it("retire sessions when APK bytes, engine bytes or JVM options change", async ( await verifyCleanup(launches); }); +it("keeps one operation's JVM environment aligned with its retained session identity", async () => { + const { service, apk, launches, provider, environment } = await setup(); + environment.JAVA_TOOL_OPTIONS = "-Xmx512m"; + const createRoot = PrivateRuntimeRoot.create.bind(PrivateRuntimeRoot); + const allocation = vi + .spyOn(PrivateRuntimeRoot, "create") + .mockImplementationOnce(async (options) => { + // Configuration and the reuse key are already admitted; launch follows + // this asynchronous workspace boundary. + environment.JAVA_TOOL_OPTIONS = "-Xmx8g"; + return createRoot(options); + }); + try { + const first = await service.execute("inspect_android_package", { + path: apk, + }); + if (!first.ok) throw first.error; + expect(allocation).toHaveBeenCalledOnce(); + environment.JAVA_TOOL_OPTIONS = "-Xmx512m"; + const second = await service.execute("inspect_android_package", { + path: apk, + }); + if (!second.ok) throw second.error; + expect(launches).toHaveLength(1); + expect(launches[0]?.javaToolOptions).toBe("-Xmx512m"); + } finally { + allocation.mockRestore(); + await provider.close(); + await verifyCleanup(launches); + } +}); + it("retires an idle engine and starts a fresh one for a later request", async () => { const { service, apk, launches, provider } = await setup(); vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); From c2b0c955cd9935886dc24fdb02c031f955476457 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Tue, 27 Oct 2026 00:28:31 +0800 Subject: [PATCH 18/25] fix(workers): reject malformed UTF-8 in JSON result files --- src/browser/WebBundleAnalysisWorker.test.ts | 111 +++++++++++++++++- src/browser/WebBundleAnalysisWorker.ts | 7 +- .../analysis/JavaScriptAnalysisWorker.ts | 10 +- src/sqlite/SqliteDatabaseWorkerExecution.ts | 5 +- .../workerFailureComposition.test.ts | 90 ++++++++++++++ .../sqlite/sqliteDatabaseProvider.test.ts | 69 +++++++++++ 6 files changed, 283 insertions(+), 9 deletions(-) diff --git a/src/browser/WebBundleAnalysisWorker.test.ts b/src/browser/WebBundleAnalysisWorker.test.ts index 118eccec86..47770d9732 100644 --- a/src/browser/WebBundleAnalysisWorker.test.ts +++ b/src/browser/WebBundleAnalysisWorker.test.ts @@ -78,6 +78,14 @@ type TestMode = readonly oomMarker?: string; readonly stderrMarker?: string; } + | { + readonly kind: "result"; + readonly encoding: + | "invalid-utf8" + | "replacement" + | "bom" + | "malformed-json"; + } | { readonly kind: "delayed-ready" } | { readonly kind: "exit-before-ready" } | { @@ -166,6 +174,33 @@ const protocolWorkerProgram = ( }); `; +const resultWorkerProgram = (encoding: string): string => ` + import { readFile, writeFile } from "node:fs/promises"; + import { createHash } from "node:crypto"; + import { createInterface } from "node:readline"; + import { getHeapStatistics } from "node:v8"; + const { analyzeCapturedWebScript } = await import(${JSON.stringify(new URL("../domain/webBundleAnalyzer.ts", import.meta.url).href)}); + const send = (value) => process.stdout.write(JSON.stringify(value) + "\\n"); + send({ kind: "ready", heap_limit_bytes: getHeapStatistics().heap_size_limit }); + createInterface({ input: process.stdin, crlfDelay: Infinity }).once("line", async (line) => { + const request = JSON.parse(line); + const script = JSON.parse(await readFile(request.id + ".source", "utf8")); + const facts = analyzeCapturedWebScript(script); + let bytes = Buffer.from(JSON.stringify(facts)); + const encoding = ${JSON.stringify(encoding)}; + if (encoding === "invalid-utf8") { + const index = bytes.indexOf(Buffer.from("result-marker")); + if (index < 0) throw new Error("Expected endpoint marker in schema-valid facts"); + bytes[index] = 0xff; + } else if (encoding === "bom") bytes = Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), bytes]); + else if (encoding === "malformed-json") bytes = bytes.subarray(0, bytes.length - 1); + await writeFile(request.id + ".result", bytes); + send({ kind: "complete", id: request.id, result: { + bytes: bytes.length, sha256: createHash("sha256").update(bytes).digest("hex"), + } }); + }); +`; + const delayedReadyProgram = ` import { createInterface } from "node:readline"; import { getHeapStatistics } from "node:v8"; @@ -203,7 +238,9 @@ const launcherFor = ( ? delayedReadyProgram : mode.kind === "exit-before-ready" ? exitBeforeReadyProgram - : protocolWorkerProgram(mode.failure); + : mode.kind === "result" + ? resultWorkerProgram(mode.encoding) + : protocolWorkerProgram(mode.failure); const spawned = await spawnOwnedProviderProcess({ ...request, arguments: [ @@ -926,3 +963,75 @@ it.each([ } }, ); + +it.each(["invalid-utf8", "bom", "malformed-json"] as const)( + "rejects %s worker result bytes with a matching descriptor", + async (encoding) => { + let root: string | undefined; + const worker = new WebBundleAnalysisWorker( + WEB_BUNDLE_WORKER_LIMITS.heapMb, + launcherFor( + { kind: "result", encoding }, + { + onSpawn: (spawned) => { + root = spawned.root; + }, + }, + ), + ); + try { + const result = await worker.analyze( + makeScript("fetch('/result-marker');"), + 10_000, + ); + const error = failureOf(result); + expect(error).toBeInstanceOf(AnalysisOutputError); + expect(error.cause).toBeInstanceOf( + encoding === "invalid-utf8" ? TypeError : SyntaxError, + ); + expect(error.cleanupIncomplete).toBe(false); + } finally { + await worker.close(); + } + expect(root).toBeDefined(); + await expect(access(root!)).rejects.toMatchObject({ + code: "ENOENT", + path: root, + }); + }, +); + +it("preserves literal replacement characters in valid UTF-8 worker facts", async () => { + let root: string | undefined; + const worker = new WebBundleAnalysisWorker( + WEB_BUNDLE_WORKER_LIMITS.heapMb, + launcherFor( + { kind: "result", encoding: "replacement" }, + { + onSpawn: (spawned) => { + root = spawned.root; + }, + }, + ), + ); + try { + const result = await worker.analyze( + makeScript("fetch('/literal-�');"), + 10_000, + ); + expect(result.ok).toBe(true); + if (!result.ok) throw new Error(result.error.error.message); + if (result.value.status !== "analyzed") + throw new Error("Expected analyzed worker facts"); + expect(result.value.observations.endpoints).toContainEqual( + expect.objectContaining({ value: "/literal-�" }), + ); + } finally { + await worker.close(); + } + expect(root).toBeDefined(); + await expect(access(root!)).rejects.toMatchObject({ + code: "ENOENT", + path: root, + }); +}); diff --git a/src/browser/WebBundleAnalysisWorker.ts b/src/browser/WebBundleAnalysisWorker.ts index 1413d9eef6..218beffa48 100644 --- a/src/browser/WebBundleAnalysisWorker.ts +++ b/src/browser/WebBundleAnalysisWorker.ts @@ -31,6 +31,7 @@ import { import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup.js"; import { ProviderCleanupError } from "../domain/providerCleanupError.js"; import { jsonParts, bufferedJsonParts } from "../domain/jsonSerialization.js"; +import { safeParseUtf8Json } from "../domain/safeJson.js"; import { javaScriptRuntime } from "../domain/runtimeVersion.js"; import { err, ok, type Result } from "../domain/result.js"; import type { IncludedScript } from "../domain/webBundleAnalyzerState.js"; @@ -787,9 +788,9 @@ export class WebBundleAnalysisWorker { OPERATION, "Web analyzer result differs from its descriptor", ); - const facts = webBundleScriptAnalysisSchema.parse( - JSON.parse(stored.bytes.toString("utf8")), - ); + const parsed = safeParseUtf8Json(stored.bytes); + if (!parsed.ok) throw parsed.cause; + const facts = webBundleScriptAnalysisSchema.parse(parsed.value); if (facts.status !== "analyzed" || facts.script_key !== script.script_key) throw new AnalysisOutputError( OPERATION, diff --git a/src/javascript/analysis/JavaScriptAnalysisWorker.ts b/src/javascript/analysis/JavaScriptAnalysisWorker.ts index c75c5703d0..58e87c9cf6 100644 --- a/src/javascript/analysis/JavaScriptAnalysisWorker.ts +++ b/src/javascript/analysis/JavaScriptAnalysisWorker.ts @@ -1,3 +1,4 @@ +import { safeParseUtf8Json } from "../../domain/safeJson.js"; import { analysisSettingLimit } from "../../domain/analysisSettingDefinitions.js"; import { WorkerFrameSignal } from "../../process/WorkerFrameSignal.js"; import type { @@ -719,17 +720,18 @@ export class JavaScriptAnalysisWorker { remediationAction: JAVASCRIPT_PARENT_CHECKPOINT_HEAP_REMEDIATION, }, ); - const text = await readFile(path, "utf8"); + const checkpoint = await readFile(path); if ( expected !== undefined && - createHash("sha256").update(text).digest("hex") !== expected.sha256 + createHash("sha256").update(checkpoint).digest("hex") !== expected.sha256 ) throw new AnalysisOutputError( OPERATION, "JavaScript static checkpoint digest does not match", ); - const value: unknown = JSON.parse(text); - const facts = javaScriptWorkerStaticAnalysisSchema.parse(value); + const decoded = safeParseUtf8Json(checkpoint); + if (!decoded.ok) throw decoded.cause; + const facts = javaScriptWorkerStaticAnalysisSchema.parse(decoded.value); const remaining = this.#remainingTotalResultBudget(); const retainedBytes = bytes * STATIC_CHECKPOINT_RETENTION_EXPANSION; this.#parentCapacityExhausted = retainedBytes > remaining; diff --git a/src/sqlite/SqliteDatabaseWorkerExecution.ts b/src/sqlite/SqliteDatabaseWorkerExecution.ts index 092f109503..985240a23c 100644 --- a/src/sqlite/SqliteDatabaseWorkerExecution.ts +++ b/src/sqlite/SqliteDatabaseWorkerExecution.ts @@ -1,3 +1,4 @@ +import { safeParseUtf8Json } from "../domain/safeJson.js"; import { bindArtifactFileCleanup, artifactFileCancellationCause, @@ -269,7 +270,9 @@ export const executeSqliteDatabaseWorker = async (request: { finalization = sqliteReplyCompletionFailure(cause, capturedOutput); file = completed; } - reply = replySchema.parse(JSON.parse(file.bytes.toString("utf8"))); + const decoded = safeParseUtf8Json(file.bytes); + if (!decoded.ok) throw decoded.cause; + reply = replySchema.parse(decoded.value); } catch (cause: unknown) { throw sqliteReplyFinalizationFailure( sqliteReplyReadFailure(cause, capturedOutput), diff --git a/tests/boundary/javascript/workerFailureComposition.test.ts b/tests/boundary/javascript/workerFailureComposition.test.ts index accc7b2c6d..94c855b2f7 100644 --- a/tests/boundary/javascript/workerFailureComposition.test.ts +++ b/tests/boundary/javascript/workerFailureComposition.test.ts @@ -1,4 +1,5 @@ import { createHash } from "node:crypto"; +import { access } from "node:fs/promises"; import { afterEach, expect, it, vi } from "vitest"; const faults = vi.hoisted(() => ({ @@ -185,3 +186,92 @@ it.each([false, true])( ); }, ); + +it.each([ + { malformed: true, advertised: true }, + { malformed: false, advertised: true }, + { malformed: true, advertised: false }, + { malformed: false, advertised: false }, +])( + "reads checkpoint bytes strictly with malformed=$malformed and advertised=$advertised", + async ({ malformed, advertised }) => { + const valid = Buffer.from( + JSON.stringify({ + ...staticCheckpointFacts, + limitations: ["literal � checkpoint"], + }), + ); + const replacementAt = valid.indexOf(Buffer.from("�")); + expect(replacementAt).toBeGreaterThanOrEqual(0); + const checkpoint = malformed + ? Buffer.concat([ + valid.subarray(0, replacementAt), + Buffer.from([0xff]), + valid.subarray(replacementAt + Buffer.byteLength("�")), + ]) + : valid; + let runtimeRoot: string | undefined; + const worker = new JavaScriptAnalysisWorker( + { heapMb: 128, timeoutMs: 30_000 }, + async (request) => { + runtimeRoot = request.cwd; + return spawnOwnedProviderProcess({ + ...request, + arguments: [ + "-e", + `${ready} + const fs=require("node:fs"); + const {createHash}=require("node:crypto"); + const checkpoint=Buffer.from(${JSON.stringify(checkpoint.toString("base64"))},"base64"); + require("node:readline").createInterface({input:process.stdin}).on("line",line=>{ + const request=JSON.parse(line); + if(request.file){ + fs.writeFileSync(request.id+".static.json",checkpoint); + if(!${advertised}) process.exit(2); + process.stdout.write(JSON.stringify({kind:"static",id:request.id,bytes:checkpoint.length,sha256:createHash("sha256").update(checkpoint).digest("hex")})+"\\n"); + } else process.stdout.write(JSON.stringify({kind:"complete",id:request.id,descriptor:null})+"\\n"); + });`, + ], + }); + }, + ); + try { + const result = await worker.analyze(file, { nodes: 100, unknowns: 100 }); + expect(result.ok).toBe(advertised && !malformed); + const retained = result.ok ? result.value : result.error; + if (malformed) { + expect(retained.javascript).toBeNull(); + if (result.ok) throw new Error("Expected malformed checkpoint failure"); + if (advertised) { + expect(result.error.error).toMatchObject({ + _tag: "AnalysisOutputError", + cause: expect.any(TypeError), + }); + expect(projectAnalysisError(result.error.error).code).toBe( + "unreadable_output", + ); + expect(result.error.error.message).not.toContain( + "digest does not match", + ); + } + } else { + expect(retained.javascript?.limitations).toEqual([ + "literal � checkpoint", + ]); + } + if (!advertised) { + if (result.ok) throw new Error("Expected worker crash failure"); + expect(result.error.error).toMatchObject({ + _tag: "ProviderAdapterError", + }); + } + } finally { + await worker.close(); + } + if (runtimeRoot === undefined) + throw new Error( + "Owned analyzer did not expose its runtime root to launcher", + ); + await expect(access(runtimeRoot)).rejects.toMatchObject({ code: "ENOENT" }); + }, +); diff --git a/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts b/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts index c2140ba382..74770b0126 100644 --- a/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts +++ b/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts @@ -1374,3 +1374,72 @@ nativeTest.each([ await expectPrivateRootRemoved(ownedRoot); }, ); + +nativeTest.each([ + { encoding: "malformed UTF-8", code: "unreadable_output" }, + { encoding: "literal replacement character", code: "capability_unavailable" }, + { encoding: "UTF-8 BOM", code: "unreadable_output" }, + { encoding: "malformed JSON", code: "unreadable_output" }, +] as const)( + "admits complete SQLite reply-file JSON without replacement decoding: $encoding", + async ({ encoding, code }) => { + const { database } = await createFixture(); + let ownedRoot = ""; + const provider = new SqliteDatabaseProvider(process.env, async (spawn) => { + ownedRoot = spawn.cwd ?? ""; + const requestPath = spawn.arguments[2]; + if (requestPath === undefined) + throw new Error("SQLite request path missing"); + const request = JSON.parse(await readFile(requestPath, "utf8")) as { + reply_path: string; + }; + const valid = Buffer.from( + JSON.stringify({ + ok: false, + reason: "unavailable", + message: "recorded \ufffd diagnostic", + }), + ); + let bytes = valid; + if (encoding === "malformed UTF-8") { + const replacementOffset = valid.indexOf(Buffer.from("\ufffd")); + expect(replacementOffset).toBeGreaterThanOrEqual(0); + bytes = Buffer.concat([ + valid.subarray(0, replacementOffset), + Buffer.from([0xff]), + valid.subarray(replacementOffset + 3), + ]); + } else if (encoding === "UTF-8 BOM") { + bytes = Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), valid]); + } else if (encoding === "malformed JSON") { + bytes = Buffer.from("{ malformed reply"); + } + await writeFile(request.reply_path, bytes, { flag: "wx", mode: 0o600 }); + return spawnOwnedProviderProcess({ + ...spawn, + arguments: [ + "-e", + "process.stderr.write('sqlite-reply-marker'); process.exit(0)", + ], + }); + }); + onTestFinished(() => provider.close()); + const result = await provider.inspect({ path: database.path }); + if (result.ok) throw new Error("Expected worker refusal"); + expect(projectAnalysisError(result.error).code).toBe(code); + expect(result.error.capturedOutput).toMatchObject({ + stderr: "sqlite-reply-marker", + truncated: false, + }); + if (encoding === "literal replacement character") + expect(projectAnalysisError(result.error).message).toBe( + "recorded \ufffd diagnostic", + ); + else + expect(result.error.cause).toBeInstanceOf( + encoding === "malformed UTF-8" ? TypeError : SyntaxError, + ); + await provider.close(); + await expectPrivateRootRemoved(ownedRoot); + }, +); From 0973649cec47a6a65edf74cc587fd5801c5c7302 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Tue, 27 Oct 2026 00:38:06 +0800 Subject: [PATCH 19/25] refactor(captures): consolidate recorded exchange body facts --- .../environment/environmentBuildReport.ts | 6 +- .../environmentCompareBody.test.ts | 22 ++ .../environment/environmentCompareProbe.ts | 22 +- .../environment/environmentSource.ts | 7 +- .../environmentStateWriteAnswer.test.ts | 24 +- .../environmentStateWriteReview.test.ts | 69 +++++- .../EnvironmentManifestAccumulator.ts | 84 +++---- .../EnvironmentStateAccumulator.ts | 229 +++++++++--------- .../WebNetworkApiSurfaceAccumulator.ts | 2 +- .../analysisView/recordedExchangeBody.ts | 72 ++++++ src/domain/analysisView/recordedMediaType.ts | 34 +++ .../analysisView/recordedResponseBody.ts | 41 +++- .../analysisView/webNetworkRouteTemplates.ts | 22 -- src/server/environment/stateIntake.ts | 2 +- 14 files changed, 410 insertions(+), 226 deletions(-) create mode 100644 src/domain/analysisView/recordedExchangeBody.ts create mode 100644 src/domain/analysisView/recordedMediaType.ts diff --git a/src/application/environment/environmentBuildReport.ts b/src/application/environment/environmentBuildReport.ts index 0ab133ce21..8e8750ebdc 100644 --- a/src/application/environment/environmentBuildReport.ts +++ b/src/application/environment/environmentBuildReport.ts @@ -1,14 +1,16 @@ import { basename } from "node:path"; import { isIP } from "node:net"; -import { isJsonLike } from "../../domain/analysisView/EnvironmentManifestAccumulator.js"; +import { + isJsonLike, + mediaTypeEssence, +} from "../../domain/analysisView/recordedMediaType.js"; import { decodeRetainedBase64, readRecordedResponseBody, type RecordedResponseBody, } from "../../domain/analysisView/recordedResponseBody.js"; import { scenarioContentMediaType } from "../../domain/analysisView/scenarioCaptureRecords.js"; -import { mediaTypeEssence } from "../../domain/analysisView/webNetworkRouteTemplates.js"; import type { BrowserScenarioEvent } from "../../domain/browserScenarioCaptureValues.js"; import type { EnvironmentBuildReport } from "../../domain/environmentBundleExport.js"; import { diff --git a/src/application/environment/environmentCompareBody.test.ts b/src/application/environment/environmentCompareBody.test.ts index 2535b688ce..fc3ef07848 100644 --- a/src/application/environment/environmentCompareBody.test.ts +++ b/src/application/environment/environmentCompareBody.test.ts @@ -10,6 +10,7 @@ import { environmentStateServeFixture, } from "../../../tests/fixtures/environmentServe.js"; import { compareEnvironment } from "./environmentCompare.js"; +import { requestBody } from "./environmentCompareProbe.js"; import { REPLAY_BODY_LIMIT_BYTES } from "./environmentSource.js"; const responseCapture = ( @@ -144,3 +145,24 @@ describe("environment comparison response representation", () => { expect(compare(json, text).probes[0]?.results[0]?.equal).toBe(false); }); }); + +it.each([ + { mimeType: " TEXT/PLAIN ; charset=utf-8", expected: { value: 1 } }, + { mimeType: "application/problem+json", expected: { value: 1 } }, + { mimeType: "text/problem+json", expected: '{"value":1}' }, +])( + "reads probe request arguments independently of body-size and parameter admission: $mimeType", + ({ mimeType, expected }) => { + const capture = environmentManifestCaptureFixture([ + { + request: { + method: "POST", + url: "http://app.example/api/value", + bodySize: "not-a-size", + postData: { mimeType, text: '{"value":1}', params: 7 }, + }, + }, + ]); + expect(requestBody(capture.records[0]!)).toEqual(expected); + }, +); diff --git a/src/application/environment/environmentCompareProbe.ts b/src/application/environment/environmentCompareProbe.ts index cdfb7ce423..b49a69d1af 100644 --- a/src/application/environment/environmentCompareProbe.ts +++ b/src/application/environment/environmentCompareProbe.ts @@ -1,11 +1,15 @@ import { z } from "zod"; -import { isJsonLike } from "../../domain/analysisView/EnvironmentManifestAccumulator.js"; + +import { readRecordedRequestBody } from "../../domain/analysisView/recordedExchangeBody.js"; +import { + isJsonLike, + mediaTypeEssence, +} from "../../domain/analysisView/recordedMediaType.js"; import { primaryOrigin, type OriginStats, } from "../../domain/analysisView/environmentManifestOrigin.js"; import { - mediaTypeEssence, readRecordedRequestLine, type RecordedRequestLine, type RequestLineExclusion, @@ -44,12 +48,6 @@ export interface ProbeStep { const BODYLESS_STATUSES: ReadonlySet = new Set([204, 205, 304]); const startedSchema = z.object({ startedDateTime: z.string() }); -const postDataSchema = z.object({ - request: z.object({ - postData: z.object({ mimeType: z.string(), text: z.string() }), - }), -}); - const methodSchema = z.object({ request: z.object({ method: z.string() }) }); const startedInstant = ( @@ -90,9 +88,11 @@ export const sendsBodies = ( export const requestBody = ( record: WebNetworkCaptureRecord, ): JsonValue | undefined => { - const parsed = postDataSchema.safeParse(record.reported); - if (!parsed.success) return undefined; - const { mimeType, text } = parsed.data.request.postData; + const facts = readRecordedRequestBody(record.reported); + if (facts.kind !== "request" || facts.postData.kind !== "object") + return undefined; + const { mimeType, text } = facts.postData; + if (mimeType === undefined || text === undefined) return undefined; return isJsonLike(mediaTypeEssence(mimeType)) ? (parseExactJson(text) ?? text) : text; diff --git a/src/application/environment/environmentSource.ts b/src/application/environment/environmentSource.ts index d80831b840..4c56cd6382 100644 --- a/src/application/environment/environmentSource.ts +++ b/src/application/environment/environmentSource.ts @@ -1,5 +1,10 @@ import { z } from "zod"; +import { + isJsonLike, + mediaTypeEssence, +} from "../../domain/analysisView/recordedMediaType.js"; + import { analysisViewResultSchema, type AnalysisViewResult, @@ -9,9 +14,7 @@ import { readRecordedResponseBody, type RecordedResponseBody, } from "../../domain/analysisView/recordedResponseBody.js"; -import { isJsonLike } from "../../domain/analysisView/EnvironmentManifestAccumulator.js"; import { - mediaTypeEssence, readRecordedRequestLine, type RecordedRequestLine, } from "../../domain/analysisView/webNetworkRouteTemplates.js"; diff --git a/src/application/environment/environmentStateWriteAnswer.test.ts b/src/application/environment/environmentStateWriteAnswer.test.ts index 43d98cc2b0..e4291004f1 100644 --- a/src/application/environment/environmentStateWriteAnswer.test.ts +++ b/src/application/environment/environmentStateWriteAnswer.test.ts @@ -76,6 +76,7 @@ const expectReloadsUnchanged = (books: Served): void => { describe("confirmed write response bodies", () => { it.each<{ name: string; + status?: number; content: Record; shape: "recorded" | "empty"; bodyRecorded: boolean; @@ -96,6 +97,23 @@ describe("confirmed write response bodies", () => { shape: "recorded", bodyRecorded: false, }, + { + name: "205 without retained text", + status: 205, + content: { mimeType: "text/html" }, + shape: "recorded", + bodyRecorded: false, + }, + ...[undefined, null].map((text) => ({ + name: `zero-size HTML with ${text === null ? "null" : "omitted"} text`, + content: { + mimeType: "text/html", + size: 0, + ...(text === null ? { text } : {}), + }, + shape: "recorded" as const, + bodyRecorded: false, + })), { name: "known empty text", content: { mimeType: "application/json", size: 0, text: "" }, @@ -104,7 +122,7 @@ describe("confirmed write response bodies", () => { }, ])( "preserves $name when serving a confirmed document write", - ({ content, shape, bodyRecorded }) => { + ({ content, shape, bodyRecorded, status = 200 }) => { const entries: Record[] = [ { request: { method: "GET", url: "http://app.example/api/settings" }, @@ -125,7 +143,7 @@ describe("confirmed write response bodies", () => { text: '{"theme":"dark"}', }, }, - response: { status: 200, content }, + response: { status, content }, }, { request: { method: "GET", url: "http://app.example/api/settings" }, @@ -169,7 +187,7 @@ describe("confirmed write response bodies", () => { const write = respond(patch, { body: { theme: "blue" } }); if (!write.ok) throw write.error; expect(write.value).toMatchObject({ - status: 200, + status, body: null, replay: { record_ordinal: 1, diff --git a/src/application/environment/environmentStateWriteReview.test.ts b/src/application/environment/environmentStateWriteReview.test.ts index 276aea7a11..bc00397240 100644 --- a/src/application/environment/environmentStateWriteReview.test.ts +++ b/src/application/environment/environmentStateWriteReview.test.ts @@ -26,7 +26,13 @@ import { webNetworkCaptureSchema } from "../../domain/webNetworkCapture.js"; const archived = (value: boolean) => ({ ...FIRST_BOOK, archived: value }); describe("bodyless action evidence", () => { - it.each([ + it.each<{ + name: string; + postData: unknown; + bodySize?: unknown; + served?: boolean; + shape?: boolean; + }>([ { name: "form", postData: { @@ -46,9 +52,45 @@ describe("bodyless action evidence", () => { }, }, { name: "unretained body", postData: undefined }, + { + name: "omitted body with zero size", + postData: undefined, + bodySize: 0, + served: true, + }, + { name: "explicit null with zero size", postData: null, bodySize: 0 }, + { + name: "params-only with zero size", + postData: { + mimeType: "application/json", + params: [{ name: "archived", value: "true" }], + }, + bodySize: 0, + }, + { + name: "empty text with nonempty params", + postData: { + mimeType: "application/json", + text: "", + params: [{ name: "archived", value: "true" }], + }, + bodySize: 0, + }, + { + name: "explicit empty body with zero size", + postData: { mimeType: "application/json", text: "", params: [] }, + bodySize: 0, + served: true, + }, + { + name: "invalid size with retained JSON", + postData: { mimeType: "application/json", text: '{"archived":true}' }, + bodySize: "unknown", + shape: true, + }, ])( - "does not use unreadable or unretained request bodies as a repeated bodyless action: $name", - ({ postData }) => { + "preserves literal request-body availability for repeated actions: $name", + ({ postData, bodySize = 12, served = false, shape = false }) => { const capture = timedCaptureEvidence([ ["GET", "/api/books", [archived(false), SECOND_BOOK]], ["GET", "/api/books/1", archived(false)], @@ -60,14 +102,18 @@ describe("bodyless action evidence", () => { ]); const result = capture.normalized_result as { records: { - reported: { request: { postData?: unknown; bodySize?: number } }; + reported: { request: { postData?: unknown; bodySize?: unknown } }; }[]; }; const request = result.records[5]!.reported.request; - request.bodySize = 12; + request.bodySize = bodySize; if (postData === undefined) delete request.postData; else request.postData = postData; - const { state } = environmentStateServeInputs( + if (shape) { + result.records[3]!.reported.request.bodySize = bodySize; + result.records[3]!.reported.request.postData = postData; + } + const { source, state } = environmentStateServeInputs( environmentStateServeFixture( analysisViewCaptureEvidence( webNetworkCaptureSchema.parse(capture.normalized_result), @@ -75,8 +121,17 @@ describe("bodyless action evidence", () => { ), ); expect(state.writes.get("post_books_archive_by_id")).toMatchObject({ - served: false, + served, }); + const tool = source.manifest.tools.find( + ({ name }) => name === "post_books_archive_by_id", + ); + expect(tool).toBeDefined(); + if (shape) + expect(tool?.body_shape).toMatchObject({ + properties: { archived: "boolean" }, + }); + else if (bodySize === 0) expect(tool?.body_shape).toBeNull(); }, ); }); diff --git a/src/domain/analysisView/EnvironmentManifestAccumulator.ts b/src/domain/analysisView/EnvironmentManifestAccumulator.ts index 9d0becd771..f261c31a91 100644 --- a/src/domain/analysisView/EnvironmentManifestAccumulator.ts +++ b/src/domain/analysisView/EnvironmentManifestAccumulator.ts @@ -1,4 +1,3 @@ -import { z } from "zod"; import type { AnalysisError } from "../analysisErrorBase.js"; import { jsonValueType, parseJsonObject } from "../jsonShape.js"; import { ok, type Result } from "../result.js"; @@ -11,7 +10,12 @@ import { type WebNetworkCaptureRecord, } from "../webNetworkCapture.js"; import { completeWithinViewCoverage } from "./analysisViewCoverage.js"; -import { recordedResponseBodyIsEmpty } from "./recordedResponseBody.js"; +import { + recordedResponseBodyIsEmpty, + readRecordedResponseDeclaration, +} from "./recordedResponseBody.js"; +import { readRecordedRequestBody } from "./recordedExchangeBody.js"; +import { isJson, isJsonLike, mediaTypeEssence } from "./recordedMediaType.js"; import type { UnsignedAnalysisView } from "./analysisView.js"; import type { ScenarioResponseBodyRetention } from "./scenarioCaptureRecords.js"; import { WebNetworkProjectionAccumulator } from "./WebNetworkProjectionAccumulator.js"; @@ -41,7 +45,6 @@ import { GROUP_BYTES, RECORD_SCRATCH_BYTES, SEGMENT_BYTES, - mediaTypeEssence, readQueryParameterNames, readRecordedRequestLine, routePartCount, @@ -97,24 +100,15 @@ interface GroupState extends ToolGroup { bodiless: boolean; } -const responseTextSchema = z.object({ - response: z.object({ - status: z.unknown().optional(), - content: z - .object({ text: z.unknown().optional(), size: z.unknown().optional() }) - .nullable() - .optional(), - }), -}); -/** Whether a record's response had no body: a 204 or 205, an empty text, or a recorded size of zero; a missing text alone may be a body not kept. */ -const emptyResponse = (record: WebNetworkCaptureRecord): boolean => { - const parsed = responseTextSchema.safeParse(record.reported); - if (!parsed.success) return false; - const { status, content } = parsed.data.response; +/** Manifest eligibility also accepts declared bodyless statuses and a recorded zero size when no text was retained. */ +const manifestResponseIsEmpty = (record: WebNetworkCaptureRecord): boolean => { + const response = readRecordedResponseDeclaration(record); + if (response === undefined) return false; + const { status, content } = response; return ( status === 204 || status === 205 || - recordedResponseBodyIsEmpty(record) || + recordedResponseBodyIsEmpty(record, response) || ((content?.text === undefined || content.text === null) && content?.size === 0) ); @@ -123,18 +117,6 @@ const emptyResponse = (record: WebNetworkCaptureRecord): boolean => { const answeredEmpty = (group: GroupState): boolean => group.bodiless && [...group.statuses].some((status) => status >= 200 && status < 300); -const postDataSchema = z.object({ - request: z.object({ - postData: z - .object({ - mimeType: z.unknown().optional(), - text: z.unknown().optional(), - }) - .nullable() - .optional(), - }), -}); -const JSON_LIKE_TEXT = new Set(["text/plain", "text/markdown"]); const ASSET_ESSENCES = new Set([ "text/html", "application/xhtml+xml", @@ -170,17 +152,6 @@ const UNKNOWNS: Manifest["unknowns"] = [ }, ]; -/** `application/json` or `application/*+json`; a `+json` suffix on another type is not JSON here. */ -const isJson = (essence: string | null): boolean => - essence === "application/json" || - (essence !== null && - essence.startsWith("application/") && - essence.endsWith("+json")); - -/** JSON, `+json`, `text/plain` or `text/markdown`: the content types a replay parses as JSON. */ -export const isJsonLike = (essence: string | null): boolean => - isJson(essence) || (essence !== null && JSON_LIKE_TEXT.has(essence)); - const isApiPath = (path: string): boolean => path.startsWith("/api/") || path.startsWith("/apis/"); @@ -260,9 +231,11 @@ export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumula this.scratch, ); if (line === null) return this.overBudget(); + const releaseGrouping = this.temporaryCheckpoint(); if (line.kind === "excluded") { this.#unrepresented += 1; - return ok(undefined); + releaseGrouping(); + return this.onRecordedRequestLine(record, line); } // Coverage uses the same admitted request line as grouping, including origin and raw path. if (scenarioResponseBody !== undefined) { @@ -300,28 +273,39 @@ export class EnvironmentManifestAccumulator extends WebNetworkProjectionAccumula ? this.#bodyShape(record.reported) : ok(null); if (!bodyShape.ok) return bodyShape; - return this.#append( + const grouped = this.#append( record.ordinal, line, route, queryNames, { key, group }, - { shape: bodyShape.value, empty: emptyResponse(record) }, + { shape: bodyShape.value, empty: manifestResponseIsEmpty(record) }, ); + if (!grouped.ok) return grouped; + // Keep the parsed line's admission while releasing grouping-only scratch. + releaseGrouping(); + return this.onRecordedRequestLine(record, line); } finally { this.releaseTemporary(); } } + /** Consume the owner's parsed request line after successful grouping, including exclusions. */ + protected onRecordedRequestLine( + _record: WebNetworkCaptureRecord, + _line: RecordedRequestLine, + ): Result { + return ok(undefined); + } + /** Read the top-level key types of the group's first recorded JSON-typed request body; nested values are never walked. */ #bodyShape(reported: unknown): Result { - const parsed = postDataSchema.safeParse(reported); - const body = parsed.success ? parsed.data.request.postData : undefined; + const request = readRecordedRequestBody(reported); + const body = request.kind === "request" ? request.postData : undefined; if ( - body === undefined || - body === null || - typeof body.text !== "string" || - typeof body.mimeType !== "string" + body?.kind !== "object" || + body.text === undefined || + body.mimeType === undefined ) return ok(null); // The media type is scanned and lowercased before it is known to be JSON; charge it first. diff --git a/src/domain/analysisView/EnvironmentStateAccumulator.ts b/src/domain/analysisView/EnvironmentStateAccumulator.ts index a8dd690bf8..2d0f2fce15 100644 --- a/src/domain/analysisView/EnvironmentStateAccumulator.ts +++ b/src/domain/analysisView/EnvironmentStateAccumulator.ts @@ -1,5 +1,4 @@ import { z } from "zod"; -import type { ScenarioResponseBodyRetention } from "./scenarioCaptureRecords.js"; import type { AnalysisError } from "../analysisErrorBase.js"; import { parseExactJson } from "../exactJsonNumber.js"; import { MAX_JSON_DEPTH, type JsonValue } from "../jsonValue.js"; @@ -14,10 +13,9 @@ import { type WebNetworkCaptureRecord, } from "../webNetworkCapture.js"; import type { UnsignedAnalysisView } from "./analysisView.js"; -import { - EnvironmentManifestAccumulator, - isJsonLike, -} from "./EnvironmentManifestAccumulator.js"; +import { EnvironmentManifestAccumulator } from "./EnvironmentManifestAccumulator.js"; +import { isJsonLike, mediaTypeEssence } from "./recordedMediaType.js"; +import { readRecordedRequestBody } from "./recordedExchangeBody.js"; import { EnvironmentStateExtraction } from "./EnvironmentStateExtraction.js"; import type { TemplateRule } from "./environmentManifestRules.js"; import { @@ -32,14 +30,14 @@ import { import { base64PrefixChars, readRecordedResponseBody, + readRecordedResponseDeclaration, recordedResponseBodyIsEmpty, type RecordedResponseBody, } from "./recordedResponseBody.js"; import { ENTRY_BYTES, RECORD_SCRATCH_BYTES, - mediaTypeEssence, - readRecordedRequestLine, + type RecordedRequestLine, textBytes, } from "./webNetworkRouteTemplates.js"; @@ -96,22 +94,6 @@ const redactedWriteSchema = z.object({ request: z.object({ method: z.string() }), response: z.object({ status: z.number() }).optional(), }); -const contentSchema = z.object({ - response: z.object({ content: z.object({ text: z.string() }) }), -}); -const requestSchema = z.object({ - request: z.object({ - postData: z.unknown().optional(), - bodySize: z.number().optional(), - }), -}); - -const requestContentSchema = z.object({ - mimeType: z.string(), - text: z.string(), - params: z.array(z.unknown()).optional(), -}); - interface RecordEntry { readonly facts: StateRecordFacts; /** Null for a write whose URL was redacted: its target is unknown, so it bounds every type. */ @@ -157,91 +139,83 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator return false; } - override add( + protected override onRecordedRequestLine( record: WebNetworkCaptureRecord, - scenarioResponseBody?: ScenarioResponseBodyRetention, + line: RecordedRequestLine, ): Result { - const grouped = super.add(record, scenarioResponseBody); - if (!grouped.ok) return grouped; - try { - const line = readRecordedRequestLine(record.reported, true, this.scratch); - if (line === null) return this.overBudget(); - const timing = timingSchema.safeParse(record.reported); - // Admit normalization and fraction parsing before materializing their strings. - if ( - timing.success && - !this.scratch( - RECORD_SCRATCH_BYTES + - textBytes(timing.data.startedDateTime.length + 6) * 3, - ) + const timing = timingSchema.safeParse(record.reported); + // Admit normalization and fraction parsing before materializing their strings. + if ( + timing.success && + !this.scratch( + RECORD_SCRATCH_BYTES + + textBytes(timing.data.startedDateTime.length + 6) * 3, ) - return this.overBudget(); - const startInstant = timing.success - ? readRecordedDateTime(timing.data.startedDateTime)?.instant - : undefined; - // Keep the established millisecond interval policy independently of sorting. - const start = - startInstant === undefined - ? record.ordinal - : startInstant.epochSecond * 1000 + - Number(startInstant.fraction.slice(0, 3).padEnd(3, "0")); - // A fraction substring may retain its normalized timestamp's backing store, - // including trailing zeroes removed from the exact comparison value. - const instantBytes = - timing.success && startInstant !== undefined - ? ENTRY_BYTES + textBytes(timing.data.startedDateTime.length + 6) - : 0; - if (line.kind === "excluded") - return line.reason === "url-redacted" - ? this.#redactedWrite( - record, - timing.success && startInstant !== undefined - ? { - start, - end: start + timing.data.time, - startInstant, - instantBytes, - } - : undefined, - ) - : ok(undefined); - // The entry keeps the line's strings and a reference to the already admitted record. - if ( - !this.retain( - ENTRY_BYTES * 8 + - instantBytes + - textBytes( - line.method.length + - line.origin.length + - line.path.length + - (line.query?.length ?? 0) + - line.rawMime.length, - ), - ) - ) - return this.overBudget(); - this.#entries.set(record.ordinal, { - facts: { - ordinal: record.ordinal, - method: line.method, - path: line.path, - query: line.query === "" ? undefined : line.query, - status: line.status, - start, - end: + ) + return this.overBudget(); + const startInstant = timing.success + ? readRecordedDateTime(timing.data.startedDateTime)?.instant + : undefined; + // Keep the established millisecond interval policy independently of sorting. + const start = + startInstant === undefined + ? record.ordinal + : startInstant.epochSecond * 1000 + + Number(startInstant.fraction.slice(0, 3).padEnd(3, "0")); + // A fraction substring may retain its normalized timestamp's backing store, + // including trailing zeroes removed from the exact comparison value. + const instantBytes = + timing.success && startInstant !== undefined + ? ENTRY_BYTES + textBytes(timing.data.startedDateTime.length + 6) + : 0; + if (line.kind === "excluded") + return line.reason === "url-redacted" + ? this.#redactedWrite( + record, timing.success && startInstant !== undefined - ? start + timing.data.time - : record.ordinal, - }, - origin: line.origin, - contentType: line.rawMime, - startInstant, - record, - }); - return ok(undefined); - } finally { - this.releaseTemporary(); - } + ? { + start, + end: start + timing.data.time, + startInstant, + instantBytes, + } + : undefined, + ) + : ok(undefined); + // The entry keeps the line's strings and a reference to the already admitted record. + if ( + !this.retain( + ENTRY_BYTES * 8 + + instantBytes + + textBytes( + line.method.length + + line.origin.length + + line.path.length + + (line.query?.length ?? 0) + + line.rawMime.length, + ), + ) + ) + return this.overBudget(); + this.#entries.set(record.ordinal, { + facts: { + ordinal: record.ordinal, + method: line.method, + path: line.path, + query: line.query === "" ? undefined : line.query, + status: line.status, + start, + end: + timing.success && startInstant !== undefined + ? start + timing.data.time + : record.ordinal, + }, + origin: line.origin, + contentType: line.rawMime, + startInstant, + record, + }); + return ok(undefined); } /** Keep a successful write whose URL was redacted, with an empty path whose root relates to every type. */ @@ -294,17 +268,22 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator // A body that is not JSON-like is refused before its decoding is charged. if (entry === undefined || !isJsonLike(mediaTypeEssence(entry.contentType))) return { kind: "unusable" }; - const content = contentSchema.safeParse(entry.record.reported); - if (!content.success) return { kind: "unusable" }; + const declaration = readRecordedResponseDeclaration(entry.record); + const content = declaration?.content; + if (typeof content?.text !== "string") return { kind: "unusable" }; // Larger bodies are never parsed; within the limit, decoding copies at most the retained text and the tree is linear in it. - const length = content.data.response.content.text.length; + const length = content.text.length; if (length > base64PrefixChars(STATE_BODY_LIMIT_BYTES)) return { kind: "unusable" }; if (!this.temporary(RECORD_SCRATCH_BYTES + textBytes(length) * 4)) return null; const value = stateBodyJson( entry.contentType, - readRecordedResponseBody(entry.record, STATE_BODY_LIMIT_BYTES), + readRecordedResponseBody( + entry.record, + STATE_BODY_LIMIT_BYTES, + declaration, + ), ); return value === undefined ? { kind: "unusable" } : { kind: "json", value }; } @@ -315,16 +294,30 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator use: (body: ParsedRequestBody) => T, ): T | null { const entry = this.#entries.get(ordinal); - const request = requestSchema.safeParse(entry?.record.reported); - if (entry === undefined || !request.success) + const request = readRecordedRequestBody(entry?.record.reported); + if ( + entry === undefined || + request.kind !== "request" || + request.bodySize.kind === "invalid" + ) return use({ kind: "unusable" }); - const { postData, bodySize } = request.data.request; - if (postData === undefined) - return use({ kind: bodySize === 0 ? "absent" : "unusable" }); - const parsed = requestContentSchema.safeParse(postData); - if (!parsed.success) return use({ kind: "unusable" }); - const { mimeType, text, params } = parsed.data; - if (bodySize === 0 && text === "" && (params?.length ?? 0) === 0) + const { postData, bodySize } = request; + const zeroSize = bodySize.kind === "number" && bodySize.value === 0; + if (postData.kind === "missing") + return use({ kind: zeroSize ? "absent" : "unusable" }); + if ( + postData.kind !== "object" || + postData.text === undefined || + postData.mimeType === undefined || + postData.params.kind === "invalid" + ) + return use({ kind: "unusable" }); + const { mimeType, text, params } = postData; + if ( + zeroSize && + text === "" && + (params.kind === "array" ? params.count : 0) === 0 + ) return use({ kind: "absent" }); if ( !isJsonLike(mediaTypeEssence(mimeType)) || @@ -344,7 +337,7 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator } } - /** Whether a record's response carried no body: a 204, or known decoded empty content. */ + /** State inference accepts a 204 or known decoded empty content, independently of manifest eligibility. */ #emptyResponse(ordinal: number): boolean { const entry = this.#entries.get(ordinal); if (entry === undefined) return false; diff --git a/src/domain/analysisView/WebNetworkApiSurfaceAccumulator.ts b/src/domain/analysisView/WebNetworkApiSurfaceAccumulator.ts index 03ce753985..5f603b26b3 100644 --- a/src/domain/analysisView/WebNetworkApiSurfaceAccumulator.ts +++ b/src/domain/analysisView/WebNetworkApiSurfaceAccumulator.ts @@ -1,3 +1,4 @@ +import { mediaTypeEssence } from "./recordedMediaType.js"; import type { AnalysisError } from "../analysisErrorBase.js"; import { compareUnicodeCodePoints as compare } from "../unicodeCodePointOrder.js"; import { ok, type Result } from "../result.js"; @@ -14,7 +15,6 @@ import { GROUP_BYTES, RECORD_SCRATCH_BYTES, SEGMENT_BYTES, - mediaTypeEssence, readQueryParameterNames, readRecordedRequestLine, scanRouteTemplate, diff --git a/src/domain/analysisView/recordedExchangeBody.ts b/src/domain/analysisView/recordedExchangeBody.ts new file mode 100644 index 0000000000..6e12f16a1b --- /dev/null +++ b/src/domain/analysisView/recordedExchangeBody.ts @@ -0,0 +1,72 @@ +/** Parsed declarations borrow source strings; they do not decode bodies or normalize media types. */ +export type RecordedRequestBody = + | { readonly kind: "request-unavailable" } + | { + readonly kind: "request"; + readonly bodySize: + | { readonly kind: "missing" | "invalid" } + | { readonly kind: "number"; readonly value: number }; + readonly postData: + | { readonly kind: "missing" | "null" | "malformed" } + | { + readonly kind: "object"; + readonly text: string | undefined; + readonly mimeType: string | undefined; + readonly params: + | { readonly kind: "missing" | "invalid" } + | { readonly kind: "array"; readonly count: number }; + }; + }; + +const object = (value: unknown): Record | undefined => + typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as Record) + : undefined; + +/** Read HAR request body declarations once without turning omission or null into absence. */ +export const readRecordedRequestBody = ( + reported: unknown, +): RecordedRequestBody => { + const request = object(object(reported)?.request); + if (request === undefined) return { kind: "request-unavailable" }; + const size = request.bodySize; + const bodySize: Extract< + RecordedRequestBody, + { kind: "request" } + >["bodySize"] = + size === undefined + ? { kind: "missing" } + : typeof size === "number" && Number.isFinite(size) + ? { kind: "number", value: size } + : { kind: "invalid" }; + const source = request.postData; + const postData = object(source); + return { + kind: "request", + bodySize, + postData: + postData === undefined + ? { + kind: + source === undefined + ? "missing" + : source === null + ? "null" + : "malformed", + } + : { + kind: "object", + text: typeof postData.text === "string" ? postData.text : undefined, + mimeType: + typeof postData.mimeType === "string" + ? postData.mimeType + : undefined, + params: + postData.params === undefined + ? { kind: "missing" } + : Array.isArray(postData.params) + ? { kind: "array", count: postData.params.length } + : { kind: "invalid" }, + }, + }; +}; diff --git a/src/domain/analysisView/recordedMediaType.ts b/src/domain/analysisView/recordedMediaType.ts new file mode 100644 index 0000000000..6a5f5a9590 --- /dev/null +++ b/src/domain/analysisView/recordedMediaType.ts @@ -0,0 +1,34 @@ +const MEDIA_TYPE_ESSENCE = + /^[A-Za-z0-9!#$%&'*+.^_`|~-]+\/[A-Za-z0-9!#$%&'*+.^_`|~-]+$/u; + +const JSON_LIKE_TEXT = new Set(["text/plain", "text/markdown"]); + +/** Lowercase type/subtype without parameters; malformed essences are unknown. */ +export const mediaTypeEssence = (reported: string): string | null => { + const separator = reported.indexOf(";"); + let start = 0; + let end = separator < 0 ? reported.length : separator; + // HTTP OWS is SP/HTAB. Scan from both ends so malformed internal runs of + // whitespace cannot cause repeated trailing-regex searches. + while (start < end && (reported[start] === " " || reported[start] === "\t")) + start += 1; + while ( + end > start && + (reported[end - 1] === " " || reported[end - 1] === "\t") + ) + end -= 1; + const essence = reported.slice(start, end); + // Validate ASCII tokens before case folding can turn Unicode into ASCII. + return MEDIA_TYPE_ESSENCE.test(essence) ? essence.toLowerCase() : null; +}; + +/** `application/json` or `application/*+json`; a `+json` suffix on another type is not JSON here. */ +export const isJson = (essence: string | null): boolean => + essence === "application/json" || + (essence !== null && + essence.startsWith("application/") && + essence.endsWith("+json")); + +/** JSON, `+json`, `text/plain` or `text/markdown`: the content types a replay parses as JSON. */ +export const isJsonLike = (essence: string | null): boolean => + isJson(essence) || (essence !== null && JSON_LIKE_TEXT.has(essence)); diff --git a/src/domain/analysisView/recordedResponseBody.ts b/src/domain/analysisView/recordedResponseBody.ts index 77b60ad214..810da48869 100644 --- a/src/domain/analysisView/recordedResponseBody.ts +++ b/src/domain/analysisView/recordedResponseBody.ts @@ -11,15 +11,32 @@ export interface RecordedResponseBody { readonly bodyCut?: true; } -const contentTextSchema = z.object({ +const responseDeclarationSchema = z.object({ response: z.object({ - content: z.object({ - text: z.unknown().optional(), - encoding: z.unknown().optional(), - }), + status: z.unknown().optional(), + content: z + .object({ + text: z.unknown().optional(), + encoding: z.unknown().optional(), + size: z.unknown().optional(), + }) + .nullable() + .optional(), }), }); +/** Retained response declarations shared by decoding and consumer-specific empty-body policies. */ +export const readRecordedResponseDeclaration = ( + record: WebNetworkCaptureRecord, +) => { + const parsed = responseDeclarationSchema.safeParse(record.reported); + return parsed.success ? parsed.data.response : undefined; +}; + +export type RecordedResponseDeclaration = ReturnType< + typeof readRecordedResponseDeclaration +>; + /** Strict, and keeping a leading byte-order mark, so callers see the recorded text unchanged. */ const strictUtf8 = (): TextDecoder => new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); @@ -53,10 +70,13 @@ export const decodeRetainedBase64 = ( export const readRecordedResponseBody = ( record: WebNetworkCaptureRecord, limitBytes: number, + declaration: RecordedResponseDeclaration = readRecordedResponseDeclaration( + record, + ), ): RecordedResponseBody => { - const content = contentTextSchema.safeParse(record.reported); - if (!content.success) return { bodyText: null }; - const { text, encoding } = content.data.response.content; + const content = declaration?.content; + if (content == null) return { bodyText: null }; + const { text, encoding } = content; if (typeof text !== "string") return { bodyText: null }; if (encoding === undefined) { // A hidden content property could have been the encoding declaration; @@ -80,8 +100,11 @@ export const readRecordedResponseBody = ( /** Prove decoded emptiness with a bounded prefix, preserving encoding and cut uncertainty. */ export const recordedResponseBodyIsEmpty = ( record: WebNetworkCaptureRecord, + declaration: RecordedResponseDeclaration = readRecordedResponseDeclaration( + record, + ), ): boolean => { - const body = readRecordedResponseBody(record, 0); + const body = readRecordedResponseBody(record, 0, declaration); return ( body.bodyText === "" && body.bodyEncoding === undefined && diff --git a/src/domain/analysisView/webNetworkRouteTemplates.ts b/src/domain/analysisView/webNetworkRouteTemplates.ts index b100881675..fe12bed20f 100644 --- a/src/domain/analysisView/webNetworkRouteTemplates.ts +++ b/src/domain/analysisView/webNetworkRouteTemplates.ts @@ -52,9 +52,6 @@ const responseSchema = z.object({ }), }); const contentSchema = z.object({ mimeType: z.unknown().optional() }); -const MEDIA_TYPE_ESSENCE = - /^[A-Za-z0-9!#$%&'*+.^_`|~-]+\/[A-Za-z0-9!#$%&'*+.^_`|~-]+$/u; - // Conservative representation allowances, not an enforced V8 heap limit. // Fixed charges include the accumulator, bucket/set entries, array growth and // page/schema projection. Text includes UTF-16 storage and two worst-case JSON @@ -67,25 +64,6 @@ export const CAPTURE_CONTAINER_BYTES = 512; const SEGMENT_SCRATCH_BYTES = 512; export const textBytes = (length: number): number => 32 + length * 14; -/** Lowercase type/subtype without parameters; malformed essences are unknown. */ -export const mediaTypeEssence = (reported: string): string | null => { - const separator = reported.indexOf(";"); - let start = 0; - let end = separator < 0 ? reported.length : separator; - // HTTP OWS is SP/HTAB. Scan from both ends so malformed internal runs of - // whitespace cannot cause repeated trailing-regex searches. - while (start < end && (reported[start] === " " || reported[start] === "\t")) - start += 1; - while ( - end > start && - (reported[end - 1] === " " || reported[end - 1] === "\t") - ) - end -= 1; - const essence = reported.slice(start, end); - // Validate ASCII tokens before case folding can turn Unicode into ASCII. - return MEDIA_TYPE_ESSENCE.test(essence) ? essence.toLowerCase() : null; -}; - /** Route parts of a template: one more than its separators. */ export const routePartCount = (template: string): number => { let count = 1; diff --git a/src/server/environment/stateIntake.ts b/src/server/environment/stateIntake.ts index c323fdf780..92e6cc25e1 100644 --- a/src/server/environment/stateIntake.ts +++ b/src/server/environment/stateIntake.ts @@ -1,11 +1,11 @@ import { fileURLToPath } from "node:url"; +import { mediaTypeEssence } from "../../domain/analysisView/recordedMediaType.js"; import { ArtifactResourceRetryFailure, type ArtifactResourceScope, } from "../../artifacts/ArtifactResourceScope.js"; import { STATE_INTAKE_LIMIT_BYTES } from "../../application/environment/environmentStateDocument.js"; -import { mediaTypeEssence } from "../../domain/analysisView/webNetworkRouteTemplates.js"; import { isExactJsonNumber } from "../../domain/exactJsonNumber.js"; import { err, ok, type Result } from "../../domain/result.js"; import { safeParseJson } from "../../domain/safeJson.js"; From 60b55be7e7ef666ec071b6bf9099f55c6aa75b45 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Tue, 27 Oct 2026 00:40:22 +0800 Subject: [PATCH 20/25] fix(native): bound PE export names before intermediate retention Charge PE alias names when storing them, before target interpretation can suppress output rows. Validate unused-slot names as bytes without retaining or decoding strings. Keep row and forwarder accounting separate so names are charged once. Preserve alias ordering, malformed unused-name issues and ordinal-only suppression. Add production service/provider retention regressions using compact shared-pointer fixtures. Validation: 43 focused parser, filesystem, CLI and stdio MCP tests; fresh build, typecheck, lint and formatting passed. Three bounded Node 24.22.1 measurement pairs showed median process RSS high-water 154000 to 93168 KiB; timing and sampled family memory have cache/startup/sampling limits. No Windows loader acceptance or OOM reproduction claim. --- src/native/exports/NativeExportParser.test.ts | 26 +++++++++++- src/native/exports/NativeExportParser.ts | 40 +++++++++++++------ .../boundary/filesystem/nativeExports.test.ts | 32 +++++++++++++++ .../binaryDiagnostics/nativeExports.ts | 23 +++++++++++ 4 files changed, 107 insertions(+), 14 deletions(-) diff --git a/src/native/exports/NativeExportParser.test.ts b/src/native/exports/NativeExportParser.test.ts index 2ce0c3dd9a..72e7c1b37f 100644 --- a/src/native/exports/NativeExportParser.test.ts +++ b/src/native/exports/NativeExportParser.test.ts @@ -50,7 +50,7 @@ const peExportNameAcrossSections = (overlapAtRva?: number): Buffer => { return bytes; }; -describe("PE export name mapping coverage", () => { +describe("PE export name coverage", () => { it("reads a bounded export name across adjacent section mappings", () => { const result = parse(peExportNameAcrossSections()); expect(result).toMatchObject({ @@ -82,6 +82,30 @@ describe("PE export name mapping coverage", () => { expect(result.issues).toMatchObject([{ code: "name_unreadable" }]); expect(result.coverage.status).toBe("partial"); }); + + it("validates unreadable names on unused slots without retaining exports", () => { + const result = parse( + peExportImage({ + functions: [null, { rva: 0x3000 }], + names: [ + { name: "Unreadable", index: 0 }, + { name: "Unused", index: 0 }, + { name: "Kept", index: 1 }, + ], + nameRvaOverride: { position: 0, rva: 0x9000 }, + }), + ); + + expect(result.exports.map(({ name }) => name)).toEqual(["Kept"]); + expect(result.issues).toMatchObject([ + { code: "name_unreadable", location: "0x9000" }, + ]); + expect(result.coverage).toMatchObject({ + status: "partial", + examined_entries: 2, + exports: 1, + }); + }); }); describe("PE export directories", () => { diff --git a/src/native/exports/NativeExportParser.ts b/src/native/exports/NativeExportParser.ts index 2927d214aa..c1313d0f75 100644 --- a/src/native/exports/NativeExportParser.ts +++ b/src/native/exports/NativeExportParser.ts @@ -36,13 +36,11 @@ const MAXIMUM_NAME_BYTES = 4096; /** Text retained across all rows; names can repeat one long string table entry. */ const MAXIMUM_RETAINED_TEXT_BYTES = 16 * 1024 * 1024; -/** Charge every retained row and its text before it is stored. */ +/** Charge retained text and rows at their owning storage boundary. */ const createRetention = () => { let rows = 0; let text = 0; - return (...strings: readonly (string | null)[]): void => { - rows++; - rowBudget(rows, "Export inventory"); + const retainText = (...strings: readonly (string | null)[]): void => { for (const value of strings) text += value?.length ?? 0; if (text > MAXIMUM_RETAINED_TEXT_BYTES) throw new BinaryResourceFailure( @@ -51,6 +49,14 @@ const createRetention = () => { `Export names exceed the ${String(MAXIMUM_RETAINED_TEXT_BYTES)}-byte retained-text allowance`, ); }; + return { + text: retainText, + row: (...strings: readonly (string | null)[]): void => { + rows++; + rowBudget(rows, "Export inventory"); + retainText(...strings); + }, + }; }; const hex = (value: bigint | number) => `0x${value.toString(16)}`; @@ -115,7 +121,7 @@ const parsePeExports = (bytes: Buffer, artifact: Artifact): NativeExports => { const mappings = imageFileMappings(image); const map = (rva: number, size: number): number => mappedFileOffset(mappings, base + BigInt(rva), size, image.zero_fills); - const cString = (rva: number): string => { + const cStringBytes = (rva: number): Buffer => { const value = readMappedCString( bytes, image, @@ -127,7 +133,7 @@ const parsePeExports = (bytes: Buffer, artifact: Artifact): NativeExports => { "malformed", `Export name at RVA ${hex(rva)} is empty`, ); - return value.toString("latin1"); + return value; }; const header = map(directory.address, PE_EXPORT_DIRECTORY_BYTES); @@ -145,7 +151,7 @@ const parsePeExports = (bytes: Buffer, artifact: Artifact): NativeExports => { const issues: NativeExportIssue[] = []; let dllName: string | null = null; try { - dllName = cString(reader.u32(header + 12)); + dllName = cStringBytes(reader.u32(header + 12)).toString("latin1"); } catch (cause) { issues.push({ code: "name_unreadable", @@ -154,6 +160,7 @@ const parsePeExports = (bytes: Buffer, artifact: Artifact): NativeExports => { }); } + const retain = createRetention(); const namesByFunction = new Map(); const namedFunctions = new Set(); for (let index = 0; index < nameCount; index++) { @@ -169,8 +176,13 @@ const parsePeExports = (bytes: Buffer, artifact: Artifact): NativeExports => { } namedFunctions.add(functionIndex); try { + const nameBytes = cStringBytes(nameRva); + // Validate unused slots too, but retain names only for populated slots. + if (reader.u32(functions + functionIndex * 4) === 0) continue; + const name = nameBytes.toString("latin1"); + retain.text(name); const list = namesByFunction.get(functionIndex) ?? []; - list.push(cString(nameRva)); + list.push(name); namesByFunction.set(functionIndex, list); } catch (cause) { issues.push({ @@ -182,7 +194,6 @@ const parsePeExports = (bytes: Buffer, artifact: Artifact): NativeExports => { } const forwarderEnd = directory.address + directory.size; - const retain = createRetention(); const exports: PeExport[] = []; for (let index = 0; index < functionCount; index++) { const rva = reader.u32(functions + index * 4); @@ -192,7 +203,10 @@ const parsePeExports = (bytes: Buffer, artifact: Artifact): NativeExports => { let target: Pick; if (rva >= directory.address && rva < forwarderEnd) { try { - target = { address: null, forwarder: cString(rva) }; + target = { + address: null, + forwarder: cStringBytes(rva).toString("latin1"), + }; } catch (cause) { issues.push({ code: "malformed_entry", @@ -205,12 +219,12 @@ const parsePeExports = (bytes: Buffer, artifact: Artifact): NativeExports => { const exportNames = namesByFunction.get(index); if (exportNames !== undefined) for (const name of exportNames) { - retain(name, target.forwarder); + retain.row(target.forwarder); exports.push({ name, ordinal, ...target }); } // A slot whose only name was unreadable is reported through its issue. else if (!namedFunctions.has(index)) { - retain(target.forwarder); + retain.row(target.forwarder); exports.push({ name: null, ordinal, ...target }); } } @@ -619,7 +633,7 @@ const parseElfExports = (bytes: Buffer, artifact: Artifact): NativeExports => { }); continue; } - retain( + retain.row( name, typeof version === "object" ? (version?.name ?? null) : null, rawNameBase64, diff --git a/tests/boundary/filesystem/nativeExports.test.ts b/tests/boundary/filesystem/nativeExports.test.ts index 4355ceb1f7..7afeba3a24 100644 --- a/tests/boundary/filesystem/nativeExports.test.ts +++ b/tests/boundary/filesystem/nativeExports.test.ts @@ -8,10 +8,42 @@ import { nativeExportsSchema } from "../../../src/domain/native/nativeExports.js import { elfExportImage, peExportImage, + peRepeatedExportNamesImage, } from "../../fixtures/binaryDiagnostics/nativeExports.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; describe("native export stable filesystem boundary", () => { + it("bounds retained PE aliases before interpreting their target", async () => { + const root = await createTestTempDirectory("rea-native-export-aliases-"); + const service = createNativeExportsService(); + try { + // A malformed forwarder suppresses all rows, but its names still occupy + // the intermediate inventory and must be bounded before EAT traversal. + for (const target of [0x3000, 0x1080, 0]) { + const path = join(root, `aliases-${String(target)}.dll`); + const bytes = peRepeatedExportNamesImage(target); + expect(bytes.length).toBeLessThan(32_000); + await writeFile(path, bytes); + const result = await service.inspect({ path }); + if (target === 0) { + if (!result.ok) throw new Error(JSON.stringify(result.error)); + expect(result.value.normalized_result).toMatchObject({ + exports: [], + issues: [], + coverage: { status: "complete", examined_entries: 1, exports: 0 }, + }); + } else { + expect(result).toMatchObject({ + ok: false, + error: { _tag: "AnalysisResourceConstraintError" }, + }); + } + } + } finally { + await service.close(); + } + }); + it("binds PE and ELF export inventories to the selected file", async () => { const root = await createTestTempDirectory("rea-native-exports-"); const service = createNativeExportsService(); diff --git a/tests/fixtures/binaryDiagnostics/nativeExports.ts b/tests/fixtures/binaryDiagnostics/nativeExports.ts index 16667a958e..12d02547bc 100644 --- a/tests/fixtures/binaryDiagnostics/nativeExports.ts +++ b/tests/fixtures/binaryDiagnostics/nativeExports.ts @@ -108,6 +108,29 @@ export const peExportImage = (options: { return bytes; }; +/** Compact PE whose name-table aliases all select one shared 4096-byte name. */ +export const peRepeatedExportNamesImage = ( + targetRva: number, + count = 4100, +): Buffer => { + const names = 0x500; + const ordinals = names + count * 4; + const text = ordinals + count * 2; + const bytes = Buffer.alloc(text + 4097); + peExportImage({ functions: [{ rva: targetRva }] }).copy(bytes); + const section = 152 + 240; + bytes.writeUInt32LE(bytes.length - SECTION_OFFSET, section + 8); + bytes.writeUInt32LE(bytes.length - SECTION_OFFSET, section + 16); + const rva = (offset: number): number => offset - SECTION_OFFSET + SECTION_RVA; + bytes.writeUInt32LE(count, SECTION_OFFSET + 24); + bytes.writeUInt32LE(rva(names), SECTION_OFFSET + 32); + bytes.writeUInt32LE(rva(ordinals), SECTION_OFFSET + 36); + for (let index = 0; index < count; index++) + bytes.writeUInt32LE(rva(text), names + index * 4); + bytes.fill(0x6e, text, text + 4096); + return bytes; +}; + /** One `.dynsym` entry; `version` is its raw `.gnu.version` half-word. */ export interface ElfFixtureSymbol { readonly name: string; From 3d74c480e019621f804cd4d68101cffc5d367287 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Tue, 27 Oct 2026 00:49:27 +0800 Subject: [PATCH 21/25] fix(verifiers): consolidate Ghidra MCP result admission --- scripts/verify-real-ghidra-6502.mjs | 8 +- .../verify-real-ghidra-address-context.mjs | 8 +- scripts/verify-real-ghidra-com.mjs | 18 ++- scripts/verify-real-ghidra-dos.mjs | 9 +- scripts/verify-real-ghidra-m68k.mjs | 8 +- scripts/verify-real-ghidra-nativeaot.mjs | 17 ++- scripts/verify-real-ghidra-raw.mjs | 17 ++- scripts/verify-real-ghidra-riscv.mjs | 19 +++- scripts/verify-real-ghidra-swift.mjs | 3 +- scripts/verify-real-ghidra-switch.mjs | 8 +- .../process/mcpVerifierResults.test.ts | 105 +++++++++++++++++- 11 files changed, 164 insertions(+), 56 deletions(-) diff --git a/scripts/verify-real-ghidra-6502.mjs b/scripts/verify-real-ghidra-6502.mjs index 87fb01d410..3417a9c7ff 100644 --- a/scripts/verify-real-ghidra-6502.mjs +++ b/scripts/verify-real-ghidra-6502.mjs @@ -18,7 +18,10 @@ import { promisify } from "node:util"; import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { parseEvidence } from "../dist/domain/evidence.js"; -import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs"; +import { + requireMcpOperationResult, + requireMcpOutput, +} from "./lib/mcp-verifier-results.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; if (process.argv.length !== 2) @@ -464,10 +467,9 @@ async function call(name, args = {}) { { name, arguments: args }, { timeout: 240000 }, ); - assert.notEqual(result.isError, true, JSON.stringify(result)); if (["open_binary", "binary_session", "close_binary"].includes(name)) return requireMcpOperationResult(result, name); - return result.structuredContent; + return requireMcpOutput(result, name); } function evidence(value, sha256) { diff --git a/scripts/verify-real-ghidra-address-context.mjs b/scripts/verify-real-ghidra-address-context.mjs index 432680b299..3d9cac7d4b 100644 --- a/scripts/verify-real-ghidra-address-context.mjs +++ b/scripts/verify-real-ghidra-address-context.mjs @@ -9,7 +9,10 @@ import { promisify } from "node:util"; import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { parseEvidence } from "../dist/domain/evidence.js"; -import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs"; +import { + requireMcpOperationResult, + requireMcpOutput, +} from "./lib/mcp-verifier-results.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; if (process.argv.length !== 2 || process.platform !== "linux") @@ -320,8 +323,7 @@ async function call(name, args = {}) { { name, arguments: args }, { timeout: 240000 }, ); - assert.notEqual(result.isError, true, JSON.stringify(result)); return ["open_binary", "binary_session", "close_binary"].includes(name) ? requireMcpOperationResult(result, name) - : result.structuredContent; + : requireMcpOutput(result, name); } diff --git a/scripts/verify-real-ghidra-com.mjs b/scripts/verify-real-ghidra-com.mjs index c63b3b0222..6139f633c2 100644 --- a/scripts/verify-real-ghidra-com.mjs +++ b/scripts/verify-real-ghidra-com.mjs @@ -1,6 +1,9 @@ #!/usr/bin/env node import assert from "node:assert/strict"; -import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs"; +import { + requireMcpLifecycleResult, + requireMcpOutput, +} from "./lib/mcp-verifier-results.mjs"; import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; @@ -339,16 +342,11 @@ async function call(name, args = {}) { { name, arguments: args }, { timeout: 240000 }, ); - assert.notEqual(value.isError, true, JSON.stringify(value)); - const projected = requireMcpOperationResult(value, name); - if (name === "open_binary" || name === "close_binary") return projected; - const evidence = parseEvidence(value.structuredContent); + if (["open_binary", "close_binary"].includes(name)) + return requireMcpLifecycleResult(value, name); + const evidence = parseEvidence(requireMcpOutput(value, name)); assert.equal(evidence.subject.digest.sha256, sha256); - assert.deepEqual( - evidence.normalized_result, - value.structuredContent.normalized_result, - ); - return projected; + return evidence.normalized_result; } async function cli(command, address, options = []) { const { stdout } = await promisify(execFile)( diff --git a/scripts/verify-real-ghidra-dos.mjs b/scripts/verify-real-ghidra-dos.mjs index ec49ca8499..1c685f8d97 100644 --- a/scripts/verify-real-ghidra-dos.mjs +++ b/scripts/verify-real-ghidra-dos.mjs @@ -1,6 +1,9 @@ #!/usr/bin/env node import assert from "node:assert/strict"; -import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs"; +import { + requireMcpOperationResult, + requireMcpOutput, +} from "./lib/mcp-verifier-results.mjs"; import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; import { @@ -555,7 +558,6 @@ async function call(name, arguments_ = {}) { { name, arguments: arguments_ }, { timeout: 240000 }, ); - assert.notEqual(result.isError, true, JSON.stringify(result)); assert.ok(result.structuredContent, `${name} omitted structured result`); return requireMcpOperationResult(result, name); } @@ -564,8 +566,7 @@ async function evidenceCall(name, arguments_) { { name, arguments: arguments_ }, { timeout: 240000 }, ); - assert.notEqual(result.isError, true, JSON.stringify(result)); - const evidence = parseEvidence(result.structuredContent); + const evidence = parseEvidence(requireMcpOutput(result, name)); assert.equal(evidence.subject.digest.sha256, sha256); assert.equal(evidence.provider.id, "ghidra"); assertProfile(evidence.analysis_profile); diff --git a/scripts/verify-real-ghidra-m68k.mjs b/scripts/verify-real-ghidra-m68k.mjs index 537ebdf6cc..17d9b26b65 100644 --- a/scripts/verify-real-ghidra-m68k.mjs +++ b/scripts/verify-real-ghidra-m68k.mjs @@ -16,7 +16,10 @@ import { promisify } from "node:util"; import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { parseEvidence } from "../dist/domain/evidence.js"; -import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs"; +import { + requireMcpOperationResult, + requireMcpOutput, +} from "./lib/mcp-verifier-results.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; if (process.argv.length !== 2) @@ -281,10 +284,9 @@ async function call(name, args = {}) { { name, arguments: args }, { timeout: 240000 }, ); - assert.notEqual(result.isError, true, JSON.stringify(result)); if (["open_binary", "binary_session", "close_binary"].includes(name)) return requireMcpOperationResult(result, name); - return result.structuredContent; + return requireMcpOutput(result, name); } function evidence(value) { diff --git a/scripts/verify-real-ghidra-nativeaot.mjs b/scripts/verify-real-ghidra-nativeaot.mjs index e6280f601a..fb5064613f 100644 --- a/scripts/verify-real-ghidra-nativeaot.mjs +++ b/scripts/verify-real-ghidra-nativeaot.mjs @@ -1,6 +1,9 @@ #!/usr/bin/env node import assert from "node:assert/strict"; -import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs"; +import { + requireMcpLifecycleResult, + requireMcpOutput, +} from "./lib/mcp-verifier-results.mjs"; import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; import { @@ -394,14 +397,8 @@ async function call(name, args = {}) { { name, arguments: args }, { timeout: 360000 }, ); - assert.notEqual(response.isError, true, JSON.stringify(response)); - const projected = requireMcpOperationResult(response, name); - if (name === "close_binary") return projected; - const evidence = parseEvidence(response.structuredContent); + if (name === "close_binary") return requireMcpLifecycleResult(response, name); + const evidence = parseEvidence(requireMcpOutput(response, name)); assert.equal(evidence.subject.digest.sha256, sha256); - assert.deepEqual( - evidence.normalized_result, - response.structuredContent.normalized_result, - ); - return projected; + return evidence.normalized_result; } diff --git a/scripts/verify-real-ghidra-raw.mjs b/scripts/verify-real-ghidra-raw.mjs index 549da44456..1b09cb60c1 100644 --- a/scripts/verify-real-ghidra-raw.mjs +++ b/scripts/verify-real-ghidra-raw.mjs @@ -12,7 +12,10 @@ import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { parseEvidence } from "../dist/domain/evidence.js"; import { inspectVerifierGhidraInstallation } from "./lib/ghidra-installation-inspection.mjs"; -import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs"; +import { + requireMcpLifecycleResult, + requireMcpOutput, +} from "./lib/mcp-verifier-results.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; if (process.argv.length !== 2) @@ -351,15 +354,9 @@ async function call(name, args = {}, targetSha256 = sha256) { { name, arguments: args }, { timeout: 240000 }, ); - assert.notEqual(result.isError, true, JSON.stringify(result)); - const projected = requireMcpOperationResult(result, name); - if ( - name === "open_binary" || - name === "close_binary" || - name === "binary_session" - ) - return projected; - const evidence = parseEvidence(result.structuredContent); + if (["open_binary", "close_binary", "binary_session"].includes(name)) + return requireMcpLifecycleResult(result, name); + const evidence = parseEvidence(requireMcpOutput(result, name)); assert.equal(evidence.subject.digest.sha256, targetSha256); assert.equal(evidence.subject.format, "raw-binary"); return evidence; diff --git a/scripts/verify-real-ghidra-riscv.mjs b/scripts/verify-real-ghidra-riscv.mjs index 766b064434..29e67d88fa 100644 --- a/scripts/verify-real-ghidra-riscv.mjs +++ b/scripts/verify-real-ghidra-riscv.mjs @@ -26,7 +26,8 @@ import { inspectVerifierGhidraInstallation } from "./lib/ghidra-installation-ins import { buildRiscvFixture } from "../tests/conformance/ghidra/riscv-fixture.mjs"; import { mcpTextValue, - requireMcpOperationResult, + requireMcpLifecycleResult, + requireMcpOutput, requireMcpToolError, } from "./lib/mcp-verifier-results.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; @@ -141,18 +142,24 @@ try { { name, arguments: args }, { timeout: 240000 }, ); - const projected = requireMcpOperationResult(reply, name); + const lifecycle = [ + "open_binary", + "close_binary", + "binary_session", + ].includes(name); + const admitted = lifecycle + ? requireMcpLifecycleResult(reply, name) + : parseEvidence(requireMcpOutput(reply, name)); const textual = JSON.parse(mcpTextValue(reply)); - assert.deepEqual(reply.structuredContent, textual); + assert.ok(reply.structuredContent, `${name} omitted structured result`); const validate = validators.get(name); assert.ok(validate, `${name} missing from advertised catalog`); assert.ok( validate(textual), `${name}: ${JSON.stringify(validate.errors)}`, ); - if (!["open_binary", "close_binary", "binary_session"].includes(name)) - assertEvidence(parseEvidence(textual)); - return projected; + if (!lifecycle) assertEvidence(admitted); + return lifecycle ? admitted : admitted.normalized_result; }; const opened = await call("open_binary", { path: fixture.path, diff --git a/scripts/verify-real-ghidra-swift.mjs b/scripts/verify-real-ghidra-swift.mjs index 09d92c4fb1..7b2c02e04f 100644 --- a/scripts/verify-real-ghidra-swift.mjs +++ b/scripts/verify-real-ghidra-swift.mjs @@ -21,7 +21,6 @@ import { ProviderProcessSupervisor, spawnOwnedProviderProcess, } from "../dist/process/ProviderProcess.js"; -import { toolContract } from "../dist/contracts/toolContracts.js"; import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; @@ -215,7 +214,7 @@ async function call(name, arguments_) { { timeout: 180_000 }, ); if (name === "analyze_swift_types") - toolContract(name).outputSchema.parse(response.structuredContent); + assert.ok(response.structuredContent, `${name} omitted structured result`); return requireMcpOperationResult(response, name); } diff --git a/scripts/verify-real-ghidra-switch.mjs b/scripts/verify-real-ghidra-switch.mjs index 5d43c2bc4c..5797969234 100644 --- a/scripts/verify-real-ghidra-switch.mjs +++ b/scripts/verify-real-ghidra-switch.mjs @@ -1,6 +1,9 @@ #!/usr/bin/env node import assert from "node:assert/strict"; -import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs"; +import { + requireMcpOperationResult, + requireMcpOutput, +} from "./lib/mcp-verifier-results.mjs"; import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; import { @@ -326,9 +329,8 @@ async function toolResult(client, name, arguments_ = {}) { { name, arguments: arguments_ }, { timeout: 240000 }, ); - assert.notEqual(result.isError, true, JSON.stringify(result)); assert.ok(result.structuredContent, `${name} omitted structured result`); - return result.structuredContent; + return requireMcpOutput(result, name); } async function call(client, name, arguments_ = {}) { const result = await client.callTool( diff --git a/tests/boundary/process/mcpVerifierResults.test.ts b/tests/boundary/process/mcpVerifierResults.test.ts index 1f0e4d5619..69c9ced8b5 100644 --- a/tests/boundary/process/mcpVerifierResults.test.ts +++ b/tests/boundary/process/mcpVerifierResults.test.ts @@ -1,6 +1,12 @@ import { toolContract } from "../../../src/contracts/toolContracts.js"; +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import { runInNewContext } from "node:vm"; +import { parse } from "@babel/parser"; import type { ToolContract } from "../../../src/contracts/toolContractTypes.js"; -import { createEvidence } from "../../../src/domain/evidence.js"; +import { createAnalysisProfile } from "../../../src/domain/analysisProfile.js"; +import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; +import { SUPPORTED_GHIDRA_VERSION } from "../../../src/ghidra/GhidraReleasePolicy.js"; import { createEvidenceBundle, type EvidenceBundle, @@ -8,6 +14,10 @@ import { import { expect, it } from "vitest"; interface VerifierResults { + readonly requireMcpOperationResult: ( + result: unknown, + operation: string, + ) => unknown; readonly requireMcpOutput: ( result: unknown, operation: string, @@ -207,6 +217,37 @@ const namedEvidence = () => rawResult: { engine_name: "entry", source: "symbol table" }, }, ); +const ghidraEvidence = () => { + const provider = { + id: "ghidra", + name: "Ghidra", + version: SUPPORTED_GHIDRA_VERSION, + }; + return createEvidence( + { + path: "/fixture/ghidra-image.elf", + sha256: "b".repeat(64), + format: "elf", + architecture: "x86_64", + }, + provider, + { + operation: "address_name", + parameters: { address: "0x401000" }, + result: "entry", + rawResult: { + address: "00401000", + symbol: "entry", + source: "Ghidra symbol table", + }, + analysisProfile: createAnalysisProfile(provider, { + loader: "ElfLoader", + language_id: "x86:LE:64:default", + compiler_spec_id: "gcc", + }), + }, + ); +}; const textReply = (value: unknown) => ({ content: [{ type: "text", text: JSON.stringify(value) }], }); @@ -241,6 +282,66 @@ it("admits matching complete text and structured Evidence and rejects either mal ).toThrow(); }); +it.each([false, true])( + "keeps Ghidra contract admission separate from Evidence authentication (structured=%s)", + async (structured) => { + const { requireMcpOutput } = await verifierResults(); + const evidence = ghidraEvidence(); + const reply = (value: unknown) => ({ + ...textReply(value), + ...(structured ? { structuredContent: value } : {}), + }); + const admitted = requireMcpOutput(reply(evidence), "address_name"); + expect(admitted).toEqual(evidence); + const authenticated = parseEvidence(admitted); + expect(authenticated).toEqual(evidence); + expect(authenticated.analysis_profile?.provider).toEqual(evidence.provider); + expect(authenticated.subject).toEqual(evidence.subject); + const tampered = { ...evidence, normalized_result: "different_entry" }; + const shapeValid = requireMcpOutput(reply(tampered), "address_name"); + expect(shapeValid).toEqual(tampered); + expect(() => parseEvidence(shapeValid)).toThrow(/semantic identifier/u); + }, +); + +it("rejects malformed and disagreeing replies at the Ghidra address-context verifier call boundary", async () => { + const source = await readFile( + new URL( + "../../../scripts/verify-real-ghidra-address-context.mjs", + import.meta.url, + ), + "utf8", + ); + const declaration = parse(source, { sourceType: "module" }).program.body.find( + (node) => node.type === "FunctionDeclaration" && node.id?.name === "call", + ); + if (declaration?.start == null || declaration.end == null) + throw new Error("Ghidra verifier call boundary missing"); + const evidence = ghidraEvidence(); + let response: unknown; + // Execute the script's actual call boundary without launching its compiler or provider. + const call = runInNewContext( + `(${source.slice(declaration.start, declaration.end)})`, + { + assert, + client: { callTool: async () => response }, + ...(await verifierResults()), + }, + ) as (name: string) => Promise; + response = { ...textReply(evidence), structuredContent: evidence }; + await expect(call("address_name")).resolves.toEqual(evidence); + const malformed = { ...evidence, normalized_result: { name: "entry" } }; + response = { ...textReply(malformed), structuredContent: malformed }; + await expect(call("address_name")).rejects.toThrow(); + response = { + ...textReply(evidence), + structuredContent: { ...evidence, normalized_result: "different_entry" }, + }; + await expect(call("address_name")).rejects.toThrow( + "text and structured results disagree", + ); +}); + it("rejects disagreement in lifecycle envelope fields that the named schema would otherwise discard", async () => { const { requireMcpOutput } = await verifierResults(); const result = retainedBundle(); @@ -260,7 +361,7 @@ it("rejects disagreement in lifecycle envelope fields that the named schema woul it("rejects disagreement in complete Evidence provenance before downstream extraction", async () => { const { requireMcpOutput } = await verifierResults(); - const evidence = namedEvidence(); + const evidence = ghidraEvidence(); expect(() => requireMcpOutput( { From 74d5cab372150fe2d816980ef34ef315c5f9f1fa Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:01:10 +0800 Subject: [PATCH 22/25] fix(native): account export and version text in UTF-8 bytes Count retained UTF-8 text rather than UTF-16 code units, including exact-byte base64 spellings. Bound the decoded version-name map independently from output expansion, releasing replaced entries from its live storage charge. Add real service/provider boundary controls for accepted and over-budget symbol names, version names and unused version metadata. Existing per-string bounds remain the allocation owner. Validation: native parser, filesystem and CLI/MCP views passed; build, typecheck, lint and formatting passed. No serialized-report size cap, OOM reproduction or performance improvement is claimed. --- src/native/exports/NativeExportParser.ts | 44 +++++++-- .../boundary/filesystem/nativeExports.test.ts | 98 +++++++++++++++++++ 2 files changed, 134 insertions(+), 8 deletions(-) diff --git a/src/native/exports/NativeExportParser.ts b/src/native/exports/NativeExportParser.ts index c1313d0f75..5302adc06a 100644 --- a/src/native/exports/NativeExportParser.ts +++ b/src/native/exports/NativeExportParser.ts @@ -36,18 +36,29 @@ const MAXIMUM_NAME_BYTES = 4096; /** Text retained across all rows; names can repeat one long string table entry. */ const MAXIMUM_RETAINED_TEXT_BYTES = 16 * 1024 * 1024; +const retainedTextBytes = (...strings: readonly (string | null)[]): number => + strings.reduce( + (bytes, value) => + bytes + (value === null ? 0 : Buffer.byteLength(value, "utf8")), + 0, + ); + +const checkRetainedText = (bytes: number): void => { + if (bytes > MAXIMUM_RETAINED_TEXT_BYTES) + throw new BinaryResourceFailure( + "structure", + MAXIMUM_RETAINED_TEXT_BYTES, + `Export names exceed the ${String(MAXIMUM_RETAINED_TEXT_BYTES)}-byte retained-text allowance`, + ); +}; + /** Charge retained text and rows at their owning storage boundary. */ const createRetention = () => { let rows = 0; let text = 0; const retainText = (...strings: readonly (string | null)[]): void => { - for (const value of strings) text += value?.length ?? 0; - if (text > MAXIMUM_RETAINED_TEXT_BYTES) - throw new BinaryResourceFailure( - "structure", - MAXIMUM_RETAINED_TEXT_BYTES, - `Export names exceed the ${String(MAXIMUM_RETAINED_TEXT_BYTES)}-byte retained-text allowance`, - ); + text += retainedTextBytes(...strings); + checkRetainedText(text); }; return { text: retainText, @@ -316,8 +327,25 @@ const readVersionNames = ( issues: NativeExportIssue[], ): Map => { const namesByIndex = new Map(); + // This intermediate map owns its storage independently of repeated text in + // output rows. Replacing an index releases its previous decoded name. + let textBytes = 0; const addName = (index: number, name: Buffer): void => { - namesByIndex.set(index, name.length === 0 ? null : decodeElfString(name)); + const decoded = name.length === 0 ? null : decodeElfString(name); + const previous = namesByIndex.get(index); + const nextTextBytes = + textBytes - + retainedTextBytes( + previous?.name ?? null, + previous?.name_bytes_base64 ?? null, + ) + + retainedTextBytes( + decoded?.name ?? null, + decoded?.name_bytes_base64 ?? null, + ); + checkRetainedText(nextTextBytes); + namesByIndex.set(index, decoded); + textBytes = nextTextBytes; }; const verneed = sections.find(({ type }) => type === SHT_GNU_VERNEED); if (verneed !== undefined) { diff --git a/tests/boundary/filesystem/nativeExports.test.ts b/tests/boundary/filesystem/nativeExports.test.ts index 7afeba3a24..a4e98b597f 100644 --- a/tests/boundary/filesystem/nativeExports.test.ts +++ b/tests/boundary/filesystem/nativeExports.test.ts @@ -87,6 +87,104 @@ describe("native export stable filesystem boundary", () => { }); }); +describe("native export text capacity", () => { + it.each(["symbol", "version"] as const)( + "bounds UTF-8 bytes of retained ELF %s text and its exact-byte spelling", + async (kind) => { + const root = await createTestTempDirectory("rea-native-export-text-"); + const service = createNativeExportsService(); + const text = "€".repeat(20_000); + const rawName = Buffer.from(text, "utf8").toString("latin1"); + try { + for (const count of [119, 160]) { + const path = join(root, `${kind}-${String(count)}.so`); + const bytes = elfExportImage({ + symbols: Array.from({ length: count }, () => + kind === "symbol" ? { name: rawName } : { name: "a", version: 2 }, + ), + ...(kind === "version" + ? { versionDefinitions: ["lib.so", rawName] } + : {}), + }); + expect(bytes.length).toBeLessThan(70_000); + await writeFile(path, bytes); + const result = await service.inspect({ path }); + if (count === 160) { + expect(result).toMatchObject({ + ok: false, + error: { _tag: "AnalysisResourceConstraintError" }, + }); + continue; + } + if (!result.ok) throw new Error(JSON.stringify(result.error)); + const report = nativeExportsSchema.parse( + result.value.normalized_result, + ); + if (report.format !== "elf") throw new Error("Expected ELF report"); + expect(report.coverage).toMatchObject({ + status: "complete", + exports: count, + }); + const row = report.exports[0]!; + const name = kind === "symbol" ? row : row.version; + expect(name).toMatchObject({ + name: text, + name_bytes_base64: Buffer.from(text, "utf8").toString("base64"), + name_utf8_decodable: true, + }); + } + } finally { + await service.close(); + } + }, + ); + + it("bounds intermediate ELF version names even when no symbols use them", async () => { + const root = await createTestTempDirectory("rea-native-export-versions-"); + const service = createNativeExportsService(); + try { + for (const count of [279, 280]) { + const path = join(root, `versions-${String(count)}.so`); + const bytes = elfExportImage({ + symbols: [], + versionDefinitions: Array(count).fill("v".repeat(60_000)), + // Definitions replace requirements at the same indices; released + // names must not remain charged against the live map's allowance. + ...(count === 279 + ? { + versionRequirements: Array.from( + { length: count }, + (_, index) => ({ + index: index + 1, + name: "w".repeat(60_000), + }), + ), + } + : {}), + }); + expect(bytes.length).toBeLessThan(140_000); + await writeFile(path, bytes); + const result = await service.inspect({ path }); + if (count === 280) + expect(result).toMatchObject({ + ok: false, + error: { _tag: "AnalysisResourceConstraintError" }, + }); + else { + if (!result.ok) throw new Error(JSON.stringify(result.error)); + expect(result.value.normalized_result).toMatchObject({ + exports: [], + issues: [], + coverage: { status: "complete", exports: 0 }, + }); + } + } + } finally { + await service.close(); + } + }); +}); + describe("native export encoding and failures", () => { it("keeps ELF symbol bytes while exposing valid UTF-8 text", async () => { const root = await createTestTempDirectory("rea-native-export-unicode-"); From 9d2c087d8306803c232286e6e3f05fd6acc7a8ad Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:01:19 +0800 Subject: [PATCH 23/25] fix(readers): preserve completed snapshots through cleanup retries Retain exact completed file path, digest and length in Python pyc and Erlang BEAM failure observations. Translate Python close failures at the provider boundary and bind them to the retained descriptor cleanup leases so later successful retries resolve prior errors. Exercise actual services and file handles with completed reads, cancellation during finalization and physical read failure followed by cancellation. Assert no fabricated snapshot on early failure and resolution of every prior cleanup report. Validation: six focused descriptor regressions plus existing filesystem, parser and MCP coverage passed; build and static checks passed. Authored format fixtures establish portable reader behavior, not CPython/Erlang runtime execution. --- src/erlang/BeamProvider.ts | 16 +- src/python/PythonPycProvider.ts | 36 +++- .../filesystem/pythonBeamCleanup.test.ts | 154 ++++++++++++++++++ 3 files changed, 202 insertions(+), 4 deletions(-) create mode 100644 tests/boundary/filesystem/pythonBeamCleanup.test.ts diff --git a/src/erlang/BeamProvider.ts b/src/erlang/BeamProvider.ts index 637f7c9944..28ed4648c2 100644 --- a/src/erlang/BeamProvider.ts +++ b/src/erlang/BeamProvider.ts @@ -1,5 +1,8 @@ import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; -import { readStableArtifact } from "../artifacts/readStableArtifact.js"; +import { + completedStableArtifact, + readStableArtifact, +} from "../artifacts/readStableArtifact.js"; import { artifactFilePrimaryFacts, type ArtifactFilePrimaryFacts, @@ -9,6 +12,7 @@ import { } from "../artifacts/ArtifactFileOperation.js"; import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup.js"; +import { fileSnapshotPartialObservationSchema } from "../domain/fileSnapshotObservation.js"; import type { BeamPort } from "../application/erlang/BeamPort.js"; import type { ExecutionOptions } from "../application/AnalysisProvider.js"; import { ArtifactOperationError } from "../domain/artifactOperationError.js"; @@ -203,6 +207,16 @@ const beamFailurePrimary = ( const beamFailure = (cause: unknown, input: InspectErlangBeamInput) => { const primary = beamFailurePrimary(artifactFilePrimaryFacts(cause), input); + const snapshot = completedStableArtifact(cause); + if (snapshot !== undefined) + primary.retainPartialObservation( + fileSnapshotPartialObservationSchema.parse({ + kind: "file-snapshot-hash", + path: input.path, + sha256: snapshot.sha256, + bytes: snapshot.bytes.length, + }), + ); if (!(cause instanceof ArtifactReaderFailure) || cause.cleanup === undefined) return primary; const cleanup = new ProviderCleanupError( diff --git a/src/python/PythonPycProvider.ts b/src/python/PythonPycProvider.ts index b463bb8186..461fc9e12a 100644 --- a/src/python/PythonPycProvider.ts +++ b/src/python/PythonPycProvider.ts @@ -6,9 +6,13 @@ import { bindArtifactFileCleanup, } from "../artifacts/ArtifactFileOperation.js"; import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; -import { readStableArtifact } from "../artifacts/readStableArtifact.js"; +import { + completedStableArtifact, + readStableArtifact, +} from "../artifacts/readStableArtifact.js"; import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup.js"; +import { fileSnapshotPartialObservationSchema } from "../domain/fileSnapshotObservation.js"; import type { PythonPycPort } from "../application/python/PythonPycPort.js"; import type { ExecutionOptions } from "../application/AnalysisProvider.js"; import { AnalysisError } from "../domain/analysisErrorBase.js"; @@ -164,6 +168,16 @@ const failureForPrimary = ( const failureFor = (cause: unknown, input: InspectPythonPycInput) => { const primary = failureForPrimary(artifactFilePrimaryFacts(cause), input); + const snapshot = completedStableArtifact(cause); + if (snapshot !== undefined) + primary.retainPartialObservation( + fileSnapshotPartialObservationSchema.parse({ + kind: "file-snapshot-hash", + path: input.path, + sha256: snapshot.sha256, + bytes: snapshot.bytes.length, + }), + ); if (!(cause instanceof ArtifactReaderFailure) || cause.cleanup === undefined) return primary; const cleanup = new ProviderCleanupError( @@ -182,8 +196,24 @@ export class PythonPycProvider implements PythonPycPort { readonly identity = PYTHON_PYC_PROVIDER_IDENTITY; readonly #resources = new ArtifactResourceScope(); - close(): Promise { - return this.#resources.close(); + async close(): Promise { + try { + await this.#resources.close(); + } catch (cause: unknown) { + if ( + !(cause instanceof ArtifactReaderFailure) || + cause.cleanup === undefined + ) + throw cause; + const error = new ProviderCleanupError( + PYTHON_PYC_PROVIDER_IDENTITY.id, + cause.cleanup.resources, + { reason: cause.cleanup.reason }, + { cause, operation: OPERATION }, + ); + bindArtifactFileCleanup(cause, error); + throw error; + } } async inspect(input: InspectPythonPycInput, options?: ExecutionOptions) { diff --git a/tests/boundary/filesystem/pythonBeamCleanup.test.ts b/tests/boundary/filesystem/pythonBeamCleanup.test.ts new file mode 100644 index 0000000000..4d8bec7057 --- /dev/null +++ b/tests/boundary/filesystem/pythonBeamCleanup.test.ts @@ -0,0 +1,154 @@ +import { createHash } from "node:crypto"; +import { fstatSync } from "node:fs"; +import type { FileHandle } from "node:fs/promises"; +import * as fileSystem from "node:fs/promises"; +import { join } from "node:path"; +import { afterEach, expect, it, vi } from "vitest"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { ProviderCleanupError } from "../../../src/domain/providerCleanupError.js"; +import { pythonPycFixture } from "../../../src/python/PythonPyc.fixture.js"; +import { PythonPycProvider } from "../../../src/python/PythonPycProvider.js"; +import { PythonPycService } from "../../../src/application/python/PythonPycService.js"; +import { BeamService } from "../../../src/application/erlang/BeamService.js"; +import { BeamProvider } from "../../../src/erlang/BeamProvider.js"; +import { erlangBeamFixture } from "../../../src/erlang/Beam.fixture.js"; + +const injected = vi.hoisted(() => ({ + opened: undefined as + | ((path: unknown, handle: FileHandle) => void) + | undefined, +})); + +vi.mock("node:fs/promises", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + open: async (...args: Parameters) => { + const handle = await actual.open(...args); + injected.opened?.(args[0], handle); + return handle; + }, + }; +}); + +afterEach(() => { + injected.opened = undefined; + vi.restoreAllMocks(); +}); + +const providers = [ + { + name: "Python pyc", + operation: "inspect_python_pyc", + bytes: pythonPycFixture(), + create: () => new PythonPycService(new PythonPycProvider()), + }, + { + name: "Erlang BEAM", + operation: "inspect_erlang_beam", + bytes: erlangBeamFixture(), + create: () => new BeamService(new BeamProvider()), + }, +]; + +const outcomes = [ + { stage: "completed read", cancelDuringClose: false, failRead: false }, + { + stage: "cancellation during close", + cancelDuringClose: true, + failRead: false, + }, + { + stage: "read failure then cancellation", + cancelDuringClose: true, + failRead: true, + }, +]; + +it.each( + providers.flatMap((provider) => + outcomes.map((outcome) => ({ ...provider, ...outcome })), + ), +)( + "$name preserves acquired facts and resolves cleanup after retries: $stage", + async ({ operation, bytes, create, cancelDuringClose, failRead }) => { + const root = await createTestTempDirectory("rea-python-beam-close-owner-"); + const path = join(root, "selected-artifact"); + await fileSystem.writeFile(path, bytes); + const releases: (() => Promise)[] = []; + const descriptors: number[] = []; + const controller = new AbortController(); + const reason = { source: "cancelled during descriptor finalization" }; + const readFailure = new Error("injected physical artifact read failure"); + let denyClose = true; + injected.opened = (openedPath, file) => { + if (String(openedPath) !== path) return; + descriptors.push(file.fd); + if (failRead) vi.spyOn(file, "read").mockRejectedValueOnce(readFailure); + const close = file.close.bind(file); + releases.push(close); + vi.spyOn(file, "close").mockImplementation(async () => { + if (cancelDuringClose) controller.abort(reason); + if (denyClose) + throw new Error("injected artifact descriptor close failure"); + await close(); + }); + }; + const service = create(); + try { + const result = await service.inspect( + { path }, + { signal: controller.signal }, + ); + if (result.ok) throw new Error("Expected incomplete descriptor cleanup"); + expect(result.error.cleanupIncomplete).toBe(true); + expect(result.error.cleanupResources).toEqual([path]); + const expectedSnapshot = failRead + ? undefined + : { + kind: "file-snapshot-hash", + path, + sha256: createHash("sha256").update(bytes).digest("hex"), + bytes: bytes.length, + }; + expect(result.error.partialObservation).toEqual(expectedSnapshot); + if (failRead) + expect(result.error.cleanupPrimary?.cause).toBe(readFailure); + else if (cancelDuringClose) { + expect(result.error.cleanupPrimary?._tag).toBe( + "AnalysisCancelledError", + ); + expect(result.error.cleanupPrimary?.cause).toBe(reason); + } + expect(fstatSync(descriptors[0]!)).toBeDefined(); + const closeErrors: ProviderCleanupError[] = []; + for (let attempt = 0; attempt < 2; attempt += 1) { + const failure: unknown = await service.close().then( + () => undefined, + (cause: unknown) => cause, + ); + expect(failure).toBeInstanceOf(ProviderCleanupError); + if (!(failure instanceof ProviderCleanupError)) + throw new Error("Expected a translated provider cleanup failure"); + expect(failure.operation).toBe(operation); + expect(failure.cleanupIncomplete).toBe(true); + expect(failure.cleanupResources).toEqual([path]); + closeErrors.push(failure); + } + expect(descriptors).toHaveLength(1); + expect(result.error.cleanupIncomplete).toBe(true); + denyClose = false; + await service.close(); + expect(() => fstatSync(descriptors[0]!)).toThrow(); + for (const failure of [result.error, ...closeErrors]) { + expect(failure.cleanupIncomplete).toBe(false); + expect(failure.cleanupResources).toEqual([]); + } + expect(result.error.partialObservation).toEqual(expectedSnapshot); + } finally { + denyClose = false; + await service.close(); + for (const close of releases) await close(); + } + }, +); From f6805e29c3c74223660686b5ca979c0508e1b195 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:02:10 +0800 Subject: [PATCH 24/25] refactor(pe): centralize resource tree facts across readers --- src/artifacts/NodeSeaArtifactReader.test.ts | 28 ++++++ src/artifacts/nodeSea/NodeSeaLocation.ts | 4 +- src/native/pe/PeResourceLayout.ts | 58 +++++++---- src/native/pe/PeResourceParser.test.ts | 101 ++++++++++++++++++++ src/native/pe/PeResourceParser.ts | 89 +++++++++++------ src/native/pe/PeResourceTreeFacts.ts | 49 ++++++++++ 6 files changed, 284 insertions(+), 45 deletions(-) create mode 100644 src/native/pe/PeResourceTreeFacts.ts diff --git a/src/artifacts/NodeSeaArtifactReader.test.ts b/src/artifacts/NodeSeaArtifactReader.test.ts index 58b7f2b449..5249a5c98b 100644 --- a/src/artifacts/NodeSeaArtifactReader.test.ts +++ b/src/artifacts/NodeSeaArtifactReader.test.ts @@ -745,3 +745,31 @@ it.each([0xfeedface, 0xfeedfacf, 0xcefaedfe, 0xcffaedfe])( } }, ); + +it("confirms SEA naming before rejecting an unrelated cyclic PE resource branch", async () => { + const fixture = peResourceFixture(true, [ + { type: 3, name: 1, language: 1033, payload: Buffer.from("other") }, + { + type: 10, + name: "NODE_SEA_BLOB", + language: 1033, + payload: buildNodeSeaBlob(populated), + }, + ]); + const other = fixture.entryOffsets.find( + (at) => fixture.bytes.readUInt32LE(at) === 3, + ); + if (other === undefined) throw new Error("Expected unrelated resource type"); + fixture.bytes.writeUInt32LE(0x80000000, other + 4); + const handle = await open( + await writeCarrier("sea-with-unrelated-cycle", fixture.bytes), + "r", + ); + try { + await expect( + locateNodeSea(handle, fixture.bytes.length), + ).rejects.toMatchObject({ reason: "format" }); + } finally { + await handle.close(); + } +}); diff --git a/src/artifacts/nodeSea/NodeSeaLocation.ts b/src/artifacts/nodeSea/NodeSeaLocation.ts index cfa25719c9..0d3a1c81b4 100644 --- a/src/artifacts/nodeSea/NodeSeaLocation.ts +++ b/src/artifacts/nodeSea/NodeSeaLocation.ts @@ -591,8 +591,9 @@ const locatePeResources = async (carrier: Carrier): Promise => { if (bytes === undefined) throw formatFailure(NODE_SEA_TRUNCATED); // A native-looking file need not be an admitted PE image. Once the layout // is admitted, resource errors remain failures rather than inferred absence. + let layout: ReturnType; try { - readPeResourceLayout(bytes); + layout = readPeResourceLayout(bytes); } catch (cause) { if (cause instanceof ArtifactReaderFailure && cause.reason === "format") return []; @@ -611,6 +612,7 @@ const locatePeResources = async (carrier: Carrier): Promise => { max_entries: MAX_INSPECT_PE_DIRECTORY_ENTRIES, }, carrier.signal, + layout, ); return resources.resources.flatMap((resource) => { if ( diff --git a/src/native/pe/PeResourceLayout.ts b/src/native/pe/PeResourceLayout.ts index 269a126916..857378183a 100644 --- a/src/native/pe/PeResourceLayout.ts +++ b/src/native/pe/PeResourceLayout.ts @@ -1,3 +1,11 @@ +import { + PE_RESOURCE_DIRECTORY_BYTES, + PE_RESOURCE_ENTRY_BYTES, + PE_RESOURCE_NAME_PREFIX_BYTES, + readPeResourceDirectoryFacts, + readPeResourceEntryFacts, + readPeResourceNameFacts, +} from "./PeResourceTreeFacts.js"; import { observeMzWindowsHeader } from "../../domain/dosMz.js"; import { readPeCoffHeader } from "../../domain/peCoffHeader.js"; import { readPeSectionHeader } from "../../domain/peSectionHeader.js"; @@ -260,7 +268,7 @@ export const hasPeResourceName = async ( let root: number; try { layout = readPeResourceLayout(headers, fileSize); - if (layout.size < 16) return false; + if (layout.size < PE_RESOURCE_DIRECTORY_BYTES) return false; root = layout.map(layout.rva, layout.size); } catch (cause) { if (cause instanceof ArtifactReaderFailure && cause.reason === "format") @@ -274,29 +282,47 @@ export const hasPeResourceName = async ( : read(root + at, length); let examined = 0; const directory = async (at: number): Promise => { - const header = await relative(at, 16); + const header = await relative(at, PE_RESOURCE_DIRECTORY_BYTES); if (header === undefined) return undefined; - const entries = header.readUInt16LE(12) + header.readUInt16LE(14); + const { entryCount: entries } = readPeResourceDirectoryFacts(header); if (entries > maxEntries - examined) return undefined; examined += entries; - return relative(at + 16, entries * 8); + return relative( + at + PE_RESOURCE_DIRECTORY_BYTES, + entries * PE_RESOURCE_ENTRY_BYTES, + ); }; const types = await directory(0); if (types === undefined) return false; const expected = Buffer.from(name, "utf16le"); - for (let at = 0; at < types.length; at += 8) { - if (types.readUInt32LE(at) !== typeId) continue; - const target = types.readUInt32LE(at + 4); - if ((target & 0x80000000) === 0) continue; - const names = await directory(target & 0x7fffffff); + for (let at = 0; at < types.length; at += PE_RESOURCE_ENTRY_BYTES) { + const type = readPeResourceEntryFacts( + types.subarray(at, at + PE_RESOURCE_ENTRY_BYTES), + ); + if (type.identity.kind !== "id" || type.identity.id !== typeId) continue; + if (type.target.kind !== "directory") continue; + const names = await directory(type.target.offset); if (names === undefined) continue; - for (let entry = 0; entry < names.length; entry += 8) { - const encoded = names.readUInt32LE(entry); - if ((encoded & 0x80000000) === 0) continue; - const nameAt = encoded & 0x7fffffff; - const length = await relative(nameAt, 2); - if (length?.readUInt16LE(0) !== name.length) continue; - const candidate = await relative(nameAt + 2, expected.length); + for ( + let entry = 0; + entry < names.length; + entry += PE_RESOURCE_ENTRY_BYTES + ) { + const { identity } = readPeResourceEntryFacts( + names.subarray(entry, entry + PE_RESOURCE_ENTRY_BYTES), + ); + if (identity.kind !== "name") continue; + const nameAt = identity.offset; + const length = await relative(nameAt, PE_RESOURCE_NAME_PREFIX_BYTES); + if ( + length === undefined || + readPeResourceNameFacts(length).utf16Units !== name.length + ) + continue; + const candidate = await relative( + nameAt + PE_RESOURCE_NAME_PREFIX_BYTES, + expected.length, + ); if (candidate?.equals(expected)) return true; } } diff --git a/src/native/pe/PeResourceParser.test.ts b/src/native/pe/PeResourceParser.test.ts index 94afbbbd89..ecf8b0a17d 100644 --- a/src/native/pe/PeResourceParser.test.ts +++ b/src/native/pe/PeResourceParser.test.ts @@ -4,6 +4,7 @@ import { inspectPeResourcesInputSchema, peResourcesSchema, } from "../../domain/native/peResources.js"; +import { hasPeResourceName } from "./PeResourceLayout.js"; import { parsePeResources } from "./PeResourceParser.js"; import { peResourceFixture } from "./PeResources.fixture.js"; @@ -560,3 +561,103 @@ describe("PE resource ownership boundaries", () => { }); }); }); + +const probeSeaName = (bytes: Buffer) => + hasPeResourceName( + bytes, + bytes.length, + async (offset, length) => + offset < 0 || length > bytes.length - offset + ? undefined + : bytes.subarray(offset, offset + length), + 10, + "NODE_SEA_BLOB", + 4096, + ); + +it("matches resource names by their exact UTF-16 bytes without losing source identity", async () => { + const fixture = peResourceFixture(false, [ + { + type: 10, + name: "NODE_SEA_BLOB", + language: 1033, + payload: Buffer.from("data"), + }, + ]); + expect(await probeSeaName(fixture.bytes)).toBe(true); + const entry = fixture.entryOffsets.find( + (at) => (fixture.bytes.readUInt32LE(at) & 0x80000000) !== 0, + ); + if (entry === undefined) throw new Error("Expected named resource fixture"); + const nameAt = 0x400 + (fixture.bytes.readUInt32LE(entry) & 0x7fffffff); + fixture.bytes[nameAt + 3] = 1; + expect(await probeSeaName(fixture.bytes)).toBe(false); + const report = await parse(fixture.bytes); + expect(report.resources[0]?.name).toMatchObject({ + kind: "name", + name: "\u014eODE_SEA_BLOB", + utf16le_hex: fixture.bytes + .subarray(nameAt + 2, nameAt + 2 + 26) + .toString("hex"), + }); + fixture.bytes.writeUInt16LE(0xffff, nameAt); + expect(await probeSeaName(fixture.bytes)).toBe(false); + await expect(parse(fixture.bytes)).rejects.toMatchObject({ + reason: "format", + }); +}); + +it("keeps name probing independent of full resource name/ID partition admission", async () => { + const fixture = peResourceFixture(false, [ + { + type: 10, + name: "NODE_SEA_BLOB", + language: 1033, + payload: Buffer.from("data"), + }, + ]); + const names = fixture.directoryOffsets[1]; + if (names === undefined) throw new Error("Expected resource names directory"); + fixture.bytes.writeUInt16LE(0, names + 12); + fixture.bytes.writeUInt16LE(1, names + 14); + expect(await probeSeaName(fixture.bytes)).toBe(true); + await expect(parse(fixture.bytes)).rejects.toMatchObject({ + reason: "format", + message: "Resource name/ID entry counts disagree with encoded identities.", + }); +}); + +it("keeps raw name probing separate from full resource-name alignment admission", async () => { + const fixture = peResourceFixture(false, [ + { + type: 10, + name: "NODE_SEA_BLOB", + language: 1033, + payload: Buffer.from("data"), + }, + ]); + const entry = fixture.entryOffsets.find( + (at) => (fixture.bytes.readUInt32LE(at) & 0x80000000) !== 0, + ); + if (entry === undefined) throw new Error("Expected named resource fixture"); + const original = 0x400 + (fixture.bytes.readUInt32LE(entry) & 0x7fffffff); + const nameBytes = 2 + fixture.bytes.readUInt16LE(original) * 2; + const previousEnd = fixture.bytes.readUInt32LE(fixture.directoryAt + 4); + const unaligned = previousEnd % 2 === 0 ? previousEnd + 1 : previousEnd; + const extendedEnd = unaligned + nameBytes; + expect(0x400 + extendedEnd).toBeLessThanOrEqual(fixture.bytes.length); + fixture.bytes.copy( + fixture.bytes, + 0x400 + unaligned, + original, + original + nameBytes, + ); + fixture.bytes.writeUInt32LE(0x80000000 + unaligned, entry); + fixture.bytes.writeUInt32LE(extendedEnd, fixture.directoryAt + 4); + fixture.bytes.writeUInt32LE(extendedEnd, fixture.sectionAt + 8); + expect(await probeSeaName(fixture.bytes)).toBe(true); + await expect(parse(fixture.bytes)).rejects.toMatchObject({ + reason: "format", + message: "Unaligned PE resource name.", + }); +}); diff --git a/src/native/pe/PeResourceParser.ts b/src/native/pe/PeResourceParser.ts index 0cd3f83f21..0d96cd158f 100644 --- a/src/native/pe/PeResourceParser.ts +++ b/src/native/pe/PeResourceParser.ts @@ -1,3 +1,12 @@ +import { + PE_RESOURCE_DIRECTORY_BYTES, + PE_RESOURCE_ENTRY_BYTES, + PE_RESOURCE_NAME_PREFIX_BYTES, + readPeResourceDirectoryFacts, + readPeResourceEntryFacts, + readPeResourceNameFacts, + type PeResourceIdentityDeclaration, +} from "./PeResourceTreeFacts.js"; import { createHash } from "node:crypto"; import { setImmediate } from "node:timers/promises"; import { assertArtifactReadActive } from "../../artifacts/ArtifactFileOperation.js"; @@ -15,6 +24,8 @@ import { requirePeRange, } from "./PeResourceLayout.js"; +type PeResourceLayout = ReturnType; + type Range = { offset: number; bytes: number }; type OccupiedRange = Range & { kind: "directory" | "name" | "data-entry" | "payload"; @@ -43,9 +54,11 @@ export const parsePeResources = async ( artifact: PeResources["artifact"], input: InspectPeResourcesInput, signal?: AbortSignal, + /** Already admitted from these same bytes; used by carrier probes that own layout failure policy. */ + admittedLayout?: PeResourceLayout, ): Promise => { assertArtifactReadActive(signal); - const layout = readPeResourceLayout(bytes); + const layout = admittedLayout ?? readPeResourceLayout(bytes); const directories: PeResources["directories"] = []; const resources: PeResources["resources"] = []; const occupied: OccupiedRange[] = []; @@ -92,23 +105,33 @@ export const parsePeResources = async ( if (size > 0) occupied.push({ ...range, kind }); return range; }; - const readIdentity = (encoded: number): PeResourceIdentity => { - if ((encoded & 0x80000000) === 0) return { kind: "id", id: encoded }; - const at = encoded & 0x7fffffff; + const readIdentity = ( + declaration: PeResourceIdentityDeclaration, + ): PeResourceIdentity => { + if (declaration.kind === "id") return { kind: "id", id: declaration.id }; + const at = declaration.offset; const previous = names.get(at); if (previous !== undefined) return previous; if (at % 2 !== 0) peFailure("Unaligned PE resource name."); - const start = relative(at, 2); - const utf16Units = bytes.readUInt16LE(start); + const start = relative(at, PE_RESOURCE_NAME_PREFIX_BYTES); + const { utf16Units, payloadBytes: length } = readPeResourceNameFacts( + bytes.subarray(start, start + PE_RESOURCE_NAME_PREFIX_BYTES), + ); reserveDecodedName(utf16Units); - const length = utf16Units * 2; - relative(at, 2 + length); - const raw = bytes.subarray(start + 2, start + 2 + length); + relative(at, PE_RESOURCE_NAME_PREFIX_BYTES + length); + const raw = bytes.subarray( + start + PE_RESOURCE_NAME_PREFIX_BYTES, + start + PE_RESOURCE_NAME_PREFIX_BYTES + length, + ); const value: PeResourceIdentity = { kind: "name", name: raw.toString("utf16le"), utf16le_hex: raw.toString("hex"), - location: recordRange(start, 2 + length, "name"), + location: recordRange( + start, + PE_RESOURCE_NAME_PREFIX_BYTES + length, + "name", + ), }; names.set(at, value); return value; @@ -120,25 +143,30 @@ export const parsePeResources = async ( ancestors: ReadonlySet, ): Promise => { if (ancestors.has(at)) peFailure("Cyclic PE resource directory."); - const header = relative(at, 16); - const named = bytes.readUInt16LE(header + 12); - const ids = bytes.readUInt16LE(header + 14); - const count = named + ids; + const header = relative(at, PE_RESOURCE_DIRECTORY_BYTES); + const facts = readPeResourceDirectoryFacts( + bytes.subarray(header, header + PE_RESOURCE_DIRECTORY_BYTES), + ); + const { namedEntries: named, idEntries: ids, entryCount: count } = facts; if (count > input.max_entries - examined) limit( "PE resource directory entry budget exceeded; no partial inventory was returned.", "capacity", ); examined += count; - relative(at, 16 + 8 * count); + relative(at, PE_RESOURCE_DIRECTORY_BYTES + PE_RESOURCE_ENTRY_BYTES * count); const directory = { path, entry_locations: entryLocations, - location: recordRange(header, 16 + 8 * count, "directory"), - characteristics: bytes.readUInt32LE(header), - timestamp: bytes.readUInt32LE(header + 4), - major_version: bytes.readUInt16LE(header + 8), - minor_version: bytes.readUInt16LE(header + 10), + location: recordRange( + header, + PE_RESOURCE_DIRECTORY_BYTES + PE_RESOURCE_ENTRY_BYTES * count, + "directory", + ), + characteristics: facts.characteristics, + timestamp: facts.timestamp, + major_version: facts.majorVersion, + minor_version: facts.minorVersion, named_entries: named, id_entries: ids, }; @@ -151,22 +179,27 @@ export const parsePeResources = async ( await setImmediate(); assertArtifactReadActive(signal); } - const entry = header + 16 + index * 8; - const encoded = bytes.readUInt32LE(entry); - if (((encoded & 0x80000000) !== 0) !== index < named) + const entry = + header + PE_RESOURCE_DIRECTORY_BYTES + index * PE_RESOURCE_ENTRY_BYTES; + const record = readPeResourceEntryFacts( + bytes.subarray(entry, entry + PE_RESOURCE_ENTRY_BYTES), + ); + if ((record.identity.kind === "name") !== index < named) peFailure( "Resource name/ID entry counts disagree with encoded identities.", ); - const identity = readIdentity(encoded); + const identity = readIdentity(record.identity); const key = identityKey(identity); if (keys.has(key)) peFailure("Duplicate identity within a resource directory."); keys.add(key); const next = [...path, identity]; - const locations = [...entryLocations, { offset: entry, bytes: 8 }]; - const target = bytes.readUInt32LE(entry + 4); - const offset = target & 0x7fffffff; - if ((target & 0x80000000) !== 0) { + const locations = [ + ...entryLocations, + { offset: entry, bytes: PE_RESOURCE_ENTRY_BYTES }, + ]; + const { offset } = record.target; + if (record.target.kind === "directory") { if (next.length >= 3) peFailure( "Unsupported resource tree deeper than type/name/language.", diff --git a/src/native/pe/PeResourceTreeFacts.ts b/src/native/pe/PeResourceTreeFacts.ts new file mode 100644 index 0000000000..536c49de53 --- /dev/null +++ b/src/native/pe/PeResourceTreeFacts.ts @@ -0,0 +1,49 @@ +/** Fixed PE resource records; callers own interval, budget and tree admission. */ +export const PE_RESOURCE_DIRECTORY_BYTES = 16; +export const PE_RESOURCE_ENTRY_BYTES = 8; +export const PE_RESOURCE_NAME_PREFIX_BYTES = 2; + +/** Decode an already-admitted IMAGE_RESOURCE_DIRECTORY record. */ +export const readPeResourceDirectoryFacts = (bytes: Buffer) => { + const namedEntries = bytes.readUInt16LE(12); + const idEntries = bytes.readUInt16LE(14); + return { + characteristics: bytes.readUInt32LE(0), + timestamp: bytes.readUInt32LE(4), + majorVersion: bytes.readUInt16LE(8), + minorVersion: bytes.readUInt16LE(10), + namedEntries, + idEntries, + entryCount: namedEntries + idEntries, + }; +}; + +export type PeResourceIdentityDeclaration = + | { readonly kind: "id"; readonly id: number } + | { readonly kind: "name"; readonly offset: number }; + +/** Offsets remain relative to the resource root; numeric IDs retain all source bits. */ +export const readPeResourceEntryFacts = (bytes: Buffer) => { + const encodedIdentity = bytes.readUInt32LE(0); + const encodedTarget = bytes.readUInt32LE(4); + const identity: PeResourceIdentityDeclaration = + (encodedIdentity & 0x80000000) === 0 + ? { kind: "id", id: encodedIdentity } + : { kind: "name", offset: encodedIdentity & 0x7fffffff }; + return { + identity, + target: { + kind: + (encodedTarget & 0x80000000) === 0 + ? ("data" as const) + : ("directory" as const), + offset: encodedTarget & 0x7fffffff, + }, + }; +}; + +/** Observe the length only; callers admit decoded-name costs before reading or decoding payload. */ +export const readPeResourceNameFacts = (bytes: Buffer) => { + const utf16Units = bytes.readUInt16LE(0); + return { utf16Units, payloadBytes: utf16Units * 2 }; +}; From 662ec2e4501e975f2d1a28958a1ff560a9e24869 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Tue, 27 Oct 2026 01:11:51 +0800 Subject: [PATCH 25/25] fix(verifiers): preserve source Evidence through shared admission --- scripts/lib/browser-network-e2e.mjs | 17 +- scripts/lib/browser-popup-e2e.mjs | 9 +- scripts/lib/browser-screenshot-e2e.mjs | 12 +- scripts/lib/mcp-verifier-results.mjs | 3 +- scripts/lib/native-value-e2e.mjs | 43 +++-- scripts/verify-apk-signature.mjs | 7 +- scripts/verify-package-mcp.mjs | 35 ++-- scripts/verify-reconstruction-readiness.mjs | 15 +- scripts/verify-wasm-artifact.mjs | 5 +- scripts/verify-wasm-interface-comparison.mjs | 7 +- scripts/verify/android/analysis.mjs | 14 +- scripts/verify/android/class-search.mjs | 10 +- scripts/verify/harmony/analysis.mjs | 14 +- .../process/mcpVerifierResults.test.ts | 150 ++++++++++++++++++ 14 files changed, 255 insertions(+), 86 deletions(-) diff --git a/scripts/lib/browser-network-e2e.mjs b/scripts/lib/browser-network-e2e.mjs index 124655a1e1..41ca248d08 100644 --- a/scripts/lib/browser-network-e2e.mjs +++ b/scripts/lib/browser-network-e2e.mjs @@ -1,3 +1,4 @@ +import { requireMcpOutput } from "./mcp-verifier-results.mjs"; import assert from "node:assert/strict"; import { execFile } from "node:child_process"; import { fileURLToPath } from "node:url"; @@ -116,9 +117,12 @@ export async function verifyBrowserNetworkEvidence( { timeout: 60_000 }, ); assert.notEqual(result.isError, true, JSON.stringify(result)); - const evidence = parseEvidence(result.structuredContent); + assert.notEqual(result.structuredContent, undefined); + const evidence = parseEvidence( + requireMcpOutput(result, "capture_browser_scenario"), + ); mcpProof = assertBrowserNetworkEvidence( - browserScenarioCaptureSchema.parse(evidence.normalized_result), + evidence.normalized_result, site.origin, ); if (attachment !== undefined) { @@ -131,12 +135,13 @@ export async function verifyBrowserNetworkEvidence( { timeout: 60_000 }, ); assert.notEqual(connected.isError, true, JSON.stringify(connected)); - const connectedEvidence = parseEvidence(connected.structuredContent); + assert.notEqual(connected.structuredContent, undefined); + const connectedEvidence = parseEvidence( + requireMcpOutput(connected, "capture_browser_scenario"), + ); assert.deepEqual( assertBrowserNetworkEvidence( - browserScenarioCaptureSchema.parse( - connectedEvidence.normalized_result, - ), + connectedEvidence.normalized_result, site.origin, "disconnected-external", ), diff --git a/scripts/lib/browser-popup-e2e.mjs b/scripts/lib/browser-popup-e2e.mjs index 74c484bf17..c62a2dfb15 100644 --- a/scripts/lib/browser-popup-e2e.mjs +++ b/scripts/lib/browser-popup-e2e.mjs @@ -1,3 +1,4 @@ +import { requireMcpOutput } from "./mcp-verifier-results.mjs"; import assert from "node:assert/strict"; import { execFile } from "node:child_process"; import { once } from "node:events"; @@ -163,11 +164,11 @@ const verifyTransports = async (executable, origin) => { { timeout: 60_000 }, ); assert.notEqual(captured.isError, true, JSON.stringify(captured)); - const evidence = parseEvidence(captured.structuredContent); - assertNetwork( - browserScenarioCaptureSchema.parse(evidence.normalized_result), - origin, + assert.notEqual(captured.structuredContent, undefined); + const evidence = parseEvidence( + requireMcpOutput(captured, "capture_browser_scenario"), ); + assertNetwork(evidence.normalized_result, origin); } finally { try { await client.close(); diff --git a/scripts/lib/browser-screenshot-e2e.mjs b/scripts/lib/browser-screenshot-e2e.mjs index 92a27116fb..e7cbf17c44 100644 --- a/scripts/lib/browser-screenshot-e2e.mjs +++ b/scripts/lib/browser-screenshot-e2e.mjs @@ -1,4 +1,7 @@ -import { requireMcpToolError } from "./mcp-verifier-results.mjs"; +import { + requireMcpOutput, + requireMcpToolError, +} from "./mcp-verifier-results.mjs"; import assert from "node:assert/strict"; import { execFile } from "node:child_process"; import { mkdtemp, readFile, rm } from "node:fs/promises"; @@ -95,10 +98,11 @@ export async function verifyLargeScreenshotE2e(endpoint, origin) { { timeout: 60000 }, ); assert.notEqual(captured.isError, true, JSON.stringify(captured)); - const mcpEvidence = parseEvidence(captured.structuredContent); - const mcpScreenshot = webScreenshotSchema.parse( - mcpEvidence.normalized_result, + assert.notEqual(captured.structuredContent, undefined); + const mcpEvidence = parseEvidence( + requireMcpOutput(captured, "capture_web_screenshot"), ); + const mcpScreenshot = mcpEvidence.normalized_result; assert.deepEqual(mcpScreenshot.artifact, screenshot.artifact); const compared = comparePngScreenshots({ before: screenshot.artifact, diff --git a/scripts/lib/mcp-verifier-results.mjs b/scripts/lib/mcp-verifier-results.mjs index a9d10876a9..efef3efd65 100644 --- a/scripts/lib/mcp-verifier-results.mjs +++ b/scripts/lib/mcp-verifier-results.mjs @@ -46,7 +46,8 @@ export const requireMcpOutput = ( if (!isDeepStrictEqual(text, result.structuredContent)) throw new Error(`${operation} text and structured results disagree`); } - return output; + // Downstream parseEvidence must authenticate source facts, not a schema projection. + return Object.hasOwn(output, "normalized_result") ? text : output; }; const operationResultKey = (contract) => { diff --git a/scripts/lib/native-value-e2e.mjs b/scripts/lib/native-value-e2e.mjs index e58e9ef6d4..88fba60cbf 100644 --- a/scripts/lib/native-value-e2e.mjs +++ b/scripts/lib/native-value-e2e.mjs @@ -14,6 +14,7 @@ import { nativeValueTraceSchema } from "../../dist/domain/native/nativeValueTrac import { requireMcpEvidenceBundle, requireMcpLifecycleResult, + requireMcpOutput, } from "./mcp-verifier-results.mjs"; import { verifyGhidraBoundaries } from "./real-ghidra-boundaries.mjs"; @@ -198,12 +199,15 @@ async function verifyNativeValueAdapters( { timeout: 180000 }, ); assert.notEqual(result.isError, true, JSON.stringify(result)); - const mcpEvidence = parseEvidence(result.structuredContent); + assert.notEqual(result.structuredContent, undefined); + const mcpEvidence = parseEvidence( + requireMcpOutput(result, "trace_native_values"), + ); assert.equal(mcpEvidence.operation, "trace_native_values"); assert.equal(mcpEvidence.provider.id, "rea-workflow"); assert.deepEqual(mcpEvidence.analysis_profile, workflowProfile); const mcpGraph = verifyGraph( - result.structuredContent?.normalized_result, + mcpEvidence.normalized_result, target, procedure, globalAddress, @@ -218,7 +222,10 @@ async function verifyNativeValueAdapters( { timeout: 180000 }, ); assert.notEqual(analyzed.isError, true, JSON.stringify(analyzed)); - const functionEvidence = parseEvidence(analyzed.structuredContent); + assert.notEqual(analyzed.structuredContent, undefined); + const functionEvidence = parseEvidence( + requireMcpOutput(analyzed, "analyze_function"), + ); assert.equal(functionEvidence.operation, "analyze_function"); assert.equal(functionEvidence.provider.id, "ghidra"); assert.deepEqual( @@ -246,10 +253,6 @@ async function verifyNativeValueAdapters( functionEvidence.analysis_profile, ); assert.deepEqual(functionProfile, upstreamProfile); - assert.deepEqual( - functionEvidence.normalized_result, - analyzed.structuredContent?.normalized_result, - ); const comparison = await client.callTool( { name: "compare_functions", @@ -261,16 +264,15 @@ async function verifyNativeValueAdapters( { timeout: 180000 }, ); assert.notEqual(comparison.isError, true, JSON.stringify(comparison)); - const comparisonEvidence = parseEvidence(comparison.structuredContent); + assert.notEqual(comparison.structuredContent, undefined); + const comparisonEvidence = parseEvidence( + requireMcpOutput(comparison, "compare_functions"), + ); assert.equal( comparisonEvidence.normalized_result.status, "unchanged", JSON.stringify(comparisonEvidence.normalized_result.dimensions), ); - assert.deepEqual( - comparisonEvidence.normalized_result, - comparison.structuredContent?.normalized_result, - ); assert.ok( comparisonEvidence.evidence_links.includes(functionEvidence.evidence_id), ); @@ -346,8 +348,9 @@ async function verifyUnresolvedCallEvidence(client) { arguments: {}, }); assert.notEqual(inventory.isError, true, JSON.stringify(inventory)); + assert.notEqual(inventory.structuredContent, undefined); const procedures = parseEvidence( - inventory.structuredContent, + requireMcpOutput(inventory, "list_procedures"), ).normalized_result; const indirect = procedures.find((item) => item.value.endsWith("rea_ghidra_inventory_indirect"), @@ -367,7 +370,10 @@ async function verifyUnresolvedCallEvidence(client) { { timeout: 180000 }, ); assert.notEqual(analyzed.isError, true, JSON.stringify(analyzed)); - const evidence = parseEvidence(analyzed.structuredContent); + assert.notEqual(analyzed.structuredContent, undefined); + const evidence = parseEvidence( + requireMcpOutput(analyzed, "analyze_function"), + ); assert.equal(evidence.relationship_coverage?.status, "partial"); assert.ok(evidence.normalized_result.unresolved_calls.length > 0); const traced = await client.callTool( @@ -382,7 +388,10 @@ async function verifyUnresolvedCallEvidence(client) { { timeout: 180000 }, ); assert.notEqual(traced.isError, true, JSON.stringify(traced)); - const trace = parseEvidence(traced.structuredContent).normalized_result; + assert.notEqual(traced.structuredContent, undefined); + const trace = parseEvidence( + requireMcpOutput(traced, "trace_call_path"), + ).normalized_result; assert.equal(trace.goal_status, "unknown"); assert.equal(trace.truncated, true); assert.ok(trace.residual_unknowns.length > 0); @@ -397,8 +406,10 @@ async function verifyUnresolvedCallEvidence(client) { { timeout: 180000 }, ); assert.notEqual(compared.isError, true, JSON.stringify(compared)); + assert.notEqual(compared.structuredContent, undefined); assert.equal( - parseEvidence(compared.structuredContent).normalized_result.status, + parseEvidence(requireMcpOutput(compared, "compare_functions")) + .normalized_result.status, "unknown", ); } diff --git a/scripts/verify-apk-signature.mjs b/scripts/verify-apk-signature.mjs index ca763b974f..b6fd2450fa 100644 --- a/scripts/verify-apk-signature.mjs +++ b/scripts/verify-apk-signature.mjs @@ -26,7 +26,10 @@ import { locateApkSigningBlock, parseZipTail, } from "../dist/domain/apkSignature/apkSigningBlock.js"; -import { mcpTextValue } from "./lib/mcp-verifier-results.mjs"; +import { + mcpTextValue, + requireMcpOutput, +} from "./lib/mcp-verifier-results.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; const execute = promisify(execFile); @@ -261,7 +264,7 @@ try { ajv.validate(advertised.outputSchema, reply.structuredContent), JSON.stringify(ajv.errors), ); - const mcp = parseEvidence(reply.structuredContent); + const mcp = parseEvidence(requireMcpOutput(reply, "inspect_apk_signature")); assert.deepEqual(cli, mcp, "CLI and MCP evidence differ"); cases += 2; return mcp.normalized_result; diff --git a/scripts/verify-package-mcp.mjs b/scripts/verify-package-mcp.mjs index aeb6d2f01d..fed54fe111 100644 --- a/scripts/verify-package-mcp.mjs +++ b/scripts/verify-package-mcp.mjs @@ -1,6 +1,7 @@ import { requireMcpEvidenceBundle, requireMcpLifecycleResult, + requireMcpOutput, requireMcpToolError, } from "./lib/mcp-verifier-results.mjs"; import assert from "node:assert/strict"; @@ -172,31 +173,23 @@ const verifyMcpInlineArtifactEvidence = async ( { name: "inspect_artifact", arguments: {} }, mcpOptions, ); - assert.notEqual(inspected.isError, true, JSON.stringify(inspected)); - const source = parseEvidence(inspected.structuredContent); - assert.deepEqual( - source.normalized_result, - inspected.structuredContent?.normalized_result, - ); - assert.deepEqual( - JSON.parse(prompts.mcpText(inspected)), - inspected.structuredContent, + assert.notEqual(inspected.structuredContent, undefined); + const source = parseEvidence( + requireMcpOutput(inspected, "inspect_artifact"), ); const compared = await client.callTool( { name: "compare_artifacts", arguments: { - left: inspected.structuredContent, - right: inspected.structuredContent, + left: source, + right: source, }, }, mcpOptions, ); - assert.notEqual(compared.isError, true, JSON.stringify(compared)); - const comparison = parseEvidence(compared.structuredContent); - assert.deepEqual( - comparison.normalized_result, - compared.structuredContent?.normalized_result, + assert.notEqual(compared.structuredContent, undefined); + const comparison = parseEvidence( + requireMcpOutput(compared, "compare_artifacts"), ); assert.ok(comparison.evidence_links.includes(source.evidence_id)); const bundleReply = await client.callTool( @@ -220,13 +213,11 @@ const verifyMcpInlineArtifactEvidence = async ( { name: "extract_artifact", arguments: {} }, mcpOptions, ); - assert.notEqual(extracted.isError, true, JSON.stringify(extracted)); - const extraction = parseEvidence(extracted.structuredContent); - assert.deepEqual( - extraction.normalized_result, - extracted.structuredContent?.normalized_result, + assert.notEqual(extracted.structuredContent, undefined); + const extraction = parseEvidence( + requireMcpOutput(extracted, "extract_artifact"), ); - const materialized = extracted.structuredContent?.normalized_result; + const materialized = extraction.normalized_result; assert.equal(typeof materialized?.output_root, "string"); outputRoot = materialized.output_root; assert.equal(materialized.containment_verified, true); diff --git a/scripts/verify-reconstruction-readiness.mjs b/scripts/verify-reconstruction-readiness.mjs index 0eb49f8dd3..99b5d0bc40 100644 --- a/scripts/verify-reconstruction-readiness.mjs +++ b/scripts/verify-reconstruction-readiness.mjs @@ -8,7 +8,7 @@ import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { tsImport } from "tsx/esm/api"; -import { toolContract } from "../dist/contracts/toolContracts.js"; +import { requireMcpOutput } from "./lib/mcp-verifier-results.mjs"; import { parseEvidence } from "../dist/domain/evidence.js"; import { createEvidenceBundle } from "../dist/domain/evidenceBundle.js"; import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs"; @@ -80,10 +80,8 @@ await withVerifierCleanup(async () => { { name: "binary_session", arguments: {} }, { toolDefinition: sessionTool }, ); - assert.notEqual(sessionReply.isError, true); - const sessionStatus = toolContract("binary_session").outputSchema.parse( - sessionReply.structuredContent, - ).result; + assert.notEqual(sessionReply.structuredContent, undefined); + const sessionStatus = requireMcpOutput(sessionReply, "binary_session").result; const javascriptMcp = await analyzeMcp(client, javascriptPath); const electronMcp = await analyzeMcp(client, electronPath); assert.equal( @@ -194,11 +192,10 @@ async function analyzeMcp(client_, inputPath) { name: "analyze_javascript_application", arguments: { input_path: inputPath }, }); - assert.notEqual(response.isError, true); - toolContract("analyze_javascript_application").outputSchema.parse( - response.structuredContent, + assert.notEqual(response.structuredContent, undefined); + return parseEvidence( + requireMcpOutput(response, "analyze_javascript_application"), ); - return parseEvidence(response.structuredContent); } function verifySyntheticEvaluator() { diff --git a/scripts/verify-wasm-artifact.mjs b/scripts/verify-wasm-artifact.mjs index 6161c269ee..89204e92ad 100644 --- a/scripts/verify-wasm-artifact.mjs +++ b/scripts/verify-wasm-artifact.mjs @@ -21,6 +21,7 @@ import { PrivateRuntimeRoot } from "../dist/process/PrivateRuntimeRoot.js"; import { parseEvidence } from "../dist/domain/evidence.js"; import { requireMcpEvidenceBundle, + requireMcpOutput, mcpTextValue, requireMcpToolError, } from "./lib/mcp-verifier-results.mjs"; @@ -278,7 +279,7 @@ try { ajv.validate(advertised.outputSchema, reply.structuredContent), JSON.stringify(ajv.errors), ); - const mcp = parseEvidence(reply.structuredContent); + const mcp = parseEvidence(requireMcpOutput(reply, "inspect_wasm_artifact")); assert.equal(cli.evidence_id, mcp.evidence_id); assert.deepEqual(cli, mcp); const report = mcp.normalized_result; @@ -516,7 +517,7 @@ try { ajv.validate(advertised.outputSchema, reply.structuredContent), JSON.stringify(ajv.errors), ); - const mcp = parseEvidence(reply.structuredContent); + const mcp = parseEvidence(requireMcpOutput(reply, "inspect_wasm_artifact")); assert.deepEqual(cli, mcp); const report = mcp.normalized_result; assert.equal(report.validation, "valid"); diff --git a/scripts/verify-wasm-interface-comparison.mjs b/scripts/verify-wasm-interface-comparison.mjs index 6b215eef38..cabc79a2ea 100644 --- a/scripts/verify-wasm-interface-comparison.mjs +++ b/scripts/verify-wasm-interface-comparison.mjs @@ -7,6 +7,7 @@ import { promisify } from "node:util"; import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { parseEvidence } from "../dist/domain/evidence.js"; +import { requireMcpOutput } from "./lib/mcp-verifier-results.mjs"; const execute = promisify(execFile); const bin = process.env.REA_WABT_BIN_DIRECTORY; @@ -53,7 +54,8 @@ const inspect = async (path) => { arguments: { path }, }); assert.notEqual(result.isError, true, JSON.stringify(result)); - return parseEvidence(result.structuredContent); + assert.notEqual(result.structuredContent, undefined); + return parseEvidence(requireMcpOutput(result, "inspect_wasm_artifact")); }; const compare = async (left, right, scope = "both") => { const input = { @@ -66,7 +68,8 @@ const compare = async (left, right, scope = "both") => { arguments: input, }); assert.notEqual(reply.isError, true, JSON.stringify(reply)); - const mcp = parseEvidence(reply.structuredContent); + assert.notEqual(reply.structuredContent, undefined); + const mcp = parseEvidence(requireMcpOutput(reply, "compare_wasm_interfaces")); const file = join(root, "comparison.json"); await writeFile(file, JSON.stringify({ left, right, scope })); const local = parseEvidence( diff --git a/scripts/verify/android/analysis.mjs b/scripts/verify/android/analysis.mjs index 515e7300ea..6b5a7db03b 100644 --- a/scripts/verify/android/analysis.mjs +++ b/scripts/verify/android/analysis.mjs @@ -1,3 +1,4 @@ +import { requireMcpOutput } from "../../lib/mcp-verifier-results.mjs"; import assert from "node:assert/strict"; import { execFile } from "node:child_process"; import { access, writeFile } from "node:fs/promises"; @@ -147,17 +148,14 @@ try { { name: task.name, arguments: task.input }, { timeout: 150_000 }, ); - assert.notEqual(response.isError, true, JSON.stringify(response)); - const structured = response.structuredContent; - assert.ok(structured !== undefined); - const evidence = parseEvidence(structured); + assert.notEqual(response.structuredContent, undefined); + const evidence = parseEvidence(requireMcpOutput(response, task.name)); assert.deepEqual( evidence.normalized_result, cliResults.get(task.name).normalized_result, ); // Raw observations retain different workspace paths and load durations. // Validate each evidence ID independently; compare portable results/intent. - assert.equal(structured.evidence_id, evidence.evidence_id); assert.deepEqual(evidence.parameters, cliResults.get(task.name).parameters); assert.deepEqual(evidence.provider, cliResults.get(task.name).provider); assert.deepEqual( @@ -169,7 +167,6 @@ try { cliResults.get(task.name).limitations, ); assert.equal(evidence.confidence, cliResults.get(task.name).confidence); - assert.deepEqual(structured.normalized_result, evidence.normalized_result); console.log(`PASS MCP/CLI parity ${task.name}`); } const overloadedInput = { @@ -210,9 +207,10 @@ try { { name: "trace_android_references", arguments: overloadedInput }, { timeout: 150_000 }, ); - assert.notEqual(overloadedMcp.isError, true, JSON.stringify(overloadedMcp)); + assert.notEqual(overloadedMcp.structuredContent, undefined); assert.deepEqual( - parseEvidence(overloadedMcp.structuredContent).normalized_result, + parseEvidence(requireMcpOutput(overloadedMcp, "trace_android_references")) + .normalized_result, overloadedEvidence.normalized_result, ); console.log("PASS real overloaded method references through CLI/MCP"); diff --git a/scripts/verify/android/class-search.mjs b/scripts/verify/android/class-search.mjs index 1f5106f6a5..29ad5bcde3 100644 --- a/scripts/verify/android/class-search.mjs +++ b/scripts/verify/android/class-search.mjs @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { parseEvidence } from "../../../dist/domain/evidence.js"; -import { requireMcpOperationResult } from "../../lib/mcp-verifier-results.mjs"; +import { requireMcpOutput } from "../../lib/mcp-verifier-results.mjs"; /** Compare native selection with an explicit full-inventory oracle on one live session. */ export const verifyAndroidClassSearch = async ({ client, path, sha256 }) => { @@ -9,11 +9,11 @@ export const verifyAndroidClassSearch = async ({ client, path, sha256 }) => { { name: "search_android_classes", arguments: { path, query } }, { timeout: 150_000 }, ); - const result = requireMcpOperationResult( - response, - "search_android_classes", + assert.notEqual(response.structuredContent, undefined); + const evidence = parseEvidence( + requireMcpOutput(response, "search_android_classes"), ); - const evidence = parseEvidence(response.structuredContent); + const result = evidence.normalized_result; assert.equal(evidence.subject.digest.sha256, sha256); assert.equal(evidence.parameters.query, query); assert.equal(result.query, query); diff --git a/scripts/verify/harmony/analysis.mjs b/scripts/verify/harmony/analysis.mjs index aaa401c755..23b2f21733 100644 --- a/scripts/verify/harmony/analysis.mjs +++ b/scripts/verify/harmony/analysis.mjs @@ -1,3 +1,4 @@ +import { requireMcpOutput } from "../../lib/mcp-verifier-results.mjs"; import assert from "node:assert/strict"; import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; @@ -172,8 +173,10 @@ try { { name: "inspect_artifact", arguments: {} }, { timeout: 150_000 }, ); - assert.notEqual(inspected.isError, true, JSON.stringify(inspected)); - const mcpInspection = parseEvidence(inspected.structuredContent); + assert.notEqual(inspected.structuredContent, undefined); + const mcpInspection = parseEvidence( + requireMcpOutput(inspected, "inspect_artifact"), + ); assert.deepEqual( mcpInspection.normalized_result, inspection.normalized_result, @@ -186,9 +189,10 @@ try { }, { timeout: 150_000 }, ); - assert.notEqual(response.isError, true, JSON.stringify(response)); - assert.ok(response.structuredContent !== undefined); - const mcpEvidence = parseEvidence(response.structuredContent); + assert.notEqual(response.structuredContent, undefined); + const mcpEvidence = parseEvidence( + requireMcpOutput(response, "project_harmony_application_graph"), + ); assert.deepEqual( mcpEvidence.normalized_result, projectionEvidence.normalized_result, diff --git a/tests/boundary/process/mcpVerifierResults.test.ts b/tests/boundary/process/mcpVerifierResults.test.ts index 69c9ced8b5..0aa4891be6 100644 --- a/tests/boundary/process/mcpVerifierResults.test.ts +++ b/tests/boundary/process/mcpVerifierResults.test.ts @@ -1,3 +1,11 @@ +import { ARTIFACT_COMPARISON_EXAMPLE } from "../../../src/contracts/artifactComparisonExample.js"; +import { artifactInventoryResultSchema } from "../../../src/domain/artifactGraph.js"; +import { digestCanonicalValue } from "../../../src/domain/canonicalDigest.js"; +import { createAnalysisExecutionEvidence } from "../../../src/domain/analysisObservation.js"; +import { + wasmArtifactFixture, + wasmArtifactExecution, +} from "../../fixtures/wasm/artifact.js"; import { toolContract } from "../../../src/contracts/toolContracts.js"; import assert from "node:assert/strict"; import { readFile } from "node:fs/promises"; @@ -423,3 +431,145 @@ it("requires canonical complete tool errors before inspecting their diagnostic f "Expected an MCP tool failure", ); }); + +it.each(["transport disagreement", "malformed normalized result"])( + "admits complete Wasm Evidence at the interface verifier boundary: %s", + async (scenario) => { + const source = await readFile( + new URL( + "../../../scripts/verify-wasm-interface-comparison.mjs", + import.meta.url, + ), + "utf8", + ); + const declaration = parse(source, { sourceType: "module" }) + .program.body.filter((node) => node.type === "VariableDeclaration") + .flatMap((node) => node.declarations) + .find( + (node) => node.id.type === "Identifier" && node.id.name === "inspect", + )?.init; + if (declaration?.start == null || declaration.end == null) + throw new Error("Wasm interface verifier inspect boundary missing"); + const report = wasmArtifactFixture(); + const execution = wasmArtifactExecution(report); + const evidence = createAnalysisExecutionEvidence( + "inspect_wasm_artifact", + { path: report.artifact.path }, + execution, + "observed", + ); + let response: unknown; + const inspect = runInNewContext( + `(${source.slice(declaration.start, declaration.end)})`, + { + assert, + client: { callTool: async () => response }, + parseEvidence, + ...(await verifierResults()), + }, + ) as (path: string) => Promise; + response = { ...textReply(evidence), structuredContent: evidence }; + await expect(inspect(report.artifact.path)).resolves.toEqual(evidence); + + if (scenario === "transport disagreement") { + const different = wasmArtifactFixture(); + different.limitations = ["Independent complete Wasm observation."]; + const structured = createAnalysisExecutionEvidence( + "inspect_wasm_artifact", + { path: different.artifact.path }, + wasmArtifactExecution(different), + "observed", + ); + expect( + parseEvidence( + toolContract("inspect_wasm_artifact").outputSchema.parse(structured), + ), + ).toEqual(structured); + response = { ...textReply(evidence), structuredContent: structured }; + await expect(inspect(report.artifact.path)).rejects.toThrow( + "text and structured results disagree", + ); + } else { + const malformed = createEvidence( + execution.subject ?? undefined, + execution.provider, + { + operation: "inspect_wasm_artifact", + parameters: { path: report.artifact.path }, + result: "authentic Evidence with an invalid Wasm result shape", + }, + ); + expect(parseEvidence(malformed)).toEqual(malformed); + response = { ...textReply(malformed), structuredContent: malformed }; + await expect(inspect(report.artifact.path)).rejects.toThrow(); + } + response = textReply(evidence); + await expect(inspect(report.artifact.path)).rejects.toThrow(); + }, +); + +it("preserves raw extraction manifest extensions for separate Evidence authentication", async () => { + const { requireMcpOutput } = await verifierResults(); + const inventoryEvidence = ARTIFACT_COMPARISON_EXAMPLE.left; + const inventory = artifactInventoryResultSchema.parse( + inventoryEvidence.normalized_result, + ); + const extractionSemantic = { + source_manifest_id: inventory.manifest.manifest_id, + selected_occurrence_ids: [], + files_sha256: digestCanonicalValue([], "Artifact"), + output_root_alias: "$OUTPUT_ROOT", + }; + const report = { + manifest: inventory.manifest, + extraction_manifest: { + extraction_id: `aex_${digestCanonicalValue(extractionSemantic, "Artifact")}`, + ...extractionSemantic, + }, + output_root: "/owned/output", + artifacts: [], + containment_verified: true, + cleanup: { attempted: true, verified: true, residual_paths: [] }, + coverage: inventory.coverage, + provenance: inventory.provenance, + integrity_contradictions: inventory.integrity_contradictions, + limitations: [], + }; + const evidenceFor = (result: unknown) => + createEvidence(undefined, inventoryEvidence.provider, { + operation: "extract_artifact", + parameters: {}, + result, + }); + const evidence = evidenceFor(report); + const extendedResult = { + ...report, + manifest: { ...report.manifest, vendor_note: "retained source extension" }, + }; + const tampered = { ...evidence, normalized_result: extendedResult }; + const reply = (value: unknown) => ({ + ...textReply(value), + structuredContent: value, + }); + // Shape projection discards the extension; it cannot authenticate its source ID. + expect(toolContract("extract_artifact").outputSchema.parse(tampered)).toEqual( + evidence, + ); + const admittedTampered = requireMcpOutput( + reply(tampered), + "extract_artifact", + ); + expect(admittedTampered).toEqual(tampered); + expect(() => parseEvidence(admittedTampered)).toThrow(/semantic identifier/u); + + const authentic = evidenceFor(extendedResult); + const projected = + toolContract("extract_artifact").outputSchema.parse(authentic); + expect(projected).toEqual({ ...authentic, normalized_result: report }); + const admittedAuthentic = requireMcpOutput( + reply(authentic), + "extract_artifact", + ); + expect(admittedAuthentic).toEqual(authentic); + expect(parseEvidence(admittedAuthentic)).toEqual(authentic); +});