diff --git a/experiments/analysis-efficiency/compare.mjs b/experiments/analysis-efficiency/compare.mjs index 8c386a4d93..01a739c11d 100644 --- a/experiments/analysis-efficiency/compare.mjs +++ b/experiments/analysis-efficiency/compare.mjs @@ -30,6 +30,7 @@ const sameSettings = [ "provider_search_path_sha256", "provider_environment_policy_sha256", "measure_source_sha256", + "native_workflow_source_sha256", "package_lock_sha256", "node_version", "sdk_client_version", @@ -40,7 +41,9 @@ const sameSettings = [ const settingDifferences = sameSettings.filter( (key) => !isDeepStrictEqual(left[key], right[key]) || - (key === "provider_search_path_sha256" && + ((key === "provider_search_path_sha256" || + (key === "native_workflow_source_sha256" && + left.native_question !== null)) && !/^[a-f0-9]{64}$/.test(left[key] ?? "")), ); // Producer/view IDs identify different delivery observations; a named recovery diff --git a/experiments/analysis-efficiency/corpus.json b/experiments/analysis-efficiency/corpus.json index 911fa88f5f..5d2ad531dd 100644 --- a/experiments/analysis-efficiency/corpus.json +++ b/experiments/analysis-efficiency/corpus.json @@ -116,6 +116,36 @@ "derivation": "selected regular ZIP member ShareX.HelpersLib.dll; no target execution" } }, + { + "id": "sharex.exe", + "file": "ShareX.exe", + "bytes": 198656, + "sha256": "46af1ccc514b395cb703633abb5a505a771942b7db9b1915f31aeea541b3685f", + "provenance": { + "project": "ShareX native executable component", + "version": "19.0.2", + "upstreamFile": "ShareX-19.0.2-portable.zip", + "url": "https://github.com/ShareX/ShareX/releases/download/v19.0.2/ShareX-19.0.2-portable.zip", + "transport_sha256": "967b5d838df5ce25fb5fd7cbd16f1bd797427e2df16907b022b74a4bc0fb9d54", + "transport_bytes": 168796487, + "derivation": "selected regular ZIP member ShareX.exe; PE32+ x86-64 with no CLR directory; no target execution" + } + }, + { + "id": "freedos.edlin.exe", + "file": "EDLIN.EXE", + "bytes": 31492, + "sha256": "96b2b6d543e2d8bdc83e313756444e103339eb5ee2c1229abe3607539b31ad6e", + "provenance": { + "project": "FreeDOS EDLIN", + "version": "2.24", + "upstreamFile": "edlin.zip", + "url": "https://www.ibiblio.org/pub/micro/pc-stuff/freedos/files/repositories/1.4/base/edlin.zip", + "transport_sha256": "9db8a0b50bc4be5c2358cd0fa1fb6fc77342c6d5175a6b1084547c370e428f8a", + "transport_bytes": 377009, + "derivation": "selected regular ZIP member BIN/EDLIN.EXE; original UPX-compressed DOS MZ; APPINFO/EDLIN.LSM reports version; no execution or unpacking" + } + }, { "id": "vscode.tree-sitter.wasm", "file": "tree-sitter.wasm", diff --git a/experiments/analysis-efficiency/measure.mjs b/experiments/analysis-efficiency/measure.mjs index 348bf02c2a..0bfd6da529 100644 --- a/experiments/analysis-efficiency/measure.mjs +++ b/experiments/analysis-efficiency/measure.mjs @@ -1,16 +1,14 @@ import { createHash } from "node:crypto"; import { createReadStream } from "node:fs"; -import { - mkdir, - open, - readFile, - readdir, - stat, - writeFile, -} from "node:fs/promises"; +import { mkdir, readFile, readdir, stat, writeFile } from "node:fs/promises"; import { dirname, isAbsolute, resolve } from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { parseArgs } from "node:util"; +import { + NATIVE_ARTIFACTS, + measureNativeEntry, + nativeEntrySeed, +} from "./native-entry.mjs"; import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { Ajv2020 } from "ajv/dist/2020.js"; @@ -38,10 +36,15 @@ if ( !isAbsolute(values.path) || !isAbsolute(values.output) || !["complete", "summary"].includes(values.delivery) || - !["entry", "entry-and-main"].includes(values["native-question"]) || + !["entry", "entry-and-main", "entry-views"].includes( + values["native-question"], + ) || !["inventory", "selected"].includes(values["javascript-question"]) || !["default", "selected"].includes(values.schemas) || - (values.artifact !== "jq" && values["native-question"] !== "entry") || + (!NATIVE_ARTIFACTS.includes(values.artifact) && + values["native-question"] !== "entry") || + (values.artifact !== "jq" && + values["native-question"] === "entry-and-main") || (values.artifact !== "obsidian.asar" && values["javascript-question"] !== "inventory") ) @@ -118,6 +121,9 @@ await writeFile( ); const fingerprints = { measure_source_sha256: await fileSha(fileURLToPath(import.meta.url)), + native_workflow_source_sha256: await fileSha( + fileURLToPath(new URL("./native-entry.mjs", import.meta.url)), + ), runtime_files_sha256: sha(JSON.stringify(runtimeFiles)), runtime_dispatcher_sha256: await fileSha( resolve(values["runtime-root"], "scripts/rea.mjs"), @@ -545,111 +551,19 @@ const workflows = { evidence_ids: [info.evidence_id], }; }, - jq: async () => { - const file = await open(values.path, "r"); - let header; - try { - header = (await file.read(Buffer.alloc(32), 0, 32, 0)).buffer; - } finally { - await file.close(); - } - if (header.subarray(0, 6).toString("hex") !== "7f454c460201") - throw new Error("Expected ELF64 little endian entry seed"); - const address = `0x${header.readBigUInt64LE(24).toString(16)}`; - await call("open_binary", { path: values.path, provider_id: "ghidra" }); - try { - const containing = await call("resolve_containing_procedure", { - address, - }); - const info = await call("analyze_function", { procedure: address }); - const n = normalized(info); - // This exact application entry is materialized in the pinned jq entry's - // observed MOV RDI, 0x401f00 instruction, not an original-source claim. - let main; - const mainEvidenceIds = []; - if (values["native-question"] === "entry-and-main") { - let sourceId; - try { - const completed = await call("analyze_function", { - procedure: "0x401f00", - }); - sourceId = completed.evidence_id; - } catch (error) { - const reference = - error instanceof ToolFailure && - error.value?.error?.code === "resource_constraint" - ? error.value.error.details?.reported_limits?.evidence_reference - : undefined; - if (reference?.kind !== "retained-evidence") throw error; - sourceId = reference.evidence_id; - } - mainEvidenceIds.push(sourceId); - main = {}; - for (const facet of [ - "procedure", - "pseudocode", - "referenced_strings", - "callees", - "callers", - "basic_blocks", - ]) { - let offset = 0; - const items = []; - const pages = []; - do { - const selected = await call("inspect_analysis_view", { - source: { kind: "retained-evidence", evidence_id: sourceId }, - view: { - kind: "native", - facet, - offset, - limit: - facet === "pseudocode" - ? 65536 - : facet === "basic_blocks" - ? 1 - : 64, - }, - }); - mainEvidenceIds.push(selected.evidence_id); - const v = normalized(selected); - items.push(v.item); - pages.push({ - coverage: v.coverage, - limitations: v.limitations, - unknowns: v.unknowns, - }); - if (v.coverage.exhausted || facet === "basic_blocks") break; - if ( - v.coverage.next_offset === null || - v.coverage.next_offset <= offset - ) - throw new Error("Native view did not advance"); - offset = v.coverage.next_offset; - } while (true); - main[facet] = { items, pages }; - } - } - return { - artifact_sha256: targetSha(info), - entry_address: address, - containing_procedure: normalized(containing), - procedure: n.procedure, - pseudocode_prefix: n.pseudocode.slice(0, 512), - caller_count: n.callers.length, - callee_count: n.callees.length, - ...(main === undefined ? {} : { main }), - analysis_profile: info.analysis_profile, - evidence_ids: [ - containing.evidence_id, - info.evidence_id, - ...mainEvidenceIds, - ], - }; - } finally { - await call("close_binary", {}); - } - }, + ...Object.fromEntries( + NATIVE_ARTIFACTS.map((artifact) => [ + artifact, + async () => + measureNativeEntry({ + seed: await nativeEntrySeed(artifact, values.path), + values, + call, + inputSha, + ToolFailure, + }), + ]), + ), }; try { @@ -740,7 +654,9 @@ const report = { artifact_sha256_after: finalInputSha, delivery: values.delivery, sdk_schema_mode: values.schemas, - native_question: values.artifact === "jq" ? values["native-question"] : null, + native_question: NATIVE_ARTIFACTS.includes(values.artifact) + ? values["native-question"] + : null, javascript_question: values.artifact === "obsidian.asar" ? values["javascript-question"] : null, provider_settings_sha256: sha(JSON.stringify(selectedProviders)), diff --git a/experiments/analysis-efficiency/native-entry.mjs b/experiments/analysis-efficiency/native-entry.mjs new file mode 100644 index 0000000000..6f099fa9f0 --- /dev/null +++ b/experiments/analysis-efficiency/native-entry.mjs @@ -0,0 +1,232 @@ +import { open, readFile } from "node:fs/promises"; +import { isDeepStrictEqual } from "node:util"; + +export const NATIVE_ARTIFACTS = ["jq", "sharex.exe", "freedos.edlin.exe"]; +const normalized = (value) => value.normalized_result; + +const retainedEntryViews = async (call, info, inputSha, ToolFailure) => { + const n = normalized(info); + const retainedViews = []; + let nativeSummaryOutcome; + for (const facet of ["procedure", "pseudocode", "callers", "callees"]) { + const selected = await call("inspect_analysis_view", { + source: { kind: "retained-evidence", evidence_id: info.evidence_id }, + view: { + kind: "native", + facet, + offset: 0, + limit: facet === "pseudocode" ? 512 : 64, + }, + }); + const view = normalized(selected); + const expected = + facet === "procedure" + ? n.procedure + : facet === "pseudocode" + ? { text: n.pseudocode.slice(0, 512), unit: "utf16-code-units" } + : n[facet].slice(0, 64); + if ( + !isDeepStrictEqual(view.item, expected) || + view.parent_evidence_id !== info.evidence_id || + view.artifact.sha256 !== inputSha || + selected.evidence_id === info.evidence_id + ) + throw new Error( + "Retained native view changed the selected source facts or identity", + ); + retainedViews.push({ + facet, + evidence_id: selected.evidence_id, + parent_evidence_id: view.parent_evidence_id, + item: view.item, + coverage: view.coverage, + limitations: view.limitations, + unknowns: view.unknowns, + }); + } + // Exercise the advertised summary recipe separately. A known refusal + // is recorded as such; it does not validate a nonexistent capability. + try { + const summary = await call("inspect_analysis_view", { + source: { kind: "retained-evidence", evidence_id: info.evidence_id }, + view: { kind: "summary" }, + }); + nativeSummaryOutcome = { + status: "supported", + evidence_id: summary.evidence_id, + }; + } catch (error) { + if ( + !(error instanceof ToolFailure) || + error.value?.error?.code !== "invalid_request" + ) + throw error; + nativeSummaryOutcome = { + status: "unsupported", + error: error.value.error, + }; + } + return { retainedViews, nativeSummaryOutcome }; +}; + +const retainedJqMain = async (call, ToolFailure) => { + // This exact application entry is materialized in the pinned jq entry's + // observed MOV RDI, 0x401f00 instruction, not an original-source claim. + let main; + const mainEvidenceIds = []; + let sourceId; + try { + const completed = await call("analyze_function", { + procedure: "0x401f00", + }); + sourceId = completed.evidence_id; + } catch (error) { + const reference = + error instanceof ToolFailure && + error.value?.error?.code === "resource_constraint" + ? error.value.error.details?.reported_limits?.evidence_reference + : undefined; + if (reference?.kind !== "retained-evidence") throw error; + sourceId = reference.evidence_id; + } + mainEvidenceIds.push(sourceId); + main = {}; + for (const facet of [ + "procedure", + "pseudocode", + "referenced_strings", + "callees", + "callers", + "basic_blocks", + ]) { + let offset = 0; + const items = []; + const pages = []; + do { + const selected = await call("inspect_analysis_view", { + source: { kind: "retained-evidence", evidence_id: sourceId }, + view: { + kind: "native", + facet, + offset, + limit: + facet === "pseudocode" ? 65536 : facet === "basic_blocks" ? 1 : 64, + }, + }); + mainEvidenceIds.push(selected.evidence_id); + const v = normalized(selected); + items.push(v.item); + pages.push({ + coverage: v.coverage, + limitations: v.limitations, + unknowns: v.unknowns, + }); + if (v.coverage.exhausted || facet === "basic_blocks") break; + if (v.coverage.next_offset === null || v.coverage.next_offset <= offset) + throw new Error("Native view did not advance"); + offset = v.coverage.next_offset; + } while (true); + main[facet] = { items, pages }; + } + return { main, mainEvidenceIds }; +}; + +export const measureNativeEntry = async ({ + seed, + values, + call, + inputSha, + ToolFailure, +}) => { + await call("open_binary", { path: values.path, provider_id: "ghidra" }); + try { + // DOS addresses follow the actual admitted load image. A file header + // offset is not a provider address or proof of runtime execution. + let loadImage; + if (seed === undefined) { + const imageEvidence = await call("inspect_native_load_image", {}); + loadImage = normalized(imageEvidence); + if (loadImage.status !== "verified") + throw new Error( + "The shipped DOS load image was not independently verified", + ); + seed = loadImage.entry.linear_address; + } + const address = seed; + const containing = await call("resolve_containing_procedure", { + address, + }); + const info = await call("analyze_function", { procedure: address }); + const n = normalized(info); + const selected = + values["native-question"] === "entry-views" + ? await retainedEntryViews(call, info, inputSha, ToolFailure) + : {}; + const { main, mainEvidenceIds = [] } = + values["native-question"] === "entry-and-main" + ? await retainedJqMain(call, ToolFailure) + : {}; + return { + artifact_sha256: info.subject?.digest.sha256, + entry_address: address, + containing_procedure: normalized(containing), + procedure: n.procedure, + pseudocode_prefix: n.pseudocode.slice(0, 512), + caller_count: n.callers.length, + callee_count: n.callees.length, + ...(main === undefined ? {} : { main }), + analysis_profile: info.analysis_profile, + ...(loadImage === undefined ? {} : { load_image: loadImage }), + ...(selected.retainedViews === undefined + ? {} + : { + retained_views: selected.retainedViews, + native_summary_outcome: selected.nativeSummaryOutcome, + }), + evidence_ids: [ + containing.evidence_id, + info.evidence_id, + ...mainEvidenceIds, + ], + }; + } finally { + await call("close_binary", {}); + } +}; + +export const nativeEntrySeed = async (artifact, path) => { + if (artifact === "jq") { + const file = await open(path, "r"); + let header; + try { + header = (await file.read(Buffer.alloc(32), 0, 32, 0)).buffer; + } finally { + await file.close(); + } + if (header.subarray(0, 6).toString("hex") !== "7f454c460201") + throw new Error("Expected ELF64 little endian entry seed"); + return `0x${header.readBigUInt64LE(24).toString(16)}`; + } + if (artifact === "sharex.exe") { + const bytes = await readFile(path); + const pe = bytes.readUInt32LE(0x3c); + const optional = pe + 24; + if ( + bytes.subarray(pe, pe + 4).toString("hex") !== "50450000" || + bytes.readUInt16LE(pe + 4) !== 0x8664 || + bytes.readUInt16LE(optional) !== 0x20b || + bytes.readUInt32LE(optional + 108) <= 14 || + bytes.readUInt32LE(optional + 112 + 14 * 8) !== 0 || + bytes.readUInt32LE(optional + 116 + 14 * 8) !== 0 + ) + throw new Error( + "Expected the pinned native PE32+ entry seed without CLR metadata", + ); + const address = + bytes.readBigUInt64LE(optional + 24) + + BigInt(bytes.readUInt32LE(optional + 16)); + return `0x${address.toString(16)}`; + } + if (artifact === "freedos.edlin.exe") return undefined; + throw new Error("Select a pinned native artifact"); +}; diff --git a/experiments/analysis-efficiency/questions/native-dos.txt b/experiments/analysis-efficiency/questions/native-dos.txt new file mode 100644 index 0000000000..48c52a9ec7 --- /dev/null +++ b/experiments/analysis-efficiency/questions/native-dos.txt @@ -0,0 +1 @@ +Statically inspect {{artifact_path}} using only the registered REA tools and Ghidra. This is the original UPX-compressed FreeDOS EDLIN 2.24 DOS MZ executable; inspect its admitted load image and entry stub, without running or unpacking it. Keep default auto-analysis enabled. Open the target with provider_id ghidra, call inspect_native_load_image, and use its verified entry.linear_address as the seed. Resolve the containing procedure, analyze that procedure once, and close the ephemeral session after obtaining the facts. If a result is clipped, use retained native facets without reanalysis. Report exact target SHA-256, entry_address, containing_procedure, complete procedure metadata including classification, signature, locals and inclusive body ranges, the first 512 UTF-16 units of pseudocode, caller_count, callee_count, real Evidence IDs and limitations. Preserve missing/partial facts and terminal-flow caveats. Verified static import and unpacking-stub pseudocode do not prove DOS execution or recovery of the editor's application logic. Do not execute targets, launch a GUI, or use shell/file tools/network or other agents. Final JSON fields: artifact_sha256, load_image_status, entry_address, containing_procedure, procedure, pseudocode_prefix, caller_count, callee_count, evidence_ids, limitations. Stop when these facts and cleanup are supported; report unsupported outcomes explicitly. diff --git a/experiments/analysis-efficiency/questions/native-pe.txt b/experiments/analysis-efficiency/questions/native-pe.txt new file mode 100644 index 0000000000..749e962003 --- /dev/null +++ b/experiments/analysis-efficiency/questions/native-pe.txt @@ -0,0 +1 @@ +Statically inspect {{artifact_path}} using only the registered REA tools and Ghidra. The independently read, pinned PE32+ header reports image base 0x140000000 and entry RVA 0x125c0, so use address 0x1400125c0 as the analyst seed. This is a native x86-64 Windows PE component with no CLR directory, inspected on Linux; do not claim Windows runtime behavior. Keep default auto-analysis enabled. Open the target with provider_id ghidra, resolve the containing procedure, analyze that procedure once, and close the ephemeral session after obtaining the facts. If a result is clipped, use retained native facets without reanalysis. Report exact target SHA-256, entry_address, containing_procedure, complete procedure metadata including classification, signature, locals and inclusive body ranges, the first 512 UTF-16 units of pseudocode, caller_count, callee_count, real Evidence IDs and limitations. Preserve terminal-flow caveats and unresolved static edges; recovered pseudocode is not original source. Do not execute targets, launch a GUI, or use shell/file tools/network or other agents. Final JSON fields: artifact_sha256, entry_address, containing_procedure, procedure, pseudocode_prefix, caller_count, callee_count, evidence_ids, limitations. The seed applies only to the pinned ShareX.exe in corpus.json. Stop when these facts and cleanup are supported.