test(native): add real PE and DOS entry view experiments

Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>

experiments/analysis-efficiency/compare.mjs

@@ -30,6 +30,7 @@ const sameSettings = [
   "provider_search_path_sha256",
   "provider_environment_policy_sha256",
   "measure_source_sha256",
+  "native_workflow_source_sha256",
   "package_lock_sha256",
   "node_version",
   "sdk_client_version",
@@ -40,7 +41,9 @@ const sameSettings = [
 const settingDifferences = sameSettings.filter(
   (key) =>
     !isDeepStrictEqual(left[key], right[key]) ||
-    (key === "provider_search_path_sha256" &&
+    ((key === "provider_search_path_sha256" ||
+      (key === "native_workflow_source_sha256" &&
+        left.native_question !== null)) &&
       !/^[a-f0-9]{64}$/.test(left[key] ?? "")),
 );
 // Producer/view IDs identify different delivery observations; a named recovery

experiments/analysis-efficiency/corpus.json

@@ -116,6 +116,36 @@
         "derivation": "selected regular ZIP member ShareX.HelpersLib.dll; no target execution"
       }
     },
+    {
+      "id": "sharex.exe",
+      "file": "ShareX.exe",
+      "bytes": 198656,
+      "sha256": "46af1ccc514b395cb703633abb5a505a771942b7db9b1915f31aeea541b3685f",
+      "provenance": {
+        "project": "ShareX native executable component",
+        "version": "19.0.2",
+        "upstreamFile": "ShareX-19.0.2-portable.zip",
+        "url": "https://github.com/ShareX/ShareX/releases/download/v19.0.2/ShareX-19.0.2-portable.zip",
+        "transport_sha256": "967b5d838df5ce25fb5fd7cbd16f1bd797427e2df16907b022b74a4bc0fb9d54",
+        "transport_bytes": 168796487,
+        "derivation": "selected regular ZIP member ShareX.exe; PE32+ x86-64 with no CLR directory; no target execution"
+      }
+    },
+    {
+      "id": "freedos.edlin.exe",
+      "file": "EDLIN.EXE",
+      "bytes": 31492,
+      "sha256": "96b2b6d543e2d8bdc83e313756444e103339eb5ee2c1229abe3607539b31ad6e",
+      "provenance": {
+        "project": "FreeDOS EDLIN",
+        "version": "2.24",
+        "upstreamFile": "edlin.zip",
+        "url": "https://www.ibiblio.org/pub/micro/pc-stuff/freedos/files/repositories/1.4/base/edlin.zip",
+        "transport_sha256": "9db8a0b50bc4be5c2358cd0fa1fb6fc77342c6d5175a6b1084547c370e428f8a",
+        "transport_bytes": 377009,
+        "derivation": "selected regular ZIP member BIN/EDLIN.EXE; original UPX-compressed DOS MZ; APPINFO/EDLIN.LSM reports version; no execution or unpacking"
+      }
+    },
     {
       "id": "vscode.tree-sitter.wasm",
       "file": "tree-sitter.wasm",

experiments/analysis-efficiency/measure.mjs

@@ -1,16 +1,14 @@
 import { createHash } from "node:crypto";
 import { createReadStream } from "node:fs";
-import {
-  mkdir,
-  open,
-  readFile,
-  readdir,
-  stat,
-  writeFile,
-} from "node:fs/promises";
+import { mkdir, readFile, readdir, stat, writeFile } from "node:fs/promises";
 import { dirname, isAbsolute, resolve } from "node:path";
 import { fileURLToPath, pathToFileURL } from "node:url";
 import { parseArgs } from "node:util";
+import {
+  NATIVE_ARTIFACTS,
+  measureNativeEntry,
+  nativeEntrySeed,
+} from "./native-entry.mjs";
 import { Client } from "@modelcontextprotocol/client";
 import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
 import { Ajv2020 } from "ajv/dist/2020.js";
@@ -38,10 +36,15 @@ if (
   !isAbsolute(values.path) ||
   !isAbsolute(values.output) ||
   !["complete", "summary"].includes(values.delivery) ||
-  !["entry", "entry-and-main"].includes(values["native-question"]) ||
+  !["entry", "entry-and-main", "entry-views"].includes(
+    values["native-question"],
+  ) ||
   !["inventory", "selected"].includes(values["javascript-question"]) ||
   !["default", "selected"].includes(values.schemas) ||
-  (values.artifact !== "jq" && values["native-question"] !== "entry") ||
+  (!NATIVE_ARTIFACTS.includes(values.artifact) &&
+    values["native-question"] !== "entry") ||
+  (values.artifact !== "jq" &&
+    values["native-question"] === "entry-and-main") ||
   (values.artifact !== "obsidian.asar" &&
     values["javascript-question"] !== "inventory")
 )
@@ -118,6 +121,9 @@ await writeFile(
 );
 const fingerprints = {
   measure_source_sha256: await fileSha(fileURLToPath(import.meta.url)),
+  native_workflow_source_sha256: await fileSha(
+    fileURLToPath(new URL("./native-entry.mjs", import.meta.url)),
+  ),
   runtime_files_sha256: sha(JSON.stringify(runtimeFiles)),
   runtime_dispatcher_sha256: await fileSha(
     resolve(values["runtime-root"], "scripts/rea.mjs"),
@@ -545,111 +551,19 @@ const workflows = {
       evidence_ids: [info.evidence_id],
     };
   },
-  jq: async () => {
-    const file = await open(values.path, "r");
-    let header;
-    try {
-      header = (await file.read(Buffer.alloc(32), 0, 32, 0)).buffer;
-    } finally {
-      await file.close();
-    }
-    if (header.subarray(0, 6).toString("hex") !== "7f454c460201")
-      throw new Error("Expected ELF64 little endian entry seed");
-    const address = `0x${header.readBigUInt64LE(24).toString(16)}`;
-    await call("open_binary", { path: values.path, provider_id: "ghidra" });
-    try {
-      const containing = await call("resolve_containing_procedure", {
-        address,
-      });
-      const info = await call("analyze_function", { procedure: address });
-      const n = normalized(info);
-      // This exact application entry is materialized in the pinned jq entry's
-      // observed MOV RDI, 0x401f00 instruction, not an original-source claim.
-      let main;
-      const mainEvidenceIds = [];
-      if (values["native-question"] === "entry-and-main") {
-        let sourceId;
-        try {
-          const completed = await call("analyze_function", {
-            procedure: "0x401f00",
-          });
-          sourceId = completed.evidence_id;
-        } catch (error) {
-          const reference =
-            error instanceof ToolFailure &&
-            error.value?.error?.code === "resource_constraint"
-              ? error.value.error.details?.reported_limits?.evidence_reference
-              : undefined;
-          if (reference?.kind !== "retained-evidence") throw error;
-          sourceId = reference.evidence_id;
-        }
-        mainEvidenceIds.push(sourceId);
-        main = {};
-        for (const facet of [
-          "procedure",
-          "pseudocode",
-          "referenced_strings",
-          "callees",
-          "callers",
-          "basic_blocks",
-        ]) {
-          let offset = 0;
-          const items = [];
-          const pages = [];
-          do {
-            const selected = await call("inspect_analysis_view", {
-              source: { kind: "retained-evidence", evidence_id: sourceId },
-              view: {
-                kind: "native",
-                facet,
-                offset,
-                limit:
-                  facet === "pseudocode"
-                    ? 65536
-                    : facet === "basic_blocks"
-                      ? 1
-                      : 64,
-              },
-            });
-            mainEvidenceIds.push(selected.evidence_id);
-            const v = normalized(selected);
-            items.push(v.item);
-            pages.push({
-              coverage: v.coverage,
-              limitations: v.limitations,
-              unknowns: v.unknowns,
-            });
-            if (v.coverage.exhausted || facet === "basic_blocks") break;
-            if (
-              v.coverage.next_offset === null ||
-              v.coverage.next_offset <= offset
-            )
-              throw new Error("Native view did not advance");
-            offset = v.coverage.next_offset;
-          } while (true);
-          main[facet] = { items, pages };
-        }
-      }
-      return {
-        artifact_sha256: targetSha(info),
-        entry_address: address,
-        containing_procedure: normalized(containing),
-        procedure: n.procedure,
-        pseudocode_prefix: n.pseudocode.slice(0, 512),
-        caller_count: n.callers.length,
-        callee_count: n.callees.length,
-        ...(main === undefined ? {} : { main }),
-        analysis_profile: info.analysis_profile,
-        evidence_ids: [
-          containing.evidence_id,
-          info.evidence_id,
-          ...mainEvidenceIds,
-        ],
-      };
-    } finally {
-      await call("close_binary", {});
-    }
-  },
+  ...Object.fromEntries(
+    NATIVE_ARTIFACTS.map((artifact) => [
+      artifact,
+      async () =>
+        measureNativeEntry({
+          seed: await nativeEntrySeed(artifact, values.path),
+          values,
+          call,
+          inputSha,
+          ToolFailure,
+        }),
+    ]),
+  ),
 };
 
 try {
@@ -740,7 +654,9 @@ const report = {
   artifact_sha256_after: finalInputSha,
   delivery: values.delivery,
   sdk_schema_mode: values.schemas,
-  native_question: values.artifact === "jq" ? values["native-question"] : null,
+  native_question: NATIVE_ARTIFACTS.includes(values.artifact)
+    ? values["native-question"]
+    : null,
   javascript_question:
     values.artifact === "obsidian.asar" ? values["javascript-question"] : null,
   provider_settings_sha256: sha(JSON.stringify(selectedProviders)),

experiments/analysis-efficiency/native-entry.mjs

@@ -0,0 +1,232 @@
+import { open, readFile } from "node:fs/promises";
+import { isDeepStrictEqual } from "node:util";
+
+export const NATIVE_ARTIFACTS = ["jq", "sharex.exe", "freedos.edlin.exe"];
+const normalized = (value) => value.normalized_result;
+
+const retainedEntryViews = async (call, info, inputSha, ToolFailure) => {
+  const n = normalized(info);
+  const retainedViews = [];
+  let nativeSummaryOutcome;
+  for (const facet of ["procedure", "pseudocode", "callers", "callees"]) {
+    const selected = await call("inspect_analysis_view", {
+      source: { kind: "retained-evidence", evidence_id: info.evidence_id },
+      view: {
+        kind: "native",
+        facet,
+        offset: 0,
+        limit: facet === "pseudocode" ? 512 : 64,
+      },
+    });
+    const view = normalized(selected);
+    const expected =
+      facet === "procedure"
+        ? n.procedure
+        : facet === "pseudocode"
+          ? { text: n.pseudocode.slice(0, 512), unit: "utf16-code-units" }
+          : n[facet].slice(0, 64);
+    if (
+      !isDeepStrictEqual(view.item, expected) ||
+      view.parent_evidence_id !== info.evidence_id ||
+      view.artifact.sha256 !== inputSha ||
+      selected.evidence_id === info.evidence_id
+    )
+      throw new Error(
+        "Retained native view changed the selected source facts or identity",
+      );
+    retainedViews.push({
+      facet,
+      evidence_id: selected.evidence_id,
+      parent_evidence_id: view.parent_evidence_id,
+      item: view.item,
+      coverage: view.coverage,
+      limitations: view.limitations,
+      unknowns: view.unknowns,
+    });
+  }
+  // Exercise the advertised summary recipe separately. A known refusal
+  // is recorded as such; it does not validate a nonexistent capability.
+  try {
+    const summary = await call("inspect_analysis_view", {
+      source: { kind: "retained-evidence", evidence_id: info.evidence_id },
+      view: { kind: "summary" },
+    });
+    nativeSummaryOutcome = {
+      status: "supported",
+      evidence_id: summary.evidence_id,
+    };
+  } catch (error) {
+    if (
+      !(error instanceof ToolFailure) ||
+      error.value?.error?.code !== "invalid_request"
+    )
+      throw error;
+    nativeSummaryOutcome = {
+      status: "unsupported",
+      error: error.value.error,
+    };
+  }
+  return { retainedViews, nativeSummaryOutcome };
+};
+
+const retainedJqMain = async (call, ToolFailure) => {
+  // This exact application entry is materialized in the pinned jq entry's
+  // observed MOV RDI, 0x401f00 instruction, not an original-source claim.
+  let main;
+  const mainEvidenceIds = [];
+  let sourceId;
+  try {
+    const completed = await call("analyze_function", {
+      procedure: "0x401f00",
+    });
+    sourceId = completed.evidence_id;
+  } catch (error) {
+    const reference =
+      error instanceof ToolFailure &&
+      error.value?.error?.code === "resource_constraint"
+        ? error.value.error.details?.reported_limits?.evidence_reference
+        : undefined;
+    if (reference?.kind !== "retained-evidence") throw error;
+    sourceId = reference.evidence_id;
+  }
+  mainEvidenceIds.push(sourceId);
+  main = {};
+  for (const facet of [
+    "procedure",
+    "pseudocode",
+    "referenced_strings",
+    "callees",
+    "callers",
+    "basic_blocks",
+  ]) {
+    let offset = 0;
+    const items = [];
+    const pages = [];
+    do {
+      const selected = await call("inspect_analysis_view", {
+        source: { kind: "retained-evidence", evidence_id: sourceId },
+        view: {
+          kind: "native",
+          facet,
+          offset,
+          limit:
+            facet === "pseudocode" ? 65536 : facet === "basic_blocks" ? 1 : 64,
+        },
+      });
+      mainEvidenceIds.push(selected.evidence_id);
+      const v = normalized(selected);
+      items.push(v.item);
+      pages.push({
+        coverage: v.coverage,
+        limitations: v.limitations,
+        unknowns: v.unknowns,
+      });
+      if (v.coverage.exhausted || facet === "basic_blocks") break;
+      if (v.coverage.next_offset === null || v.coverage.next_offset <= offset)
+        throw new Error("Native view did not advance");
+      offset = v.coverage.next_offset;
+    } while (true);
+    main[facet] = { items, pages };
+  }
+  return { main, mainEvidenceIds };
+};
+
+export const measureNativeEntry = async ({
+  seed,
+  values,
+  call,
+  inputSha,
+  ToolFailure,
+}) => {
+  await call("open_binary", { path: values.path, provider_id: "ghidra" });
+  try {
+    // DOS addresses follow the actual admitted load image. A file header
+    // offset is not a provider address or proof of runtime execution.
+    let loadImage;
+    if (seed === undefined) {
+      const imageEvidence = await call("inspect_native_load_image", {});
+      loadImage = normalized(imageEvidence);
+      if (loadImage.status !== "verified")
+        throw new Error(
+          "The shipped DOS load image was not independently verified",
+        );
+      seed = loadImage.entry.linear_address;
+    }
+    const address = seed;
+    const containing = await call("resolve_containing_procedure", {
+      address,
+    });
+    const info = await call("analyze_function", { procedure: address });
+    const n = normalized(info);
+    const selected =
+      values["native-question"] === "entry-views"
+        ? await retainedEntryViews(call, info, inputSha, ToolFailure)
+        : {};
+    const { main, mainEvidenceIds = [] } =
+      values["native-question"] === "entry-and-main"
+        ? await retainedJqMain(call, ToolFailure)
+        : {};
+    return {
+      artifact_sha256: info.subject?.digest.sha256,
+      entry_address: address,
+      containing_procedure: normalized(containing),
+      procedure: n.procedure,
+      pseudocode_prefix: n.pseudocode.slice(0, 512),
+      caller_count: n.callers.length,
+      callee_count: n.callees.length,
+      ...(main === undefined ? {} : { main }),
+      analysis_profile: info.analysis_profile,
+      ...(loadImage === undefined ? {} : { load_image: loadImage }),
+      ...(selected.retainedViews === undefined
+        ? {}
+        : {
+            retained_views: selected.retainedViews,
+            native_summary_outcome: selected.nativeSummaryOutcome,
+          }),
+      evidence_ids: [
+        containing.evidence_id,
+        info.evidence_id,
+        ...mainEvidenceIds,
+      ],
+    };
+  } finally {
+    await call("close_binary", {});
+  }
+};
+
+export const nativeEntrySeed = async (artifact, path) => {
+  if (artifact === "jq") {
+    const file = await open(path, "r");
+    let header;
+    try {
+      header = (await file.read(Buffer.alloc(32), 0, 32, 0)).buffer;
+    } finally {
+      await file.close();
+    }
+    if (header.subarray(0, 6).toString("hex") !== "7f454c460201")
+      throw new Error("Expected ELF64 little endian entry seed");
+    return `0x${header.readBigUInt64LE(24).toString(16)}`;
+  }
+  if (artifact === "sharex.exe") {
+    const bytes = await readFile(path);
+    const pe = bytes.readUInt32LE(0x3c);
+    const optional = pe + 24;
+    if (
+      bytes.subarray(pe, pe + 4).toString("hex") !== "50450000" ||
+      bytes.readUInt16LE(pe + 4) !== 0x8664 ||
+      bytes.readUInt16LE(optional) !== 0x20b ||
+      bytes.readUInt32LE(optional + 108) <= 14 ||
+      bytes.readUInt32LE(optional + 112 + 14 * 8) !== 0 ||
+      bytes.readUInt32LE(optional + 116 + 14 * 8) !== 0
+    )
+      throw new Error(
+        "Expected the pinned native PE32+ entry seed without CLR metadata",
+      );
+    const address =
+      bytes.readBigUInt64LE(optional + 24) +
+      BigInt(bytes.readUInt32LE(optional + 16));
+    return `0x${address.toString(16)}`;
+  }
+  if (artifact === "freedos.edlin.exe") return undefined;
+  throw new Error("Select a pinned native artifact");
+};

experiments/analysis-efficiency/questions/native-dos.txt

@@ -0,0 +1 @@
+Statically inspect {{artifact_path}} using only the registered REA tools and Ghidra. This is the original UPX-compressed FreeDOS EDLIN 2.24 DOS MZ executable; inspect its admitted load image and entry stub, without running or unpacking it. Keep default auto-analysis enabled. Open the target with provider_id ghidra, call inspect_native_load_image, and use its verified entry.linear_address as the seed. Resolve the containing procedure, analyze that procedure once, and close the ephemeral session after obtaining the facts. If a result is clipped, use retained native facets without reanalysis. Report exact target SHA-256, entry_address, containing_procedure, complete procedure metadata including classification, signature, locals and inclusive body ranges, the first 512 UTF-16 units of pseudocode, caller_count, callee_count, real Evidence IDs and limitations. Preserve missing/partial facts and terminal-flow caveats. Verified static import and unpacking-stub pseudocode do not prove DOS execution or recovery of the editor's application logic. Do not execute targets, launch a GUI, or use shell/file tools/network or other agents. Final JSON fields: artifact_sha256, load_image_status, entry_address, containing_procedure, procedure, pseudocode_prefix, caller_count, callee_count, evidence_ids, limitations. Stop when these facts and cleanup are supported; report unsupported outcomes explicitly.

experiments/analysis-efficiency/questions/native-pe.txt

@@ -0,0 +1 @@
+Statically inspect {{artifact_path}} using only the registered REA tools and Ghidra. The independently read, pinned PE32+ header reports image base 0x140000000 and entry RVA 0x125c0, so use address 0x1400125c0 as the analyst seed. This is a native x86-64 Windows PE component with no CLR directory, inspected on Linux; do not claim Windows runtime behavior. Keep default auto-analysis enabled. Open the target with provider_id ghidra, resolve the containing procedure, analyze that procedure once, and close the ephemeral session after obtaining the facts. If a result is clipped, use retained native facets without reanalysis. Report exact target SHA-256, entry_address, containing_procedure, complete procedure metadata including classification, signature, locals and inclusive body ranges, the first 512 UTF-16 units of pseudocode, caller_count, callee_count, real Evidence IDs and limitations. Preserve terminal-flow caveats and unresolved static edges; recovered pseudocode is not original source. Do not execute targets, launch a GUI, or use shell/file tools/network or other agents. Final JSON fields: artifact_sha256, entry_address, containing_procedure, procedure, pseudocode_prefix, caller_count, callee_count, evidence_ids, limitations. The seed applies only to the pinned ShareX.exe in corpus.json. Stop when these facts and cleanup are supported.