fix(experiments): bind native claims to observed source evidence

Check derived view identities against all earlier producer references. Grade DOS load-image claims only from completed, referenced producer responses, including their actual target and entry identity. Focused synthetic negative transcripts and original real transcript regrades verify the boundary without model reruns.

Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>

experiments/analysis-efficiency/native-entry-views.md

@@ -154,6 +154,11 @@ parent ID, while `retained_views` holds its item, coverage, limitations and unkn
 DOS load-image facts retain their source ID and must match the selected carrier
 digest before their entry address can be used; a stale result closes the session
 without querying a procedure.
+Derived view IDs are checked against every prior producer reference as well as
+other derived views. DOS model grades require the actual completed load-image
+response and its reported source ID; status, digest, entry and supplied metadata
+are checked against that observation. Missing, failed or unreferenced responses
+cannot establish load-image verification.
 The comparator preserves both runs' reference maps and compares the facts. Existing
 `entry`/`entry-and-main` answers keep their ID arrays. Historical SDK receipts above
 retain their original pre-review representation; no replies or measurements were

experiments/analysis-efficiency/native-entry.mjs

@@ -4,11 +4,17 @@ import { isDeepStrictEqual } from "node:util";
 export const NATIVE_ARTIFACTS = ["jq", "sharex.exe", "freedos.edlin.exe"];
 const normalized = (value) => value.normalized_result;
 
-const retainedEntryViews = async (call, info, inputSha, ToolFailure) => {
+const retainedEntryViews = async (
+  call,
+  info,
+  inputSha,
+  ToolFailure,
+  sourceEvidenceIds,
+) => {
   const n = normalized(info);
   const retainedViews = [];
   const retainedViewEvidence = [];
-  const observedEvidenceIds = new Set([info.evidence_id]);
+  const observedEvidenceIds = new Set([info.evidence_id, ...sourceEvidenceIds]);
   let nativeSummaryOutcome, summaryEvidenceId;
   for (const facet of ["procedure", "pseudocode", "callers", "callees"]) {
     const selected = await call("inspect_analysis_view", {
@@ -175,7 +181,10 @@ export const measureNativeEntry = async ({
     const n = normalized(info);
     const selected =
       values["native-question"] === "entry-views"
-        ? await retainedEntryViews(call, info, inputSha, ToolFailure)
+        ? await retainedEntryViews(call, info, inputSha, ToolFailure, [
+            containing.evidence_id,
+            ...(loadImageEvidenceId === undefined ? [] : [loadImageEvidenceId]),
+          ])
         : {};
     const { main, mainEvidenceIds = [] } =
       values["native-question"] === "entry-and-main"

experiments/analysis-efficiency/review-model.py

@@ -216,15 +216,27 @@ def main():
         check('procedure.body_ranges', procedure.get('body_ranges', body.get('ranges', body.get('ranges_inclusive'))), expected['body']['ranges'])
         check('procedure.total_bytes', procedure.get('total_bytes', body.get('total_bytes')), expected['body']['total_bytes'])
         if 'load_image' in reference:
-            load = reference['load_image']
+            sources = [source for source in producer_results.get('inspect_native_load_image', [])
+                       if isinstance(source, dict)
+                       and source.get('operation') == 'inspect_native_load_image'
+                       and isinstance(source.get('evidence_id'), str)
+                       and source.get('evidence_id') in reported_ids
+                       and isinstance(source.get('normalized_result'), dict)]
+            checks['load_image_source_observed_and_referenced'] = bool(sources)
+            load = sources[-1]['normalized_result'] if sources else {}
+            entry = load.get('entry')
+            entry = entry if isinstance(entry, dict) else {}
+            check('load_image.target_sha256', load.get('target_sha256'), reference['artifact_sha256'])
+            check('load_image.source_status', load.get('status'), reference['load_image']['status'])
+            check('load_image.source_entry', entry.get('linear_address'), reference['entry_address'])
             status = answer.get('load_image_status')
-            check('load_image_status', status.get('status') if isinstance(status, dict) else status, load['status'])
+            check('load_image_status', status.get('status') if isinstance(status, dict) else status, load.get('status'))
             if isinstance(status, dict):
                 for key in ['format', 'load_segment', 'header_bytes', 'module_bytes', 'overlay_bytes']:
                     if key in status:
-                        check('load_image.' + key, status[key], load[key])
+                        check('load_image.' + key, status[key], load.get(key))
                 if 'entry_linear_address' in status:
-                    check('load_image.entry_linear_address', status['entry_linear_address'], load['entry']['linear_address'])
+                    check('load_image.entry_linear_address', status['entry_linear_address'], entry.get('linear_address'))
         calls = summary['completed_mcp_calls']
         checks['default_analysis_not_disabled'] = all(not call['arguments'].get('skip_auto_analysis', False) for call in calls if call['tool'] == 'open_binary')
         checks['session_close_completed'] = any(call['tool'] == 'close_binary' and call['status'] == 'completed' for call in calls)

tests/boundary/cli/nativeEfficiencyComparison.test.ts

@@ -1,5 +1,5 @@
 import { execFile } from "node:child_process";
-import { mkdir, writeFile } from "node:fs/promises";
+import { mkdir, readFile, writeFile } from "node:fs/promises";
 import { join, resolve } from "node:path";
 import { promisify } from "node:util";
 
@@ -16,7 +16,8 @@ import {measureNativeEntry} from './experiments/analysis-efficiency/native-entry
 const sha='0'.repeat(64);
 const duplicate=process.argv[1]==='duplicate';
 const stale=process.argv[1]==='stale-dos';
-const dos=process.argv[1]==='dos'||stale;
+const collision=process.argv[1];
+const dos=process.argv[1]==='dos'||stale||collision==='load-collision';
 const dossier={procedure:{address:'0x1000'},pseudocode:'return;',callers:[],callees:[]};
 class ToolFailure extends Error {}
 const build=async digit=>{
@@ -26,9 +27,9 @@ const build=async digit=>{
     if(stale)process.stdout.write(name+'\\n');
     if(name==='open_binary'||name==='close_binary')return {};
     if(name==='inspect_native_load_image')return {evidence_id:'ev_'+digit.repeat(63)+'6',normalized_result:{status:'verified',target_sha256:stale?'f'.repeat(64):sha,entry:{linear_address:'0x1000'}}};
-    if(name==='resolve_containing_procedure')return {evidence_id:parent,normalized_result:{found:true}};
+    if(name==='resolve_containing_procedure')return {evidence_id:'ev_'+digit.repeat(63)+'7',normalized_result:{found:true}};
     if(name==='analyze_function')return {evidence_id:parent,subject:{digest:{sha256:sha}},normalized_result:dossier};
-    const id='ev_'+digit.repeat(63)+(duplicate?1:++ordinal);
+    const id='ev_'+digit.repeat(63)+(collision==='containing-collision'?7:collision==='load-collision'?6:duplicate?1:++ordinal);
     if(args.view.kind==='summary')return {evidence_id:id};
     const facet=args.view.facet;
     const item=facet==='pseudocode'?{text:dossier.pseudocode,unit:'utf16-code-units'}:dossier[facet];
@@ -39,6 +40,155 @@ const build=async digit=>{
 process.stdout.write(JSON.stringify([await build('1'),await build('2')]));
 `;
 
+it.each([
+  "observed",
+  "missing",
+  "errored",
+  "wrong-target",
+  "wrong-entry",
+  "unreferenced",
+])(
+  "binds the DOS model grade to its observed load image: %s",
+  async (scenario) => {
+    // Synthetic transcripts exercise grading; they are not real-provider or
+    // resource measurements. Real DOS transcripts are replayed separately.
+    const root = await createTestTempDirectory("rea-dos-grader-");
+    const trial = join(root, "trial");
+    const referenceDirectory = join(root, "reference");
+    await mkdir(trial);
+    await mkdir(referenceDirectory);
+    const sha = "0".repeat(64);
+    const imageId = "ev_" + "1".repeat(64);
+    const procedureId = "ev_" + "2".repeat(64);
+    const procedure = {
+      address: "0x1000",
+      name: "entry",
+      classification: "observed",
+      signature: null,
+      locals: [],
+      body: { ranges: [], total_bytes: 0 },
+    };
+    const loadImage = {
+      status: "verified",
+      target_sha256: sha,
+      entry: { linear_address: "0x1000" },
+    };
+    const reference = {
+      artifact_sha256: sha,
+      entry_address: "0x1000",
+      caller_count: 0,
+      callee_count: 0,
+      pseudocode_prefix: "return;",
+      containing_procedure: { procedure },
+      procedure,
+      load_image: loadImage,
+    };
+    const answer = {
+      ...reference,
+      load_image_status: "verified",
+      evidence_ids: [
+        procedureId,
+        ...(["missing", "errored", "unreferenced"].includes(scenario)
+          ? []
+          : [imageId]),
+      ],
+    };
+    const event = (tool: string, status: string, value: unknown) => ({
+      type: "item.completed",
+      item: {
+        type: "mcp_tool_call",
+        tool,
+        status,
+        arguments: {},
+        result: { structured_content: value },
+      },
+    });
+    const events = [
+      event("analyze_function", "completed", { evidence_id: procedureId }),
+    ];
+    if (scenario !== "missing")
+      events.push(
+        event(
+          "inspect_native_load_image",
+          scenario === "errored" ? "failed" : "completed",
+          scenario === "errored"
+            ? { error: { code: "unsupported_target" } }
+            : {
+                operation: "inspect_native_load_image",
+                evidence_id: imageId,
+                normalized_result: {
+                  ...loadImage,
+                  target_sha256:
+                    scenario === "wrong-target" ? "f".repeat(64) : sha,
+                  entry: {
+                    linear_address:
+                      scenario === "wrong-entry" ? "0x2000" : "0x1000",
+                  },
+                },
+              },
+        ),
+      );
+    const resourceLog = join(root, "resource.log");
+    const output = join(root, "review.json");
+    for (const [path, value] of [
+      [
+        join(trial, "summary.json"),
+        {
+          completed_mcp_calls: [
+            { tool: "close_binary", status: "completed", arguments: {} },
+          ],
+        },
+      ],
+      [join(trial, "trial-settings.json"), {}],
+      [join(trial, "cleanup.json"), { profile_removed: true }],
+      [join(trial, "answer.txt"), answer],
+      [join(referenceDirectory, "answer.json"), reference],
+      [
+        resourceLog,
+        {
+          resource_scope: "synthetic-grader-fixture",
+          abort: null,
+          resource_counters_verified: true,
+          resource_events_final: {
+            "memory.events": { high: 0, max: 0, oom: 0, oom_kill: 0 },
+            "pids.events": { max: 0 },
+          },
+        },
+      ],
+    ] as const)
+      await writeFile(path, JSON.stringify(value));
+    await writeFile(
+      join(trial, "events.jsonl"),
+      events.map((value) => JSON.stringify(value)).join("\n"),
+    );
+    const command = execute("python3", [
+      resolve("experiments/analysis-efficiency/review-model.py"),
+      "--domain",
+      "native",
+      "--trial",
+      trial,
+      "--reference",
+      referenceDirectory,
+      "--resource-log",
+      resourceLog,
+      "--output",
+      output,
+    ]);
+    if (scenario === "observed") await command;
+    else await expect(command).rejects.toMatchObject({ code: 1 });
+    const report = JSON.parse(await readFile(output, "utf8"));
+    expect(report.all_selected_checks_pass).toBe(scenario === "observed");
+    if (["missing", "errored", "unreferenced"].includes(scenario))
+      expect(report.checks.load_image_source_observed_and_referenced).toBe(
+        false,
+      );
+    if (scenario === "wrong-target")
+      expect(report.checks["load_image.target_sha256"]).toBe(false);
+    if (scenario === "wrong-entry")
+      expect(report.checks["load_image.source_entry"]).toBe(false);
+  },
+);
+
 it("retains the DOS load-image source for its facts and derived entry seed", async () => {
   const { stdout } = await execute(process.execPath, [
     "--input-type=module",
@@ -70,6 +220,24 @@ it("rejects a stale DOS load image before using its entry and closes the session
   });
 });
 
+it.each(["containing-collision", "load-collision"])(
+  "rejects a retained view that reuses an earlier producer identity: %s",
+  async (collision) => {
+    await expect(
+      execute(process.execPath, [
+        "--input-type=module",
+        "-e",
+        generate,
+        collision,
+      ]),
+    ).rejects.toMatchObject({
+      stderr: expect.stringContaining(
+        "Retained native view reused an Evidence identity",
+      ),
+    });
+  },
+);
+
 it("rejects a producer that reuses a view identity across facets", async () => {
   await expect(
     execute(process.execPath, [