diff --git a/experiments/analysis-efficiency/native-entry-views.md b/experiments/analysis-efficiency/native-entry-views.md index 05388707b8..1c02fbfd14 100644 --- a/experiments/analysis-efficiency/native-entry-views.md +++ b/experiments/analysis-efficiency/native-entry-views.md @@ -154,6 +154,11 @@ parent ID, while `retained_views` holds its item, coverage, limitations and unkn DOS load-image facts retain their source ID and must match the selected carrier digest before their entry address can be used; a stale result closes the session without querying a procedure. +Derived view IDs are checked against every prior producer reference as well as +other derived views. DOS model grades require the actual completed load-image +response and its reported source ID; status, digest, entry and supplied metadata +are checked against that observation. Missing, failed or unreferenced responses +cannot establish load-image verification. The comparator preserves both runs' reference maps and compares the facts. Existing `entry`/`entry-and-main` answers keep their ID arrays. Historical SDK receipts above retain their original pre-review representation; no replies or measurements were diff --git a/experiments/analysis-efficiency/native-entry.mjs b/experiments/analysis-efficiency/native-entry.mjs index 3f61e5ea21..84bc934ff6 100644 --- a/experiments/analysis-efficiency/native-entry.mjs +++ b/experiments/analysis-efficiency/native-entry.mjs @@ -4,11 +4,17 @@ import { isDeepStrictEqual } from "node:util"; export const NATIVE_ARTIFACTS = ["jq", "sharex.exe", "freedos.edlin.exe"]; const normalized = (value) => value.normalized_result; -const retainedEntryViews = async (call, info, inputSha, ToolFailure) => { +const retainedEntryViews = async ( + call, + info, + inputSha, + ToolFailure, + sourceEvidenceIds, +) => { const n = normalized(info); const retainedViews = []; const retainedViewEvidence = []; - const observedEvidenceIds = new Set([info.evidence_id]); + const observedEvidenceIds = new Set([info.evidence_id, ...sourceEvidenceIds]); let nativeSummaryOutcome, summaryEvidenceId; for (const facet of ["procedure", "pseudocode", "callers", "callees"]) { const selected = await call("inspect_analysis_view", { @@ -175,7 +181,10 @@ export const measureNativeEntry = async ({ const n = normalized(info); const selected = values["native-question"] === "entry-views" - ? await retainedEntryViews(call, info, inputSha, ToolFailure) + ? await retainedEntryViews(call, info, inputSha, ToolFailure, [ + containing.evidence_id, + ...(loadImageEvidenceId === undefined ? [] : [loadImageEvidenceId]), + ]) : {}; const { main, mainEvidenceIds = [] } = values["native-question"] === "entry-and-main" diff --git a/experiments/analysis-efficiency/review-model.py b/experiments/analysis-efficiency/review-model.py index 3f716b40d3..c31a012902 100644 --- a/experiments/analysis-efficiency/review-model.py +++ b/experiments/analysis-efficiency/review-model.py @@ -216,15 +216,27 @@ def main(): check('procedure.body_ranges', procedure.get('body_ranges', body.get('ranges', body.get('ranges_inclusive'))), expected['body']['ranges']) check('procedure.total_bytes', procedure.get('total_bytes', body.get('total_bytes')), expected['body']['total_bytes']) if 'load_image' in reference: - load = reference['load_image'] + sources = [source for source in producer_results.get('inspect_native_load_image', []) + if isinstance(source, dict) + and source.get('operation') == 'inspect_native_load_image' + and isinstance(source.get('evidence_id'), str) + and source.get('evidence_id') in reported_ids + and isinstance(source.get('normalized_result'), dict)] + checks['load_image_source_observed_and_referenced'] = bool(sources) + load = sources[-1]['normalized_result'] if sources else {} + entry = load.get('entry') + entry = entry if isinstance(entry, dict) else {} + check('load_image.target_sha256', load.get('target_sha256'), reference['artifact_sha256']) + check('load_image.source_status', load.get('status'), reference['load_image']['status']) + check('load_image.source_entry', entry.get('linear_address'), reference['entry_address']) status = answer.get('load_image_status') - check('load_image_status', status.get('status') if isinstance(status, dict) else status, load['status']) + check('load_image_status', status.get('status') if isinstance(status, dict) else status, load.get('status')) if isinstance(status, dict): for key in ['format', 'load_segment', 'header_bytes', 'module_bytes', 'overlay_bytes']: if key in status: - check('load_image.' + key, status[key], load[key]) + check('load_image.' + key, status[key], load.get(key)) if 'entry_linear_address' in status: - check('load_image.entry_linear_address', status['entry_linear_address'], load['entry']['linear_address']) + check('load_image.entry_linear_address', status['entry_linear_address'], entry.get('linear_address')) calls = summary['completed_mcp_calls'] checks['default_analysis_not_disabled'] = all(not call['arguments'].get('skip_auto_analysis', False) for call in calls if call['tool'] == 'open_binary') checks['session_close_completed'] = any(call['tool'] == 'close_binary' and call['status'] == 'completed' for call in calls) diff --git a/tests/boundary/cli/nativeEfficiencyComparison.test.ts b/tests/boundary/cli/nativeEfficiencyComparison.test.ts index 1f9b1bf63c..65542f0ea2 100644 --- a/tests/boundary/cli/nativeEfficiencyComparison.test.ts +++ b/tests/boundary/cli/nativeEfficiencyComparison.test.ts @@ -1,5 +1,5 @@ import { execFile } from "node:child_process"; -import { mkdir, writeFile } from "node:fs/promises"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; import { join, resolve } from "node:path"; import { promisify } from "node:util"; @@ -16,7 +16,8 @@ import {measureNativeEntry} from './experiments/analysis-efficiency/native-entry const sha='0'.repeat(64); const duplicate=process.argv[1]==='duplicate'; const stale=process.argv[1]==='stale-dos'; -const dos=process.argv[1]==='dos'||stale; +const collision=process.argv[1]; +const dos=process.argv[1]==='dos'||stale||collision==='load-collision'; const dossier={procedure:{address:'0x1000'},pseudocode:'return;',callers:[],callees:[]}; class ToolFailure extends Error {} const build=async digit=>{ @@ -26,9 +27,9 @@ const build=async digit=>{ if(stale)process.stdout.write(name+'\\n'); if(name==='open_binary'||name==='close_binary')return {}; if(name==='inspect_native_load_image')return {evidence_id:'ev_'+digit.repeat(63)+'6',normalized_result:{status:'verified',target_sha256:stale?'f'.repeat(64):sha,entry:{linear_address:'0x1000'}}}; - if(name==='resolve_containing_procedure')return {evidence_id:parent,normalized_result:{found:true}}; + if(name==='resolve_containing_procedure')return {evidence_id:'ev_'+digit.repeat(63)+'7',normalized_result:{found:true}}; if(name==='analyze_function')return {evidence_id:parent,subject:{digest:{sha256:sha}},normalized_result:dossier}; - const id='ev_'+digit.repeat(63)+(duplicate?1:++ordinal); + const id='ev_'+digit.repeat(63)+(collision==='containing-collision'?7:collision==='load-collision'?6:duplicate?1:++ordinal); if(args.view.kind==='summary')return {evidence_id:id}; const facet=args.view.facet; const item=facet==='pseudocode'?{text:dossier.pseudocode,unit:'utf16-code-units'}:dossier[facet]; @@ -39,6 +40,155 @@ const build=async digit=>{ process.stdout.write(JSON.stringify([await build('1'),await build('2')])); `; +it.each([ + "observed", + "missing", + "errored", + "wrong-target", + "wrong-entry", + "unreferenced", +])( + "binds the DOS model grade to its observed load image: %s", + async (scenario) => { + // Synthetic transcripts exercise grading; they are not real-provider or + // resource measurements. Real DOS transcripts are replayed separately. + const root = await createTestTempDirectory("rea-dos-grader-"); + const trial = join(root, "trial"); + const referenceDirectory = join(root, "reference"); + await mkdir(trial); + await mkdir(referenceDirectory); + const sha = "0".repeat(64); + const imageId = "ev_" + "1".repeat(64); + const procedureId = "ev_" + "2".repeat(64); + const procedure = { + address: "0x1000", + name: "entry", + classification: "observed", + signature: null, + locals: [], + body: { ranges: [], total_bytes: 0 }, + }; + const loadImage = { + status: "verified", + target_sha256: sha, + entry: { linear_address: "0x1000" }, + }; + const reference = { + artifact_sha256: sha, + entry_address: "0x1000", + caller_count: 0, + callee_count: 0, + pseudocode_prefix: "return;", + containing_procedure: { procedure }, + procedure, + load_image: loadImage, + }; + const answer = { + ...reference, + load_image_status: "verified", + evidence_ids: [ + procedureId, + ...(["missing", "errored", "unreferenced"].includes(scenario) + ? [] + : [imageId]), + ], + }; + const event = (tool: string, status: string, value: unknown) => ({ + type: "item.completed", + item: { + type: "mcp_tool_call", + tool, + status, + arguments: {}, + result: { structured_content: value }, + }, + }); + const events = [ + event("analyze_function", "completed", { evidence_id: procedureId }), + ]; + if (scenario !== "missing") + events.push( + event( + "inspect_native_load_image", + scenario === "errored" ? "failed" : "completed", + scenario === "errored" + ? { error: { code: "unsupported_target" } } + : { + operation: "inspect_native_load_image", + evidence_id: imageId, + normalized_result: { + ...loadImage, + target_sha256: + scenario === "wrong-target" ? "f".repeat(64) : sha, + entry: { + linear_address: + scenario === "wrong-entry" ? "0x2000" : "0x1000", + }, + }, + }, + ), + ); + const resourceLog = join(root, "resource.log"); + const output = join(root, "review.json"); + for (const [path, value] of [ + [ + join(trial, "summary.json"), + { + completed_mcp_calls: [ + { tool: "close_binary", status: "completed", arguments: {} }, + ], + }, + ], + [join(trial, "trial-settings.json"), {}], + [join(trial, "cleanup.json"), { profile_removed: true }], + [join(trial, "answer.txt"), answer], + [join(referenceDirectory, "answer.json"), reference], + [ + resourceLog, + { + resource_scope: "synthetic-grader-fixture", + abort: null, + resource_counters_verified: true, + resource_events_final: { + "memory.events": { high: 0, max: 0, oom: 0, oom_kill: 0 }, + "pids.events": { max: 0 }, + }, + }, + ], + ] as const) + await writeFile(path, JSON.stringify(value)); + await writeFile( + join(trial, "events.jsonl"), + events.map((value) => JSON.stringify(value)).join("\n"), + ); + const command = execute("python3", [ + resolve("experiments/analysis-efficiency/review-model.py"), + "--domain", + "native", + "--trial", + trial, + "--reference", + referenceDirectory, + "--resource-log", + resourceLog, + "--output", + output, + ]); + if (scenario === "observed") await command; + else await expect(command).rejects.toMatchObject({ code: 1 }); + const report = JSON.parse(await readFile(output, "utf8")); + expect(report.all_selected_checks_pass).toBe(scenario === "observed"); + if (["missing", "errored", "unreferenced"].includes(scenario)) + expect(report.checks.load_image_source_observed_and_referenced).toBe( + false, + ); + if (scenario === "wrong-target") + expect(report.checks["load_image.target_sha256"]).toBe(false); + if (scenario === "wrong-entry") + expect(report.checks["load_image.source_entry"]).toBe(false); + }, +); + it("retains the DOS load-image source for its facts and derived entry seed", async () => { const { stdout } = await execute(process.execPath, [ "--input-type=module", @@ -70,6 +220,24 @@ it("rejects a stale DOS load image before using its entry and closes the session }); }); +it.each(["containing-collision", "load-collision"])( + "rejects a retained view that reuses an earlier producer identity: %s", + async (collision) => { + await expect( + execute(process.execPath, [ + "--input-type=module", + "-e", + generate, + collision, + ]), + ).rejects.toMatchObject({ + stderr: expect.stringContaining( + "Retained native view reused an Evidence identity", + ), + }); + }, +); + it("rejects a producer that reuses a view identity across facets", async () => { await expect( execute(process.execPath, [