diff --git a/experiments/atomic-codemode/README.md b/experiments/atomic-codemode/README.md index 4a7ae4281b..43f165fb7e 100644 --- a/experiments/atomic-codemode/README.md +++ b/experiments/atomic-codemode/README.md @@ -101,7 +101,7 @@ separates argument SSA values from passing storage. `values(is_input:true)` selects retained callee inputs for exact storage comparisons. These candidate links preserve snapshot identities, missing specifications, unknown storage and runtime-binding limits. The [cross-function task and oracle](call-oracle.md) -are fixed before its real-model trials; the native wire format is now version 2. +are fixed before its real-model trials; the graph wire format is version 3, with independent version-1 call facts. `references` now selects incoming exact-address references or outgoing instruction/function references, with native kind filtering, field selection and @@ -160,3 +160,9 @@ CLI/API also verifies the VM aggregates on unpatched stock Kuna 1.833. These are scoped positive results alongside the earlier mixed adapter comparisons, not proof of general superiority. Details and retained failed/partial attempts are in [trials.md](trials.md). + +Recovered calls and their boundary values now survive full-graph capacity +failure. The real VM capture preserves 416 call operations and 1,294 boundary +values while its graph remains unavailable. Values clearly state their partial +scope; definition IDs do not imply readable defining operations. See the +[boundary question and oracle](boundary-oracle.md) for the next pinned model task. diff --git a/experiments/atomic-codemode/api.mjs b/experiments/atomic-codemode/api.mjs index a3f4de5c4a..e35ff2c871 100644 --- a/experiments/atomic-codemode/api.mjs +++ b/experiments/atomic-codemode/api.mjs @@ -83,7 +83,7 @@ export const methods = { 'await rea.references({addresses:["0x2000"]})', ), snapshot: definition( - "Analyze an explicit function entry once; return a retained capture handle, facets and coverage. Reuses the source within this target lease. A valid code facet remains available when recovered IR exceeds capacity or is malformed; inspect facets before graph reads.", + "Analyze an explicit function entry once; return a retained capture handle, facets and coverage. Reuses the source within this target lease. Code, recovered calls and call-boundary values remain independently available when full recovered IR exceeds capacity; inspect facets before reads.", { entry: hex }, "snapshot", 'await rea.snapshot({entry:"0x1000"})', @@ -151,7 +151,7 @@ export const methods = { 'await rea.relations({snapshot:s,kind:"uses",seeds:[op.id]})', ), values: definition( - "Read identified native SSA values in a batch; is_input filters retained operand/output values, not an original source signature. Offset is an address-space offset, not a computed arbitrary-width value. Width, constants, function inputs, annotation status and defining operation are separate facts. Requires native IR.", + "Read identified native SSA values in a batch; is_input filters retained operand/output values, not an original source signature. Offset is an address-space offset, not a computed arbitrary-width value. Width, constants, function inputs, annotation status and defining operation are separate facts. Requires the pinned native exporter.", { snapshot, ids: z.array(z.string()).max(128).optional(), @@ -176,7 +176,7 @@ export const methods = { "await rea.values({snapshot:s,ids:op.inputs.map(i=>i.value_id).filter(Boolean)})", ), calls: definition( - "Select structured recovered call sites and argument bindings. Target comes from native call specifications, never the fspec annotation offset or display text. Each argument separates its SSA value from its passing storage. Input slots are not original source parameter ordinals; missing storage and indirect/runtime targets remain unknown. To investigate across functions, snapshot the selected callee and select values with is_input:true, comparing exact space/offset/width. Requires native IR.", + "Select structured recovered call sites and argument bindings. Target comes from native call specifications, never the fspec annotation offset or display text. Each argument separates its SSA value from its passing storage. Calls and their boundary values survive full graph capacity failure. target_value_id retains input zero independently of target resolution; it may be an fspec annotation or an unresolved indirect target value. A null output describes this recovered operation, not proof of a void source signature. Input slots are not original source parameter ordinals; missing storage and indirect/runtime targets remain unknown. To investigate across functions, snapshot the selected callee and select values with is_input:true, comparing exact space/offset/width. Requires the pinned native exporter.", { snapshot, ids: z.array(z.string()).max(128).optional(), @@ -190,6 +190,8 @@ export const methods = { "output", "prototype", "stack_offset", + "target_value_id", + "is_dead", ]), ) .optional(), @@ -199,7 +201,7 @@ export const methods = { 'await rea.calls({snapshot:s,origin_addresses:["0x1100"]})', ), blocks: definition( - "Read native recovered blocks by stable snapshot-local ID, with ordered predecessors, successor edges and member operation IDs. origin_start describes optimized IR provenance, not a machine block extent. Requires native IR. Runtime successors remain unknown.", + "Read native recovered blocks by stable snapshot-local ID, with ordered predecessors, successor edges and member operation IDs. origin_start describes optimized IR provenance, not a machine block extent. Requires an available native recovered IR graph. Runtime successors remain unknown.", { snapshot, ids: z.array(z.number().int().nonnegative()).max(128).optional(), @@ -261,15 +263,15 @@ const returns = { references: "data:{results:[{ok,address,source:{evidence_id},items:[{source_row,from,to,kind,fromFunction:{address,name}|null,toFunction:{address,name}|null,instruction}],total,next_offset,coverage}|{ok:false,address,error}]}. source_row is the zero-based row in retained native xrefs; Evidence identity plus source_row identifies an occurrence. Coverage preserves examined/excluded/unattributed counts and unknown function scope/discovery. fields projects optional columns; endpoints/kind/source_row stay.", snapshot: - "data:{snapshot:string,entry,source:{evidence_id,sha256},facets:{code:{availability},recovered_ir:{availability,error?}},coverage}. An unavailable graph has unknown/null counts, never a fabricated empty graph; code remains readable through the same handle.", + "data:{snapshot:string,entry,source:{evidence_id,sha256},facets:{code:{availability},recovered_ir:{availability,error?},calls:{availability,count?,error?},values:{availability,scope?,count?,error?}},coverage}. An unavailable graph has unknown/null counts, never a fabricated empty graph; code remains readable through the same handle.", operations: "data:{items:[{id,origin_address,block,source_line,kind,statement,definitions,constants,inputs?,output?,opcode?}],total,next_offset,evidence_id,coverage}. Native inputs:[{input_index,value_id,phi_predecessor,phi_predecessor_edge}], output:value_id|null; definitions:[{definition,use,input_index,value_id,phi_predecessor,phi_predecessor_edge}]. Diagnostic definitions use printed occurrence instead. fields projects columns; identity remains.", relations: "data:{items:definition/use edges OR CFG edges{from,to,operation,polarity?,condition_value?,condition_test?} OR controllers{controller,dependent,edge},total,next_offset,evidence_id,coverage,relation_coverage?}. definitions/uses seeds are operation-ID strings; incoming/outgoing/controllers seeds are numeric block IDs. Native def/use edges include input_index,value_id,phi_predecessor; diagnostic edges include occurrence only.", values: - "data:{items:[{id,space,offset,size_bytes,is_constant,is_input,is_annotation,definition,type_name}],total,next_offset,evidence_id,coverage}; definitions are operation IDs, offsets are exact hex strings.", + "data:{items:[{id,space,offset,size_bytes,is_constant,is_input,is_annotation,definition,type_name}],total,next_offset,evidence_id,coverage}; definitions are operation IDs, offsets are exact hex strings. When the graph is unavailable, only call-boundary values are retained; coverage states that scope and operations/definitions may be unreadable.", calls: - "data:{items:[{id,origin_address,block,opcode,spec_available,target:{space,offset}|null,target_name,arguments:[{input_index,value_id,storage:{space,offset,size_bytes}|null,storage_authority}],output}],total,next_offset,missing,evidence_id,coverage}. fields selects target/target_name/arguments/output/prototype/stack_offset. Calls include all retained call operations; null facts are unknown, not proof of no arguments or runtime binding. Candidate cross-function links require matching target identity and exact passing/input storage; join/subregister/alias interpretation remains unknown.", + "data:{items:[{id,origin_address,block,opcode,spec_available,is_dead,target:{space,offset}|null,target_name,arguments:[{input_index,value_id,storage:{space,offset,size_bytes}|null,storage_authority}],output}],total,next_offset,missing,evidence_id,coverage}. fields selects target/target_name/arguments/output/prototype/stack_offset/target_value_id/is_dead. Values may be limited to the call boundary when the graph is unavailable; coverage states that scope. Calls include all retained call operations; null facts are unknown, not proof of no arguments or runtime binding. Candidate cross-function links require matching target identity and exact passing/input storage; join/subregister/alias interpretation remains unknown.", blocks: "data:{items:[{id,origin_start,operations,predecessors,successors:[{target,index,polarity,condition_value,condition_test}]}],total,next_offset,evidence_id,coverage}; IDs are local to this snapshot.", slice: diff --git a/experiments/atomic-codemode/boundary-oracle.md b/experiments/atomic-codemode/boundary-oracle.md new file mode 100644 index 0000000000..fb7c623c69 --- /dev/null +++ b/experiments/atomic-codemode/boundary-oracle.md @@ -0,0 +1,53 @@ +# Calls and values surviving full-graph capacity + +The question is `sqlite-boundary-question.txt`. Fix it and this acceptance oracle +before either real-model run. Both lanes use the same complete fourteen-method +API, preserved engine, model/effort request, common guidance and resource limits. +This task requires VM decompilation once per fresh lease; it is different work +from the earlier shallow-reference question. Do not compare their task times as +an adapter speedup. Actual client JavaScript dispatch is available in both lanes. + +Artifact: Debian `libsqlite3-0:amd64` version `3.40.1-2+deb12u2`, +`libsqlite3.so.0.8.6`, SHA-256 +`2e6eef9a727f081f0d453b4e5e6cbd8b9ef8b6f86cbf7681cbad444d3b0b55c8`. +The selected VM implementation entry is `0xea6d0`, not its same-name PLT entry. + +Pre-model public MCP capture with the new exporter establishes: + +- Code is available. Full graph is unavailable with `atomic operation capacity + exceeded`. Calls remain available: 416 retained block-member operations, + consisting of 395 CALL and 21 CALLIND. The 395 direct targets are known, all + 21 indirect targets remain unknown. Call-boundary values are available (1,294), + covering targets, arguments and outputs only, with 21,349 operations scanned + over 1,229 recovered blocks. These do not supply a complete definition/use graph. +- At `0xf115c`, recovered call ID `0xf115c:6a84` targets `ram:0x296c0`. + Input zero is `v1000118d6`, `fspec:0x102f4`, annotation true, width 8. + Three recovered inputs use passing storage `register:0x38/8`, `0x30/8`, + `0x10/8` (RDI, RSI, RDX in this verified x86-64 lane). +- At `0xf238b`, call ID `0xf238b:7130` has the same resolved target. + Input zero is `v100012a80`, `fspec:0x1030c`, annotation true, width 8. + Two recovered inputs use passing storage `register:0x38/8`, `0x30/8`. +- Both first arguments are the same snapshot-local SSA value `v30000004b`, + `register:0x38/8`, `is_input:true`, definition null. Other retained argument + values have defining operation IDs outside this boundary view. Exact SSA + identity is supported; source parameter ordinals and general aliases are not. + +Independent GNU objdump decoding from the exported VM entry verifies the two +machine CALL targets and register setup. The earlier complete direct-call pair +oracle contains both. Of the 396 physical direct immediate calls, the only one +absent from the recovered 395 is `0xf2493 -> 0x29370` (stack check failure). +The question does not require rediscovering this extra call; the recovered-call +count must not be claimed as complete machine/runtime coverage. + +Required final facts: correct selected entry and facet availability/scope; both +call-count groups and target-unknown distinction; both resolved targets separated +from annotation offsets; argument slots/locations; shared first-input identity; +selected machine checks; explicit unreadable definition/use and alias/runtime +limits; real Evidence and subject digest. Do not require the model to report +every additional native metadata counter. Check that its API source arguments +show only one VM analysis and all relevant reads reuse its snapshot/Evidence. + +Raw public verification is retained as `facets-v3-mcp.json` and its Evidence +directory under `.cache/atomic-codemode/`. The ordinary VM record (C, mappings, +variables and other ordinary metadata) equals the prior captured engine output. +The independent oracle and precomputed answer are absent from the model workspace. diff --git a/experiments/atomic-codemode/evaluation.md b/experiments/atomic-codemode/evaluation.md index 5786b28694..819dc68292 100644 --- a/experiments/atomic-codemode/evaluation.md +++ b/experiments/atomic-codemode/evaluation.md @@ -148,9 +148,10 @@ with JavaScript dispatch. The evidence supports usable composable call/input facts, without proving that this server adapter is faster or cheaper. Broader complex-software superiority and provider parity remain unproven. -Two remaining boundaries deserve independent experiments. First, calls currently -live inside the full recovered IR facet: exceeding graph capacity loses a small -call view that might otherwise fit. Second, shallow outgoing references should serve a callee/reference list before +Two remaining boundaries deserve independent experiments. First, the initial call view lived inside full recovered IR, so graph capacity +could hide a small otherwise useful call view. The next exporter revision now +captures calls and their boundary values independently; real VM verification +retains both while the graph exceeds capacity. Second, shallow outgoing references should serve a callee/reference list before requesting deeper argument/value recovery. The subsequent implementation now exposes Kuna's existing `xrefs --from` through the shared parser, with native kind filters, explicit scope and paging. Real VM known-target references match an diff --git a/experiments/atomic-codemode/kuna-native-ir.patch b/experiments/atomic-codemode/kuna-native-ir.patch index 7455fb74e4..7c69cbaeda 100644 --- a/experiments/atomic-codemode/kuna-native-ir.patch +++ b/experiments/atomic-codemode/kuna-native-ir.patch @@ -1,48 +1,98 @@ diff --git a/decompiler/crates/kuna-cli/src/atomic_json.rs b/decompiler/crates/kuna-cli/src/atomic_json.rs new file mode 100644 -index 00000000..c2f6f1ad +index 00000000..ebfe18d8 --- /dev/null +++ b/decompiler/crates/kuna-cli/src/atomic_json.rs -@@ -0,0 +1,71 @@ +@@ -0,0 +1,166 @@ +//! Opt-in atomic documents share the ordinary record producer and JSON writer. +//! A graph that cannot fit must not discard otherwise readable reconstructed C. + +use crate::jsonfmt::{dumps_compact_bounded, Json}; + -+pub(crate) fn render(mut document: Json, limit: usize) -> Result { -+ let budget = limit -+ .checked_sub(1) -+ .ok_or("atomic JSON output budget is zero")?; -+ if let Some(mut text) = dumps_compact_bounded(&document, budget) { -+ text.push('\n'); -+ return Ok(text); -+ } -+ if let Json::Object(fields) = &mut document { ++// Ownership is moved out while trying bounded alternatives, never cloned. ++struct RemovedFacet { ++ function: usize, ++ field: usize, ++ value: (String, Json), ++ added_error: bool, ++} ++fn take_facet(document: &mut Json, name: &str, limit: usize) -> Vec { ++ let mut removed = Vec::new(); ++ if let Json::Object(fields) = document { + if let Some(Json::Array(functions)) = fields + .iter_mut() + .find_map(|(key, value)| (key == "functions").then_some(value)) + { -+ for function in functions { -+ if let Json::Object(record) = function { -+ let before = record.len(); -+ record.retain(|(key, _)| key != "atomic_facts"); -+ if record.len() != before { -+ record.push(( -+ "atomic_facts_error".into(), -+ Json::Str(format!( -+ "atomic encoded document exceeds {limit} byte capacity; code retained" -+ )), -+ )); ++ for (function, record) in functions.iter_mut().enumerate() { ++ if let Json::Object(fields) = record { ++ if let Some(field) = fields.iter().position(|(key, _)| key == name) { ++ let value = fields.remove(field); ++ let error = format!("{name}_error"); ++ let added_error = !fields.iter().any(|(key, _)| key == &error); ++ if added_error { ++ fields.push((error, Json::Str(format!("atomic encoded document exceeds {limit} byte capacity; {name} omitted, code retained")))); ++ } ++ removed.push(RemovedFacet { ++ function, ++ field, ++ value, ++ added_error, ++ }); + } + } + } + } + } -+ let mut text = dumps_compact_bounded(&document, budget).ok_or_else(|| { -+ format!("atomic JSON exceeds {limit} byte capacity even without recovered IR") -+ })?; ++ removed ++} ++fn restore_facet(document: &mut Json, removed: Vec) { ++ if let Json::Object(fields) = document { ++ if let Some(Json::Array(functions)) = fields ++ .iter_mut() ++ .find_map(|(key, value)| (key == "functions").then_some(value)) ++ { ++ for facet in removed { ++ if let Json::Object(fields) = &mut functions[facet.function] { ++ if facet.added_error { ++ let error = format!("{}_error", facet.value.0); ++ fields.retain(|(key, _)| key != &error); ++ } ++ fields.insert(facet.field, facet.value); ++ } ++ } ++ } ++ } ++} ++fn bounded(document: &Json, budget: usize) -> Option { ++ let mut text = dumps_compact_bounded(document, budget)?; + text.push('\n'); -+ Ok(text) ++ Some(text) ++} ++pub(crate) fn render(mut document: Json, limit: usize) -> Result { ++ let budget = limit ++ .checked_sub(1) ++ .ok_or("atomic JSON output budget is zero")?; ++ if let Some(text) = bounded(&document, budget) { ++ return Ok(text); ++ } ++ // Preserve existing graph availability if either facet alone would fit. ++ let calls = take_facet(&mut document, "atomic_calls", limit); ++ if !calls.is_empty() { ++ if let Some(text) = bounded(&document, budget) { ++ return Ok(text); ++ } ++ } ++ restore_facet(&mut document, calls); ++ let graph = take_facet(&mut document, "atomic_facts", limit); ++ if !graph.is_empty() { ++ if let Some(text) = bounded(&document, budget) { ++ return Ok(text); ++ } ++ } ++ take_facet(&mut document, "atomic_calls", limit); ++ bounded(&document, budget).ok_or_else(|| { ++ format!("atomic JSON exceeds {limit} byte capacity even without recovered facets") ++ }) +} + +#[cfg(test)] @@ -75,6 +125,51 @@ index 00000000..c2f6f1ad + assert!(render(original, 1).is_err()); + } +} ++ ++#[cfg(test)] ++mod independent_facet_tests { ++ use super::*; ++ fn document(graph: usize, calls: usize) -> Json { ++ Json::Object(vec![( ++ "functions".into(), ++ Json::Array(vec![Json::Object(vec![ ++ ("code".into(), Json::Str("retained C".into())), ++ ("ordinary".into(), Json::Number("42".into())), ++ ("atomic_facts".into(), Json::Str("x".repeat(graph))), ++ ("atomic_calls".into(), Json::Str("y".repeat(calls))), ++ ])]), ++ )]) ++ } ++ #[test] ++ fn each_facet_survives_the_others_encoded_overflow() { ++ for (graph, calls, keep) in [(2000, 1, "atomic_calls"), (1, 2000, "atomic_facts")] { ++ let rendered = render(document(graph, calls), 512).unwrap(); ++ assert!(rendered.len() <= 512); ++ let d: serde_json::Value = serde_json::from_str(&rendered).unwrap(); ++ let f = &d["functions"][0]; ++ assert!(f.get(keep).is_some()); ++ assert_eq!(f["ordinary"], 42); ++ assert_eq!(f["code"], "retained C"); ++ } ++ } ++ #[test] ++ fn jointly_oversized_facets_preserve_the_existing_graph_when_it_fits() { ++ let rendered = render(document(1000, 1000), 1600).unwrap(); ++ let d: serde_json::Value = serde_json::from_str(&rendered).unwrap(); ++ assert!(d["functions"][0].get("atomic_facts").is_some()); ++ assert!(d["functions"][0].get("atomic_calls").is_none()); ++ } ++ #[test] ++ fn base_record_survives_both_overflows_or_fails_explicitly() { ++ let rendered = render(document(2000, 2000), 512).unwrap(); ++ let d: serde_json::Value = serde_json::from_str(&rendered).unwrap(); ++ let f = &d["functions"][0]; ++ assert!(f.get("atomic_facts").is_none() && f.get("atomic_calls").is_none()); ++ assert!(f["atomic_facts_error"].is_string() && f["atomic_calls_error"].is_string()); ++ assert_eq!(f["code"], "retained C"); ++ assert!(render(document(1, 1), 1).is_err()); ++ } ++} diff --git a/decompiler/crates/kuna-cli/src/decompile.rs b/decompiler/crates/kuna-cli/src/decompile.rs index ad249b04..62074996 100644 --- a/decompiler/crates/kuna-cli/src/decompile.rs @@ -213,7 +308,7 @@ index ad249b04..62074996 100644 failure = Some(format!("--assert directive rejected in {}", args.binary)); } diff --git a/decompiler/crates/kuna-cli/src/decompile_all.rs b/decompiler/crates/kuna-cli/src/decompile_all.rs -index 698c7243..dfd9ef01 100644 +index 698c7243..4309f5dc 100644 --- a/decompiler/crates/kuna-cli/src/decompile_all.rs +++ b/decompiler/crates/kuna-cli/src/decompile_all.rs @@ -744,6 +744,7 @@ fn run_jobs_worker(args: &Args) -> Result<(), String> { @@ -269,24 +364,34 @@ index 698c7243..dfd9ef01 100644 /// [`render_result_json`] plus the inventory `total` a NARROWED whole-binary run /// answers with. /// -@@ -2152,6 +2168,17 @@ fn result_json( +@@ -2152,6 +2168,10 @@ fn result_json( // Always present, `[]` unless the option is on. ("types".into(), types), ]; -+ if let Some(facts) = f.detail.as_ref().and_then(|d| d.atomic_facts.as_ref()) { -+ match facts { -+ Ok(facts) => { -+ match serde_json::to_value(facts) { -+ Ok(value) => record.push(("atomic_facts".into(), Json::from(value))), -+ Err(error) => record.push(("atomic_facts_error".into(), Json::Str(format!("atomic facts serialization failed: {error}")))), -+ } -+ } -+ Err(reason) => record.push(("atomic_facts_error".into(), Json::Str(reason.clone()))), -+ } ++ if let Some(detail) = f.detail.as_ref() { ++ append_atomic_facet(&mut record, "atomic_facts", detail.atomic_facts.as_ref()); ++ append_atomic_facet(&mut record, "atomic_calls", detail.atomic_calls.as_ref()); + } if !f.stack_objects.is_empty() { record.push(("stack_objects".into(), Json::Array( f.stack_objects.iter().map(stack_object_json).collect(), +@@ -3519,3 +3539,16 @@ mod triage_tests { + } + + } ++ ++// Named independent facets share serialization/failure semantics; a bad facet ++// cannot erase the ordinary C record or another valid facet. ++fn append_atomic_facet(record: &mut Vec<(String, Json)>, name: &str, facts: Option<&Result>) { ++ if let Some(facts) = facts { ++ let encoded = facts.as_ref().map_err(Clone::clone) ++ .and_then(|facts| serde_json::to_value(facts).map_err(|error| format!("atomic facet serialization failed: {error}"))); ++ match encoded { ++ Ok(value) => record.push((name.into(), Json::from(value))), ++ Err(reason) => record.push((format!("{name}_error"), Json::Str(reason))), ++ } ++ } ++} diff --git a/decompiler/crates/kuna-cli/src/decompile_all/callee_first.rs b/decompiler/crates/kuna-cli/src/decompile_all/callee_first.rs index 3eb2a6e4..2b951ba2 100644 --- a/decompiler/crates/kuna-cli/src/decompile_all/callee_first.rs @@ -586,18 +691,19 @@ index e40b18c9..b8cae233 100644 }; diff --git a/decompiler/crates/kuna-console/src/inspect.rs b/decompiler/crates/kuna-console/src/inspect.rs -index 74024b10..1ecc4fae 100644 +index 74024b10..36fc51a1 100644 --- a/decompiler/crates/kuna-console/src/inspect.rs +++ b/decompiler/crates/kuna-console/src/inspect.rs -@@ -24,6 +24,7 @@ pub const FUNCTION_ROW_CAP: usize = 4096; +@@ -24,6 +24,8 @@ pub const FUNCTION_ROW_CAP: usize = 4096; pub struct FuncDetail { pub tokens: Vec, pub tokens_error: Option, + pub atomic_facts: Option>, ++ pub atomic_calls: Option>, } impl FuncDetail { -@@ -40,13 +41,13 @@ impl FuncDetail { +@@ -40,13 +42,13 @@ impl FuncDetail { ) -> Self { let mut tokens = kuna_srcmap::resolve(fd, var_refs, raw, untrimmed, variables); if let Err(error) = kuna_srcmap::verify(&tokens, code) { @@ -614,7 +720,7 @@ index 74024b10..1ecc4fae 100644 } diff --git a/decompiler/crates/kuna-console/src/project.rs b/decompiler/crates/kuna-console/src/project.rs -index bc401db2..775ca348 100644 +index bc401db2..a3c5fa95 100644 --- a/decompiler/crates/kuna-console/src/project.rs +++ b/decompiler/crates/kuna-console/src/project.rs @@ -245,6 +245,8 @@ pub struct DecompileOptions { @@ -650,7 +756,7 @@ index bc401db2..775ca348 100644 want_pointer_arguments, .. } = *opts; -@@ -838,11 +843,15 @@ pub fn decompile_pulled( +@@ -838,11 +843,16 @@ pub fn decompile_pulled( } } let var_refs = provenance.apply_to_variables_with_refs(&fd, &mut variables); @@ -661,8 +767,9 @@ index bc401db2..775ca348 100644 )) }); + if want_atomic_facts { -+ detail.get_or_insert_with(Default::default).atomic_facts = -+ Some(kuna_decomp::atomic_facts::collect(prog.arch(), &fd)); ++ let detail = detail.get_or_insert_with(Default::default); ++ detail.atomic_calls = Some(kuna_decomp::atomic_calls::collect_all(&fd)); ++ detail.atomic_facts = Some(kuna_decomp::atomic_facts::collect(prog.arch(), &fd)); + } for variable in &mut variables { for address in &mut variable.addresses { @@ -686,19 +793,128 @@ index 40b093b6..8f03b654 100644 [build-dependencies] diff --git a/decompiler/crates/kuna-decomp/src/substrate/atomic_calls.rs b/decompiler/crates/kuna-decomp/src/substrate/atomic_calls.rs new file mode 100644 -index 00000000..c9d9ec70 +index 00000000..91ce7f60 --- /dev/null +++ b/decompiler/crates/kuna-decomp/src/substrate/atomic_calls.rs -@@ -0,0 +1,125 @@ +@@ -0,0 +1,247 @@ +//! Recovered call relations. Argument values and passing storage are separate. +use crate::{ -+ atomic_facts::{operation_id, value_id}, ++ atomic_values::{self, operation_id, value_id, ValueFact}, + context::OpId, + fspec::{FuncCallSpecs, OFFSET_UNKNOWN}, + funcdata::Funcdata, +}; +use kuna_base::address::Address; ++use kuna_num::opcodes::OpCode; +use serde::Serialize; ++use std::collections::{BTreeSet, HashMap}; ++ ++#[derive(Debug, Clone, Serialize)] ++pub struct CallFacts { ++ pub schema_version: u32, ++ pub items: Vec, ++ pub values: Vec, ++ pub coverage: CallCoverage, ++} ++#[derive(Debug, Clone, Serialize)] ++pub struct CallCoverage { ++ pub scope: &'static str, ++ pub blocks_examined: i32, ++ pub operations_examined: usize, ++ pub value_scope: &'static str, ++ pub machine_call_equivalence: &'static str, ++} ++ ++pub fn collect_all(fd: &Funcdata) -> Result { ++ const MAX_CALLS: usize = 12_000; ++ const MAX_VALUES: usize = 36_000; ++ const MAX_INPUTS: usize = 50_000; ++ if fd.bblocks_get_size() <= 0 { ++ return Err("recovered block membership is unavailable".into()); ++ } ++ if fd.num_calls() < 0 || fd.num_calls() as usize > MAX_CALLS { ++ return Err("atomic call-spec capacity exceeded".into()); ++ } ++ let mut specs = HashMap::new(); ++ for i in 0..fd.num_calls() { ++ let spec = fd.get_call_specs(i); ++ if specs.insert(spec.get_op(), spec).is_some() { ++ return Err("duplicate native call-spec operation".into()); ++ } ++ } ++ let mut items = Vec::new(); ++ let mut keys = BTreeSet::new(); ++ let mut inputs = 0usize; ++ let mut text_bytes = 0usize; ++ let mut operations_examined = 0usize; ++ for i in 0..fd.bblocks_get_size() { ++ let block = fd.bblocks_get_block(i); ++ let expected = fd.bb_op_len(block); ++ let mut examined = 0; ++ let mut cursor = fd.bb_op_head(block); ++ // Walk native membership directly; do not allocate a Vec of every op in ++ // a large block merely to select its calls. Length also detects cycles. ++ while let Some(key) = cursor { ++ if examined >= expected { ++ return Err("cyclic native block operation list".into()); ++ } ++ examined += 1; ++ operations_examined += 1; ++ let op = fd.obank().get(key).ok_or("stale block operation")?; ++ if op.get_parent() != Some(block) { ++ return Err("call scan block ownership mismatch".into()); ++ } ++ cursor = op.basic_neighbours().1; ++ if !matches!(op.code(), OpCode::CPUI_CALL | OpCode::CPUI_CALLIND) { ++ continue; ++ } ++ if items.len() >= MAX_CALLS { ++ return Err("atomic call capacity exceeded".into()); ++ } ++ inputs = inputs ++ .checked_add(op.num_input() as usize) ++ .ok_or("atomic call input count overflow")?; ++ if inputs > MAX_INPUTS { ++ return Err("atomic call input capacity exceeded".into()); ++ } ++ let spec = specs.get(&key).copied(); ++ if let Some(spec) = spec { ++ atomic_values::charge_text(&mut text_bytes, spec.get_name().len())?; ++ if spec.proto().has_model() { ++ atomic_values::charge_text( ++ &mut text_bytes, ++ spec.proto().get_model_name().len(), ++ )?; ++ } ++ } ++ for index in 0..op.num_input() { ++ if let Some(v) = op.get_in(index) { ++ keys.insert(v); ++ } ++ } ++ if let Some(v) = op.get_out() { ++ keys.insert(v); ++ } ++ if keys.len() > MAX_VALUES { ++ return Err("atomic call value capacity exceeded".into()); ++ } ++ items.push(collect(fd, key, spec)?); ++ } ++ if examined != expected { ++ return Err("native block operation count mismatch".into()); ++ } ++ } ++ let values = atomic_values::collect(fd, keys, &mut text_bytes)?; ++ Ok(CallFacts { ++ schema_version: 1, items, values, ++ coverage: CallCoverage { ++ scope: "all recovered block-member CALL/CALLIND operations", ++ blocks_examined: fd.bblocks_get_size(), operations_examined, ++ value_scope: "call inputs including target and outputs only; definitions may be outside this facet", ++ machine_call_equivalence: "unknown", ++ }, ++ }) ++} + +#[derive(Debug, Clone, Serialize)] +pub struct Location { @@ -729,6 +945,11 @@ index 00000000..c9d9ec70 +#[derive(Debug, Clone, Serialize)] +pub struct CallFact { + pub id: String, ++ pub origin_address: String, ++ pub block: i32, ++ pub opcode: String, ++ pub is_dead: bool, ++ pub target_value_id: Option, + pub spec_available: bool, + pub target: Option, + pub target_name: Option, @@ -753,7 +974,7 @@ index 00000000..c9d9ec70 + }) +} + -+pub fn collect(fd: &Funcdata, key: OpId, spec: Option<&FuncCallSpecs>) -> Result { ++fn collect(fd: &Funcdata, key: OpId, spec: Option<&FuncCallSpecs>) -> Result { + let op = fd.obank().get(key).ok_or("stale call operation")?; + let arguments = (1..op.num_input()) + .map(|input_index| { @@ -791,6 +1012,14 @@ index 00000000..c9d9ec70 + .collect(); + Ok(CallFact { + id: operation_id(fd, key)?, ++ origin_address: format!("0x{:x}", op.get_seq_num().get_addr().get_offset()), ++ block: fd ++ .bblocks_ref() ++ .block(op.get_parent().ok_or("call has no recovered block")?) ++ .get_index(), ++ opcode: format!("{:?}", op.code()), ++ is_dead: op.is_dead(), ++ target_value_id: op.get_in(0).map(value_id), + spec_available: spec.is_some(), + target: spec.and_then(|s| location(s.get_entry_address())), + target_name: spec @@ -817,21 +1046,20 @@ index 00000000..c9d9ec70 +} diff --git a/decompiler/crates/kuna-decomp/src/substrate/atomic_facts.rs b/decompiler/crates/kuna-decomp/src/substrate/atomic_facts.rs new file mode 100644 -index 00000000..822f9d8e +index 00000000..3dfad1aa --- /dev/null +++ b/decompiler/crates/kuna-decomp/src/substrate/atomic_facts.rs -@@ -0,0 +1,295 @@ +@@ -0,0 +1,231 @@ +//! Opt-in, read-only export of the recovered function IR for the REA experiment. +//! IDs are local to this captured model; addresses alone are not SSA identities. +use crate::{ + architecture::Architecture, -+ context::{OpId, VarnodeId}, ++ atomic_values::{self, operation_id, value_id, ValueFact}, + funcdata::Funcdata, +}; +use kuna_num::opcodes::OpCode; +use serde::Serialize; -+use slotmap::Key; -+use std::collections::{BTreeSet, HashMap}; ++use std::collections::BTreeSet; + +#[derive(Debug, Clone, Serialize)] +pub struct FunctionFacts { @@ -839,7 +1067,6 @@ index 00000000..822f9d8e + pub blocks: Vec, + pub operations: Vec, + pub values: Vec, -+ pub calls: Vec, + pub coverage: Coverage, +} +#[derive(Debug, Clone, Serialize)] @@ -885,30 +1112,6 @@ index 00000000..822f9d8e + pub phi_predecessor: Option, + pub phi_predecessor_edge: Option, +} -+#[derive(Debug, Clone, Serialize)] -+pub struct ValueFact { -+ pub id: String, -+ pub space: Option, -+ pub offset: String, -+ pub size_bytes: i32, -+ pub is_constant: bool, -+ pub is_input: bool, -+ pub is_annotation: bool, -+ pub definition: Option, -+ pub type_name: String, -+} -+pub(super) fn value_id(id: VarnodeId) -> String { -+ format!("v{:x}", id.data().as_ffi()) -+} -+pub(super) fn operation_id(fd: &Funcdata, id: OpId) -> Result { -+ let operation = fd.obank().get(id).ok_or("stale operation identity")?; -+ let seq = operation.get_seq_num(); -+ Ok(format!( -+ "0x{:x}:{:x}", -+ seq.get_addr().get_offset(), -+ seq.get_time() -+ )) -+} + +pub fn collect(arch: &Architecture, fd: &Funcdata) -> Result { + // Bounds cover allocations before JSON serialization, independently of the @@ -925,18 +1128,6 @@ index 00000000..822f9d8e + } + let mut blocks = Vec::new(); + let mut operations = Vec::new(); -+ if fd.num_calls() < 0 || fd.num_calls() as usize > MAX_OPERATIONS { -+ return Err("atomic call-spec capacity exceeded".into()); -+ } -+ // One pass: repeatedly looking up a spec by operation would be quadratic. -+ let mut specs = HashMap::new(); -+ for i in 0..fd.num_calls() { -+ let spec = fd.get_call_specs(i); -+ if specs.insert(spec.get_op(), spec).is_some() { -+ return Err("duplicate native call-spec operation".into()); -+ } -+ } -+ let mut calls = Vec::new(); + let mut value_keys = BTreeSet::new(); + let mut retained_ops = BTreeSet::new(); + let mut input_count = 0usize; @@ -1013,13 +1204,6 @@ index 00000000..822f9d8e + if input_count > MAX_INPUTS { + return Err("atomic input-edge capacity exceeded".into()); + } -+ if matches!(op.code(), OpCode::CPUI_CALL | OpCode::CPUI_CALLIND) { -+ calls.push(crate::atomic_calls::collect( -+ fd, -+ key, -+ specs.get(&key).copied(), -+ )?); -+ } + if op.code() == OpCode::CPUI_BRANCHIND { + coverage.indirect_branch_operations.push(id.clone()); + } @@ -1081,40 +1265,97 @@ index 00000000..822f9d8e + }); + } + } -+ let mut values = Vec::with_capacity(value_keys.len()); -+ let mut missing = BTreeSet::new(); -+ for key in value_keys { ++ let values = atomic_values::collect(fd, value_keys, &mut display_bytes)?; ++ let missing: BTreeSet<_> = values ++ .iter() ++ .filter_map(|v| v.definition.as_ref()) ++ .filter(|id| !retained_ops.contains(*id)) ++ .cloned() ++ .collect(); ++ coverage.omitted_definition_ids = missing.into_iter().collect(); ++ Ok(FunctionFacts { ++ schema_version: 3, ++ blocks, ++ operations, ++ values, ++ coverage, ++ }) ++} +diff --git a/decompiler/crates/kuna-decomp/src/substrate/atomic_values.rs b/decompiler/crates/kuna-decomp/src/substrate/atomic_values.rs +new file mode 100644 +index 00000000..fc509847 +--- /dev/null ++++ b/decompiler/crates/kuna-decomp/src/substrate/atomic_values.rs +@@ -0,0 +1,70 @@ ++//! One read-only interpretation of native value and operation identities. ++use crate::{ ++ context::{OpId, VarnodeId}, ++ funcdata::Funcdata, ++}; ++use serde::Serialize; ++use slotmap::Key; ++use std::collections::BTreeSet; ++ ++pub const MAX_TEXT_BYTES: usize = 2 * 1024 * 1024; ++#[derive(Debug, Clone, Serialize)] ++pub struct ValueFact { ++ pub id: String, ++ pub space: Option, ++ pub offset: String, ++ pub size_bytes: i32, ++ pub is_constant: bool, ++ pub is_input: bool, ++ pub is_annotation: bool, ++ pub definition: Option, ++ pub type_name: String, ++} ++pub fn value_id(id: VarnodeId) -> String { ++ format!("v{:x}", id.data().as_ffi()) ++} ++pub fn operation_id(fd: &Funcdata, id: OpId) -> Result { ++ let operation = fd.obank().get(id).ok_or("stale operation identity")?; ++ let seq = operation.get_seq_num(); ++ Ok(format!( ++ "0x{:x}:{:x}", ++ seq.get_addr().get_offset(), ++ seq.get_time() ++ )) ++} ++pub fn charge_text(used: &mut usize, bytes: usize) -> Result<(), String> { ++ *used = used ++ .checked_add(bytes) ++ .ok_or("atomic text count overflow")?; ++ if *used > MAX_TEXT_BYTES { ++ return Err("atomic text capacity exceeded".into()); ++ } ++ Ok(()) ++} ++pub fn collect( ++ fd: &Funcdata, ++ keys: BTreeSet, ++ text_bytes: &mut usize, ++) -> Result, String> { ++ let mut values = Vec::with_capacity(keys.len()); ++ for key in keys { + let value = fd.vbank().get(key).ok_or("stale input/output value")?; -+ let definition = value.get_def().map(|id| operation_id(fd, id)).transpose()?; -+ if let Some(ref id) = definition { -+ if !retained_ops.contains(id) { -+ missing.insert(id.clone()); -+ } -+ } ++ let type_name = value.get_type().get_name(); ++ charge_text(text_bytes, type_name.len())?; + values.push(ValueFact { + id: value_id(key), + space: value + .get_addr() + .get_space() -+ .map(|space| space.get_name().to_string()), ++ .map(|s| s.get_name().to_string()), + offset: format!("0x{:x}", value.get_offset()), + size_bytes: value.get_size(), + is_constant: value.is_constant(), + is_input: value.is_input(), + is_annotation: value.is_annotation(), -+ definition, -+ type_name: value.get_type().get_name().to_string(), ++ definition: value.get_def().map(|id| operation_id(fd, id)).transpose()?, ++ type_name: type_name.to_string(), + }); + } -+ coverage.omitted_definition_ids = missing.into_iter().collect(); -+ Ok(FunctionFacts { -+ schema_version: 2, -+ blocks, -+ operations, -+ values, -+ calls, -+ coverage, -+ }) ++ Ok(values) +} diff --git a/decompiler/crates/kuna-decomp/src/substrate/funcdata_printraw.rs b/decompiler/crates/kuna-decomp/src/substrate/funcdata_printraw.rs index 39927ceb..844950a6 100644 @@ -1130,23 +1371,24 @@ index 39927ceb..844950a6 100644 let num_input = fd.obank().get(op).ok_or_else(|| "render_op: stale op".to_string())?.num_input(); let in_at = |i: int4| -> Option { fd.obank().get(op).and_then(|o| o.get_in(i)) }; diff --git a/decompiler/crates/kuna-decomp/src/substrate/mod.rs b/decompiler/crates/kuna-decomp/src/substrate/mod.rs -index 00528cb5..be1b84d3 100644 +index 00528cb5..96b10a6c 100644 --- a/decompiler/crates/kuna-decomp/src/substrate/mod.rs +++ b/decompiler/crates/kuna-decomp/src/substrate/mod.rs -@@ -11,6 +11,8 @@ pub mod funcdata_block; +@@ -11,6 +11,9 @@ pub mod funcdata_block; pub mod funcdata_encode; pub mod funcdata_op; pub mod funcdata_printraw; +pub mod atomic_facts; +pub mod atomic_calls; ++pub mod atomic_values; pub mod funcdata_varnode; pub mod dtype; pub mod context; diff --git a/docs/spec/00-overview.md b/docs/spec/00-overview.md -index ce499c4f..3c8e3470 100644 +index ce499c4f..976878d0 100644 --- a/docs/spec/00-overview.md +++ b/docs/spec/00-overview.md -@@ -3431,3 +3431,39 @@ Reserving it creates no analysis varnode and changes no physical mapping or +@@ -3431,3 +3431,50 @@ Reserving it creates no analysis varnode and changes no physical mapping or emitted C. Ghidra receives temporary storage for the logical local, rather than a persistable ordinary stack-variable location; ordinary symbol storage and register-plus-stack parameter JOINs keep their existing encoding. @@ -1178,11 +1420,22 @@ index ce499c4f..3c8e3470 100644 +the command reports failure instead of truncating it or analyzing the target +again. This flag requires `--atomic-facts --json`; ordinary output is unchanged. + -+Schema version 2 adds a structured record for every retained CALL/CALLIND. -+The `FuncCallSpecs` owner supplies the recovered entry, name and prototype -+properties; an fspec annotation's offset is never treated as a callee address. -+Arguments retain their operation input slot and SSA value separately from their -+passing storage. Storage comes from finalized native argument storage only when -+its arity matches, or from a locked prototype. Shifted arguments, placeholders -+and unavailable storage stay unknown. These facts describe the recovered model, -+not original source parameter ordinals, dynamic binding or runtime execution. ++Graph schema version 3 and an independent `atomic_calls` version-1 facet are ++captured from the same live function after C rendering. The call facet scans ++all native block members without allocating a copy of every block's operations; ++it has its own call/input/value/text limits. Full-graph capacity failure leaves ++calls and their boundary values available. Shared value extraction owns the ++native value identities, flags, widths and definitions; boundary values do not ++claim a complete function value/definition graph. ++ ++`FuncCallSpecs` supplies recovered targets, names, prototype flags and passing ++storage. Target input zero remains an SSA value (often an fspec annotation), ++never a callee code address. CALLIND targets can remain unknown. Storage uses ++matching-arity finalized locations or locked prototype storage; shifted inputs ++and placeholders stay unknown. Input slots are not source parameter ordinals. ++ ++The encoded writer attempts bounded facet alternatives without cloning their ++contents. Oversized graph output preserves fitting calls; oversized calls ++preserve a fitting graph. If neither fits, valid C and both failure reasons ++remain. If C itself cannot fit, output fails explicitly without reanalysis. ++Facet errors do not fabricate empty graph/call/value collections or change C. diff --git a/experiments/atomic-codemode/native-call-contract.mjs b/experiments/atomic-codemode/native-call-contract.mjs new file mode 100644 index 0000000000..6d9294b9df --- /dev/null +++ b/experiments/atomic-codemode/native-call-contract.mjs @@ -0,0 +1,57 @@ +import { z } from "zod"; +import { nativeCall } from "./native-calls.mjs"; +import { + ordinal, + value, + rowParser, + readNativeFacet, + requireFact, +} from "./native-contract.mjs"; + +const callFacet = z.strictObject({ + schema_version: z.literal(1), + items: z.array(nativeCall).max(12000), + values: z.array(value).max(36000), + coverage: z.strictObject({ + scope: z.string(), + blocks_examined: ordinal.min(1), + operations_examined: z + .number() + .int() + .nonnegative() + .max(Number.MAX_SAFE_INTEGER), + value_scope: z.string(), + machine_call_equivalence: z.literal("unknown"), + }), +}); +const parseCall = rowParser(nativeCall, { + arguments: (row) => nativeCall.shape.arguments.element.parse(row), +}); +const parseFacet = rowParser(callFacet, { + items: parseCall, + values: (row) => value.parse(row), +}); +export const readCallFacet = (fn) => + readNativeFacet(fn, "atomic_calls", "recovered calls", (raw) => { + requireFact( + raw && typeof raw === "object" && raw.schema_version === 1, + "Unsupported native call facet schema; expected 1", + ); + for (const [name, limit] of [ + ["items", 12000], + ["values", 36000], + ]) + requireFact( + Array.isArray(raw[name]) && raw[name].length <= limit, + `Native call ${name} exceeds capacity or has invalid shape`, + ); + requireFact( + raw.items.reduce( + (sum, row) => + sum + (Array.isArray(row?.arguments) ? row.arguments.length : 0), + 0, + ) <= 50000, + "Native call inputs exceed capacity", + ); + return parseFacet(raw); + }); diff --git a/experiments/atomic-codemode/native-call-views.mjs b/experiments/atomic-codemode/native-call-views.mjs new file mode 100644 index 0000000000..8229e85f06 --- /dev/null +++ b/experiments/atomic-codemode/native-call-views.mjs @@ -0,0 +1,145 @@ +import { callCoverage } from "./native-calls.mjs"; +import { requireFact } from "./native-contract.mjs"; +import { uniqueNative, readNativeIdentities } from "./native-reads.mjs"; +import { freezeFacts } from "./fact-reads.mjs"; + +export function indexCallFacet(facts) { + const calls = uniqueNative(facts.items), + values = uniqueNative(facts.values), + used = new Set(); + requireFact( + calls.size <= facts.coverage.operations_examined, + "Native call count exceeds examined operation coverage", + ); + const reference = (id) => { + requireFact( + id === null || values.has(id), + "Dangling native call-boundary value", + ); + if (id !== null) used.add(id); + }; + for (const row of calls.values()) { + requireFact( + row.id.split(":")[0] === row.origin_address, + "Native call origin/identity mismatch", + ); + reference(row.target_value_id); + reference(row.output); + requireFact( + row.output === null || values.get(row.output).definition === row.id, + "Native call output/definition mismatch", + ); + for (const [i, a] of row.arguments.entries()) { + reference(a.value_id); + requireFact( + a.input_index === i + 1 && + (a.storage === null) === (a.storage_authority === null), + "Native call input/storage binding mismatch", + ); + } + requireFact( + row.spec_available + ? row.prototype !== null + : row.target === null && + row.target_name === null && + row.prototype === null && + row.stack_offset === null && + row.arguments.every((a) => a.storage === null), + "Native missing call specification has invented facts", + ); + } + requireFact( + used.size === values.size, + "Native call boundary contains unrelated values", + ); + for (const value of values.values()) + if (value.definition !== null && calls.has(value.definition)) + requireFact( + calls.get(value.definition).output === value.id, + "Native boundary value/known-call definition mismatch", + ); + return { calls, values }; +} +export function validateCallGraph(callIndex, graphIndex) { + const { calls, values } = callIndex, + { operations } = graphIndex; + for (const row of calls.values()) { + const op = operations.get(row.id); + requireFact( + op && + op.address === row.origin_address && + op.block === row.block && + op.opcode === row.opcode && + op.is_dead === row.is_dead && + op.output === row.output, + "Native call/graph ownership mismatch", + ); + requireFact( + row.target_value_id === (op.inputs[0]?.value_id ?? null) && + row.arguments.length === Math.max(0, op.inputs.length - 1) && + row.arguments.every((a, i) => a.value_id === op.inputs[i + 1].value_id), + "Native call/graph operand mismatch", + ); + } + for (const value of values.values()) + requireFact( + Object.entries(value).every( + ([key, v]) => graphIndex.values.get(value.id)?.[key] === v, + ), + "Native call/graph value mismatch", + ); + for (const op of operations.values()) + requireFact( + !["CPUI_CALL", "CPUI_CALLIND"].includes(op.opcode) || calls.has(op.id), + "Native call operation omitted from call coverage", + ); +} +export function createCallViews(facts, indexes) { + const { calls, values } = indexes; + freezeFacts(facts); + const coverage = freezeFacts({ + ...callCoverage, + ...facts.coverage, + definition_operations: "not collected in call-boundary facet", + }); + const valueCoverage = freezeFacts({ + ...coverage, + scope: facts.coverage.value_scope, + }); + return { + calls: (input) => ({ + ...readNativeIdentities( + calls, + { + fields: [ + "spec_available", + "opcode", + "is_dead", + "target", + "target_name", + "arguments", + "output", + ], + ...input, + ...(input.fields && { + fields: ["spec_available", "opcode", "is_dead", ...input.fields], + }), + }, + (r) => + !input.origin_addresses || + input.origin_addresses.includes(r.origin_address), + ), + coverage, + }), + values: (input) => ({ + ...readNativeIdentities( + values, + input, + (r) => input.is_input === undefined || r.is_input === input.is_input, + ), + coverage: valueCoverage, + }), + count: calls.size, + value_count: values.size, + }; +} diff --git a/experiments/atomic-codemode/native-calls.mjs b/experiments/atomic-codemode/native-calls.mjs index 731dea64c4..5909724a05 100644 --- a/experiments/atomic-codemode/native-calls.mjs +++ b/experiments/atomic-codemode/native-calls.mjs @@ -1,35 +1,32 @@ // The native owner supplies targets and passing locations; never parse display. import { z } from "zod"; -import { FactFailure } from "./fact-error.mjs"; -import { freezeFacts } from "./fact-reads.mjs"; -const hex = z.string().regex(/^0x(?:0|[1-9a-f][0-9a-f]{0,15})$/u); +import { hex, ordinal, valueId, operationId } from "./native-contract.mjs"; const location = z.strictObject({ space: z.string().min(1), offset: hex }); const storage = location.extend({ size_bytes: z.number().int().positive().max(0x7fffffff), }); export const nativeCall = z.strictObject({ - id: z.string().regex(/^0x[0-9a-f]{1,16}:[0-9a-f]{1,8}$/u), + id: operationId, + origin_address: hex, + block: ordinal, + opcode: z.enum(["CPUI_CALL", "CPUI_CALLIND"]), + is_dead: z.boolean(), + target_value_id: valueId.nullable(), spec_available: z.boolean(), target: location.nullable(), target_name: z.string().min(1).nullable(), arguments: z.array( z.strictObject({ input_index: z.number().int().positive().max(0x7fffffff), - value_id: z - .string() - .regex(/^v[0-9a-f]{1,16}$/u) - .nullable(), + value_id: valueId.nullable(), storage: storage.nullable(), storage_authority: z .enum(["final-input-storage", "locked-prototype"]) .nullable(), }), ), - output: z - .string() - .regex(/^v[0-9a-f]{1,16}$/u) - .nullable(), + output: valueId.nullable(), prototype: z .strictObject({ model: z.string().nullable(), @@ -42,65 +39,13 @@ export const nativeCall = z.strictObject({ stack_offset: hex.nullable(), }); -const requireFact = (valid, reason) => { - if (!valid) throw new FactFailure("invalid_provider_output", reason); -}; -export function indexCalls(rows, operations) { - const calls = new Map(); - for (const row of rows) { - const op = operations.get(row.id); - requireFact( - op && - ["CPUI_CALL", "CPUI_CALLIND"].includes(op.opcode) && - !calls.has(row.id), - "Native call ownership or identity mismatch", - ); - requireFact( - row.output === op.output && - row.arguments.length === Math.max(0, op.inputs.length - 1), - "Native call argument/output arity mismatch", - ); - for (const [i, argument] of row.arguments.entries()) - requireFact( - argument.input_index === i + 1 && - argument.value_id === op.inputs[i + 1].value_id && - (argument.storage === null) === (argument.storage_authority === null), - "Native call input/storage binding mismatch", - ); - requireFact( - row.spec_available - ? row.prototype !== null - : row.target === null && - row.target_name === null && - row.prototype === null && - row.stack_offset === null && - row.arguments.every((a) => a.storage === null), - "Native missing call specification has invented facts", - ); - calls.set( - row.id, - freezeFacts({ - ...row, - origin_address: op.address, - block: op.block, - opcode: op.opcode, - }), - ); - } - for (const op of operations.values()) - requireFact( - !["CPUI_CALL", "CPUI_CALLIND"].includes(op.opcode) || calls.has(op.id), - "Native call operation omitted from call coverage", - ); - return calls; -} - export const callCoverage = Object.freeze({ scope: "all retained CALL/CALLIND operations in this recovered function", target_authority: "native recovered call specification; not runtime binding", argument_authority: "native finalized passing storage or locked prototype; null remains unknown", source_parameter_ordinals: "unknown", + source_signature: "unknown", machine_call_equivalence: "unknown", memory_alias_dependencies: "unknown", }); diff --git a/experiments/atomic-codemode/native-calls.test.mjs b/experiments/atomic-codemode/native-calls.test.mjs index 31c31e72fe..17aaf3ebc4 100644 --- a/experiments/atomic-codemode/native-calls.test.mjs +++ b/experiments/atomic-codemode/native-calls.test.mjs @@ -42,8 +42,55 @@ const callOperations = () => [ output: null, }, ]; +const fixtureValues = () => [ + value("v0", "fspec", "0x10000", { is_annotation: true }), + value("v1", "const", "0x2a", { is_constant: true }), + value("v2", "register", "0x30", { is_input: true }), +]; function fixture() { - const operations = callOperations(); + const operations = callOperations(), + values = fixtureValues(); + const rows = [ + { + id: "0x1100:0", + spec_available: true, + target: { space: "ram", offset: "0x2000" }, + target_name: "callee", + arguments: [ + { + input_index: 1, + value_id: "v1", + storage: { space: "register", offset: "0x38", size_bytes: 8 }, + storage_authority: "final-input-storage", + }, + { + input_index: 2, + value_id: "v2", + storage: null, + storage_authority: null, + }, + ], + output: null, + prototype: { + model: "__cdecl", + model_unknown: false, + inputs_locked: false, + variadic: false, + parameter_shift: 0, + }, + stack_offset: null, + }, + { + id: "0x1110:0", + spec_available: false, + target: null, + target_name: null, + arguments: [], + output: null, + prototype: null, + stack_offset: null, + }, + ]; return { binary: "/input", count: 1, @@ -55,7 +102,7 @@ function fixture() { code: "retained code", error: null, atomic_facts: { - schema_version: 2, + schema_version: 3, blocks: [ { id: 0, @@ -66,52 +113,7 @@ function fixture() { }, ], operations, - values: [ - value("v0", "fspec", "0x10000", { is_annotation: true }), - value("v1", "const", "0x2a", { is_constant: true }), - value("v2", "register", "0x30", { is_input: true }), - ], - calls: [ - { - id: "0x1100:0", - spec_available: true, - target: { space: "ram", offset: "0x2000" }, - target_name: "callee", - arguments: [ - { - input_index: 1, - value_id: "v1", - storage: { space: "register", offset: "0x38", size_bytes: 8 }, - storage_authority: "final-input-storage", - }, - { - input_index: 2, - value_id: "v2", - storage: null, - storage_authority: null, - }, - ], - output: null, - prototype: { - model: "__cdecl", - model_unknown: false, - inputs_locked: false, - variadic: false, - parameter_shift: 0, - }, - stack_offset: null, - }, - { - id: "0x1110:0", - spec_available: false, - target: null, - target_name: null, - arguments: [], - output: null, - prototype: null, - stack_offset: null, - }, - ], + values, coverage: { scope: "native graph", machine_semantic_equivalence: "unknown", @@ -120,6 +122,26 @@ function fixture() { issues: [], }, }, + atomic_calls: { + schema_version: 1, + items: rows.map((row, i) => ({ + ...row, + origin_address: operations[i].address, + block: 0, + opcode: operations[i].opcode, + is_dead: false, + target_value_id: operations[i].inputs[0].value_id, + })), + values: structuredClone(values), + coverage: { + scope: "all recovered block-member CALL/CALLIND operations", + blocks_examined: 1, + operations_examined: 2, + value_scope: + "call inputs including target and outputs only; definitions may be outside this facet", + machine_call_equivalence: "unknown", + }, + }, }, ], }; @@ -163,6 +185,7 @@ test("selected call pages and function input values compose without full reports "block", "spec_available", "opcode", + "is_dead", "prototype", ].sort(), ); @@ -177,39 +200,88 @@ test("selected call pages and function input values compose without full reports assert.equal(model.values({ is_input: true, ids: ["v1"] }).total, 0); }); -test("missing, duplicate or forged call relations reject the graph while retaining code", () => { +test("bad call facets preserve valid graph and code; cross-facet conflicts disable call composition", () => { const changes = [ - (facts) => { - facts.calls.pop(); + (f) => { + f.items.pop(); }, - (facts) => { - facts.calls.push(facts.calls[0]); + (f) => { + f.items.push(f.items[0]); }, - (facts) => { - facts.calls[0].arguments[0].value_id = "v2"; + (f) => { + f.items[0].arguments[0].value_id = "v2"; }, - (facts) => { - facts.calls[0].arguments[0].input_index = 0; + (f) => { + f.items[0].arguments[0].input_index = 0; }, - (facts) => { - facts.calls[0].arguments[0].storage_authority = null; + (f) => { + f.items[0].arguments[0].storage_authority = null; }, - (facts) => { - facts.calls[1].target = { space: "ram", offset: "0x2000" }; + (f) => { + f.items[1].target = { space: "ram", offset: "0x2000" }; }, - (facts) => { - facts.schema_version = 1; + (f) => { + f.values[1].offset = "0x2b"; + }, + (f) => { + f.schema_version = 2; }, ]; for (const change of changes) { const doc = fixture(); - change(doc.functions[0].atomic_facts); + change(doc.functions[0].atomic_calls); const model = parseNativeIR(JSON.stringify(doc), "0x1000"); assert.equal(model.code, "retained code"); - assert.equal(model.facets.recovered_ir.availability, "unavailable"); + assert.equal(model.facets.recovered_ir.availability, "available"); + assert.equal(model.facets.calls.availability, "unavailable"); assert.throws( () => model.calls({}), - (error) => error.code === "invalid_provider_output", + (e) => e.code === "invalid_provider_output", ); + assert.equal(model.values({ ids: ["v1"] }).items[0].offset, "0x2a"); } }); + +test("graph capacity failure retains calls, boundary values and explicit unreadable definitions", () => { + const doc = fixture(), + fn = doc.functions[0]; + delete fn.atomic_facts; + fn.atomic_facts_error = "atomic operation capacity exceeded"; + const model = parseNativeIR(JSON.stringify(doc), "0x1000"); + assert.equal(model.facets.recovered_ir.availability, "unavailable"); + assert.equal(model.facets.calls.availability, "available"); + assert.equal(model.facets.values.scope, "call-boundary-only"); + assert.equal(model.calls({}).total, 2); + const values = model.values({ ids: ["v0", "v1", "missing"] }); + assert.equal(values.items[0].space, "fspec"); + assert.equal(values.items[1].offset, "0x2a"); + assert.deepEqual(values.missing, ["missing"]); + assert.match(values.coverage.value_scope, /only/); + assert.equal(values.coverage.scope, values.coverage.value_scope); + assert.throws( + () => model.select({}), + (e) => e.code === "capability_unavailable", + ); + assert.equal( + model.calls({ ids: ["0x1110:0"], fields: ["target_value_id"] }).items[0] + .target_value_id, + "v2", + ); +}); + +test("malformed graph is isolated from valid calls; call-only malformed identities still fail", () => { + const doc = fixture(); + doc.functions[0].atomic_facts.operations[0].output = "vbad"; + let model = parseNativeIR(JSON.stringify(doc), "0x1000"); + assert.equal(model.facets.recovered_ir.availability, "unavailable"); + assert.equal(model.facets.calls.availability, "available"); + assert.equal(model.values({ is_input: true }).items[0].id, "v2"); + doc.functions[0].atomic_calls.items[0].target_value_id = "vdead"; + model = parseNativeIR(JSON.stringify(doc), "0x1000"); + assert.equal(model.facets.calls.availability, "unavailable"); + assert.throws( + () => model.values({}), + (e) => e.code === "invalid_provider_output", + ); + assert.equal(model.code, "retained code"); +}); diff --git a/experiments/atomic-codemode/native-contract.mjs b/experiments/atomic-codemode/native-contract.mjs new file mode 100644 index 0000000000..2a52ffc504 --- /dev/null +++ b/experiments/atomic-codemode/native-contract.mjs @@ -0,0 +1,89 @@ +// Shared native identity/value contracts and bounded row/facet decoding. +import { z } from "zod"; +import { FactFailure } from "./fact-error.mjs"; +import { ok, err } from "../../dist/domain/result.js"; +export const requireFact = (condition, reason) => { + if (!condition) throw new FactFailure("invalid_provider_output", reason); +}; +export const hex = z.string().regex(/^0x(?:0|[1-9a-f][0-9a-f]{0,15})$/u); +export const ordinal = z.number().int().nonnegative().max(0x7fffffff); +export const valueId = z.string().regex(/^v[0-9a-f]{1,16}$/u); +export const operationId = z + .string() + .regex(/^0x[0-9a-f]{1,16}:[0-9a-f]{1,8}$/u); +export const value = z.strictObject({ + id: valueId, + space: z.string().nullable(), + offset: hex, + size_bytes: z.number().int().positive().max(0x7fffffff), + is_constant: z.boolean(), + is_input: z.boolean(), + is_annotation: z.boolean(), + definition: operationId.nullable(), + type_name: z.string(), +}); +export const rowParser = (schema, arrays) => { + const metadata = schema.omit( + Object.fromEntries(Object.keys(arrays).map((key) => [key, true])), + ); + return (raw) => { + const fields = { ...raw }; + for (const key of Object.keys(arrays)) delete fields[key]; + const result = metadata.parse(fields); + for (const [key, parse] of Object.entries(arrays)) { + requireFact(Array.isArray(raw[key]), `Native ${key} must be an array`); + result[key] = raw[key].map(parse); + } + return result; + }; +}; +export const formatFailure = (error) => { + if (error instanceof FactFailure) return error; + const details = + error instanceof z.ZodError + ? { + issues: error.issues.slice(0, 8).map(({ path, code, message }) => ({ + path, + code, + message: message.slice(0, 160), + })), + } + : { reason: String(error.message).slice(0, 256) }; + return new FactFailure( + "invalid_provider_output", + "Native IR source does not match the declared format", + details, + ); +}; + +export const readNativeFacet = (fn, name, label, parse) => { + try { + const reason = fn[`${name}_error`]; + if (reason !== undefined) { + requireFact( + fn[name] === undefined, + `Native ${label} is both present and unavailable`, + ); + return err( + new FactFailure( + "capability_unavailable", + `Native ${label} facet is unavailable`, + { + native_error: reason.slice(0, 1024), + native_error_truncated: reason.length > 1024, + }, + ), + ); + } + if (fn[name] === undefined) + return err( + new FactFailure( + "capability_unavailable", + `Native ${label} facet was not captured`, + ), + ); + return ok(parse(fn[name])); + } catch (error) { + return err(formatFailure(error)); + } +}; diff --git a/experiments/atomic-codemode/native-engine.md b/experiments/atomic-codemode/native-engine.md index c4c584ebd9..f74fa9e9ef 100644 --- a/experiments/atomic-codemode/native-engine.md +++ b/experiments/atomic-codemode/native-engine.md @@ -320,3 +320,69 @@ early-stop budget and are terminated before OOM; they are not reported as passed The changed-test typecheck uses 70.1 MiB and standalone full lint 919.0 MiB. Full-repository typechecking and full upstream Kuna gates remain unverified. The [next question/oracle](reference-oracle.md) are committed before model trials. + +## Independent call and boundary-value facets + +Graph schema 3 removes embedded call records. An independent `atomic_calls` +version-1 facet now carries every retained block-member CALL/CALLIND, target-input +SSA identity, origin/block/opcode/dead status, call specs and boundary values. +`atomic_values.rs` owns common native value interpretation. The call scan walks +native block membership without allocating an operation vector for every block; +its own call/input/value/text budgets do not depend on the 12,000-op graph cap. +The default native path still does not collect either facet and C analysis phases +are unchanged. All captures occur after the same C rendering. + +The bounded serializer retains fitting facets independently and preserves ordinary +C/metadata. It tries the full document, graph without calls, calls without graph, +then the base record with both reasons. This keeps existing graph availability +when both facets individually fit but their combination does not. Facet contents +are moved for these bounded alternatives rather than cloned; no reanalysis or +clipped collection labeled complete is introduced. + +The JS owner validates each facet independently with shared ID/value contracts +and row-wise admission. When both parse, call identities, origins, operands, +outputs and boundary values must agree with the graph owner. Bad call facts do +not erase a valid graph; a bad/oversized graph does not erase valid calls. Code +survives either failure. `values` uses the complete retained graph when present, +otherwise the explicitly scoped call-boundary set; absent IDs in that set do not +prove absence from the native function. Defining operation identities may remain +unreadable, and source signatures, source parameter ordinals, aliases and runtime +binding stay unknown. `is_dead` is always retained on call rows, like an +instruction decode status. CLI, SDK and both MCP adapters share these views. + +The new preserved engine is +`e7d3f0cc8d8aae814bd6a2625a567d6aaad96c3fe2405a4f08dc8e065b851588`. +Its full rebuild took 315.815 family wall / 315.305 CPU seconds, 2,119.4 MiB +kernel process-family peak. Four focused native atomic-JSON tests pass, including +both overflow directions, both-overflow C preservation and graph preference when +the combined document is too large (95.981 / 95.631 seconds including test-target +compilation, 626.8 MiB). The final CLI rebuild after tests is 26.203 / 25.952 +seconds, 492.7 MiB. It is copied outside Cargo outputs and pinned by full digest. +All these runs verify cleanup and OOM 0; full upstream gates remain unexecuted. + +Real public MCP verification keeps VM C available, reports graph operation +capacity failure, and now exposes 416 calls plus 1,294 boundary values. The call +scan examines 21,349 operations over 1,229 blocks. It preserves 395 direct known +targets and 21 unresolved indirect calls; all direct pairs match independent +machine pairs except the omitted `0xf2493 -> 0x29370` stack-check call. All 21 +indirect origin addresses also match independent disassembly. Selected +`0xf115c`/`0xf238b` calls target `0x296c0`; their fspec inputs remain separate +annotation values. Repeated snapshots reuse the same handle and source, and an +explicit unavailable graph read fails without hiding these retained facts. + +The VM source grows from 273,703 to 793,791 bytes to retain this new data; the +wrapper grows from 202,206 to 208,083 bytes. Their ordinary records, including C, +variables and line mappings, are identical to the prior capture. The wrapper's +blocks/operations/values/coverage also remain identical. These are fidelity and +source-volume observations, not model-token or speed results. + +The real reference/VM verifier passes in 49.471 wall / 49.130 CPU seconds, +615.8 MiB. The existing cross-function call/input verifier passes in 8.377 / +8.094 seconds, 292.9 MiB; encoded Pragma/parser C/graph checks pass in 12.403 / +12.195 seconds, 335.0 MiB. All verify cleanup and OOM 0. The updated contract +suite passes 47 JS tests, including independent facet failure, cross-facet +conflicts, input-zero identity and boundary-only value reads. `check-spec` passes +in default lenient mode, lint and changed-file formatting pass, and the patch +applies forward to the pinned upstream revision and reverses from the clone. +Full repository typechecking retains its previously recorded resource limitation. +The [boundary question/oracle](boundary-oracle.md) are fixed before model runs. diff --git a/experiments/atomic-codemode/native-ir-contract.mjs b/experiments/atomic-codemode/native-ir-contract.mjs index 380f666490..67614dbb18 100644 --- a/experiments/atomic-codemode/native-ir-contract.mjs +++ b/experiments/atomic-codemode/native-ir-contract.mjs @@ -1,17 +1,19 @@ // Native JSON decoding and resource admission; row-wise validation bounds errors. import { z } from "zod"; import { FactFailure } from "./fact-error.mjs"; -import { ok, err } from "../../dist/domain/result.js"; import { NATIVE_OUTPUT_BYTES } from "./limits.mjs"; -import { nativeCall } from "./native-calls.mjs"; -const requireFact = (condition, reason) => { - if (!condition) throw new FactFailure("invalid_provider_output", reason); -}; - -const hex = z.string().regex(/^0x(?:0|[1-9a-f][0-9a-f]{0,15})$/u); -const ordinal = z.number().int().nonnegative().max(0x7fffffff); -const valueId = z.string().regex(/^v[0-9a-f]{1,16}$/u); -const operationId = z.string().regex(/^0x[0-9a-f]{1,16}:[0-9a-f]{1,8}$/u); +import { readCallFacet } from "./native-call-contract.mjs"; +import { + hex, + ordinal, + valueId, + operationId, + value, + rowParser, + requireFact, + formatFailure, + readNativeFacet, +} from "./native-contract.mjs"; const edgeId = z.string().regex(/^\d+:\d+$/u); const edge = z.strictObject({ id: edgeId, @@ -38,19 +40,8 @@ const operation = z.strictObject({ ), output: valueId.nullable(), }); -const value = z.strictObject({ - id: valueId, - space: z.string().nullable(), - offset: hex, - size_bytes: z.number().int().positive().max(0x7fffffff), - is_constant: z.boolean(), - is_input: z.boolean(), - is_annotation: z.boolean(), - definition: operationId.nullable(), - type_name: z.string(), -}); const factsSchema = z.strictObject({ - schema_version: z.literal(2), + schema_version: z.literal(3), blocks: z .array( z.strictObject({ @@ -65,7 +56,6 @@ const factsSchema = z.strictObject({ .max(12000), operations: z.array(operation).max(12000), values: z.array(value).max(36000), - calls: z.array(nativeCall).max(12000), coverage: z.strictObject({ scope: z.string(), machine_semantic_equivalence: z.literal("unknown"), @@ -87,6 +77,8 @@ const documentSchema = z.object({ error: z.null(), atomic_facts: z.unknown().optional(), atomic_facts_error: z.string().optional(), + atomic_calls: z.unknown().optional(), + atomic_calls_error: z.string().optional(), }), ) .length(1), @@ -94,21 +86,6 @@ const documentSchema = z.object({ // Parse one row at a time: a bad provider must not expand thousands of malformed // records into a giant Zod issue list before the caller can reject the source. -const rowParser = (schema, arrays) => { - const metadata = schema.omit( - Object.fromEntries(Object.keys(arrays).map((key) => [key, true])), - ); - return (raw) => { - const fields = { ...raw }; - for (const key of Object.keys(arrays)) delete fields[key]; - const result = metadata.parse(fields); - for (const [key, parse] of Object.entries(arrays)) { - requireFact(Array.isArray(raw[key]), `Native ${key} must be an array`); - result[key] = raw[key].map(parse); - } - return result; - }; -}; const nativeBlock = factsSchema.shape.blocks.element; const parseBlock = rowParser(nativeBlock, { operations: (id) => operationId.parse(id), @@ -118,14 +95,10 @@ const parseBlock = rowParser(nativeBlock, { const parseOperation = rowParser(operation, { inputs: (row) => operation.shape.inputs.element.parse(row), }); -const parseCall = rowParser(nativeCall, { - arguments: (row) => nativeCall.shape.arguments.element.parse(row), -}); const parseFacts = rowParser(factsSchema, { blocks: parseBlock, operations: parseOperation, values: (row) => value.parse(row), - calls: parseCall, }); function preflight(facts) { @@ -134,8 +107,8 @@ function preflight(facts) { "Native atomic facts facet is unavailable; inspect the retained source for its reason and C", ); requireFact( - facts.schema_version === 2, - "Unsupported native atomic facts schema version; expected 2", + facts.schema_version === 3, + "Unsupported native atomic facts schema version; expected 3", ); requireFact( facts.blocks?.length > 0, @@ -145,7 +118,6 @@ function preflight(facts) { ["blocks", 12000], ["operations", 12000], ["values", 36000], - ["calls", 12000], ]) requireFact( Array.isArray(facts[key]) && facts[key].length <= limit, @@ -168,60 +140,15 @@ function preflight(facts) { 0, ); requireFact( - inputs <= 50000 && - members <= 12000 && - edges <= 100000 && - facts.calls.reduce( - (sum, call) => - sum + (Array.isArray(call?.arguments) ? call.arguments.length : 0), - 0, - ) <= 50000, + inputs <= 50000 && members <= 12000 && edges <= 100000, "Native relationships exceed resource capacity", ); } -const formatFailure = (error) => { - if (error instanceof FactFailure) return error; - const details = - error instanceof z.ZodError - ? { - issues: error.issues.slice(0, 8).map(({ path, code, message }) => ({ - path, - code, - message: message.slice(0, 160), - })), - } - : { reason: String(error.message).slice(0, 256) }; - return new FactFailure( - "invalid_provider_output", - "Native IR source does not match the declared format", - details, - ); -}; - -const readGraph = (fn) => { - try { - if (fn.atomic_facts_error !== undefined) { - requireFact( - fn.atomic_facts === undefined, - "Native graph is both present and reported unavailable", - ); - return err( - new FactFailure( - "capability_unavailable", - "Native recovered IR facet is unavailable", - { - native_error: fn.atomic_facts_error.slice(0, 1024), - native_error_truncated: fn.atomic_facts_error.length > 1024, - }, - ), - ); - } - preflight(fn.atomic_facts); - return ok(parseFacts(fn.atomic_facts)); - } catch (error) { - return err(formatFailure(error)); - } -}; +const readGraph = (fn) => + readNativeFacet(fn, "atomic_facts", "recovered IR", (raw) => { + preflight(raw); + return parseFacts(raw); + }); export function parseNativeFunction(text, entry, expectedInput) { try { @@ -247,7 +174,12 @@ export function parseNativeFunction(text, entry, expectedInput) { "subject_mismatch", "Native function entry differs from requested subject", ); - return { code: fn.code, language: doc.language, facts: readGraph(fn) }; + return { + code: fn.code, + language: doc.language, + facts: readGraph(fn), + calls: readCallFacet(fn), + }; } catch (error) { throw formatFailure(error); } diff --git a/experiments/atomic-codemode/native-ir.mjs b/experiments/atomic-codemode/native-ir.mjs index 3ea96fe229..6c78e8ffe8 100644 --- a/experiments/atomic-codemode/native-ir.mjs +++ b/experiments/atomic-codemode/native-ir.mjs @@ -5,19 +5,21 @@ import { parseNativeFunction } from "./native-ir-contract.mjs"; import { backwardSlice } from "../atomic-re-tools/kuna-ssa.mjs"; import { controlDependencies } from "./control-dependence.mjs"; import { freezeFacts as freeze } from "./fact-reads.mjs"; -import { indexCalls, callCoverage } from "./native-calls.mjs"; +import { + indexCallFacet, + validateCallGraph, + createCallViews, +} from "./native-call-views.mjs"; +import { requireFact } from "./native-contract.mjs"; +import { + uniqueNative as unique, + nativePage as page, + readNativeIdentities as readIdentities, +} from "./native-reads.mjs"; const index = (map, key, row) => { if (!map.has(key)) map.set(key, []); map.get(key).push(row); }; -const unique = (rows) => { - const map = new Map(); - for (const row of rows) { - if (map.has(row.id)) throw new Error(`Duplicate native identity ${row.id}`); - map.set(row.id, row); - } - return map; -}; const kind = (opcode) => ({ CPUI_LOAD: "load", @@ -30,35 +32,6 @@ const kind = (opcode) => CPUI_BRANCHIND: "switch", CPUI_RETURN: "return", })[opcode] ?? "value"; -const page = (rows, { offset = 0, limit = 20, fields } = {}) => ({ - items: rows - .slice(offset, offset + limit) - .map((row) => - fields - ? Object.fromEntries( - [...new Set(["id", "origin_address", "block", ...fields])] - .filter((key) => Object.hasOwn(row, key)) - .map((key) => [key, row[key]]), - ) - : row, - ), - total: rows.length, - next_offset: offset + limit < rows.length ? offset + limit : null, -}); -const requireFact = (condition, reason) => { - if (!condition) throw new FactFailure("invalid_provider_output", reason); -}; -const readIdentities = (index, input, filter = () => true) => ({ - ...page( - (input.ids - ? input.ids.flatMap((id) => index.get(id) ?? []) - : [...index.values()] - ).filter(filter), - input, - ), - missing: input.ids?.filter((id) => !index.has(id)) ?? [], -}); - function validateBlocks(operations, values, blocks) { const membership = new Set(); for (const block of blocks.values()) { @@ -149,7 +122,6 @@ function indexNative(facts) { validateBlocks(operations, values, blocks); validateOperations(operations, values, blocks); validateValues(facts, operations, values); - const calls = indexCalls(facts.calls, operations); const users = new Map(), incoming = new Map(), outgoing = new Map(), @@ -215,21 +187,12 @@ function indexNative(facts) { outgoing, constants, normalized, - calls, }; } function createViews(facts, code, indexes) { - const { - values, - blocks, - users, - incoming, - outgoing, - constants, - normalized, - calls, - } = indexes; + const { values, blocks, users, incoming, outgoing, constants, normalized } = + indexes; const coverage = freeze({ ...facts.coverage, operations: normalized.size, @@ -254,35 +217,14 @@ function createViews(facts, code, indexes) { fields: ["kind", "statement", "definitions", "constants"], ...input, }), - values: (input) => - readIdentities( + values: (input) => ({ + ...readIdentities( values, input, (value) => input.is_input === undefined || value.is_input === input.is_input, ), - calls: (input) => ({ - ...readIdentities( - calls, - { - fields: [ - "spec_available", - "opcode", - "target", - "target_name", - "arguments", - "output", - ], - ...input, - ...(input.fields && { - fields: ["spec_available", "opcode", ...input.fields], - }), - }, - (row) => - !input.origin_addresses || - input.origin_addresses.includes(row.origin_address), - ), - coverage: callCoverage, + coverage, }), blocks: (input) => readIdentities(blocks, input), edges: (input) => { @@ -380,50 +322,64 @@ const projectedFailure = (error) => ({ details: error.details ?? {}, }); +const captureFacets = (graph, calls, graphError, callError) => + freeze({ + code: { availability: "available" }, + recovered_ir: graph + ? { availability: "available" } + : { availability: "unavailable", error: graphError }, + calls: calls + ? { availability: "available", count: calls.count } + : { availability: "unavailable", error: callError }, + values: graph + ? { availability: "available", scope: "complete-for-recovered-graph" } + : calls + ? { + availability: "available", + scope: "call-boundary-only", + count: calls.value_count, + } + : { availability: "unavailable", error: callError }, + }); + export function parseNativeIR(text, entry, expectedInput) { const capture = parseNativeFunction(text, entry, expectedInput); - let graph = capture.facts; - if (graph.ok) { + let graph, graphIndex, graphError, calls, callError; + if (capture.facts.ok) { try { - freeze(graph.value); - const model = createViews( - graph.value, - capture.code, - indexNative(graph.value), - ); - return Object.freeze({ - ...model, - language: capture.language, - facets: freeze({ - code: { availability: "available" }, - recovered_ir: { availability: "available" }, - }), - }); + freeze(capture.facts.value); + graphIndex = indexNative(capture.facts.value); + graph = createViews(capture.facts.value, capture.code, graphIndex); } catch (error) { - graph = { ok: false, error }; + graphError = projectedFailure(error); } - } - const error = projectedFailure(graph.error); - const unavailable = () => { + } else graphError = projectedFailure(capture.facts.error); + if (capture.calls.ok) { + try { + const callIndex = indexCallFacet(capture.calls.value); + if (graph && graphIndex) validateCallGraph(callIndex, graphIndex); + calls = createCallViews(capture.calls.value, callIndex); + } catch (error) { + callError = projectedFailure(error); + } + } else callError = projectedFailure(capture.calls.error); + const unavailable = (facet, error) => () => { throw new FactFailure(error.code, error.message, { ...error.details, - facet: "recovered_ir", + facet, }); }; - return Object.freeze({ + const unavailableGraph = unavailable("recovered_ir", graphError ?? {}); + const unavailableCalls = unavailable("calls", callError ?? {}); + const fallback = { code: capture.code, - language: capture.language, - facets: freeze({ - code: { availability: "available" }, - recovered_ir: { availability: "unavailable", error }, - }), coverage: freeze({ scope: "native reconstructed code; recovered IR facet unavailable", operations: null, blocks: null, control_flow: "unknown", machine_semantic_equivalence: "unknown", - issues: [error.message], + issues: [graphError?.message], }), ...Object.fromEntries( [ @@ -433,12 +389,17 @@ export function parseNativeIR(text, entry, expectedInput) { "uses", "incoming", "select", - "values", - "calls", "blocks", "edges", "slice", - ].map((name) => [name, unavailable]), + ].map((name) => [name, unavailableGraph]), ), + }; + return Object.freeze({ + ...(graph ?? fallback), + language: capture.language, + calls: calls?.calls ?? unavailableCalls, + values: graph?.values ?? calls?.values ?? unavailableCalls, + facets: captureFacets(graph, calls, graphError, callError), }); } diff --git a/experiments/atomic-codemode/native-ir.test.mjs b/experiments/atomic-codemode/native-ir.test.mjs index 3d4d5abdbc..57e5c45f29 100644 --- a/experiments/atomic-codemode/native-ir.test.mjs +++ b/experiments/atomic-codemode/native-ir.test.mjs @@ -53,8 +53,7 @@ function diamond() { error: null, code: "reconstructed C", atomic_facts: { - schema_version: 2, - calls: [], + schema_version: 3, blocks: [ { id: 0, diff --git a/experiments/atomic-codemode/native-reads.mjs b/experiments/atomic-codemode/native-reads.mjs new file mode 100644 index 0000000000..104291a506 --- /dev/null +++ b/experiments/atomic-codemode/native-reads.mjs @@ -0,0 +1,26 @@ +import { pageFacts } from "./fact-reads.mjs"; +import { requireFact } from "./native-contract.mjs"; + +export const nativePage = ( + rows, + input = {}, + identity = ["id", "origin_address", "block"], +) => pageFacts(rows, { offset: 0, limit: 20, ...input }, identity); +export const uniqueNative = (rows) => { + const index = new Map(); + for (const row of rows) { + requireFact(!index.has(row.id), `Duplicate native identity ${row.id}`); + index.set(row.id, row); + } + return index; +}; +export const readNativeIdentities = (index, input, filter = () => true) => ({ + ...nativePage( + (input.ids + ? input.ids.flatMap((id) => index.get(id) ?? []) + : [...index.values()] + ).filter(filter), + input, + ), + missing: input.ids?.filter((id) => !index.has(id)) ?? [], +}); diff --git a/experiments/atomic-codemode/provider.mjs b/experiments/atomic-codemode/provider.mjs index 69419c426d..e24d6e8e2a 100644 --- a/experiments/atomic-codemode/provider.mjs +++ b/experiments/atomic-codemode/provider.mjs @@ -20,6 +20,13 @@ import { } from "../atomic-re-tools/kuna-ssa.mjs"; const sha256 = (value) => createHash("sha256").update(value).digest("hex"); +const missingNativeFacet = () => ({ + availability: "unavailable", + error: { + code: "capability_unavailable", + message: "Requires the pinned native exporter", + }, +}); export class KunaFactOwner { #lifetime = new AbortController(); #sources = new Map(); @@ -87,7 +94,7 @@ export class KunaFactOwner { engine_version: owner.#installation.version, address_space: "native-default", analysis_model: owner.#nativeIR - ? "experimental-native-ir-v2" + ? "experimental-native-ir-v3" : "diagnostic-ssa-1.833", profile: owner.#nativeIR ? "pinned engine with embedded specs; fresh private engine cache per lease; byte reads suppress global discovery; other native CLI defaults; no annotations" @@ -274,6 +281,8 @@ export class KunaFactOwner { facets: parsed.facets ?? { code: { availability: "available" }, recovered_ir: { availability: "available", coverage: "partial" }, + calls: missingNativeFacet(), + values: missingNativeFacet(), }, }; } @@ -402,10 +411,7 @@ export class KunaFactOwner { } async values(input) { this.#requireNativeIR(input.snapshot); - return this.#query(input, (model) => ({ - ...model.values(input), - coverage: model.coverage, - })); + return this.#query(input, (model) => model.values(input)); } async calls(input) { this.#requireNativeIR(input.snapshot); diff --git a/experiments/atomic-codemode/sqlite-boundary-question.txt b/experiments/atomic-codemode/sqlite-boundary-question.txt new file mode 100644 index 0000000000..8dd5c72d9c --- /dev/null +++ b/experiments/atomic-codemode/sqlite-boundary-question.txt @@ -0,0 +1,17 @@ +Analyze sqlite3VdbeExec in {{artifact_path}} statically. Use one retained function +capture to inspect recovered call sites and their boundary values even if the +complete recovered IR graph is unavailable. Report each facet's availability and +the scope of any retained value set; do not treat it as a complete function graph. + +Aggregate the retained recovered calls by CALL versus CALLIND and known versus +unknown target. At call origins 0xf115c and 0xf238b, report the resolved target, +the input-zero value identity and its annotation/address-space facts, recovered +argument slots and passing locations. Determine whether the first argument is +the same retained SSA value at both sites and whether it is a function input. +Verify the selected machine CALL targets and essential argument-register setup +with focused instruction windows. Distinguish internal fspec offsets, SSA value +locations, passing storage, and code addresses. + +Explain which definition/use or alias claims remain unavailable. Reuse the same +capture for selected reads rather than reanalyzing the VM. Keep the answer focused +and give real Evidence references, target digest and static/runtime limitations. diff --git a/experiments/atomic-codemode/verify-calls.mjs b/experiments/atomic-codemode/verify-calls.mjs index 14cb22afd6..2f31bb1ef6 100644 --- a/experiments/atomic-codemode/verify-calls.mjs +++ b/experiments/atomic-codemode/verify-calls.mjs @@ -56,7 +56,7 @@ try { opened.context.artifact_sha256, "2e6eef9a727f081f0d453b4e5e6cbd8b9ef8b6f86cbf7681cbad444d3b0b55c8", ); - assert.equal(opened.context.analysis_model, "experimental-native-ir-v2"); + assert.equal(opened.context.analysis_model, "experimental-native-ir-v3"); const wrapper = await read("snapshot", { entry: "0x65180" }); const selected = await read("calls", { snapshot: wrapper.snapshot, diff --git a/experiments/atomic-codemode/verify-mcp.mjs b/experiments/atomic-codemode/verify-mcp.mjs index 6d6e69f032..256949f259 100644 --- a/experiments/atomic-codemode/verify-mcp.mjs +++ b/experiments/atomic-codemode/verify-mcp.mjs @@ -83,7 +83,7 @@ return {snapshot:s.data,code:c.data,graph};`, } } } else if (scenario === "sqlite-partial") { - assert.equal(opened.context.analysis_model, "experimental-native-ir-v2"); + assert.equal(opened.context.analysis_model, "experimental-native-ir-v3"); const result = await call("execute_javascript", { code: ` const read=async(p)=>{const r=await p;if(!r.ok)throw new Error(JSON.stringify(r));return r.data;}; @@ -141,7 +141,7 @@ return {literal,entry,snapshot:s,consumer_blocks:[...new Set(consumers.data.item assert.equal(result.value.projected, true); assert.equal(result.value.coverage.status, "partial"); assert.ok(result.value.guards.some((s) => s.includes("/ #0x1f"))); - if (opened.context.analysis_model === "experimental-native-ir-v2") { + if (opened.context.analysis_model === "experimental-native-ir-v3") { const native = await call("execute_javascript", { code: `const s=${JSON.stringify(result.value.snapshot)}; const read=async(p)=>{const r=await p;if(!r.ok)throw new Error(JSON.stringify(r));return r.data;}; diff --git a/experiments/atomic-codemode/verify-references.mjs b/experiments/atomic-codemode/verify-references.mjs index 2d2c6ec65c..d8eaa60cf7 100644 --- a/experiments/atomic-codemode/verify-references.mjs +++ b/experiments/atomic-codemode/verify-references.mjs @@ -142,6 +142,47 @@ try { assert.equal(vm.facets.code.availability, "available"); assert.equal(vm.facets.recovered_ir.availability, "unavailable"); assert.match(vm.facets.recovered_ir.error.details.native_error, /capacity/); + assert.equal(vm.facets.calls.availability, "available"); + assert.equal(vm.facets.values.scope, "call-boundary-only"); + const selectedCalls = await read("calls", { + snapshot: vm.snapshot, + origin_addresses: ["0xf238b", "0xf115c"], + fields: ["target", "target_value_id", "arguments"], + }); + assert.equal(selectedCalls.total, 2); + assert.ok(selectedCalls.items.every((c) => c.target?.offset === "0x296c0")); + assert.ok(selectedCalls.coverage.operations_examined > 12000); + const ids = [ + ...new Set( + selectedCalls.items + .flatMap((c) => [ + c.target_value_id, + ...c.arguments.map((a) => a.value_id), + ]) + .filter(Boolean), + ), + ]; + const boundary = await read("values", { snapshot: vm.snapshot, ids }); + assert.equal(boundary.missing.length, 0); + assert.match(boundary.coverage.value_scope, /only/); + for (const row of selectedCalls.items) + assert.ok( + boundary.items.some( + (v) => + v.id === row.target_value_id && + v.space === "fspec" && + v.is_annotation, + ), + ); + const unavailable = await call("execute_javascript", { + code: `return await rea.operations({snapshot:${JSON.stringify(vm.snapshot)},limit:1});`, + }); + assert.equal(unavailable.value.ok, false); + assert.equal(unavailable.value.error.code, "capability_unavailable"); + assert.equal( + (await read("snapshot", { entry: "0xea6d0" })).snapshot, + vm.snapshot, + ); } assert.equal((await call("close_binary", {})).ok, true); } catch (error) {