From e2205313a59be8ddb77d99d219d08da152fcb567 Mon Sep 17 00:00:00 2001 From: N0zoM1z0 <161784452+N0zoM1z0@users.noreply.github.com> Date: Fri, 16 Oct 2026 13:47:33 +0000 Subject: [PATCH] feat(experiment): expose bounded full recovered graphs Preserve complete graph facts with a bounded lossless native wire codec and shared paged Evidence views. Verify both MCP adapters and retained legacy records, then freeze a control/data-flow question and oracle for real agents. [AUTOMATED] Exploratory native patch; no upstream integration or PR. Co-authored-by: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com> Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> --- experiments/atomic-codemode/README.md | 20 +- experiments/atomic-codemode/api.mjs | 3 +- .../atomic-codemode/kuna-native-ir.patch | 498 ++++++++++++++---- .../atomic-codemode/native-calls.test.mjs | 16 + .../atomic-codemode/native-compute-study.md | 10 +- experiments/atomic-codemode/native-engine.md | 7 +- .../atomic-codemode/native-evidence-reads.mjs | 49 ++ .../native-evidence-reads.test.mjs | 86 +++ .../atomic-codemode/native-graph-encoding.mjs | 36 ++ .../native-graph-encoding.test.mjs | 66 +++ .../native-graph-transport-study.md | 114 ++++ .../atomic-codemode/native-ir-contract.mjs | 119 ++++- .../atomic-codemode/native-ir.test.mjs | 66 ++- .../native-savepoint-study.js.txt | 50 ++ experiments/atomic-codemode/provider.mjs | 17 +- experiments/atomic-codemode/reference.mjs | 2 +- .../savepoint-control-oracle.md | 95 ++++ .../sqlite-savepoint-control-question.txt | 30 ++ .../atomic-codemode/verify-discovery.mjs | 6 + .../atomic-codemode/verify-full-graph.mjs | 227 ++++++++ 20 files changed, 1361 insertions(+), 156 deletions(-) create mode 100644 experiments/atomic-codemode/native-evidence-reads.mjs create mode 100644 experiments/atomic-codemode/native-evidence-reads.test.mjs create mode 100644 experiments/atomic-codemode/native-graph-encoding.mjs create mode 100644 experiments/atomic-codemode/native-graph-encoding.test.mjs create mode 100644 experiments/atomic-codemode/native-graph-transport-study.md create mode 100644 experiments/atomic-codemode/native-savepoint-study.js.txt create mode 100644 experiments/atomic-codemode/savepoint-control-oracle.md create mode 100644 experiments/atomic-codemode/sqlite-savepoint-control-question.txt create mode 100644 experiments/atomic-codemode/verify-full-graph.mjs diff --git a/experiments/atomic-codemode/README.md b/experiments/atomic-codemode/README.md index a467372f1f..aecde5001f 100644 --- a/experiments/atomic-codemode/README.md +++ b/experiments/atomic-codemode/README.md @@ -52,8 +52,11 @@ evaluator. The ambient environment and credentials are not logged. Current bounds: 64 MiB guest heap, 32 KiB code and script value/error payloads, 64 host calls per execution, 120 seconds wall deadline and 2 seconds of guest synchronous execution; native requests have existing supervision, an 8 MiB capture budget and 1 GiB function-memory limit. Retained source storage is 64 MiB -per target, with two resident indexes (at most 2 MiB diagnostic or 8 MiB native -source each). Sources are +per target, with two resident indexes (each derived from at most 2 MiB diagnostic +or 8 MiB native wire source). Compressed native graphs admit at most 32 MiB decoded +JSON plus independent row/text/relationship bounds. One decoded Evidence text view +up to 32 MiB is also retained per lease; these source bounds are not parsed-heap +size guarantees. Sources are content-digested and retained until close; explicit evaluator copies survive close. Capacity failures are explicit, not clipped facts. Leases and snapshot handles prevent cross-session joins. @@ -276,7 +279,12 @@ the now finer fact/query boundary. The [native phase study](native-compute-study.md) now measures the VM drive at about 34 seconds, C rendering at 0.22 seconds and projection at 0.03 seconds. Repeated new native processes do not gain the API owner's warm result reuse. -It also demonstrates offline lossless encoding of all VM facts below the existing -transport budget; native codec/decoder bounds and model verification remain -pending. This may close full-graph/definition coverage without another analysis -or a live-function lifecycle, but is not yet a callable capability. +Its subsequent [bounded codec implementation](native-graph-transport-study.md) +now returns the complete 21,349-operation VM graph in about 3.79 MB, inside the +unchanged 8 MiB wire budget. Selected definitions/CFG reads and decoded Evidence +work through both real MCP adapters; all graph fields and ordinary C/calls match +the prior complete golden. Same-lease reuse is about 67 ms after a 39.7-second +fresh capture, with resource limits and no OOM. The [next question/oracle](savepoint-control-oracle.md) +start from names/error messages and exercise the new condition/CFG coverage. +Actual model results remain pending; source fidelity is not machine equivalence, +provider parity or a general speedup. diff --git a/experiments/atomic-codemode/api.mjs b/experiments/atomic-codemode/api.mjs index f1ec40b4ed..e1e0d21b2e 100644 --- a/experiments/atomic-codemode/api.mjs +++ b/experiments/atomic-codemode/api.mjs @@ -253,9 +253,10 @@ export const methods = { "await rea.code({snapshot:s,offset:100,limit:64})", ), evidence: definition( - "Read complete retained native source in UTF-16-code-unit pages by its evidence_id; no reanalysis. Available until close_binary.", + "Read complete retained native source in UTF-16-code-unit pages by evidence_id; no reanalysis. component:source preserves exact original wire JSON. component:recovered_ir reads the decoded graph source, including lossless compressed graphs; source hash remains the original capture and view has its own hash/byte count. This is source inspection, not a claim that malformed graph facts are valid. Available until close_binary.", { evidence_id: z.string().startsWith("ev_"), + component: z.enum(["source", "recovered_ir"]).default("source"), offset: page.offset, limit: z.number().int().min(1).max(16384).default(4096), }, diff --git a/experiments/atomic-codemode/kuna-native-ir.patch b/experiments/atomic-codemode/kuna-native-ir.patch index f7f9babfc6..1e10aa667e 100644 --- a/experiments/atomic-codemode/kuna-native-ir.patch +++ b/experiments/atomic-codemode/kuna-native-ir.patch @@ -1,9 +1,176 @@ +diff --git a/decompiler/Cargo.lock b/decompiler/Cargo.lock +index 157e7bd2..6caa4604 100644 +--- a/decompiler/Cargo.lock ++++ b/decompiler/Cargo.lock +@@ -23,6 +23,12 @@ version = "0.7.8" + source = "registry+https://github.com/rust-lang/crates.io-index" + checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + ++[[package]] ++name = "base64" ++version = "0.22.1" ++source = "registry+https://github.com/rust-lang/crates.io-index" ++checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" ++ + [[package]] + name = "bitflags" + version = "2.13.0" +@@ -265,6 +271,8 @@ dependencies = [ + name = "kuna-cli" + version = "0.1.0" + dependencies = [ ++ "base64", ++ "flate2", + "kuna-analysis", + "kuna-base", + "kuna-console", +diff --git a/decompiler/crates/kuna-cli/Cargo.toml b/decompiler/crates/kuna-cli/Cargo.toml +index 06231a77..db79599c 100644 +--- a/decompiler/crates/kuna-cli/Cargo.toml ++++ b/decompiler/crates/kuna-cli/Cargo.toml +@@ -9,6 +9,8 @@ license.workspace = true + publish.workspace = true + + [dependencies] ++base64 = "=0.22.1" ++flate2 = "=1.1.9" + kuna-decomp.workspace = true + kuna-base.workspace = true + kuna-console.workspace = true +diff --git a/decompiler/crates/kuna-cli/src/atomic_compression.rs b/decompiler/crates/kuna-cli/src/atomic_compression.rs +new file mode 100644 +index 00000000..48631d39 +--- /dev/null ++++ b/decompiler/crates/kuna-cli/src/atomic_compression.rs +@@ -0,0 +1,122 @@ ++//! Bounded lossless wire encoding of logical schema-3 atomic graphs. ++use crate::jsonfmt::{dumps_compact_bounded, Json}; ++use base64::{engine::general_purpose::STANDARD, Engine as _}; ++use flate2::{write::GzEncoder, Compression}; ++use std::io::{self, Write}; ++ ++const MAX_RAW_BYTES: usize = 32 * 1024 * 1024; ++struct BoundedBytes { ++ data: Vec, ++ limit: usize, ++} ++impl Write for BoundedBytes { ++ fn write(&mut self, bytes: &[u8]) -> io::Result { ++ let next = self ++ .data ++ .len() ++ .checked_add(bytes.len()) ++ .filter(|n| *n <= self.limit) ++ .ok_or_else(|| io::Error::other("atomic compression capacity exceeded"))?; ++ if next > self.data.capacity() { ++ let capacity = next ++ .max(self.data.capacity().saturating_mul(2)) ++ .min(self.limit); ++ self.data ++ .try_reserve_exact(capacity - self.data.len()) ++ .map_err(|_| io::Error::other("atomic compression allocation failed"))?; ++ } ++ self.data.extend_from_slice(bytes); ++ Ok(bytes.len()) ++ } ++ fn flush(&mut self) -> io::Result<()> { ++ Ok(()) ++ } ++} ++ ++pub(crate) fn encode(graph: &Json, limit: usize) -> Option<(Json, usize)> { ++ let Json::Object(fields) = graph else { ++ return None; ++ }; ++ if !fields ++ .iter() ++ .any(|(key, v)| key == "schema_version" && *v == Json::Number("3".into())) ++ { ++ return None; ++ } ++ let raw = dumps_compact_bounded(graph, MAX_RAW_BYTES)?; ++ let mut envelope = Json::Object(vec![ ++ ("schema_version".into(), Json::Number("4".into())), ++ ("encoding".into(), Json::Str("gzip-base64".into())), ++ ("raw_bytes".into(), Json::Number(raw.len().to_string())), ++ ("data".into(), Json::Str(String::new())), ++ ]); ++ let overhead = dumps_compact_bounded(&envelope, limit)?.len(); ++ let payload_limit = limit.checked_sub(overhead)?; ++ let compressed_limit = (payload_limit / 4).checked_mul(3)?; ++ let mut gzip = GzEncoder::new( ++ BoundedBytes { ++ data: Vec::new(), ++ limit: compressed_limit, ++ }, ++ Compression::fast(), ++ ); ++ gzip.write_all(raw.as_bytes()).ok()?; ++ let compressed = gzip.finish().ok()?.data; ++ let encoded_len = compressed ++ .len() ++ .checked_add(2)? ++ .checked_div(3)? ++ .checked_mul(4)?; ++ if encoded_len > payload_limit { ++ return None; ++ } ++ let encoded = STANDARD.encode(compressed); ++ if let Json::Object(fields) = &mut envelope { ++ let (_, data) = fields.iter_mut().find(|(key, _)| key == "data")?; ++ *data = Json::Str(encoded); ++ } ++ Some((envelope, overhead + encoded_len)) ++} ++ ++#[cfg(test)] ++mod tests { ++ use super::*; ++ use std::io::Read; ++ #[test] ++ fn lossless_frame_accounts_for_escaping_and_base64_expansion() { ++ let graph = Json::Object(vec![ ++ ("schema_version".into(), Json::Number("3".into())), ++ ("statement".into(), Json::Str("δΈ­ζ–‡πŸ˜€\0".repeat(1000))), ++ ]); ++ let (encoded, size) = encode(&graph, 2000).unwrap(); ++ let text = dumps_compact_bounded(&encoded, 2000).unwrap(); ++ assert_eq!(text.len(), size); ++ let frame: serde_json::Value = serde_json::from_str(&text).unwrap(); ++ let bytes = STANDARD.decode(frame["data"].as_str().unwrap()).unwrap(); ++ let mut restored = String::new(); ++ flate2::read::GzDecoder::new(bytes.as_slice()) ++ .read_to_string(&mut restored) ++ .unwrap(); ++ assert_eq!( ++ restored, ++ dumps_compact_bounded(&graph, MAX_RAW_BYTES).unwrap() ++ ); ++ assert_eq!(frame["raw_bytes"].as_u64().unwrap(), restored.len() as u64); ++ assert!(encode(&graph, 1).is_none()); ++ assert!(encode(&graph, size - 1).is_none()); ++ } ++ #[test] ++ fn archive_is_single_layer_and_raw_allocation_has_an_independent_bound() { ++ assert!(encode(&Json::Str("not a graph".into()), 1000).is_none()); ++ let nested = Json::Object(vec![("schema_version".into(), Json::Number("4".into()))]); ++ assert!(encode(&nested, 1000).is_none()); ++ let oversized = Json::Object(vec![ ++ ("schema_version".into(), Json::Number("3".into())), ++ ( ++ "statement".into(), ++ Json::Str("\0".repeat(MAX_RAW_BYTES / 6 + 1)), ++ ), ++ ]); ++ assert!(encode(&oversized, 1000).is_none()); ++ } ++} diff --git a/decompiler/crates/kuna-cli/src/atomic_json.rs b/decompiler/crates/kuna-cli/src/atomic_json.rs new file mode 100644 -index 00000000..527ee788 +index 00000000..7077bc56 --- /dev/null +++ b/decompiler/crates/kuna-cli/src/atomic_json.rs -@@ -0,0 +1,249 @@ +@@ -0,0 +1,359 @@ +//! Opt-in atomic documents share the ordinary record producer and JSON writer. +//! A graph that cannot fit must not discard otherwise readable reconstructed C. + @@ -21,35 +188,40 @@ index 00000000..527ee788 + Capacity, + RedundantWithGraph, +} ++fn function_records(document: &mut Json) -> Option<&mut Vec> { ++ let Json::Object(fields) = document else { ++ return None; ++ }; ++ let (_, value) = fields.iter_mut().find(|(key, _)| key == "functions")?; ++ let Json::Array(functions) = value else { ++ return None; ++ }; ++ Some(functions) ++} +fn take_facet(document: &mut Json, name: &str, limit: usize, why: Omission) -> Vec { + let mut removed = Vec::new(); -+ if let Json::Object(fields) = document { -+ if let Some(Json::Array(functions)) = fields -+ .iter_mut() -+ .find_map(|(key, value)| (key == "functions").then_some(value)) -+ { -+ for (function, record) in functions.iter_mut().enumerate() { -+ if let Json::Object(fields) = record { -+ if matches!(why, Omission::RedundantWithGraph) -+ && !fields.iter().any(|(key, _)| key == "atomic_facts") -+ { -+ continue; -+ } -+ if let Some(field) = fields.iter().position(|(key, _)| key == name) { -+ let value = fields.remove(field); -+ let error = format!("{name}_error"); -+ let added_error = matches!(why, Omission::Capacity) -+ && !fields.iter().any(|(key, _)| key == &error); -+ if added_error { -+ fields.push((error, Json::Str(format!("atomic encoded document exceeds {limit} byte capacity; {name} omitted, code retained")))); -+ } -+ removed.push(RemovedFacet { -+ function, -+ field, -+ value, -+ added_error, -+ }); ++ if let Some(functions) = function_records(document) { ++ for (function, record) in functions.iter_mut().enumerate() { ++ if let Json::Object(fields) = record { ++ if matches!(why, Omission::RedundantWithGraph) ++ && !fields.iter().any(|(key, _)| key == "atomic_facts") ++ { ++ continue; ++ } ++ if let Some(field) = fields.iter().position(|(key, _)| key == name) { ++ let value = fields.remove(field); ++ let error = format!("{name}_error"); ++ let added_error = matches!(why, Omission::Capacity) ++ && !fields.iter().any(|(key, _)| key == &error); ++ if added_error { ++ fields.push((error, Json::Str(format!("atomic encoded document exceeds {limit} byte capacity; {name} omitted, code retained")))); + } ++ removed.push(RemovedFacet { ++ function, ++ field, ++ value, ++ added_error, ++ }); + } + } + } @@ -57,22 +229,62 @@ index 00000000..527ee788 + removed +} +fn restore_facet(document: &mut Json, removed: Vec) { -+ if let Json::Object(fields) = document { -+ if let Some(Json::Array(functions)) = fields -+ .iter_mut() -+ .find_map(|(key, value)| (key == "functions").then_some(value)) -+ { -+ for facet in removed { -+ if let Json::Object(fields) = &mut functions[facet.function] { -+ if facet.added_error { -+ let error = format!("{}_error", facet.value.0); -+ fields.retain(|(key, _)| key != &error); ++ if let Some(functions) = function_records(document) { ++ for facet in removed { ++ if let Json::Object(fields) = &mut functions[facet.function] { ++ if facet.added_error { ++ let error = format!("{}_error", facet.value.0); ++ fields.retain(|(key, _)| key != &error); ++ } ++ fields.insert(facet.field, facet.value); ++ } ++ } ++ } ++} ++struct GraphAlternative { ++ function: usize, ++ value: Json, ++} ++fn compress_graphs(document: &mut Json, budget: usize) -> Vec { ++ let mut originals = Vec::new(); ++ let mut remaining = budget; ++ if let Some(functions) = function_records(document) { ++ for (function, record) in functions.iter_mut().enumerate() { ++ if let Json::Object(fields) = record { ++ if let Some((_, graph)) = fields.iter_mut().find(|(key, _)| key == "atomic_facts") { ++ if let Some((encoded, size)) = ++ crate::atomic_compression::encode(graph, remaining) ++ { ++ remaining -= size; ++ originals.push(GraphAlternative { ++ function, ++ value: std::mem::replace(graph, encoded), ++ }); + } -+ fields.insert(facet.field, facet.value); + } + } + } + } ++ originals ++} ++fn exchange_graphs( ++ document: &mut Json, ++ alternatives: Vec, ++) -> Vec { ++ let mut previous = Vec::new(); ++ if let Some(functions) = function_records(document) { ++ for alternative in alternatives { ++ if let Json::Object(fields) = &mut functions[alternative.function] { ++ if let Some((_, graph)) = fields.iter_mut().find(|(key, _)| key == "atomic_facts") { ++ previous.push(GraphAlternative { ++ function: alternative.function, ++ value: std::mem::replace(graph, alternative.value), ++ }); ++ } ++ } ++ } ++ } ++ previous +} +fn bounded(document: &Json, budget: usize) -> Option { + let mut text = dumps_compact_bounded(document, budget)?; @@ -94,11 +306,26 @@ index 00000000..527ee788 + if let Some(text) = bounded(&document, budget) { + return Ok(text); + } ++ let originals = compress_graphs(&mut document, budget); ++ let alternatives = if originals.is_empty() { ++ Vec::new() ++ } else { ++ if let Some(text) = bounded(&document, budget) { ++ return Ok(text); ++ } ++ exchange_graphs(&mut document, originals) ++ }; + let calls = take_facet(&mut document, "atomic_calls", limit, Omission::Capacity); + if !calls.is_empty() { + if let Some(text) = bounded(&document, budget) { + return Ok(text); + } ++ if !alternatives.is_empty() { ++ drop(exchange_graphs(&mut document, alternatives)); ++ if let Some(text) = bounded(&document, budget) { ++ return Ok(text); ++ } ++ } + } + restore_facet(&mut document, calls); + take_facet(&mut document, "atomic_facts", limit, Omission::Capacity); @@ -164,6 +391,56 @@ index 00000000..527ee788 +} + +#[cfg(test)] ++mod compressed_graph_tests { ++ use super::*; ++ use base64::{engine::general_purpose::STANDARD, Engine as _}; ++ use std::io::Read; ++ #[test] ++ fn compressed_graph_retains_calls_and_code_before_losing_any_facet() { ++ let graph = Json::Object(vec![ ++ ("schema_version".into(), Json::Number("3".into())), ++ ("payload".into(), Json::Str("x".repeat(100_000))), ++ ]); ++ let original = Json::Object(vec![( ++ "functions".into(), ++ Json::Array(vec![Json::Object(vec![ ++ ("code".into(), Json::Str("retained C".into())), ++ ("atomic_facts".into(), graph.clone()), ++ ("atomic_calls".into(), Json::Str("independent calls".into())), ++ ("atomic_projection".into(), Json::Str("fallback".into())), ++ ])]), ++ )]); ++ let uncompressed: serde_json::Value = ++ serde_json::from_str(&render(original.clone(), 200_000).unwrap()).unwrap(); ++ assert_eq!( ++ uncompressed["functions"][0]["atomic_facts"]["schema_version"], ++ 3 ++ ); ++ let text = render(original.clone(), 4096).unwrap(); ++ assert!(text.len() <= 4096); ++ let document: serde_json::Value = serde_json::from_str(&text).unwrap(); ++ let function = &document["functions"][0]; ++ assert_eq!(function["code"], "retained C"); ++ assert_eq!(function["atomic_calls"], "independent calls"); ++ assert!(function.get("atomic_projection").is_none()); ++ assert!(function.get("atomic_facts_error").is_none()); ++ let frame = &function["atomic_facts"]; ++ assert_eq!(frame["schema_version"], 4); ++ let compressed = STANDARD.decode(frame["data"].as_str().unwrap()).unwrap(); ++ let mut restored = String::new(); ++ flate2::read::GzDecoder::new(compressed.as_slice()) ++ .read_to_string(&mut restored) ++ .unwrap(); ++ assert_eq!(restored, dumps_compact_bounded(&graph, 200_000).unwrap()); ++ let tiny: serde_json::Value = ++ serde_json::from_str(&render(original, 512).unwrap()).unwrap(); ++ assert!(tiny["functions"][0].get("atomic_facts").is_none()); ++ assert_eq!(tiny["functions"][0]["atomic_projection"], "fallback"); ++ assert_eq!(tiny["functions"][0]["atomic_calls"], "independent calls"); ++ } ++} ++ ++#[cfg(test)] +mod independent_facet_tests { + use super::*; + fn document(graph: usize, calls: usize) -> Json { @@ -283,7 +560,7 @@ index ad249b04..62074996 100644 @@ -979,7 +988,13 @@ pub fn main(argv: &[String]) -> Result { return Err("--base requires --raw-image".into()); } - + + if atomic_output_bytes.is_some() && !atomic_facts { + return Err("--atomic-output-bytes requires --atomic-facts".into()); + } @@ -306,7 +583,7 @@ index ad249b04..62074996 100644 @@ -1007,6 +1024,10 @@ pub fn main(argv: &[String]) -> Result { })); } - + + if atomic_facts { + return Err("--atomic-facts requires --json".into()); + } @@ -363,7 +640,7 @@ index ad249b04..62074996 100644 @@ -1217,7 +1252,7 @@ fn decompile_json(args: &AllArgs, target: &str) -> Result<(String, Option), } - + +impl From for Json { + fn from(value: serde_json::Value) -> Self { + match value { @@ -515,7 +792,7 @@ index 198b85a6..71bd9675 100644 @@ -50,6 +63,12 @@ pub fn dumps_compact(v: &Json) -> String { writer::render(v, Format::Compact) } - + +/// Enforce the caller's encoded-byte budget before growing the output buffer. +/// Uses the same writer and escaping as the other modes; no length estimator. +pub fn dumps_compact_bounded(v: &Json, limit: usize) -> Option { @@ -531,22 +808,22 @@ index c4eca1db..06b9d024 100644 +++ b/decompiler/crates/kuna-cli/src/jsonfmt/writer.rs @@ -1,7 +1,7 @@ //! JSON layout and ASCII-safe escaping shared by all CLI output modes. - + use super::Json; -use std::fmt::Write as _; +use std::fmt::{self, Write}; - + #[derive(Clone, Copy, PartialEq, Eq)] pub(super) enum Format { @@ -12,108 +12,142 @@ pub(super) enum Format { - + pub(super) fn render(value: &Json, format: Format) -> String { let mut out = String::new(); - write_value(&mut out, value, 0, format); + write_value(&mut out, value, 0, format).expect("String writer is infallible"); out } - + +struct BoundedString { + text: String, + limit: usize, @@ -594,7 +871,7 @@ index c4eca1db..06b9d024 100644 + Ok(()) } } - + -fn write_value(out: &mut String, value: &Json, indent: usize, format: Format) { +fn write_value(out: &mut impl Write, value: &Json, indent: usize, format: Format) -> fmt::Result { match value { @@ -640,7 +917,7 @@ index c4eca1db..06b9d024 100644 } } } - + fn write_pairs<'a>( - out: &mut String, + out: &mut impl Write, @@ -677,7 +954,7 @@ index c4eca1db..06b9d024 100644 - out.push('}'); + out.write_char('}') } - + -fn write_string(out: &mut String, s: &str) { - out.push('"'); +fn write_string(out: &mut impl Write, s: &str) -> fmt::Result { @@ -751,14 +1028,15 @@ index 6710e12a..ef90468c 100644 + ); +} diff --git a/decompiler/crates/kuna-cli/src/main.rs b/decompiler/crates/kuna-cli/src/main.rs -index 2345743c..40fff9ec 100644 +index 2345743c..3da94435 100644 --- a/decompiler/crates/kuna-cli/src/main.rs +++ b/decompiler/crates/kuna-cli/src/main.rs -@@ -1,6 +1,7 @@ +@@ -1,6 +1,8 @@ //! Command dispatch for the kuna CLI. - + mod args; +mod atomic_json; ++mod atomic_compression; mod callgraph; mod catalog; mod crypto; @@ -773,7 +1051,7 @@ index e40b18c9..b8cae233 100644 + want_atomic_facts: false, want_pointer_arguments: false, }; - + diff --git a/decompiler/crates/kuna-console/src/inspect.rs b/decompiler/crates/kuna-console/src/inspect.rs index 74024b10..72e4ec24 100644 --- a/decompiler/crates/kuna-console/src/inspect.rs @@ -786,7 +1064,7 @@ index 74024b10..72e4ec24 100644 + pub atomic_calls: Option>, + pub atomic_projection: Option>, } - + impl FuncDetail { @@ -40,13 +43,13 @@ impl FuncDetail { ) -> Self { @@ -803,7 +1081,7 @@ index 74024b10..72e4ec24 100644 + FuncDetail { tokens, tokens_error: None, ..Default::default() } } } - + diff --git a/decompiler/crates/kuna-console/src/project.rs b/decompiler/crates/kuna-console/src/project.rs index bc401db2..1b53ccf1 100644 --- a/decompiler/crates/kuna-console/src/project.rs @@ -873,17 +1151,17 @@ index 40b093b6..8f03b654 100644 smallvec.workspace = true slotmap.workspace = true +serde = { version = "1", features = ["derive"] } - + [dev-dependencies] # Verifier-only: the datatest engine (item w10-structure-printbody). regex = { workspace = true } -serde = { version = "1", features = ["derive"] } toml = { version = "1.1", default-features = false, features = ["parse", "serde", "std"] } - + [build-dependencies] diff --git a/decompiler/crates/kuna-decomp/src/substrate/atomic_calls.rs b/decompiler/crates/kuna-decomp/src/substrate/atomic_calls.rs new file mode 100644 -index 00000000..821ca6f0 +index 00000000..2b7ced64 --- /dev/null +++ b/decompiler/crates/kuna-decomp/src/substrate/atomic_calls.rs @@ -0,0 +1,216 @@ @@ -934,7 +1212,7 @@ index 00000000..821ca6f0 + let mut items = Vec::new(); + let mut keys = BTreeSet::new(); + let mut inputs = 0usize; -+ let mut text_bytes = 0usize; ++ let mut text = atomic_values::TextBudget::new(2 * 1024 * 1024); + for &key in &membership.calls { + let op = fd.obank().get(key).ok_or("stale call operation")?; + inputs = inputs @@ -945,9 +1223,9 @@ index 00000000..821ca6f0 + } + let spec = specs.get(&key).copied(); + if let Some(spec) = spec { -+ atomic_values::charge_text(&mut text_bytes, spec.get_name().len())?; ++ text.charge(spec.get_name().len())?; + if spec.proto().has_model() { -+ atomic_values::charge_text(&mut text_bytes, spec.proto().get_model_name().len())?; ++ text.charge(spec.proto().get_model_name().len())?; + } + } + for index in 0..op.num_input() { @@ -963,7 +1241,7 @@ index 00000000..821ca6f0 + } + items.push(collect(fd, key, spec)?); + } -+ let values = atomic_values::collect(fd, keys, &mut text_bytes)?; ++ let values = atomic_values::collect(fd, keys, &mut text)?; + Ok(CallFacts { + schema_version: 1, items, values, + coverage: CallCoverage { @@ -1105,10 +1383,10 @@ index 00000000..821ca6f0 +} diff --git a/decompiler/crates/kuna-decomp/src/substrate/atomic_facts.rs b/decompiler/crates/kuna-decomp/src/substrate/atomic_facts.rs new file mode 100644 -index 00000000..78d5fc50 +index 00000000..6b9afe69 --- /dev/null +++ b/decompiler/crates/kuna-decomp/src/substrate/atomic_facts.rs -@@ -0,0 +1,155 @@ +@@ -0,0 +1,156 @@ +//! Opt-in, read-only export of the recovered function IR for the REA experiment. +//! IDs are local to this captured model; addresses alone are not SSA identities. +use crate::{ @@ -1162,10 +1440,11 @@ index 00000000..78d5fc50 +) -> Result { + // Bounds cover allocations before JSON serialization, independently of the + // transport byte budget. Overflow is an unavailable facet, not clipped IR. -+ const MAX_OPERATIONS: usize = 12_000; ++ const MAX_OPERATIONS: usize = 32_000; + const MAX_VALUES: usize = 36_000; -+ const MAX_INPUTS: usize = 50_000; -+ const MAX_DISPLAY_BYTES: usize = 2 * 1024 * 1024; ++ const MAX_INPUTS: usize = 100_000; ++ const MAX_DISPLAY_BYTES: usize = 4 * 1024 * 1024; ++ const MAX_CFG_EDGES: usize = 50_000; + if fd.bblocks_get_size() <= 0 { + return Err("recovered basic-block graph is unavailable".into()); + } @@ -1197,7 +1476,7 @@ index 00000000..78d5fc50 + edge_count = edge_count + .checked_add(block.size_out() as usize) + .ok_or("atomic CFG edge count overflow")?; -+ if edge_count > MAX_INPUTS { ++ if edge_count > MAX_CFG_EDGES { + return Err("atomic CFG edge capacity exceeded".into()); + } + let tail = members.last().and_then(|id| fd.obank().get(*id)); @@ -1336,10 +1615,10 @@ index 00000000..6628ddd7 +} diff --git a/decompiler/crates/kuna-decomp/src/substrate/atomic_operations.rs b/decompiler/crates/kuna-decomp/src/substrate/atomic_operations.rs new file mode 100644 -index 00000000..7d56cce5 +index 00000000..30d6be39 --- /dev/null +++ b/decompiler/crates/kuna-decomp/src/substrate/atomic_operations.rs -@@ -0,0 +1,127 @@ +@@ -0,0 +1,119 @@ +//! Shared read-only operation/value normalization for full and selected facts. +use crate::{ + architecture::Architecture, @@ -1372,10 +1651,9 @@ index 00000000..7d56cce5 +pub struct OperationBudget { + max_values: usize, + max_inputs: usize, -+ max_display: usize, + keys: BTreeSet, + input_count: usize, -+ display_bytes: usize, ++ text: atomic_values::TextBudget, + pub issues: Vec, +} +impl OperationBudget { @@ -1383,10 +1661,9 @@ index 00000000..7d56cce5 + Self { + max_values, + max_inputs, -+ max_display, + keys: BTreeSet::new(), + input_count: 0, -+ display_bytes: 0, ++ text: atomic_values::TextBudget::new(max_display), + issues: Vec::new(), + } + } @@ -1445,13 +1722,7 @@ index 00000000..7d56cce5 + } + let mut statement = String::new(); + crate::funcdata_printraw::render_op(&mut statement, arch, fd, key)?; -+ self.display_bytes = self -+ .display_bytes -+ .checked_add(statement.len()) -+ .ok_or("atomic display count overflow")?; -+ if self.display_bytes > self.max_display { -+ return Err("atomic display capacity exceeded".into()); -+ } ++ self.text.charge(statement.len())?; + Ok(OperationFact { + id, + address: format!("0x{:x}", op.get_seq_num().get_addr().get_offset()), @@ -1464,7 +1735,7 @@ index 00000000..7d56cce5 + }) + } + pub fn values(&mut self, fd: &Funcdata) -> Result, String> { -+ atomic_values::collect(fd, std::mem::take(&mut self.keys), &mut self.display_bytes) ++ atomic_values::collect(fd, std::mem::take(&mut self.keys), &mut self.text) + } +} diff --git a/decompiler/crates/kuna-decomp/src/substrate/atomic_projection.rs b/decompiler/crates/kuna-decomp/src/substrate/atomic_projection.rs @@ -1592,10 +1863,10 @@ index 00000000..097b153a +} diff --git a/decompiler/crates/kuna-decomp/src/substrate/atomic_values.rs b/decompiler/crates/kuna-decomp/src/substrate/atomic_values.rs new file mode 100644 -index 00000000..fc509847 +index 00000000..6d23954c --- /dev/null +++ b/decompiler/crates/kuna-decomp/src/substrate/atomic_values.rs -@@ -0,0 +1,70 @@ +@@ -0,0 +1,80 @@ +//! One read-only interpretation of native value and operation identities. +use crate::{ + context::{OpId, VarnodeId}, @@ -1605,7 +1876,26 @@ index 00000000..fc509847 +use slotmap::Key; +use std::collections::BTreeSet; + -+pub const MAX_TEXT_BYTES: usize = 2 * 1024 * 1024; ++pub struct TextBudget { ++ used: usize, ++ limit: usize, ++} ++impl TextBudget { ++ pub fn new(limit: usize) -> Self { ++ Self { used: 0, limit } ++ } ++ pub fn charge(&mut self, bytes: usize) -> Result<(), String> { ++ let next = self ++ .used ++ .checked_add(bytes) ++ .ok_or("atomic text count overflow")?; ++ if next > self.limit { ++ return Err("atomic text capacity exceeded".into()); ++ } ++ self.used = next; ++ Ok(()) ++ } ++} +#[derive(Debug, Clone, Serialize)] +pub struct ValueFact { + pub id: String, @@ -1630,25 +1920,16 @@ index 00000000..fc509847 + seq.get_time() + )) +} -+pub fn charge_text(used: &mut usize, bytes: usize) -> Result<(), String> { -+ *used = used -+ .checked_add(bytes) -+ .ok_or("atomic text count overflow")?; -+ if *used > MAX_TEXT_BYTES { -+ return Err("atomic text capacity exceeded".into()); -+ } -+ Ok(()) -+} +pub fn collect( + fd: &Funcdata, + keys: BTreeSet, -+ text_bytes: &mut usize, ++ text: &mut TextBudget, +) -> Result, String> { + let mut values = Vec::with_capacity(keys.len()); + for key in keys { + let value = fd.vbank().get(key).ok_or("stale input/output value")?; + let type_name = value.get_type().get_name(); -+ charge_text(text_bytes, type_name.len())?; ++ text.charge(type_name.len())?; + values.push(ValueFact { + id: value_id(key), + space: value @@ -1697,10 +1978,10 @@ index 00528cb5..b019a174 100644 pub mod dtype; pub mod context; diff --git a/docs/spec/00-overview.md b/docs/spec/00-overview.md -index ce499c4f..5a98b2c3 100644 +index ce499c4f..b3b53dcf 100644 --- a/docs/spec/00-overview.md +++ b/docs/spec/00-overview.md -@@ -3431,3 +3431,68 @@ Reserving it creates no analysis varnode and changes no physical mapping or +@@ -3431,3 +3431,79 @@ Reserving it creates no analysis varnode and changes no physical mapping or emitted C. Ghidra receives temporary storage for the logical local, rather than a persistable ordinary stack-variable location; ordinary symbol storage and register-plus-stack parameter JOINs keep their existing encoding. @@ -1747,7 +2028,18 @@ index ce499c4f..5a98b2c3 100644 +and placeholders stay unknown. Input slots are not source parameter ordinals. + +The encoded writer attempts bounded facet alternatives without cloning their -+contents. A full graph omits its redundant operation projection; if encoded ++contents. A raw schema-3 graph that fits keeps its existing format. Before ++dropping useful facts, an oversized graph can use a schema-4 envelope with ++`encoding: "gzip-base64"`, the exact `raw_bytes` and canonical padded `data`. ++Its decompressed JSON is the unchanged logical schema-3 graph. Raw encoding ++stops at 32 MiB; compressed output, base64 expansion and envelope overhead are ++bounded before allocation against the caller's encoded budget. All graphs in a ++document share that encoded budget. Optional compression does not analyze again. ++The independent full-graph row limits are 32,000 operations, 36,000 values, ++100,000 inputs, 4 MiB display/type text, 12,000 blocks and 50,000 CFG edges. ++They bound construction separately from wire size; exceeding them is unavailable, ++not a complete clipped graph. C and independent calls keep their own contracts. ++A full graph omits its redundant operation projection; if encoded +capacity makes that graph unavailable, the independent projection is restored. +A fitting graph has priority, then fitting calls, then the projection. Failed +facets retain named errors. If C itself cannot fit, output fails explicitly diff --git a/experiments/atomic-codemode/native-calls.test.mjs b/experiments/atomic-codemode/native-calls.test.mjs index d4ca9addfd..e4b1da8678 100644 --- a/experiments/atomic-codemode/native-calls.test.mjs +++ b/experiments/atomic-codemode/native-calls.test.mjs @@ -2,6 +2,22 @@ import test from "node:test"; import assert from "node:assert/strict"; import { parseNativeIR } from "./native-ir.mjs"; +test("malformed compressed graph leaves independent call facts and code readable", () => { + const doc = fixture(); + const originalCode = doc.functions[0].code; + doc.functions[0].atomic_facts = { + schema_version: 4, + encoding: "gzip-base64", + raw_bytes: 64, + data: "AAAA", + }; + const model = parseNativeIR(JSON.stringify(doc), "0x1000"); + assert.equal(model.code, originalCode); + assert.equal(model.facets.recovered_ir.availability, "unavailable"); + assert.equal(model.facets.calls.availability, "available"); + assert.equal(model.calls({}).total, 2); +}); + const input = (input_index, value_id) => ({ input_index, value_id, diff --git a/experiments/atomic-codemode/native-compute-study.md b/experiments/atomic-codemode/native-compute-study.md index 2f33640de3..78d595f417 100644 --- a/experiments/atomic-codemode/native-compute-study.md +++ b/experiments/atomic-codemode/native-compute-study.md @@ -6,7 +6,9 @@ or establish model-token savings. ## Phase instrument and actual measurements -Apply `kuna-native-ir.patch`, then `kuna-atomic-timing-diagnostic.patch`, to Kuna +Use experiment commit `7d0da075e` and its versions of both patches to reproduce +these historical measurements. Apply `kuna-native-ir.patch`, then +`kuna-atomic-timing-diagnostic.patch`, to Kuna `fe01d6294eb0d1053ce343c8943646df88e143fc`. Reuse existing compiled specs and the one-job opt-level-1 embed-specs build in `native-engine.md`. Preserved engine: `c755184455b8fe8950a5194788d954bc0476602779b6dd614cb893a230f6fde7`; @@ -114,5 +116,7 @@ allocation/encoding growth, bounded inflation, exact format/length/row admission malformed-graph isolation from calls/C, complete Evidence access, actual CPU and memory, fallback behavior and real-model workflows. The shared owner must return selected facts rather than send an encoded graph to the agent. The current -public engine still has a bounded call-rooted projection, without a complete VM -CFG or arbitrary expansion of unretained definitions. +public engine at this sizing checkpoint still has a bounded call-rooted +projection. The [subsequent implementation](native-graph-transport-study.md) +now verifies full recovered-graph access through bounded native/Node encoding +and both public MCP adapters; real-model verification remains separate. diff --git a/experiments/atomic-codemode/native-engine.md b/experiments/atomic-codemode/native-engine.md index 3dbe441736..aff2497880 100644 --- a/experiments/atomic-codemode/native-engine.md +++ b/experiments/atomic-codemode/native-engine.md @@ -670,7 +670,8 @@ profiled. The [subsequent native computation study](native-compute-study.md) now measures them: VM drive about 34 seconds, C rendering 0.22 seconds and projection 0.03 seconds. Skipping C is a small opportunity for this workload. An offline lossless encoding probe also puts the earlier full 17.8 MB diagnostic record -inside the unchanged 8 MiB transport. That codec is not implemented in the public -engine; allocation/inflation bounds, shared decoding, Evidence and real-model -behavior still need testing. Current model comparisons share the same +inside the unchanged 8 MiB transport. The [subsequent codec study](native-graph-transport-study.md) +now implements and verifies this through both public experimental MCP adapters, +including complete decoded Evidence and shared text-budget ownership. Actual +model behavior remains to be tested. Model comparisons share the same computation/fact owners, avoiding a capability difference between adapters. diff --git a/experiments/atomic-codemode/native-evidence-reads.mjs b/experiments/atomic-codemode/native-evidence-reads.mjs new file mode 100644 index 0000000000..d91854e0b9 --- /dev/null +++ b/experiments/atomic-codemode/native-evidence-reads.mjs @@ -0,0 +1,49 @@ +// One bounded decoded source view; the source owner revalidates bytes per read. +import { createHash } from "node:crypto"; +import { readNativeGraphEvidence } from "./native-ir-contract.mjs"; + +export class NativeEvidenceReads { + #graph; + constructor(readSource) { + this.readSource = readSource; + } + clear() { + this.#graph = undefined; + } + async read(input) { + const { text: original, source } = await this.readSource(input.evidence_id); + let text = original, + view; + if (input.component === "recovered_ir") { + const key = `${input.evidence_id}:${source.sha256}`; + if (this.#graph?.key !== key) { + const decoded = readNativeGraphEvidence(text); + this.#graph = { + key, + ...decoded, + sha256: createHash("sha256").update(decoded.text).digest("hex"), + bytes: Buffer.byteLength(decoded.text), + }; + } + const decoded = this.#graph; + text = decoded.text; + view = { + component: "recovered_ir", + encoding: decoded.encoding, + sha256: decoded.sha256, + bytes: decoded.bytes, + authority: + "uninterpreted retained graph source; fact validity is separate", + }; + } + const offset = input.offset ?? 0, + limit = input.limit ?? 4096; + return { + source, + ...(view ? { view } : {}), + text: text.slice(offset, offset + limit), + total_code_units: text.length, + next_offset: offset + limit < text.length ? offset + limit : null, + }; + } +} diff --git a/experiments/atomic-codemode/native-evidence-reads.test.mjs b/experiments/atomic-codemode/native-evidence-reads.test.mjs new file mode 100644 index 0000000000..974d20097a --- /dev/null +++ b/experiments/atomic-codemode/native-evidence-reads.test.mjs @@ -0,0 +1,86 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { createHash } from "node:crypto"; +import { gzipSync } from "node:zlib"; +import { NativeEvidenceReads } from "./native-evidence-reads.mjs"; + +const hash = (s) => createHash("sha256").update(s).digest("hex"); +function captured(text) { + const bytes = Buffer.from(text); + const wire = JSON.stringify({ + binary: "/input", + count: 1, + language: "c", + error: null, + functions: [ + { + address_hex: "0x1000", + code: "retained C", + error: null, + atomic_facts: { + schema_version: 4, + encoding: "gzip-base64", + raw_bytes: bytes.length, + data: gzipSync(bytes).toString("base64"), + }, + }, + ], + }); + return { + text: wire, + source: { + evidence_id: "ev_owned", + sha256: hash(wire), + bytes: Buffer.byteLength(wire), + }, + }; +} +test("decoded Evidence pages preserve their own byte identity and raw source provenance", async () => { + const logical = "{malformed JSON with δΈ­ζ–‡ and πŸ˜€", + source = captured(logical); + const reader = new NativeEvidenceReads(async () => source); + const raw = await reader.read({ evidence_id: "ev_owned", limit: 1e5 }); + assert.equal(raw.text, source.text); + assert.equal(raw.view, undefined); + const chunks = []; + for (let offset = 0; offset < logical.length; offset += 3) { + const page = await reader.read({ + evidence_id: "ev_owned", + component: "recovered_ir", + offset, + limit: 3, + }); + chunks.push(page.text); + assert.equal(page.source.sha256, source.source.sha256); + assert.equal(page.view.sha256, hash(logical)); + assert.equal(page.view.bytes, Buffer.byteLength(logical)); + assert.equal(page.total_code_units, logical.length); + } + assert.equal(chunks.join(""), logical); +}); +test("decoded view reuse still requires source-owner validation and follows content identity", async () => { + let source = captured('{"schema_version":3}'), + failure = false, + reads = 0; + const reader = new NativeEvidenceReads(async () => { + reads++; + if (failure) throw new Error("source_changed"); + return source; + }); + const query = { evidence_id: "ev_owned", component: "recovered_ir" }; + await reader.read(query); + failure = true; + await assert.rejects(reader.read(query), /source_changed/); + failure = false; + source = captured('{"schema_version":3,"new":true}'); + assert.equal( + (await reader.read(query)).text, + '{"schema_version":3,"new":true}', + ); + reader.clear(); + assert.equal( + (await reader.read(query)).view.sha256, + hash('{"schema_version":3,"new":true}'), + ); + assert.equal(reads, 4); +}); diff --git a/experiments/atomic-codemode/native-graph-encoding.mjs b/experiments/atomic-codemode/native-graph-encoding.mjs new file mode 100644 index 0000000000..1b2c97cf66 --- /dev/null +++ b/experiments/atomic-codemode/native-graph-encoding.mjs @@ -0,0 +1,36 @@ +// The wire owner validates the archive before normal graph-row admission. +import { gunzipSync } from "node:zlib"; +import { z } from "zod"; +import { requireFact } from "./native-contract.mjs"; +import { NATIVE_OUTPUT_BYTES } from "./limits.mjs"; + +export const MAX_GRAPH_JSON_BYTES = 32 * 1024 * 1024; +const envelope = z.strictObject({ + schema_version: z.literal(4), + encoding: z.literal("gzip-base64"), + raw_bytes: z.number().int().min(1).max(MAX_GRAPH_JSON_BYTES), + data: z.string().max(NATIVE_OUTPUT_BYTES), +}); +export function decodeGraphArchive(raw) { + if (raw?.schema_version !== 4) return undefined; + const encoded = envelope.parse(raw); + requireFact( + encoded.data.length > 0 && + encoded.data.length % 4 === 0 && + /^[A-Za-z0-9+/]*={0,2}$/u.test(encoded.data), + "Native compressed graph has invalid base64", + ); + const bytes = Buffer.from(encoded.data, "base64"); + requireFact( + bytes.toString("base64") === encoded.data, + "Native compressed graph has noncanonical base64", + ); + // The native-declared length is admitted before inflation, not checked only + // after an unbounded allocation. Node also verifies the gzip framing/CRC. + const decoded = gunzipSync(bytes, { maxOutputLength: encoded.raw_bytes }); + requireFact( + decoded.length === encoded.raw_bytes, + "Native compressed graph length differs from its declaration", + ); + return new TextDecoder("utf-8", { fatal: true }).decode(decoded); +} diff --git a/experiments/atomic-codemode/native-graph-encoding.test.mjs b/experiments/atomic-codemode/native-graph-encoding.test.mjs new file mode 100644 index 0000000000..866d9848e6 --- /dev/null +++ b/experiments/atomic-codemode/native-graph-encoding.test.mjs @@ -0,0 +1,66 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { gzipSync } from "node:zlib"; +import { + decodeGraphArchive, + MAX_GRAPH_JSON_BYTES, +} from "./native-graph-encoding.mjs"; + +const encodeBytes = (bytes) => ({ + schema_version: 4, + encoding: "gzip-base64", + raw_bytes: bytes.length, + data: gzipSync(bytes, { level: 1 }).toString("base64"), +}); +const encode = (value) => encodeBytes(Buffer.from(JSON.stringify(value))); +test("bounded gzip preserves logical JSON fields and legacy graphs without a second interpretation", () => { + const graph = { + schema_version: 3, + statement: "δΈ­ζ–‡\u0000πŸ˜€", + offset: "0xffffffffffffffff", + }; + assert.equal(decodeGraphArchive(encode(graph)), JSON.stringify(graph)); + assert.equal(decodeGraphArchive(graph), undefined); +}); +test("admission rejects unsupported envelopes and malformed base64 before inflation", () => { + const good = encode({ schema_version: 3 }); + for (const change of [ + { encoding: "deflate" }, + { raw_bytes: 0 }, + { raw_bytes: MAX_GRAPH_JSON_BYTES + 1 }, + { unexpected: true }, + { data: "AAAA\n" }, + { data: "A===" }, + { data: "?AAA" }, + { data: "A".repeat(8 * 1024 * 1024 + 1) }, + ]) + assert.throws(() => decodeGraphArchive({ ...good, ...change })); + assert.throws( + () => decodeGraphArchive({ ...good, data: "AB==" }), + /noncanonical/, + ); +}); +test("inflation respects declared size and verifies framing without interpreting logical source", () => { + const graph = { schema_version: 3, statement: "x".repeat(1024 * 1024) }; + const good = encode(graph); + assert.throws(() => decodeGraphArchive({ ...good, raw_bytes: 64 }), { + code: "ERR_BUFFER_TOO_LARGE", + }); + assert.throws( + () => decodeGraphArchive({ ...good, raw_bytes: good.raw_bytes + 1 }), + /differs/, + ); + const corrupt = Buffer.from(good.data, "base64"); + corrupt[corrupt.length - 8] ^= 1; + assert.throws(() => + decodeGraphArchive({ ...good, data: corrupt.toString("base64") }), + ); + assert.throws(() => + decodeGraphArchive({ + ...good, + data: Buffer.from(good.data, "base64").subarray(0, -5).toString("base64"), + }), + ); + assert.throws(() => decodeGraphArchive(encodeBytes(Buffer.from([0xff])))); + assert.equal(decodeGraphArchive(encodeBytes(Buffer.from("{"))), "{"); +}); diff --git a/experiments/atomic-codemode/native-graph-transport-study.md b/experiments/atomic-codemode/native-graph-transport-study.md new file mode 100644 index 0000000000..87c90c8f25 --- /dev/null +++ b/experiments/atomic-codemode/native-graph-transport-study.md @@ -0,0 +1,114 @@ +# Bounded full recovered graphs + +This implements the preceding native encoding probe as an opt-in experiment, +not stock Kuna support, an integrated release or provider parity. + +## Format and owners + +Logical graph schema 3 stays unchanged. A fitting ordinary record keeps its +existing representation. Before dropping facts, the writer can encode only the +graph in a schema-4 `{encoding:"gzip-base64",raw_bytes,data}` envelope. Ordinary +C/metadata and independent calls keep their fields. Existing graph/call/ +projection/code fallback and explicit failure still apply if encodings do not +fit; no graph is clipped as complete. + +Raw serialization stops at 32 MiB. Envelope overhead and base64 expansion are +accounted for before allocation; gzip output and encoded alternatives share the +unchanged caller budget of 8 MiB. Alternatives move ownership instead of cloning +graphs. Construction separately admits 32,000 operations, 36,000 values, 100,000 +inputs, 12,000 blocks, 50,000 CFG edges and 4 MiB display/type text. Calls and +projections keep their earlier limits. + +Each collector now owns one `TextBudget` for combined display/type charges. The +initial larger graph exposed a hidden second 2 MiB limit in value collection; +the first real VM probe rejected it with `atomic text capacity exceeded` while +preserving calls/C/projection. Passing the collector-owned budget through value +extraction removes this competing policy. Final graph text is 2,345,629 bytes. + +`native-graph-encoding.mjs` owns envelope admission, canonical base64, gzip +integrity, exact lengths and fatal UTF-8 decoding. Inflation uses Node's +[`maxOutputLength`](https://github.com/nodejs/node/blob/main/doc/api/zlib.md#class-options) +with an admitted 32 MiB maximum. Logical schema/row/relationship/text admission +and existing native identity/CFG/phi checks then operate on the same graph. +Nested encodings fail logical admission; bad graphs preserve independent calls/C. + +`evidence` defaults to the exact source wire record and original hash. +`component:"recovered_ir"` pages decoded graph source with a separate view hash +and byte count. Well-framed malformed logical facts remain inspectable without +being accepted as facts. `NativeEvidenceReads` owns one decoded view up to 32 MiB +per lease; the source owner revalidates bytes on every read. Content identity +keys the view; successful close clears it. All adapters share these owners. + +## Real verification + +Kuna upstream: `fe01d6294eb0d1053ce343c8943646df88e143fc`. +Final preserved engine: +`677b9795d49a65e459054ecbd3c0df37d603b541d72df9f74b46778947ed3936`. +Patch SHA-256: +`9f5754a15bd971326d14085f4c515348735d140be3bf36a371e7cf336fc291f3`. +Binaries are copied outside mutable Cargo outputs, not hardlinked or committed. + +On the pinned Debian SQLite target, both actual MCP adapters under Node 24.18.0 +verify 21,349 operations, 35,755 values, 1,229 blocks and zero omitted definitions. +The complete graph deeply equals the earlier diagnostic golden, and ordinary +C/metadata plus all 416 recovered calls are unchanged. Selected old projection +frontiers now resolve, selected CFG/PHI reads work, both complete Evidence views +verify and repeated snapshots reuse one source. This establishes producer +fidelity, not independent machine/IR equivalence or runtime feasibility. + +The actual wire document is **3,788,881 bytes**; decoded graph is **17,008,223 +bytes**. The public byte budget remains 8,388,608. Raw receipts/source copies are +`full-graph-codec-*` under `.cache/atomic-codemode/`. + +| Executed scope | Family wall seconds | CPU seconds | Kernel peak MiB | +| ---------------------------------------- | ------------------: | ----------: | --------------: | +| Initial build | 441.295 | 391.223 | 2,194.8 | +| Focused native tests, including compile | 98.589 | 98.034 | 632.9 | +| Formatted cached rebuild | 26.999 | 26.749 | 499.3 | +| Collector-budget rebuild | 361.041 | 342.220 | 2,192.9 | +| First probe: hidden text-cap failure | 39.279 | 39.010 | 650.9 | +| Exploratory success under system Node 20 | 44.858 | 44.480 | 576.6 | +| Final Code Mode MCP, pinned Node 24 | 42.422 | 42.111 | 651.0 | +| Final direct MCP, pinned Node 24 | 41.496 | 41.252 | 644.2 | +| CLI: incorrect unique-name assumption | 1.229 | 1.037 | 132.5 | +| CLI: corrected string/xref composition | 43.512 | 43.217 | 584.2 | +| Full-graph cache benchmark | 41.912 | 39.955 | 588.4 | +| Both catalogs/reference schemas | 3.644 | 1.716 | 142.7 | +| Three small functions / call composition | 8.579 | 8.358 | 300.4 | +| Legacy engine / current reader | 39.813 | 39.425 | 642.5 | + +All resource runs use one CPU, 4 GiB hard / 3 GiB early-stop bounds, verified +cleanup and OOM 0. Kernel peak includes charged file cache, not RSS. The system +Node probe is outside pinned-toolchain validation; the verifier now rejects a +wrong Node before native analysis. The first failed probe used preserved +pre-fix `fb717dde...`. Native focused tests pass ten cases; shared JS passes 65. +Changed JS lint/format, lenient native `check-spec`, applied-source whitespace +and patch forward/reverse checks pass. Patch-context spaces are format markers. +The three small-function records retain their prior ordinary representations; +the current reader also accepts the old engine's explicit full-graph refusal, +independent calls and 4,096-operation projection through real MCP reads. +Native full corpus/stages/workspace and whole-REA gates remain unexecuted or +previously resource-limited; no upstream/PR/integration readiness is claimed. + +Fresh-lease snapshot: 39,677.7 ms / 38.726 family CPU seconds. Same-lease warm: +67.1 ms / 0.068 seconds, identical handle/source. Cumulative peak is 588.4 MiB, +an upper bound on isolated warm memory. OS cache is uncontrolled; no cross-session +cache or model-token benefit is inferred from this scripted benchmark. + +## From names/messages to conditions + +The inventory has two identical VDBE names (`0x288f0`, `0xea6d0`) without aliases. +The initial CLI assumed uniqueness. Its corrected workflow composes strings, +incoming references and owning-function addresses before one VM snapshot; then +constant producers, uses, derived controllers and selected values/definitions. +It obtains both savepoint guards in sixteen host reads without the C report. +The final example also checks pagination before treating selected facts as a +complete set. A retained-source replay derives the two controller relations in +about 8.9 ms wall; isolated replay CPU/peak memory were not measured. + +The [question/oracle](savepoint-control-oracle.md) are fixed before model trials, +with function name/messages but no supplied code addresses. Selected independent +machine evidence checks local guards, load/spill/setup and shared call. Recovered +CFG/use coverage still does not establish full machine flow, memory aliases, +nontermination or feasible execution. Real-model costs/answers remain to be +measured; public scripted checks do not establish them. diff --git a/experiments/atomic-codemode/native-ir-contract.mjs b/experiments/atomic-codemode/native-ir-contract.mjs index 475fcf7d1d..2c4c6cc744 100644 --- a/experiments/atomic-codemode/native-ir-contract.mjs +++ b/experiments/atomic-codemode/native-ir-contract.mjs @@ -5,6 +5,10 @@ import { NATIVE_OUTPUT_BYTES } from "./limits.mjs"; import { readCallFacet } from "./native-call-contract.mjs"; import { readProjectionFacet } from "./native-projection-contract.mjs"; import { operation, parseOperation } from "./native-operation-contract.mjs"; +import { + decodeGraphArchive, + MAX_GRAPH_JSON_BYTES, +} from "./native-graph-encoding.mjs"; import { hex, ordinal, @@ -40,7 +44,7 @@ const factsSchema = z.strictObject({ ) .min(1) .max(12000), - operations: z.array(operation).max(12000), + operations: z.array(operation).max(32000), values: z.array(value).max(36000), coverage: z.strictObject({ scope: z.string(), @@ -111,7 +115,7 @@ function preflight(facts) { ); for (const [key, limit] of [ ["blocks", 12000], - ["operations", 12000], + ["operations", 32000], ["values", 36000], ]) requireFact( @@ -119,7 +123,7 @@ function preflight(facts) { `Native ${key} exceeds resource capacity or has invalid shape`, ); for (const [rows, limit] of [ - [facts.coverage?.indirect_branch_operations, 12000], + [facts.coverage?.indirect_branch_operations, 32000], [facts.coverage?.omitted_definition_ids, 36000], [facts.coverage?.issues, 12000], ]) @@ -144,40 +148,69 @@ function preflight(facts) { 0, ); requireFact( - inputs <= 50000 && members <= 12000 && edges <= 100000, + inputs <= 100000 && members <= 32000 && edges <= 100000, "Native relationships exceed resource capacity", ); + const displayBytes = + facts.operations.reduce( + (sum, op) => + sum + + (typeof op?.statement === "string" + ? Buffer.byteLength(op.statement) + : 0), + 0, + ) + + facts.values.reduce( + (sum, value) => + sum + + (typeof value?.type_name === "string" + ? Buffer.byteLength(value.type_name) + : 0), + 0, + ); + requireFact( + displayBytes <= 4 * 1024 * 1024, + "Native graph display exceeds resource capacity", + ); } const readGraph = (fn) => readNativeFacet(fn, "atomic_facts", "recovered IR", (raw) => { - preflight(raw); - return parseFacts(raw); + const decoded = decodeGraphArchive(raw); + const graph = decoded === undefined ? raw : JSON.parse(decoded); + preflight(graph); + return parseFacts(graph); }); +function readSource(text) { + requireFact( + Buffer.byteLength(text) <= NATIVE_OUTPUT_BYTES, + "Native JSON source exceeds transport budget", + ); + return JSON.parse(text); +} +function readFunction(raw, entry, expectedInput) { + requireFact( + Array.isArray(raw?.functions) && raw.functions.length === 1, + "Native source must describe exactly one function", + ); + const doc = documentSchema.parse(raw), + fn = doc.functions[0]; + if (expectedInput !== undefined && doc.binary !== expectedInput) + throw new FactFailure( + "subject_mismatch", + "Native source names a different command input", + { expected: expectedInput, observed: doc.binary }, + ); + if (entry !== undefined && fn.address_hex !== entry) + throw new FactFailure( + "subject_mismatch", + "Native function entry differs from requested subject", + ); + return { doc, fn }; +} export function parseNativeFunction(text, entry, expectedInput) { try { - requireFact( - Buffer.byteLength(text) <= NATIVE_OUTPUT_BYTES, - "Native JSON source exceeds transport budget", - ); - const raw = JSON.parse(text); - requireFact( - Array.isArray(raw?.functions) && raw.functions.length === 1, - "Native source must describe exactly one function", - ); - const doc = documentSchema.parse(raw), - fn = doc.functions[0]; - if (expectedInput !== undefined && doc.binary !== expectedInput) - throw new FactFailure( - "subject_mismatch", - "Native source names a different command input", - { expected: expectedInput, observed: doc.binary }, - ); - if (fn.address_hex !== entry) - throw new FactFailure( - "subject_mismatch", - "Native function entry differs from requested subject", - ); + const { doc, fn } = readFunction(readSource(text), entry, expectedInput); return { code: fn.code, language: doc.language, @@ -189,3 +222,33 @@ export function parseNativeFunction(text, entry, expectedInput) { throw formatFailure(error); } } + +export function readNativeGraphEvidence(text) { + try { + const raw = readSource(text); + if (raw?.functions === undefined) + throw new FactFailure( + "capability_unavailable", + "This retained source is not a native function record", + ); + const { fn } = readFunction(raw); + if (fn.atomic_facts === undefined) + throw new FactFailure( + "capability_unavailable", + "Recovered graph source was not captured", + { native_error: fn.atomic_facts_error?.slice(0, 1024) }, + ); + const decoded = decodeGraphArchive(fn.atomic_facts); + const view = decoded ?? JSON.stringify(fn.atomic_facts); + requireFact( + Buffer.byteLength(view) <= MAX_GRAPH_JSON_BYTES, + "Native graph Evidence view exceeds decoded capacity", + ); + return { + text: view, + encoding: decoded === undefined ? "json" : "gzip-base64", + }; + } catch (error) { + throw formatFailure(error); + } +} diff --git a/experiments/atomic-codemode/native-ir.test.mjs b/experiments/atomic-codemode/native-ir.test.mjs index 660e2614cd..4a32649dd1 100644 --- a/experiments/atomic-codemode/native-ir.test.mjs +++ b/experiments/atomic-codemode/native-ir.test.mjs @@ -1,6 +1,8 @@ import test from "node:test"; import assert from "node:assert/strict"; +import { gzipSync } from "node:zlib"; import { parseNativeIR } from "./native-ir.mjs"; +import { readNativeGraphEvidence } from "./native-ir-contract.mjs"; import { controlDependencies } from "./control-dependence.mjs"; const id = (n) => `0x${n}:0`; @@ -121,6 +123,68 @@ function diamond() { }; } +test("compressed and ordinary graphs have identical parsed facts and malformed archives preserve code", () => { + const doc = diamond(), + raw = Buffer.from(JSON.stringify(doc.functions[0].atomic_facts)); + const ordinary = parseNativeIR(JSON.stringify(doc), "0x1000"); + doc.functions[0].atomic_facts = { + schema_version: 4, + encoding: "gzip-base64", + raw_bytes: raw.length, + data: gzipSync(raw).toString("base64"), + }; + const encoded = parseNativeIR(JSON.stringify(doc), "0x1000"); + assert.deepEqual(encoded.coverage, ordinary.coverage); + assert.deepEqual( + encoded.select({ fields: ["inputs", "output", "definitions"] }), + ordinary.select({ fields: ["inputs", "output", "definitions"] }), + ); + assert.deepEqual(encoded.blocks({}), ordinary.blocks({})); + assert.deepEqual(readNativeGraphEvidence(JSON.stringify(doc)), { + text: raw.toString(), + encoding: "gzip-base64", + }); + for (const change of [ + { raw_bytes: 2 }, + { data: "broken" }, + { raw_bytes: 32 * 1024 * 1024 + 1 }, + ]) { + const bad = structuredClone(doc); + Object.assign(bad.functions[0].atomic_facts, change); + const model = parseNativeIR(JSON.stringify(bad), "0x1000"); + assert.equal(model.code, "reconstructed C"); + assert.equal(model.facets.recovered_ir.availability, "unavailable"); + assert.throws(() => model.select({})); + } + for (const text of [ + "{malformed logical JSON", + JSON.stringify({ schema_version: 4, data: "nested" }), + ]) { + const bad = structuredClone(doc), + bytes = Buffer.from(text); + bad.functions[0].atomic_facts = { + schema_version: 4, + encoding: "gzip-base64", + raw_bytes: bytes.length, + data: gzipSync(bytes).toString("base64"), + }; + const source = JSON.stringify(bad); + assert.equal( + parseNativeIR(source, "0x1000").facets.recovered_ir.availability, + "unavailable", + ); + assert.equal( + readNativeGraphEvidence(source).text, + text, + "Malformed logical facts remain inspectable through owned decoded source", + ); + } + assert.throws( + () => readNativeGraphEvidence('{"hex":"0102"}'), + (error) => error.code === "capability_unavailable", + ); +}); + test("native identity, implicit CFG and operand/phi bindings do not depend on display text", () => { const model = parseNativeIR(JSON.stringify(diamond()), "0x1000"); assert.equal(model.operation("0x3:0").origin_address, "0x3"); @@ -248,7 +312,7 @@ test("native facet failure preserves C without fabricating an empty graph; sourc test("oversized or malformed native arrays fail before diagnostic expansion", () => { const capacity = diamond(); capacity.functions[0].atomic_facts.operations = Array.from( - { length: 12001 }, + { length: 32001 }, () => ({}), ); assert.throws( diff --git a/experiments/atomic-codemode/native-savepoint-study.js.txt b/experiments/atomic-codemode/native-savepoint-study.js.txt new file mode 100644 index 0000000000..5ca4b593f2 --- /dev/null +++ b/experiments/atomic-codemode/native-savepoint-study.js.txt @@ -0,0 +1,50 @@ +const read = async (name,input) => { + const result=await rea[name](input); + if(!result.ok)throw new Error(JSON.stringify(result.error)); + return result.data; +}; +const complete=(page)=>{ + if(page.next_offset!==null)throw new Error("Continue the selected fact page before drawing a complete conclusion"); + return page; +}; +const found=await read("functions",{query:"sqlite3VdbeExec",exact:true}); +complete(found); +const formats=[]; +for(const query of ["cannot open savepoint - SQL statements in progress","cannot release savepoint - SQL statements in progress"]) { + const strings=await read("strings",{query}); + complete(strings); + const addresses=[...new Set(strings.items.filter((s)=>s.value===query).map((s)=>s.address))]; + if(addresses.length!==1)throw new Error("Ambiguous format: "+JSON.stringify(strings)); + const references=await read("references",{addresses,direction:"incoming",fields:["fromFunction","instruction"]}); + for(const result of references.results) { + if(!result.ok)throw new Error(JSON.stringify(result.error)); + complete(result); + } + formats.push({query,address:addresses[0],string_evidence_id:strings.source.evidence_id,references}); +} +const owners=formats.map((f)=>new Set(f.references.results.flatMap((r)=>r.ok?r.items.map((ref)=>ref.fromFunction?.address).filter(Boolean):[]))); +const candidates=found.items.filter((f)=>owners.every((set)=>set.has(f.address))); +if(candidates.length!==1)throw new Error("Unresolved format-owning function: "+JSON.stringify({found,formats})); +const capture=await read("snapshot",{entry:candidates[0].address}); +const snapshot=capture.snapshot; +for(const format of formats) { + const producers=await read("operations",{snapshot,constant:format.address,fields:["opcode","output"]}); + complete(producers); + const setup=producers.items.filter((op)=>op.opcode==="CPUI_PTRSUB"); + if(setup.length!==1)throw new Error("Ambiguous format producer: "+JSON.stringify(producers)); + const uses=await read("relations",{snapshot,kind:"uses",seeds:[setup[0].id]}); + complete(uses); + Object.assign(format,{producer:setup[0],uses}); +} +const controls=await read("relations",{snapshot,kind:"controllers",seeds:formats.map((f)=>f.producer.block)}); +complete(controls); +const branchIDs=[...new Set(controls.items.flatMap((relation)=>relation.edge.condition_value?[relation.edge.condition_value]:[]))]; +const conditions=await read("values",{snapshot,ids:branchIDs}); +complete(conditions); +const definitions=await read("operations",{snapshot,ids:conditions.items.map((v)=>v.definition).filter(Boolean),fields:["opcode","inputs","output","constants"]}); +complete(definitions); +const values=await read("values",{snapshot,ids:[...new Set(definitions.items.flatMap((op)=>op.inputs.map((i)=>i.value_id).filter(Boolean)))]}); +complete(values); +const evidence=await read("evidence",{evidence_id:capture.source.evidence_id,component:"recovered_ir",limit:128}); +if((await read("snapshot",{entry:candidates[0].address})).snapshot!==snapshot)throw new Error("Capture was not reused"); +return {capture,formats,controls,conditions,definitions,values,evidence}; diff --git a/experiments/atomic-codemode/provider.mjs b/experiments/atomic-codemode/provider.mjs index 82fc531c43..3e63be60d0 100644 --- a/experiments/atomic-codemode/provider.mjs +++ b/experiments/atomic-codemode/provider.mjs @@ -8,6 +8,7 @@ import { KunaTargetCommands } from "../../dist/kuna/KunaTargetCommands.js"; import { referenceReader } from "./reference-reads.mjs"; import { PrivateRuntimeRoot } from "../../dist/process/PrivateRuntimeRoot.js"; import { parseNativeIR } from "./native-ir.mjs"; +import { NativeEvidenceReads } from "./native-evidence-reads.mjs"; import { DEFAULT_NATIVE_OPERATION_FIELDS } from "./native-operation-views.mjs"; import { FactFailure, requireResult } from "./fact-error.mjs"; import { windowReaders } from "./window-readers.mjs"; @@ -42,6 +43,10 @@ export class KunaFactOwner { #strings; #functions; #captureDirectory; + #evidence = new NativeEvidenceReads(async (id) => ({ + text: await this.#sourceText(id), + source: this.#sources.get(id).source, + })); #nativeIR = false; context; static async open(path, environment = process.env) { @@ -462,16 +467,7 @@ export class KunaFactOwner { }; } async evidence(input) { - const text = await this.#sourceText(input.evidence_id); - return { - source: this.#sources.get(input.evidence_id).source, - text: text.slice(input.offset, input.offset + input.limit), - total_code_units: text.length, - next_offset: - input.offset + input.limit < text.length - ? input.offset + input.limit - : null, - }; + return this.#evidence.read(input); } async close() { this.#lifetime.abort(); @@ -484,6 +480,7 @@ export class KunaFactOwner { this.#snapshots.clear(); this.#strings = undefined; this.#functions = undefined; + this.#evidence.clear(); } cancel() { this.#lifetime.abort(); diff --git a/experiments/atomic-codemode/reference.mjs b/experiments/atomic-codemode/reference.mjs index 7206b6e7c9..99d6d07037 100644 --- a/experiments/atomic-codemode/reference.mjs +++ b/experiments/atomic-codemode/reference.mjs @@ -36,7 +36,7 @@ const returns = { "data:{operations:[operation rows as above],coverage:{status,boundaries:[{id,reason}],missing,missing_reasons?,source_scope?,source_issues,runtime_reachability,memory_alias_dependencies},value_frontier?:[{value_id?|operation?,input_index?,reason}],evidence_id}", code: "data:{evidence_id,start_line,language:string|null,text,total_lines,next_offset,coverage}; language is native-reported or unknown. Read/filter full reconstructed text within Code Mode without returning it all.", evidence: - "data:{source:{evidence_id,sha256,bytes,arguments},text,total_code_units,next_offset}; source has complete native output, including code before the SSA graph.", + "data:{source:{evidence_id,sha256,bytes,arguments},view?:{component,encoding,sha256,bytes,authority},text,total_code_units,next_offset}. Default component:source preserves exact native wire output. component:recovered_ir exposes complete decoded graph JSON with a separate view hash, including compressed graphs; source hash still identifies the original wire capture. This source view preserves malformed logical facts without asserting their validity. One decoded Evidence view up to 32 MiB is cached per lease and source freshness is checked on reads.", }; const referenceSchemas = new WeakMap(); const schemaFor = (input) => { diff --git a/experiments/atomic-codemode/savepoint-control-oracle.md b/experiments/atomic-codemode/savepoint-control-oracle.md new file mode 100644 index 0000000000..923eb63b5f --- /dev/null +++ b/experiments/atomic-codemode/savepoint-control-oracle.md @@ -0,0 +1,95 @@ +# Savepoint format/control oracle + +Pinned before model trials; not provided to measured models. This question gives +a function name and messages, not entry, instruction, graph or value addresses. +It is still a scoped local investigation, not general unseeded reconstruction. +Both atomic adapters share the same native facts and authority. No traditional +provider cell exists for this question, whose required graph identities are not +a fair provider-independent outcome for such a cell. + +Target: Debian SQLite `3.40.1-2+deb12u2` amd64, +`/usr/lib/x86_64-linux-gnu/libsqlite3.so.0` -> `.0.8.6`, SHA-256 +`2e6eef9a727f081f0d453b4e5e6cbd8b9ef8b6f86cbf7681cbad444d3b0b55c8`. +Preserved engine: `677b9795d49a65e459054ecbd3c0df37d603b541d72df9f74b46778947ed3936`. + +The exact-name inventory has entries `0x288f0` (16 bytes) and `0xea6d0` (32,384 +bytes), with identical names and no aliases. The inventory alone does not label +PLT versus implementation. Incoming data references from the two strings identify +the owning `sqlite3VdbeExec` entry `0xea6d0`; the CLI reproducer verifies this +composition. Do not select by catalog order or claim the name establishes binding. + +| Format | Address / NUL-inclusive bytes | Producer / block | Merge input | +| ----------------------------------------------------- | ----------------------------- | ------------------- | -------------------------- | +| cannot open savepoint - SQL statements in progress | 0x12c500 / 51 | 0xf2381:712c / 1142 | 0, v100012a75, edge 1142:0 | +| cannot release savepoint - SQL statements in progress | 0x12c538 / 54 | 0xf2437:753c / 1158 | 1, v10001352e, edge 1158:0 | + +Both `CPUI_PTRSUB` format producers use constant-zero input 0 and the eight-byte +address constant at input 1. Both feed PHI `0xf238b:ceb7` in block 1159, output +`v100033f25`, passed in RSI (eight-byte register storage at `0x30`) at recovered +input index 2 of shared CALL `0xf238b:7130`. The native target is `ram:0x296c0`, +label `sqlite3VdbeError`, independently verified as the machine direct CALL target; +runtime binding and complete source signatures remain unknown. + +## Local recovered conditions + +| Setup | Controller / CBRANCH | True edge / destination | False edge / destination | Condition | +| ------------------- | ------------------------ | ----------------------- | ------------------------ | ------------------- | +| open, block 1142 | block 1141, 0xedc01:7065 | 1141:1 -> 1142 | 1141:0 -> 1143 | v100012864, ne-zero | +| release, block 1158 | block 1157, 0xf0389:71b8 | 1157:1 -> 1158 | 1157:0 -> 1161 | v100012bd9, ne-zero | + +The open condition is one-byte `v100012864`, defined by +`0xedc01:7064 CPUI_INT_SLESS`: input 0 is four-byte constant zero +`v30001285f`; input 1 is four-byte `v100012840`, defined by +`0xedbf8:7056 CPUI_LOAD`. Thus the local recovered guard is a **signed** positive +loaded 32-bit value, not an unsigned nonzero test. + +The release condition is one-byte `v100012bd9`, defined by +`0xf0384:71b3 CPUI_INT_EQUAL`: input 0 is four-byte `v10001280a`, defined by +`0xedbe5:7041 CPUI_LOAD`; input 1 is four-byte constant one `v3000393a3`. +Its local guard is equality to one. Both load contents and aliases remain unknown. +Models need not invent source member names for either memory object. + +Derived controllers return precisely the two relations above for dependents +1142/1158 in the actual retained-source replay. Coverage is complete for finite +paths to recovered zero-successor exits; nontermination and runtime reachability +remain unknown. These are direct/local control relations, not all transitive +earlier guards or proof of feasible execution. + +## Independent selected machine evidence + +GNU objdump over the same ELF independently supplies these bytes/transfers; +ELF file/virtual mappings also verify their raw bytes. The complete recovered +graph, including these identities, deeply equals the earlier diagnostic native +golden. That producer comparison is not independent machine/IR equivalence. + +| Address | Bytes | Machine fact | +| ------- | -------------- | ------------------------------- | +| 0xedbe5 | 8b7004 | MOV ESI,[RAX+4] | +| 0xedbec | 89742440 | MOV [RSP+0x40],ESI | +| 0xedbf8 | 418b9dd8000000 | MOV EBX,[R13+0xd8] | +| 0xedbff | 85db | TEST EBX,EBX | +| 0xedc01 | 0f8f6b470000 | JG 0xf2372; fallthrough 0xedc07 | +| 0xf0384 | 837c244001 | CMP dword [RSP+0x40],1 | +| 0xf0389 | 0f849e200000 | JE 0xf242d; fallthrough 0xf038f | +| 0xf2381 | 488d3578a10300 | LEA RSI,[0x12c500] | +| 0xf238b | e83073f3ff | CALL 0x296c0 | +| 0xf2437 | 488d35faa00300 | LEA RSI,[0x12c538] | +| 0xf2446 | e940ffffff | JMP 0xf238b | + +The earlier load at `0xedbe5` and spill at `0xedbec` correspond to the native +value later compared at `0xf0384`; selected bytes do not prove all intervening +memory/runtime behavior. Machine setup/branch windows and native optimized CFG +edges are separate evidence representations. PHI is an IR operation at a shared +origin, not a machine instruction. + +Expected recovered coverage is now a full block-member graph: 21,349 operations, +35,755 values, 1,229 blocks and zero omitted definition IDs. It includes 416 +recovered calls; 395 known CALL targets and 21 unresolved CALLIND targets. This +does not establish full machine-flow/call coverage or memory alias resolution. +The previous call-rooted projection lacked both CBRANCH/Boolean definitions and +the open-path LOAD, as well as all CFG access; those absences remain recorded. + +Grade the complete answer and actual trace, not keywords alone: both formats and +shared call/PHI alternatives, both edges for each local condition, precise Boolean +and memory widths/signedness, focused machine checks, real source IDs/digest, +scope/earlier-guard/alias/nontermination/runtime limits, and retained-capture reuse. diff --git a/experiments/atomic-codemode/sqlite-savepoint-control-question.txt b/experiments/atomic-codemode/sqlite-savepoint-control-question.txt new file mode 100644 index 0000000000..528e9e2ae9 --- /dev/null +++ b/experiments/atomic-codemode/sqlite-savepoint-control-question.txt @@ -0,0 +1,30 @@ +Investigate sqlite3VdbeExec in {{artifact_path}} statically, starting from these +observed error formats: +- cannot open savepoint - SQL statements in progress +- cannot release savepoint - SQL statements in progress + +Locate both retained format sources and determine whether they reach the same +recovered error call, including how their values merge. Names can identify +multiple callable entries; establish the format-owning implementation from +references or byte evidence before deep analysis. Reuse one retained VM capture +and selected structured facts rather than fetching the whole C report or graph. + +For each format-setup block, find the nearest recovered conditional predecessor +controlling entry to that setup. Report both successor edges, their identities, +polarity and condition value/test. Trace each condition through its Boolean +definition to constants and LOAD boundaries, preserving operand indexes and +widths. Stop at those memory reads; do not guess source field names or arbitrary +runtime contents. Distinct SSA values, storage offsets and computed contents are +different facts. + +Verify the selected setup sites, shared machine CALL, conditional branch targets +and relevant load/spill instructions with focused instruction windows. Explain +the static local guards, including comparison signedness, without claiming these +are all earlier path conditions or a demonstrated SQL trigger. Distinguish native +edge/PHI identities from machine instruction addresses and transfers. + +Keep the answer focused, with real Evidence IDs and target digest. State the +available recovered-graph scope and remaining alias, complete-machine-flow, +nontermination and runtime-feasibility limits. If using derived controllers, +retain their finite-recovered-exit-path semantics. Complete retained Evidence +has a paged source/decoded-graph view; return only the facts needed here. diff --git a/experiments/atomic-codemode/verify-discovery.mjs b/experiments/atomic-codemode/verify-discovery.mjs index 5d66a0ed73..bf03b285eb 100644 --- a/experiments/atomic-codemode/verify-discovery.mjs +++ b/experiments/atomic-codemode/verify-discovery.mjs @@ -44,6 +44,7 @@ for (const direct of [false, true]) { "values", "instructions", "bytes", + "evidence", ]; const selected = await call(ids.map((name) => prefix + name).join(" ")); assert.deepEqual( @@ -78,6 +79,11 @@ for (const direct of [false, true]) { input_indices: [0], }, ], + [ + "evidence", + { evidence_id: "ev_example", component: "recovered_ir" }, + { evidence_id: "ev_example", component: "unknown" }, + ], ]) { const method = selected.methods.find((m) => m.method === name); const validators = [ajv.compile(method.input_schema)]; diff --git a/experiments/atomic-codemode/verify-full-graph.mjs b/experiments/atomic-codemode/verify-full-graph.mjs new file mode 100644 index 0000000000..e48f299278 --- /dev/null +++ b/experiments/atomic-codemode/verify-full-graph.mjs @@ -0,0 +1,227 @@ +// Real public retained-graph reads; producer fidelity is not machine equivalence. +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import { resolve, join } from "node:path"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import Ajv2020 from "ajv/dist/2020.js"; +import { readNativeGraphEvidence } from "./native-ir-contract.mjs"; +import { + createVerifierRun, + completeVerifierRun, + assertVerifierCleanupComplete, +} from "../../scripts/lib/verifier-run.mjs"; + +const [binary, output, goldenPath, partialPath] = process.argv.slice(2); +assert.equal( + process.versions.node, + (await readFile(resolve(".nvmrc"), "utf8")).trim(), + "Use the repository's pinned Node before running native analysis", +); +const golden = JSON.parse(await readFile(goldenPath, "utf8")).functions[0]; +const graph = golden.atomic_facts; +const partial = JSON.parse(await readFile(partialPath, "utf8")).functions[0] + .atomic_projection; +const formerFrontier = partial.coverage.frontier + .slice(0, 3) + .map((row) => row.definition); +const run = createVerifierRun(), + calls = [], + failures = []; +const direct = process.env.REA_ATOMIC_ADAPTER === "direct"; +const client = new Client({ + name: "rea-full-graph-verifier", + version: "0.1.0", +}); +const transport = new StdioClientTransport({ + command: process.execPath, + args: [resolve("experiments/atomic-codemode/server.mjs")], + env: { ...process.env }, + stderr: "pipe", +}); +const call = async (name, input) => { + const reply = await client.callTool( + { name, arguments: input }, + { timeout: 180000 }, + ); + const value = JSON.parse(reply.content[0].text); + calls.push({ + name, + input, + value, + content_bytes: Buffer.byteLength(reply.content[0].text), + }); + assert.equal(value.ok, true, JSON.stringify(value)); + return value; +}; +const read = async (name, input) => { + const value = direct + ? await call(`facts_${name}`, input) + : ( + await call("execute_javascript", { + code: `return await rea.${name}(${JSON.stringify(input)});`, + }) + ).value; + assert.equal(value.ok, true, JSON.stringify(value)); + return value.data; +}; +let sourceReview; +try { + await client.connect(transport); + const catalog = await client.listTools(), + ajv = new Ajv2020({ strict: false }); + for (const tool of catalog.tools) ajv.compile(tool.inputSchema); + await call("open_binary", { path: binary }); + const capture = await read("snapshot", { entry: "0xea6d0" }), + snapshot = capture.snapshot; + assert.equal(capture.facets.recovered_ir.availability, "available"); + assert.equal(capture.facets.operations.count, 21349); + assert.equal(capture.facets.values.count, 35755); + assert.equal(capture.coverage.blocks, 1229); + assert.equal(capture.coverage.omitted_definition_count, 0); + assert.equal(capture.coverage.runtime_successors, "unknown"); + const definitions = await read("operations", { + snapshot, + ids: formerFrontier, + fields: ["opcode", "output"], + }); + assert.deepEqual( + definitions.items.map((op) => op.id), + formerFrontier, + ); + for (const op of definitions.items) { + const expected = graph.operations.find((row) => row.id === op.id); + assert.equal(op.opcode, expected.opcode); + assert.equal(op.output, expected.output); + } + const blocks = await read("blocks", { + snapshot, + ids: [1154, 1159], + fields: ["predecessors", "successors", "operations"], + }); + for (const block of blocks.items) { + const expected = graph.blocks.find((row) => row.id === block.id); + for (const field of ["predecessors", "successors", "operations"]) + assert.deepEqual(block[field], expected[field]); + } + const edges = await read("relations", { + snapshot, + kind: "incoming", + seeds: [1154, 1159], + limit: 128, + }); + const expectedEdges = [1154, 1159].flatMap((seed) => + graph.blocks.flatMap((block) => + block.successors + .filter((edge) => edge.target === seed) + .map((edge) => ({ + ...edge, + from: block.id, + to: edge.target, + operation: block.operations.at(-1) ?? null, + kind: edge.polarity ? "conditional" : "recovered", + })), + ), + ); + assert.deepEqual(edges.items, expectedEdges); + const operands = await read("relations", { + snapshot, + kind: "operands", + seeds: ["0xea998:14649"], + input_indices: [0, 105, 155], + }); + assert.equal(operands.items.length, 3); + const retained = await read("calls", { + snapshot, + origin_addresses: ["0xf115c", "0xf238b"], + fields: ["arguments", "target"], + }); + assert.equal(retained.items.length, 2); + assert.equal(retained.evidence_id, capture.source.evidence_id); + const raw = await read("evidence", { + evidence_id: capture.source.evidence_id, + limit: 128, + }); + const decoded = await read("evidence", { + evidence_id: capture.source.evidence_id, + component: "recovered_ir", + limit: 256, + }); + assert.equal(raw.view, undefined); + assert.equal(decoded.view.encoding, "gzip-base64"); + assert.equal(decoded.source.sha256, capture.source.sha256); + const text = await readFile( + join( + process.env.REA_ATOMIC_EVIDENCE_DIR, + `${capture.source.evidence_id}.txt`, + ), + "utf8", + ); + assert.equal( + createHash("sha256").update(text).digest("hex"), + capture.source.sha256, + ); + assert.ok(Buffer.byteLength(text) <= 8 * 1024 * 1024); + const fn = JSON.parse(text).functions[0]; + const view = readNativeGraphEvidence(text); + assert.deepEqual(JSON.parse(view.text), graph); + assert.equal( + decoded.view.sha256, + createHash("sha256").update(view.text).digest("hex"), + ); + assert.equal(decoded.view.bytes, Buffer.byteLength(view.text)); + assert.equal(decoded.text, view.text.slice(0, 256)); + const { atomic_facts: _ignored, ...ordinary } = fn; + const { atomic_facts: _oldGraph, ...goldenOrdinary } = golden; + assert.deepEqual(ordinary, goldenOrdinary); + const next = await read("evidence", { + evidence_id: capture.source.evidence_id, + component: "recovered_ir", + offset: decoded.next_offset, + limit: 64, + }); + assert.deepEqual(next.view, decoded.view); + assert.equal( + next.text, + view.text.slice(decoded.next_offset, decoded.next_offset + 64), + ); + assert.equal( + (await read("snapshot", { entry: "0xea6d0" })).snapshot, + snapshot, + ); + sourceReview = { + source_bytes: Buffer.byteLength(text), + decoded_graph_bytes: Buffer.byteLength(view.text), + logical_graph_equals_diagnostic_golden: true, + ordinary_record_and_calls_equal_golden: true, + former_projection_frontier_readable: formerFrontier, + complete_evidence_views_verified: true, + }; + await call("close_binary", {}); +} catch (error) { + failures.push(error.stack ?? String(error)); +} finally { + await client.close(); + await transport.close(); +} +const lifecycle = await completeVerifierRun(run); +assertVerifierCleanupComplete(lifecycle.process_lineage); +await writeFile( + output, + JSON.stringify({ + direct, + node_version: process.version, + calls, + failures, + sourceReview, + lifecycle, + scope: + "Real public MCP plus complete retained producer-gold comparison; no model or runtime/machine-equivalence claim.", + }), + { flag: "wx" }, +); +console.log( + JSON.stringify({ output, calls: calls.length, failures, sourceReview }), +); +if (failures.length) process.exitCode = 1;