From 47d3c6b1a15c166091b47666c9cac6dbd882454c Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Sun, 18 Oct 2026 10:32:58 +0800 Subject: [PATCH] test(perf): verify authentic retained investigation queries (#4814) Verify retained queries against authentic SQLite ELF and npm histories, retain failed RPC/cleanup/output costs, and make mixed-history verification repeatable. Record the separate validation-reuse bottleneck in #4810. Co-authored-by: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com> Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> --- experiments/analysis-efficiency/README.md | 7 + .../analysis-efficiency/agent-trial.py | 1 + .../retained-investigation-results.json | 635 ++++++++++++++++++ .../retained-investigation-workflows.md | 201 ++++++ .../retained-investigation.mjs | 310 +++++++++ .../retained-mixed-verification.mjs | 163 +++++ .../retained-query-calls.mjs | 46 ++ .../retained-query-finalization.mjs | 81 +++ .../test_retained_queries.py | 166 +++++ 9 files changed, 1610 insertions(+) create mode 100644 experiments/analysis-efficiency/retained-investigation-results.json create mode 100644 experiments/analysis-efficiency/retained-investigation-workflows.md create mode 100644 experiments/analysis-efficiency/retained-investigation.mjs create mode 100644 experiments/analysis-efficiency/retained-mixed-verification.mjs create mode 100644 experiments/analysis-efficiency/retained-query-calls.mjs create mode 100644 experiments/analysis-efficiency/retained-query-finalization.mjs create mode 100644 experiments/analysis-efficiency/test_retained_queries.py diff --git a/experiments/analysis-efficiency/README.md b/experiments/analysis-efficiency/README.md index 295a2aa765..227ed379c4 100644 --- a/experiments/analysis-efficiency/README.md +++ b/experiments/analysis-efficiency/README.md @@ -1,5 +1,12 @@ # Analysis efficiency measurements +For authentic old Evidence and mixed investigations, see the +[retained-query checkpoint](retained-investigation-workflows.md). +`retained-investigation.mjs` compares existing complete delivery with selected +pages, including capacity-refusal recovery and original-record round trips. +It records negative model results and separates imported-format validation work +from the query contract. + For genuine Fiddler captures, see the [SAZ selection checkpoint](saz-selection-workflows.md). `measure.mjs --saz-selection baseline|focused|retained` compares default complete delivery (including retained-reference recovery), caller-selected ordinals, and diff --git a/experiments/analysis-efficiency/agent-trial.py b/experiments/analysis-efficiency/agent-trial.py index 810fc55902..7b668c1933 100644 --- a/experiments/analysis-efficiency/agent-trial.py +++ b/experiments/analysis-efficiency/agent-trial.py @@ -187,6 +187,7 @@ def main(): 'inspect_managed_members', 'inspect_managed_native_boundaries', 'inspect_artifact', 'inspect_wasm_artifact', 'inspect_evm_interface', 'get_evidence_bundle', 'export_evidence_bundle', + 'import_evidence_bundle', 'list_unknowns', 'verify_unknown_resolution', 'capture_browser_scenario', 'resolve_containing_procedure', 'analyze_function', 'close_binary']: config += '[mcp_servers.rea.tools.' + tool + ']\napproval_mode = "approve"\n' diff --git a/experiments/analysis-efficiency/retained-investigation-results.json b/experiments/analysis-efficiency/retained-investigation-results.json new file mode 100644 index 0000000000..261166dc71 --- /dev/null +++ b/experiments/analysis-efficiency/retained-investigation-results.json @@ -0,0 +1,635 @@ +{ + "runtime_revision": "f4322b15ed1423d736bd2572cf7a8eadf2b466e1", + "host": "Linux x86-64", + "node": "24.18.0", + "npm": "11.16.0", + "codex_cli": "0.161.0", + "requested_model": "gpt-6.1-sol", + "reasoning": "high", + "resolved_model_identity": "unknown", + "code_mode": false, + "client_enabled_tools": [ + "import_evidence_bundle", + "get_evidence_bundle", + "inspect_analysis_view", + "list_unknowns" + ], + "catalog": { + "tools": 207, + "serialized_bytes": 3152125, + "sha256": "4f31d9d56c400bc48f5639a9dc4b4cdad7faf862e4be81641d94fa70552438e4" + }, + "bounds": { + "cpu_quota_percent": 100, + "family_hard_mib": 8192, + "early_stop_mib": 7168, + "swap": false, + "tasks": 192 + }, + "counter_scope": "Payload-exit counters with observer alive; kernel family peaks include file cache; terminal whole-scope counters and remote model compute unknown", + "sources": { + "native": { + "target_sha256": "2e6eef9a727f081f0d453b4e5e6cbd8b9ef8b6f86cbf7681cbad444d3b0b55c8", + "sections": 26, + "symbols": 1479, + "relocations": 2963, + "all_symbol_locations_names_values_agree_with_independent_ELF_parser": true + }, + "javascript": { + "package": "npm", + "version": "11.16.0", + "source_files": [ + { + "path": "package.json", + "sha256": "60ff022e32b5d8291d3e09849fd0cee6bbbff4e4d8352ad258214b1adf070714" + }, + { + "path": "bin/npm-cli.js", + "sha256": "8e5f6f3429f8cdbe693cdc29904e9d5a7b127a494bd15c804bd54c7403bfcbe7" + }, + { + "path": "lib/cli.js", + "sha256": "67666f06479f9b0bbc01412c198caadd4287d34d5ed74a02871ea78f186451e7" + }, + { + "path": "lib/cli/entry.js", + "sha256": "667a44c1fa8795fdbba8ad843736ca7f8f24e64511985e9a7060b002123560c6" + }, + { + "path": "lib/commands/install.js", + "sha256": "f7a60e8b40ab1bece76a2441b8b15378d199f05c6050eebe455cd9961f67b816" + } + ], + "lib_cli_module": { + "node_id": "jag_node_131257ad52f3333971c2a5ed325c64357e52604489e42ccc67a8849770cb5b10", + "kind": "javascript-module", + "path": "lib/cli.js" + }, + "source_bytes_checked": true, + "inventory_completeness": "producer-relative, not independently established", + "unselected_sources": 1416 + } + }, + "sdk": [ + { + "name": "sdk-native-complete-1", + "status": "passed", + "answer_sha256": "4e28c2c2a25620576d27abe846a05835ec49386b889779e33410c31971523b54", + "query_ms": 1328.7, + "query_reply_bytes": 11224227, + "rpc_count": 7, + "elapsed_seconds": 14.117, + "cpu_seconds": 13.915, + "peak_family_mib": 1160.8, + "memory_events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "payload_cleanup_verified": true, + "response_capacity_refused": false, + "warm_page_ms": [] + }, + { + "name": "sdk-native-complete-2", + "status": "passed", + "answer_sha256": "4e28c2c2a25620576d27abe846a05835ec49386b889779e33410c31971523b54", + "query_ms": 1385.4, + "query_reply_bytes": 11224227, + "rpc_count": 7, + "elapsed_seconds": 14.722, + "cpu_seconds": 14.485, + "peak_family_mib": 1109.9, + "memory_events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "payload_cleanup_verified": true, + "response_capacity_refused": false, + "warm_page_ms": [] + }, + { + "name": "sdk-native-focused-1", + "status": "passed", + "answer_sha256": "4e28c2c2a25620576d27abe846a05835ec49386b889779e33410c31971523b54", + "query_ms": 529.7, + "query_reply_bytes": 167985, + "rpc_count": 25, + "elapsed_seconds": 12.907, + "cpu_seconds": 12.679, + "peak_family_mib": 934.8, + "memory_events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "payload_cleanup_verified": true, + "response_capacity_refused": false, + "warm_page_ms": [44.1, 59.6, 24.0, 30.8, 26.7, 47.4] + }, + { + "name": "sdk-native-focused-2", + "status": "passed", + "answer_sha256": "4e28c2c2a25620576d27abe846a05835ec49386b889779e33410c31971523b54", + "query_ms": 572.2, + "query_reply_bytes": 167985, + "rpc_count": 25, + "elapsed_seconds": 13.316, + "cpu_seconds": 13.116, + "peak_family_mib": 1000.0, + "memory_events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "payload_cleanup_verified": true, + "response_capacity_refused": false, + "warm_page_ms": [44.0, 64.6, 26.8, 33.2, 29.8, 50.4] + }, + { + "name": "sdk-javascript-complete-1", + "status": "passed", + "answer_sha256": "0fe3ad249eb1a7233a4966a56652c173667c48fdaf395b6c08fc6901ef3b46f7", + "query_ms": 11241.5, + "query_reply_bytes": 212856, + "rpc_count": 31, + "elapsed_seconds": 26.3, + "cpu_seconds": 26.1, + "peak_family_mib": 1333.5, + "memory_events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "payload_cleanup_verified": true, + "response_capacity_refused": true, + "warm_page_ms": [1124.6, 1073.8, 1474.7, 1074.7, 1081.6, 1109.5] + }, + { + "name": "sdk-javascript-complete-2", + "status": "passed", + "answer_sha256": "0fe3ad249eb1a7233a4966a56652c173667c48fdaf395b6c08fc6901ef3b46f7", + "query_ms": 11938.1, + "query_reply_bytes": 212856, + "rpc_count": 31, + "elapsed_seconds": 27.105, + "cpu_seconds": 26.87, + "peak_family_mib": 1320.9, + "memory_events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "payload_cleanup_verified": true, + "response_capacity_refused": true, + "warm_page_ms": [1155.4, 1096.2, 1489.3, 1121.3, 1088.3, 1484.6] + }, + { + "name": "sdk-javascript-focused-2", + "status": "passed", + "answer_sha256": "0fe3ad249eb1a7233a4966a56652c173667c48fdaf395b6c08fc6901ef3b46f7", + "query_ms": 10967.3, + "query_reply_bytes": 208350, + "rpc_count": 28, + "elapsed_seconds": 25.708, + "cpu_seconds": 25.505, + "peak_family_mib": 1344.0, + "memory_events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "payload_cleanup_verified": true, + "response_capacity_refused": false, + "warm_page_ms": [1342.7, 1085.1, 1055.0, 1034.6, 1082.7, 1440.7] + }, + { + "name": "sdk-javascript-focused-3", + "status": "passed", + "answer_sha256": "0fe3ad249eb1a7233a4966a56652c173667c48fdaf395b6c08fc6901ef3b46f7", + "query_ms": 10990.4, + "query_reply_bytes": 208350, + "rpc_count": 28, + "elapsed_seconds": 26.197, + "cpu_seconds": 25.98, + "peak_family_mib": 1336.0, + "memory_events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "payload_cleanup_verified": true, + "response_capacity_refused": false, + "warm_page_ms": [1338.1, 1088.5, 1069.1, 1038.3, 1087.5, 1483.0] + }, + { + "name": "sdk-mixed-native", + "status": "passed", + "answer_sha256": "4e28c2c2a25620576d27abe846a05835ec49386b889779e33410c31971523b54", + "query_ms": 560.0, + "query_reply_bytes": 167991, + "rpc_count": 25, + "elapsed_seconds": 16.933, + "cpu_seconds": 16.714, + "peak_family_mib": 1261.4, + "memory_events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "payload_cleanup_verified": true, + "response_capacity_refused": false, + "warm_page_ms": [46.9, 51.0, 36.3, 24.4, 38.6, 43.8] + }, + { + "name": "sdk-mixed-javascript", + "status": "passed", + "answer_sha256": "0fe3ad249eb1a7233a4966a56652c173667c48fdaf395b6c08fc6901ef3b46f7", + "query_ms": 10904.5, + "query_reply_bytes": 208356, + "rpc_count": 28, + "elapsed_seconds": 26.5, + "cpu_seconds": 26.297, + "peak_family_mib": 1346.5, + "memory_events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "payload_cleanup_verified": true, + "response_capacity_refused": false, + "warm_page_ms": [1321.9, 1063.3, 1070.8, 1309.9, 1041.7, 1074.2] + } + ], + "model_trials": [ + { + "name": "model-javascript-complete-1", + "facts_pass": true, + "elapsed_seconds": 76.49, + "cpu_seconds": 24.687, + "peak_family_mib": 818.7, + "input_tokens": 454491, + "cached_input_tokens": 396288, + "uncached_input_tokens": 58203, + "output_tokens": 2033, + "reported_reasoning_output_tokens": 207, + "mcp_calls_including_discovery": 9, + "payload_cleanup_verified": true, + "memory_events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + } + }, + { + "name": "model-javascript-complete-2", + "facts_pass": true, + "elapsed_seconds": 58.335, + "cpu_seconds": 14.993, + "peak_family_mib": 971.4, + "input_tokens": 328066, + "cached_input_tokens": 280704, + "uncached_input_tokens": 47362, + "output_tokens": 1676, + "reported_reasoning_output_tokens": 129, + "mcp_calls_including_discovery": 9, + "payload_cleanup_verified": true, + "memory_events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + } + }, + { + "name": "model-javascript-focused-1", + "facts_pass": false, + "elapsed_seconds": 34.206, + "cpu_seconds": 16.31, + "peak_family_mib": 811.9, + "input_tokens": 167470, + "cached_input_tokens": 136320, + "uncached_input_tokens": 31150, + "output_tokens": 633, + "reported_reasoning_output_tokens": 172, + "mcp_calls_including_discovery": 4, + "payload_cleanup_verified": true, + "memory_events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + } + }, + { + "name": "model-javascript-focused-2", + "facts_pass": true, + "elapsed_seconds": 66.455, + "cpu_seconds": 14.337, + "peak_family_mib": 955.3, + "input_tokens": 339528, + "cached_input_tokens": 307456, + "uncached_input_tokens": 32072, + "output_tokens": 2051, + "reported_reasoning_output_tokens": 169, + "mcp_calls_including_discovery": 10, + "payload_cleanup_verified": true, + "memory_events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + } + }, + { + "name": "model-javascript-guided-complete-1", + "facts_pass": true, + "elapsed_seconds": 69.206, + "cpu_seconds": 13.65, + "peak_family_mib": 888.8, + "input_tokens": 168862, + "cached_input_tokens": 121728, + "uncached_input_tokens": 47134, + "output_tokens": 2385, + "reported_reasoning_output_tokens": 737, + "mcp_calls_including_discovery": 6, + "payload_cleanup_verified": true, + "memory_events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + } + }, + { + "name": "model-javascript-guided-complete-2", + "facts_pass": false, + "elapsed_seconds": 29.187, + "cpu_seconds": 15.105, + "peak_family_mib": 697.4, + "input_tokens": 28696, + "cached_input_tokens": 23296, + "uncached_input_tokens": 5400, + "output_tokens": 610, + "reported_reasoning_output_tokens": 379, + "mcp_calls_including_discovery": 0, + "payload_cleanup_verified": true, + "memory_events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + } + }, + { + "name": "model-javascript-guided-focused-1", + "facts_pass": true, + "elapsed_seconds": 53.464, + "cpu_seconds": 12.059, + "peak_family_mib": 944.5, + "input_tokens": 124218, + "cached_input_tokens": 92416, + "uncached_input_tokens": 31802, + "output_tokens": 1746, + "reported_reasoning_output_tokens": 330, + "mcp_calls_including_discovery": 5, + "payload_cleanup_verified": true, + "memory_events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + } + }, + { + "name": "model-javascript-guided-focused-2", + "facts_pass": true, + "elapsed_seconds": 58.526, + "cpu_seconds": 22.852, + "peak_family_mib": 924.2, + "input_tokens": 129134, + "cached_input_tokens": 97536, + "uncached_input_tokens": 31598, + "output_tokens": 1478, + "reported_reasoning_output_tokens": 69, + "mcp_calls_including_discovery": 5, + "payload_cleanup_verified": true, + "memory_events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + } + }, + { + "name": "model-native-complete-1", + "facts_pass": true, + "elapsed_seconds": 52.103, + "cpu_seconds": 8.686, + "peak_family_mib": 1028.5, + "input_tokens": 287913, + "cached_input_tokens": 249984, + "uncached_input_tokens": 37929, + "output_tokens": 1724, + "reported_reasoning_output_tokens": 121, + "mcp_calls_including_discovery": 4, + "payload_cleanup_verified": true, + "memory_events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + } + }, + { + "name": "model-native-complete-2", + "facts_pass": true, + "elapsed_seconds": 58.261, + "cpu_seconds": 9.102, + "peak_family_mib": 837.4, + "input_tokens": 348976, + "cached_input_tokens": 286976, + "uncached_input_tokens": 62000, + "output_tokens": 2009, + "reported_reasoning_output_tokens": 246, + "mcp_calls_including_discovery": 5, + "payload_cleanup_verified": true, + "memory_events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + } + }, + { + "name": "model-native-focused-1", + "facts_pass": true, + "elapsed_seconds": 62.078, + "cpu_seconds": 10.198, + "peak_family_mib": 823.9, + "input_tokens": 335445, + "cached_input_tokens": 293120, + "uncached_input_tokens": 42325, + "output_tokens": 1924, + "reported_reasoning_output_tokens": 87, + "mcp_calls_including_discovery": 8, + "payload_cleanup_verified": true, + "memory_events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + } + }, + { + "name": "model-native-focused-2", + "facts_pass": true, + "elapsed_seconds": 57.933, + "cpu_seconds": 8.058, + "peak_family_mib": 821.3, + "input_tokens": 286938, + "cached_input_tokens": 256384, + "uncached_input_tokens": 30554, + "output_tokens": 1831, + "reported_reasoning_output_tokens": 106, + "mcp_calls_including_discovery": 8, + "payload_cleanup_verified": true, + "memory_events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + } + } + ], + "failed_trials": { + "model-javascript-focused-1": "Wrong connector discovery; approval-policy refusal; no source facts established.", + "model-javascript-guided-complete-2": "Premature agent refusal with no REA RPC; asserted tool unavailability not established by a connection/discovery observation." + }, + "pilot_failures": [ + "Private pilot oracle read view.item instead of view.items; corrected before the committed SDK cohorts. No product failure inferred." + ], + "decision": "Existing query APIs suffice for these bounded questions; no general token/time advantage established. Separate imported immutable-format validation optimization is tracked in #4810.", + "review_followup_verification": { + "review-mixed-native": { + "status": "passed", + "checks": { + "heads_checked": 64, + "pages": 32, + "revised_unknown_id": "unk_057c8f2dbf30ec98851a66ee054366bc6e2e168874581e6f99cb80673d0ff9b8", + "revision_checked": 2, + "stale_update_rejected": true, + "tampered_import_rejected_atomically": true, + "unknowns_after_close": 0, + "old_reference_refused": true + }, + "original_records_preserved": true, + "elapsed_seconds": 18.444, + "cpu_seconds": 18.16515, + "peak_family_mib": 1401.6328125, + "oom_events_at_payload_exit": 0, + "payload_cleanup_verified": true + }, + "review-mixed-javascript": { + "status": "passed", + "checks": { + "heads_checked": 64, + "pages": 32, + "revised_unknown_id": "unk_057c8f2dbf30ec98851a66ee054366bc6e2e168874581e6f99cb80673d0ff9b8", + "revision_checked": 2, + "stale_update_rejected": true, + "tampered_import_rejected_atomically": true, + "unknowns_after_close": 0, + "old_reference_refused": true + }, + "original_records_preserved": true, + "elapsed_seconds": 27.617, + "cpu_seconds": 27.275299, + "peak_family_mib": 1444.30859375, + "oom_events_at_payload_exit": 0, + "payload_cleanup_verified": true + }, + "deterministic_receipt_tests": { + "passed": 4, + "scope": "Rejected SDK call, protocol error, receipt-write failure, simultaneous call/write failure; not real-provider evidence" + }, + "cleanup_receipt_regressions": { + "passed": 6, + "scope": "Real stream/file refusals, independent peer close, and injected partial publication at the filesystem boundary. Original failure and call costs survive; failed cleanup retains workspace ownership. Harness failure checks, not real-provider evidence.", + "baseline_reproduction": { + "revision": "b025a919b36ec091ca888339df6b24e78e89b79b", + "method": "Execute the prior committed finalization block with deterministic workspace or log rejection; not a production-provider reproduction.", + "workspace_error": { + "file_receipt_attempts": 0, + "stdout_receipt_attempts": 0 + }, + "log_error": { + "file_receipt_attempts": 0, + "stdout_receipt_attempts": 0 + } + }, + "partial_retry_baseline": { + "revision": "5d6259e658ee54fc9829b9143a8721a2ab426e80", + "regression_exit_code": 1, + "observed_saved_report": "partial", + "method": "Run the unchanged partial-write regression against the prior committed finalizer with deterministic filesystem-boundary failure injection." + } + } + } +} diff --git a/experiments/analysis-efficiency/retained-investigation-workflows.md b/experiments/analysis-efficiency/retained-investigation-workflows.md new file mode 100644 index 0000000000..071f7077d4 --- /dev/null +++ b/experiments/analysis-efficiency/retained-investigation-workflows.md @@ -0,0 +1,201 @@ +# Retained investigation query checkpoint + +[#3134](https://github.com/morluto/rea/issues/3134) measured existing retained +queries on Linux. Selected reads preserved the requested facts, original +Evidence and Unknown revisions, including a mixed investigation. Existing +contracts suffice for these questions; this checkpoint adds no production API, +cache, persistence or default change. + +Imported JavaScript pages repeatedly validated their entire source graph. That +separate, observed bottleneck is owned by +[#4810](https://github.com/morluto/rea/issues/4810). Smaller replies did not +establish a general model-token or whole-task latency improvement. + +The [curated measurements](retained-investigation-results.json) retain individual +SDK/model costs and unsuccessful attempts. Raw bundles, transcripts, profiles +and detailed receipts stay private and are not build inputs. + +## Sources and questions + +Runtime: `f4322b15ed1423d736bd2572cf7a8eadf2b466e1`, Linux x86-64, +Node 24.18.0/npm 11.16.0. The SDK discovered all 207 advertised tools +(3,152,125 serialized catalog bytes). Model trials used Codex CLI 0.161.0, +requested `gpt-6.1-sol`/high, direct MCP, and the same four enabled tool names: +`import_evidence_bundle`, `get_evidence_bundle`, `inspect_analysis_view`, and +`list_unknowns`. The resolved service-side model identity is unknown. These are +selected-client-tool trials, not an evaluation of full-catalog agent discovery. + +Two public source workflows produced authentic bundles through production MCP: + +- Native: `inspect_binary_layout` on Debian's `libsqlite3.so.0.8.6`, SHA-256 + `2e6eef9a727f081f0d453b4e5e6cbd8b9ef8b6f86cbf7681cbad444d3b0b55c8`. + Pwntools 4.15.0/pyelftools 0.33/Unicorn 2.1.2 observed 26 sections, 1,479 + symbols and 2,963 relocations. An independent ELF64 decoder checked section + and relocation counts and every symbol's table/entry index, value, raw name + bytes and source entry range against the original file. +- JavaScript: shipped npm 11.16.0, directory inventory digest + `f8e576a7d162542c616168b25a95a5f62b9accf29467cc37e149905ad4b8e603`. + `analyze_javascript_application` selected exactly `package.json`, + `bin/npm-cli.js`, `lib/cli.js`, `lib/cli/entry.js`, and + `lib/commands/install.js`. It parsed four JavaScript files; 1,416 other + relevant files remained unselected. The complete retained result contained + 2,822 application nodes, 1,796 semantic nodes and 13,682 semantic unknowns. + Source-file digests and the `lib/cli.js` source-module identity were checked + against shipped bytes. Inventory/discovery completeness remains producer-relative. + +Analyst-recorded Unknowns came from the actual source limitations: five native +and seven JavaScript questions. These registry heads are distinct from the +13,682 semantic-graph unknowns. The exported histories contained 15 and 17 +records, respectively, and approximately 1.75 and 21.06 MB of canonical JSON. +No synthetic payload arrays were used. + +The SDK task read the first six original symbol/module rows and the relevant +current Unknown heads. Complete delivery and selected delivery answered the +same question; selected delivery intentionally exercised two-row pages. Each +fresh server performed three traversals, exported again and closed. The model +task additionally requested native identity/counts, or the five JavaScript source +choices and exact `lib/cli.js` source module, with explicit runtime uncertainty. +Compare arms within each declared task, not SDK costs against model costs. + +## Correctness, capacity and lifetime + +Both fresh-server SDK repetitions passed in both delivery arms. Selected rows, +ordering and counts agreed with original records; all original records and +Unknown revisions survived export/import, including links, omitted fields, +nulls and scalar types. Query calls did not rerun either producer. The resumed +servers had no native producer configuration. + +The JavaScript complete bundle exceeded the default 10 MiB receive-buffer +boundary. Its typed capacity refusal preserved the complete retained data and a +usable connection. Complete-arm costs include that refusal and subsequent +selected-view recovery; they are not timings for a successful complete transfer. + +A mixed history combined both authentic investigations and promoted 64 actual +semantic unknowns to separately identified analyst follow-ups. One follow-up +was updated to investigating. It contained 97 records, 76 heads and 77 revisions. +Both source queries preserved all original records/revisions. Thirty-two +two-head pages verified every semantic-follow-up head, including revision two. +A stale update and a tampered bundle import were rejected without changing the +ledger. Export reproduced the complete original mixed bundle. Closing cleared +records/Unknowns and made the old reference unavailable; no implicit persistence +or producer restart was introduced. + +Public CLI `inspect-evidence-bundle` summary and inline `inspect-analysis-view` +summary also passed on both authentic sources. Portable CLI input still requires +complete input admission/validation; small output does not remove that cost. + +## SDK costs + +Each row spans two fresh MCP servers. Query time is the sum of SDK RPC elapsed +times over three traversals. Local post-call contract validation, oracle checks +and transcript writes are included only in the whole harness, along with startup, +full discovery, file reads, import, export and close. OS caches were uncontrolled/warm. A fresh imported +session is not a cold native-analysis measurement. + +| Source / arm | Three traversals | Query reply bytes | Whole elapsed | Local family CPU | Family peak | +| ------------------------------ | ---------------: | ----------------: | --------------: | ---------------: | --------------: | +| Native complete | 1.329–1.385 s | 11,224,227 | 14.117–14.722 s | 13.915–14.485 s | 1,110–1,161 MiB | +| Native selected | 0.530–0.572 s | 167,985 | 12.907–13.316 s | 12.679–13.116 s | 935–1,000 MiB | +| JavaScript complete + recovery | 11.242–11.938 s | 212,856 | 26.300–27.105 s | 26.100–26.870 s | 1,321–1,334 MiB | +| JavaScript selected | 10.967–10.990 s | 208,350 | 25.708–26.197 s | 25.505–25.980 s | 1,336–1,344 MiB | + +Mixed-history selected tasks took 16.933 s native and 26.500 s JavaScript, +including the larger import and complete round-trip checks. Their family peaks +were 1,261 and 1,347 MiB. This does not establish scaling to thousands of +registry heads or every provider. + +Original-session npm warm pages measured 8–31 ms; after import, repeated pages +cost about 1.1–1.4 s. Source inspection shows that imported immutable Evidence +does not carry the format owner's private graph proofs, so each view repeats +full validation. Native layout views also parse the complete layout on each +page. #4810 owns equivalent before/after verification of a correction; this +checkpoint claims no speedup from an unimplemented cache. + +## Actual model tasks + +All twelve attempts retained their costs. Ten supplied every checked fact and +current Unknown identity/status; runtime and unselected-source coverage stayed +unknown. One initial JavaScript selected attempt searched the wrong connector +and received an approval-policy refusal. One guided complete attempt stopped +without an REA RPC and asserted tool unavailability without a supporting +connection/discovery observation. Both are failed tasks, despite CLI exit zero. + +| Cohort / arm | Successful tasks | Total input tokens | Elapsed | +| ------------------------------------- | ---------------: | -----------------------: | -----------------------: | +| Native complete | 2/2 | 287,913 / 348,976 | 52.103 / 58.261 s | +| Native selected, two-head pages | 2/2 | 335,445 / 286,938 | 62.078 / 57.933 s | +| JavaScript complete + recovery | 2/2 | 454,491 / 328,066 | 76.490 / 58.335 s | +| JavaScript selected, two-head pages | 1/2 | 167,470 failed / 339,528 | 34.206 failed / 66.455 s | +| JavaScript guided complete + recovery | 1/2 | 168,862 / 28,696 failed | 69.206 / 29.187 failed s | +| JavaScript guided selected | 2/2 | 124,218 / 129,134 | 53.464 / 58.526 s | + +The separate guided JavaScript cohort named the exact configured tools and +allowed one sufficiently large Unknown page for this bounded question. Both +arms received the same discovery guidance. Its single completed complete-arm +sample is insufficient for a stable repeated-pair benefit claim. Native total +input/time overlapped; extra pages and model turns can consume the reply saving. +Do not pool failed tasks as fast answers or attribute guidance changes to REA. + +The JSON report records total/cached/uncached input, output, local CPU and peak +memory per attempt. These are client-reported usage counters, not tokenizer +estimates; output includes the client's reported reasoning accounting without +adding it again. Schema/discovery, errors, retries and continuations remain in +the task. Remote model compute, full-catalog model behavior and other hosts were +not measured. + +Heavy runs were serial, one CPU, 8 GiB family hard limit, 7 GiB early stop, +no swap. Kernel peaks include charged file cache. Counter observations were +taken at payload exit while the owned observer remained alive; terminal +whole-scope counters remain unknown. Observed payload OOM/capacity events were +zero, all payloads exited and cleanup was verified. The expected MCP response +refusal is a separate transport-capacity outcome. + +## Reproduce + +Build the pinned runtime, acquire the matching real artifacts and produce the +two authentic histories with the public operations above. Export before close. +Keep bundles, transcripts and model profiles private. For a known source ID: + +```sh +node experiments/analysis-efficiency/retained-investigation.mjs \ + --runtime-root /absolute/built-rea \ + --revision f4322b15ed1423d736bd2572cf7a8eadf2b466e1 \ + --bundle /absolute/authentic-history.json \ + --evidence-id ev_ORIGINAL_SHA256 \ + --scenario native --delivery focused \ + --output /absolute/new-receipt-directory +``` + +Run under the existing process-family resource wrapper, with sufficient bounds +for the selected source. Repeat with `--delivery complete`; use +`--scenario javascript` for module rows. The task's Unknown domain is +`retained-native` or `retained-javascript`. Output is never overwritten. +`agent-trial.py --mode direct` supports the same four enabled names; its success +exit only establishes client completion. Grade original IDs/digests, exact +facts, coverage and Unknowns from the retained transcript and final answer. + +For a privately supplied authentic mixed bundle, add `--verify-mixed`. Besides +the normal source query and original-record round trip, this pages every current +head in `javascript-semantic-followup`, checks the current updated revision, +submits a deliberately stale update and a tampered copy of the bundle, verifies +atomic refusal, then checks Unknown/reference lifetime after close. The bundle +must contain genuine follow-ups and at least one updated head. The checkpoint's +64-head bundle passed this mode for both source scenarios. Its extra verification +costs are separate from the original SDK comparison rows above. + +Rejected SDK calls are recorded before propagating the original failure, with +request, phase and elapsed time. Reply bytes stay null when no usable reply was +received. Receipt-write failures remain distinct; simultaneous call/write +failures retain both causes. Deterministic failure tests run with +`python3 -m unittest discover -s experiments/analysis-efficiency -p test_retained_queries.py`. + +Finalization independently attempts client/transport close, log completion and +both receipt outputs. Cleanup failures preserve the original task failure and +call costs, record the failing phase and retain the workspace for retry. A file +receipt refusal still permits stdout; a broken stdout write is recorded in a +file retry. Files publish through a sibling temporary file and rename; an +interrupted retry preserves the previous receipt. Failed publication files stay +in the owned output directory. Cleanup or receipt-output failures make the +command fail. Six additional regressions exercise stream/file errors, partial +publication and peer cleanup; these are harness failure checks, not real-provider +evidence. diff --git a/experiments/analysis-efficiency/retained-investigation.mjs b/experiments/analysis-efficiency/retained-investigation.mjs new file mode 100644 index 0000000000..05772374dc --- /dev/null +++ b/experiments/analysis-efficiency/retained-investigation.mjs @@ -0,0 +1,310 @@ +/** Measure public retained queries against an authentic, previously exported bundle. */ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { createWriteStream } from "node:fs"; +import { + appendFile, + mkdir, + mkdtemp, + readFile, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, isAbsolute, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { parseArgs } from "node:util"; +import { finished } from "node:stream/promises"; +import canonicalize from "canonicalize"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { + requireMcpOperationResult, + requireMcpToolError, +} from "../../scripts/lib/mcp-verifier-results.mjs"; +import { createRetainedQueryCaller } from "./retained-query-calls.mjs"; +import { finalizeRetainedQueryReceipt } from "./retained-query-finalization.mjs"; +import { + verifyMixedRetainedHistory, + verifyMixedRetainedClosure, + retainedUnknownHeads, +} from "./retained-mixed-verification.mjs"; + +const { values } = parseArgs({ + options: { + "runtime-root": { + type: "string", + default: resolve(dirname(fileURLToPath(import.meta.url)), "../.."), + }, + revision: { type: "string" }, + bundle: { type: "string" }, + "evidence-id": { type: "string" }, + scenario: { type: "string" }, + delivery: { type: "string" }, + output: { type: "string" }, + "verify-mixed": { type: "boolean", default: false }, + }, +}); +for (const key of ["runtime-root", "bundle", "output"]) + assert.ok(values[key] && isAbsolute(values[key]), `${key} must be absolute`); +assert.ok(values.revision && values["evidence-id"]); +assert.ok(["native", "javascript"].includes(values.scenario)); +assert.ok(["complete", "focused"].includes(values.delivery)); +const root = values["runtime-root"]; +const output = values.output; +await mkdir(output, { recursive: false, mode: 0o700 }); +const digest = (bytes) => createHash("sha256").update(bytes).digest("hex"); +const sourceBytes = await readFile(values.bundle); +const original = JSON.parse(sourceBytes); +const source = original.records.find( + (record) => record.evidence_id === values["evidence-id"], +); +const operation = + values.scenario === "native" + ? "inspect_binary_layout" + : "analyze_javascript_application"; +assert.equal(source?.operation, operation); +const unknownDomain = `retained-${values.scenario}`; +const expectedUnknowns = retainedUnknownHeads(original.unknowns, unknownDomain); +// These oracles check selected original facts, not producer discovery completeness. +const expectedRows = + values.scenario === "native" + ? source.normalized_result.symbols + : source.normalized_result.graph.nodes + .filter((node) => + ["javascript-asset", "javascript-module", "source-module"].includes( + node.kind, + ), + ) + .map((node) => { + const candidates = node.observations.flatMap((observation) => { + const property = observation.properties.path; + const where = observation.evidence.location; + return [ + ...(typeof property === "string" ? [property] : []), + ...(where.available && where.value.kind === "artifact-path" + ? [where.value.path] + : where.available && where.value.kind === "source-range" + ? [where.value.source] + : []), + ]; + }); + return { + node_id: node.node_id, + kind: node.kind, + path: + node.identity.strategy === "canonical-path" + ? node.identity.path + : node.identity.strategy === "source-map-original" + ? node.identity.original_source + : (candidates[0] ?? null), + }; + }); +assert.ok(expectedRows.length >= 6, "This task needs six original rows"); +const runtimeEntrySha256 = digest( + await readFile(join(root, "dist/mcp/main.js")), +); +const account = await mkdtemp(join(tmpdir(), "rea-retained-query-")); +const transport = new StdioClientTransport({ + command: process.execPath, + args: [join(root, "scripts/rea.mjs"), "mcp"], + env: { + PATH: process.env.PATH, + HOME: account, + NODE_OPTIONS: "--max-old-space-size=2048", + REA_LOG_LEVEL: "silent", + }, + stderr: "pipe", +}); +const serverLog = createWriteStream(join(output, "server.log")); +const serverLogFinished = finished(serverLog); +serverLogFinished.catch(() => {}); +transport.stderr?.pipe(serverLog); +const client = new Client({ + name: "retained-investigation-measurement", + version: "1", +}); +const report = { + revision: values.revision, + runtime_entry_sha256: runtimeEntrySha256, + bundle_sha256: digest(sourceBytes), + bundle_bytes: sourceBytes.length, + source_evidence_id: source.evidence_id, + scenario: values.scenario, + delivery: values.delivery, + verify_mixed: values["verify-mixed"], + calls: [], + status: "running", +}; +const call = createRetainedQueryCaller(client, async (receipt, event) => { + report.calls.push(receipt); + await appendFile( + join(output, "transcript.jsonl"), + JSON.stringify(event) + "\n", + ); +}); +const result = (reply, name) => requireMcpOperationResult(reply, name); +try { + await client.connect(transport); + const started = performance.now(); + const catalog = await client.listTools(); + report.catalog = { + elapsed_ms: performance.now() - started, + tools: catalog.tools.length, + bytes: Buffer.byteLength(JSON.stringify(catalog)), + sha256: digest(JSON.stringify(catalog)), + }; + assert.equal(catalog.nextCursor, undefined); + await writeFile(join(output, "catalog.json"), JSON.stringify(catalog)); + result( + await call("import_evidence_bundle", { path: values.bundle }, "import"), + "import_evidence_bundle", + ); + const answers = []; + for (let repeat = 0; repeat < 3; repeat += 1) { + let answer; + if (values.delivery === "complete") { + const reply = await call("get_evidence_bundle", {}, `complete-${repeat}`); + if (reply.isError !== true) { + const bundle = result(reply, "get_evidence_bundle"); + assert.deepEqual( + bundle.records.find( + (record) => record.evidence_id === source.evidence_id, + ), + source, + ); + assert.deepEqual(bundle.unknowns, original.unknowns); + answer = [expectedRows.slice(0, 6), expectedUnknowns]; + } else { + // Keep capacity refusals and every recovery RPC in the completed task. + const failure = requireMcpToolError(reply); + assert.equal(failure.code, "resource_constraint"); + assert.equal(failure.details?.resource, "transport"); + report.complete_delivery_refused = true; + } + } + if (answer === undefined) { + const metadata = result( + await call( + "get_evidence_bundle", + { + detail: "summary", + filters: { evidence_id: source.evidence_id }, + }, + `discovery-${repeat}`, + ), + "get_evidence_bundle", + ); + assert.equal(metadata.matching_records, 1); + assert.equal(metadata.records[0].evidence_id, source.evidence_id); + const rows = []; + for (const offset of [0, 2, 4]) { + const view = result( + await call( + "inspect_analysis_view", + { + source: { + kind: "retained-evidence", + evidence_id: source.evidence_id, + }, + view: { + kind: "page", + collection: + values.scenario === "native" ? "symbols" : "modules", + offset, + limit: 2, + }, + }, + `page-${repeat}-${offset}`, + ), + "inspect_analysis_view", + ); + assert.equal(view.parent_evidence_id, source.evidence_id); + assert.equal(view.coverage.total, expectedRows.length); + assert.equal(view.coverage.examined, 2); + assert.equal( + view.coverage.next_offset, + offset + 2 < expectedRows.length ? offset + 2 : null, + ); + assert.deepEqual(view.items, expectedRows.slice(offset, offset + 2)); + rows.push(...view.items); + } + const unknowns = []; + let offset = 0; + do { + const page = result( + await call( + "list_unknowns", + { domain: unknownDomain, offset, limit: 2 }, + `unknowns-${repeat}-${offset}`, + ), + "list_unknowns", + ); + assert.equal(page.total, expectedUnknowns.length); + assert.equal(page.offset, offset); + assert.deepEqual( + page.items, + expectedUnknowns.slice(offset, offset + 2), + ); + assert.equal( + page.next_offset, + offset + 2 < expectedUnknowns.length ? offset + 2 : null, + ); + unknowns.push(...page.items); + offset = page.next_offset; + } while (offset !== null); + assert.deepEqual(unknowns, expectedUnknowns); + answer = [rows, unknowns]; + } + answers.push(digest(canonicalize(answer))); + } + assert.equal(new Set(answers).size, 1); + report.answer_sha256 = answers[0]; + if (values["verify-mixed"]) + report.mixed_verification = await verifyMixedRetainedHistory({ + call, + original, + sourceId: source.evidence_id, + output, + }); + const roundtrip = join(output, "roundtrip.json"); + result( + await call("export_evidence_bundle", { path: roundtrip }, "export"), + "export_evidence_bundle", + ); + const exported = JSON.parse(await readFile(roundtrip, "utf8")); + const exportedRecords = new Map( + exported.records.map((record) => [record.evidence_id, record]), + ); + for (const record of original.records) + assert.deepEqual(exportedRecords.get(record.evidence_id), record); + assert.deepEqual(exported.unknowns, original.unknowns); + report.original_records_preserved = true; + result(await call("close_binary", {}, "close"), "close_binary"); + const closed = result( + await call("get_evidence_bundle", { detail: "summary" }, "after-close"), + "get_evidence_bundle", + ); + assert.equal(closed.total_retained_records, 0); + report.close_cleared_records = true; + if (values["verify-mixed"]) + Object.assign( + report.mixed_verification, + await verifyMixedRetainedClosure(call, source.evidence_id), + ); + assert.equal(digest(await readFile(values.bundle)), report.bundle_sha256); + report.status = "passed"; +} catch (cause) { + report.status = "failed"; + report.failure = { name: cause.name, message: cause.message }; +} finally { + const finalized = await finalizeRetainedQueryReceipt({ + report, + client, + transport, + account, + serverLog, + serverLogFinished, + output, + }); + if (report.status !== "passed" || !finalized) process.exitCode = 1; +} diff --git a/experiments/analysis-efficiency/retained-mixed-verification.mjs b/experiments/analysis-efficiency/retained-mixed-verification.mjs new file mode 100644 index 0000000000..38ebb7a674 --- /dev/null +++ b/experiments/analysis-efficiency/retained-mixed-verification.mjs @@ -0,0 +1,163 @@ +import assert from "node:assert/strict"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { + requireMcpOperationResult as result, + requireMcpToolError, +} from "../../scripts/lib/mcp-verifier-results.mjs"; + +/** Expected current heads from the original exported revisions, not displayed pages. */ +export const retainedUnknownHeads = (unknowns, domain) => { + const heads = new Map(); + for (const unknown of unknowns) { + const previous = heads.get(unknown.unknown_id); + if (previous === undefined || previous.revision < unknown.revision) + heads.set(unknown.unknown_id, unknown); + } + return [...heads.values()] + .filter((unknown) => unknown.domain === domain) + .sort((a, b) => + a.unknown_id < b.unknown_id ? -1 : a.unknown_id > b.unknown_id ? 1 : 0, + ); +}; + +const verifyFollowupPages = async (call, original) => { + const expected = retainedUnknownHeads( + original.unknowns, + "javascript-semantic-followup", + ); + assert.ok( + expected.length > 0, + "Supply an authentic mixed history with semantic follow-ups", + ); + const revised = expected.find((unknown) => unknown.revision > 1); + assert.ok(revised, "Mixed verification requires a retained updated Unknown"); + let offset = 0; + let pages = 0; + const observed = []; + do { + const page = result( + await call( + "list_unknowns", + { + domain: "javascript-semantic-followup", + offset, + limit: 2, + }, + `mixed-heads-${offset}`, + ), + "list_unknowns", + ); + assert.equal(page.total, expected.length); + assert.equal(page.offset, offset); + assert.deepEqual(page.items, expected.slice(offset, offset + 2)); + assert.equal( + page.next_offset, + offset + 2 < expected.length ? offset + 2 : null, + ); + observed.push(...page.items); + offset = page.next_offset; + pages += 1; + } while (offset !== null); + assert.deepEqual(observed, expected); + return { revised, headsChecked: expected.length, pages }; +}; + +const verifyMutationRefusals = async ({ + call, + original, + sourceId, + output, + revised, +}) => { + const before = result( + await call( + "get_evidence_bundle", + { detail: "summary" }, + "mixed-before-refusals", + ), + "get_evidence_bundle", + ); + const stale = await call( + "update_unknown", + { + unknown_id: revised.unknown_id, + expected_revision: revised.revision - 1, + status: revised.status, + severity: revised.severity, + supporting_evidence_ids: revised.supporting_evidence_ids, + contradicting_evidence_ids: revised.contradicting_evidence_ids, + required_authority: revised.required_authority, + required_confidence: revised.required_confidence, + required_environment: revised.required_environment, + recommended_probes: revised.recommended_probes, + relationships: revised.relationships, + resolution: revised.resolution, + }, + "mixed-stale-update", + ); + assert.equal(requireMcpToolError(stale).code, "revision_conflict"); + const tampered = structuredClone(original); + const target = tampered.records.find( + (record) => record.evidence_id === sourceId, + ); + assert.ok(target); + target.normalized_result = { checkpoint_tampered: true }; + const path = join(output, "tampered-import.json"); + await writeFile(path, JSON.stringify(tampered), { flag: "wx" }); + const rejected = await call( + "import_evidence_bundle", + { path }, + "mixed-tampered-import", + ); + assert.equal( + requireMcpToolError(rejected).code, + "evidence_integrity_mismatch", + ); + const after = result( + await call( + "get_evidence_bundle", + { detail: "summary" }, + "mixed-after-refusals", + ), + "get_evidence_bundle", + ); + assert.deepEqual(after, before); +}; + +/** Verify original mixed-history heads, CAS rejection and atomic import failure. */ +export const verifyMixedRetainedHistory = async (options) => { + const { revised, headsChecked, pages } = await verifyFollowupPages( + options.call, + options.original, + ); + await verifyMutationRefusals({ ...options, revised }); + return { + heads_checked: headsChecked, + pages, + revised_unknown_id: revised.unknown_id, + revision_checked: revised.revision, + stale_update_rejected: true, + tampered_import_rejected_atomically: true, + }; +}; + +/** Closing must neither retain old Unknowns nor rehydrate a closed source. */ +export const verifyMixedRetainedClosure = async (call, sourceId) => { + const unknowns = result( + await call("list_unknowns", {}, "mixed-after-close-heads"), + "list_unknowns", + ); + assert.equal(unknowns.total, 0); + const missing = await call( + "inspect_analysis_view", + { + source: { kind: "retained-evidence", evidence_id: sourceId }, + view: { kind: "summary" }, + }, + "mixed-after-close-reference", + ); + const failure = requireMcpToolError(missing); + assert.equal(failure.details?.reason, "missing"); + return { unknowns_after_close: 0, old_reference_refused: true }; +}; diff --git a/experiments/analysis-efficiency/retained-query-calls.mjs b/experiments/analysis-efficiency/retained-query-calls.mjs new file mode 100644 index 0000000000..d8ba3aea09 --- /dev/null +++ b/experiments/analysis-efficiency/retained-query-calls.mjs @@ -0,0 +1,46 @@ +/** Keep settled and rejected RPCs in the same receipt before returning control. */ +export const createRetainedQueryCaller = + (client, record) => async (name, input, phase) => { + const start = performance.now(); + let reply; + try { + reply = await client.callTool( + { name, arguments: input }, + { timeout: 300000 }, + ); + } catch (cause) { + const failure = + cause instanceof Error + ? { name: cause.name, message: cause.message } + : { name: typeof cause, message: String(cause) }; + const receipt = { + name, + input, + phase, + elapsed_ms: performance.now() - start, + reply_bytes: null, + is_error: true, + failure, + }; + try { + await record(receipt, { name, input, phase, failure }); + } catch (recordingFailure) { + throw new AggregateError( + [cause, recordingFailure], + "MCP call and receipt recording both failed", + ); + } + throw cause; + } + const receipt = { + name, + input, + phase, + elapsed_ms: performance.now() - start, + reply_bytes: Buffer.byteLength(JSON.stringify(reply)), + is_error: reply.isError === true, + }; + // A receipt-write failure is distinct from a rejected MCP call. + await record(receipt, { name, input, phase, reply }); + return reply; + }; diff --git a/experiments/analysis-efficiency/retained-query-finalization.mjs b/experiments/analysis-efficiency/retained-query-finalization.mjs new file mode 100644 index 0000000000..599210d2cd --- /dev/null +++ b/experiments/analysis-efficiency/retained-query-finalization.mjs @@ -0,0 +1,81 @@ +import { rename, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +const attempt = async (failures, phase, action) => { + try { + await action(); + return true; + } catch (cause) { + failures.push(`${phase}: ${String(cause)}`); + return false; + } +}; + +const writeReceipt = (stream, text) => + new Promise((resolve, reject) => { + const onError = (cause) => reject(cause); + stream.once("error", onError); + try { + stream.write(text, (cause) => { + if (cause !== null && cause !== undefined) { + // A Writable emits its error after notifying the write callback. + reject(cause); + } else { + stream.off("error", onError); + resolve(); + } + }); + } catch (cause) { + stream.off("error", onError); + reject(cause); + } + }); + +/** Preserve task costs through independent cleanup and receipt-output failures. */ +export const finalizeRetainedQueryReceipt = async ({ + report, + client, + transport, + account, + serverLog, + serverLogFinished, + output, + stdout = process.stdout, +}) => { + report.cleanup_failures = []; + await Promise.all([ + attempt(report.cleanup_failures, "client", () => client.close()), + attempt(report.cleanup_failures, "transport", () => transport.close()), + ]); + await attempt(report.cleanup_failures, "server-log", async () => { + if (!serverLog.writableEnded) serverLog.end(); + await serverLogFinished; + }); + if (report.cleanup_failures.length === 0) + await attempt(report.cleanup_failures, "workspace", () => + rm(account, { recursive: true }), + ); + if (report.cleanup_failures.length > 0) report.retained_workspace = account; + + report.receipt_failures = []; + let publication = 0; + const saveReport = async () => { + // Keep failed publications in the owned output directory for inspection. + // A partial retry must never truncate an already published cost receipt. + const pending = join(output, `.report-${++publication}.json.pending`); + await writeFile(pending, JSON.stringify(report, null, 2) + "\n", { + flag: "wx", + }); + await rename(pending, join(output, "report.json")); + }; + await attempt(report.receipt_failures, "report-file", saveReport); + const stdoutWritten = await attempt(report.receipt_failures, "stdout", () => + writeReceipt(stdout, JSON.stringify(report) + "\n"), + ); + // Preserve a broken stdout diagnostic in the file that may already exist. + if (!stdoutWritten) + await attempt(report.receipt_failures, "report-file-retry", saveReport); + return ( + report.cleanup_failures.length === 0 && report.receipt_failures.length === 0 + ); +}; diff --git a/experiments/analysis-efficiency/test_retained_queries.py b/experiments/analysis-efficiency/test_retained_queries.py new file mode 100644 index 0000000000..50d9f58579 --- /dev/null +++ b/experiments/analysis-efficiency/test_retained_queries.py @@ -0,0 +1,166 @@ +"""Deterministic RPC receipt failures; not real-provider or model evidence.""" +import json +from pathlib import Path +import subprocess +import unittest + + +class RetainedQueryReceipts(unittest.TestCase): + def run_node(self, body): + module = Path(__file__).with_name('retained-query-calls.mjs').resolve().as_uri() + finalizer = Path(__file__).with_name('retained-query-finalization.mjs').resolve().as_uri() + source = ('import assert from "node:assert/strict";\n' + 'const {createRetainedQueryCaller} = await import(' + json.dumps(module) + ');\n' + 'const {finalizeRetainedQueryReceipt} = await import(' + json.dumps(finalizer) + ');\n' + body) + result = subprocess.run(['node', '--input-type=module', '-'], input=source, + text=True, capture_output=True, timeout=15) + self.assertEqual(result.returncode, 0, result.stderr) + + def run_finalization(self, body): + self.run_node(''' +import {mkdtemp,mkdir,readFile,writeFile,rm,stat} from 'node:fs/promises'; +import {join} from 'node:path'; +import {tmpdir} from 'node:os'; +import {Writable} from 'node:stream'; +import {finished} from 'node:stream/promises'; +const root=await mkdtemp(join(tmpdir(),'rea-receipt-test-')); +const account=join(root,'account');const output=join(root,'output'); +await mkdir(account);await mkdir(output); +const chunks=[]; +const stdout=new Writable({write(chunk,encoding,done){chunks.push(chunk.toString());done();}}); +const serverLog=new Writable({write(chunk,encoding,done){done();}}); +const serverLogFinished=finished(serverLog);serverLogFinished.catch(()=>{}); +const originalFailure={name:'Error',message:'original request failed'}; +const report={status:'failed',failure:originalFailure,calls:[{phase:'page',elapsed_ms:17}]}; +const input={report,client:{close:async()=>{}},transport:{close:async()=>{}}, + account,output,stdout,serverLog,serverLogFinished}; +const saved=async()=>JSON.parse(await readFile(join(output,'report.json'),'utf8')); +try { +''' + body + ''' +} finally { + serverLog.destroy();await serverLogFinished.catch(()=>{});stdout.destroy(); + await rm(root,{recursive:true,force:true}); +} +''') + + def test_rejected_call_preserves_request_phase_cost_and_original_failure(self): + self.run_node(''' +const failure = new Error('owned server closed'); +const input = {offset:0, limit:2}; const rows=[]; const events=[]; +const call = createRetainedQueryCaller({callTool:async request=>{ + assert.strictEqual(request.arguments,input);throw failure; +}}, async(row,event)=>{rows.push(row);events.push(event);}); +await assert.rejects(call('list_unknowns',input,'resume-2'), e=>e===failure); +assert.equal(rows.length,1);assert.equal(events.length,1); +assert.strictEqual(rows[0].input,input);assert.equal(rows[0].name,'list_unknowns'); +assert.equal(rows[0].phase,'resume-2');assert.ok(rows[0].elapsed_ms>=0); +assert.equal(rows[0].reply_bytes,null);assert.equal(rows[0].is_error,true); +assert.deepEqual(events[0].failure,{name:'Error',message:'owned server closed'}); +''') + + def test_protocol_error_is_a_received_reply_with_known_bytes(self): + self.run_node(''' +const reply={isError:true,content:[{type:'text',text:'capacity refusal'}]};const rows=[]; +const call=createRetainedQueryCaller({callTool:async()=>reply},async row=>rows.push(row)); +assert.strictEqual(await call('get_evidence_bundle',{},'complete'),reply); +assert.equal(rows.length,1);assert.equal(rows[0].is_error,true); +assert.equal(rows[0].reply_bytes,Buffer.byteLength(JSON.stringify(reply))); +assert.equal(Object.hasOwn(rows[0],'failure'),false); +''') + + def test_receipt_write_error_does_not_reclassify_a_completed_rpc(self): + self.run_node(''' +const failure=new Error('receipt disk unavailable');const rows=[]; +const call=createRetainedQueryCaller({callTool:async()=>({content:[]})},async row=>{ + rows.push(row);throw failure; +}); +await assert.rejects(call('list_unknowns',{},'read'),e=>e===failure); +assert.equal(rows.length,1);assert.equal(rows[0].is_error,false); +assert.equal(Object.hasOwn(rows[0],'failure'),false); +''') + + def test_failed_recording_preserves_the_prior_rpc_failure(self): + self.run_node(''' +const rpc=new Error('request timeout');const disk=new Error('receipt close failed'); +const call=createRetainedQueryCaller({callTool:async()=>{throw rpc;}},async()=>{throw disk;}); +await assert.rejects(call('inspect_analysis_view',{},'selected'),e=> + e instanceof AggregateError && e.errors[0]===rpc && e.errors[1]===disk); +''') + + def test_log_stream_failure_keeps_costs_both_receipts_and_workspace(self): + self.run_finalization(''' +const log=new Writable({write(chunk,encoding,done){done(new Error('log disk failed'));}}); +const logFinished=finished(log);logFinished.catch(()=>{});log.write('diagnostic'); +assert.equal(await finalizeRetainedQueryReceipt({...input,serverLog:log, + serverLogFinished:logFinished}),false); +assert.match(report.cleanup_failures[0],/server-log:.*log disk failed/); +assert.strictEqual(report.failure,originalFailure); +assert.equal(report.calls[0].elapsed_ms,17);assert.equal(report.retained_workspace,account); +assert.equal((await stat(account)).isDirectory(),true); +assert.deepEqual(await saved(),report);assert.deepEqual(JSON.parse(chunks.join('')),report); +''') + + def test_workspace_removal_error_still_writes_the_original_failure(self): + self.run_finalization(''' +const blocked=join(root,'owned-file');await writeFile(blocked,'owned'); +const malformedAccount=join(blocked,'child'); +assert.equal(await finalizeRetainedQueryReceipt({...input,account:malformedAccount}),false); +assert.match(report.cleanup_failures[0],/^workspace:/); +assert.equal(report.retained_workspace,malformedAccount); +assert.deepEqual((await saved()).failure,originalFailure); +assert.deepEqual(JSON.parse(chunks.join('')),report); +''') + + def test_report_file_refusal_preserves_the_stdout_receipt(self): + self.run_finalization(''' +await mkdir(join(output,'report.json')); +assert.equal(await finalizeRetainedQueryReceipt(input),false); +assert.deepEqual(report.cleanup_failures,[]); +assert.match(report.receipt_failures[0],/^report-file:/); +assert.deepEqual(JSON.parse(chunks.join('')),report); +assert.deepEqual(report.failure,originalFailure); +''') + + def test_stdout_write_error_is_handled_and_saved_in_the_file(self): + self.run_finalization(''' +const failedStdout=new Writable({write(chunk,encoding,done){done(new Error('broken pipe'));}}); +assert.equal(await finalizeRetainedQueryReceipt({...input,stdout:failedStdout}),false); +assert.match(report.receipt_failures[0],/stdout:.*broken pipe/); +assert.deepEqual(await saved(),report);assert.deepEqual(report.failure,originalFailure); +''') + + def test_partial_retry_after_stdout_failure_preserves_the_prior_receipt(self): + self.run_finalization(''' +const fs=await import('node:fs/promises'); +const {syncBuiltinESMExports}=await import('node:module'); +const originalWrite=fs.default.writeFile;let writes=0;let before; +fs.default.writeFile=async(...args)=>{ + if(++writes===2){await originalWrite(args[0],'partial',args[2]);throw new Error('disk quota');} + return originalWrite(...args); +}; +syncBuiltinESMExports(); +try { + const failedStdout=new Writable({async write(chunk,encoding,done){ + before=await readFile(join(output,'report.json'),'utf8');done(new Error('broken pipe')); + }}); + assert.equal(await finalizeRetainedQueryReceipt({...input,stdout:failedStdout}),false); + assert.equal(writes,2);assert.equal(report.receipt_failures.length,2); + assert.match(report.receipt_failures[1],/report-file-retry:.*disk quota/); + assert.equal(await readFile(join(output,'report.json'),'utf8'),before); + assert.deepEqual((await saved()).failure,originalFailure); + assert.equal((await saved()).calls[0].elapsed_ms,17); +} finally {fs.default.writeFile=originalWrite;syncBuiltinESMExports();} +''') + + def test_failed_client_close_keeps_ownership_and_attempts_peer_cleanup(self): + self.run_finalization(''' +const closed=[]; +assert.equal(await finalizeRetainedQueryReceipt({...input, + client:{close(){closed.push('client');throw new Error('close failed');}}, + transport:{close:async()=>{closed.push('transport');}}}),false); +assert.deepEqual(closed,['client','transport']); +assert.match(report.cleanup_failures[0],/client:.*close failed/); +assert.equal(report.retained_workspace,account);assert.equal(serverLog.writableEnded,true); +assert.equal((await stat(account)).isDirectory(),true); +assert.deepEqual(await saved(),report);assert.deepEqual(JSON.parse(chunks.join('')),report); +''')