mirror of
https://github.com/morluto/rea.git
synced 2026-10-26 17:35:11 +00:00
test(perf): verify authentic retained investigation queries (#4814)
Verify retained queries against authentic SQLite ELF and npm histories, retain failed RPC/cleanup/output costs, and make mixed-history verification repeatable. Record the separate validation-reuse bottleneck in #4810. Co-authored-by: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com> Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com>
This commit is contained in:
9 files changed
+1610
No files matched your search
@@ -1,5 +1,12 @@
|
|||||||
# Analysis efficiency measurements
|
# Analysis efficiency measurements
|
||||||
|
|
||||||
|
For authentic old Evidence and mixed investigations, see the
|
||||||
|
[retained-query checkpoint](retained-investigation-workflows.md).
|
||||||
|
`retained-investigation.mjs` compares existing complete delivery with selected
|
||||||
|
pages, including capacity-refusal recovery and original-record round trips.
|
||||||
|
It records negative model results and separates imported-format validation work
|
||||||
|
from the query contract.
|
||||||
|
|
||||||
For genuine Fiddler captures, see the [SAZ selection checkpoint](saz-selection-workflows.md).
|
For genuine Fiddler captures, see the [SAZ selection checkpoint](saz-selection-workflows.md).
|
||||||
`measure.mjs --saz-selection baseline|focused|retained` compares default complete
|
`measure.mjs --saz-selection baseline|focused|retained` compares default complete
|
||||||
delivery (including retained-reference recovery), caller-selected ordinals, and
|
delivery (including retained-reference recovery), caller-selected ordinals, and
|
||||||
|
|||||||
@@ -187,6 +187,7 @@ def main():
|
|||||||
'inspect_managed_members', 'inspect_managed_native_boundaries',
|
'inspect_managed_members', 'inspect_managed_native_boundaries',
|
||||||
'inspect_artifact', 'inspect_wasm_artifact',
|
'inspect_artifact', 'inspect_wasm_artifact',
|
||||||
'inspect_evm_interface', 'get_evidence_bundle', 'export_evidence_bundle',
|
'inspect_evm_interface', 'get_evidence_bundle', 'export_evidence_bundle',
|
||||||
|
'import_evidence_bundle', 'list_unknowns', 'verify_unknown_resolution',
|
||||||
'capture_browser_scenario',
|
'capture_browser_scenario',
|
||||||
'resolve_containing_procedure', 'analyze_function', 'close_binary']:
|
'resolve_containing_procedure', 'analyze_function', 'close_binary']:
|
||||||
config += '[mcp_servers.rea.tools.' + tool + ']\napproval_mode = "approve"\n'
|
config += '[mcp_servers.rea.tools.' + tool + ']\napproval_mode = "approve"\n'
|
||||||
|
|||||||
@@ -0,0 +1,635 @@
|
|||||||
|
{
|
||||||
|
"runtime_revision": "f4322b15ed1423d736bd2572cf7a8eadf2b466e1",
|
||||||
|
"host": "Linux x86-64",
|
||||||
|
"node": "24.18.0",
|
||||||
|
"npm": "11.16.0",
|
||||||
|
"codex_cli": "0.161.0",
|
||||||
|
"requested_model": "gpt-6.1-sol",
|
||||||
|
"reasoning": "high",
|
||||||
|
"resolved_model_identity": "unknown",
|
||||||
|
"code_mode": false,
|
||||||
|
"client_enabled_tools": [
|
||||||
|
"import_evidence_bundle",
|
||||||
|
"get_evidence_bundle",
|
||||||
|
"inspect_analysis_view",
|
||||||
|
"list_unknowns"
|
||||||
|
],
|
||||||
|
"catalog": {
|
||||||
|
"tools": 207,
|
||||||
|
"serialized_bytes": 3152125,
|
||||||
|
"sha256": "4f31d9d56c400bc48f5639a9dc4b4cdad7faf862e4be81641d94fa70552438e4"
|
||||||
|
},
|
||||||
|
"bounds": {
|
||||||
|
"cpu_quota_percent": 100,
|
||||||
|
"family_hard_mib": 8192,
|
||||||
|
"early_stop_mib": 7168,
|
||||||
|
"swap": false,
|
||||||
|
"tasks": 192
|
||||||
|
},
|
||||||
|
"counter_scope": "Payload-exit counters with observer alive; kernel family peaks include file cache; terminal whole-scope counters and remote model compute unknown",
|
||||||
|
"sources": {
|
||||||
|
"native": {
|
||||||
|
"target_sha256": "2e6eef9a727f081f0d453b4e5e6cbd8b9ef8b6f86cbf7681cbad444d3b0b55c8",
|
||||||
|
"sections": 26,
|
||||||
|
"symbols": 1479,
|
||||||
|
"relocations": 2963,
|
||||||
|
"all_symbol_locations_names_values_agree_with_independent_ELF_parser": true
|
||||||
|
},
|
||||||
|
"javascript": {
|
||||||
|
"package": "npm",
|
||||||
|
"version": "11.16.0",
|
||||||
|
"source_files": [
|
||||||
|
{
|
||||||
|
"path": "package.json",
|
||||||
|
"sha256": "60ff022e32b5d8291d3e09849fd0cee6bbbff4e4d8352ad258214b1adf070714"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"path": "bin/npm-cli.js",
|
||||||
|
"sha256": "8e5f6f3429f8cdbe693cdc29904e9d5a7b127a494bd15c804bd54c7403bfcbe7"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"path": "lib/cli.js",
|
||||||
|
"sha256": "67666f06479f9b0bbc01412c198caadd4287d34d5ed74a02871ea78f186451e7"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"path": "lib/cli/entry.js",
|
||||||
|
"sha256": "667a44c1fa8795fdbba8ad843736ca7f8f24e64511985e9a7060b002123560c6"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"path": "lib/commands/install.js",
|
||||||
|
"sha256": "f7a60e8b40ab1bece76a2441b8b15378d199f05c6050eebe455cd9961f67b816"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"lib_cli_module": {
|
||||||
|
"node_id": "jag_node_131257ad52f3333971c2a5ed325c64357e52604489e42ccc67a8849770cb5b10",
|
||||||
|
"kind": "javascript-module",
|
||||||
|
"path": "lib/cli.js"
|
||||||
|
},
|
||||||
|
"source_bytes_checked": true,
|
||||||
|
"inventory_completeness": "producer-relative, not independently established",
|
||||||
|
"unselected_sources": 1416
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"sdk": [
|
||||||
|
{
|
||||||
|
"name": "sdk-native-complete-1",
|
||||||
|
"status": "passed",
|
||||||
|
"answer_sha256": "4e28c2c2a25620576d27abe846a05835ec49386b889779e33410c31971523b54",
|
||||||
|
"query_ms": 1328.7,
|
||||||
|
"query_reply_bytes": 11224227,
|
||||||
|
"rpc_count": 7,
|
||||||
|
"elapsed_seconds": 14.117,
|
||||||
|
"cpu_seconds": 13.915,
|
||||||
|
"peak_family_mib": 1160.8,
|
||||||
|
"memory_events": {
|
||||||
|
"low": 0,
|
||||||
|
"high": 0,
|
||||||
|
"max": 0,
|
||||||
|
"oom": 0,
|
||||||
|
"oom_kill": 0,
|
||||||
|
"oom_group_kill": 0
|
||||||
|
},
|
||||||
|
"payload_cleanup_verified": true,
|
||||||
|
"response_capacity_refused": false,
|
||||||
|
"warm_page_ms": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "sdk-native-complete-2",
|
||||||
|
"status": "passed",
|
||||||
|
"answer_sha256": "4e28c2c2a25620576d27abe846a05835ec49386b889779e33410c31971523b54",
|
||||||
|
"query_ms": 1385.4,
|
||||||
|
"query_reply_bytes": 11224227,
|
||||||
|
"rpc_count": 7,
|
||||||
|
"elapsed_seconds": 14.722,
|
||||||
|
"cpu_seconds": 14.485,
|
||||||
|
"peak_family_mib": 1109.9,
|
||||||
|
"memory_events": {
|
||||||
|
"low": 0,
|
||||||
|
"high": 0,
|
||||||
|
"max": 0,
|
||||||
|
"oom": 0,
|
||||||
|
"oom_kill": 0,
|
||||||
|
"oom_group_kill": 0
|
||||||
|
},
|
||||||
|
"payload_cleanup_verified": true,
|
||||||
|
"response_capacity_refused": false,
|
||||||
|
"warm_page_ms": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "sdk-native-focused-1",
|
||||||
|
"status": "passed",
|
||||||
|
"answer_sha256": "4e28c2c2a25620576d27abe846a05835ec49386b889779e33410c31971523b54",
|
||||||
|
"query_ms": 529.7,
|
||||||
|
"query_reply_bytes": 167985,
|
||||||
|
"rpc_count": 25,
|
||||||
|
"elapsed_seconds": 12.907,
|
||||||
|
"cpu_seconds": 12.679,
|
||||||
|
"peak_family_mib": 934.8,
|
||||||
|
"memory_events": {
|
||||||
|
"low": 0,
|
||||||
|
"high": 0,
|
||||||
|
"max": 0,
|
||||||
|
"oom": 0,
|
||||||
|
"oom_kill": 0,
|
||||||
|
"oom_group_kill": 0
|
||||||
|
},
|
||||||
|
"payload_cleanup_verified": true,
|
||||||
|
"response_capacity_refused": false,
|
||||||
|
"warm_page_ms": [44.1, 59.6, 24.0, 30.8, 26.7, 47.4]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "sdk-native-focused-2",
|
||||||
|
"status": "passed",
|
||||||
|
"answer_sha256": "4e28c2c2a25620576d27abe846a05835ec49386b889779e33410c31971523b54",
|
||||||
|
"query_ms": 572.2,
|
||||||
|
"query_reply_bytes": 167985,
|
||||||
|
"rpc_count": 25,
|
||||||
|
"elapsed_seconds": 13.316,
|
||||||
|
"cpu_seconds": 13.116,
|
||||||
|
"peak_family_mib": 1000.0,
|
||||||
|
"memory_events": {
|
||||||
|
"low": 0,
|
||||||
|
"high": 0,
|
||||||
|
"max": 0,
|
||||||
|
"oom": 0,
|
||||||
|
"oom_kill": 0,
|
||||||
|
"oom_group_kill": 0
|
||||||
|
},
|
||||||
|
"payload_cleanup_verified": true,
|
||||||
|
"response_capacity_refused": false,
|
||||||
|
"warm_page_ms": [44.0, 64.6, 26.8, 33.2, 29.8, 50.4]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "sdk-javascript-complete-1",
|
||||||
|
"status": "passed",
|
||||||
|
"answer_sha256": "0fe3ad249eb1a7233a4966a56652c173667c48fdaf395b6c08fc6901ef3b46f7",
|
||||||
|
"query_ms": 11241.5,
|
||||||
|
"query_reply_bytes": 212856,
|
||||||
|
"rpc_count": 31,
|
||||||
|
"elapsed_seconds": 26.3,
|
||||||
|
"cpu_seconds": 26.1,
|
||||||
|
"peak_family_mib": 1333.5,
|
||||||
|
"memory_events": {
|
||||||
|
"low": 0,
|
||||||
|
"high": 0,
|
||||||
|
"max": 0,
|
||||||
|
"oom": 0,
|
||||||
|
"oom_kill": 0,
|
||||||
|
"oom_group_kill": 0
|
||||||
|
},
|
||||||
|
"payload_cleanup_verified": true,
|
||||||
|
"response_capacity_refused": true,
|
||||||
|
"warm_page_ms": [1124.6, 1073.8, 1474.7, 1074.7, 1081.6, 1109.5]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "sdk-javascript-complete-2",
|
||||||
|
"status": "passed",
|
||||||
|
"answer_sha256": "0fe3ad249eb1a7233a4966a56652c173667c48fdaf395b6c08fc6901ef3b46f7",
|
||||||
|
"query_ms": 11938.1,
|
||||||
|
"query_reply_bytes": 212856,
|
||||||
|
"rpc_count": 31,
|
||||||
|
"elapsed_seconds": 27.105,
|
||||||
|
"cpu_seconds": 26.87,
|
||||||
|
"peak_family_mib": 1320.9,
|
||||||
|
"memory_events": {
|
||||||
|
"low": 0,
|
||||||
|
"high": 0,
|
||||||
|
"max": 0,
|
||||||
|
"oom": 0,
|
||||||
|
"oom_kill": 0,
|
||||||
|
"oom_group_kill": 0
|
||||||
|
},
|
||||||
|
"payload_cleanup_verified": true,
|
||||||
|
"response_capacity_refused": true,
|
||||||
|
"warm_page_ms": [1155.4, 1096.2, 1489.3, 1121.3, 1088.3, 1484.6]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "sdk-javascript-focused-2",
|
||||||
|
"status": "passed",
|
||||||
|
"answer_sha256": "0fe3ad249eb1a7233a4966a56652c173667c48fdaf395b6c08fc6901ef3b46f7",
|
||||||
|
"query_ms": 10967.3,
|
||||||
|
"query_reply_bytes": 208350,
|
||||||
|
"rpc_count": 28,
|
||||||
|
"elapsed_seconds": 25.708,
|
||||||
|
"cpu_seconds": 25.505,
|
||||||
|
"peak_family_mib": 1344.0,
|
||||||
|
"memory_events": {
|
||||||
|
"low": 0,
|
||||||
|
"high": 0,
|
||||||
|
"max": 0,
|
||||||
|
"oom": 0,
|
||||||
|
"oom_kill": 0,
|
||||||
|
"oom_group_kill": 0
|
||||||
|
},
|
||||||
|
"payload_cleanup_verified": true,
|
||||||
|
"response_capacity_refused": false,
|
||||||
|
"warm_page_ms": [1342.7, 1085.1, 1055.0, 1034.6, 1082.7, 1440.7]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "sdk-javascript-focused-3",
|
||||||
|
"status": "passed",
|
||||||
|
"answer_sha256": "0fe3ad249eb1a7233a4966a56652c173667c48fdaf395b6c08fc6901ef3b46f7",
|
||||||
|
"query_ms": 10990.4,
|
||||||
|
"query_reply_bytes": 208350,
|
||||||
|
"rpc_count": 28,
|
||||||
|
"elapsed_seconds": 26.197,
|
||||||
|
"cpu_seconds": 25.98,
|
||||||
|
"peak_family_mib": 1336.0,
|
||||||
|
"memory_events": {
|
||||||
|
"low": 0,
|
||||||
|
"high": 0,
|
||||||
|
"max": 0,
|
||||||
|
"oom": 0,
|
||||||
|
"oom_kill": 0,
|
||||||
|
"oom_group_kill": 0
|
||||||
|
},
|
||||||
|
"payload_cleanup_verified": true,
|
||||||
|
"response_capacity_refused": false,
|
||||||
|
"warm_page_ms": [1338.1, 1088.5, 1069.1, 1038.3, 1087.5, 1483.0]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "sdk-mixed-native",
|
||||||
|
"status": "passed",
|
||||||
|
"answer_sha256": "4e28c2c2a25620576d27abe846a05835ec49386b889779e33410c31971523b54",
|
||||||
|
"query_ms": 560.0,
|
||||||
|
"query_reply_bytes": 167991,
|
||||||
|
"rpc_count": 25,
|
||||||
|
"elapsed_seconds": 16.933,
|
||||||
|
"cpu_seconds": 16.714,
|
||||||
|
"peak_family_mib": 1261.4,
|
||||||
|
"memory_events": {
|
||||||
|
"low": 0,
|
||||||
|
"high": 0,
|
||||||
|
"max": 0,
|
||||||
|
"oom": 0,
|
||||||
|
"oom_kill": 0,
|
||||||
|
"oom_group_kill": 0
|
||||||
|
},
|
||||||
|
"payload_cleanup_verified": true,
|
||||||
|
"response_capacity_refused": false,
|
||||||
|
"warm_page_ms": [46.9, 51.0, 36.3, 24.4, 38.6, 43.8]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "sdk-mixed-javascript",
|
||||||
|
"status": "passed",
|
||||||
|
"answer_sha256": "0fe3ad249eb1a7233a4966a56652c173667c48fdaf395b6c08fc6901ef3b46f7",
|
||||||
|
"query_ms": 10904.5,
|
||||||
|
"query_reply_bytes": 208356,
|
||||||
|
"rpc_count": 28,
|
||||||
|
"elapsed_seconds": 26.5,
|
||||||
|
"cpu_seconds": 26.297,
|
||||||
|
"peak_family_mib": 1346.5,
|
||||||
|
"memory_events": {
|
||||||
|
"low": 0,
|
||||||
|
"high": 0,
|
||||||
|
"max": 0,
|
||||||
|
"oom": 0,
|
||||||
|
"oom_kill": 0,
|
||||||
|
"oom_group_kill": 0
|
||||||
|
},
|
||||||
|
"payload_cleanup_verified": true,
|
||||||
|
"response_capacity_refused": false,
|
||||||
|
"warm_page_ms": [1321.9, 1063.3, 1070.8, 1309.9, 1041.7, 1074.2]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"model_trials": [
|
||||||
|
{
|
||||||
|
"name": "model-javascript-complete-1",
|
||||||
|
"facts_pass": true,
|
||||||
|
"elapsed_seconds": 76.49,
|
||||||
|
"cpu_seconds": 24.687,
|
||||||
|
"peak_family_mib": 818.7,
|
||||||
|
"input_tokens": 454491,
|
||||||
|
"cached_input_tokens": 396288,
|
||||||
|
"uncached_input_tokens": 58203,
|
||||||
|
"output_tokens": 2033,
|
||||||
|
"reported_reasoning_output_tokens": 207,
|
||||||
|
"mcp_calls_including_discovery": 9,
|
||||||
|
"payload_cleanup_verified": true,
|
||||||
|
"memory_events": {
|
||||||
|
"low": 0,
|
||||||
|
"high": 0,
|
||||||
|
"max": 0,
|
||||||
|
"oom": 0,
|
||||||
|
"oom_kill": 0,
|
||||||
|
"oom_group_kill": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "model-javascript-complete-2",
|
||||||
|
"facts_pass": true,
|
||||||
|
"elapsed_seconds": 58.335,
|
||||||
|
"cpu_seconds": 14.993,
|
||||||
|
"peak_family_mib": 971.4,
|
||||||
|
"input_tokens": 328066,
|
||||||
|
"cached_input_tokens": 280704,
|
||||||
|
"uncached_input_tokens": 47362,
|
||||||
|
"output_tokens": 1676,
|
||||||
|
"reported_reasoning_output_tokens": 129,
|
||||||
|
"mcp_calls_including_discovery": 9,
|
||||||
|
"payload_cleanup_verified": true,
|
||||||
|
"memory_events": {
|
||||||
|
"low": 0,
|
||||||
|
"high": 0,
|
||||||
|
"max": 0,
|
||||||
|
"oom": 0,
|
||||||
|
"oom_kill": 0,
|
||||||
|
"oom_group_kill": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "model-javascript-focused-1",
|
||||||
|
"facts_pass": false,
|
||||||
|
"elapsed_seconds": 34.206,
|
||||||
|
"cpu_seconds": 16.31,
|
||||||
|
"peak_family_mib": 811.9,
|
||||||
|
"input_tokens": 167470,
|
||||||
|
"cached_input_tokens": 136320,
|
||||||
|
"uncached_input_tokens": 31150,
|
||||||
|
"output_tokens": 633,
|
||||||
|
"reported_reasoning_output_tokens": 172,
|
||||||
|
"mcp_calls_including_discovery": 4,
|
||||||
|
"payload_cleanup_verified": true,
|
||||||
|
"memory_events": {
|
||||||
|
"low": 0,
|
||||||
|
"high": 0,
|
||||||
|
"max": 0,
|
||||||
|
"oom": 0,
|
||||||
|
"oom_kill": 0,
|
||||||
|
"oom_group_kill": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "model-javascript-focused-2",
|
||||||
|
"facts_pass": true,
|
||||||
|
"elapsed_seconds": 66.455,
|
||||||
|
"cpu_seconds": 14.337,
|
||||||
|
"peak_family_mib": 955.3,
|
||||||
|
"input_tokens": 339528,
|
||||||
|
"cached_input_tokens": 307456,
|
||||||
|
"uncached_input_tokens": 32072,
|
||||||
|
"output_tokens": 2051,
|
||||||
|
"reported_reasoning_output_tokens": 169,
|
||||||
|
"mcp_calls_including_discovery": 10,
|
||||||
|
"payload_cleanup_verified": true,
|
||||||
|
"memory_events": {
|
||||||
|
"low": 0,
|
||||||
|
"high": 0,
|
||||||
|
"max": 0,
|
||||||
|
"oom": 0,
|
||||||
|
"oom_kill": 0,
|
||||||
|
"oom_group_kill": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "model-javascript-guided-complete-1",
|
||||||
|
"facts_pass": true,
|
||||||
|
"elapsed_seconds": 69.206,
|
||||||
|
"cpu_seconds": 13.65,
|
||||||
|
"peak_family_mib": 888.8,
|
||||||
|
"input_tokens": 168862,
|
||||||
|
"cached_input_tokens": 121728,
|
||||||
|
"uncached_input_tokens": 47134,
|
||||||
|
"output_tokens": 2385,
|
||||||
|
"reported_reasoning_output_tokens": 737,
|
||||||
|
"mcp_calls_including_discovery": 6,
|
||||||
|
"payload_cleanup_verified": true,
|
||||||
|
"memory_events": {
|
||||||
|
"low": 0,
|
||||||
|
"high": 0,
|
||||||
|
"max": 0,
|
||||||
|
"oom": 0,
|
||||||
|
"oom_kill": 0,
|
||||||
|
"oom_group_kill": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "model-javascript-guided-complete-2",
|
||||||
|
"facts_pass": false,
|
||||||
|
"elapsed_seconds": 29.187,
|
||||||
|
"cpu_seconds": 15.105,
|
||||||
|
"peak_family_mib": 697.4,
|
||||||
|
"input_tokens": 28696,
|
||||||
|
"cached_input_tokens": 23296,
|
||||||
|
"uncached_input_tokens": 5400,
|
||||||
|
"output_tokens": 610,
|
||||||
|
"reported_reasoning_output_tokens": 379,
|
||||||
|
"mcp_calls_including_discovery": 0,
|
||||||
|
"payload_cleanup_verified": true,
|
||||||
|
"memory_events": {
|
||||||
|
"low": 0,
|
||||||
|
"high": 0,
|
||||||
|
"max": 0,
|
||||||
|
"oom": 0,
|
||||||
|
"oom_kill": 0,
|
||||||
|
"oom_group_kill": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "model-javascript-guided-focused-1",
|
||||||
|
"facts_pass": true,
|
||||||
|
"elapsed_seconds": 53.464,
|
||||||
|
"cpu_seconds": 12.059,
|
||||||
|
"peak_family_mib": 944.5,
|
||||||
|
"input_tokens": 124218,
|
||||||
|
"cached_input_tokens": 92416,
|
||||||
|
"uncached_input_tokens": 31802,
|
||||||
|
"output_tokens": 1746,
|
||||||
|
"reported_reasoning_output_tokens": 330,
|
||||||
|
"mcp_calls_including_discovery": 5,
|
||||||
|
"payload_cleanup_verified": true,
|
||||||
|
"memory_events": {
|
||||||
|
"low": 0,
|
||||||
|
"high": 0,
|
||||||
|
"max": 0,
|
||||||
|
"oom": 0,
|
||||||
|
"oom_kill": 0,
|
||||||
|
"oom_group_kill": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "model-javascript-guided-focused-2",
|
||||||
|
"facts_pass": true,
|
||||||
|
"elapsed_seconds": 58.526,
|
||||||
|
"cpu_seconds": 22.852,
|
||||||
|
"peak_family_mib": 924.2,
|
||||||
|
"input_tokens": 129134,
|
||||||
|
"cached_input_tokens": 97536,
|
||||||
|
"uncached_input_tokens": 31598,
|
||||||
|
"output_tokens": 1478,
|
||||||
|
"reported_reasoning_output_tokens": 69,
|
||||||
|
"mcp_calls_including_discovery": 5,
|
||||||
|
"payload_cleanup_verified": true,
|
||||||
|
"memory_events": {
|
||||||
|
"low": 0,
|
||||||
|
"high": 0,
|
||||||
|
"max": 0,
|
||||||
|
"oom": 0,
|
||||||
|
"oom_kill": 0,
|
||||||
|
"oom_group_kill": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "model-native-complete-1",
|
||||||
|
"facts_pass": true,
|
||||||
|
"elapsed_seconds": 52.103,
|
||||||
|
"cpu_seconds": 8.686,
|
||||||
|
"peak_family_mib": 1028.5,
|
||||||
|
"input_tokens": 287913,
|
||||||
|
"cached_input_tokens": 249984,
|
||||||
|
"uncached_input_tokens": 37929,
|
||||||
|
"output_tokens": 1724,
|
||||||
|
"reported_reasoning_output_tokens": 121,
|
||||||
|
"mcp_calls_including_discovery": 4,
|
||||||
|
"payload_cleanup_verified": true,
|
||||||
|
"memory_events": {
|
||||||
|
"low": 0,
|
||||||
|
"high": 0,
|
||||||
|
"max": 0,
|
||||||
|
"oom": 0,
|
||||||
|
"oom_kill": 0,
|
||||||
|
"oom_group_kill": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "model-native-complete-2",
|
||||||
|
"facts_pass": true,
|
||||||
|
"elapsed_seconds": 58.261,
|
||||||
|
"cpu_seconds": 9.102,
|
||||||
|
"peak_family_mib": 837.4,
|
||||||
|
"input_tokens": 348976,
|
||||||
|
"cached_input_tokens": 286976,
|
||||||
|
"uncached_input_tokens": 62000,
|
||||||
|
"output_tokens": 2009,
|
||||||
|
"reported_reasoning_output_tokens": 246,
|
||||||
|
"mcp_calls_including_discovery": 5,
|
||||||
|
"payload_cleanup_verified": true,
|
||||||
|
"memory_events": {
|
||||||
|
"low": 0,
|
||||||
|
"high": 0,
|
||||||
|
"max": 0,
|
||||||
|
"oom": 0,
|
||||||
|
"oom_kill": 0,
|
||||||
|
"oom_group_kill": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "model-native-focused-1",
|
||||||
|
"facts_pass": true,
|
||||||
|
"elapsed_seconds": 62.078,
|
||||||
|
"cpu_seconds": 10.198,
|
||||||
|
"peak_family_mib": 823.9,
|
||||||
|
"input_tokens": 335445,
|
||||||
|
"cached_input_tokens": 293120,
|
||||||
|
"uncached_input_tokens": 42325,
|
||||||
|
"output_tokens": 1924,
|
||||||
|
"reported_reasoning_output_tokens": 87,
|
||||||
|
"mcp_calls_including_discovery": 8,
|
||||||
|
"payload_cleanup_verified": true,
|
||||||
|
"memory_events": {
|
||||||
|
"low": 0,
|
||||||
|
"high": 0,
|
||||||
|
"max": 0,
|
||||||
|
"oom": 0,
|
||||||
|
"oom_kill": 0,
|
||||||
|
"oom_group_kill": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "model-native-focused-2",
|
||||||
|
"facts_pass": true,
|
||||||
|
"elapsed_seconds": 57.933,
|
||||||
|
"cpu_seconds": 8.058,
|
||||||
|
"peak_family_mib": 821.3,
|
||||||
|
"input_tokens": 286938,
|
||||||
|
"cached_input_tokens": 256384,
|
||||||
|
"uncached_input_tokens": 30554,
|
||||||
|
"output_tokens": 1831,
|
||||||
|
"reported_reasoning_output_tokens": 106,
|
||||||
|
"mcp_calls_including_discovery": 8,
|
||||||
|
"payload_cleanup_verified": true,
|
||||||
|
"memory_events": {
|
||||||
|
"low": 0,
|
||||||
|
"high": 0,
|
||||||
|
"max": 0,
|
||||||
|
"oom": 0,
|
||||||
|
"oom_kill": 0,
|
||||||
|
"oom_group_kill": 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"failed_trials": {
|
||||||
|
"model-javascript-focused-1": "Wrong connector discovery; approval-policy refusal; no source facts established.",
|
||||||
|
"model-javascript-guided-complete-2": "Premature agent refusal with no REA RPC; asserted tool unavailability not established by a connection/discovery observation."
|
||||||
|
},
|
||||||
|
"pilot_failures": [
|
||||||
|
"Private pilot oracle read view.item instead of view.items; corrected before the committed SDK cohorts. No product failure inferred."
|
||||||
|
],
|
||||||
|
"decision": "Existing query APIs suffice for these bounded questions; no general token/time advantage established. Separate imported immutable-format validation optimization is tracked in #4810.",
|
||||||
|
"review_followup_verification": {
|
||||||
|
"review-mixed-native": {
|
||||||
|
"status": "passed",
|
||||||
|
"checks": {
|
||||||
|
"heads_checked": 64,
|
||||||
|
"pages": 32,
|
||||||
|
"revised_unknown_id": "unk_057c8f2dbf30ec98851a66ee054366bc6e2e168874581e6f99cb80673d0ff9b8",
|
||||||
|
"revision_checked": 2,
|
||||||
|
"stale_update_rejected": true,
|
||||||
|
"tampered_import_rejected_atomically": true,
|
||||||
|
"unknowns_after_close": 0,
|
||||||
|
"old_reference_refused": true
|
||||||
|
},
|
||||||
|
"original_records_preserved": true,
|
||||||
|
"elapsed_seconds": 18.444,
|
||||||
|
"cpu_seconds": 18.16515,
|
||||||
|
"peak_family_mib": 1401.6328125,
|
||||||
|
"oom_events_at_payload_exit": 0,
|
||||||
|
"payload_cleanup_verified": true
|
||||||
|
},
|
||||||
|
"review-mixed-javascript": {
|
||||||
|
"status": "passed",
|
||||||
|
"checks": {
|
||||||
|
"heads_checked": 64,
|
||||||
|
"pages": 32,
|
||||||
|
"revised_unknown_id": "unk_057c8f2dbf30ec98851a66ee054366bc6e2e168874581e6f99cb80673d0ff9b8",
|
||||||
|
"revision_checked": 2,
|
||||||
|
"stale_update_rejected": true,
|
||||||
|
"tampered_import_rejected_atomically": true,
|
||||||
|
"unknowns_after_close": 0,
|
||||||
|
"old_reference_refused": true
|
||||||
|
},
|
||||||
|
"original_records_preserved": true,
|
||||||
|
"elapsed_seconds": 27.617,
|
||||||
|
"cpu_seconds": 27.275299,
|
||||||
|
"peak_family_mib": 1444.30859375,
|
||||||
|
"oom_events_at_payload_exit": 0,
|
||||||
|
"payload_cleanup_verified": true
|
||||||
|
},
|
||||||
|
"deterministic_receipt_tests": {
|
||||||
|
"passed": 4,
|
||||||
|
"scope": "Rejected SDK call, protocol error, receipt-write failure, simultaneous call/write failure; not real-provider evidence"
|
||||||
|
},
|
||||||
|
"cleanup_receipt_regressions": {
|
||||||
|
"passed": 6,
|
||||||
|
"scope": "Real stream/file refusals, independent peer close, and injected partial publication at the filesystem boundary. Original failure and call costs survive; failed cleanup retains workspace ownership. Harness failure checks, not real-provider evidence.",
|
||||||
|
"baseline_reproduction": {
|
||||||
|
"revision": "b025a919b36ec091ca888339df6b24e78e89b79b",
|
||||||
|
"method": "Execute the prior committed finalization block with deterministic workspace or log rejection; not a production-provider reproduction.",
|
||||||
|
"workspace_error": {
|
||||||
|
"file_receipt_attempts": 0,
|
||||||
|
"stdout_receipt_attempts": 0
|
||||||
|
},
|
||||||
|
"log_error": {
|
||||||
|
"file_receipt_attempts": 0,
|
||||||
|
"stdout_receipt_attempts": 0
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"partial_retry_baseline": {
|
||||||
|
"revision": "5d6259e658ee54fc9829b9143a8721a2ab426e80",
|
||||||
|
"regression_exit_code": 1,
|
||||||
|
"observed_saved_report": "partial",
|
||||||
|
"method": "Run the unchanged partial-write regression against the prior committed finalizer with deterministic filesystem-boundary failure injection."
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,201 @@
|
|||||||
|
# Retained investigation query checkpoint
|
||||||
|
|
||||||
|
[#3134](https://github.com/morluto/rea/issues/3134) measured existing retained
|
||||||
|
queries on Linux. Selected reads preserved the requested facts, original
|
||||||
|
Evidence and Unknown revisions, including a mixed investigation. Existing
|
||||||
|
contracts suffice for these questions; this checkpoint adds no production API,
|
||||||
|
cache, persistence or default change.
|
||||||
|
|
||||||
|
Imported JavaScript pages repeatedly validated their entire source graph. That
|
||||||
|
separate, observed bottleneck is owned by
|
||||||
|
[#4810](https://github.com/morluto/rea/issues/4810). Smaller replies did not
|
||||||
|
establish a general model-token or whole-task latency improvement.
|
||||||
|
|
||||||
|
The [curated measurements](retained-investigation-results.json) retain individual
|
||||||
|
SDK/model costs and unsuccessful attempts. Raw bundles, transcripts, profiles
|
||||||
|
and detailed receipts stay private and are not build inputs.
|
||||||
|
|
||||||
|
## Sources and questions
|
||||||
|
|
||||||
|
Runtime: `f4322b15ed1423d736bd2572cf7a8eadf2b466e1`, Linux x86-64,
|
||||||
|
Node 24.18.0/npm 11.16.0. The SDK discovered all 207 advertised tools
|
||||||
|
(3,152,125 serialized catalog bytes). Model trials used Codex CLI 0.161.0,
|
||||||
|
requested `gpt-6.1-sol`/high, direct MCP, and the same four enabled tool names:
|
||||||
|
`import_evidence_bundle`, `get_evidence_bundle`, `inspect_analysis_view`, and
|
||||||
|
`list_unknowns`. The resolved service-side model identity is unknown. These are
|
||||||
|
selected-client-tool trials, not an evaluation of full-catalog agent discovery.
|
||||||
|
|
||||||
|
Two public source workflows produced authentic bundles through production MCP:
|
||||||
|
|
||||||
|
- Native: `inspect_binary_layout` on Debian's `libsqlite3.so.0.8.6`, SHA-256
|
||||||
|
`2e6eef9a727f081f0d453b4e5e6cbd8b9ef8b6f86cbf7681cbad444d3b0b55c8`.
|
||||||
|
Pwntools 4.15.0/pyelftools 0.33/Unicorn 2.1.2 observed 26 sections, 1,479
|
||||||
|
symbols and 2,963 relocations. An independent ELF64 decoder checked section
|
||||||
|
and relocation counts and every symbol's table/entry index, value, raw name
|
||||||
|
bytes and source entry range against the original file.
|
||||||
|
- JavaScript: shipped npm 11.16.0, directory inventory digest
|
||||||
|
`f8e576a7d162542c616168b25a95a5f62b9accf29467cc37e149905ad4b8e603`.
|
||||||
|
`analyze_javascript_application` selected exactly `package.json`,
|
||||||
|
`bin/npm-cli.js`, `lib/cli.js`, `lib/cli/entry.js`, and
|
||||||
|
`lib/commands/install.js`. It parsed four JavaScript files; 1,416 other
|
||||||
|
relevant files remained unselected. The complete retained result contained
|
||||||
|
2,822 application nodes, 1,796 semantic nodes and 13,682 semantic unknowns.
|
||||||
|
Source-file digests and the `lib/cli.js` source-module identity were checked
|
||||||
|
against shipped bytes. Inventory/discovery completeness remains producer-relative.
|
||||||
|
|
||||||
|
Analyst-recorded Unknowns came from the actual source limitations: five native
|
||||||
|
and seven JavaScript questions. These registry heads are distinct from the
|
||||||
|
13,682 semantic-graph unknowns. The exported histories contained 15 and 17
|
||||||
|
records, respectively, and approximately 1.75 and 21.06 MB of canonical JSON.
|
||||||
|
No synthetic payload arrays were used.
|
||||||
|
|
||||||
|
The SDK task read the first six original symbol/module rows and the relevant
|
||||||
|
current Unknown heads. Complete delivery and selected delivery answered the
|
||||||
|
same question; selected delivery intentionally exercised two-row pages. Each
|
||||||
|
fresh server performed three traversals, exported again and closed. The model
|
||||||
|
task additionally requested native identity/counts, or the five JavaScript source
|
||||||
|
choices and exact `lib/cli.js` source module, with explicit runtime uncertainty.
|
||||||
|
Compare arms within each declared task, not SDK costs against model costs.
|
||||||
|
|
||||||
|
## Correctness, capacity and lifetime
|
||||||
|
|
||||||
|
Both fresh-server SDK repetitions passed in both delivery arms. Selected rows,
|
||||||
|
ordering and counts agreed with original records; all original records and
|
||||||
|
Unknown revisions survived export/import, including links, omitted fields,
|
||||||
|
nulls and scalar types. Query calls did not rerun either producer. The resumed
|
||||||
|
servers had no native producer configuration.
|
||||||
|
|
||||||
|
The JavaScript complete bundle exceeded the default 10 MiB receive-buffer
|
||||||
|
boundary. Its typed capacity refusal preserved the complete retained data and a
|
||||||
|
usable connection. Complete-arm costs include that refusal and subsequent
|
||||||
|
selected-view recovery; they are not timings for a successful complete transfer.
|
||||||
|
|
||||||
|
A mixed history combined both authentic investigations and promoted 64 actual
|
||||||
|
semantic unknowns to separately identified analyst follow-ups. One follow-up
|
||||||
|
was updated to investigating. It contained 97 records, 76 heads and 77 revisions.
|
||||||
|
Both source queries preserved all original records/revisions. Thirty-two
|
||||||
|
two-head pages verified every semantic-follow-up head, including revision two.
|
||||||
|
A stale update and a tampered bundle import were rejected without changing the
|
||||||
|
ledger. Export reproduced the complete original mixed bundle. Closing cleared
|
||||||
|
records/Unknowns and made the old reference unavailable; no implicit persistence
|
||||||
|
or producer restart was introduced.
|
||||||
|
|
||||||
|
Public CLI `inspect-evidence-bundle` summary and inline `inspect-analysis-view`
|
||||||
|
summary also passed on both authentic sources. Portable CLI input still requires
|
||||||
|
complete input admission/validation; small output does not remove that cost.
|
||||||
|
|
||||||
|
## SDK costs
|
||||||
|
|
||||||
|
Each row spans two fresh MCP servers. Query time is the sum of SDK RPC elapsed
|
||||||
|
times over three traversals. Local post-call contract validation, oracle checks
|
||||||
|
and transcript writes are included only in the whole harness, along with startup,
|
||||||
|
full discovery, file reads, import, export and close. OS caches were uncontrolled/warm. A fresh imported
|
||||||
|
session is not a cold native-analysis measurement.
|
||||||
|
|
||||||
|
| Source / arm | Three traversals | Query reply bytes | Whole elapsed | Local family CPU | Family peak |
|
||||||
|
| ------------------------------ | ---------------: | ----------------: | --------------: | ---------------: | --------------: |
|
||||||
|
| Native complete | 1.329–1.385 s | 11,224,227 | 14.117–14.722 s | 13.915–14.485 s | 1,110–1,161 MiB |
|
||||||
|
| Native selected | 0.530–0.572 s | 167,985 | 12.907–13.316 s | 12.679–13.116 s | 935–1,000 MiB |
|
||||||
|
| JavaScript complete + recovery | 11.242–11.938 s | 212,856 | 26.300–27.105 s | 26.100–26.870 s | 1,321–1,334 MiB |
|
||||||
|
| JavaScript selected | 10.967–10.990 s | 208,350 | 25.708–26.197 s | 25.505–25.980 s | 1,336–1,344 MiB |
|
||||||
|
|
||||||
|
Mixed-history selected tasks took 16.933 s native and 26.500 s JavaScript,
|
||||||
|
including the larger import and complete round-trip checks. Their family peaks
|
||||||
|
were 1,261 and 1,347 MiB. This does not establish scaling to thousands of
|
||||||
|
registry heads or every provider.
|
||||||
|
|
||||||
|
Original-session npm warm pages measured 8–31 ms; after import, repeated pages
|
||||||
|
cost about 1.1–1.4 s. Source inspection shows that imported immutable Evidence
|
||||||
|
does not carry the format owner's private graph proofs, so each view repeats
|
||||||
|
full validation. Native layout views also parse the complete layout on each
|
||||||
|
page. #4810 owns equivalent before/after verification of a correction; this
|
||||||
|
checkpoint claims no speedup from an unimplemented cache.
|
||||||
|
|
||||||
|
## Actual model tasks
|
||||||
|
|
||||||
|
All twelve attempts retained their costs. Ten supplied every checked fact and
|
||||||
|
current Unknown identity/status; runtime and unselected-source coverage stayed
|
||||||
|
unknown. One initial JavaScript selected attempt searched the wrong connector
|
||||||
|
and received an approval-policy refusal. One guided complete attempt stopped
|
||||||
|
without an REA RPC and asserted tool unavailability without a supporting
|
||||||
|
connection/discovery observation. Both are failed tasks, despite CLI exit zero.
|
||||||
|
|
||||||
|
| Cohort / arm | Successful tasks | Total input tokens | Elapsed |
|
||||||
|
| ------------------------------------- | ---------------: | -----------------------: | -----------------------: |
|
||||||
|
| Native complete | 2/2 | 287,913 / 348,976 | 52.103 / 58.261 s |
|
||||||
|
| Native selected, two-head pages | 2/2 | 335,445 / 286,938 | 62.078 / 57.933 s |
|
||||||
|
| JavaScript complete + recovery | 2/2 | 454,491 / 328,066 | 76.490 / 58.335 s |
|
||||||
|
| JavaScript selected, two-head pages | 1/2 | 167,470 failed / 339,528 | 34.206 failed / 66.455 s |
|
||||||
|
| JavaScript guided complete + recovery | 1/2 | 168,862 / 28,696 failed | 69.206 / 29.187 failed s |
|
||||||
|
| JavaScript guided selected | 2/2 | 124,218 / 129,134 | 53.464 / 58.526 s |
|
||||||
|
|
||||||
|
The separate guided JavaScript cohort named the exact configured tools and
|
||||||
|
allowed one sufficiently large Unknown page for this bounded question. Both
|
||||||
|
arms received the same discovery guidance. Its single completed complete-arm
|
||||||
|
sample is insufficient for a stable repeated-pair benefit claim. Native total
|
||||||
|
input/time overlapped; extra pages and model turns can consume the reply saving.
|
||||||
|
Do not pool failed tasks as fast answers or attribute guidance changes to REA.
|
||||||
|
|
||||||
|
The JSON report records total/cached/uncached input, output, local CPU and peak
|
||||||
|
memory per attempt. These are client-reported usage counters, not tokenizer
|
||||||
|
estimates; output includes the client's reported reasoning accounting without
|
||||||
|
adding it again. Schema/discovery, errors, retries and continuations remain in
|
||||||
|
the task. Remote model compute, full-catalog model behavior and other hosts were
|
||||||
|
not measured.
|
||||||
|
|
||||||
|
Heavy runs were serial, one CPU, 8 GiB family hard limit, 7 GiB early stop,
|
||||||
|
no swap. Kernel peaks include charged file cache. Counter observations were
|
||||||
|
taken at payload exit while the owned observer remained alive; terminal
|
||||||
|
whole-scope counters remain unknown. Observed payload OOM/capacity events were
|
||||||
|
zero, all payloads exited and cleanup was verified. The expected MCP response
|
||||||
|
refusal is a separate transport-capacity outcome.
|
||||||
|
|
||||||
|
## Reproduce
|
||||||
|
|
||||||
|
Build the pinned runtime, acquire the matching real artifacts and produce the
|
||||||
|
two authentic histories with the public operations above. Export before close.
|
||||||
|
Keep bundles, transcripts and model profiles private. For a known source ID:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
node experiments/analysis-efficiency/retained-investigation.mjs \
|
||||||
|
--runtime-root /absolute/built-rea \
|
||||||
|
--revision f4322b15ed1423d736bd2572cf7a8eadf2b466e1 \
|
||||||
|
--bundle /absolute/authentic-history.json \
|
||||||
|
--evidence-id ev_ORIGINAL_SHA256 \
|
||||||
|
--scenario native --delivery focused \
|
||||||
|
--output /absolute/new-receipt-directory
|
||||||
|
```
|
||||||
|
|
||||||
|
Run under the existing process-family resource wrapper, with sufficient bounds
|
||||||
|
for the selected source. Repeat with `--delivery complete`; use
|
||||||
|
`--scenario javascript` for module rows. The task's Unknown domain is
|
||||||
|
`retained-native` or `retained-javascript`. Output is never overwritten.
|
||||||
|
`agent-trial.py --mode direct` supports the same four enabled names; its success
|
||||||
|
exit only establishes client completion. Grade original IDs/digests, exact
|
||||||
|
facts, coverage and Unknowns from the retained transcript and final answer.
|
||||||
|
|
||||||
|
For a privately supplied authentic mixed bundle, add `--verify-mixed`. Besides
|
||||||
|
the normal source query and original-record round trip, this pages every current
|
||||||
|
head in `javascript-semantic-followup`, checks the current updated revision,
|
||||||
|
submits a deliberately stale update and a tampered copy of the bundle, verifies
|
||||||
|
atomic refusal, then checks Unknown/reference lifetime after close. The bundle
|
||||||
|
must contain genuine follow-ups and at least one updated head. The checkpoint's
|
||||||
|
64-head bundle passed this mode for both source scenarios. Its extra verification
|
||||||
|
costs are separate from the original SDK comparison rows above.
|
||||||
|
|
||||||
|
Rejected SDK calls are recorded before propagating the original failure, with
|
||||||
|
request, phase and elapsed time. Reply bytes stay null when no usable reply was
|
||||||
|
received. Receipt-write failures remain distinct; simultaneous call/write
|
||||||
|
failures retain both causes. Deterministic failure tests run with
|
||||||
|
`python3 -m unittest discover -s experiments/analysis-efficiency -p test_retained_queries.py`.
|
||||||
|
|
||||||
|
Finalization independently attempts client/transport close, log completion and
|
||||||
|
both receipt outputs. Cleanup failures preserve the original task failure and
|
||||||
|
call costs, record the failing phase and retain the workspace for retry. A file
|
||||||
|
receipt refusal still permits stdout; a broken stdout write is recorded in a
|
||||||
|
file retry. Files publish through a sibling temporary file and rename; an
|
||||||
|
interrupted retry preserves the previous receipt. Failed publication files stay
|
||||||
|
in the owned output directory. Cleanup or receipt-output failures make the
|
||||||
|
command fail. Six additional regressions exercise stream/file errors, partial
|
||||||
|
publication and peer cleanup; these are harness failure checks, not real-provider
|
||||||
|
evidence.
|
||||||
@@ -0,0 +1,310 @@
|
|||||||
|
/** Measure public retained queries against an authentic, previously exported bundle. */
|
||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { createHash } from "node:crypto";
|
||||||
|
import { createWriteStream } from "node:fs";
|
||||||
|
import {
|
||||||
|
appendFile,
|
||||||
|
mkdir,
|
||||||
|
mkdtemp,
|
||||||
|
readFile,
|
||||||
|
writeFile,
|
||||||
|
} from "node:fs/promises";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { dirname, isAbsolute, join, resolve } from "node:path";
|
||||||
|
import { fileURLToPath } from "node:url";
|
||||||
|
import { parseArgs } from "node:util";
|
||||||
|
import { finished } from "node:stream/promises";
|
||||||
|
import canonicalize from "canonicalize";
|
||||||
|
import { Client } from "@modelcontextprotocol/client";
|
||||||
|
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
||||||
|
import {
|
||||||
|
requireMcpOperationResult,
|
||||||
|
requireMcpToolError,
|
||||||
|
} from "../../scripts/lib/mcp-verifier-results.mjs";
|
||||||
|
import { createRetainedQueryCaller } from "./retained-query-calls.mjs";
|
||||||
|
import { finalizeRetainedQueryReceipt } from "./retained-query-finalization.mjs";
|
||||||
|
import {
|
||||||
|
verifyMixedRetainedHistory,
|
||||||
|
verifyMixedRetainedClosure,
|
||||||
|
retainedUnknownHeads,
|
||||||
|
} from "./retained-mixed-verification.mjs";
|
||||||
|
|
||||||
|
const { values } = parseArgs({
|
||||||
|
options: {
|
||||||
|
"runtime-root": {
|
||||||
|
type: "string",
|
||||||
|
default: resolve(dirname(fileURLToPath(import.meta.url)), "../.."),
|
||||||
|
},
|
||||||
|
revision: { type: "string" },
|
||||||
|
bundle: { type: "string" },
|
||||||
|
"evidence-id": { type: "string" },
|
||||||
|
scenario: { type: "string" },
|
||||||
|
delivery: { type: "string" },
|
||||||
|
output: { type: "string" },
|
||||||
|
"verify-mixed": { type: "boolean", default: false },
|
||||||
|
},
|
||||||
|
});
|
||||||
|
for (const key of ["runtime-root", "bundle", "output"])
|
||||||
|
assert.ok(values[key] && isAbsolute(values[key]), `${key} must be absolute`);
|
||||||
|
assert.ok(values.revision && values["evidence-id"]);
|
||||||
|
assert.ok(["native", "javascript"].includes(values.scenario));
|
||||||
|
assert.ok(["complete", "focused"].includes(values.delivery));
|
||||||
|
const root = values["runtime-root"];
|
||||||
|
const output = values.output;
|
||||||
|
await mkdir(output, { recursive: false, mode: 0o700 });
|
||||||
|
const digest = (bytes) => createHash("sha256").update(bytes).digest("hex");
|
||||||
|
const sourceBytes = await readFile(values.bundle);
|
||||||
|
const original = JSON.parse(sourceBytes);
|
||||||
|
const source = original.records.find(
|
||||||
|
(record) => record.evidence_id === values["evidence-id"],
|
||||||
|
);
|
||||||
|
const operation =
|
||||||
|
values.scenario === "native"
|
||||||
|
? "inspect_binary_layout"
|
||||||
|
: "analyze_javascript_application";
|
||||||
|
assert.equal(source?.operation, operation);
|
||||||
|
const unknownDomain = `retained-${values.scenario}`;
|
||||||
|
const expectedUnknowns = retainedUnknownHeads(original.unknowns, unknownDomain);
|
||||||
|
// These oracles check selected original facts, not producer discovery completeness.
|
||||||
|
const expectedRows =
|
||||||
|
values.scenario === "native"
|
||||||
|
? source.normalized_result.symbols
|
||||||
|
: source.normalized_result.graph.nodes
|
||||||
|
.filter((node) =>
|
||||||
|
["javascript-asset", "javascript-module", "source-module"].includes(
|
||||||
|
node.kind,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.map((node) => {
|
||||||
|
const candidates = node.observations.flatMap((observation) => {
|
||||||
|
const property = observation.properties.path;
|
||||||
|
const where = observation.evidence.location;
|
||||||
|
return [
|
||||||
|
...(typeof property === "string" ? [property] : []),
|
||||||
|
...(where.available && where.value.kind === "artifact-path"
|
||||||
|
? [where.value.path]
|
||||||
|
: where.available && where.value.kind === "source-range"
|
||||||
|
? [where.value.source]
|
||||||
|
: []),
|
||||||
|
];
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
node_id: node.node_id,
|
||||||
|
kind: node.kind,
|
||||||
|
path:
|
||||||
|
node.identity.strategy === "canonical-path"
|
||||||
|
? node.identity.path
|
||||||
|
: node.identity.strategy === "source-map-original"
|
||||||
|
? node.identity.original_source
|
||||||
|
: (candidates[0] ?? null),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
assert.ok(expectedRows.length >= 6, "This task needs six original rows");
|
||||||
|
const runtimeEntrySha256 = digest(
|
||||||
|
await readFile(join(root, "dist/mcp/main.js")),
|
||||||
|
);
|
||||||
|
const account = await mkdtemp(join(tmpdir(), "rea-retained-query-"));
|
||||||
|
const transport = new StdioClientTransport({
|
||||||
|
command: process.execPath,
|
||||||
|
args: [join(root, "scripts/rea.mjs"), "mcp"],
|
||||||
|
env: {
|
||||||
|
PATH: process.env.PATH,
|
||||||
|
HOME: account,
|
||||||
|
NODE_OPTIONS: "--max-old-space-size=2048",
|
||||||
|
REA_LOG_LEVEL: "silent",
|
||||||
|
},
|
||||||
|
stderr: "pipe",
|
||||||
|
});
|
||||||
|
const serverLog = createWriteStream(join(output, "server.log"));
|
||||||
|
const serverLogFinished = finished(serverLog);
|
||||||
|
serverLogFinished.catch(() => {});
|
||||||
|
transport.stderr?.pipe(serverLog);
|
||||||
|
const client = new Client({
|
||||||
|
name: "retained-investigation-measurement",
|
||||||
|
version: "1",
|
||||||
|
});
|
||||||
|
const report = {
|
||||||
|
revision: values.revision,
|
||||||
|
runtime_entry_sha256: runtimeEntrySha256,
|
||||||
|
bundle_sha256: digest(sourceBytes),
|
||||||
|
bundle_bytes: sourceBytes.length,
|
||||||
|
source_evidence_id: source.evidence_id,
|
||||||
|
scenario: values.scenario,
|
||||||
|
delivery: values.delivery,
|
||||||
|
verify_mixed: values["verify-mixed"],
|
||||||
|
calls: [],
|
||||||
|
status: "running",
|
||||||
|
};
|
||||||
|
const call = createRetainedQueryCaller(client, async (receipt, event) => {
|
||||||
|
report.calls.push(receipt);
|
||||||
|
await appendFile(
|
||||||
|
join(output, "transcript.jsonl"),
|
||||||
|
JSON.stringify(event) + "\n",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
const result = (reply, name) => requireMcpOperationResult(reply, name);
|
||||||
|
try {
|
||||||
|
await client.connect(transport);
|
||||||
|
const started = performance.now();
|
||||||
|
const catalog = await client.listTools();
|
||||||
|
report.catalog = {
|
||||||
|
elapsed_ms: performance.now() - started,
|
||||||
|
tools: catalog.tools.length,
|
||||||
|
bytes: Buffer.byteLength(JSON.stringify(catalog)),
|
||||||
|
sha256: digest(JSON.stringify(catalog)),
|
||||||
|
};
|
||||||
|
assert.equal(catalog.nextCursor, undefined);
|
||||||
|
await writeFile(join(output, "catalog.json"), JSON.stringify(catalog));
|
||||||
|
result(
|
||||||
|
await call("import_evidence_bundle", { path: values.bundle }, "import"),
|
||||||
|
"import_evidence_bundle",
|
||||||
|
);
|
||||||
|
const answers = [];
|
||||||
|
for (let repeat = 0; repeat < 3; repeat += 1) {
|
||||||
|
let answer;
|
||||||
|
if (values.delivery === "complete") {
|
||||||
|
const reply = await call("get_evidence_bundle", {}, `complete-${repeat}`);
|
||||||
|
if (reply.isError !== true) {
|
||||||
|
const bundle = result(reply, "get_evidence_bundle");
|
||||||
|
assert.deepEqual(
|
||||||
|
bundle.records.find(
|
||||||
|
(record) => record.evidence_id === source.evidence_id,
|
||||||
|
),
|
||||||
|
source,
|
||||||
|
);
|
||||||
|
assert.deepEqual(bundle.unknowns, original.unknowns);
|
||||||
|
answer = [expectedRows.slice(0, 6), expectedUnknowns];
|
||||||
|
} else {
|
||||||
|
// Keep capacity refusals and every recovery RPC in the completed task.
|
||||||
|
const failure = requireMcpToolError(reply);
|
||||||
|
assert.equal(failure.code, "resource_constraint");
|
||||||
|
assert.equal(failure.details?.resource, "transport");
|
||||||
|
report.complete_delivery_refused = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (answer === undefined) {
|
||||||
|
const metadata = result(
|
||||||
|
await call(
|
||||||
|
"get_evidence_bundle",
|
||||||
|
{
|
||||||
|
detail: "summary",
|
||||||
|
filters: { evidence_id: source.evidence_id },
|
||||||
|
},
|
||||||
|
`discovery-${repeat}`,
|
||||||
|
),
|
||||||
|
"get_evidence_bundle",
|
||||||
|
);
|
||||||
|
assert.equal(metadata.matching_records, 1);
|
||||||
|
assert.equal(metadata.records[0].evidence_id, source.evidence_id);
|
||||||
|
const rows = [];
|
||||||
|
for (const offset of [0, 2, 4]) {
|
||||||
|
const view = result(
|
||||||
|
await call(
|
||||||
|
"inspect_analysis_view",
|
||||||
|
{
|
||||||
|
source: {
|
||||||
|
kind: "retained-evidence",
|
||||||
|
evidence_id: source.evidence_id,
|
||||||
|
},
|
||||||
|
view: {
|
||||||
|
kind: "page",
|
||||||
|
collection:
|
||||||
|
values.scenario === "native" ? "symbols" : "modules",
|
||||||
|
offset,
|
||||||
|
limit: 2,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
`page-${repeat}-${offset}`,
|
||||||
|
),
|
||||||
|
"inspect_analysis_view",
|
||||||
|
);
|
||||||
|
assert.equal(view.parent_evidence_id, source.evidence_id);
|
||||||
|
assert.equal(view.coverage.total, expectedRows.length);
|
||||||
|
assert.equal(view.coverage.examined, 2);
|
||||||
|
assert.equal(
|
||||||
|
view.coverage.next_offset,
|
||||||
|
offset + 2 < expectedRows.length ? offset + 2 : null,
|
||||||
|
);
|
||||||
|
assert.deepEqual(view.items, expectedRows.slice(offset, offset + 2));
|
||||||
|
rows.push(...view.items);
|
||||||
|
}
|
||||||
|
const unknowns = [];
|
||||||
|
let offset = 0;
|
||||||
|
do {
|
||||||
|
const page = result(
|
||||||
|
await call(
|
||||||
|
"list_unknowns",
|
||||||
|
{ domain: unknownDomain, offset, limit: 2 },
|
||||||
|
`unknowns-${repeat}-${offset}`,
|
||||||
|
),
|
||||||
|
"list_unknowns",
|
||||||
|
);
|
||||||
|
assert.equal(page.total, expectedUnknowns.length);
|
||||||
|
assert.equal(page.offset, offset);
|
||||||
|
assert.deepEqual(
|
||||||
|
page.items,
|
||||||
|
expectedUnknowns.slice(offset, offset + 2),
|
||||||
|
);
|
||||||
|
assert.equal(
|
||||||
|
page.next_offset,
|
||||||
|
offset + 2 < expectedUnknowns.length ? offset + 2 : null,
|
||||||
|
);
|
||||||
|
unknowns.push(...page.items);
|
||||||
|
offset = page.next_offset;
|
||||||
|
} while (offset !== null);
|
||||||
|
assert.deepEqual(unknowns, expectedUnknowns);
|
||||||
|
answer = [rows, unknowns];
|
||||||
|
}
|
||||||
|
answers.push(digest(canonicalize(answer)));
|
||||||
|
}
|
||||||
|
assert.equal(new Set(answers).size, 1);
|
||||||
|
report.answer_sha256 = answers[0];
|
||||||
|
if (values["verify-mixed"])
|
||||||
|
report.mixed_verification = await verifyMixedRetainedHistory({
|
||||||
|
call,
|
||||||
|
original,
|
||||||
|
sourceId: source.evidence_id,
|
||||||
|
output,
|
||||||
|
});
|
||||||
|
const roundtrip = join(output, "roundtrip.json");
|
||||||
|
result(
|
||||||
|
await call("export_evidence_bundle", { path: roundtrip }, "export"),
|
||||||
|
"export_evidence_bundle",
|
||||||
|
);
|
||||||
|
const exported = JSON.parse(await readFile(roundtrip, "utf8"));
|
||||||
|
const exportedRecords = new Map(
|
||||||
|
exported.records.map((record) => [record.evidence_id, record]),
|
||||||
|
);
|
||||||
|
for (const record of original.records)
|
||||||
|
assert.deepEqual(exportedRecords.get(record.evidence_id), record);
|
||||||
|
assert.deepEqual(exported.unknowns, original.unknowns);
|
||||||
|
report.original_records_preserved = true;
|
||||||
|
result(await call("close_binary", {}, "close"), "close_binary");
|
||||||
|
const closed = result(
|
||||||
|
await call("get_evidence_bundle", { detail: "summary" }, "after-close"),
|
||||||
|
"get_evidence_bundle",
|
||||||
|
);
|
||||||
|
assert.equal(closed.total_retained_records, 0);
|
||||||
|
report.close_cleared_records = true;
|
||||||
|
if (values["verify-mixed"])
|
||||||
|
Object.assign(
|
||||||
|
report.mixed_verification,
|
||||||
|
await verifyMixedRetainedClosure(call, source.evidence_id),
|
||||||
|
);
|
||||||
|
assert.equal(digest(await readFile(values.bundle)), report.bundle_sha256);
|
||||||
|
report.status = "passed";
|
||||||
|
} catch (cause) {
|
||||||
|
report.status = "failed";
|
||||||
|
report.failure = { name: cause.name, message: cause.message };
|
||||||
|
} finally {
|
||||||
|
const finalized = await finalizeRetainedQueryReceipt({
|
||||||
|
report,
|
||||||
|
client,
|
||||||
|
transport,
|
||||||
|
account,
|
||||||
|
serverLog,
|
||||||
|
serverLogFinished,
|
||||||
|
output,
|
||||||
|
});
|
||||||
|
if (report.status !== "passed" || !finalized) process.exitCode = 1;
|
||||||
|
}
|
||||||
@@ -0,0 +1,163 @@
|
|||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { writeFile } from "node:fs/promises";
|
||||||
|
import { join } from "node:path";
|
||||||
|
import {
|
||||||
|
requireMcpOperationResult as result,
|
||||||
|
requireMcpToolError,
|
||||||
|
} from "../../scripts/lib/mcp-verifier-results.mjs";
|
||||||
|
|
||||||
|
/** Expected current heads from the original exported revisions, not displayed pages. */
|
||||||
|
export const retainedUnknownHeads = (unknowns, domain) => {
|
||||||
|
const heads = new Map();
|
||||||
|
for (const unknown of unknowns) {
|
||||||
|
const previous = heads.get(unknown.unknown_id);
|
||||||
|
if (previous === undefined || previous.revision < unknown.revision)
|
||||||
|
heads.set(unknown.unknown_id, unknown);
|
||||||
|
}
|
||||||
|
return [...heads.values()]
|
||||||
|
.filter((unknown) => unknown.domain === domain)
|
||||||
|
.sort((a, b) =>
|
||||||
|
a.unknown_id < b.unknown_id ? -1 : a.unknown_id > b.unknown_id ? 1 : 0,
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
const verifyFollowupPages = async (call, original) => {
|
||||||
|
const expected = retainedUnknownHeads(
|
||||||
|
original.unknowns,
|
||||||
|
"javascript-semantic-followup",
|
||||||
|
);
|
||||||
|
assert.ok(
|
||||||
|
expected.length > 0,
|
||||||
|
"Supply an authentic mixed history with semantic follow-ups",
|
||||||
|
);
|
||||||
|
const revised = expected.find((unknown) => unknown.revision > 1);
|
||||||
|
assert.ok(revised, "Mixed verification requires a retained updated Unknown");
|
||||||
|
let offset = 0;
|
||||||
|
let pages = 0;
|
||||||
|
const observed = [];
|
||||||
|
do {
|
||||||
|
const page = result(
|
||||||
|
await call(
|
||||||
|
"list_unknowns",
|
||||||
|
{
|
||||||
|
domain: "javascript-semantic-followup",
|
||||||
|
offset,
|
||||||
|
limit: 2,
|
||||||
|
},
|
||||||
|
`mixed-heads-${offset}`,
|
||||||
|
),
|
||||||
|
"list_unknowns",
|
||||||
|
);
|
||||||
|
assert.equal(page.total, expected.length);
|
||||||
|
assert.equal(page.offset, offset);
|
||||||
|
assert.deepEqual(page.items, expected.slice(offset, offset + 2));
|
||||||
|
assert.equal(
|
||||||
|
page.next_offset,
|
||||||
|
offset + 2 < expected.length ? offset + 2 : null,
|
||||||
|
);
|
||||||
|
observed.push(...page.items);
|
||||||
|
offset = page.next_offset;
|
||||||
|
pages += 1;
|
||||||
|
} while (offset !== null);
|
||||||
|
assert.deepEqual(observed, expected);
|
||||||
|
return { revised, headsChecked: expected.length, pages };
|
||||||
|
};
|
||||||
|
|
||||||
|
const verifyMutationRefusals = async ({
|
||||||
|
call,
|
||||||
|
original,
|
||||||
|
sourceId,
|
||||||
|
output,
|
||||||
|
revised,
|
||||||
|
}) => {
|
||||||
|
const before = result(
|
||||||
|
await call(
|
||||||
|
"get_evidence_bundle",
|
||||||
|
{ detail: "summary" },
|
||||||
|
"mixed-before-refusals",
|
||||||
|
),
|
||||||
|
"get_evidence_bundle",
|
||||||
|
);
|
||||||
|
const stale = await call(
|
||||||
|
"update_unknown",
|
||||||
|
{
|
||||||
|
unknown_id: revised.unknown_id,
|
||||||
|
expected_revision: revised.revision - 1,
|
||||||
|
status: revised.status,
|
||||||
|
severity: revised.severity,
|
||||||
|
supporting_evidence_ids: revised.supporting_evidence_ids,
|
||||||
|
contradicting_evidence_ids: revised.contradicting_evidence_ids,
|
||||||
|
required_authority: revised.required_authority,
|
||||||
|
required_confidence: revised.required_confidence,
|
||||||
|
required_environment: revised.required_environment,
|
||||||
|
recommended_probes: revised.recommended_probes,
|
||||||
|
relationships: revised.relationships,
|
||||||
|
resolution: revised.resolution,
|
||||||
|
},
|
||||||
|
"mixed-stale-update",
|
||||||
|
);
|
||||||
|
assert.equal(requireMcpToolError(stale).code, "revision_conflict");
|
||||||
|
const tampered = structuredClone(original);
|
||||||
|
const target = tampered.records.find(
|
||||||
|
(record) => record.evidence_id === sourceId,
|
||||||
|
);
|
||||||
|
assert.ok(target);
|
||||||
|
target.normalized_result = { checkpoint_tampered: true };
|
||||||
|
const path = join(output, "tampered-import.json");
|
||||||
|
await writeFile(path, JSON.stringify(tampered), { flag: "wx" });
|
||||||
|
const rejected = await call(
|
||||||
|
"import_evidence_bundle",
|
||||||
|
{ path },
|
||||||
|
"mixed-tampered-import",
|
||||||
|
);
|
||||||
|
assert.equal(
|
||||||
|
requireMcpToolError(rejected).code,
|
||||||
|
"evidence_integrity_mismatch",
|
||||||
|
);
|
||||||
|
const after = result(
|
||||||
|
await call(
|
||||||
|
"get_evidence_bundle",
|
||||||
|
{ detail: "summary" },
|
||||||
|
"mixed-after-refusals",
|
||||||
|
),
|
||||||
|
"get_evidence_bundle",
|
||||||
|
);
|
||||||
|
assert.deepEqual(after, before);
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Verify original mixed-history heads, CAS rejection and atomic import failure. */
|
||||||
|
export const verifyMixedRetainedHistory = async (options) => {
|
||||||
|
const { revised, headsChecked, pages } = await verifyFollowupPages(
|
||||||
|
options.call,
|
||||||
|
options.original,
|
||||||
|
);
|
||||||
|
await verifyMutationRefusals({ ...options, revised });
|
||||||
|
return {
|
||||||
|
heads_checked: headsChecked,
|
||||||
|
pages,
|
||||||
|
revised_unknown_id: revised.unknown_id,
|
||||||
|
revision_checked: revised.revision,
|
||||||
|
stale_update_rejected: true,
|
||||||
|
tampered_import_rejected_atomically: true,
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Closing must neither retain old Unknowns nor rehydrate a closed source. */
|
||||||
|
export const verifyMixedRetainedClosure = async (call, sourceId) => {
|
||||||
|
const unknowns = result(
|
||||||
|
await call("list_unknowns", {}, "mixed-after-close-heads"),
|
||||||
|
"list_unknowns",
|
||||||
|
);
|
||||||
|
assert.equal(unknowns.total, 0);
|
||||||
|
const missing = await call(
|
||||||
|
"inspect_analysis_view",
|
||||||
|
{
|
||||||
|
source: { kind: "retained-evidence", evidence_id: sourceId },
|
||||||
|
view: { kind: "summary" },
|
||||||
|
},
|
||||||
|
"mixed-after-close-reference",
|
||||||
|
);
|
||||||
|
const failure = requireMcpToolError(missing);
|
||||||
|
assert.equal(failure.details?.reason, "missing");
|
||||||
|
return { unknowns_after_close: 0, old_reference_refused: true };
|
||||||
|
};
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
/** Keep settled and rejected RPCs in the same receipt before returning control. */
|
||||||
|
export const createRetainedQueryCaller =
|
||||||
|
(client, record) => async (name, input, phase) => {
|
||||||
|
const start = performance.now();
|
||||||
|
let reply;
|
||||||
|
try {
|
||||||
|
reply = await client.callTool(
|
||||||
|
{ name, arguments: input },
|
||||||
|
{ timeout: 300000 },
|
||||||
|
);
|
||||||
|
} catch (cause) {
|
||||||
|
const failure =
|
||||||
|
cause instanceof Error
|
||||||
|
? { name: cause.name, message: cause.message }
|
||||||
|
: { name: typeof cause, message: String(cause) };
|
||||||
|
const receipt = {
|
||||||
|
name,
|
||||||
|
input,
|
||||||
|
phase,
|
||||||
|
elapsed_ms: performance.now() - start,
|
||||||
|
reply_bytes: null,
|
||||||
|
is_error: true,
|
||||||
|
failure,
|
||||||
|
};
|
||||||
|
try {
|
||||||
|
await record(receipt, { name, input, phase, failure });
|
||||||
|
} catch (recordingFailure) {
|
||||||
|
throw new AggregateError(
|
||||||
|
[cause, recordingFailure],
|
||||||
|
"MCP call and receipt recording both failed",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
throw cause;
|
||||||
|
}
|
||||||
|
const receipt = {
|
||||||
|
name,
|
||||||
|
input,
|
||||||
|
phase,
|
||||||
|
elapsed_ms: performance.now() - start,
|
||||||
|
reply_bytes: Buffer.byteLength(JSON.stringify(reply)),
|
||||||
|
is_error: reply.isError === true,
|
||||||
|
};
|
||||||
|
// A receipt-write failure is distinct from a rejected MCP call.
|
||||||
|
await record(receipt, { name, input, phase, reply });
|
||||||
|
return reply;
|
||||||
|
};
|
||||||
@@ -0,0 +1,81 @@
|
|||||||
|
import { rename, rm, writeFile } from "node:fs/promises";
|
||||||
|
import { join } from "node:path";
|
||||||
|
|
||||||
|
const attempt = async (failures, phase, action) => {
|
||||||
|
try {
|
||||||
|
await action();
|
||||||
|
return true;
|
||||||
|
} catch (cause) {
|
||||||
|
failures.push(`${phase}: ${String(cause)}`);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const writeReceipt = (stream, text) =>
|
||||||
|
new Promise((resolve, reject) => {
|
||||||
|
const onError = (cause) => reject(cause);
|
||||||
|
stream.once("error", onError);
|
||||||
|
try {
|
||||||
|
stream.write(text, (cause) => {
|
||||||
|
if (cause !== null && cause !== undefined) {
|
||||||
|
// A Writable emits its error after notifying the write callback.
|
||||||
|
reject(cause);
|
||||||
|
} else {
|
||||||
|
stream.off("error", onError);
|
||||||
|
resolve();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
} catch (cause) {
|
||||||
|
stream.off("error", onError);
|
||||||
|
reject(cause);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
/** Preserve task costs through independent cleanup and receipt-output failures. */
|
||||||
|
export const finalizeRetainedQueryReceipt = async ({
|
||||||
|
report,
|
||||||
|
client,
|
||||||
|
transport,
|
||||||
|
account,
|
||||||
|
serverLog,
|
||||||
|
serverLogFinished,
|
||||||
|
output,
|
||||||
|
stdout = process.stdout,
|
||||||
|
}) => {
|
||||||
|
report.cleanup_failures = [];
|
||||||
|
await Promise.all([
|
||||||
|
attempt(report.cleanup_failures, "client", () => client.close()),
|
||||||
|
attempt(report.cleanup_failures, "transport", () => transport.close()),
|
||||||
|
]);
|
||||||
|
await attempt(report.cleanup_failures, "server-log", async () => {
|
||||||
|
if (!serverLog.writableEnded) serverLog.end();
|
||||||
|
await serverLogFinished;
|
||||||
|
});
|
||||||
|
if (report.cleanup_failures.length === 0)
|
||||||
|
await attempt(report.cleanup_failures, "workspace", () =>
|
||||||
|
rm(account, { recursive: true }),
|
||||||
|
);
|
||||||
|
if (report.cleanup_failures.length > 0) report.retained_workspace = account;
|
||||||
|
|
||||||
|
report.receipt_failures = [];
|
||||||
|
let publication = 0;
|
||||||
|
const saveReport = async () => {
|
||||||
|
// Keep failed publications in the owned output directory for inspection.
|
||||||
|
// A partial retry must never truncate an already published cost receipt.
|
||||||
|
const pending = join(output, `.report-${++publication}.json.pending`);
|
||||||
|
await writeFile(pending, JSON.stringify(report, null, 2) + "\n", {
|
||||||
|
flag: "wx",
|
||||||
|
});
|
||||||
|
await rename(pending, join(output, "report.json"));
|
||||||
|
};
|
||||||
|
await attempt(report.receipt_failures, "report-file", saveReport);
|
||||||
|
const stdoutWritten = await attempt(report.receipt_failures, "stdout", () =>
|
||||||
|
writeReceipt(stdout, JSON.stringify(report) + "\n"),
|
||||||
|
);
|
||||||
|
// Preserve a broken stdout diagnostic in the file that may already exist.
|
||||||
|
if (!stdoutWritten)
|
||||||
|
await attempt(report.receipt_failures, "report-file-retry", saveReport);
|
||||||
|
return (
|
||||||
|
report.cleanup_failures.length === 0 && report.receipt_failures.length === 0
|
||||||
|
);
|
||||||
|
};
|
||||||
@@ -0,0 +1,166 @@
|
|||||||
|
"""Deterministic RPC receipt failures; not real-provider or model evidence."""
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import subprocess
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
|
||||||
|
class RetainedQueryReceipts(unittest.TestCase):
|
||||||
|
def run_node(self, body):
|
||||||
|
module = Path(__file__).with_name('retained-query-calls.mjs').resolve().as_uri()
|
||||||
|
finalizer = Path(__file__).with_name('retained-query-finalization.mjs').resolve().as_uri()
|
||||||
|
source = ('import assert from "node:assert/strict";\n'
|
||||||
|
'const {createRetainedQueryCaller} = await import(' + json.dumps(module) + ');\n'
|
||||||
|
'const {finalizeRetainedQueryReceipt} = await import(' + json.dumps(finalizer) + ');\n' + body)
|
||||||
|
result = subprocess.run(['node', '--input-type=module', '-'], input=source,
|
||||||
|
text=True, capture_output=True, timeout=15)
|
||||||
|
self.assertEqual(result.returncode, 0, result.stderr)
|
||||||
|
|
||||||
|
def run_finalization(self, body):
|
||||||
|
self.run_node('''
|
||||||
|
import {mkdtemp,mkdir,readFile,writeFile,rm,stat} from 'node:fs/promises';
|
||||||
|
import {join} from 'node:path';
|
||||||
|
import {tmpdir} from 'node:os';
|
||||||
|
import {Writable} from 'node:stream';
|
||||||
|
import {finished} from 'node:stream/promises';
|
||||||
|
const root=await mkdtemp(join(tmpdir(),'rea-receipt-test-'));
|
||||||
|
const account=join(root,'account');const output=join(root,'output');
|
||||||
|
await mkdir(account);await mkdir(output);
|
||||||
|
const chunks=[];
|
||||||
|
const stdout=new Writable({write(chunk,encoding,done){chunks.push(chunk.toString());done();}});
|
||||||
|
const serverLog=new Writable({write(chunk,encoding,done){done();}});
|
||||||
|
const serverLogFinished=finished(serverLog);serverLogFinished.catch(()=>{});
|
||||||
|
const originalFailure={name:'Error',message:'original request failed'};
|
||||||
|
const report={status:'failed',failure:originalFailure,calls:[{phase:'page',elapsed_ms:17}]};
|
||||||
|
const input={report,client:{close:async()=>{}},transport:{close:async()=>{}},
|
||||||
|
account,output,stdout,serverLog,serverLogFinished};
|
||||||
|
const saved=async()=>JSON.parse(await readFile(join(output,'report.json'),'utf8'));
|
||||||
|
try {
|
||||||
|
''' + body + '''
|
||||||
|
} finally {
|
||||||
|
serverLog.destroy();await serverLogFinished.catch(()=>{});stdout.destroy();
|
||||||
|
await rm(root,{recursive:true,force:true});
|
||||||
|
}
|
||||||
|
''')
|
||||||
|
|
||||||
|
def test_rejected_call_preserves_request_phase_cost_and_original_failure(self):
|
||||||
|
self.run_node('''
|
||||||
|
const failure = new Error('owned server closed');
|
||||||
|
const input = {offset:0, limit:2}; const rows=[]; const events=[];
|
||||||
|
const call = createRetainedQueryCaller({callTool:async request=>{
|
||||||
|
assert.strictEqual(request.arguments,input);throw failure;
|
||||||
|
}}, async(row,event)=>{rows.push(row);events.push(event);});
|
||||||
|
await assert.rejects(call('list_unknowns',input,'resume-2'), e=>e===failure);
|
||||||
|
assert.equal(rows.length,1);assert.equal(events.length,1);
|
||||||
|
assert.strictEqual(rows[0].input,input);assert.equal(rows[0].name,'list_unknowns');
|
||||||
|
assert.equal(rows[0].phase,'resume-2');assert.ok(rows[0].elapsed_ms>=0);
|
||||||
|
assert.equal(rows[0].reply_bytes,null);assert.equal(rows[0].is_error,true);
|
||||||
|
assert.deepEqual(events[0].failure,{name:'Error',message:'owned server closed'});
|
||||||
|
''')
|
||||||
|
|
||||||
|
def test_protocol_error_is_a_received_reply_with_known_bytes(self):
|
||||||
|
self.run_node('''
|
||||||
|
const reply={isError:true,content:[{type:'text',text:'capacity refusal'}]};const rows=[];
|
||||||
|
const call=createRetainedQueryCaller({callTool:async()=>reply},async row=>rows.push(row));
|
||||||
|
assert.strictEqual(await call('get_evidence_bundle',{},'complete'),reply);
|
||||||
|
assert.equal(rows.length,1);assert.equal(rows[0].is_error,true);
|
||||||
|
assert.equal(rows[0].reply_bytes,Buffer.byteLength(JSON.stringify(reply)));
|
||||||
|
assert.equal(Object.hasOwn(rows[0],'failure'),false);
|
||||||
|
''')
|
||||||
|
|
||||||
|
def test_receipt_write_error_does_not_reclassify_a_completed_rpc(self):
|
||||||
|
self.run_node('''
|
||||||
|
const failure=new Error('receipt disk unavailable');const rows=[];
|
||||||
|
const call=createRetainedQueryCaller({callTool:async()=>({content:[]})},async row=>{
|
||||||
|
rows.push(row);throw failure;
|
||||||
|
});
|
||||||
|
await assert.rejects(call('list_unknowns',{},'read'),e=>e===failure);
|
||||||
|
assert.equal(rows.length,1);assert.equal(rows[0].is_error,false);
|
||||||
|
assert.equal(Object.hasOwn(rows[0],'failure'),false);
|
||||||
|
''')
|
||||||
|
|
||||||
|
def test_failed_recording_preserves_the_prior_rpc_failure(self):
|
||||||
|
self.run_node('''
|
||||||
|
const rpc=new Error('request timeout');const disk=new Error('receipt close failed');
|
||||||
|
const call=createRetainedQueryCaller({callTool:async()=>{throw rpc;}},async()=>{throw disk;});
|
||||||
|
await assert.rejects(call('inspect_analysis_view',{},'selected'),e=>
|
||||||
|
e instanceof AggregateError && e.errors[0]===rpc && e.errors[1]===disk);
|
||||||
|
''')
|
||||||
|
|
||||||
|
def test_log_stream_failure_keeps_costs_both_receipts_and_workspace(self):
|
||||||
|
self.run_finalization('''
|
||||||
|
const log=new Writable({write(chunk,encoding,done){done(new Error('log disk failed'));}});
|
||||||
|
const logFinished=finished(log);logFinished.catch(()=>{});log.write('diagnostic');
|
||||||
|
assert.equal(await finalizeRetainedQueryReceipt({...input,serverLog:log,
|
||||||
|
serverLogFinished:logFinished}),false);
|
||||||
|
assert.match(report.cleanup_failures[0],/server-log:.*log disk failed/);
|
||||||
|
assert.strictEqual(report.failure,originalFailure);
|
||||||
|
assert.equal(report.calls[0].elapsed_ms,17);assert.equal(report.retained_workspace,account);
|
||||||
|
assert.equal((await stat(account)).isDirectory(),true);
|
||||||
|
assert.deepEqual(await saved(),report);assert.deepEqual(JSON.parse(chunks.join('')),report);
|
||||||
|
''')
|
||||||
|
|
||||||
|
def test_workspace_removal_error_still_writes_the_original_failure(self):
|
||||||
|
self.run_finalization('''
|
||||||
|
const blocked=join(root,'owned-file');await writeFile(blocked,'owned');
|
||||||
|
const malformedAccount=join(blocked,'child');
|
||||||
|
assert.equal(await finalizeRetainedQueryReceipt({...input,account:malformedAccount}),false);
|
||||||
|
assert.match(report.cleanup_failures[0],/^workspace:/);
|
||||||
|
assert.equal(report.retained_workspace,malformedAccount);
|
||||||
|
assert.deepEqual((await saved()).failure,originalFailure);
|
||||||
|
assert.deepEqual(JSON.parse(chunks.join('')),report);
|
||||||
|
''')
|
||||||
|
|
||||||
|
def test_report_file_refusal_preserves_the_stdout_receipt(self):
|
||||||
|
self.run_finalization('''
|
||||||
|
await mkdir(join(output,'report.json'));
|
||||||
|
assert.equal(await finalizeRetainedQueryReceipt(input),false);
|
||||||
|
assert.deepEqual(report.cleanup_failures,[]);
|
||||||
|
assert.match(report.receipt_failures[0],/^report-file:/);
|
||||||
|
assert.deepEqual(JSON.parse(chunks.join('')),report);
|
||||||
|
assert.deepEqual(report.failure,originalFailure);
|
||||||
|
''')
|
||||||
|
|
||||||
|
def test_stdout_write_error_is_handled_and_saved_in_the_file(self):
|
||||||
|
self.run_finalization('''
|
||||||
|
const failedStdout=new Writable({write(chunk,encoding,done){done(new Error('broken pipe'));}});
|
||||||
|
assert.equal(await finalizeRetainedQueryReceipt({...input,stdout:failedStdout}),false);
|
||||||
|
assert.match(report.receipt_failures[0],/stdout:.*broken pipe/);
|
||||||
|
assert.deepEqual(await saved(),report);assert.deepEqual(report.failure,originalFailure);
|
||||||
|
''')
|
||||||
|
|
||||||
|
def test_partial_retry_after_stdout_failure_preserves_the_prior_receipt(self):
|
||||||
|
self.run_finalization('''
|
||||||
|
const fs=await import('node:fs/promises');
|
||||||
|
const {syncBuiltinESMExports}=await import('node:module');
|
||||||
|
const originalWrite=fs.default.writeFile;let writes=0;let before;
|
||||||
|
fs.default.writeFile=async(...args)=>{
|
||||||
|
if(++writes===2){await originalWrite(args[0],'partial',args[2]);throw new Error('disk quota');}
|
||||||
|
return originalWrite(...args);
|
||||||
|
};
|
||||||
|
syncBuiltinESMExports();
|
||||||
|
try {
|
||||||
|
const failedStdout=new Writable({async write(chunk,encoding,done){
|
||||||
|
before=await readFile(join(output,'report.json'),'utf8');done(new Error('broken pipe'));
|
||||||
|
}});
|
||||||
|
assert.equal(await finalizeRetainedQueryReceipt({...input,stdout:failedStdout}),false);
|
||||||
|
assert.equal(writes,2);assert.equal(report.receipt_failures.length,2);
|
||||||
|
assert.match(report.receipt_failures[1],/report-file-retry:.*disk quota/);
|
||||||
|
assert.equal(await readFile(join(output,'report.json'),'utf8'),before);
|
||||||
|
assert.deepEqual((await saved()).failure,originalFailure);
|
||||||
|
assert.equal((await saved()).calls[0].elapsed_ms,17);
|
||||||
|
} finally {fs.default.writeFile=originalWrite;syncBuiltinESMExports();}
|
||||||
|
''')
|
||||||
|
|
||||||
|
def test_failed_client_close_keeps_ownership_and_attempts_peer_cleanup(self):
|
||||||
|
self.run_finalization('''
|
||||||
|
const closed=[];
|
||||||
|
assert.equal(await finalizeRetainedQueryReceipt({...input,
|
||||||
|
client:{close(){closed.push('client');throw new Error('close failed');}},
|
||||||
|
transport:{close:async()=>{closed.push('transport');}}}),false);
|
||||||
|
assert.deepEqual(closed,['client','transport']);
|
||||||
|
assert.match(report.cleanup_failures[0],/client:.*close failed/);
|
||||||
|
assert.equal(report.retained_workspace,account);assert.equal(serverLog.writableEnded,true);
|
||||||
|
assert.equal((await stat(account)).isDirectory(),true);
|
||||||
|
assert.deepEqual(await saved(),report);assert.deepEqual(JSON.parse(chunks.join('')),report);
|
||||||
|
''')
|
||||||
Reference in new issue
Block a user