mirror of
https://github.com/morluto/rea.git
synced 2026-10-26 17:35:11 +00:00
test(perf): verify authentic retained investigation queries (#4814)
Verify retained queries against authentic SQLite ELF and npm histories, retain failed RPC/cleanup/output costs, and make mixed-history verification repeatable. Record the separate validation-reuse bottleneck in #4810. Co-authored-by: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com> Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com>
This commit is contained in:
9 files changed
+1610
No files matched your search
@@ -1,5 +1,12 @@
|
||||
# Analysis efficiency measurements
|
||||
|
||||
For authentic old Evidence and mixed investigations, see the
|
||||
[retained-query checkpoint](retained-investigation-workflows.md).
|
||||
`retained-investigation.mjs` compares existing complete delivery with selected
|
||||
pages, including capacity-refusal recovery and original-record round trips.
|
||||
It records negative model results and separates imported-format validation work
|
||||
from the query contract.
|
||||
|
||||
For genuine Fiddler captures, see the [SAZ selection checkpoint](saz-selection-workflows.md).
|
||||
`measure.mjs --saz-selection baseline|focused|retained` compares default complete
|
||||
delivery (including retained-reference recovery), caller-selected ordinals, and
|
||||
|
||||
@@ -187,6 +187,7 @@ def main():
|
||||
'inspect_managed_members', 'inspect_managed_native_boundaries',
|
||||
'inspect_artifact', 'inspect_wasm_artifact',
|
||||
'inspect_evm_interface', 'get_evidence_bundle', 'export_evidence_bundle',
|
||||
'import_evidence_bundle', 'list_unknowns', 'verify_unknown_resolution',
|
||||
'capture_browser_scenario',
|
||||
'resolve_containing_procedure', 'analyze_function', 'close_binary']:
|
||||
config += '[mcp_servers.rea.tools.' + tool + ']\napproval_mode = "approve"\n'
|
||||
|
||||
@@ -0,0 +1,635 @@
|
||||
{
|
||||
"runtime_revision": "f4322b15ed1423d736bd2572cf7a8eadf2b466e1",
|
||||
"host": "Linux x86-64",
|
||||
"node": "24.18.0",
|
||||
"npm": "11.16.0",
|
||||
"codex_cli": "0.161.0",
|
||||
"requested_model": "gpt-6.1-sol",
|
||||
"reasoning": "high",
|
||||
"resolved_model_identity": "unknown",
|
||||
"code_mode": false,
|
||||
"client_enabled_tools": [
|
||||
"import_evidence_bundle",
|
||||
"get_evidence_bundle",
|
||||
"inspect_analysis_view",
|
||||
"list_unknowns"
|
||||
],
|
||||
"catalog": {
|
||||
"tools": 207,
|
||||
"serialized_bytes": 3152125,
|
||||
"sha256": "4f31d9d56c400bc48f5639a9dc4b4cdad7faf862e4be81641d94fa70552438e4"
|
||||
},
|
||||
"bounds": {
|
||||
"cpu_quota_percent": 100,
|
||||
"family_hard_mib": 8192,
|
||||
"early_stop_mib": 7168,
|
||||
"swap": false,
|
||||
"tasks": 192
|
||||
},
|
||||
"counter_scope": "Payload-exit counters with observer alive; kernel family peaks include file cache; terminal whole-scope counters and remote model compute unknown",
|
||||
"sources": {
|
||||
"native": {
|
||||
"target_sha256": "2e6eef9a727f081f0d453b4e5e6cbd8b9ef8b6f86cbf7681cbad444d3b0b55c8",
|
||||
"sections": 26,
|
||||
"symbols": 1479,
|
||||
"relocations": 2963,
|
||||
"all_symbol_locations_names_values_agree_with_independent_ELF_parser": true
|
||||
},
|
||||
"javascript": {
|
||||
"package": "npm",
|
||||
"version": "11.16.0",
|
||||
"source_files": [
|
||||
{
|
||||
"path": "package.json",
|
||||
"sha256": "60ff022e32b5d8291d3e09849fd0cee6bbbff4e4d8352ad258214b1adf070714"
|
||||
},
|
||||
{
|
||||
"path": "bin/npm-cli.js",
|
||||
"sha256": "8e5f6f3429f8cdbe693cdc29904e9d5a7b127a494bd15c804bd54c7403bfcbe7"
|
||||
},
|
||||
{
|
||||
"path": "lib/cli.js",
|
||||
"sha256": "67666f06479f9b0bbc01412c198caadd4287d34d5ed74a02871ea78f186451e7"
|
||||
},
|
||||
{
|
||||
"path": "lib/cli/entry.js",
|
||||
"sha256": "667a44c1fa8795fdbba8ad843736ca7f8f24e64511985e9a7060b002123560c6"
|
||||
},
|
||||
{
|
||||
"path": "lib/commands/install.js",
|
||||
"sha256": "f7a60e8b40ab1bece76a2441b8b15378d199f05c6050eebe455cd9961f67b816"
|
||||
}
|
||||
],
|
||||
"lib_cli_module": {
|
||||
"node_id": "jag_node_131257ad52f3333971c2a5ed325c64357e52604489e42ccc67a8849770cb5b10",
|
||||
"kind": "javascript-module",
|
||||
"path": "lib/cli.js"
|
||||
},
|
||||
"source_bytes_checked": true,
|
||||
"inventory_completeness": "producer-relative, not independently established",
|
||||
"unselected_sources": 1416
|
||||
}
|
||||
},
|
||||
"sdk": [
|
||||
{
|
||||
"name": "sdk-native-complete-1",
|
||||
"status": "passed",
|
||||
"answer_sha256": "4e28c2c2a25620576d27abe846a05835ec49386b889779e33410c31971523b54",
|
||||
"query_ms": 1328.7,
|
||||
"query_reply_bytes": 11224227,
|
||||
"rpc_count": 7,
|
||||
"elapsed_seconds": 14.117,
|
||||
"cpu_seconds": 13.915,
|
||||
"peak_family_mib": 1160.8,
|
||||
"memory_events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
},
|
||||
"payload_cleanup_verified": true,
|
||||
"response_capacity_refused": false,
|
||||
"warm_page_ms": []
|
||||
},
|
||||
{
|
||||
"name": "sdk-native-complete-2",
|
||||
"status": "passed",
|
||||
"answer_sha256": "4e28c2c2a25620576d27abe846a05835ec49386b889779e33410c31971523b54",
|
||||
"query_ms": 1385.4,
|
||||
"query_reply_bytes": 11224227,
|
||||
"rpc_count": 7,
|
||||
"elapsed_seconds": 14.722,
|
||||
"cpu_seconds": 14.485,
|
||||
"peak_family_mib": 1109.9,
|
||||
"memory_events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
},
|
||||
"payload_cleanup_verified": true,
|
||||
"response_capacity_refused": false,
|
||||
"warm_page_ms": []
|
||||
},
|
||||
{
|
||||
"name": "sdk-native-focused-1",
|
||||
"status": "passed",
|
||||
"answer_sha256": "4e28c2c2a25620576d27abe846a05835ec49386b889779e33410c31971523b54",
|
||||
"query_ms": 529.7,
|
||||
"query_reply_bytes": 167985,
|
||||
"rpc_count": 25,
|
||||
"elapsed_seconds": 12.907,
|
||||
"cpu_seconds": 12.679,
|
||||
"peak_family_mib": 934.8,
|
||||
"memory_events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
},
|
||||
"payload_cleanup_verified": true,
|
||||
"response_capacity_refused": false,
|
||||
"warm_page_ms": [44.1, 59.6, 24.0, 30.8, 26.7, 47.4]
|
||||
},
|
||||
{
|
||||
"name": "sdk-native-focused-2",
|
||||
"status": "passed",
|
||||
"answer_sha256": "4e28c2c2a25620576d27abe846a05835ec49386b889779e33410c31971523b54",
|
||||
"query_ms": 572.2,
|
||||
"query_reply_bytes": 167985,
|
||||
"rpc_count": 25,
|
||||
"elapsed_seconds": 13.316,
|
||||
"cpu_seconds": 13.116,
|
||||
"peak_family_mib": 1000.0,
|
||||
"memory_events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
},
|
||||
"payload_cleanup_verified": true,
|
||||
"response_capacity_refused": false,
|
||||
"warm_page_ms": [44.0, 64.6, 26.8, 33.2, 29.8, 50.4]
|
||||
},
|
||||
{
|
||||
"name": "sdk-javascript-complete-1",
|
||||
"status": "passed",
|
||||
"answer_sha256": "0fe3ad249eb1a7233a4966a56652c173667c48fdaf395b6c08fc6901ef3b46f7",
|
||||
"query_ms": 11241.5,
|
||||
"query_reply_bytes": 212856,
|
||||
"rpc_count": 31,
|
||||
"elapsed_seconds": 26.3,
|
||||
"cpu_seconds": 26.1,
|
||||
"peak_family_mib": 1333.5,
|
||||
"memory_events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
},
|
||||
"payload_cleanup_verified": true,
|
||||
"response_capacity_refused": true,
|
||||
"warm_page_ms": [1124.6, 1073.8, 1474.7, 1074.7, 1081.6, 1109.5]
|
||||
},
|
||||
{
|
||||
"name": "sdk-javascript-complete-2",
|
||||
"status": "passed",
|
||||
"answer_sha256": "0fe3ad249eb1a7233a4966a56652c173667c48fdaf395b6c08fc6901ef3b46f7",
|
||||
"query_ms": 11938.1,
|
||||
"query_reply_bytes": 212856,
|
||||
"rpc_count": 31,
|
||||
"elapsed_seconds": 27.105,
|
||||
"cpu_seconds": 26.87,
|
||||
"peak_family_mib": 1320.9,
|
||||
"memory_events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
},
|
||||
"payload_cleanup_verified": true,
|
||||
"response_capacity_refused": true,
|
||||
"warm_page_ms": [1155.4, 1096.2, 1489.3, 1121.3, 1088.3, 1484.6]
|
||||
},
|
||||
{
|
||||
"name": "sdk-javascript-focused-2",
|
||||
"status": "passed",
|
||||
"answer_sha256": "0fe3ad249eb1a7233a4966a56652c173667c48fdaf395b6c08fc6901ef3b46f7",
|
||||
"query_ms": 10967.3,
|
||||
"query_reply_bytes": 208350,
|
||||
"rpc_count": 28,
|
||||
"elapsed_seconds": 25.708,
|
||||
"cpu_seconds": 25.505,
|
||||
"peak_family_mib": 1344.0,
|
||||
"memory_events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
},
|
||||
"payload_cleanup_verified": true,
|
||||
"response_capacity_refused": false,
|
||||
"warm_page_ms": [1342.7, 1085.1, 1055.0, 1034.6, 1082.7, 1440.7]
|
||||
},
|
||||
{
|
||||
"name": "sdk-javascript-focused-3",
|
||||
"status": "passed",
|
||||
"answer_sha256": "0fe3ad249eb1a7233a4966a56652c173667c48fdaf395b6c08fc6901ef3b46f7",
|
||||
"query_ms": 10990.4,
|
||||
"query_reply_bytes": 208350,
|
||||
"rpc_count": 28,
|
||||
"elapsed_seconds": 26.197,
|
||||
"cpu_seconds": 25.98,
|
||||
"peak_family_mib": 1336.0,
|
||||
"memory_events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
},
|
||||
"payload_cleanup_verified": true,
|
||||
"response_capacity_refused": false,
|
||||
"warm_page_ms": [1338.1, 1088.5, 1069.1, 1038.3, 1087.5, 1483.0]
|
||||
},
|
||||
{
|
||||
"name": "sdk-mixed-native",
|
||||
"status": "passed",
|
||||
"answer_sha256": "4e28c2c2a25620576d27abe846a05835ec49386b889779e33410c31971523b54",
|
||||
"query_ms": 560.0,
|
||||
"query_reply_bytes": 167991,
|
||||
"rpc_count": 25,
|
||||
"elapsed_seconds": 16.933,
|
||||
"cpu_seconds": 16.714,
|
||||
"peak_family_mib": 1261.4,
|
||||
"memory_events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
},
|
||||
"payload_cleanup_verified": true,
|
||||
"response_capacity_refused": false,
|
||||
"warm_page_ms": [46.9, 51.0, 36.3, 24.4, 38.6, 43.8]
|
||||
},
|
||||
{
|
||||
"name": "sdk-mixed-javascript",
|
||||
"status": "passed",
|
||||
"answer_sha256": "0fe3ad249eb1a7233a4966a56652c173667c48fdaf395b6c08fc6901ef3b46f7",
|
||||
"query_ms": 10904.5,
|
||||
"query_reply_bytes": 208356,
|
||||
"rpc_count": 28,
|
||||
"elapsed_seconds": 26.5,
|
||||
"cpu_seconds": 26.297,
|
||||
"peak_family_mib": 1346.5,
|
||||
"memory_events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
},
|
||||
"payload_cleanup_verified": true,
|
||||
"response_capacity_refused": false,
|
||||
"warm_page_ms": [1321.9, 1063.3, 1070.8, 1309.9, 1041.7, 1074.2]
|
||||
}
|
||||
],
|
||||
"model_trials": [
|
||||
{
|
||||
"name": "model-javascript-complete-1",
|
||||
"facts_pass": true,
|
||||
"elapsed_seconds": 76.49,
|
||||
"cpu_seconds": 24.687,
|
||||
"peak_family_mib": 818.7,
|
||||
"input_tokens": 454491,
|
||||
"cached_input_tokens": 396288,
|
||||
"uncached_input_tokens": 58203,
|
||||
"output_tokens": 2033,
|
||||
"reported_reasoning_output_tokens": 207,
|
||||
"mcp_calls_including_discovery": 9,
|
||||
"payload_cleanup_verified": true,
|
||||
"memory_events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "model-javascript-complete-2",
|
||||
"facts_pass": true,
|
||||
"elapsed_seconds": 58.335,
|
||||
"cpu_seconds": 14.993,
|
||||
"peak_family_mib": 971.4,
|
||||
"input_tokens": 328066,
|
||||
"cached_input_tokens": 280704,
|
||||
"uncached_input_tokens": 47362,
|
||||
"output_tokens": 1676,
|
||||
"reported_reasoning_output_tokens": 129,
|
||||
"mcp_calls_including_discovery": 9,
|
||||
"payload_cleanup_verified": true,
|
||||
"memory_events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "model-javascript-focused-1",
|
||||
"facts_pass": false,
|
||||
"elapsed_seconds": 34.206,
|
||||
"cpu_seconds": 16.31,
|
||||
"peak_family_mib": 811.9,
|
||||
"input_tokens": 167470,
|
||||
"cached_input_tokens": 136320,
|
||||
"uncached_input_tokens": 31150,
|
||||
"output_tokens": 633,
|
||||
"reported_reasoning_output_tokens": 172,
|
||||
"mcp_calls_including_discovery": 4,
|
||||
"payload_cleanup_verified": true,
|
||||
"memory_events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "model-javascript-focused-2",
|
||||
"facts_pass": true,
|
||||
"elapsed_seconds": 66.455,
|
||||
"cpu_seconds": 14.337,
|
||||
"peak_family_mib": 955.3,
|
||||
"input_tokens": 339528,
|
||||
"cached_input_tokens": 307456,
|
||||
"uncached_input_tokens": 32072,
|
||||
"output_tokens": 2051,
|
||||
"reported_reasoning_output_tokens": 169,
|
||||
"mcp_calls_including_discovery": 10,
|
||||
"payload_cleanup_verified": true,
|
||||
"memory_events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "model-javascript-guided-complete-1",
|
||||
"facts_pass": true,
|
||||
"elapsed_seconds": 69.206,
|
||||
"cpu_seconds": 13.65,
|
||||
"peak_family_mib": 888.8,
|
||||
"input_tokens": 168862,
|
||||
"cached_input_tokens": 121728,
|
||||
"uncached_input_tokens": 47134,
|
||||
"output_tokens": 2385,
|
||||
"reported_reasoning_output_tokens": 737,
|
||||
"mcp_calls_including_discovery": 6,
|
||||
"payload_cleanup_verified": true,
|
||||
"memory_events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "model-javascript-guided-complete-2",
|
||||
"facts_pass": false,
|
||||
"elapsed_seconds": 29.187,
|
||||
"cpu_seconds": 15.105,
|
||||
"peak_family_mib": 697.4,
|
||||
"input_tokens": 28696,
|
||||
"cached_input_tokens": 23296,
|
||||
"uncached_input_tokens": 5400,
|
||||
"output_tokens": 610,
|
||||
"reported_reasoning_output_tokens": 379,
|
||||
"mcp_calls_including_discovery": 0,
|
||||
"payload_cleanup_verified": true,
|
||||
"memory_events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "model-javascript-guided-focused-1",
|
||||
"facts_pass": true,
|
||||
"elapsed_seconds": 53.464,
|
||||
"cpu_seconds": 12.059,
|
||||
"peak_family_mib": 944.5,
|
||||
"input_tokens": 124218,
|
||||
"cached_input_tokens": 92416,
|
||||
"uncached_input_tokens": 31802,
|
||||
"output_tokens": 1746,
|
||||
"reported_reasoning_output_tokens": 330,
|
||||
"mcp_calls_including_discovery": 5,
|
||||
"payload_cleanup_verified": true,
|
||||
"memory_events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "model-javascript-guided-focused-2",
|
||||
"facts_pass": true,
|
||||
"elapsed_seconds": 58.526,
|
||||
"cpu_seconds": 22.852,
|
||||
"peak_family_mib": 924.2,
|
||||
"input_tokens": 129134,
|
||||
"cached_input_tokens": 97536,
|
||||
"uncached_input_tokens": 31598,
|
||||
"output_tokens": 1478,
|
||||
"reported_reasoning_output_tokens": 69,
|
||||
"mcp_calls_including_discovery": 5,
|
||||
"payload_cleanup_verified": true,
|
||||
"memory_events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "model-native-complete-1",
|
||||
"facts_pass": true,
|
||||
"elapsed_seconds": 52.103,
|
||||
"cpu_seconds": 8.686,
|
||||
"peak_family_mib": 1028.5,
|
||||
"input_tokens": 287913,
|
||||
"cached_input_tokens": 249984,
|
||||
"uncached_input_tokens": 37929,
|
||||
"output_tokens": 1724,
|
||||
"reported_reasoning_output_tokens": 121,
|
||||
"mcp_calls_including_discovery": 4,
|
||||
"payload_cleanup_verified": true,
|
||||
"memory_events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "model-native-complete-2",
|
||||
"facts_pass": true,
|
||||
"elapsed_seconds": 58.261,
|
||||
"cpu_seconds": 9.102,
|
||||
"peak_family_mib": 837.4,
|
||||
"input_tokens": 348976,
|
||||
"cached_input_tokens": 286976,
|
||||
"uncached_input_tokens": 62000,
|
||||
"output_tokens": 2009,
|
||||
"reported_reasoning_output_tokens": 246,
|
||||
"mcp_calls_including_discovery": 5,
|
||||
"payload_cleanup_verified": true,
|
||||
"memory_events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "model-native-focused-1",
|
||||
"facts_pass": true,
|
||||
"elapsed_seconds": 62.078,
|
||||
"cpu_seconds": 10.198,
|
||||
"peak_family_mib": 823.9,
|
||||
"input_tokens": 335445,
|
||||
"cached_input_tokens": 293120,
|
||||
"uncached_input_tokens": 42325,
|
||||
"output_tokens": 1924,
|
||||
"reported_reasoning_output_tokens": 87,
|
||||
"mcp_calls_including_discovery": 8,
|
||||
"payload_cleanup_verified": true,
|
||||
"memory_events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "model-native-focused-2",
|
||||
"facts_pass": true,
|
||||
"elapsed_seconds": 57.933,
|
||||
"cpu_seconds": 8.058,
|
||||
"peak_family_mib": 821.3,
|
||||
"input_tokens": 286938,
|
||||
"cached_input_tokens": 256384,
|
||||
"uncached_input_tokens": 30554,
|
||||
"output_tokens": 1831,
|
||||
"reported_reasoning_output_tokens": 106,
|
||||
"mcp_calls_including_discovery": 8,
|
||||
"payload_cleanup_verified": true,
|
||||
"memory_events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
}
|
||||
}
|
||||
],
|
||||
"failed_trials": {
|
||||
"model-javascript-focused-1": "Wrong connector discovery; approval-policy refusal; no source facts established.",
|
||||
"model-javascript-guided-complete-2": "Premature agent refusal with no REA RPC; asserted tool unavailability not established by a connection/discovery observation."
|
||||
},
|
||||
"pilot_failures": [
|
||||
"Private pilot oracle read view.item instead of view.items; corrected before the committed SDK cohorts. No product failure inferred."
|
||||
],
|
||||
"decision": "Existing query APIs suffice for these bounded questions; no general token/time advantage established. Separate imported immutable-format validation optimization is tracked in #4810.",
|
||||
"review_followup_verification": {
|
||||
"review-mixed-native": {
|
||||
"status": "passed",
|
||||
"checks": {
|
||||
"heads_checked": 64,
|
||||
"pages": 32,
|
||||
"revised_unknown_id": "unk_057c8f2dbf30ec98851a66ee054366bc6e2e168874581e6f99cb80673d0ff9b8",
|
||||
"revision_checked": 2,
|
||||
"stale_update_rejected": true,
|
||||
"tampered_import_rejected_atomically": true,
|
||||
"unknowns_after_close": 0,
|
||||
"old_reference_refused": true
|
||||
},
|
||||
"original_records_preserved": true,
|
||||
"elapsed_seconds": 18.444,
|
||||
"cpu_seconds": 18.16515,
|
||||
"peak_family_mib": 1401.6328125,
|
||||
"oom_events_at_payload_exit": 0,
|
||||
"payload_cleanup_verified": true
|
||||
},
|
||||
"review-mixed-javascript": {
|
||||
"status": "passed",
|
||||
"checks": {
|
||||
"heads_checked": 64,
|
||||
"pages": 32,
|
||||
"revised_unknown_id": "unk_057c8f2dbf30ec98851a66ee054366bc6e2e168874581e6f99cb80673d0ff9b8",
|
||||
"revision_checked": 2,
|
||||
"stale_update_rejected": true,
|
||||
"tampered_import_rejected_atomically": true,
|
||||
"unknowns_after_close": 0,
|
||||
"old_reference_refused": true
|
||||
},
|
||||
"original_records_preserved": true,
|
||||
"elapsed_seconds": 27.617,
|
||||
"cpu_seconds": 27.275299,
|
||||
"peak_family_mib": 1444.30859375,
|
||||
"oom_events_at_payload_exit": 0,
|
||||
"payload_cleanup_verified": true
|
||||
},
|
||||
"deterministic_receipt_tests": {
|
||||
"passed": 4,
|
||||
"scope": "Rejected SDK call, protocol error, receipt-write failure, simultaneous call/write failure; not real-provider evidence"
|
||||
},
|
||||
"cleanup_receipt_regressions": {
|
||||
"passed": 6,
|
||||
"scope": "Real stream/file refusals, independent peer close, and injected partial publication at the filesystem boundary. Original failure and call costs survive; failed cleanup retains workspace ownership. Harness failure checks, not real-provider evidence.",
|
||||
"baseline_reproduction": {
|
||||
"revision": "b025a919b36ec091ca888339df6b24e78e89b79b",
|
||||
"method": "Execute the prior committed finalization block with deterministic workspace or log rejection; not a production-provider reproduction.",
|
||||
"workspace_error": {
|
||||
"file_receipt_attempts": 0,
|
||||
"stdout_receipt_attempts": 0
|
||||
},
|
||||
"log_error": {
|
||||
"file_receipt_attempts": 0,
|
||||
"stdout_receipt_attempts": 0
|
||||
}
|
||||
},
|
||||
"partial_retry_baseline": {
|
||||
"revision": "5d6259e658ee54fc9829b9143a8721a2ab426e80",
|
||||
"regression_exit_code": 1,
|
||||
"observed_saved_report": "partial",
|
||||
"method": "Run the unchanged partial-write regression against the prior committed finalizer with deterministic filesystem-boundary failure injection."
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
# Retained investigation query checkpoint
|
||||
|
||||
[#3134](https://github.com/morluto/rea/issues/3134) measured existing retained
|
||||
queries on Linux. Selected reads preserved the requested facts, original
|
||||
Evidence and Unknown revisions, including a mixed investigation. Existing
|
||||
contracts suffice for these questions; this checkpoint adds no production API,
|
||||
cache, persistence or default change.
|
||||
|
||||
Imported JavaScript pages repeatedly validated their entire source graph. That
|
||||
separate, observed bottleneck is owned by
|
||||
[#4810](https://github.com/morluto/rea/issues/4810). Smaller replies did not
|
||||
establish a general model-token or whole-task latency improvement.
|
||||
|
||||
The [curated measurements](retained-investigation-results.json) retain individual
|
||||
SDK/model costs and unsuccessful attempts. Raw bundles, transcripts, profiles
|
||||
and detailed receipts stay private and are not build inputs.
|
||||
|
||||
## Sources and questions
|
||||
|
||||
Runtime: `f4322b15ed1423d736bd2572cf7a8eadf2b466e1`, Linux x86-64,
|
||||
Node 24.18.0/npm 11.16.0. The SDK discovered all 207 advertised tools
|
||||
(3,152,125 serialized catalog bytes). Model trials used Codex CLI 0.161.0,
|
||||
requested `gpt-6.1-sol`/high, direct MCP, and the same four enabled tool names:
|
||||
`import_evidence_bundle`, `get_evidence_bundle`, `inspect_analysis_view`, and
|
||||
`list_unknowns`. The resolved service-side model identity is unknown. These are
|
||||
selected-client-tool trials, not an evaluation of full-catalog agent discovery.
|
||||
|
||||
Two public source workflows produced authentic bundles through production MCP:
|
||||
|
||||
- Native: `inspect_binary_layout` on Debian's `libsqlite3.so.0.8.6`, SHA-256
|
||||
`2e6eef9a727f081f0d453b4e5e6cbd8b9ef8b6f86cbf7681cbad444d3b0b55c8`.
|
||||
Pwntools 4.15.0/pyelftools 0.33/Unicorn 2.1.2 observed 26 sections, 1,479
|
||||
symbols and 2,963 relocations. An independent ELF64 decoder checked section
|
||||
and relocation counts and every symbol's table/entry index, value, raw name
|
||||
bytes and source entry range against the original file.
|
||||
- JavaScript: shipped npm 11.16.0, directory inventory digest
|
||||
`f8e576a7d162542c616168b25a95a5f62b9accf29467cc37e149905ad4b8e603`.
|
||||
`analyze_javascript_application` selected exactly `package.json`,
|
||||
`bin/npm-cli.js`, `lib/cli.js`, `lib/cli/entry.js`, and
|
||||
`lib/commands/install.js`. It parsed four JavaScript files; 1,416 other
|
||||
relevant files remained unselected. The complete retained result contained
|
||||
2,822 application nodes, 1,796 semantic nodes and 13,682 semantic unknowns.
|
||||
Source-file digests and the `lib/cli.js` source-module identity were checked
|
||||
against shipped bytes. Inventory/discovery completeness remains producer-relative.
|
||||
|
||||
Analyst-recorded Unknowns came from the actual source limitations: five native
|
||||
and seven JavaScript questions. These registry heads are distinct from the
|
||||
13,682 semantic-graph unknowns. The exported histories contained 15 and 17
|
||||
records, respectively, and approximately 1.75 and 21.06 MB of canonical JSON.
|
||||
No synthetic payload arrays were used.
|
||||
|
||||
The SDK task read the first six original symbol/module rows and the relevant
|
||||
current Unknown heads. Complete delivery and selected delivery answered the
|
||||
same question; selected delivery intentionally exercised two-row pages. Each
|
||||
fresh server performed three traversals, exported again and closed. The model
|
||||
task additionally requested native identity/counts, or the five JavaScript source
|
||||
choices and exact `lib/cli.js` source module, with explicit runtime uncertainty.
|
||||
Compare arms within each declared task, not SDK costs against model costs.
|
||||
|
||||
## Correctness, capacity and lifetime
|
||||
|
||||
Both fresh-server SDK repetitions passed in both delivery arms. Selected rows,
|
||||
ordering and counts agreed with original records; all original records and
|
||||
Unknown revisions survived export/import, including links, omitted fields,
|
||||
nulls and scalar types. Query calls did not rerun either producer. The resumed
|
||||
servers had no native producer configuration.
|
||||
|
||||
The JavaScript complete bundle exceeded the default 10 MiB receive-buffer
|
||||
boundary. Its typed capacity refusal preserved the complete retained data and a
|
||||
usable connection. Complete-arm costs include that refusal and subsequent
|
||||
selected-view recovery; they are not timings for a successful complete transfer.
|
||||
|
||||
A mixed history combined both authentic investigations and promoted 64 actual
|
||||
semantic unknowns to separately identified analyst follow-ups. One follow-up
|
||||
was updated to investigating. It contained 97 records, 76 heads and 77 revisions.
|
||||
Both source queries preserved all original records/revisions. Thirty-two
|
||||
two-head pages verified every semantic-follow-up head, including revision two.
|
||||
A stale update and a tampered bundle import were rejected without changing the
|
||||
ledger. Export reproduced the complete original mixed bundle. Closing cleared
|
||||
records/Unknowns and made the old reference unavailable; no implicit persistence
|
||||
or producer restart was introduced.
|
||||
|
||||
Public CLI `inspect-evidence-bundle` summary and inline `inspect-analysis-view`
|
||||
summary also passed on both authentic sources. Portable CLI input still requires
|
||||
complete input admission/validation; small output does not remove that cost.
|
||||
|
||||
## SDK costs
|
||||
|
||||
Each row spans two fresh MCP servers. Query time is the sum of SDK RPC elapsed
|
||||
times over three traversals. Local post-call contract validation, oracle checks
|
||||
and transcript writes are included only in the whole harness, along with startup,
|
||||
full discovery, file reads, import, export and close. OS caches were uncontrolled/warm. A fresh imported
|
||||
session is not a cold native-analysis measurement.
|
||||
|
||||
| Source / arm | Three traversals | Query reply bytes | Whole elapsed | Local family CPU | Family peak |
|
||||
| ------------------------------ | ---------------: | ----------------: | --------------: | ---------------: | --------------: |
|
||||
| Native complete | 1.329–1.385 s | 11,224,227 | 14.117–14.722 s | 13.915–14.485 s | 1,110–1,161 MiB |
|
||||
| Native selected | 0.530–0.572 s | 167,985 | 12.907–13.316 s | 12.679–13.116 s | 935–1,000 MiB |
|
||||
| JavaScript complete + recovery | 11.242–11.938 s | 212,856 | 26.300–27.105 s | 26.100–26.870 s | 1,321–1,334 MiB |
|
||||
| JavaScript selected | 10.967–10.990 s | 208,350 | 25.708–26.197 s | 25.505–25.980 s | 1,336–1,344 MiB |
|
||||
|
||||
Mixed-history selected tasks took 16.933 s native and 26.500 s JavaScript,
|
||||
including the larger import and complete round-trip checks. Their family peaks
|
||||
were 1,261 and 1,347 MiB. This does not establish scaling to thousands of
|
||||
registry heads or every provider.
|
||||
|
||||
Original-session npm warm pages measured 8–31 ms; after import, repeated pages
|
||||
cost about 1.1–1.4 s. Source inspection shows that imported immutable Evidence
|
||||
does not carry the format owner's private graph proofs, so each view repeats
|
||||
full validation. Native layout views also parse the complete layout on each
|
||||
page. #4810 owns equivalent before/after verification of a correction; this
|
||||
checkpoint claims no speedup from an unimplemented cache.
|
||||
|
||||
## Actual model tasks
|
||||
|
||||
All twelve attempts retained their costs. Ten supplied every checked fact and
|
||||
current Unknown identity/status; runtime and unselected-source coverage stayed
|
||||
unknown. One initial JavaScript selected attempt searched the wrong connector
|
||||
and received an approval-policy refusal. One guided complete attempt stopped
|
||||
without an REA RPC and asserted tool unavailability without a supporting
|
||||
connection/discovery observation. Both are failed tasks, despite CLI exit zero.
|
||||
|
||||
| Cohort / arm | Successful tasks | Total input tokens | Elapsed |
|
||||
| ------------------------------------- | ---------------: | -----------------------: | -----------------------: |
|
||||
| Native complete | 2/2 | 287,913 / 348,976 | 52.103 / 58.261 s |
|
||||
| Native selected, two-head pages | 2/2 | 335,445 / 286,938 | 62.078 / 57.933 s |
|
||||
| JavaScript complete + recovery | 2/2 | 454,491 / 328,066 | 76.490 / 58.335 s |
|
||||
| JavaScript selected, two-head pages | 1/2 | 167,470 failed / 339,528 | 34.206 failed / 66.455 s |
|
||||
| JavaScript guided complete + recovery | 1/2 | 168,862 / 28,696 failed | 69.206 / 29.187 failed s |
|
||||
| JavaScript guided selected | 2/2 | 124,218 / 129,134 | 53.464 / 58.526 s |
|
||||
|
||||
The separate guided JavaScript cohort named the exact configured tools and
|
||||
allowed one sufficiently large Unknown page for this bounded question. Both
|
||||
arms received the same discovery guidance. Its single completed complete-arm
|
||||
sample is insufficient for a stable repeated-pair benefit claim. Native total
|
||||
input/time overlapped; extra pages and model turns can consume the reply saving.
|
||||
Do not pool failed tasks as fast answers or attribute guidance changes to REA.
|
||||
|
||||
The JSON report records total/cached/uncached input, output, local CPU and peak
|
||||
memory per attempt. These are client-reported usage counters, not tokenizer
|
||||
estimates; output includes the client's reported reasoning accounting without
|
||||
adding it again. Schema/discovery, errors, retries and continuations remain in
|
||||
the task. Remote model compute, full-catalog model behavior and other hosts were
|
||||
not measured.
|
||||
|
||||
Heavy runs were serial, one CPU, 8 GiB family hard limit, 7 GiB early stop,
|
||||
no swap. Kernel peaks include charged file cache. Counter observations were
|
||||
taken at payload exit while the owned observer remained alive; terminal
|
||||
whole-scope counters remain unknown. Observed payload OOM/capacity events were
|
||||
zero, all payloads exited and cleanup was verified. The expected MCP response
|
||||
refusal is a separate transport-capacity outcome.
|
||||
|
||||
## Reproduce
|
||||
|
||||
Build the pinned runtime, acquire the matching real artifacts and produce the
|
||||
two authentic histories with the public operations above. Export before close.
|
||||
Keep bundles, transcripts and model profiles private. For a known source ID:
|
||||
|
||||
```sh
|
||||
node experiments/analysis-efficiency/retained-investigation.mjs \
|
||||
--runtime-root /absolute/built-rea \
|
||||
--revision f4322b15ed1423d736bd2572cf7a8eadf2b466e1 \
|
||||
--bundle /absolute/authentic-history.json \
|
||||
--evidence-id ev_ORIGINAL_SHA256 \
|
||||
--scenario native --delivery focused \
|
||||
--output /absolute/new-receipt-directory
|
||||
```
|
||||
|
||||
Run under the existing process-family resource wrapper, with sufficient bounds
|
||||
for the selected source. Repeat with `--delivery complete`; use
|
||||
`--scenario javascript` for module rows. The task's Unknown domain is
|
||||
`retained-native` or `retained-javascript`. Output is never overwritten.
|
||||
`agent-trial.py --mode direct` supports the same four enabled names; its success
|
||||
exit only establishes client completion. Grade original IDs/digests, exact
|
||||
facts, coverage and Unknowns from the retained transcript and final answer.
|
||||
|
||||
For a privately supplied authentic mixed bundle, add `--verify-mixed`. Besides
|
||||
the normal source query and original-record round trip, this pages every current
|
||||
head in `javascript-semantic-followup`, checks the current updated revision,
|
||||
submits a deliberately stale update and a tampered copy of the bundle, verifies
|
||||
atomic refusal, then checks Unknown/reference lifetime after close. The bundle
|
||||
must contain genuine follow-ups and at least one updated head. The checkpoint's
|
||||
64-head bundle passed this mode for both source scenarios. Its extra verification
|
||||
costs are separate from the original SDK comparison rows above.
|
||||
|
||||
Rejected SDK calls are recorded before propagating the original failure, with
|
||||
request, phase and elapsed time. Reply bytes stay null when no usable reply was
|
||||
received. Receipt-write failures remain distinct; simultaneous call/write
|
||||
failures retain both causes. Deterministic failure tests run with
|
||||
`python3 -m unittest discover -s experiments/analysis-efficiency -p test_retained_queries.py`.
|
||||
|
||||
Finalization independently attempts client/transport close, log completion and
|
||||
both receipt outputs. Cleanup failures preserve the original task failure and
|
||||
call costs, record the failing phase and retain the workspace for retry. A file
|
||||
receipt refusal still permits stdout; a broken stdout write is recorded in a
|
||||
file retry. Files publish through a sibling temporary file and rename; an
|
||||
interrupted retry preserves the previous receipt. Failed publication files stay
|
||||
in the owned output directory. Cleanup or receipt-output failures make the
|
||||
command fail. Six additional regressions exercise stream/file errors, partial
|
||||
publication and peer cleanup; these are harness failure checks, not real-provider
|
||||
evidence.
|
||||
@@ -0,0 +1,310 @@
|
||||
/** Measure public retained queries against an authentic, previously exported bundle. */
|
||||
import assert from "node:assert/strict";
|
||||
import { createHash } from "node:crypto";
|
||||
import { createWriteStream } from "node:fs";
|
||||
import {
|
||||
appendFile,
|
||||
mkdir,
|
||||
mkdtemp,
|
||||
readFile,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, isAbsolute, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { parseArgs } from "node:util";
|
||||
import { finished } from "node:stream/promises";
|
||||
import canonicalize from "canonicalize";
|
||||
import { Client } from "@modelcontextprotocol/client";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
||||
import {
|
||||
requireMcpOperationResult,
|
||||
requireMcpToolError,
|
||||
} from "../../scripts/lib/mcp-verifier-results.mjs";
|
||||
import { createRetainedQueryCaller } from "./retained-query-calls.mjs";
|
||||
import { finalizeRetainedQueryReceipt } from "./retained-query-finalization.mjs";
|
||||
import {
|
||||
verifyMixedRetainedHistory,
|
||||
verifyMixedRetainedClosure,
|
||||
retainedUnknownHeads,
|
||||
} from "./retained-mixed-verification.mjs";
|
||||
|
||||
const { values } = parseArgs({
|
||||
options: {
|
||||
"runtime-root": {
|
||||
type: "string",
|
||||
default: resolve(dirname(fileURLToPath(import.meta.url)), "../.."),
|
||||
},
|
||||
revision: { type: "string" },
|
||||
bundle: { type: "string" },
|
||||
"evidence-id": { type: "string" },
|
||||
scenario: { type: "string" },
|
||||
delivery: { type: "string" },
|
||||
output: { type: "string" },
|
||||
"verify-mixed": { type: "boolean", default: false },
|
||||
},
|
||||
});
|
||||
for (const key of ["runtime-root", "bundle", "output"])
|
||||
assert.ok(values[key] && isAbsolute(values[key]), `${key} must be absolute`);
|
||||
assert.ok(values.revision && values["evidence-id"]);
|
||||
assert.ok(["native", "javascript"].includes(values.scenario));
|
||||
assert.ok(["complete", "focused"].includes(values.delivery));
|
||||
const root = values["runtime-root"];
|
||||
const output = values.output;
|
||||
await mkdir(output, { recursive: false, mode: 0o700 });
|
||||
const digest = (bytes) => createHash("sha256").update(bytes).digest("hex");
|
||||
const sourceBytes = await readFile(values.bundle);
|
||||
const original = JSON.parse(sourceBytes);
|
||||
const source = original.records.find(
|
||||
(record) => record.evidence_id === values["evidence-id"],
|
||||
);
|
||||
const operation =
|
||||
values.scenario === "native"
|
||||
? "inspect_binary_layout"
|
||||
: "analyze_javascript_application";
|
||||
assert.equal(source?.operation, operation);
|
||||
const unknownDomain = `retained-${values.scenario}`;
|
||||
const expectedUnknowns = retainedUnknownHeads(original.unknowns, unknownDomain);
|
||||
// These oracles check selected original facts, not producer discovery completeness.
|
||||
const expectedRows =
|
||||
values.scenario === "native"
|
||||
? source.normalized_result.symbols
|
||||
: source.normalized_result.graph.nodes
|
||||
.filter((node) =>
|
||||
["javascript-asset", "javascript-module", "source-module"].includes(
|
||||
node.kind,
|
||||
),
|
||||
)
|
||||
.map((node) => {
|
||||
const candidates = node.observations.flatMap((observation) => {
|
||||
const property = observation.properties.path;
|
||||
const where = observation.evidence.location;
|
||||
return [
|
||||
...(typeof property === "string" ? [property] : []),
|
||||
...(where.available && where.value.kind === "artifact-path"
|
||||
? [where.value.path]
|
||||
: where.available && where.value.kind === "source-range"
|
||||
? [where.value.source]
|
||||
: []),
|
||||
];
|
||||
});
|
||||
return {
|
||||
node_id: node.node_id,
|
||||
kind: node.kind,
|
||||
path:
|
||||
node.identity.strategy === "canonical-path"
|
||||
? node.identity.path
|
||||
: node.identity.strategy === "source-map-original"
|
||||
? node.identity.original_source
|
||||
: (candidates[0] ?? null),
|
||||
};
|
||||
});
|
||||
assert.ok(expectedRows.length >= 6, "This task needs six original rows");
|
||||
const runtimeEntrySha256 = digest(
|
||||
await readFile(join(root, "dist/mcp/main.js")),
|
||||
);
|
||||
const account = await mkdtemp(join(tmpdir(), "rea-retained-query-"));
|
||||
const transport = new StdioClientTransport({
|
||||
command: process.execPath,
|
||||
args: [join(root, "scripts/rea.mjs"), "mcp"],
|
||||
env: {
|
||||
PATH: process.env.PATH,
|
||||
HOME: account,
|
||||
NODE_OPTIONS: "--max-old-space-size=2048",
|
||||
REA_LOG_LEVEL: "silent",
|
||||
},
|
||||
stderr: "pipe",
|
||||
});
|
||||
const serverLog = createWriteStream(join(output, "server.log"));
|
||||
const serverLogFinished = finished(serverLog);
|
||||
serverLogFinished.catch(() => {});
|
||||
transport.stderr?.pipe(serverLog);
|
||||
const client = new Client({
|
||||
name: "retained-investigation-measurement",
|
||||
version: "1",
|
||||
});
|
||||
const report = {
|
||||
revision: values.revision,
|
||||
runtime_entry_sha256: runtimeEntrySha256,
|
||||
bundle_sha256: digest(sourceBytes),
|
||||
bundle_bytes: sourceBytes.length,
|
||||
source_evidence_id: source.evidence_id,
|
||||
scenario: values.scenario,
|
||||
delivery: values.delivery,
|
||||
verify_mixed: values["verify-mixed"],
|
||||
calls: [],
|
||||
status: "running",
|
||||
};
|
||||
const call = createRetainedQueryCaller(client, async (receipt, event) => {
|
||||
report.calls.push(receipt);
|
||||
await appendFile(
|
||||
join(output, "transcript.jsonl"),
|
||||
JSON.stringify(event) + "\n",
|
||||
);
|
||||
});
|
||||
const result = (reply, name) => requireMcpOperationResult(reply, name);
|
||||
try {
|
||||
await client.connect(transport);
|
||||
const started = performance.now();
|
||||
const catalog = await client.listTools();
|
||||
report.catalog = {
|
||||
elapsed_ms: performance.now() - started,
|
||||
tools: catalog.tools.length,
|
||||
bytes: Buffer.byteLength(JSON.stringify(catalog)),
|
||||
sha256: digest(JSON.stringify(catalog)),
|
||||
};
|
||||
assert.equal(catalog.nextCursor, undefined);
|
||||
await writeFile(join(output, "catalog.json"), JSON.stringify(catalog));
|
||||
result(
|
||||
await call("import_evidence_bundle", { path: values.bundle }, "import"),
|
||||
"import_evidence_bundle",
|
||||
);
|
||||
const answers = [];
|
||||
for (let repeat = 0; repeat < 3; repeat += 1) {
|
||||
let answer;
|
||||
if (values.delivery === "complete") {
|
||||
const reply = await call("get_evidence_bundle", {}, `complete-${repeat}`);
|
||||
if (reply.isError !== true) {
|
||||
const bundle = result(reply, "get_evidence_bundle");
|
||||
assert.deepEqual(
|
||||
bundle.records.find(
|
||||
(record) => record.evidence_id === source.evidence_id,
|
||||
),
|
||||
source,
|
||||
);
|
||||
assert.deepEqual(bundle.unknowns, original.unknowns);
|
||||
answer = [expectedRows.slice(0, 6), expectedUnknowns];
|
||||
} else {
|
||||
// Keep capacity refusals and every recovery RPC in the completed task.
|
||||
const failure = requireMcpToolError(reply);
|
||||
assert.equal(failure.code, "resource_constraint");
|
||||
assert.equal(failure.details?.resource, "transport");
|
||||
report.complete_delivery_refused = true;
|
||||
}
|
||||
}
|
||||
if (answer === undefined) {
|
||||
const metadata = result(
|
||||
await call(
|
||||
"get_evidence_bundle",
|
||||
{
|
||||
detail: "summary",
|
||||
filters: { evidence_id: source.evidence_id },
|
||||
},
|
||||
`discovery-${repeat}`,
|
||||
),
|
||||
"get_evidence_bundle",
|
||||
);
|
||||
assert.equal(metadata.matching_records, 1);
|
||||
assert.equal(metadata.records[0].evidence_id, source.evidence_id);
|
||||
const rows = [];
|
||||
for (const offset of [0, 2, 4]) {
|
||||
const view = result(
|
||||
await call(
|
||||
"inspect_analysis_view",
|
||||
{
|
||||
source: {
|
||||
kind: "retained-evidence",
|
||||
evidence_id: source.evidence_id,
|
||||
},
|
||||
view: {
|
||||
kind: "page",
|
||||
collection:
|
||||
values.scenario === "native" ? "symbols" : "modules",
|
||||
offset,
|
||||
limit: 2,
|
||||
},
|
||||
},
|
||||
`page-${repeat}-${offset}`,
|
||||
),
|
||||
"inspect_analysis_view",
|
||||
);
|
||||
assert.equal(view.parent_evidence_id, source.evidence_id);
|
||||
assert.equal(view.coverage.total, expectedRows.length);
|
||||
assert.equal(view.coverage.examined, 2);
|
||||
assert.equal(
|
||||
view.coverage.next_offset,
|
||||
offset + 2 < expectedRows.length ? offset + 2 : null,
|
||||
);
|
||||
assert.deepEqual(view.items, expectedRows.slice(offset, offset + 2));
|
||||
rows.push(...view.items);
|
||||
}
|
||||
const unknowns = [];
|
||||
let offset = 0;
|
||||
do {
|
||||
const page = result(
|
||||
await call(
|
||||
"list_unknowns",
|
||||
{ domain: unknownDomain, offset, limit: 2 },
|
||||
`unknowns-${repeat}-${offset}`,
|
||||
),
|
||||
"list_unknowns",
|
||||
);
|
||||
assert.equal(page.total, expectedUnknowns.length);
|
||||
assert.equal(page.offset, offset);
|
||||
assert.deepEqual(
|
||||
page.items,
|
||||
expectedUnknowns.slice(offset, offset + 2),
|
||||
);
|
||||
assert.equal(
|
||||
page.next_offset,
|
||||
offset + 2 < expectedUnknowns.length ? offset + 2 : null,
|
||||
);
|
||||
unknowns.push(...page.items);
|
||||
offset = page.next_offset;
|
||||
} while (offset !== null);
|
||||
assert.deepEqual(unknowns, expectedUnknowns);
|
||||
answer = [rows, unknowns];
|
||||
}
|
||||
answers.push(digest(canonicalize(answer)));
|
||||
}
|
||||
assert.equal(new Set(answers).size, 1);
|
||||
report.answer_sha256 = answers[0];
|
||||
if (values["verify-mixed"])
|
||||
report.mixed_verification = await verifyMixedRetainedHistory({
|
||||
call,
|
||||
original,
|
||||
sourceId: source.evidence_id,
|
||||
output,
|
||||
});
|
||||
const roundtrip = join(output, "roundtrip.json");
|
||||
result(
|
||||
await call("export_evidence_bundle", { path: roundtrip }, "export"),
|
||||
"export_evidence_bundle",
|
||||
);
|
||||
const exported = JSON.parse(await readFile(roundtrip, "utf8"));
|
||||
const exportedRecords = new Map(
|
||||
exported.records.map((record) => [record.evidence_id, record]),
|
||||
);
|
||||
for (const record of original.records)
|
||||
assert.deepEqual(exportedRecords.get(record.evidence_id), record);
|
||||
assert.deepEqual(exported.unknowns, original.unknowns);
|
||||
report.original_records_preserved = true;
|
||||
result(await call("close_binary", {}, "close"), "close_binary");
|
||||
const closed = result(
|
||||
await call("get_evidence_bundle", { detail: "summary" }, "after-close"),
|
||||
"get_evidence_bundle",
|
||||
);
|
||||
assert.equal(closed.total_retained_records, 0);
|
||||
report.close_cleared_records = true;
|
||||
if (values["verify-mixed"])
|
||||
Object.assign(
|
||||
report.mixed_verification,
|
||||
await verifyMixedRetainedClosure(call, source.evidence_id),
|
||||
);
|
||||
assert.equal(digest(await readFile(values.bundle)), report.bundle_sha256);
|
||||
report.status = "passed";
|
||||
} catch (cause) {
|
||||
report.status = "failed";
|
||||
report.failure = { name: cause.name, message: cause.message };
|
||||
} finally {
|
||||
const finalized = await finalizeRetainedQueryReceipt({
|
||||
report,
|
||||
client,
|
||||
transport,
|
||||
account,
|
||||
serverLog,
|
||||
serverLogFinished,
|
||||
output,
|
||||
});
|
||||
if (report.status !== "passed" || !finalized) process.exitCode = 1;
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
requireMcpOperationResult as result,
|
||||
requireMcpToolError,
|
||||
} from "../../scripts/lib/mcp-verifier-results.mjs";
|
||||
|
||||
/** Expected current heads from the original exported revisions, not displayed pages. */
|
||||
export const retainedUnknownHeads = (unknowns, domain) => {
|
||||
const heads = new Map();
|
||||
for (const unknown of unknowns) {
|
||||
const previous = heads.get(unknown.unknown_id);
|
||||
if (previous === undefined || previous.revision < unknown.revision)
|
||||
heads.set(unknown.unknown_id, unknown);
|
||||
}
|
||||
return [...heads.values()]
|
||||
.filter((unknown) => unknown.domain === domain)
|
||||
.sort((a, b) =>
|
||||
a.unknown_id < b.unknown_id ? -1 : a.unknown_id > b.unknown_id ? 1 : 0,
|
||||
);
|
||||
};
|
||||
|
||||
const verifyFollowupPages = async (call, original) => {
|
||||
const expected = retainedUnknownHeads(
|
||||
original.unknowns,
|
||||
"javascript-semantic-followup",
|
||||
);
|
||||
assert.ok(
|
||||
expected.length > 0,
|
||||
"Supply an authentic mixed history with semantic follow-ups",
|
||||
);
|
||||
const revised = expected.find((unknown) => unknown.revision > 1);
|
||||
assert.ok(revised, "Mixed verification requires a retained updated Unknown");
|
||||
let offset = 0;
|
||||
let pages = 0;
|
||||
const observed = [];
|
||||
do {
|
||||
const page = result(
|
||||
await call(
|
||||
"list_unknowns",
|
||||
{
|
||||
domain: "javascript-semantic-followup",
|
||||
offset,
|
||||
limit: 2,
|
||||
},
|
||||
`mixed-heads-${offset}`,
|
||||
),
|
||||
"list_unknowns",
|
||||
);
|
||||
assert.equal(page.total, expected.length);
|
||||
assert.equal(page.offset, offset);
|
||||
assert.deepEqual(page.items, expected.slice(offset, offset + 2));
|
||||
assert.equal(
|
||||
page.next_offset,
|
||||
offset + 2 < expected.length ? offset + 2 : null,
|
||||
);
|
||||
observed.push(...page.items);
|
||||
offset = page.next_offset;
|
||||
pages += 1;
|
||||
} while (offset !== null);
|
||||
assert.deepEqual(observed, expected);
|
||||
return { revised, headsChecked: expected.length, pages };
|
||||
};
|
||||
|
||||
const verifyMutationRefusals = async ({
|
||||
call,
|
||||
original,
|
||||
sourceId,
|
||||
output,
|
||||
revised,
|
||||
}) => {
|
||||
const before = result(
|
||||
await call(
|
||||
"get_evidence_bundle",
|
||||
{ detail: "summary" },
|
||||
"mixed-before-refusals",
|
||||
),
|
||||
"get_evidence_bundle",
|
||||
);
|
||||
const stale = await call(
|
||||
"update_unknown",
|
||||
{
|
||||
unknown_id: revised.unknown_id,
|
||||
expected_revision: revised.revision - 1,
|
||||
status: revised.status,
|
||||
severity: revised.severity,
|
||||
supporting_evidence_ids: revised.supporting_evidence_ids,
|
||||
contradicting_evidence_ids: revised.contradicting_evidence_ids,
|
||||
required_authority: revised.required_authority,
|
||||
required_confidence: revised.required_confidence,
|
||||
required_environment: revised.required_environment,
|
||||
recommended_probes: revised.recommended_probes,
|
||||
relationships: revised.relationships,
|
||||
resolution: revised.resolution,
|
||||
},
|
||||
"mixed-stale-update",
|
||||
);
|
||||
assert.equal(requireMcpToolError(stale).code, "revision_conflict");
|
||||
const tampered = structuredClone(original);
|
||||
const target = tampered.records.find(
|
||||
(record) => record.evidence_id === sourceId,
|
||||
);
|
||||
assert.ok(target);
|
||||
target.normalized_result = { checkpoint_tampered: true };
|
||||
const path = join(output, "tampered-import.json");
|
||||
await writeFile(path, JSON.stringify(tampered), { flag: "wx" });
|
||||
const rejected = await call(
|
||||
"import_evidence_bundle",
|
||||
{ path },
|
||||
"mixed-tampered-import",
|
||||
);
|
||||
assert.equal(
|
||||
requireMcpToolError(rejected).code,
|
||||
"evidence_integrity_mismatch",
|
||||
);
|
||||
const after = result(
|
||||
await call(
|
||||
"get_evidence_bundle",
|
||||
{ detail: "summary" },
|
||||
"mixed-after-refusals",
|
||||
),
|
||||
"get_evidence_bundle",
|
||||
);
|
||||
assert.deepEqual(after, before);
|
||||
};
|
||||
|
||||
/** Verify original mixed-history heads, CAS rejection and atomic import failure. */
|
||||
export const verifyMixedRetainedHistory = async (options) => {
|
||||
const { revised, headsChecked, pages } = await verifyFollowupPages(
|
||||
options.call,
|
||||
options.original,
|
||||
);
|
||||
await verifyMutationRefusals({ ...options, revised });
|
||||
return {
|
||||
heads_checked: headsChecked,
|
||||
pages,
|
||||
revised_unknown_id: revised.unknown_id,
|
||||
revision_checked: revised.revision,
|
||||
stale_update_rejected: true,
|
||||
tampered_import_rejected_atomically: true,
|
||||
};
|
||||
};
|
||||
|
||||
/** Closing must neither retain old Unknowns nor rehydrate a closed source. */
|
||||
export const verifyMixedRetainedClosure = async (call, sourceId) => {
|
||||
const unknowns = result(
|
||||
await call("list_unknowns", {}, "mixed-after-close-heads"),
|
||||
"list_unknowns",
|
||||
);
|
||||
assert.equal(unknowns.total, 0);
|
||||
const missing = await call(
|
||||
"inspect_analysis_view",
|
||||
{
|
||||
source: { kind: "retained-evidence", evidence_id: sourceId },
|
||||
view: { kind: "summary" },
|
||||
},
|
||||
"mixed-after-close-reference",
|
||||
);
|
||||
const failure = requireMcpToolError(missing);
|
||||
assert.equal(failure.details?.reason, "missing");
|
||||
return { unknowns_after_close: 0, old_reference_refused: true };
|
||||
};
|
||||
@@ -0,0 +1,46 @@
|
||||
/** Keep settled and rejected RPCs in the same receipt before returning control. */
|
||||
export const createRetainedQueryCaller =
|
||||
(client, record) => async (name, input, phase) => {
|
||||
const start = performance.now();
|
||||
let reply;
|
||||
try {
|
||||
reply = await client.callTool(
|
||||
{ name, arguments: input },
|
||||
{ timeout: 300000 },
|
||||
);
|
||||
} catch (cause) {
|
||||
const failure =
|
||||
cause instanceof Error
|
||||
? { name: cause.name, message: cause.message }
|
||||
: { name: typeof cause, message: String(cause) };
|
||||
const receipt = {
|
||||
name,
|
||||
input,
|
||||
phase,
|
||||
elapsed_ms: performance.now() - start,
|
||||
reply_bytes: null,
|
||||
is_error: true,
|
||||
failure,
|
||||
};
|
||||
try {
|
||||
await record(receipt, { name, input, phase, failure });
|
||||
} catch (recordingFailure) {
|
||||
throw new AggregateError(
|
||||
[cause, recordingFailure],
|
||||
"MCP call and receipt recording both failed",
|
||||
);
|
||||
}
|
||||
throw cause;
|
||||
}
|
||||
const receipt = {
|
||||
name,
|
||||
input,
|
||||
phase,
|
||||
elapsed_ms: performance.now() - start,
|
||||
reply_bytes: Buffer.byteLength(JSON.stringify(reply)),
|
||||
is_error: reply.isError === true,
|
||||
};
|
||||
// A receipt-write failure is distinct from a rejected MCP call.
|
||||
await record(receipt, { name, input, phase, reply });
|
||||
return reply;
|
||||
};
|
||||
@@ -0,0 +1,81 @@
|
||||
import { rename, rm, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
const attempt = async (failures, phase, action) => {
|
||||
try {
|
||||
await action();
|
||||
return true;
|
||||
} catch (cause) {
|
||||
failures.push(`${phase}: ${String(cause)}`);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
const writeReceipt = (stream, text) =>
|
||||
new Promise((resolve, reject) => {
|
||||
const onError = (cause) => reject(cause);
|
||||
stream.once("error", onError);
|
||||
try {
|
||||
stream.write(text, (cause) => {
|
||||
if (cause !== null && cause !== undefined) {
|
||||
// A Writable emits its error after notifying the write callback.
|
||||
reject(cause);
|
||||
} else {
|
||||
stream.off("error", onError);
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
} catch (cause) {
|
||||
stream.off("error", onError);
|
||||
reject(cause);
|
||||
}
|
||||
});
|
||||
|
||||
/** Preserve task costs through independent cleanup and receipt-output failures. */
|
||||
export const finalizeRetainedQueryReceipt = async ({
|
||||
report,
|
||||
client,
|
||||
transport,
|
||||
account,
|
||||
serverLog,
|
||||
serverLogFinished,
|
||||
output,
|
||||
stdout = process.stdout,
|
||||
}) => {
|
||||
report.cleanup_failures = [];
|
||||
await Promise.all([
|
||||
attempt(report.cleanup_failures, "client", () => client.close()),
|
||||
attempt(report.cleanup_failures, "transport", () => transport.close()),
|
||||
]);
|
||||
await attempt(report.cleanup_failures, "server-log", async () => {
|
||||
if (!serverLog.writableEnded) serverLog.end();
|
||||
await serverLogFinished;
|
||||
});
|
||||
if (report.cleanup_failures.length === 0)
|
||||
await attempt(report.cleanup_failures, "workspace", () =>
|
||||
rm(account, { recursive: true }),
|
||||
);
|
||||
if (report.cleanup_failures.length > 0) report.retained_workspace = account;
|
||||
|
||||
report.receipt_failures = [];
|
||||
let publication = 0;
|
||||
const saveReport = async () => {
|
||||
// Keep failed publications in the owned output directory for inspection.
|
||||
// A partial retry must never truncate an already published cost receipt.
|
||||
const pending = join(output, `.report-${++publication}.json.pending`);
|
||||
await writeFile(pending, JSON.stringify(report, null, 2) + "\n", {
|
||||
flag: "wx",
|
||||
});
|
||||
await rename(pending, join(output, "report.json"));
|
||||
};
|
||||
await attempt(report.receipt_failures, "report-file", saveReport);
|
||||
const stdoutWritten = await attempt(report.receipt_failures, "stdout", () =>
|
||||
writeReceipt(stdout, JSON.stringify(report) + "\n"),
|
||||
);
|
||||
// Preserve a broken stdout diagnostic in the file that may already exist.
|
||||
if (!stdoutWritten)
|
||||
await attempt(report.receipt_failures, "report-file-retry", saveReport);
|
||||
return (
|
||||
report.cleanup_failures.length === 0 && report.receipt_failures.length === 0
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,166 @@
|
||||
"""Deterministic RPC receipt failures; not real-provider or model evidence."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import unittest
|
||||
|
||||
|
||||
class RetainedQueryReceipts(unittest.TestCase):
|
||||
def run_node(self, body):
|
||||
module = Path(__file__).with_name('retained-query-calls.mjs').resolve().as_uri()
|
||||
finalizer = Path(__file__).with_name('retained-query-finalization.mjs').resolve().as_uri()
|
||||
source = ('import assert from "node:assert/strict";\n'
|
||||
'const {createRetainedQueryCaller} = await import(' + json.dumps(module) + ');\n'
|
||||
'const {finalizeRetainedQueryReceipt} = await import(' + json.dumps(finalizer) + ');\n' + body)
|
||||
result = subprocess.run(['node', '--input-type=module', '-'], input=source,
|
||||
text=True, capture_output=True, timeout=15)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
|
||||
def run_finalization(self, body):
|
||||
self.run_node('''
|
||||
import {mkdtemp,mkdir,readFile,writeFile,rm,stat} from 'node:fs/promises';
|
||||
import {join} from 'node:path';
|
||||
import {tmpdir} from 'node:os';
|
||||
import {Writable} from 'node:stream';
|
||||
import {finished} from 'node:stream/promises';
|
||||
const root=await mkdtemp(join(tmpdir(),'rea-receipt-test-'));
|
||||
const account=join(root,'account');const output=join(root,'output');
|
||||
await mkdir(account);await mkdir(output);
|
||||
const chunks=[];
|
||||
const stdout=new Writable({write(chunk,encoding,done){chunks.push(chunk.toString());done();}});
|
||||
const serverLog=new Writable({write(chunk,encoding,done){done();}});
|
||||
const serverLogFinished=finished(serverLog);serverLogFinished.catch(()=>{});
|
||||
const originalFailure={name:'Error',message:'original request failed'};
|
||||
const report={status:'failed',failure:originalFailure,calls:[{phase:'page',elapsed_ms:17}]};
|
||||
const input={report,client:{close:async()=>{}},transport:{close:async()=>{}},
|
||||
account,output,stdout,serverLog,serverLogFinished};
|
||||
const saved=async()=>JSON.parse(await readFile(join(output,'report.json'),'utf8'));
|
||||
try {
|
||||
''' + body + '''
|
||||
} finally {
|
||||
serverLog.destroy();await serverLogFinished.catch(()=>{});stdout.destroy();
|
||||
await rm(root,{recursive:true,force:true});
|
||||
}
|
||||
''')
|
||||
|
||||
def test_rejected_call_preserves_request_phase_cost_and_original_failure(self):
|
||||
self.run_node('''
|
||||
const failure = new Error('owned server closed');
|
||||
const input = {offset:0, limit:2}; const rows=[]; const events=[];
|
||||
const call = createRetainedQueryCaller({callTool:async request=>{
|
||||
assert.strictEqual(request.arguments,input);throw failure;
|
||||
}}, async(row,event)=>{rows.push(row);events.push(event);});
|
||||
await assert.rejects(call('list_unknowns',input,'resume-2'), e=>e===failure);
|
||||
assert.equal(rows.length,1);assert.equal(events.length,1);
|
||||
assert.strictEqual(rows[0].input,input);assert.equal(rows[0].name,'list_unknowns');
|
||||
assert.equal(rows[0].phase,'resume-2');assert.ok(rows[0].elapsed_ms>=0);
|
||||
assert.equal(rows[0].reply_bytes,null);assert.equal(rows[0].is_error,true);
|
||||
assert.deepEqual(events[0].failure,{name:'Error',message:'owned server closed'});
|
||||
''')
|
||||
|
||||
def test_protocol_error_is_a_received_reply_with_known_bytes(self):
|
||||
self.run_node('''
|
||||
const reply={isError:true,content:[{type:'text',text:'capacity refusal'}]};const rows=[];
|
||||
const call=createRetainedQueryCaller({callTool:async()=>reply},async row=>rows.push(row));
|
||||
assert.strictEqual(await call('get_evidence_bundle',{},'complete'),reply);
|
||||
assert.equal(rows.length,1);assert.equal(rows[0].is_error,true);
|
||||
assert.equal(rows[0].reply_bytes,Buffer.byteLength(JSON.stringify(reply)));
|
||||
assert.equal(Object.hasOwn(rows[0],'failure'),false);
|
||||
''')
|
||||
|
||||
def test_receipt_write_error_does_not_reclassify_a_completed_rpc(self):
|
||||
self.run_node('''
|
||||
const failure=new Error('receipt disk unavailable');const rows=[];
|
||||
const call=createRetainedQueryCaller({callTool:async()=>({content:[]})},async row=>{
|
||||
rows.push(row);throw failure;
|
||||
});
|
||||
await assert.rejects(call('list_unknowns',{},'read'),e=>e===failure);
|
||||
assert.equal(rows.length,1);assert.equal(rows[0].is_error,false);
|
||||
assert.equal(Object.hasOwn(rows[0],'failure'),false);
|
||||
''')
|
||||
|
||||
def test_failed_recording_preserves_the_prior_rpc_failure(self):
|
||||
self.run_node('''
|
||||
const rpc=new Error('request timeout');const disk=new Error('receipt close failed');
|
||||
const call=createRetainedQueryCaller({callTool:async()=>{throw rpc;}},async()=>{throw disk;});
|
||||
await assert.rejects(call('inspect_analysis_view',{},'selected'),e=>
|
||||
e instanceof AggregateError && e.errors[0]===rpc && e.errors[1]===disk);
|
||||
''')
|
||||
|
||||
def test_log_stream_failure_keeps_costs_both_receipts_and_workspace(self):
|
||||
self.run_finalization('''
|
||||
const log=new Writable({write(chunk,encoding,done){done(new Error('log disk failed'));}});
|
||||
const logFinished=finished(log);logFinished.catch(()=>{});log.write('diagnostic');
|
||||
assert.equal(await finalizeRetainedQueryReceipt({...input,serverLog:log,
|
||||
serverLogFinished:logFinished}),false);
|
||||
assert.match(report.cleanup_failures[0],/server-log:.*log disk failed/);
|
||||
assert.strictEqual(report.failure,originalFailure);
|
||||
assert.equal(report.calls[0].elapsed_ms,17);assert.equal(report.retained_workspace,account);
|
||||
assert.equal((await stat(account)).isDirectory(),true);
|
||||
assert.deepEqual(await saved(),report);assert.deepEqual(JSON.parse(chunks.join('')),report);
|
||||
''')
|
||||
|
||||
def test_workspace_removal_error_still_writes_the_original_failure(self):
|
||||
self.run_finalization('''
|
||||
const blocked=join(root,'owned-file');await writeFile(blocked,'owned');
|
||||
const malformedAccount=join(blocked,'child');
|
||||
assert.equal(await finalizeRetainedQueryReceipt({...input,account:malformedAccount}),false);
|
||||
assert.match(report.cleanup_failures[0],/^workspace:/);
|
||||
assert.equal(report.retained_workspace,malformedAccount);
|
||||
assert.deepEqual((await saved()).failure,originalFailure);
|
||||
assert.deepEqual(JSON.parse(chunks.join('')),report);
|
||||
''')
|
||||
|
||||
def test_report_file_refusal_preserves_the_stdout_receipt(self):
|
||||
self.run_finalization('''
|
||||
await mkdir(join(output,'report.json'));
|
||||
assert.equal(await finalizeRetainedQueryReceipt(input),false);
|
||||
assert.deepEqual(report.cleanup_failures,[]);
|
||||
assert.match(report.receipt_failures[0],/^report-file:/);
|
||||
assert.deepEqual(JSON.parse(chunks.join('')),report);
|
||||
assert.deepEqual(report.failure,originalFailure);
|
||||
''')
|
||||
|
||||
def test_stdout_write_error_is_handled_and_saved_in_the_file(self):
|
||||
self.run_finalization('''
|
||||
const failedStdout=new Writable({write(chunk,encoding,done){done(new Error('broken pipe'));}});
|
||||
assert.equal(await finalizeRetainedQueryReceipt({...input,stdout:failedStdout}),false);
|
||||
assert.match(report.receipt_failures[0],/stdout:.*broken pipe/);
|
||||
assert.deepEqual(await saved(),report);assert.deepEqual(report.failure,originalFailure);
|
||||
''')
|
||||
|
||||
def test_partial_retry_after_stdout_failure_preserves_the_prior_receipt(self):
|
||||
self.run_finalization('''
|
||||
const fs=await import('node:fs/promises');
|
||||
const {syncBuiltinESMExports}=await import('node:module');
|
||||
const originalWrite=fs.default.writeFile;let writes=0;let before;
|
||||
fs.default.writeFile=async(...args)=>{
|
||||
if(++writes===2){await originalWrite(args[0],'partial',args[2]);throw new Error('disk quota');}
|
||||
return originalWrite(...args);
|
||||
};
|
||||
syncBuiltinESMExports();
|
||||
try {
|
||||
const failedStdout=new Writable({async write(chunk,encoding,done){
|
||||
before=await readFile(join(output,'report.json'),'utf8');done(new Error('broken pipe'));
|
||||
}});
|
||||
assert.equal(await finalizeRetainedQueryReceipt({...input,stdout:failedStdout}),false);
|
||||
assert.equal(writes,2);assert.equal(report.receipt_failures.length,2);
|
||||
assert.match(report.receipt_failures[1],/report-file-retry:.*disk quota/);
|
||||
assert.equal(await readFile(join(output,'report.json'),'utf8'),before);
|
||||
assert.deepEqual((await saved()).failure,originalFailure);
|
||||
assert.equal((await saved()).calls[0].elapsed_ms,17);
|
||||
} finally {fs.default.writeFile=originalWrite;syncBuiltinESMExports();}
|
||||
''')
|
||||
|
||||
def test_failed_client_close_keeps_ownership_and_attempts_peer_cleanup(self):
|
||||
self.run_finalization('''
|
||||
const closed=[];
|
||||
assert.equal(await finalizeRetainedQueryReceipt({...input,
|
||||
client:{close(){closed.push('client');throw new Error('close failed');}},
|
||||
transport:{close:async()=>{closed.push('transport');}}}),false);
|
||||
assert.deepEqual(closed,['client','transport']);
|
||||
assert.match(report.cleanup_failures[0],/client:.*close failed/);
|
||||
assert.equal(report.retained_workspace,account);assert.equal(serverLog.writableEnded,true);
|
||||
assert.equal((await stat(account)).isDirectory(),true);
|
||||
assert.deepEqual(await saved(),report);assert.deepEqual(JSON.parse(chunks.join('')),report);
|
||||
''')
|
||||
Reference in new issue
Block a user