From 4e716b06078d725a53234503cbefbb340731e809 Mon Sep 17 00:00:00 2001 From: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com> Date: Tue, 20 Oct 2026 11:32:18 +0800 Subject: [PATCH] perf(analysis-view): reuse validated immutable captures (#5349) Reuse the format owner's full validation for exact authenticated immutable captures across retained queries. Preserve ordinary mutable-input validation and per-request subject, coverage and budget checks. Refs #3133. Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> --- experiments/analysis-efficiency/README.md | 5 + .../retained-capture-validation.json | 894 ++++++++++++++++++ .../retained-capture-validation.md | 238 +++++ .../retained-capture-validation.mjs | 251 +++++ .../WebNetworkProjectionAccumulator.ts | 4 +- .../webNetworkCaptureView.test.ts | 24 + .../analysisView/webNetworkCaptureView.ts | 4 +- src/domain/webNetworkCapture.test.ts | 118 +++ src/domain/webNetworkCapture.ts | 23 + 9 files changed, 1557 insertions(+), 4 deletions(-) create mode 100644 experiments/analysis-efficiency/retained-capture-validation.json create mode 100644 experiments/analysis-efficiency/retained-capture-validation.md create mode 100644 experiments/analysis-efficiency/retained-capture-validation.mjs create mode 100644 src/domain/webNetworkCapture.test.ts diff --git a/experiments/analysis-efficiency/README.md b/experiments/analysis-efficiency/README.md index 5330f13574..3d9b728aa5 100644 --- a/experiments/analysis-efficiency/README.md +++ b/experiments/analysis-efficiency/README.md @@ -1,5 +1,10 @@ # Analysis efficiency measurements +The [retained-capture validation checkpoint](retained-capture-validation.md) +reconciles the earlier audit queue and measures immutable historical-capture +validation reuse on public compact and large qlogs, including first-view costs, +small-input limits and unchanged Evidence delivery. + For Hopper literal string queries, see the [decoded-record checkpoint](hopper-string-records.md). It measures record reuse through real Linux CLI/MCP workflows, preserving native recognition gaps, guard-stopped attempts and negative whole-task costs. diff --git a/experiments/analysis-efficiency/retained-capture-validation.json b/experiments/analysis-efficiency/retained-capture-validation.json new file mode 100644 index 0000000000..18737ec4a8 --- /dev/null +++ b/experiments/analysis-efficiency/retained-capture-validation.json @@ -0,0 +1,894 @@ +{ + "baseline_revision": "5fa4defcbc0e55ef1ec84c888b918abe945303fe", + "baseline_runtime_sha256": "8068d5a85e07880998f232ed9f02ba8f5a4bd914164e4097e00104213c05a1c3", + "candidate_measured_runtime_sha256": "3de94a0ca596c96c48e9f5f52cd7c064f5bfccaece38166550d7fec884ab2f04", + "cohorts": [ + { + "name": "baseline-compact-import-1", + "capture_sha256": "2dbe00348c2f6fcd81f829564718e276052d4aaea77f2a6745cbd147869aec6f", + "events": 132, + "workflow_ms": 5249.381685, + "scope_seconds": 5.818, + "payload_cpu_seconds": 5.6285, + "payload_family_peak_bytes": 692121600, + "first_view_ms": 1354.982591, + "warm_14_queries_ms": 174.149147000001, + "catalog": { + "elapsed_ms": 1247.9706180000003, + "sha256": "655cb1871fc82a011ea15352bfd1856e28d6d0171269951d3f3a1e1bd694e74f", + "bytes": 3254305 + }, + "source_id": "ev_df45171a0cf0d3c691124f0cc44a8d6b38e0e8c9d3d298ea611ffdac084c304e", + "answers_sha256": "641f85866e87c2768b4cf56911640e4f9e3f3340ac44cd7a468c5329fdd9448c", + "original_record_sha256": "6c9f269acba239804406d278cfc3cce1fd7d0880b312c785df3a791a2c157a4d", + "call_count": 17, + "rpc_reply_bytes": 161234, + "query_reply_bytes": 160659, + "resource_events_at_payload_exit": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "final_whole_scope_counters_verified": false, + "abort": null + }, + { + "name": "baseline-compact-import-2", + "capture_sha256": "2dbe00348c2f6fcd81f829564718e276052d4aaea77f2a6745cbd147869aec6f", + "events": 132, + "workflow_ms": 5668.385286000001, + "scope_seconds": 6.178, + "payload_cpu_seconds": 5.960291, + "payload_family_peak_bytes": 664920064, + "first_view_ms": 1400.6959980000001, + "warm_14_queries_ms": 183.19255900000007, + "catalog": { + "elapsed_ms": 1496.2133520000002, + "sha256": "655cb1871fc82a011ea15352bfd1856e28d6d0171269951d3f3a1e1bd694e74f", + "bytes": 3254305 + }, + "source_id": "ev_df45171a0cf0d3c691124f0cc44a8d6b38e0e8c9d3d298ea611ffdac084c304e", + "answers_sha256": "641f85866e87c2768b4cf56911640e4f9e3f3340ac44cd7a468c5329fdd9448c", + "original_record_sha256": "6c9f269acba239804406d278cfc3cce1fd7d0880b312c785df3a791a2c157a4d", + "call_count": 17, + "rpc_reply_bytes": 161234, + "query_reply_bytes": 160659, + "resource_events_at_payload_exit": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "final_whole_scope_counters_verified": false, + "abort": null + }, + { + "name": "baseline-compact-import-3", + "capture_sha256": "2dbe00348c2f6fcd81f829564718e276052d4aaea77f2a6745cbd147869aec6f", + "events": 132, + "workflow_ms": 6465.653956, + "scope_seconds": 6.988, + "payload_cpu_seconds": 6.788176, + "payload_family_peak_bytes": 690696192, + "first_view_ms": 1959.8266800000001, + "warm_14_queries_ms": 271.20134300000063, + "catalog": { + "elapsed_ms": 1479.3080370000002, + "sha256": "655cb1871fc82a011ea15352bfd1856e28d6d0171269951d3f3a1e1bd694e74f", + "bytes": 3254305 + }, + "source_id": "ev_df45171a0cf0d3c691124f0cc44a8d6b38e0e8c9d3d298ea611ffdac084c304e", + "answers_sha256": "641f85866e87c2768b4cf56911640e4f9e3f3340ac44cd7a468c5329fdd9448c", + "original_record_sha256": "6c9f269acba239804406d278cfc3cce1fd7d0880b312c785df3a791a2c157a4d", + "call_count": 17, + "rpc_reply_bytes": 161234, + "query_reply_bytes": 160659, + "resource_events_at_payload_exit": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "final_whole_scope_counters_verified": false, + "abort": null + }, + { + "name": "baseline-large-import-1", + "capture_sha256": "4854882fdeec3d66701779cfde1c614a09e3d5e3b90afdea3fc3d2566a0e6e3f", + "events": 30753, + "workflow_ms": 19253.191338, + "scope_seconds": 19.884, + "payload_cpu_seconds": 19.622432, + "payload_family_peak_bytes": 1791569920, + "first_view_ms": 3034.7453300000016, + "warm_14_queries_ms": 6399.140819999997, + "catalog": { + "elapsed_ms": 1299.400326, + "sha256": "655cb1871fc82a011ea15352bfd1856e28d6d0171269951d3f3a1e1bd694e74f", + "bytes": 3254305 + }, + "source_id": "ev_57091094e6142a58af3b1716e8735a963d451a81eb79f214679542ca65e12022", + "answers_sha256": "263db399005814e40a7e962f5a46d3eff95e4372cd541fe3241d050be41d3d60", + "original_record_sha256": "46a8a61e38a78cf2000c5cda35695bef7c8198776be6d8a3ae54853a5f003a8a", + "call_count": 17, + "rpc_reply_bytes": 161114, + "query_reply_bytes": 160539, + "resource_events_at_payload_exit": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "final_whole_scope_counters_verified": false, + "abort": null + }, + { + "name": "baseline-large-import-2", + "capture_sha256": "4854882fdeec3d66701779cfde1c614a09e3d5e3b90afdea3fc3d2566a0e6e3f", + "events": 30753, + "workflow_ms": 19464.788056999998, + "scope_seconds": 20.089, + "payload_cpu_seconds": 19.791543, + "payload_family_peak_bytes": 1783373824, + "first_view_ms": 2902.7044369999985, + "warm_14_queries_ms": 6482.219604, + "catalog": { + "elapsed_ms": 1361.999307, + "sha256": "655cb1871fc82a011ea15352bfd1856e28d6d0171269951d3f3a1e1bd694e74f", + "bytes": 3254305 + }, + "source_id": "ev_57091094e6142a58af3b1716e8735a963d451a81eb79f214679542ca65e12022", + "answers_sha256": "263db399005814e40a7e962f5a46d3eff95e4372cd541fe3241d050be41d3d60", + "original_record_sha256": "46a8a61e38a78cf2000c5cda35695bef7c8198776be6d8a3ae54853a5f003a8a", + "call_count": 17, + "rpc_reply_bytes": 161114, + "query_reply_bytes": 160539, + "resource_events_at_payload_exit": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "final_whole_scope_counters_verified": false, + "abort": null + }, + { + "name": "baseline-large-import-3", + "capture_sha256": "4854882fdeec3d66701779cfde1c614a09e3d5e3b90afdea3fc3d2566a0e6e3f", + "events": 30753, + "workflow_ms": 19570.145603, + "scope_seconds": 20.181, + "payload_cpu_seconds": 19.910178, + "payload_family_peak_bytes": 1784766464, + "first_view_ms": 2891.966555000001, + "warm_14_queries_ms": 6393.723106999998, + "catalog": { + "elapsed_ms": 1375.4531000000002, + "sha256": "655cb1871fc82a011ea15352bfd1856e28d6d0171269951d3f3a1e1bd694e74f", + "bytes": 3254305 + }, + "source_id": "ev_57091094e6142a58af3b1716e8735a963d451a81eb79f214679542ca65e12022", + "answers_sha256": "263db399005814e40a7e962f5a46d3eff95e4372cd541fe3241d050be41d3d60", + "original_record_sha256": "46a8a61e38a78cf2000c5cda35695bef7c8198776be6d8a3ae54853a5f003a8a", + "call_count": 17, + "rpc_reply_bytes": 161114, + "query_reply_bytes": 160539, + "resource_events_at_payload_exit": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "final_whole_scope_counters_verified": false, + "abort": null + }, + { + "name": "candidate-compact-import-1", + "capture_sha256": "2dbe00348c2f6fcd81f829564718e276052d4aaea77f2a6745cbd147869aec6f", + "events": 132, + "workflow_ms": 5479.5708540000005, + "scope_seconds": 6.08, + "payload_cpu_seconds": 5.85, + "payload_family_peak_bytes": 693051392, + "first_view_ms": 1621.3823879999995, + "warm_14_queries_ms": 106.31982299999981, + "catalog": { + "elapsed_ms": 1213.6661829999998, + "sha256": "655cb1871fc82a011ea15352bfd1856e28d6d0171269951d3f3a1e1bd694e74f", + "bytes": 3254305 + }, + "source_id": "ev_df45171a0cf0d3c691124f0cc44a8d6b38e0e8c9d3d298ea611ffdac084c304e", + "answers_sha256": "641f85866e87c2768b4cf56911640e4f9e3f3340ac44cd7a468c5329fdd9448c", + "original_record_sha256": "6c9f269acba239804406d278cfc3cce1fd7d0880b312c785df3a791a2c157a4d", + "call_count": 17, + "rpc_reply_bytes": 161236, + "query_reply_bytes": 160659, + "resource_events_at_payload_exit": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "final_whole_scope_counters_verified": false, + "abort": null + }, + { + "name": "candidate-compact-import-2", + "capture_sha256": "2dbe00348c2f6fcd81f829564718e276052d4aaea77f2a6745cbd147869aec6f", + "events": 132, + "workflow_ms": 5673.527286, + "scope_seconds": 6.272, + "payload_cpu_seconds": 6.059065, + "payload_family_peak_bytes": 694616064, + "first_view_ms": 1462.453802, + "warm_14_queries_ms": 108.05108499999824, + "catalog": { + "elapsed_ms": 1309.1064109999998, + "sha256": "655cb1871fc82a011ea15352bfd1856e28d6d0171269951d3f3a1e1bd694e74f", + "bytes": 3254305 + }, + "source_id": "ev_df45171a0cf0d3c691124f0cc44a8d6b38e0e8c9d3d298ea611ffdac084c304e", + "answers_sha256": "641f85866e87c2768b4cf56911640e4f9e3f3340ac44cd7a468c5329fdd9448c", + "original_record_sha256": "6c9f269acba239804406d278cfc3cce1fd7d0880b312c785df3a791a2c157a4d", + "call_count": 17, + "rpc_reply_bytes": 161236, + "query_reply_bytes": 160659, + "resource_events_at_payload_exit": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "final_whole_scope_counters_verified": false, + "abort": null + }, + { + "name": "candidate-compact-import-3", + "capture_sha256": "2dbe00348c2f6fcd81f829564718e276052d4aaea77f2a6745cbd147869aec6f", + "events": 132, + "workflow_ms": 5393.434348000001, + "scope_seconds": 5.976, + "payload_cpu_seconds": 5.690881, + "payload_family_peak_bytes": 681385984, + "first_view_ms": 1448.6816159999994, + "warm_14_queries_ms": 131.2696420000002, + "catalog": { + "elapsed_ms": 1323.0185740000002, + "sha256": "655cb1871fc82a011ea15352bfd1856e28d6d0171269951d3f3a1e1bd694e74f", + "bytes": 3254305 + }, + "source_id": "ev_df45171a0cf0d3c691124f0cc44a8d6b38e0e8c9d3d298ea611ffdac084c304e", + "answers_sha256": "641f85866e87c2768b4cf56911640e4f9e3f3340ac44cd7a468c5329fdd9448c", + "original_record_sha256": "6c9f269acba239804406d278cfc3cce1fd7d0880b312c785df3a791a2c157a4d", + "call_count": 17, + "rpc_reply_bytes": 161236, + "query_reply_bytes": 160659, + "resource_events_at_payload_exit": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "final_whole_scope_counters_verified": false, + "abort": null + }, + { + "name": "candidate-large-import-1", + "capture_sha256": "4854882fdeec3d66701779cfde1c614a09e3d5e3b90afdea3fc3d2566a0e6e3f", + "events": 30753, + "workflow_ms": 14052.698892, + "scope_seconds": 14.742, + "payload_cpu_seconds": 14.496087, + "payload_family_peak_bytes": 1173385216, + "first_view_ms": 4077.395645999999, + "warm_14_queries_ms": 113.30538300000626, + "catalog": { + "elapsed_ms": 1469.9327679999997, + "sha256": "655cb1871fc82a011ea15352bfd1856e28d6d0171269951d3f3a1e1bd694e74f", + "bytes": 3254305 + }, + "source_id": "ev_57091094e6142a58af3b1716e8735a963d451a81eb79f214679542ca65e12022", + "answers_sha256": "263db399005814e40a7e962f5a46d3eff95e4372cd541fe3241d050be41d3d60", + "original_record_sha256": "46a8a61e38a78cf2000c5cda35695bef7c8198776be6d8a3ae54853a5f003a8a", + "call_count": 17, + "rpc_reply_bytes": 161116, + "query_reply_bytes": 160539, + "resource_events_at_payload_exit": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "final_whole_scope_counters_verified": false, + "abort": null + }, + { + "name": "candidate-large-import-2", + "capture_sha256": "4854882fdeec3d66701779cfde1c614a09e3d5e3b90afdea3fc3d2566a0e6e3f", + "events": 30753, + "workflow_ms": 13581.232348000001, + "scope_seconds": 14.135, + "payload_cpu_seconds": 13.936527, + "payload_family_peak_bytes": 1177927680, + "first_view_ms": 3845.8420860000006, + "warm_14_queries_ms": 114.86873200000264, + "catalog": { + "elapsed_ms": 1438.3021010000002, + "sha256": "655cb1871fc82a011ea15352bfd1856e28d6d0171269951d3f3a1e1bd694e74f", + "bytes": 3254305 + }, + "source_id": "ev_57091094e6142a58af3b1716e8735a963d451a81eb79f214679542ca65e12022", + "answers_sha256": "263db399005814e40a7e962f5a46d3eff95e4372cd541fe3241d050be41d3d60", + "original_record_sha256": "46a8a61e38a78cf2000c5cda35695bef7c8198776be6d8a3ae54853a5f003a8a", + "call_count": 17, + "rpc_reply_bytes": 161116, + "query_reply_bytes": 160539, + "resource_events_at_payload_exit": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "final_whole_scope_counters_verified": false, + "abort": null + }, + { + "name": "candidate-large-import-3", + "capture_sha256": "4854882fdeec3d66701779cfde1c614a09e3d5e3b90afdea3fc3d2566a0e6e3f", + "events": 30753, + "workflow_ms": 14680.945012, + "scope_seconds": 15.248, + "payload_cpu_seconds": 15.020055, + "payload_family_peak_bytes": 1201561600, + "first_view_ms": 4190.895545000001, + "warm_14_queries_ms": 110.60082600000169, + "catalog": { + "elapsed_ms": 1362.8905800000002, + "sha256": "655cb1871fc82a011ea15352bfd1856e28d6d0171269951d3f3a1e1bd694e74f", + "bytes": 3254305 + }, + "source_id": "ev_57091094e6142a58af3b1716e8735a963d451a81eb79f214679542ca65e12022", + "answers_sha256": "263db399005814e40a7e962f5a46d3eff95e4372cd541fe3241d050be41d3d60", + "original_record_sha256": "46a8a61e38a78cf2000c5cda35695bef7c8198776be6d8a3ae54853a5f003a8a", + "call_count": 17, + "rpc_reply_bytes": 161116, + "query_reply_bytes": 160539, + "resource_events_at_payload_exit": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "final_whole_scope_counters_verified": false, + "abort": null + } + ], + "actual_model_token_usage": null, + "tokenizer_estimate": null, + "token_claim": "Unmeasured actual model usage; identical catalog and selected view replies, no token reduction claimed.", + "captures": [ + { + "id": "compact", + "url": "https://quic.aiortc.org/logs/dadb9dba427e797c.qlog", + "bytes": 29034, + "events": 132, + "sha256": "2dbe00348c2f6fcd81f829564718e276052d4aaea77f2a6745cbd147869aec6f" + }, + { + "id": "large", + "url": "https://quic.aiortc.org/logs/6e037439a5f79d47.qlog", + "bytes": 5922676, + "events": 30753, + "sha256": "4854882fdeec3d66701779cfde1c614a09e3d5e3b90afdea3fc3d2566a0e6e3f" + } + ], + "host": { + "platform": "Linux x86_64", + "kernel": "6.1.0-53-cloud-amd64", + "node": "24.18.0", + "npm": "11.16.0", + "sdk": "2.3.1", + "cpu_affinity": [15], + "query_memory_hard_bytes": 4294967296, + "query_memory_early_stop_bytes": 3221225472, + "swap_bytes": 0, + "os_cache": "warm, not flushed" + }, + "incomplete_attempts": [ + { + "phase": "initial baseline producer harness, each recording", + "reason": "Oracle initially expected the raw QUIC reset credential; actual correctly redacted transport credential. Fixed oracle checks the one exact permitted redaction. Original exported Evidence retained and used unchanged for all imported cohorts.", + "included_in_performance_cohorts": false + }, + { + "phase": "first candidate build", + "reason": "3 GiB early memory stop; payload did not finish; no OOM or hard-limit events. Serial build subsequently passed with 6 GiB hard/5 GiB early limits.", + "included_in_performance_cohorts": false + }, + { + "phase": "first full static check", + "reason": "Typecheck reached the 5 GiB early stop under a 6 GiB hard limit. No OOM or hard-limit events; task incomplete. Subsequent static checks use 8 GiB hard/7 GiB early, one CPU and at least 16 GiB available host memory.", + "included_in_performance_cohorts": false + }, + { + "phase": "intermediate full static check", + "reason": "The driver was being revised and had one formatting discrepancy. Formatter corrected it; the subsequent complete npm run check passed.", + "included_in_performance_cohorts": false + }, + { + "phase": "first latest-main pre-push check", + "reason": "Host available memory temporarily fell below the 8 GiB early-stop floor. Guard stopped the typecheck and push did not complete; no OOM/limit events. Host availability recovered before subsequent validation.", + "included_in_performance_cohorts": false + } + ], + "harness_finalization_note": "The final driver reuses the existing tested RPC finalizer and streams stderr instead of buffering it; the 12 timing cohorts used the earlier finalization path. RPC arguments and compared runtime implementations are unchanged. Final-driver replays are separate verification, not relabeled timing samples.", + "executed_checks": { + "source_tests": { + "files": 10, + "passed": 189 + }, + "compiled_tests": { + "files": 4, + "passed": 43, + "skipped": 1, + "skip_reason": "The qlog guard-unavailable refusal case runs only on hosts without decoder memory containment; this Linux host has it. The actual decoder heap-exhaustion case passed." + }, + "rpc_finalization_regressions": { + "passed": 10 + }, + "static_check": "npm run check passed, including typecheck, lint/module boundaries, formatting, knip, metadata and architecture guards", + "test_discovery": "1711 test files discovered exactly once" + }, + "final_driver_verification": [ + { + "sample": "compact", + "producer_status": "passed", + "selected_view_evidence_semantically_equal_to_imported": true, + "cli_evidence_equal_to_mcp": true, + "source_id": "ev_df45171a0cf0d3c691124f0cc44a8d6b38e0e8c9d3d298ea611ffdac084c304e", + "original_record_sha256": "6c9f269acba239804406d278cfc3cce1fd7d0880b312c785df3a791a2c157a4d", + "cleanup_failures": [], + "receipt_failures": [] + }, + { + "sample": "large", + "producer_status": "passed", + "selected_view_evidence_semantically_equal_to_imported": true, + "cli_evidence_equal_to_mcp": true, + "source_id": "ev_57091094e6142a58af3b1716e8735a963d451a81eb79f214679542ca65e12022", + "original_record_sha256": "46a8a61e38a78cf2000c5cda35695bef7c8198776be6d8a3ae54853a5f003a8a", + "cleanup_failures": [], + "receipt_failures": [] + } + ], + "latest_main_integration": { + "base_revision": "d72021b0dabedcc6fbc7bc6e0286fd359c604271", + "candidate_source_revision": "cd1c3758022e42b83bf7954aa3dc663148651b8b", + "cohorts": [ + { + "name": "integrated-baseline-compact", + "scope_seconds": 6.574, + "payload_cpu_seconds": 6.370822, + "payload_family_peak_bytes": 695435264, + "first_view_ms": 1340.7030219999997, + "warm_14_queries_ms": 178.7094820000002, + "catalog": { + "elapsed_ms": 1916.610111, + "sha256": "8a646b95cc5607ee57e5a3c0ef480b77e4cec7e5168498e72f511a8af40daf61", + "bytes": 3255374 + }, + "answers_sha256": "641f85866e87c2768b4cf56911640e4f9e3f3340ac44cd7a468c5329fdd9448c", + "original_record_sha256": "6c9f269acba239804406d278cfc3cce1fd7d0880b312c785df3a791a2c157a4d", + "payload_cleanup_verified": true, + "resource_events_at_payload_exit": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "final_whole_scope_counters_verified": false + }, + { + "name": "integrated-baseline-large", + "scope_seconds": 19.466, + "payload_cpu_seconds": 19.132667, + "payload_family_peak_bytes": 1795719168, + "first_view_ms": 2920.709574999999, + "warm_14_queries_ms": 6315.634482000003, + "catalog": { + "elapsed_ms": 1277.3594239999998, + "sha256": "8a646b95cc5607ee57e5a3c0ef480b77e4cec7e5168498e72f511a8af40daf61", + "bytes": 3255374 + }, + "answers_sha256": "263db399005814e40a7e962f5a46d3eff95e4372cd541fe3241d050be41d3d60", + "original_record_sha256": "46a8a61e38a78cf2000c5cda35695bef7c8198776be6d8a3ae54853a5f003a8a", + "payload_cleanup_verified": true, + "resource_events_at_payload_exit": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "final_whole_scope_counters_verified": false + }, + { + "name": "integrated-candidate-compact", + "scope_seconds": 5.664, + "payload_cpu_seconds": 5.444678, + "payload_family_peak_bytes": 695730176, + "first_view_ms": 1347.5685030000004, + "warm_14_queries_ms": 108.42142599999988, + "catalog": { + "elapsed_ms": 1173.273788, + "sha256": "8a646b95cc5607ee57e5a3c0ef480b77e4cec7e5168498e72f511a8af40daf61", + "bytes": 3255374 + }, + "answers_sha256": "641f85866e87c2768b4cf56911640e4f9e3f3340ac44cd7a468c5329fdd9448c", + "original_record_sha256": "6c9f269acba239804406d278cfc3cce1fd7d0880b312c785df3a791a2c157a4d", + "payload_cleanup_verified": true, + "resource_events_at_payload_exit": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "final_whole_scope_counters_verified": false + }, + { + "name": "integrated-candidate-large", + "scope_seconds": 13.719, + "payload_cpu_seconds": 13.524082, + "payload_family_peak_bytes": 1177186304, + "first_view_ms": 3807.763484, + "warm_14_queries_ms": 105.97323599999982, + "catalog": { + "elapsed_ms": 1297.2051929999998, + "sha256": "8a646b95cc5607ee57e5a3c0ef480b77e4cec7e5168498e72f511a8af40daf61", + "bytes": 3255374 + }, + "answers_sha256": "263db399005814e40a7e962f5a46d3eff95e4372cd541fe3241d050be41d3d60", + "original_record_sha256": "46a8a61e38a78cf2000c5cda35695bef7c8198776be6d8a3ae54853a5f003a8a", + "payload_cleanup_verified": true, + "resource_events_at_payload_exit": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "final_whole_scope_counters_verified": false + } + ], + "scope": "One integration pair per recording; separate from the original three-run distributions. Current baseline and candidate include Evidence-file source support. Both real CLI inline projections preserve original MCP Evidence.", + "baseline_runtime_sha256": "e0e0bb47340bd6a676ef6e939e49f36bd64f4263353afb40cc75bafa79f3d8ba", + "candidate_runtime_sha256": "8d6accf22f658b8815a8d52a85e0e5f910394456bd898dc3440f57e278f14b2f", + "source_combination_control": { + "candidate_combined": { + "passed": 251, + "failed": 1, + "files": 14 + }, + "unmodified_main_default_order": { + "passed": 243, + "failed": 0, + "files": 13 + }, + "unmodified_main_environment_first": { + "passed": 62, + "failed": 1, + "files": 2 + }, + "failure": "File authentication cancellation test returned EvidenceFileError instead of AnalysisCancelledError. Both unmodified main and candidate reproduce it when environmentReplay loads the service before the spy-owning test file.", + "control": "Temporary BaseSequencer override sorted environmentReplay before AnalysisViewService; production and test files on main stayed unchanged. A first attempt to pass a sequencer module string directly was rejected during runner setup and executed no tests.", + "scope": "Existing order-sensitive module spy, not established production cancellation failure. No unrelated repair included.", + "separate_current_runs": { + "service_file_passed": 48, + "other_source_files_passed": 204, + "other_source_files": 13, + "note": "Two separate processes avoid the existing shared-module spy order dependency. The mixed-run failure and unmodified-main control remain recorded." + } + } + }, + "final_scope_note": "Reuse is connected only to retained capture and HAR grouping views. Final call-site review restored ordinary canonical schema admission for one-shot producer/environment-file reads, avoiding extra proof work without a measured repeated-read benefit. Earlier frozen timing runtimes retain their original identities; final focused replay is qualified separately.", + "final_narrow_scope_verification": { + "source": "Retained views/grouping only; producer/environment schema call sites restored. All active query outputs and original records match baseline.", + "source_checks": "252 assertions passed in two invocations; mixed-run cancellation-spy failure retained with unmodified-main reproduction.", + "additional_mcp_assertions_passed": 20, + "static_check": "Full npm run check passed on final narrowed code.", + "cli": "Both public-recording CLI Evidence projections exactly match MCP.", + "runs": [ + { + "name": "final-narrow-compact", + "scope_seconds": 11.687, + "payload_cpu_seconds": 10.892926, + "payload_family_peak_bytes": 694239232, + "first_view_ms": 2877.247516999999, + "warm_14_queries_ms": 232.99256799999966, + "catalog_elapsed_ms": 2558.570081, + "answers_sha256": "641f85866e87c2768b4cf56911640e4f9e3f3340ac44cd7a468c5329fdd9448c", + "original_record_sha256": "6c9f269acba239804406d278cfc3cce1fd7d0880b312c785df3a791a2c157a4d", + "payload_cleanup_verified": true, + "resource_events_at_payload_exit": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "final_whole_scope_counters_verified": false + }, + { + "name": "final-narrow-large", + "scope_seconds": 31.854, + "payload_cpu_seconds": 28.414057, + "payload_family_peak_bytes": 1236717568, + "first_view_ms": 10157.196946, + "warm_14_queries_ms": 234.07879700000558, + "catalog_elapsed_ms": 2642.0467449999996, + "answers_sha256": "263db399005814e40a7e962f5a46d3eff95e4372cd541fe3241d050be41d3d60", + "original_record_sha256": "46a8a61e38a78cf2000c5cda35695bef7c8198776be6d8a3ae54853a5f003a8a", + "payload_cleanup_verified": true, + "resource_events_at_payload_exit": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "final_whole_scope_counters_verified": false + }, + { + "name": "final-control-baseline", + "scope_seconds": 18.87, + "payload_cpu_seconds": 18.518563, + "payload_family_peak_bytes": 1805721600, + "first_view_ms": 2797.2868439999993, + "warm_14_queries_ms": 5997.660557999998, + "catalog_elapsed_ms": 1202.264228, + "answers_sha256": "263db399005814e40a7e962f5a46d3eff95e4372cd541fe3241d050be41d3d60", + "original_record_sha256": "46a8a61e38a78cf2000c5cda35695bef7c8198776be6d8a3ae54853a5f003a8a", + "payload_cleanup_verified": true, + "resource_events_at_payload_exit": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "final_whole_scope_counters_verified": false + }, + { + "name": "final-control-candidate", + "scope_seconds": 14.242, + "payload_cpu_seconds": 14.047643, + "payload_family_peak_bytes": 1202728960, + "first_view_ms": 4040.8927329999988, + "warm_14_queries_ms": 107.68954900000426, + "catalog_elapsed_ms": 1348.444556, + "answers_sha256": "263db399005814e40a7e962f5a46d3eff95e4372cd541fe3241d050be41d3d60", + "original_record_sha256": "46a8a61e38a78cf2000c5cda35695bef7c8198776be6d8a3ae54853a5f003a8a", + "payload_cleanup_verified": true, + "resource_events_at_payload_exit": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "final_whole_scope_counters_verified": false + } + ], + "qualification": "One final adjacent baseline/candidate pair, not a new distribution. The earlier slower unpaired compact/large runs are retained; their broad phase slowdown is not causally attributed." + }, + "review_oracle_correction": { + "root_cause": "Returned-row iteration alone did not reject an empty or short page with unchanged coverage metadata.", + "fix": "Require exactly two selected records before checking row facts; production code unchanged.", + "archived_transcript_recheck": { + "oracle": "Exact page-validation block from the revised driver, evaluated against archived production replies", + "completed_transcripts": 22, + "pages": 198, + "records": 396, + "all_passed": true, + "negative_controls": [ + { + "returned_records": 0, + "coverage_unchanged": true, + "rejected": true + }, + { + "returned_records": 1, + "coverage_unchanged": true, + "rejected": true + } + ] + }, + "real_mcp_replays": [ + { + "name": "review-compact", + "capture_sha256": "2dbe00348c2f6fcd81f829564718e276052d4aaea77f2a6745cbd147869aec6f", + "answers_sha256": "641f85866e87c2768b4cf56911640e4f9e3f3340ac44cd7a468c5329fdd9448c", + "call_count": 17, + "scope_seconds": 5.37, + "payload_cpu_seconds": 5.194994, + "payload_family_peak_bytes": 679886848, + "payload_cleanup_verified": true, + "resource_events_at_payload_exit": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "final_whole_scope_counters_verified": false, + "abort": null + }, + { + "name": "review-large", + "capture_sha256": "4854882fdeec3d66701779cfde1c614a09e3d5e3b90afdea3fc3d2566a0e6e3f", + "answers_sha256": "263db399005814e40a7e962f5a46d3eff95e4372cd541fe3241d050be41d3d60", + "call_count": 17, + "scope_seconds": 14.242, + "payload_cpu_seconds": 13.966121, + "payload_family_peak_bytes": 1196273664, + "payload_cleanup_verified": true, + "resource_events_at_payload_exit": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "final_whole_scope_counters_verified": false, + "abort": null + } + ] + } +} diff --git a/experiments/analysis-efficiency/retained-capture-validation.md b/experiments/analysis-efficiency/retained-capture-validation.md new file mode 100644 index 0000000000..2323554d17 --- /dev/null +++ b/experiments/analysis-efficiency/retained-capture-validation.md @@ -0,0 +1,238 @@ +# Retained historical capture validation + +On main `5fa4defcbc0e55ef1ec84c888b918abe945303fe` (2026-10-20), repeated +historical capture pages still validated and copied the complete capture. +The immutable-parent work in [#4962](https://github.com/morluto/rea/pull/4962) +covered native layouts and JavaScript, rather than historical captures. +This continuation of [#3133](https://github.com/morluto/rea/issues/3133) +removes that repeated work without changing its query contract. + +## Reconciled audit scope + +The earlier directory list is a starting point, not a list of unfinished fixes. +Current history and the existing evidence support these narrower conclusions: + +| Scope | Existing work and remaining limit | +| --------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Shared Evidence and queries | [#3134](https://github.com/morluto/rea/issues/3134) and [#4810](https://github.com/morluto/rea/issues/4810) are closed. Authentic mixed histories and immutable native/JavaScript validation reuse have dedicated [measurements](immutable-parent-validation.md). This capture owner still repeated complete validation. | +| Apktool | [#4777](https://github.com/morluto/rea/issues/4777) is closed. Shared decoding and optional primary-string omission have [three-APK evidence](apktool-string-tables.md). Do not implement another decode cache from the old issue list. | +| Hopper | [#5017](https://github.com/morluto/rea/pull/5017) already reuses decoded string records. Its [checkpoint](hopper-string-records.md) preserves faster warm queries and the negative whole-task result. | +| JEB | The [Linux CE audit](jeb-linux-native.md) verifies selected real CLI/MCP reads. It does not justify another wrapper cache; discovery and other platforms remain separate coverage gaps. | +| IDA | Existing [native Windows verification](ida-native-windows.md) qualifies selected correctness/authority paths. It does not establish a comparable full performance audit. | +| JavaScript/browser/Electron | Prior experiments and retained pages already exist. [#4415](https://github.com/morluto/rea/issues/4415) remains exploratory; no safe general Joplin preparation speedup is established. | +| Other static formats | Managed, Flutter, SQLite, WASM, EVM, PCAP, signals and SAZ already have checkpoints in this directory. Extend a specific measured question rather than create duplicate selectors or caches. | +| Other providers, languages and infrastructure | Rizin/Cutter/GDB/objdump, Go/Rust, ADB/Frida/inspector and startup/resource owners still need bounded real-workflow questions. Historical local improvements do not prove complete coverage or establish another bottleneck. | + +This reconciles the cited issues and checked-in evidence; it is not a new +performance qualification of all `src/` directories or provider capabilities. + +## Owning change and lifetime + +`safeParseOwnedWebNetworkCapture` belongs to the capture format owner. A +successful complete schema parse may authenticate the exact source only if +`canBorrowParsedJsonSnapshot` verifies an existing realm-private sealing proof, +unchanged values, schema normalization, descriptors and alias topology. +The existing immutable JSON primitives supply that proof; an external freeze +or caller flag cannot supply it. + +One weak-set entry records the successful interpretation, without retaining +another report, index or source reference. Its lifetime follows the source's +reachability. Exact-object identity, not a path, digest string or live-state +assumption, keys reuse. Mutable, merely frozen, malformed and normalized inputs +keep the ordinary validation path. Failures are not cached. Validation is +synchronous and publishes the proof only after complete success. + +Capture pages and HAR grouping use this same owner. Producer admission and +one-shot environment-file loading retain ordinary owner-schema validation, +avoiding an extra reuse-proof traversal where no repeated result is available. +Each consumer still checks its own subject, +operation, format, coverage, declarations and resource budget. In particular, +validating a report does not authenticate a different parent's subject. +No provider rerun, new tool, schema, default, target execution or persisted cache +is introduced. Existing conservative grouping admission budgets remain intact. + +## Real recordings and completed task + +Two unchanged public recordings from the aioquic server's +[log listing](https://quic.aiortc.org/logs) were downloaded on 2026-10-20: + +| Recording | Bytes | Events | SHA-256 | +| ------------------------------------------------------------- | --------: | -----: | ------------------------------------------------------------------ | +| [compact](https://quic.aiortc.org/logs/dadb9dba427e797c.qlog) | 29,034 | 132 | `2dbe00348c2f6fcd81f829564718e276052d4aaea77f2a6745cbd147869aec6f` | +| [large](https://quic.aiortc.org/logs/6e037439a5f79d47.qlog) | 5,922,676 | 30,753 | `4854882fdeec3d66701779cfde1c614a09e3d5e3b90afdea3fc3d2566a0e6e3f` | + +Both report JSON qlog 0.3 with one server trace. The producer version and +deployment authenticity remain unknown. No recorded destination was contacted. + +Production stdio MCP created authentic complete Evidence, exported it, and +fresh servers imported that same unchanged bundle for each comparison. Each +task discovers the complete live catalog, imports, performs three traversals +of summary/three two-record complete pages/trace facts, exports again and closes +the transport. This is 17 RPCs and one catalog discovery, with 14 warm selected +queries after the first view. The question is the first six source events and +trace event total, revisited within one retained investigation. + +An independent JSON read checks event counts, original event objects, ordinals +and trace/event pointers. It explicitly expects only the QUIC reset-token +transport-credential redaction in those selected events and checks its sidecar. +All 15 selected-view Evidence records, parent identities and original-record +round trips agree exactly between implementations. This does not independently +qualify every transport fact or prove network completeness. + +## Measurements + +Three fresh-server tasks per implementation per recording ran serially, +baseline/candidate first and candidate/baseline in each later cohort. +The [curated receipt](retained-capture-validation.json) preserves every run, +runtime fingerprints, source and answer identities, resource observations and +incomplete attempts. Raw captures, bundles, replies and logs stay outside Git. + +Linux x86-64, Node 24.18.0/npm 11.16.0, MCP SDK 2.3.1. Each client uses the +exact named live tool definition for output validation. First-view time includes +client/schema initialization as well as capture validation; it is not isolated +parser time. OS file caches were warm and unflushed. No cold-disk run occurred. + +Medians across three completed tasks: + +| Recording / metric | Before | After | +| ------------------------------------------------- | ---------: | ---------: | +| Large: first selected view | 2.903 s | 4.077 s | +| Large: 14 warm queries | 6.399 s | 0.113 s | +| Large: complete owned scope wall | 20.089 s | 14.742 s | +| Large: complete payload-family CPU | 19.792 s | 14.496 s | +| Large: kernel family peak, including file cache | 1702.1 MiB | 1123.4 MiB | +| Compact: first selected view | 1.401 s | 1.462 s | +| Compact: 14 warm queries | 0.183 s | 0.108 s | +| Compact: complete owned scope wall | 6.178 s | 6.080 s | +| Compact: complete payload-family CPU | 5.960 s | 5.850 s | +| Compact: kernel family peak, including file cache | 658.7 MiB | 660.9 MiB | + +Large warm totals range 6.394–6.482 s before and 0.111–0.115 s after. +The extra first-parse structural comparison costs time; repeated views recover +that cost in this large-recording task. Small-task wall times overlap, so no +small-input whole-task benefit or universal capture speedup is established. +The weak-set bookkeeping's exact heap size and post-GC retained heap were not +measured; the implementation retains no second capture representation. + +The catalog is unchanged at 3,254,305 serialized bytes. The 15 query replies are +unchanged at 160,659 bytes for compact and 160,539 for large, including the SDK +result envelopes. Actual model token usage and tokenizer estimates are +unmeasured; no token reduction or model-quality improvement is claimed. + +Query scopes use one logical CPU, 4 GiB hard/3 GiB early memory limits, no swap +and 192 tasks. All 12 completed observations verify payload exit, cleanup and +zero OOM/limit events. Final counters after the transient scope disappears +remain unknown; payload-exit counters are recorded while its observer is alive. +An initial build stopped at 3 GiB before completion, without OOM. The subsequent +serial build used 6 GiB hard/5 GiB early limits and peaked at 3264.9 MiB. +Two initial producer harnesses stopped on the oracle's incorrect expectation +that a reset credential would remain raw. Their original exports remain intact; +they are not completed performance tasks or product failures. + +## Reproduction and validation + +Build each runtime with the pinned toolchain and preserve a separate baseline. +Keep each capture's absolute selected path identical across compared runtimes: + +```sh +python3 experiments/analysis-efficiency/resource-run.py --observe-payload-exit -- \ + node experiments/analysis-efficiency/retained-capture-validation.mjs \ + --runtime /absolute/baseline --capture /absolute/input.qlog \ + --output /absolute/new-producer-output + +python3 experiments/analysis-efficiency/resource-run.py --observe-payload-exit -- \ + node experiments/analysis-efficiency/retained-capture-validation.mjs \ + --runtime /absolute/candidate --capture /absolute/input.qlog \ + --bundle /absolute/new-producer-output/original.json \ + --output /absolute/new-import-output +``` + +Executed source verification passed 189 assertions across ten files, covering +capture schemas/views/services, subject rechecks, forged freezes, normalization, +mutation, invalid ordinals, HAR budgets, environment loading and replay. +Four compiled CLI/MCP/decoder/acceptance files passed 43 assertions. One +guard-unavailable refusal case was skipped because this Linux host has decoder +memory containment; the actual decoder heap-exhaustion and recovery case passed. +The ten existing RPC/receipt/finalization regressions passed. `npm run check` +passed, and test discovery found all 1711 test files exactly once. + +Final-driver replays completed both public producer tasks with no cleanup or +receipt errors. Their selected Evidence matches imported-source queries, and +both real CLI projections exactly match the MCP Evidence. The final driver +uses the existing tested finalizer and streams stderr; these are separate +verification runs, not relabeled samples from the earlier timing cohorts. +One intermediate check encountered an unformatted driver revision; formatting +was corrected before the successful full check. The first full typecheck also +hit the 5 GiB early stop; its successful run used 8 GiB hard/7 GiB early limits +and peaked at 5392.8 MiB without OOM. Queries remained within the smaller limit. + +Timing claims cover these qlog tasks only. Current HAR/SAZ/mitmproxy performance, +other hosts, real model token costs and post-GC retained heap remain unmeasured. + +## Latest-main integration + +Main advanced to `d72021b0dabedcc6fbc7bc6e0286fd359c604271` before publication, +adding Evidence-file sources and template path pairs. The change rebased without +conflicts. The original timing distributions above retain their original base. + +One additional pair per recording compares this current main with the rebased +candidate, using the final driver and original authentic bundles. Both preserve +all selected Evidence and original records. For large, 14 warm queries cost +6.316 → 0.106 s; complete scope wall is 19.466 → 13.719 s; payload-family CPU +19.133 → 13.524 s; family peak 1712.5 → 1122.7 MiB. Compact scope wall is +6.574 → 5.664 s in this single pair, without establishing a small-task trend. +Both real CLI projections again exactly match the original MCP Evidence. +Individual counters are in the receipt's `latest_main_integration` section. + +Evidence-file reads still authenticate a fresh object on each read. This +optimization is not a file-path cache and does not reuse an earlier file's +validation after its bytes change. + +The first integrated 14-file source run passed 251/252 assertions and failed +the newly merged file-authentication cancellation injection in +`AnalysisViewService.test.ts`. An unmodified-main 13-file control passed +243/243 with a different service-file order. A controlled unmodified-main run +placing `environmentReplay.test.ts` before `AnalysisViewService.test.ts` +reproduced exactly that cancellation failure (62 passes, one failure). +The service project shares modules between files; the source test installs a +module spy after another file may already have loaded the service. This is +source-derived reasoning for the order sensitivity, rather than a reproduced +production cancellation defect. No unrelated test or production repair is +included, and the combined failure remains part of the verification record. + +The service file passed all 48 assertions in its own process, and the other +13 selected source files passed 204 assertions. The final narrowed implementation +passed the full `npm run check`, plus 20 additional compiled MCP assertions, +including identical retained/inline/Evidence-file manifest projection. + +After limiting reuse to retained/grouping queries, both public imports and real +CLI projections passed again. This first unpaired replay was slower across +catalog discovery, import and first-view work: large scope wall was 31.854 s, +CPU 28.414 s, with 0.234 s for 14 warm queries. Compact scope wall was 11.687 s. +The cause of this broad slowdown is unestablished; these runs are not discarded. + +A subsequent adjacent baseline/candidate large-recording control used the final +code and same inputs. Scope wall was 18.870 → 14.242 s; CPU 18.519 → 14.048 s; +kernel family peak 1722.1 → 1147.0 MiB; 14 warm queries 5.998 → 0.108 s. +First view was slower at 2.797 → 4.041 s. All original and selected Evidence +agree. This final single pair requalifies the narrowed code; it is not a new +latency distribution or an explanation of the slower unpaired replay. + +## Review-driven oracle correction + +Review found that the original page oracle checked only returned rows: a short +page could pass with unchanged coverage metadata. The driver now also requires +both selected records before checking their facts. This fixes the oracle at its +owner, without changing production code or introducing a second validator. + +The revised driver's exact page-validation block rechecked all 22 completed +production transcripts: 198 pages and 396 records passed. Controls truncating +an authentic response to zero or one record, with coverage unchanged, both +failed the new assertion. These are archived-response controls, not additional +real-provider qualification. Earlier timings remain unchanged observations. + +Both real MCP import/query/export tasks were then repeated with the revised +driver. Compact scope wall/CPU/peak were 5.370 s / 5.195 s / 648.4 MiB; large +were 14.242 s / 13.966 s / 1140.9 MiB. Both preserve the prior answer digests, +verify cleanup, and record zero OOM or limit events at payload exit. These +candidate-only correctness replays do not create another before/after pair. diff --git a/experiments/analysis-efficiency/retained-capture-validation.mjs b/experiments/analysis-efficiency/retained-capture-validation.mjs new file mode 100644 index 0000000000..88c96964fa --- /dev/null +++ b/experiments/analysis-efficiency/retained-capture-validation.mjs @@ -0,0 +1,251 @@ +/** Public stdio MCP capture paging; keep original recordings and transcripts outside Git. */ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { mkdir, readFile, writeFile, mkdtemp } from "node:fs/promises"; +import { createWriteStream } from "node:fs"; +import { finished } from "node:stream/promises"; +import { join, resolve } from "node:path"; +import { tmpdir } from "node:os"; +import { parseArgs } from "node:util"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { createRetainedQueryCaller } from "./retained-query-calls.mjs"; +import { finalizeRetainedQueryReceipt } from "./retained-query-finalization.mjs"; + +const { values } = parseArgs({ + options: Object.fromEntries( + ["runtime", "capture", "output", "bundle"].map((name) => [ + name, + { type: "string" }, + ]), + ), +}); +for (const name of ["runtime", "capture", "output"]) + assert.ok( + values[name] && values[name] === resolve(values[name]), + `${name} must be absolute`, + ); +const hash = (value) => createHash("sha256").update(value).digest("hex"); +const output = values.output; +await mkdir(output, { mode: 0o700 }); +const recording = await readFile(values.capture); +const raw = JSON.parse(recording); +const events = raw.traces.flatMap((trace, t) => + trace.events.map((reported, e) => ({ + reported, + pointer: `/traces/${t}/events/${e}`, + })), +); +assert.ok(events.length >= 6); +const home = await mkdtemp(join(tmpdir(), "rea-capture-validation-")); +const client = new Client({ + name: "retained-capture-validation", + version: "1", +}); +const transport = new StdioClientTransport({ + command: process.execPath, + args: [join(values.runtime, "scripts/rea.mjs"), "mcp"], + env: { + HOME: home, + PATH: process.env.PATH, + NODE_OPTIONS: "--max-old-space-size=2048", + REA_LOG_LEVEL: "silent", + }, + stderr: "pipe", +}); +const serverLog = createWriteStream(join(output, "server.log")); +const serverLogFinished = finished(serverLog); +serverLogFinished.catch(() => {}); +transport.stderr?.pipe(serverLog); +const report = { + capture_sha256: hash(recording), + capture_bytes: recording.length, + events: events.length, + mode: values.bundle ? "imported" : "producer", + calls: [], +}; +const transcript = []; +const call = createRetainedQueryCaller(client, (receipt, event) => { + report.calls.push(receipt); + transcript.push(event); +}); +const result = async (name, args, phase) => { + const response = await call(name, args, phase); + assert.notEqual(response.isError, true, JSON.stringify(response)); + assert.ok(response.structuredContent); + return response.structuredContent; +}; +const started = performance.now(); +try { + await client.connect(transport); + const discovery = performance.now(); + const catalog = await client.listTools(); + assert.equal( + catalog.nextCursor, + undefined, + "This driver requires a complete catalog page", + ); + report.catalog = { + elapsed_ms: performance.now() - discovery, + sha256: hash(JSON.stringify(catalog)), + bytes: Buffer.byteLength(JSON.stringify(catalog)), + }; + await writeFile(join(output, "catalog.json"), JSON.stringify(catalog)); + // Compile the exact live named contract for each call, avoiding client-wide compilation. + const definitions = new Map(catalog.tools.map((tool) => [tool.name, tool])); + const originalCallTool = client.callTool.bind(client); + client.callTool = (input, options) => + originalCallTool(input, { + ...options, + toolDefinition: definitions.get(input.name), + }); + let original; + if (values.bundle) { + const bundle = JSON.parse(await readFile(values.bundle, "utf8")); + original = bundle.records.find( + (record) => record.operation === "inspect_web_network_capture", + ); + assert.ok(original); + await result("import_evidence_bundle", { path: values.bundle }, "import"); + } else { + const evidence = await result( + "inspect_web_network_capture", + { + capture_path: values.capture, + format: "qlog", + detail: "summary", + }, + "produce-summary", + ); + report.summary_id = evidence.evidence_id; + const path = join(output, "original.json"); + await result("export_evidence_bundle", { path }, "export-original"); + const bundle = JSON.parse(await readFile(path, "utf8")); + original = bundle.records.find( + (record) => record.operation === "inspect_web_network_capture", + ); + assert.ok(original); + } + assert.equal( + original.normalized_result.artifact.sha256, + report.capture_sha256, + ); + assert.equal(original.normalized_result.total_records, events.length); + const source = { + kind: "retained-evidence", + evidence_id: original.evidence_id, + }; + report.source_id = original.evidence_id; + const answers = []; + for (let pass = 0; pass < 3; pass += 1) { + const summary = await result( + "inspect_analysis_view", + { source, view: { kind: "summary" } }, + `summary-${pass}`, + ); + assert.equal( + summary.normalized_result.summary.total_records, + events.length, + ); + answers.push(summary); + for (const offset of [0, 2, 4]) { + const evidence = await result( + "inspect_analysis_view", + { + source, + view: { kind: "capture", detail: "complete", offset, limit: 2 }, + }, + `page-${pass}-${offset}`, + ); + const view = evidence.normalized_result; + assert.equal(view.parent_evidence_id, original.evidence_id); + assert.equal(view.coverage.total, events.length); + assert.equal(view.coverage.next_offset, offset + 2); + assert.equal( + view.items.length, + 2, + "Every selected record must be returned", + ); + for (const [i, row] of view.items.entries()) { + assert.equal(row.ordinal, offset + i); + assert.equal(row.location.pointer, events[offset + i].pointer); + const expected = structuredClone(events[offset + i].reported); + // QUIC reset tokens are transport authentication material, not reported facts. + const resetToken = Object.hasOwn( + expected.data ?? {}, + "stateless_reset_token", + ); + if (resetToken) expected.data.stateless_reset_token = null; + assert.deepEqual( + row.redactions, + resetToken + ? [ + { + pointer: "/data/stateless_reset_token", + reason: "transport-credential", + }, + ] + : [], + ); + assert.deepEqual(row.reported, expected); + } + answers.push(evidence); + } + const traces = await result( + "inspect_analysis_view", + { + source, + view: { kind: "qlog", facet: "traces", offset: 0, limit: 50 }, + }, + `traces-${pass}`, + ); + assert.equal(traces.normalized_result.page.items.length, raw.traces.length); + for (const [i, row] of traces.normalized_result.page.items.entries()) { + assert.equal(row.trace_index, i); + assert.equal(row.event_count, raw.traces[i].events.length); + } + answers.push(traces); + } + const roundtrip = join(output, "roundtrip.json"); + await result( + "export_evidence_bundle", + { path: roundtrip }, + "export-roundtrip", + ); + const exported = JSON.parse(await readFile(roundtrip, "utf8")); + assert.deepEqual( + exported.records.find( + (record) => record.evidence_id === original.evidence_id, + ), + original, + ); + assert.equal(hash(await readFile(values.capture)), report.capture_sha256); + report.answers_sha256 = hash(JSON.stringify(answers)); + report.original_record_sha256 = hash(JSON.stringify(original)); + report.status = "passed"; +} catch (cause) { + report.status = "failed"; + report.failure = String(cause?.stack ?? cause); + process.exitCode = 1; +} finally { + report.elapsed_before_cleanup_ms = performance.now() - started; + try { + await writeFile( + join(output, "transcript.json"), + JSON.stringify(transcript), + ); + } catch (cause) { + report.status = "failed"; + report.transcript_failure = String(cause); + } + const finalized = await finalizeRetainedQueryReceipt({ + report, + client, + transport, + account: home, + serverLog, + serverLogFinished, + output, + }); + if (report.status !== "passed" || !finalized) process.exitCode = 1; +} diff --git a/src/domain/analysisView/WebNetworkProjectionAccumulator.ts b/src/domain/analysisView/WebNetworkProjectionAccumulator.ts index d9c48b7070..89a8b0d4d7 100644 --- a/src/domain/analysisView/WebNetworkProjectionAccumulator.ts +++ b/src/domain/analysisView/WebNetworkProjectionAccumulator.ts @@ -7,7 +7,7 @@ import type { AnalysisError } from "../analysisErrorBase.js"; import { compareUnicodeCodePoints as compare } from "../unicodeCodePointOrder.js"; import { err, ok, type Result } from "../result.js"; import { - webNetworkCaptureSchema, + safeParseOwnedWebNetworkCapture, type WebNetworkCapture, } from "../webNetworkCapture.js"; import { admitAnalysisViewData } from "./analysisViewDataAdmission.js"; @@ -116,7 +116,7 @@ export const admitHarCaptureParent = ( ); const admitted = accumulator.admitCaptureClone(parent.normalizedResult); if (!admitted.ok) return admitted; - const parsed = webNetworkCaptureSchema.safeParse(parent.normalizedResult); + const parsed = safeParseOwnedWebNetworkCapture(parent.normalizedResult); if (!parsed.success) return err( analysisViewInputError([ diff --git a/src/domain/analysisView/webNetworkCaptureView.test.ts b/src/domain/analysisView/webNetworkCaptureView.test.ts index 7035c2b95f..ea87b86863 100644 --- a/src/domain/analysisView/webNetworkCaptureView.test.ts +++ b/src/domain/analysisView/webNetworkCaptureView.test.ts @@ -1,6 +1,7 @@ import { expect, it } from "vitest"; import { createHash } from "node:crypto"; import { createEvidence } from "../evidence.js"; +import { freezeJsonSnapshot } from "../immutableJson.js"; import { webNetworkCaptureSchema } from "../webNetworkCapture.js"; import { projectAnalysisView } from "./analysisView.js"; @@ -77,6 +78,29 @@ const capture = () => { }; }; +it("checks each parent's subject even after its immutable capture has been validated", () => { + const { parent } = capture(); + const source = { + ...parent, + normalizedResult: freezeJsonSnapshot(parent.normalizedResult), + }; + expect(projectAnalysisView(source, { kind: "summary" }).ok).toBe(true); + expect( + projectAnalysisView({ ...source, subject: null }, { kind: "summary" }).ok, + ).toBe(false); + if (source.subject === null) throw new Error("Expected fixture subject"); + expect( + projectAnalysisView( + { + ...source, + subject: { ...source.subject, local_path: "/different.saz" }, + }, + { kind: "summary" }, + ).ok, + ).toBe(false); + expect(projectAnalysisView(source, { kind: "summary" }).ok).toBe(true); +}); + it("keeps selected-record page coverage distinct from capture coverage and unsafe number absence", () => { const { parent, report } = capture(); const summary = projectAnalysisView(parent, { kind: "summary" }); diff --git a/src/domain/analysisView/webNetworkCaptureView.ts b/src/domain/analysisView/webNetworkCaptureView.ts index fce1ddf720..02da9d59b1 100644 --- a/src/domain/analysisView/webNetworkCaptureView.ts +++ b/src/domain/analysisView/webNetworkCaptureView.ts @@ -8,7 +8,7 @@ import { err, ok, type Result } from "../result.js"; import { webNetworkCaptureRecordSchema, webNetworkCaptureBinaryDescriptorSchema, - webNetworkCaptureSchema, + safeParseOwnedWebNetworkCapture, webNetworkCaptureArtifactSchema, webNetworkCaptureDecoderSchema, WEB_NETWORK_CAPTURE_FORMATS, @@ -388,7 +388,7 @@ const preview = (record: WebNetworkCaptureRecord, format: string) => { const parseCaptureParent = ( parent: AnalysisViewParent, ): Result => { - const parsed = webNetworkCaptureSchema.safeParse(parent.normalizedResult); + const parsed = safeParseOwnedWebNetworkCapture(parent.normalizedResult); if (!parsed.success) return err( analysisInputErrorFromIssues( diff --git a/src/domain/webNetworkCapture.test.ts b/src/domain/webNetworkCapture.test.ts new file mode 100644 index 0000000000..990ca99274 --- /dev/null +++ b/src/domain/webNetworkCapture.test.ts @@ -0,0 +1,118 @@ +import { expect, it } from "vitest"; +import { + freezeJsonSnapshot, + immutableJsonSnapshotMember, +} from "./immutableJson.js"; +import { + safeParseOwnedWebNetworkCapture, + webNetworkCaptureSchema, +} from "./webNetworkCapture.js"; + +const capture = (format = "har") => ({ + format, + artifact: { path: "/capture", sha256: "a".repeat(64), bytes: 10 }, + decoder: { id: "fixture", name: "Fixture", version: "1" }, + container: { + reported: {}, + numeric_literals: [], + redactions: [], + records_pointer: "/log/entries", + }, + total_records: 3, + records: [ + { + ordinal: 2, + location: { kind: "json-pointer", pointer: "/log/entries/2" }, + reported: { method: "GET", timestamp: null }, + binary_fields: [], + numeric_literals: [ + { + pointer: "/time", + producer_type: "json-number", + literal: "9007199254740993", + }, + ], + redactions: [], + limitations: [], + }, + ], + runtime_attribution: "unknown", + limitations: [], +}); + +it.each(["har", "saz", "mitmproxy"])( + "borrows a fully validated immutable %s capture, retaining exact source facts", + (format) => { + const source = freezeJsonSnapshot(capture(format)); + const expected = webNetworkCaptureSchema.parse(source); + for (let page = 0; page < 2; page += 1) { + const parsed = safeParseOwnedWebNetworkCapture(source); + if (!parsed.success) throw parsed.error; + expect(parsed.data).toBe(source); + expect(parsed.data).toEqual(expected); + } + }, +); + +it.each(["mutable", "shallow-frozen"])( + "revalidates a %s capture after a nested ordinal changes", + (kind) => { + const source = capture(); + if (kind === "shallow-frozen") Object.freeze(source); + expect(safeParseOwnedWebNetworkCapture(source).success).toBe(true); + source.records[0]!.ordinal = 3; + expect(safeParseOwnedWebNetworkCapture(source)).toEqual( + webNetworkCaptureSchema.safeParse(source), + ); + expect(safeParseOwnedWebNetworkCapture(source).success).toBe(false); + source.records[0]!.ordinal = 0; + expect(safeParseOwnedWebNetworkCapture(source).success).toBe(true); + }, +); + +it("validates a child of a sealed portable envelope and rejects malformed sealed siblings", () => { + const malformed = capture(); + malformed.records.push({ ...malformed.records[0]! }); + const envelope = freezeJsonSnapshot({ valid: capture(), malformed }); + const valid = immutableJsonSnapshotMember(envelope, "valid"); + const parsed = safeParseOwnedWebNetworkCapture(valid); + expect(parsed.success && parsed.data).toBe(valid); + const invalid = immutableJsonSnapshotMember(envelope, "malformed"); + for (let attempt = 0; attempt < 2; attempt += 1) { + const result = safeParseOwnedWebNetworkCapture(invalid); + expect(result.success).toBe(false); + expect(result).toEqual(webNetworkCaptureSchema.safeParse(invalid)); + } +}); + +it("does not treat external deep freezing as the owner's validation proof", () => { + const source = capture(); + const freeze = (value: unknown): void => { + if (value === null || typeof value !== "object") return; + for (const child of Object.values(value)) freeze(child); + Object.freeze(value); + }; + freeze(source); + for (let page = 0; page < 2; page += 1) { + const parsed = safeParseOwnedWebNetworkCapture(source); + if (!parsed.success) throw parsed.error; + expect(parsed.data).not.toBe(source); + expect(parsed.data).toEqual(webNetworkCaptureSchema.parse(source)); + } +}); + +it("retains schema normalization when an owned source contains a hidden reported field", () => { + const source = capture(); + Object.defineProperty(source.container, "reported", { + value: Object.defineProperty({}, "hidden", { value: 7 }), + enumerable: true, + }); + const sealed = freezeJsonSnapshot(source); + const parsed = safeParseOwnedWebNetworkCapture(sealed); + if (!parsed.success) throw parsed.error; + expect(parsed.data).not.toBe(sealed); + expect(parsed.data).toEqual(webNetworkCaptureSchema.parse(sealed)); + expect( + Object.hasOwn(parsed.data.container.reported as object, "hidden"), + ).toBe(false); +}); diff --git a/src/domain/webNetworkCapture.ts b/src/domain/webNetworkCapture.ts index 77752f8e53..1f40d3d1b7 100644 --- a/src/domain/webNetworkCapture.ts +++ b/src/domain/webNetworkCapture.ts @@ -2,6 +2,7 @@ import { z } from "zod"; import { jsonValueSchema } from "./jsonValue.js"; import { localPathStringSchema } from "./localPath.js"; import { qlogCaptureFactsSchema } from "./qlogCapture.js"; +import { canBorrowParsedJsonSnapshot } from "./immutableJson.js"; /** Complete historical evidence budgets; exceeding a budget returns no partial capture. */ export const WEB_NETWORK_CAPTURE_LIMITS = { @@ -216,3 +217,25 @@ export type WebNetworkCapture = z.output; export type WebNetworkCaptureRecord = z.output< typeof webNetworkCaptureRecordSchema >; + +const validatedImmutableCaptures = new WeakSet(); + +/** Reuse successful validation of the exact owned snapshot, without a retained copy. */ +export const safeParseOwnedWebNetworkCapture = ( + input: unknown, +): z.ZodSafeParseResult => { + if ( + typeof input === "object" && + input !== null && + validatedImmutableCaptures.has(input) + ) + return { success: true, data: input as WebNetworkCapture }; + const parsed = webNetworkCaptureSchema.safeParse(input); + if (parsed.success && canBorrowParsedJsonSnapshot(input, parsed.data)) { + // Borrow only when full validation preserves values, schema normalization + // and alias topology. Merely frozen or mutable objects never gain a proof. + validatedImmutableCaptures.add(input); + return { success: true, data: input }; + } + return parsed; +};