mirror of
https://github.com/morluto/rea.git
synced 2026-10-26 17:35:11 +00:00
perf(javascript): query selected source-to-bundle facts (#6128)
Select completed source-to-bundle mappings and candidates through the shared view owner, preserve reference-file provenance, and retain full Evidence. Reuse bounded JSON reading, heap admission, and cancellation before session writes. Verified public Redux Toolkit source/map facts with an independent oracle, equivalent CLI/MCP workflows, focused negative controls, and complete CI/review. Historical paired SDK context estimates fall about 64%; timing/resource tradeoffs and final verification receipts are recorded without a general speedup or actual model-token claim. Fixes #6112.
This commit is contained in:
gensokyo-dev authored and GitHub committed
1 parent
a0214e0e0e
commit
0583fc69be
29 files changed
+3527
-129
No files matched your search
@@ -0,0 +1,34 @@
|
||||
import argparse, json, statistics
|
||||
from pathlib import Path
|
||||
import tiktoken
|
||||
parser=argparse.ArgumentParser()
|
||||
parser.add_argument('root', type=Path)
|
||||
parser.add_argument('--pattern', default='paired-*')
|
||||
parser.add_argument('--output', type=Path)
|
||||
args=parser.parse_args()
|
||||
root=args.root
|
||||
enc=tiktoken.get_encoding('cl100k_base')
|
||||
compact=lambda v: json.dumps(v,ensure_ascii=False,separators=(',',':'))
|
||||
count=lambda v: len(enc.encode(v,disallowed_special=()))
|
||||
results=[]
|
||||
for case in sorted(root.glob(args.pattern)):
|
||||
if not case.is_dir(): continue
|
||||
report=json.loads((case/'report.json').read_text())
|
||||
assert report['status']=='observed' and report['cleanup_verified']
|
||||
scope=next(json.loads(line) for line in reversed(case.with_name(case.name+'.log').read_text().splitlines()) if line.startswith('{"resource_scope":'))
|
||||
observed=scope['payload_resource_observation']
|
||||
assert scope['abort'] is None and scope['payload_resource_observation_verified'] and observed['payload_cleanup_verified'] and observed['payload_exit_code']==0
|
||||
catalog=json.loads((case/'catalog.json').read_text())
|
||||
transcript=json.loads((case/'transcript.json').read_text())
|
||||
inputs=[{k: t[k] for k in ['name','description','inputSchema']} for t in catalog['tools']]
|
||||
arg=sum(count(compact({'name':r['name'],'arguments':r['arguments']})) for r in transcript)
|
||||
text=sum(count(b['text']) for r in transcript for b in r['reply']['content'] if b['type']=='text')
|
||||
results.append({'case':case.name,'mode':report['mode'],'output_schema_profile':report.get('output_schema_profile','full'),'runtime_revision':report['runtime_revision'],'runtime_worktree_status':report['runtime_worktree_status'],'token_estimates':{'input_catalog_once':count(compact(inputs)),'arguments':arg,'text_replies':text,'whole_task':count(compact(inputs))+arg+text},'actual_model_usage':None,'wall_seconds':scope['elapsed_seconds'],'cpu_seconds':observed['cpu_stat']['usage_usec']/1e6,'peak_process_family_bytes_including_page_cache':observed['memory_peak_bytes'],'calls':report['calls'],'retained_records':report['retained_records'],'proof':json.loads((case/'independent-proof.json').read_text()),'resource_events':observed['resource_events'],'cleanup_verified':True})
|
||||
summary={}
|
||||
for mode in ['complete','selected']:
|
||||
rows=[r for r in results if r['mode']==mode]
|
||||
summary[mode]={key:statistics.median(r[key] for r in rows) for key in ['wall_seconds','cpu_seconds','peak_process_family_bytes_including_page_cache']}
|
||||
summary[mode]['whole_task_tokens_estimated']=[r['token_estimates']['whole_task'] for r in rows]
|
||||
out={'status':'passed','tokenizer':'cl100k_base, tiktoken 0.12.0; estimates only; one input catalog plus arguments plus MCP text replies; not actual model usage','cold_warm':'Fresh server process and isolated HOME each run; alternating complete/selected, selected/complete, complete/selected. OS page caches are warm/uncontrolled; not an OS-cold claim.','tarball_sha256':'f2517c466b50424b661a27e48039f15db7783e5822f46b5c7d39f20104ac67a9','preparation_scope':'Reference-source import is performed once outside these repeated SDK trials and reported separately. Both workflows use the explicit existing semantic cap of one for the static correspondence question; full semantic coverage is not claimed.','summary':summary,'runs':results}
|
||||
(args.output or root/'costs.json').write_text(json.dumps(out,indent=2)+'\n')
|
||||
print(json.dumps(summary,indent=2))
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Verify published source/map facts independently of REA's parsers."""
|
||||
import argparse
|
||||
import collections
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path, PurePosixPath
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("case", type=Path)
|
||||
parser.add_argument("reference", type=Path)
|
||||
args = parser.parse_args()
|
||||
root = args.case.parent
|
||||
bundle = json.loads((args.case / "evidence.json").read_text())
|
||||
reference = json.loads(args.reference.read_text())
|
||||
oracle = json.loads((root / "oracle.json").read_text())
|
||||
application = next(r for r in bundle["records"] if r["operation"] == "analyze_javascript_application")
|
||||
comparison = next(r for r in bundle["records"] if r["operation"] == "compare_source_to_bundle")["normalized_result"]
|
||||
expected = {(r["map"], r["source_index"]): r for r in oracle["rows"]}
|
||||
nodes = [n for n in application["normalized_result"]["graph"]["nodes"] if n["kind"] == "source-module"]
|
||||
assert len(nodes) == len(expected) == 99
|
||||
by_path = collections.defaultdict(set)
|
||||
by_basename = collections.defaultdict(set)
|
||||
seen = set()
|
||||
for node in nodes:
|
||||
observation = node["observations"][0]
|
||||
location = observation["source_map_reference"]
|
||||
key = (PurePosixPath(location["map_path"]).name, observation["properties"]["source_index"])
|
||||
row = expected[key]
|
||||
assert key not in seen
|
||||
seen.add(key)
|
||||
assert node["identity"]["original_source"] == row["map_source"] == location["source_name"]
|
||||
assert node["identity"]["source_sha256"] == row["content_sha256"] == observation["properties"]["content_sha256"]
|
||||
assert node["identity"]["source_map_sha256"] == hashlib.sha256((root / "artifact/package/dist" / row["map"]).read_bytes()).hexdigest()
|
||||
path = "src/" + row["source_path"]
|
||||
by_path[path].add(node["node_id"])
|
||||
by_basename[PurePosixPath(path).name].add(node["node_id"])
|
||||
actual_files = {p.relative_to(root / "reference").as_posix(): hashlib.sha256(p.read_bytes()).hexdigest()
|
||||
for p in (root / "reference/src").rglob("*") if p.is_file()}
|
||||
entries = {e["path"]: e for e in reference["entries"] if e["kind"] == "file"}
|
||||
assert set(entries) == set(actual_files)
|
||||
assert all(entries[p]["sha256"] == sha for p, sha in actual_files.items())
|
||||
items = {item["source_path"]: item for item in comparison["items"]}
|
||||
assert set(items) == set(actual_files)
|
||||
candidate_count = 0
|
||||
for path, item in items.items():
|
||||
assert item["source_sha256"] == actual_files[path]
|
||||
candidates = {candidate["current_node_id"] for candidate in item["candidates"]}
|
||||
assert candidates == by_basename[PurePosixPath(path).name], path
|
||||
candidate_count += len(candidates)
|
||||
assert set(item["current_node_ids"]) == by_path[path]
|
||||
assert item["status"] == ("split" if by_path[path] else "unknown")
|
||||
assert all(signal["kind"] != "exact-source-digest"
|
||||
for candidate in item["candidates"] for signal in candidate["signals"])
|
||||
assert candidate_count == comparison["coverage"]["candidate_evaluations"] == 210
|
||||
assert comparison["coverage"]["status"] == "partial"
|
||||
proof = {"status": "passed", "method": "Independent Python JSON/SHA-256/POSIX path and basename joins; no REA parser or matcher.",
|
||||
"source_files": len(actual_files), "embedded_nodes": len(nodes), "candidate_relations": candidate_count,
|
||||
"strong_location_rows": sum(bool(by_path[p]) for p in actual_files),
|
||||
"limitation": "Static multiple-location projections; no semantic-equivalence or removal claim."}
|
||||
with (args.case / "independent-proof.json").open("x") as output:
|
||||
json.dump(proof, output, indent=2)
|
||||
output.write("\n")
|
||||
print(json.dumps(proof))
|
||||
@@ -0,0 +1,548 @@
|
||||
{
|
||||
"status": "passed",
|
||||
"tokenizer": "cl100k_base, tiktoken 0.12.0; estimates only; one input catalog plus arguments plus MCP text replies; not actual model usage",
|
||||
"cold_warm": "Fresh server process and isolated HOME each run; alternating complete/selected, selected/complete, complete/selected. OS page caches are warm/uncontrolled; not an OS-cold claim.",
|
||||
"tarball_sha256": "f2517c466b50424b661a27e48039f15db7783e5822f46b5c7d39f20104ac67a9",
|
||||
"preparation_scope": "Reference-source import is performed once outside these repeated SDK trials and reported separately. Both workflows use the explicit existing semantic cap of one for the static correspondence question; full semantic coverage is not claimed.",
|
||||
"summary": {
|
||||
"complete": {
|
||||
"wall_seconds": 11.597,
|
||||
"cpu_seconds": 11.243998,
|
||||
"peak_process_family_bytes_including_page_cache": 659202048,
|
||||
"whole_task_tokens_estimated": [152668, 152668, 152668]
|
||||
},
|
||||
"selected": {
|
||||
"wall_seconds": 10.89,
|
||||
"cpu_seconds": 10.569455,
|
||||
"peak_process_family_bytes_including_page_cache": 656396288,
|
||||
"whole_task_tokens_estimated": [54841, 54841, 54841]
|
||||
}
|
||||
},
|
||||
"runs": [
|
||||
{
|
||||
"case": "off-paired-1-complete",
|
||||
"mode": "complete",
|
||||
"output_schema_profile": "off",
|
||||
"runtime_revision": "01788bac863ac786bfbb648cad69b11e23b40a89",
|
||||
"runtime_worktree_status": "",
|
||||
"token_estimates": {
|
||||
"input_catalog_once": 47987,
|
||||
"arguments": 37273,
|
||||
"text_replies": 67408,
|
||||
"whole_task": 152668
|
||||
},
|
||||
"actual_model_usage": null,
|
||||
"wall_seconds": 12.215,
|
||||
"cpu_seconds": 11.872135,
|
||||
"peak_process_family_bytes_including_page_cache": 650854400,
|
||||
"calls": [
|
||||
{
|
||||
"name": "analyze_javascript_application",
|
||||
"elapsed_ms": 7275.801749,
|
||||
"is_error": false,
|
||||
"reply_bytes": 21651
|
||||
},
|
||||
{
|
||||
"name": "compare_source_to_bundle",
|
||||
"elapsed_ms": 81.34299699999974,
|
||||
"is_error": false,
|
||||
"reply_bytes": 427799
|
||||
},
|
||||
{
|
||||
"name": "export_evidence_bundle",
|
||||
"elapsed_ms": 74.90714800000023,
|
||||
"is_error": false,
|
||||
"reply_bytes": 342
|
||||
}
|
||||
],
|
||||
"retained_records": [
|
||||
{
|
||||
"evidence_id": "ev_094eb596bdb817bd982fd99d2324665598ae2e393727b655e338480593f3b8fe",
|
||||
"operation": "compare_source_to_bundle"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_4e4bc466268275396594c26f6a657f8d2fe77b85275be57983c41ea10366db7f",
|
||||
"operation": "analyze_javascript_application"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_a903f44a3fcc9a0bdf13e7cd4e0befb6d5b7efb4c594837a125e04e0d98f9c78",
|
||||
"operation": "record_unknown"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_e060ca3f86616ab4c6cf8fe50ec812a5f732e0d2156d31343fb13d1e90357089",
|
||||
"operation": "inspect_analysis_view"
|
||||
}
|
||||
],
|
||||
"proof": {
|
||||
"status": "passed",
|
||||
"method": "Independent Python JSON/SHA-256/POSIX path and basename joins; no REA parser or matcher.",
|
||||
"source_files": 180,
|
||||
"embedded_nodes": 99,
|
||||
"candidate_relations": 210,
|
||||
"strong_location_rows": 33,
|
||||
"limitation": "Static multiple-location projections; no semantic-equivalence or removal claim."
|
||||
},
|
||||
"resource_events": {
|
||||
"memory.events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
},
|
||||
"pids.events": {
|
||||
"max": 0
|
||||
}
|
||||
},
|
||||
"cleanup_verified": true
|
||||
},
|
||||
{
|
||||
"case": "off-paired-2-selected",
|
||||
"mode": "selected",
|
||||
"output_schema_profile": "off",
|
||||
"runtime_revision": "543b273d7adf581f6fd52c2938020207e455d3df",
|
||||
"runtime_worktree_status": " M experiments/analysis-efficiency/source-to-bundle-costs.py\n M experiments/analysis-efficiency/source-to-bundle.mjs\n M src/cli/sourceToBundleCommand.ts\n M tests/acceptance/applications/applicationWorkflowCli.test.ts\n M tests/boundary/mcp/sourceToBundleViewsMcp.test.ts\n",
|
||||
"token_estimates": {
|
||||
"input_catalog_once": 48100,
|
||||
"arguments": 298,
|
||||
"text_replies": 6443,
|
||||
"whole_task": 54841
|
||||
},
|
||||
"actual_model_usage": null,
|
||||
"wall_seconds": 11.196,
|
||||
"cpu_seconds": 10.7977,
|
||||
"peak_process_family_bytes_including_page_cache": 650358784,
|
||||
"calls": [
|
||||
{
|
||||
"name": "analyze_javascript_application",
|
||||
"elapsed_ms": 6273.303819000001,
|
||||
"is_error": false,
|
||||
"reply_bytes": 21651
|
||||
},
|
||||
{
|
||||
"name": "compare_source_to_bundle",
|
||||
"elapsed_ms": 81.66274700000031,
|
||||
"is_error": false,
|
||||
"reply_bytes": 13199
|
||||
},
|
||||
{
|
||||
"name": "inspect_analysis_view",
|
||||
"elapsed_ms": 20.52900899999986,
|
||||
"is_error": false,
|
||||
"reply_bytes": 19393
|
||||
},
|
||||
{
|
||||
"name": "export_evidence_bundle",
|
||||
"elapsed_ms": 83.38822599999912,
|
||||
"is_error": false,
|
||||
"reply_bytes": 342
|
||||
}
|
||||
],
|
||||
"retained_records": [
|
||||
{
|
||||
"evidence_id": "ev_02b198af5997beaf606bf300768cc1ac8cd608fcd0860acbe9b791dbd5bae5dc",
|
||||
"operation": "inspect_analysis_view"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_094eb596bdb817bd982fd99d2324665598ae2e393727b655e338480593f3b8fe",
|
||||
"operation": "compare_source_to_bundle"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_4e4bc466268275396594c26f6a657f8d2fe77b85275be57983c41ea10366db7f",
|
||||
"operation": "analyze_javascript_application"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_95d8fe5e75c019d86fc17bafeea32445e82eec06e0ec07d0ecdafd1e8bf8ef34",
|
||||
"operation": "inspect_analysis_view"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_a903f44a3fcc9a0bdf13e7cd4e0befb6d5b7efb4c594837a125e04e0d98f9c78",
|
||||
"operation": "record_unknown"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_e060ca3f86616ab4c6cf8fe50ec812a5f732e0d2156d31343fb13d1e90357089",
|
||||
"operation": "inspect_analysis_view"
|
||||
}
|
||||
],
|
||||
"proof": {
|
||||
"status": "passed",
|
||||
"method": "Independent Python JSON/SHA-256/POSIX path and basename joins; no REA parser or matcher.",
|
||||
"source_files": 180,
|
||||
"embedded_nodes": 99,
|
||||
"candidate_relations": 210,
|
||||
"strong_location_rows": 33,
|
||||
"limitation": "Static multiple-location projections; no semantic-equivalence or removal claim."
|
||||
},
|
||||
"resource_events": {
|
||||
"memory.events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
},
|
||||
"pids.events": {
|
||||
"max": 0
|
||||
}
|
||||
},
|
||||
"cleanup_verified": true
|
||||
},
|
||||
{
|
||||
"case": "off-paired-3-selected",
|
||||
"mode": "selected",
|
||||
"output_schema_profile": "off",
|
||||
"runtime_revision": "543b273d7adf581f6fd52c2938020207e455d3df",
|
||||
"runtime_worktree_status": " M experiments/analysis-efficiency/source-to-bundle-costs.py\n M experiments/analysis-efficiency/source-to-bundle.mjs\n M src/cli/sourceToBundleCommand.ts\n M tests/acceptance/applications/applicationWorkflowCli.test.ts\n M tests/boundary/mcp/sourceToBundleViewsMcp.test.ts\n",
|
||||
"token_estimates": {
|
||||
"input_catalog_once": 48100,
|
||||
"arguments": 298,
|
||||
"text_replies": 6443,
|
||||
"whole_task": 54841
|
||||
},
|
||||
"actual_model_usage": null,
|
||||
"wall_seconds": 10.89,
|
||||
"cpu_seconds": 10.569455,
|
||||
"peak_process_family_bytes_including_page_cache": 656396288,
|
||||
"calls": [
|
||||
{
|
||||
"name": "analyze_javascript_application",
|
||||
"elapsed_ms": 6153.662525000002,
|
||||
"is_error": false,
|
||||
"reply_bytes": 21651
|
||||
},
|
||||
{
|
||||
"name": "compare_source_to_bundle",
|
||||
"elapsed_ms": 78.89196699999957,
|
||||
"is_error": false,
|
||||
"reply_bytes": 13199
|
||||
},
|
||||
{
|
||||
"name": "inspect_analysis_view",
|
||||
"elapsed_ms": 20.370968999999604,
|
||||
"is_error": false,
|
||||
"reply_bytes": 19393
|
||||
},
|
||||
{
|
||||
"name": "export_evidence_bundle",
|
||||
"elapsed_ms": 81.88148700000056,
|
||||
"is_error": false,
|
||||
"reply_bytes": 342
|
||||
}
|
||||
],
|
||||
"retained_records": [
|
||||
{
|
||||
"evidence_id": "ev_02b198af5997beaf606bf300768cc1ac8cd608fcd0860acbe9b791dbd5bae5dc",
|
||||
"operation": "inspect_analysis_view"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_094eb596bdb817bd982fd99d2324665598ae2e393727b655e338480593f3b8fe",
|
||||
"operation": "compare_source_to_bundle"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_4e4bc466268275396594c26f6a657f8d2fe77b85275be57983c41ea10366db7f",
|
||||
"operation": "analyze_javascript_application"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_95d8fe5e75c019d86fc17bafeea32445e82eec06e0ec07d0ecdafd1e8bf8ef34",
|
||||
"operation": "inspect_analysis_view"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_a903f44a3fcc9a0bdf13e7cd4e0befb6d5b7efb4c594837a125e04e0d98f9c78",
|
||||
"operation": "record_unknown"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_e060ca3f86616ab4c6cf8fe50ec812a5f732e0d2156d31343fb13d1e90357089",
|
||||
"operation": "inspect_analysis_view"
|
||||
}
|
||||
],
|
||||
"proof": {
|
||||
"status": "passed",
|
||||
"method": "Independent Python JSON/SHA-256/POSIX path and basename joins; no REA parser or matcher.",
|
||||
"source_files": 180,
|
||||
"embedded_nodes": 99,
|
||||
"candidate_relations": 210,
|
||||
"strong_location_rows": 33,
|
||||
"limitation": "Static multiple-location projections; no semantic-equivalence or removal claim."
|
||||
},
|
||||
"resource_events": {
|
||||
"memory.events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
},
|
||||
"pids.events": {
|
||||
"max": 0
|
||||
}
|
||||
},
|
||||
"cleanup_verified": true
|
||||
},
|
||||
{
|
||||
"case": "off-paired-4-complete",
|
||||
"mode": "complete",
|
||||
"output_schema_profile": "off",
|
||||
"runtime_revision": "01788bac863ac786bfbb648cad69b11e23b40a89",
|
||||
"runtime_worktree_status": "",
|
||||
"token_estimates": {
|
||||
"input_catalog_once": 47987,
|
||||
"arguments": 37273,
|
||||
"text_replies": 67408,
|
||||
"whole_task": 152668
|
||||
},
|
||||
"actual_model_usage": null,
|
||||
"wall_seconds": 11.597,
|
||||
"cpu_seconds": 11.243998,
|
||||
"peak_process_family_bytes_including_page_cache": 659202048,
|
||||
"calls": [
|
||||
{
|
||||
"name": "analyze_javascript_application",
|
||||
"elapsed_ms": 6342.5286989999995,
|
||||
"is_error": false,
|
||||
"reply_bytes": 21651
|
||||
},
|
||||
{
|
||||
"name": "compare_source_to_bundle",
|
||||
"elapsed_ms": 90.04261700000097,
|
||||
"is_error": false,
|
||||
"reply_bytes": 427799
|
||||
},
|
||||
{
|
||||
"name": "export_evidence_bundle",
|
||||
"elapsed_ms": 97.75641600000017,
|
||||
"is_error": false,
|
||||
"reply_bytes": 342
|
||||
}
|
||||
],
|
||||
"retained_records": [
|
||||
{
|
||||
"evidence_id": "ev_094eb596bdb817bd982fd99d2324665598ae2e393727b655e338480593f3b8fe",
|
||||
"operation": "compare_source_to_bundle"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_4e4bc466268275396594c26f6a657f8d2fe77b85275be57983c41ea10366db7f",
|
||||
"operation": "analyze_javascript_application"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_a903f44a3fcc9a0bdf13e7cd4e0befb6d5b7efb4c594837a125e04e0d98f9c78",
|
||||
"operation": "record_unknown"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_e060ca3f86616ab4c6cf8fe50ec812a5f732e0d2156d31343fb13d1e90357089",
|
||||
"operation": "inspect_analysis_view"
|
||||
}
|
||||
],
|
||||
"proof": {
|
||||
"status": "passed",
|
||||
"method": "Independent Python JSON/SHA-256/POSIX path and basename joins; no REA parser or matcher.",
|
||||
"source_files": 180,
|
||||
"embedded_nodes": 99,
|
||||
"candidate_relations": 210,
|
||||
"strong_location_rows": 33,
|
||||
"limitation": "Static multiple-location projections; no semantic-equivalence or removal claim."
|
||||
},
|
||||
"resource_events": {
|
||||
"memory.events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
},
|
||||
"pids.events": {
|
||||
"max": 0
|
||||
}
|
||||
},
|
||||
"cleanup_verified": true
|
||||
},
|
||||
{
|
||||
"case": "off-paired-5-complete",
|
||||
"mode": "complete",
|
||||
"output_schema_profile": "off",
|
||||
"runtime_revision": "01788bac863ac786bfbb648cad69b11e23b40a89",
|
||||
"runtime_worktree_status": "",
|
||||
"token_estimates": {
|
||||
"input_catalog_once": 47987,
|
||||
"arguments": 37273,
|
||||
"text_replies": 67408,
|
||||
"whole_task": 152668
|
||||
},
|
||||
"actual_model_usage": null,
|
||||
"wall_seconds": 10.899,
|
||||
"cpu_seconds": 10.597762,
|
||||
"peak_process_family_bytes_including_page_cache": 663457792,
|
||||
"calls": [
|
||||
{
|
||||
"name": "analyze_javascript_application",
|
||||
"elapsed_ms": 6138.704318,
|
||||
"is_error": false,
|
||||
"reply_bytes": 21651
|
||||
},
|
||||
{
|
||||
"name": "compare_source_to_bundle",
|
||||
"elapsed_ms": 87.58297700000003,
|
||||
"is_error": false,
|
||||
"reply_bytes": 427799
|
||||
},
|
||||
{
|
||||
"name": "export_evidence_bundle",
|
||||
"elapsed_ms": 76.88620699999956,
|
||||
"is_error": false,
|
||||
"reply_bytes": 342
|
||||
}
|
||||
],
|
||||
"retained_records": [
|
||||
{
|
||||
"evidence_id": "ev_094eb596bdb817bd982fd99d2324665598ae2e393727b655e338480593f3b8fe",
|
||||
"operation": "compare_source_to_bundle"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_4e4bc466268275396594c26f6a657f8d2fe77b85275be57983c41ea10366db7f",
|
||||
"operation": "analyze_javascript_application"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_a903f44a3fcc9a0bdf13e7cd4e0befb6d5b7efb4c594837a125e04e0d98f9c78",
|
||||
"operation": "record_unknown"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_e060ca3f86616ab4c6cf8fe50ec812a5f732e0d2156d31343fb13d1e90357089",
|
||||
"operation": "inspect_analysis_view"
|
||||
}
|
||||
],
|
||||
"proof": {
|
||||
"status": "passed",
|
||||
"method": "Independent Python JSON/SHA-256/POSIX path and basename joins; no REA parser or matcher.",
|
||||
"source_files": 180,
|
||||
"embedded_nodes": 99,
|
||||
"candidate_relations": 210,
|
||||
"strong_location_rows": 33,
|
||||
"limitation": "Static multiple-location projections; no semantic-equivalence or removal claim."
|
||||
},
|
||||
"resource_events": {
|
||||
"memory.events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
},
|
||||
"pids.events": {
|
||||
"max": 0
|
||||
}
|
||||
},
|
||||
"cleanup_verified": true
|
||||
},
|
||||
{
|
||||
"case": "off-paired-6-selected",
|
||||
"mode": "selected",
|
||||
"output_schema_profile": "off",
|
||||
"runtime_revision": "543b273d7adf581f6fd52c2938020207e455d3df",
|
||||
"runtime_worktree_status": " M experiments/analysis-efficiency/source-to-bundle-costs.py\n M experiments/analysis-efficiency/source-to-bundle.mjs\n M src/cli/sourceToBundleCommand.ts\n M tests/acceptance/applications/applicationWorkflowCli.test.ts\n M tests/boundary/mcp/sourceToBundleViewsMcp.test.ts\n",
|
||||
"token_estimates": {
|
||||
"input_catalog_once": 48100,
|
||||
"arguments": 298,
|
||||
"text_replies": 6443,
|
||||
"whole_task": 54841
|
||||
},
|
||||
"actual_model_usage": null,
|
||||
"wall_seconds": 10.802,
|
||||
"cpu_seconds": 10.466533,
|
||||
"peak_process_family_bytes_including_page_cache": 660979712,
|
||||
"calls": [
|
||||
{
|
||||
"name": "analyze_javascript_application",
|
||||
"elapsed_ms": 6126.120589999999,
|
||||
"is_error": false,
|
||||
"reply_bytes": 21651
|
||||
},
|
||||
{
|
||||
"name": "compare_source_to_bundle",
|
||||
"elapsed_ms": 79.81148600000051,
|
||||
"is_error": false,
|
||||
"reply_bytes": 13199
|
||||
},
|
||||
{
|
||||
"name": "inspect_analysis_view",
|
||||
"elapsed_ms": 18.2459889999991,
|
||||
"is_error": false,
|
||||
"reply_bytes": 19393
|
||||
},
|
||||
{
|
||||
"name": "export_evidence_bundle",
|
||||
"elapsed_ms": 82.41327699999965,
|
||||
"is_error": false,
|
||||
"reply_bytes": 342
|
||||
}
|
||||
],
|
||||
"retained_records": [
|
||||
{
|
||||
"evidence_id": "ev_02b198af5997beaf606bf300768cc1ac8cd608fcd0860acbe9b791dbd5bae5dc",
|
||||
"operation": "inspect_analysis_view"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_094eb596bdb817bd982fd99d2324665598ae2e393727b655e338480593f3b8fe",
|
||||
"operation": "compare_source_to_bundle"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_4e4bc466268275396594c26f6a657f8d2fe77b85275be57983c41ea10366db7f",
|
||||
"operation": "analyze_javascript_application"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_95d8fe5e75c019d86fc17bafeea32445e82eec06e0ec07d0ecdafd1e8bf8ef34",
|
||||
"operation": "inspect_analysis_view"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_a903f44a3fcc9a0bdf13e7cd4e0befb6d5b7efb4c594837a125e04e0d98f9c78",
|
||||
"operation": "record_unknown"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_e060ca3f86616ab4c6cf8fe50ec812a5f732e0d2156d31343fb13d1e90357089",
|
||||
"operation": "inspect_analysis_view"
|
||||
}
|
||||
],
|
||||
"proof": {
|
||||
"status": "passed",
|
||||
"method": "Independent Python JSON/SHA-256/POSIX path and basename joins; no REA parser or matcher.",
|
||||
"source_files": 180,
|
||||
"embedded_nodes": 99,
|
||||
"candidate_relations": 210,
|
||||
"strong_location_rows": 33,
|
||||
"limitation": "Static multiple-location projections; no semantic-equivalence or removal claim."
|
||||
},
|
||||
"resource_events": {
|
||||
"memory.events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
},
|
||||
"pids.events": {
|
||||
"max": 0
|
||||
}
|
||||
},
|
||||
"cleanup_verified": true
|
||||
}
|
||||
],
|
||||
"baseline_revision": "01788bac8",
|
||||
"configuration_scope": "Both cohorts explicitly use existing REA_MCP_OUTPUT_SCHEMA_PROFILE=off. REA still canonical-validates successful results; clients do not receive output schemas. Defaults remain unchanged. Pilot off-1/off-2 and warm calibration excluded.",
|
||||
"implementation_files_sha256": {
|
||||
"src/application/javascript/SourceToBundleReference.ts": "00f590540b1d8a1755badd7318a4f65b472671e505989c98338f2c63dbe5f7fe",
|
||||
"src/cli/applicationCommands.ts": "7a436590d7d8f38c059b1ef9a008a20b6021eb04a6caafb72d473149f8762a00",
|
||||
"src/cli/sourceToBundleCommand.ts": "943ee98cd4abb6674a8f3d90d84d28744ff5a262200a57a45fc1a5b7fba9d105",
|
||||
"src/contracts/applicationToolContracts.ts": "3b910f1ad1baf13b428d2d108a0020b6235a06dc0bd2bfa2f4b3024ccf57100c",
|
||||
"src/contracts/javascript/applicationWorkflowInputContracts.ts": "4256f7de82fabf3cc946ca607b8429defa4e8b9f6d3f4be935c20f70741c2d42",
|
||||
"src/domain/analysisView/analysisView.ts": "56107291635491f3606d57fb85a2b2b175a4689d291d3855a899cb201203148a",
|
||||
"src/domain/analysisView/sourceToBundleView.ts": "a8cf9a894a4d0383ae109cfc6671c3ba9aa2b75b2b95e229b1297e55f8393de5",
|
||||
"src/server/createServer.ts": "400764b1ef9bad9d7250cc179736f20daf4cadcde622bbea34e30861b9b23e04",
|
||||
"src/server/registerApplicationTools/compareSourceToBundle.ts": "8796d139d011ccc89fecfc01dc0fea48b57d6da3823f5148120e34f868a9e0a0",
|
||||
"src/server/registerApplicationTools/helpers.ts": "c49ec1a9a28d531696012af5d731818deab3fe59f71f4bdcceb3fbb79736824a",
|
||||
"src/server/registerApplicationTools/types.ts": "959dbfc7162048da2bf477aaa545cb10c766f4e64a25ec5ff370d37244fc0eb7"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
"""Prepare a pinned public source/bundle corpus without extracting or executing package scripts."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path, PurePosixPath
|
||||
import tarfile
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('tarball', type=Path)
|
||||
parser.add_argument('output', type=Path)
|
||||
args = parser.parse_args()
|
||||
expected = 'f2517c466b50424b661a27e48039f15db7783e5822f46b5c7d39f20104ac67a9'
|
||||
assert hashlib.sha256(args.tarball.read_bytes()).hexdigest() == expected
|
||||
args.output.mkdir()
|
||||
bundles = ['redux-toolkit.browser.mjs', 'redux-toolkit.legacy-esm.js', 'redux-toolkit.modern.mjs']
|
||||
selected = {f'package/dist/{name}{suffix}' for name in bundles for suffix in ['', '.map']}
|
||||
with tarfile.open(args.tarball) as archive:
|
||||
for member in archive:
|
||||
if not member.isfile():
|
||||
continue
|
||||
path = PurePosixPath(member.name)
|
||||
if path.is_absolute() or '..' in path.parts:
|
||||
raise ValueError('Unsafe archive member')
|
||||
if not (member.name.startswith('package/src/') or member.name in selected):
|
||||
continue
|
||||
data = archive.extractfile(member).read()
|
||||
destination = args.output / 'artifact' / str(path)
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
destination.write_bytes(data)
|
||||
if member.name.startswith('package/src/'):
|
||||
layout = args.output / 'reference' / str(path.relative_to('package'))
|
||||
else:
|
||||
layout = args.output / 'artifact/shipped' / str(path.relative_to('package'))
|
||||
layout.parent.mkdir(parents=True, exist_ok=True)
|
||||
layout.write_bytes(data)
|
||||
rows = []
|
||||
for name in sorted(bundles):
|
||||
document = json.loads((args.output / 'artifact/package/dist' / (name + '.map')).read_text())
|
||||
assert len(document['sources']) == len(document['sourcesContent']) == 33
|
||||
for index, (source, content) in enumerate(zip(document['sources'], document['sourcesContent'])):
|
||||
assert source.startswith('../src/')
|
||||
relative = source.removeprefix('../src/')
|
||||
embedded_sha = hashlib.sha256(content.encode()).hexdigest()
|
||||
source_sha = hashlib.sha256((args.output / 'reference/src' / relative).read_bytes()).hexdigest()
|
||||
rows.append({'map': name + '.map', 'source_index': index, 'source_path': relative,
|
||||
'map_source': source, 'content_sha256': embedded_sha,
|
||||
'source_sha256': source_sha, 'exact': embedded_sha == source_sha})
|
||||
assert len(list((args.output / 'reference/src').rglob('*.*'))) == 180
|
||||
assert len(rows) == 99 and not any(row['exact'] for row in rows)
|
||||
(args.output / 'oracle.json').write_text(json.dumps({'independent_method': 'Python JSON and SHA-256 over published map sourcesContent versus published source files; no REA parser.', 'rows': rows}, indent=2) + '\n')
|
||||
print(json.dumps({'tarball_sha256': expected, 'source_files': 180, 'embedded_nodes': 99, 'executed_target_code': False}))
|
||||
@@ -0,0 +1,916 @@
|
||||
{
|
||||
"status": "passed",
|
||||
"tokenizer": "cl100k_base, tiktoken 0.12.0; estimates only; one input catalog plus arguments plus MCP text replies; not actual model usage",
|
||||
"cold_warm": "Fresh server process and isolated HOME each run; alternating complete/selected, selected/complete, complete/selected. OS page caches are warm/uncontrolled; not an OS-cold claim.",
|
||||
"tarball_sha256": "f2517c466b50424b661a27e48039f15db7783e5822f46b5c7d39f20104ac67a9",
|
||||
"preparation_scope": "Reference-source import is performed once outside these repeated SDK trials and reported separately. Both workflows use the explicit existing semantic cap of one for the static correspondence question; full semantic coverage is not claimed.",
|
||||
"summary": {
|
||||
"complete": {
|
||||
"wall_seconds": 11.899,
|
||||
"cpu_seconds": 11.551076,
|
||||
"peak_process_family_bytes_including_page_cache": 698986496,
|
||||
"whole_task_tokens_estimated": [152592, 152592, 152592]
|
||||
},
|
||||
"selected": {
|
||||
"wall_seconds": 14.733,
|
||||
"cpu_seconds": 14.376874,
|
||||
"peak_process_family_bytes_including_page_cache": 996917248,
|
||||
"whole_task_tokens_estimated": [54765, 54765, 54765]
|
||||
}
|
||||
},
|
||||
"runs": [
|
||||
{
|
||||
"case": "paired-1-complete",
|
||||
"mode": "complete",
|
||||
"runtime_revision": "aa51e4343963fdd0b97ec2ebdc3c41579fd9cb38",
|
||||
"runtime_worktree_status": "",
|
||||
"token_estimates": {
|
||||
"input_catalog_once": 47913,
|
||||
"arguments": 37272,
|
||||
"text_replies": 67407,
|
||||
"whole_task": 152592
|
||||
},
|
||||
"actual_model_usage": null,
|
||||
"wall_seconds": 12.618,
|
||||
"cpu_seconds": 12.259151,
|
||||
"peak_process_family_bytes_including_page_cache": 699125760,
|
||||
"calls": [
|
||||
{
|
||||
"name": "analyze_javascript_application",
|
||||
"elapsed_ms": 6728.166864999999,
|
||||
"is_error": false,
|
||||
"reply_bytes": 21651
|
||||
},
|
||||
{
|
||||
"name": "compare_source_to_bundle",
|
||||
"elapsed_ms": 205.6020669999998,
|
||||
"is_error": false,
|
||||
"reply_bytes": 427799
|
||||
},
|
||||
{
|
||||
"name": "export_evidence_bundle",
|
||||
"elapsed_ms": 77.03590900000017,
|
||||
"is_error": false,
|
||||
"reply_bytes": 334
|
||||
}
|
||||
],
|
||||
"retained_records": [
|
||||
{
|
||||
"evidence_id": "ev_094eb596bdb817bd982fd99d2324665598ae2e393727b655e338480593f3b8fe",
|
||||
"operation": "compare_source_to_bundle"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_4e4bc466268275396594c26f6a657f8d2fe77b85275be57983c41ea10366db7f",
|
||||
"operation": "analyze_javascript_application"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_a903f44a3fcc9a0bdf13e7cd4e0befb6d5b7efb4c594837a125e04e0d98f9c78",
|
||||
"operation": "record_unknown"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_e060ca3f86616ab4c6cf8fe50ec812a5f732e0d2156d31343fb13d1e90357089",
|
||||
"operation": "inspect_analysis_view"
|
||||
}
|
||||
],
|
||||
"proof": {
|
||||
"status": "passed",
|
||||
"method": "Independent Python JSON/SHA-256/POSIX path and basename joins; no REA parser or matcher.",
|
||||
"source_files": 180,
|
||||
"embedded_nodes": 99,
|
||||
"candidate_relations": 210,
|
||||
"strong_location_rows": 33,
|
||||
"limitation": "Static multiple-location projections; no semantic-equivalence or removal claim."
|
||||
},
|
||||
"resource_events": {
|
||||
"memory.events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
},
|
||||
"pids.events": {
|
||||
"max": 0
|
||||
}
|
||||
},
|
||||
"cleanup_verified": true
|
||||
},
|
||||
{
|
||||
"case": "paired-2-selected",
|
||||
"mode": "selected",
|
||||
"runtime_revision": "aa51e4343963fdd0b97ec2ebdc3c41579fd9cb38",
|
||||
"runtime_worktree_status": " M src/cli/applicationCommands.ts\n M src/contracts/applicationToolContracts.ts\n M src/contracts/javascript/applicationWorkflowInputContracts.ts\n M src/domain/analysisView/analysisView.ts\n M src/server/createServer.ts\n M src/server/registerApplicationTools/compareSourceToBundle.ts\n M src/server/registerApplicationTools/helpers.ts\n M src/server/registerApplicationTools/types.ts\n M tests/acceptance/applications/applicationWorkflowCli.test.ts\n?? experiments/analysis-efficiency/source-to-bundle.mjs\n?? src/application/javascript/SourceToBundleReference.ts\n?? src/cli/sourceToBundleCommand.ts\n?? src/domain/analysisView/sourceToBundleView.ts\n?? tests/boundary/mcp/sourceToBundleViewsMcp.test.ts\n",
|
||||
"token_estimates": {
|
||||
"input_catalog_once": 48026,
|
||||
"arguments": 297,
|
||||
"text_replies": 6442,
|
||||
"whole_task": 54765
|
||||
},
|
||||
"actual_model_usage": null,
|
||||
"wall_seconds": 15.327,
|
||||
"cpu_seconds": 14.954805,
|
||||
"peak_process_family_bytes_including_page_cache": 997220352,
|
||||
"calls": [
|
||||
{
|
||||
"name": "analyze_javascript_application",
|
||||
"elapsed_ms": 7057.07386,
|
||||
"is_error": false,
|
||||
"reply_bytes": 21651
|
||||
},
|
||||
{
|
||||
"name": "compare_source_to_bundle",
|
||||
"elapsed_ms": 261.767695999999,
|
||||
"is_error": false,
|
||||
"reply_bytes": 13199
|
||||
},
|
||||
{
|
||||
"name": "inspect_analysis_view",
|
||||
"elapsed_ms": 2391.5776320000004,
|
||||
"is_error": false,
|
||||
"reply_bytes": 19393
|
||||
},
|
||||
{
|
||||
"name": "export_evidence_bundle",
|
||||
"elapsed_ms": 179.67298699999992,
|
||||
"is_error": false,
|
||||
"reply_bytes": 334
|
||||
}
|
||||
],
|
||||
"retained_records": [
|
||||
{
|
||||
"evidence_id": "ev_02b198af5997beaf606bf300768cc1ac8cd608fcd0860acbe9b791dbd5bae5dc",
|
||||
"operation": "inspect_analysis_view"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_094eb596bdb817bd982fd99d2324665598ae2e393727b655e338480593f3b8fe",
|
||||
"operation": "compare_source_to_bundle"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_4e4bc466268275396594c26f6a657f8d2fe77b85275be57983c41ea10366db7f",
|
||||
"operation": "analyze_javascript_application"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_95d8fe5e75c019d86fc17bafeea32445e82eec06e0ec07d0ecdafd1e8bf8ef34",
|
||||
"operation": "inspect_analysis_view"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_a903f44a3fcc9a0bdf13e7cd4e0befb6d5b7efb4c594837a125e04e0d98f9c78",
|
||||
"operation": "record_unknown"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_e060ca3f86616ab4c6cf8fe50ec812a5f732e0d2156d31343fb13d1e90357089",
|
||||
"operation": "inspect_analysis_view"
|
||||
}
|
||||
],
|
||||
"proof": {
|
||||
"status": "passed",
|
||||
"method": "Independent Python JSON/SHA-256/POSIX path and basename joins; no REA parser or matcher.",
|
||||
"source_files": 180,
|
||||
"embedded_nodes": 99,
|
||||
"candidate_relations": 210,
|
||||
"strong_location_rows": 33,
|
||||
"limitation": "Static multiple-location projections; no semantic-equivalence or removal claim."
|
||||
},
|
||||
"resource_events": {
|
||||
"memory.events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
},
|
||||
"pids.events": {
|
||||
"max": 0
|
||||
}
|
||||
},
|
||||
"cleanup_verified": true
|
||||
},
|
||||
{
|
||||
"case": "paired-3-selected",
|
||||
"mode": "selected",
|
||||
"runtime_revision": "aa51e4343963fdd0b97ec2ebdc3c41579fd9cb38",
|
||||
"runtime_worktree_status": " M src/cli/applicationCommands.ts\n M src/contracts/applicationToolContracts.ts\n M src/contracts/javascript/applicationWorkflowInputContracts.ts\n M src/domain/analysisView/analysisView.ts\n M src/server/createServer.ts\n M src/server/registerApplicationTools/compareSourceToBundle.ts\n M src/server/registerApplicationTools/helpers.ts\n M src/server/registerApplicationTools/types.ts\n M tests/acceptance/applications/applicationWorkflowCli.test.ts\n?? experiments/analysis-efficiency/source-to-bundle-oracle.py\n?? experiments/analysis-efficiency/source-to-bundle.mjs\n?? src/application/javascript/SourceToBundleReference.ts\n?? src/cli/sourceToBundleCommand.ts\n?? src/domain/analysisView/sourceToBundleView.ts\n?? tests/boundary/mcp/sourceToBundleViewsMcp.test.ts\n",
|
||||
"token_estimates": {
|
||||
"input_catalog_once": 48026,
|
||||
"arguments": 297,
|
||||
"text_replies": 6442,
|
||||
"whole_task": 54765
|
||||
},
|
||||
"actual_model_usage": null,
|
||||
"wall_seconds": 14.117,
|
||||
"cpu_seconds": 13.83662,
|
||||
"peak_process_family_bytes_including_page_cache": 996917248,
|
||||
"calls": [
|
||||
{
|
||||
"name": "analyze_javascript_application",
|
||||
"elapsed_ms": 6560.107647999999,
|
||||
"is_error": false,
|
||||
"reply_bytes": 21651
|
||||
},
|
||||
{
|
||||
"name": "compare_source_to_bundle",
|
||||
"elapsed_ms": 248.57295700000032,
|
||||
"is_error": false,
|
||||
"reply_bytes": 13199
|
||||
},
|
||||
{
|
||||
"name": "inspect_analysis_view",
|
||||
"elapsed_ms": 2105.5653280000006,
|
||||
"is_error": false,
|
||||
"reply_bytes": 19393
|
||||
},
|
||||
{
|
||||
"name": "export_evidence_bundle",
|
||||
"elapsed_ms": 163.21972800000003,
|
||||
"is_error": false,
|
||||
"reply_bytes": 334
|
||||
}
|
||||
],
|
||||
"retained_records": [
|
||||
{
|
||||
"evidence_id": "ev_02b198af5997beaf606bf300768cc1ac8cd608fcd0860acbe9b791dbd5bae5dc",
|
||||
"operation": "inspect_analysis_view"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_094eb596bdb817bd982fd99d2324665598ae2e393727b655e338480593f3b8fe",
|
||||
"operation": "compare_source_to_bundle"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_4e4bc466268275396594c26f6a657f8d2fe77b85275be57983c41ea10366db7f",
|
||||
"operation": "analyze_javascript_application"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_95d8fe5e75c019d86fc17bafeea32445e82eec06e0ec07d0ecdafd1e8bf8ef34",
|
||||
"operation": "inspect_analysis_view"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_a903f44a3fcc9a0bdf13e7cd4e0befb6d5b7efb4c594837a125e04e0d98f9c78",
|
||||
"operation": "record_unknown"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_e060ca3f86616ab4c6cf8fe50ec812a5f732e0d2156d31343fb13d1e90357089",
|
||||
"operation": "inspect_analysis_view"
|
||||
}
|
||||
],
|
||||
"proof": {
|
||||
"status": "passed",
|
||||
"method": "Independent Python JSON/SHA-256/POSIX path and basename joins; no REA parser or matcher.",
|
||||
"source_files": 180,
|
||||
"embedded_nodes": 99,
|
||||
"candidate_relations": 210,
|
||||
"strong_location_rows": 33,
|
||||
"limitation": "Static multiple-location projections; no semantic-equivalence or removal claim."
|
||||
},
|
||||
"resource_events": {
|
||||
"memory.events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
},
|
||||
"pids.events": {
|
||||
"max": 0
|
||||
}
|
||||
},
|
||||
"cleanup_verified": true
|
||||
},
|
||||
{
|
||||
"case": "paired-4-complete",
|
||||
"mode": "complete",
|
||||
"runtime_revision": "aa51e4343963fdd0b97ec2ebdc3c41579fd9cb38",
|
||||
"runtime_worktree_status": "",
|
||||
"token_estimates": {
|
||||
"input_catalog_once": 47913,
|
||||
"arguments": 37272,
|
||||
"text_replies": 67407,
|
||||
"whole_task": 152592
|
||||
},
|
||||
"actual_model_usage": null,
|
||||
"wall_seconds": 11.899,
|
||||
"cpu_seconds": 11.551076,
|
||||
"peak_process_family_bytes_including_page_cache": 687702016,
|
||||
"calls": [
|
||||
{
|
||||
"name": "analyze_javascript_application",
|
||||
"elapsed_ms": 6484.389229,
|
||||
"is_error": false,
|
||||
"reply_bytes": 21651
|
||||
},
|
||||
{
|
||||
"name": "compare_source_to_bundle",
|
||||
"elapsed_ms": 216.41979700000047,
|
||||
"is_error": false,
|
||||
"reply_bytes": 427799
|
||||
},
|
||||
{
|
||||
"name": "export_evidence_bundle",
|
||||
"elapsed_ms": 83.66305800000009,
|
||||
"is_error": false,
|
||||
"reply_bytes": 334
|
||||
}
|
||||
],
|
||||
"retained_records": [
|
||||
{
|
||||
"evidence_id": "ev_094eb596bdb817bd982fd99d2324665598ae2e393727b655e338480593f3b8fe",
|
||||
"operation": "compare_source_to_bundle"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_4e4bc466268275396594c26f6a657f8d2fe77b85275be57983c41ea10366db7f",
|
||||
"operation": "analyze_javascript_application"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_a903f44a3fcc9a0bdf13e7cd4e0befb6d5b7efb4c594837a125e04e0d98f9c78",
|
||||
"operation": "record_unknown"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_e060ca3f86616ab4c6cf8fe50ec812a5f732e0d2156d31343fb13d1e90357089",
|
||||
"operation": "inspect_analysis_view"
|
||||
}
|
||||
],
|
||||
"proof": {
|
||||
"status": "passed",
|
||||
"method": "Independent Python JSON/SHA-256/POSIX path and basename joins; no REA parser or matcher.",
|
||||
"source_files": 180,
|
||||
"embedded_nodes": 99,
|
||||
"candidate_relations": 210,
|
||||
"strong_location_rows": 33,
|
||||
"limitation": "Static multiple-location projections; no semantic-equivalence or removal claim."
|
||||
},
|
||||
"resource_events": {
|
||||
"memory.events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
},
|
||||
"pids.events": {
|
||||
"max": 0
|
||||
}
|
||||
},
|
||||
"cleanup_verified": true
|
||||
},
|
||||
{
|
||||
"case": "paired-5-complete",
|
||||
"mode": "complete",
|
||||
"runtime_revision": "aa51e4343963fdd0b97ec2ebdc3c41579fd9cb38",
|
||||
"runtime_worktree_status": "",
|
||||
"token_estimates": {
|
||||
"input_catalog_once": 47913,
|
||||
"arguments": 37272,
|
||||
"text_replies": 67407,
|
||||
"whole_task": 152592
|
||||
},
|
||||
"actual_model_usage": null,
|
||||
"wall_seconds": 11.608,
|
||||
"cpu_seconds": 11.240981,
|
||||
"peak_process_family_bytes_including_page_cache": 698986496,
|
||||
"calls": [
|
||||
{
|
||||
"name": "analyze_javascript_application",
|
||||
"elapsed_ms": 6322.663412,
|
||||
"is_error": false,
|
||||
"reply_bytes": 21651
|
||||
},
|
||||
{
|
||||
"name": "compare_source_to_bundle",
|
||||
"elapsed_ms": 206.36909699999887,
|
||||
"is_error": false,
|
||||
"reply_bytes": 427799
|
||||
},
|
||||
{
|
||||
"name": "export_evidence_bundle",
|
||||
"elapsed_ms": 84.77780899999925,
|
||||
"is_error": false,
|
||||
"reply_bytes": 334
|
||||
}
|
||||
],
|
||||
"retained_records": [
|
||||
{
|
||||
"evidence_id": "ev_094eb596bdb817bd982fd99d2324665598ae2e393727b655e338480593f3b8fe",
|
||||
"operation": "compare_source_to_bundle"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_4e4bc466268275396594c26f6a657f8d2fe77b85275be57983c41ea10366db7f",
|
||||
"operation": "analyze_javascript_application"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_a903f44a3fcc9a0bdf13e7cd4e0befb6d5b7efb4c594837a125e04e0d98f9c78",
|
||||
"operation": "record_unknown"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_e060ca3f86616ab4c6cf8fe50ec812a5f732e0d2156d31343fb13d1e90357089",
|
||||
"operation": "inspect_analysis_view"
|
||||
}
|
||||
],
|
||||
"proof": {
|
||||
"status": "passed",
|
||||
"method": "Independent Python JSON/SHA-256/POSIX path and basename joins; no REA parser or matcher.",
|
||||
"source_files": 180,
|
||||
"embedded_nodes": 99,
|
||||
"candidate_relations": 210,
|
||||
"strong_location_rows": 33,
|
||||
"limitation": "Static multiple-location projections; no semantic-equivalence or removal claim."
|
||||
},
|
||||
"resource_events": {
|
||||
"memory.events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
},
|
||||
"pids.events": {
|
||||
"max": 0
|
||||
}
|
||||
},
|
||||
"cleanup_verified": true
|
||||
},
|
||||
{
|
||||
"case": "paired-6-selected",
|
||||
"mode": "selected",
|
||||
"runtime_revision": "aa51e4343963fdd0b97ec2ebdc3c41579fd9cb38",
|
||||
"runtime_worktree_status": " M src/cli/applicationCommands.ts\n M src/contracts/applicationToolContracts.ts\n M src/contracts/javascript/applicationWorkflowInputContracts.ts\n M src/domain/analysisView/analysisView.ts\n M src/server/createServer.ts\n M src/server/registerApplicationTools/compareSourceToBundle.ts\n M src/server/registerApplicationTools/helpers.ts\n M src/server/registerApplicationTools/types.ts\n M tests/acceptance/applications/applicationWorkflowCli.test.ts\n?? experiments/analysis-efficiency/source-to-bundle-oracle.py\n?? experiments/analysis-efficiency/source-to-bundle-prepare.py\n?? experiments/analysis-efficiency/source-to-bundle.mjs\n?? src/application/javascript/SourceToBundleReference.ts\n?? src/cli/sourceToBundleCommand.ts\n?? src/domain/analysisView/sourceToBundleView.ts\n?? tests/boundary/mcp/sourceToBundleViewsMcp.test.ts\n",
|
||||
"token_estimates": {
|
||||
"input_catalog_once": 48026,
|
||||
"arguments": 297,
|
||||
"text_replies": 6442,
|
||||
"whole_task": 54765
|
||||
},
|
||||
"actual_model_usage": null,
|
||||
"wall_seconds": 14.733,
|
||||
"cpu_seconds": 14.376874,
|
||||
"peak_process_family_bytes_including_page_cache": 992952320,
|
||||
"calls": [
|
||||
{
|
||||
"name": "analyze_javascript_application",
|
||||
"elapsed_ms": 6604.5753270000005,
|
||||
"is_error": false,
|
||||
"reply_bytes": 21651
|
||||
},
|
||||
{
|
||||
"name": "compare_source_to_bundle",
|
||||
"elapsed_ms": 257.93621600000006,
|
||||
"is_error": false,
|
||||
"reply_bytes": 13199
|
||||
},
|
||||
{
|
||||
"name": "inspect_analysis_view",
|
||||
"elapsed_ms": 2186.2845859999998,
|
||||
"is_error": false,
|
||||
"reply_bytes": 19393
|
||||
},
|
||||
{
|
||||
"name": "export_evidence_bundle",
|
||||
"elapsed_ms": 170.3188169999994,
|
||||
"is_error": false,
|
||||
"reply_bytes": 334
|
||||
}
|
||||
],
|
||||
"retained_records": [
|
||||
{
|
||||
"evidence_id": "ev_02b198af5997beaf606bf300768cc1ac8cd608fcd0860acbe9b791dbd5bae5dc",
|
||||
"operation": "inspect_analysis_view"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_094eb596bdb817bd982fd99d2324665598ae2e393727b655e338480593f3b8fe",
|
||||
"operation": "compare_source_to_bundle"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_4e4bc466268275396594c26f6a657f8d2fe77b85275be57983c41ea10366db7f",
|
||||
"operation": "analyze_javascript_application"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_95d8fe5e75c019d86fc17bafeea32445e82eec06e0ec07d0ecdafd1e8bf8ef34",
|
||||
"operation": "inspect_analysis_view"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_a903f44a3fcc9a0bdf13e7cd4e0befb6d5b7efb4c594837a125e04e0d98f9c78",
|
||||
"operation": "record_unknown"
|
||||
},
|
||||
{
|
||||
"evidence_id": "ev_e060ca3f86616ab4c6cf8fe50ec812a5f732e0d2156d31343fb13d1e90357089",
|
||||
"operation": "inspect_analysis_view"
|
||||
}
|
||||
],
|
||||
"proof": {
|
||||
"status": "passed",
|
||||
"method": "Independent Python JSON/SHA-256/POSIX path and basename joins; no REA parser or matcher.",
|
||||
"source_files": 180,
|
||||
"embedded_nodes": 99,
|
||||
"candidate_relations": 210,
|
||||
"strong_location_rows": 33,
|
||||
"limitation": "Static multiple-location projections; no semantic-equivalence or removal claim."
|
||||
},
|
||||
"resource_events": {
|
||||
"memory.events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
},
|
||||
"pids.events": {
|
||||
"max": 0
|
||||
}
|
||||
},
|
||||
"cleanup_verified": true
|
||||
}
|
||||
],
|
||||
"implementation_files_sha256": {
|
||||
"src/application/javascript/SourceToBundleReference.ts": "00f590540b1d8a1755badd7318a4f65b472671e505989c98338f2c63dbe5f7fe",
|
||||
"src/cli/applicationCommands.ts": "7a436590d7d8f38c059b1ef9a008a20b6021eb04a6caafb72d473149f8762a00",
|
||||
"src/cli/sourceToBundleCommand.ts": "70887d0a060c2b315df2eeaabb5889f5e47e61da79dbc9ca076e49b8858c3571",
|
||||
"src/contracts/applicationToolContracts.ts": "3b910f1ad1baf13b428d2d108a0020b6235a06dc0bd2bfa2f4b3024ccf57100c",
|
||||
"src/contracts/javascript/applicationWorkflowInputContracts.ts": "4256f7de82fabf3cc946ca607b8429defa4e8b9f6d3f4be935c20f70741c2d42",
|
||||
"src/domain/analysisView/analysisView.ts": "56107291635491f3606d57fb85a2b2b175a4689d291d3855a899cb201203148a",
|
||||
"src/domain/analysisView/sourceToBundleView.ts": "a8cf9a894a4d0383ae109cfc6671c3ba9aa2b75b2b95e229b1297e55f8393de5",
|
||||
"src/server/createServer.ts": "400764b1ef9bad9d7250cc179736f20daf4cadcde622bbea34e30861b9b23e04",
|
||||
"src/server/registerApplicationTools/compareSourceToBundle.ts": "8796d139d011ccc89fecfc01dc0fea48b57d6da3823f5148120e34f868a9e0a0",
|
||||
"src/server/registerApplicationTools/helpers.ts": "c49ec1a9a28d531696012af5d731818deab3fe59f71f4bdcceb3fbb79736824a",
|
||||
"src/server/registerApplicationTools/types.ts": "959dbfc7162048da2bf477aaa545cb10c766f4e64a25ec5ff370d37244fc0eb7"
|
||||
},
|
||||
"separate_preparation": {
|
||||
"reference_import": {
|
||||
"wall_seconds": 4.757,
|
||||
"cpu_seconds": 4.602731,
|
||||
"peak_process_family_bytes_including_page_cache": 383381504,
|
||||
"cleanup_verified": true
|
||||
},
|
||||
"application_default": {
|
||||
"wall_seconds": 25.824,
|
||||
"cpu_seconds": 25.433681,
|
||||
"peak_process_family_bytes_including_page_cache": 1100005376,
|
||||
"cleanup_verified": true
|
||||
},
|
||||
"application_explicit_semantic_one": {
|
||||
"wall_seconds": 12.933,
|
||||
"cpu_seconds": 12.600848,
|
||||
"peak_process_family_bytes_including_page_cache": 731721728,
|
||||
"cleanup_verified": true
|
||||
},
|
||||
"semantic_scope": "Comparison facts exactly agree except lineage across default and semantic-one; full semantic coverage is not equivalent. These two one-run SDK observations are not repeatable speedup evidence."
|
||||
},
|
||||
"native_cli_control": {
|
||||
"wall_seconds": 2.842,
|
||||
"cpu_seconds": 2.669896,
|
||||
"peak_process_family_bytes_including_page_cache": 374730752,
|
||||
"cleanup_verified": true,
|
||||
"comparison_bytes": 205231,
|
||||
"external_three_row_projection_bytes": 22108,
|
||||
"scope": "Public CLI full Evidence exactly equals MCP. An external Python projection is already possible; raw SDK context reduction does not establish actual model savings over code composition. Byte sizes are not token counts."
|
||||
},
|
||||
"warm_continuation_calibration": {
|
||||
"calls": [
|
||||
{
|
||||
"name": "analyze_javascript_application",
|
||||
"elapsed_ms": 6546.054383000001,
|
||||
"is_error": false,
|
||||
"reply_bytes": 21651
|
||||
},
|
||||
{
|
||||
"name": "compare_source_to_bundle",
|
||||
"elapsed_ms": 250.82817400000022,
|
||||
"is_error": false,
|
||||
"reply_bytes": 13199
|
||||
},
|
||||
{
|
||||
"name": "inspect_analysis_view",
|
||||
"elapsed_ms": 2071.093723,
|
||||
"is_error": false,
|
||||
"reply_bytes": 19393
|
||||
},
|
||||
{
|
||||
"name": "inspect_analysis_view",
|
||||
"elapsed_ms": 71.82136900000114,
|
||||
"is_error": false,
|
||||
"reply_bytes": 19393
|
||||
},
|
||||
{
|
||||
"name": "export_evidence_bundle",
|
||||
"elapsed_ms": 97.2455460000001,
|
||||
"is_error": false,
|
||||
"reply_bytes": 332
|
||||
}
|
||||
],
|
||||
"scope": "One additional fresh-process trial, excluded from paired whole-task medians. Second inspect_analysis_view returns identical Evidence with an already reused SDK validator; full parent remains retained.",
|
||||
"wall_seconds": 14.218,
|
||||
"cpu_seconds": 13.858723,
|
||||
"peak_process_family_bytes_including_page_cache": 993660928,
|
||||
"cleanup_verified": true
|
||||
},
|
||||
"subsequent_cli_error_envelope_fix": {
|
||||
"file": "src/cli/sourceToBundleCommand.ts",
|
||||
"sha256": "943ee98cd4abb6674a8f3d90d84d28744ff5a262200a57a45fc1a5b7fba9d105",
|
||||
"scope": "Preserves the previous error: Application workflow failed field around finishAnalysisCommand failures. Original paired MCP measurements predate this CLI-only negative-result fix. Actual CLI wrong-application output now exactly equals the baseline, including all projected diagnostic fields. MCP producer/view implementation is unchanged."
|
||||
},
|
||||
"subsequent_review_guards": {
|
||||
"scope": "Additional post-review checks, excluded from the original paired medians. Shared JSON authentication headroom now precedes graph validation; cancellation dispatch follows validation and all pure comparison/view computation before any session writes. Candidate pages now have cross-source boundary regression coverage. Matching/scoring and complete comparison Evidence are unchanged.",
|
||||
"implementation_files_sha256": {
|
||||
"src/application/JsonAuthenticationHeadroom.ts": "9f3cd40e0dc204d30a1c979c9e18535344963edbd2e75c47f633fa72a6447f96",
|
||||
"src/application/analysisView/evidenceFileSource.ts": "5b7adedb474de0ac9a1576f46a801db8232d473189689129ed3aece13322a42a",
|
||||
"src/application/javascript/SourceToBundleReference.ts": "a65aa4b5853001d133a4af385dc70c051144dfc7e4e22cd0b919bce5fa65ccc4",
|
||||
"src/cli/applicationCommands.ts": "7a436590d7d8f38c059b1ef9a008a20b6021eb04a6caafb72d473149f8762a00",
|
||||
"src/cli/sourceToBundleCommand.ts": "943ee98cd4abb6674a8f3d90d84d28744ff5a262200a57a45fc1a5b7fba9d105",
|
||||
"src/contracts/applicationToolContracts.ts": "3b910f1ad1baf13b428d2d108a0020b6235a06dc0bd2bfa2f4b3024ccf57100c",
|
||||
"src/contracts/javascript/applicationWorkflowInputContracts.ts": "4256f7de82fabf3cc946ca607b8429defa4e8b9f6d3f4be935c20f70741c2d42",
|
||||
"src/domain/analysisView/analysisView.ts": "56107291635491f3606d57fb85a2b2b175a4689d291d3855a899cb201203148a",
|
||||
"src/domain/analysisView/sourceToBundleView.ts": "a8cf9a894a4d0383ae109cfc6671c3ba9aa2b75b2b95e229b1297e55f8393de5",
|
||||
"src/server/createServer.ts": "400764b1ef9bad9d7250cc179736f20daf4cadcde622bbea34e30861b9b23e04",
|
||||
"src/server/registerApplicationTools/compareSourceToBundle.ts": "a5a3f125c9fb31b39fe50ae9135b9f3d76a2ce08e245fcb0a352ec550bef0083",
|
||||
"src/server/registerApplicationTools/helpers.ts": "c49ec1a9a28d531696012af5d731818deab3fe59f71f4bdcceb3fbb79736824a",
|
||||
"src/server/registerApplicationTools/types.ts": "959dbfc7162048da2bf477aaa545cb10c766f4e64a25ec5ff370d37244fc0eb7"
|
||||
},
|
||||
"regressions": {
|
||||
"shared_headroom_and_evidence_reader_tests": 7,
|
||||
"first_post_review_guard_suites": 12,
|
||||
"final_cancellation_and_cross_source_page_suites": 65,
|
||||
"negative_control_disabled_guards": {
|
||||
"expected_failed_tests": 3,
|
||||
"scope": "All three new guard regressions fail when the admission and cancellation gates are temporarily disabled; production source restored before successful checks."
|
||||
}
|
||||
},
|
||||
"verification": [
|
||||
{
|
||||
"profile": "full",
|
||||
"calls": [
|
||||
{
|
||||
"name": "analyze_javascript_application",
|
||||
"elapsed_ms": 6483.442064,
|
||||
"is_error": false,
|
||||
"reply_bytes": 21651
|
||||
},
|
||||
{
|
||||
"name": "compare_source_to_bundle",
|
||||
"elapsed_ms": 252.73620699999992,
|
||||
"is_error": false,
|
||||
"reply_bytes": 13199
|
||||
},
|
||||
{
|
||||
"name": "inspect_analysis_view",
|
||||
"elapsed_ms": 2205.9964070000005,
|
||||
"is_error": false,
|
||||
"reply_bytes": 19393
|
||||
},
|
||||
{
|
||||
"name": "export_evidence_bundle",
|
||||
"elapsed_ms": 85.7624880000003,
|
||||
"is_error": false,
|
||||
"reply_bytes": 332
|
||||
}
|
||||
],
|
||||
"wall_seconds": 14.217,
|
||||
"cpu_seconds": 13.843447,
|
||||
"peak_process_family_bytes_including_page_cache": 976711680,
|
||||
"complete_parent_byte_equal_to_original": true,
|
||||
"whole_task_cl100k_estimate": 54835,
|
||||
"independent_proof": {
|
||||
"status": "passed",
|
||||
"method": "Independent Python JSON/SHA-256/POSIX path and basename joins; no REA parser or matcher.",
|
||||
"source_files": 180,
|
||||
"embedded_nodes": 99,
|
||||
"candidate_relations": 210,
|
||||
"strong_location_rows": 33,
|
||||
"limitation": "Static multiple-location projections; no semantic-equivalence or removal claim."
|
||||
},
|
||||
"cleanup_verified": true,
|
||||
"resource_events": {
|
||||
"memory.events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
},
|
||||
"pids.events": {
|
||||
"max": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"profile": "off",
|
||||
"calls": [
|
||||
{
|
||||
"name": "analyze_javascript_application",
|
||||
"elapsed_ms": 6463.359534,
|
||||
"is_error": false,
|
||||
"reply_bytes": 21651
|
||||
},
|
||||
{
|
||||
"name": "compare_source_to_bundle",
|
||||
"elapsed_ms": 94.13822900000014,
|
||||
"is_error": false,
|
||||
"reply_bytes": 13199
|
||||
},
|
||||
{
|
||||
"name": "inspect_analysis_view",
|
||||
"elapsed_ms": 17.32437199999913,
|
||||
"is_error": false,
|
||||
"reply_bytes": 19393
|
||||
},
|
||||
{
|
||||
"name": "export_evidence_bundle",
|
||||
"elapsed_ms": 79.37489800000003,
|
||||
"is_error": false,
|
||||
"reply_bytes": 330
|
||||
}
|
||||
],
|
||||
"wall_seconds": 11.297,
|
||||
"cpu_seconds": 10.913648,
|
||||
"peak_process_family_bytes_including_page_cache": 656244736,
|
||||
"complete_parent_byte_equal_to_original": true,
|
||||
"whole_task_cl100k_estimate": 54835,
|
||||
"independent_proof": {
|
||||
"status": "passed",
|
||||
"method": "Independent Python JSON/SHA-256/POSIX path and basename joins; no REA parser or matcher.",
|
||||
"source_files": 180,
|
||||
"embedded_nodes": 99,
|
||||
"candidate_relations": 210,
|
||||
"strong_location_rows": 33,
|
||||
"limitation": "Static multiple-location projections; no semantic-equivalence or removal claim."
|
||||
},
|
||||
"cleanup_verified": true,
|
||||
"resource_events": {
|
||||
"memory.events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
},
|
||||
"pids.events": {
|
||||
"max": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"subsequent_reference_provenance": {
|
||||
"scope": "Final post-review verification, excluded from historical paired medians. File calls now preserve reference_source kind/path/requested_path from the same reader. Full Evidence identity intentionally distinguishes chosen file origins; inline output and normalized comparison facts are unchanged. Read-I/O cancellations report the comparison operation and preserve their cause.",
|
||||
"implementation_files_sha256": {
|
||||
"src/application/JsonAuthenticationHeadroom.ts": "9f3cd40e0dc204d30a1c979c9e18535344963edbd2e75c47f633fa72a6447f96",
|
||||
"src/application/JsonFiles.ts": "dfc6eaf9b6d0b7f8d4e12d2c39f9e0c5288a29a16fae0451f4c7be1ee2e31f65",
|
||||
"src/application/analysisView/evidenceFileSource.ts": "5b7adedb474de0ac9a1576f46a801db8232d473189689129ed3aece13322a42a",
|
||||
"src/application/javascript/JavaScriptApplicationWorkflowService.ts": "657e36d2a20aee2722c76d6494d092db2d06ce5cfa6e9ba0506bcf13aa4e9d1d",
|
||||
"src/application/javascript/SourceToBundleReference.ts": "8206198666a249a6d545ec79684b9788b7937603a7d9968ffba007ce9abe30f7",
|
||||
"src/cli/applicationCommands.ts": "7a436590d7d8f38c059b1ef9a008a20b6021eb04a6caafb72d473149f8762a00",
|
||||
"src/cli/sourceToBundleCommand.ts": "b905a11c377c159658c1b77e9ecfde2a2961604f1366cfc67509341f4a924fa9",
|
||||
"src/contracts/applicationToolContracts.ts": "3b910f1ad1baf13b428d2d108a0020b6235a06dc0bd2bfa2f4b3024ccf57100c",
|
||||
"src/contracts/javascript/applicationWorkflowInputContracts.ts": "4256f7de82fabf3cc946ca607b8429defa4e8b9f6d3f4be935c20f70741c2d42",
|
||||
"src/domain/analysisView/analysisView.ts": "56107291635491f3606d57fb85a2b2b175a4689d291d3855a899cb201203148a",
|
||||
"src/domain/analysisView/sourceToBundleView.ts": "a8cf9a894a4d0383ae109cfc6671c3ba9aa2b75b2b95e229b1297e55f8393de5",
|
||||
"src/server/createServer.ts": "400764b1ef9bad9d7250cc179736f20daf4cadcde622bbea34e30861b9b23e04",
|
||||
"src/server/registerApplicationTools/compareSourceToBundle.ts": "e14f07f6114f7ce4ec7d97034713074125d3133c876f12a8656a9322017882ee",
|
||||
"src/server/registerApplicationTools/helpers.ts": "c49ec1a9a28d531696012af5d731818deab3fe59f71f4bdcceb3fbb79736824a",
|
||||
"src/server/registerApplicationTools/types.ts": "959dbfc7162048da2bf477aaa545cb10c766f4e64a25ec5ff370d37244fc0eb7"
|
||||
},
|
||||
"regressions": {
|
||||
"mcp_profiles_and_source_guards_passed": 6,
|
||||
"cli_and_shared_json_ownership_selected_tests_passed": 29,
|
||||
"negative_control_without_provenance_and_cancellation_attribution": 3
|
||||
},
|
||||
"verification": [
|
||||
{
|
||||
"profile": "full",
|
||||
"calls": [
|
||||
{
|
||||
"name": "analyze_javascript_application",
|
||||
"elapsed_ms": 6576.780192,
|
||||
"is_error": false,
|
||||
"reply_bytes": 21651
|
||||
},
|
||||
{
|
||||
"name": "compare_source_to_bundle",
|
||||
"elapsed_ms": 264.797908999999,
|
||||
"is_error": false,
|
||||
"reply_bytes": 13199
|
||||
},
|
||||
{
|
||||
"name": "inspect_analysis_view",
|
||||
"elapsed_ms": 2090.7180249999983,
|
||||
"is_error": false,
|
||||
"reply_bytes": 19393
|
||||
},
|
||||
{
|
||||
"name": "export_evidence_bundle",
|
||||
"elapsed_ms": 164.20850099999916,
|
||||
"is_error": false,
|
||||
"reply_bytes": 330
|
||||
}
|
||||
],
|
||||
"wall_seconds": 14.417,
|
||||
"cpu_seconds": 14.027529,
|
||||
"peak_process_family_bytes_including_page_cache": 995631104,
|
||||
"normalized_comparison_byte_equal_to_original": true,
|
||||
"file_provenance_recorded": {
|
||||
"kind": "file",
|
||||
"path": "/tmp/rea-perf-source-bundle-20261021/reference-main-aa51.json",
|
||||
"requested_path": "/tmp/rea-perf-source-bundle-20261021/reference-main-aa51.json"
|
||||
},
|
||||
"evidence_id_differs_due_to_file_provenance": true,
|
||||
"whole_task_cl100k_estimate": 54896,
|
||||
"independent_proof": {
|
||||
"status": "passed",
|
||||
"method": "Independent Python JSON/SHA-256/POSIX path and basename joins; no REA parser or matcher.",
|
||||
"source_files": 180,
|
||||
"embedded_nodes": 99,
|
||||
"candidate_relations": 210,
|
||||
"strong_location_rows": 33,
|
||||
"limitation": "Static multiple-location projections; no semantic-equivalence or removal claim."
|
||||
},
|
||||
"cleanup_verified": true,
|
||||
"resource_events": {
|
||||
"memory.events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
},
|
||||
"pids.events": {
|
||||
"max": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"profile": "off",
|
||||
"calls": [
|
||||
{
|
||||
"name": "analyze_javascript_application",
|
||||
"elapsed_ms": 6315.5019010000005,
|
||||
"is_error": false,
|
||||
"reply_bytes": 21651
|
||||
},
|
||||
{
|
||||
"name": "compare_source_to_bundle",
|
||||
"elapsed_ms": 91.56914900000083,
|
||||
"is_error": false,
|
||||
"reply_bytes": 13199
|
||||
},
|
||||
{
|
||||
"name": "inspect_analysis_view",
|
||||
"elapsed_ms": 24.95524800000021,
|
||||
"is_error": false,
|
||||
"reply_bytes": 19393
|
||||
},
|
||||
{
|
||||
"name": "export_evidence_bundle",
|
||||
"elapsed_ms": 81.6187800000007,
|
||||
"is_error": false,
|
||||
"reply_bytes": 328
|
||||
}
|
||||
],
|
||||
"wall_seconds": 11.104,
|
||||
"cpu_seconds": 10.819556,
|
||||
"peak_process_family_bytes_including_page_cache": 658169856,
|
||||
"normalized_comparison_byte_equal_to_original": true,
|
||||
"file_provenance_recorded": {
|
||||
"kind": "file",
|
||||
"path": "/tmp/rea-perf-source-bundle-20261021/reference-main-aa51.json",
|
||||
"requested_path": "/tmp/rea-perf-source-bundle-20261021/reference-main-aa51.json"
|
||||
},
|
||||
"evidence_id_differs_due_to_file_provenance": true,
|
||||
"whole_task_cl100k_estimate": 54896,
|
||||
"independent_proof": {
|
||||
"status": "passed",
|
||||
"method": "Independent Python JSON/SHA-256/POSIX path and basename joins; no REA parser or matcher.",
|
||||
"source_files": 180,
|
||||
"embedded_nodes": 99,
|
||||
"candidate_relations": 210,
|
||||
"strong_location_rows": 33,
|
||||
"limitation": "Static multiple-location projections; no semantic-equivalence or removal claim."
|
||||
},
|
||||
"cleanup_verified": true,
|
||||
"resource_events": {
|
||||
"memory.events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
},
|
||||
"pids.events": {
|
||||
"max": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"native_cli": {
|
||||
"legacy_inline_evidence_equal_to_baseline": true,
|
||||
"file_cli_mcp_evidence_identical_for_same_origin": true,
|
||||
"normalized_comparison_equal_to_baseline": true,
|
||||
"selected_file_parent_resolves": true
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
# Selected source-to-bundle facts
|
||||
|
||||
Issue #6112. Baseline: `main@aa51e4343963fdd0b97ec2ebdc3c41579fd9cb38`.
|
||||
Measured implementation: the production-file SHA-256 manifest in
|
||||
[source-to-bundle.json](source-to-bundle.json). These are working-tree
|
||||
measurements, not a claim that the baseline commit already contains the feature.
|
||||
The original cohort predates the CLI error-envelope and review guard fixes,
|
||||
pinned separately in the JSON. The CLI invalid-application output was restored
|
||||
exactly. Post-guard SDK tasks verified both output profiles before the later
|
||||
reference-file provenance fix. Final tasks preserve normalized comparison facts
|
||||
while full Evidence IDs now commit the selected file origin. Historical paired
|
||||
medians are not presented as new measurements of the later guard changes.
|
||||
|
||||
## Analyst question and correctness
|
||||
|
||||
For `src/createSlice.ts`, `src/createReducer.ts` and
|
||||
`src/query/core/buildThunks.ts`, report source correspondence and candidate counts,
|
||||
then inspect the exact supporting candidate signals for the two split files.
|
||||
Keep the complete comparison available. This is static location correspondence;
|
||||
it does not establish behavioral equivalence or removal.
|
||||
|
||||
Public corpus: [Redux Toolkit 2.2.7 npm tarball](https://registry.npmjs.org/@reduxjs/toolkit/-/toolkit-2.2.7.tgz),
|
||||
1,184,767 bytes; SHA-256
|
||||
`f2517c466b50424b661a27e48039f15db7783e5822f46b5c7d39f20104ac67a9`.
|
||||
The published `src/` and three root `dist/` bundles/maps are copied byte-for-byte
|
||||
into separate roots, retaining the `src/` and `dist/` prefixes. No target code or
|
||||
package scripts execute. REA runs its own static analysis helpers.
|
||||
|
||||
An independent Python JSON/SHA-256/path oracle checks all 180 source-file hashes,
|
||||
99 embedded source nodes and 210 candidate relations in every completed task.
|
||||
All 99 embedded source strings differ from the published source bytes; path
|
||||
correspondence must not become an exact-source or semantic-equivalence claim.
|
||||
The complete report has 33 split and 147 unknown rows. The selected files have
|
||||
3, 3 and 0 candidates respectively. Selected mapping/candidate identities,
|
||||
original indices, signals, mapped flags and source coverage agree with the full
|
||||
retained comparison. Global classification runs before selection.
|
||||
|
||||
## Completed-task costs
|
||||
|
||||
Linux, Node 24.18.0, one CPU, 4 GiB hard / 3 GiB early-stop process-family guard,
|
||||
192-task ceiling and no swap. Six SDK tasks alternate complete/selected,
|
||||
selected/complete, complete/selected. Each starts a fresh server with an isolated
|
||||
HOME. OS page caches are warm/uncontrolled: these are process-cold observations,
|
||||
not OS-cold measurements. All six exit successfully, release their processes and
|
||||
record no OOM, resource stop or task-limit event.
|
||||
|
||||
Each task includes catalog discovery, application preparation, comparison and
|
||||
full Evidence export. Selected tasks additionally query candidate signals via
|
||||
`inspect_analysis_view`. Both cohorts use the existing explicit semantic-node
|
||||
cap of one because the question only needs source/map correspondence. This is
|
||||
not equivalent semantic-graph coverage; the omission metadata remains visible.
|
||||
|
||||
| Whole task | Existing full inline graph/report | File graph + selected mappings/candidates |
|
||||
| ----------------------------------------- | --------------------------------: | ----------------------------------------: |
|
||||
| Input catalog, cl100k estimate | 47,913 | 48,026 |
|
||||
| All call arguments, estimate | 37,272 | 297 |
|
||||
| All MCP text replies, estimate | 67,407 | 6,442 |
|
||||
| Total estimated context | 152,592 | 54,765 |
|
||||
| Wall seconds, median | 11.899 | 14.733 |
|
||||
| Process-family CPU seconds, median | 11.551 | 14.377 |
|
||||
| Kernel process-family peak MiB, median | 666.6 | 950.7 |
|
||||
| Comparison reply bytes, text + structured | 427,799 | 13,199 |
|
||||
| Additional candidate reply bytes | 0 | 19,393 |
|
||||
|
||||
Context estimates fall 64.1%, including the slightly larger catalog and extra
|
||||
call. Tokenizer: `tiktoken==0.12.0`, `cl100k_base`; one name/description/input-schema
|
||||
catalog plus every argument and MCP text reply. Output schemas, transport
|
||||
framing, model prompts/reasoning and actual billed usage are not included.
|
||||
There was no model invocation; these are not actual model token savings.
|
||||
|
||||
This workflow trades extra cold-client time/memory for substantially smaller
|
||||
context. Comparison itself takes approximately 206–216 ms before and 249–262 ms
|
||||
with file validation and selected delivery. The first candidate-view call takes
|
||||
2.1–2.4 seconds. A separate calibration repeats the identical candidate query:
|
||||
2,071 ms first, 72 ms second, with identical Evidence. This is consistent with
|
||||
one-time SDK validator work; it is not a general speedup or a server-only profile.
|
||||
No comparison cache was added: the measured matching computation is already
|
||||
small, and a file cache would need additional freshness/identity rules.
|
||||
|
||||
Reference import is a separate one-time public CLI step: 4.757 s wall,
|
||||
4.603 s CPU, 365.6 MiB peak. Add that same preparation cost to either first task;
|
||||
download/extraction and compilation are excluded. An additional baseline with
|
||||
default semantic coverage retains 22,090 semantic nodes and exports 42,870,148
|
||||
bytes; explicit cap one retains one and exports 2,896,937 bytes. All comparison
|
||||
facts agree after excluding expected Evidence lineage. Those single-run
|
||||
observations demonstrate question-specific existing task selection, not a new
|
||||
semantic-analysis optimization or equal-coverage performance claim.
|
||||
|
||||
## Existing output-schema selection
|
||||
|
||||
A second six-task cohort explicitly selects the existing
|
||||
`REA_MCP_OUTPUT_SCHEMA_PROFILE=off` for **both** workflows, with baseline
|
||||
`main@01788bac8`. REA still validates successful results against the canonical
|
||||
contracts and returns complete structured content; clients do not receive output
|
||||
schemas to compile. This is an explicit caller configuration, not a new default.
|
||||
The independent oracle and selected/full fact checks pass in every run.
|
||||
|
||||
| Whole task, output profile off | Full graph/report | Selected facts |
|
||||
| -------------------------------------- | ----------------: | -------------: |
|
||||
| Total estimated context | 152,668 | 54,841 |
|
||||
| Wall seconds, median | 11.597 | 10.890 |
|
||||
| Process-family CPU seconds, median | 11.244 | 10.569 |
|
||||
| Kernel process-family peak MiB, median | 628.7 | 626.0 |
|
||||
| Candidate query milliseconds | not needed | 18–21 |
|
||||
|
||||
The context reduction remains approximately 64%. Whole-task ranges overlap
|
||||
(complete 10.899–12.215 s; selected 10.802–11.196 s), so the apparent median
|
||||
improvement does not establish a stable speedup. Unlike the default full-output
|
||||
cohort, this configured workflow avoids the extra cold SDK validator cost and
|
||||
has similar task memory/time. Two initial pilot cases are excluded. All six
|
||||
observed runs, explicit configuration and implementation hashes are retained in
|
||||
[source-to-bundle-output-off.json](source-to-bundle-output-off.json).
|
||||
|
||||
## Existing CLI/code-composition control
|
||||
|
||||
The public CLI on the same saved application/reference produces complete Evidence
|
||||
identical to MCP (205,231 output bytes). The measured CLI comparison step is
|
||||
2.842 s wall, 2.670 s CPU, 357.4 MiB process-family peak. A separate Python
|
||||
projection of the three rows already yields 22,108 bytes. A native agent can
|
||||
redirect full JSON to a file and select facts in code, so raw SDK savings do not
|
||||
establish savings over every native client. These byte sizes are not token counts;
|
||||
actual native-agent context and external projection CPU are unmeasured.
|
||||
|
||||
The new CLI reference-file form preserves the normalized facts and records its
|
||||
file origin. Its saved Evidence
|
||||
can be queried by `inspect-analysis-view` using the same source-to-bundle
|
||||
projector as MCP. The interface supplies reusable, authenticated facts and
|
||||
relationships instead of requiring each caller to maintain an ad-hoc projection.
|
||||
|
||||
## Reproduce
|
||||
|
||||
Use pinned Node/npm, `npm ci` and `npm run build:cached` for baseline and feature
|
||||
worktrees. Install dependencies once per lockfile; do not run npm through a shared
|
||||
node_modules symlink. Run heavy steps serially with the checked-in resource guard.
|
||||
Set absolute paths for `BASE_RUNTIME`, `FEATURE_RUNTIME` and `LAB`.
|
||||
|
||||
```sh
|
||||
curl -fL 'https://registry.npmjs.org/@reduxjs/toolkit/-/toolkit-2.2.7.tgz' -o /tmp/toolkit.tgz
|
||||
python3 experiments/analysis-efficiency/source-to-bundle-prepare.py /tmp/toolkit.tgz "$LAB"
|
||||
python3 experiments/analysis-efficiency/resource-run.py --observe-payload-exit -- \
|
||||
node "$BASE_RUNTIME/scripts/rea.mjs" import-reference-source "$LAB/reference" --json \
|
||||
> "$LAB/reference.json" 2> "$LAB/reference-import.log"
|
||||
```
|
||||
|
||||
For each of the six ordered trials, use a new output directory; set `RUNTIME` to
|
||||
the baseline for `MODE=complete`, feature for `MODE=selected`. Capture both stdout
|
||||
and stderr in the adjacent `.log` so the resource receipt remains available.
|
||||
|
||||
```sh
|
||||
python3 experiments/analysis-efficiency/resource-run.py --observe-payload-exit -- \
|
||||
node experiments/analysis-efficiency/source-to-bundle.mjs \
|
||||
--runtime "$RUNTIME" --corpus "$LAB" --reference "$LAB/reference.json" \
|
||||
--output "$LAB/paired-1-complete" --mode "$MODE" \
|
||||
> "$LAB/paired-1-complete.log" 2>&1
|
||||
python3 experiments/analysis-efficiency/source-to-bundle-oracle.py \
|
||||
"$LAB/paired-1-complete" "$LAB/reference.json"
|
||||
uv run --no-project --with tiktoken==0.12.0 python \
|
||||
experiments/analysis-efficiency/source-to-bundle-costs.py "$LAB"
|
||||
```
|
||||
|
||||
Use a separate output name and `--mode selected --warm-continuation` for the
|
||||
validator calibration; exclude it from paired medians. For the configured
|
||||
cohort, add `--output-profile off`, use new `off-paired-*` directories and the
|
||||
01788bac8 control, then run the cost reader with
|
||||
`--pattern 'off-paired-*' --output "$LAB/costs-off.json"`. Retain the tarball,
|
||||
reference, transcripts, bundles, independent proofs and guard receipts locally.
|
||||
Never commit target binaries, dependency trees, credentials or generated builds.
|
||||
|
||||
## Review guards and regression controls
|
||||
|
||||
The file reader admits JSON assembly; a shared application guard additionally
|
||||
reserves live heap headroom before schema validation clones data and computes the
|
||||
root commitment. Evidence-file and historical-graph readers use the same owner,
|
||||
with their original operation-specific diagnostics. Validation yields before
|
||||
checking a pending cancellation, including validation failures. MCP computes its
|
||||
selected view before yielding and committing any application source, comparison,
|
||||
residual unknown or view to the session.
|
||||
|
||||
Seven shared headroom/Evidence-reader tests pass. A separate 65-test focused run
|
||||
covers file cancellation, graph paths, actual SDK cancellation during validation
|
||||
and comparison, and cross-source pages with candidate counts 2/1/2. The latter
|
||||
checks original indices, totals and next offsets across boundaries and filters.
|
||||
All three new guard regressions fail in a negative control with those gates
|
||||
temporarily removed, then pass after restoration. Memory admission is an estimate;
|
||||
the reported hard process-family ceilings come from the experiment guard.
|
||||
|
||||
Post-guard public-package SDK tasks, before the later provenance fix, retained all
|
||||
180 source-file / 99 embedded-node / 210 relation oracle checks and byte-identical
|
||||
complete comparison Evidence. Their receipts, source hashes and costs are under
|
||||
`subsequent_review_guards` in the main JSON; they are excluded from paired medians.
|
||||
|
||||
## Reference-file provenance and cancellation attribution
|
||||
|
||||
The final file form records `parameters.reference_source` with `kind: file`, the
|
||||
exact caller `requested_path` and the reader's absolute selected `path`. The
|
||||
shared file owner supplies that path without a second interpretation; POSIX
|
||||
`..` and symlink semantics remain intact. Identical graphs in different chosen
|
||||
files have the same normalized facts/mapping IDs and distinct Evidence IDs.
|
||||
Inline legacy Evidence remains identical. Actual public-package CLI file
|
||||
Evidence equals MCP file Evidence for the same origin; relative CLI paths also
|
||||
have maintained coverage.
|
||||
|
||||
Reader cancellation is reattributed to `compare_source_to_bundle`, preserving the
|
||||
original cause. Six MCP/profile/guard tests and 29 selected CLI/shared-JSON
|
||||
ownership tests pass. Removing provenance recording and cancellation attribution
|
||||
makes three tests fail as expected. Additional Evidence-reader/file cancellation
|
||||
checks pass. Final real-package tasks estimate 54,896 cl100k tokens in both output
|
||||
profiles, preserve all independent oracle facts, and retain the new provenance;
|
||||
these are additional verification runs, not new paired medians. The main JSON's
|
||||
`subsequent_reference_provenance` records final source hashes and receipts.
|
||||
@@ -0,0 +1,251 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { resolve } from "node:path";
|
||||
import { parseArgs } from "node:util";
|
||||
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { Client } from "@modelcontextprotocol/client";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
||||
const { values } = parseArgs({
|
||||
options: {
|
||||
runtime: { type: "string" },
|
||||
corpus: { type: "string" },
|
||||
reference: { type: "string" },
|
||||
output: { type: "string" },
|
||||
mode: { type: "string" },
|
||||
"output-profile": { type: "string", default: "full" },
|
||||
"warm-continuation": { type: "boolean", default: false },
|
||||
},
|
||||
});
|
||||
for (const key of ["runtime", "corpus", "reference", "output"])
|
||||
assert.ok(
|
||||
values[key] && resolve(values[key]) === values[key],
|
||||
key + " must be absolute",
|
||||
);
|
||||
assert.ok(["complete", "selected"].includes(values.mode));
|
||||
assert.ok(["full", "off"].includes(values["output-profile"]));
|
||||
const root = values.corpus,
|
||||
runtime = values.runtime,
|
||||
output = values.output;
|
||||
await mkdir(output);
|
||||
const reference = JSON.parse(await readFile(values.reference, "utf8"));
|
||||
const sha = (value) => createHash("sha256").update(value).digest("hex");
|
||||
const revision = execFileSync("git", ["rev-parse", "HEAD"], {
|
||||
cwd: runtime,
|
||||
encoding: "utf8",
|
||||
}).trim();
|
||||
const sourceState = execFileSync("git", ["status", "--porcelain"], {
|
||||
cwd: runtime,
|
||||
encoding: "utf8",
|
||||
});
|
||||
const transport = new StdioClientTransport({
|
||||
command: process.execPath,
|
||||
args: [join(runtime, "scripts/rea.mjs"), "mcp"],
|
||||
cwd: output,
|
||||
env: {
|
||||
PATH: process.env.PATH,
|
||||
HOME: output,
|
||||
REA_LOG_LEVEL: "silent",
|
||||
REA_MCP_OUTPUT_SCHEMA_PROFILE: values["output-profile"],
|
||||
UV_THREADPOOL_SIZE: "1",
|
||||
NODE_OPTIONS: "--max-old-space-size=2048",
|
||||
NODE_DISABLE_COMPILE_CACHE: "1",
|
||||
},
|
||||
stderr: "pipe",
|
||||
});
|
||||
const client = new Client({ name: "source-bundle-baseline", version: "1" });
|
||||
let stderr = "";
|
||||
transport.stderr?.on("data", (chunk) => {
|
||||
stderr += chunk;
|
||||
});
|
||||
const report = {
|
||||
status: "incomplete",
|
||||
runtime_revision: revision,
|
||||
runtime_worktree_status: sourceState,
|
||||
mode: values.mode,
|
||||
output_schema_profile: values["output-profile"],
|
||||
reference_sha256: sha(await readFile(values.reference)),
|
||||
node_version: process.version,
|
||||
calls: [],
|
||||
cleanup_verified: false,
|
||||
};
|
||||
const transcript = [];
|
||||
const began = performance.now();
|
||||
try {
|
||||
await client.connect(transport);
|
||||
const catalog = await client.listTools();
|
||||
await writeFile(join(output, "catalog.json"), JSON.stringify(catalog));
|
||||
const definitions = new Map(catalog.tools.map((t) => [t.name, t]));
|
||||
const call = async (name, args) => {
|
||||
const start = performance.now();
|
||||
const reply = await client.callTool(
|
||||
{ name, arguments: args },
|
||||
{ toolDefinition: definitions.get(name) },
|
||||
);
|
||||
transcript.push({ name, arguments: args, reply });
|
||||
report.calls.push({
|
||||
name,
|
||||
elapsed_ms: performance.now() - start,
|
||||
is_error: reply.isError === true,
|
||||
reply_bytes: Buffer.byteLength(JSON.stringify(reply)),
|
||||
});
|
||||
return reply;
|
||||
};
|
||||
const analyzed = await call("analyze_javascript_application", {
|
||||
input_path: join(root, "artifact/shipped"),
|
||||
detail: "summary",
|
||||
max_semantic_nodes: 1,
|
||||
max_semantic_nodes_per_file: 1,
|
||||
});
|
||||
assert.notEqual(analyzed.isError, true, JSON.stringify(analyzed));
|
||||
report.preparation_summary = analyzed.structuredContent.normalized_result;
|
||||
const parentId = report.preparation_summary.parent_evidence_id;
|
||||
assert.ok(parentId);
|
||||
const compared = await call("compare_source_to_bundle", {
|
||||
reference:
|
||||
values.mode === "complete"
|
||||
? reference
|
||||
: { kind: "file", path: values.reference },
|
||||
application: { kind: "retained-evidence", evidence_id: parentId },
|
||||
...(values.mode === "complete"
|
||||
? {}
|
||||
: {
|
||||
view: {
|
||||
kind: "source-to-bundle",
|
||||
collection: "mappings",
|
||||
source_paths: [
|
||||
"src/createSlice.ts",
|
||||
"src/createReducer.ts",
|
||||
"src/query/core/buildThunks.ts",
|
||||
],
|
||||
},
|
||||
}),
|
||||
});
|
||||
assert.notEqual(compared.isError, true, JSON.stringify(compared));
|
||||
if (values.mode === "selected") {
|
||||
const selected = compared.structuredContent.normalized_result;
|
||||
assert.equal(selected.items.length, 3);
|
||||
assert.deepEqual(
|
||||
selected.items.map((r) => [r.source_path, r.status]),
|
||||
[
|
||||
["src/createReducer.ts", "split"],
|
||||
["src/createSlice.ts", "split"],
|
||||
["src/query/core/buildThunks.ts", "unknown"],
|
||||
],
|
||||
);
|
||||
const candidates = await call("inspect_analysis_view", {
|
||||
source: {
|
||||
kind: "retained-evidence",
|
||||
evidence_id: selected.parent_evidence_id,
|
||||
},
|
||||
view: {
|
||||
kind: "source-to-bundle",
|
||||
collection: "candidates",
|
||||
source_paths: ["src/createSlice.ts", "src/createReducer.ts"],
|
||||
fields: ["score", "confidence", "signals"],
|
||||
},
|
||||
});
|
||||
assert.notEqual(candidates.isError, true, JSON.stringify(candidates));
|
||||
assert.equal(
|
||||
candidates.structuredContent.normalized_result.items.length,
|
||||
6,
|
||||
);
|
||||
if (values["warm-continuation"]) {
|
||||
const repeated = await call(
|
||||
"inspect_analysis_view",
|
||||
transcript.at(-1).arguments,
|
||||
);
|
||||
assert.deepEqual(repeated, candidates);
|
||||
}
|
||||
}
|
||||
report.comparison_reply = compared.structuredContent;
|
||||
const exported = await call("export_evidence_bundle", {
|
||||
path: join(output, "evidence.json"),
|
||||
});
|
||||
assert.notEqual(exported.isError, true);
|
||||
const bundle = JSON.parse(
|
||||
await readFile(join(output, "evidence.json"), "utf8"),
|
||||
);
|
||||
const comparison = bundle.records.find(
|
||||
(r) => r.operation === "compare_source_to_bundle",
|
||||
);
|
||||
report.retained_records = bundle.records.map((r) => ({
|
||||
evidence_id: r.evidence_id,
|
||||
operation: r.operation,
|
||||
}));
|
||||
if (comparison) {
|
||||
report.comparison_id = comparison.normalized_result.comparison_id;
|
||||
report.comparison_summary = comparison.normalized_result.summary;
|
||||
report.coverage = comparison.normalized_result.coverage;
|
||||
report.selected_items = comparison.normalized_result.items.filter((r) =>
|
||||
[
|
||||
"src/createSlice.ts",
|
||||
"src/createReducer.ts",
|
||||
"src/query/core/buildThunks.ts",
|
||||
].includes(r.source_path),
|
||||
);
|
||||
report.complete_comparison_bytes = Buffer.byteLength(
|
||||
JSON.stringify(comparison),
|
||||
);
|
||||
}
|
||||
report.status = "observed";
|
||||
assert.equal(report.comparison_summary.split, 33);
|
||||
assert.equal(report.comparison_summary.unknown, 147);
|
||||
const views = transcript.filter(
|
||||
(r) =>
|
||||
["compare_source_to_bundle", "inspect_analysis_view"].includes(r.name) &&
|
||||
r.reply.structuredContent.normalized_result.parent_evidence_id,
|
||||
);
|
||||
for (const view of views) {
|
||||
const result = view.reply.structuredContent.normalized_result;
|
||||
assert.equal(result.parent_evidence_id, comparison.evidence_id);
|
||||
assert.deepEqual(
|
||||
result.source_coverage,
|
||||
comparison.normalized_result.coverage,
|
||||
);
|
||||
for (const row of result.items) {
|
||||
const original = comparison.normalized_result.items[row.source_index];
|
||||
assert.equal(row.mapping_id, original.mapping_id);
|
||||
assert.equal(row.source_path, original.source_path);
|
||||
if (result.view.collection === "candidates") {
|
||||
const candidate = original.candidates[row.candidate_index];
|
||||
assert.equal(row.current_node_id, candidate.current_node_id);
|
||||
assert.deepEqual(row.signals, candidate.signals);
|
||||
assert.equal(
|
||||
row.mapped,
|
||||
original.current_node_ids.includes(candidate.current_node_id),
|
||||
);
|
||||
} else {
|
||||
assert.equal(row.status, original.status);
|
||||
assert.equal(row.candidate_count, original.candidates.length);
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await client.close();
|
||||
await transport.close();
|
||||
report.cleanup_verified = transport.pid === null;
|
||||
report.elapsed_ms = performance.now() - began;
|
||||
await writeFile(
|
||||
join(output, "report.json"),
|
||||
JSON.stringify(report, null, 2) + "\n",
|
||||
);
|
||||
await writeFile(join(output, "transcript.json"), JSON.stringify(transcript));
|
||||
await writeFile(join(output, "server.log"), stderr);
|
||||
}
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
status: report.status,
|
||||
calls: report.calls,
|
||||
comparison_summary: report.comparison_summary,
|
||||
coverage: report.coverage,
|
||||
selected: report.selected_items?.map((r) => ({
|
||||
path: r.source_path,
|
||||
status: r.status,
|
||||
candidates: r.candidates.length,
|
||||
})),
|
||||
complete_comparison_bytes: report.complete_comparison_bytes,
|
||||
cleanup_verified: report.cleanup_verified,
|
||||
}),
|
||||
);
|
||||
Reference in new issue
Block a user