feat(erlang): query selected BEAM table facts (#6943)

* feat(erlang): query selected BEAM table facts

* docs(perf): record BEAM selected-view measurements
This commit is contained in:
gensokyo-dev authored and GitHub committed 2026-10-24 12:00:18 +08:00
1 parent c2caae88f8
commit ead9fdcda9
20 files changed
+4206 -66

No files matched your search

@@ -0,0 +1,111 @@
"""Verify completed BEAM selection/export tasks and count a declared token projection."""
import argparse
import hashlib
import json
from pathlib import Path
import statistics
import tiktoken
def compact(value):
return json.dumps(value, ensure_ascii=False, separators=(',', ':'), allow_nan=False)
def digest(path):
return hashlib.sha256(path.read_bytes()).hexdigest()
def inspect(path, encoding):
report = json.loads((path / 'report.json').read_text())
catalog = json.loads((path / 'catalog.json').read_text())
transcript = json.loads((path / 'transcript.json').read_text())
scope = next(json.loads(line) for line in reversed(path.with_suffix('.log').read_text().splitlines())
if line.startswith('{"resource_scope":'))
observed = scope.get('payload_resource_observation') or {}
events = observed.get('resource_events', {})
assert report['status'] == 'passed' and report['runtime_identity_sha256']
assert report['complete_source_export_verified'] is True
assert not report['cleanup_failures'] and not report['receipt_failures']
assert scope['abort'] is None and scope['payload_resource_observation_verified'] is True
assert observed['payload_exit_code'] == 0 and observed['payload_cleanup_verified'] is True
assert all(events.get('memory.events', {}).get(key) == 0 for key in ['high', 'max', 'oom', 'oom_kill'])
assert events.get('pids.events', {}).get('max') == 0
assert len(catalog['tools']) == report['catalog_tools'] == 221
assert transcript[0]['name'] == 'inspect_erlang_beam'
assert transcript[-1]['name'] == 'export_evidence_bundle'
assert all(event['name'] == 'inspect_analysis_view' for event in transcript[1:-1])
assert all('reply' in event and not event['reply'].get('isError', False) for event in transcript)
count = lambda text: len(encoding.encode(text, disallowed_special=()))
directory = [{key: tool[key] for key in ['name', 'description', 'inputSchema']} for tool in catalog['tools']]
request = transcript[0]['input']
question = report['question']
assert ('detail' in request) == (report['lane'] == 'selected')
arguments = sum(count(compact({'name': event['name'], 'arguments': event['input']})) for event in transcript)
replies = sum(count(block['text']) for event in transcript for block in event['reply']['content'] if block['type'] == 'text')
inputs, prompt = count(compact(directory)), count(question)
instructions = count(report['server_instructions'])
return dict(case=path.name, lane=report['lane'], artifact=report['artifact'],
answer_count=report['answer_count'], answer_sha256=report['answer_sha256'],
parent_evidence_id=report['parent_evidence_id'], source_report_sha256=report['source_report_sha256'],
runtime_identity_sha256=report['runtime_identity_sha256'], catalog_sha256=digest(path / 'catalog.json'),
calls=report['calls'], phases=report['phases'], wall_seconds=scope['elapsed_seconds'],
cpu_seconds=observed['cpu_stat']['usage_usec'] / 1e6,
kernel_family_peak_bytes=observed['memory_peak_bytes'],
events_at_payload_exit=events, payload_cleanup_verified=True,
final_whole_scope_events=scope.get('resource_events_final'),
token_proxy=dict(question=prompt, server_instructions=instructions, input_catalog_once=inputs, arguments=arguments,
text_replies=replies, combined=prompt + instructions + inputs + arguments + replies,
complete_transport_catalog=count(compact(catalog))),
receipt_sha256={file.name: digest(file) for file in [path / 'report.json', path / 'catalog.json',
path / 'transcript.json', path / 'evidence.json', path.with_suffix('.log')]})
def summarize(rows):
result = {}
for artifact in sorted({row['artifact']['sha256'] for row in rows}):
cases = [row for row in rows if row['artifact']['sha256'] == artifact]
for field in ['answer_count', 'answer_sha256', 'parent_evidence_id', 'source_report_sha256']:
assert len({row[field] for row in cases}) == 1, field
result[artifact] = {}
for lane in ['complete', 'selected']:
samples = [row for row in cases if row['lane'] == lane]
assert len(samples) == 3
for field in ['runtime_identity_sha256', 'catalog_sha256']:
assert len({row[field] for row in samples}) == 1, field
metrics = {key: [row[key] for row in samples] for key in ['wall_seconds', 'cpu_seconds', 'kernel_family_peak_bytes']}
metrics['projected_tokens'] = [row['token_proxy']['combined'] for row in samples]
metrics['mcp_reply_bytes'] = [sum(call['reply_bytes'] for call in row['calls']) for row in samples]
result[artifact][lane] = {key: dict(median=statistics.median(values), minimum=min(values), maximum=max(values))
for key, values in metrics.items()}
return result
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--cohort', type=Path, required=True)
parser.add_argument('--output', type=Path, required=True)
args = parser.parse_args()
manifest = json.loads((args.cohort / 'manifest.json').read_text())
index = json.loads((args.cohort / 'index.json').read_text())
assert len(index) == 18 and all(case['exit'] == 0 for case in index)
for file, expected in manifest['driver_files'].items():
assert digest(Path(file)) == expected, file
identities = {}
for lane, path in manifest['runtime_identities'].items():
identity = json.loads(Path(path).read_text())
assert identity['source_clean'] is True
identities[lane] = dict(sha256=digest(Path(path)), source_revision=identity['source_revision'])
encoding = tiktoken.get_encoding('cl100k_base')
rows = [inspect(args.cohort / case['name'], encoding) for case in index]
assert all(row['runtime_identity_sha256'] == identities[row['lane']]['sha256'] for row in rows)
receipt = dict(tiktoken=tiktoken.__version__, encoding=encoding.name, actual_model_usage=None,
cohort=manifest, commands=index, runtime_identities=identities,
projection='Scripted fresh SDK tasks, not model runs. One fixed question, server instructions and complete input directory (name/description/inputSchema), every argument and MCP text reply including export. Raw catalog including output schemas reported separately. No framing, host injection, history replay, inference quality or billable usage measured.',
resources='CPU quota100%, payload affinity0-1, initial observer15. Early stop3GiB, hard4GiB, swap0. Fresh HOME/client/server; Node compile cache disabled, OS caches unflushed. Whole process-family CPU and kernel memory peak include file cache, runtime identity checks, hashing, assertions, SDK/server serialization and cleanup. Oracle generation and tokenization outside timed tasks. Payload-exit counters verified; final whole-scope counters unavailable.',
summary=summarize(rows), recordings=rows)
args.output.write_text(json.dumps(receipt, ensure_ascii=False, indent=2) + '\n')
if __name__ == '__main__':
main()
@@ -0,0 +1,108 @@
"""Independent struct oracle for the fixed public OTP 27 benchmark artifacts.
Reads bytes only. It deliberately refuses compact atom layouts; format-wide
coverage belongs to the production reader and its separate regressions.
"""
from pathlib import Path
import getopt
import hashlib
import json
import struct
import sys
def decode(path):
data = path.read_bytes()
assert len(data) <= 64 * 1024 * 1024
u32 = lambda at: struct.unpack_from('>I', data, at)[0]
assert data[:4] == b'FOR1' and data[8:12] == b'BEAM'
assert u32(4) + 8 == len(data)
chunks, selected, at = [], {}, 12
while at < len(data):
name = data[at:at + 4].decode('ascii')
size = u32(at + 4)
end = at + 8 + size
padded = (end + 3) & ~3
assert end <= padded <= len(data)
chunks.append(dict(index=len(chunks), name=name,
location=dict(offset=at, bytes=padded - at),
payload=dict(location=dict(offset=at + 8, bytes=size),
bytes=size, sha256=hashlib.sha256(data[at + 8:end]).hexdigest())))
selected[name] = (at + 8, end)
at = padded
assert at == len(data)
atom_name = 'AtU8' if 'AtU8' in selected else 'Atom'
start, end = selected[atom_name]
n = u32(start)
assert n < 65536, 'Fixed OTP 27 oracle does not decode compact atom tables'
at, atoms = start + 4, []
for i in range(n):
size = data[at]
at += 1
name = data[at:at + size].decode('utf-8' if atom_name == 'AtU8' else 'latin1')
atoms.append(dict(index=i + 1, name=name))
at += size
assert at == end
def atom(i):
return atoms[i - 1]['name'] if 0 < i <= len(atoms) else None
def table(name, keys):
if name not in selected:
return None
start, end = selected[name]
if name == 'FunT' and start == end:
return None
n, width = u32(start), len(keys)
assert start + 4 + n * 4 * width == end
rows = []
for i in range(n):
values = struct.unpack_from('>' + 'I' * width, data, start + 4 + i * width * 4)
row = dict(zip(keys, values))
if 'module_index' in row:
row['module'] = atom(row['module_index'])
row['function'] = atom(row['function_index'])
rows.append(row)
return rows
exports = table('ExpT', ['function_index', 'arity', 'label'])
imports = table('ImpT', ['module_index', 'function_index', 'arity'])
lambdas = table('FunT', ['function_index', 'arity', 'label', 'index', 'free_vars', 'old_unique'])
code = None
if 'Code' in selected:
start, end = selected['Code']
size = u32(start)
assert size >= 16 and start + 4 + size <= end
code = dict(zip(['instruction_set', 'opcode_max', 'labels', 'functions'],
struct.unpack_from('>IIII', data, start + 4)))
code.update(extra_header_bytes=size - 16,
instructions=dict(offset=start + 4 + size, bytes=end - start - 4 - size))
return dict(artifact=dict(path=str(path), sha256=hashlib.sha256(data).hexdigest(), bytes=len(data)),
form=dict(location=dict(offset=0, bytes=len(data)), size=len(data) - 8),
chunks=chunks, atoms=atoms, atom_encoding='utf8' if atom_name == 'AtU8' else 'latin1',
exports=exports, imports=imports, lambdas=lambdas, code=code,
module=atoms[0]['name'],
coverage=dict(status='complete', examined_chunks=len(chunks), atoms=len(atoms)))
def main():
options, remaining = getopt.getopt(sys.argv[1:], '', ['input=', 'output=', 'name='])
assert not remaining
inputs = [v for k, v in options if k == '--input']
outputs = [v for k, v in options if k == '--output']
names = [v for k, v in options if k == '--name']
assert len(inputs) == len(outputs) == 1 and names
path = Path(inputs[0])
assert path.is_absolute()
expected = decode(path)
answer = [dict(source_index=i, **row) for i, row in enumerate(expected['exports'] or [])
if row['function'] in names]
result = dict(oracle='Independent Python struct decoding of fixed OTP27 bytes; no BEAM execution. '
'Every table, chunk range/hash and Code header checked. Other encodings/malformed inputs unqualified.',
source='https://www.erlang.org/doc/apps/stdlib/beam_lib.html',
expected=expected, names=names, answer=answer)
Path(outputs[0]).write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n')
if __name__ == '__main__':
main()
@@ -0,0 +1,534 @@
{
"method": "3alternating freshsessions perlane; SDK cachesentirecatalog thenusesdefaultvalidator withouttoolDefinition override. Diagnostic client startup control, separate from18exactnamedtasks. Samequestion/fullsourceexports/oracle; allpayloadexit/cleanupverified.",
"summary": {
"716d78394b7c3da983e36aac7aadd6673e5041677ba7de20413cc2704d958d55": {
"complete": {
"wall_seconds": {
"median": 19.764,
"minimum": 17.257,
"maximum": 21.577
},
"cpu_seconds": {
"median": 16.276557,
"minimum": 15.287268,
"maximum": 16.561897
},
"kernel_family_peak_bytes": {
"median": 1073950720,
"minimum": 1041948672,
"maximum": 1078980608
},
"projected_tokens": {
"median": 76907,
"minimum": 76907,
"maximum": 76907
},
"mcp_reply_bytes": {
"median": 147003,
"minimum": 147003,
"maximum": 147003
}
},
"selected": {
"wall_seconds": {
"median": 19.375,
"minimum": 18.052,
"maximum": 22.397
},
"cpu_seconds": {
"median": 16.867381,
"minimum": 16.139353,
"maximum": 17.069666
},
"kernel_family_peak_bytes": {
"median": 1105092608,
"minimum": 1059520512,
"maximum": 1117036544
},
"projected_tokens": {
"median": 58090,
"minimum": 58090,
"maximum": 58090
},
"mcp_reply_bytes": {
"median": 7469,
"minimum": 7469,
"maximum": 7469
}
}
}
},
"recordings": [
{
"case": "erl_lint-1-complete",
"lane": "complete",
"artifact": {
"path": "/tmp/rea-perf-static-tool-audit-20261024/real-beam/erl_lint.beam",
"sha256": "716d78394b7c3da983e36aac7aadd6673e5041677ba7de20413cc2704d958d55",
"bytes": 233968
},
"answer_count": 4,
"answer_sha256": "34ce3091dbea9ac8cd20831cf6443c7a1de6e02b753246cafa3e0e22b83907c9",
"parent_evidence_id": "ev_df4bfa8b8f0391ad6291542fdccf9062ae3bb8dbabf01c2b166ca6f3493051fc",
"source_report_sha256": "db64350ae25bff9e77f5e167f4a4ac095ed872e09c15de5ce3b2728c078a16d2",
"runtime_identity_sha256": "1b4b40330a2e21d31c06a07d10937cb6a3b37cd0bcbf3d129380341d486b916e",
"catalog_sha256": "f4a683209df5bed1ed25ae42c690fdd803348742132e88f77ba2e4e8333ca562",
"calls": [
{
"name": "inspect_erlang_beam",
"input": {
"path": "/tmp/rea-perf-static-tool-audit-20261024/real-beam/erl_lint.beam"
},
"phase": "inspect",
"elapsed_ms": 18092.161941,
"reply_bytes": 146627,
"is_error": false
},
{
"name": "export_evidence_bundle",
"input": {
"path": "/tmp/rea-perf-static-tool-audit-20261024/sdk-cache-cohort/erl_lint-1-complete/evidence.json"
},
"phase": "export",
"elapsed_ms": 52.40052199999991,
"reply_bytes": 376,
"is_error": false
}
],
"phases": {
"discovery_ms": 1018.838796
},
"wall_seconds": 21.577,
"cpu_seconds": 16.561897,
"kernel_family_peak_bytes": 1073950720,
"events_at_payload_exit": {
"memory.events": {
"low": 0,
"high": 0,
"max": 0,
"oom": 0,
"oom_kill": 0,
"oom_group_kill": 0
},
"pids.events": {
"max": 0
}
},
"payload_cleanup_verified": true,
"final_whole_scope_events": null,
"token_proxy": {
"question": 32,
"server_instructions": 513,
"input_catalog_once": 55409,
"arguments": 78,
"text_replies": 20875,
"combined": 76907,
"complete_transport_catalog": 829075
},
"receipt_sha256": {
"report.json": "3717e681ac50a70db8bd7303ac18d87bd2028d4321104a793421695a82579290",
"catalog.json": "f4a683209df5bed1ed25ae42c690fdd803348742132e88f77ba2e4e8333ca562",
"transcript.json": "c718ac6e9b6c63533bc28d18c90c73a03abc1697516ae216a444b6dd55827255",
"evidence.json": "ca66367cd1bb325d7d4e47f958c86234d01aaf749b68778ac5212cac246aacb3",
"erl_lint-1-complete.log": "a90d1345cd09e90865c8a75f732abe2561c2299d0084718e91205f2cb904f38d"
}
},
{
"case": "erl_lint-1-selected",
"lane": "selected",
"artifact": {
"path": "/tmp/rea-perf-static-tool-audit-20261024/real-beam/erl_lint.beam",
"sha256": "716d78394b7c3da983e36aac7aadd6673e5041677ba7de20413cc2704d958d55",
"bytes": 233968
},
"answer_count": 4,
"answer_sha256": "34ce3091dbea9ac8cd20831cf6443c7a1de6e02b753246cafa3e0e22b83907c9",
"parent_evidence_id": "ev_df4bfa8b8f0391ad6291542fdccf9062ae3bb8dbabf01c2b166ca6f3493051fc",
"source_report_sha256": "db64350ae25bff9e77f5e167f4a4ac095ed872e09c15de5ce3b2728c078a16d2",
"runtime_identity_sha256": "a74d203a5a12d7eb82aaa1e0fda6595b9647af2255f318bf116953777052460f",
"catalog_sha256": "9c09aa2944f10c806df78d1cfa2e1daeddc0a487c432701cb67fa65e14213b0c",
"calls": [
{
"name": "inspect_erlang_beam",
"input": {
"path": "/tmp/rea-perf-static-tool-audit-20261024/real-beam/erl_lint.beam",
"detail": "summary",
"view": {
"kind": "beam",
"collection": "exports",
"names": ["module", "exprs", "expr"],
"fields": ["function_index", "function", "arity", "label"],
"offset": 0,
"limit": 64
}
},
"phase": "inspect",
"elapsed_ms": 18394.333454,
"reply_bytes": 7093,
"is_error": false
},
{
"name": "export_evidence_bundle",
"input": {
"path": "/tmp/rea-perf-static-tool-audit-20261024/sdk-cache-cohort/erl_lint-1-selected/evidence.json"
},
"phase": "export",
"elapsed_ms": 58.59494199999972,
"reply_bytes": 376,
"is_error": false
}
],
"phases": {
"discovery_ms": 1056.9951270000001
},
"wall_seconds": 22.397,
"cpu_seconds": 17.069666,
"kernel_family_peak_bytes": 1117036544,
"events_at_payload_exit": {
"memory.events": {
"low": 0,
"high": 0,
"max": 0,
"oom": 0,
"oom_kill": 0,
"oom_group_kill": 0
},
"pids.events": {
"max": 0
}
},
"payload_cleanup_verified": true,
"final_whole_scope_events": null,
"token_proxy": {
"question": 32,
"server_instructions": 513,
"input_catalog_once": 56359,
"arguments": 120,
"text_replies": 1066,
"combined": 58090,
"complete_transport_catalog": 834792
},
"receipt_sha256": {
"report.json": "c3c17495f33ee4ea9daabd7a353f698fa0a804451c4bbdbe9eb1adf905ab483a",
"catalog.json": "9c09aa2944f10c806df78d1cfa2e1daeddc0a487c432701cb67fa65e14213b0c",
"transcript.json": "fd8c589a6e0dbcd0783ef5a526b51db7a3c208cd4067986ed99804bda06c4498",
"evidence.json": "2373bc2e57e40aac881f3ef91fa77187b098dd61d6cf44d37a4b8fe56d022b40",
"erl_lint-1-selected.log": "fdd55def718ffa7c972b3cb9b6be1b7f66a7fa5fb1c371cc492051ffb0d08f62"
}
},
{
"case": "erl_lint-2-selected",
"lane": "selected",
"artifact": {
"path": "/tmp/rea-perf-static-tool-audit-20261024/real-beam/erl_lint.beam",
"sha256": "716d78394b7c3da983e36aac7aadd6673e5041677ba7de20413cc2704d958d55",
"bytes": 233968
},
"answer_count": 4,
"answer_sha256": "34ce3091dbea9ac8cd20831cf6443c7a1de6e02b753246cafa3e0e22b83907c9",
"parent_evidence_id": "ev_df4bfa8b8f0391ad6291542fdccf9062ae3bb8dbabf01c2b166ca6f3493051fc",
"source_report_sha256": "db64350ae25bff9e77f5e167f4a4ac095ed872e09c15de5ce3b2728c078a16d2",
"runtime_identity_sha256": "a74d203a5a12d7eb82aaa1e0fda6595b9647af2255f318bf116953777052460f",
"catalog_sha256": "9c09aa2944f10c806df78d1cfa2e1daeddc0a487c432701cb67fa65e14213b0c",
"calls": [
{
"name": "inspect_erlang_beam",
"input": {
"path": "/tmp/rea-perf-static-tool-audit-20261024/real-beam/erl_lint.beam",
"detail": "summary",
"view": {
"kind": "beam",
"collection": "exports",
"names": ["module", "exprs", "expr"],
"fields": ["function_index", "function", "arity", "label"],
"offset": 0,
"limit": 64
}
},
"phase": "inspect",
"elapsed_ms": 15135.096698,
"reply_bytes": 7093,
"is_error": false
},
{
"name": "export_evidence_bundle",
"input": {
"path": "/tmp/rea-perf-static-tool-audit-20261024/sdk-cache-cohort/erl_lint-2-selected/evidence.json"
},
"phase": "export",
"elapsed_ms": 37.63045200000124,
"reply_bytes": 376,
"is_error": false
}
],
"phases": {
"discovery_ms": 1335.57118
},
"wall_seconds": 19.375,
"cpu_seconds": 16.139353,
"kernel_family_peak_bytes": 1059520512,
"events_at_payload_exit": {
"memory.events": {
"low": 0,
"high": 0,
"max": 0,
"oom": 0,
"oom_kill": 0,
"oom_group_kill": 0
},
"pids.events": {
"max": 0
}
},
"payload_cleanup_verified": true,
"final_whole_scope_events": null,
"token_proxy": {
"question": 32,
"server_instructions": 513,
"input_catalog_once": 56359,
"arguments": 120,
"text_replies": 1066,
"combined": 58090,
"complete_transport_catalog": 834792
},
"receipt_sha256": {
"report.json": "d25a255b106e1bb232237474915e2900e313ba354aded0d27dad81ea74cfe0c6",
"catalog.json": "9c09aa2944f10c806df78d1cfa2e1daeddc0a487c432701cb67fa65e14213b0c",
"transcript.json": "2c0f92c25f97bb825b99133b84d792c7aac315b4c100986d75cb01939af6ce77",
"evidence.json": "2373bc2e57e40aac881f3ef91fa77187b098dd61d6cf44d37a4b8fe56d022b40",
"erl_lint-2-selected.log": "011553e956860fb3ce9d4e54ca7d4a2baf88a260786d82f60b1d0b70974fdbb7"
}
},
{
"case": "erl_lint-2-complete",
"lane": "complete",
"artifact": {
"path": "/tmp/rea-perf-static-tool-audit-20261024/real-beam/erl_lint.beam",
"sha256": "716d78394b7c3da983e36aac7aadd6673e5041677ba7de20413cc2704d958d55",
"bytes": 233968
},
"answer_count": 4,
"answer_sha256": "34ce3091dbea9ac8cd20831cf6443c7a1de6e02b753246cafa3e0e22b83907c9",
"parent_evidence_id": "ev_df4bfa8b8f0391ad6291542fdccf9062ae3bb8dbabf01c2b166ca6f3493051fc",
"source_report_sha256": "db64350ae25bff9e77f5e167f4a4ac095ed872e09c15de5ce3b2728c078a16d2",
"runtime_identity_sha256": "1b4b40330a2e21d31c06a07d10937cb6a3b37cd0bcbf3d129380341d486b916e",
"catalog_sha256": "f4a683209df5bed1ed25ae42c690fdd803348742132e88f77ba2e4e8333ca562",
"calls": [
{
"name": "inspect_erlang_beam",
"input": {
"path": "/tmp/rea-perf-static-tool-audit-20261024/real-beam/erl_lint.beam"
},
"phase": "inspect",
"elapsed_ms": 15133.678018999999,
"reply_bytes": 146627,
"is_error": false
},
{
"name": "export_evidence_bundle",
"input": {
"path": "/tmp/rea-perf-static-tool-audit-20261024/sdk-cache-cohort/erl_lint-2-complete/evidence.json"
},
"phase": "export",
"elapsed_ms": 44.53432299999986,
"reply_bytes": 376,
"is_error": false
}
],
"phases": {
"discovery_ms": 1000.9708420000002
},
"wall_seconds": 19.764,
"cpu_seconds": 15.287268,
"kernel_family_peak_bytes": 1041948672,
"events_at_payload_exit": {
"memory.events": {
"low": 0,
"high": 0,
"max": 0,
"oom": 0,
"oom_kill": 0,
"oom_group_kill": 0
},
"pids.events": {
"max": 0
}
},
"payload_cleanup_verified": true,
"final_whole_scope_events": null,
"token_proxy": {
"question": 32,
"server_instructions": 513,
"input_catalog_once": 55409,
"arguments": 78,
"text_replies": 20875,
"combined": 76907,
"complete_transport_catalog": 829075
},
"receipt_sha256": {
"report.json": "61aa7d7c78db145af4399b22d57aa39c2008c479530b28a49d401a6d564e46f7",
"catalog.json": "f4a683209df5bed1ed25ae42c690fdd803348742132e88f77ba2e4e8333ca562",
"transcript.json": "3729bcc5547ab06f80b578c24bbb7937a438b0c190ee6680d0d7868cb5885159",
"evidence.json": "ca66367cd1bb325d7d4e47f958c86234d01aaf749b68778ac5212cac246aacb3",
"erl_lint-2-complete.log": "7bb8d637df3cb7ba542345c9d93ea09fe967d11c90cf5383083db7cd2c5fafb8"
}
},
{
"case": "erl_lint-3-complete",
"lane": "complete",
"artifact": {
"path": "/tmp/rea-perf-static-tool-audit-20261024/real-beam/erl_lint.beam",
"sha256": "716d78394b7c3da983e36aac7aadd6673e5041677ba7de20413cc2704d958d55",
"bytes": 233968
},
"answer_count": 4,
"answer_sha256": "34ce3091dbea9ac8cd20831cf6443c7a1de6e02b753246cafa3e0e22b83907c9",
"parent_evidence_id": "ev_df4bfa8b8f0391ad6291542fdccf9062ae3bb8dbabf01c2b166ca6f3493051fc",
"source_report_sha256": "db64350ae25bff9e77f5e167f4a4ac095ed872e09c15de5ce3b2728c078a16d2",
"runtime_identity_sha256": "1b4b40330a2e21d31c06a07d10937cb6a3b37cd0bcbf3d129380341d486b916e",
"catalog_sha256": "f4a683209df5bed1ed25ae42c690fdd803348742132e88f77ba2e4e8333ca562",
"calls": [
{
"name": "inspect_erlang_beam",
"input": {
"path": "/tmp/rea-perf-static-tool-audit-20261024/real-beam/erl_lint.beam"
},
"phase": "inspect",
"elapsed_ms": 13658.772345000001,
"reply_bytes": 146627,
"is_error": false
},
{
"name": "export_evidence_bundle",
"input": {
"path": "/tmp/rea-perf-static-tool-audit-20261024/sdk-cache-cohort/erl_lint-3-complete/evidence.json"
},
"phase": "export",
"elapsed_ms": 48.42420299999867,
"reply_bytes": 376,
"is_error": false
}
],
"phases": {
"discovery_ms": 1121.564358
},
"wall_seconds": 17.257,
"cpu_seconds": 16.276557,
"kernel_family_peak_bytes": 1078980608,
"events_at_payload_exit": {
"memory.events": {
"low": 0,
"high": 0,
"max": 0,
"oom": 0,
"oom_kill": 0,
"oom_group_kill": 0
},
"pids.events": {
"max": 0
}
},
"payload_cleanup_verified": true,
"final_whole_scope_events": null,
"token_proxy": {
"question": 32,
"server_instructions": 513,
"input_catalog_once": 55409,
"arguments": 78,
"text_replies": 20875,
"combined": 76907,
"complete_transport_catalog": 829075
},
"receipt_sha256": {
"report.json": "d683aab80d34c4f0fb474981c5715a4ba1ee706bf4e74be10bf69c40358ec011",
"catalog.json": "f4a683209df5bed1ed25ae42c690fdd803348742132e88f77ba2e4e8333ca562",
"transcript.json": "12380e1e67368b8df7787d54be828c953d4e2ef63007c04cd9fc912b42cc42b5",
"evidence.json": "ca66367cd1bb325d7d4e47f958c86234d01aaf749b68778ac5212cac246aacb3",
"erl_lint-3-complete.log": "404f609e4dc10bbf0d8f1d3076751b2cb8b271ff401b00ea161a00944a55d558"
}
},
{
"case": "erl_lint-3-selected",
"lane": "selected",
"artifact": {
"path": "/tmp/rea-perf-static-tool-audit-20261024/real-beam/erl_lint.beam",
"sha256": "716d78394b7c3da983e36aac7aadd6673e5041677ba7de20413cc2704d958d55",
"bytes": 233968
},
"answer_count": 4,
"answer_sha256": "34ce3091dbea9ac8cd20831cf6443c7a1de6e02b753246cafa3e0e22b83907c9",
"parent_evidence_id": "ev_df4bfa8b8f0391ad6291542fdccf9062ae3bb8dbabf01c2b166ca6f3493051fc",
"source_report_sha256": "db64350ae25bff9e77f5e167f4a4ac095ed872e09c15de5ce3b2728c078a16d2",
"runtime_identity_sha256": "a74d203a5a12d7eb82aaa1e0fda6595b9647af2255f318bf116953777052460f",
"catalog_sha256": "9c09aa2944f10c806df78d1cfa2e1daeddc0a487c432701cb67fa65e14213b0c",
"calls": [
{
"name": "inspect_erlang_beam",
"input": {
"path": "/tmp/rea-perf-static-tool-audit-20261024/real-beam/erl_lint.beam",
"detail": "summary",
"view": {
"kind": "beam",
"collection": "exports",
"names": ["module", "exprs", "expr"],
"fields": ["function_index", "function", "arity", "label"],
"offset": 0,
"limit": 64
}
},
"phase": "inspect",
"elapsed_ms": 14405.966256000002,
"reply_bytes": 7093,
"is_error": false
},
{
"name": "export_evidence_bundle",
"input": {
"path": "/tmp/rea-perf-static-tool-audit-20261024/sdk-cache-cohort/erl_lint-3-selected/evidence.json"
},
"phase": "export",
"elapsed_ms": 31.32568199999878,
"reply_bytes": 376,
"is_error": false
}
],
"phases": {
"discovery_ms": 1107.032897
},
"wall_seconds": 18.052,
"cpu_seconds": 16.867381,
"kernel_family_peak_bytes": 1105092608,
"events_at_payload_exit": {
"memory.events": {
"low": 0,
"high": 0,
"max": 0,
"oom": 0,
"oom_kill": 0,
"oom_group_kill": 0
},
"pids.events": {
"max": 0
}
},
"payload_cleanup_verified": true,
"final_whole_scope_events": null,
"token_proxy": {
"question": 32,
"server_instructions": 513,
"input_catalog_once": 56359,
"arguments": 120,
"text_replies": 1066,
"combined": 58090,
"complete_transport_catalog": 834792
},
"receipt_sha256": {
"report.json": "732e5b4f880d3a981d5362ece88578a2da9e68e80e97eca2dec2ae5856ebbdee",
"catalog.json": "9c09aa2944f10c806df78d1cfa2e1daeddc0a487c432701cb67fa65e14213b0c",
"transcript.json": "e62dc35d939b21418989093a8addadbe533d358a046feb7d99cecab10cedc2d4",
"evidence.json": "2373bc2e57e40aac881f3ef91fa77187b098dd61d6cf44d37a4b8fe56d022b40",
"erl_lint-3-selected.log": "2f35909a9ad5efa9e8ddeffe665887e8a9e9c97c13b4a030b765dd1ceecb14ef"
}
}
]
}
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,125 @@
# Selected BEAM exports: completed-task measurements
Issue #6938. The analyst asks for every exported function with a small set of
exact names, preserving arities, original table/atom indexes and labels. Both
lanes export and validate the **complete** parent report, not only the answer.
Baseline `e09b06369d923e9cccedd91f6839ba643dfbd893`; selected delivery
`bcc3f7611` (full SHA and source-clean runtime identities in the JSON receipt).
The reader, format budgets and complete Evidence identity are unchanged.
Node 24.22.1, SDK client/core/server 2.3.1, npm 11.16.0; Linux x86-64. The
18-task cohort alternates three fresh complete/selected sessions per artifact.
Fresh HOME/client/server and disabled Node compile cache; OS caches unflushed.
This is a scripted client task, not an actual model investigation.
## Results
Medians, before → after. Seconds and kernel process-family MiB (includes file
cache), not parent-only RSS. Ranges and every receipt hash are in
[the 18-task JSON](erlang-beam-views.json).
| Real artifact | Whole-task estimated tokens | Wall s | Family CPU s | Family peak MiB |
| ------------- | --------------------------: | ------------: | ------------: | --------------: |
| erl_lint | 76,907 → 58,090 | 6.773 → 6.870 | 5.581 → 5.312 | 667.0 → 630.9 |
| erl_parse | 72,499 → 58,087 | 6.367 → 6.974 | 5.627 → 5.555 | 666.1 → 657.5 |
| pg2 | 57,733 → 58,018 | 6.365 → 7.576 | 5.295 → 5.615 | 648.2 → 626.9 |
For erl_lint the estimate drops 24.5%; erl_parse drops 19.9%. Small pg2 grows 0.5%
(285 tokens), so selected delivery does not benefit every input. Complete stays
the default. No stable general elapsed/CPU/memory improvement is established;
the small selected task has a higher wall/CPU median. Full parsing/retention and
cold client setup remain. Matching source-report, parent-ID and answer digests
agree across all six tasks for each artifact.
The projection counts one question, server instructions, all 221 input-tool
names/descriptions/schemas once, and every argument/text reply including export.
`tiktoken 0.14.0/cl100k_base` serialization estimates are not actual model usage.
Input directory cost grows 55,409→56,359tokens; the complete transport catalog
including output schemas grows 829,075→834,792tokens and is reported separately.
Protocol framing, host injection/history replay, output-schema presentation to
a model and inference quality are unmeasured. Reported context reduction is
specific to this declared client projection.
## SDK cached-catalog control
Six additional alternating fresh tasks use the SDK's cached catalog/default
validator without an explicit `toolDefinition`. They answer/export the same
question and pass the same independent oracle and cleanup checks. They are
separate from the 18-task primary cohort; do not pool their startup costs.
Median wall 19.764→19.375s (ranges 17.257–21.577 / 18.052–22.397);
family CPU 16.277→16.867s (ranges 15.287–16.562 / 16.139–17.070);
family peak 1024.2→1053.9MiB.
No general cached-client startup improvement is claimed. See
[all six control receipts](erlang-beam-sdk-cache.json).
## Correctness and resources
The unmodified Debian `erlang-base_27.3.4.1+dfsg-1+deb13u3_amd64.deb` supplies
actual OTP 27 compiler/stdlib modules: erl_lint 233,968 bytes (915 atoms, 119 imports,
112 lambdas), erl_parse 608,240 bytes (1,035 atoms), and pg2 1,152 bytes (4 atoms).
The package URL/SHA256, every selected artifact digest, source revision, driver
hash and runtime fingerprint are retained in the JSON records. The independent
[Python struct oracle](erlang-beam-oracle.py) checks every decoded table, chunk
range/payload digest and Code header, then the selected export answer. It reads
bytes only and deliberately qualifies these fixed OTP 27 layouts, not every
historical/compact format. No Erlang installation, emulator or target execution
was used. Default CLI and selected CLI on all three inputs also matched every
oracle fact and parent ID and published exactly the complete source Evidence.
Each of the 24 SDK tasks exited 0 with payload cleanup verified and no guard stop,
OOM or task-cap event. Aggregate CPUQuota 100%, payload affinity0–1,3 GiB early
stop/4 GiB hard limit, no swap, TasksMax 192. CPU/peak counters are observed after
the payload exits while the observer lives; final whole-scope counters remain
unavailable. Costs include SDK discovery/schema validation, runtime/dependency
hashing, assertions, source retention/export and cleanup. Sample acquisition,
oracle generation and tokenization occur outside timed tasks. Node, all compiled dist
files, selected entrypoints, 59 actual SDK files and driver files are hashed
before/after; the source revisions were clean when fingerprints were created.
Public regressions additionally exercise all five collections, duplicate names
and arities, selectors/fields/pages, missing versus empty tables, unresolved atom
names, unavailable retention, source deletion, offline bundles and refusal before
file reads. They establish those boundaries; real new-view qualification here is
OTP 27 on Linux, not every platform or provider.
## Reproducing the question
Build clean pinned checkouts with the repository's pinned toolchain and acquire
the public package linked in the JSON. Keep extracted files outside the repo.
Generate each oracle, for example:
```sh
python3 experiments/analysis-efficiency/erlang-beam-oracle.py \
--input /samples/erl_lint.beam --output /runs/erl_lint-oracle.json \
--name module --name exprs --name expr
node experiments/analysis-efficiency/erlang-beam-views.mjs \
--runtime /built/rea --identity /runs/runtime-identity.json \
--oracle /runs/erl_lint-oracle.json --output /runs/erl_lint-selected \
--lane selected
```
A runtime fingerprint supplies `compiled_files` and `entrypoints` with relative
paths/SHA256, `driver_files` relative to this experiment directory, actual SDK
`dependency_files` with absolute paths/SHA256, and `node_sha256`. Source revision
and clean-state fields record preparation. Create fingerprints before timing;
the driver verifies them before and after each task. The publication records
fingerprint/receipt hashes and full commands; the original complete fingerprint
files, SDK transcripts, complete Evidence bundles and resource logs remain at
`/tmp/rea-perf-static-tool-audit-20261024/`. The local cohort controller is retained
there too. These local receipts are not shipped binaries or installed providers.
Wrap each invocation with `resource-run.py --observe-payload-exit`, setting the
pinned Node PATH/heap, process affinity and resource policy as in its recorded
command. Use a fresh output directory/session each time, alternate lanes three
times per artifact, and count with
[erlang-beam-costs.py](erlang-beam-costs.py). `--page-size` can qualify extra
retained-page calls; `--sdk-cache` selects the separate cached-catalog control.
The18 published primary tasks use the exact discovered toolDefinition/default
validator and 64-row pages, requiring two calls each including full-source export.
The initial exploratory baseline used earlier main `e2463372`; it is not mixed
into this table. Failed development checks remain in the local record: an
incorrect formatter path and copied service import, followed by exact complete
contract narrowing in old test consumers; corrected type/format/public checks
passed. These were not successful benchmark samples.
@@ -0,0 +1,255 @@
/** Compare a completed selected-export question through production stdio MCP. */
import assert from "node:assert/strict";
import { createHash } from "node:crypto";
import { createWriteStream } from "node:fs";
import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { finished } from "node:stream/promises";
import { parseArgs } from "node:util";
import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
const { values } = parseArgs({
options: {
...Object.fromEntries(
["runtime", "oracle", "output", "identity"].map((k) => [
k,
{ type: "string" },
]),
),
lane: { type: "string", default: "complete" },
"page-size": { type: "string", default: "64" },
"sdk-cache": { type: "boolean", default: false },
},
});
assert.ok(["complete", "selected"].includes(values.lane));
const pageSize = Number(values["page-size"]);
assert.ok(Number.isSafeInteger(pageSize) && pageSize > 0);
for (const key of ["runtime", "oracle", "output", "identity"])
assert.equal(values[key], resolve(values[key]));
const { createRetainedQueryCaller } =
await import("./retained-query-calls.mjs");
const { finalizeRetainedQueryReceipt } =
await import("./retained-query-finalization.mjs");
const hash = (x) => createHash("sha256").update(x).digest("hex"),
oracleBytes = await readFile(values.oracle),
expected = JSON.parse(oracleBytes),
identityBytes = await readFile(values.identity),
identity = JSON.parse(identityBytes);
assert.equal(
hash(await readFile(expected.expected.artifact.path)),
expected.expected.artifact.sha256,
);
await mkdir(values.output, { mode: 0o700 });
const account = await mkdtemp(join(tmpdir(), "rea-beam-task-"));
const client = new Client({ name: "beam-selected-exports", version: "1" });
const transport = new StdioClientTransport({
command: process.execPath,
args: [join(values.runtime, "scripts/rea.mjs"), "mcp"],
maxBufferSize: 10485760,
env: {
HOME: account,
TMPDIR: account,
PATH: process.env.PATH,
REA_LOG_LEVEL: "silent",
NODE_OPTIONS: "--max-old-space-size=2048",
NODE_DISABLE_COMPILE_CACHE: "1",
UV_THREADPOOL_SIZE: "2",
},
stderr: "pipe",
});
const serverLog = createWriteStream(join(values.output, "server.log")),
serverLogFinished = finished(serverLog);
serverLogFinished.catch(() => {});
transport.stderr?.pipe(serverLog);
const report = {
status: "running",
lane: values.lane,
artifact: expected.expected.artifact,
calls: [],
phases: {},
question: `Return every exported function named ${expected.names.join(", ")} with its original export-table index, function atom index, arity and code label. Preserve complete source Evidence.`,
client: values["sdk-cache"]
? "Production stdio SDK, cached catalog definitions and default validator"
: "Production stdio SDK, exact discovered toolDefinition and default validator",
runtime_identity_sha256: hash(identityBytes),
oracle_sha256: hash(oracleBytes),
driver_sha256: hash(await readFile(new URL(import.meta.url))),
};
const verifyRuntime = async () => {
assert.equal(hash(await readFile(values.identity)), hash(identityBytes));
assert.equal(hash(await readFile(values.oracle)), hash(oracleBytes));
for (const r of [...identity.compiled_files, ...identity.entrypoints])
assert.equal(
hash(await readFile(join(values.runtime, r.path))),
r.sha256,
r.path,
);
for (const r of identity.dependency_files ?? [])
assert.equal(hash(await readFile(r.path)), r.sha256, r.path);
for (const r of identity.driver_files)
assert.equal(
hash(await readFile(new URL(r.path, import.meta.url))),
r.sha256,
r.path,
);
assert.equal(hash(await readFile(process.execPath)), identity.node_sha256);
assert.equal(
hash(await readFile(new URL(import.meta.url))),
report.driver_sha256,
);
};
const transcript = [];
let definitions;
const caller = createRetainedQueryCaller(
{
callTool(params, options) {
return client.callTool(
params,
values["sdk-cache"]
? options
: {
...options,
toolDefinition: definitions.get(params.name),
},
);
},
},
(receipt, event) => {
report.calls.push(receipt);
transcript.push(event);
},
);
const observe = async (name, input, phase) => {
const result = await caller(name, input, phase);
assert.notEqual(result.isError, true, JSON.stringify(result));
assert.ok(result.structuredContent);
return result.structuredContent;
};
const verifySource = (source) => {
for (const [key, value] of Object.entries(expected.expected))
assert.deepEqual(source[key], value, key);
assert.equal(source.coverage.status, "complete");
};
try {
await verifyRuntime();
const started = performance.now();
await client.connect(transport);
report.server_instructions = client.getInstructions() ?? "";
const catalog = await client.listTools();
definitions = new Map(catalog.tools.map((t) => [t.name, t]));
report.catalog_tools = catalog.tools.length;
await writeFile(join(values.output, "catalog.json"), JSON.stringify(catalog));
report.phases.discovery_ms = performance.now() - started;
const selected = values.lane === "selected";
const selectedView = (offset) => ({
kind: "beam",
collection: "exports",
names: expected.names,
fields: ["function_index", "function", "arity", "label"],
offset,
limit: pageSize,
});
const response = await observe(
"inspect_erlang_beam",
{
path: expected.expected.artifact.path,
...(selected ? { detail: "summary", view: selectedView(0) } : {}),
},
"inspect",
);
report.parent_evidence_id = selected
? response.normalized_result.parent_evidence_id
: response.evidence_id;
let answer;
if (!selected) {
verifySource(response.normalized_result);
answer = (response.normalized_result.exports ?? [])
.map((r, i) => ({ source_index: i, ...r }))
.filter((r) => expected.names.includes(r.function));
} else {
answer = [];
let offset = 0,
page = response.normalized_result;
for (;;) {
assert.equal(page.kind, "beam");
assert.equal(page.page.collection, "exports");
assert.deepEqual(page.beam.artifact, expected.expected.artifact);
assert.deepEqual(page.beam.coverage, expected.expected.coverage);
assert.equal(page.beam.counts.exports, expected.expected.exports.length);
assert.equal(page.coverage.total, expected.answer.length);
answer.push(...page.page.items);
if (page.coverage.next_offset === null) break;
assert.ok(page.coverage.next_offset > offset);
offset = page.coverage.next_offset;
page = (
await observe(
"inspect_analysis_view",
{
source: {
kind: "retained-evidence",
evidence_id: report.parent_evidence_id,
},
view: selectedView(offset),
},
"selected-page",
)
).normalized_result;
}
}
assert.deepEqual(answer, expected.answer);
report.answer_count = answer.length;
report.answer_sha256 = hash(
JSON.stringify(
answer.map((r) =>
Object.fromEntries(
Object.entries(r).sort(([a], [b]) => a.localeCompare(b)),
),
),
),
);
await writeFile(join(values.output, "answer.json"), JSON.stringify(answer));
await observe(
"export_evidence_bundle",
{ path: join(values.output, "evidence.json") },
"export",
);
const bundle = JSON.parse(
await readFile(join(values.output, "evidence.json")),
);
const parent = bundle.records.find(
(r) => r.evidence_id === report.parent_evidence_id,
);
assert.ok(parent);
assert.equal(parent.operation, "inspect_erlang_beam");
verifySource(parent.normalized_result);
report.source_report_sha256 = hash(JSON.stringify(parent.normalized_result));
report.complete_source_export_verified = true;
report.status = "passed";
} catch (cause) {
report.status = "failed";
report.error = String(cause);
} finally {
try {
await verifyRuntime();
} catch (cause) {
report.status = "failed";
report.runtime_identity_failure = String(cause);
}
await finalizeRetainedQueryReceipt({
client,
transport,
serverLog,
serverLogFinished,
account,
report,
output: values.output,
transcript,
});
if (
report.status !== "passed" ||
report.cleanup_failures.length ||
report.receipt_failures.length
)
process.exitCode = 1;
}