mirror of
https://github.com/morluto/rea.git
synced 2026-10-26 17:35:11 +00:00
feat(erlang): query selected BEAM table facts (#6943)
* feat(erlang): query selected BEAM table facts * docs(perf): record BEAM selected-view measurements
This commit is contained in:
1 parent
c2caae88f8
commit
ead9fdcda9
20 files changed
+4206
-66
No files matched your search
@@ -0,0 +1,111 @@
|
||||
"""Verify completed BEAM selection/export tasks and count a declared token projection."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import statistics
|
||||
|
||||
import tiktoken
|
||||
|
||||
|
||||
def compact(value):
|
||||
return json.dumps(value, ensure_ascii=False, separators=(',', ':'), allow_nan=False)
|
||||
|
||||
|
||||
def digest(path):
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def inspect(path, encoding):
|
||||
report = json.loads((path / 'report.json').read_text())
|
||||
catalog = json.loads((path / 'catalog.json').read_text())
|
||||
transcript = json.loads((path / 'transcript.json').read_text())
|
||||
scope = next(json.loads(line) for line in reversed(path.with_suffix('.log').read_text().splitlines())
|
||||
if line.startswith('{"resource_scope":'))
|
||||
observed = scope.get('payload_resource_observation') or {}
|
||||
events = observed.get('resource_events', {})
|
||||
assert report['status'] == 'passed' and report['runtime_identity_sha256']
|
||||
assert report['complete_source_export_verified'] is True
|
||||
assert not report['cleanup_failures'] and not report['receipt_failures']
|
||||
assert scope['abort'] is None and scope['payload_resource_observation_verified'] is True
|
||||
assert observed['payload_exit_code'] == 0 and observed['payload_cleanup_verified'] is True
|
||||
assert all(events.get('memory.events', {}).get(key) == 0 for key in ['high', 'max', 'oom', 'oom_kill'])
|
||||
assert events.get('pids.events', {}).get('max') == 0
|
||||
assert len(catalog['tools']) == report['catalog_tools'] == 221
|
||||
assert transcript[0]['name'] == 'inspect_erlang_beam'
|
||||
assert transcript[-1]['name'] == 'export_evidence_bundle'
|
||||
assert all(event['name'] == 'inspect_analysis_view' for event in transcript[1:-1])
|
||||
assert all('reply' in event and not event['reply'].get('isError', False) for event in transcript)
|
||||
count = lambda text: len(encoding.encode(text, disallowed_special=()))
|
||||
directory = [{key: tool[key] for key in ['name', 'description', 'inputSchema']} for tool in catalog['tools']]
|
||||
request = transcript[0]['input']
|
||||
question = report['question']
|
||||
assert ('detail' in request) == (report['lane'] == 'selected')
|
||||
arguments = sum(count(compact({'name': event['name'], 'arguments': event['input']})) for event in transcript)
|
||||
replies = sum(count(block['text']) for event in transcript for block in event['reply']['content'] if block['type'] == 'text')
|
||||
inputs, prompt = count(compact(directory)), count(question)
|
||||
instructions = count(report['server_instructions'])
|
||||
return dict(case=path.name, lane=report['lane'], artifact=report['artifact'],
|
||||
answer_count=report['answer_count'], answer_sha256=report['answer_sha256'],
|
||||
parent_evidence_id=report['parent_evidence_id'], source_report_sha256=report['source_report_sha256'],
|
||||
runtime_identity_sha256=report['runtime_identity_sha256'], catalog_sha256=digest(path / 'catalog.json'),
|
||||
calls=report['calls'], phases=report['phases'], wall_seconds=scope['elapsed_seconds'],
|
||||
cpu_seconds=observed['cpu_stat']['usage_usec'] / 1e6,
|
||||
kernel_family_peak_bytes=observed['memory_peak_bytes'],
|
||||
events_at_payload_exit=events, payload_cleanup_verified=True,
|
||||
final_whole_scope_events=scope.get('resource_events_final'),
|
||||
token_proxy=dict(question=prompt, server_instructions=instructions, input_catalog_once=inputs, arguments=arguments,
|
||||
text_replies=replies, combined=prompt + instructions + inputs + arguments + replies,
|
||||
complete_transport_catalog=count(compact(catalog))),
|
||||
receipt_sha256={file.name: digest(file) for file in [path / 'report.json', path / 'catalog.json',
|
||||
path / 'transcript.json', path / 'evidence.json', path.with_suffix('.log')]})
|
||||
|
||||
|
||||
def summarize(rows):
|
||||
result = {}
|
||||
for artifact in sorted({row['artifact']['sha256'] for row in rows}):
|
||||
cases = [row for row in rows if row['artifact']['sha256'] == artifact]
|
||||
for field in ['answer_count', 'answer_sha256', 'parent_evidence_id', 'source_report_sha256']:
|
||||
assert len({row[field] for row in cases}) == 1, field
|
||||
result[artifact] = {}
|
||||
for lane in ['complete', 'selected']:
|
||||
samples = [row for row in cases if row['lane'] == lane]
|
||||
assert len(samples) == 3
|
||||
for field in ['runtime_identity_sha256', 'catalog_sha256']:
|
||||
assert len({row[field] for row in samples}) == 1, field
|
||||
metrics = {key: [row[key] for row in samples] for key in ['wall_seconds', 'cpu_seconds', 'kernel_family_peak_bytes']}
|
||||
metrics['projected_tokens'] = [row['token_proxy']['combined'] for row in samples]
|
||||
metrics['mcp_reply_bytes'] = [sum(call['reply_bytes'] for call in row['calls']) for row in samples]
|
||||
result[artifact][lane] = {key: dict(median=statistics.median(values), minimum=min(values), maximum=max(values))
|
||||
for key, values in metrics.items()}
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--cohort', type=Path, required=True)
|
||||
parser.add_argument('--output', type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
manifest = json.loads((args.cohort / 'manifest.json').read_text())
|
||||
index = json.loads((args.cohort / 'index.json').read_text())
|
||||
assert len(index) == 18 and all(case['exit'] == 0 for case in index)
|
||||
for file, expected in manifest['driver_files'].items():
|
||||
assert digest(Path(file)) == expected, file
|
||||
identities = {}
|
||||
for lane, path in manifest['runtime_identities'].items():
|
||||
identity = json.loads(Path(path).read_text())
|
||||
assert identity['source_clean'] is True
|
||||
identities[lane] = dict(sha256=digest(Path(path)), source_revision=identity['source_revision'])
|
||||
encoding = tiktoken.get_encoding('cl100k_base')
|
||||
rows = [inspect(args.cohort / case['name'], encoding) for case in index]
|
||||
assert all(row['runtime_identity_sha256'] == identities[row['lane']]['sha256'] for row in rows)
|
||||
receipt = dict(tiktoken=tiktoken.__version__, encoding=encoding.name, actual_model_usage=None,
|
||||
cohort=manifest, commands=index, runtime_identities=identities,
|
||||
projection='Scripted fresh SDK tasks, not model runs. One fixed question, server instructions and complete input directory (name/description/inputSchema), every argument and MCP text reply including export. Raw catalog including output schemas reported separately. No framing, host injection, history replay, inference quality or billable usage measured.',
|
||||
resources='CPU quota100%, payload affinity0-1, initial observer15. Early stop3GiB, hard4GiB, swap0. Fresh HOME/client/server; Node compile cache disabled, OS caches unflushed. Whole process-family CPU and kernel memory peak include file cache, runtime identity checks, hashing, assertions, SDK/server serialization and cleanup. Oracle generation and tokenization outside timed tasks. Payload-exit counters verified; final whole-scope counters unavailable.',
|
||||
summary=summarize(rows), recordings=rows)
|
||||
args.output.write_text(json.dumps(receipt, ensure_ascii=False, indent=2) + '\n')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,108 @@
|
||||
"""Independent struct oracle for the fixed public OTP 27 benchmark artifacts.
|
||||
|
||||
Reads bytes only. It deliberately refuses compact atom layouts; format-wide
|
||||
coverage belongs to the production reader and its separate regressions.
|
||||
"""
|
||||
from pathlib import Path
|
||||
import getopt
|
||||
import hashlib
|
||||
import json
|
||||
import struct
|
||||
import sys
|
||||
|
||||
|
||||
def decode(path):
|
||||
data = path.read_bytes()
|
||||
assert len(data) <= 64 * 1024 * 1024
|
||||
u32 = lambda at: struct.unpack_from('>I', data, at)[0]
|
||||
assert data[:4] == b'FOR1' and data[8:12] == b'BEAM'
|
||||
assert u32(4) + 8 == len(data)
|
||||
chunks, selected, at = [], {}, 12
|
||||
while at < len(data):
|
||||
name = data[at:at + 4].decode('ascii')
|
||||
size = u32(at + 4)
|
||||
end = at + 8 + size
|
||||
padded = (end + 3) & ~3
|
||||
assert end <= padded <= len(data)
|
||||
chunks.append(dict(index=len(chunks), name=name,
|
||||
location=dict(offset=at, bytes=padded - at),
|
||||
payload=dict(location=dict(offset=at + 8, bytes=size),
|
||||
bytes=size, sha256=hashlib.sha256(data[at + 8:end]).hexdigest())))
|
||||
selected[name] = (at + 8, end)
|
||||
at = padded
|
||||
assert at == len(data)
|
||||
atom_name = 'AtU8' if 'AtU8' in selected else 'Atom'
|
||||
start, end = selected[atom_name]
|
||||
n = u32(start)
|
||||
assert n < 65536, 'Fixed OTP 27 oracle does not decode compact atom tables'
|
||||
at, atoms = start + 4, []
|
||||
for i in range(n):
|
||||
size = data[at]
|
||||
at += 1
|
||||
name = data[at:at + size].decode('utf-8' if atom_name == 'AtU8' else 'latin1')
|
||||
atoms.append(dict(index=i + 1, name=name))
|
||||
at += size
|
||||
assert at == end
|
||||
|
||||
def atom(i):
|
||||
return atoms[i - 1]['name'] if 0 < i <= len(atoms) else None
|
||||
|
||||
def table(name, keys):
|
||||
if name not in selected:
|
||||
return None
|
||||
start, end = selected[name]
|
||||
if name == 'FunT' and start == end:
|
||||
return None
|
||||
n, width = u32(start), len(keys)
|
||||
assert start + 4 + n * 4 * width == end
|
||||
rows = []
|
||||
for i in range(n):
|
||||
values = struct.unpack_from('>' + 'I' * width, data, start + 4 + i * width * 4)
|
||||
row = dict(zip(keys, values))
|
||||
if 'module_index' in row:
|
||||
row['module'] = atom(row['module_index'])
|
||||
row['function'] = atom(row['function_index'])
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
exports = table('ExpT', ['function_index', 'arity', 'label'])
|
||||
imports = table('ImpT', ['module_index', 'function_index', 'arity'])
|
||||
lambdas = table('FunT', ['function_index', 'arity', 'label', 'index', 'free_vars', 'old_unique'])
|
||||
code = None
|
||||
if 'Code' in selected:
|
||||
start, end = selected['Code']
|
||||
size = u32(start)
|
||||
assert size >= 16 and start + 4 + size <= end
|
||||
code = dict(zip(['instruction_set', 'opcode_max', 'labels', 'functions'],
|
||||
struct.unpack_from('>IIII', data, start + 4)))
|
||||
code.update(extra_header_bytes=size - 16,
|
||||
instructions=dict(offset=start + 4 + size, bytes=end - start - 4 - size))
|
||||
return dict(artifact=dict(path=str(path), sha256=hashlib.sha256(data).hexdigest(), bytes=len(data)),
|
||||
form=dict(location=dict(offset=0, bytes=len(data)), size=len(data) - 8),
|
||||
chunks=chunks, atoms=atoms, atom_encoding='utf8' if atom_name == 'AtU8' else 'latin1',
|
||||
exports=exports, imports=imports, lambdas=lambdas, code=code,
|
||||
module=atoms[0]['name'],
|
||||
coverage=dict(status='complete', examined_chunks=len(chunks), atoms=len(atoms)))
|
||||
|
||||
|
||||
def main():
|
||||
options, remaining = getopt.getopt(sys.argv[1:], '', ['input=', 'output=', 'name='])
|
||||
assert not remaining
|
||||
inputs = [v for k, v in options if k == '--input']
|
||||
outputs = [v for k, v in options if k == '--output']
|
||||
names = [v for k, v in options if k == '--name']
|
||||
assert len(inputs) == len(outputs) == 1 and names
|
||||
path = Path(inputs[0])
|
||||
assert path.is_absolute()
|
||||
expected = decode(path)
|
||||
answer = [dict(source_index=i, **row) for i, row in enumerate(expected['exports'] or [])
|
||||
if row['function'] in names]
|
||||
result = dict(oracle='Independent Python struct decoding of fixed OTP27 bytes; no BEAM execution. '
|
||||
'Every table, chunk range/hash and Code header checked. Other encodings/malformed inputs unqualified.',
|
||||
source='https://www.erlang.org/doc/apps/stdlib/beam_lib.html',
|
||||
expected=expected, names=names, answer=answer)
|
||||
Path(outputs[0]).write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,534 @@
|
||||
{
|
||||
"method": "3alternating freshsessions perlane; SDK cachesentirecatalog thenusesdefaultvalidator withouttoolDefinition override. Diagnostic client startup control, separate from18exactnamedtasks. Samequestion/fullsourceexports/oracle; allpayloadexit/cleanupverified.",
|
||||
"summary": {
|
||||
"716d78394b7c3da983e36aac7aadd6673e5041677ba7de20413cc2704d958d55": {
|
||||
"complete": {
|
||||
"wall_seconds": {
|
||||
"median": 19.764,
|
||||
"minimum": 17.257,
|
||||
"maximum": 21.577
|
||||
},
|
||||
"cpu_seconds": {
|
||||
"median": 16.276557,
|
||||
"minimum": 15.287268,
|
||||
"maximum": 16.561897
|
||||
},
|
||||
"kernel_family_peak_bytes": {
|
||||
"median": 1073950720,
|
||||
"minimum": 1041948672,
|
||||
"maximum": 1078980608
|
||||
},
|
||||
"projected_tokens": {
|
||||
"median": 76907,
|
||||
"minimum": 76907,
|
||||
"maximum": 76907
|
||||
},
|
||||
"mcp_reply_bytes": {
|
||||
"median": 147003,
|
||||
"minimum": 147003,
|
||||
"maximum": 147003
|
||||
}
|
||||
},
|
||||
"selected": {
|
||||
"wall_seconds": {
|
||||
"median": 19.375,
|
||||
"minimum": 18.052,
|
||||
"maximum": 22.397
|
||||
},
|
||||
"cpu_seconds": {
|
||||
"median": 16.867381,
|
||||
"minimum": 16.139353,
|
||||
"maximum": 17.069666
|
||||
},
|
||||
"kernel_family_peak_bytes": {
|
||||
"median": 1105092608,
|
||||
"minimum": 1059520512,
|
||||
"maximum": 1117036544
|
||||
},
|
||||
"projected_tokens": {
|
||||
"median": 58090,
|
||||
"minimum": 58090,
|
||||
"maximum": 58090
|
||||
},
|
||||
"mcp_reply_bytes": {
|
||||
"median": 7469,
|
||||
"minimum": 7469,
|
||||
"maximum": 7469
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"recordings": [
|
||||
{
|
||||
"case": "erl_lint-1-complete",
|
||||
"lane": "complete",
|
||||
"artifact": {
|
||||
"path": "/tmp/rea-perf-static-tool-audit-20261024/real-beam/erl_lint.beam",
|
||||
"sha256": "716d78394b7c3da983e36aac7aadd6673e5041677ba7de20413cc2704d958d55",
|
||||
"bytes": 233968
|
||||
},
|
||||
"answer_count": 4,
|
||||
"answer_sha256": "34ce3091dbea9ac8cd20831cf6443c7a1de6e02b753246cafa3e0e22b83907c9",
|
||||
"parent_evidence_id": "ev_df4bfa8b8f0391ad6291542fdccf9062ae3bb8dbabf01c2b166ca6f3493051fc",
|
||||
"source_report_sha256": "db64350ae25bff9e77f5e167f4a4ac095ed872e09c15de5ce3b2728c078a16d2",
|
||||
"runtime_identity_sha256": "1b4b40330a2e21d31c06a07d10937cb6a3b37cd0bcbf3d129380341d486b916e",
|
||||
"catalog_sha256": "f4a683209df5bed1ed25ae42c690fdd803348742132e88f77ba2e4e8333ca562",
|
||||
"calls": [
|
||||
{
|
||||
"name": "inspect_erlang_beam",
|
||||
"input": {
|
||||
"path": "/tmp/rea-perf-static-tool-audit-20261024/real-beam/erl_lint.beam"
|
||||
},
|
||||
"phase": "inspect",
|
||||
"elapsed_ms": 18092.161941,
|
||||
"reply_bytes": 146627,
|
||||
"is_error": false
|
||||
},
|
||||
{
|
||||
"name": "export_evidence_bundle",
|
||||
"input": {
|
||||
"path": "/tmp/rea-perf-static-tool-audit-20261024/sdk-cache-cohort/erl_lint-1-complete/evidence.json"
|
||||
},
|
||||
"phase": "export",
|
||||
"elapsed_ms": 52.40052199999991,
|
||||
"reply_bytes": 376,
|
||||
"is_error": false
|
||||
}
|
||||
],
|
||||
"phases": {
|
||||
"discovery_ms": 1018.838796
|
||||
},
|
||||
"wall_seconds": 21.577,
|
||||
"cpu_seconds": 16.561897,
|
||||
"kernel_family_peak_bytes": 1073950720,
|
||||
"events_at_payload_exit": {
|
||||
"memory.events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
},
|
||||
"pids.events": {
|
||||
"max": 0
|
||||
}
|
||||
},
|
||||
"payload_cleanup_verified": true,
|
||||
"final_whole_scope_events": null,
|
||||
"token_proxy": {
|
||||
"question": 32,
|
||||
"server_instructions": 513,
|
||||
"input_catalog_once": 55409,
|
||||
"arguments": 78,
|
||||
"text_replies": 20875,
|
||||
"combined": 76907,
|
||||
"complete_transport_catalog": 829075
|
||||
},
|
||||
"receipt_sha256": {
|
||||
"report.json": "3717e681ac50a70db8bd7303ac18d87bd2028d4321104a793421695a82579290",
|
||||
"catalog.json": "f4a683209df5bed1ed25ae42c690fdd803348742132e88f77ba2e4e8333ca562",
|
||||
"transcript.json": "c718ac6e9b6c63533bc28d18c90c73a03abc1697516ae216a444b6dd55827255",
|
||||
"evidence.json": "ca66367cd1bb325d7d4e47f958c86234d01aaf749b68778ac5212cac246aacb3",
|
||||
"erl_lint-1-complete.log": "a90d1345cd09e90865c8a75f732abe2561c2299d0084718e91205f2cb904f38d"
|
||||
}
|
||||
},
|
||||
{
|
||||
"case": "erl_lint-1-selected",
|
||||
"lane": "selected",
|
||||
"artifact": {
|
||||
"path": "/tmp/rea-perf-static-tool-audit-20261024/real-beam/erl_lint.beam",
|
||||
"sha256": "716d78394b7c3da983e36aac7aadd6673e5041677ba7de20413cc2704d958d55",
|
||||
"bytes": 233968
|
||||
},
|
||||
"answer_count": 4,
|
||||
"answer_sha256": "34ce3091dbea9ac8cd20831cf6443c7a1de6e02b753246cafa3e0e22b83907c9",
|
||||
"parent_evidence_id": "ev_df4bfa8b8f0391ad6291542fdccf9062ae3bb8dbabf01c2b166ca6f3493051fc",
|
||||
"source_report_sha256": "db64350ae25bff9e77f5e167f4a4ac095ed872e09c15de5ce3b2728c078a16d2",
|
||||
"runtime_identity_sha256": "a74d203a5a12d7eb82aaa1e0fda6595b9647af2255f318bf116953777052460f",
|
||||
"catalog_sha256": "9c09aa2944f10c806df78d1cfa2e1daeddc0a487c432701cb67fa65e14213b0c",
|
||||
"calls": [
|
||||
{
|
||||
"name": "inspect_erlang_beam",
|
||||
"input": {
|
||||
"path": "/tmp/rea-perf-static-tool-audit-20261024/real-beam/erl_lint.beam",
|
||||
"detail": "summary",
|
||||
"view": {
|
||||
"kind": "beam",
|
||||
"collection": "exports",
|
||||
"names": ["module", "exprs", "expr"],
|
||||
"fields": ["function_index", "function", "arity", "label"],
|
||||
"offset": 0,
|
||||
"limit": 64
|
||||
}
|
||||
},
|
||||
"phase": "inspect",
|
||||
"elapsed_ms": 18394.333454,
|
||||
"reply_bytes": 7093,
|
||||
"is_error": false
|
||||
},
|
||||
{
|
||||
"name": "export_evidence_bundle",
|
||||
"input": {
|
||||
"path": "/tmp/rea-perf-static-tool-audit-20261024/sdk-cache-cohort/erl_lint-1-selected/evidence.json"
|
||||
},
|
||||
"phase": "export",
|
||||
"elapsed_ms": 58.59494199999972,
|
||||
"reply_bytes": 376,
|
||||
"is_error": false
|
||||
}
|
||||
],
|
||||
"phases": {
|
||||
"discovery_ms": 1056.9951270000001
|
||||
},
|
||||
"wall_seconds": 22.397,
|
||||
"cpu_seconds": 17.069666,
|
||||
"kernel_family_peak_bytes": 1117036544,
|
||||
"events_at_payload_exit": {
|
||||
"memory.events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
},
|
||||
"pids.events": {
|
||||
"max": 0
|
||||
}
|
||||
},
|
||||
"payload_cleanup_verified": true,
|
||||
"final_whole_scope_events": null,
|
||||
"token_proxy": {
|
||||
"question": 32,
|
||||
"server_instructions": 513,
|
||||
"input_catalog_once": 56359,
|
||||
"arguments": 120,
|
||||
"text_replies": 1066,
|
||||
"combined": 58090,
|
||||
"complete_transport_catalog": 834792
|
||||
},
|
||||
"receipt_sha256": {
|
||||
"report.json": "c3c17495f33ee4ea9daabd7a353f698fa0a804451c4bbdbe9eb1adf905ab483a",
|
||||
"catalog.json": "9c09aa2944f10c806df78d1cfa2e1daeddc0a487c432701cb67fa65e14213b0c",
|
||||
"transcript.json": "fd8c589a6e0dbcd0783ef5a526b51db7a3c208cd4067986ed99804bda06c4498",
|
||||
"evidence.json": "2373bc2e57e40aac881f3ef91fa77187b098dd61d6cf44d37a4b8fe56d022b40",
|
||||
"erl_lint-1-selected.log": "fdd55def718ffa7c972b3cb9b6be1b7f66a7fa5fb1c371cc492051ffb0d08f62"
|
||||
}
|
||||
},
|
||||
{
|
||||
"case": "erl_lint-2-selected",
|
||||
"lane": "selected",
|
||||
"artifact": {
|
||||
"path": "/tmp/rea-perf-static-tool-audit-20261024/real-beam/erl_lint.beam",
|
||||
"sha256": "716d78394b7c3da983e36aac7aadd6673e5041677ba7de20413cc2704d958d55",
|
||||
"bytes": 233968
|
||||
},
|
||||
"answer_count": 4,
|
||||
"answer_sha256": "34ce3091dbea9ac8cd20831cf6443c7a1de6e02b753246cafa3e0e22b83907c9",
|
||||
"parent_evidence_id": "ev_df4bfa8b8f0391ad6291542fdccf9062ae3bb8dbabf01c2b166ca6f3493051fc",
|
||||
"source_report_sha256": "db64350ae25bff9e77f5e167f4a4ac095ed872e09c15de5ce3b2728c078a16d2",
|
||||
"runtime_identity_sha256": "a74d203a5a12d7eb82aaa1e0fda6595b9647af2255f318bf116953777052460f",
|
||||
"catalog_sha256": "9c09aa2944f10c806df78d1cfa2e1daeddc0a487c432701cb67fa65e14213b0c",
|
||||
"calls": [
|
||||
{
|
||||
"name": "inspect_erlang_beam",
|
||||
"input": {
|
||||
"path": "/tmp/rea-perf-static-tool-audit-20261024/real-beam/erl_lint.beam",
|
||||
"detail": "summary",
|
||||
"view": {
|
||||
"kind": "beam",
|
||||
"collection": "exports",
|
||||
"names": ["module", "exprs", "expr"],
|
||||
"fields": ["function_index", "function", "arity", "label"],
|
||||
"offset": 0,
|
||||
"limit": 64
|
||||
}
|
||||
},
|
||||
"phase": "inspect",
|
||||
"elapsed_ms": 15135.096698,
|
||||
"reply_bytes": 7093,
|
||||
"is_error": false
|
||||
},
|
||||
{
|
||||
"name": "export_evidence_bundle",
|
||||
"input": {
|
||||
"path": "/tmp/rea-perf-static-tool-audit-20261024/sdk-cache-cohort/erl_lint-2-selected/evidence.json"
|
||||
},
|
||||
"phase": "export",
|
||||
"elapsed_ms": 37.63045200000124,
|
||||
"reply_bytes": 376,
|
||||
"is_error": false
|
||||
}
|
||||
],
|
||||
"phases": {
|
||||
"discovery_ms": 1335.57118
|
||||
},
|
||||
"wall_seconds": 19.375,
|
||||
"cpu_seconds": 16.139353,
|
||||
"kernel_family_peak_bytes": 1059520512,
|
||||
"events_at_payload_exit": {
|
||||
"memory.events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
},
|
||||
"pids.events": {
|
||||
"max": 0
|
||||
}
|
||||
},
|
||||
"payload_cleanup_verified": true,
|
||||
"final_whole_scope_events": null,
|
||||
"token_proxy": {
|
||||
"question": 32,
|
||||
"server_instructions": 513,
|
||||
"input_catalog_once": 56359,
|
||||
"arguments": 120,
|
||||
"text_replies": 1066,
|
||||
"combined": 58090,
|
||||
"complete_transport_catalog": 834792
|
||||
},
|
||||
"receipt_sha256": {
|
||||
"report.json": "d25a255b106e1bb232237474915e2900e313ba354aded0d27dad81ea74cfe0c6",
|
||||
"catalog.json": "9c09aa2944f10c806df78d1cfa2e1daeddc0a487c432701cb67fa65e14213b0c",
|
||||
"transcript.json": "2c0f92c25f97bb825b99133b84d792c7aac315b4c100986d75cb01939af6ce77",
|
||||
"evidence.json": "2373bc2e57e40aac881f3ef91fa77187b098dd61d6cf44d37a4b8fe56d022b40",
|
||||
"erl_lint-2-selected.log": "011553e956860fb3ce9d4e54ca7d4a2baf88a260786d82f60b1d0b70974fdbb7"
|
||||
}
|
||||
},
|
||||
{
|
||||
"case": "erl_lint-2-complete",
|
||||
"lane": "complete",
|
||||
"artifact": {
|
||||
"path": "/tmp/rea-perf-static-tool-audit-20261024/real-beam/erl_lint.beam",
|
||||
"sha256": "716d78394b7c3da983e36aac7aadd6673e5041677ba7de20413cc2704d958d55",
|
||||
"bytes": 233968
|
||||
},
|
||||
"answer_count": 4,
|
||||
"answer_sha256": "34ce3091dbea9ac8cd20831cf6443c7a1de6e02b753246cafa3e0e22b83907c9",
|
||||
"parent_evidence_id": "ev_df4bfa8b8f0391ad6291542fdccf9062ae3bb8dbabf01c2b166ca6f3493051fc",
|
||||
"source_report_sha256": "db64350ae25bff9e77f5e167f4a4ac095ed872e09c15de5ce3b2728c078a16d2",
|
||||
"runtime_identity_sha256": "1b4b40330a2e21d31c06a07d10937cb6a3b37cd0bcbf3d129380341d486b916e",
|
||||
"catalog_sha256": "f4a683209df5bed1ed25ae42c690fdd803348742132e88f77ba2e4e8333ca562",
|
||||
"calls": [
|
||||
{
|
||||
"name": "inspect_erlang_beam",
|
||||
"input": {
|
||||
"path": "/tmp/rea-perf-static-tool-audit-20261024/real-beam/erl_lint.beam"
|
||||
},
|
||||
"phase": "inspect",
|
||||
"elapsed_ms": 15133.678018999999,
|
||||
"reply_bytes": 146627,
|
||||
"is_error": false
|
||||
},
|
||||
{
|
||||
"name": "export_evidence_bundle",
|
||||
"input": {
|
||||
"path": "/tmp/rea-perf-static-tool-audit-20261024/sdk-cache-cohort/erl_lint-2-complete/evidence.json"
|
||||
},
|
||||
"phase": "export",
|
||||
"elapsed_ms": 44.53432299999986,
|
||||
"reply_bytes": 376,
|
||||
"is_error": false
|
||||
}
|
||||
],
|
||||
"phases": {
|
||||
"discovery_ms": 1000.9708420000002
|
||||
},
|
||||
"wall_seconds": 19.764,
|
||||
"cpu_seconds": 15.287268,
|
||||
"kernel_family_peak_bytes": 1041948672,
|
||||
"events_at_payload_exit": {
|
||||
"memory.events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
},
|
||||
"pids.events": {
|
||||
"max": 0
|
||||
}
|
||||
},
|
||||
"payload_cleanup_verified": true,
|
||||
"final_whole_scope_events": null,
|
||||
"token_proxy": {
|
||||
"question": 32,
|
||||
"server_instructions": 513,
|
||||
"input_catalog_once": 55409,
|
||||
"arguments": 78,
|
||||
"text_replies": 20875,
|
||||
"combined": 76907,
|
||||
"complete_transport_catalog": 829075
|
||||
},
|
||||
"receipt_sha256": {
|
||||
"report.json": "61aa7d7c78db145af4399b22d57aa39c2008c479530b28a49d401a6d564e46f7",
|
||||
"catalog.json": "f4a683209df5bed1ed25ae42c690fdd803348742132e88f77ba2e4e8333ca562",
|
||||
"transcript.json": "3729bcc5547ab06f80b578c24bbb7937a438b0c190ee6680d0d7868cb5885159",
|
||||
"evidence.json": "ca66367cd1bb325d7d4e47f958c86234d01aaf749b68778ac5212cac246aacb3",
|
||||
"erl_lint-2-complete.log": "7bb8d637df3cb7ba542345c9d93ea09fe967d11c90cf5383083db7cd2c5fafb8"
|
||||
}
|
||||
},
|
||||
{
|
||||
"case": "erl_lint-3-complete",
|
||||
"lane": "complete",
|
||||
"artifact": {
|
||||
"path": "/tmp/rea-perf-static-tool-audit-20261024/real-beam/erl_lint.beam",
|
||||
"sha256": "716d78394b7c3da983e36aac7aadd6673e5041677ba7de20413cc2704d958d55",
|
||||
"bytes": 233968
|
||||
},
|
||||
"answer_count": 4,
|
||||
"answer_sha256": "34ce3091dbea9ac8cd20831cf6443c7a1de6e02b753246cafa3e0e22b83907c9",
|
||||
"parent_evidence_id": "ev_df4bfa8b8f0391ad6291542fdccf9062ae3bb8dbabf01c2b166ca6f3493051fc",
|
||||
"source_report_sha256": "db64350ae25bff9e77f5e167f4a4ac095ed872e09c15de5ce3b2728c078a16d2",
|
||||
"runtime_identity_sha256": "1b4b40330a2e21d31c06a07d10937cb6a3b37cd0bcbf3d129380341d486b916e",
|
||||
"catalog_sha256": "f4a683209df5bed1ed25ae42c690fdd803348742132e88f77ba2e4e8333ca562",
|
||||
"calls": [
|
||||
{
|
||||
"name": "inspect_erlang_beam",
|
||||
"input": {
|
||||
"path": "/tmp/rea-perf-static-tool-audit-20261024/real-beam/erl_lint.beam"
|
||||
},
|
||||
"phase": "inspect",
|
||||
"elapsed_ms": 13658.772345000001,
|
||||
"reply_bytes": 146627,
|
||||
"is_error": false
|
||||
},
|
||||
{
|
||||
"name": "export_evidence_bundle",
|
||||
"input": {
|
||||
"path": "/tmp/rea-perf-static-tool-audit-20261024/sdk-cache-cohort/erl_lint-3-complete/evidence.json"
|
||||
},
|
||||
"phase": "export",
|
||||
"elapsed_ms": 48.42420299999867,
|
||||
"reply_bytes": 376,
|
||||
"is_error": false
|
||||
}
|
||||
],
|
||||
"phases": {
|
||||
"discovery_ms": 1121.564358
|
||||
},
|
||||
"wall_seconds": 17.257,
|
||||
"cpu_seconds": 16.276557,
|
||||
"kernel_family_peak_bytes": 1078980608,
|
||||
"events_at_payload_exit": {
|
||||
"memory.events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
},
|
||||
"pids.events": {
|
||||
"max": 0
|
||||
}
|
||||
},
|
||||
"payload_cleanup_verified": true,
|
||||
"final_whole_scope_events": null,
|
||||
"token_proxy": {
|
||||
"question": 32,
|
||||
"server_instructions": 513,
|
||||
"input_catalog_once": 55409,
|
||||
"arguments": 78,
|
||||
"text_replies": 20875,
|
||||
"combined": 76907,
|
||||
"complete_transport_catalog": 829075
|
||||
},
|
||||
"receipt_sha256": {
|
||||
"report.json": "d683aab80d34c4f0fb474981c5715a4ba1ee706bf4e74be10bf69c40358ec011",
|
||||
"catalog.json": "f4a683209df5bed1ed25ae42c690fdd803348742132e88f77ba2e4e8333ca562",
|
||||
"transcript.json": "12380e1e67368b8df7787d54be828c953d4e2ef63007c04cd9fc912b42cc42b5",
|
||||
"evidence.json": "ca66367cd1bb325d7d4e47f958c86234d01aaf749b68778ac5212cac246aacb3",
|
||||
"erl_lint-3-complete.log": "404f609e4dc10bbf0d8f1d3076751b2cb8b271ff401b00ea161a00944a55d558"
|
||||
}
|
||||
},
|
||||
{
|
||||
"case": "erl_lint-3-selected",
|
||||
"lane": "selected",
|
||||
"artifact": {
|
||||
"path": "/tmp/rea-perf-static-tool-audit-20261024/real-beam/erl_lint.beam",
|
||||
"sha256": "716d78394b7c3da983e36aac7aadd6673e5041677ba7de20413cc2704d958d55",
|
||||
"bytes": 233968
|
||||
},
|
||||
"answer_count": 4,
|
||||
"answer_sha256": "34ce3091dbea9ac8cd20831cf6443c7a1de6e02b753246cafa3e0e22b83907c9",
|
||||
"parent_evidence_id": "ev_df4bfa8b8f0391ad6291542fdccf9062ae3bb8dbabf01c2b166ca6f3493051fc",
|
||||
"source_report_sha256": "db64350ae25bff9e77f5e167f4a4ac095ed872e09c15de5ce3b2728c078a16d2",
|
||||
"runtime_identity_sha256": "a74d203a5a12d7eb82aaa1e0fda6595b9647af2255f318bf116953777052460f",
|
||||
"catalog_sha256": "9c09aa2944f10c806df78d1cfa2e1daeddc0a487c432701cb67fa65e14213b0c",
|
||||
"calls": [
|
||||
{
|
||||
"name": "inspect_erlang_beam",
|
||||
"input": {
|
||||
"path": "/tmp/rea-perf-static-tool-audit-20261024/real-beam/erl_lint.beam",
|
||||
"detail": "summary",
|
||||
"view": {
|
||||
"kind": "beam",
|
||||
"collection": "exports",
|
||||
"names": ["module", "exprs", "expr"],
|
||||
"fields": ["function_index", "function", "arity", "label"],
|
||||
"offset": 0,
|
||||
"limit": 64
|
||||
}
|
||||
},
|
||||
"phase": "inspect",
|
||||
"elapsed_ms": 14405.966256000002,
|
||||
"reply_bytes": 7093,
|
||||
"is_error": false
|
||||
},
|
||||
{
|
||||
"name": "export_evidence_bundle",
|
||||
"input": {
|
||||
"path": "/tmp/rea-perf-static-tool-audit-20261024/sdk-cache-cohort/erl_lint-3-selected/evidence.json"
|
||||
},
|
||||
"phase": "export",
|
||||
"elapsed_ms": 31.32568199999878,
|
||||
"reply_bytes": 376,
|
||||
"is_error": false
|
||||
}
|
||||
],
|
||||
"phases": {
|
||||
"discovery_ms": 1107.032897
|
||||
},
|
||||
"wall_seconds": 18.052,
|
||||
"cpu_seconds": 16.867381,
|
||||
"kernel_family_peak_bytes": 1105092608,
|
||||
"events_at_payload_exit": {
|
||||
"memory.events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
},
|
||||
"pids.events": {
|
||||
"max": 0
|
||||
}
|
||||
},
|
||||
"payload_cleanup_verified": true,
|
||||
"final_whole_scope_events": null,
|
||||
"token_proxy": {
|
||||
"question": 32,
|
||||
"server_instructions": 513,
|
||||
"input_catalog_once": 56359,
|
||||
"arguments": 120,
|
||||
"text_replies": 1066,
|
||||
"combined": 58090,
|
||||
"complete_transport_catalog": 834792
|
||||
},
|
||||
"receipt_sha256": {
|
||||
"report.json": "732e5b4f880d3a981d5362ece88578a2da9e68e80e97eca2dec2ae5856ebbdee",
|
||||
"catalog.json": "9c09aa2944f10c806df78d1cfa2e1daeddc0a487c432701cb67fa65e14213b0c",
|
||||
"transcript.json": "e62dc35d939b21418989093a8addadbe533d358a046feb7d99cecab10cedc2d4",
|
||||
"evidence.json": "2373bc2e57e40aac881f3ef91fa77187b098dd61d6cf44d37a4b8fe56d022b40",
|
||||
"erl_lint-3-selected.log": "2f35909a9ad5efa9e8ddeffe665887e8a9e9c97c13b4a030b765dd1ceecb14ef"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,125 @@
|
||||
# Selected BEAM exports: completed-task measurements
|
||||
|
||||
Issue #6938. The analyst asks for every exported function with a small set of
|
||||
exact names, preserving arities, original table/atom indexes and labels. Both
|
||||
lanes export and validate the **complete** parent report, not only the answer.
|
||||
|
||||
Baseline `e09b06369d923e9cccedd91f6839ba643dfbd893`; selected delivery
|
||||
`bcc3f7611` (full SHA and source-clean runtime identities in the JSON receipt).
|
||||
The reader, format budgets and complete Evidence identity are unchanged.
|
||||
Node 24.22.1, SDK client/core/server 2.3.1, npm 11.16.0; Linux x86-64. The
|
||||
18-task cohort alternates three fresh complete/selected sessions per artifact.
|
||||
Fresh HOME/client/server and disabled Node compile cache; OS caches unflushed.
|
||||
This is a scripted client task, not an actual model investigation.
|
||||
|
||||
## Results
|
||||
|
||||
Medians, before → after. Seconds and kernel process-family MiB (includes file
|
||||
cache), not parent-only RSS. Ranges and every receipt hash are in
|
||||
[the 18-task JSON](erlang-beam-views.json).
|
||||
|
||||
| Real artifact | Whole-task estimated tokens | Wall s | Family CPU s | Family peak MiB |
|
||||
| ------------- | --------------------------: | ------------: | ------------: | --------------: |
|
||||
| erl_lint | 76,907 → 58,090 | 6.773 → 6.870 | 5.581 → 5.312 | 667.0 → 630.9 |
|
||||
| erl_parse | 72,499 → 58,087 | 6.367 → 6.974 | 5.627 → 5.555 | 666.1 → 657.5 |
|
||||
| pg2 | 57,733 → 58,018 | 6.365 → 7.576 | 5.295 → 5.615 | 648.2 → 626.9 |
|
||||
|
||||
For erl_lint the estimate drops 24.5%; erl_parse drops 19.9%. Small pg2 grows 0.5%
|
||||
(285 tokens), so selected delivery does not benefit every input. Complete stays
|
||||
the default. No stable general elapsed/CPU/memory improvement is established;
|
||||
the small selected task has a higher wall/CPU median. Full parsing/retention and
|
||||
cold client setup remain. Matching source-report, parent-ID and answer digests
|
||||
agree across all six tasks for each artifact.
|
||||
|
||||
The projection counts one question, server instructions, all 221 input-tool
|
||||
names/descriptions/schemas once, and every argument/text reply including export.
|
||||
`tiktoken 0.14.0/cl100k_base` serialization estimates are not actual model usage.
|
||||
Input directory cost grows 55,409→56,359tokens; the complete transport catalog
|
||||
including output schemas grows 829,075→834,792tokens and is reported separately.
|
||||
Protocol framing, host injection/history replay, output-schema presentation to
|
||||
a model and inference quality are unmeasured. Reported context reduction is
|
||||
specific to this declared client projection.
|
||||
|
||||
## SDK cached-catalog control
|
||||
|
||||
Six additional alternating fresh tasks use the SDK's cached catalog/default
|
||||
validator without an explicit `toolDefinition`. They answer/export the same
|
||||
question and pass the same independent oracle and cleanup checks. They are
|
||||
separate from the 18-task primary cohort; do not pool their startup costs.
|
||||
|
||||
Median wall 19.764→19.375s (ranges 17.257–21.577 / 18.052–22.397);
|
||||
family CPU 16.277→16.867s (ranges 15.287–16.562 / 16.139–17.070);
|
||||
family peak 1024.2→1053.9MiB.
|
||||
No general cached-client startup improvement is claimed. See
|
||||
[all six control receipts](erlang-beam-sdk-cache.json).
|
||||
|
||||
## Correctness and resources
|
||||
|
||||
The unmodified Debian `erlang-base_27.3.4.1+dfsg-1+deb13u3_amd64.deb` supplies
|
||||
actual OTP 27 compiler/stdlib modules: erl_lint 233,968 bytes (915 atoms, 119 imports,
|
||||
112 lambdas), erl_parse 608,240 bytes (1,035 atoms), and pg2 1,152 bytes (4 atoms).
|
||||
The package URL/SHA256, every selected artifact digest, source revision, driver
|
||||
hash and runtime fingerprint are retained in the JSON records. The independent
|
||||
[Python struct oracle](erlang-beam-oracle.py) checks every decoded table, chunk
|
||||
range/payload digest and Code header, then the selected export answer. It reads
|
||||
bytes only and deliberately qualifies these fixed OTP 27 layouts, not every
|
||||
historical/compact format. No Erlang installation, emulator or target execution
|
||||
was used. Default CLI and selected CLI on all three inputs also matched every
|
||||
oracle fact and parent ID and published exactly the complete source Evidence.
|
||||
|
||||
Each of the 24 SDK tasks exited 0 with payload cleanup verified and no guard stop,
|
||||
OOM or task-cap event. Aggregate CPUQuota 100%, payload affinity0–1,3 GiB early
|
||||
stop/4 GiB hard limit, no swap, TasksMax 192. CPU/peak counters are observed after
|
||||
the payload exits while the observer lives; final whole-scope counters remain
|
||||
unavailable. Costs include SDK discovery/schema validation, runtime/dependency
|
||||
hashing, assertions, source retention/export and cleanup. Sample acquisition,
|
||||
oracle generation and tokenization occur outside timed tasks. Node, all compiled dist
|
||||
files, selected entrypoints, 59 actual SDK files and driver files are hashed
|
||||
before/after; the source revisions were clean when fingerprints were created.
|
||||
|
||||
Public regressions additionally exercise all five collections, duplicate names
|
||||
and arities, selectors/fields/pages, missing versus empty tables, unresolved atom
|
||||
names, unavailable retention, source deletion, offline bundles and refusal before
|
||||
file reads. They establish those boundaries; real new-view qualification here is
|
||||
OTP 27 on Linux, not every platform or provider.
|
||||
|
||||
## Reproducing the question
|
||||
|
||||
Build clean pinned checkouts with the repository's pinned toolchain and acquire
|
||||
the public package linked in the JSON. Keep extracted files outside the repo.
|
||||
Generate each oracle, for example:
|
||||
|
||||
```sh
|
||||
python3 experiments/analysis-efficiency/erlang-beam-oracle.py \
|
||||
--input /samples/erl_lint.beam --output /runs/erl_lint-oracle.json \
|
||||
--name module --name exprs --name expr
|
||||
node experiments/analysis-efficiency/erlang-beam-views.mjs \
|
||||
--runtime /built/rea --identity /runs/runtime-identity.json \
|
||||
--oracle /runs/erl_lint-oracle.json --output /runs/erl_lint-selected \
|
||||
--lane selected
|
||||
```
|
||||
|
||||
A runtime fingerprint supplies `compiled_files` and `entrypoints` with relative
|
||||
paths/SHA256, `driver_files` relative to this experiment directory, actual SDK
|
||||
`dependency_files` with absolute paths/SHA256, and `node_sha256`. Source revision
|
||||
and clean-state fields record preparation. Create fingerprints before timing;
|
||||
the driver verifies them before and after each task. The publication records
|
||||
fingerprint/receipt hashes and full commands; the original complete fingerprint
|
||||
files, SDK transcripts, complete Evidence bundles and resource logs remain at
|
||||
`/tmp/rea-perf-static-tool-audit-20261024/`. The local cohort controller is retained
|
||||
there too. These local receipts are not shipped binaries or installed providers.
|
||||
|
||||
Wrap each invocation with `resource-run.py --observe-payload-exit`, setting the
|
||||
pinned Node PATH/heap, process affinity and resource policy as in its recorded
|
||||
command. Use a fresh output directory/session each time, alternate lanes three
|
||||
times per artifact, and count with
|
||||
[erlang-beam-costs.py](erlang-beam-costs.py). `--page-size` can qualify extra
|
||||
retained-page calls; `--sdk-cache` selects the separate cached-catalog control.
|
||||
The18 published primary tasks use the exact discovered toolDefinition/default
|
||||
validator and 64-row pages, requiring two calls each including full-source export.
|
||||
|
||||
The initial exploratory baseline used earlier main `e2463372`; it is not mixed
|
||||
into this table. Failed development checks remain in the local record: an
|
||||
incorrect formatter path and copied service import, followed by exact complete
|
||||
contract narrowing in old test consumers; corrected type/format/public checks
|
||||
passed. These were not successful benchmark samples.
|
||||
@@ -0,0 +1,255 @@
|
||||
/** Compare a completed selected-export question through production stdio MCP. */
|
||||
import assert from "node:assert/strict";
|
||||
import { createHash } from "node:crypto";
|
||||
import { createWriteStream } from "node:fs";
|
||||
import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import { finished } from "node:stream/promises";
|
||||
import { parseArgs } from "node:util";
|
||||
import { Client } from "@modelcontextprotocol/client";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
||||
const { values } = parseArgs({
|
||||
options: {
|
||||
...Object.fromEntries(
|
||||
["runtime", "oracle", "output", "identity"].map((k) => [
|
||||
k,
|
||||
{ type: "string" },
|
||||
]),
|
||||
),
|
||||
lane: { type: "string", default: "complete" },
|
||||
"page-size": { type: "string", default: "64" },
|
||||
"sdk-cache": { type: "boolean", default: false },
|
||||
},
|
||||
});
|
||||
assert.ok(["complete", "selected"].includes(values.lane));
|
||||
const pageSize = Number(values["page-size"]);
|
||||
assert.ok(Number.isSafeInteger(pageSize) && pageSize > 0);
|
||||
for (const key of ["runtime", "oracle", "output", "identity"])
|
||||
assert.equal(values[key], resolve(values[key]));
|
||||
const { createRetainedQueryCaller } =
|
||||
await import("./retained-query-calls.mjs");
|
||||
const { finalizeRetainedQueryReceipt } =
|
||||
await import("./retained-query-finalization.mjs");
|
||||
const hash = (x) => createHash("sha256").update(x).digest("hex"),
|
||||
oracleBytes = await readFile(values.oracle),
|
||||
expected = JSON.parse(oracleBytes),
|
||||
identityBytes = await readFile(values.identity),
|
||||
identity = JSON.parse(identityBytes);
|
||||
assert.equal(
|
||||
hash(await readFile(expected.expected.artifact.path)),
|
||||
expected.expected.artifact.sha256,
|
||||
);
|
||||
await mkdir(values.output, { mode: 0o700 });
|
||||
const account = await mkdtemp(join(tmpdir(), "rea-beam-task-"));
|
||||
const client = new Client({ name: "beam-selected-exports", version: "1" });
|
||||
const transport = new StdioClientTransport({
|
||||
command: process.execPath,
|
||||
args: [join(values.runtime, "scripts/rea.mjs"), "mcp"],
|
||||
maxBufferSize: 10485760,
|
||||
env: {
|
||||
HOME: account,
|
||||
TMPDIR: account,
|
||||
PATH: process.env.PATH,
|
||||
REA_LOG_LEVEL: "silent",
|
||||
NODE_OPTIONS: "--max-old-space-size=2048",
|
||||
NODE_DISABLE_COMPILE_CACHE: "1",
|
||||
UV_THREADPOOL_SIZE: "2",
|
||||
},
|
||||
stderr: "pipe",
|
||||
});
|
||||
const serverLog = createWriteStream(join(values.output, "server.log")),
|
||||
serverLogFinished = finished(serverLog);
|
||||
serverLogFinished.catch(() => {});
|
||||
transport.stderr?.pipe(serverLog);
|
||||
const report = {
|
||||
status: "running",
|
||||
lane: values.lane,
|
||||
artifact: expected.expected.artifact,
|
||||
calls: [],
|
||||
phases: {},
|
||||
question: `Return every exported function named ${expected.names.join(", ")} with its original export-table index, function atom index, arity and code label. Preserve complete source Evidence.`,
|
||||
client: values["sdk-cache"]
|
||||
? "Production stdio SDK, cached catalog definitions and default validator"
|
||||
: "Production stdio SDK, exact discovered toolDefinition and default validator",
|
||||
runtime_identity_sha256: hash(identityBytes),
|
||||
oracle_sha256: hash(oracleBytes),
|
||||
driver_sha256: hash(await readFile(new URL(import.meta.url))),
|
||||
};
|
||||
const verifyRuntime = async () => {
|
||||
assert.equal(hash(await readFile(values.identity)), hash(identityBytes));
|
||||
assert.equal(hash(await readFile(values.oracle)), hash(oracleBytes));
|
||||
for (const r of [...identity.compiled_files, ...identity.entrypoints])
|
||||
assert.equal(
|
||||
hash(await readFile(join(values.runtime, r.path))),
|
||||
r.sha256,
|
||||
r.path,
|
||||
);
|
||||
for (const r of identity.dependency_files ?? [])
|
||||
assert.equal(hash(await readFile(r.path)), r.sha256, r.path);
|
||||
for (const r of identity.driver_files)
|
||||
assert.equal(
|
||||
hash(await readFile(new URL(r.path, import.meta.url))),
|
||||
r.sha256,
|
||||
r.path,
|
||||
);
|
||||
assert.equal(hash(await readFile(process.execPath)), identity.node_sha256);
|
||||
assert.equal(
|
||||
hash(await readFile(new URL(import.meta.url))),
|
||||
report.driver_sha256,
|
||||
);
|
||||
};
|
||||
const transcript = [];
|
||||
let definitions;
|
||||
const caller = createRetainedQueryCaller(
|
||||
{
|
||||
callTool(params, options) {
|
||||
return client.callTool(
|
||||
params,
|
||||
values["sdk-cache"]
|
||||
? options
|
||||
: {
|
||||
...options,
|
||||
toolDefinition: definitions.get(params.name),
|
||||
},
|
||||
);
|
||||
},
|
||||
},
|
||||
(receipt, event) => {
|
||||
report.calls.push(receipt);
|
||||
transcript.push(event);
|
||||
},
|
||||
);
|
||||
const observe = async (name, input, phase) => {
|
||||
const result = await caller(name, input, phase);
|
||||
assert.notEqual(result.isError, true, JSON.stringify(result));
|
||||
assert.ok(result.structuredContent);
|
||||
return result.structuredContent;
|
||||
};
|
||||
const verifySource = (source) => {
|
||||
for (const [key, value] of Object.entries(expected.expected))
|
||||
assert.deepEqual(source[key], value, key);
|
||||
assert.equal(source.coverage.status, "complete");
|
||||
};
|
||||
try {
|
||||
await verifyRuntime();
|
||||
const started = performance.now();
|
||||
await client.connect(transport);
|
||||
report.server_instructions = client.getInstructions() ?? "";
|
||||
const catalog = await client.listTools();
|
||||
definitions = new Map(catalog.tools.map((t) => [t.name, t]));
|
||||
report.catalog_tools = catalog.tools.length;
|
||||
await writeFile(join(values.output, "catalog.json"), JSON.stringify(catalog));
|
||||
report.phases.discovery_ms = performance.now() - started;
|
||||
const selected = values.lane === "selected";
|
||||
const selectedView = (offset) => ({
|
||||
kind: "beam",
|
||||
collection: "exports",
|
||||
names: expected.names,
|
||||
fields: ["function_index", "function", "arity", "label"],
|
||||
offset,
|
||||
limit: pageSize,
|
||||
});
|
||||
const response = await observe(
|
||||
"inspect_erlang_beam",
|
||||
{
|
||||
path: expected.expected.artifact.path,
|
||||
...(selected ? { detail: "summary", view: selectedView(0) } : {}),
|
||||
},
|
||||
"inspect",
|
||||
);
|
||||
report.parent_evidence_id = selected
|
||||
? response.normalized_result.parent_evidence_id
|
||||
: response.evidence_id;
|
||||
let answer;
|
||||
if (!selected) {
|
||||
verifySource(response.normalized_result);
|
||||
answer = (response.normalized_result.exports ?? [])
|
||||
.map((r, i) => ({ source_index: i, ...r }))
|
||||
.filter((r) => expected.names.includes(r.function));
|
||||
} else {
|
||||
answer = [];
|
||||
let offset = 0,
|
||||
page = response.normalized_result;
|
||||
for (;;) {
|
||||
assert.equal(page.kind, "beam");
|
||||
assert.equal(page.page.collection, "exports");
|
||||
assert.deepEqual(page.beam.artifact, expected.expected.artifact);
|
||||
assert.deepEqual(page.beam.coverage, expected.expected.coverage);
|
||||
assert.equal(page.beam.counts.exports, expected.expected.exports.length);
|
||||
assert.equal(page.coverage.total, expected.answer.length);
|
||||
answer.push(...page.page.items);
|
||||
if (page.coverage.next_offset === null) break;
|
||||
assert.ok(page.coverage.next_offset > offset);
|
||||
offset = page.coverage.next_offset;
|
||||
page = (
|
||||
await observe(
|
||||
"inspect_analysis_view",
|
||||
{
|
||||
source: {
|
||||
kind: "retained-evidence",
|
||||
evidence_id: report.parent_evidence_id,
|
||||
},
|
||||
view: selectedView(offset),
|
||||
},
|
||||
"selected-page",
|
||||
)
|
||||
).normalized_result;
|
||||
}
|
||||
}
|
||||
assert.deepEqual(answer, expected.answer);
|
||||
report.answer_count = answer.length;
|
||||
report.answer_sha256 = hash(
|
||||
JSON.stringify(
|
||||
answer.map((r) =>
|
||||
Object.fromEntries(
|
||||
Object.entries(r).sort(([a], [b]) => a.localeCompare(b)),
|
||||
),
|
||||
),
|
||||
),
|
||||
);
|
||||
await writeFile(join(values.output, "answer.json"), JSON.stringify(answer));
|
||||
await observe(
|
||||
"export_evidence_bundle",
|
||||
{ path: join(values.output, "evidence.json") },
|
||||
"export",
|
||||
);
|
||||
const bundle = JSON.parse(
|
||||
await readFile(join(values.output, "evidence.json")),
|
||||
);
|
||||
const parent = bundle.records.find(
|
||||
(r) => r.evidence_id === report.parent_evidence_id,
|
||||
);
|
||||
assert.ok(parent);
|
||||
assert.equal(parent.operation, "inspect_erlang_beam");
|
||||
verifySource(parent.normalized_result);
|
||||
report.source_report_sha256 = hash(JSON.stringify(parent.normalized_result));
|
||||
report.complete_source_export_verified = true;
|
||||
report.status = "passed";
|
||||
} catch (cause) {
|
||||
report.status = "failed";
|
||||
report.error = String(cause);
|
||||
} finally {
|
||||
try {
|
||||
await verifyRuntime();
|
||||
} catch (cause) {
|
||||
report.status = "failed";
|
||||
report.runtime_identity_failure = String(cause);
|
||||
}
|
||||
await finalizeRetainedQueryReceipt({
|
||||
client,
|
||||
transport,
|
||||
serverLog,
|
||||
serverLogFinished,
|
||||
account,
|
||||
report,
|
||||
output: values.output,
|
||||
transcript,
|
||||
});
|
||||
if (
|
||||
report.status !== "passed" ||
|
||||
report.cleanup_failures.length ||
|
||||
report.receipt_failures.length
|
||||
)
|
||||
process.exitCode = 1;
|
||||
}
|
||||
Reference in new issue
Block a user