mirror of
https://github.com/morluto/rea.git
synced 2026-10-26 17:35:11 +00:00
feat(cli): summarize managed members with durable Evidence paging (#5167)
Fixes #5148. Add opt-in durable managed member summaries and authenticated paging while preserving complete default output, original Evidence, cleanup ownership and queued cancellation. Verified with bounded real Newtonsoft/Redux/Rust tasks, independent oracles, 556 focused tests and the expanded domain gate; paired token estimates and latency tradeoffs are recorded in experiments/analysis-efficiency/managed-cli.md.
This commit is contained in:
1 parent
d8948efb81
commit
57fd3ea5d8
48 files changed
+4671
-196
No files matched your search
@@ -0,0 +1,84 @@
|
||||
"""Summarize completed CLI trials; token counts are estimates, not model usage."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import statistics
|
||||
from pathlib import Path
|
||||
import tiktoken
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('root', type=Path)
|
||||
parser.add_argument('--output', type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
root = args.root
|
||||
inputs = json.loads((root / 'measurement-inputs.json').read_text())
|
||||
expected = inputs['expected_runtime_bindings']
|
||||
assert expected['complete']['git_revision'] == inputs['baseline_revision']
|
||||
assert expected['selected']['git_revision'] == inputs['feature_parent_revision']
|
||||
assert expected['complete']['root'] != expected['selected']['root']
|
||||
assert expected['complete']['compiled_cli_sha256'] != expected['selected']['compiled_cli_sha256']
|
||||
assert expected['complete']['source_diff_sha256'] == hashlib.sha256(b'').hexdigest()
|
||||
assert expected['complete']['untracked_source_sha256'] == hashlib.sha256(b'[]').hexdigest()
|
||||
enc = tiktoken.get_encoding('cl100k_base')
|
||||
count = lambda text: len(enc.encode(text, disallowed_special=()))
|
||||
compact = lambda obj: json.dumps(obj, ensure_ascii=False, separators=(',', ':'))
|
||||
runs = []
|
||||
for case in sorted(root.glob('pair-*')):
|
||||
if not case.is_dir():
|
||||
continue
|
||||
report = json.loads((case / 'report.json').read_text())
|
||||
assert report['status'] == 'passed' and report['cleanup_verified']
|
||||
assert report['runtime_binding_verified']
|
||||
assert report['runtime_binding'] == expected[report['mode']], case.name
|
||||
assert str(Path(report['runtime']).resolve()) == report['runtime_binding']['root']
|
||||
receipt = next(json.loads(line) for line in reversed(case.with_name(case.name + '.log').read_text().splitlines()) if line.startswith('{"resource_scope":'))
|
||||
obs = receipt['payload_resource_observation']
|
||||
assert receipt['abort'] is None and receipt['payload_resource_observation_verified']
|
||||
assert obs['payload_exit_code'] == 0 and obs['payload_cleanup_verified']
|
||||
assert obs['resource_events']['memory.events']['oom_kill'] == 0
|
||||
tokens = {'question_and_recipe': count(report['question'] + '\n' + report['recipe']),
|
||||
'arguments': sum(count(compact(call['arguments'])) for call in report['calls']),
|
||||
'help_and_replies': sum(count((case / call['stdout_file']).read_text()) for call in report['calls'])}
|
||||
tokens['whole_task'] = sum(tokens.values())
|
||||
runs.append({'case': case.name, 'mode': report['mode'], 'cache': report['cache'], 'runtime': report['runtime'], 'runtime_binding': report['runtime_binding'], 'runtime_binding_audit_ms': report['runtime_binding_audit_ms'],
|
||||
'wall_seconds': receipt['elapsed_seconds'], 'cpu_seconds': obs['cpu_stat']['usage_usec'] / 1e6,
|
||||
'peak_process_family_bytes_including_page_cache': obs['memory_peak_bytes'],
|
||||
'tokens_estimated': tokens, 'calls': report['calls'],
|
||||
'parent_evidence_id': report['parent_evidence_id'], 'parent_semantic_sha256': report['parent_semantic_sha256'],
|
||||
'answer_sha256': report['answer_sha256'], 'oracle_verified': report['oracle_verified'],
|
||||
'cleanup_verified': True, 'resource_events': obs['resource_events']})
|
||||
assert len(runs) == 12
|
||||
for key in ['answer_sha256', 'parent_semantic_sha256', 'parent_evidence_id']:
|
||||
assert len({row[key] for row in runs}) == 1, key
|
||||
summary = {}
|
||||
for cache in ['cold', 'warm']:
|
||||
summary[cache] = {}
|
||||
for mode in ['complete', 'selected']:
|
||||
rows = [r for r in runs if r['mode'] == mode and r['cache'] == cache]
|
||||
assert len(rows) == 3
|
||||
summary[cache][mode] = {key: {'median': statistics.median(r[key] for r in rows), 'min': min(r[key] for r in rows), 'max': max(r[key] for r in rows)} for key in ['wall_seconds', 'cpu_seconds', 'peak_process_family_bytes_including_page_cache']}
|
||||
summary[cache][mode]['whole_task_tokens_estimated'] = [r['tokens_estimated']['whole_task'] for r in rows]
|
||||
result = {'status': 'passed', 'tokenizer': 'tiktoken==0.12.0/cl100k_base; estimates, not model usage',
|
||||
'context_scope': 'Question, supplied command recipe, actual selected-command help, all logical CLI argv and all stdout. No model invoked; reasoning, framing, irrelevant tool discovery and actual native-agent usage unmeasured.',
|
||||
'cache_scope': 'cold disables Node compile cache; warm primes the identical workflow once per runtime/mode before timing. Fresh processes and isolated HOME each trial; OS page cache uncontrolled.',
|
||||
'resource_scope': 'Whole task driver plus all CLI children; includes pre/post source-and-compiled-runtime binding verification, bundle publication/authentication, help, extra calls and local answer/oracle checking. Kernel cgroup peak includes file cache; not sampled RSS. Download/build/oracle preparation and priming excluded.',
|
||||
'acquisition': json.loads((root / 'acquisition.json').read_text()), 'measurement_inputs': inputs,
|
||||
'summary': summary, 'runs': runs}
|
||||
native = root / 'native-control'
|
||||
if native.exists():
|
||||
binding = json.loads((native / 'runtime-binding.json').read_text())
|
||||
assert binding['runtime_binding_verified'] and binding['runtime_binding'] == expected['complete']
|
||||
assert str(Path(binding['runtime']).resolve()) == binding['runtime_binding']['root']
|
||||
receipt = next(json.loads(line) for line in reversed((root / 'native-control.log').read_text().splitlines()) if line.startswith('{"resource_scope":'))
|
||||
obs = receipt['payload_resource_observation']
|
||||
assert receipt['abort'] is None and receipt['payload_resource_observation_verified'] and obs['payload_exit_code'] == 0 and obs['payload_cleanup_verified']
|
||||
baseline_case = root / next(run['case'] for run in runs if run['mode'] == 'complete')
|
||||
assert json.loads((native / 'answer.json').read_text()) == json.loads((baseline_case / 'answer.json').read_text())
|
||||
control_source = Path(__file__).with_name('managed-cli-local-control.py').read_text()
|
||||
result['existing_native_control'] = {'runtime': binding['runtime'], 'runtime_binding': binding['runtime_binding'], 'runtime_binding_audit_ms': binding['runtime_binding_audit_ms'], 'repetitions': 1,
|
||||
'tokens_estimated_script_plus_help_plus_answer': count(control_source) + count((native / 'help.txt').read_text()) + count((native / 'answer.json').read_text()),
|
||||
'token_scope': 'Projection source, original CLI help and returned answer; excludes shared question and launcher argv. No actual model usage measured.',
|
||||
'wall_seconds': receipt['elapsed_seconds'], 'cpu_seconds': obs['cpu_stat']['usage_usec'] / 1e6,
|
||||
'peak_process_family_bytes_including_page_cache': obs['memory_peak_bytes'], 'answer_matches': True, 'cleanup_verified': True}
|
||||
args.output.write_text(json.dumps(result, indent=2) + '\n')
|
||||
print(json.dumps(summary, indent=2))
|
||||
@@ -0,0 +1,44 @@
|
||||
"""Existing native code-composition control: save complete stdout, return selected facts."""
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('runtime', type=Path)
|
||||
parser.add_argument('target', type=Path)
|
||||
parser.add_argument('output', type=Path)
|
||||
parser.add_argument('--measurement-inputs', type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
args.output.mkdir()
|
||||
node = 'node'
|
||||
inputs = json.loads(args.measurement_inputs.read_text())
|
||||
expected = inputs['expected_runtime_bindings']['complete']
|
||||
assert expected['git_revision'] == inputs['baseline_revision']
|
||||
helper = Path(__file__).with_name('managed-cli-runtime.mjs')
|
||||
def capture_binding():
|
||||
return json.loads(subprocess.check_output([node, str(helper), str(args.runtime)], text=True))
|
||||
binding_started = time.monotonic()
|
||||
binding = capture_binding()
|
||||
binding_audit_ms = {'before': (time.monotonic() - binding_started) * 1000}
|
||||
assert binding == expected
|
||||
|
||||
entry = str(args.runtime / 'scripts/rea.mjs')
|
||||
help_result = subprocess.run([node, entry, 'inspect-managed-members', '--help'], check=True, capture_output=True, text=True)
|
||||
(args.output / 'help.txt').write_text(help_result.stdout)
|
||||
with (args.output / 'parent.json').open('w') as file:
|
||||
subprocess.run([node, entry, 'inspect-managed-members', str(args.target), '--type', 'Newtonsoft.Json.JsonConvert', '--json'], check=True, stdout=file)
|
||||
parent = json.loads((args.output / 'parent.json').read_text())
|
||||
data = parent['normalized_result']
|
||||
answer = {'parent_evidence_id': parent['evidence_id'], 'artifact_sha256': data['artifact']['sha256'],
|
||||
'selection': data['selection'], 'source_coverage': data['coverage'],
|
||||
'total_methods': len(data['methods']), 'total_member_refs': len(data['member_refs']),
|
||||
'methods': data['methods'][:8], 'member_refs': data['member_refs'][:16],
|
||||
'limitation': 'MemberRef rows are assembly-wide; methods are selected-type rows. Page exhaustion is not an absence claim.'}
|
||||
(args.output / 'answer.json').write_text(json.dumps(answer))
|
||||
binding_finished = time.monotonic()
|
||||
assert capture_binding() == binding
|
||||
binding_audit_ms['after'] = (time.monotonic() - binding_finished) * 1000
|
||||
(args.output / 'runtime-binding.json').write_text(json.dumps({'runtime': str(args.runtime), 'runtime_binding': binding, 'runtime_binding_verified': True, 'runtime_binding_audit_ms': binding_audit_ms}))
|
||||
print(json.dumps(answer))
|
||||
@@ -0,0 +1,18 @@
|
||||
import json,hashlib,argparse
|
||||
from pathlib import Path
|
||||
import dnfile
|
||||
parser=argparse.ArgumentParser();parser.add_argument('root',type=Path);root=parser.parse_args().root;path=root/'Newtonsoft.Json.dll'
|
||||
pe=dnfile.dnPE(str(path));assert pe.net is not None
|
||||
counts={name:len(getattr(pe.net.mdtables,name).rows) if getattr(pe.net.mdtables,name,None) else 0 for name in ['TypeDef','MethodDef','MemberRef','Field','Property','Event','Param','StandAloneSig','TypeSpec','ExportedType']}
|
||||
selected=[]
|
||||
for index,row in enumerate(pe.net.mdtables.TypeDef.rows,1):
|
||||
name=(str(row.TypeNamespace)+'.' if str(row.TypeNamespace) else '')+str(row.TypeName)
|
||||
if name!='Newtonsoft.Json.JsonConvert':continue
|
||||
selected.append({'token':f'0x{0x02000000|index:08x}','name':name,'methods':[{'token':f'0x{0x06000000|m.row_index:08x}','name':str(m.row.Name),'rva':m.row.Rva,'signature_hex':m.row.Signature.value.hex()} for m in row.MethodList]})
|
||||
assert len(selected)==1
|
||||
refs=[{'token':f'0x{0x0a000000|i:08x}','name':str(row.Name),'signature_hex':row.Signature.value.hex()} for i,row in enumerate(pe.net.mdtables.MemberRef.rows[:16],1)]
|
||||
assembly=pe.net.mdtables.Assembly.rows[0]
|
||||
oracle={'decoder':'dnfile==0.18.0 (independent Python PE/CLI parser)','assembly_sha256':hashlib.sha256(path.read_bytes()).hexdigest(),'assembly_name':str(assembly.Name),'assembly_version':'.'.join(str(getattr(assembly,k)) for k in ['MajorVersion','MinorVersion','BuildNumber','RevisionNumber']),'row_counts':counts,'selected_types':selected,'member_refs_first_16':refs,'target_execution':False}
|
||||
(root/'oracle.json').write_text(json.dumps(oracle,indent=2)+'\n')
|
||||
print(json.dumps({'assembly':oracle['assembly_name'],'version':oracle['assembly_version'],'row_counts':counts,'selected_type':selected[0]['name'],'selected_methods':len(selected[0]['methods']),'target_execution':False}))
|
||||
pe.close()
|
||||
@@ -0,0 +1,87 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFile, readdir, realpath } from "node:fs/promises";
|
||||
import { join, relative } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
const sha = (value) => createHash("sha256").update(value).digest("hex");
|
||||
|
||||
/** Bind a trial to its source checkout and the compiled CLI actually launched. */
|
||||
export const captureRuntimeBinding = async (runtime) => {
|
||||
const root = await realpath(runtime);
|
||||
const git = (args) =>
|
||||
execFileSync("git", args, { cwd: root, maxBuffer: 16 * 1024 * 1024 });
|
||||
const gitRevision = git(["rev-parse", "HEAD"]).toString().trim();
|
||||
const sourcePaths = ["src", "scripts", "package.json", "package-lock.json"];
|
||||
const sourceDiff = git([
|
||||
"diff",
|
||||
"--no-ext-diff",
|
||||
"--no-textconv",
|
||||
"--binary",
|
||||
"HEAD",
|
||||
"--",
|
||||
...sourcePaths,
|
||||
]);
|
||||
const untracked = git([
|
||||
"ls-files",
|
||||
"--others",
|
||||
"--exclude-standard",
|
||||
"-z",
|
||||
"--",
|
||||
...sourcePaths,
|
||||
])
|
||||
.toString()
|
||||
.split("\0")
|
||||
.filter(Boolean)
|
||||
.sort();
|
||||
const untrackedEntries = [];
|
||||
for (const name of untracked)
|
||||
untrackedEntries.push([name, sha(await readFile(join(root, name)))]);
|
||||
const files = ["scripts/rea.mjs", "package.json", "package-lock.json"];
|
||||
const visit = async (directory) => {
|
||||
const entries = await readdir(directory, { withFileTypes: true });
|
||||
for (const entry of entries.sort((a, b) =>
|
||||
a.name < b.name ? -1 : a.name > b.name ? 1 : 0,
|
||||
)) {
|
||||
const path = join(directory, entry.name);
|
||||
assert.ok(!entry.isSymbolicLink(), `Unbound compiled CLI link: ${path}`);
|
||||
if (entry.isDirectory()) await visit(path);
|
||||
else if (entry.isFile() && /\.(?:[cm]?js|json)$/.test(entry.name))
|
||||
files.push(relative(root, path));
|
||||
}
|
||||
};
|
||||
await visit(join(root, "dist/cli-runtime"));
|
||||
assert.ok(
|
||||
files.includes(join("dist", "cli-runtime", "cli.js")),
|
||||
"Missing compiled CLI entry",
|
||||
);
|
||||
const hash = createHash("sha256");
|
||||
let bytes = 0;
|
||||
for (const name of files.sort()) {
|
||||
const value = await readFile(join(root, name));
|
||||
bytes += value.length;
|
||||
hash.update(JSON.stringify([name, value.length, sha(value)]) + "\n");
|
||||
}
|
||||
return {
|
||||
binding_version: 1,
|
||||
root,
|
||||
node: process.version,
|
||||
git_revision: gitRevision,
|
||||
source_diff_sha256: sha(sourceDiff),
|
||||
untracked_source_sha256: sha(JSON.stringify(untrackedEntries)),
|
||||
compiled_cli_sha256: hash.digest("hex"),
|
||||
compiled_cli_files: files.length,
|
||||
compiled_cli_bytes: bytes,
|
||||
};
|
||||
};
|
||||
|
||||
export const assertRuntimeBinding = (actual, expected, mode) => {
|
||||
assert.ok(expected, `Missing expected runtime binding for ${mode}`);
|
||||
assert.deepEqual(actual, expected, `Runtime binding differs for ${mode}`);
|
||||
};
|
||||
|
||||
if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
|
||||
console.log(
|
||||
JSON.stringify(await captureRuntimeBinding(process.argv[2]), null, 2),
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
{
|
||||
"status": "passed",
|
||||
"scope": "Single completed SDK regression after shared JSON reader/admission changes; not a new paired source-to-bundle performance comparison. Existing semantic-node cap one answers static correspondence only; no equal semantic-graph coverage claim. Corpus/reference preparation performed earlier and excluded.",
|
||||
"artifact": {
|
||||
"name": "Redux Toolkit 2.2.7",
|
||||
"tarball_sha256": "f2517c466b50424b661a27e48039f15db7783e5822f46b5c7d39f20104ac67a9"
|
||||
},
|
||||
"runtime_parent_revision": "4fab2dfda16236dc407aa179920780d6a970dda3",
|
||||
"production_manifest": "managed-cli.json#measurement_inputs.production_sources_sha256",
|
||||
"input_schema_profile": "default compact",
|
||||
"output_schema_profile": "full",
|
||||
"tokens_estimated": {
|
||||
"input_catalog_once": 49054,
|
||||
"arguments": 307,
|
||||
"text_replies": 7017,
|
||||
"whole_task": 56378
|
||||
},
|
||||
"token_scope": "tiktoken==0.12.0/cl100k_base; one name/description/input-schema catalog plus actual arguments and MCP text replies; excludes output schemas, framing, model prompts/reasoning and actual model usage. Not actual native-agent token usage.",
|
||||
"actual_model_usage": null,
|
||||
"wall_seconds": 21.38,
|
||||
"cpu_seconds": 21.022683,
|
||||
"peak_process_family_bytes_including_page_cache": 974041088,
|
||||
"calls": [
|
||||
{
|
||||
"name": "analyze_javascript_application",
|
||||
"elapsed_ms": 9837.265845000002,
|
||||
"is_error": false,
|
||||
"reply_bytes": 25941
|
||||
},
|
||||
{
|
||||
"name": "compare_source_to_bundle",
|
||||
"elapsed_ms": 256.3669919999993,
|
||||
"is_error": false,
|
||||
"reply_bytes": 13199
|
||||
},
|
||||
{
|
||||
"name": "inspect_analysis_view",
|
||||
"elapsed_ms": 3012.057904000001,
|
||||
"is_error": false,
|
||||
"reply_bytes": 19393
|
||||
},
|
||||
{
|
||||
"name": "export_evidence_bundle",
|
||||
"elapsed_ms": 148.9011160000009,
|
||||
"is_error": false,
|
||||
"reply_bytes": 364
|
||||
}
|
||||
],
|
||||
"independent_proof": {
|
||||
"status": "passed",
|
||||
"method": "Independent Python JSON/SHA-256/POSIX path and basename joins; no REA parser or matcher.",
|
||||
"source_files": 180,
|
||||
"embedded_nodes": 99,
|
||||
"candidate_relations": 210,
|
||||
"strong_location_rows": 33,
|
||||
"limitation": "Static multiple-location projections; no semantic-equivalence or removal claim."
|
||||
},
|
||||
"resource_events": {
|
||||
"memory.events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0
|
||||
},
|
||||
"pids.events": {
|
||||
"max": 0
|
||||
}
|
||||
},
|
||||
"cleanup_verified": true
|
||||
}
|
||||
@@ -0,0 +1,266 @@
|
||||
import {
|
||||
captureRuntimeBinding,
|
||||
assertRuntimeBinding,
|
||||
} from "./managed-cli-runtime.mjs";
|
||||
import assert from "node:assert/strict";
|
||||
import { execFile } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFile, writeFile, mkdir } from "node:fs/promises";
|
||||
import { join, resolve } from "node:path";
|
||||
const execute = promisify(execFile);
|
||||
const [runtime, root, output, mode, cache = "cold"] = process.argv.slice(2);
|
||||
for (const value of [runtime, root, output])
|
||||
assert.equal(resolve(value), value);
|
||||
assert.ok(["complete", "selected"].includes(mode));
|
||||
assert.ok(["cold", "warm"].includes(cache));
|
||||
await mkdir(output);
|
||||
await mkdir(join(output, "tmp"));
|
||||
const oracle = JSON.parse(await readFile(join(root, "oracle.json"), "utf8"));
|
||||
const target = join(root, "Newtonsoft.Json.dll");
|
||||
const hash = (value) => createHash("sha256").update(value).digest("hex");
|
||||
assert.equal(hash(await readFile(target)), oracle.assembly_sha256);
|
||||
const stable = (value) =>
|
||||
Array.isArray(value)
|
||||
? value.map(stable)
|
||||
: value !== null && typeof value === "object"
|
||||
? Object.fromEntries(
|
||||
Object.keys(value)
|
||||
.sort()
|
||||
.map((key) => [key, stable(value[key])]),
|
||||
)
|
||||
: value;
|
||||
const report = {
|
||||
status: "incomplete",
|
||||
runtime,
|
||||
mode,
|
||||
cache,
|
||||
node: process.version,
|
||||
question:
|
||||
"Report identity/selection/source coverage for JsonConvert, its total method count and the first eight complete method records, plus the first sixteen assembly-wide MemberRef records and continuation coverage.",
|
||||
recipe:
|
||||
mode === "selected"
|
||||
? "Use --detail summary --evidence-bundle FILE; pass the exact parent_evidence_id to inspect-analysis-view --evidence-bundle FILE with source kind retained-evidence and view {kind:managed-members,collection:methods|member_refs,offset:0,limit:8|16}."
|
||||
: "Use inspect-managed-members --type Newtonsoft.Json.JsonConvert; the completed full Evidence supplies the requested rows and remains accessible.",
|
||||
calls: [],
|
||||
cleanup_verified: false,
|
||||
};
|
||||
const began = performance.now();
|
||||
const environment = {
|
||||
PATH: process.env.PATH,
|
||||
HOME: output,
|
||||
TMPDIR: join(output, "tmp"),
|
||||
LANG: "C.UTF-8",
|
||||
REA_LOG_LEVEL: "silent",
|
||||
UV_THREADPOOL_SIZE: "1",
|
||||
NODE_OPTIONS: "--max-old-space-size=2048",
|
||||
...(cache === "cold"
|
||||
? { NODE_DISABLE_COMPILE_CACHE: "1" }
|
||||
: { NODE_COMPILE_CACHE: join(root, `compile-cache-${mode}`) }),
|
||||
};
|
||||
const call = async (label, args, json = true) => {
|
||||
const started = performance.now();
|
||||
const result = await execute(
|
||||
process.execPath,
|
||||
[join(runtime, "scripts/rea.mjs"), ...args],
|
||||
{
|
||||
cwd: output,
|
||||
env: environment,
|
||||
maxBuffer: 16 * 1024 * 1024,
|
||||
timeout: 120000,
|
||||
},
|
||||
);
|
||||
await writeFile(join(output, label + ".stdout"), result.stdout);
|
||||
await writeFile(join(output, label + ".stderr"), result.stderr);
|
||||
report.calls.push({
|
||||
label,
|
||||
arguments: ["rea", ...args],
|
||||
elapsed_ms: performance.now() - started,
|
||||
stdout_bytes: Buffer.byteLength(result.stdout),
|
||||
stdout_file: label + ".stdout",
|
||||
});
|
||||
return json ? JSON.parse(result.stdout) : result.stdout;
|
||||
};
|
||||
let parent, summary, methods, refs;
|
||||
try {
|
||||
const inputs = JSON.parse(
|
||||
await readFile(join(root, "measurement-inputs.json"), "utf8"),
|
||||
);
|
||||
const expected = inputs.expected_runtime_bindings?.[mode];
|
||||
assert.ok(expected, `Missing expected runtime for ${mode}`);
|
||||
assert.equal(
|
||||
expected.git_revision,
|
||||
mode === "complete"
|
||||
? inputs.baseline_revision
|
||||
: inputs.feature_parent_revision,
|
||||
);
|
||||
assert.notEqual(
|
||||
inputs.expected_runtime_bindings.complete.root,
|
||||
inputs.expected_runtime_bindings.selected.root,
|
||||
);
|
||||
assert.notEqual(
|
||||
inputs.expected_runtime_bindings.complete.compiled_cli_sha256,
|
||||
inputs.expected_runtime_bindings.selected.compiled_cli_sha256,
|
||||
);
|
||||
const bindingStarted = performance.now();
|
||||
report.runtime_binding = await captureRuntimeBinding(runtime);
|
||||
report.runtime_binding_audit_ms = {
|
||||
before: performance.now() - bindingStarted,
|
||||
};
|
||||
assertRuntimeBinding(report.runtime_binding, expected, mode);
|
||||
await call("help-members", ["inspect-managed-members", "--help"], false);
|
||||
if (mode === "complete") {
|
||||
parent = await call("complete", [
|
||||
"inspect-managed-members",
|
||||
target,
|
||||
"--type",
|
||||
"Newtonsoft.Json.JsonConvert",
|
||||
"--json",
|
||||
]);
|
||||
methods = parent.normalized_result.methods.slice(0, 8);
|
||||
refs = parent.normalized_result.member_refs.slice(0, 16);
|
||||
} else {
|
||||
await call("help-view", ["inspect-analysis-view", "--help"], false);
|
||||
const bundlePath = join(output, "parent.json");
|
||||
summary = await call("summary", [
|
||||
"inspect-managed-members",
|
||||
target,
|
||||
"--type",
|
||||
"Newtonsoft.Json.JsonConvert",
|
||||
"--detail",
|
||||
"summary",
|
||||
"--evidence-bundle",
|
||||
bundlePath,
|
||||
"--json",
|
||||
]);
|
||||
const id = summary.normalized_result.parent_evidence_id;
|
||||
assert.ok(id);
|
||||
const bundle = JSON.parse(await readFile(bundlePath, "utf8"));
|
||||
assert.equal(bundle.records.length, 1);
|
||||
parent = bundle.records[0];
|
||||
assert.equal(parent.evidence_id, id);
|
||||
for (const [collection, limit] of [
|
||||
["methods", 8],
|
||||
["member_refs", 16],
|
||||
]) {
|
||||
const value = await call(collection, [
|
||||
"inspect-analysis-view",
|
||||
JSON.stringify({
|
||||
source: { kind: "retained-evidence", evidence_id: id },
|
||||
view: { kind: "managed-members", collection, offset: 0, limit },
|
||||
}),
|
||||
"--evidence-bundle",
|
||||
bundlePath,
|
||||
"--json",
|
||||
]);
|
||||
const page = value.normalized_result;
|
||||
assert.equal(page.parent_evidence_id, id);
|
||||
assert.deepEqual(page.source_coverage, parent.normalized_result.coverage);
|
||||
assert.deepEqual(
|
||||
page.page.rows,
|
||||
parent.normalized_result[collection].slice(0, limit),
|
||||
);
|
||||
assert.equal(
|
||||
page.coverage.total,
|
||||
parent.normalized_result[collection].length,
|
||||
);
|
||||
assert.equal(page.coverage.next_offset, limit);
|
||||
assert.equal(page.coverage.exhausted, false);
|
||||
if (collection === "methods") methods = page.page.rows;
|
||||
else refs = page.page.rows;
|
||||
}
|
||||
}
|
||||
const data = parent.normalized_result;
|
||||
assert.equal(data.artifact.sha256, oracle.assembly_sha256);
|
||||
assert.equal(data.methods.length, 72);
|
||||
assert.equal(data.member_refs.length, 2149);
|
||||
assert.equal(data.types.length, 1);
|
||||
assert.equal(data.types[0].token, oracle.selected_types[0].token);
|
||||
for (let i = 0; i < data.methods.length; i++) {
|
||||
const row = data.methods[i],
|
||||
wanted = oracle.selected_types[0].methods[i];
|
||||
assert.equal(row.token, wanted.token);
|
||||
assert.equal(row.name, wanted.name);
|
||||
assert.equal(row.rva, wanted.rva);
|
||||
assert.equal(
|
||||
row.signature.raw_sha256,
|
||||
hash(Buffer.from(wanted.signature_hex, "hex")),
|
||||
);
|
||||
}
|
||||
for (let i = 0; i < refs.length; i++) {
|
||||
const row = refs[i],
|
||||
wanted = oracle.member_refs_first_16[i];
|
||||
assert.equal(row.token, wanted.token);
|
||||
assert.equal(row.name, wanted.name);
|
||||
assert.equal(
|
||||
row.signature.raw_sha256,
|
||||
hash(Buffer.from(wanted.signature_hex, "hex")),
|
||||
);
|
||||
}
|
||||
assert.equal(methods.length, 8);
|
||||
assert.equal(refs.length, 16);
|
||||
if (summary) {
|
||||
assert.equal(summary.normalized_result.summary.reported_counts.methods, 72);
|
||||
assert.equal(
|
||||
summary.normalized_result.summary.reported_counts.member_refs,
|
||||
2149,
|
||||
);
|
||||
assert.deepEqual(summary.normalized_result.summary.coverage, data.coverage);
|
||||
assert.deepEqual(
|
||||
summary.normalized_result.summary.selection,
|
||||
data.selection,
|
||||
);
|
||||
}
|
||||
const answer = {
|
||||
parent_evidence_id: parent.evidence_id,
|
||||
artifact_sha256: oracle.assembly_sha256,
|
||||
selection: data.selection,
|
||||
source_coverage: data.coverage,
|
||||
total_methods: data.methods.length,
|
||||
total_member_refs: data.member_refs.length,
|
||||
methods,
|
||||
member_refs: refs,
|
||||
limitation:
|
||||
"MemberRef rows are assembly-wide; methods are selected-type rows. Page exhaustion is not an absence claim.",
|
||||
};
|
||||
report.parent_evidence_id = parent.evidence_id;
|
||||
report.parent_semantic_sha256 = hash(JSON.stringify(stable(parent)));
|
||||
report.answer_sha256 = hash(JSON.stringify(stable(answer)));
|
||||
report.oracle_verified = {
|
||||
all_selected_method_metadata: 72,
|
||||
returned_methods: 8,
|
||||
returned_member_refs: 16,
|
||||
};
|
||||
const bindingFinished = performance.now();
|
||||
assertRuntimeBinding(
|
||||
await captureRuntimeBinding(runtime),
|
||||
report.runtime_binding,
|
||||
mode,
|
||||
);
|
||||
report.runtime_binding_audit_ms.after = performance.now() - bindingFinished;
|
||||
report.runtime_binding_verified = true;
|
||||
report.status = "passed";
|
||||
await writeFile(join(output, "parent-evidence.json"), JSON.stringify(parent));
|
||||
await writeFile(join(output, "answer.json"), JSON.stringify(answer));
|
||||
} finally {
|
||||
report.elapsed_ms = performance.now() - began;
|
||||
report.cleanup_verified = true;
|
||||
await writeFile(
|
||||
join(output, "report.json"),
|
||||
JSON.stringify(report, null, 2) + "\n",
|
||||
);
|
||||
}
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
status: report.status,
|
||||
mode,
|
||||
cache,
|
||||
parent: report.parent_evidence_id,
|
||||
answer: report.answer_sha256,
|
||||
calls: report.calls.map((c) => ({
|
||||
label: c.label,
|
||||
elapsed_ms: c.elapsed_ms,
|
||||
bytes: c.stdout_bytes,
|
||||
})),
|
||||
}),
|
||||
);
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,219 @@
|
||||
# Portable managed CLI member views
|
||||
|
||||
Issue #5148 / PR #5167. Baseline `main@3dbe314ada9f6dcc994ca4655dc0faf082503847`;
|
||||
feature integrates the original `7b3ed4d025edd24d062cde0327a53bd311e174fc`.
|
||||
The measured production-file SHA-256 manifest is in [managed-cli.json](managed-cli.json).
|
||||
These are working-tree measurements. Each trial verifies both source and the
|
||||
complete compiled CLI graph before and after execution against its explicitly
|
||||
selected build. The two roles have distinct checkout roots and compiled digests.
|
||||
Identity-audit time is included in whole-task costs and recorded separately.
|
||||
The audit reads compiled files before timing individual CLI calls, warming those
|
||||
filesystem pages; cold means disabled Node compile cache, not disk-cold startup.
|
||||
|
||||
## Question and independent checks
|
||||
|
||||
Report identity, selected type, source coverage, the total method count, the first
|
||||
eight complete method records, and the first sixteen assembly-wide MemberRef
|
||||
records, including continuation coverage. Keep complete original Evidence available.
|
||||
The question deliberately does not ask for every method or reference.
|
||||
|
||||
Public artifact: [Newtonsoft.Json 13.0.3](https://www.nuget.org/packages/Newtonsoft.Json/13.0.3),
|
||||
from the NuGet flat-container archive. The 2,441,966-byte archive SHA-256 is
|
||||
`872fc189e638ab1056555b03aaa38f68bcb54286e221aa646eb1129babf63c77`.
|
||||
Selected member `lib/net6.0/Newtonsoft.Json.dll`, 712,464 bytes, SHA-256
|
||||
`22c649f75fce5be7c7ccda8880473b634ef69ecf33f5d1ab8ad892caf47d5a07`.
|
||||
Package version 13.0.3 is distinct from metadata assembly version 13.0.0.0.
|
||||
No library code is executed; REA parses PE/CLI statically.
|
||||
|
||||
An independent Python oracle uses `dnfile==0.18.0` and `pefile==2024.8.26`.
|
||||
It finds 494 TypeDefs, 4,208 MethodDefs and 2,149 MemberRefs. The selected
|
||||
`Newtonsoft.Json.JsonConvert` has 72 methods. Every trial checks all 72 method
|
||||
tokens, names, RVAs and original signature hashes, plus the sixteen returned
|
||||
MemberRef tokens, names and signature hashes, against this oracle. This does
|
||||
not independently establish complete CIL semantics or opcode correctness.
|
||||
The full parent Evidence and requested answers are identical across all twelve
|
||||
trials; returned rows equal the exact parent slices. Selection and source
|
||||
coverage remain distinct from page exhaustion. MemberRefs remain assembly-wide.
|
||||
|
||||
## Completed-task cost
|
||||
|
||||
Linux x64, Node 24.18.0/npm 11.16.0. Three alternating pairs per cache condition,
|
||||
serial, one CPU; 4 GiB hard / 3 GiB early-stop family guard, no swap, 192-task
|
||||
ceiling. Every task includes actual selected-command help, all CLI calls,
|
||||
provider parsing, bundle publication/authentication, local answer extraction
|
||||
and oracle comparisons, plus both runtime-identity audits. Build, download, oracle preparation and warm priming
|
||||
are excluded. OS page cache is uncontrolled. No build/test overlapped timing.
|
||||
|
||||
Complete mode starts two CLI processes (help plus analysis). Selected mode starts
|
||||
five (two help commands, summary/publication, methods page, MemberRef page).
|
||||
Cold disables the Node compile cache; warm primes the exact workflow once per
|
||||
runtime/mode, then still starts fresh processes with isolated HOME each trial.
|
||||
All trials complete with verified process cleanup and no OOM/resource-stop event.
|
||||
|
||||
| Whole task, median | Complete | Summary + two pages |
|
||||
| ----------------------------------------------- | -------: | ------------------: |
|
||||
| Estimated context tokens, both cache conditions | 451,373 | 13,078 |
|
||||
| Cold wall seconds | 9.969 | 14.801 |
|
||||
| Cold family CPU seconds | 9.743 | 14.555 |
|
||||
| Cold kernel family peak MiB | 393.281 | 400.098 |
|
||||
| Warm wall seconds | 10.339 | 24.141 |
|
||||
| Warm family CPU seconds | 10.097 | 23.842 |
|
||||
| Warm kernel family peak MiB | 401.809 | 407.164 |
|
||||
|
||||
Context estimates include the question, supplied command recipe, all logical
|
||||
argv, actual help and stdout. Tokenizer `tiktoken==0.12.0`, `cl100k_base`.
|
||||
The reduction is 97.1% for this fixed task. There is no model invocation;
|
||||
actual billed/native-agent usage, reasoning and transport framing are unmeasured.
|
||||
The memory counter is the kernel cgroup process-family peak including file
|
||||
cache, not sampled RSS. Ranges and individual receipts are in the JSON.
|
||||
|
||||
This is an opt-in context/portability improvement with extra latency and CPU,
|
||||
not a speed or memory optimization. Complete provider parsing and full authentication of file-supplied bundles
|
||||
remain. Fresh managed Evidence is sealed at its producing factory and reused
|
||||
without another payload clone, including low-headroom error projection. Complete output remains the default and is preferable
|
||||
for small outputs or questions requiring every row. No cache or parser cap
|
||||
was introduced.
|
||||
|
||||
## Existing native code composition
|
||||
|
||||
The unchanged baseline can already redirect full stdout to a file and select
|
||||
rows in code. [The measured control](managed-cli-local-control.py) returns the
|
||||
identical answer and retains the complete original Evidence. One cold run:
|
||||
11.249 s wall, 11.000 s family CPU,
|
||||
375.3 MiB kernel peak. Projection source,
|
||||
original help and returned answer together estimate 10,555 tokens; the shared
|
||||
question and launcher argv are excluded. This single-run control does not
|
||||
establish stable performance or actual native-agent usage. It does demonstrate
|
||||
that raw CLI reply savings are not universal savings over code composition.
|
||||
The additive interface supplies authenticated reusable projections through the
|
||||
same application/domain owner as MCP, without caller-maintained JSON slicing.
|
||||
|
||||
## Correctness and resource boundaries
|
||||
|
||||
The public CLI regression pages all eight managed collections after process
|
||||
exit, target deletion and a working-directory change. Complete parent identity,
|
||||
original row order, signatures, source/page coverage and unknowns survive.
|
||||
Existing outputs are not replaced; failed publication retains completed facts.
|
||||
|
||||
The Evidence factory parses mutable caller data into independent trees, computes
|
||||
identity, then seals its fresh owned snapshot with active ancestry/iterators.
|
||||
It does not retain a set/array of every completed object. Existing sealed roots
|
||||
can be shared; mutable input aliases are cloned independently. General graph
|
||||
sealing keeps its separate cycle, sharing, hidden-field and atomic-failure rules.
|
||||
Managed and derived-view factories seal their own completed Evidence, so CLI
|
||||
error projection can borrow it without reauthentication or another payload copy.
|
||||
The view owner retains strict JSON normalization for optional-field contracts;
|
||||
the CLI no longer copies its already validated completed projection again.
|
||||
|
||||
Shared heap admission traverses by depth rather than queueing every report row
|
||||
or allocating an entries array. It reserves parsed working copies and actual
|
||||
escaping expansion before cloning/hashing, reporting UTF-16 storage explicitly.
|
||||
Canonical bundle equality streams existing ordinary JSON encoder parts, preserving
|
||||
native key order, omission, negative zero and Unicode semantics without two
|
||||
whole-document strings. All production bundle readers explicitly pass their
|
||||
operation: CLI import/export/compare/inspection, retained views, MCP import/
|
||||
comparison and both native-matching read passes. Errors retain the original
|
||||
cause, cleanup primary and lease through retry.
|
||||
|
||||
One process primitive dispatches queued callbacks and then their OS signal
|
||||
notifications. The managed producer checks it before accepting complete Evidence.
|
||||
Publication/read stages use it before their final checks. Inline, file and bundle
|
||||
CLI views share the existing command finalizer and actual resource scope; the
|
||||
finalizer checks interruption after close while preserving any primary/cleanup
|
||||
failure. Existing inline error labels remain intact. Synchronous analysis is
|
||||
cooperatively cancellable between stages, not preemptible.
|
||||
|
||||
Four compiled publication cases (SIGINT/SIGTERM × event/actual POSIX signal)
|
||||
fail without their checkpoint and pass restored. Eight additional compiled
|
||||
completion cases cover default complete delivery and inline projection; all fail
|
||||
with only their new completion checkpoints removed and pass restored. Fixture
|
||||
hooks match the selected target path or exact parent ID. Two real Newtonsoft
|
||||
publication runs and four real default/inline completion runs retain the identical
|
||||
parent or independently checked selected summary, exit 130/143 and preserve
|
||||
committed receipts where applicable. No target library code is executed.
|
||||
|
||||
All 556 focused tests across 36 files pass, including direct
|
||||
finalizer/cleanup ownership, WASM/Android cancellation and native matching.
|
||||
All six `npm run check` tasks plus architecture guards, and all six
|
||||
`npm run docs:check` tasks plus five README tests pass. The expanded domain
|
||||
project passes 5,617 tests in 287 files with
|
||||
3 skipped cases; this overlaps the focused gate. Its first
|
||||
batched shard hit two existing 30-second timeouts in unchanged large synthetic
|
||||
tests. Both passed isolated and in the fresh shard rerun under the same timeout;
|
||||
the cause of the batch variation is unverified. No production/timeout change
|
||||
was made for those cases. An earlier unsplit expansion was manually stopped
|
||||
before its heap ceiling; that incomplete attempt is not counted as a pass.
|
||||
|
||||
Eight producing/operation negative controls also catch graph-sized sealing/
|
||||
admission scratch and wrong operation attribution, including native reload.
|
||||
Native built-in instrumentation is confined to the isolated process-global lane.
|
||||
Initial test-only spy/probe/schema-import issues were corrected before these
|
||||
passing gates. No actual heap exhaustion is induced. Simulated providers do not
|
||||
qualify foreign engines; Windows OS signals remain unverified locally.
|
||||
|
||||
## Shared-reader real-artifact regression
|
||||
|
||||
The existing Redux Toolkit 2.2.7 source/bundle task also completes through the
|
||||
updated reader with default full output schemas and compact input schemas.
|
||||
An independent Python SHA-256/path oracle verifies 180 historical files,
|
||||
99 embedded source nodes and 210 candidate relations (33 strong-location rows,
|
||||
147 unknown). Original signals and partial source coverage remain visible.
|
||||
This uses the existing explicit semantic-node cap of one for static correspondence;
|
||||
it does not establish equivalent semantic-graph coverage or behavioral equivalence.
|
||||
|
||||
One completed task: 21.380 s wall,
|
||||
21.023 s family CPU,
|
||||
928.9 MiB kernel family peak,
|
||||
56,378 estimated context tokens (one input catalog,
|
||||
all arguments and MCP text replies; excludes output schemas, framing and model
|
||||
reasoning). Corpus/reference preparation was performed earlier and excluded.
|
||||
This is a regression verification with resource observation, not a new paired
|
||||
performance comparison. Actual model usage remains unmeasured. Details:
|
||||
[managed-cli-shared-read.json](managed-cli-shared-read.json).
|
||||
|
||||
## Reproduction
|
||||
|
||||
Extract the selected DLL to a private lab as `Newtonsoft.Json.dll`. Create an
|
||||
`acquisition.json` containing the URL, hashes, sizes and selected ZIP member
|
||||
shown above; never commit the DLL. Prepare the independent oracle:
|
||||
|
||||
```bash
|
||||
uv run --no-project --with dnfile==0.18.0 --with pefile==2024.8.26 python \
|
||||
experiments/analysis-efficiency/managed-cli-oracle.py /absolute/lab
|
||||
```
|
||||
|
||||
Build baseline and feature separately with the pinned toolchain. Run each under
|
||||
the shared resource guard, alternating complete/selected, selected/complete,
|
||||
complete/selected for cold and warm. Use a new output directory for every run:
|
||||
|
||||
```bash
|
||||
python3 experiments/analysis-efficiency/resource-run.py --observe-payload-exit -- \
|
||||
node experiments/analysis-efficiency/managed-cli-task.mjs \
|
||||
/absolute/runtime /absolute/lab /absolute/lab/pair-cold-1-selected selected cold
|
||||
```
|
||||
|
||||
Save each guard output to the adjacent `pair-*.log`. Warm mode uses the lab's
|
||||
mode-specific compile-cache directory; prime each identical workflow once first.
|
||||
Supply a `measurement-inputs.json` with exact revisions and production hashes,
|
||||
plus `expected_runtime_bindings.complete` and `.selected`. Capture each with
|
||||
`node experiments/analysis-efficiency/managed-cli-runtime.mjs /absolute/runtime`
|
||||
after building. Bind complete to the clean baseline revision, selected to the
|
||||
feature parent revision plus tracked/untracked source digests, with different
|
||||
roots and compiled-graph digests. The collector rejects a wrong build before any
|
||||
CLI call, rechecks it afterward, and aggregation independently enforces identity.
|
||||
A real wrong-runtime attempt is rejected before analysis. A copied trial with a
|
||||
tampered revision fails aggregation; removing only that consumer guard accepts it.
|
||||
The twelve-trial aggregation reproduces without private pilots. The native
|
||||
control also requires `--measurement-inputs /absolute/lab/measurement-inputs.json`
|
||||
and performs the same before/after identity checks.
|
||||
Then aggregate with:
|
||||
|
||||
```bash
|
||||
uv run --no-project --with tiktoken==0.12.0 python \
|
||||
experiments/analysis-efficiency/managed-cli-costs.py /absolute/lab \
|
||||
--output /absolute/results.json
|
||||
```
|
||||
|
||||
The original issue reporter's executable, foreign operating systems and actual
|
||||
model sessions remain unverified. Setup pilots and earlier unbound cohorts, including one failed setup run
|
||||
while compiled artifacts were absent after a typecheck failure, are excluded.
|
||||
Reference in new issue
Block a user