feat(python): query selected pyc code-object facts (#6928)

* feat(python): query retained pyc code-object facts

* test(python): bind complete pyc selection benchmark tasks

* test(python): include server instructions in token accounting

* docs(perf): record pyc selected-facts task measurements

* test(python): keep oracle dependencies separate from selected targets

* docs(perf): record guarded oracle requalification

* test(python): narrow complete Evidence through its owning contract
This commit is contained in:
gensokyo-dev authored and GitHub committed 2026-10-24 10:23:06 +08:00
1 parent 315b23403f
commit e2463372ea
22 files changed
+3664 -156

No files matched your search

@@ -0,0 +1,114 @@
"""Verify completed PYC selection/export tasks and count a declared token projection."""
import argparse
import hashlib
import json
from pathlib import Path
import statistics
import tiktoken
def compact(value):
return json.dumps(value, ensure_ascii=False, separators=(',', ':'), allow_nan=False)
def digest(path):
return hashlib.sha256(path.read_bytes()).hexdigest()
def inspect(path, encoding):
report = json.loads((path / 'report.json').read_text())
catalog = json.loads((path / 'catalog.json').read_text())
transcript = json.loads((path / 'transcript.json').read_text())
scope = next(json.loads(line) for line in reversed(path.with_suffix('.log').read_text().splitlines())
if line.startswith('{"resource_scope":'))
observed = scope.get('payload_resource_observation') or {}
events = observed.get('resource_events', {})
assert report['status'] == 'passed' and report['runtime_identity_sha256']
assert report['complete_source_export_verified'] is True
assert not report['cleanup_failures'] and not report['receipt_failures']
assert scope['abort'] is None and scope['payload_resource_observation_verified'] is True
assert observed['payload_exit_code'] == 0 and observed['payload_cleanup_verified'] is True
assert all(events.get('memory.events', {}).get(key) == 0 for key in ['high', 'max', 'oom', 'oom_kill'])
assert events.get('pids.events', {}).get('max') == 0
assert len(catalog['tools']) == report['catalog_tools'] == 221
assert [event['name'] for event in transcript] == ['inspect_python_pyc', 'export_evidence_bundle']
assert all('reply' in event and not event['reply'].get('isError', False) for event in transcript)
count = lambda text: len(encoding.encode(text, disallowed_special=()))
directory = [{key: tool[key] for key in ['name', 'description', 'inputSchema']} for tool in catalog['tools']]
request = transcript[0]['input']
# Both lanes answer the same question; selected arguments disclose the exact fields/names.
question = (f"For {report['artifact']['path']}, return every matching code object for "
f"{', '.join(report['selected_names'])}, including its original source_index, name, qualname, "
"filename, firstlineno, bytecode bytes/SHA-256 and nested_code constant-reference indices. "
"Keep same-name alternatives, preserve source coverage and export complete source Evidence. "
"Do not execute the bytecode or infer runtime calls.")
assert ('detail' in request) == (report['lane'] == 'selected')
arguments = sum(count(compact({'name': event['name'], 'arguments': event['input']})) for event in transcript)
replies = sum(count(block['text']) for event in transcript for block in event['reply']['content'] if block['type'] == 'text')
inputs, prompt = count(compact(directory)), count(question)
instructions = count(report['server_instructions'])
return dict(case=path.name, lane=report['lane'], artifact=report['artifact'],
answer_count=report['answer_count'], answer_sha256=report['answer_sha256'],
parent_evidence_id=report['parent_evidence_id'], source_report_sha256=report['source_report_sha256'],
runtime_identity_sha256=report['runtime_identity_sha256'], catalog_sha256=digest(path / 'catalog.json'),
calls=report['calls'], phases=report['phases'], wall_seconds=scope['elapsed_seconds'],
cpu_seconds=observed['cpu_stat']['usage_usec'] / 1e6,
kernel_family_peak_bytes=observed['memory_peak_bytes'],
events_at_payload_exit=events, payload_cleanup_verified=True,
final_whole_scope_events=scope.get('resource_events_final'),
token_proxy=dict(question=prompt, server_instructions=instructions, input_catalog_once=inputs, arguments=arguments,
text_replies=replies, combined=prompt + instructions + inputs + arguments + replies,
complete_transport_catalog=count(compact(catalog))),
receipt_sha256={file.name: digest(file) for file in [path / 'report.json', path / 'catalog.json',
path / 'transcript.json', path / 'evidence.json', path.with_suffix('.log')]})
def summarize(rows):
result = {}
for artifact in sorted({row['artifact']['sha256'] for row in rows}):
cases = [row for row in rows if row['artifact']['sha256'] == artifact]
for field in ['answer_count', 'answer_sha256', 'parent_evidence_id', 'source_report_sha256']:
assert len({row[field] for row in cases}) == 1, field
result[artifact] = {}
for lane in ['complete', 'selected']:
samples = [row for row in cases if row['lane'] == lane]
assert len(samples) == 3
for field in ['runtime_identity_sha256', 'catalog_sha256']:
assert len({row[field] for row in samples}) == 1, field
metrics = {key: [row[key] for row in samples] for key in ['wall_seconds', 'cpu_seconds', 'kernel_family_peak_bytes']}
metrics['projected_tokens'] = [row['token_proxy']['combined'] for row in samples]
metrics['mcp_reply_bytes'] = [sum(call['reply_bytes'] for call in row['calls']) for row in samples]
result[artifact][lane] = {key: dict(median=statistics.median(values), minimum=min(values), maximum=max(values))
for key, values in metrics.items()}
return result
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--cohort', type=Path, required=True)
parser.add_argument('--output', type=Path, required=True)
args = parser.parse_args()
manifest = json.loads((args.cohort / 'manifest.json').read_text())
index = json.loads((args.cohort / 'index.json').read_text())
assert len(index) == 18 and all(case['exit'] == 0 for case in index)
for file, expected in manifest['driver_files'].items():
assert digest(Path(file)) == expected, file
identities = {}
for lane, path in manifest['runtime_identities'].items():
identity = json.loads(Path(path).read_text())
assert identity['source_clean'] is True
identities[lane] = dict(sha256=digest(Path(path)), source_revision=identity['source_revision'])
encoding = tiktoken.get_encoding('cl100k_base')
rows = [inspect(args.cohort / case['name'], encoding) for case in index]
assert all(row['runtime_identity_sha256'] == identities[row['lane']]['sha256'] for row in rows)
receipt = dict(tiktoken=tiktoken.__version__, encoding=encoding.name, actual_model_usage=None,
cohort=manifest, commands=index, runtime_identities=identities,
projection='Scripted fresh SDK tasks, not model runs. One fixed question, server instructions and complete input directory (name/description/inputSchema), every argument and MCP text reply including export. Raw catalog including output schemas reported separately. No framing, host injection, history replay, inference quality or billable usage measured.',
resources='CPU quota100%, payload affinity0-1, initial observer15. Early stop3GiB, hard4GiB, swap0. Fresh HOME/client/server; Node compile cache disabled, OS caches unflushed. Whole process-family CPU and kernel memory peak include file cache, runtime identity checks, hashing, assertions, SDK/server serialization and cleanup. Oracle generation and tokenization outside timed tasks. Payload-exit counters verified; final whole-scope counters unavailable.',
summary=summarize(rows), recordings=rows)
args.output.write_text(json.dumps(receipt, ensure_ascii=False, indent=2) + '\n')
if __name__ == '__main__':
main()
@@ -0,0 +1,103 @@
"""Independent CPython code-object facts; never execute target bytecode."""
import getopt
import hashlib
import importlib.util
import json
import marshal
from pathlib import Path
import py_compile
import sys
import types
FIELDS = ['name', 'qualname', 'filename', 'firstlineno', 'bytecode', 'nested_code']
NAMES = ['parse_args', 'parse_known_args', '_parse_known_args']
def describe(path, names):
raw = path.read_bytes()
assert raw[:4] == importlib.util.MAGIC_NUMBER, "Use the matching CPython producer for this oracle"
root = marshal.loads(raw[16:])
assert isinstance(root, types.CodeType)
objects, indices, children = [], {}, {}
def nested(value, visited):
identity = id(value)
if identity in visited:
return []
visited.add(identity)
if isinstance(value, types.CodeType):
return [value]
if isinstance(value, (tuple, list, frozenset, set)):
return [child for item in value for child in nested(item, visited)]
if isinstance(value, dict):
return [child for item in value.items() for child in nested(item, visited)]
return []
def visit(code):
if id(code) in indices:
return
indices[id(code)] = len(objects)
objects.append(code)
children[id(code)] = nested(code.co_consts, set())
for child in children[id(code)]:
visit(child)
visit(root)
rows = [dict(source_index=index, name=code.co_name,
qualname=code.co_qualname, filename=code.co_filename,
firstlineno=code.co_firstlineno,
bytecode=dict(bytes=len(code.co_code),
sha256=hashlib.sha256(code.co_code).hexdigest()),
nested_code=[indices[id(child)] for child in children[id(code)]])
for index, code in enumerate(objects)]
return dict(artifact=dict(path=str(path.resolve()), bytes=len(raw),
sha256=hashlib.sha256(raw).hexdigest()),
python=sys.version, names=names, fields=FIELDS, rows=rows,
answer=[row for row in rows if row['name'] in names],
bytecode_execution=False)
# Keep the argument parser independent of the argparse/typing targets.
options, trailing = getopt.getopt(sys.argv[1:], '',
['path=', 'compile-control=', 'output=', 'name='])
if trailing:
raise SystemExit('Unexpected positional arguments; repeat --name NAME for selectors')
values, selected_names = {}, []
for key, value in options:
if key == '--name':
selected_names.append(value)
elif key in values:
raise SystemExit('Duplicate option: ' + key)
else:
values[key] = value
if '--output' not in values:
raise SystemExit('--output is required')
path = Path(values['--path']) if '--path' in values else None
compile_control = Path(values['--compile-control']) if '--compile-control' in values else None
output = Path(values['--output'])
names = selected_names or NAMES
assert all(module not in sys.modules for module in ['argparse', 'typing']), 'Oracle dependency loaded a target module'
assert sys.implementation.name == 'cpython' and sys.version_info >= (3, 11)
if compile_control:
directory = compile_control
directory.mkdir(parents=True, exist_ok=False)
source = directory / 'control.py'
source.write_text('''class First:
def parse_args(self):
def inner():
return "a" + "😀" * 40
return inner
def parse_known_args(self):
return 1
class Second:
def parse_args(self):
return 2
def _parse_known_args():
return 3
''', encoding='utf8')
path = directory / 'control.pyc'
py_compile.compile(str(source), cfile=str(path), dfile='control.py',
doraise=True, invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH)
assert path is not None
output.write_text(json.dumps(describe(path, names), indent=2) + '\n')
assert all(module not in sys.modules for module in ['argparse', 'typing']), 'Oracle loaded a target module'
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,59 @@
# Selected Python pyc facts
Issue [#6922](https://github.com/morluto/rea/issues/6922) asks for selected code-object facts without delivering the whole report. The existing static parser already owns the facts. The change adds a typed `python-pyc` view to the shared retained-Evidence projector, and opt-in `detail: "summary", view: {...}` delivery through the existing CLI/MCP tool. The default complete result and parent Evidence identity remain unchanged.
Names, original indexes, fields and matching-row pages compose. Every same-name alternative survives under its parent-scoped `source_index`; an empty selector selects nothing. `nested_code` keeps original code-constant reference indexes and does not claim runtime calls. Source coverage is separate from page coverage. Complete Evidence remains exportable, and saved bundles support offline views after the original file is deleted.
The immutable-result reader is shared with native-export views instead of adding a second cache implementation. Each owning schema has a weak cache keyed by the validated, owner-sealed result object. It never caches a path or current filesystem state. Mutable/unowned values and failed parses are revalidated; schema-normalized facts stay separate from the source. CLI delivery JSON parsing also has one owner, shared with the existing JavaScript route commands.
## Question and independent oracle
For each target, return every code object with one of three exact names, including `source_index`, `name`, `qualname`, `filename`, `firstlineno`, bytecode byte length/SHA-256 and `nested_code`. Preserve source coverage and export the complete source Evidence. Both lanes perform discovery, one inspection and one Evidence export. The selected lane includes the view in the first inspection; this question fits one page with the default limit of 64.
| Input | Source bytes | All objects | Selected rows | Names |
| ----------------------------------------- | -----------: | ----------: | ------------: | ----------------------------------------------------------- |
| CPython 3.11.2 `argparse.cpython-311.pyc` | 113,662 | 194 | 3 | `parse_args`, `parse_known_args`, `_parse_known_args` |
| CPython 3.11.2 `typing.cpython-311.pyc` | 157,945 | 307 | 3 | `get_type_hints`, `_eval_type`, `_type_check` |
| Inert compiler-produced control | 1,396 | 8 | 4 | argparse names, including two distinct `parse_args` objects |
`python-pyc-oracle.py` uses the matching CPython 3.11.2 magic and an independent marshal/CodeType graph walk. It checks the selected facts for **all** objects in the exported source, as well as every returned answer row. The small control is compiled, never executed. The production lanes never execute target bytecode. The original oracle CLI imported the system argparse module as its argument-parser dependency (outside timed tasks); it did not explicitly execute the selected marshal CodeType, but the broader claim that the oracle never loaded target bytecode was inaccurate. Review replaced that dependency with `getopt`, added before/after guards for argparse/typing module loading, and regenerated all three oracles: their complete JSON facts are identical. The current oracle neither imports either selected standard-library module nor executes the decoded CodeType. At `062f94ce3`, a newly compiled control reproduces the original code facts and pyc SHA-256; all three selected stdio/export tasks pass again with the regenerated oracles and a current runtime/driver identity manifest. These observations qualify the three concrete CPython 3.11 artifacts, not every supported historical marshal layout or an agent's investigation quality.
Target SHA-256 values, full commands, runtime/driver identities and receipt digests are in [the machine-readable results](python-pyc-views.json). All six sessions per target produce the same answer digest, parent Evidence ID and complete normalized-source digest.
## Fixed-version measurements
Baseline: `6b3eee1ccf4c51f23bef7a0007c7b8e892f662db`. Candidate: `29ff0893e` (full revision in the results). Three alternating fresh sessions per lane and target, 18 completed tasks. The order reverses in the second pair. Linux x86-64, Node 24.22.1, npm 11.16.0; production stdio MCP with SDK 2.3.1, its exact discovered tool definitions and default output validator. No validator patch or private SDK hook is used.
Each task has a fresh HOME/client/server, disables Node compile caches, and leaves OS caches unflushed. These are fresh-session measurements with warm/uncontrolled OS caches; there is no separate warm-server cohort. Runtime files, installed client modules, entrypoints, the Node executable and driver files are checked before/after tasks. Oracle generation and tokenization are outside timed tasks. Whole-task costs include discovery, SDK schema compilation, runtime checks, source hashing, comparisons, serialization, export and cleanup.
| Median | argparse complete → selected | typing complete → selected | Small control complete → selected |
| ---------------------------------------------------- | ---------------------------: | -------------------------: | --------------------------------: |
| Estimated task tokens | 104,415 → 57,300 | 128,858 → 57,316 | 57,719 → 57,332 |
| Serialized MCP reply bytes, both calls | 338,446 → 9,011 | 495,172 → 8,799 | 14,891 → 9,083 |
| Elapsed seconds | 5.760 → 5.559 | 5.761 → 5.761 | 5.358 → 5.459 |
| Process-family CPU seconds | 5.532 → 5.369 | 5.532 → 5.566 | 5.180 → 5.247 |
| Kernel process-family peak MiB, including file cache | 649.1 → 663.7 | 685.0 → 642.5 | 644.8 → 654.3 |
The declared token projection falls **45.1%** for argparse and **55.5%** for typing, versus **0.7%** for the small control. No stable general elapsed-time, CPU or memory improvement is established. The parser and complete-source retention still do their full work. Wall-time ranges overlap for every target; typing selected ranges 5.560–6.466 seconds versus 5.660–5.963 complete. Other ranges and individual samples remain in the results.
Token estimates use `tiktoken==0.14.0` / `cl100k_base`. The projection counts the question, server instructions, all 221 discovered tool names/descriptions/input schemas once, every argument and every MCP text reply, including export. Candidate input-directory cost increases from 54,883 to 55,283 estimated tokens. Complete raw catalog transport, including output schemas, is separately counted: 822,529 → 826,451 tokens. It is not added to the model-input directory projection. Structured content is validated and retained; the context projection counts the text reply once. Actual model usage, inference quality, history replay, protocol framing and host-added context are unmeasured.
Each task uses one aggregate CPU quota, payload affinity CPUs 0–1, a 3 GiB early stop, 4 GiB hard limit, no swap and 192-task limit. The observer initially runs on CPU 15; the receipt's initial affinity does not describe payload placement. All 18 payloads exit zero, confirm cleanup, and show zero memory/task limit or OOM events at payload exit. Final whole-scope counters are unavailable and remain null. Peak kernel memory includes file cache and differs from a single process's RSS.
## Verification and integration scope
Executed against the measured base integration:
- 46 source tests: analysis views, schema-specific immutable-result ownership, PYC delivery and existing JavaScript delivery contracts. Converted input schemas validate under draft-2020-12 and draft-07 and agree with runtime parsing on valid/invalid combinations.
- 27 public CLI/MCP tests across PYC and native-export suites: direct selected delivery, full source bundles, retained/offline queries after deleting the source, duplicate names, paging, empty/intersecting selectors, source references, view digests, invalid delivery and existing complete PYC behavior.
- Six serialized static tasks (typecheck, lint, format, unused exports, metadata and their compile dependency) and architecture guards passed. Test-family peak was about 2.48 GiB, with no guard stop/OOM and verified payload cleanup.
The candidate subsequently integrated main `76500f12b54e07b1ea700a403b2593890d63b42a`. The table remains a measurement of the fixed revisions above. At integration `c172fd330`, `npm run docs:generate` and `npm run docs:check` passed (including 9 README consistency tests); the six serialized static tasks and architecture guards passed again, and the two affected retained-view public suites passed all 5 tests. The combined integration job peaked at about 2.59 GiB with verified payload cleanup and no resource stop/OOM. Real-artifact qualification is recorded separately; it does not turn these timings into measurements of a later revision.
Retained failed attempts: an initial test mislabeled magic 3531 as 3.11 (the owner maps it to 3.12); the assertion now compares the complete owner's header. A CLI validation return initially bypassed `logCliCommand`, producing exit zero; validation now runs inside that command owner, before opening the provider, and the public negative regression passes. An oversized acceptance function was split into fixture/selector helpers. A final test invocation used a nonexistent JavaScript test filename and was rejected before running tests; the corrected invocation produced the 46/27 passing results above. An early cached build had an unverified transient cleanup observation; its scope was subsequently inactive. The final baseline build and all measured tasks verify payload cleanup. None of these failed attempts are counted as successful measurements.
## Reproduction
Use the pinned toolchain and dependencies. Build detached baseline and candidate runtimes, and generate independent facts with `python3 python-pyc-oracle.py --path ABSOLUTE_PYC --name NAME (repeatable) --output ORACLE.json`; `--compile-control NEW_DIRECTORY` creates the inert small control. Run `python-pyc-views.mjs --runtime RUNTIME --oracle ORACLE --output NEW_OUTPUT --lane complete|selected`, through `resource-run.py --observe-payload-exit` and the recorded affinity/toolchain wrapper. The optional `--identity` manifest pins compiled files/entrypoints, the Node executable and driver files; the before/after checks are included in task costs. Preserve raw catalog, transcript, complete bundle, report and guard log for every run.
Repeat the recorded alternating order. A cohort manifest binds runtime identities and driver hashes; its index retains each exact command and exit. Aggregate with `uv run --no-project --with tiktoken==0.14.0 python python-pyc-costs.py --cohort COHORT --output RESULTS.json`. The aggregator rejects missing/failed tasks, failed cleanup/resource observations, changed identities, unequal answers/source digests and unexpected call counts. Raw local receipts are under `/tmp/rea-perf-pyc-views-20261023/export-cohort`; they are not required runtime files.
@@ -0,0 +1,272 @@
/** Same selected code-object question through production stdio MCP delivery. */
import assert from "node:assert/strict";
import { createHash } from "node:crypto";
import { createWriteStream } from "node:fs";
import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { finished } from "node:stream/promises";
import { parseArgs } from "node:util";
import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { createRetainedQueryCaller } from "./retained-query-calls.mjs";
import { finalizeRetainedQueryReceipt } from "./retained-query-finalization.mjs";
const { values } = parseArgs({
options: {
runtime: { type: "string" },
identity: { type: "string" },
oracle: { type: "string" },
output: { type: "string" },
lane: { type: "string", default: "complete" },
"page-size": { type: "string", default: "64" },
"two-step": { type: "boolean", default: false },
"probe-support": { type: "boolean", default: false },
},
});
for (const key of ["runtime", "oracle", "output"])
assert.ok(values[key] && values[key] === resolve(values[key]), key);
assert.ok(["complete", "selected"].includes(values.lane));
const pageSize = Number(values["page-size"]);
assert.ok(Number.isSafeInteger(pageSize) && pageSize > 0);
const expected = JSON.parse(await readFile(values.oracle, "utf8"));
const hash = (value) => createHash("sha256").update(value).digest("hex");
assert.equal(
hash(await readFile(expected.artifact.path)),
expected.artifact.sha256,
);
await mkdir(values.output, { mode: 0o700 });
const account = await mkdtemp(join(tmpdir(), "rea-pyc-views-"));
const client = new Client({ name: "pyc-views", version: "1" });
const transport = new StdioClientTransport({
command: process.execPath,
args: [join(values.runtime, "scripts/rea.mjs"), "mcp"],
maxBufferSize: 10485760,
env: {
HOME: account,
TMPDIR: account,
PATH: process.env.PATH,
REA_LOG_LEVEL: "silent",
NODE_OPTIONS: "--max-old-space-size=2048",
NODE_DISABLE_COMPILE_CACHE: "1",
UV_THREADPOOL_SIZE: "2",
},
stderr: "pipe",
});
const serverLog = createWriteStream(join(values.output, "server.log"));
const serverLogFinished = finished(serverLog);
serverLogFinished.catch(() => {});
transport.stderr?.pipe(serverLog);
const report = {
status: "running",
lane: values.lane,
artifact: expected.artifact,
selected_names: expected.names,
calls: [],
phases: {},
client: "SDK exact discovered toolDefinition, default validator",
task: "Return every matching code object for exact names with original indices and selected fields; export complete source Evidence.",
bytecode_execution: false,
page_size: pageSize,
direct_first_view: values.lane === "selected" && !values["two-step"],
};
const verifyRuntime = async (phase) => {
if (values.identity === undefined) return;
const started = performance.now();
const bytes = await readFile(values.identity);
const identity = JSON.parse(bytes);
for (const entry of [...identity.compiled_files, ...identity.entrypoints])
assert.equal(
hash(await readFile(join(values.runtime, entry.path))),
entry.sha256,
entry.path,
);
for (const entry of identity.driver_files)
assert.equal(
hash(await readFile(new URL(entry.path, import.meta.url))),
entry.sha256,
);
assert.equal(hash(await readFile(process.execPath)), identity.node_sha256);
report.runtime_identity_sha256 ??= hash(bytes);
assert.equal(hash(bytes), report.runtime_identity_sha256);
report.phases[`runtime_validation_${phase}_ms`] = performance.now() - started;
};
const transcript = [];
let definitions;
const caller = createRetainedQueryCaller(
{
callTool(params, options) {
return client.callTool(params, {
...options,
toolDefinition: definitions.get(params.name),
});
},
},
(receipt, event) => {
report.calls.push(receipt);
transcript.push(event);
},
);
const observe = async (name, input, phase) => {
const result = await caller(name, input, phase);
if (result.isError === true) assert.fail(JSON.stringify(result));
assert.ok(result.structuredContent);
return result.structuredContent;
};
const columns = new Set(["source_index", ...expected.fields]);
const select = (row) =>
Object.fromEntries(Object.entries(row).filter(([key]) => columns.has(key)));
try {
await verifyRuntime("before");
const started = performance.now();
await client.connect(transport);
report.server_instructions = client.getInstructions() ?? "";
const catalog = await client.listTools();
definitions = new Map(catalog.tools.map((tool) => [tool.name, tool]));
await writeFile(join(values.output, "catalog.json"), JSON.stringify(catalog));
report.phases.discovery_ms = performance.now() - started;
report.catalog_tools = catalog.tools.length;
const selected = values.lane === "selected";
const selectedView = (offset) => ({
kind: "python-pyc",
collection: "code_objects",
names: expected.names,
fields: expected.fields,
offset,
limit: pageSize,
});
const response = await observe(
"inspect_python_pyc",
{
path: expected.artifact.path,
...(selected
? {
detail: "summary",
...(values["two-step"] ? {} : { view: selectedView(0) }),
}
: {}),
},
"inspect",
);
const parentId = selected
? response.normalized_result.parent_evidence_id
: response.evidence_id;
report.parent_evidence_id = parentId;
let answer;
if (!selected) {
const source = response.normalized_result;
assert.deepEqual(source.artifact, expected.artifact);
assert.equal(source.coverage.status, "complete");
const rows = source.code_objects.map((row, source_index) =>
select({ ...row, source_index }),
);
assert.deepEqual(rows, expected.rows);
answer = rows.filter((row) => expected.names.includes(row.name));
} else {
answer = [];
let offset = 0;
let page = values["two-step"] ? undefined : response.normalized_result;
do {
page ??= (
await observe(
"inspect_analysis_view",
{
source: { kind: "retained-evidence", evidence_id: parentId },
view: selectedView(offset),
},
"selected-page",
)
).normalized_result;
assert.equal(page.kind, "python-pyc");
assert.deepEqual(page.pyc.artifact, expected.artifact);
assert.equal(page.pyc.coverage.status, "complete");
assert.equal(page.pyc.coverage.code_objects, expected.rows.length);
assert.equal(page.coverage.total, expected.answer.length);
answer.push(...page.items);
if (page.coverage.next_offset === null) break;
assert.ok(page.coverage.next_offset > offset);
assert.ok(page.coverage.next_offset <= expected.answer.length);
offset = page.coverage.next_offset;
page = undefined;
} while (true);
}
assert.deepEqual(answer, expected.answer);
report.answer_sha256 = hash(JSON.stringify(answer));
report.answer_count = answer.length;
await writeFile(join(values.output, "answer.json"), JSON.stringify(answer));
await observe(
"export_evidence_bundle",
{ path: join(values.output, "evidence.json") },
"export",
);
const bundle = JSON.parse(
await readFile(join(values.output, "evidence.json"), "utf8"),
);
const parent = bundle.records.find(
(record) => record.evidence_id === parentId,
);
assert.ok(parent);
assert.equal(parent.operation, "inspect_python_pyc");
assert.deepEqual(parent.normalized_result.artifact, expected.artifact);
assert.equal(parent.normalized_result.coverage.status, "complete");
assert.deepEqual(
parent.normalized_result.code_objects.map((row, source_index) =>
select({ ...row, source_index }),
),
expected.rows,
);
report.source_report_sha256 = hash(JSON.stringify(parent.normalized_result));
report.complete_source_export_verified = true;
if (values["probe-support"]) {
report.support = {};
for (const [label, name, input] of [
[
"summary_delivery",
"inspect_python_pyc",
{ path: expected.artifact.path, detail: "summary" },
],
[
"summary_view",
"inspect_analysis_view",
{
source: { kind: "retained-evidence", evidence_id: parentId },
view: { kind: "summary" },
},
],
]) {
try {
const reply = await caller(name, input, "capability-probe");
report.support[label] = { is_error: reply.isError === true, reply };
} catch (cause) {
report.support[label] = { is_error: true, message: String(cause) };
}
}
}
report.status = "passed";
} catch (cause) {
report.status = "failed";
report.error = String(cause);
} finally {
try {
await verifyRuntime("after");
} catch (cause) {
report.status = "failed";
report.runtime_identity_failure = String(cause);
}
await finalizeRetainedQueryReceipt({
client,
transport,
serverLog,
serverLogFinished,
account,
report,
output: values.output,
transcript,
});
if (
report.status !== "passed" ||
report.cleanup_failures.length ||
report.receipt_failures.length
)
process.exitCode = 1;
}