mirror of
https://github.com/morluto/rea.git
synced 2026-11-07 07:16:13 +00:00
gensokyo-dev
c467cf8685
State the existing discovery page and exact-name bounds using the validation constants. Isolate account apps and plugins in catalog trials by default, with an explicit opt-in and one shared feature policy for config and receipts.
345 lines
20 KiB
Python
345 lines
20 KiB
Python
"""Evaluate one real Codex CLI task with an isolated REA MCP registration."""
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import signal
|
|
import subprocess
|
|
import tempfile
|
|
import time
|
|
|
|
from codex_context import capture_tool_context
|
|
from runtime_identity import file_digest, runtime_fingerprint
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--mode', choices=['direct', 'client-code'], default='client-code',
|
|
help='client-code enables code_mode_host; direct is a host-disabled diagnostic')
|
|
parser.add_argument('--runtime-root', type=Path, required=True)
|
|
parser.add_argument('--revision', required=True)
|
|
parser.add_argument('--prompt', type=Path, required=True)
|
|
parser.add_argument('--artifact-path', help='Replace {{artifact_path}} in a question template')
|
|
parser.add_argument('--output', type=Path, required=True)
|
|
parser.add_argument('--model', default='gpt-5.6-terra')
|
|
parser.add_argument('--reasoning', default='high')
|
|
parser.add_argument('--timeout', type=int, default=900)
|
|
parser.add_argument('--codex', default='/usr/bin/codex')
|
|
parser.add_argument('--providers', type=Path)
|
|
parser.add_argument('--enabled-tools', type=Path)
|
|
parser.add_argument('--catalog-load', action='store_true',
|
|
help='Measure a no-tool response with identical instructions across catalogs')
|
|
parser.add_argument('--no-mcp', action='store_true',
|
|
help='No-server control; requires --catalog-load')
|
|
parser.add_argument('--profile-parent', type=Path,
|
|
help='External disposable-profile parent (default: cache beside the source Codex home)')
|
|
parser.add_argument('--authority', choices=['static', 'browser'], default='static')
|
|
parser.add_argument('--include-account-tools', action='store_true',
|
|
help='Allow account apps/plugins; default isolates REA from their catalogs')
|
|
parser.add_argument('--capture-tool-context', action='store_true',
|
|
help='Retain owned tool/usage items; excludes account/ambient state')
|
|
parser.add_argument('--guide-enabled-tools', action='store_true',
|
|
help='Opt-in exact-name discovery guidance for selected REA tools')
|
|
args = parser.parse_args()
|
|
if args.no_mcp and not args.catalog_load:
|
|
parser.error('--no-mcp requires --catalog-load')
|
|
if args.catalog_load and (args.guide_enabled_tools or args.artifact_path):
|
|
parser.error('--catalog-load excludes tool guidance and artifact tasks')
|
|
if args.guide_enabled_tools and args.enabled_tools is None:
|
|
parser.error('--guide-enabled-tools requires --enabled-tools')
|
|
enabled_tools = None
|
|
if args.enabled_tools:
|
|
enabled_tools = json.loads(args.enabled_tools.read_text())
|
|
if not isinstance(enabled_tools, list) or not all(isinstance(name, str) and name for name in enabled_tools) or len(enabled_tools) != len(set(enabled_tools)):
|
|
raise ValueError('enabled-tools must be a unique list of exact tool names')
|
|
if args.guide_enabled_tools and not enabled_tools:
|
|
parser.error('--guide-enabled-tools requires a nonempty selected tool list')
|
|
def terminate(signum, _frame):
|
|
# The outer resource scope terminates descendants together. Ensure the
|
|
# runner's copied credentials are also removed on a resource abort.
|
|
raise SystemExit(128 + signum)
|
|
signal.signal(signal.SIGTERM, terminate)
|
|
if not args.runtime_root.is_absolute() or not args.output.is_absolute():
|
|
raise ValueError('Runtime and output paths must be absolute')
|
|
repository = args.runtime_root
|
|
driver_repository = Path(__file__).resolve().parents[3]
|
|
experiment = Path(__file__).resolve().parent
|
|
node = shutil.which('node')
|
|
expected_node = (repository / '.nvmrc').read_text().strip().removeprefix('v')
|
|
if subprocess.check_output([node, '--version'], text=True).strip() != 'v' + expected_node:
|
|
raise SystemExit('Use runtime-pinned Node ' + expected_node + ' on PATH')
|
|
provider_env = {'REA_LOG_LEVEL': 'silent', 'NODE_OPTIONS': '--max-old-space-size=512'}
|
|
if args.providers:
|
|
declared = json.loads(args.providers.read_text())
|
|
if not isinstance(declared, dict) or not all(
|
|
isinstance(value, str) for value in declared.values()
|
|
):
|
|
raise ValueError('--providers must contain an object of explicit string settings')
|
|
provider_env.update(declared)
|
|
provider_policy_text = (experiment / 'provider-environment.json').read_text()
|
|
provider_policy = json.loads(provider_policy_text)
|
|
undeclared = sorted(key for key in os.environ if (
|
|
any(key.startswith(prefix) for prefix in provider_policy['namespace_prefixes'])
|
|
or key in provider_policy['additional_keys']
|
|
) and key not in provider_env)
|
|
if undeclared:
|
|
raise ValueError('Declare inherited provider settings explicitly in --providers: '
|
|
+ ', '.join(undeclared) + '. Values are not logged.')
|
|
source_auth_home = (Path(os.environ['CODEX_HOME']) if 'CODEX_HOME' in os.environ
|
|
else Path.home() / '.codex')
|
|
source_auth = source_auth_home / 'auth.json'
|
|
args.output.mkdir(parents=True, exist_ok=False, mode=0o700)
|
|
runtime_files, runtime_digest = runtime_fingerprint(repository)
|
|
(args.output / 'runtime-files.json').write_text(json.dumps(runtime_files))
|
|
# Output artifacts may live inside a checkout. Execution must not: Codex
|
|
# discovers ancestor AGENTS.md and skills independently of its user config.
|
|
profile_parent = (args.profile_parent or source_auth_home.parent / '.cache' / 'rea-agent-trials').resolve()
|
|
if any(profile_parent.is_relative_to(root.resolve()) for root in [repository, driver_repository]):
|
|
raise ValueError('--profile-parent must be outside the runtime and driver checkouts')
|
|
profile_parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
profile = Path(tempfile.mkdtemp(prefix='profile-', dir=profile_parent))
|
|
trial_codex_home = profile / 'codex'
|
|
account = profile / 'account'
|
|
workspace = profile / 'workspace'
|
|
for directory in [trial_codex_home, account, workspace]:
|
|
directory.mkdir(mode=0o700)
|
|
auth = trial_codex_home / 'auth.json'
|
|
started = time.monotonic()
|
|
timeout = False
|
|
exit_code = None
|
|
environment = dict(os.environ)
|
|
environment.update({
|
|
'CODEX_HOME': str(trial_codex_home), 'HOME': str(account),
|
|
'XDG_CONFIG_HOME': str(account / '.config'),
|
|
'XDG_CACHE_HOME': str(account / '.cache'),
|
|
'XDG_DATA_HOME': str(account / '.local/share'),
|
|
'NODE_OPTIONS': '--max-old-space-size=512',
|
|
})
|
|
target_digest = file_digest(Path(args.artifact_path)) if args.artifact_path else None
|
|
command_args = [str(repository / 'scripts/rea.mjs'), 'mcp']
|
|
instruction = (
|
|
'Use the registered REA MCP server for this investigation. Static analysis '
|
|
'only: do not execute target code, fetch URLs, launch a GUI, or modify targets. '
|
|
'Preserve observations, derivations, inferences, limitations, and real Evidence IDs. '
|
|
'An Evidence value has evidence_id, normalized_result and limitations; '
|
|
'non-Evidence session values have result. '
|
|
'The analyzed target digest is Evidence.subject.digest.sha256. '
|
|
'normalized_result.engine.artifact_sha256 identifies the analysis engine '
|
|
'artifact and must never be substituted for the target digest. '
|
|
'Prefer summary/focused views where appropriate. Do not invent successful facts '
|
|
'after a tool error. No other agents. Stop once the requested facts are supported.'
|
|
)
|
|
if args.authority == 'browser':
|
|
instruction = (
|
|
'Use the registered REA tools for the explicitly selected public browser scenario. '
|
|
'Launching the selected headless browser with a fresh empty profile and observing '
|
|
'the supplied public URL is authorized. Keep all actions within that scenario. '
|
|
'No injected code, credentials, unrelated targets, shell/file tools or other agents. '
|
|
'Preserve real Evidence IDs, observation completeness, failed captures and limitations. '
|
|
'Do not infer original source or global behavior from one observed session.'
|
|
)
|
|
if args.mode == 'client-code':
|
|
instruction += (
|
|
' Use the client JavaScript executor to compose registered MCP calls when '
|
|
'possible and select the requested normalized_result fields before returning '
|
|
'intermediate output to model context. Keep real Evidence IDs and limitations.'
|
|
)
|
|
if args.guide_enabled_tools:
|
|
instruction += (
|
|
' Configured REA tool function names for this trial: '
|
|
+ json.dumps(['mcp__rea__' + name for name in enabled_tools])
|
|
+ '. For discovery select only these exact names from ALL_TOOLS and '
|
|
'return only their metadata. Preserve the original analyst question, '
|
|
'dependency order and source Evidence.'
|
|
)
|
|
if args.catalog_load:
|
|
instruction = (
|
|
'This is a catalog loading measurement. Do not call any tools, search, '
|
|
'or delegate. Reply with exactly the text requested by the user.'
|
|
)
|
|
quote = json.dumps
|
|
# A disposable CODEX_HOME does not disable account-connected apps. Keep
|
|
# the requested feature policy identical in the config and its receipt.
|
|
features = {
|
|
'multi_agent': False, 'shell_snapshot': False, 'shell_tool': False,
|
|
'unified_exec': False, 'code_mode_host': args.mode == 'client-code',
|
|
'apps': args.include_account_tools, 'plugins': args.include_account_tools,
|
|
}
|
|
config = '\n'.join([
|
|
'approval_policy = "never"', 'sandbox_mode = "read-only"',
|
|
'web_search = "disabled"', 'model = ' + quote(args.model),
|
|
'model_reasoning_effort = ' + quote(args.reasoning),
|
|
'developer_instructions = ' + quote(instruction),
|
|
'[features]',
|
|
*[name + ' = ' + str(enabled).lower() for name, enabled in features.items()],
|
|
'[history]', 'persistence = "none"',
|
|
]) + '\n'
|
|
if not args.no_mcp:
|
|
config += '\n'.join([
|
|
'[mcp_servers.rea]', 'required = true', 'command = ' + quote(node),
|
|
'args = ' + quote(command_args), 'startup_timeout_sec = 60',
|
|
'tool_timeout_sec = 600',
|
|
*([] if enabled_tools is None else ['enabled_tools = ' + quote(enabled_tools)]),
|
|
'[mcp_servers.rea.env]',
|
|
*[key + ' = ' + quote(value) for key, value in provider_env.items()],
|
|
]) + '\n'
|
|
# MCP read-only hints cover normal static tools. Explicitly authorize only
|
|
# the session-opening primitive needed by the requested native tasks.
|
|
approved_tools = ['open_binary', 'analyze_javascript_application', 'inspect_analysis_view',
|
|
'inspect_android_package', 'search_android_classes',
|
|
'inspect_android_class', 'inspect_android_method',
|
|
'inspect_firmware_regions', 'inspect_managed_artifact',
|
|
'inspect_sqlite_database', 'inspect_pcap_capture',
|
|
'inspect_web_network_capture',
|
|
'inspect_packet_capture', 'inspect_packet_fields',
|
|
'decode_signal_recording', 'inspect_signal_decoders',
|
|
'inspect_managed_members', 'inspect_managed_native_boundaries',
|
|
'inspect_artifact', 'inspect_wasm_artifact',
|
|
'compare_functions', 'build_call_path', 'compare_artifacts',
|
|
'compare_process_captures', 'find_changed_behavior',
|
|
'compare_managed_members', 'project_managed_application_graph',
|
|
'compare_wasm_interfaces', 'project_apple_application_graph',
|
|
'project_android_application_graph', 'project_harmony_application_graph',
|
|
'trace_application_feature', 'trace_javascript_semantics',
|
|
'compare_application_versions', 'compare_javascript_export_shapes',
|
|
'reconcile_javascript_runtime',
|
|
'inspect_evm_interface', 'get_evidence_bundle', 'export_evidence_bundle',
|
|
'import_evidence_bundle', 'list_unknowns', 'verify_unknown_resolution',
|
|
'record_unknown', 'update_unknown', 'reset_investigation', 'binary_session',
|
|
'capture_browser_scenario',
|
|
'resolve_containing_procedure', 'analyze_function', 'close_binary']
|
|
if not args.no_mcp:
|
|
for tool in approved_tools:
|
|
config += '[mcp_servers.rea.tools.' + tool + ']\napproval_mode = "approve"\n'
|
|
try:
|
|
if source_auth.exists():
|
|
shutil.copyfile(source_auth, auth)
|
|
auth.chmod(0o600)
|
|
(trial_codex_home / 'config.toml').write_text(config)
|
|
(trial_codex_home / 'config.toml').chmod(0o600)
|
|
prompt = args.prompt.read_text()
|
|
template_digest = hashlib.sha256(prompt.encode()).hexdigest()
|
|
if '{{artifact_path}}' in prompt:
|
|
if not args.artifact_path or not os.path.isabs(args.artifact_path):
|
|
raise ValueError('A question template requires an absolute --artifact-path')
|
|
prompt = prompt.replace('{{artifact_path}}', args.artifact_path)
|
|
elif args.artifact_path:
|
|
raise ValueError('--artifact-path requires the literal template marker')
|
|
(args.output / 'prompt.txt').write_text(prompt)
|
|
(args.output / 'trial-settings.json').write_text(json.dumps({
|
|
'mode': args.mode, 'model': args.model, 'reasoning': args.reasoning,
|
|
'capture_tool_context': args.capture_tool_context,
|
|
'guide_enabled_tools': args.guide_enabled_tools,
|
|
'catalog_load': args.catalog_load, 'mcp_enabled': not args.no_mcp,
|
|
'driver_commit': subprocess.check_output(
|
|
['git', 'rev-parse', 'HEAD'], cwd=driver_repository, text=True).strip(),
|
|
'runtime_revision': args.revision,
|
|
'runtime_root': str(repository),
|
|
'workspace': str(workspace),
|
|
'workspace_isolated_from_runtime': not workspace.is_relative_to(repository.resolve()),
|
|
'selected_artifact_path': args.artifact_path,
|
|
'selected_artifact_sha256_before': target_digest,
|
|
'enabled_tools': enabled_tools,
|
|
'authority': args.authority,
|
|
'runtime_files_sha256': runtime_digest,
|
|
'runtime_dispatcher_sha256': file_digest(repository / 'scripts/rea.mjs'),
|
|
'package_lock_sha256': file_digest(repository / 'package-lock.json'),
|
|
'experiment_sources_sha256': {
|
|
name: hashlib.sha256((experiment / name).read_bytes()).hexdigest()
|
|
for name in ['agent-trial.py', 'codex_context.py', 'runtime_identity.py', 'provider-environment.json']
|
|
},
|
|
'developer_instructions': instruction,
|
|
'developer_instructions_sha256': hashlib.sha256(instruction.encode()).hexdigest(),
|
|
'prompt_sha256': hashlib.sha256(prompt.encode()).hexdigest(),
|
|
'question_template_sha256': template_digest,
|
|
'provider_settings_sha256': hashlib.sha256(
|
|
json.dumps(provider_env, sort_keys=True).encode()).hexdigest(),
|
|
'provider_search_path_sha256': hashlib.sha256(
|
|
provider_env.get('PATH', os.environ.get('PATH', '')).encode()).hexdigest(),
|
|
'provider_environment_policy_sha256': hashlib.sha256(
|
|
provider_policy_text.encode()).hexdigest(),
|
|
'features': features,
|
|
}, indent=2) + '\n')
|
|
invocation = [args.codex, '--no-daemon', 'exec', '--strict-config',
|
|
*([] if args.capture_tool_context else ['--ephemeral']),
|
|
'--skip-git-repo-check', '--ignore-rules', '--json',
|
|
'--color', 'never', '-C', str(workspace),
|
|
'--output-last-message', str(args.output.resolve() / 'answer.txt'), '-']
|
|
with (args.output / 'events.jsonl').open('w') as events, (args.output / 'stderr.log').open('w') as errors:
|
|
child = subprocess.Popen(invocation, env=environment, cwd=workspace,
|
|
stdin=subprocess.PIPE, stdout=events, stderr=errors, text=True)
|
|
child.stdin.write(prompt)
|
|
child.stdin.close()
|
|
try:
|
|
exit_code = child.wait(timeout=args.timeout)
|
|
except subprocess.TimeoutExpired:
|
|
timeout = True
|
|
child.terminate()
|
|
try:
|
|
exit_code = child.wait(timeout=10)
|
|
except subprocess.TimeoutExpired:
|
|
child.kill()
|
|
exit_code = child.wait()
|
|
usage = []
|
|
tools = []
|
|
event_types = {}
|
|
with (args.output / 'events.jsonl').open() as event_file:
|
|
for line in event_file:
|
|
if not line.strip():
|
|
continue
|
|
event = json.loads(line)
|
|
kind = event.get('type', 'unknown')
|
|
event_types[kind] = event_types.get(kind, 0) + 1
|
|
if 'usage' in event:
|
|
usage.append(event['usage'])
|
|
item = event.get('item', {})
|
|
if item.get('type') == 'mcp_tool_call' and kind == 'item.completed':
|
|
tools.append({
|
|
**{k: item.get(k) for k in ['server', 'tool', 'arguments', 'status', 'error']},
|
|
'codex_event_result_json_utf8_bytes': len(json.dumps(
|
|
item.get('result'), separators=(',', ':'), ensure_ascii=False
|
|
).encode()),
|
|
})
|
|
final_target_digest = file_digest(Path(args.artifact_path)) if args.artifact_path else None
|
|
try:
|
|
_, final_runtime_digest = runtime_fingerprint(repository)
|
|
except OSError:
|
|
final_runtime_digest = None
|
|
summary = {
|
|
'mode': args.mode, 'requested_model': args.model, 'reasoning': args.reasoning,
|
|
'runtime_revision': args.revision,
|
|
'runtime_files_sha256_before': runtime_digest,
|
|
'runtime_files_sha256_after': final_runtime_digest,
|
|
'runtime_bytes_unchanged': runtime_digest == final_runtime_digest,
|
|
'catalog_load': args.catalog_load, 'mcp_enabled': not args.no_mcp,
|
|
'capture_tool_context_requested': args.capture_tool_context,
|
|
'selected_artifact_sha256_before': target_digest,
|
|
'selected_artifact_sha256_after': final_target_digest,
|
|
'selected_artifact_bytes_unchanged': target_digest == final_target_digest if target_digest else None,
|
|
'model_identity_scope': 'requested alias; resolved server-side model identity is not inferred',
|
|
'codex_version': subprocess.check_output([args.codex, '--version'], text=True).strip(),
|
|
'exit_code': exit_code, 'timeout': timeout,
|
|
'wall_seconds': round(time.monotonic() - started, 3),
|
|
'usage': usage, 'completed_mcp_calls': tools, 'event_types': event_types,
|
|
'factual_correctness': 'not_assessed',
|
|
'scope': 'real_cli_trial_requires_transcript_and_fact_review',
|
|
}
|
|
(args.output / 'summary.json').write_text(json.dumps(summary, indent=2) + '\n')
|
|
print(json.dumps(summary))
|
|
finally:
|
|
try:
|
|
if args.capture_tool_context:
|
|
capture_tool_context(trial_codex_home, args.output)
|
|
finally:
|
|
# Capture failure must never prevent copied-credential cleanup.
|
|
shutil.rmtree(profile)
|
|
(args.output / 'cleanup.json').write_text(json.dumps({'profile_removed': not profile.exists()}) + '\n')
|
|
raise SystemExit(124 if timeout else 125 if final_runtime_digest != runtime_digest else exit_code)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|