From c23e16162914e6bffc9549959c3faef0a66f6027 Mon Sep 17 00:00:00 2001 From: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:09:28 +0800 Subject: [PATCH] perf(mcp): record catalog identities for all schema profiles (#5415) Share compiled schema conversion while recording all four built-in catalog profiles. Preserve canonical hashes, output validation and custom-budget fallback. Refs #723. Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> --- .../analysis-efficiency/mcp-profile-costs.py | 74 ++ .../mcp-profile-digest.mjs | 25 + .../mcp-profile-identity.json | 808 ++++++++++++++++++ .../mcp-profile-identity.md | 131 +++ .../mcp-profile-identity.py | 199 +++++ scripts/record-catalog-digests.mjs | 19 +- src/catalogIdentity.ts | 108 ++- src/recordedCatalogDigests.ts | 8 +- tests/boundary/mcp/mcpDoctor.test.ts | 36 + tests/boundary/mcp/serverIdentity.test.ts | 42 +- 10 files changed, 1395 insertions(+), 55 deletions(-) create mode 100644 experiments/analysis-efficiency/mcp-profile-costs.py create mode 100644 experiments/analysis-efficiency/mcp-profile-digest.mjs create mode 100644 experiments/analysis-efficiency/mcp-profile-identity.json create mode 100644 experiments/analysis-efficiency/mcp-profile-identity.md create mode 100644 experiments/analysis-efficiency/mcp-profile-identity.py diff --git a/experiments/analysis-efficiency/mcp-profile-costs.py b/experiments/analysis-efficiency/mcp-profile-costs.py new file mode 100644 index 0000000000..64cb1292d0 --- /dev/null +++ b/experiments/analysis-efficiency/mcp-profile-costs.py @@ -0,0 +1,74 @@ +"""Summarize recorded Codex discovery tasks; tiktoken==0.12.0 is a proxy only.""" +import argparse +import hashlib +import json +from pathlib import Path + +import tiktoken + + +def compact(value): + return json.dumps(value, ensure_ascii=False, separators=(",", ":"), allow_nan=False) + + +def inspect(path, encoding): + report = json.loads((path / "report.json").read_text()) + catalog = json.loads((path / "catalog.json").read_text()) + transcript = json.loads((path / "transcript.json").read_text()) + log = path.with_suffix(".log") + scope = next(json.loads(line) for line in reversed(log.read_text().splitlines()) + if line.startswith('{"resource_scope":')) + observed = scope.get("payload_resource_observation") or {} + completed = (report["status"] == "passed" and not report["cleanup_failures"] + and report["appserver_exit_code"] == 0 and scope["abort"] is None + and scope.get("payload_resource_observation_verified") is True + and observed.get("payload_cleanup_verified") is True + and observed.get("payload_exit_code") == 0) + tools = list(catalog["data"][0]["tools"].values()) + messages = [event["message"] for event in transcript] + ids = {m["id"] for m in messages if m.get("method") == "mcpServer/tool/call"} + replies = [m["result"] for m in messages if m.get("id") in ids and "result" in m] + arguments = [m["params"] for m in messages if m.get("method") == "mcpServer/tool/call"] + count = lambda text: len(encoding.encode(text, disallowed_special=())) + input_catalog = count(compact([{k: t[k] for k in ["name", "description", "inputSchema"]} for t in tools])) + argument_tokens = sum(count(compact(a)) for a in arguments) + text_tokens = sum(count(c["text"]) for r in replies for c in r["content"] if c["type"] == "text") + return { + "recording": path.name, "completed_with_verified_cleanup": completed, + "input_profile": report["input_profile"], "output_profile": report["output_profile"], + "catalog": report["catalog"], "calls": report["calls"], + # Codex returns tools as a map; JSON member order varies by process. + "catalog_semantic_sha256": hashlib.sha256(json.dumps( + catalog, ensure_ascii=False, sort_keys=True, separators=(",", ":"), + allow_nan=False).encode()).hexdigest(), + "digests": [r["structuredContent"]["result"]["server_identity"]["catalog"]["digests"] for r in replies], + "token_proxy": {"input_facing_catalog": input_catalog, "arguments": argument_tokens, + "text_replies": text_tokens, "combined": input_catalog + argument_tokens + text_tokens, + "complete_discovered_catalog": count(compact(catalog))}, + "wall_seconds": scope["elapsed_seconds"], + "cpu_seconds": observed["cpu_stat"]["usage_usec"] / 1e6, + "kernel_family_peak_bytes_including_file_cache": observed.get("memory_peak_bytes"), + "observed_resource_events": observed.get("resource_events"), + "final_whole_scope_events": scope.get("resource_events_final"), + "sha256": {p.name: hashlib.sha256(p.read_bytes()).hexdigest() + for p in [path / "report.json", path / "catalog.json", path / "transcript.json", log]}, + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("cases", type=Path, nargs="+") + args = parser.parse_args() + encoding = tiktoken.get_encoding("cl100k_base") + records = [inspect(path, encoding) for path in args.cases] + result = {"tiktoken": tiktoken.__version__, "encoding": encoding.name, + "actual_model_usage": None, + "unmeasured": ["actual host model schema injection and rendering", "model input/output/cached tokens", "model reasoning and application investigation"], + "recordings": records} + args.output.write_text(json.dumps(result, indent=2) + "\n") + assert all(r["completed_with_verified_cleanup"] for r in records) + + +if __name__ == "__main__": + main() diff --git a/experiments/analysis-efficiency/mcp-profile-digest.mjs b/experiments/analysis-efficiency/mcp-profile-digest.mjs new file mode 100644 index 0000000000..ed780af903 --- /dev/null +++ b/experiments/analysis-efficiency/mcp-profile-digest.mjs @@ -0,0 +1,25 @@ +import { pathToFileURL } from "node:url"; +import { resolve } from "node:path"; +const [root, input, output] = process.argv.slice(2); +const start = performance.now(); +const { createCatalogIdentity } = await import( + pathToFileURL(resolve(root, "dist/catalogIdentity.js")) +); +const imported = performance.now(); +const cpu = process.cpuUsage(); +const identity = createCatalogIdentity( + input === "compact" ? { budgetBytes: 13 * 1024 } : undefined, + output, +); +const digests = identity.digests; +const end = performance.now(); +console.log( + JSON.stringify({ + input, + output, + import_ms: imported - start, + digest_ms: end - imported, + digest_cpu_us: process.cpuUsage(cpu), + digests, + }), +); diff --git a/experiments/analysis-efficiency/mcp-profile-identity.json b/experiments/analysis-efficiency/mcp-profile-identity.json new file mode 100644 index 0000000000..742d3b74d3 --- /dev/null +++ b/experiments/analysis-efficiency/mcp-profile-identity.json @@ -0,0 +1,808 @@ +{ + "issue": 723, + "source_base": "d532b18123916a9e88f98c7880ba4f984e23d715", + "runtimes": { + "baseline-runtime-d532b1812": { + "source_revision": "d532b18123916a9e88f98c7880ba4f984e23d715", + "archive_sha256": "79832e2571112400892ec1aae429b030de38c8c672a8acfa6680a2568362fed3", + "manifest_sha256": "89cbc1688e4702a47b9a5cc66ac38d18de46b74eb550e44227f0ec2119c5e7af" + }, + "candidate-runtime-1": { + "source_base": "d532b18123916a9e88f98c7880ba4f984e23d715", + "patch_sha256": "cfb0eee1ee7f3ceecff7cb77ac63e569dfcfc020e4252cb11869644628f4f743", + "archive_sha256": "7e4275b1db784545335a4eab373a3db11b1eb5cc6bac7611521a698b0cffce56", + "manifest_sha256": "356b3baa153650b330cea6bc7e28163565f3201b8d80aa2d6bb69f66faa8705a" + } + }, + "node": "24.18.0", + "npm": "11.16.0", + "mcp_sdk": "2.3.1", + "client": "Codex app-server 0.163.0", + "negotiated_protocol": "2025-06-18", + "model_turns_started": 0, + "actual_model_usage": null, + "task": "Complete 209-tool discovery, then verify no target open, no cleanup pending, no bound provider, package version and tool count twice.", + "run_order": [ + "baseline-d532-1", + "candidate-1-1 (reversed profiles)", + "baseline-d532-2 (reversed profiles)", + "candidate-1-2" + ], + "resource_bounds": { + "cpu_affinity": [15], + "cpu_quota_percent": 100, + "hard_memory_gib": 4, + "early_stop_gib": 3, + "node_heap_gib": 2, + "swap": 0, + "task_ceiling": 192, + "accounting": "Kernel cgroup process-family peak including file cache, Codex, controller and server; CPU counters at payload exit with only observer alive. Final whole-scope counters unavailable.", + "oom_events": 0, + "verified_payload_cleanup_recordings": 16 + }, + "tiktoken": "0.12.0", + "encoding": "cl100k_base", + "token_proxy_limit": "Input-facing catalog + RPC params + text result serialization; no model turn/usage, schema injection, prompts or reasoning measured. Identity paths/timestamps cause small reply token differences.", + "scripts_sha256": { + "codex-host-probe.py": "d0da9b39e0343b5e84f05328f56df3e82b712fd3fb32ee375cf3735a448fb80e", + "digest-probe.mjs": "e2f8950014bb40806aef147196ca5d432c6400941d3901e1c0a0e78ce5266bcf", + "build-digest-probe.mjs": "46060b75c014ca39f9efe317a0445a0220ea7166a08dbdacf04706670e958e6d", + "run-cohort.py": "1cc5db864b4fe1b2d657ddfe72d468d9e72bddf5118b0bf10c9fce344ec45e9c" + }, + "profile_digest_oracle": { + "full": { + "full": { + "tools_sha256": "c3e7950408c9005ae3ae0633966609bf1c0f72a36392bc0e0c0fdfe9ea584ae6", + "prompts_sha256": "7152916ab48bd2a9677a5ad9cdbde0a51dc53c113934952ad096e93f5f18aa7e", + "combined_sha256": "e0b2715af8312fb565f14d75296adc947b94252e53643f08d831ca8fc8aee941" + }, + "off": { + "tools_sha256": "5ad649af53472a1d789b26997cb26a4e8d5d1f2a97c587ad1a6b16030ef5d764", + "prompts_sha256": "7152916ab48bd2a9677a5ad9cdbde0a51dc53c113934952ad096e93f5f18aa7e", + "combined_sha256": "cd84cff526daa360e30e299002e7d14427fde9c2b00f59f622fccf89ff2535c7" + } + }, + "compact": { + "full": { + "tools_sha256": "774c1b7c2c2f57beea9bed62171f046da0f17f2b3bbd7900a90855ecdd26ba79", + "prompts_sha256": "7152916ab48bd2a9677a5ad9cdbde0a51dc53c113934952ad096e93f5f18aa7e", + "combined_sha256": "2cbc9ae351fe4f1204a4c2b5e2e92fff8a2ecd87bc14c7ef068d6691c17dff15" + }, + "off": { + "tools_sha256": "efe7a17bad5617171162414df88529ff54807b870b5d510e2838ce18e2ba8bd8", + "prompts_sha256": "7152916ab48bd2a9677a5ad9cdbde0a51dc53c113934952ad096e93f5f18aa7e", + "combined_sha256": "07d379697201b70d45579d0811e137a0df088be88498fc9fae6806adba26c976" + } + } + }, + "recordings": [ + { + "recording": "baseline-d532-1-full-full", + "verified_complete": true, + "profiles": ["full", "full"], + "catalog_tools": 209, + "catalog_bytes": 3322571, + "catalog_semantic_sha256": "799cc997a9573b5c3dfce3ac22e1fb5eb1637e81b489392f37cc81767b334359", + "discovery_ms": 5074.2612921167165, + "first_status_ms": 1699.8079400509596, + "warm_status_ms": 981.8687399383634, + "wall_seconds": 8.919, + "cpu_seconds": 8.480822, + "kernel_family_peak_bytes": 802246656, + "token_proxy": { + "input_facing_catalog": 187005, + "arguments": 86, + "text_replies": 22702, + "combined": 209793, + "complete_discovered_catalog": 845872 + }, + "raw_sha256": { + "report.json": "f36f889ab241f704977241443ad55176de6607805f6c82013a287e968dbb428d", + "catalog.json": "ad5dcfb828d4524f97b494884deed42389ab3e29e2f7839a12b5e95bd1c50ff1", + "transcript.json": "7d019f379100f45ff1c3f55e66d79e233f7b736886be1823da11e4b843d5fdc1", + "baseline-d532-1-full-full.log": "3709d55fe5412f49ee880bfee4be818711c977b026ad3133a3f8abb91111f9a7" + } + }, + { + "recording": "baseline-d532-2-full-full", + "verified_complete": true, + "profiles": ["full", "full"], + "catalog_tools": 209, + "catalog_bytes": 3322571, + "catalog_semantic_sha256": "799cc997a9573b5c3dfce3ac22e1fb5eb1637e81b489392f37cc81767b334359", + "discovery_ms": 5847.158181015402, + "first_status_ms": 1123.013640055433, + "warm_status_ms": 911.4040599670261, + "wall_seconds": 8.905, + "cpu_seconds": 8.452489, + "kernel_family_peak_bytes": 803254272, + "token_proxy": { + "input_facing_catalog": 187004, + "arguments": 86, + "text_replies": 22702, + "combined": 209792, + "complete_discovered_catalog": 845872 + }, + "raw_sha256": { + "report.json": "b85792959c6838010ebd622d1cf213d2eb1cd9562da14fc49a2555bda03a0eee", + "catalog.json": "f588658a13bc1b559008508954f2ce7f34815b326686b4c167c0c27b521cad1b", + "transcript.json": "5326f37ffdeae98ad4679cf53f462a41fddd71eb9891040753e716489dcc1b68", + "baseline-d532-2-full-full.log": "147000d226f6cfb3450af51ade419b20c6b0b7d7409c22e876c98830762b39eb" + } + }, + { + "recording": "baseline-d532-1-compact-full", + "verified_complete": true, + "profiles": ["compact", "full"], + "catalog_tools": 209, + "catalog_bytes": 2723927, + "catalog_semantic_sha256": "d18f02920b84d39b90dc488db89a8f403f6ec35139a39a3329375a18078af1ea", + "discovery_ms": 11450.75011299923, + "first_status_ms": 4571.937991073355, + "warm_status_ms": 1836.9912200141698, + "wall_seconds": 18.99, + "cpu_seconds": 16.522676, + "kernel_family_peak_bytes": 808927232, + "token_proxy": { + "input_facing_catalog": 43507, + "arguments": 82, + "text_replies": 22714, + "combined": 66303, + "complete_discovered_catalog": 702182 + }, + "raw_sha256": { + "report.json": "b27141e7af05b3dd43930c0df30f4872709d8db72cc953478150084e99e032f4", + "catalog.json": "8ff7b639d3409b52761d483283c674e3670e42169ddfda0c1e2a87a207c4dcbd", + "transcript.json": "1f91daa9703495ca6519459124e9564f0c96f5542b61563bf10029a1f6337926", + "baseline-d532-1-compact-full.log": "fb187f2b9255e4fc1ecd2c15f9df3c7f1efc76aaba528589f5eed81e7e80e835" + } + }, + { + "recording": "baseline-d532-2-compact-full", + "verified_complete": true, + "profiles": ["compact", "full"], + "catalog_tools": 209, + "catalog_bytes": 2723927, + "catalog_semantic_sha256": "d18f02920b84d39b90dc488db89a8f403f6ec35139a39a3329375a18078af1ea", + "discovery_ms": 5926.411310909316, + "first_status_ms": 2899.308810941875, + "warm_status_ms": 1365.2960499748588, + "wall_seconds": 11.148, + "cpu_seconds": 10.613801, + "kernel_family_peak_bytes": 816377856, + "token_proxy": { + "input_facing_catalog": 43507, + "arguments": 82, + "text_replies": 22714, + "combined": 66303, + "complete_discovered_catalog": 702182 + }, + "raw_sha256": { + "report.json": "df856c2a486c513923b3c67db30bc60ca28ba3b1d0e3418d28d9d6429eca9366", + "catalog.json": "407ff33fd8e57f2842b76ce190c2089930add6c9a4f618cc90320045f6a67ed2", + "transcript.json": "d11dfb4c28c5355e733943e3504d3b4e11f056f0f7625e835e5cdc916a451716", + "baseline-d532-2-compact-full.log": "0485a806e815ecb91862d37cb9200b7cb0877a3d8c69882e8299100d13769a57" + } + }, + { + "recording": "baseline-d532-1-full-off", + "verified_complete": true, + "profiles": ["full", "off"], + "catalog_tools": 209, + "catalog_bytes": 832191, + "catalog_semantic_sha256": "e537c521b5dd0ac369dee706815579d9d7803497a8fc7f7d683fec267042b81c", + "discovery_ms": 11546.154452953488, + "first_status_ms": 4629.906311165541, + "warm_status_ms": 2057.3992100544274, + "wall_seconds": 19.588, + "cpu_seconds": 16.963204, + "kernel_family_peak_bytes": 709857280, + "token_proxy": { + "input_facing_catalog": 187005, + "arguments": 86, + "text_replies": 22710, + "combined": 209801, + "complete_discovered_catalog": 195042 + }, + "raw_sha256": { + "report.json": "4963559eafd97d6aa72e2265651d9f6d3d0cd3a04965677d6fdedfe0378e7c11", + "catalog.json": "aef1822a088446790e14373a6b9b1039cda640598d19620fdd776f6d7c49d890", + "transcript.json": "203f7f00eca7617cdaa8c7fd45fdde3d4502cd29cf36ee0c724181ad3178410b", + "baseline-d532-1-full-off.log": "84e5a67f6f1e97fb3c429d96d7b3de33e08eddc56b21b453fa2b3abf19dbd15b" + } + }, + { + "recording": "baseline-d532-2-full-off", + "verified_complete": true, + "profiles": ["full", "off"], + "catalog_tools": 209, + "catalog_bytes": 832191, + "catalog_semantic_sha256": "e537c521b5dd0ac369dee706815579d9d7803497a8fc7f7d683fec267042b81c", + "discovery_ms": 5198.2202511280775, + "first_status_ms": 2726.173529867083, + "warm_status_ms": 1434.7798710223287, + "wall_seconds": 10.325, + "cpu_seconds": 9.75455, + "kernel_family_peak_bytes": 703328256, + "token_proxy": { + "input_facing_catalog": 187005, + "arguments": 86, + "text_replies": 22710, + "combined": 209801, + "complete_discovered_catalog": 195042 + }, + "raw_sha256": { + "report.json": "e603556898f606d6804c73a227ef6d809f09f516316255b316e5afb473170ff9", + "catalog.json": "8200cccc7a0d7bdcf288cab0225269acadbd532aed69da7d7e8a3b4cf34b9c16", + "transcript.json": "4d2a862a24282cac2581a36ca46fd99474835cba247203556c04015ea02096fe", + "baseline-d532-2-full-off.log": "90b720806f3f5756408501a1f9d5dd724108647706e38aca10cbc0a35e56a37a" + } + }, + { + "recording": "baseline-d532-1-compact-off", + "verified_complete": true, + "profiles": ["compact", "off"], + "catalog_tools": 209, + "catalog_bytes": 233547, + "catalog_semantic_sha256": "0b81bdd81251ddd527a9c9ecbb5e6048e9c8a3be1f6195465a803d206f03fe73", + "discovery_ms": 12976.338863139972, + "first_status_ms": 6152.441991027445, + "warm_status_ms": 3121.8184099998325, + "wall_seconds": 23.47, + "cpu_seconds": 19.234341, + "kernel_family_peak_bytes": 665423872, + "token_proxy": { + "input_facing_catalog": 43507, + "arguments": 82, + "text_replies": 22690, + "combined": 66279, + "complete_discovered_catalog": 51352 + }, + "raw_sha256": { + "report.json": "c3c21635cf212eba3902ad64f956244ddfe122425fa3b39ec280b5296ebe23c6", + "catalog.json": "e5a4444ea91913e68705a6bd6afee2b8ea0a43b2f91f8628ada597e57afab5db", + "transcript.json": "58df156c85a8b1ec325945db08e4e84e8174dfd739f48e4d83b2a3537f7a52b9", + "baseline-d532-1-compact-off.log": "8d5b2ddcf5b31ad4635269837e217dd2242a91d82ea961d01e2ab3a2076b9e35" + } + }, + { + "recording": "baseline-d532-2-compact-off", + "verified_complete": true, + "profiles": ["compact", "off"], + "catalog_tools": 209, + "catalog_bytes": 233547, + "catalog_semantic_sha256": "0b81bdd81251ddd527a9c9ecbb5e6048e9c8a3be1f6195465a803d206f03fe73", + "discovery_ms": 5907.094460912049, + "first_status_ms": 2939.780990127474, + "warm_status_ms": 1040.7937010750175, + "wall_seconds": 10.742, + "cpu_seconds": 10.176561, + "kernel_family_peak_bytes": 667668480, + "token_proxy": { + "input_facing_catalog": 43507, + "arguments": 82, + "text_replies": 22690, + "combined": 66279, + "complete_discovered_catalog": 51352 + }, + "raw_sha256": { + "report.json": "14aed217fcad50915b211e4ee9894ebf9cb4abd8d4c4a4e007cc6fe2b397d44a", + "catalog.json": "e55a5b2cba59b09ac4f63831c2af6682d162bf5b05259b3c0f9db8eb2bc67e44", + "transcript.json": "e9a9ac8b657da35501e577bf1d1e18c89923c5bcdbb1307c8254d512bbdc7b53", + "baseline-d532-2-compact-off.log": "e7c0c68bd1088f11107a6713df1efb74f01ff3479d1913977fe22935814c912c" + } + }, + { + "recording": "candidate-1-1-full-full", + "verified_complete": true, + "profiles": ["full", "full"], + "catalog_tools": 209, + "catalog_bytes": 3322571, + "catalog_semantic_sha256": "799cc997a9573b5c3dfce3ac22e1fb5eb1637e81b489392f37cc81767b334359", + "discovery_ms": 7035.046560922638, + "first_status_ms": 1280.1523089874536, + "warm_status_ms": 962.2103709261864, + "wall_seconds": 10.309, + "cpu_seconds": 9.82015, + "kernel_family_peak_bytes": 806117376, + "token_proxy": { + "input_facing_catalog": 187004, + "arguments": 84, + "text_replies": 22690, + "combined": 209778, + "complete_discovered_catalog": 845872 + }, + "raw_sha256": { + "report.json": "4e6d4cb20d4b435b47164535930c60902eed44c4efec0665d9eeb2bd0635fd39", + "catalog.json": "f0700961a2a999671107dc5e5274098a5d97047e834654c7c3c7ee0c420a91b5", + "transcript.json": "4d0b64606c913325ce83bd797b2286d0e9758f09c526dacfce422756cf5212d8", + "candidate-1-1-full-full.log": "9d37d44776cfd9f41811e0a608c79ed63257383218e01f602317c2dc5a9de10f" + } + }, + { + "recording": "candidate-1-2-full-full", + "verified_complete": true, + "profiles": ["full", "full"], + "catalog_tools": 209, + "catalog_bytes": 3322571, + "catalog_semantic_sha256": "799cc997a9573b5c3dfce3ac22e1fb5eb1637e81b489392f37cc81767b334359", + "discovery_ms": 5431.887832004577, + "first_status_ms": 1290.1408409234136, + "warm_status_ms": 1058.6841800250113, + "wall_seconds": 8.819, + "cpu_seconds": 8.06667, + "kernel_family_peak_bytes": 801071104, + "token_proxy": { + "input_facing_catalog": 187004, + "arguments": 78, + "text_replies": 22690, + "combined": 209772, + "complete_discovered_catalog": 845872 + }, + "raw_sha256": { + "report.json": "e2853484e579376a8edb168f84ad2add2ffdd30ce78e123b2dfcdd17d524344e", + "catalog.json": "5c918daeb3f8b10625927a87f3f95367a2a2823cc7c098aa8ad3d8d5b47aa6ad", + "transcript.json": "1e22387d227134741eac842b8000c5bb73a42e2368c3c53e21bc1d45c17ed819", + "candidate-1-2-full-full.log": "877a3bdfe67875bb5ab1981850da2fe519428708184eef55a28c5fa18bb727ea" + } + }, + { + "recording": "candidate-1-1-compact-full", + "verified_complete": true, + "profiles": ["compact", "full"], + "catalog_tools": 209, + "catalog_bytes": 2723927, + "catalog_semantic_sha256": "d18f02920b84d39b90dc488db89a8f403f6ec35139a39a3329375a18078af1ea", + "discovery_ms": 5128.74938105233, + "first_status_ms": 1351.6553710214794, + "warm_status_ms": 1321.080510970205, + "wall_seconds": 8.815, + "cpu_seconds": 8.478922, + "kernel_family_peak_bytes": 781271040, + "token_proxy": { + "input_facing_catalog": 43507, + "arguments": 80, + "text_replies": 22702, + "combined": 66289, + "complete_discovered_catalog": 702182 + }, + "raw_sha256": { + "report.json": "ee38fcca794a959271af78a6c65377f411fdbe3b7b97068835b36586ec804ddf", + "catalog.json": "db8f1cbaa97c09672537d87ac48457bdb599ae439bca2d8a9216a032edbbdd88", + "transcript.json": "22d7a8b00909ad82e40ca7beb21d793bff61dacd88ba0d345ef82eeacf177516", + "candidate-1-1-compact-full.log": "ae17a479c9f094e522e15b58617302bf098464a4f55cea21cc913d18ba02afd2" + } + }, + { + "recording": "candidate-1-2-compact-full", + "verified_complete": true, + "profiles": ["compact", "full"], + "catalog_tools": 209, + "catalog_bytes": 2723927, + "catalog_semantic_sha256": "d18f02920b84d39b90dc488db89a8f403f6ec35139a39a3329375a18078af1ea", + "discovery_ms": 5686.0627620480955, + "first_status_ms": 1266.2633110303432, + "warm_status_ms": 1025.3590901847929, + "wall_seconds": 8.893, + "cpu_seconds": 8.191988, + "kernel_family_peak_bytes": 780988416, + "token_proxy": { + "input_facing_catalog": 43507, + "arguments": 82, + "text_replies": 22702, + "combined": 66291, + "complete_discovered_catalog": 702182 + }, + "raw_sha256": { + "report.json": "1ae8930195ef5d5b6d5118ffaad3d0d2bedaaf114b3887a28357504be023821f", + "catalog.json": "7e65fcac2fe7257a2f1c058f2b9ad1200772072fd7cabd4ea315bf4c48307303", + "transcript.json": "5a0220b5fe36ae31af91f9eb7f8006290fcf790c774096b838f7287c472e94c1", + "candidate-1-2-compact-full.log": "2e4baa5d7e627be27f87739d03f2a4e762ae1675a63e14cc7b5adf883c24192c" + } + }, + { + "recording": "candidate-1-1-full-off", + "verified_complete": true, + "profiles": ["full", "off"], + "catalog_tools": 209, + "catalog_bytes": 832191, + "catalog_semantic_sha256": "e537c521b5dd0ac369dee706815579d9d7803497a8fc7f7d683fec267042b81c", + "discovery_ms": 5637.611771002412, + "first_status_ms": 1178.2668600790203, + "warm_status_ms": 1036.6536700166762, + "wall_seconds": 8.791, + "cpu_seconds": 8.201682, + "kernel_family_peak_bytes": 656801792, + "token_proxy": { + "input_facing_catalog": 187005, + "arguments": 82, + "text_replies": 22698, + "combined": 209785, + "complete_discovered_catalog": 195042 + }, + "raw_sha256": { + "report.json": "cd706d0ab9a3dd9662ec73ecc45deefdbd40a4a42cd9ee2735be0f8798cc1933", + "catalog.json": "9d1fd380d490edf4b6403e3daed015b72bc654b96dfb06dbe7d776e6a6146d76", + "transcript.json": "0e3f0c66a2b638729d341af8d8a379576d267b53c1dcea5563e1afa4f9245cc8", + "candidate-1-1-full-off.log": "1baf3976c762923680cd17cbef36778b3982e90005eb34bb8ec3ce8030ebd3fb" + } + }, + { + "recording": "candidate-1-2-full-off", + "verified_complete": true, + "profiles": ["full", "off"], + "catalog_tools": 209, + "catalog_bytes": 832191, + "catalog_semantic_sha256": "e537c521b5dd0ac369dee706815579d9d7803497a8fc7f7d683fec267042b81c", + "discovery_ms": 5316.555531928316, + "first_status_ms": 1383.681601146236, + "warm_status_ms": 1108.902801061049, + "wall_seconds": 8.698, + "cpu_seconds": 8.19166, + "kernel_family_peak_bytes": 662822912, + "token_proxy": { + "input_facing_catalog": 187004, + "arguments": 82, + "text_replies": 22698, + "combined": 209784, + "complete_discovered_catalog": 195042 + }, + "raw_sha256": { + "report.json": "89c5f579cb8f8a1860b3f605042ffc51db4aaae69f7a22213256f93e42b91672", + "catalog.json": "0301c40f667a17a9238ad8d9abdbb8e8c7b07a7d0dcc6ae062f5885e0b19dfc7", + "transcript.json": "7e884d99d3335707f9108a63528fd2830b1c21e7888cfc320852d75924d833cb", + "candidate-1-2-full-off.log": "ba34788aa92aa60e79e832659c48f304396d1df41f493a509ff474a152fa7c6c" + } + }, + { + "recording": "candidate-1-1-compact-off", + "verified_complete": true, + "profiles": ["compact", "off"], + "catalog_tools": 209, + "catalog_bytes": 233547, + "catalog_semantic_sha256": "0b81bdd81251ddd527a9c9ecbb5e6048e9c8a3be1f6195465a803d206f03fe73", + "discovery_ms": 4922.329220920801, + "first_status_ms": 813.2044598460197, + "warm_status_ms": 1415.0590698700398, + "wall_seconds": 7.982, + "cpu_seconds": 7.401054, + "kernel_family_peak_bytes": 629260288, + "token_proxy": { + "input_facing_catalog": 43507, + "arguments": 80, + "text_replies": 22678, + "combined": 66265, + "complete_discovered_catalog": 51352 + }, + "raw_sha256": { + "report.json": "89ae06a9c0e8f4c9b85aa711fc6f71204798c990e582f5702713f78612c4fce2", + "catalog.json": "d6a231b2babd7885d58f9712881f96fa9dc2b763cc7efd802bc5676dd71fe28a", + "transcript.json": "65fbdb48a9bc2aa65a815ea405d1210e298f908ddbab5e60f815c4e01a0f0e98", + "candidate-1-1-compact-off.log": "76501971a3698b13435b96647b59ee092689397012d453e62c821e4ecf40e988" + } + }, + { + "recording": "candidate-1-2-compact-off", + "verified_complete": true, + "profiles": ["compact", "off"], + "catalog_tools": 209, + "catalog_bytes": 233547, + "catalog_semantic_sha256": "0b81bdd81251ddd527a9c9ecbb5e6048e9c8a3be1f6195465a803d206f03fe73", + "discovery_ms": 4839.713891968131, + "first_status_ms": 1174.165149917826, + "warm_status_ms": 1409.5584598835558, + "wall_seconds": 8.289, + "cpu_seconds": 7.764389, + "kernel_family_peak_bytes": 627363840, + "token_proxy": { + "input_facing_catalog": 43507, + "arguments": 80, + "text_replies": 22678, + "combined": 66265, + "complete_discovered_catalog": 51352 + }, + "raw_sha256": { + "report.json": "a4ef1220b73c39421ccfc22c2102c613fd0a33900cbc8683dad1dc3bc034cf5a", + "catalog.json": "39ae1d7385fe8684f037e354ec2ab1126243fd31667be1b1cc559a9d1e40149c", + "transcript.json": "d60eae5fdbfb4095f040d54103a6e9b1d2ff8b8f64126976d14b0a3e024969f7", + "candidate-1-2-compact-off.log": "0e195ca040565e29e1f6d25b087a706d0d500ff1752a24bff54ad30704bd6d6b" + } + } + ], + "digest_phases": [ + { + "recording": "baseline-d532-1-compact-full-digest", + "input": "compact", + "output": "full", + "import_ms": 3827.9035209999997, + "digest_ms": 2916.01332, + "digest_cpu_us": { + "user": 2587474, + "system": 151462 + }, + "raw_sha256": "28a8e0b5c847978be9d71e4d5ed6953758e266e697b820f45f1d2141861c95e3" + }, + { + "recording": "baseline-d532-1-compact-off-digest", + "input": "compact", + "output": "off", + "import_ms": 3873.5359620000004, + "digest_ms": 3790.342601, + "digest_cpu_us": { + "user": 3121749, + "system": 212457 + }, + "raw_sha256": "b88dd91f159a1763d547f1e26637afaf3a03b28faad3295c2deff8c90e7d7707" + }, + { + "recording": "baseline-d532-1-full-full-digest", + "input": "full", + "output": "full", + "import_ms": 1766.4657209999998, + "digest_ms": 0.06419000000005326, + "digest_cpu_us": { + "user": 65, + "system": 12 + }, + "raw_sha256": "0f18c0d63def7df73b6183240c49011aca9cd85899a221fdb01908da1d79120f" + }, + { + "recording": "baseline-d532-1-full-off-digest", + "input": "full", + "output": "off", + "import_ms": 5377.792571, + "digest_ms": 3209.784441, + "digest_cpu_us": { + "user": 2585090, + "system": 252879 + }, + "raw_sha256": "2de37e6aac00f37003f619c4b4a5a747637e0b188e9892d5a37a091ae568400a" + }, + { + "recording": "baseline-d532-2-compact-full-digest", + "input": "compact", + "output": "full", + "import_ms": 1807.87952, + "digest_ms": 1512.0799399999999, + "digest_cpu_us": { + "user": 1421207, + "system": 88507 + }, + "raw_sha256": "a330124a6cec474e5337aa5a40c33d31fa0bdce304554c1c68f79954bf5cdd33" + }, + { + "recording": "baseline-d532-2-compact-off-digest", + "input": "compact", + "output": "off", + "import_ms": 1854.8867300000002, + "digest_ms": 1402.711411, + "digest_cpu_us": { + "user": 1284446, + "system": 116933 + }, + "raw_sha256": "02f19af3b9836796d85f81effd5e86bab0aa4978eeae0c59b8c35e57d03dd17b" + }, + { + "recording": "baseline-d532-2-full-full-digest", + "input": "full", + "output": "full", + "import_ms": 1572.84796, + "digest_ms": 0.0849799999998595, + "digest_cpu_us": { + "user": 85, + "system": 15 + }, + "raw_sha256": "a9fca1656b8763f5ef305548eceb7bbd27cdeb426c6d6934a891c82823a54f31" + }, + { + "recording": "baseline-d532-2-full-off-digest", + "input": "full", + "output": "off", + "import_ms": 1814.550491, + "digest_ms": 1280.248369, + "digest_cpu_us": { + "user": 1166910, + "system": 108321 + }, + "raw_sha256": "350a276f6f408de517db2b19af7cd7490c243b71338854d4acb54418f1a00b64" + }, + { + "recording": "candidate-1-1-compact-full-digest", + "input": "compact", + "output": "full", + "import_ms": 1760.5044, + "digest_ms": 0.08051999999997861, + "digest_cpu_us": { + "user": 76, + "system": 14 + }, + "raw_sha256": "3ebc2dac6cff195b54918e5afaedc0b73fe9f1a63ef12ebb253d9a1aca683c8c" + }, + { + "recording": "candidate-1-1-compact-off-digest", + "input": "compact", + "output": "off", + "import_ms": 1922.20413, + "digest_ms": 0.07567999999992026, + "digest_cpu_us": { + "user": 75, + "system": 11 + }, + "raw_sha256": "d5916e1e5760f9c7ce5a42ee8b7635477464a5a7cb7d379309666357b6fbe35f" + }, + { + "recording": "candidate-1-1-full-full-digest", + "input": "full", + "output": "full", + "import_ms": 1777.54332, + "digest_ms": 0.08724099999994905, + "digest_cpu_us": { + "user": 93, + "system": 13 + }, + "raw_sha256": "ed68a8c592c9556b7a279a90b7324d5549f9dc197fe593d27c55c5de24827481" + }, + { + "recording": "candidate-1-1-full-off-digest", + "input": "full", + "output": "off", + "import_ms": 2062.896171, + "digest_ms": 0.06988900000033027, + "digest_cpu_us": { + "user": 69, + "system": 12 + }, + "raw_sha256": "166dcdaa064c334dcc882429c2434c0353b0f6d2ccdea986138a2149ccccf10a" + }, + { + "recording": "candidate-1-2-compact-full-digest", + "input": "compact", + "output": "full", + "import_ms": 1813.35421, + "digest_ms": 0.06941099999994549, + "digest_cpu_us": { + "user": 73, + "system": 10 + }, + "raw_sha256": "86497ee4962fd03fcad4f2bcc31e95f5f57d9c4270cc0578d4b43a076ec01b53" + }, + { + "recording": "candidate-1-2-compact-off-digest", + "input": "compact", + "output": "off", + "import_ms": 2024.0317209999998, + "digest_ms": 0.08607000000029075, + "digest_cpu_us": { + "user": 89, + "system": 13 + }, + "raw_sha256": "82b47ed9bc203f1417f5f76de808d493c171f95598934fc9ebbd6f63be43b938" + }, + { + "recording": "candidate-1-2-full-full-digest", + "input": "full", + "output": "full", + "import_ms": 1917.6121309999999, + "digest_ms": 0.07732000000009975, + "digest_cpu_us": { + "user": 77, + "system": 11 + }, + "raw_sha256": "410395499363c55e53fe4d8488b5e83c54170786fe1814389fd3bf4ec317c6ac" + }, + { + "recording": "candidate-1-2-full-off-digest", + "input": "full", + "output": "off", + "import_ms": 1715.96008, + "digest_ms": 0.08907999999996719, + "digest_cpu_us": { + "user": 104, + "system": 15 + }, + "raw_sha256": "f6ea0fffc2974e02bdab1348736533c8dbd677c2311e339a945d760bd9337f9a" + } + ], + "build_digest_phase": [ + { + "phase": "before", + "operation": "createCatalogDigests", + "import_ms": 2103.978981, + "compute_ms": 1640.0212299999998, + "compute_cpu_us": { + "user": 1487510, + "system": 148234 + }, + "raw_sha256": "3a4bfe5e4297e95cf0ab3b832bb34c81edeca02e7f884db7dd5a6944c2ee1674" + }, + { + "phase": "after", + "operation": "createCatalogDigestProfiles", + "import_ms": 1752.475121, + "compute_ms": 2379.7143199999996, + "compute_cpu_us": { + "user": 2232836, + "system": 140061 + }, + "raw_sha256": "b25120373c7433457e30ff00925ca2771c0d61ba96828fe89e63be651b4f6e4a" + } + ], + "failed_attempts": [ + { + "recording": "codex-host-full-1", + "outcome": "Harness expected counts.tools rather than canonical counts.mcp_tools; corrected oracle, not a completed task.", + "raw_sha256": { + "report.json": "ac3d531a9dad31e31cb9a5957ed1f5e6415534fda1e52c64bd93a07a36ef7d0f", + "codex-host-full-1.log": "7c5f13e2976efef59306b9a2fabd2e2031277d3c5aed2b74f4ff2ad80c25a61c" + } + }, + { + "recording": "codex-host-matrix-compact-off-1", + "outcome": "Functional checks passed but MCP child remained at payload exit. Excluded from qualified cohort. Controller now waits for allocated scope to drain without targeting a PID.", + "raw_sha256": { + "report.json": "1cbc9df5aff972538629ed695271f0c3a66fcfe546208b392d09b3b7cf72f97c", + "codex-host-matrix-compact-off-1.log": "997e13a9ea7b3c5247d5037279fbef1d0dfd238eed0781748cc2ddf3afc706e7" + } + } + ], + "unverified": [ + "Model selection/injection and actual model tokens", + "Provider analysis, target execution and end-to-end reverse-engineering application quality", + "Source-mode speedup: absent build records still use dynamic fallback", + "Other MCP clients/OS compatibility not established by this Codex experiment" + ], + "validation": [ + { + "check": "check-profiles", + "payload_exit_code": 1, + "owned_cleanup_verified": true, + "raw_sha256": "04dd1ad8142e61e9c2960ec55a3a7f0421f4c22f5cb5b9702761a0d7f29d3cc4", + "source_head": "pre-rebase candidate", + "outcome": "Unused build-only export identified; existing output-policy coverage extended to the profile builder." + }, + { + "check": "build-latest", + "payload_exit_code": 0, + "owned_cleanup_verified": true, + "raw_sha256": "f430e38ecb205aa7bde17b48ff7ad97805b18511dbbfee8bdbab605f4f93b86e", + "source_head": "7ad06628df042a7c8034155a98fc51c35fa0704c", + "outcome": "passed" + }, + { + "check": "tests-latest", + "payload_exit_code": 0, + "owned_cleanup_verified": true, + "raw_sha256": "58b06f5be6ae5c0cb871cba6788b3f84581289a47ffe26aaad8897b606c94e2d", + "source_head": "7ad06628df042a7c8034155a98fc51c35fa0704c", + "outcome": "passed" + }, + { + "check": "check-latest", + "payload_exit_code": 0, + "owned_cleanup_verified": true, + "raw_sha256": "2373fec55a74515f6e3036d46becd4ed76149d9a70731a64c5dddacaeeb920cc", + "source_head": "7ad06628df042a7c8034155a98fc51c35fa0704c", + "outcome": "passed" + }, + { + "check": "docs-latest", + "payload_exit_code": 0, + "owned_cleanup_verified": true, + "raw_sha256": "23ef004935364b7a1ad2634610b67c1d739bde4f4c455552a473a05b346920ee", + "source_head": "7ad06628df042a7c8034155a98fc51c35fa0704c", + "outcome": "passed" + } + ], + "post_rebase_qualification": { + "head": "7ad06628df042a7c8034155a98fc51c35fa0704c", + "base": "e84468c0b", + "client": "Codex app-server 0.163.0", + "profiles": ["compact", "off"], + "outcome": "Complete discovery and first/warm status facts pass; allocated scope drains; no model turns. Qualification only; not relabeled as the pinned before/after measurements.", + "raw_sha256": { + "final-latest-compact-off/report.json": "ba6f549fe087f55dc48011daf92bd18f4bfc2a32e6185081ab2b7436bf39b1b8", + "final-latest-compact-off/transcript.json": "461a25723c2a4bb4c18675ffa613e98a8592a4e933054fa8643f1da73fc4686f", + "final-latest-compact-off.log": "4efc56e69b8f0d1111345bce10d0d0792b510e18901f52d10e538374257e1d61" + } + } +} diff --git a/experiments/analysis-efficiency/mcp-profile-identity.md b/experiments/analysis-efficiency/mcp-profile-identity.md new file mode 100644 index 0000000000..812473e677 --- /dev/null +++ b/experiments/analysis-efficiency/mcp-profile-identity.md @@ -0,0 +1,131 @@ +# MCP schema profile identity startup + +Continues [#723](https://github.com/morluto/rea/issues/723). The default catalog +identity was already recorded at build time (#4041). Compact input and omitted +output advertisement profiles still converted and hashed the complete catalog +on demand. This change records the four built-in combinations at the same +owning build step. Canonical tool facts and output-schema conversion are shared +while building those records. Only immutable hash triples reach the runtime: +the generated module grows from 411 to 1,320 bytes. + +Exact full/compact and full/off selections choose their matching record. Source +execution and custom compact budgets retain lazy dynamic calculation, including +snapshotting the caller's budget. The build consumes the compiled contracts and +pinned converter before bundling; rebuilding invalidates the records. No live +provider state is cached. Output-off identities still cover canonical output +contracts, as before. All four digest triples match the pre-change runtime. + +## Completed task and oracle + +A fresh **actual Codex app-server 0.163.0** discovers every tool, then reports +whether a target is open, cleanup is pending, a provider is bound, and whether +the package version and catalog count match, twice in the same session. +The oracle is the selected package metadata, the pinned 209-tool contract, +no-target state, text/structured-result agreement and original pre-change +identities. Canonicalized actual-client catalogs agree before/after within +each profile; JSON object order in Codex's tool map varies by process. + +The [driver](mcp-profile-identity.py) uses the installed client's version-specific +`initialize`, `thread/start`, `mcpServerStatus/list` and `mcpServer/tool/call` +protocol ([official app-server documentation](https://learn.chatgpt.com/docs/app-server)). +It uses a fresh temporary account/configuration and starts **no model turns**. +This establishes actual-client schema discovery and direct tool-call +compatibility, not model analysis quality or actual model token savings. +Existing compact/off choices remain explicit; neither becomes a new default. + +Baseline: main `d532b18123916a9e88f98c7880ba4f984e23d715`. Both packaged runtimes +are frozen before measurement; their source patch, package/manifests and raw +transcripts/resource observations are retained outside Git, pinned by the +[receipt](mcp-profile-identity.json). Later validation does not relabel timings. +Node 24.18.0/npm 11.16.0, MCP SDK 2.3.1, negotiated protocol `2025-06-18`, Linux +x86-64. Two repetitions alternate baseline/candidate cohorts and reverse profile +order. Every recording starts a fresh server and includes discovery, first and +warm status calls, verification and cleanup. OS caches are unflushed. + +The existing guard admits one CPU, a 4 GiB hard/3 GiB early memory boundary, +2 GiB Node heaps, no swap and 192 tasks. Kernel family peaks include file cache, +Codex, REA and controller; they are not provider heap measurements. All 16 +qualified recordings exit successfully with only the observer remaining, no +resource stop and zero observed OOM events. Final whole-scope counters are +unavailable. The host's CPU is shared with unrelated work; ranges retain the +observed variation. + +## Measurements + +Ranges show both repetitions. Times include the whole task and shutdown. + +| Input / output profile | Wall seconds before → after | CPU seconds before → after | Family peak MiB before → after | +| ---------------------- | --------------------------- | -------------------------- | ------------------------------ | +| full / full | 8.91–8.92 → 8.82–10.31 | 8.45–8.48 → 8.07–9.82 | 765–766 → 764–769 | +| compact / full | 11.15–18.99 → 8.82–8.89 | 10.61–16.52 → 8.19–8.48 | 772–779 → 745–745 | +| full / off | 10.33–19.59 → 8.70–8.79 | 9.76–16.96 → 8.19–8.20 | 671–677 → 626–632 | +| compact / off | 10.74–23.47 → 7.98–8.29 | 10.18–19.23 → 7.40–7.76 | 635–637 → 598–600 | + +First/warm call times remain separately recorded in the receipt. Across the +three non-default profiles the first call ranges from 2.73–6.15 s before to +0.81–1.38 s after; warm calls range from 1.04–3.12 s before to 1.03–1.42 s after. +The default profile has no established gain; one candidate repetition is slower. + +Fresh-process [identity probes](mcp-profile-digest.mjs) isolate the lazy read +from module loading. The non-default digest computation takes 1.28–3.79 s +before and under 0.2 ms after. Building all four identities shares schema +conversion: isolated compute CPU increases from 1.636 s (one identity) to +2.373 s (four). Full uncached builds were 42.59 → 81.09 s wall, 41.59 → 68.27 s +CPU, 3318.5 → 3216.4 MiB peak. These single builds include TypeScript, bundling +and concurrent host contention, so their difference is not attributed solely +to digest generation. The isolated phase is one observation per version. + +There is **no token reduction from this change**. Each profile's complete +catalog size and serialization-token estimate are unchanged. Full/full remains +3,322,571 bytes as exposed by this client; compact/off remains 233,547 bytes +through existing configuration. The input-facing catalog plus both argument +and text replies is about 209.8k proxy tokens with full input and 66.3k with +compact input, before and after. Small differences come from fresh timestamps +and runtime paths. The [cost script](mcp-profile-costs.py) uses +`tiktoken==0.12.0`/`cl100k_base`; these estimates are not actual model usage or a +claim about what the host injects. Output schemas, hidden prompts and model +reasoning must not be inferred from that projection. + +## Validation and remaining scope + +Executed: four files / 46 tests pass, including real production stdio doctor +for all four profiles, source catalog/schema consistency, output-off identity +coverage, malformed replies, startup cancellation/cleanup, generated immutable +record reuse and caller-selected custom budgets. The pre-change runtime is +also an independent oracle for all four complete hash triples. + +After rebasing onto main `e84468c0b`, the four files / 46 tests pass again, +as do `npm run check` (types, lint, format, knip, metadata and architecture) +and `npm run docs:check` (including five README consistency tests). Actual +Codex compact/off discovery and both status calls pass again with verified +cleanup. This final qualification does not replace the pinned timing cohort. +An initial static check identified the build-only export as untracked; the +existing output-policy regression now directly exercises its four profiles. + +Earlier exploratory attempts remain excluded: one controller expected the +wrong count key; one functional compact/off run left a child alive at payload +exit. The corrected controller waits for its allocated scope to drain without +killing individual PIDs; all later qualified runs verify that drain. Neither +failed attempt establishes a production regression. + +SDK 2.3.1 already supports a named `toolDefinition`, used by REA's doctor to +avoid preparing every output validator for its one probe. Generic external SDK +callers still incur their own catalog preparation. Input presentation and +output refs already have owning reuse; these were not duplicate fixes. +Target/provider investigations, other clients/platforms and actual model +analysis cost remain unverified by this no-target startup task. + +To reproduce with the pinned client and a built runtime, run under the existing +resource guard: + +```sh +python3 experiments/analysis-efficiency/resource-run.py --observe-payload-exit -- \ + python3 experiments/analysis-efficiency/mcp-profile-identity.py \ + --output /tmp/rea-profile-case --runtime /absolute/built/rea \ + --node /absolute/node24/bin/node --codex /absolute/codex-0.163.0 \ + --input-profile compact --output-profile off +``` + +Retain the resource output as `/tmp/rea-profile-case.log` for the cost script. +The driver is pinned to Codex 0.163.0's experimental protocol and the 209-tool +catalog in this receipt; update its oracle deliberately for another revision. diff --git a/experiments/analysis-efficiency/mcp-profile-identity.py b/experiments/analysis-efficiency/mcp-profile-identity.py new file mode 100644 index 0000000000..78c9dcc862 --- /dev/null +++ b/experiments/analysis-efficiency/mcp-profile-identity.py @@ -0,0 +1,199 @@ +"""Exercise real Codex MCP discovery/calls without starting a model turn.""" +import argparse +import asyncio +import hashlib +import json +import os +from pathlib import Path +import re +import time +import traceback + + +async def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--runtime", type=Path, required=True) + parser.add_argument("--codex", type=Path, required=True) + parser.add_argument("--node", type=Path, required=True) + parser.add_argument("--input-profile", choices=["full", "compact"], default="full") + parser.add_argument("--output-profile", choices=["full", "off"], default="full") + args = parser.parse_args() + args.output.mkdir(mode=0o700) + audit_home = args.output / "account" + codex_state = audit_home / ".codex" + mcp_home = audit_home / "mcp" + workspace = audit_home / "workspace" + for directory in [codex_state, mcp_home, workspace]: + directory.mkdir(parents=True) + task_path = f"{args.node.parent}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + mcp_env = {"HOME": str(mcp_home), "PATH": task_path, + "NODE_OPTIONS": "--max-old-space-size=2048", "UV_THREADPOOL_SIZE": "1", + "REA_LOG_LEVEL": "silent", "REA_MCP_INPUT_SCHEMA_PROFILE": args.input_profile, + "REA_MCP_OUTPUT_SCHEMA_PROFILE": args.output_profile} + config = '[analytics]\nenabled = false\n[mcp_servers.rea_audit]\n' + config += f'command = {json.dumps(str(args.node))}\n' + config += f'args = {json.dumps([str(args.runtime / "scripts/rea.mjs"), "mcp"])}\n' + config += 'startup_timeout_sec = 60\ntool_timeout_sec = 60\n' + config += '[mcp_servers.rea_audit.env]\n' + config += ''.join(f'{key} = {json.dumps(value)}\n' for key, value in mcp_env.items()) + (codex_state / "config.toml").write_text(config) + (args.output / "declared-config.toml").write_text(config) + environment = {"HOME": str(audit_home), "CODEX_HOME": str(codex_state), "PATH": task_path, + "NODE_OPTIONS": "--max-old-space-size=2048", "UV_THREADPOOL_SIZE": "1"} + report = {"question": "Discover the complete canonical tool catalog and report no-target session/cleanup state, selected provider, package version and catalog tool count twice.", + "client": "Codex app-server 0.163.0", "model_turns_started": 0, + "actual_model_usage": None, "input_profile": args.input_profile, + "output_profile": args.output_profile, "calls": [], "observations": [], + "unexpected_server_requests": [], "cleanup_failures": []} + transcript = [] + pending = {} + next_id = 0 + stderr = (args.output / "server.log").open("wb") + process = await asyncio.create_subprocess_exec(str(args.codex), "app-server", "--stdio", + stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, stderr=stderr, + cwd=workspace, env=environment, limit=16 * 1024 * 1024) + + async def send(message): + encoded = (json.dumps(message, separators=(",", ":")) + "\n").encode() + transcript.append({"direction": "client", "bytes": len(encoded), "message": message}) + process.stdin.write(encoded) + await process.stdin.drain() + + async def read_messages(): + try: + while line := await process.stdout.readline(): + value = json.loads(line) + transcript.append({"direction": "server", "bytes": len(line), "message": value}) + if "id" in value and "method" not in value: + future = pending.pop(value["id"], None) + if future and not future.done(): + future.set_result(value) + elif "id" in value: + report["unexpected_server_requests"].append(value["method"]) + await send({"id": value["id"], "error": {"code": -32601, "message": "Unexpected request in read-only discovery probe"}}) + raise RuntimeError("app-server stdout closed") + except Exception as error: + for future in pending.values(): + if not future.done(): + future.set_exception(error) + + reader = asyncio.create_task(read_messages()) + + async def request(method, params): + nonlocal next_id + assert method in ["initialize", "thread/start", "mcpServerStatus/list", "mcpServer/tool/call"] + next_id += 1 + message = {"id": next_id, "method": method, "params": params} + future = asyncio.get_running_loop().create_future() + pending[next_id] = future + start = time.monotonic() + await send(message) + reply = await asyncio.wait_for(future, 90) + report["calls"].append({"method": method, "elapsed_ms": (time.monotonic() - start) * 1000, + "error": reply.get("error")}) + if "error" in reply: + raise RuntimeError(json.dumps(reply["error"])[:4000]) + return reply["result"] + + try: + await request("initialize", {"clientInfo": {"name": "rea_host_probe", "title": "REA host compatibility probe", "version": "1"}, + "capabilities": {"experimentalApi": True, "explicitGatewayOauth": True}}) + await send({"method": "initialized"}) + created = await request("thread/start", {"cwd": str(workspace), "ephemeral": True, + "approvalPolicy": "never", "sandbox": "read-only"}) + thread_id = created["thread"]["id"] + assert not created["thread"].get("turns", []) + inventory = await request("mcpServerStatus/list", {"threadId": thread_id, + "serverName": "rea_audit", "detail": "toolsAndAuthOnly", "limit": 10}) + assert inventory.get("nextCursor") is None + assert len(inventory["data"]) == 1 + server = inventory["data"][0] + assert server["name"] == "rea_audit" and server.get("toolsError") is None + tools = list(server["tools"].values()) + names = [tool["name"] for tool in tools] + assert len(names) == len(set(names)) == 209 + assert "binary_session" in names + output_schemas = sum(tool.get("outputSchema") is not None for tool in tools) + assert output_schemas == (209 if args.output_profile == "full" else 0) + raw = json.dumps(inventory, separators=(",", ":"), ensure_ascii=False).encode() + (args.output / "catalog.json").write_bytes(raw) + report["catalog"] = {"tools": len(tools), "output_schemas": output_schemas, + "bytes": len(raw), "sha256": hashlib.sha256(raw).hexdigest(), + "server_info": server.get("serverInfo"), "runtime_status": server.get("runtimeStatus")} + version = json.loads((args.runtime / "package.json").read_text())["version"] + for phase in ["first", "warm"]: + reply = await request("mcpServer/tool/call", {"threadId": thread_id, + "server": "rea_audit", "tool": "binary_session", "arguments": {}}) + assert reply.get("isError") is not True + structured = reply["structuredContent"] + texts = [c["text"] for c in reply["content"] if c.get("type") == "text"] + assert len(texts) == 1 and json.loads(texts[0]) == structured + facts = structured["result"] + assert facts["open"] is False and facts["cleanup_pending"] is False + assert facts["analysis_provider_binding"] is None + assert facts["server_identity"]["package"]["version"] == version + assert facts["server_identity"]["catalog"]["counts"]["mcp_tools"] == 209 + report["observations"].append({"phase": phase, "open": facts["open"], + "cleanup_pending": facts["cleanup_pending"], "selected_provider": facts["analysis_provider_binding"], + "package_version": version, "catalog_tools": 209, + "negotiated_protocol": facts["server_identity"]["negotiated_protocol_version"], + "structured_bytes": len(json.dumps(structured, separators=(",", ":"), ensure_ascii=False).encode())}) + selected_facts = lambda observation: {key: value for key, value in observation.items() + if key not in ["phase", "structured_bytes"]} + assert selected_facts(report["observations"][0]) == selected_facts(report["observations"][1]) + assert not report["unexpected_server_requests"] + report["status"] = "passed" + except Exception: + report["status"] = "failed" + report["failure"] = traceback.format_exc()[-6000:] + finally: + process.stdin.close() + try: + await asyncio.wait_for(process.wait(), 15) + except asyncio.TimeoutError: + report["cleanup_failures"].append("app-server did not exit after stdin close; terminated owned process") + process.terminate() + try: + await asyncio.wait_for(process.wait(), 5) + except asyncio.TimeoutError: + process.kill() + await process.wait() + await reader + # app-server can acknowledge exit before its MCP child finishes EOF + # cleanup. Wait on the allocated resource scope; never target a PID. + try: + groups = [line[3:] for line in Path("/proc/self/cgroup").read_text().splitlines() + if line.startswith("0::")] + assert len(groups) == 1 + group = Path("/sys/fs/cgroup") / groups[0].lstrip("/") + assert re.fullmatch(r"rea-efficiency-[0-9a-f]{32}\.scope", group.name) + observer = os.getppid() + observer_args = Path(f"/proc/{observer}/cmdline").read_bytes().split(b"\0") + assert any(Path(os.fsdecode(value)).name == "resource_payload.py" for value in observer_args if value) + expected = {os.getpid(), observer} + until = time.monotonic() + 10 + while True: + actors = {int(value) for value in (group / "cgroup.procs").read_text().splitlines()} + assert expected <= actors + remaining = sorted(actors - expected) + if not remaining or time.monotonic() >= until: + break + await asyncio.sleep(0.05) + report["owned_scope_drain"] = {"remaining_processes": remaining} + if remaining: + report["cleanup_failures"].append("owned MCP child did not settle after app-server exit") + except (OSError, ValueError, AssertionError) as error: + report["owned_scope_drain"] = {"status": "unavailable", "reason": type(error).__name__} + report["cleanup_failures"].append("allocated resource-scope cleanup could not be verified") + stderr.close() + report["appserver_exit_code"] = process.returncode + if process.returncode != 0: + report["cleanup_failures"].append(f"app-server exit code: {process.returncode}") + (args.output / "transcript.json").write_text(json.dumps(transcript) + "\n") + (args.output / "report.json").write_text(json.dumps(report, indent=2) + "\n") + print(json.dumps({key: report.get(key) for key in ["status", "failure", "catalog", "observations", "cleanup_failures", "appserver_exit_code"]})) + return 0 if report["status"] == "passed" and not report["cleanup_failures"] else 1 + + +raise SystemExit(asyncio.run(main())) diff --git a/scripts/record-catalog-digests.mjs b/scripts/record-catalog-digests.mjs index 82807c121f..6e3525b9b3 100644 --- a/scripts/record-catalog-digests.mjs +++ b/scripts/record-catalog-digests.mjs @@ -2,18 +2,31 @@ import { writeFile } from "node:fs/promises"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; -// Record the default catalog digests into the compiled runtime. Run after +// Record built-in catalog profile digests into the compiled runtime. Run after // `tsc` and before bundling, so the digests come from the same compiled // contracts and pinned schema converter that the runtime serves. const compiledRoot = resolve( dirname(fileURLToPath(import.meta.url)), "../dist", ); -const { createCatalogDigests } = await import( +const { createCatalogDigestProfiles } = await import( pathToFileURL(join(compiledRoot, "catalogIdentity.js")).href ); +// Retain only hashes, with every shared record frozen. Schemas remain owned by +// the compiled contracts and are converted once for all four identities. +const profiles = Object.entries(createCatalogDigestProfiles()) + .map( + ([input, outputs]) => + `${JSON.stringify(input)}: Object.freeze({${Object.entries(outputs) + .map( + ([output, digests]) => + `${JSON.stringify(output)}: Object.freeze(${JSON.stringify(digests)})`, + ) + .join(",")}})`, + ) + .join(","); await writeFile( join(compiledRoot, "recordedCatalogDigests.js"), - `/** Recorded from the compiled default catalog by scripts/record-catalog-digests.mjs; do not edit. */\nexport const RECORDED_CATALOG_DIGESTS = Object.freeze(${JSON.stringify(createCatalogDigests())});\n`, + `/** Recorded from compiled catalog profiles by scripts/record-catalog-digests.mjs; do not edit. */\nexport const RECORDED_CATALOG_DIGEST_PROFILES = Object.freeze({${profiles}});\n`, "utf8", ); diff --git a/src/catalogIdentity.ts b/src/catalogIdentity.ts index 80425d26ee..8429783a90 100644 --- a/src/catalogIdentity.ts +++ b/src/catalogIdentity.ts @@ -17,7 +17,11 @@ import { type CompactInputSchemaPresentation, } from "./contracts/compactInputSchema.js"; import type { McpOutputSchemaProfile } from "./config/mcpOutputSchemaProfile.js"; -import { RECORDED_CATALOG_DIGESTS } from "./recordedCatalogDigests.js"; +import { + COMPACT_INPUT_SCHEMA_BUDGET_BYTES, + type McpInputSchemaProfile, +} from "./config/mcpInputSchemaProfile.js"; +import { RECORDED_CATALOG_DIGEST_PROFILES } from "./recordedCatalogDigests.js"; const sortByName = ( contracts: readonly ToolContract[], @@ -34,10 +38,7 @@ const sortedToolContracts = sortByName(TOOL_CONTRACTS); * that predates the untrusted-content framing must not digest equal to one * that emits `_meta.provenance`, or a stale process would report aligned. */ -const createToolCatalog = ( - contracts: readonly ToolContract[], - compactPresentation: CompactInputSchemaPresentation | undefined, -) => +const createToolCatalog = (contracts: readonly ToolContract[]) => contracts.map((contract) => ({ name: contract.name, title: contract.title, @@ -46,15 +47,26 @@ const createToolCatalog = ( effects: { ...contract.effects }, returns_target_content: contract.returnsTargetContent, annotations: contract.annotations, - input_schema: presentAdvertisedInputSchema( - advertisedInputJsonSchema(contract), - compactPresentation, - ), + input_schema: advertisedInputJsonSchema(contract), output_schema: toolOutputSchemaWithMetadata(contract)[ "~standard" ].jsonSchema.output({ target: "draft-2020-12" }), })); +const presentToolCatalog = ( + catalog: ReturnType, + presentation: CompactInputSchemaPresentation | undefined, +) => + presentation === undefined + ? catalog + : catalog.map((tool) => ({ + ...tool, + input_schema: presentAdvertisedInputSchema( + tool.input_schema, + presentation, + ), + })); + const createPromptCatalog = (contracts: readonly PromptContract[]) => contracts .map((contract) => ({ @@ -76,22 +88,15 @@ const digest = (value: unknown): string => { return createHash("sha256").update(encoded).digest("hex"); }; -/** - * Catalog digests over the given tool contracts. Exported so tests can prove - * which contract fields the identity covers; production uses - * {@link CATALOG_IDENTITY}. - */ -export const createCatalogDigests = ( - contracts: readonly ToolContract[] = TOOL_CONTRACTS, - promptContracts: readonly PromptContract[] = PROMPT_CONTRACTS, - compactPresentation: CompactInputSchemaPresentation | undefined = undefined, - outputSchemaProfile: McpOutputSchemaProfile = "full", +const digestCatalogs = ( + catalog: ReturnType, + prompts: ReturnType, + outputSchemaProfile: McpOutputSchemaProfile, ) => { const tools = { output_schema_profile: outputSchemaProfile, - catalog: createToolCatalog(sortByName(contracts), compactPresentation), + catalog, }; - const prompts = createPromptCatalog(promptContracts); return { tools_sha256: digest(tools), prompts_sha256: digest(prompts), @@ -103,7 +108,50 @@ export const createCatalogDigests = ( } as const; }; -export type CatalogDigests = ReturnType; +/** + * Catalog digests over the given tool contracts. Exported so tests can prove + * which contract fields the identity covers; production uses + * {@link CATALOG_IDENTITY}. + */ +export const createCatalogDigests = ( + contracts: readonly ToolContract[] = TOOL_CONTRACTS, + promptContracts: readonly PromptContract[] = PROMPT_CONTRACTS, + compactPresentation: CompactInputSchemaPresentation | undefined = undefined, + outputSchemaProfile: McpOutputSchemaProfile = "full", +) => + digestCatalogs( + presentToolCatalog( + createToolCatalog(sortByName(contracts)), + compactPresentation, + ), + createPromptCatalog(promptContracts), + outputSchemaProfile, + ); + +export type CatalogDigests = ReturnType; +export type CatalogDigestProfiles = Readonly< + Record< + McpInputSchemaProfile, + Readonly> + > +>; + +/** Build-time identities share schema conversion across the four supported profiles. */ +export const createCatalogDigestProfiles = ( + contracts: readonly ToolContract[] = TOOL_CONTRACTS, + promptContracts: readonly PromptContract[] = PROMPT_CONTRACTS, +): CatalogDigestProfiles => { + const full = createToolCatalog(sortByName(contracts)); + const compact = presentToolCatalog(full, { + budgetBytes: COMPACT_INPUT_SCHEMA_BUDGET_BYTES, + }); + const prompts = createPromptCatalog(promptContracts); + const outputProfiles = (catalog: typeof full) => ({ + full: digestCatalogs(catalog, prompts, "full"), + off: digestCatalogs(catalog, prompts, "off"), + }); + return { full: outputProfiles(full), compact: outputProfiles(compact) }; +}; export const createCatalogIdentity = ( compactPresentation: CompactInputSchemaPresentation | undefined = undefined, @@ -113,12 +161,18 @@ export const createCatalogIdentity = ( compactPresentation === undefined ? undefined : { budgetBytes: compactPresentation.budgetBytes }; - // Building and canonicalizing every advertised schema dominates doctor and - // server startup, so the compiled runtime records the default digests once. + // Only the exact built-in presentations have build-time identities. Source + // runs and custom budgets still compute from their selected schemas lazily. + const inputProfile = + selectedPresentation === undefined + ? "full" + : selectedPresentation.budgetBytes === COMPACT_INPUT_SCHEMA_BUDGET_BYTES + ? "compact" + : undefined; let catalogDigests: CatalogDigests | undefined = - selectedPresentation === undefined && outputSchemaProfile === "full" - ? RECORDED_CATALOG_DIGESTS - : undefined; + inputProfile === undefined + ? undefined + : RECORDED_CATALOG_DIGEST_PROFILES?.[inputProfile][outputSchemaProfile]; return { counts: { cli_commands: CLI_COMMAND_NAMES.length, diff --git a/src/recordedCatalogDigests.ts b/src/recordedCatalogDigests.ts index fc37c24d50..fc8ded8edb 100644 --- a/src/recordedCatalogDigests.ts +++ b/src/recordedCatalogDigests.ts @@ -1,8 +1,10 @@ -import type { CatalogDigests } from "./catalogIdentity.js"; +import type { CatalogDigestProfiles } from "./catalogIdentity.js"; /** - * Digests of the default catalog (full input and output schemas), recorded + * Digests of the four built-in input/output schema profiles, recorded * from the compiled contracts by `scripts/record-catalog-digests.mjs` after * compilation. Source runs leave this undefined and compute them on first use. */ -export const RECORDED_CATALOG_DIGESTS: CatalogDigests | undefined = undefined; +export const RECORDED_CATALOG_DIGEST_PROFILES: + | CatalogDigestProfiles + | undefined = undefined; diff --git a/tests/boundary/mcp/mcpDoctor.test.ts b/tests/boundary/mcp/mcpDoctor.test.ts index 12919a7fb2..f7cecc5bec 100644 --- a/tests/boundary/mcp/mcpDoctor.test.ts +++ b/tests/boundary/mcp/mcpDoctor.test.ts @@ -1,7 +1,9 @@ import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { execFile } from "node:child_process"; import { readFile, writeFile } from "node:fs/promises"; import { join, resolve } from "node:path"; +import { promisify } from "node:util"; import { afterEach, describe, expect, it, vi } from "vitest"; @@ -22,6 +24,40 @@ import { MCP_STARTUP_POLICY } from "../../../src/mcpStartupPolicy.js"; afterEach(() => vi.useRealTimers()); describe("production MCP doctor", () => { + it("reuses immutable compiled profile digests and preserves custom-budget identity", async () => { + // Exercise the generated module with Node, outside Vitest's source loader. + await promisify(execFile)( + process.execPath, + [ + "--input-type=module", + "--eval", + ` + import assert from "node:assert/strict"; + import { createCatalogIdentity, createCatalogDigests } from "./dist/catalogIdentity.js"; + import { RECORDED_CATALOG_DIGEST_PROFILES as profiles } from "./dist/recordedCatalogDigests.js"; + import { COMPACT_INPUT_SCHEMA_BUDGET_BYTES as budgetBytes } from "./dist/config/mcpInputSchemaProfile.js"; + assert.ok(Object.isFrozen(profiles)); + for (const input of ["full", "compact"]) { + assert.ok(Object.isFrozen(profiles[input])); + for (const output of ["full", "off"]) { + const identity = createCatalogIdentity(input === "compact" ? { budgetBytes } : undefined, output); + assert.equal(identity.digests, profiles[input][output]); + assert.ok(Object.isFrozen(identity.digests)); + } + } + const selection = { budgetBytes: 1 }; + const custom = createCatalogIdentity(selection, "off"); + selection.budgetBytes = budgetBytes; + const expected = createCatalogDigests(undefined, undefined, { budgetBytes: 1 }, "off"); + assert.deepEqual(custom.digests, expected); + assert.notEqual(custom.digests.tools_sha256, profiles.compact.off.tools_sha256); + assert.equal(custom.digests, custom.digests); + `, + ], + { timeout: 30_000 }, + ); + }, 35_000); + it("preserves the full-output envelope for startup failures", async () => { const result = await runProductionMcpDoctorCli( ["--json", "--full-output"], diff --git a/tests/boundary/mcp/serverIdentity.test.ts b/tests/boundary/mcp/serverIdentity.test.ts index 2deef7abe5..68a726c29c 100644 --- a/tests/boundary/mcp/serverIdentity.test.ts +++ b/tests/boundary/mcp/serverIdentity.test.ts @@ -12,6 +12,7 @@ import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { machoImage } from "../../../src/artifacts/apple/MachoImage.fixture.js"; import { CATALOG_IDENTITY, + createCatalogDigestProfiles, createCatalogDigests, } from "../../../src/catalogIdentity.js"; import { CLI_COMMAND_NAMES } from "../../../src/cliCommandNames.js"; @@ -141,17 +142,7 @@ describe("server and catalog identity", () => { }); it("binds output schema profile and canonical output contracts into identity", () => { - const full = createCatalogDigests(); - const off = createCatalogDigests( - TOOL_CONTRACTS, - PROMPT_CONTRACTS, - undefined, - "off", - ); - expect(off.tools_sha256).not.toBe(full.tools_sha256); - expect(off.combined_sha256).not.toBe(full.combined_sha256); - expect(off.prompts_sha256).toBe(full.prompts_sha256); - + const profiles = createCatalogDigestProfiles(); const changedOutput = TOOL_CONTRACTS.map((contract, index) => index === 0 ? { @@ -160,17 +151,24 @@ describe("server and catalog identity", () => { } : contract, ); - const changedFull = createCatalogDigests(changedOutput); - const changedOff = createCatalogDigests( - changedOutput, - PROMPT_CONTRACTS, - undefined, - "off", - ); - expect(changedFull.tools_sha256).not.toBe(full.tools_sha256); - expect(changedOff.tools_sha256).not.toBe(off.tools_sha256); - expect(changedFull.combined_sha256).not.toBe(full.combined_sha256); - expect(changedOff.combined_sha256).not.toBe(off.combined_sha256); + const changed = createCatalogDigestProfiles(changedOutput); + for (const input of ["full", "compact"] as const) { + const { full, off } = profiles[input]; + expect(off.tools_sha256).not.toBe(full.tools_sha256); + expect(off.combined_sha256).not.toBe(full.combined_sha256); + expect(off.prompts_sha256).toBe(full.prompts_sha256); + for (const output of ["full", "off"] as const) { + expect(changed[input][output].tools_sha256).not.toBe( + profiles[input][output].tools_sha256, + ); + expect(changed[input][output].combined_sha256).not.toBe( + profiles[input][output].combined_sha256, + ); + expect(changed[input][output].prompts_sha256).toBe( + profiles[input][output].prompts_sha256, + ); + } + } }); it("changes catalog identity when advertised input examples change", () => {