From aa51e4343963fdd0b97ec2ebdc3c41579fd9cb38 Mon Sep 17 00:00:00 2001 From: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com> Date: Thu, 22 Oct 2026 00:17:11 +0800 Subject: [PATCH] perf(mcp)!: default to compact input schema advertisements (#5975) Use compact MCP input advertisements by default while preserving explicit full, canonical validation, output contracts and complete Evidence. Require an explicit selected catalog for runtime identity and honor both profiles in CLI status. Real ripgrep/Zellij SDK tasks, independent ELF oracle, actual client trials, profile parity workflows and CI are recorded in experiments/analysis-efficiency/mcp-input-catalog.md. SDK token estimates improve; actual Codex token savings and general latency/memory gains are not established. Refs #5861 BREAKING CHANGE: MCP input advertisements default to compact. Set REA_MCP_INPUT_SCHEMA_PROFILE=full for complete advertised guidance and nested constraints. --- README.md | 6 + docs/client-verification.md | 4 +- docs/mcp-contracts.md | 11 +- .../mcp-input-catalog-costs.py | 90 + .../mcp-input-catalog-oracle.py | 54 + .../mcp-input-catalog.json | 2794 +++++++++++++++++ .../analysis-efficiency/mcp-input-catalog.md | 228 ++ .../analysis-efficiency/mcp-input-catalog.mjs | 176 ++ scripts/generate-skill-metadata.mjs | 4 +- scripts/lib/catalog-builders.mjs | 3 +- scripts/lib/product-catalog.mjs | 2 +- scripts/lib/release-package-canary.mjs | 4 +- scripts/verify-claude-client.mjs | 22 +- scripts/verify-package-pack.mjs | 4 +- scripts/verify-reconstruction-readiness.mjs | 17 +- src/application/DirectAnalysisStatus.ts | 8 + src/application/Doctor.fixture.ts | 4 +- src/application/Doctor.ts | 11 +- src/application/Setup.fixture.ts | 4 +- src/catalogIdentity.ts | 14 +- src/config/mcpInputSchemaProfile.ts | 10 +- src/config/parseConfig.ts | 3 + src/config/types.ts | 2 + src/server/EvidenceMcpServer.ts | 7 +- src/serverIdentity.ts | 9 +- tests/acceptance/applications/runtime.test.ts | 4 +- .../filesystem/dependencyInstall.test.ts | 2 +- .../filesystem/generatedSkill.test.ts | 2 +- .../mcp/absoluteLocalPathSchema.test.ts | 10 +- tests/boundary/mcp/analysisViewMcp.test.ts | 2 + .../mcp/applicationEvidenceMcp.test.ts | 9 +- .../mcp/applicationExportShapeMcp.test.ts | 6 +- .../mcp/artifactMcp.inlineInventory.test.ts | 302 +- tests/boundary/mcp/browserMcp.test.ts | 90 +- tests/boundary/mcp/browserScenarioMcp.test.ts | 12 +- .../boundary/mcp/contractPresentation.test.ts | 7 +- .../mcp/functionCallDirectionsMcp.test.ts | 44 +- tests/boundary/mcp/godotPckMcp.test.ts | 47 +- .../mcp/javascriptFactsViewMcp.test.ts | 21 +- tests/boundary/mcp/mcpDoctor.test.ts | 45 +- .../boundary/mcp/outputSchemaProfile.test.ts | 4 +- .../retainedApplicationEvidenceMcp.test.ts | 67 +- tests/boundary/mcp/serverIdentity.test.ts | 41 +- .../process/packageTarListing.test.ts | 4 +- .../contracts/mcpInputSchemaProfile.test.ts | 8 +- 45 files changed, 3887 insertions(+), 331 deletions(-) create mode 100644 experiments/analysis-efficiency/mcp-input-catalog-costs.py create mode 100644 experiments/analysis-efficiency/mcp-input-catalog-oracle.py create mode 100644 experiments/analysis-efficiency/mcp-input-catalog.json create mode 100644 experiments/analysis-efficiency/mcp-input-catalog.md create mode 100644 experiments/analysis-efficiency/mcp-input-catalog.mjs diff --git a/README.md b/README.md index c47b6399cb..b0d470e45e 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,12 @@ Setup supports Claude Code, Codex, Cursor, Gemini CLI, Grok Build and [installation and setup](docs/installation.md) for provider configuration and manual MCP registration. +MCP uses compact input schemas by default to reduce context overhead. Set +`REA_MCP_INPUT_SCHEMA_PROFILE=full` in your MCP registration when you need the +complete advertised field guidance and nested validation details. Both profiles +keep the full tool set and canonical server validation. See +"Compact input schema profile" in the MCP contracts guide. + ### Ask your agent ```text diff --git a/docs/client-verification.md b/docs/client-verification.md index c5f1bf944b..6759ab5180 100644 --- a/docs/client-verification.md +++ b/docs/client-verification.md @@ -17,8 +17,8 @@ shared form would leave later fields unknown instead. The existing `REA_MCP_INPUT_SCHEMA_PROFILE=compact` profile lets callers stay below its character limit by advertising reduced nested validation structure, while REA continues to validate calls against the complete canonical input schema. -Compact applies to the server's advertised inputs and is not an automatic -full-profile fallback; see the [compact input schema profile](mcp-contracts.md#compact-input-schema-profile) +Compact is the default for the server's advertised inputs; an explicit `full` +choice is preserved. See the [compact input schema profile](mcp-contracts.md#compact-input-schema-profile) for details of its reduced forms. Custom renderer hosts can raise `inputMaxChars` through `renderToolSignature` or diff --git a/docs/mcp-contracts.md b/docs/mcp-contracts.md index 7b6d37328a..eec6ac9b43 100644 --- a/docs/mcp-contracts.md +++ b/docs/mcp-contracts.md @@ -1020,14 +1020,15 @@ verify the compact advertisement through the same client boundary. Some function-calling providers reject a request while any single tool's `parameters` schema exceeds a literal serialized size cap; Moonshot clients -such as kimi-cli fail every request while the default catalog is attached +such as kimi-cli have reported failures with the full catalog attached ([#1484](https://github.com/morluto/rea/issues/1484)). Setting the server's -`REA_MCP_INPUT_SCHEMA_PROFILE` environment variable to `compact` renders every -advertised input schema within a 13,312-byte budget, measured to sit below +`REA_MCP_INPUT_SCHEMA_PROFILE` environment variable to `compact` (the default) +renders every advertised input schema within a 13,312-byte budget, measured to sit below Moonshot's observed cap (community measurements: a 13.7 KB schema is accepted while a ~15 KB one is rejected). The setting is read at startup; restart the -server to apply changes, as with `REA_MCP_MAX_RESPONSE_BYTES`. The default -value `full` keeps the complete advertisement unchanged. Compact presentation +server to apply changes, as with `REA_MCP_MAX_RESPONSE_BYTES`. Explicitly select +`full` to retain complete advertised input guidance and nested validation detail. +Existing explicit profile choices are preserved. Compact presentation also reduces schemas whose expanded JSON tree exceeds ten levels, the limit reported by strict-depth MCP clients in [#2813](https://github.com/morluto/rea/issues/2813). WorkBuddy/CodeBuddy gateways have also rejected full REA catalogs with an diff --git a/experiments/analysis-efficiency/mcp-input-catalog-costs.py b/experiments/analysis-efficiency/mcp-input-catalog-costs.py new file mode 100644 index 0000000000..bf089ac431 --- /dev/null +++ b/experiments/analysis-efficiency/mcp-input-catalog-costs.py @@ -0,0 +1,90 @@ +"""Account for catalog, complete SDK tasks and separately recorded model trials.""" +import argparse +import hashlib +import json +from pathlib import Path + +import tiktoken + + +def compact(value): + return json.dumps(value, ensure_ascii=False, separators=(",", ":"), allow_nan=False) + + +def resource(case): + path = case.with_name(case.name + ".log") + scope = next(json.loads(line) for line in reversed(path.read_text().splitlines()) + if line.startswith('{"resource_scope":')) + observation = scope["payload_resource_observation"] + assert scope["abort"] is None and scope["payload_resource_observation_verified"] + assert observation["payload_cleanup_verified"] and observation["payload_exit_code"] == 0 + assert all(observation["resource_events"]["memory.events"][key] == 0 + for key in ["high", "max", "oom", "oom_kill"]) + return {"wall_seconds": scope["elapsed_seconds"], + "cpu_seconds": observation["cpu_stat"]["usage_usec"] / 1e6, + "kernel_family_peak_bytes_including_page_cache": observation["memory_peak_bytes"], + "resource_stop": scope["abort"], "cleanup_verified": True, + "final_whole_scope_counters": scope["resource_events_final"], + "guard_sources_sha256": scope["payload_observer_sources_sha256"], + "log_sha256": hashlib.sha256(path.read_bytes()).hexdigest()} + + +def sdk(case, count): + report = json.loads((case / "report.json").read_text()) + assert report["status"] == "passed" and report["cleanup_verified"] + catalog = json.loads((case / "catalog.json").read_text()) + calls = json.loads((case / "transcript.json").read_text()) + inputs = [{key: tool[key] for key in ["name", "description", "inputSchema"]} + for tool in catalog["tools"]] + arguments = sum(count(compact({"name": call["name"], "arguments": call["arguments"]})) + for call in calls) + text = sum(count(block["text"]) for call in calls for block in call["reply"]["content"] + if block["type"] == "text") + return {"case": case.name, "kind": "SDK", "profile": report["profile"], + "artifact_sha256": report["artifact_sha256"], + "tool_count": len(catalog["tools"]), "calls": report["calls"], + "parent_evidence_id": report["parent_evidence_id"], + "parent_semantic_sha256": report["parent_semantic_sha256"], + "token_estimates": {"input_catalog_once": count(compact(inputs)), + "arguments": arguments, "text_replies": text, + "whole_task": count(compact(inputs)) + arguments + text, + "raw_catalog_including_output_schemas": count(compact(catalog))}, + "actual_model_usage": None, **resource(case), + "receipt_sha256": {name: hashlib.sha256((case / name).read_bytes()).hexdigest() + for name in ["report.json", "catalog.json", "transcript.json"]}} + + +def model(case): + summary = json.loads((case / "summary.json").read_text()) + assert summary["exit_code"] == 0 and not summary["timeout"] + assert summary["selected_artifact_bytes_unchanged"] + assert json.loads((case / "cleanup.json").read_text())["profile_removed"] + return {"case": case.name, "kind": "model", "client": summary["codex_version"], + "requested_model": summary["requested_model"], "reasoning": summary["reasoning"], + "model_identity_scope": summary["model_identity_scope"], + "actual_model_usage": summary["usage"], + "artifact_sha256": summary["selected_artifact_sha256_before"], + "calls": summary["completed_mcp_calls"], + "factual_correctness": "requires independent oracle and answer review", + **resource(case), + "receipt_sha256": {name: hashlib.sha256((case / name).read_bytes()).hexdigest() + for name in ["summary.json", "answer.txt", "events.jsonl", + "trial-settings.json", "context-receipt.json"]}} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("cases", type=Path, nargs="+") + args = parser.parse_args() + encoding = tiktoken.get_encoding("cl100k_base") + count = lambda value: len(encoding.encode(value, disallowed_special=())) + result = {"tokenizer": f"tiktoken {tiktoken.__version__}", "encoding": encoding.name, + "estimate_scope": "One complete name/description/input-schema catalog, every RPC argument and text reply; raw output-schema transport counted separately. Model usage is separate and is not inferred from estimates.", + "records": [sdk(case, count) if (case / "report.json").exists() else model(case) + for case in args.cases]} + args.output.write_text(json.dumps(result, indent=2) + "\n") + + +if __name__ == "__main__": + main() diff --git a/experiments/analysis-efficiency/mcp-input-catalog-oracle.py b/experiments/analysis-efficiency/mcp-input-catalog-oracle.py new file mode 100644 index 0000000000..1806edbea1 --- /dev/null +++ b/experiments/analysis-efficiency/mcp-input-catalog-oracle.py @@ -0,0 +1,54 @@ +"""Independent ELF64 little-endian / zlib oracle for the pinned Alpine corpus.""" +import argparse +import hashlib +import json +from pathlib import Path +import struct +import zlib + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("artifact", type=Path) + parser.add_argument("output", type=Path) + args = parser.parse_args() + assert args.artifact.stat().st_size <= 256 * 1024 ** 2 + binary = args.artifact.read_bytes() + assert binary[:6] == b"\x7fELF\x02\x01" + table = struct.unpack_from(" `view.offset: Too small: expected number to be >=0`\n\nCalled `inspect_rust_binary` exactly once. No retry or binary execution occurred. The error returned no Evidence ID.", + "scope": "Independent ELF/zlib/JSON package facts and derived counts, real returned Evidence ID, digest, static-only and runtime-dependency limitations; negative cases preserve the server refusal." + }, + { + "case": "model-compact-negative", + "passed": true, + "answer": "Yes. The canonical REA server rejected offset `-1` with `invalid_request` / `invalid_input`:\n\n> `view.offset: Too small: expected number to be >=0`\n\nCalled `inspect_rust_binary` exactly once. No corrected retry or binary execution occurred. The error returned no Evidence ID.", + "scope": "Independent ELF/zlib/JSON package facts and derived counts, real returned Evidence ID, digest, static-only and runtime-dependency limitations; negative cases preserve the server refusal." + } + ], + "failed_attempts": [ + { + "case": "sdk-full-ripgrep-1", + "reason": "The first benchmark used evidence_id at the root instead of source.kind=retained-evidence. The server correctly refused the harness input; corrected before accepted measurements.", + "excluded_from_performance_cohorts": true, + "log_sha256": "acd72e2210cb906ed07cb061390e57bb03ff46d47568f054eb1ced0ba2139d66" + } + ], + "model_question_templates": { + "direct": "Use REA to inspect the local Rust binary {{artifact_path}} without executing it. Return the exact name, version, dependency kind, and original index of every observed serde package. Keep the answer concise, cite the real parent Evidence ID and artifact digest, and state any limitation. Choose the smallest sufficient response.\n", + "broad": "Determine which embedded dependency metadata REA can recover from {{artifact_path}} without executing it. Report the root package name/version, total package and edge counts, whether serde is present, its observed version(s), the artifact SHA256 and supporting Evidence ID. Distinguish observed metadata from inferences; keep the answer concise.\n", + "negative": "Use REA inspect_rust_binary once on {{artifact_path}} with view {\"collection\":\"packages\",\"offset\":-1}. Report whether the canonical server rejects the invalid offset. Do not retry with a corrected offset and do not execute the binary.\n" + }, + "latest_main_integration": { + "base_revision": "25e1a24a3ed3671a0148757555f18d16c29d936a", + "comparison_scope": "One explicit-full/default pair on the rebased candidate; not added to original before/after cohorts.", + "records": [ + { + "case": "integration-full", + "kind": "SDK", + "profile": "full", + "artifact_sha256": "9a86e9ca47902a869fdf1886f4cf17714f4fa5655427128beadff498cfb392db", + "tool_count": 213, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 365.156704, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2194.9995639999997, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 8.802002000000357, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 5.010089999999764, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 28.393213999999716, + "is_error": false + } + ], + "parent_evidence_id": "ev_8c9124795a5cf45c2bf816bbfa0ced9c3025550e72d4e3facf5fb2e266d3dadb", + "parent_semantic_sha256": "df1930ee07a13eb4b47c91f2834c4b0f8d99089e051a02083c7c3cf03bb30467", + "token_estimates": { + "input_catalog_once": 200425, + "arguments": 287, + "text_replies": 3560, + "whole_task": 204272, + "raw_catalog_including_output_schemas": 903497 + }, + "actual_model_usage": null, + "wall_seconds": 7.171, + "cpu_seconds": 7.019359, + "kernel_family_peak_bytes_including_page_cache": 874827776, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "guard_sources_sha256": { + "resource-run.py": "7a9642ab86ba34290564f636eb080cacf38b4185a254028680b238bff172993b", + "resource_payload.py": "6f089e645459eba80c5d73e39c21606c78173a44daf49fabc3b88c4631ce51c6" + }, + "log_sha256": "f22505fae66da7a5083cbb93388dce93d9dfd9b7da0de66f48e70a43ff6e8ba4", + "receipt_sha256": { + "report.json": "a4782431a109d8501b78ac41db27e71e6f82c3b333dd37aed893c3b12aae49d1", + "catalog.json": "1640dd811f3bb6cf2b21a6d0e7b9b750bc45f514f56470f99800917da161e706", + "transcript.json": "2a34d1b655bcf1873c8cb3156568525ce41ef53f29973eda0df393aca51d306d" + }, + "observed_resource_events": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "bounds": { + "hard_memory_bytes": 4294967296, + "early_stop_bytes": 3221225472, + "cpu_quota_percent": 100, + "cpu_affinity": [15], + "task_ceiling": 192 + } + }, + { + "case": "integration-default", + "kind": "SDK", + "profile": "default", + "artifact_sha256": "9a86e9ca47902a869fdf1886f4cf17714f4fa5655427128beadff498cfb392db", + "tool_count": 213, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 361.093202, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2230.8176569999996, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 7.458020999999462, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 4.059271000000081, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 28.007474000000002, + "is_error": false + } + ], + "parent_evidence_id": "ev_8c9124795a5cf45c2bf816bbfa0ced9c3025550e72d4e3facf5fb2e266d3dadb", + "parent_semantic_sha256": "df1930ee07a13eb4b47c91f2834c4b0f8d99089e051a02083c7c3cf03bb30467", + "token_estimates": { + "input_catalog_once": 47868, + "arguments": 287, + "text_replies": 3560, + "whole_task": 51715, + "raw_catalog_including_output_schemas": 750743 + }, + "actual_model_usage": null, + "wall_seconds": 7.173, + "cpu_seconds": 6.994262, + "kernel_family_peak_bytes_including_page_cache": 872333312, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "guard_sources_sha256": { + "resource-run.py": "7a9642ab86ba34290564f636eb080cacf38b4185a254028680b238bff172993b", + "resource_payload.py": "6f089e645459eba80c5d73e39c21606c78173a44daf49fabc3b88c4631ce51c6" + }, + "log_sha256": "56c3438b3e12771d202ecbe60b775a9eb0000ee8b4fb9e629692a8d45790daca", + "receipt_sha256": { + "report.json": "928db422930cb420d04a165ef20f2ba569bfe9f156fbf049e1f2d600172d5ade", + "catalog.json": "56a48174c029aa3abfb9a56e38a40d021919899c5736f129d24c684cfaf01962", + "transcript.json": "98e2d713f1e3ae8b9561e1b1eea282e04eaf0d15eb515d75a23f2a905e0b8564" + }, + "observed_resource_events": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "bounds": { + "hard_memory_bytes": 4294967296, + "early_stop_bytes": 3221225472, + "cpu_quota_percent": 100, + "cpu_affinity": [15], + "task_ceiling": 192 + } + } + ] + }, + "executed_validation": { + "initial_focused": "10 files: 88 passed, 1 skipped, 2 failed (stale default-identity assertion and 30s canonical schema timeout).", + "corrected_rerun": "Both failed suites: 26 passed with serial files and unchanged deadlines; canonical schema timeout cause unproven.", + "latest_integration": "3 identity/profile/production doctor files: 32 passed.", + "check": "Passed on original base and latest-main integration.", + "docs_check_first_attempt": "Failed because the new README anchor differed from translated README link inventories; reused the existing MCP guide link.", + "docs_checks": "Initial checks caught a new/duplicate README link absent from translated inventories; the introduction now names the existing guide without adding a link.", + "docs_check_final": "Passed after preserving the translated README link inventory." + }, + "review_followup": { + "root_cause": "The previous ambiguous CATALOG_IDENTITY denoted full advertisement while consumers assumed it represented the default live server.", + "structural_fix": "Explicit FULL_CATALOG_IDENTITY and DEFAULT_CATALOG_IDENTITY replace the ambiguous export; counts-only canonical consumers retain full identity; default diagnostics use the default; readiness records the typed live server catalog; Claude expectations use the shared input/output profile parsers.", + "source_sha256": { + "scripts/generate-skill-metadata.mjs": "e88f47e80a6d0e6b7eb14a464b7e5f3e44993529ecbc0ed6604c9569edb68ae7", + "scripts/lib/catalog-builders.mjs": "bb0cf8ebfe5a7d0ff090c1ddae6dd8760577b3b824cc1fd7cd9f2e83b558919e", + "scripts/lib/product-catalog.mjs": "20182284c3e1df7b37b858affb75028aa24724901147b638f4a07190908d9b9b", + "scripts/lib/release-package-canary.mjs": "c0f3e59832a0cce00829a3bf1e033588d9d840736d1305dbbad636506b2ac6d3", + "scripts/verify-claude-client.mjs": "8b098bbc00e9cb4877f86564245a16db696a2799fe6c48fe0e65f324629da4d7", + "scripts/verify-package-pack.mjs": "11b61499b846e896bc65689ca69c1c45db1ab4f0b14614518371a1dd59368a92", + "scripts/verify-reconstruction-readiness.mjs": "a7f4614694e0cc196b726b989a5bf5b0fdae574c48393e1a981699bc665f9d28", + "src/application/Doctor.fixture.ts": "e98481ff468f208561dd8b8132b8326317a088706a7ce366f5dd0dc521031292", + "src/application/Doctor.ts": "cd10aab92516c234e41c1597b35a126be289aae5dfaef0d964ef27ae5c648855", + "src/application/Setup.fixture.ts": "175487d3f6916852acac7635fccfb61d1616aca28d198278ec7d31a517b68c77", + "src/catalogIdentity.ts": "54e67816c09ab1abb12955ed0e87f4aa5e6f8607668aadb24ad1dc74e0ca85a3", + "src/server/EvidenceMcpServer.ts": "4f5913307b381815e7e319d777f4efe89fbee98da87978925028239b8773c88b", + "src/serverIdentity.ts": "b90af4f999ab0aadec352dfd3244b42f77b835542ec588577c8f3c1438f005df", + "tests/acceptance/applications/runtime.test.ts": "b57883f459018053893a268554a05910bf1aaa480599c6f7cad79cbbd0f9a1cf", + "tests/boundary/filesystem/dependencyInstall.test.ts": "055bb0cc573fd0a3b81155177e0dd913ee28bc776fbe370ef53076119e2b5cbd", + "tests/boundary/filesystem/generatedSkill.test.ts": "3197d6a2274fd505631b274329811c5b25c9f870c0bc6952f04f203b7cf360b2", + "tests/boundary/mcp/mcpDoctor.test.ts": "c88a665e8560ac97755ca809e6f45b2ad2354809a2eb7100470783a764f9decb", + "tests/boundary/mcp/outputSchemaProfile.test.ts": "d84e1e0636b811d48dde5d9b532de937f9fd3db49f394e20650e8258aa52ba85", + "tests/boundary/mcp/serverIdentity.test.ts": "dbe59d532b4a53635e502e93577fbd9f23d754ad31e715af73d461ada126934f", + "tests/boundary/process/packageTarListing.test.ts": "20581fed2bbbc0c3a3756560026cb138a66074e20f8c159e191d42de4e000f8d" + }, + "validation": { + "review-tests": { + "passed": true, + "cpu_seconds": 137.239542, + "family_peak_bytes": 3058552832, + "log_sha256": "b380cc2270668d047245b3426cd143271658a7ad5da1aad3db073c356605c26c" + }, + "review-check": { + "passed": true, + "cpu_seconds": 28.654837, + "family_peak_bytes": 1334308864, + "log_sha256": "b3182a46e432b0b0503088abec8abf8d0224c3da8025eb4f0e8777ac789e5e27" + }, + "review-docs": { + "passed": true, + "cpu_seconds": 14.641149, + "family_peak_bytes": 507789312, + "log_sha256": "2f4aea6a070f6012ee7587ead635342cf9d318699b9701a8de09ef79021878d2" + }, + "readiness-default": { + "passed": true, + "cpu_seconds": 29.813437, + "family_peak_bytes": 1241534464, + "log_sha256": "e864436ba2db543f6941bfb83ce58d37fdf1c17ca5ff38a78674e7c812b9abb7" + }, + "readiness-full": { + "passed": true, + "cpu_seconds": 28.197651, + "family_peak_bytes": 1157550080, + "log_sha256": "f18fc53b6985aeef4d951cb2183eac7c12ea5c85d3426908a1766490ddf26da9" + }, + "readiness-compact-off": { + "passed": true, + "cpu_seconds": 18.102913, + "family_peak_bytes": 861978624, + "log_sha256": "0fa90210e8e125f4f0c09db69a594f62ffc1f1c11fc71b848f701bd99e13baa0" + }, + "review-sdk-default": { + "passed": true, + "cpu_seconds": 7.375525, + "family_peak_bytes": 861224960, + "log_sha256": "e95b4d93d772530b00a59ea09a4966a3b8eac9a99e640680f53af86f6288cf69" + } + }, + "focused_tests": "9 files: 106 passed, 2 skipped.", + "readiness_profiles": ["default", "full", "compact input / output off"], + "sdk_catalog_and_complete_parent_unchanged": true, + "native_selected_identity_proof": [ + { + "profile": "default", + "passed": true, + "saved_native_status_sha256": "b8fa284ad96587530d2664835a83f139b5ded29ddd4e5ab9fdaa15efc9500e00", + "catalog": { + "counts": { + "cli_commands": 160, + "mcp_tools": 213, + "mcp_prompts": 6 + }, + "digests": { + "tools_sha256": "b8c3712bb47a049ccd249b0de3afacdc27da0aea465baaadc8b7f17eef61daf3", + "prompts_sha256": "84c719b1aa48a7b12ad29e16278fc4c4254d9364f6620792cba6ceaef5b4a98c", + "combined_sha256": "0389e5636823b38f9c8a1561554e0c32d22a43528e792dc0995583ba37c460e4" + } + }, + "readiness_catalog_digest": "0389e5636823b38f9c8a1561554e0c32d22a43528e792dc0995583ba37c460e4" + }, + { + "profile": "full", + "passed": true, + "saved_native_status_sha256": "d1cb209b015da2a6662a56bcd167bf515fda8dd0baa1145b38d254f4eec0725c", + "catalog": { + "counts": { + "cli_commands": 160, + "mcp_tools": 213, + "mcp_prompts": 6 + }, + "digests": { + "tools_sha256": "7362fced4ad2726682316d0f1e8acf65e90c59192ef163c7cc4ce00c3d756767", + "prompts_sha256": "84c719b1aa48a7b12ad29e16278fc4c4254d9364f6620792cba6ceaef5b4a98c", + "combined_sha256": "5ab16853c717a6e98473cb6c5ac1871e186b9784f32ce1ac54d59ccaf1d3634e" + } + }, + "readiness_catalog_digest": "5ab16853c717a6e98473cb6c5ac1871e186b9784f32ce1ac54d59ccaf1d3634e" + } + ], + "native_client_limit": { + "client": "Claude Code 2.1.296", + "model": "loopback Anthropic fixture; no live API claim", + "end_to_end_passed": false, + "reason": "The unchanged verifier requires binary_session inline; native Claude persists the ~55KB result. Same failure reproduced on untouched baseline product code after preparing its packaged skill.", + "probes": [ + { + "case": "claude-default", + "fixture_failure": "AssertionError [ERR_ASSERTION]: Claude must return the inline binary_session result before continuing", + "payload_exit_code": 1, + "cleanup_verified": true, + "saved_status_catalog": { + "counts": { + "cli_commands": 160, + "mcp_tools": 213, + "mcp_prompts": 6 + }, + "digests": { + "tools_sha256": "b8c3712bb47a049ccd249b0de3afacdc27da0aea465baaadc8b7f17eef61daf3", + "prompts_sha256": "84c719b1aa48a7b12ad29e16278fc4c4254d9364f6620792cba6ceaef5b4a98c", + "combined_sha256": "0389e5636823b38f9c8a1561554e0c32d22a43528e792dc0995583ba37c460e4" + } + }, + "saved_status_bytes": 55471 + }, + { + "case": "claude-full", + "fixture_failure": "AssertionError [ERR_ASSERTION]: Claude must return the inline binary_session result before continuing", + "payload_exit_code": 1, + "cleanup_verified": true, + "saved_status_catalog": { + "counts": { + "cli_commands": 160, + "mcp_tools": 213, + "mcp_prompts": 6 + }, + "digests": { + "tools_sha256": "7362fced4ad2726682316d0f1e8acf65e90c59192ef163c7cc4ce00c3d756767", + "prompts_sha256": "84c719b1aa48a7b12ad29e16278fc4c4254d9364f6620792cba6ceaef5b4a98c", + "combined_sha256": "5ab16853c717a6e98473cb6c5ac1871e186b9784f32ce1ac54d59ccaf1d3634e" + } + }, + "saved_status_bytes": 55471 + }, + { + "case": "claude-baseline-full-complete", + "fixture_failure": "AssertionError [ERR_ASSERTION]: Claude must return the inline binary_session result before continuing", + "payload_exit_code": 1, + "cleanup_verified": true, + "saved_status_catalog": { + "counts": { + "cli_commands": 160, + "mcp_tools": 213, + "mcp_prompts": 6 + }, + "digests": { + "tools_sha256": "3e27909ade9f9f6e61f361fd0501c9a712b7412666b00758448195749ccb680e", + "prompts_sha256": "84c719b1aa48a7b12ad29e16278fc4c4254d9364f6620792cba6ceaef5b4a98c", + "combined_sha256": "71e51816c31c8e9f0dd6956c69b7f36e00ce53e0d5579b583b944979036028ed" + } + }, + "saved_status_bytes": 55473 + } + ] + } + }, + "ci_followup": { + "integration_base": "6d356b84344659085f2b912857b5e27fe0160c9b", + "first_ci_run": 39601241054, + "cause": "Three tests assumed the old full default in nested input rejection/property guidance.", + "fix": "Parameterize affected ZIP retention, browser endpoint and literal-seed workflows for full and compact; retain strict full advertisement assertions and enforce canonical valid/invalid behavior for both.", + "serial_tests": "4 files, 27 passed.", + "validation": { + "ci-regression-tests-serial": { + "passed": true, + "family_peak_bytes": 2452987904, + "cpu_seconds": 216.800823, + "log_sha256": "8fe90cbb0c3c283559ae66be66035117e7884e3565b1637aaf61307400efa73f" + }, + "check-before-review-push": { + "passed": true, + "family_peak_bytes": 1326555136, + "cpu_seconds": 39.301628, + "log_sha256": "ce739f3aca7279ef2da53faba08f6e74581a0801e2014c7eb6d6aa402f782e4c" + }, + "docs-before-review-push": { + "passed": true, + "family_peak_bytes": 511111168, + "cpu_seconds": 11.300216, + "log_sha256": "7ed6e5274552bf34d89c2b70cdb792b8274b0dbbf80e61884657bb386d238cae" + } + }, + "incomplete_attempt": { + "log_sha256": "22b6035a7b87f0642ab96afc8b25ef29911b69d37082849ef753f4302fe1bf4d", + "reason": "Parallel files reached the 3 GiB early-stop threshold (4 GiB hard limit), no OOM; incomplete, not a passing run. Serial files passed without raising limits." + }, + "source_sha256": { + "tests/boundary/mcp/artifactMcp.inlineInventory.test.ts": "c6c875dd49fc76f7b7b4aae8b9a1f3d379748278bbd0ec2c15f7ae2a7d0ed364", + "tests/boundary/mcp/browserMcp.test.ts": "bf9d0ad7f8061d2d7963de6a126ef4ed108a9e4eccc8107cedcd4360ac7fe096", + "tests/boundary/mcp/retainedApplicationEvidenceMcp.test.ts": "1d0d7121d339100e72f454d2b0a34a307a2e3609df3c416dca3674844761a6ba" + } + }, + "accounting_followup": { + "original_cost_script_sha256": "aa9e71a730e8f9a853fe0e285e09796b84babc50ad685e18e7bc52326263adf6", + "current_cost_script_sha256": "ad32e6726f892be16cc7d352118f7e9f9f0a85649d9edd1049b5d73483df8078", + "change": "Document receipt redirection and accounting; append .log to the entire directory basename rather than replacing its suffix.", + "original_records_reprocessed_without_changes": true, + "additional_real_dotted_directory_validation": { + "case": "reproduce.sdk.1", + "kind": "SDK", + "profile": "default", + "artifact_sha256": "9a86e9ca47902a869fdf1886f4cf17714f4fa5655427128beadff498cfb392db", + "tool_count": 213, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 502.6376340000006, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2992.545244000001, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 9.582700000000841, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 6.301149999999325, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 37.41663999999946, + "is_error": false + } + ], + "parent_evidence_id": "ev_8c9124795a5cf45c2bf816bbfa0ced9c3025550e72d4e3facf5fb2e266d3dadb", + "parent_semantic_sha256": "df1930ee07a13eb4b47c91f2834c4b0f8d99089e051a02083c7c3cf03bb30467", + "token_estimates": { + "input_catalog_once": 47868, + "arguments": 289, + "text_replies": 3562, + "whole_task": 51719, + "raw_catalog_including_output_schemas": 750743 + }, + "actual_model_usage": null, + "wall_seconds": 8.687, + "cpu_seconds": 8.489351, + "kernel_family_peak_bytes_including_page_cache": 870670336, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "guard_sources_sha256": { + "resource-run.py": "7a9642ab86ba34290564f636eb080cacf38b4185a254028680b238bff172993b", + "resource_payload.py": "6f089e645459eba80c5d73e39c21606c78173a44daf49fabc3b88c4631ce51c6" + }, + "log_sha256": "3d3390f3886b8905d5735ae484e0f142017716482d7e838ae5f5648a4513b79e", + "receipt_sha256": { + "report.json": "fa45533fd6909cfa1267f1b9c12af2c3d402ada69e95e97049f6cfb8149f9687", + "catalog.json": "56a48174c029aa3abfb9a56e38a40d021919899c5736f129d24c684cfaf01962", + "transcript.json": "ae7e65f061e7628d2c21562e1da0fe81168f73e6e124bf32b9d5fecffe88078d" + } + }, + "included_in_original_performance_cohorts": false + }, + "full_presentation_test_followup": { + "base_revision": "6d356b84344659085f2b912857b5e27fe0160c9b", + "change": "Full schema/example conformance cases explicitly select full presentation. Default source comparison and JavaScript facts workflows retain actual canonical refusals; their producer-schema assertions remain strict. Regenerated ignored catalog metadata before the corrected run.", + "executed_runs": [ + { + "log": "ci-second-regressions.log", + "log_sha256": "ede521d31bfcc3db2799166dd3a7a7a433245452bf46182d4000889c6f8f0618", + "outcome": "27 passed; five full-presentation assumptions failed, including stale generated metadata", + "resource_receipt": { + "resource_scope": "rea-efficiency-6c78d365ef9d42d98dfa7a4407a60724.scope", + "peak_cgroup_bytes_sampled": 2150170624, + "elapsed_seconds": 186.422, + "abort": null, + "hard_memory_bytes": 4294967296, + "early_stop_bytes": 3221225472, + "cpu_quota_percent": 200, + "cpu_affinity": [14, 15], + "task_ceiling": 192, + "peak_tasks_sampled": 67, + "resource_events_last_observed": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "resource_events_final": null, + "resource_counters_verified": false, + "cpu_stat_last_observed": { + "usage_usec": 219960849, + "user_usec": 202742520, + "system_usec": 17218328, + "nr_periods": 1863, + "nr_throttled": 33, + "throttled_usec": 44521, + "nr_bursts": 0, + "burst_usec": 0 + }, + "payload_resource_observation": { + "phase": "payload-exited-observer-alive", + "ownership_token": "rea-efficiency-6c78d365ef9d42d98dfa7a4407a60724.scope", + "payload_exit_code": 1, + "observer_pid": 771268, + "remaining_processes": [771268], + "payload_cleanup_verified": true, + "cpu_stat": { + "usage_usec": 219959328, + "user_usec": 202741119, + "system_usec": 17218209, + "nr_periods": 1863, + "nr_throttled": 33, + "throttled_usec": 44521, + "nr_bursts": 0, + "burst_usec": 0 + }, + "memory_peak_bytes": 2152132608, + "control_group": "user.slice/user-1004.slice/user@1004.service/app.slice/rea-efficiency-6c78d365ef9d42d98dfa7a4407a60724.scope", + "resource_events": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + } + }, + "payload_resource_observation_verified": true, + "payload_observation_scope": "Counters at payload exit while observer remains alive; not final whole-scope counters.", + "payload_observer_sources_sha256": { + "resource-run.py": "378868d8ea3589e3acd640397fa4a543e0a2845768bb35a41119051797690343", + "resource_payload.py": "6f089e645459eba80c5d73e39c21606c78173a44daf49fabc3b88c4631ce51c6" + } + } + }, + { + "log": "ci-second-regressions-fixed.log", + "log_sha256": "a11f10bfb05303eb9b797f9a701d5d86e686ee0aa049804a5668a140596bcf0e", + "outcome": "31 passed; one 20-second child-process timeout", + "resource_receipt": { + "resource_scope": "rea-efficiency-978bb49765c14b3cb1922953928da222.scope", + "peak_cgroup_bytes_sampled": 2606223360, + "elapsed_seconds": 251.442, + "abort": null, + "hard_memory_bytes": 4294967296, + "early_stop_bytes": 3221225472, + "cpu_quota_percent": 200, + "cpu_affinity": [14, 15], + "task_ceiling": 192, + "peak_tasks_sampled": 67, + "resource_events_last_observed": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "resource_events_final": null, + "resource_counters_verified": false, + "cpu_stat_last_observed": { + "usage_usec": 294221945, + "user_usec": 270973501, + "system_usec": 23248443, + "nr_periods": 2513, + "nr_throttled": 44, + "throttled_usec": 38486, + "nr_bursts": 0, + "burst_usec": 0 + }, + "payload_resource_observation": { + "phase": "payload-exited-observer-alive", + "ownership_token": "rea-efficiency-978bb49765c14b3cb1922953928da222.scope", + "payload_exit_code": 1, + "observer_pid": 789544, + "remaining_processes": [789544], + "payload_cleanup_verified": true, + "cpu_stat": { + "usage_usec": 294220614, + "user_usec": 270972276, + "system_usec": 23248338, + "nr_periods": 2513, + "nr_throttled": 44, + "throttled_usec": 38486, + "nr_bursts": 0, + "burst_usec": 0 + }, + "memory_peak_bytes": 2606747648, + "control_group": "user.slice/user-1004.slice/user@1004.service/app.slice/rea-efficiency-978bb49765c14b3cb1922953928da222.scope", + "resource_events": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + } + }, + "payload_resource_observation_verified": true, + "payload_observation_scope": "Counters at payload exit while observer remains alive; not final whole-scope counters.", + "payload_observer_sources_sha256": { + "resource-run.py": "378868d8ea3589e3acd640397fa4a543e0a2845768bb35a41119051797690343", + "resource_payload.py": "6f089e645459eba80c5d73e39c21606c78173a44daf49fabc3b88c4631ce51c6" + } + } + }, + { + "log": "contract-presentation-fresh.log", + "log_sha256": "cd65b5322e28731cc276dd7251884104db24bbb8dd4f542733756b87d3c2d1fd", + "outcome": "one passed in a fresh process with unchanged deadlines", + "resource_receipt": { + "resource_scope": "rea-efficiency-4e6e2188b9c040edaa35cb43654a1ed8.scope", + "peak_cgroup_bytes_sampled": 1054052352, + "elapsed_seconds": 15.652, + "abort": null, + "hard_memory_bytes": 4294967296, + "early_stop_bytes": 3221225472, + "cpu_quota_percent": 200, + "cpu_affinity": [14, 15], + "task_ceiling": 192, + "peak_tasks_sampled": 33, + "resource_events_last_observed": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + }, + "resource_events_final": null, + "resource_counters_verified": false, + "cpu_stat_last_observed": { + "usage_usec": 18652902, + "user_usec": 16999452, + "system_usec": 1653450, + "nr_periods": 156, + "nr_throttled": 0, + "throttled_usec": 0, + "nr_bursts": 0, + "burst_usec": 0 + }, + "payload_resource_observation": { + "phase": "payload-exited-observer-alive", + "ownership_token": "rea-efficiency-4e6e2188b9c040edaa35cb43654a1ed8.scope", + "payload_exit_code": 0, + "observer_pid": 793414, + "remaining_processes": [793414], + "payload_cleanup_verified": true, + "cpu_stat": { + "usage_usec": 18652236, + "user_usec": 16998845, + "system_usec": 1653390, + "nr_periods": 156, + "nr_throttled": 0, + "throttled_usec": 0, + "nr_bursts": 0, + "burst_usec": 0 + }, + "memory_peak_bytes": 1055846400, + "control_group": "user.slice/user-1004.slice/user@1004.service/app.slice/rea-efficiency-4e6e2188b9c040edaa35cb43654a1ed8.scope", + "resource_events": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + } + }, + "payload_resource_observation_verified": true, + "payload_observation_scope": "Counters at payload exit while observer remains alive; not final whole-scope counters.", + "payload_observer_sources_sha256": { + "resource-run.py": "378868d8ea3589e3acd640397fa4a543e0a2845768bb35a41119051797690343", + "resource_payload.py": "6f089e645459eba80c5d73e39c21606c78173a44daf49fabc3b88c4631ce51c6" + } + } + } + ], + "all_five_files_have_passing_checks": true, + "combined_run_passed": false, + "timeout_cause": "Not established; fresh-process retry passed with unchanged deadlines.", + "included_in_performance_cohorts": false + }, + "godot_profile_followup": { + "ci_head": "c8ed337568c8c28ccbcb5a59de3e35759ee8f250", + "ci_result": "Portable tests: 15,703 passed, 29 skipped; one Godot full-schema assumption failed.", + "change": "Run the retained PCK inventory workflow under full and compact; assert full advertisement constraints and canonical invalid-field refusal under both.", + "local_result": "Three passed, including both profiles and encrypted-directory unknown coverage.", + "log_sha256": "01a2c366dff66262d145368832bb5e4344768bfeda5fcce579afbff44a0be864", + "elapsed_seconds": 75.687, + "payload_resource_observation": { + "phase": "payload-exited-observer-alive", + "ownership_token": "rea-efficiency-c5daece4d14c4998abd5732974cbf5e6.scope", + "payload_exit_code": 0, + "observer_pid": 811836, + "remaining_processes": [811836], + "payload_cleanup_verified": true, + "cpu_stat": { + "usage_usec": 83425505, + "user_usec": 75026217, + "system_usec": 8399287, + "nr_periods": 755, + "nr_throttled": 1, + "throttled_usec": 33, + "nr_bursts": 0, + "burst_usec": 0 + }, + "memory_peak_bytes": 1815400448, + "control_group": "user.slice/user-1004.slice/user@1004.service/app.slice/rea-efficiency-c5daece4d14c4998abd5732974cbf5e6.scope", + "resource_events": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + } + }, + "included_in_performance_cohorts": false + }, + "cli_identity_followup": { + "source_derived_root_cause": "Direct CLI status omitted the selected catalog, triggering a compact fallback even for explicit full. Parsed output profile was also discarded by AppConfig.", + "structural_change": "CLI status derives both input/output profiles from parsed config. Runtime identity requires a caller-supplied catalog; both production callers pass it explicitly. No ambient config reinterpretation at the identity owner.", + "actual_public_entrypoints": [ + "stdio binary_session", + "rea capabilities --json", + "rea providers --json" + ], + "profile_cases": [ + "default compact/full", + "full/full", + "compact/full", + "full/off", + "compact/off" + ], + "executed_runs": [ + { + "log": "cli-profile-mcpDoctor.log", + "log_sha256": "7f5c79d2dfd1589a47041eeb4886858858560fc8cfc0860cfa496aef7b860dde", + "passed_tests": 14, + "elapsed_seconds": 83.973, + "payload_resource_observation": { + "phase": "payload-exited-observer-alive", + "ownership_token": "rea-efficiency-d7bf0512b44140cf91635a1a692ff825.scope", + "payload_exit_code": 0, + "observer_pid": 821163, + "remaining_processes": [821163], + "payload_cleanup_verified": true, + "cpu_stat": { + "usage_usec": 100650679, + "user_usec": 87453243, + "system_usec": 13197435, + "nr_periods": 834, + "nr_throttled": 10, + "throttled_usec": 4235, + "nr_bursts": 0, + "burst_usec": 0 + }, + "memory_peak_bytes": 1524985856, + "control_group": "user.slice/user-1004.slice/user@1004.service/app.slice/rea-efficiency-d7bf0512b44140cf91635a1a692ff825.scope", + "resource_events": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + } + } + }, + { + "log": "cli-profile-serverIdentity.log", + "log_sha256": "aa75647dd78a6b4df22dc2d0f2816991b941800d7af72664b57c973950084d1f", + "passed_tests": 11, + "elapsed_seconds": 49.302, + "payload_resource_observation": { + "phase": "payload-exited-observer-alive", + "ownership_token": "rea-efficiency-8f537b501576473b9b5e32f0e7627f6b.scope", + "payload_exit_code": 0, + "observer_pid": 823954, + "remaining_processes": [823954], + "payload_cleanup_verified": true, + "cpu_stat": { + "usage_usec": 54311975, + "user_usec": 50014222, + "system_usec": 4297752, + "nr_periods": 492, + "nr_throttled": 1, + "throttled_usec": 3981, + "nr_bursts": 0, + "burst_usec": 0 + }, + "memory_peak_bytes": 1885233152, + "control_group": "user.slice/user-1004.slice/user@1004.service/app.slice/rea-efficiency-8f537b501576473b9b5e32f0e7627f6b.scope", + "resource_events": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + } + } + } + ], + "included_in_performance_cohorts": false + }, + "latest_main_8e_integration": { + "base_revision": "8e3bf56e1", + "tested_code_revision": "2a6a0812886e1a4e022042947a59575bed1bc954", + "scope": "Separate integration after upstream browser Evidence schema changes; not a repeated performance cohort.", + "real_sdk_costs": { + "case": "latest-main-8e-sdk", + "kind": "SDK", + "profile": "default", + "artifact_sha256": "9a86e9ca47902a869fdf1886f4cf17714f4fa5655427128beadff498cfb392db", + "tool_count": 213, + "calls": [ + { + "name": "inspect_rust_binary", + "elapsed_ms": 370.39350399999967, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 2317.916096, + "is_error": false + }, + { + "name": "inspect_analysis_view", + "elapsed_ms": 9.645430000000488, + "is_error": false + }, + { + "name": "inspect_rust_binary", + "elapsed_ms": 5.3366600000008475, + "is_error": true + }, + { + "name": "export_evidence_bundle", + "elapsed_ms": 30.598637999999482, + "is_error": false + } + ], + "parent_evidence_id": "ev_8c9124795a5cf45c2bf816bbfa0ced9c3025550e72d4e3facf5fb2e266d3dadb", + "parent_semantic_sha256": "df1930ee07a13eb4b47c91f2834c4b0f8d99089e051a02083c7c3cf03bb30467", + "token_estimates": { + "input_catalog_once": 47868, + "arguments": 290, + "text_replies": 3563, + "whole_task": 51721, + "raw_catalog_including_output_schemas": 750781 + }, + "actual_model_usage": null, + "wall_seconds": 7.785, + "cpu_seconds": 7.533643, + "kernel_family_peak_bytes_including_page_cache": 873308160, + "resource_stop": null, + "cleanup_verified": true, + "final_whole_scope_counters": null, + "guard_sources_sha256": { + "resource-run.py": "7a9642ab86ba34290564f636eb080cacf38b4185a254028680b238bff172993b", + "resource_payload.py": "6f089e645459eba80c5d73e39c21606c78173a44daf49fabc3b88c4631ce51c6" + }, + "log_sha256": "528bc7a1373c01facb84706db0848ed66905e8dfecba17275c3bb2b866a5205c", + "receipt_sha256": { + "report.json": "69e9fb06030e7b00a0f3323a1ccf8e2ed53172e36ebfa98d6e2e6abbd33d9faa", + "catalog.json": "3be1d142d47c9a103cbbab0dcc48e556fe7acb52f812f930ba92b59540beb2d4", + "transcript.json": "dc2bbf5bb7e5391dd2b510ad8126402c6a796a19d02f2d04beec321fbb7c64e1" + } + }, + "full_parent_equal_to_previous_review_check": true, + "all_catalog_metadata_except_output_schemas_equal": true, + "public_workflow_checks": "Fourteen production CLI/MCP profile and lifecycle tests passed. Full absolute-path advertisement conformance and default canonical path refusals passed (two tests).", + "path_followup": "CI e255 exposed a remaining full nested-path advertisement assumption. Only that conformance case selects full; the actual relative-path refusal workflow still exercises the omitted/default profile.", + "validation_runs": [ + { + "log": "latest-main-8e-doctor.log", + "log_sha256": "c7bbb538b8fd409ce2f3381af8166c3f1c66ff2f34b03abe095f99fa5016a367", + "elapsed_seconds": 103.153, + "payload_resource_observation": { + "phase": "payload-exited-observer-alive", + "ownership_token": "rea-efficiency-47112168490343be9cf3738adf32b051.scope", + "payload_exit_code": 0, + "observer_pid": 852216, + "remaining_processes": [852216], + "payload_cleanup_verified": true, + "cpu_stat": { + "usage_usec": 120761531, + "user_usec": 104690859, + "system_usec": 16070672, + "nr_periods": 1026, + "nr_throttled": 7, + "throttled_usec": 15275, + "nr_bursts": 0, + "burst_usec": 0 + }, + "memory_peak_bytes": 1406656512, + "control_group": "user.slice/user-1004.slice/user@1004.service/app.slice/rea-efficiency-47112168490343be9cf3738adf32b051.scope", + "resource_events": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + } + } + }, + { + "log": "absolute-path-profiles.log", + "log_sha256": "f28b63b8969152fdeec3595ae66ed6ccb4733c9cd21542d0b32d6bb269ceb8a0", + "elapsed_seconds": 18.742, + "payload_resource_observation": { + "phase": "payload-exited-observer-alive", + "ownership_token": "rea-efficiency-e46c4b026e1b4023917aceaaf79eb563.scope", + "payload_exit_code": 0, + "observer_pid": 856870, + "remaining_processes": [856870], + "payload_cleanup_verified": true, + "cpu_stat": { + "usage_usec": 18867565, + "user_usec": 16387068, + "system_usec": 2480497, + "nr_periods": 185, + "nr_throttled": 0, + "throttled_usec": 0, + "nr_bursts": 0, + "burst_usec": 0 + }, + "memory_peak_bytes": 969953280, + "control_group": "user.slice/user-1004.slice/user@1004.service/app.slice/rea-efficiency-e46c4b026e1b4023917aceaaf79eb563.scope", + "resource_events": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + } + } + } + ], + "included_in_performance_cohorts": false + }, + "remaining_constraint_followup": { + "base_revision": "8e3bf56e11bcb56d0e8754fdfa32a7a9f004ca13", + "ci_observation": "Unchanged 2608 retry passed the prior Java-cleanup failure, then found build_call_path nested coverage constraints assumed a full advertisement.", + "change": "Function-call direction workflow now exercises full/compact with canonical invalid coverage refusals. Browser scenario keeps default actual advertisement for valid calls, full producer schema negatives, and typed canonical refusal.", + "audit_scope": "Compared full/compact input catalogs and reviewed negative schema assertions. Fresh isolated processes cover function-call directions, ADB start, browser scenarios, JavaScript source selection and managed workflows.", + "passing_test_count": 14, + "failed_local_attempt_preserved": true, + "executed_runs": [ + { + "log": "final-negative-functionCallDirectionsMcp.log", + "log_sha256": "2f5a35b3e0df25daff235f052862a319737455062ca879a700d110f6c4c5d5b2", + "test_summary": "2 passed (2)", + "elapsed_seconds": 53.207, + "payload_resource_observation": { + "phase": "payload-exited-observer-alive", + "ownership_token": "rea-efficiency-03b95305041144949970ac47169a4978.scope", + "payload_exit_code": 0, + "observer_pid": 917614, + "remaining_processes": [917614], + "payload_cleanup_verified": true, + "cpu_stat": { + "usage_usec": 58322344, + "user_usec": 51463945, + "system_usec": 6858398, + "nr_periods": 530, + "nr_throttled": 1, + "throttled_usec": 14, + "nr_bursts": 0, + "burst_usec": 0 + }, + "memory_peak_bytes": 1652879360, + "control_group": "user.slice/user-1004.slice/user@1004.service/app.slice/rea-efficiency-03b95305041144949970ac47169a4978.scope", + "resource_events": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + } + } + }, + { + "log": "final-negative-adbStartMcp.log", + "log_sha256": "e3aacf3a950607a210349ae111043d80477021ffeecd6e98897700898fc0f2e2", + "test_summary": "2 passed (2)", + "elapsed_seconds": 38.274, + "payload_resource_observation": { + "phase": "payload-exited-observer-alive", + "ownership_token": "rea-efficiency-65b1f5db0a1d468abd392502314307c4.scope", + "payload_exit_code": 0, + "observer_pid": 918208, + "remaining_processes": [918208], + "payload_cleanup_verified": true, + "cpu_stat": { + "usage_usec": 42667409, + "user_usec": 36781789, + "system_usec": 5885619, + "nr_periods": 381, + "nr_throttled": 2, + "throttled_usec": 4057, + "nr_bursts": 0, + "burst_usec": 0 + }, + "memory_peak_bytes": 1313853440, + "control_group": "user.slice/user-1004.slice/user@1004.service/app.slice/rea-efficiency-65b1f5db0a1d468abd392502314307c4.scope", + "resource_events": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + } + } + }, + { + "log": "final-negative-browserScenarioMcp.log", + "log_sha256": "87b905b1b03a47b146eba003674bbe054c75cd6074540c0b907e5e24e7a302b5", + "test_summary": "1 failed | 1 passed (2)", + "elapsed_seconds": 39.577, + "payload_resource_observation": { + "phase": "payload-exited-observer-alive", + "ownership_token": "rea-efficiency-da25c46186ea43f6a5c96db67d61f8ff.scope", + "payload_exit_code": 1, + "observer_pid": 919050, + "remaining_processes": [919050], + "payload_cleanup_verified": true, + "cpu_stat": { + "usage_usec": 43818953, + "user_usec": 38886090, + "system_usec": 4932862, + "nr_periods": 394, + "nr_throttled": 2, + "throttled_usec": 3972, + "nr_bursts": 0, + "burst_usec": 0 + }, + "memory_peak_bytes": 1394446336, + "control_group": "user.slice/user-1004.slice/user@1004.service/app.slice/rea-efficiency-da25c46186ea43f6a5c96db67d61f8ff.scope", + "resource_events": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + } + } + }, + { + "log": "final-negative-fixed-browserScenarioMcp.log", + "log_sha256": "1d4ae533fa12bc1769c9c8b9cf8a5acb81855f68d8a6232e19a27a3a9330c4ad", + "test_summary": "2 passed (2)", + "elapsed_seconds": 28.236, + "payload_resource_observation": { + "phase": "payload-exited-observer-alive", + "ownership_token": "rea-efficiency-9424832752364fc295201a0fdfc2f97d.scope", + "payload_exit_code": 0, + "observer_pid": 923267, + "remaining_processes": [923267], + "payload_cleanup_verified": true, + "cpu_stat": { + "usage_usec": 32676191, + "user_usec": 28426282, + "system_usec": 4249908, + "nr_periods": 281, + "nr_throttled": 0, + "throttled_usec": 0, + "nr_bursts": 0, + "burst_usec": 0 + }, + "memory_peak_bytes": 2045136896, + "control_group": "user.slice/user-1004.slice/user@1004.service/app.slice/rea-efficiency-9424832752364fc295201a0fdfc2f97d.scope", + "resource_events": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + } + } + }, + { + "log": "final-negative-fixed-javascriptSourceSelection.log", + "log_sha256": "3a56ef922664064aec099f4cbd9741131f998487574eaac69ba2af1441f8edbc", + "test_summary": "6 passed (6)", + "elapsed_seconds": 36.907, + "payload_resource_observation": { + "phase": "payload-exited-observer-alive", + "ownership_token": "rea-efficiency-6456af0972c948109302b1121f6f1322.scope", + "payload_exit_code": 0, + "observer_pid": 923954, + "remaining_processes": [923954], + "payload_cleanup_verified": true, + "cpu_stat": { + "usage_usec": 45054409, + "user_usec": 40333838, + "system_usec": 4720571, + "nr_periods": 367, + "nr_throttled": 3, + "throttled_usec": 4210, + "nr_bursts": 0, + "burst_usec": 0 + }, + "memory_peak_bytes": 1351868416, + "control_group": "user.slice/user-1004.slice/user@1004.service/app.slice/rea-efficiency-6456af0972c948109302b1121f6f1322.scope", + "resource_events": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + } + } + }, + { + "log": "final-negative-fixed-managedMcp.log", + "log_sha256": "4fcf48f9087eed10a34a81cff4037ea2f47d4fe0335cc0222e80d787fd8e9163", + "test_summary": "2 passed (2)", + "elapsed_seconds": 32.557, + "payload_resource_observation": { + "phase": "payload-exited-observer-alive", + "ownership_token": "rea-efficiency-005cd6a613b84d1dbba465a19719d7a0.scope", + "payload_exit_code": 0, + "observer_pid": 925544, + "remaining_processes": [925544], + "payload_cleanup_verified": true, + "cpu_stat": { + "usage_usec": 36764028, + "user_usec": 33981848, + "system_usec": 2782180, + "nr_periods": 324, + "nr_throttled": 0, + "throttled_usec": 0, + "nr_bursts": 0, + "burst_usec": 0 + }, + "memory_peak_bytes": 1497161728, + "control_group": "user.slice/user-1004.slice/user@1004.service/app.slice/rea-efficiency-005cd6a613b84d1dbba465a19719d7a0.scope", + "resource_events": { + "memory.events": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "pids.events": { + "max": 0 + } + } + } + } + ], + "included_in_performance_cohorts": false + } +} diff --git a/experiments/analysis-efficiency/mcp-input-catalog.md b/experiments/analysis-efficiency/mcp-input-catalog.md new file mode 100644 index 0000000000..7f5ca3f47f --- /dev/null +++ b/experiments/analysis-efficiency/mcp-input-catalog.md @@ -0,0 +1,228 @@ +# Default MCP input catalog cost + +Continues [#5861](https://github.com/morluto/rea/issues/5861). Baseline is +`705058f3e2df82503f1c601fc464aea7cbb626b8`. The candidate selects the existing +compact input presentation when the setting is omitted. Explicit `full` and +`compact` choices, all 213 tools, descriptions, output contracts, canonical +validation, provider behavior and complete Evidence remain available. + +This is a change to the default advertisement. Callers requiring full field +guidance and nested validation declarations can set +`REA_MCP_INPUT_SCHEMA_PROFILE=full`. Compact drops nested annotation prose and +reduces structurally oversized/deep schemas; it is not an equivalent complete +input declaration. The canonical server validator still admits or rejects the +same inputs. No new profile, tool-selection API, cache or discovery protocol is +introduced. Output schemas retain their existing default. + +## Completed SDK tasks + +The [receipt](mcp-input-catalog.json) pins sources, publisher URLs/digests, +individual calls and resource observations. The unchanged Alpine v3.21 x86_64 +releases are ripgrep 14.1.1-r0 (3,416,168 bytes) and Zellij 0.40.0-r0 +(17,151,864 bytes). Neither binary was executed. The independent +[oracle](mcp-input-catalog-oracle.py) reads ELF section headers, decompresses +`.dep-v0` with Python zlib, and derives original package indices and dependency +edges from the embedded JSON, independently of REA. + +Each [production stdio SDK task](mcp-input-catalog.mjs) discovers and validates +the complete advertised input catalog, requests exact serde package fields, +reads the same retained page twice, probes a negative offset, exports complete +Evidence, compares every package and edge with the oracle, and closes the +transport. RPCs bind the exact advertised named tool definition. Tool metadata +outside input schemas and complete parent Evidence are equal before/after. +The real negative input is refused in every task. + +Three fresh-process pairs per artifact alternate order (before/after, +after/before, before/after). Both runtimes use Node 24.18.0, npm 11.16.0 and +MCP SDK 2.3.1 on Linux x86_64. Node compile caching is disabled; OS caches are +not flushed and unrelated host load is uncontrolled. Artifact acquisition, +builds and oracle preparation are outside the measured analysis task. + +| Artifact | Default before/after | Whole-task estimated tokens | Wall median seconds | CPU median seconds | Family peak median MiB | +| -------- | -------------------- | --------------------------: | ------------------: | -----------------: | ---------------------: | +| ripgrep | full | 204,358 | 7.379 | 7.159 | 804.3 | +| ripgrep | compact | 51,801 | 7.274 | 7.036 | 815.5 | +| Zellij | full | 204,278 | 7.373 | 7.162 | 839.0 | +| Zellij | compact | 51,721 | 7.374 | 7.131 | 827.0 | + +The input-facing catalog alone changes from 200,425 to 47,868 estimated tokens +(76.1% lower). Each whole task includes the catalog once, all five calls' +arguments and text replies: about 74.7% lower. These are offline +`tiktoken==0.12.0` / `cl100k_base` estimates, not model usage. The raw SDK catalog +including output schemas changes from 903,377 to 750,623 estimated tokens; +clients exposing that entire representation have a smaller proportional gain. +JSON-RPC envelopes and client-specific conversion/replay are unmeasured here. + +Wall ranges overlap (ripgrep 7.27–7.68 vs 7.07–7.27 seconds; Zellij 7.17–7.48 vs +7.07–7.38). No general elapsed, CPU or memory improvement is established. +Fresh processes include first-use cost and two same-session historical reads; +the latter are not fresh live-state observations. + +## Actual model tasks and the limit of the claim + +Six real Codex CLI 0.161.0 tasks used fresh isolated profiles, the complete +catalog, no enabled-tool filter, and requested `gpt-6.1-sol` / medium reasoning. +The resolved server-side model identity is unknown. Each profile completed a +direct ripgrep package question, a broad Zellij metadata question, and an +intentional invalid-offset question. Artifact bytes were unchanged and owned +profiles were removed. Answers match independent package/index/count/digest +facts, cite actual returned Evidence or its parent, preserve embedded-metadata +limitations, and accurately report the negative cases without retrying. + +| Question | Actual input tokens full → compact | Calls full → compact | Guarded wall seconds full → compact | +| -------------- | ---------------------------------: | -------------------: | ----------------------------------: | +| Direct package | 169,419 → 173,274 | 2 → 2 | 33.854 → 31.448 | +| Broad metadata | 126,522 → 166,082 | 1 → 2 | 28.434 → 34.989 | +| Invalid offset | 167,283 → 140,530 | 1 → 1 | 32.983 → 29.641 | + +These are one pair per question on the baseline runtime's explicit full/compact +variants, not repeated model-performance cohorts. Compact used an extra call +for the broad question. **Actual Codex token, latency or task-quality gains are +not established.** Recorded client usage includes cached input; it does not +identify the exact rendered catalog contribution. The change addresses the +default declaration cost for clients carrying the complete catalog; it does not +promise identical savings for clients performing tool search or other schema +conversion. Explicit full remains useful when the additional guidance avoids +extra calls. + +All analysis/model runs use one CPU, 4 GiB family hard / 3 GiB early memory +limits, no swap and 192 tasks. CPU and memory.peak are kernel cgroup counters +observed at payload exit, including the controller and charged page cache. +All accepted tasks have authenticated payload-exit observations, successful +exit and cleanup, and zero observed high/max/OOM/PID-limit events. Final +whole-scope counters are unavailable after the transient scope disappears. + +The first SDK harness attempt used an incorrect retained-source argument and +was correctly refused. It is retained as a failed attempt and excluded from +accepted comparisons; the harness was corrected before the repeated cohorts. + +## Reproduction + +Generate an oracle with Python 3 and the pinned extracted binary: + +```bash +python3 experiments/analysis-efficiency/mcp-input-catalog-oracle.py /absolute/ripgrep /absolute/oracle.json +python3 experiments/analysis-efficiency/resource-run.py --observe-payload-exit -- \ + node experiments/analysis-efficiency/mcp-input-catalog.mjs \ + --runtime /absolute/runtime --artifact /absolute/ripgrep \ + --oracle /absolute/oracle.json --output /absolute/new-output \ + > /absolute/new-output.log 2>&1 + +# In an analysis environment with tiktoken==0.12.0 installed: +python3 experiments/analysis-efficiency/mcp-input-catalog-costs.py \ + --output /absolute/costs.json /absolute/new-output +``` + +Repeat against baseline/candidate runtimes with the setting omitted. Explicit +`--profile full` and `--profile compact` verify caller overrides. The existing +`agent-trial.py` runs the recorded model question files and explicit provider +settings; authentication/profile files and raw binaries are never committed. +Use [the cost script](mcp-input-catalog-costs.py) with the pinned tokenizer to +account for complete SDK tasks and separately recorded model usage. + +Keep the sibling log: accounting requires its authenticated resource receipt. +Its name appends `.log` to the entire case-directory basename, including dots. +A real `reproduce.sdk.1` Zellij task and its accounting completed successfully; +reprocessing every original record with this path fix leaves all costs equal. +This additional check is not part of the original performance cohorts. + +## Latest-main integration and validation + +Rebased onto `25e1a24a3ed3671a0148757555f18d16c29d936a`. A separate real Zellij +pair on that candidate, explicit full versus omitted/default, passed all facts, +full parent Evidence and negative checks. Catalog input estimates remain +200,425 versus 47,868 tokens; these integration runs do not replace or extend +the earlier repeated cohorts. Explicit full on the candidate also reproduced +the pre-change full catalog exactly. + +The initial ten-file local run passed 88 tests, skipped one and failed two: a +live-identity assertion still expected the old default digest, and the complete +canonical-schema suite timed out at 30 seconds. The assertion now checks the +default compact identity. Both affected suites passed all 26 tests with serial +files and unchanged deadlines; the timeout's cause is not established. After +rebasing, 32 profile/identity/production-doctor tests passed. Static checks +passed on both bases. Documentation checks caught a new README link absent from +translated link inventories (including a duplicate-link attempt); the +introduction now names the existing guide without changing the link inventory. +Final documentation and CI results qualify the PR. + +## Review follow-up: ownership of catalog identity + +The former `CATALOG_IDENTITY` meant the full advertisement, while two consumers +treated it as the default live identity. The owner now exports explicit +`FULL_CATALOG_IDENTITY` and `DEFAULT_CATALOG_IDENTITY`; the ambiguous export is +removed. Canonical build/packaging consumers use full identity, default +diagnostics use default identity, and custom/explicit profiles keep their own +identity. Readiness captures and validates `binary_session` from the actual +server rather than assuming a profile. Claude's verifier derives its expected +identity using the shared input/output profile parsers. + +Nine related test files passed 106 tests with two skips. Static and +documentation gates passed again. Actual readiness tasks passed with default, +explicit full, and compact input/output-off settings. The final real Zellij SDK +task reproduced the earlier complete parent and advertised catalog exactly. + +Real Claude Code 2.1.296 with the loopback fixture discovered all 213 schemas +and returned `binary_session` for both profiles. It persisted the 55,471-byte +result, preventing the verifier's existing inline-only recognizer from +continuing. The untouched baseline reproduces the same failure (55,473-byte +result) after preparing its matching packaged skill. This is not a passing +end-to-end Claude analysis claim; the unrelated recognizer is left unchanged. +The actual persisted native responses were independently parsed with the named +production output contract: both selected catalog identities match, and the +completed readiness reports record the same digests. Receipts retain the +baseline/candidate failures, cleanup and this narrower successful check. + +First-head CI found three remaining tests that assumed a full default +advertisement. The ZIP retained-inventory, browser-endpoint and literal-seed +cases now exercise full and compact: full retains its strict declaration and +guidance assertions; both keep canonical refusal and valid workflow checks. +Four affected files passed all 27 tests serially on main +`6d356b84344659085f2b912857b5e27fe0160c9b`; static/documentation gates passed. +An earlier parallel-file attempt was stopped at the 3 GiB early boundary before +OOM and is incomplete. Serial execution stayed below the same boundary without +raising the 4 GiB family limit. No performance cohort includes these test runs. + +Additional CI cases now select full presentation when testing nested schemas +or literal advertised examples. Default source-comparison and JavaScript-fact +workflows still exercise canonical invalid-input refusal. After regenerating +the ignored catalog metadata, four files passed 31 tests; the full-catalog +ordering case hit its 20-second child deadline. That case passed alone in a +fresh process with the same deadline (14.85-second test duration). All five +files have passing checks; the combined run did not pass and its timeout cause +is not established. Family peaks were 2.43 GiB for the combined run and 1.0 GiB +for the standalone run, within unchanged limits. These validation attempts are +separate from the performance cohorts. + +The next CI portable aggregate passed 15,703 tests and skipped 29, with one +remaining Godot inventory advertisement assertion failing. That workflow now +runs under both full and compact, preserving canonical invalid-field refusal +and retained facts after source removal. Its three local tests passed within +the same resource limits, including encrypted-directory unknown coverage. + +A further review found that CLI status relied on an implicit default catalog, +so explicit full input advertised a different identity through CLI and MCP. +CLI status now derives input/output presentation from parsed configuration; +configuration retains the output profile too. Runtime identity requires an +explicit catalog from each caller. Two suites passed 25 tests, including real +production stdio, `capabilities` and `providers` comparisons for default, full, +compact and both output-off combinations. This target-free verification is +separate from real artifact measurements and does not qualify native engines. + +Rebased again onto `8e3bf56e1` after upstream browser Evidence schema changes. +A separate completed real Zellij SDK task preserved its complete parent and +every catalog field except output schemas; whole-task estimate remains 51,721 +tokens. Fourteen production CLI/MCP profile/lifecycle tests passed. Two +absolute-path tests also passed: full advertisement conformance explicitly +selects full, while actual relative-path refusals keep the default compact +presentation. This fixes the remaining nested-path assertion found by CI. +These integration checks do not replace the original performance cohorts. + +A later unchanged-head CI retry passed the earlier Ghidra Java-cleanup failure +but exposed a full-schema assumption in call-direction coverage. That workflow +now tests full and compact with actual canonical invalid-input refusals. A +fresh-process audit also found a browser scenario cross-field assertion; it +keeps default valid calls and typed refusals while checking full producer +constraints separately. Five isolated suites passed 14 tests across call paths, +ADB start, browser scenarios, source selection and managed analysis. The first +browser attempt failed and remains recorded; no performance cohort changed. diff --git a/experiments/analysis-efficiency/mcp-input-catalog.mjs b/experiments/analysis-efficiency/mcp-input-catalog.mjs new file mode 100644 index 0000000000..45672b4f86 --- /dev/null +++ b/experiments/analysis-efficiency/mcp-input-catalog.mjs @@ -0,0 +1,176 @@ +// Compare complete public MCP tasks with caller-selected input advertisements. +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { parseArgs } from "node:util"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { Ajv2020 } from "ajv/dist/2020.js"; + +const { values } = parseArgs({ + options: { + runtime: { type: "string" }, + artifact: { type: "string" }, + oracle: { type: "string" }, + output: { type: "string" }, + profile: { type: "string", default: "default" }, + }, +}); +for (const key of ["runtime", "artifact", "oracle", "output"]) + assert.ok( + values[key] && resolve(values[key]) === values[key], + `${key} must be absolute`, + ); +assert.ok(["default", "full", "compact"].includes(values.profile)); +await mkdir(values.output, { recursive: false }); +const hash = (value) => createHash("sha256").update(value).digest("hex"); +const oracle = JSON.parse(await readFile(values.oracle, "utf8")); +const artifactSha = hash(await readFile(values.artifact)); +assert.equal(artifactSha, oracle.artifact_sha256); +const environment = { + PATH: process.env.PATH, + HOME: values.output, + REA_LOG_LEVEL: "silent", + UV_THREADPOOL_SIZE: "1", + NODE_OPTIONS: "--max-old-space-size=2048", + NODE_DISABLE_COMPILE_CACHE: "1", + ...(values.profile === "default" + ? {} + : { REA_MCP_INPUT_SCHEMA_PROFILE: values.profile }), +}; +const transport = new StdioClientTransport({ + command: process.execPath, + args: [join(values.runtime, "scripts/rea.mjs"), "mcp"], + cwd: values.output, + env: environment, + stderr: "pipe", +}); +const client = new Client({ name: "rea-catalog-task", version: "1" }); +const report = { + profile: values.profile, + artifact_sha256: artifactSha, + calls: [], + checks: [], + status: "incomplete", + cleanup_verified: false, +}; +const transcript = []; +let stderr = ""; +transport.stderr?.on("data", (value) => { + stderr += value; +}); +const began = performance.now(); +try { + await client.connect(transport); + const catalog = await client.listTools(); + assert.equal(catalog.nextCursor, undefined); + const definitions = new Map(catalog.tools.map((tool) => [tool.name, tool])); + assert.equal(definitions.size, catalog.tools.length); + const ajv = new Ajv2020({ strict: false, validateFormats: false }); + for (const tool of catalog.tools) + assert.ok( + ajv.validateSchema(tool.inputSchema), + `Invalid schema: ${tool.name}`, + ); + await writeFile(join(values.output, "catalog.json"), JSON.stringify(catalog)); + report.tool_names = [...definitions.keys()]; + const call = async (name, args) => { + const definition = definitions.get(name); + assert.ok(definition, `Missing named tool ${name}`); + const start = performance.now(); + const reply = await client.callTool( + { name, arguments: args }, + { toolDefinition: definition }, + ); + report.calls.push({ + name, + elapsed_ms: performance.now() - start, + is_error: reply.isError === true, + }); + transcript.push({ name, arguments: args, reply }); + return reply; + }; + const input = { + path: values.artifact, + view: { + collection: "packages", + names: ["serde"], + fields: ["name", "version", "kind"], + }, + }; + const first = await call("inspect_rust_binary", input); + assert.notEqual(first.isError, true); + const selected = first.structuredContent; + assert.ok(selected?.normalized_result?.page); + assert.deepEqual(selected.normalized_result.page.rows, oracle.serde_rows); + assert.equal(selected.subject.digest.sha256, artifactSha); + const parentId = selected.normalized_result.parent_evidence_id; + for (let index = 0; index < 2; index++) { + const reply = await call("inspect_analysis_view", { + source: { kind: "retained-evidence", evidence_id: parentId }, + view: { + kind: "rust", + collection: "packages", + names: ["serde"], + fields: ["name", "version", "kind"], + }, + }); + assert.notEqual(reply.isError, true); + assert.deepEqual( + reply.structuredContent.normalized_result.page.rows, + oracle.serde_rows, + ); + assert.equal( + reply.structuredContent.normalized_result.parent_evidence_id, + parentId, + ); + } + const malformed = await call("inspect_rust_binary", { + path: values.artifact, + view: { collection: "packages", offset: -1 }, + }); + assert.equal(malformed.isError, true); + report.checks.push( + "Independent package rows and digest; repeated retained identity; canonical negative-offset refusal", + ); + const exported = await call("export_evidence_bundle", { + path: join(values.output, "evidence.json"), + }); + assert.notEqual(exported.isError, true); + const bundle = JSON.parse( + await readFile(join(values.output, "evidence.json"), "utf8"), + ); + const parent = bundle.records.find( + (record) => record.evidence_id === parentId, + ); + assert.ok(parent, "Complete original Evidence is retained"); + assert.equal(parent.subject.digest.sha256, artifactSha); + assert.deepEqual( + parent.normalized_result.auditable_metadata.packages, + oracle.packages, + ); + assert.deepEqual( + parent.normalized_result.auditable_metadata.dependency_edges, + oracle.edges, + ); + report.parent_evidence_id = parentId; + report.parent_semantic_sha256 = hash(JSON.stringify(parent)); + report.answer = oracle.serde_rows; + report.status = "passed"; +} finally { + await client.close(); + await transport.close(); + report.cleanup_verified = transport.pid === null; + assert.ok(report.cleanup_verified); + report.elapsed_ms = performance.now() - began; + await writeFile( + join(values.output, "report.json"), + JSON.stringify(report, null, 2) + "\n", + ); + await writeFile( + join(values.output, "transcript.json"), + JSON.stringify(transcript), + ); + await writeFile(join(values.output, "server.log"), stderr); +} diff --git a/scripts/generate-skill-metadata.mjs b/scripts/generate-skill-metadata.mjs index 3ca0a467db..f705501ad5 100644 --- a/scripts/generate-skill-metadata.mjs +++ b/scripts/generate-skill-metadata.mjs @@ -17,7 +17,7 @@ const current = ( await readFile(join(sourceRoot, "SKILL.md"), "utf8") ).replaceAll("\r\n", "\n"); const cacheBust = String(Date.now()); -const { CATALOG_IDENTITY } = await import( +const { FULL_CATALOG_IDENTITY } = await import( `${pathToFileURL(join(root, "dist/catalogIdentity.js")).href}?${cacheBust}` ); const { PRODUCT_IDENTITY } = await import( @@ -36,7 +36,7 @@ const pinnedSpecifier = PRODUCT_IDENTITY.registrationPackageSpecifier; const source = current .replace( versionLine, - `$&\n tool_count: ${String(CATALOG_IDENTITY.counts.mcp_tools)}`, + `$&\n tool_count: ${String(FULL_CATALOG_IDENTITY.counts.mcp_tools)}`, ) .replaceAll(latestSpecifier, pinnedSpecifier); const paths = await filePaths(sourceRoot); diff --git a/scripts/lib/catalog-builders.mjs b/scripts/lib/catalog-builders.mjs index cedce74583..d4b51ead22 100644 --- a/scripts/lib/catalog-builders.mjs +++ b/scripts/lib/catalog-builders.mjs @@ -143,7 +143,8 @@ export const toolFamilyCatalog = (sources) => { ); if ( familyTotal !== sources.toolContracts.TOOL_CONTRACTS.length || - familyTotal !== sources.catalogIdentity.CATALOG_IDENTITY.counts.mcp_tools + familyTotal !== + sources.catalogIdentity.FULL_CATALOG_IDENTITY.counts.mcp_tools ) throw new Error("Tool family inventory does not match TOOL_CONTRACTS"); return { total: familyTotal, families }; diff --git a/scripts/lib/product-catalog.mjs b/scripts/lib/product-catalog.mjs index dadffaf7d6..159a55a5d8 100644 --- a/scripts/lib/product-catalog.mjs +++ b/scripts/lib/product-catalog.mjs @@ -43,7 +43,7 @@ export const createProductCatalog = async (root) => { aliases: cli.aliases, }, runtime_catalog: { - counts: sources.catalogIdentity.CATALOG_IDENTITY.counts, + counts: sources.catalogIdentity.FULL_CATALOG_IDENTITY.counts, digests: { // Full runtime schema digests belong to server identity, not this facts projection. providers_sha256: digest(providers), diff --git a/scripts/lib/release-package-canary.mjs b/scripts/lib/release-package-canary.mjs index 1270f3282e..b00bb9788c 100644 --- a/scripts/lib/release-package-canary.mjs +++ b/scripts/lib/release-package-canary.mjs @@ -5,7 +5,7 @@ import { readFile } from "node:fs/promises"; import { join } from "node:path"; import { promisify } from "node:util"; -import { CATALOG_IDENTITY } from "../../dist/catalogIdentity.js"; +import { FULL_CATALOG_IDENTITY } from "../../dist/catalogIdentity.js"; import { TOOL_CONTRACTS } from "../../dist/contracts/toolContracts.js"; import { requireMcpLifecycleResult, @@ -104,7 +104,7 @@ export async function verifyInstalledCanaryPackage( ); return { mcpTools: toolNames.length, - canonicalMcpTools: CATALOG_IDENTITY.counts.mcp_tools, + canonicalMcpTools: FULL_CATALOG_IDENTITY.counts.mcp_tools, }; } catch (cause) { throw new Error(`Installed package MCP verification failed: ${stderr}`, { diff --git a/scripts/verify-claude-client.mjs b/scripts/verify-claude-client.mjs index 9a7298920d..10a17ee3c0 100644 --- a/scripts/verify-claude-client.mjs +++ b/scripts/verify-claude-client.mjs @@ -5,7 +5,12 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; import Ajv2020 from "ajv/dist/2020.js"; -import { CATALOG_IDENTITY } from "../dist/catalogIdentity.js"; +import { createCatalogIdentity } from "../dist/catalogIdentity.js"; +import { + compactPresentationForMcpInputSchemaProfile, + parseMcpInputSchemaProfile, +} from "../dist/config/mcpInputSchemaProfile.js"; +import { parseMcpOutputSchemaProfile } from "../dist/config/mcpOutputSchemaProfile.js"; import { TOOL_CONTRACTS, toolContract, @@ -166,10 +171,21 @@ const fixture = await createAnthropicModelFixture({ ...parsedStatus, result: expandMcpSessionStatus(parsedStatus.result), }; + const inputProfile = parseMcpInputSchemaProfile( + environment.REA_MCP_INPUT_SCHEMA_PROFILE, + ); + const outputProfile = parseMcpOutputSchemaProfile( + environment.REA_MCP_OUTPUT_SCHEMA_PROFILE, + ); + assert.ok(inputProfile.ok); + assert.ok(outputProfile.ok); assert.deepEqual( status.result.server_identity.catalog, - CATALOG_IDENTITY, - "binary_session must return the exact compact catalog counts and digests", + createCatalogIdentity( + compactPresentationForMcpInputSchemaProfile(inputProfile.value), + outputProfile.value ?? "full", + ), + "binary_session must return the selected input/output profile's catalog identity", ); assert.equal( Object.hasOwn(status.result.server_identity.catalog, "tools"), diff --git a/scripts/verify-package-pack.mjs b/scripts/verify-package-pack.mjs index c1e4edb00c..9111f67887 100644 --- a/scripts/verify-package-pack.mjs +++ b/scripts/verify-package-pack.mjs @@ -1,6 +1,6 @@ import { mkdir } from "node:fs/promises"; import { join } from "node:path"; -import { CATALOG_IDENTITY } from "../dist/catalogIdentity.js"; +import { FULL_CATALOG_IDENTITY } from "../dist/catalogIdentity.js"; import { exec, execNpm } from "./lib/verify-package-core.mjs"; import { parseTarListing } from "./lib/verify-package-listing.mjs"; @@ -52,7 +52,7 @@ export async function verifyPackagePack({ root, workspace }) { ).stdout; if ( !skill.includes( - ` tool_count: ${String(CATALOG_IDENTITY.counts.mcp_tools)}\n`, + ` tool_count: ${String(FULL_CATALOG_IDENTITY.counts.mcp_tools)}\n`, ) || skill.includes(" catalog_digest:") ) diff --git a/scripts/verify-reconstruction-readiness.mjs b/scripts/verify-reconstruction-readiness.mjs index 940a39fdd4..afcb27812e 100644 --- a/scripts/verify-reconstruction-readiness.mjs +++ b/scripts/verify-reconstruction-readiness.mjs @@ -7,7 +7,7 @@ import { promisify } from "node:util"; import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; -import { CATALOG_IDENTITY } from "../dist/catalogIdentity.js"; +import { toolContract } from "../dist/contracts/toolContracts.js"; import { RECONSTRUCTION_READINESS_EXAMPLE } from "../dist/contracts/reconstructionReadinessExample.js"; import { parseEvidence } from "../dist/domain/evidence.js"; import { createEvidenceBundle } from "../dist/domain/evidenceBundle.js"; @@ -52,6 +52,18 @@ const client = new Client({ try { await client.connect(transport); + const catalog = await client.listTools(); + const sessionTool = catalog.tools.find( + (tool) => tool.name === "binary_session", + ); + assert.ok(sessionTool, "The live server must advertise binary_session"); + const sessionReply = await client.callTool( + { name: "binary_session", arguments: {} }, + { toolDefinition: sessionTool }, + ); + const sessionStatus = toolContract("binary_session").outputSchema.parse( + sessionReply.structuredContent, + ).result; const javascriptMcp = await analyzeMcp(client, javascriptPath); const electronMcp = await analyzeMcp(client, electronPath); assert.equal( @@ -68,7 +80,8 @@ try { const input = structuredClone(RECONSTRUCTION_READINESS_EXAMPLE); input.identity.cli_version = PACKAGE_METADATA.version; input.identity.server_version = PACKAGE_METADATA.version; - input.identity.catalog_digest = CATALOG_IDENTITY.digests.combined_sha256; + input.identity.catalog_digest = + sessionStatus.server_identity.catalog.digests.combined_sha256; const records = [ ...input.evidence_bundle.records, nativeEvidence, diff --git a/src/application/DirectAnalysisStatus.ts b/src/application/DirectAnalysisStatus.ts index 26c479df6c..8a17414396 100644 --- a/src/application/DirectAnalysisStatus.ts +++ b/src/application/DirectAnalysisStatus.ts @@ -8,6 +8,8 @@ import { providerCapabilitySet } from "../contracts/toolOutputSchemaPrimitives.j import { providerCapabilitySets } from "./binary/BinarySessionStatus.js"; import { sqliteDatabaseCapability } from "./sqlite/SqliteDatabaseCapability.js"; import { CURRENT_ENTRYPOINT_PATH } from "../process/CurrentEntrypointPath.js"; +import { createCatalogIdentity } from "../catalogIdentity.js"; +import { compactPresentationForMcpInputSchemaProfile } from "../config/mcpInputSchemaProfile.js"; /** Read target-free provider status and operation availability. */ export const runSessionStatus = async ( @@ -33,6 +35,12 @@ export const runSessionStatus = async ( server_identity: createServerIdentity({ startedAt: new Date().toISOString(), commandPath: CURRENT_ENTRYPOINT_PATH, + catalog: createCatalogIdentity( + compactPresentationForMcpInputSchemaProfile( + config.value.mcpInputSchemaProfile, + ), + config.value.mcpOutputSchemaProfile ?? "full", + ), }), }; } finally { diff --git a/src/application/Doctor.fixture.ts b/src/application/Doctor.fixture.ts index 6d49e5ac90..c2708d738b 100644 --- a/src/application/Doctor.fixture.ts +++ b/src/application/Doctor.fixture.ts @@ -1,4 +1,4 @@ -import { CATALOG_IDENTITY } from "../catalogIdentity.js"; +import { DEFAULT_CATALOG_IDENTITY } from "../catalogIdentity.js"; import { PRODUCT_IDENTITY } from "../identity.js"; import { ok } from "../domain/result.js"; import type { DoctorHost } from "./Doctor.js"; @@ -36,7 +36,7 @@ export const createDoctorHostFixture = ( installedSkillIdentity: () => Promise.resolve({ version: PRODUCT_IDENTITY.skillVersion, - toolCount: CATALOG_IDENTITY.counts.mcp_tools, + toolCount: DEFAULT_CATALOG_IDENTITY.counts.mcp_tools, canonical: true, }), ...overrides, diff --git a/src/application/Doctor.ts b/src/application/Doctor.ts index 588c5d4acd..cbb3353228 100644 --- a/src/application/Doctor.ts +++ b/src/application/Doctor.ts @@ -45,7 +45,7 @@ import { type InstalledSkillIdentity, type SkillPathIssue, } from "./SetupSkill.js"; -import { CATALOG_IDENTITY } from "../catalogIdentity.js"; +import { DEFAULT_CATALOG_IDENTITY } from "../catalogIdentity.js"; import { PRODUCT_IDENTITY, SDK_IDENTITY } from "../identity.js"; import { readClientRegistrationStatuses, @@ -178,7 +178,8 @@ interface DoctorIdentity { readonly cli_package_version: string; readonly expected_skill_version: string; readonly sdk: typeof SDK_IDENTITY; - readonly catalog: typeof CATALOG_IDENTITY; + /** Package default advertisement; the active server reports its selected profile. */ + readonly catalog: typeof DEFAULT_CATALOG_IDENTITY; readonly live_server: { readonly state: "unknown"; readonly remediation: string; @@ -390,8 +391,8 @@ const collectDoctorIdentity = async ( expected_skill_version: PRODUCT_IDENTITY.skillVersion, sdk: SDK_IDENTITY, catalog: { - counts: CATALOG_IDENTITY.counts, - digests: CATALOG_IDENTITY.digests, + counts: DEFAULT_CATALOG_IDENTITY.counts, + digests: DEFAULT_CATALOG_IDENTITY.digests, }, live_server: { state: "unknown", @@ -422,7 +423,7 @@ const skillIdentityAligned = ( identity: InstalledSkillIdentity | undefined, ): boolean => identity?.version === PRODUCT_IDENTITY.skillVersion && - identity.toolCount === CATALOG_IDENTITY.counts.mcp_tools && + identity.toolCount === DEFAULT_CATALOG_IDENTITY.counts.mcp_tools && identity.canonical; const skillIdentityState = ( diff --git a/src/application/Setup.fixture.ts b/src/application/Setup.fixture.ts index 07adf87cca..0bd462280b 100644 --- a/src/application/Setup.fixture.ts +++ b/src/application/Setup.fixture.ts @@ -18,7 +18,7 @@ import type { } from "./Doctor.js"; import type { LinuxDistribution } from "./LinuxHopper.js"; import { PRODUCT_IDENTITY, SDK_IDENTITY } from "../identity.js"; -import { CATALOG_IDENTITY } from "../catalogIdentity.js"; +import { DEFAULT_CATALOG_IDENTITY } from "../catalogIdentity.js"; import type { ClientRegistrationStatus } from "./ClientRegistrationStatus.js"; import { setupRegistrationCommand } from "./SetupHost.js"; import { CURRENT_ENTRYPOINT_PATH } from "../process/CurrentEntrypointPath.js"; @@ -228,7 +228,7 @@ export class FakeSetupHost implements SetupHost { cli_package_version: PRODUCT_IDENTITY.packageVersion, expected_skill_version: PRODUCT_IDENTITY.skillVersion, sdk: SDK_IDENTITY, - catalog: CATALOG_IDENTITY, + catalog: DEFAULT_CATALOG_IDENTITY, live_server: { state: "unknown", remediation: "Inspect the running server.", diff --git a/src/catalogIdentity.ts b/src/catalogIdentity.ts index 8429783a90..afd53aa764 100644 --- a/src/catalogIdentity.ts +++ b/src/catalogIdentity.ts @@ -19,6 +19,7 @@ import { import type { McpOutputSchemaProfile } from "./config/mcpOutputSchemaProfile.js"; import { COMPACT_INPUT_SCHEMA_BUDGET_BYTES, + compactPresentationForMcpInputSchemaProfile, type McpInputSchemaProfile, } from "./config/mcpInputSchemaProfile.js"; import { RECORDED_CATALOG_DIGEST_PROFILES } from "./recordedCatalogDigests.js"; @@ -111,7 +112,7 @@ const digestCatalogs = ( /** * Catalog digests over the given tool contracts. Exported so tests can prove * which contract fields the identity covers; production uses - * {@link CATALOG_IDENTITY}. + * {@link FULL_CATALOG_IDENTITY}. */ export const createCatalogDigests = ( contracts: readonly ToolContract[] = TOOL_CONTRACTS, @@ -191,7 +192,12 @@ export const createCatalogIdentity = ( }; /** - * Stable, schema-sensitive identity; schema digests are recorded at build time - * or computed on first use. + * Full input/output advertisement identity. Digests are recorded at build time + * or computed on first use; this is independent of the configured default. */ -export const CATALOG_IDENTITY = createCatalogIdentity(); +export const FULL_CATALOG_IDENTITY = createCatalogIdentity(); + +/** Default MCP advertisement; explicit full/custom selections have their own identity. */ +export const DEFAULT_CATALOG_IDENTITY = createCatalogIdentity( + compactPresentationForMcpInputSchemaProfile(undefined), +); diff --git a/src/config/mcpInputSchemaProfile.ts b/src/config/mcpInputSchemaProfile.ts index bfdc017e4f..1acb5b2a1b 100644 --- a/src/config/mcpInputSchemaProfile.ts +++ b/src/config/mcpInputSchemaProfile.ts @@ -12,16 +12,16 @@ export const COMPACT_INPUT_SCHEMA_BUDGET_BYTES = 13 * 1024; export const mcpInputSchemaProfileSchema = z.enum(["full", "compact"]); -/** Select the exact compact presentation budget shared by registration and identity. */ +/** Select the default compact advertisement, preserving an explicit full choice. */ export const compactPresentationForMcpInputSchemaProfile = ( profile: "full" | "compact" | undefined, ): { readonly budgetBytes: number } | undefined => - profile === "compact" - ? { budgetBytes: COMPACT_INPUT_SCHEMA_BUDGET_BYTES } - : undefined; + profile === "full" + ? undefined + : { budgetBytes: COMPACT_INPUT_SCHEMA_BUDGET_BYTES }; const profileConstraint = - 'REA_MCP_INPUT_SCHEMA_PROFILE must be "full" (default) or "compact". The compact profile advertises bounded input schemas with fewer nested annotations for providers with schema compatibility constraints.'; + 'REA_MCP_INPUT_SCHEMA_PROFILE must be "compact" (default) or "full". The compact profile advertises bounded input schemas with fewer nested annotations; full preserves complete advertised input guidance.'; export type McpInputSchemaProfile = z.infer; diff --git a/src/config/parseConfig.ts b/src/config/parseConfig.ts index 8fa259d682..30765ee018 100644 --- a/src/config/parseConfig.ts +++ b/src/config/parseConfig.ts @@ -77,6 +77,9 @@ export const parseConfig = ( ...(env.REA_MCP_INPUT_SCHEMA_PROFILE === undefined ? {} : { mcpInputSchemaProfile: env.REA_MCP_INPUT_SCHEMA_PROFILE }), + ...(env.REA_MCP_OUTPUT_SCHEMA_PROFILE === undefined + ? {} + : { mcpOutputSchemaProfile: env.REA_MCP_OUTPUT_SCHEMA_PROFILE }), analysisProvider: env.REA_ANALYSIS_PROVIDER, ...(env.REA_IDA_MCP_CONFIG === undefined ? {} diff --git a/src/config/types.ts b/src/config/types.ts index c398d6f6fd..0110007eb2 100644 --- a/src/config/types.ts +++ b/src/config/types.ts @@ -4,10 +4,12 @@ import type { AnalysisProviderSelector } from "../contracts/providerSelection.js import type { GhidraLanguageOverride } from "./ghidraLanguageOverride.js"; import type { McpInputSchemaProfile } from "./mcpInputSchemaProfile.js"; +import type { McpOutputSchemaProfile } from "./mcpOutputSchemaProfile.js"; export interface AppConfig { readonly mcpMaxResponseBytes?: number; readonly mcpInputSchemaProfile?: McpInputSchemaProfile; + readonly mcpOutputSchemaProfile?: McpOutputSchemaProfile; readonly analysisProvider: AnalysisProviderSelector; readonly idaMcpConfigPath?: string; readonly ghidraInstallDir: string | undefined; diff --git a/src/server/EvidenceMcpServer.ts b/src/server/EvidenceMcpServer.ts index 2339edc8cb..47fe27c9e8 100644 --- a/src/server/EvidenceMcpServer.ts +++ b/src/server/EvidenceMcpServer.ts @@ -26,7 +26,10 @@ import { presentAdvertisedInputSchema, type CompactInputSchemaPresentation, } from "../contracts/compactInputSchema.js"; -import { CATALOG_IDENTITY, createCatalogIdentity } from "../catalogIdentity.js"; +import { + FULL_CATALOG_IDENTITY, + createCatalogIdentity, +} from "../catalogIdentity.js"; import { transformAdvertisedInputJsonSchema } from "../contracts/toolSchemaMetadata.js"; import type { EvidenceWriter } from "../application/investigation/InvestigationRecordPort.js"; import { @@ -165,7 +168,7 @@ export class EvidenceMcpServer extends RequestTrackingMcpServer { : { budgetBytes: compactPresentation.budgetBytes }; this.catalogIdentity = this.compactPresentation === undefined && !this.omitOutputSchemas - ? CATALOG_IDENTITY + ? FULL_CATALOG_IDENTITY : createCatalogIdentity( this.compactPresentation, this.omitOutputSchemas ? "off" : "full", diff --git a/src/serverIdentity.ts b/src/serverIdentity.ts index 12d88b5f47..9ee037dcbd 100644 --- a/src/serverIdentity.ts +++ b/src/serverIdentity.ts @@ -1,10 +1,7 @@ import { dirname, basename, resolve } from "node:path"; import { fileURLToPath } from "node:url"; -import { - CATALOG_IDENTITY, - type createCatalogIdentity, -} from "./catalogIdentity.js"; +import type { createCatalogIdentity } from "./catalogIdentity.js"; import { PRODUCT_IDENTITY, SDK_IDENTITY } from "./identity.js"; /** Optional observations supplied by a client comparing its expected server. */ @@ -34,9 +31,9 @@ export const createServerIdentity = (input: { readonly expected?: ExpectedServerIdentity; readonly client?: ConnectedClientIdentity; readonly protocolVersion?: string | undefined; - readonly catalog?: ReturnType; + readonly catalog: ReturnType; }) => { - const catalog = input.catalog ?? CATALOG_IDENTITY; + const catalog = input.catalog; const serverPath = input.commandPath || undefined; const reasons: string[] = []; if ( diff --git a/tests/acceptance/applications/runtime.test.ts b/tests/acceptance/applications/runtime.test.ts index d085d27713..e755d64d07 100644 --- a/tests/acceptance/applications/runtime.test.ts +++ b/tests/acceptance/applications/runtime.test.ts @@ -5,7 +5,7 @@ import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { describe, expect, it } from "vitest"; import { z } from "zod"; -import { CATALOG_IDENTITY } from "../../../src/catalogIdentity.js"; +import { FULL_CATALOG_IDENTITY } from "../../../src/catalogIdentity.js"; import { TOOL_CONTRACTS } from "../../../src/contracts/toolContracts.js"; import { evidenceResultOf, @@ -51,7 +51,7 @@ const expectAvailableToolInventory = async ( expect( availability.length, "compiled tool inventory is stale; run npm run build:cached", - ).toBe(CATALOG_IDENTITY.counts.mcp_tools); + ).toBe(FULL_CATALOG_IDENTITY.counts.mcp_tools); expect(new Set(listed.tools.map(({ name }) => name))).toEqual( new Set(TOOL_CONTRACTS.map(({ name }) => name)), ); diff --git a/tests/boundary/filesystem/dependencyInstall.test.ts b/tests/boundary/filesystem/dependencyInstall.test.ts index 34609de849..06db663fc0 100644 --- a/tests/boundary/filesystem/dependencyInstall.test.ts +++ b/tests/boundary/filesystem/dependencyInstall.test.ts @@ -231,7 +231,7 @@ const createBuildCacheFixture = async (): Promise => { await writeFile("resource-controls.json", JSON.stringify(Object.fromEntries(["GOMAXPROCS", "GOMEMLIMIT", "UV_THREADPOOL_SIZE", "REA_TEST_UNLISTED_ENV"].map((name) => [name, process.env[name] ?? null])))); await mkdir("dist", { recursive: true }); await writeFile("dist/cache-fixture.js", "built"); -await writeFile("dist/catalogIdentity.js", "export const CATALOG_IDENTITY = { counts: { mcp_tools: 1 } };\\n"); +await writeFile("dist/catalogIdentity.js", "export const FULL_CATALOG_IDENTITY = { counts: { mcp_tools: 1 } };\\n"); await writeFile("dist/identity.js", ${JSON.stringify(`export const PRODUCT_IDENTITY = ${JSON.stringify({ packageSpecifier: `${packageJson.name}@latest`, registrationPackageSpecifier: `${packageJson.name}@${packageJson.version}` })};\n`)}); `, ); diff --git a/tests/boundary/filesystem/generatedSkill.test.ts b/tests/boundary/filesystem/generatedSkill.test.ts index 2b7ebf754b..b08d98981a 100644 --- a/tests/boundary/filesystem/generatedSkill.test.ts +++ b/tests/boundary/filesystem/generatedSkill.test.ts @@ -38,7 +38,7 @@ async function fixture(eol = "\n") { const identity = async (count: number, digest: string) => writeFile( join(root, "dist/catalogIdentity.js"), - `export const CATALOG_IDENTITY = ${JSON.stringify({ counts: { mcp_tools: count }, digests: { combined_sha256: digest } })};\n`, + `export const FULL_CATALOG_IDENTITY = ${JSON.stringify({ counts: { mcp_tools: count }, digests: { combined_sha256: digest } })};\n`, ); await identity(139, "a".repeat(64)); const generate = (...args: string[]) => diff --git a/tests/boundary/mcp/absoluteLocalPathSchema.test.ts b/tests/boundary/mcp/absoluteLocalPathSchema.test.ts index 2340dc1e17..58ae31f7df 100644 --- a/tests/boundary/mcp/absoluteLocalPathSchema.test.ts +++ b/tests/boundary/mcp/absoluteLocalPathSchema.test.ts @@ -14,7 +14,10 @@ interface PathInputCase { readonly invalid: Readonly>; } -const createMcp = async (firmwareCalls: string[]) => { +const createMcp = async ( + firmwareCalls: string[], + profile?: "full" | "compact", +) => { const session = createTestBinarySession(() => { throw new Error("Invalid path input must not acquire a binary provider"); }); @@ -22,6 +25,7 @@ const createMcp = async (firmwareCalls: string[]) => { { kind: "session", session }, { logger: silentLogger, + environment: { ...process.env, REA_MCP_INPUT_SCHEMA_PROFILE: profile }, firmwareAnalysis: { execute: async (request) => { firmwareCalls.push(request.operation); @@ -174,9 +178,9 @@ const inputCases = ( ]; }; -it("advertises host-absolute path constraints after SDK schema conversion", async () => { +it("advertises full host-absolute path constraints after SDK schema conversion", async () => { const firmwareCalls: string[] = []; - const { client, close } = await createMcp(firmwareCalls); + const { client, close } = await createMcp(firmwareCalls, "full"); try { const listed = await client.listTools(); const ajv = new Ajv2020({ strict: false, validateFormats: false }); diff --git a/tests/boundary/mcp/analysisViewMcp.test.ts b/tests/boundary/mcp/analysisViewMcp.test.ts index 749e9665b5..96bfae29e5 100644 --- a/tests/boundary/mcp/analysisViewMcp.test.ts +++ b/tests/boundary/mcp/analysisViewMcp.test.ts @@ -61,6 +61,8 @@ const connect = async ( const server = createServer( { kind: "session", session }, { + // These schema conformance cases exercise full nested constraints. + environment: { ...process.env, REA_MCP_INPUT_SCHEMA_PROFILE: "full" }, ...(binaryLayout === undefined ? {} : { binaryLayout }), ...(webNetworkCapture === undefined ? {} : { webNetworkCapture }), }, diff --git a/tests/boundary/mcp/applicationEvidenceMcp.test.ts b/tests/boundary/mcp/applicationEvidenceMcp.test.ts index 9e49735b40..3c0ca94c71 100644 --- a/tests/boundary/mcp/applicationEvidenceMcp.test.ts +++ b/tests/boundary/mcp/applicationEvidenceMcp.test.ts @@ -26,6 +26,8 @@ import { type ApplicationMcpHarness, } from "../../fixtures/applicationMcpHarness.js"; import { createHistoricalSourceGraph } from "../../../src/domain/referenceSourceGraph.js"; +import { advertisedInputJsonSchema } from "../../../src/contracts/toolSchemaMetadata.js"; +import { toolContract } from "../../../src/contracts/toolContracts.js"; async function runUnicodeSourceMapComparison( harness: ApplicationMcpHarness, @@ -257,7 +259,12 @@ async function runInlineWorkflowScenarios( exclusions: [{ path: "vendor/removed.ts", reason: "project-ignored" }], }, }; - expect(validate(missingPattern)).toBe(false); + expect( + ajv.validate( + advertisedInputJsonSchema(toolContract("compare_source_to_bundle")), + missingPattern, + ), + ).toBe(false); const rejectedExclusion = await harness.client.callTool({ name: "compare_source_to_bundle", arguments: missingPattern, diff --git a/tests/boundary/mcp/applicationExportShapeMcp.test.ts b/tests/boundary/mcp/applicationExportShapeMcp.test.ts index 8ace1e8af9..4252bbe489 100644 --- a/tests/boundary/mcp/applicationExportShapeMcp.test.ts +++ b/tests/boundary/mcp/applicationExportShapeMcp.test.ts @@ -26,7 +26,11 @@ it("executes the advertised export presence example with matching analyzed Evide ); if (example === undefined) throw new Error("Missing advertised presence example"); - const { client, close } = await createApplicationMcpHarness(); + const { client, close } = await createApplicationMcpHarness({ + serverOptions: { + environment: { ...process.env, REA_MCP_INPUT_SCHEMA_PROFILE: "full" }, + }, + }); onTestFinished(close); const listing = await client.listTools(); const advertised = listing.tools.find( diff --git a/tests/boundary/mcp/artifactMcp.inlineInventory.test.ts b/tests/boundary/mcp/artifactMcp.inlineInventory.test.ts index c691f63777..33be84d85e 100644 --- a/tests/boundary/mcp/artifactMcp.inlineInventory.test.ts +++ b/tests/boundary/mcp/artifactMcp.inlineInventory.test.ts @@ -188,7 +188,11 @@ it("preserves keyed archive identities and rejects inconsistent inventory carrie } }); -const createInventoryViewReader = async (client: Client, parent: Evidence) => { +const createInventoryViewReader = async ( + client: Client, + parent: Evidence, + profile: "full" | "compact", +) => { const advertised = (await client.listTools()).tools.find( (tool) => tool.name === "inspect_analysis_view", ); @@ -199,12 +203,18 @@ const createInventoryViewReader = async (client: Client, parent: Evidence) => { const outputSchema: Record = advertised.outputSchema; const validateInput = ajv.compile(inputSchema); const validateOutput = ajv.compile(outputSchema); - expect( - validateInput({ - source: { kind: "retained-evidence", evidence_id: parent.evidence_id }, - view: { kind: "inventory", collection: "nodes", logical_paths: [] }, - }), - ).toBe(false); + const malformed = { + source: { kind: "retained-evidence", evidence_id: parent.evidence_id }, + view: { kind: "inventory", collection: "nodes", logical_paths: [] }, + }; + if (profile === "full") expect(validateInput(malformed)).toBe(false); + const rejected = await client.callTool({ + name: "inspect_analysis_view", + arguments: malformed, + }); + expect(parseMcpToolError(rejected)).toMatchObject({ + error: { code: "invalid_request", category: "invalid_input" }, + }); const readView = async ( view: Record, source = { @@ -322,147 +332,155 @@ const assertSummaryDelivery = async ( ); }; -it("keeps complete ZIP output and reads retained inventory views after the source is removed", async () => { - const directory = await createTestTempDirectory("rea-artifact-inline-"); - const archive = join(directory, "many.zip"); - const writer = new ZipWriter(new Uint8ArrayWriter()); - const fileCount = 520; - for (let index = 0; index < fileCount; index += 1) - await writer.add( - `files/${String(index)}.txt`, - new TextReader(`file-${String(index)}`), - ); - await writeFile(archive, await writer.close()); +it.each(["full", "compact"] as const)( + "keeps complete ZIP output and reads retained inventory views after the source is removed (%s)", + async (profile) => { + const directory = await createTestTempDirectory("rea-artifact-inline-"); + const archive = join(directory, "many.zip"); + const writer = new ZipWriter(new Uint8ArrayWriter()); + const fileCount = 520; + for (let index = 0; index < fileCount; index += 1) + await writer.add( + `files/${String(index)}.txt`, + new TextReader(`file-${String(index)}`), + ); + await writeFile(archive, await writer.close()); - const session = createTestBinarySession(new ArtifactProvider(process.env)); - const server = createServer({ kind: "session", session }); - const client = new Client({ name: "artifact-inline-test", version: "1" }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - try { - await server.connect(serverTransport); - await client.connect(clientTransport); - const opened = await client.callTool({ - name: "open_binary", - arguments: { path: archive }, - }); - expect(opened.isError).not.toBe(true); - const result = await client.callTool({ - name: "inspect_artifact", - arguments: {}, - }); - expect(result.isError, JSON.stringify(result.structuredContent)).not.toBe( - true, - ); - const parent = parseEvidence(result.structuredContent); - await assertSummaryDelivery(client, session, parent); - const parsedInspection = artifactInspectionResultSchema.parse( - parent.normalized_result, - ); - const inventoryEvidence = parsedInspection.substeps[0]?.evidence; - if (inventoryEvidence === undefined) - throw new Error("Missing inventory Evidence"); - const graph = artifactInventoryResultSchema.parse( - inventoryEvidence.normalized_result, - ); - expect(graph).toMatchObject({ - integrity_contradictions: [], - coverage: { scope: "container-entries", status: "complete" }, - }); - expect(graph.occurrences).toHaveLength(fileCount + 1); - // Follow-ups must use retained source facts, not reopen this archive. - await unlink(archive); - const readView = await createInventoryViewReader(client, parent); - const summary = await readView({ kind: "summary" }); - expect(summary.result).toMatchObject({ - kind: "summary", - parent_evidence_id: parent.evidence_id, - artifact: { path: archive, sha256: graph.manifest.root_sha256 }, - summary: { - manifest: graph.manifest, - source_complete: true, - available_counts: { occurrences: fileCount + 1 }, - observed_entry_kind_counts: { file: fileCount + 1 }, - inventory_evidence_ids: [inventoryEvidence.evidence_id], - }, - }); - expect(summary.evidence.evidence_links).toEqual([parent.evidence_id]); - const paths = ["files/0.txt", "files/519.txt", "missing.txt"]; - const selected = await readView({ - kind: "inventory", - collection: "occurrences", - logical_paths: paths, - offset: 0, - limit: 1, - }); - expect(selected.result).toMatchObject({ - kind: "inventory", - source_complete: true, - items: graph.occurrences - .filter((row) => paths.includes(row.logical_path)) - .slice(0, 1), - coverage: { examined: 1, total: 2, next_offset: 1, exhausted: false }, - }); - const next = await readView({ - kind: "inventory", - collection: "occurrences", - logical_paths: paths, - offset: 1, - limit: 1, - }); - expect(next.result).toMatchObject({ - coverage: { examined: 1, next_offset: null, exhausted: true }, - }); - const none = await readView({ - kind: "inventory", - collection: "occurrences", - logical_paths: [], - }); - expect(none.result).toMatchObject({ - items: [], - coverage: { total: 0, exhausted: true }, - }); - - const partial = createEvidence( - { path: archive, sha256: graph.manifest.root_sha256, format: "zip" }, - inventoryEvidence.provider, + const session = createTestBinarySession(new ArtifactProvider(process.env)); + const server = createServer( + { kind: "session", session }, { - operation: "inventory_artifact", - parameters: {}, - result: jsonValueSchema.parse({ - ...graph, - occurrences: graph.occurrences.slice(0, 1), - }), + environment: { ...process.env, REA_MCP_INPUT_SCHEMA_PROFILE: profile }, }, ); - const partialReply = await client.callTool({ - name: "inspect_analysis_view", - arguments: { - source: { kind: "inline", evidence: partial }, - view: { - kind: "inventory", - collection: "occurrences", - logical_paths: ["missing.txt"], + const client = new Client({ name: "artifact-inline-test", version: "1" }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + try { + await server.connect(serverTransport); + await client.connect(clientTransport); + const opened = await client.callTool({ + name: "open_binary", + arguments: { path: archive }, + }); + expect(opened.isError).not.toBe(true); + const result = await client.callTool({ + name: "inspect_artifact", + arguments: {}, + }); + expect(result.isError, JSON.stringify(result.structuredContent)).not.toBe( + true, + ); + const parent = parseEvidence(result.structuredContent); + await assertSummaryDelivery(client, session, parent); + const parsedInspection = artifactInspectionResultSchema.parse( + parent.normalized_result, + ); + const inventoryEvidence = parsedInspection.substeps[0]?.evidence; + if (inventoryEvidence === undefined) + throw new Error("Missing inventory Evidence"); + const graph = artifactInventoryResultSchema.parse( + inventoryEvidence.normalized_result, + ); + expect(graph).toMatchObject({ + integrity_contradictions: [], + coverage: { scope: "container-entries", status: "complete" }, + }); + expect(graph.occurrences).toHaveLength(fileCount + 1); + // Follow-ups must use retained source facts, not reopen this archive. + await unlink(archive); + const readView = await createInventoryViewReader(client, parent, profile); + const summary = await readView({ kind: "summary" }); + expect(summary.result).toMatchObject({ + kind: "summary", + parent_evidence_id: parent.evidence_id, + artifact: { path: archive, sha256: graph.manifest.root_sha256 }, + summary: { + manifest: graph.manifest, + source_complete: true, + available_counts: { occurrences: fileCount + 1 }, + observed_entry_kind_counts: { file: fileCount + 1 }, + inventory_evidence_ids: [inventoryEvidence.evidence_id], }, - }, - }); - expect( - parseEvidence(partialReply.structuredContent).normalized_result, - ).toMatchObject({ - source_complete: false, - items: [], - unknowns: expect.arrayContaining([ - expect.stringContaining("Missing matches do not establish absence"), - ]), - }); + }); + expect(summary.evidence.evidence_links).toEqual([parent.evidence_id]); + const paths = ["files/0.txt", "files/519.txt", "missing.txt"]; + const selected = await readView({ + kind: "inventory", + collection: "occurrences", + logical_paths: paths, + offset: 0, + limit: 1, + }); + expect(selected.result).toMatchObject({ + kind: "inventory", + source_complete: true, + items: graph.occurrences + .filter((row) => paths.includes(row.logical_path)) + .slice(0, 1), + coverage: { examined: 1, total: 2, next_offset: 1, exhausted: false }, + }); + const next = await readView({ + kind: "inventory", + collection: "occurrences", + logical_paths: paths, + offset: 1, + limit: 1, + }); + expect(next.result).toMatchObject({ + coverage: { examined: 1, next_offset: null, exhausted: true }, + }); + const none = await readView({ + kind: "inventory", + collection: "occurrences", + logical_paths: [], + }); + expect(none.result).toMatchObject({ + items: [], + coverage: { total: 0, exhausted: true }, + }); - await assertPortableRoundtrip(client, directory, parent, summary.result); - } finally { - await client.close(); - await server.close(); - await session.close(); - } -}); + const partial = createEvidence( + { path: archive, sha256: graph.manifest.root_sha256, format: "zip" }, + inventoryEvidence.provider, + { + operation: "inventory_artifact", + parameters: {}, + result: jsonValueSchema.parse({ + ...graph, + occurrences: graph.occurrences.slice(0, 1), + }), + }, + ); + const partialReply = await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source: { kind: "inline", evidence: partial }, + view: { + kind: "inventory", + collection: "occurrences", + logical_paths: ["missing.txt"], + }, + }, + }); + expect( + parseEvidence(partialReply.structuredContent).normalized_result, + ).toMatchObject({ + source_complete: false, + items: [], + unknowns: expect.arrayContaining([ + expect.stringContaining("Missing matches do not establish absence"), + ]), + }); + + await assertPortableRoundtrip(client, directory, parent, summary.result); + } finally { + await client.close(); + await server.close(); + await session.close(); + } + }, +); it("refuses summary delivery without a retained Evidence owner and preserves complete delivery", async () => { const directory = await createTestTempDirectory("rea-inventory-fixed-"); diff --git a/tests/boundary/mcp/browserMcp.test.ts b/tests/boundary/mcp/browserMcp.test.ts index f26981cb0c..48593e4ef3 100644 --- a/tests/boundary/mcp/browserMcp.test.ts +++ b/tests/boundary/mcp/browserMcp.test.ts @@ -435,46 +435,53 @@ it("delivers the source-map response URL through authenticated MCP Evidence", as ).toBe(true); }); -it("records a browser-socket endpoint redacted and reconciles its Evidence", async () => { - const browser = await startFakeCdpBrowser(); - browsers.push(browser); - const connected = await connectBrowser(); - const tool = (await connected.client.listTools()).tools.find( - ({ name }) => name === "inspect_web_page", - ); - expect(tool?.inputSchema.properties?.cdp_endpoint).toMatchObject({ - description: expect.stringContaining("/devtools/browser/ID"), - }); - const inspected = await connected.client.callTool({ - name: "inspect_web_page", - arguments: { - cdp_endpoint: browser.browserWebSocketUrl, - target_id: "allowed-page", - observation_ms: 0, - include_script_sources: true, - }, - }); - expect(inspected.isError).not.toBe(true); - expect(inspected.structuredContent).toMatchObject({ - parameters: { - cdp_endpoint: browser.browserWebSocketUrl.replace(/fake$/u, "[REDACTED]"), - }, - }); - expect(JSON.stringify(inspected.structuredContent)).not.toContain( - "/devtools/browser/fake", - ); - const reconciled = await connected.client.callTool({ - name: "reconcile_javascript_runtime", - arguments: { - static_layers: JAVASCRIPT_RUNTIME_RECONCILIATION_EXAMPLE.static_layers, - runtime_observations: [evidenceFor(inspected.structuredContent)], - }, - }); - expect(reconciled.isError).not.toBe(true); - expect(reconciled.structuredContent).toMatchObject({ - normalized_result: { summary: { runtime_scripts: 1 } }, - }); -}); +it.each(["full", "compact"] as const)( + "records a browser-socket endpoint redacted and reconciles its Evidence (%s)", + async (profile) => { + const browser = await startFakeCdpBrowser(); + browsers.push(browser); + const connected = await connectBrowser(profile); + const tool = (await connected.client.listTools()).tools.find( + ({ name }) => name === "inspect_web_page", + ); + if (profile === "full") + expect(tool?.inputSchema.properties?.cdp_endpoint).toMatchObject({ + description: expect.stringContaining("/devtools/browser/ID"), + }); + const inspected = await connected.client.callTool({ + name: "inspect_web_page", + arguments: { + cdp_endpoint: browser.browserWebSocketUrl, + target_id: "allowed-page", + observation_ms: 0, + include_script_sources: true, + }, + }); + expect(inspected.isError).not.toBe(true); + expect(inspected.structuredContent).toMatchObject({ + parameters: { + cdp_endpoint: browser.browserWebSocketUrl.replace( + /fake$/u, + "[REDACTED]", + ), + }, + }); + expect(JSON.stringify(inspected.structuredContent)).not.toContain( + "/devtools/browser/fake", + ); + const reconciled = await connected.client.callTool({ + name: "reconcile_javascript_runtime", + arguments: { + static_layers: JAVASCRIPT_RUNTIME_RECONCILIATION_EXAMPLE.static_layers, + runtime_observations: [evidenceFor(inspected.structuredContent)], + }, + }); + expect(reconciled.isError).not.toBe(true); + expect(reconciled.structuredContent).toMatchObject({ + normalized_result: { summary: { runtime_scripts: 1 } }, + }); + }, +); it("does not attach to a target outside the request's allowed origin scope", async () => { const browser = await startFakeCdpBrowser(); @@ -538,7 +545,7 @@ it("projects legal oversized PNG dimensions as a memory constraint", async () => }); }); -const connectBrowser = async () => { +const connectBrowser = async (profile?: "full" | "compact") => { const session = createTestBinarySession(() => ({ execute: () => Promise.resolve(observed(null)), close: () => Promise.resolve(resultOk(null)), @@ -546,6 +553,7 @@ const connectBrowser = async () => { const server = createServer( { kind: "session", session }, { + environment: { ...process.env, REA_MCP_INPUT_SCHEMA_PROFILE: profile }, browserObservation: new CdpBrowserProvider(), availabilityPolicy: () => ({ processCaptureEnabled: false, diff --git a/tests/boundary/mcp/browserScenarioMcp.test.ts b/tests/boundary/mcp/browserScenarioMcp.test.ts index e0b5bb1e3a..6ed007354f 100644 --- a/tests/boundary/mcp/browserScenarioMcp.test.ts +++ b/tests/boundary/mcp/browserScenarioMcp.test.ts @@ -22,6 +22,8 @@ import { err, ok, type Result } from "../../../src/domain/result.js"; import { createServer } from "../../../src/server/createServer.js"; import { observed } from "../../fixtures/analysisExecution.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; +import { advertisedInputJsonSchema } from "../../../src/contracts/toolSchemaMetadata.js"; +import { toolContract } from "../../../src/contracts/toolContracts.js"; const providerIdentity: ProviderIdentity = { id: "test-browser-scenario", @@ -257,6 +259,10 @@ describe("browser scenario MCP tool", () => { strict: false, validateFormats: false, }).compile(comparisonTool.inputSchema); + const validateFullComparison = new Ajv2020({ + strict: false, + validateFormats: false, + }).compile(advertisedInputJsonSchema(toolContract("compare_web_captures"))); expect( validateComparison(comparisonInput), JSON.stringify(validateComparison.errors), @@ -266,12 +272,14 @@ describe("browser scenario MCP tool", () => { { before_scenario: captureResult, after: captureResult }, { ...comparisonInput, before: captureResult, after: captureResult }, ]) { - expect(validateComparison(malformed)).toBe(false); + expect(validateFullComparison(malformed)).toBe(false); const rejected = await client.callTool({ name: "compare_web_captures", arguments: malformed, }); - expect(rejected.isError).toBe(true); + expect(parseMcpToolError(rejected)).toMatchObject({ + error: { code: "invalid_request", category: "invalid_input" }, + }); } const compared = await client.callTool({ name: "compare_web_captures", diff --git a/tests/boundary/mcp/contractPresentation.test.ts b/tests/boundary/mcp/contractPresentation.test.ts index 9d1d9d6bb0..fdb0b5c545 100644 --- a/tests/boundary/mcp/contractPresentation.test.ts +++ b/tests/boundary/mcp/contractPresentation.test.ts @@ -25,8 +25,13 @@ const resultSchema = z.object({ ), }); -it("preserves advertised schemas and handler meaning when presentation order changes", async () => { +it("preserves full advertised schemas and handler meaning when presentation order changes", async () => { const { stdout } = await execute(process.execPath, [fixture, "reversed"], { + env: { + ...process.env, + REA_MCP_INPUT_SCHEMA_PROFILE: "full", + REA_MCP_OUTPUT_SCHEMA_PROFILE: "full", + }, timeout: 20_000, maxBuffer: 16 * 1024 * 1024, }); diff --git a/tests/boundary/mcp/functionCallDirectionsMcp.test.ts b/tests/boundary/mcp/functionCallDirectionsMcp.test.ts index b4034d036a..3084c070e1 100644 --- a/tests/boundary/mcp/functionCallDirectionsMcp.test.ts +++ b/tests/boundary/mcp/functionCallDirectionsMcp.test.ts @@ -8,6 +8,7 @@ import { functionDossierSchema } from "../../../src/domain/hopperValues.js"; import type { RelationshipCoverage } from "../../../src/domain/relationshipCoverage.js"; import { createServer } from "../../../src/server/createServer.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; const observe = (relationshipCoverage: RelationshipCoverage) => { const template = FUNCTION_COMPARISON_EXAMPLE.left; @@ -32,11 +33,15 @@ const observe = (relationshipCoverage: RelationshipCoverage) => { ); }; -it("preserves directional gaps through advertised schemas and public path reconstruction", async () => { +const profiles = ["full", "compact"] as const; +it.each(profiles)("preserves directional gaps (%s)", async (profile) => { const session = createTestBinarySession(() => { throw new Error("Offline call paths must not launch a provider"); }); - const server = createServer({ kind: "session", session }); + const server = createServer( + { kind: "session", session }, + { environment: { ...process.env, REA_MCP_INPUT_SCHEMA_PROFILE: profile } }, + ); const client = new Client({ name: "function-call-directions", version: "1" }); const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); @@ -89,21 +94,28 @@ it("preserves directional gaps through advertised schemas and public path recons ["callees", "callees"], ["callers", "callees", "callers"], ["incoming"], - ]) - expect( - validate({ - ...input, - functions: [ - { - ...observation, - relationship_coverage: { - ...coverage, - limited_directions, - }, + ]) { + const malformed = { + ...input, + functions: [ + { + ...observation, + relationship_coverage: { + ...coverage, + limited_directions, }, - ], - }), - ).toBe(false); + }, + ], + }; + if (profile === "full") expect(validate(malformed)).toBe(false); + const rejected = await client.callTool({ + name: "build_call_path", + arguments: malformed, + }); + expect(parseMcpToolError(rejected)).toMatchObject({ + error: { code: "invalid_request", category: "invalid_input" }, + }); + } const result = await client.callTool({ name: "build_call_path", arguments: input, diff --git a/tests/boundary/mcp/godotPckMcp.test.ts b/tests/boundary/mcp/godotPckMcp.test.ts index bb0ffcd4d7..b7b2a7dc85 100644 --- a/tests/boundary/mcp/godotPckMcp.test.ts +++ b/tests/boundary/mcp/godotPckMcp.test.ts @@ -2,9 +2,11 @@ import { writeFile, unlink } from "node:fs/promises"; import { join } from "node:path"; import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { Ajv2020 } from "ajv/dist/2020.js"; +import type { ValidateFunction } from "ajv"; import { expect, it } from "vitest"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { buildGodotPck } from "../../fixtures/godotPck.js"; import { ArtifactProvider } from "../../../src/artifacts/ArtifactProvider.js"; import { createServer } from "../../../src/server/createServer.js"; @@ -36,7 +38,28 @@ const viewValidators = async (client: Client) => { return { validateInput, validateOutput }; }; -it("validates advertised schemas, batches connected facts and reuses one retained inventory after unlink", async () => { +const assertInvalidNodeField = async ( + client: Client, + evidenceId: string, + validateFullInput?: ValidateFunction, +) => { + const arguments_ = { + source: { kind: "retained-evidence", evidence_id: evidenceId }, + view: { kind: "inventory", collection: "nodes", fields: ["logical_path"] }, + }; + if (validateFullInput !== undefined) + expect(validateFullInput(arguments_)).toBe(false); + const rejected = await client.callTool({ + name: "inspect_analysis_view", + arguments: arguments_, + }); + expect(parseMcpToolError(rejected)).toMatchObject({ + error: { code: "invalid_request", category: "invalid_input" }, + }); +}; + +const profiles = ["full", "compact"] as const; +it.each(profiles)("reuses retained PCK inventory (%s)", async (profile) => { const root = await createTestTempDirectory("rea-pck-mcp-"); const path = join(root, "game.pck"); await writeFile( @@ -47,7 +70,12 @@ it("validates advertised schemas, batches connected facts and reuses one retaine ]), ); const session = createTestBinarySession(new ArtifactProvider({})); - const server = createServer({ kind: "session", session }); + const server = createServer( + { kind: "session", session }, + { + environment: { ...process.env, REA_MCP_INPUT_SCHEMA_PROFILE: profile }, + }, + ); const client = new Client({ name: "pck-contract-test", version: "1" }); const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); @@ -113,16 +141,11 @@ it("validates advertised schemas, batches connected facts and reuses one retaine expect(evidence.evidence_links).toContain(parent.evidence_id); return analysisViewResultSchema.parse(evidence.normalized_result); }; - expect( - validateInput({ - source, - view: { - kind: "inventory", - collection: "nodes", - fields: ["logical_path"], - }, - }), - ).toBe(false); + await assertInvalidNodeField( + client, + parent.evidence_id, + profile === "full" ? validateInput : undefined, + ); const wanted = graph.occurrences.filter((row) => row.logical_path !== "."); const occurrences = await view({ collection: "occurrences", diff --git a/tests/boundary/mcp/javascriptFactsViewMcp.test.ts b/tests/boundary/mcp/javascriptFactsViewMcp.test.ts index f4f03bd06e..3316cd1e3c 100644 --- a/tests/boundary/mcp/javascriptFactsViewMcp.test.ts +++ b/tests/boundary/mcp/javascriptFactsViewMcp.test.ts @@ -5,6 +5,9 @@ import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarnes import { createTestBinarySession } from "../../fixtures/binarySession.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { advertisedMcpToolSchemas } from "../../fixtures/advertisedMcpTool.js"; +import { advertisedInputJsonSchema } from "../../../src/contracts/toolSchemaMetadata.js"; +import { toolContract } from "../../../src/contracts/toolContracts.js"; +import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { analysisViewJavaScriptAnalysis, analysisViewJavaScriptEvidence, @@ -79,12 +82,18 @@ it("pages selected real filesystem observations with shared identities and porta "inspect_analysis_view", ); expect(ajv.validate(inputSchema, { source, view: query })).toBe(true); - expect( - ajv.validate(inputSchema, { - source, - view: { ...query, property_keys: [""] }, - }), - ).toBe(false); + const malformed = { source, view: { ...query, property_keys: [""] } }; + const fullInputSchema = advertisedInputJsonSchema( + toolContract("inspect_analysis_view"), + ); + expect(ajv.validate(fullInputSchema, malformed)).toBe(false); + const rejected = await harness.client.callTool({ + name: "inspect_analysis_view", + arguments: malformed, + }); + expect(parseMcpToolError(rejected)).toMatchObject({ + error: { code: "invalid_request", category: "invalid_input" }, + }); const full = await facts(harness, source, query); expect(ajv.validate(outputSchema, full.wire.structuredContent)).toBe(true); const portable = await facts( diff --git a/tests/boundary/mcp/mcpDoctor.test.ts b/tests/boundary/mcp/mcpDoctor.test.ts index f7cecc5bec..60504df323 100644 --- a/tests/boundary/mcp/mcpDoctor.test.ts +++ b/tests/boundary/mcp/mcpDoctor.test.ts @@ -15,7 +15,7 @@ import { runProductionMcpDoctorCli, } from "../../../src/mcpDoctor.js"; import { - CATALOG_IDENTITY, + FULL_CATALOG_IDENTITY, createCatalogIdentity, } from "../../../src/catalogIdentity.js"; import { compactPresentationForMcpInputSchemaProfile } from "../../../src/config/mcpInputSchemaProfile.js"; @@ -23,6 +23,25 @@ import { MCP_STARTUP_POLICY } from "../../../src/mcpStartupPolicy.js"; afterEach(() => vi.useRealTimers()); +const assertCliCatalogIdentity = async ( + environment: NodeJS.ProcessEnv, + identity: ReturnType, +) => { + for (const command of ["capabilities", "providers"]) { + const { stdout } = await promisify(execFile)( + process.execPath, + [resolve("scripts/rea.mjs"), command, "--json"], + { env: environment, timeout: 15_000, maxBuffer: 1024 * 1024 }, + ); + const cliStatus: unknown = JSON.parse(stdout); + expect(cliStatus, command).toMatchObject({ + server_identity: { + catalog: { counts: identity.counts, digests: identity.digests }, + }, + }); + } +}; + describe("production MCP doctor", () => { it("reuses immutable compiled profile digests and preserves custom-budget identity", async () => { // Exercise the generated module with Node, outside Vitest's source loader. @@ -81,13 +100,23 @@ describe("production MCP doctor", () => { }); it.each([ + { + name: "default compact input", + environment: { + REA_MCP_INPUT_SCHEMA_PROFILE: undefined, + REA_MCP_OUTPUT_SCHEMA_PROFILE: "full", + }, + identity: createCatalogIdentity( + compactPresentationForMcpInputSchemaProfile(undefined), + ), + }, { name: "full", environment: { REA_MCP_INPUT_SCHEMA_PROFILE: "full", REA_MCP_OUTPUT_SCHEMA_PROFILE: "full", }, - identity: CATALOG_IDENTITY, + identity: FULL_CATALOG_IDENTITY, }, { name: "compact", @@ -119,7 +148,7 @@ describe("production MCP doctor", () => { ), }, ])( - "connects to the production stdio child and verifies the $name catalog", + "verifies the $name catalog through production stdio and both CLI inventories", async ({ environment, identity }) => { const result = await runProductionMcpDoctor({ command: process.execPath, @@ -145,7 +174,7 @@ describe("production MCP doctor", () => { result.inventory?.tools.observed, ); expect(result.inventory?.tools.expected).toBe( - CATALOG_IDENTITY.counts.mcp_tools, + FULL_CATALOG_IDENTITY.counts.mcp_tools, ); expect(result.identity?.expected_catalog_digest).toBe( identity.digests.combined_sha256, @@ -153,6 +182,10 @@ describe("production MCP doctor", () => { expect(result.identity?.observed_catalog_digest).toBe( identity.digests.combined_sha256, ); + await assertCliCatalogIdentity( + { ...process.env, ...environment }, + identity, + ); }, 30_000, ); @@ -213,10 +246,10 @@ describe("production MCP doctor probe validation", () => { }); expect(result.healthy).toBe(mode === "valid"); expect(result.inventory?.tools.observed).toBe( - CATALOG_IDENTITY.counts.mcp_tools, + FULL_CATALOG_IDENTITY.counts.mcp_tools, ); expect(result.inventory?.prompts.observed).toBe( - CATALOG_IDENTITY.counts.mcp_prompts, + FULL_CATALOG_IDENTITY.counts.mcp_prompts, ); expect(result.checks).toContainEqual({ name: "initialize", diff --git a/tests/boundary/mcp/outputSchemaProfile.test.ts b/tests/boundary/mcp/outputSchemaProfile.test.ts index a298ef0918..b647b44aa8 100644 --- a/tests/boundary/mcp/outputSchemaProfile.test.ts +++ b/tests/boundary/mcp/outputSchemaProfile.test.ts @@ -10,7 +10,7 @@ import { EvidenceMcpServer } from "../../../src/server/EvidenceMcpServer.js"; import { toolRegistrationOptions } from "../../../src/server/toolRegistrationOptions.js"; import { createToolResultDelivery } from "../../../src/server/toolResult.js"; import type { ToolContract } from "../../../src/contracts/toolContractTypes.js"; -import { CATALOG_IDENTITY } from "../../../src/catalogIdentity.js"; +import { FULL_CATALOG_IDENTITY } from "../../../src/catalogIdentity.js"; const createServer = (omitOutputSchemas: boolean) => new EvidenceMcpServer( @@ -25,7 +25,7 @@ const createServer = (omitOutputSchemas: boolean) => it("omits only output schema advertisements and keeps canonical MCP output validation", async () => { const server = createServer(true); expect(server.catalogIdentity.digests.combined_sha256).not.toBe( - CATALOG_IDENTITY.digests.combined_sha256, + FULL_CATALOG_IDENTITY.digests.combined_sha256, ); const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); diff --git a/tests/boundary/mcp/retainedApplicationEvidenceMcp.test.ts b/tests/boundary/mcp/retainedApplicationEvidenceMcp.test.ts index ebc7062a95..d449b51d4a 100644 --- a/tests/boundary/mcp/retainedApplicationEvidenceMcp.test.ts +++ b/tests/boundary/mcp/retainedApplicationEvidenceMcp.test.ts @@ -24,11 +24,14 @@ const application = JAVASCRIPT_FEATURE_TRACE_EXAMPLE.application; const pair = JAVASCRIPT_APPLICATION_VERSION_COMPARISON_EXAMPLE; const exportPair = JAVASCRIPT_EXPORT_SHAPE_COMPARISON_EXAMPLE; -async function connect() { +async function connect(profile?: "full" | "compact") { const session = createTestBinarySession(() => { throw new Error("references must not launch a provider"); }); - const server = createServer({ kind: "session", session }); + const server = createServer( + { kind: "session", session }, + { environment: { ...process.env, REA_MCP_INPUT_SCHEMA_PROFILE: profile } }, + ); const client = new Client({ name: "retained-evidence-test", version: "1" }); resources.push(client, server, session); const [a, b] = InMemoryTransport.createLinkedPair(); @@ -37,33 +40,39 @@ async function connect() { return { client, session }; } -it("advertises the primitive literal seeds accepted by semantic tracing", async () => { - const { client } = await connect(); - const { tools } = await client.listTools(); - const tool = tools.find(({ name }) => name === "trace_javascript_semantics"); - if (tool === undefined) throw new Error("Semantic trace tool is missing"); - const validate = new Ajv2020({ - strict: false, - validateFormats: false, - }).compile(tool.inputSchema); - for (const value of ["literal", 42, true, null, {}, []]) { - const primitive = value === null || typeof value !== "object"; - const arguments_ = { - application, - query: { - seed: { kind: "literal", value }, - direction: "backward-provenance", - }, - }; - expect(validate(arguments_), JSON.stringify(value)).toBe(primitive); - const result = await client.callTool({ - name: tool.name, - arguments: arguments_, - }); - if (primitive) expect(result.isError).not.toBe(true); - else expect(result.isError).toBe(true); - } -}); +it.each(["full", "compact"] as const)( + "advertises the primitive literal seeds accepted by semantic tracing (%s)", + async (profile) => { + const { client } = await connect(profile); + const { tools } = await client.listTools(); + const tool = tools.find( + ({ name }) => name === "trace_javascript_semantics", + ); + if (tool === undefined) throw new Error("Semantic trace tool is missing"); + const validate = new Ajv2020({ + strict: false, + validateFormats: false, + }).compile(tool.inputSchema); + for (const value of ["literal", 42, true, null, {}, []]) { + const primitive = value === null || typeof value !== "object"; + const arguments_ = { + application, + query: { + seed: { kind: "literal", value }, + direction: "backward-provenance", + }, + }; + if (primitive || profile === "full") + expect(validate(arguments_), JSON.stringify(value)).toBe(primitive); + const result = await client.callTool({ + name: tool.name, + arguments: arguments_, + }); + if (primitive) expect(result.isError).not.toBe(true); + else expect(result.isError).toBe(true); + } + }, +); it("runs the same trace/compare workflows with inline and retained application Evidence", async () => { const { client, session } = await connect(); diff --git a/tests/boundary/mcp/serverIdentity.test.ts b/tests/boundary/mcp/serverIdentity.test.ts index 68a726c29c..a8e4ef5057 100644 --- a/tests/boundary/mcp/serverIdentity.test.ts +++ b/tests/boundary/mcp/serverIdentity.test.ts @@ -11,10 +11,13 @@ import { createTestBinarySession } from "../../fixtures/binarySession.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { machoImage } from "../../../src/artifacts/apple/MachoImage.fixture.js"; import { - CATALOG_IDENTITY, + FULL_CATALOG_IDENTITY, + DEFAULT_CATALOG_IDENTITY, createCatalogDigestProfiles, createCatalogDigests, + createCatalogIdentity, } from "../../../src/catalogIdentity.js"; +import { COMPACT_INPUT_SCHEMA_BUDGET_BYTES } from "../../../src/config/mcpInputSchemaProfile.js"; import { CLI_COMMAND_NAMES } from "../../../src/cliCommandNames.js"; import { PACKAGE_METADATA } from "../../../src/generatedPackageMetadata.js"; import { PRODUCT_IDENTITY, SDK_IDENTITY } from "../../../src/identity.js"; @@ -112,16 +115,18 @@ describe("server and catalog identity", () => { packageJson.dependencies["@modelcontextprotocol/server"], ); expect(new Set(CLI_COMMAND_NAMES).size).toBe(CLI_COMMAND_NAMES.length); - expect(CATALOG_IDENTITY.counts).toEqual({ + expect(FULL_CATALOG_IDENTITY.counts).toEqual({ cli_commands: CLI_COMMAND_NAMES.length, mcp_tools: TOOL_CONTRACTS.length, mcp_prompts: PROMPT_CONTRACTS.length, }); - expect(CATALOG_IDENTITY.digests.combined_sha256).toMatch(/^[a-f0-9]{64}$/u); + expect(FULL_CATALOG_IDENTITY.digests.combined_sha256).toMatch( + /^[a-f0-9]{64}$/u, + ); }); it("covers each tool's target-content policy in the catalog digest", () => { - expect(createCatalogDigests()).toEqual(CATALOG_IDENTITY.digests); + expect(createCatalogDigests()).toEqual(FULL_CATALOG_IDENTITY.digests); const [first] = TOOL_CONTRACTS; if (first === undefined) throw new Error("expected a tool contract"); const toggled = TOOL_CONTRACTS.map((contract) => @@ -131,18 +136,20 @@ describe("server and catalog identity", () => { ); const digests = createCatalogDigests(toggled); expect(digests.combined_sha256).not.toBe( - CATALOG_IDENTITY.digests.combined_sha256, + FULL_CATALOG_IDENTITY.digests.combined_sha256, ); expect(digests.tools_sha256).not.toBe( - CATALOG_IDENTITY.digests.tools_sha256, + FULL_CATALOG_IDENTITY.digests.tools_sha256, ); expect(digests.prompts_sha256).toBe( - CATALOG_IDENTITY.digests.prompts_sha256, + FULL_CATALOG_IDENTITY.digests.prompts_sha256, ); }); it("binds output schema profile and canonical output contracts into identity", () => { const profiles = createCatalogDigestProfiles(); + expect(FULL_CATALOG_IDENTITY.digests).toEqual(profiles.full.full); + expect(DEFAULT_CATALOG_IDENTITY.digests).toEqual(profiles.compact.full); const changedOutput = TOOL_CONTRACTS.map((contract, index) => index === 0 ? { @@ -187,22 +194,25 @@ describe("server and catalog identity", () => { it("reports unknown without a live comparison and distinguishes aligned from stale", () => { const unknown = createServerIdentity({ + catalog: DEFAULT_CATALOG_IDENTITY, startedAt: "2026-07-13T00:00:00.000Z", commandPath: normalizeEntrypointPath(""), }); expect(unknown.alignment.state).toBe("unknown"); expect(unknown.server.command_path).toBeNull(); const aligned = createServerIdentity({ + catalog: DEFAULT_CATALOG_IDENTITY, startedAt: "2026-07-13T00:00:00.000Z", commandPath: "/packaged/rea.mjs", expected: { package_version: PRODUCT_IDENTITY.packageVersion, - catalog_digest: CATALOG_IDENTITY.digests.combined_sha256, + catalog_digest: DEFAULT_CATALOG_IDENTITY.digests.combined_sha256, server_path: "/packaged/rea.mjs", }, }); expect(aligned.alignment).toMatchObject({ state: "aligned", reasons: [] }); const stale = createServerIdentity({ + catalog: DEFAULT_CATALOG_IDENTITY, startedAt: "2026-07-13T00:00:00.000Z", commandPath: "/packaged/rea.mjs", expected: { @@ -218,6 +228,7 @@ describe("server and catalog identity", () => { it("keeps registration alignment unknown when the server entrypoint is absent", () => { const identity = createServerIdentity({ + catalog: DEFAULT_CATALOG_IDENTITY, startedAt: "2026-07-13T00:00:00.000Z", commandPath: undefined, expected: { server_path: "/registered/rea.mjs" }, @@ -307,12 +318,14 @@ describe("guided prompt catalog identity", () => { ); const digests = createCatalogDigests(TOOL_CONTRACTS, changed); expect(digests.prompts_sha256).not.toBe( - CATALOG_IDENTITY.digests.prompts_sha256, + FULL_CATALOG_IDENTITY.digests.prompts_sha256, ); expect(digests.combined_sha256).not.toBe( - CATALOG_IDENTITY.digests.combined_sha256, + FULL_CATALOG_IDENTITY.digests.combined_sha256, + ); + expect(digests.tools_sha256).toBe( + FULL_CATALOG_IDENTITY.digests.tools_sha256, ); - expect(digests.tools_sha256).toBe(CATALOG_IDENTITY.digests.tools_sha256); }); }); @@ -390,7 +403,7 @@ describe("live server identity over MCP", () => { reasons: [], }); expect(server.catalogIdentity.digests.tools_sha256).not.toBe( - CATALOG_IDENTITY.digests.tools_sha256, + FULL_CATALOG_IDENTITY.digests.tools_sha256, ); } finally { await Promise.allSettled([ @@ -538,7 +551,9 @@ const assertLiveIdentity = async (client: Client): Promise => { }, client: { name: "identity-test", version: "9" }, negotiated_protocol_version: expect.any(String), - catalog: CATALOG_IDENTITY, + catalog: createCatalogIdentity({ + budgetBytes: COMPACT_INPUT_SCHEMA_BUDGET_BYTES, + }), alignment: { state: "unknown" }, }, }, diff --git a/tests/boundary/process/packageTarListing.test.ts b/tests/boundary/process/packageTarListing.test.ts index 58e48cbd1c..e876811bac 100644 --- a/tests/boundary/process/packageTarListing.test.ts +++ b/tests/boundary/process/packageTarListing.test.ts @@ -35,7 +35,7 @@ async function verifyListing(listing: string) { `import childProcess from "node:child_process"; import { syncBuiltinESMExports } from "node:module"; import { promisify } from "node:util"; -import { CATALOG_IDENTITY } from ${JSON.stringify(catalogUrl)}; +import { FULL_CATALOG_IDENTITY } from ${JSON.stringify(catalogUrl)}; let bridgeParsed = false; childProcess.execFile = () => { throw new Error("Expected promisified execution"); }; childProcess.execFile[promisify.custom] = async (command, args) => { @@ -45,7 +45,7 @@ childProcess.execFile[promisify.custom] = async (command, args) => { else if (command === "tar" && args[0] === "-xOf" && args[2] === "package/package.json") stdout = JSON.stringify({ bin: { rea: "scripts/rea.mjs", "rea-agents": "scripts/rea.mjs" }, mcpName: "io.github.morluto/rea", dependencies: { "@lydell/node-pty": "1.1.0" } }); else if (command === "tar" && args[0] === "-xOf" && args[2] === "package/skills/reverse-engineer-anything/SKILL.md") - stdout = " tool_count: " + CATALOG_IDENTITY.counts.mcp_tools + "\\n"; + stdout = " tool_count: " + FULL_CATALOG_IDENTITY.counts.mcp_tools + "\\n"; else if (command === "tar" && args[0] === "-xf") stdout = ""; else if (command === "python3") { bridgeParsed = true; stdout = ""; } else throw new Error("Unexpected command: " + command + " " + JSON.stringify(args)); diff --git a/tests/conformance/contracts/mcpInputSchemaProfile.test.ts b/tests/conformance/contracts/mcpInputSchemaProfile.test.ts index 661f545087..48b3769d27 100644 --- a/tests/conformance/contracts/mcpInputSchemaProfile.test.ts +++ b/tests/conformance/contracts/mcpInputSchemaProfile.test.ts @@ -31,10 +31,10 @@ describe("MCP input schema profile configuration", () => { }, ); - it("selects the same result-affecting budget for the compact profile", () => { - expect(compactPresentationForMcpInputSchemaProfile(undefined)).toBe( - undefined, - ); + it("defaults to compact while preserving an explicit full choice", () => { + expect(compactPresentationForMcpInputSchemaProfile(undefined)).toEqual({ + budgetBytes: COMPACT_INPUT_SCHEMA_BUDGET_BYTES, + }); expect(compactPresentationForMcpInputSchemaProfile("full")).toBe(undefined); expect(compactPresentationForMcpInputSchemaProfile("compact")).toEqual({ budgetBytes: COMPACT_INPUT_SCHEMA_BUDGET_BYTES,