refactor(evaluation): move agent grading into development ownership

Keep the maintained manual evaluator, fixture rubrics and semantic tests together under tests/evaluation/model-evals instead of shipping them in dist. Load the same typed owners from the manual runner and update the architecture and client guide. Grading/report contracts and executable logic are unchanged.

docs/architecture.mermaid

@@ -86,8 +86,8 @@ graph TD
         TESTS["tests/<br/>Vitest suite<br/>fixtures/ holds fake launcher, bridge, and CDP seams"]
     end
 
-    subgraph "Evaluation"
+    subgraph "Development Evaluation"
-        EVALUATION["src/evaluation/<br/>Agent release evaluation<br/>session transcript scoring"]
+        EVALUATION["tests/evaluation/model-evals/<br/>Agent release evaluation<br/>session transcript scoring"]
     end
 
     DISPATCH --> CLI

docs/client-verification.md

@@ -458,6 +458,11 @@ real local Codex CLI with:
 npm run verify:agent
 ```
 
+The verifier's scenarios, transcript scoring, fixture rubrics, and report gates
+are development verification code under `tests/evaluation/model-evals/`. The
+source-owned evaluator runs alongside the compiled REA tools; it is excluded
+from the shipped runtime package.
+
 The lane runs scenarios sequentially in standalone Codex (`--no-daemon`) with a
 disposable HOME and CODEX_HOME under `~/.cache/rea-agent-evaluations/`. Set
 `REA_AGENT_EVAL_ROOT` to choose another parent directory outside the OS temporary
@@ -487,7 +492,7 @@ boundary instead of stopping at Codex approval. Its scenario contract accepts
 one `provider_unavailable` open refusal without a close, or one successful open
 followed by a successful close. Other errors, an unclosed session, and a close
 without acquisition fail. These outcomes and the prompt are owned together in
-`src/evaluation/AgentEvaluationScenarios.ts`; generic successful-tool sequences
+`tests/evaluation/model-evals/AgentEvaluationScenarios.ts`; generic successful-tool sequences
 still apply to scenarios that require them. The provider scenario checks routing,
 reported limitations, and cleanup; free-form answer correctness remains
 unassessed. It does not validate a real native engine or authorize runtime execution.

knip.json

@@ -10,7 +10,6 @@
     "src/composition/javascriptAnalysisWorkerProcess.ts",
     "src/composition/webBundleAnalysisWorkerProcess.ts",
     "src/contracts/errorSchemas.ts",
-    "src/evaluation/AgentEvaluationFixtures.ts",
     "scripts/verify/fridaBindingsCompatibility.ts",
     "scripts/verify/termux/browser.mjs",
     "scripts/verify-real-ghidra-noreturn.mjs",

scripts/verify-agent-experience.mjs

@@ -11,26 +11,44 @@ import {
 import { homedir } from "node:os";
 import { delimiter, dirname, join, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
+import { tsImport } from "tsx/esm/api";
 
-import { evaluateCodexEvents } from "../dist/evaluation/CodexAgentEval.js";
-import {
-  MISSING_TARGET_SCENARIO,
-  nativeProviderScenario,
-} from "../dist/evaluation/AgentEvaluationScenarios.js";
-import {
-  createAgentEvaluationFixtures,
-  agentFixtureClaims,
-  agentFixtureAnswerInstructions,
-} from "../dist/evaluation/AgentEvaluationFixtures.js";
-import {
-  agentEvaluationPassed,
-  summarizeFactualCorrectness,
-} from "../dist/evaluation/AgentEvaluationReport.js";
 import { PRODUCT_IDENTITY } from "../dist/identity.js";
 import { createAgentEvaluationCommands } from "./lib/agent-evaluation-commands.mjs";
 import { closeVerifierResources } from "./lib/verifier-cleanup.mjs";
 import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs";
 
+const { evaluateCodexEvents } = await tsImport(
+  new URL("../tests/evaluation/model-evals/CodexAgentEval.ts", import.meta.url)
+    .href,
+  import.meta.url,
+);
+const { MISSING_TARGET_SCENARIO, nativeProviderScenario } = await tsImport(
+  new URL(
+    "../tests/evaluation/model-evals/AgentEvaluationScenarios.ts",
+    import.meta.url,
+  ).href,
+  import.meta.url,
+);
+const {
+  createAgentEvaluationFixtures,
+  agentFixtureClaims,
+  agentFixtureAnswerInstructions,
+} = await tsImport(
+  new URL(
+    "../tests/evaluation/model-evals/AgentEvaluationFixtures.ts",
+    import.meta.url,
+  ).href,
+  import.meta.url,
+);
+const { agentEvaluationPassed, summarizeFactualCorrectness } = await tsImport(
+  new URL(
+    "../tests/evaluation/model-evals/AgentEvaluationReport.ts",
+    import.meta.url,
+  ).href,
+  import.meta.url,
+);
+
 const repositoryRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
 const verifierRun = createVerifierRun();
 const timeoutMs = Number(process.env.REA_AGENT_EVAL_TIMEOUT_MS ?? 480_000);

tests/boundary/mcp/agentKnownAnswerMcp.test.ts

@@ -5,12 +5,12 @@ import { describe, expect, it } from "vitest";
 import { createPackage } from "@electron/asar";
 
 import { parseEvidence, type Evidence } from "../../../src/domain/evidence.js";
-import { evaluateCodexEvents } from "../../../src/evaluation/CodexAgentEval.js";
+import { evaluateCodexEvents } from "../../evaluation/model-evals/CodexAgentEval.js";
 import {
   createAgentEvaluationFixtures,
   agentFixtureClaims,
-} from "../../../src/evaluation/AgentEvaluationFixtures.js";
+} from "../../evaluation/model-evals/AgentEvaluationFixtures.js";
-import type { FixtureClaimExpectation } from "../../../src/evaluation/KnownAnswerEvaluation.js";
+import type { FixtureClaimExpectation } from "../../evaluation/model-evals/KnownAnswerEvaluation.js";
 import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js";
 import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
 

tests/evaluation/model-evals/AgentEvaluationFixtures.ts

@@ -4,24 +4,24 @@ import { join } from "node:path";
 
 import { createPackage } from "@electron/asar";
 
-import type { JsonValue } from "../domain/jsonValue.js";
+import type { JsonValue } from "../../../src/domain/jsonValue.js";
-import { javascriptApplicationAnalysisResultSchema } from "../domain/javascript/javascriptApplicationAnalysis.js";
+import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js";
-import type { ApplicationNode } from "../domain/javascript/javascriptApplicationGraphSchemas.js";
+import type { ApplicationNode } from "../../../src/domain/javascript/javascriptApplicationGraphSchemas.js";
-import { javaScriptExportShapeComparisonResultSchema } from "../domain/javascript/javascriptExportShapeComparisonSchemas.js";
+import { javaScriptExportShapeComparisonResultSchema } from "../../../src/domain/javascript/javascriptExportShapeComparisonSchemas.js";
 import {
   applicationFeatureTraceResultSchema,
   featureSeedMatchMode,
-} from "../domain/javascript/javascriptFeatureTraceSchemas.js";
+} from "../../../src/domain/javascript/javascriptFeatureTraceSchemas.js";
-import { javaScriptSemanticTraceResultSchema } from "../domain/javascript/javascriptSemanticTraceSchemas.js";
+import { javaScriptSemanticTraceResultSchema } from "../../../src/domain/javascript/javascriptSemanticTraceSchemas.js";
 import {
   managedArtifactInspectionSchema,
   managedMemberInspectionSchema,
-} from "../domain/managed/managedArtifact.js";
+} from "../../../src/domain/managed/managedArtifact.js";
 import {
   managedTokenScopesMatch,
   readManagedInspectionEvidence,
-} from "../domain/managed/managedInspectionEvidence.js";
+} from "../../../src/domain/managed/managedInspectionEvidence.js";
-import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js";
+import { compareUnicodeCodePoints } from "../../../src/domain/unicodeCodePointOrder.js";
 import type {
   FixtureClaimContext,
   FixtureClaimExpectation,

tests/evaluation/model-evals/CodexAgentEval.ts

@@ -5,8 +5,8 @@ import {
   type KnownAnswerAssessment,
   type ParsedCodexToolCall,
 } from "./KnownAnswerEvaluation.js";
-import { canonicalJson } from "../domain/comparisonSemantics.js";
+import { canonicalJson } from "../../../src/domain/comparisonSemantics.js";
-import { TOOL_CONTRACTS } from "../contracts/toolContracts.js";
+import { TOOL_CONTRACTS } from "../../../src/contracts/toolContracts.js";
 
 /** One REA MCP invocation observed in a Codex JSONL transcript. */
 interface CodexMcpCall {

tests/evaluation/model-evals/KnownAnswerEvaluation.ts

@@ -6,9 +6,12 @@ import {
   evidenceAuthoritySchema,
   evidenceSchema,
   type Evidence,
-} from "../domain/evidence.js";
+} from "../../../src/domain/evidence.js";
-import { jsonValueSchema, type JsonValue } from "../domain/jsonValue.js";
+import {
-import { canonicalJson } from "../domain/comparisonSemantics.js";
+  jsonValueSchema,
+  type JsonValue,
+} from "../../../src/domain/jsonValue.js";
+import { canonicalJson } from "../../../src/domain/comparisonSemantics.js";
 
 /** Only authenticated, successful, non-conflicting transcript deliveries. */
 export interface FixtureClaimContext {

tests/evaluation/model-evals/agentEvaluationReport.test.ts

@@ -6,7 +6,7 @@ import { describe, expect, it } from "vitest";
 import {
   agentEvaluationPassed,
   summarizeFactualCorrectness,
-} from "../../../src/evaluation/AgentEvaluationReport.js";
+} from "./AgentEvaluationReport.js";
 
 const routingSuccess = {
   exitCode: 0,

tests/evaluation/model-evals/agentEvaluationScenarios.test.ts

@@ -1,11 +1,11 @@
 import { describe, expect, it } from "vitest";
 
-import { evaluateCodexEvents } from "../../../src/evaluation/CodexAgentEval.js";
+import { evaluateCodexEvents } from "./CodexAgentEval.js";
-import { agentEvaluationPassed } from "../../../src/evaluation/AgentEvaluationReport.js";
+import { agentEvaluationPassed } from "./AgentEvaluationReport.js";
 import {
   MISSING_TARGET_SCENARIO,
   nativeProviderScenario,
-} from "../../../src/evaluation/AgentEvaluationScenarios.js";
+} from "./AgentEvaluationScenarios.js";
 
 const call = (id: string, tool: string, errorCode?: string) => ({
   type: "item.completed",

tests/evaluation/model-evals/codexAgentEval.test.ts

@@ -2,7 +2,7 @@ import { describe, expect, it } from "vitest";
 
 import { createEvidence } from "../../../src/domain/evidence.js";
 import type { JsonValue } from "../../../src/domain/jsonValue.js";
-import { evaluateCodexEvents } from "../../../src/evaluation/CodexAgentEval.js";
+import { evaluateCodexEvents } from "./CodexAgentEval.js";
 
 const observation = (
   operation = "inspect_managed_artifact",

tests/evaluation/model-evals/knownAnswerEvaluation.test.ts

@@ -4,18 +4,18 @@ import { describe, expect, it } from "vitest";
 import { toolContract } from "../../../src/contracts/toolContracts.js";
 import { createEvidence, type Evidence } from "../../../src/domain/evidence.js";
 import type { JsonValue } from "../../../src/domain/jsonValue.js";
-import { evaluateCodexEvents } from "../../../src/evaluation/CodexAgentEval.js";
+import { evaluateCodexEvents } from "./CodexAgentEval.js";
 import {
   agentFixtureAnswerInstructions,
   agentFixtureClaims,
-} from "../../../src/evaluation/AgentEvaluationFixtures.js";
+} from "./AgentEvaluationFixtures.js";
 import { inspectManagedArtifactBytes } from "../../../src/dotnet/ManagedArtifactInspector.js";
 import { inspectManagedMembersBytes } from "../../../src/dotnet/ManagedMemberInspector.js";
 import {
   buildManagedPeFixture,
   managedPeFixtureTarget,
 } from "../../../src/dotnet/ManagedPe.fixture.js";
-import type { FixtureClaimExpectation } from "../../../src/evaluation/KnownAnswerEvaluation.js";
+import type { FixtureClaimExpectation } from "./KnownAnswerEvaluation.js";
 import { ToolResultDelivery } from "../../../src/server/toolResult.js";
 
 const delivery = new ToolResultDelivery(STDIO_DEFAULT_MAX_BUFFER_SIZE);