From 3ad465a6669c482606ede5edbf104af04ee0ed0d Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 24 Oct 2026 11:49:47 +0800 Subject: [PATCH] refactor(evaluation): move agent grading into development ownership Keep the maintained manual evaluator, fixture rubrics and semantic tests together under tests/evaluation/model-evals instead of shipping them in dist. Load the same typed owners from the manual runner and update the architecture and client guide. Grading/report contracts and executable logic are unchanged. --- docs/architecture.mermaid | 4 +- docs/client-verification.md | 7 ++- knip.json | 1 - scripts/verify-agent-experience.mjs | 46 +++++++++++++------ .../boundary/mcp/agentKnownAnswerMcp.test.ts | 6 +-- .../model-evals}/AgentEvaluationFixtures.ts | 18 ++++---- .../model-evals}/AgentEvaluationReport.ts | 0 .../model-evals}/AgentEvaluationScenarios.ts | 0 .../evaluation/model-evals}/CodexAgentEval.ts | 4 +- .../model-evals}/KnownAnswerEvaluation.ts | 9 ++-- .../model-evals/agentEvaluationReport.test.ts | 2 +- .../agentEvaluationScenarios.test.ts | 6 +-- .../model-evals/codexAgentEval.test.ts | 2 +- .../model-evals/knownAnswerEvaluation.test.ts | 6 +-- 14 files changed, 68 insertions(+), 43 deletions(-) rename {src/evaluation => tests/evaluation/model-evals}/AgentEvaluationFixtures.ts (96%) rename {src/evaluation => tests/evaluation/model-evals}/AgentEvaluationReport.ts (100%) rename {src/evaluation => tests/evaluation/model-evals}/AgentEvaluationScenarios.ts (100%) rename {src/evaluation => tests/evaluation/model-evals}/CodexAgentEval.ts (98%) rename {src/evaluation => tests/evaluation/model-evals}/KnownAnswerEvaluation.ts (98%) diff --git a/docs/architecture.mermaid b/docs/architecture.mermaid index 6ab14e680c..d38bd957f2 100644 --- a/docs/architecture.mermaid +++ b/docs/architecture.mermaid @@ -86,8 +86,8 @@ graph TD TESTS["tests/
Vitest suite
fixtures/ holds fake launcher, bridge, and CDP seams"] end - subgraph "Evaluation" - EVALUATION["src/evaluation/
Agent release evaluation
session transcript scoring"] + subgraph "Development Evaluation" + EVALUATION["tests/evaluation/model-evals/
Agent release evaluation
session transcript scoring"] end DISPATCH --> CLI diff --git a/docs/client-verification.md b/docs/client-verification.md index e1496e5e92..6561752e74 100644 --- a/docs/client-verification.md +++ b/docs/client-verification.md @@ -458,6 +458,11 @@ real local Codex CLI with: npm run verify:agent ``` +The verifier's scenarios, transcript scoring, fixture rubrics, and report gates +are development verification code under `tests/evaluation/model-evals/`. The +source-owned evaluator runs alongside the compiled REA tools; it is excluded +from the shipped runtime package. + The lane runs scenarios sequentially in standalone Codex (`--no-daemon`) with a disposable HOME and CODEX_HOME under `~/.cache/rea-agent-evaluations/`. Set `REA_AGENT_EVAL_ROOT` to choose another parent directory outside the OS temporary @@ -487,7 +492,7 @@ boundary instead of stopping at Codex approval. Its scenario contract accepts one `provider_unavailable` open refusal without a close, or one successful open followed by a successful close. Other errors, an unclosed session, and a close without acquisition fail. These outcomes and the prompt are owned together in -`src/evaluation/AgentEvaluationScenarios.ts`; generic successful-tool sequences +`tests/evaluation/model-evals/AgentEvaluationScenarios.ts`; generic successful-tool sequences still apply to scenarios that require them. The provider scenario checks routing, reported limitations, and cleanup; free-form answer correctness remains unassessed. It does not validate a real native engine or authorize runtime execution. diff --git a/knip.json b/knip.json index 2d5837f975..4d24b67dd5 100644 --- a/knip.json +++ b/knip.json @@ -10,7 +10,6 @@ "src/composition/javascriptAnalysisWorkerProcess.ts", "src/composition/webBundleAnalysisWorkerProcess.ts", "src/contracts/errorSchemas.ts", - "src/evaluation/AgentEvaluationFixtures.ts", "scripts/verify/fridaBindingsCompatibility.ts", "scripts/verify/termux/browser.mjs", "scripts/verify-real-ghidra-noreturn.mjs", diff --git a/scripts/verify-agent-experience.mjs b/scripts/verify-agent-experience.mjs index fbb5b47ccb..ffac2db067 100644 --- a/scripts/verify-agent-experience.mjs +++ b/scripts/verify-agent-experience.mjs @@ -11,26 +11,44 @@ import { import { homedir } from "node:os"; import { delimiter, dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { tsImport } from "tsx/esm/api"; -import { evaluateCodexEvents } from "../dist/evaluation/CodexAgentEval.js"; -import { - MISSING_TARGET_SCENARIO, - nativeProviderScenario, -} from "../dist/evaluation/AgentEvaluationScenarios.js"; -import { - createAgentEvaluationFixtures, - agentFixtureClaims, - agentFixtureAnswerInstructions, -} from "../dist/evaluation/AgentEvaluationFixtures.js"; -import { - agentEvaluationPassed, - summarizeFactualCorrectness, -} from "../dist/evaluation/AgentEvaluationReport.js"; import { PRODUCT_IDENTITY } from "../dist/identity.js"; import { createAgentEvaluationCommands } from "./lib/agent-evaluation-commands.mjs"; import { closeVerifierResources } from "./lib/verifier-cleanup.mjs"; import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs"; +const { evaluateCodexEvents } = await tsImport( + new URL("../tests/evaluation/model-evals/CodexAgentEval.ts", import.meta.url) + .href, + import.meta.url, +); +const { MISSING_TARGET_SCENARIO, nativeProviderScenario } = await tsImport( + new URL( + "../tests/evaluation/model-evals/AgentEvaluationScenarios.ts", + import.meta.url, + ).href, + import.meta.url, +); +const { + createAgentEvaluationFixtures, + agentFixtureClaims, + agentFixtureAnswerInstructions, +} = await tsImport( + new URL( + "../tests/evaluation/model-evals/AgentEvaluationFixtures.ts", + import.meta.url, + ).href, + import.meta.url, +); +const { agentEvaluationPassed, summarizeFactualCorrectness } = await tsImport( + new URL( + "../tests/evaluation/model-evals/AgentEvaluationReport.ts", + import.meta.url, + ).href, + import.meta.url, +); + const repositoryRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); const verifierRun = createVerifierRun(); const timeoutMs = Number(process.env.REA_AGENT_EVAL_TIMEOUT_MS ?? 480_000); diff --git a/tests/boundary/mcp/agentKnownAnswerMcp.test.ts b/tests/boundary/mcp/agentKnownAnswerMcp.test.ts index 11e8d365a3..a2f5f67971 100644 --- a/tests/boundary/mcp/agentKnownAnswerMcp.test.ts +++ b/tests/boundary/mcp/agentKnownAnswerMcp.test.ts @@ -5,12 +5,12 @@ import { describe, expect, it } from "vitest"; import { createPackage } from "@electron/asar"; import { parseEvidence, type Evidence } from "../../../src/domain/evidence.js"; -import { evaluateCodexEvents } from "../../../src/evaluation/CodexAgentEval.js"; +import { evaluateCodexEvents } from "../../evaluation/model-evals/CodexAgentEval.js"; import { createAgentEvaluationFixtures, agentFixtureClaims, -} from "../../../src/evaluation/AgentEvaluationFixtures.js"; -import type { FixtureClaimExpectation } from "../../../src/evaluation/KnownAnswerEvaluation.js"; +} from "../../evaluation/model-evals/AgentEvaluationFixtures.js"; +import type { FixtureClaimExpectation } from "../../evaluation/model-evals/KnownAnswerEvaluation.js"; import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; diff --git a/src/evaluation/AgentEvaluationFixtures.ts b/tests/evaluation/model-evals/AgentEvaluationFixtures.ts similarity index 96% rename from src/evaluation/AgentEvaluationFixtures.ts rename to tests/evaluation/model-evals/AgentEvaluationFixtures.ts index 2a1e65bfe2..3fb8d88c04 100644 --- a/src/evaluation/AgentEvaluationFixtures.ts +++ b/tests/evaluation/model-evals/AgentEvaluationFixtures.ts @@ -4,24 +4,24 @@ import { join } from "node:path"; import { createPackage } from "@electron/asar"; -import type { JsonValue } from "../domain/jsonValue.js"; -import { javascriptApplicationAnalysisResultSchema } from "../domain/javascript/javascriptApplicationAnalysis.js"; -import type { ApplicationNode } from "../domain/javascript/javascriptApplicationGraphSchemas.js"; -import { javaScriptExportShapeComparisonResultSchema } from "../domain/javascript/javascriptExportShapeComparisonSchemas.js"; +import type { JsonValue } from "../../../src/domain/jsonValue.js"; +import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; +import type { ApplicationNode } from "../../../src/domain/javascript/javascriptApplicationGraphSchemas.js"; +import { javaScriptExportShapeComparisonResultSchema } from "../../../src/domain/javascript/javascriptExportShapeComparisonSchemas.js"; import { applicationFeatureTraceResultSchema, featureSeedMatchMode, -} from "../domain/javascript/javascriptFeatureTraceSchemas.js"; -import { javaScriptSemanticTraceResultSchema } from "../domain/javascript/javascriptSemanticTraceSchemas.js"; +} from "../../../src/domain/javascript/javascriptFeatureTraceSchemas.js"; +import { javaScriptSemanticTraceResultSchema } from "../../../src/domain/javascript/javascriptSemanticTraceSchemas.js"; import { managedArtifactInspectionSchema, managedMemberInspectionSchema, -} from "../domain/managed/managedArtifact.js"; +} from "../../../src/domain/managed/managedArtifact.js"; import { managedTokenScopesMatch, readManagedInspectionEvidence, -} from "../domain/managed/managedInspectionEvidence.js"; -import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; +} from "../../../src/domain/managed/managedInspectionEvidence.js"; +import { compareUnicodeCodePoints } from "../../../src/domain/unicodeCodePointOrder.js"; import type { FixtureClaimContext, FixtureClaimExpectation, diff --git a/src/evaluation/AgentEvaluationReport.ts b/tests/evaluation/model-evals/AgentEvaluationReport.ts similarity index 100% rename from src/evaluation/AgentEvaluationReport.ts rename to tests/evaluation/model-evals/AgentEvaluationReport.ts diff --git a/src/evaluation/AgentEvaluationScenarios.ts b/tests/evaluation/model-evals/AgentEvaluationScenarios.ts similarity index 100% rename from src/evaluation/AgentEvaluationScenarios.ts rename to tests/evaluation/model-evals/AgentEvaluationScenarios.ts diff --git a/src/evaluation/CodexAgentEval.ts b/tests/evaluation/model-evals/CodexAgentEval.ts similarity index 98% rename from src/evaluation/CodexAgentEval.ts rename to tests/evaluation/model-evals/CodexAgentEval.ts index 8dfc14df22..64cd50b2a3 100644 --- a/src/evaluation/CodexAgentEval.ts +++ b/tests/evaluation/model-evals/CodexAgentEval.ts @@ -5,8 +5,8 @@ import { type KnownAnswerAssessment, type ParsedCodexToolCall, } from "./KnownAnswerEvaluation.js"; -import { canonicalJson } from "../domain/comparisonSemantics.js"; -import { TOOL_CONTRACTS } from "../contracts/toolContracts.js"; +import { canonicalJson } from "../../../src/domain/comparisonSemantics.js"; +import { TOOL_CONTRACTS } from "../../../src/contracts/toolContracts.js"; /** One REA MCP invocation observed in a Codex JSONL transcript. */ interface CodexMcpCall { diff --git a/src/evaluation/KnownAnswerEvaluation.ts b/tests/evaluation/model-evals/KnownAnswerEvaluation.ts similarity index 98% rename from src/evaluation/KnownAnswerEvaluation.ts rename to tests/evaluation/model-evals/KnownAnswerEvaluation.ts index 6b51d4284f..38d86e2c6f 100644 --- a/src/evaluation/KnownAnswerEvaluation.ts +++ b/tests/evaluation/model-evals/KnownAnswerEvaluation.ts @@ -6,9 +6,12 @@ import { evidenceAuthoritySchema, evidenceSchema, type Evidence, -} from "../domain/evidence.js"; -import { jsonValueSchema, type JsonValue } from "../domain/jsonValue.js"; -import { canonicalJson } from "../domain/comparisonSemantics.js"; +} from "../../../src/domain/evidence.js"; +import { + jsonValueSchema, + type JsonValue, +} from "../../../src/domain/jsonValue.js"; +import { canonicalJson } from "../../../src/domain/comparisonSemantics.js"; /** Only authenticated, successful, non-conflicting transcript deliveries. */ export interface FixtureClaimContext { diff --git a/tests/evaluation/model-evals/agentEvaluationReport.test.ts b/tests/evaluation/model-evals/agentEvaluationReport.test.ts index 562fd591ea..db403720ad 100644 --- a/tests/evaluation/model-evals/agentEvaluationReport.test.ts +++ b/tests/evaluation/model-evals/agentEvaluationReport.test.ts @@ -6,7 +6,7 @@ import { describe, expect, it } from "vitest"; import { agentEvaluationPassed, summarizeFactualCorrectness, -} from "../../../src/evaluation/AgentEvaluationReport.js"; +} from "./AgentEvaluationReport.js"; const routingSuccess = { exitCode: 0, diff --git a/tests/evaluation/model-evals/agentEvaluationScenarios.test.ts b/tests/evaluation/model-evals/agentEvaluationScenarios.test.ts index eab15fdf6c..6c3af8392c 100644 --- a/tests/evaluation/model-evals/agentEvaluationScenarios.test.ts +++ b/tests/evaluation/model-evals/agentEvaluationScenarios.test.ts @@ -1,11 +1,11 @@ import { describe, expect, it } from "vitest"; -import { evaluateCodexEvents } from "../../../src/evaluation/CodexAgentEval.js"; -import { agentEvaluationPassed } from "../../../src/evaluation/AgentEvaluationReport.js"; +import { evaluateCodexEvents } from "./CodexAgentEval.js"; +import { agentEvaluationPassed } from "./AgentEvaluationReport.js"; import { MISSING_TARGET_SCENARIO, nativeProviderScenario, -} from "../../../src/evaluation/AgentEvaluationScenarios.js"; +} from "./AgentEvaluationScenarios.js"; const call = (id: string, tool: string, errorCode?: string) => ({ type: "item.completed", diff --git a/tests/evaluation/model-evals/codexAgentEval.test.ts b/tests/evaluation/model-evals/codexAgentEval.test.ts index b72bf2ff42..e4fe0d6051 100644 --- a/tests/evaluation/model-evals/codexAgentEval.test.ts +++ b/tests/evaluation/model-evals/codexAgentEval.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from "vitest"; import { createEvidence } from "../../../src/domain/evidence.js"; import type { JsonValue } from "../../../src/domain/jsonValue.js"; -import { evaluateCodexEvents } from "../../../src/evaluation/CodexAgentEval.js"; +import { evaluateCodexEvents } from "./CodexAgentEval.js"; const observation = ( operation = "inspect_managed_artifact", diff --git a/tests/evaluation/model-evals/knownAnswerEvaluation.test.ts b/tests/evaluation/model-evals/knownAnswerEvaluation.test.ts index a9a4cc32b2..67b6945aeb 100644 --- a/tests/evaluation/model-evals/knownAnswerEvaluation.test.ts +++ b/tests/evaluation/model-evals/knownAnswerEvaluation.test.ts @@ -4,18 +4,18 @@ import { describe, expect, it } from "vitest"; import { toolContract } from "../../../src/contracts/toolContracts.js"; import { createEvidence, type Evidence } from "../../../src/domain/evidence.js"; import type { JsonValue } from "../../../src/domain/jsonValue.js"; -import { evaluateCodexEvents } from "../../../src/evaluation/CodexAgentEval.js"; +import { evaluateCodexEvents } from "./CodexAgentEval.js"; import { agentFixtureAnswerInstructions, agentFixtureClaims, -} from "../../../src/evaluation/AgentEvaluationFixtures.js"; +} from "./AgentEvaluationFixtures.js"; import { inspectManagedArtifactBytes } from "../../../src/dotnet/ManagedArtifactInspector.js"; import { inspectManagedMembersBytes } from "../../../src/dotnet/ManagedMemberInspector.js"; import { buildManagedPeFixture, managedPeFixtureTarget, } from "../../../src/dotnet/ManagedPe.fixture.js"; -import type { FixtureClaimExpectation } from "../../../src/evaluation/KnownAnswerEvaluation.js"; +import type { FixtureClaimExpectation } from "./KnownAnswerEvaluation.js"; import { ToolResultDelivery } from "../../../src/server/toolResult.js"; const delivery = new ToolResultDelivery(STDIO_DEFAULT_MAX_BUFFER_SIZE);