From c2c8ad3d701e4ca7384cb6bb541e3b36c0d34a76 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 20 Nov 2026 11:29:08 -0500 Subject: [PATCH] test(evaluation): consolidate transcript and claim grounding checks (#12883) * test(evaluation): isolate transcript routing and citation failures * test(evaluation): require precise claim grounding failures --- .../model-evals/codexAgentEval.test.ts | 166 ++++++------------ .../model-evals/knownAnswerEvaluation.test.ts | 89 ++++------ 2 files changed, 85 insertions(+), 170 deletions(-) diff --git a/tests/evaluation/model-evals/codexAgentEval.test.ts b/tests/evaluation/model-evals/codexAgentEval.test.ts index 5696602c33..bf6988065a 100644 --- a/tests/evaluation/model-evals/codexAgentEval.test.ts +++ b/tests/evaluation/model-evals/codexAgentEval.test.ts @@ -97,41 +97,6 @@ describe("Codex agent release evaluation", () => { }); expect(metrics.reaCalls).toHaveLength(1); }); - - it("requires the final answer to cite produced Evidence when requested", () => { - const evidence = observation(); - const evidenceId = evidence.evidence_id; - const events = [ - { - type: "item.completed", - item: { - id: "item-tool", - type: "mcp_tool_call", - server: "rea", - tool: "inspect_managed_artifact", - arguments: { path: "/tmp/app.dll" }, - result: { structuredContent: evidence }, - }, - }, - { - type: "item.completed", - item: { - type: "agent_message", - text: `The observed managed metadata is supported by evidence ${evidenceId}; runtime behavior remains unknown without runtime authority.`, - }, - }, - ]; - - expect( - evaluateCodexEvents(events, "inspect_managed_artifact", { - requireEvidence: true, - }), - ).toMatchObject({ - evidenceIds: [evidenceId], - finalCitesEvidence: true, - answerHeuristicsMet: true, - }); - }); }); describe("Codex completed tool call parsing", () => { @@ -380,24 +345,6 @@ describe("Codex Evidence delivery", () => { }); describe("Codex request accounting", () => { - it("flags identical calls with distinct item identities", () => { - const event = (id: string) => ({ - type: "item.completed", - item: { - id, - type: "mcp_tool_call", - server: "rea", - tool: "binary_session", - arguments: {}, - }, - }); - - expect( - evaluateCodexEvents([event("item-1"), event("item-2")], "binary_session") - .repeatedCallCount, - ).toBe(1); - }); - it("distinguishes session verification after a successful close from an unchanged retry", () => { const call = (id: string, tool: string, status = "completed") => ({ type: "item.completed", @@ -649,39 +596,25 @@ describe("Codex agent completion evaluation", () => { }); }); - it("requires scenario-specific answer terms when configured", () => { - const event = (text: string) => [ - { - type: "item.completed", - item: { - type: "mcp_tool_call", - server: "rea", - tool: "analyze_javascript_application", - arguments: {}, + it("requires a term from every configured answer group", () => { + expect( + evaluateCodexEvents( + [ + { + type: "item.completed", + item: { + type: "agent_message", + text: "Observed evidence identifies profileApi, but runtime behavior remains unknown without more authority and detail.", + }, + }, + ], + null, + { + requiredAnswerTermGroups: [ + ["profileApi"], + ["preload", "contextBridge"], + ], }, - }, - { type: "item.completed", item: { type: "agent_message", text } }, - ]; - const options = { - requiredAnswerTermGroups: [["profileApi"], ["preload", "contextBridge"]], - }; - - expect( - evaluateCodexEvents( - event( - "Observed evidence connects profileApi through the preload bridge; runtime behavior remains unknown without authority.", - ), - "analyze_javascript_application", - options, - ), - ).toMatchObject({ answerTermCoverageMet: true, answerHeuristicsMet: true }); - expect( - evaluateCodexEvents( - event( - "Observed artifact evidence is available, but application behavior remains unknown without more authority and detail.", - ), - "analyze_javascript_application", - options, ), ).toMatchObject({ answerTermCoverageMet: false, @@ -755,39 +688,48 @@ describe("Codex agent workflow evaluation", () => { ], forbidInputValidationFailures: true, }; - const valid = evaluateCodexEvents( - [ - call("left", "analyze_javascript_application"), - call("right", "analyze_javascript_application"), - call("compare", "compare_javascript_export_shapes"), - final, - ], - "analyze_javascript_application", - options, - ); - expect(valid).toMatchObject({ + const left = call("left", "analyze_javascript_application"); + const right = call("right", "analyze_javascript_application"); + const compare = call("compare", "compare_javascript_export_shapes"); + const rejected = call("bad", "compare_javascript_export_shapes", { + structured_content: { error: { code: "invalid_request" } }, + isError: true, + }); + const evaluate = (events: unknown[]) => + evaluateCodexEvents(events, "analyze_javascript_application", options); + + expect(evaluate([left, right, compare, final])).toMatchObject({ requiredToolSubsequenceMet: true, inputValidationFailureCount: 0, + finalCitesEvidence: true, answerHeuristicsMet: true, }); - - const invalid = evaluateCodexEvents( - [ - call("left", "analyze_javascript_application"), - call("bad", "compare_javascript_export_shapes", { - structured_content: { - error: { code: "invalid_request" }, - }, - isError: true, - }), - final, - ], - "analyze_javascript_application", - options, - ); - expect(invalid).toMatchObject({ + expect(evaluate([left, compare, right, final])).toMatchObject({ + requiredToolSubsequenceMet: false, + inputValidationFailureCount: 0, + answerHeuristicsMet: false, + }); + expect(evaluate([left, right, rejected, final])).toMatchObject({ requiredToolSubsequenceMet: false, inputValidationFailureCount: 1, + }); + expect(evaluate([left, right, compare, rejected, final])).toMatchObject({ + requiredToolSubsequenceMet: true, + inputValidationFailureCount: 1, + finalCitesEvidence: true, + answerHeuristicsMet: false, + }); + const uncited = { + ...final, + item: { + ...final.item, + text: final.item.text.replace(evidenceId, "cited elsewhere"), + }, + }; + expect(evaluate([left, right, compare, uncited])).toMatchObject({ + requiredToolSubsequenceMet: true, + inputValidationFailureCount: 0, + finalCitesEvidence: false, answerHeuristicsMet: false, }); }); diff --git a/tests/evaluation/model-evals/knownAnswerEvaluation.test.ts b/tests/evaluation/model-evals/knownAnswerEvaluation.test.ts index de4c068ba1..d4d890d4f0 100644 --- a/tests/evaluation/model-evals/knownAnswerEvaluation.test.ts +++ b/tests/evaluation/model-evals/knownAnswerEvaluation.test.ts @@ -111,6 +111,13 @@ const assess = ( }, ); const answer = (claims: readonly unknown[]) => JSON.stringify({ claims }); +const claimFailures = ( + events: readonly unknown[], + evidence: Evidence, + rubric?: readonly FixtureClaimExpectation[], +) => + assess(events, answer([claimFor(evidence)]), rubric).factualAssessment + .claims[0]?.failures; // Synthetic transcript cases exercise grading, not an SDK-built assembly oracle. const managedTranscript = () => { @@ -445,11 +452,10 @@ describe("managed member grounding", () => { // Removing exact fixture comparison or accepted-source validation must break these regressions. describe("known-answer fixture grading", () => { - it("passes closed claims against authenticated production MCP Evidence", () => { + it("passes closed claims against authenticated MCP Evidence with equivalent result aliases", () => { const evidence = createObservation(); - expect( - assess([delivered(evidence)], answer([claimFor(evidence)])), - ).toMatchObject({ + const event = delivered(evidence, { output: resultFor(evidence) }); + expect(assess([event], answer([claimFor(evidence)]))).toMatchObject({ factualCorrectness: "passed", factualAssessment: { scope: "configured_fixture_claims", @@ -479,15 +485,6 @@ describe("known-answer fixture grading", () => { ).toBe("failed"); }); - it("retains not_assessed without a fixture rubric", () => { - expect( - evaluateCodexEvents( - [final("Observed metadata remains unknown.")], - "analyze_javascript_application", - ).factualCorrectness, - ).toBe("not_assessed"); - }); - it.each([{ rubric: [] }, { rubric: [expected(), expected()] }])( "fails closed for an empty or duplicate rubric", ({ rubric }) => { @@ -513,10 +510,9 @@ describe("known-answer fixture grading", () => { }, }, ]; - expect( - assess([delivered(evidence)], answer([claimFor(evidence)]), rubric) - .factualCorrectness, - ).toBe("failed"); + expect(claimFailures([delivered(evidence)], evidence, rubric)).toEqual([ + "selector_failed", + ]); }); }); @@ -596,10 +592,9 @@ describe("successful Evidence grounding", () => { { tool: "other_tool" }, ])("rejects a foreign or unsuccessful completion %j", (overrides) => { const evidence = createObservation(); - expect( - assess([delivered(evidence, overrides)], answer([claimFor(evidence)])) - .factualCorrectness, - ).toBe("failed"); + expect(claimFailures([delivered(evidence, overrides)], evidence)).toEqual([ + "unknown_evidence", + ]); }); it("rejects IDs present only in arguments or unrelated metadata", () => { @@ -610,9 +605,7 @@ describe("successful Evidence grounding", () => { structuredContent: { metadata: { evidence_id: evidence.evidence_id } }, }, }); - expect( - assess([event], answer([claimFor(evidence)])).factualCorrectness, - ).toBe("failed"); + expect(claimFailures([event], evidence)).toEqual(["unknown_evidence"]); }); it("rejects a tampered envelope even with a valid-looking Evidence ID", () => { @@ -625,9 +618,7 @@ describe("successful Evidence grounding", () => { }, }, }); - expect( - assess([event], answer([claimFor(evidence)])).factualCorrectness, - ).toBe("failed"); + expect(claimFailures([event], evidence)).toEqual(["unknown_evidence"]); }); it.each(["id", "result", "error"])( @@ -652,9 +643,7 @@ describe("successful Evidence grounding", () => { ...(field === "error" ? { isError: true } : {}), }, }); - expect( - assess([event], answer([claimFor(evidence)])).factualCorrectness, - ).toBe("failed"); + expect(claimFailures([event], evidence)).toEqual(["unknown_evidence"]); }, ); @@ -675,20 +664,10 @@ describe("successful Evidence grounding", () => { }, }; const event = delivered(evidence, { output }); - expect( - assess([event], answer([claimFor(evidence)])).factualCorrectness, - ).toBe("failed"); + expect(claimFailures([event], evidence)).toEqual(["unknown_evidence"]); }, ); - it("accepts JSON-equal separately allocated outer result/output aliases", () => { - const evidence = createObservation(); - const event = delivered(evidence, { output: resultFor(evidence) }); - expect( - assess([event], answer([claimFor(evidence)])).factualCorrectness, - ).toBe("passed"); - }); - it("accepts the documented snake-case structured-content transcript representation", () => { const evidence = createObservation(); const event = delivered(evidence, { @@ -714,20 +693,18 @@ describe("successful Evidence grounding", () => { it("rejects authentic unrelated Evidence containing the same value elsewhere", () => { const evidence = createObservation({ unrelated: expected().expectedValue }); - expect( - assess([delivered(evidence)], answer([claimFor(evidence)])) - .factualCorrectness, - ).toBe("failed"); + expect(claimFailures([delivered(evidence)], evidence)).toEqual([ + "source_assertion_mismatch", + ]); }); }); describe("fixture identity and provenance binding", () => { it("rejects Evidence for a different subject despite an equal supported value", () => { const evidence = createObservation(undefined, "/targets/other.asar"); - expect( - assess([delivered(evidence)], answer([claimFor(evidence)])) - .factualCorrectness, - ).toBe("failed"); + expect(claimFailures([delivered(evidence)], evidence)).toEqual([ + "source_subject_mismatch", + ]); }); it("rejects a wrong required source parameter", () => { @@ -739,10 +716,9 @@ describe("fixture identity and provenance binding", () => { source: { ...expectation.source, parameters: { format: "directory" } }, }, ]; - expect( - assess([delivered(evidence)], answer([claimFor(evidence)]), rubric) - .factualCorrectness, - ).toBe("failed"); + expect(claimFailures([delivered(evidence)], evidence, rubric)).toEqual([ + "source_parameters_mismatch", + ]); }); it("rejects conflicting subject diagnostics for the same semantic Evidence ID", () => { @@ -754,11 +730,8 @@ describe("fixture identity and provenance binding", () => { }), }; expect( - assess( - [delivered(evidence), delivered(conflicting)], - answer([claimFor(evidence)]), - ).factualCorrectness, - ).toBe("failed"); + claimFailures([delivered(evidence), delivered(conflicting)], evidence), + ).toEqual(["conflicting_evidence"]); }); it("requires linked source Evidence to be delivered and match the configured artifact", () => {