test(evaluation): consolidate transcript and claim grounding checks (#12883)

* test(evaluation): isolate transcript routing and citation failures

* test(evaluation): require precise claim grounding failures
This commit is contained in:
morluto authored and GitHub committed 2026-11-20 11:29:08 -05:00
1 parent 4449d13047
commit c2c8ad3d70
2 files changed
+85 -170

No files matched your search

@@ -97,41 +97,6 @@ describe("Codex agent release evaluation", () => {
});
expect(metrics.reaCalls).toHaveLength(1);
});
it("requires the final answer to cite produced Evidence when requested", () => {
const evidence = observation();
const evidenceId = evidence.evidence_id;
const events = [
{
type: "item.completed",
item: {
id: "item-tool",
type: "mcp_tool_call",
server: "rea",
tool: "inspect_managed_artifact",
arguments: { path: "/tmp/app.dll" },
result: { structuredContent: evidence },
},
},
{
type: "item.completed",
item: {
type: "agent_message",
text: `The observed managed metadata is supported by evidence ${evidenceId}; runtime behavior remains unknown without runtime authority.`,
},
},
];
expect(
evaluateCodexEvents(events, "inspect_managed_artifact", {
requireEvidence: true,
}),
).toMatchObject({
evidenceIds: [evidenceId],
finalCitesEvidence: true,
answerHeuristicsMet: true,
});
});
});
describe("Codex completed tool call parsing", () => {
@@ -380,24 +345,6 @@ describe("Codex Evidence delivery", () => {
});
describe("Codex request accounting", () => {
it("flags identical calls with distinct item identities", () => {
const event = (id: string) => ({
type: "item.completed",
item: {
id,
type: "mcp_tool_call",
server: "rea",
tool: "binary_session",
arguments: {},
},
});
expect(
evaluateCodexEvents([event("item-1"), event("item-2")], "binary_session")
.repeatedCallCount,
).toBe(1);
});
it("distinguishes session verification after a successful close from an unchanged retry", () => {
const call = (id: string, tool: string, status = "completed") => ({
type: "item.completed",
@@ -649,39 +596,25 @@ describe("Codex agent completion evaluation", () => {
});
});
it("requires scenario-specific answer terms when configured", () => {
const event = (text: string) => [
{
type: "item.completed",
item: {
type: "mcp_tool_call",
server: "rea",
tool: "analyze_javascript_application",
arguments: {},
it("requires a term from every configured answer group", () => {
expect(
evaluateCodexEvents(
[
{
type: "item.completed",
item: {
type: "agent_message",
text: "Observed evidence identifies profileApi, but runtime behavior remains unknown without more authority and detail.",
},
},
],
null,
{
requiredAnswerTermGroups: [
["profileApi"],
["preload", "contextBridge"],
],
},
},
{ type: "item.completed", item: { type: "agent_message", text } },
];
const options = {
requiredAnswerTermGroups: [["profileApi"], ["preload", "contextBridge"]],
};
expect(
evaluateCodexEvents(
event(
"Observed evidence connects profileApi through the preload bridge; runtime behavior remains unknown without authority.",
),
"analyze_javascript_application",
options,
),
).toMatchObject({ answerTermCoverageMet: true, answerHeuristicsMet: true });
expect(
evaluateCodexEvents(
event(
"Observed artifact evidence is available, but application behavior remains unknown without more authority and detail.",
),
"analyze_javascript_application",
options,
),
).toMatchObject({
answerTermCoverageMet: false,
@@ -755,39 +688,48 @@ describe("Codex agent workflow evaluation", () => {
],
forbidInputValidationFailures: true,
};
const valid = evaluateCodexEvents(
[
call("left", "analyze_javascript_application"),
call("right", "analyze_javascript_application"),
call("compare", "compare_javascript_export_shapes"),
final,
],
"analyze_javascript_application",
options,
);
expect(valid).toMatchObject({
const left = call("left", "analyze_javascript_application");
const right = call("right", "analyze_javascript_application");
const compare = call("compare", "compare_javascript_export_shapes");
const rejected = call("bad", "compare_javascript_export_shapes", {
structured_content: { error: { code: "invalid_request" } },
isError: true,
});
const evaluate = (events: unknown[]) =>
evaluateCodexEvents(events, "analyze_javascript_application", options);
expect(evaluate([left, right, compare, final])).toMatchObject({
requiredToolSubsequenceMet: true,
inputValidationFailureCount: 0,
finalCitesEvidence: true,
answerHeuristicsMet: true,
});
const invalid = evaluateCodexEvents(
[
call("left", "analyze_javascript_application"),
call("bad", "compare_javascript_export_shapes", {
structured_content: {
error: { code: "invalid_request" },
},
isError: true,
}),
final,
],
"analyze_javascript_application",
options,
);
expect(invalid).toMatchObject({
expect(evaluate([left, compare, right, final])).toMatchObject({
requiredToolSubsequenceMet: false,
inputValidationFailureCount: 0,
answerHeuristicsMet: false,
});
expect(evaluate([left, right, rejected, final])).toMatchObject({
requiredToolSubsequenceMet: false,
inputValidationFailureCount: 1,
});
expect(evaluate([left, right, compare, rejected, final])).toMatchObject({
requiredToolSubsequenceMet: true,
inputValidationFailureCount: 1,
finalCitesEvidence: true,
answerHeuristicsMet: false,
});
const uncited = {
...final,
item: {
...final.item,
text: final.item.text.replace(evidenceId, "cited elsewhere"),
},
};
expect(evaluate([left, right, compare, uncited])).toMatchObject({
requiredToolSubsequenceMet: true,
inputValidationFailureCount: 0,
finalCitesEvidence: false,
answerHeuristicsMet: false,
});
});
@@ -111,6 +111,13 @@ const assess = (
},
);
const answer = (claims: readonly unknown[]) => JSON.stringify({ claims });
const claimFailures = (
events: readonly unknown[],
evidence: Evidence,
rubric?: readonly FixtureClaimExpectation[],
) =>
assess(events, answer([claimFor(evidence)]), rubric).factualAssessment
.claims[0]?.failures;
// Synthetic transcript cases exercise grading, not an SDK-built assembly oracle.
const managedTranscript = () => {
@@ -445,11 +452,10 @@ describe("managed member grounding", () => {
// Removing exact fixture comparison or accepted-source validation must break these regressions.
describe("known-answer fixture grading", () => {
it("passes closed claims against authenticated production MCP Evidence", () => {
it("passes closed claims against authenticated MCP Evidence with equivalent result aliases", () => {
const evidence = createObservation();
expect(
assess([delivered(evidence)], answer([claimFor(evidence)])),
).toMatchObject({
const event = delivered(evidence, { output: resultFor(evidence) });
expect(assess([event], answer([claimFor(evidence)]))).toMatchObject({
factualCorrectness: "passed",
factualAssessment: {
scope: "configured_fixture_claims",
@@ -479,15 +485,6 @@ describe("known-answer fixture grading", () => {
).toBe("failed");
});
it("retains not_assessed without a fixture rubric", () => {
expect(
evaluateCodexEvents(
[final("Observed metadata remains unknown.")],
"analyze_javascript_application",
).factualCorrectness,
).toBe("not_assessed");
});
it.each([{ rubric: [] }, { rubric: [expected(), expected()] }])(
"fails closed for an empty or duplicate rubric",
({ rubric }) => {
@@ -513,10 +510,9 @@ describe("known-answer fixture grading", () => {
},
},
];
expect(
assess([delivered(evidence)], answer([claimFor(evidence)]), rubric)
.factualCorrectness,
).toBe("failed");
expect(claimFailures([delivered(evidence)], evidence, rubric)).toEqual([
"selector_failed",
]);
});
});
@@ -596,10 +592,9 @@ describe("successful Evidence grounding", () => {
{ tool: "other_tool" },
])("rejects a foreign or unsuccessful completion %j", (overrides) => {
const evidence = createObservation();
expect(
assess([delivered(evidence, overrides)], answer([claimFor(evidence)]))
.factualCorrectness,
).toBe("failed");
expect(claimFailures([delivered(evidence, overrides)], evidence)).toEqual([
"unknown_evidence",
]);
});
it("rejects IDs present only in arguments or unrelated metadata", () => {
@@ -610,9 +605,7 @@ describe("successful Evidence grounding", () => {
structuredContent: { metadata: { evidence_id: evidence.evidence_id } },
},
});
expect(
assess([event], answer([claimFor(evidence)])).factualCorrectness,
).toBe("failed");
expect(claimFailures([event], evidence)).toEqual(["unknown_evidence"]);
});
it("rejects a tampered envelope even with a valid-looking Evidence ID", () => {
@@ -625,9 +618,7 @@ describe("successful Evidence grounding", () => {
},
},
});
expect(
assess([event], answer([claimFor(evidence)])).factualCorrectness,
).toBe("failed");
expect(claimFailures([event], evidence)).toEqual(["unknown_evidence"]);
});
it.each(["id", "result", "error"])(
@@ -652,9 +643,7 @@ describe("successful Evidence grounding", () => {
...(field === "error" ? { isError: true } : {}),
},
});
expect(
assess([event], answer([claimFor(evidence)])).factualCorrectness,
).toBe("failed");
expect(claimFailures([event], evidence)).toEqual(["unknown_evidence"]);
},
);
@@ -675,20 +664,10 @@ describe("successful Evidence grounding", () => {
},
};
const event = delivered(evidence, { output });
expect(
assess([event], answer([claimFor(evidence)])).factualCorrectness,
).toBe("failed");
expect(claimFailures([event], evidence)).toEqual(["unknown_evidence"]);
},
);
it("accepts JSON-equal separately allocated outer result/output aliases", () => {
const evidence = createObservation();
const event = delivered(evidence, { output: resultFor(evidence) });
expect(
assess([event], answer([claimFor(evidence)])).factualCorrectness,
).toBe("passed");
});
it("accepts the documented snake-case structured-content transcript representation", () => {
const evidence = createObservation();
const event = delivered(evidence, {
@@ -714,20 +693,18 @@ describe("successful Evidence grounding", () => {
it("rejects authentic unrelated Evidence containing the same value elsewhere", () => {
const evidence = createObservation({ unrelated: expected().expectedValue });
expect(
assess([delivered(evidence)], answer([claimFor(evidence)]))
.factualCorrectness,
).toBe("failed");
expect(claimFailures([delivered(evidence)], evidence)).toEqual([
"source_assertion_mismatch",
]);
});
});
describe("fixture identity and provenance binding", () => {
it("rejects Evidence for a different subject despite an equal supported value", () => {
const evidence = createObservation(undefined, "/targets/other.asar");
expect(
assess([delivered(evidence)], answer([claimFor(evidence)]))
.factualCorrectness,
).toBe("failed");
expect(claimFailures([delivered(evidence)], evidence)).toEqual([
"source_subject_mismatch",
]);
});
it("rejects a wrong required source parameter", () => {
@@ -739,10 +716,9 @@ describe("fixture identity and provenance binding", () => {
source: { ...expectation.source, parameters: { format: "directory" } },
},
];
expect(
assess([delivered(evidence)], answer([claimFor(evidence)]), rubric)
.factualCorrectness,
).toBe("failed");
expect(claimFailures([delivered(evidence)], evidence, rubric)).toEqual([
"source_parameters_mismatch",
]);
});
it("rejects conflicting subject diagnostics for the same semantic Evidence ID", () => {
@@ -754,11 +730,8 @@ describe("fixture identity and provenance binding", () => {
}),
};
expect(
assess(
[delivered(evidence), delivered(conflicting)],
answer([claimFor(evidence)]),
).factualCorrectness,
).toBe("failed");
claimFailures([delivered(evidence), delivered(conflicting)], evidence),
).toEqual(["conflicting_evidence"]);
});
it("requires linked source Evidence to be delivered and match the configured artifact", () => {