mirror of
https://github.com/morluto/rea.git
synced 2026-11-21 14:05:49 +00:00
test(evaluation): consolidate transcript and claim grounding checks (#12883)
* test(evaluation): isolate transcript routing and citation failures * test(evaluation): require precise claim grounding failures
This commit is contained in:
1 parent
4449d13047
commit
c2c8ad3d70
2 files changed
+85
-170
No files matched your search
@@ -97,41 +97,6 @@ describe("Codex agent release evaluation", () => {
|
||||
});
|
||||
expect(metrics.reaCalls).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("requires the final answer to cite produced Evidence when requested", () => {
|
||||
const evidence = observation();
|
||||
const evidenceId = evidence.evidence_id;
|
||||
const events = [
|
||||
{
|
||||
type: "item.completed",
|
||||
item: {
|
||||
id: "item-tool",
|
||||
type: "mcp_tool_call",
|
||||
server: "rea",
|
||||
tool: "inspect_managed_artifact",
|
||||
arguments: { path: "/tmp/app.dll" },
|
||||
result: { structuredContent: evidence },
|
||||
},
|
||||
},
|
||||
{
|
||||
type: "item.completed",
|
||||
item: {
|
||||
type: "agent_message",
|
||||
text: `The observed managed metadata is supported by evidence ${evidenceId}; runtime behavior remains unknown without runtime authority.`,
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
expect(
|
||||
evaluateCodexEvents(events, "inspect_managed_artifact", {
|
||||
requireEvidence: true,
|
||||
}),
|
||||
).toMatchObject({
|
||||
evidenceIds: [evidenceId],
|
||||
finalCitesEvidence: true,
|
||||
answerHeuristicsMet: true,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("Codex completed tool call parsing", () => {
|
||||
@@ -380,24 +345,6 @@ describe("Codex Evidence delivery", () => {
|
||||
});
|
||||
|
||||
describe("Codex request accounting", () => {
|
||||
it("flags identical calls with distinct item identities", () => {
|
||||
const event = (id: string) => ({
|
||||
type: "item.completed",
|
||||
item: {
|
||||
id,
|
||||
type: "mcp_tool_call",
|
||||
server: "rea",
|
||||
tool: "binary_session",
|
||||
arguments: {},
|
||||
},
|
||||
});
|
||||
|
||||
expect(
|
||||
evaluateCodexEvents([event("item-1"), event("item-2")], "binary_session")
|
||||
.repeatedCallCount,
|
||||
).toBe(1);
|
||||
});
|
||||
|
||||
it("distinguishes session verification after a successful close from an unchanged retry", () => {
|
||||
const call = (id: string, tool: string, status = "completed") => ({
|
||||
type: "item.completed",
|
||||
@@ -649,39 +596,25 @@ describe("Codex agent completion evaluation", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("requires scenario-specific answer terms when configured", () => {
|
||||
const event = (text: string) => [
|
||||
{
|
||||
type: "item.completed",
|
||||
item: {
|
||||
type: "mcp_tool_call",
|
||||
server: "rea",
|
||||
tool: "analyze_javascript_application",
|
||||
arguments: {},
|
||||
it("requires a term from every configured answer group", () => {
|
||||
expect(
|
||||
evaluateCodexEvents(
|
||||
[
|
||||
{
|
||||
type: "item.completed",
|
||||
item: {
|
||||
type: "agent_message",
|
||||
text: "Observed evidence identifies profileApi, but runtime behavior remains unknown without more authority and detail.",
|
||||
},
|
||||
},
|
||||
],
|
||||
null,
|
||||
{
|
||||
requiredAnswerTermGroups: [
|
||||
["profileApi"],
|
||||
["preload", "contextBridge"],
|
||||
],
|
||||
},
|
||||
},
|
||||
{ type: "item.completed", item: { type: "agent_message", text } },
|
||||
];
|
||||
const options = {
|
||||
requiredAnswerTermGroups: [["profileApi"], ["preload", "contextBridge"]],
|
||||
};
|
||||
|
||||
expect(
|
||||
evaluateCodexEvents(
|
||||
event(
|
||||
"Observed evidence connects profileApi through the preload bridge; runtime behavior remains unknown without authority.",
|
||||
),
|
||||
"analyze_javascript_application",
|
||||
options,
|
||||
),
|
||||
).toMatchObject({ answerTermCoverageMet: true, answerHeuristicsMet: true });
|
||||
expect(
|
||||
evaluateCodexEvents(
|
||||
event(
|
||||
"Observed artifact evidence is available, but application behavior remains unknown without more authority and detail.",
|
||||
),
|
||||
"analyze_javascript_application",
|
||||
options,
|
||||
),
|
||||
).toMatchObject({
|
||||
answerTermCoverageMet: false,
|
||||
@@ -755,39 +688,48 @@ describe("Codex agent workflow evaluation", () => {
|
||||
],
|
||||
forbidInputValidationFailures: true,
|
||||
};
|
||||
const valid = evaluateCodexEvents(
|
||||
[
|
||||
call("left", "analyze_javascript_application"),
|
||||
call("right", "analyze_javascript_application"),
|
||||
call("compare", "compare_javascript_export_shapes"),
|
||||
final,
|
||||
],
|
||||
"analyze_javascript_application",
|
||||
options,
|
||||
);
|
||||
expect(valid).toMatchObject({
|
||||
const left = call("left", "analyze_javascript_application");
|
||||
const right = call("right", "analyze_javascript_application");
|
||||
const compare = call("compare", "compare_javascript_export_shapes");
|
||||
const rejected = call("bad", "compare_javascript_export_shapes", {
|
||||
structured_content: { error: { code: "invalid_request" } },
|
||||
isError: true,
|
||||
});
|
||||
const evaluate = (events: unknown[]) =>
|
||||
evaluateCodexEvents(events, "analyze_javascript_application", options);
|
||||
|
||||
expect(evaluate([left, right, compare, final])).toMatchObject({
|
||||
requiredToolSubsequenceMet: true,
|
||||
inputValidationFailureCount: 0,
|
||||
finalCitesEvidence: true,
|
||||
answerHeuristicsMet: true,
|
||||
});
|
||||
|
||||
const invalid = evaluateCodexEvents(
|
||||
[
|
||||
call("left", "analyze_javascript_application"),
|
||||
call("bad", "compare_javascript_export_shapes", {
|
||||
structured_content: {
|
||||
error: { code: "invalid_request" },
|
||||
},
|
||||
isError: true,
|
||||
}),
|
||||
final,
|
||||
],
|
||||
"analyze_javascript_application",
|
||||
options,
|
||||
);
|
||||
expect(invalid).toMatchObject({
|
||||
expect(evaluate([left, compare, right, final])).toMatchObject({
|
||||
requiredToolSubsequenceMet: false,
|
||||
inputValidationFailureCount: 0,
|
||||
answerHeuristicsMet: false,
|
||||
});
|
||||
expect(evaluate([left, right, rejected, final])).toMatchObject({
|
||||
requiredToolSubsequenceMet: false,
|
||||
inputValidationFailureCount: 1,
|
||||
});
|
||||
expect(evaluate([left, right, compare, rejected, final])).toMatchObject({
|
||||
requiredToolSubsequenceMet: true,
|
||||
inputValidationFailureCount: 1,
|
||||
finalCitesEvidence: true,
|
||||
answerHeuristicsMet: false,
|
||||
});
|
||||
const uncited = {
|
||||
...final,
|
||||
item: {
|
||||
...final.item,
|
||||
text: final.item.text.replace(evidenceId, "cited elsewhere"),
|
||||
},
|
||||
};
|
||||
expect(evaluate([left, right, compare, uncited])).toMatchObject({
|
||||
requiredToolSubsequenceMet: true,
|
||||
inputValidationFailureCount: 0,
|
||||
finalCitesEvidence: false,
|
||||
answerHeuristicsMet: false,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -111,6 +111,13 @@ const assess = (
|
||||
},
|
||||
);
|
||||
const answer = (claims: readonly unknown[]) => JSON.stringify({ claims });
|
||||
const claimFailures = (
|
||||
events: readonly unknown[],
|
||||
evidence: Evidence,
|
||||
rubric?: readonly FixtureClaimExpectation[],
|
||||
) =>
|
||||
assess(events, answer([claimFor(evidence)]), rubric).factualAssessment
|
||||
.claims[0]?.failures;
|
||||
|
||||
// Synthetic transcript cases exercise grading, not an SDK-built assembly oracle.
|
||||
const managedTranscript = () => {
|
||||
@@ -445,11 +452,10 @@ describe("managed member grounding", () => {
|
||||
|
||||
// Removing exact fixture comparison or accepted-source validation must break these regressions.
|
||||
describe("known-answer fixture grading", () => {
|
||||
it("passes closed claims against authenticated production MCP Evidence", () => {
|
||||
it("passes closed claims against authenticated MCP Evidence with equivalent result aliases", () => {
|
||||
const evidence = createObservation();
|
||||
expect(
|
||||
assess([delivered(evidence)], answer([claimFor(evidence)])),
|
||||
).toMatchObject({
|
||||
const event = delivered(evidence, { output: resultFor(evidence) });
|
||||
expect(assess([event], answer([claimFor(evidence)]))).toMatchObject({
|
||||
factualCorrectness: "passed",
|
||||
factualAssessment: {
|
||||
scope: "configured_fixture_claims",
|
||||
@@ -479,15 +485,6 @@ describe("known-answer fixture grading", () => {
|
||||
).toBe("failed");
|
||||
});
|
||||
|
||||
it("retains not_assessed without a fixture rubric", () => {
|
||||
expect(
|
||||
evaluateCodexEvents(
|
||||
[final("Observed metadata remains unknown.")],
|
||||
"analyze_javascript_application",
|
||||
).factualCorrectness,
|
||||
).toBe("not_assessed");
|
||||
});
|
||||
|
||||
it.each([{ rubric: [] }, { rubric: [expected(), expected()] }])(
|
||||
"fails closed for an empty or duplicate rubric",
|
||||
({ rubric }) => {
|
||||
@@ -513,10 +510,9 @@ describe("known-answer fixture grading", () => {
|
||||
},
|
||||
},
|
||||
];
|
||||
expect(
|
||||
assess([delivered(evidence)], answer([claimFor(evidence)]), rubric)
|
||||
.factualCorrectness,
|
||||
).toBe("failed");
|
||||
expect(claimFailures([delivered(evidence)], evidence, rubric)).toEqual([
|
||||
"selector_failed",
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -596,10 +592,9 @@ describe("successful Evidence grounding", () => {
|
||||
{ tool: "other_tool" },
|
||||
])("rejects a foreign or unsuccessful completion %j", (overrides) => {
|
||||
const evidence = createObservation();
|
||||
expect(
|
||||
assess([delivered(evidence, overrides)], answer([claimFor(evidence)]))
|
||||
.factualCorrectness,
|
||||
).toBe("failed");
|
||||
expect(claimFailures([delivered(evidence, overrides)], evidence)).toEqual([
|
||||
"unknown_evidence",
|
||||
]);
|
||||
});
|
||||
|
||||
it("rejects IDs present only in arguments or unrelated metadata", () => {
|
||||
@@ -610,9 +605,7 @@ describe("successful Evidence grounding", () => {
|
||||
structuredContent: { metadata: { evidence_id: evidence.evidence_id } },
|
||||
},
|
||||
});
|
||||
expect(
|
||||
assess([event], answer([claimFor(evidence)])).factualCorrectness,
|
||||
).toBe("failed");
|
||||
expect(claimFailures([event], evidence)).toEqual(["unknown_evidence"]);
|
||||
});
|
||||
|
||||
it("rejects a tampered envelope even with a valid-looking Evidence ID", () => {
|
||||
@@ -625,9 +618,7 @@ describe("successful Evidence grounding", () => {
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(
|
||||
assess([event], answer([claimFor(evidence)])).factualCorrectness,
|
||||
).toBe("failed");
|
||||
expect(claimFailures([event], evidence)).toEqual(["unknown_evidence"]);
|
||||
});
|
||||
|
||||
it.each(["id", "result", "error"])(
|
||||
@@ -652,9 +643,7 @@ describe("successful Evidence grounding", () => {
|
||||
...(field === "error" ? { isError: true } : {}),
|
||||
},
|
||||
});
|
||||
expect(
|
||||
assess([event], answer([claimFor(evidence)])).factualCorrectness,
|
||||
).toBe("failed");
|
||||
expect(claimFailures([event], evidence)).toEqual(["unknown_evidence"]);
|
||||
},
|
||||
);
|
||||
|
||||
@@ -675,20 +664,10 @@ describe("successful Evidence grounding", () => {
|
||||
},
|
||||
};
|
||||
const event = delivered(evidence, { output });
|
||||
expect(
|
||||
assess([event], answer([claimFor(evidence)])).factualCorrectness,
|
||||
).toBe("failed");
|
||||
expect(claimFailures([event], evidence)).toEqual(["unknown_evidence"]);
|
||||
},
|
||||
);
|
||||
|
||||
it("accepts JSON-equal separately allocated outer result/output aliases", () => {
|
||||
const evidence = createObservation();
|
||||
const event = delivered(evidence, { output: resultFor(evidence) });
|
||||
expect(
|
||||
assess([event], answer([claimFor(evidence)])).factualCorrectness,
|
||||
).toBe("passed");
|
||||
});
|
||||
|
||||
it("accepts the documented snake-case structured-content transcript representation", () => {
|
||||
const evidence = createObservation();
|
||||
const event = delivered(evidence, {
|
||||
@@ -714,20 +693,18 @@ describe("successful Evidence grounding", () => {
|
||||
|
||||
it("rejects authentic unrelated Evidence containing the same value elsewhere", () => {
|
||||
const evidence = createObservation({ unrelated: expected().expectedValue });
|
||||
expect(
|
||||
assess([delivered(evidence)], answer([claimFor(evidence)]))
|
||||
.factualCorrectness,
|
||||
).toBe("failed");
|
||||
expect(claimFailures([delivered(evidence)], evidence)).toEqual([
|
||||
"source_assertion_mismatch",
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("fixture identity and provenance binding", () => {
|
||||
it("rejects Evidence for a different subject despite an equal supported value", () => {
|
||||
const evidence = createObservation(undefined, "/targets/other.asar");
|
||||
expect(
|
||||
assess([delivered(evidence)], answer([claimFor(evidence)]))
|
||||
.factualCorrectness,
|
||||
).toBe("failed");
|
||||
expect(claimFailures([delivered(evidence)], evidence)).toEqual([
|
||||
"source_subject_mismatch",
|
||||
]);
|
||||
});
|
||||
|
||||
it("rejects a wrong required source parameter", () => {
|
||||
@@ -739,10 +716,9 @@ describe("fixture identity and provenance binding", () => {
|
||||
source: { ...expectation.source, parameters: { format: "directory" } },
|
||||
},
|
||||
];
|
||||
expect(
|
||||
assess([delivered(evidence)], answer([claimFor(evidence)]), rubric)
|
||||
.factualCorrectness,
|
||||
).toBe("failed");
|
||||
expect(claimFailures([delivered(evidence)], evidence, rubric)).toEqual([
|
||||
"source_parameters_mismatch",
|
||||
]);
|
||||
});
|
||||
|
||||
it("rejects conflicting subject diagnostics for the same semantic Evidence ID", () => {
|
||||
@@ -754,11 +730,8 @@ describe("fixture identity and provenance binding", () => {
|
||||
}),
|
||||
};
|
||||
expect(
|
||||
assess(
|
||||
[delivered(evidence), delivered(conflicting)],
|
||||
answer([claimFor(evidence)]),
|
||||
).factualCorrectness,
|
||||
).toBe("failed");
|
||||
claimFailures([delivered(evidence), delivered(conflicting)], evidence),
|
||||
).toEqual(["conflicting_evidence"]);
|
||||
});
|
||||
|
||||
it("requires linked source Evidence to be delivered and match the configured artifact", () => {
|
||||
|
||||
Reference in new issue
Block a user