Files
reverse-engineering-anything/tests/evaluation/model-evals/agentEvaluationReport.test.ts
T

139 lines
3.9 KiB
TypeScript

import { describe, expect, it } from "vitest";
import {
agentEvaluationPassed,
assertAgentEvaluationReportPassed,
summarizeFactualCorrectness,
} from "./AgentEvaluationReport.js";
const routingSuccess = {
exitCode: 0,
naturalUse: true,
correctFirstTool: true,
repeatedCallCount: 0,
inputValidationFailureCount: 0,
requiredToolSubsequenceMet: true,
inputTokens: 100,
answerHeuristicsMet: true,
epistemicCuePresent: true,
};
it("reports only rejected fixture outcomes, preserving their order", () => {
const passing = {
id: "passing-fixture",
...routingSuccess,
factualCorrectness: "passed" as const,
};
expect(() => assertAgentEvaluationReportPassed([passing])).not.toThrow();
expect(() =>
assertAgentEvaluationReportPassed([
{ ...passing, id: "incorrect-fixture", factualCorrectness: "failed" },
passing,
{ ...passing, id: "refused-scenario", scenarioOutcomePassed: false },
]),
).toThrow(
/^Agent routing\/workflow\/answer checks failed: incorrect-fixture, refused-scenario$/u,
);
});
describe("scoped factual report", () => {
it("requires a validated target clarification and zero REA calls for the negative route", () => {
const clarification = {
...routingSuccess,
expectedFirstTool: null,
naturalUse: false,
factualCorrectness: "not_assessed" as const,
answerHeuristicsMet: false,
epistemicCuePresent: false,
targetClarificationPassed: true,
};
expect(agentEvaluationPassed(clarification)).toBe(true);
for (const change of [
{ naturalUse: true },
{ targetClarificationPassed: false },
])
expect(agentEvaluationPassed({ ...clarification, ...change })).toBe(
false,
);
});
it("accepts correct structured facts without superficial prose cues", () => {
expect(
agentEvaluationPassed({
...routingSuccess,
answerHeuristicsMet: false,
epistemicCuePresent: false,
factualCorrectness: "passed",
}),
).toBe(true);
});
it("retains workflow and execution checks for factually correct answers", () => {
for (const change of [
{ exitCode: 1 },
{ naturalUse: false },
{ correctFirstTool: false },
{ repeatedCallCount: 1 },
{ inputValidationFailureCount: 1 },
{ requiredToolSubsequenceMet: false },
{ inputTokens: 0 },
])
expect(
agentEvaluationPassed({
...routingSuccess,
factualCorrectness: "passed",
...change,
}),
).toBe(false);
});
it("retains heuristic checks only for unassessed scenarios", () => {
expect(
agentEvaluationPassed({
...routingSuccess,
factualCorrectness: "not_assessed",
}),
).toBe(true);
expect(
agentEvaluationPassed({
...routingSuccess,
factualCorrectness: "not_assessed",
answerHeuristicsMet: false,
}),
).toBe(false);
expect(
agentEvaluationPassed({
...routingSuccess,
factualCorrectness: "not_assessed",
epistemicCuePresent: false,
}),
).toBe(false);
});
it("reports assessed failures separately from unassessed scenarios", () => {
expect(
summarizeFactualCorrectness([
{ factualCorrectness: "passed" },
{ factualCorrectness: "failed" },
{ factualCorrectness: "not_assessed" },
]),
).toEqual({
status: "failed",
scope: "configured_fixture_claims",
counts: { assessed: 2, passed: 1, failed: 1, notAssessed: 1 },
});
expect(
summarizeFactualCorrectness([
{ factualCorrectness: "passed" },
{ factualCorrectness: "not_assessed" },
]).status,
).toBe("passed");
expect(
summarizeFactualCorrectness([{ factualCorrectness: "not_assessed" }])
.status,
).toBe("not_assessed");
expect(summarizeFactualCorrectness([]).status).toBe("not_assessed");
});
});