mirror of
https://github.com/morluto/rea.git
synced 2026-11-06 14:56:32 +00:00
Ysopwnable
2653b194b6
Guide task-centered recovery from observed client, registration, PATH and provider blockers. Install the recovery reference, document exact JavaScript module selectors, and require clarification before searching for an unnamed app. Validate public setup in isolated Codex accounts and preserve failed or unrecognized tool outcomes in the existing evaluator. Refs: #1587 BREAKING CHANGE: Optional verify:agent reports now use schemaVersion 4. Consumers must accept expectedFirstTool: null and targetClarificationPassed for missing-target scenarios. Production CLI/MCP contracts are unchanged.
64 lines
2.3 KiB
TypeScript
64 lines
2.3 KiB
TypeScript
/** Factual outcome for configured fixtures, independent of routing heuristics. */
|
|
export type FixtureCorrectness = "passed" | "failed" | "not_assessed";
|
|
|
|
/** Measurements consumed by the evaluation runner's release gate. */
|
|
export interface AgentEvaluationGateResult {
|
|
/** Explicit null requires a target clarification with no REA calls. */
|
|
readonly expectedFirstTool?: string | null;
|
|
readonly targetClarificationPassed?: boolean;
|
|
readonly exitCode: number;
|
|
readonly naturalUse: boolean;
|
|
readonly correctFirstTool: boolean;
|
|
readonly repeatedCallCount: number;
|
|
readonly inputValidationFailureCount: number;
|
|
readonly requiredToolSubsequenceMet: boolean;
|
|
readonly inputTokens: number;
|
|
readonly answerHeuristicsMet: boolean;
|
|
readonly epistemicCuePresent: boolean;
|
|
readonly factualCorrectness: FixtureCorrectness;
|
|
}
|
|
|
|
/** Require configured facts to pass while preserving workflow and execution gates. */
|
|
export const agentEvaluationPassed = (
|
|
result: AgentEvaluationGateResult,
|
|
): boolean =>
|
|
result.exitCode === 0 &&
|
|
(result.expectedFirstTool === null
|
|
? !result.naturalUse
|
|
: result.naturalUse) &&
|
|
result.correctFirstTool &&
|
|
result.repeatedCallCount === 0 &&
|
|
result.inputValidationFailureCount === 0 &&
|
|
result.requiredToolSubsequenceMet &&
|
|
result.inputTokens > 0 &&
|
|
(result.expectedFirstTool === null
|
|
? result.targetClarificationPassed === true
|
|
: result.factualCorrectness === "passed" ||
|
|
(result.factualCorrectness === "not_assessed" &&
|
|
result.answerHeuristicsMet &&
|
|
result.epistemicCuePresent));
|
|
|
|
/** Keep assessed fixture outcomes and scenarios without truth oracles distinct. */
|
|
export const summarizeFactualCorrectness = (
|
|
results: readonly { readonly factualCorrectness: FixtureCorrectness }[],
|
|
) => {
|
|
const passed = results.filter(
|
|
({ factualCorrectness }) => factualCorrectness === "passed",
|
|
).length;
|
|
const failed = results.filter(
|
|
({ factualCorrectness }) => factualCorrectness === "failed",
|
|
).length;
|
|
const status: FixtureCorrectness =
|
|
failed > 0 ? "failed" : passed > 0 ? "passed" : "not_assessed";
|
|
return {
|
|
status,
|
|
scope: "configured_fixture_claims",
|
|
counts: {
|
|
assessed: passed + failed,
|
|
passed,
|
|
failed,
|
|
notAssessed: results.length - passed - failed,
|
|
},
|
|
};
|
|
};
|