Files
ECC/scripts/ci/evaluate-context-routing.js
T

155 lines
7.2 KiB
JavaScript

#!/usr/bin/env node
/**
* Routing-quality gate over the routing prompt corpus from community PR #2945.
*
* Wrong implicit loads fail in both sets. Ranking is gated on the direct set
* only; the paraphrased set is reported so it can improve without blocking.
*/
'use strict';
const fs = require('node:fs');
const path = require('node:path');
const { resolveTaskContext } = require('../lib/context-selection');
const { DEFAULT_REPO_ROOT } = require('../lib/context-profile-support');
const CORPUS = path.join(DEFAULT_REPO_ROOT, 'tests/fixtures/context-selection/routing-prompts.json');
const SCHEMA = 'ecc.routing-quality.v1';
const SETS = Object.freeze(['direct', 'paraphrased']);
const THRESHOLDS = Object.freeze({ 'direct.top3Rate': 0.88, 'direct.top1Rate': 0.80 });
const METRICS = Object.freeze(['top1Rate', 'top3Rate', 'admissionPrecision']);
const rate = (count, total) => (total ? Number((count / total).toFixed(3)) : null);
function score(rows) {
const sets = {};
const wrongLoads = [];
for (const set of SETS) {
const entries = rows.filter(entry => entry.set === set);
let top1 = 0;
let top3 = 0;
let admitted = 0;
let wrong = 0;
for (const entry of entries) {
if (entry.expected.includes(entry.candidateIds[0])) top1 += 1;
if (entry.candidateIds.slice(0, 3).some(id => entry.expected.includes(id))) top3 += 1;
if (!entry.selectedIds.length) continue;
admitted += 1;
if (entry.selectedIds.some(id => !entry.expected.includes(id))) {
wrong += 1;
wrongLoads.push({ set, query: entry.query, selectedIds: entry.selectedIds, expected: entry.expected });
}
}
sets[set] = { prompts: entries.length, top1, top3, admitted, wrongLoads: wrong,
top1Rate: rate(top1, entries.length), top3Rate: rate(top3, entries.length),
admissionPrecision: rate(admitted - wrong, admitted) };
}
const breaches = [];
for (const set of SETS) {
if (sets[set].wrongLoads) breaches.push({ metric: `${set}.wrongLoads`, actual: sets[set].wrongLoads, required: 0 });
}
for (const [metric, required] of Object.entries(THRESHOLDS)) {
const [set, key] = metric.split('.');
const { prompts } = sets[set];
// Gate on the unrounded ratio; the rounded rate is for display only.
const ratio = prompts ? sets[set][key.replace(/Rate$/, '')] / prompts : null;
if (ratio === null || ratio < required) breaches.push({ metric, actual: sets[set][key], required });
}
return { schemaVersion: SCHEMA, status: breaches.length ? 'fail' : 'pass',
thresholds: { ...THRESHOLDS, wrongLoads: 0 }, sets, breaches, wrongLoads };
}
/** Reads the corpus and rejects a missing or empty set, or a malformed prompt, before any resolve. */
function readCorpus(corpusPath) {
const corpus = JSON.parse(fs.readFileSync(corpusPath, 'utf8'));
for (const set of SETS) {
const prompts = corpus && corpus[set];
const valid = Array.isArray(prompts) && prompts.length > 0 && prompts.every(prompt => prompt
&& typeof prompt.query === 'string' && Array.isArray(prompt.expected) && prompt.expected.length > 0);
if (!valid) throw new Error(`routing corpus ${corpusPath}: "${set}" must be a non-empty array of { query, expected } prompts`);
}
return corpus;
}
/** One resolver pass over the corpus; every prompt is a fresh task in auto mode.
* The registry digest comes from the resolver's own receipt, so the gate adds no registry load. */
function evaluate({ repoRoot = DEFAULT_REPO_ROOT, corpusPath = CORPUS } = {}) {
const corpus = readCorpus(corpusPath);
const rows = [];
let registryDigest = null;
for (const set of SETS) {
for (const [index, { query, expected }] of corpus[set].entries()) {
const result = resolveTaskContext({ repoRoot, selectionMode: 'auto', load: false,
task: { sessionId: 'routing-eval', taskId: `${set}-${index}`, revision: 1, phase: 'evaluate', query } });
registryDigest = registryDigest || result.receipt.registryDigest;
rows.push({ set, query, expected, candidateIds: result.candidates.map(candidate => candidate.id), selectedIds: result.selectedIds });
}
}
return { ...score(rows), registryDigest, node: process.version };
}
function baselineDelta(current, previous) {
const lines = [];
for (const set of SETS) {
for (const key of METRICS) {
const now = current.sets[set][key];
const before = previous.sets?.[set]?.[key];
if (typeof now !== 'number' || typeof before !== 'number') {
lines.push(`${set}.${key}: ${before ?? 'n/a'} -> ${now ?? 'n/a'}`);
continue;
}
const change = now - before;
lines.push(`${set}.${key}: ${before.toFixed(3)} -> ${now.toFixed(3)} (${change < 0 ? '-' : '+'}${Math.abs(change).toFixed(3)})`);
}
}
return lines;
}
function summary(result) {
const { direct, paraphrased } = result.sets;
return `Routing quality ${result.status}: direct top-1 ${direct.top1}/${direct.prompts}, top-3 ${direct.top3}/${direct.prompts}, `
+ `${direct.admitted} implicit loads; paraphrased top-3 ${paraphrased.top3}/${paraphrased.prompts} (reported, not gated); `
+ `${result.wrongLoads.length} wrong loads.`;
}
/** Reads a previous receipt before the corpus run, so a bad path or file fails fast. */
function readBaseline(io, file) {
if (!file) return null;
const previous = JSON.parse(io.readFile(file));
const isReceipt = previous !== null && typeof previous === 'object' && !Array.isArray(previous)
&& previous.schemaVersion === SCHEMA
&& SETS.every(set => previous.sets && previous.sets[set] !== null && typeof previous.sets[set] === 'object');
if (!isReceipt) throw new Error(`${file} is not an ${SCHEMA} receipt`);
return previous;
}
function main(args = process.argv.slice(2), dependencies = {}) {
const io = { evaluate, log: console.log, error: console.error, readFile: file => fs.readFileSync(file, 'utf8'), ...dependencies };
try {
let json = false;
let baseline = null;
for (let index = 0; index < args.length; index++) {
if (args[index] === '--json') json = true;
else if (args[index] === '--baseline' && args[index + 1] && !args[index + 1].startsWith('--')) baseline = args[++index];
else throw new Error(args[index] === '--baseline' ? '--baseline requires a receipt path' : `Unknown argument: ${args[index]}`);
}
const previous = readBaseline(io, baseline);
const result = io.evaluate();
if (json) io.log(JSON.stringify(result, null, 2));
else {
io.log(summary(result));
for (const breach of result.breaches) io.error(`Gate breached: ${breach.metric} is ${breach.actual}, requires ${breach.metric.endsWith('wrongLoads') ? '0' : `>= ${breach.required}`}`);
for (const wrong of result.wrongLoads) io.error(`Wrong load (${wrong.set}): "${wrong.query}" -> ${wrong.selectedIds.join(', ')}`);
}
// With --json the delta goes to stderr so stdout stays a single JSON document.
if (previous) for (const line of baselineDelta(result, previous)) (json ? io.error : io.log)(line);
return result.status === 'pass' ? 0 : 1;
} catch (error) {
io.error(`Routing quality evaluation failed: ${error.message}`);
return 1;
}
}
if (require.main === module) process.exitCode = main();
module.exports = { THRESHOLDS, baselineDelta, evaluate, main, score };