mirror of
https://github.com/affaan-m/ECC.git
synced 2026-11-17 13:32:18 +00:00
155 lines
7.2 KiB
JavaScript
155 lines
7.2 KiB
JavaScript
#!/usr/bin/env node
|
|
/**
|
|
* Routing-quality gate over the routing prompt corpus from community PR #2945.
|
|
*
|
|
* Wrong implicit loads fail in both sets. Ranking is gated on the direct set
|
|
* only; the paraphrased set is reported so it can improve without blocking.
|
|
*/
|
|
|
|
'use strict';
|
|
|
|
const fs = require('node:fs');
|
|
const path = require('node:path');
|
|
const { resolveTaskContext } = require('../lib/context-selection');
|
|
const { DEFAULT_REPO_ROOT } = require('../lib/context-profile-support');
|
|
|
|
const CORPUS = path.join(DEFAULT_REPO_ROOT, 'tests/fixtures/context-selection/routing-prompts.json');
|
|
const SCHEMA = 'ecc.routing-quality.v1';
|
|
const SETS = Object.freeze(['direct', 'paraphrased']);
|
|
const THRESHOLDS = Object.freeze({ 'direct.top3Rate': 0.88, 'direct.top1Rate': 0.80 });
|
|
const METRICS = Object.freeze(['top1Rate', 'top3Rate', 'admissionPrecision']);
|
|
|
|
const rate = (count, total) => (total ? Number((count / total).toFixed(3)) : null);
|
|
|
|
function score(rows) {
|
|
const sets = {};
|
|
const wrongLoads = [];
|
|
for (const set of SETS) {
|
|
const entries = rows.filter(entry => entry.set === set);
|
|
let top1 = 0;
|
|
let top3 = 0;
|
|
let admitted = 0;
|
|
let wrong = 0;
|
|
for (const entry of entries) {
|
|
if (entry.expected.includes(entry.candidateIds[0])) top1 += 1;
|
|
if (entry.candidateIds.slice(0, 3).some(id => entry.expected.includes(id))) top3 += 1;
|
|
if (!entry.selectedIds.length) continue;
|
|
admitted += 1;
|
|
if (entry.selectedIds.some(id => !entry.expected.includes(id))) {
|
|
wrong += 1;
|
|
wrongLoads.push({ set, query: entry.query, selectedIds: entry.selectedIds, expected: entry.expected });
|
|
}
|
|
}
|
|
sets[set] = { prompts: entries.length, top1, top3, admitted, wrongLoads: wrong,
|
|
top1Rate: rate(top1, entries.length), top3Rate: rate(top3, entries.length),
|
|
admissionPrecision: rate(admitted - wrong, admitted) };
|
|
}
|
|
const breaches = [];
|
|
for (const set of SETS) {
|
|
if (sets[set].wrongLoads) breaches.push({ metric: `${set}.wrongLoads`, actual: sets[set].wrongLoads, required: 0 });
|
|
}
|
|
for (const [metric, required] of Object.entries(THRESHOLDS)) {
|
|
const [set, key] = metric.split('.');
|
|
const { prompts } = sets[set];
|
|
// Gate on the unrounded ratio; the rounded rate is for display only.
|
|
const ratio = prompts ? sets[set][key.replace(/Rate$/, '')] / prompts : null;
|
|
if (ratio === null || ratio < required) breaches.push({ metric, actual: sets[set][key], required });
|
|
}
|
|
return { schemaVersion: SCHEMA, status: breaches.length ? 'fail' : 'pass',
|
|
thresholds: { ...THRESHOLDS, wrongLoads: 0 }, sets, breaches, wrongLoads };
|
|
}
|
|
|
|
/** Reads the corpus and rejects a missing or empty set, or a malformed prompt, before any resolve. */
|
|
function readCorpus(corpusPath) {
|
|
const corpus = JSON.parse(fs.readFileSync(corpusPath, 'utf8'));
|
|
for (const set of SETS) {
|
|
const prompts = corpus && corpus[set];
|
|
const valid = Array.isArray(prompts) && prompts.length > 0 && prompts.every(prompt => prompt
|
|
&& typeof prompt.query === 'string' && Array.isArray(prompt.expected) && prompt.expected.length > 0);
|
|
if (!valid) throw new Error(`routing corpus ${corpusPath}: "${set}" must be a non-empty array of { query, expected } prompts`);
|
|
}
|
|
return corpus;
|
|
}
|
|
|
|
/** One resolver pass over the corpus; every prompt is a fresh task in auto mode.
|
|
* The registry digest comes from the resolver's own receipt, so the gate adds no registry load. */
|
|
function evaluate({ repoRoot = DEFAULT_REPO_ROOT, corpusPath = CORPUS } = {}) {
|
|
const corpus = readCorpus(corpusPath);
|
|
const rows = [];
|
|
let registryDigest = null;
|
|
for (const set of SETS) {
|
|
for (const [index, { query, expected }] of corpus[set].entries()) {
|
|
const result = resolveTaskContext({ repoRoot, selectionMode: 'auto', load: false,
|
|
task: { sessionId: 'routing-eval', taskId: `${set}-${index}`, revision: 1, phase: 'evaluate', query } });
|
|
registryDigest = registryDigest || result.receipt.registryDigest;
|
|
rows.push({ set, query, expected, candidateIds: result.candidates.map(candidate => candidate.id), selectedIds: result.selectedIds });
|
|
}
|
|
}
|
|
return { ...score(rows), registryDigest, node: process.version };
|
|
}
|
|
|
|
function baselineDelta(current, previous) {
|
|
const lines = [];
|
|
for (const set of SETS) {
|
|
for (const key of METRICS) {
|
|
const now = current.sets[set][key];
|
|
const before = previous.sets?.[set]?.[key];
|
|
if (typeof now !== 'number' || typeof before !== 'number') {
|
|
lines.push(`${set}.${key}: ${before ?? 'n/a'} -> ${now ?? 'n/a'}`);
|
|
continue;
|
|
}
|
|
const change = now - before;
|
|
lines.push(`${set}.${key}: ${before.toFixed(3)} -> ${now.toFixed(3)} (${change < 0 ? '-' : '+'}${Math.abs(change).toFixed(3)})`);
|
|
}
|
|
}
|
|
return lines;
|
|
}
|
|
|
|
function summary(result) {
|
|
const { direct, paraphrased } = result.sets;
|
|
return `Routing quality ${result.status}: direct top-1 ${direct.top1}/${direct.prompts}, top-3 ${direct.top3}/${direct.prompts}, `
|
|
+ `${direct.admitted} implicit loads; paraphrased top-3 ${paraphrased.top3}/${paraphrased.prompts} (reported, not gated); `
|
|
+ `${result.wrongLoads.length} wrong loads.`;
|
|
}
|
|
|
|
/** Reads a previous receipt before the corpus run, so a bad path or file fails fast. */
|
|
function readBaseline(io, file) {
|
|
if (!file) return null;
|
|
const previous = JSON.parse(io.readFile(file));
|
|
const isReceipt = previous !== null && typeof previous === 'object' && !Array.isArray(previous)
|
|
&& previous.schemaVersion === SCHEMA
|
|
&& SETS.every(set => previous.sets && previous.sets[set] !== null && typeof previous.sets[set] === 'object');
|
|
if (!isReceipt) throw new Error(`${file} is not an ${SCHEMA} receipt`);
|
|
return previous;
|
|
}
|
|
|
|
function main(args = process.argv.slice(2), dependencies = {}) {
|
|
const io = { evaluate, log: console.log, error: console.error, readFile: file => fs.readFileSync(file, 'utf8'), ...dependencies };
|
|
try {
|
|
let json = false;
|
|
let baseline = null;
|
|
for (let index = 0; index < args.length; index++) {
|
|
if (args[index] === '--json') json = true;
|
|
else if (args[index] === '--baseline' && args[index + 1] && !args[index + 1].startsWith('--')) baseline = args[++index];
|
|
else throw new Error(args[index] === '--baseline' ? '--baseline requires a receipt path' : `Unknown argument: ${args[index]}`);
|
|
}
|
|
const previous = readBaseline(io, baseline);
|
|
const result = io.evaluate();
|
|
if (json) io.log(JSON.stringify(result, null, 2));
|
|
else {
|
|
io.log(summary(result));
|
|
for (const breach of result.breaches) io.error(`Gate breached: ${breach.metric} is ${breach.actual}, requires ${breach.metric.endsWith('wrongLoads') ? '0' : `>= ${breach.required}`}`);
|
|
for (const wrong of result.wrongLoads) io.error(`Wrong load (${wrong.set}): "${wrong.query}" -> ${wrong.selectedIds.join(', ')}`);
|
|
}
|
|
// With --json the delta goes to stderr so stdout stays a single JSON document.
|
|
if (previous) for (const line of baselineDelta(result, previous)) (json ? io.error : io.log)(line);
|
|
return result.status === 'pass' ? 0 : 1;
|
|
} catch (error) {
|
|
io.error(`Routing quality evaluation failed: ${error.message}`);
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
if (require.main === module) process.exitCode = main();
|
|
module.exports = { THRESHOLDS, baselineDelta, evaluate, main, score };
|