Files
ECC/tests/scripts/evaluate-context-routing.test.js
T

141 lines
7.1 KiB
JavaScript

'use strict';
const assert = require('node:assert/strict');
const test = require('node:test');
const evaluator = () => require('../../scripts/ci/evaluate-context-routing');
const row = (set, expected, candidateIds, selectedIds = []) => ({ set, query: `${set} ${candidateIds.join(' ')}`, expected, candidateIds, selectedIds });
function healthyRows() {
const rows = [];
for (let index = 0; index < 50; index++) rows.push(row('direct', ['skill:a'], ['skill:a', 'skill:b', 'skill:c']));
rows.push(row('direct', ['skill:a'], ['skill:b', 'skill:a', 'skill:c']));
rows.push(row('direct', ['skill:a'], ['skill:b', 'skill:c', 'skill:d']));
for (let index = 0; index < 25; index++) rows.push(row('paraphrased', ['skill:a'], ['skill:z']));
return rows;
}
test('healthy rows pass every gate and report per-set rates', () => {
const result = evaluator().score(healthyRows());
assert.equal(result.status, 'pass');
assert.deepEqual(result.breaches, []);
assert.equal(result.sets.direct.prompts, 52);
assert.equal(result.sets.direct.top1, 50);
assert.equal(result.sets.direct.top3, 51);
assert.equal(result.sets.paraphrased.top3Rate, 0);
});
test('a planted ranking regression fails and names the breached threshold', () => {
const rows = healthyRows().map((entry, index) => (entry.set === 'direct' && index < 10
? { ...entry, candidateIds: ['skill:x', 'skill:y', 'skill:z'] } : entry));
const result = evaluator().score(rows);
assert.equal(result.status, 'fail');
assert.ok(result.breaches.some(breach => breach.metric === 'direct.top3Rate'));
assert.ok(result.breaches.some(breach => breach.metric === 'direct.top1Rate'));
});
test('one wrong implicit load fails regardless of ranking quality', () => {
const rows = healthyRows();
rows[0] = { ...rows[0], selectedIds: ['skill:b'] };
const result = evaluator().score(rows);
assert.equal(result.status, 'fail');
assert.deepEqual(result.breaches.map(breach => breach.metric), ['direct.wrongLoads']);
assert.equal(result.wrongLoads.length, 1);
assert.equal(result.sets.direct.admissionPrecision, 0);
});
test('paraphrased ranking is reported, never gated; its wrong loads still fail', () => {
const rows = healthyRows();
assert.equal(evaluator().score(rows).status, 'pass');
const loaded = rows.map(entry => (entry.set === 'paraphrased' ? { ...entry, selectedIds: ['skill:z'] } : entry));
const result = evaluator().score(loaded);
assert.deepEqual(result.breaches.map(breach => breach.metric), ['paraphrased.wrongLoads']);
});
test('no implicit loads leaves admission precision undefined instead of perfect', () => {
assert.equal(evaluator().score(healthyRows()).sets.direct.admissionPrecision, null);
});
/** Direct rows whose expected skill sits at ranks[index]; rows past ranks.length miss it. */
function rankedRows(total, ranks) {
return Array.from({ length: total }, (_, index) => {
const candidateIds = ['skill:w', 'skill:x', 'skill:y', 'skill:z'];
if (index < ranks.length) candidateIds[ranks[index]] = 'skill:a';
return row('direct', ['skill:a'], candidateIds);
});
}
test('top-3 counts a hit at the third rank but not the fourth', () => {
const { direct } = evaluator().score(rankedRows(4, [0, 2, 3])).sets;
assert.equal(direct.top1, 1);
assert.equal(direct.top3, 2);
});
test('thresholds compare unrounded ratios and pass exactly at 0.88 and 0.80', () => {
const { score } = evaluator();
const atThreshold = score(rankedRows(25, [...Array(20).fill(0), 1, 1]));
assert.equal(atThreshold.sets.direct.top1Rate, 0.8);
assert.equal(atThreshold.sets.direct.top3Rate, 0.88);
assert.deepEqual(atThreshold.breaches, []);
const roundsUp = score(rankedRows(2500, [...Array(1999).fill(0), ...Array(501).fill(1)]));
assert.deepEqual(roundsUp.breaches, [{ metric: 'direct.top1Rate', actual: 0.8, required: 0.8 }]);
});
test('thresholds are pinned and an empty corpus fails closed', () => {
const { THRESHOLDS, score } = evaluator();
assert.deepEqual({ ...THRESHOLDS }, { 'direct.top3Rate': 0.88, 'direct.top1Rate': 0.8 });
const result = score([]);
assert.equal(result.status, 'fail');
assert.deepEqual(result.breaches.map(breach => [breach.metric, breach.actual]), [['direct.top3Rate', null], ['direct.top1Rate', null]]);
});
test('--baseline prints a per-metric delta against a previous receipt', () => {
const current = evaluator().score(healthyRows());
const previous = JSON.parse(JSON.stringify(current));
previous.sets.direct.top1Rate = 1;
const lines = evaluator().baselineDelta(current, previous);
assert.ok(lines.some(line => /direct\.top1Rate/.test(line) && /-0\.038/.test(line)));
assert.ok(lines.some(line => /direct\.top3Rate/.test(line) && /\+0\.000/.test(line)));
});
test('the CLI emits JSON, applies a baseline and exits by gate status without re-running the corpus', () => {
const output = [];
const io = { log: text => output.push(text), error: text => output.push(text), readFile: () => JSON.stringify(evaluator().score(healthyRows())) };
const passing = { evaluate: () => evaluator().score(healthyRows()) };
assert.equal(evaluator().main(['--json'], { ...passing, ...io }), 0);
assert.equal(JSON.parse(output.at(-1)).status, 'pass');
output.length = 0;
assert.equal(evaluator().main(['--baseline', 'previous.json'], { ...passing, ...io }), 0);
assert.ok(output.some(line => /direct\.top1Rate/.test(line)));
const failing = { evaluate: () => evaluator().score(healthyRows().map(entry => ({ ...entry, selectedIds: ['skill:q'] }))) };
assert.equal(evaluator().main([], { ...failing, ...io }), 1);
assert.equal(evaluator().main(['--write'], { ...passing, ...io }), 1);
assert.equal(evaluator().main(['--baseline'], { ...passing, ...io }), 1);
});
test('the CLI reads the baseline before the corpus run and keeps --json stdout parseable', () => {
const stdout = [];
const stderr = [];
let runs = 0;
const io = { log: text => stdout.push(text), error: text => stderr.push(text),
evaluate: () => { runs += 1; return evaluator().score(healthyRows()); } };
const receipt = JSON.stringify(evaluator().score(healthyRows()));
assert.equal(evaluator().main(['--json', '--baseline', 'previous.json'], { ...io, readFile: () => receipt }), 0);
assert.equal(JSON.parse(stdout.join('\n')).status, 'pass');
assert.ok(stderr.some(line => /direct\.top1Rate/.test(line)));
runs = 0;
assert.equal(evaluator().main(['--baseline', 'missing.json'], { ...io, readFile: () => { throw new Error('ENOENT'); } }), 1);
assert.equal(evaluator().main(['--baseline', 'bad.json'], { ...io, readFile: () => '{bad' }), 1);
assert.equal(evaluator().main(['--baseline', 'null.json'], { ...io, readFile: () => 'null' }), 1);
assert.equal(runs, 0);
assert.equal(evaluator().main([], { ...io, evaluate: () => { throw new Error('boom'); } }), 1);
assert.ok(stderr.includes('Routing quality evaluation failed: boom'));
});
test('the gate has an npm entry point but stays out of the default test chain', () => {
const { scripts } = require('../../package.json');
assert.equal(scripts['routing:check'], 'node scripts/ci/evaluate-context-routing.js');
assert.ok(!scripts.test.includes('evaluate-context-routing'));
});