mirror of
https://github.com/affaan-m/ECC.git
synced 2026-11-17 05:22:17 +00:00
174 lines
8.9 KiB
JavaScript
174 lines
8.9 KiB
JavaScript
'use strict';
|
|
|
|
const assert = require('node:assert/strict');
|
|
const test = require('node:test');
|
|
|
|
const evaluator = () => require('../../scripts/ci/evaluate-context-routing');
|
|
|
|
const row = (set, expected, candidateIds, selectedIds = []) => ({ set, query: `${set} ${candidateIds.join(' ')}`, expected, candidateIds, selectedIds });
|
|
|
|
function healthyRows() {
|
|
const rows = [];
|
|
for (let index = 0; index < 50; index++) rows.push(row('direct', ['skill:a'], ['skill:a', 'skill:b', 'skill:c']));
|
|
rows.push(row('direct', ['skill:a'], ['skill:b', 'skill:a', 'skill:c']));
|
|
rows.push(row('direct', ['skill:a'], ['skill:b', 'skill:c', 'skill:d']));
|
|
for (let index = 0; index < 25; index++) rows.push(row('paraphrased', ['skill:a'], ['skill:z']));
|
|
return rows;
|
|
}
|
|
|
|
test('healthy rows pass every gate and report per-set rates', () => {
|
|
const result = evaluator().score(healthyRows());
|
|
assert.equal(result.status, 'pass');
|
|
assert.deepEqual(result.breaches, []);
|
|
assert.equal(result.sets.direct.prompts, 52);
|
|
assert.equal(result.sets.direct.top1, 50);
|
|
assert.equal(result.sets.direct.top3, 51);
|
|
assert.equal(result.sets.paraphrased.top3Rate, 0);
|
|
});
|
|
|
|
test('a planted ranking regression fails and names the breached threshold', () => {
|
|
const rows = healthyRows().map((entry, index) => (entry.set === 'direct' && index < 10
|
|
? { ...entry, candidateIds: ['skill:x', 'skill:y', 'skill:z'] } : entry));
|
|
const result = evaluator().score(rows);
|
|
assert.equal(result.status, 'fail');
|
|
assert.ok(result.breaches.some(breach => breach.metric === 'direct.top3Rate'));
|
|
assert.ok(result.breaches.some(breach => breach.metric === 'direct.top1Rate'));
|
|
});
|
|
|
|
test('one wrong implicit load fails regardless of ranking quality', () => {
|
|
const rows = healthyRows();
|
|
rows[0] = { ...rows[0], selectedIds: ['skill:b'] };
|
|
const result = evaluator().score(rows);
|
|
assert.equal(result.status, 'fail');
|
|
assert.deepEqual(result.breaches.map(breach => breach.metric), ['direct.wrongLoads']);
|
|
assert.equal(result.wrongLoads.length, 1);
|
|
assert.equal(result.sets.direct.admissionPrecision, 0);
|
|
});
|
|
|
|
test('paraphrased ranking is reported, never gated; its wrong loads still fail', () => {
|
|
const rows = healthyRows();
|
|
assert.equal(evaluator().score(rows).status, 'pass');
|
|
const loaded = rows.map(entry => (entry.set === 'paraphrased' ? { ...entry, selectedIds: ['skill:z'] } : entry));
|
|
const result = evaluator().score(loaded);
|
|
assert.deepEqual(result.breaches.map(breach => breach.metric), ['paraphrased.wrongLoads']);
|
|
});
|
|
|
|
test('no implicit loads leaves admission precision undefined instead of perfect', () => {
|
|
assert.equal(evaluator().score(healthyRows()).sets.direct.admissionPrecision, null);
|
|
});
|
|
|
|
/** Direct rows whose expected skill sits at ranks[index]; rows past ranks.length miss it. */
|
|
function rankedRows(total, ranks) {
|
|
return Array.from({ length: total }, (_, index) => {
|
|
const candidateIds = ['skill:w', 'skill:x', 'skill:y', 'skill:z'];
|
|
if (index < ranks.length) candidateIds[ranks[index]] = 'skill:a';
|
|
return row('direct', ['skill:a'], candidateIds);
|
|
});
|
|
}
|
|
|
|
test('top-3 counts a hit at the third rank but not the fourth', () => {
|
|
const { direct } = evaluator().score(rankedRows(4, [0, 2, 3])).sets;
|
|
assert.equal(direct.top1, 1);
|
|
assert.equal(direct.top3, 2);
|
|
});
|
|
|
|
test('thresholds compare unrounded ratios and pass exactly at 0.88 and 0.80', () => {
|
|
const { score } = evaluator();
|
|
const atThreshold = score(rankedRows(25, [...Array(20).fill(0), 1, 1]));
|
|
assert.equal(atThreshold.sets.direct.top1Rate, 0.8);
|
|
assert.equal(atThreshold.sets.direct.top3Rate, 0.88);
|
|
assert.deepEqual(atThreshold.breaches, []);
|
|
const roundsUp = score(rankedRows(2500, [...Array(1999).fill(0), ...Array(501).fill(1)]));
|
|
assert.deepEqual(roundsUp.breaches, [{ metric: 'direct.top1Rate', actual: 0.8, required: 0.8 }]);
|
|
});
|
|
|
|
test('thresholds are pinned and an empty corpus fails closed', () => {
|
|
const { THRESHOLDS, score } = evaluator();
|
|
assert.deepEqual({ ...THRESHOLDS }, { 'direct.top3Rate': 0.88, 'direct.top1Rate': 0.8 });
|
|
const result = score([]);
|
|
assert.equal(result.status, 'fail');
|
|
assert.deepEqual(result.breaches.map(breach => [breach.metric, breach.actual]), [['direct.top3Rate', null], ['direct.top1Rate', null]]);
|
|
});
|
|
|
|
test('--baseline prints a per-metric delta against a previous receipt', () => {
|
|
const current = evaluator().score(healthyRows());
|
|
const previous = JSON.parse(JSON.stringify(current));
|
|
previous.sets.direct.top1Rate = 1;
|
|
const lines = evaluator().baselineDelta(current, previous);
|
|
assert.ok(lines.some(line => /direct\.top1Rate/.test(line) && /-0\.038/.test(line)));
|
|
assert.ok(lines.some(line => /direct\.top3Rate/.test(line) && /\+0\.000/.test(line)));
|
|
});
|
|
|
|
test('the CLI emits JSON, applies a baseline and exits by gate status without re-running the corpus', () => {
|
|
const output = [];
|
|
const io = { log: text => output.push(text), error: text => output.push(text), readFile: () => JSON.stringify(evaluator().score(healthyRows())) };
|
|
const passing = { evaluate: () => evaluator().score(healthyRows()) };
|
|
assert.equal(evaluator().main(['--json'], { ...passing, ...io }), 0);
|
|
assert.equal(JSON.parse(output.at(-1)).status, 'pass');
|
|
output.length = 0;
|
|
assert.equal(evaluator().main(['--baseline', 'previous.json'], { ...passing, ...io }), 0);
|
|
assert.ok(output.some(line => /direct\.top1Rate/.test(line)));
|
|
const failing = { evaluate: () => evaluator().score(healthyRows().map(entry => ({ ...entry, selectedIds: ['skill:q'] }))) };
|
|
assert.equal(evaluator().main([], { ...failing, ...io }), 1);
|
|
assert.equal(evaluator().main(['--write'], { ...passing, ...io }), 1);
|
|
assert.equal(evaluator().main(['--baseline'], { ...passing, ...io }), 1);
|
|
});
|
|
|
|
test('the CLI reads the baseline before the corpus run and keeps --json stdout parseable', () => {
|
|
const stdout = [];
|
|
const stderr = [];
|
|
let runs = 0;
|
|
const io = { log: text => stdout.push(text), error: text => stderr.push(text),
|
|
evaluate: () => { runs += 1; return evaluator().score(healthyRows()); } };
|
|
const receipt = JSON.stringify(evaluator().score(healthyRows()));
|
|
assert.equal(evaluator().main(['--json', '--baseline', 'previous.json'], { ...io, readFile: () => receipt }), 0);
|
|
assert.equal(JSON.parse(stdout.join('\n')).status, 'pass');
|
|
assert.ok(stderr.some(line => /direct\.top1Rate/.test(line)));
|
|
runs = 0;
|
|
assert.equal(evaluator().main(['--baseline', 'missing.json'], { ...io, readFile: () => { throw new Error('ENOENT'); } }), 1);
|
|
assert.equal(evaluator().main(['--baseline', 'bad.json'], { ...io, readFile: () => '{bad' }), 1);
|
|
assert.equal(evaluator().main(['--baseline', 'null.json'], { ...io, readFile: () => 'null' }), 1);
|
|
assert.equal(runs, 0);
|
|
assert.equal(evaluator().main([], { ...io, evaluate: () => { throw new Error('boom'); } }), 1);
|
|
assert.ok(stderr.includes('Routing quality evaluation failed: boom'));
|
|
});
|
|
|
|
test('a baseline that is not a routing-quality receipt fails before the corpus run', () => {
|
|
let runs = 0;
|
|
const io = { log: () => {}, error: () => {}, evaluate: () => { runs += 1; return evaluator().score(healthyRows()); } };
|
|
for (const text of ['{}', '[]', '{"schemaVersion":"other.v1","sets":{"direct":{},"paraphrased":{}}}', '{"schemaVersion":"ecc.routing-quality.v1"}']) {
|
|
assert.equal(evaluator().main(['--baseline', 'previous.json'], { ...io, readFile: () => text }), 1, text);
|
|
}
|
|
assert.equal(runs, 0);
|
|
const nullRates = JSON.stringify(evaluator().score([]));
|
|
assert.equal(evaluator().main(['--baseline', 'previous.json'], { ...io, readFile: () => nullRates }), 0);
|
|
});
|
|
|
|
test('an incomplete corpus fails before any prompt is resolved', () => {
|
|
const fs = require('node:fs');
|
|
const os = require('node:os');
|
|
const path = require('node:path');
|
|
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-routing-corpus-'));
|
|
const prompt = { query: 'review this pull request', expected: ['skill:a'] };
|
|
try {
|
|
for (const corpus of [{}, { paraphrased: [prompt] }, { direct: [], paraphrased: [] }, { direct: [prompt], paraphrased: [] },
|
|
{ direct: [{ query: 'x' }], paraphrased: [prompt] }, { direct: [{ query: 42, expected: ['skill:a'] }], paraphrased: [prompt] },
|
|
{ direct: [{ query: 'x', expected: [] }], paraphrased: [prompt] }]) {
|
|
const corpusPath = path.join(directory, 'corpus.json');
|
|
fs.writeFileSync(corpusPath, JSON.stringify(corpus));
|
|
const errors = [];
|
|
assert.equal(evaluator().main([], { log: () => {}, error: text => errors.push(text), evaluate: () => evaluator().evaluate({ corpusPath }) }), 1, JSON.stringify(corpus));
|
|
assert.match(errors.join('\n'), /routing corpus/);
|
|
}
|
|
} finally {
|
|
fs.rmSync(directory, { recursive: true, force: true, maxRetries: 3 });
|
|
}
|
|
});
|
|
|
|
test('the gate has an npm entry point but stays out of the default test chain', () => {
|
|
const { scripts } = require('../../package.json');
|
|
assert.equal(scripts['routing:check'], 'node scripts/ci/evaluate-context-routing.js');
|
|
assert.ok(!scripts.test.includes('evaluate-context-routing'));
|
|
assert.ok(!scripts.test.includes('routing:check'));
|
|
});
|