From e9fe714e2420c7fdf0c14db440a2ee753c0c2ebd Mon Sep 17 00:00:00 2001 From: montjeffrey Date: Thu, 8 Oct 2026 15:50:43 +0000 Subject: [PATCH] test(context): gate routing quality on the shared corpus pass --- tests/lib/context-selection.test.js | 14 +- .../scripts/evaluate-context-routing.test.js | 140 ++++++++++++++++++ 2 files changed, 146 insertions(+), 8 deletions(-) create mode 100644 tests/scripts/evaluate-context-routing.test.js diff --git a/tests/lib/context-selection.test.js b/tests/lib/context-selection.test.js index 53755c62e..73f680a85 100644 --- a/tests/lib/context-selection.test.js +++ b/tests/lib/context-selection.test.js @@ -320,14 +320,12 @@ test('actual registry: function-word overlap alone never auto-selects a skill', assert.equal(result.fallback, null); }); -test('actual registry: auto selection loads only expected skills across the routing prompt corpus', () => { - const corpus = require('../fixtures/context-selection/routing-prompts.json'); - const wrong = []; - for (const [index, { query, expected }] of [...corpus.direct, ...corpus.paraphrased].entries()) { - const result = resolveTaskContext({ task: task({ taskId: `corpus-${index}`, query }), load: false }); - if (result.selectedIds.some(id => !expected.includes(id))) wrong.push(`${query} -> ${result.selectedIds.join(', ')}`); - } - assert.deepEqual(wrong, []); +test('actual registry: the routing corpus clears every quality gate with no wrong loads', () => { + const result = require('../../scripts/ci/evaluate-context-routing').evaluate(); + assert.deepEqual(result.wrongLoads, []); + assert.deepEqual(result.breaches, []); + assert.equal(result.status, 'pass'); + assert.equal(result.sets.direct.prompts + result.sets.paraphrased.prompts, 77); }); test('actual registry: a simple factual question needs no context', () => { diff --git a/tests/scripts/evaluate-context-routing.test.js b/tests/scripts/evaluate-context-routing.test.js new file mode 100644 index 000000000..b1ecefe05 --- /dev/null +++ b/tests/scripts/evaluate-context-routing.test.js @@ -0,0 +1,140 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const test = require('node:test'); + +const evaluator = () => require('../../scripts/ci/evaluate-context-routing'); + +const row = (set, expected, candidateIds, selectedIds = []) => ({ set, query: `${set} ${candidateIds.join(' ')}`, expected, candidateIds, selectedIds }); + +function healthyRows() { + const rows = []; + for (let index = 0; index < 50; index++) rows.push(row('direct', ['skill:a'], ['skill:a', 'skill:b', 'skill:c'])); + rows.push(row('direct', ['skill:a'], ['skill:b', 'skill:a', 'skill:c'])); + rows.push(row('direct', ['skill:a'], ['skill:b', 'skill:c', 'skill:d'])); + for (let index = 0; index < 25; index++) rows.push(row('paraphrased', ['skill:a'], ['skill:z'])); + return rows; +} + +test('healthy rows pass every gate and report per-set rates', () => { + const result = evaluator().score(healthyRows()); + assert.equal(result.status, 'pass'); + assert.deepEqual(result.breaches, []); + assert.equal(result.sets.direct.prompts, 52); + assert.equal(result.sets.direct.top1, 50); + assert.equal(result.sets.direct.top3, 51); + assert.equal(result.sets.paraphrased.top3Rate, 0); +}); + +test('a planted ranking regression fails and names the breached threshold', () => { + const rows = healthyRows().map((entry, index) => (entry.set === 'direct' && index < 10 + ? { ...entry, candidateIds: ['skill:x', 'skill:y', 'skill:z'] } : entry)); + const result = evaluator().score(rows); + assert.equal(result.status, 'fail'); + assert.ok(result.breaches.some(breach => breach.metric === 'direct.top3Rate')); + assert.ok(result.breaches.some(breach => breach.metric === 'direct.top1Rate')); +}); + +test('one wrong implicit load fails regardless of ranking quality', () => { + const rows = healthyRows(); + rows[0] = { ...rows[0], selectedIds: ['skill:b'] }; + const result = evaluator().score(rows); + assert.equal(result.status, 'fail'); + assert.deepEqual(result.breaches.map(breach => breach.metric), ['direct.wrongLoads']); + assert.equal(result.wrongLoads.length, 1); + assert.equal(result.sets.direct.admissionPrecision, 0); +}); + +test('paraphrased ranking is reported, never gated; its wrong loads still fail', () => { + const rows = healthyRows(); + assert.equal(evaluator().score(rows).status, 'pass'); + const loaded = rows.map(entry => (entry.set === 'paraphrased' ? { ...entry, selectedIds: ['skill:z'] } : entry)); + const result = evaluator().score(loaded); + assert.deepEqual(result.breaches.map(breach => breach.metric), ['paraphrased.wrongLoads']); +}); + +test('no implicit loads leaves admission precision undefined instead of perfect', () => { + assert.equal(evaluator().score(healthyRows()).sets.direct.admissionPrecision, null); +}); + +/** Direct rows whose expected skill sits at ranks[index]; rows past ranks.length miss it. */ +function rankedRows(total, ranks) { + return Array.from({ length: total }, (_, index) => { + const candidateIds = ['skill:w', 'skill:x', 'skill:y', 'skill:z']; + if (index < ranks.length) candidateIds[ranks[index]] = 'skill:a'; + return row('direct', ['skill:a'], candidateIds); + }); +} + +test('top-3 counts a hit at the third rank but not the fourth', () => { + const { direct } = evaluator().score(rankedRows(4, [0, 2, 3])).sets; + assert.equal(direct.top1, 1); + assert.equal(direct.top3, 2); +}); + +test('thresholds compare unrounded ratios and pass exactly at 0.88 and 0.80', () => { + const { score } = evaluator(); + const atThreshold = score(rankedRows(25, [...Array(20).fill(0), 1, 1])); + assert.equal(atThreshold.sets.direct.top1Rate, 0.8); + assert.equal(atThreshold.sets.direct.top3Rate, 0.88); + assert.deepEqual(atThreshold.breaches, []); + const roundsUp = score(rankedRows(2500, [...Array(1999).fill(0), ...Array(501).fill(1)])); + assert.deepEqual(roundsUp.breaches, [{ metric: 'direct.top1Rate', actual: 0.8, required: 0.8 }]); +}); + +test('thresholds are pinned and an empty corpus fails closed', () => { + const { THRESHOLDS, score } = evaluator(); + assert.deepEqual({ ...THRESHOLDS }, { 'direct.top3Rate': 0.88, 'direct.top1Rate': 0.8 }); + const result = score([]); + assert.equal(result.status, 'fail'); + assert.deepEqual(result.breaches.map(breach => [breach.metric, breach.actual]), [['direct.top3Rate', null], ['direct.top1Rate', null]]); +}); + +test('--baseline prints a per-metric delta against a previous receipt', () => { + const current = evaluator().score(healthyRows()); + const previous = JSON.parse(JSON.stringify(current)); + previous.sets.direct.top1Rate = 1; + const lines = evaluator().baselineDelta(current, previous); + assert.ok(lines.some(line => /direct\.top1Rate/.test(line) && /-0\.038/.test(line))); + assert.ok(lines.some(line => /direct\.top3Rate/.test(line) && /\+0\.000/.test(line))); +}); + +test('the CLI emits JSON, applies a baseline and exits by gate status without re-running the corpus', () => { + const output = []; + const io = { log: text => output.push(text), error: text => output.push(text), readFile: () => JSON.stringify(evaluator().score(healthyRows())) }; + const passing = { evaluate: () => evaluator().score(healthyRows()) }; + assert.equal(evaluator().main(['--json'], { ...passing, ...io }), 0); + assert.equal(JSON.parse(output.at(-1)).status, 'pass'); + output.length = 0; + assert.equal(evaluator().main(['--baseline', 'previous.json'], { ...passing, ...io }), 0); + assert.ok(output.some(line => /direct\.top1Rate/.test(line))); + const failing = { evaluate: () => evaluator().score(healthyRows().map(entry => ({ ...entry, selectedIds: ['skill:q'] }))) }; + assert.equal(evaluator().main([], { ...failing, ...io }), 1); + assert.equal(evaluator().main(['--write'], { ...passing, ...io }), 1); + assert.equal(evaluator().main(['--baseline'], { ...passing, ...io }), 1); +}); + +test('the CLI reads the baseline before the corpus run and keeps --json stdout parseable', () => { + const stdout = []; + const stderr = []; + let runs = 0; + const io = { log: text => stdout.push(text), error: text => stderr.push(text), + evaluate: () => { runs += 1; return evaluator().score(healthyRows()); } }; + const receipt = JSON.stringify(evaluator().score(healthyRows())); + assert.equal(evaluator().main(['--json', '--baseline', 'previous.json'], { ...io, readFile: () => receipt }), 0); + assert.equal(JSON.parse(stdout.join('\n')).status, 'pass'); + assert.ok(stderr.some(line => /direct\.top1Rate/.test(line))); + runs = 0; + assert.equal(evaluator().main(['--baseline', 'missing.json'], { ...io, readFile: () => { throw new Error('ENOENT'); } }), 1); + assert.equal(evaluator().main(['--baseline', 'bad.json'], { ...io, readFile: () => '{bad' }), 1); + assert.equal(evaluator().main(['--baseline', 'null.json'], { ...io, readFile: () => 'null' }), 1); + assert.equal(runs, 0); + assert.equal(evaluator().main([], { ...io, evaluate: () => { throw new Error('boom'); } }), 1); + assert.ok(stderr.includes('Routing quality evaluation failed: boom')); +}); + +test('the gate has an npm entry point but stays out of the default test chain', () => { + const { scripts } = require('../../package.json'); + assert.equal(scripts['routing:check'], 'node scripts/ci/evaluate-context-routing.js'); + assert.ok(!scripts.test.includes('evaluate-context-routing')); +});