mirror of
https://github.com/affaan-m/ECC.git
synced 2026-11-17 13:32:18 +00:00
feat(context): gate routing quality in CI
This commit is contained in:
1 parent
e9fe714e24
commit
2ac828c955
3 files changed
+148
No files matched your search
@@ -142,3 +142,12 @@ Independent review reproduced literal loads hidden after object-literal division
|
||||
The maintained parser in the existing pinned TypeScript development dependency now reads source without evaluating it. The gate traverses literal `require`, native `module.require`, dynamic `import`, static imports/re-exports, and literal `path.join(__dirname, ...)` calls. Existing regex, template, escape and nested-dynamic cases remain covered. Parse diagnostics fail the projection closed. ESM URL queries and fragments identify the same planned file; encoded separators and malformed escapes fail rather than normalize into a safe path.
|
||||
|
||||
Focused validation passes 67 cases under Node 24.9.0 and Node 18.20.8. The actual CLI projects the current 304-skill registry into 10 profile/layout combinations, with 3,016 planned files, 65 JavaScript scripts and 35/35 literal relative specifiers resolved, zero escapes and zero warnings. Script execution remains deferred. Dynamic expressions are reported as warnings; arbitrary loader aliases, shadowed bindings and provider-native runtime behavior remain outside this literal-dependency gate.
|
||||
|
||||
## Routing-quality gate
|
||||
|
||||
The 77-prompt routing corpus from #2945 now gates ranking quality as well as wrong loads. Before this, the corpus test asserted only wrong loads, the other real-registry checks pinned individual prompts, and the evaluation pilot needs an opt-in model provider (in CI it runs on a synthetic fixture and reports `insufficient-sample`). Nothing scored top-1 or top-3 across the corpus, so a ranking regression there passed unnoticed.
|
||||
|
||||
- RED: 13 cases fail before the evaluator and its npm script exist, the 12 scorer and CLI cases plus the replaced corpus test. GREEN: the 12 cases take milliseconds each against planted rows; a planted ten-prompt ranking regression breaches both direct thresholds, one wrong load fails on its own, and paraphrased ranking never gates while paraphrased wrong loads do. Thresholds compare unrounded ratios, so exactly 0.88 and 0.80 pass and 1,999/2,500 does not; a hit at the fourth rank is not a top-3 hit; a missing or malformed `--baseline` fails before the corpus run.
|
||||
- Baseline on Node 24.13.0, Windows x64, registry digest `6f213dc84593...` (314 skills, routing policy v5): direct top-1 44/52 (0.846), top-3 48/52 (0.923), 17 implicit loads all correct; paraphrased top-1 2/25, top-3 3/25, no implicit loads. The first build (Node 22.22.0, 293 skills, before the v5 anchor) measured the same direct counts, 19 implicit loads and paraphrased top-3 4/25.
|
||||
- Thresholds: direct top-3 at least 0.88 (46/52) and top-1 at least 0.80 (42/52), two prompts of slack each; zero wrong loads in both sets. Twenty-one skills and a trigger manifest update landed between the two builds without moving any direct count, so routine skill additions are not expected to trip the gate.
|
||||
- One resolver pass replaces the earlier wrong-load test instead of adding a second. Instrumented runs of the old and new corpus tests each make 77 `resolveTaskContext` calls and 77 registry loads; the registry digest is read from the resolver's receipt, not a separate load. The first build, on Node 22.22.0, measured the test file at 19.0 s against 17.6 s (about 230 ms per resolve). Each push runs this file six times (five full test cells and the coverage job), so a separate pass would have added about two CPU-minutes per push. `npm run routing:check` runs the gate locally and stays out of the default `npm test` chain for the same reason. It accepts `--json` and `--baseline <receipt>`; with `--json`, the baseline delta goes to stderr so stdout stays one JSON document.
|
||||
@@ -509,6 +509,7 @@
|
||||
"catalog:check": "node scripts/ci/catalog.js --text",
|
||||
"context-profiles:check": "node scripts/ci/validate-context-profiles.js",
|
||||
"carrier-closure:check": "node scripts/ci/validate-carrier-closure.js",
|
||||
"routing:check": "node scripts/ci/evaluate-context-routing.js",
|
||||
"catalog:sync": "node scripts/ci/catalog.js --write --text",
|
||||
"command-registry:generate": "node scripts/ci/generate-command-registry.js",
|
||||
"command-registry:write": "node scripts/ci/generate-command-registry.js --write",
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Routing-quality gate over the routing prompt corpus from community PR #2945.
|
||||
*
|
||||
* Wrong implicit loads fail in both sets. Ranking is gated on the direct set
|
||||
* only; the paraphrased set is reported so it can improve without blocking.
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const { resolveTaskContext } = require('../lib/context-selection');
|
||||
const { DEFAULT_REPO_ROOT } = require('../lib/context-profile-support');
|
||||
|
||||
const CORPUS = path.join(DEFAULT_REPO_ROOT, 'tests/fixtures/context-selection/routing-prompts.json');
|
||||
const SETS = Object.freeze(['direct', 'paraphrased']);
|
||||
const THRESHOLDS = Object.freeze({ 'direct.top3Rate': 0.88, 'direct.top1Rate': 0.80 });
|
||||
const METRICS = Object.freeze(['top1Rate', 'top3Rate', 'admissionPrecision']);
|
||||
|
||||
const rate = (count, total) => (total ? Number((count / total).toFixed(3)) : null);
|
||||
|
||||
function score(rows) {
|
||||
const sets = {};
|
||||
const wrongLoads = [];
|
||||
for (const set of SETS) {
|
||||
const entries = rows.filter(entry => entry.set === set);
|
||||
let top1 = 0;
|
||||
let top3 = 0;
|
||||
let admitted = 0;
|
||||
let wrong = 0;
|
||||
for (const entry of entries) {
|
||||
if (entry.expected.includes(entry.candidateIds[0])) top1 += 1;
|
||||
if (entry.candidateIds.slice(0, 3).some(id => entry.expected.includes(id))) top3 += 1;
|
||||
if (!entry.selectedIds.length) continue;
|
||||
admitted += 1;
|
||||
if (entry.selectedIds.some(id => !entry.expected.includes(id))) {
|
||||
wrong += 1;
|
||||
wrongLoads.push({ set, query: entry.query, selectedIds: entry.selectedIds, expected: entry.expected });
|
||||
}
|
||||
}
|
||||
sets[set] = { prompts: entries.length, top1, top3, admitted, wrongLoads: wrong,
|
||||
top1Rate: rate(top1, entries.length), top3Rate: rate(top3, entries.length),
|
||||
admissionPrecision: rate(admitted - wrong, admitted) };
|
||||
}
|
||||
const breaches = [];
|
||||
for (const set of SETS) {
|
||||
if (sets[set].wrongLoads) breaches.push({ metric: `${set}.wrongLoads`, actual: sets[set].wrongLoads, required: 0 });
|
||||
}
|
||||
for (const [metric, required] of Object.entries(THRESHOLDS)) {
|
||||
const [set, key] = metric.split('.');
|
||||
const { prompts } = sets[set];
|
||||
// Gate on the unrounded ratio; the rounded rate is for display only.
|
||||
const ratio = prompts ? sets[set][key.replace(/Rate$/, '')] / prompts : null;
|
||||
if (ratio === null || ratio < required) breaches.push({ metric, actual: sets[set][key], required });
|
||||
}
|
||||
return { schemaVersion: 'ecc.routing-quality.v1', status: breaches.length ? 'fail' : 'pass',
|
||||
thresholds: { ...THRESHOLDS, wrongLoads: 0 }, sets, breaches, wrongLoads };
|
||||
}
|
||||
|
||||
/** One resolver pass over the corpus; every prompt is a fresh task in auto mode.
|
||||
* The registry digest comes from the resolver's own receipt, so the gate adds no registry load. */
|
||||
function evaluate({ repoRoot = DEFAULT_REPO_ROOT, corpusPath = CORPUS } = {}) {
|
||||
const corpus = JSON.parse(fs.readFileSync(corpusPath, 'utf8'));
|
||||
const rows = [];
|
||||
let registryDigest = null;
|
||||
for (const set of SETS) {
|
||||
for (const [index, { query, expected }] of corpus[set].entries()) {
|
||||
const result = resolveTaskContext({ repoRoot, selectionMode: 'auto', load: false,
|
||||
task: { sessionId: 'routing-eval', taskId: `${set}-${index}`, revision: 1, phase: 'evaluate', query } });
|
||||
registryDigest = registryDigest || result.receipt.registryDigest;
|
||||
rows.push({ set, query, expected, candidateIds: result.candidates.map(candidate => candidate.id), selectedIds: result.selectedIds });
|
||||
}
|
||||
}
|
||||
return { ...score(rows), registryDigest, node: process.version };
|
||||
}
|
||||
|
||||
function baselineDelta(current, previous) {
|
||||
const lines = [];
|
||||
for (const set of SETS) {
|
||||
for (const key of METRICS) {
|
||||
const now = current.sets[set][key];
|
||||
const before = previous.sets?.[set]?.[key];
|
||||
if (typeof now !== 'number' || typeof before !== 'number') {
|
||||
lines.push(`${set}.${key}: ${before ?? 'n/a'} -> ${now ?? 'n/a'}`);
|
||||
continue;
|
||||
}
|
||||
const change = now - before;
|
||||
lines.push(`${set}.${key}: ${before.toFixed(3)} -> ${now.toFixed(3)} (${change < 0 ? '-' : '+'}${Math.abs(change).toFixed(3)})`);
|
||||
}
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
function summary(result) {
|
||||
const { direct, paraphrased } = result.sets;
|
||||
return `Routing quality ${result.status}: direct top-1 ${direct.top1}/${direct.prompts}, top-3 ${direct.top3}/${direct.prompts}, `
|
||||
+ `${direct.admitted} implicit loads; paraphrased top-3 ${paraphrased.top3}/${paraphrased.prompts} (reported, not gated); `
|
||||
+ `${result.wrongLoads.length} wrong loads.`;
|
||||
}
|
||||
|
||||
/** Reads a previous receipt before the corpus run, so a bad path or file fails fast. */
|
||||
function readBaseline(io, file) {
|
||||
if (!file) return null;
|
||||
const previous = JSON.parse(io.readFile(file));
|
||||
if (previous === null || typeof previous !== 'object') throw new Error(`${file} is not a routing-quality receipt`);
|
||||
return previous;
|
||||
}
|
||||
|
||||
function main(args = process.argv.slice(2), dependencies = {}) {
|
||||
const io = { evaluate, log: console.log, error: console.error, readFile: file => fs.readFileSync(file, 'utf8'), ...dependencies };
|
||||
try {
|
||||
let json = false;
|
||||
let baseline = null;
|
||||
for (let index = 0; index < args.length; index++) {
|
||||
if (args[index] === '--json') json = true;
|
||||
else if (args[index] === '--baseline' && args[index + 1] && !args[index + 1].startsWith('--')) baseline = args[++index];
|
||||
else throw new Error(args[index] === '--baseline' ? '--baseline requires a receipt path' : `Unknown argument: ${args[index]}`);
|
||||
}
|
||||
const previous = readBaseline(io, baseline);
|
||||
const result = io.evaluate();
|
||||
if (json) io.log(JSON.stringify(result, null, 2));
|
||||
else {
|
||||
io.log(summary(result));
|
||||
for (const breach of result.breaches) io.error(`Gate breached: ${breach.metric} is ${breach.actual}, requires ${breach.metric.endsWith('wrongLoads') ? '0' : `>= ${breach.required}`}`);
|
||||
for (const wrong of result.wrongLoads) io.error(`Wrong load (${wrong.set}): "${wrong.query}" -> ${wrong.selectedIds.join(', ')}`);
|
||||
}
|
||||
// With --json the delta goes to stderr so stdout stays a single JSON document.
|
||||
if (previous) for (const line of baselineDelta(result, previous)) (json ? io.error : io.log)(line);
|
||||
return result.status === 'pass' ? 0 : 1;
|
||||
} catch (error) {
|
||||
io.error(`Routing quality evaluation failed: ${error.message}`);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (require.main === module) process.exitCode = main();
|
||||
module.exports = { THRESHOLDS, baselineDelta, evaluate, main, score };
|
||||
Reference in new issue
Block a user