Merge #632: reject duplicate behavioral evaluation IDs

scripts/run-evals-test.js

@@ -642,3 +642,31 @@ test('fixture baseline commits ignore inherited user signing requirements', () =
     fs.rmSync(temp, { recursive: true, force: true });
   }
 });
+
+test('rejects duplicate behavioral IDs before they can overwrite a grading slot', () => {
+  const root = makeSandbox();
+  try {
+    writeSkill(root, 'alpha-skill', 'Handles alpha widgets. Use when changing alpha widgets.');
+    const evalCase = completeCase('alpha-skill', 'change alpha widget');
+    evalCase.evals.push({ ...behavioralEval(), prompt: 'A different evaluation' });
+    writeJson(path.join(root, 'evals', 'cases', 'alpha-skill.json'), evalCase);
+    const result = run(root);
+    assert.equal(result.status, 1, result.stdout + result.stderr);
+    assert.match(result.stdout, /duplicate behavioral eval id=1/);
+    const dryRun = run(root, ['--behavioral', 'alpha-skill', '--dry-run']);
+    assert.equal(dryRun.status, 1, dryRun.stdout + dryRun.stderr);
+    assert.match(dryRun.stderr, /duplicate behavioral eval id=1/);
+  } finally { fs.rmSync(root, { recursive: true, force: true }); }
+});
+
+test('accepts distinct behavioral IDs including zero', () => {
+  const root = makeSandbox();
+  try {
+    writeSkill(root, 'alpha-skill', 'Handles alpha widgets. Use when changing alpha widgets.');
+    const evalCase = completeCase('alpha-skill', 'change alpha widget');
+    evalCase.evals.push({ ...behavioralEval(), id: 0 });
+    writeJson(path.join(root, 'evals', 'cases', 'alpha-skill.json'), evalCase);
+    const result = run(root);
+    assert.equal(result.status, 0, result.stdout + result.stderr);
+  } finally { fs.rmSync(root, { recursive: true, force: true }); }
+});

scripts/run-evals.js

@@ -193,6 +193,16 @@ function resolveFixturePath(root, rel) {
   return resolvedPath;
 }
 
+function duplicateEvalIds(evals) {
+  const seen = new Set();
+  const duplicates = new Set();
+  for (const ev of evals) {
+    if (seen.has(ev.id)) duplicates.add(ev.id);
+    seen.add(ev.id);
+  }
+  return [...duplicates];
+}
+
 // ---------- tier 2 ----------
 
 function runDeterministic(minRank1) {
@@ -235,6 +245,11 @@ function runDeterministic(minRank1) {
       continue;
     }
 
+    for (const id of duplicateEvalIds(d.evals || [])) {
+      console.log(`  ✗  ${c.file}: duplicate behavioral eval id=${id}`);
+      errors++;
+    }
+
     // Schema: behavioral evals (skill-creator evals.json shape)
     for (const ev of d.evals || []) {
       const kind = ev.kind || 'execution';
@@ -523,6 +538,11 @@ function runBehavioral(skillName, dryRun) {
     console.error(`"${skillName}" has no behavioral evals`);
     process.exit(1);
   }
+  const duplicateIds = duplicateEvalIds(d.evals);
+  if (duplicateIds.length) {
+    console.error(`"${skillName}" has duplicate behavioral eval id=${duplicateIds.join(', ')}; each grading slot needs a unique id`);
+    process.exit(1);
+  }
   if (!dryRun) fs.mkdirSync(RESULTS_DIR, { recursive: true });
   let failures = 0;