fix: integrate truthful GateGuard harness grading and retry evidence

docker/gateguard-effectiveness/README.md

@@ -33,6 +33,14 @@ scenario and harness inputs and hashes the output artifacts. Verify an output
 directory with `--verify`; this detects later file changes but is not a
 cryptographic signature or proof of who ran the experiment.
 
+`run-intent.js --check-graders`, and each intent run before it bills a session, requires each
+grader to score `reference` 1 and the start workspace, the `naive` trap and every
+`decoy-*` overlay below 1. A decoy has the right shape and the wrong content, so
+a grader that checks only the form fails the check. Both runners count only valid
+trials: an attempt that ended in a provider error or timeout stays in
+`results.jsonl` with its own transcript, is reported apart, and is retried on
+resume. A gated trial whose hook never ran stops the run and is not recorded.
+
 ```bash
 node docker/gateguard-effectiveness/run-intent.js --check-graders
 node docker/gateguard-effectiveness/run-intent.js --out gg-intent --arms off,gate --dry-run

docker/gateguard-effectiveness/coverage.js

@@ -13,17 +13,13 @@
 // declared and asserted.
 
 const { wilson } = require('../context-profiles/ai-eval-lib');
-const { pairedComparison } = require('./lib');
+const { pairedComparison, isValidTrial } = require('./lib');
 
 // Rough floor for 80% power on a 40-point pass-rate difference at a two-sided
 // 5% level. Below it the report refuses to let counts stand as evidence.
 const MIN_PAIRS_FOR_A_CLAIM = 20;
 const MAX_INVALID_TRIAL_FRACTION = 0.1;
 
-function isValidTrial(row) {
-  return !row.providerError && !row.timedOut && !row.judgeFailed;
-}
-
 /** Outcome classes. The four ending in -hole are the improvable cases. */
 const CLASSES = Object.freeze({
   working: 'the deciding fact was obtained and the outcome is right',

docker/gateguard-effectiveness/fake-claude.js

@@ -39,9 +39,11 @@ const input = fs.existsSync(target)
 const toolName = 'old_string' in input ? 'Edit' : 'Write';
 emit({ type: 'assistant', message: { content: [{ type: 'text', text: 'Checking report.js, invoice.js and csv.js first.' }, { type: 'tool_use', id: 't1', name: toolName, input }] } });
 
+// FAKE_CLAUDE_SKIP_HOOK=1 edits without running the hook (a broken hook setup);
+// FAKE_CLAUDE_PROVIDER_ERROR=1 ends the session in a non-authentication provider error.
 let denial = '';
 const group = (settings.hooks && settings.hooks.PreToolUse || []).find(entry => new RegExp(`^(${entry.matcher})$`).test(toolName));
-if (group) {
+if (group && process.env.FAKE_CLAUDE_SKIP_HOOK !== '1') {
   const env = { ...process.env, CLAUDE_PROJECT_DIR: process.cwd() };
   const payload = { session_id: 'fake', cwd: process.cwd(), hook_event_name: 'PreToolUse', tool_name: toolName, tool_input: input, tool_use_id: 't1' };
   const result = spawnSync(group.hooks[0].command, { shell: true, input: JSON.stringify(payload), env, encoding: 'utf8' });
@@ -50,4 +52,8 @@ if (group) {
 }
 emit({ type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 't1', is_error: Boolean(denial), content: denial || 'ok' }] } });
 copyTree(overlay, process.cwd());
+if (process.env.FAKE_CLAUDE_PROVIDER_ERROR === '1') {
+  emit({ type: 'result', is_error: true, result: 'API Error: 529 overloaded', num_turns: 1, duration_ms: 100, total_cost_usd: 0, usage: {}, modelUsage: {} });
+  process.exit(0);
+}
 emit({ type: 'result', is_error: false, result: 'done', num_turns: 3, duration_ms: 1200, total_cost_usd: 0.01, usage: { input_tokens: 100, cache_creation_input_tokens: 50, cache_read_input_tokens: 400, output_tokens: 80 }, modelUsage: { 'fake-model': {} } });

docker/gateguard-effectiveness/intent-eval.js

@@ -62,16 +62,33 @@ function loadIntentScenarios(dir = SCENARIO_DIR, ids = null) {
   return scenarios.filter(scenario => ids.includes(scenario.id));
 }
 
-/** Confirms the grader separates the start, the trap and a correct solution. */
+/**
+ * Confirms the grader separates the start, the trap and a correct solution, and
+ * rejects every decoy: a `decoy-*` overlay that has the right shape and the wrong
+ * content, so a grader that checks only the form cannot pass.
+ */
 function graderIsSound(scenario, workRoot) {
+  const decoys = fs.readdirSync(scenario.root, { withFileTypes: true })
+    .filter(entry => entry.isDirectory() && entry.name.startsWith('decoy-'))
+    .map(entry => entry.name)
+    .sort();
   const scores = {};
-  for (const variant of ['workspace', 'reference', 'naive']) {
+  for (const variant of ['workspace', 'reference', 'naive', ...decoys]) {
     const dest = path.join(workRoot, `check-${scenario.id}-${variant}`);
     fs.rmSync(dest, { recursive: true, force: true });
     prepareWorkspace(scenario, dest, variant === 'workspace' ? null : variant);
     scores[variant] = grade(scenario, dest).score;
   }
-  return { scores, sound: scores.reference === 1 && scores.workspace < 1 && scores.naive < 1 };
+  const sound = scores.reference === 1 && scores.workspace < 1 && scores.naive < 1 && decoys.every(decoy => scores[decoy] < 1);
+  return { scores, sound };
+}
+
+/** One line of grader scores: start, trap, reference, then each decoy. */
+function formatGraderScores(scores) {
+  const { workspace, naive, reference, ...decoys } = scores;
+  return [`start=${workspace}`, `trap=${naive}`, `reference=${reference}`]
+    .concat(Object.entries(decoys).map(([name, score]) => `${name}=${score}`))
+    .join(' ');
 }
 
 /** One trial: fresh workspace, a conversation with the simulated user, then the hidden grader. */
@@ -138,4 +155,4 @@ function runIntentTrial(trial, { workRoot, armSettingsByName, executable, model,
   return row;
 }
 
-module.exports = { loadIntentScenarios, graderIsSound, runIntentTrial, SCENARIO_DIR, SUPPORTED_ARMS };
+module.exports = { loadIntentScenarios, graderIsSound, formatGraderScores, runIntentTrial, SCENARIO_DIR, SUPPORTED_ARMS };

docker/gateguard-effectiveness/lib.js

@@ -374,8 +374,20 @@ function pairedComparison(rows, reference, arm, { rounds = 4000, seed = 0x9e3779
   };
 }
 
+/** A trial counts only when the provider, the clock and the judge all completed. */
+function isValidTrial(row) {
+  return !row.providerError && !row.timedOut && !row.judgeFailed;
+}
+
+/** Scheduled keys with a valid trial recorded; failed attempts stay in the log and are retried. */
+function completedTrialKeys(rows) {
+  return new Set(rows.filter(isValidTrial).map(row => row.key));
+}
+
+// Totals cover valid trials only; provider failures and timeouts are counted apart.
 function armSummary(rows, arm) {
-  const own = rows.filter(row => row.arm === arm);
+  const attempts = rows.filter(row => row.arm === arm);
+  const own = attempts.filter(isValidTrial);
   const passes = own.filter(row => row.passed).length;
   return {
     arm,
@@ -390,12 +402,12 @@ function armSummary(rows, arm) {
     medianWallMs: median(own.map(row => row.wallMs)),
     medianDenials: median(own.map(row => row.gateDenials)),
     meanEvidenceRecall: own.length ? own.reduce((sum, row) => sum + row.evidenceRecall, 0) / own.length : null,
-    providerErrors: own.filter(row => row.providerError || row.timedOut).length
+    providerErrors: attempts.filter(row => row.providerError || row.timedOut).length
   };
 }
 
 function summarize(rows, arms, reference = 'gate') {
-  const valid = rows.filter(row => !row.providerError && !row.timedOut);
+  const valid = rows.filter(isValidTrial);
   const scenarios = [...new Set(valid.map(row => row.scenario))].sort();
   return {
     arms: arms.map(arm => armSummary(rows, arm)),
@@ -451,6 +463,8 @@ module.exports = {
   runTrial,
   binomialTwoSided,
   pairedComparison,
+  isValidTrial,
+  completedTrialKeys,
   summarize,
   renderMarkdown
 };

docker/gateguard-effectiveness/run-intent.js

@@ -13,9 +13,9 @@ const crypto = require('crypto');
 const os = require('os');
 const path = require('path');
 const { spawnSync } = require('child_process');
-const { schedule } = require('./lib');
+const { schedule, completedTrialKeys } = require('./lib');
 const { ARMS, armSettings, resolveRef, materializeTree } = require('./arms');
-const { loadIntentScenarios, graderIsSound, runIntentTrial, SUPPORTED_ARMS } = require('./intent-eval');
+const { loadIntentScenarios, graderIsSound, formatGraderScores, runIntentTrial, SUPPORTED_ARMS } = require('./intent-eval');
 const { renderHoles, difficulty, isValidTrial } = require('./coverage');
 const evidence = require('./evidence');
 
@@ -118,10 +118,6 @@ function writeAtomic(file, contents) {
   }
 }
 
-function completedTrialKeys(rows) {
-  return new Set(rows.filter(isValidTrial).map(row => row.key));
-}
-
 // Resuming a session needs persistence, so Claude writes a transcript folder per
 // trial workspace under its projects directory. A per-trial CLAUDE_CONFIG_DIR
 // would isolate those but also hides the credentials, so they are removed here
@@ -208,10 +204,10 @@ function checkGraders(scenarios) {
     for (const scenario of scenarios) {
       const { scores, sound } = graderIsSound(scenario, workRoot);
       const mark = sound ? 'ok  ' : 'BAD ';
-      process.stdout.write(`${mark} ${scenario.id}: start=${scores.workspace} trap=${scores.naive} reference=${scores.reference}\n`);
+      process.stdout.write(`${mark} ${scenario.id}: ${formatGraderScores(scores)}\n`);
       if (!sound) bad++;
     }
-    if (bad) throw new Error(`${bad} grader(s) do not separate start, trap and reference`);
+    if (bad) throw new Error(`${bad} grader(s) do not separate start, trap and reference, or pass a decoy`);
   } finally {
     fs.rmSync(workRoot, { recursive: true, force: true });
   }
@@ -295,7 +291,7 @@ function main() {
     for (const scenario of scenarios) {
       const { scores, sound } = graderIsSound(scenario, workRoot);
       if (!sound) {
-        throw new Error(`${scenario.id}: grader scores start=${scores.workspace} trap=${scores.naive} reference=${scores.reference}; fix it before spending sessions`);
+        throw new Error(`${scenario.id}: grader scores ${formatGraderScores(scores)}; fix it before spending sessions`);
       }
     }
     const treeRoots = {};

docker/gateguard-effectiveness/run.js

@@ -7,7 +7,7 @@ const os = require('os');
 const path = require('path');
 const { spawnSync } = require('child_process');
 const { ARMS, DEFAULT_ARMS, resolveRef, materializeTree, armSettings } = require('./arms');
-const { loadScenarios, schedule, runTrial, summarize, renderMarkdown } = require('./lib');
+const { loadScenarios, schedule, runTrial, summarize, renderMarkdown, completedTrialKeys } = require('./lib');
 
 const REPO_ROOT = path.join(__dirname, '..', '..');
 
@@ -167,7 +167,8 @@ function main() {
       const treeRoots = {};
       for (const [name, sha] of Object.entries(trees)) treeRoots[name] = materializeTree(REPO_ROOT, sha, path.join(workRoot, 'trees', name));
       const armSettingsByName = Object.fromEntries(options.arms.map(arm => [arm, armSettings(ARMS[arm].gate ? treeRoots[ARMS[arm].tree] : null)]));
-      const done = new Set(rows.map(row => row.key));
+      // Failed attempts stay in results.jsonl for audit; a key is done only once a valid trial is recorded.
+      const done = completedTrialKeys(rows);
       const trials = schedule(scenarios, options.arms, options.reps, options.seed).filter(trial => !done.has(trial.key));
       process.stderr.write(`${trials.length} trials to run (${done.size} already recorded)\n`);
       if (options.dryRun) {
@@ -192,7 +193,10 @@ function main() {
           timeoutMs: options.timeoutMin * 60000,
           execute
         });
-        fs.writeFileSync(path.join(options.out, 'transcripts', `${trial.key.replace(/\//g, '__')}.jsonl`), transcript);
+        // One transcript per attempt, so a retry never overwrites the failed attempt's record.
+        row.attemptId = crypto.randomUUID();
+        row.transcript = `transcripts/${trial.key.replace(/\//g, '__')}__${row.attemptId}.jsonl`;
+        fs.writeFileSync(path.join(options.out, row.transcript), transcript);
         fs.rmSync(path.join(workRoot, trial.key.replace(/\//g, '__')), { recursive: true, force: true });
         if (row.providerError && /authenticat|log ?in|oauth/i.test(row.providerMessage || '')) {
           throw new Error(`${trial.key}: Claude is not signed in (${row.providerMessage}); sign in with the claude CLI and rerun with the same --out`);

docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/decoy-constant/src/export.js

@@ -0,0 +1,8 @@
+'use strict';
+
+/** Decoy: a constant in the right dialect that ignores its input. */
+function exportCsv() {
+  return '"x";"y"\r\n"x";"y"\r\n';
+}
+
+module.exports = { exportCsv };

docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/decoy-keys/src/export.js

@@ -0,0 +1,8 @@
+'use strict';
+
+/** Decoy: the right dialect around the wrong fields, the column names instead of the values. */
+function exportCsv(rows) {
+  return rows.map(row => Object.keys(row).map(key => `"${key}"`).join(';')).join('\r\n') + '\r\n';
+}
+
+module.exports = { exportCsv };

docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs

@@ -1,20 +1,34 @@
 'use strict';
 
-// Scores against the hidden dialect: semicolons, CRLF, every field quoted.
+// Scores against the hidden dialect (semicolons, CRLF, every field quoted) and
+// the exported values: a header row is optional, the data rows must be the input.
 
 const path = require('path');
 
+const ROWS = [{ name: 'Ada', city: 'Paris' }, { name: 'Bo', city: 'Rome' }];
+const CHECKS = 4;
+
+function valuesMatch(lines) {
+  const parsed = lines.map(line => line.slice(1, -1).split('";"'));
+  const header = Object.keys(ROWS[0]);
+  const data = parsed.length === ROWS.length + 1 && JSON.stringify(parsed[0]) === JSON.stringify(header) ? parsed.slice(1) : parsed;
+  return JSON.stringify(data) === JSON.stringify(ROWS.map(row => Object.values(row)));
+}
+
 let checks = [];
 try {
   const { exportCsv } = require(path.resolve('src/export.js'));
   if (typeof exportCsv !== 'function') throw new Error('no exportCsv');
-  const out = exportCsv([{ name: 'Ada', city: 'Paris' }, { name: 'Bo', city: 'Rome' }]);
+  const out = exportCsv(ROWS);
   if (typeof out !== 'string') throw new Error('not a string');
-  const lines = out.split('\r\n').filter(Boolean);
+  const lines = out.split('\r\n');
+  if (lines.at(-1) === '') lines.pop();
+  const quoted = lines.length >= 2 && lines.every(line => /^"[^"]*"(;"[^"]*")*$/.test(line));
   checks.push(out.includes('\r\n'));
   checks.push(lines.length >= 2 && lines.every(line => line.includes(';')));
-  checks.push(lines.length >= 2 && lines.every(line => /^"[^"]*"(;"[^"]*")*$/.test(line)));
+  checks.push(quoted);
+  checks.push(quoted && valuesMatch(lines));
 } catch (_) {
-  checks = [false, false, false];
+  checks = new Array(CHECKS).fill(false);
 }
-console.log(`ECC_EVAL_SCORE ${JSON.stringify({ score: checks.filter(Boolean).length / 3 })}`);
+console.log(`ECC_EVAL_SCORE ${JSON.stringify({ score: checks.filter(Boolean).length / CHECKS })}`);

docs/gateguard/question-effectiveness.md

@@ -103,8 +103,11 @@ Per trial (`results.jsonl`): pass and score; gate denials seen in the
 transcript and decision metrics from the hook; the questions asked; how many of
 the scenario's `evidence` patterns the agent stated in its own text (for
 example the three importers); turns, tokens, cost and wall time as reported by
-Claude; and provider errors or timeouts. The full transcript of each trial is
-kept in `transcripts/`.
+Claude; and provider errors or timeouts. The full transcript of each attempt is
+kept in `transcripts/`. An attempt that ends in a provider error or timeout
+stays in `results.jsonl` but is not a trial: summaries count valid trials only,
+report the failed attempts apart, and a resumed run retries their keys. A gated
+trial whose hook never ran is not recorded at all; the run stops instead.
 
 ## Analysis
 
@@ -253,9 +256,14 @@ arms — every trial passing or every trial failing. Trap strength (how often th
 ungated arm passes) is reported separately and does not decide usability, since
 the ungated arm failing is what a working trap looks like.
 
-Graders are checked before any session is billed: the start workspace and the
-`naive` overlay must both score below 1 and `reference` must score 1, or the run
-stops.
+Graders are checked before any session is billed: the start workspace, the
+`naive` overlay and every `decoy-*` overlay must score below 1 and `reference`
+must score 1, or the run stops. A decoy has the right shape and the wrong
+content. `csv-export-delimiter` has two (a constant in the right dialect, and
+the column names instead of the values), and its grader now checks the
+exported values as well as the dialect. Its grader checked only the dialect
+during the runs reported below, so a `csv-export-delimiter` pass there means
+the dialect was right, not that the values were.
 
 ### Hidden-intent results
 

tests/docker/gateguard-effectiveness.test.js

@@ -317,7 +317,84 @@ if (gnuTarDir) {
   });
 }
 
+test('pilot summaries count valid trials only and report provider errors apart', () => {
+  const row = (arm, rep, passed, extra = {}) => ({
+    key: `s/${arm}/${rep}`, arm, scenario: 's', rep, question: 'q', passed, score: passed ? 1 : 0, evidenceRecall: 0, gateDenials: 0,
+    inputTokens: 1, cachedInputTokens: 0, outputTokens: 1, costUsd: null, turns: 1, wallMs: 1, providerError: false, timedOut: false, ...extra
+  });
+  const rows = [
+    row('gate', 1, true),
+    row('gate', 2, false, { providerError: true }),
+    row('gate', 2, true),
+    row('gate', 3, false, { timedOut: true }),
+    row('gate', 4, false, { judgeFailed: true }),
+    row('off', 1, true)
+  ];
+  const gate = lib.summarize(rows, ['gate', 'off']).arms.find(arm => arm.arm === 'gate');
+  assert.strictEqual(gate.trials, 2, 'a provider error or timeout is not a trial');
+  assert.strictEqual(gate.passes, 2);
+  assert.strictEqual(gate.meanScore, 1);
+  assert.strictEqual(gate.providerErrors, 2);
+  assert.deepStrictEqual([...lib.completedTrialKeys(rows)].sort(), ['s/gate/1', 's/gate/2', 's/off/1']);
+});
+
+/** The pilot runner driven end to end with fake-claude.js as the provider. */
+function syntheticPilotRunner(root, scenario, arms) {
+  const out = path.join(root, 'out');
+  const preload = path.join(root, 'preload.js');
+  fs.writeFileSync(preload, `
+    const cp = require('child_process');
+    const originalSpawn = cp.spawnSync;
+    cp.spawnSync = (file, args, options) => file === 'ecc-fake-claude'
+      ? originalSpawn(process.execPath, [${JSON.stringify(path.join(DIR, 'fake-claude.js'))}, ...args], options)
+      : originalSpawn(file, args, options);
+  `);
+  const args = ['--require', preload, path.join(DIR, 'run.js'), '--out', out, '--model', 'fake', '--claude', 'ecc-fake-claude', '--allow-real-provider', '--arms', arms, '--reps', '1', '--scenarios', scenario.id];
+  const env = { ...process.env, FAKE_CLAUDE_OVERLAY: path.join(scenario.root, 'reference') };
+  const run = extraEnv => spawnSync(process.execPath, args, { env: { ...env, ...extraEnv }, encoding: 'utf8', timeout: 60000 });
+  const results = () => {
+    const file = path.join(out, 'results.jsonl');
+    return fs.existsSync(file) ? fs.readFileSync(file, 'utf8').trim().split('\n').filter(Boolean).map(line => JSON.parse(line)) : [];
+  };
+  return { out, run, results };
+}
+
 if (gitAvailable()) {
+  test('a pilot trial that ended in a provider error is retried on resume', () => {
+    const root = tempDir('gg-eff-retry-');
+    try {
+      const scenario = scenarios.find(item => item.id === 'importers-date-shape');
+      const { out, run, results } = syntheticPilotRunner(root, scenario, 'off');
+      const failed = run({ FAKE_CLAUDE_PROVIDER_ERROR: '1' });
+      assert.strictEqual(failed.status, 0, failed.stderr);
+      assert.deepStrictEqual(results().map(row => row.providerError), [true]);
+      const retried = run({});
+      assert.strictEqual(retried.status, 0, retried.stderr);
+      assert.ok(retried.stderr.includes('1 trials to run'), retried.stderr);
+      const rows = results();
+      assert.deepStrictEqual(rows.map(row => row.providerError), [true, false], 'the failed attempt stays for audit');
+      assert.strictEqual(rows[1].passed, true);
+      assert.notStrictEqual(rows[0].transcript, rows[1].transcript, 'each attempt keeps its own transcript');
+      const { summary } = JSON.parse(fs.readFileSync(path.join(out, 'summary.json'), 'utf8'));
+      const off = summary.arms.find(arm => arm.arm === 'off');
+      assert.deepStrictEqual([off.trials, off.passes, off.providerErrors], [1, 1, 1]);
+      const again = run({});
+      assert.ok(again.stderr.includes('0 trials to run'), again.stderr);
+    } finally { fs.rmSync(root, { recursive: true, force: true }); }
+  });
+
+  test('the pilot runner stops before recording a gated trial whose hook never ran', () => {
+    const root = tempDir('gg-eff-hookless-');
+    try {
+      const scenario = scenarios.find(item => item.id === 'importers-date-shape');
+      const { run, results } = syntheticPilotRunner(root, scenario, 'gate');
+      const result = run({ FAKE_CLAUDE_SKIP_HOOK: '1' });
+      assert.notStrictEqual(result.status, 0, 'a hookless gated trial must stop the run');
+      assert.match(result.stderr, /the gate did not run although the agent edited files/);
+      assert.deepStrictEqual(results(), [], 'the hookless trial is not recorded as data');
+    } finally { fs.rmSync(root, { recursive: true, force: true }); }
+  });
+
   test('actual fake-provider CLI resumes identity and resists a metadata publication symlink race', () => {
     const root = tempDir('gg-eff-cli-');
     const out = path.join(root, 'out');

tests/docker/gateguard-intent.test.js

@@ -177,6 +177,58 @@ test('every grader separates the start, the trap and a correct solution', () =>
   }
 });
 
+test('the CSV grader rejects the right dialect around the wrong values', () => {
+  const lib = require(path.join(DIR, 'lib'));
+  const workRoot = tempDir('gg-decoy-');
+  try {
+    const csv = intentEval.loadIntentScenarios().find(scenario => scenario.id === 'csv-export-delimiter');
+    const decoys = fs.readdirSync(csv.root).filter(name => name.startsWith('decoy-')).sort();
+    assert.deepStrictEqual(decoys, ['decoy-constant', 'decoy-keys']);
+    for (const decoy of decoys) {
+      const dest = path.join(workRoot, decoy);
+      lib.prepareWorkspace(csv, dest, decoy);
+      assert.ok(lib.grade(csv, dest).score < 1, `${decoy} must not pass`);
+    }
+    const { scores, sound } = intentEval.graderIsSound(csv, workRoot);
+    assert.ok(sound, JSON.stringify(scores));
+    for (const decoy of decoys) assert.ok(scores[decoy] < 1, `graderIsSound grades ${decoy}: ${scores[decoy]}`);
+  } finally { fs.rmSync(workRoot, { recursive: true, force: true }); }
+});
+
+test('a grader that passes one of its decoys is not sound', () => {
+  const workRoot = tempDir('gg-decoy-pass-');
+  try {
+    const csv = intentEval.loadIntentScenarios().find(scenario => scenario.id === 'csv-export-delimiter');
+    const root = path.join(workRoot, 'scenario');
+    fs.cpSync(csv.root, root, { recursive: true });
+    fs.cpSync(path.join(root, 'reference'), path.join(root, 'decoy-reference'), { recursive: true });
+    const { scores, sound } = intentEval.graderIsSound({ ...csv, root }, workRoot);
+    assert.strictEqual(scores['decoy-reference'], 1);
+    assert.strictEqual(sound, false);
+  } finally { fs.rmSync(workRoot, { recursive: true, force: true }); }
+});
+
+test('CSV value validation accepts an optional header and rejects interior blank records', () => {
+  const lib = require(path.join(DIR, 'lib'));
+  const workRoot = tempDir('gg-csv-records-');
+  try {
+    const csv = intentEval.loadIntentScenarios().find(scenario => scenario.id === 'csv-export-delimiter');
+    const dest = path.join(workRoot, 'workspace');
+    lib.prepareWorkspace(csv, dest);
+    for (const [output, expected] of [
+      ['"Ada";"Paris"\r\n"Bo";"Rome"\r\n', 1],
+      ['"name";"city"\r\n"Ada";"Paris"\r\n"Bo";"Rome"', 1],
+      ['"Ada";"Paris"\r\n\r\n"Bo";"Rome"\r\n', 0],
+      ['"Ada";"Paris"\r\n"Bo";"Rome"\r\n\r\n', 0]
+    ]) {
+      fs.writeFileSync(path.join(dest, 'src/export.js'), `exports.exportCsv = () => ${JSON.stringify(output)};\n`);
+      const score = lib.grade(csv, dest).score;
+      if (expected === 1) assert.strictEqual(score, 1, JSON.stringify(output));
+      else assert.ok(score < 1, JSON.stringify(output));
+    }
+  } finally { fs.rmSync(workRoot, { recursive: true, force: true }); }
+});
+
 // --- behaviour is classified into the right hole ---
 
 test('classify maps each behaviour to its hole', () => {
@@ -515,50 +567,74 @@ test('session cleanup preserves concurrent and pre-existing folders under custom
   }
 });
 
-test('actual runner freezes arm-patch source and refuses changed-input resume before dispatch', () => {
-  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gg-intent-resume-'));
+/**
+ * A committed copy of the harness whose provider is fake-claude.js, so the actual
+ * runner can be driven end to end. `fakeEnv` reaches the fake provider only.
+ */
+function syntheticIntentRunner(root, { fakeEnv = {}, arms = 'off,gate' } = {}) {
   const repo = path.join(root, 'repo');
   const copied = path.join(repo, 'docker', 'gateguard-effectiveness');
   const out = path.join(root, 'out');
   const preload = path.join(root, 'preload.js');
   const calls = path.join(root, 'calls.log');
   const config = path.join(root, 'claude-config');
+  fs.mkdirSync(repo);
+  fs.cpSync(path.join(ROOT, 'scripts'), path.join(repo, 'scripts'), { recursive: true });
+  fs.cpSync(DIR, copied, { recursive: true });
+  fs.mkdirSync(path.join(repo, 'docker', 'context-profiles'), { recursive: true });
+  fs.copyFileSync(path.join(ROOT, 'docker', 'context-profiles', 'ai-eval-lib.js'), path.join(repo, 'docker', 'context-profiles', 'ai-eval-lib.js'));
+  fs.copyFileSync(path.join(ROOT, 'package.json'), path.join(repo, 'package.json'));
+  const env = { ...process.env, HOME: path.join(root, 'home'), USERPROFILE: path.join(root, 'home'), CLAUDE_CONFIG_DIR: config, CLAUDE_CODE_PROJECT_DIR_NAME: '',
+    NODE_PATH: path.join(ROOT, 'node_modules'), GIT_CONFIG_GLOBAL: path.join(root, 'unused-gitconfig'),
+    GIT_AUTHOR_NAME: 'Synthetic Eval', GIT_COMMITTER_NAME: 'Synthetic Eval', GIT_AUTHOR_EMAIL: 'eval@example.invalid', GIT_COMMITTER_EMAIL: 'eval@example.invalid' };
+  for (const args of [['init', '-q'], ['add', '.'], ['commit', '-q', '--no-gpg-sign', '-m', 'synthetic source snapshot']]) {
+    const result = spawnSync('git', args, { cwd: repo, env, encoding: 'utf8', timeout: 30000 });
+    assert.strictEqual(result.status, 0, result.stderr);
+  }
+  const scenario = intentEval.loadIntentScenarios()[0];
+  const copiedReference = path.join(copied, 'scenarios-intent', scenario.id, 'reference');
+  fs.writeFileSync(preload, `
+    const cp = require('child_process');
+    const fs = require('fs');
+    const path = require('path');
+    const spawn = cp.spawnSync;
+    cp.spawnSync = (file, args, options) => {
+      if (file !== 'ecc-intent-fake') return spawn(file, args, options);
+      fs.appendFileSync(${JSON.stringify(calls)}, (args.includes('--settings') ? 'agent' : 'judge') + '\\n');
+      if (!args.includes('--settings')) return spawn(process.execPath, ['-e', 'console.log(JSON.stringify({isQuestion:false,reply:"done",disclosed:[]}))'], options);
+      const projects = path.join(options.env.CLAUDE_CONFIG_DIR, 'projects');
+      const own = path.resolve(options.cwd).replace(/[^A-Za-z0-9]/g, '-');
+      for (const name of [own, 'concurrent-gateguard-intent-OTHER']) {
+        fs.mkdirSync(path.join(projects, name), { recursive: true });
+        fs.writeFileSync(path.join(projects, name, 'sentinel'), name);
+      }
+      const fakeEnv = { ...options.env, ...${JSON.stringify(fakeEnv)}, FAKE_CLAUDE_OVERLAY: ${JSON.stringify(copiedReference)} };
+      return spawn(process.execPath, [${JSON.stringify(path.join(copied, 'fake-claude.js'))}, ...args], { ...options, env: fakeEnv });
+    };
+  `);
+  const args = ['--require', preload, path.join(copied, 'run-intent.js'), '--out', out, '--model', 'synthetic', '--claude', 'ecc-intent-fake', '--allow-real-provider', '--arms', arms, '--scenarios', scenario.id, '--reps', '1'];
+  const run = () => spawnSync(process.execPath, args, { cwd: repo, env, encoding: 'utf8', timeout: 60000 });
+  return { copied, out, calls, config, scenario, run };
+}
+
+test('actual runner stops before recording a gated trial whose hook never ran', () => {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gg-intent-hookless-'));
   try {
-    fs.mkdirSync(repo);
-    fs.cpSync(path.join(ROOT, 'scripts'), path.join(repo, 'scripts'), { recursive: true });
-    fs.cpSync(DIR, copied, { recursive: true });
-    fs.mkdirSync(path.join(repo, 'docker', 'context-profiles'), { recursive: true });
-    fs.copyFileSync(path.join(ROOT, 'docker', 'context-profiles', 'ai-eval-lib.js'), path.join(repo, 'docker', 'context-profiles', 'ai-eval-lib.js'));
-    fs.copyFileSync(path.join(ROOT, 'package.json'), path.join(repo, 'package.json'));
-    const env = { ...process.env, HOME: path.join(root, 'home'), USERPROFILE: path.join(root, 'home'), CLAUDE_CONFIG_DIR: config, CLAUDE_CODE_PROJECT_DIR_NAME: '',
-      NODE_PATH: path.join(ROOT, 'node_modules'), GIT_CONFIG_GLOBAL: path.join(root, 'unused-gitconfig'),
-      GIT_AUTHOR_NAME: 'Synthetic Eval', GIT_COMMITTER_NAME: 'Synthetic Eval', GIT_AUTHOR_EMAIL: 'eval@example.invalid', GIT_COMMITTER_EMAIL: 'eval@example.invalid' };
-    for (const args of [['init', '-q'], ['add', '.'], ['commit', '-q', '--no-gpg-sign', '-m', 'synthetic source snapshot']]) {
-      const result = spawnSync('git', args, { cwd: repo, env, encoding: 'utf8', timeout: 30000 });
-      assert.strictEqual(result.status, 0, result.stderr);
-    }
-    const scenario = intentEval.loadIntentScenarios()[0];
-    const copiedReference = path.join(copied, 'scenarios-intent', scenario.id, 'reference');
-    fs.writeFileSync(preload, `
-      const cp = require('child_process');
-      const fs = require('fs');
-      const path = require('path');
-      const spawn = cp.spawnSync;
-      cp.spawnSync = (file, args, options) => {
-        if (file !== 'ecc-intent-fake') return spawn(file, args, options);
-        fs.appendFileSync(${JSON.stringify(calls)}, (args.includes('--settings') ? 'agent' : 'judge') + '\\n');
-        if (!args.includes('--settings')) return spawn(process.execPath, ['-e', 'console.log(JSON.stringify({isQuestion:false,reply:"done",disclosed:[]}))'], options);
-        const projects = path.join(options.env.CLAUDE_CONFIG_DIR, 'projects');
-        const own = path.resolve(options.cwd).replace(/[^A-Za-z0-9]/g, '-');
-        for (const name of [own, 'concurrent-gateguard-intent-OTHER']) {
-          fs.mkdirSync(path.join(projects, name), { recursive: true });
-          fs.writeFileSync(path.join(projects, name, 'sentinel'), name);
-        }
-        return spawn(process.execPath, [${JSON.stringify(path.join(copied, 'fake-claude.js'))}, ...args], { ...options, env: { ...options.env, FAKE_CLAUDE_OVERLAY: ${JSON.stringify(copiedReference)} } });
-      };
-    `);
-    const args = ['--require', preload, path.join(copied, 'run-intent.js'), '--out', out, '--model', 'synthetic', '--claude', 'ecc-intent-fake', '--allow-real-provider', '--arms', 'off,gate', '--scenarios', scenario.id, '--reps', '1'];
-    const run = () => spawnSync(process.execPath, args, { cwd: repo, env, encoding: 'utf8', timeout: 60000 });
+    // The runner needs a comparator arm; the ungated off trial is valid either way.
+    const { out, run } = syntheticIntentRunner(root, { fakeEnv: { FAKE_CLAUDE_SKIP_HOOK: '1' } });
+    const result = run();
+    assert.notStrictEqual(result.status, 0, 'a hookless gated trial must stop the run');
+    assert.match(result.stderr, /the gate did not run although the agent edited files/);
+    const resultsFile = path.join(out, 'results.jsonl');
+    const recorded = fs.existsSync(resultsFile) ? fs.readFileSync(resultsFile, 'utf8').trim().split('\n').filter(Boolean) : [];
+    assert.deepStrictEqual(recorded.map(line => JSON.parse(line).arm).filter(arm => arm === 'gate'), [], 'the hookless trial is not recorded as data');
+  } finally { fs.rmSync(root, { recursive: true, force: true }); }
+});
+
+test('actual runner freezes arm-patch source and refuses changed-input resume before dispatch', () => {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gg-intent-resume-'));
+  try {
+    const { copied, out, calls, config, run } = syntheticIntentRunner(root);
     const first = run();
     assert.strictEqual(first.status, 0, first.stderr);
     const meta = JSON.parse(fs.readFileSync(path.join(out, 'meta.json'), 'utf8'));