test(gateguard): pin grader decoys, pilot retries and the hookless-trial abort

Three #3481 review findings still hold on main 8ce348fb:

- csv-export-delimiter's grader checks only the dialect (CRLF, semicolons,
  quoting), so a constant '"x";"y"' export or one emitting the column
  names scores 1. Add decoy-constant and decoy-keys overlays and pin that
  each scores below 1 and that graderIsSound reports them; a scenario whose
  decoy passes is not sound.
- The pilot runner (run.js) marks errored keys done, so they are never
  retried, and armSummary counts provider-error and timed-out rows in
  trials, passes and means. Pin the summary on valid rows only, and drive
  run.js end to end: a provider error is retried on resume, keeps its own
  row and transcript, and the summary reports it apart.
- Disabling run-intent.js's abort on an unobserved hook left every test
  passing. Drive both runners end to end with a gated trial whose hook
  never runs: each must stop and record nothing for it.

fake-claude.js gains FAKE_CLAUDE_SKIP_HOOK and FAKE_CLAUDE_PROVIDER_ERROR.
The intent end-to-end setup moves into syntheticIntentRunner, shared with
the existing resume test.

Against main: 2 intent failures (the decoy cases) and 2 pilot failures
(summary, retry). The two abort tests pass on main, since both aborts
exist; each fails when its runner's abort is disabled
(if (false && !row.hookObserved)).

docker/gateguard-effectiveness/fake-claude.js

@@ -39,9 +39,11 @@ const input = fs.existsSync(target)
 const toolName = 'old_string' in input ? 'Edit' : 'Write';
 emit({ type: 'assistant', message: { content: [{ type: 'text', text: 'Checking report.js, invoice.js and csv.js first.' }, { type: 'tool_use', id: 't1', name: toolName, input }] } });
 
+// FAKE_CLAUDE_SKIP_HOOK=1 edits without running the hook (a broken hook setup);
+// FAKE_CLAUDE_PROVIDER_ERROR=1 ends the session in a non-authentication provider error.
 let denial = '';
 const group = (settings.hooks && settings.hooks.PreToolUse || []).find(entry => new RegExp(`^(${entry.matcher})$`).test(toolName));
-if (group) {
+if (group && process.env.FAKE_CLAUDE_SKIP_HOOK !== '1') {
   const env = { ...process.env, CLAUDE_PROJECT_DIR: process.cwd() };
   const payload = { session_id: 'fake', cwd: process.cwd(), hook_event_name: 'PreToolUse', tool_name: toolName, tool_input: input, tool_use_id: 't1' };
   const result = spawnSync(group.hooks[0].command, { shell: true, input: JSON.stringify(payload), env, encoding: 'utf8' });
@@ -50,4 +52,8 @@ if (group) {
 }
 emit({ type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 't1', is_error: Boolean(denial), content: denial || 'ok' }] } });
 copyTree(overlay, process.cwd());
+if (process.env.FAKE_CLAUDE_PROVIDER_ERROR === '1') {
+  emit({ type: 'result', is_error: true, result: 'API Error: 529 overloaded', num_turns: 1, duration_ms: 100, total_cost_usd: 0, usage: {}, modelUsage: {} });
+  process.exit(0);
+}
 emit({ type: 'result', is_error: false, result: 'done', num_turns: 3, duration_ms: 1200, total_cost_usd: 0.01, usage: { input_tokens: 100, cache_creation_input_tokens: 50, cache_read_input_tokens: 400, output_tokens: 80 }, modelUsage: { 'fake-model': {} } });

docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/decoy-constant/src/export.js

@@ -0,0 +1,8 @@
+'use strict';
+
+/** Decoy: a constant in the right dialect that ignores its input. */
+function exportCsv() {
+  return '"x";"y"\r\n"x";"y"\r\n';
+}
+
+module.exports = { exportCsv };

docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/decoy-keys/src/export.js

@@ -0,0 +1,8 @@
+'use strict';
+
+/** Decoy: the right dialect around the wrong fields, the column names instead of the values. */
+function exportCsv(rows) {
+  return rows.map(row => Object.keys(row).map(key => `"${key}"`).join(';')).join('\r\n') + '\r\n';
+}
+
+module.exports = { exportCsv };

tests/docker/gateguard-effectiveness.test.js

@@ -317,7 +317,83 @@ if (gnuTarDir) {
   });
 }
 
+test('pilot summaries count valid trials only and report provider errors apart', () => {
+  const row = (arm, rep, passed, extra = {}) => ({
+    key: `s/${arm}/${rep}`, arm, scenario: 's', rep, question: 'q', passed, score: passed ? 1 : 0, evidenceRecall: 0, gateDenials: 0,
+    inputTokens: 1, cachedInputTokens: 0, outputTokens: 1, costUsd: null, turns: 1, wallMs: 1, providerError: false, timedOut: false, ...extra
+  });
+  const rows = [
+    row('gate', 1, true),
+    row('gate', 2, false, { providerError: true }),
+    row('gate', 2, true),
+    row('gate', 3, false, { timedOut: true }),
+    row('off', 1, true)
+  ];
+  const gate = lib.summarize(rows, ['gate', 'off']).arms.find(arm => arm.arm === 'gate');
+  assert.strictEqual(gate.trials, 2, 'a provider error or timeout is not a trial');
+  assert.strictEqual(gate.passes, 2);
+  assert.strictEqual(gate.meanScore, 1);
+  assert.strictEqual(gate.providerErrors, 2);
+  assert.deepStrictEqual([...lib.completedTrialKeys(rows)].sort(), ['s/gate/1', 's/gate/2', 's/off/1']);
+});
+
+/** The pilot runner driven end to end with fake-claude.js as the provider. */
+function syntheticPilotRunner(root, scenario, arms) {
+  const out = path.join(root, 'out');
+  const preload = path.join(root, 'preload.js');
+  fs.writeFileSync(preload, `
+    const cp = require('child_process');
+    const originalSpawn = cp.spawnSync;
+    cp.spawnSync = (file, args, options) => file === 'ecc-fake-claude'
+      ? originalSpawn(process.execPath, [${JSON.stringify(path.join(DIR, 'fake-claude.js'))}, ...args], options)
+      : originalSpawn(file, args, options);
+  `);
+  const args = ['--require', preload, path.join(DIR, 'run.js'), '--out', out, '--model', 'fake', '--claude', 'ecc-fake-claude', '--allow-real-provider', '--arms', arms, '--reps', '1', '--scenarios', scenario.id];
+  const env = { ...process.env, FAKE_CLAUDE_OVERLAY: path.join(scenario.root, 'reference') };
+  const run = extraEnv => spawnSync(process.execPath, args, { env: { ...env, ...extraEnv }, encoding: 'utf8', timeout: 60000 });
+  const results = () => {
+    const file = path.join(out, 'results.jsonl');
+    return fs.existsSync(file) ? fs.readFileSync(file, 'utf8').trim().split('\n').filter(Boolean).map(line => JSON.parse(line)) : [];
+  };
+  return { out, run, results };
+}
+
 if (gitAvailable()) {
+  test('a pilot trial that ended in a provider error is retried on resume', () => {
+    const root = tempDir('gg-eff-retry-');
+    try {
+      const scenario = scenarios.find(item => item.id === 'importers-date-shape');
+      const { out, run, results } = syntheticPilotRunner(root, scenario, 'off');
+      const failed = run({ FAKE_CLAUDE_PROVIDER_ERROR: '1' });
+      assert.strictEqual(failed.status, 0, failed.stderr);
+      assert.deepStrictEqual(results().map(row => row.providerError), [true]);
+      const retried = run({});
+      assert.strictEqual(retried.status, 0, retried.stderr);
+      assert.ok(retried.stderr.includes('1 trials to run'), retried.stderr);
+      const rows = results();
+      assert.deepStrictEqual(rows.map(row => row.providerError), [true, false], 'the failed attempt stays for audit');
+      assert.strictEqual(rows[1].passed, true);
+      assert.notStrictEqual(rows[0].transcript, rows[1].transcript, 'each attempt keeps its own transcript');
+      const { summary } = JSON.parse(fs.readFileSync(path.join(out, 'summary.json'), 'utf8'));
+      const off = summary.arms.find(arm => arm.arm === 'off');
+      assert.deepStrictEqual([off.trials, off.passes, off.providerErrors], [1, 1, 1]);
+      const again = run({});
+      assert.ok(again.stderr.includes('0 trials to run'), again.stderr);
+    } finally { fs.rmSync(root, { recursive: true, force: true }); }
+  });
+
+  test('the pilot runner stops before recording a gated trial whose hook never ran', () => {
+    const root = tempDir('gg-eff-hookless-');
+    try {
+      const scenario = scenarios.find(item => item.id === 'importers-date-shape');
+      const { run, results } = syntheticPilotRunner(root, scenario, 'gate');
+      const result = run({ FAKE_CLAUDE_SKIP_HOOK: '1' });
+      assert.notStrictEqual(result.status, 0, 'a hookless gated trial must stop the run');
+      assert.match(result.stderr, /the gate did not run although the agent edited files/);
+      assert.deepStrictEqual(results(), [], 'the hookless trial is not recorded as data');
+    } finally { fs.rmSync(root, { recursive: true, force: true }); }
+  });
+
   test('actual fake-provider CLI resumes identity and resists a metadata publication symlink race', () => {
     const root = tempDir('gg-eff-cli-');
     const out = path.join(root, 'out');

tests/docker/gateguard-intent.test.js

@@ -177,6 +177,33 @@ test('every grader separates the start, the trap and a correct solution', () =>
   }
 });
 
+test('the CSV grader rejects the right dialect around the wrong values', () => {
+  const lib = require(path.join(DIR, 'lib'));
+  const workRoot = tempDir('gg-decoy-');
+  const csv = intentEval.loadIntentScenarios().find(scenario => scenario.id === 'csv-export-delimiter');
+  const decoys = fs.readdirSync(csv.root).filter(name => name.startsWith('decoy-')).sort();
+  assert.deepStrictEqual(decoys, ['decoy-constant', 'decoy-keys']);
+  for (const decoy of decoys) {
+    const dest = path.join(workRoot, decoy);
+    lib.prepareWorkspace(csv, dest, decoy);
+    assert.ok(lib.grade(csv, dest).score < 1, `${decoy} must not pass`);
+  }
+  const { scores, sound } = intentEval.graderIsSound(csv, workRoot);
+  assert.ok(sound, JSON.stringify(scores));
+  for (const decoy of decoys) assert.ok(scores[decoy] < 1, `graderIsSound grades ${decoy}: ${scores[decoy]}`);
+});
+
+test('a grader that passes one of its decoys is not sound', () => {
+  const workRoot = tempDir('gg-decoy-pass-');
+  const csv = intentEval.loadIntentScenarios().find(scenario => scenario.id === 'csv-export-delimiter');
+  const root = path.join(workRoot, 'scenario');
+  fs.cpSync(csv.root, root, { recursive: true });
+  fs.cpSync(path.join(root, 'reference'), path.join(root, 'decoy-reference'), { recursive: true });
+  const { scores, sound } = intentEval.graderIsSound({ ...csv, root }, workRoot);
+  assert.strictEqual(scores['decoy-reference'], 1);
+  assert.strictEqual(sound, false);
+});
+
 // --- behaviour is classified into the right hole ---
 
 test('classify maps each behaviour to its hole', () => {
@@ -515,50 +542,74 @@ test('session cleanup preserves concurrent and pre-existing folders under custom
   }
 });
 
-test('actual runner freezes arm-patch source and refuses changed-input resume before dispatch', () => {
-  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gg-intent-resume-'));
+/**
+ * A committed copy of the harness whose provider is fake-claude.js, so the actual
+ * runner can be driven end to end. `fakeEnv` reaches the fake provider only.
+ */
+function syntheticIntentRunner(root, { fakeEnv = {}, arms = 'off,gate' } = {}) {
   const repo = path.join(root, 'repo');
   const copied = path.join(repo, 'docker', 'gateguard-effectiveness');
   const out = path.join(root, 'out');
   const preload = path.join(root, 'preload.js');
   const calls = path.join(root, 'calls.log');
   const config = path.join(root, 'claude-config');
+  fs.mkdirSync(repo);
+  fs.cpSync(path.join(ROOT, 'scripts'), path.join(repo, 'scripts'), { recursive: true });
+  fs.cpSync(DIR, copied, { recursive: true });
+  fs.mkdirSync(path.join(repo, 'docker', 'context-profiles'), { recursive: true });
+  fs.copyFileSync(path.join(ROOT, 'docker', 'context-profiles', 'ai-eval-lib.js'), path.join(repo, 'docker', 'context-profiles', 'ai-eval-lib.js'));
+  fs.copyFileSync(path.join(ROOT, 'package.json'), path.join(repo, 'package.json'));
+  const env = { ...process.env, HOME: path.join(root, 'home'), USERPROFILE: path.join(root, 'home'), CLAUDE_CONFIG_DIR: config, CLAUDE_CODE_PROJECT_DIR_NAME: '',
+    NODE_PATH: path.join(ROOT, 'node_modules'), GIT_CONFIG_GLOBAL: path.join(root, 'unused-gitconfig'),
+    GIT_AUTHOR_NAME: 'Synthetic Eval', GIT_COMMITTER_NAME: 'Synthetic Eval', GIT_AUTHOR_EMAIL: 'eval@example.invalid', GIT_COMMITTER_EMAIL: 'eval@example.invalid' };
+  for (const args of [['init', '-q'], ['add', '.'], ['commit', '-q', '--no-gpg-sign', '-m', 'synthetic source snapshot']]) {
+    const result = spawnSync('git', args, { cwd: repo, env, encoding: 'utf8', timeout: 30000 });
+    assert.strictEqual(result.status, 0, result.stderr);
+  }
+  const scenario = intentEval.loadIntentScenarios()[0];
+  const copiedReference = path.join(copied, 'scenarios-intent', scenario.id, 'reference');
+  fs.writeFileSync(preload, `
+    const cp = require('child_process');
+    const fs = require('fs');
+    const path = require('path');
+    const spawn = cp.spawnSync;
+    cp.spawnSync = (file, args, options) => {
+      if (file !== 'ecc-intent-fake') return spawn(file, args, options);
+      fs.appendFileSync(${JSON.stringify(calls)}, (args.includes('--settings') ? 'agent' : 'judge') + '\\n');
+      if (!args.includes('--settings')) return spawn(process.execPath, ['-e', 'console.log(JSON.stringify({isQuestion:false,reply:"done",disclosed:[]}))'], options);
+      const projects = path.join(options.env.CLAUDE_CONFIG_DIR, 'projects');
+      const own = path.resolve(options.cwd).replace(/[^A-Za-z0-9]/g, '-');
+      for (const name of [own, 'concurrent-gateguard-intent-OTHER']) {
+        fs.mkdirSync(path.join(projects, name), { recursive: true });
+        fs.writeFileSync(path.join(projects, name, 'sentinel'), name);
+      }
+      const fakeEnv = { ...options.env, ...${JSON.stringify(fakeEnv)}, FAKE_CLAUDE_OVERLAY: ${JSON.stringify(copiedReference)} };
+      return spawn(process.execPath, [${JSON.stringify(path.join(copied, 'fake-claude.js'))}, ...args], { ...options, env: fakeEnv });
+    };
+  `);
+  const args = ['--require', preload, path.join(copied, 'run-intent.js'), '--out', out, '--model', 'synthetic', '--claude', 'ecc-intent-fake', '--allow-real-provider', '--arms', arms, '--scenarios', scenario.id, '--reps', '1'];
+  const run = () => spawnSync(process.execPath, args, { cwd: repo, env, encoding: 'utf8', timeout: 60000 });
+  return { copied, out, calls, config, scenario, run };
+}
+
+test('actual runner stops before recording a gated trial whose hook never ran', () => {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gg-intent-hookless-'));
   try {
-    fs.mkdirSync(repo);
-    fs.cpSync(path.join(ROOT, 'scripts'), path.join(repo, 'scripts'), { recursive: true });
-    fs.cpSync(DIR, copied, { recursive: true });
-    fs.mkdirSync(path.join(repo, 'docker', 'context-profiles'), { recursive: true });
-    fs.copyFileSync(path.join(ROOT, 'docker', 'context-profiles', 'ai-eval-lib.js'), path.join(repo, 'docker', 'context-profiles', 'ai-eval-lib.js'));
-    fs.copyFileSync(path.join(ROOT, 'package.json'), path.join(repo, 'package.json'));
-    const env = { ...process.env, HOME: path.join(root, 'home'), USERPROFILE: path.join(root, 'home'), CLAUDE_CONFIG_DIR: config, CLAUDE_CODE_PROJECT_DIR_NAME: '',
-      NODE_PATH: path.join(ROOT, 'node_modules'), GIT_CONFIG_GLOBAL: path.join(root, 'unused-gitconfig'),
-      GIT_AUTHOR_NAME: 'Synthetic Eval', GIT_COMMITTER_NAME: 'Synthetic Eval', GIT_AUTHOR_EMAIL: 'eval@example.invalid', GIT_COMMITTER_EMAIL: 'eval@example.invalid' };
-    for (const args of [['init', '-q'], ['add', '.'], ['commit', '-q', '--no-gpg-sign', '-m', 'synthetic source snapshot']]) {
-      const result = spawnSync('git', args, { cwd: repo, env, encoding: 'utf8', timeout: 30000 });
-      assert.strictEqual(result.status, 0, result.stderr);
-    }
-    const scenario = intentEval.loadIntentScenarios()[0];
-    const copiedReference = path.join(copied, 'scenarios-intent', scenario.id, 'reference');
-    fs.writeFileSync(preload, `
-      const cp = require('child_process');
-      const fs = require('fs');
-      const path = require('path');
-      const spawn = cp.spawnSync;
-      cp.spawnSync = (file, args, options) => {
-        if (file !== 'ecc-intent-fake') return spawn(file, args, options);
-        fs.appendFileSync(${JSON.stringify(calls)}, (args.includes('--settings') ? 'agent' : 'judge') + '\\n');
-        if (!args.includes('--settings')) return spawn(process.execPath, ['-e', 'console.log(JSON.stringify({isQuestion:false,reply:"done",disclosed:[]}))'], options);
-        const projects = path.join(options.env.CLAUDE_CONFIG_DIR, 'projects');
-        const own = path.resolve(options.cwd).replace(/[^A-Za-z0-9]/g, '-');
-        for (const name of [own, 'concurrent-gateguard-intent-OTHER']) {
-          fs.mkdirSync(path.join(projects, name), { recursive: true });
-          fs.writeFileSync(path.join(projects, name, 'sentinel'), name);
-        }
-        return spawn(process.execPath, [${JSON.stringify(path.join(copied, 'fake-claude.js'))}, ...args], { ...options, env: { ...options.env, FAKE_CLAUDE_OVERLAY: ${JSON.stringify(copiedReference)} } });
-      };
-    `);
-    const args = ['--require', preload, path.join(copied, 'run-intent.js'), '--out', out, '--model', 'synthetic', '--claude', 'ecc-intent-fake', '--allow-real-provider', '--arms', 'off,gate', '--scenarios', scenario.id, '--reps', '1'];
-    const run = () => spawnSync(process.execPath, args, { cwd: repo, env, encoding: 'utf8', timeout: 60000 });
+    // The runner needs a comparator arm; the ungated off trial is valid either way.
+    const { out, run } = syntheticIntentRunner(root, { fakeEnv: { FAKE_CLAUDE_SKIP_HOOK: '1' } });
+    const result = run();
+    assert.notStrictEqual(result.status, 0, 'a hookless gated trial must stop the run');
+    assert.match(result.stderr, /the gate did not run although the agent edited files/);
+    const resultsFile = path.join(out, 'results.jsonl');
+    const recorded = fs.existsSync(resultsFile) ? fs.readFileSync(resultsFile, 'utf8').trim().split('\n').filter(Boolean) : [];
+    assert.deepStrictEqual(recorded.map(line => JSON.parse(line).arm).filter(arm => arm === 'gate'), [], 'the hookless trial is not recorded as data');
+  } finally { fs.rmSync(root, { recursive: true, force: true }); }
+});
+
+test('actual runner freezes arm-patch source and refuses changed-input resume before dispatch', () => {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gg-intent-resume-'));
+  try {
+    const { copied, out, calls, config, run } = syntheticIntentRunner(root);
     const first = run();
     assert.strictEqual(first.status, 0, first.stderr);
     const meta = JSON.parse(fs.readFileSync(path.join(out, 'meta.json'), 'utf8'));