feat(gateguard): measure question effectiveness against a hidden user intent

The existing harness cannot measure what it set out to measure. Every trap
hides its deciding fact in the repository, which agents reliably discover, and
trials run one-shot under "do not ask clarifying questions", so a fact-forcing
gate is evaluated with no user to answer it. Its scenarios also declare the
question the gate ought to raise, chosen by reading the gate's own taxonomy, so
the suite passes by construction and can never find a missing question.

Add a second runner that measures effectiveness instead of coverage:

- scenarios-intent/ keeps the deciding fact in intent.json, as a choice between
  plausible conventions the repository cannot settle, so an agent that does not
  ask can only guess and the ungated arm cannot sit at the ceiling.
- user-sim.js plays that user with a separate cheap model: it answers the
  question in front of it, never volunteers, and reports which fact ids it
  disclosed, so asking something and asking the deciding thing are told apart.
  An unparseable judge is a recorded failure, not a silent stonewall.
- session.js runs multi-turn sessions joined by --session-id and --resume, and
  tells every arm the user is reachable, so the comparison is symmetric and the
  gate's contribution is whether it makes the agent ask the deciding question.
- coverage.js classifies each trial from behaviour alone into working, lucky, or
  a follow-through, targeting, silence or coverage hole, and reports the
  questions the gate actually raised rather than asserting declared ones.
- run-intent.js refuses minus-target, which needs targets these scenarios
  deliberately lack, and checks every grader separates the start, the trap and
  the reference before a session is billed.

A scenario counts as carrying no information only when nothing varies across
arms. Trap strength is reported separately: the ungated arm failing is what a
working trap looks like, so it must not make a scenario unusable.

docker/gateguard-effectiveness/README.md

@@ -4,7 +4,27 @@ Development-only harness that runs real Claude sessions on trapped coding
 tasks with and without GateGuard's questions. Design, arms, measures and
 results: [docs/gateguard/question-effectiveness.md](../../docs/gateguard/question-effectiveness.md).
 
+Two runners, measuring different things.
+
+`run.js` — cost baseline. One-shot autonomous sessions over `scenarios/`, where
+each trap hides its deciding fact in the repository. The pilot returned no
+outcome difference between any arm, so treat this as a measure of the gate's
+turn and token cost, not of its effect.
+
 ```bash
 node docker/gateguard-effectiveness/run.js --out gg-eff --dry-run
 node docker/gateguard-effectiveness/run.js --out gg-eff --model <model> --allow-real-provider
 ```
+
+`run-intent.js` — effectiveness. Multi-turn sessions over `scenarios-intent/`,
+where the deciding fact exists only in a simulated user's hidden intent, so an
+agent that does not ask can only guess. Scenarios are authored blind to the
+gate's question taxonomy, and the report classifies each trial into a hole worth
+fixing.
+
+```bash
+node docker/gateguard-effectiveness/run-intent.js --check-graders
+node docker/gateguard-effectiveness/run-intent.js --out gg-intent --arms off,gate --dry-run
+node docker/gateguard-effectiveness/run-intent.js --out gg-intent --model <model> --allow-real-provider
+node docker/gateguard-effectiveness/run-intent.js --out gg-intent --summarize
+```

docker/gateguard-effectiveness/coverage.js

@@ -0,0 +1,172 @@
+'use strict';
+
+// Hole finding.
+//
+// The first evaluation asserted that the gate asks each scenario's declared
+// targetQuestions. Those targets were chosen by reading the gate's own
+// taxonomy, so the check passed by construction and could never find a missing
+// question. Nothing here reads the taxonomy. Every class below is computed from
+// what the trial *did*: whether the gate fired, whether the agent asked the
+// user, whether the deciding fact came out, and whether the outcome was right.
+//
+// The questions a scenario provokes are therefore discovered and reported, not
+// declared and asserted.
+
+/** Outcome classes. The four ending in -hole are the improvable cases. */
+const CLASSES = Object.freeze({
+  working: 'the deciding fact was obtained and the outcome is right',
+  lucky: 'right outcome without ever asking: the scenario does not force the question',
+  'follow-through-hole': 'the deciding fact was obtained and the outcome is still wrong',
+  'targeting-hole': 'the agent asked, but not for the deciding fact',
+  'silence-hole': 'the gate fired and the agent still did not ask',
+  'coverage-hole': 'a deciding ambiguity the gate never engaged, and the outcome is wrong'
+});
+
+/** Classifies one trial from its behaviour alone. */
+function classify(row) {
+  const gateFired = Number(row.gateDenials) > 0;
+  if (row.disclosedDecisive) return row.passed ? 'working' : 'follow-through-hole';
+  if (row.asked) return 'targeting-hole';
+  if (row.passed) return 'lucky';
+  return gateFired ? 'silence-hole' : 'coverage-hole';
+}
+
+function tally(rows) {
+  const counts = {};
+  for (const key of Object.keys(CLASSES)) counts[key] = 0;
+  for (const row of rows) counts[classify(row)]++;
+  return counts;
+}
+
+/**
+ * Trap strength: how often the ungated arm passes.
+ *
+ * A trap the ungated arm always beats is not a trap. This is reported for its
+ * own sake and does NOT decide whether a scenario is usable, because the
+ * ungated arm failing is what a working trap looks like.
+ */
+function difficulty(rows, { baselineArm = 'off' } = {}) {
+  const byScenario = {};
+  for (const row of rows.filter(row => row.arm === baselineArm)) {
+    (byScenario[row.scenario] ||= []).push(row);
+  }
+  return Object.entries(byScenario)
+    .map(([scenario, rs]) => {
+      const passed = rs.filter(row => row.passed).length;
+      const rate = passed / rs.length;
+      return { scenario, n: rs.length, passed, rate, label: rate === 1 ? 'ceiling' : 'traps' };
+    })
+    .sort((a, b) => a.scenario.localeCompare(b.scenario));
+}
+
+/**
+ * Whether a scenario separates any arm from any other, judged across every arm.
+ *
+ * Only a scenario where nothing varies is uninformative: every trial passing
+ * (the trap does not bite) or every trial failing (nothing reaches it). Judging
+ * this from the ungated arm alone would discard exactly the scenarios that
+ * work, since there the ungated arm is meant to fail.
+ */
+function informative(rows) {
+  const byScenario = {};
+  for (const row of rows) (byScenario[row.scenario] ||= []).push(row);
+  return Object.entries(byScenario)
+    .map(([scenario, rs]) => {
+      const passed = rs.filter(row => row.passed).length;
+      const label = passed === rs.length ? 'ceiling' : passed === 0 ? 'floor' : 'informative';
+      return { scenario, n: rs.length, passed, label };
+    })
+    .sort((a, b) => a.scenario.localeCompare(b.scenario));
+}
+
+/** Questions the gate actually raised per scenario, discovered from the runs. */
+function discoveredQuestions(rows) {
+  const byScenario = {};
+  for (const row of rows) {
+    const set = (byScenario[row.scenario] ||= new Set());
+    for (const question of row.questionsAsked || []) set.add(question);
+  }
+  return Object.entries(byScenario)
+    .map(([scenario, set]) => ({ scenario, questions: [...set].sort() }))
+    .sort((a, b) => a.scenario.localeCompare(b.scenario));
+}
+
+/**
+ * Scenarios where the gate asked nothing at all, across every gated arm.
+ * These are the strongest candidates for a missing question.
+ */
+function unengaged(rows, { baselineArm = 'off' } = {}) {
+  const gated = rows.filter(row => row.arm !== baselineArm);
+  const byScenario = {};
+  for (const row of gated) {
+    const entry = (byScenario[row.scenario] ||= { fired: 0, n: 0 });
+    entry.n++;
+    if (Number(row.gateDenials) > 0) entry.fired++;
+  }
+  return Object.entries(byScenario)
+    .filter(([, entry]) => entry.fired === 0)
+    .map(([scenario]) => scenario)
+    .sort();
+}
+
+function pct(value) {
+  return `${Math.round(value * 100)}%`;
+}
+
+function renderHoles(rows, { baselineArm = 'off' } = {}) {
+  const arms = [...new Set(rows.map(row => row.arm))].sort();
+  const classes = Object.keys(CLASSES);
+  const lines = [];
+
+  const strength = difficulty(rows, { baselineArm });
+  const verdicts = informative(rows);
+
+  lines.push('### Trap strength and usability', '');
+  lines.push('| Scenario | Ungated passed | Rate | Trap | Separates arms |', '| --- | ---: | ---: | --- | --- |');
+  const verdictOf = Object.fromEntries(verdicts.map(row => [row.scenario, row.label]));
+  for (const row of strength) {
+    lines.push(`| ${row.scenario} | ${row.passed}/${row.n} | ${pct(row.rate)} | ${row.label} | ${verdictOf[row.scenario]} |`);
+  }
+  const dead = verdicts.filter(row => row.label !== 'informative');
+  lines.push('');
+  lines.push(
+    dead.length
+      ? `${dead.length} of ${verdicts.length} scenarios carry no information (${dead.map(row => `${row.scenario}: ${row.label}`).join('; ')}). Arm comparisons below exclude them.`
+      : `All ${verdicts.length} scenarios separate at least one arm from another.`
+  );
+
+  const usableScenarios = new Set(verdicts.filter(row => row.label === 'informative').map(row => row.scenario));
+  const usable = rows.filter(row => usableScenarios.has(row.scenario));
+
+  lines.push('', '### Outcome classes', '');
+  lines.push(`| Arm | ${classes.join(' | ')} |`, `| --- | ${classes.map(() => '---:').join(' | ')} |`);
+  for (const arm of arms) {
+    const counts = tally(usable.filter(row => row.arm === arm));
+    lines.push(`| ${arm} | ${classes.map(key => counts[key]).join(' | ')} |`);
+  }
+
+  lines.push('', '### Holes to work on', '');
+  const holes = classes.filter(key => key.endsWith('-hole'));
+  const totals = tally(usable);
+  const present = holes.filter(key => totals[key] > 0);
+  if (!present.length) {
+    lines.push('No holes on the informative scenarios in this run.');
+  } else {
+    for (const key of present) lines.push(`- **${key}** (${totals[key]}): ${CLASSES[key]}`);
+  }
+
+  const missing = unengaged(rows, { baselineArm });
+  if (missing.length) {
+    lines.push('', `Gate never fired on: ${missing.join(', ')} — candidates for a question the taxonomy lacks.`);
+  }
+
+  lines.push('', '### Questions the gate actually raised', '');
+  lines.push('| Scenario | Observed questions |', '| --- | --- |');
+  for (const row of discoveredQuestions(rows)) {
+    lines.push(`| ${row.scenario} | ${row.questions.length ? row.questions.join(', ') : '_none_'} |`);
+  }
+
+  return `${lines.join('\n')}\n`;
+}
+
+module.exports = { CLASSES, classify, tally, difficulty, informative, discoveredQuestions, unengaged, renderHoles };

docker/gateguard-effectiveness/intent-eval.js

@@ -0,0 +1,128 @@
+'use strict';
+
+// Loader and trial runner for hidden-intent scenarios.
+//
+// Deliberately absent from task.json: targetQuestions. The first generation of
+// scenarios declared which question the gate ought to raise, chosen by reading
+// the gate's own taxonomy, and the suite then asserted the gate raised it. That
+// is a tautology, so it never found a missing question. Here the scenario
+// declares only the trap and the user's hidden intent; which questions fire is
+// observed from the run and classified in coverage.js.
+
+const fs = require('fs');
+const path = require('path');
+const { prepareWorkspace, commitWorkspace, grade, readMetrics } = require('./lib');
+const { ARMS, TRIAL_FILE, armSettings, trialPatch } = require('./arms');
+const { runConversation } = require('./session');
+const { classify } = require('./coverage');
+
+const SCENARIO_DIR = path.join(__dirname, 'scenarios-intent');
+
+// minus-target needs declared target questions, which these scenarios do not have.
+const SUPPORTED_ARMS = Object.freeze(['off', 'gate', 'placebo', 'main']);
+
+function loadIntentScenarios(dir = SCENARIO_DIR, ids = null) {
+  const scenarios = fs
+    .readdirSync(dir, { withFileTypes: true })
+    .filter(entry => entry.isDirectory())
+    .map(entry => {
+      const root = path.join(dir, entry.name);
+      const spec = JSON.parse(fs.readFileSync(path.join(root, 'task.json'), 'utf8'));
+      const intent = JSON.parse(fs.readFileSync(path.join(root, 'intent.json'), 'utf8'));
+      if (spec.id !== entry.name) throw new Error(`scenario ${entry.name} declares id ${spec.id}`);
+      if (spec.targetQuestions) {
+        throw new Error(`scenario ${spec.id} declares targetQuestions; hidden-intent scenarios are authored blind to the taxonomy`);
+      }
+      for (const key of ['trap', 'prompt']) {
+        if (typeof spec[key] !== 'string' || !spec[key]) throw new Error(`scenario ${spec.id} is missing ${key}`);
+      }
+      if (!Array.isArray(spec.evidence) || !spec.evidence.length) throw new Error(`scenario ${spec.id} has no evidence`);
+      if (typeof intent.ambiguity !== 'string' || !intent.ambiguity) throw new Error(`scenario ${spec.id} intent has no ambiguity`);
+      if (typeof intent.stonewall !== 'string' || !intent.stonewall) throw new Error(`scenario ${spec.id} intent has no stonewall`);
+      if (!Array.isArray(intent.facts) || !intent.facts.some(fact => fact.decisive)) {
+        throw new Error(`scenario ${spec.id} intent has no decisive fact`);
+      }
+      for (const fact of intent.facts) {
+        if (!fact.id || typeof fact.fact !== 'string' || !fact.fact) throw new Error(`scenario ${spec.id} has a malformed fact`);
+      }
+      return {
+        ...spec,
+        root,
+        intent,
+        evidence: spec.evidence.map(source => new RegExp(source, 'i')),
+        grader: fs.readFileSync(path.join(root, 'grader.cjs'), 'utf8')
+      };
+    })
+    .sort((a, b) => a.id.localeCompare(b.id));
+  if (!ids) return scenarios;
+  const unknown = ids.filter(id => !scenarios.some(scenario => scenario.id === id));
+  if (unknown.length) throw new Error(`unknown scenario: ${unknown.join(', ')}`);
+  return scenarios.filter(scenario => ids.includes(scenario.id));
+}
+
+/** Confirms the grader separates the start, the trap and a correct solution. */
+function graderIsSound(scenario, workRoot) {
+  const scores = {};
+  for (const variant of ['workspace', 'reference', 'naive']) {
+    const dest = path.join(workRoot, `check-${scenario.id}-${variant}`);
+    fs.rmSync(dest, { recursive: true, force: true });
+    prepareWorkspace(scenario, dest, variant === 'workspace' ? null : variant);
+    scores[variant] = grade(scenario, dest).score;
+  }
+  return { scores, sound: scores.reference === 1 && scores.workspace < 1 && scores.naive < 1 };
+}
+
+/** One trial: fresh workspace, a conversation with the simulated user, then the hidden grader. */
+function runIntentTrial(trial, { workRoot, armSettingsByName, executable, model, judgeModel, maxTurns, timeoutMs, userTurns, execute, judge }) {
+  if (!SUPPORTED_ARMS.includes(trial.arm)) throw new Error(`arm ${trial.arm} is not supported for hidden-intent scenarios`);
+  const dir = path.join(workRoot, trial.key.replace(/\//g, '__'));
+  fs.rmSync(dir, { recursive: true, force: true });
+  const cwd = prepareWorkspace(trial.scenario, path.join(dir, 'repo'));
+  const git = commitWorkspace(cwd);
+  const stateDir = path.join(dir, 'state');
+  fs.mkdirSync(stateDir, { recursive: true });
+  const settingsPath = path.join(dir, 'settings.json');
+  fs.writeFileSync(settingsPath, `${JSON.stringify(armSettingsByName[trial.arm], null, 2)}\n`);
+  fs.writeFileSync(path.join(dir, TRIAL_FILE), `${JSON.stringify({ stateDir, patch: trialPatch(trial.arm, trial.scenario) })}\n`);
+
+  const conversation = runConversation(
+    { cwd, stateDir, settingsPath, prompt: trial.scenario.prompt, intent: trial.scenario.intent },
+    { executable, model, judgeModel, maxTurns, timeoutMs, userTurns, execute, judge }
+  );
+  const metrics = readMetrics(stateDir);
+  const denials = metrics.filter(event => event.decision === 'deny' || event.decision === 'routine-deny');
+  const outcome = grade(trial.scenario, cwd);
+
+  const row = {
+    key: trial.key,
+    scenario: trial.scenario.id,
+    arm: trial.arm,
+    rep: trial.rep,
+    git,
+    gated: ARMS[trial.arm].gate,
+    passed: outcome.passed,
+    score: outcome.score,
+    asked: conversation.asked,
+    userTurns: conversation.userTurns,
+    disclosed: conversation.disclosed,
+    disclosedDecisive: conversation.disclosedDecisive,
+    judgeFailed: conversation.judgeFailed,
+    gateDenials: conversation.gateDenials,
+    questionsAsked: [...new Set(denials.flatMap(event => event.questions || []))].sort(),
+    editCalls: conversation.editCalls,
+    tools: conversation.tools,
+    exitStatus: conversation.exitStatus,
+    timedOut: conversation.timedOut,
+    inputTokens: conversation.inputTokens,
+    cachedInputTokens: conversation.cachedInputTokens,
+    outputTokens: conversation.outputTokens,
+    costUsd: conversation.costUsd,
+    turns: conversation.turns,
+    durationMs: conversation.durationMs,
+    exchange: conversation.exchange
+  };
+  row.outcomeClass = classify(row);
+  return row;
+}
+
+module.exports = { loadIntentScenarios, graderIsSound, runIntentTrial, SCENARIO_DIR, SUPPORTED_ARMS };

docker/gateguard-effectiveness/run-intent.js

@@ -0,0 +1,231 @@
+#!/usr/bin/env node
+'use strict';
+
+// Hidden-intent effectiveness run.
+//
+// Separate from run.js on purpose: run.js measures the gate's cost in one-shot
+// autonomous sessions and remains the cost baseline. This runner measures
+// whether the gate's questions obtain a fact the repository does not hold, and
+// reports the holes rather than asserting coverage.
+
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const { spawnSync } = require('child_process');
+const { schedule } = require('./lib');
+const { ARMS, armSettings, resolveRef, materializeTree } = require('./arms');
+const { loadIntentScenarios, graderIsSound, runIntentTrial, SUPPORTED_ARMS } = require('./intent-eval');
+const { renderHoles, difficulty } = require('./coverage');
+
+const REPO_ROOT = path.resolve(__dirname, '..', '..');
+
+const USAGE = [
+  'Usage: node docker/gateguard-effectiveness/run-intent.js --out <dir> --model <model> --allow-real-provider [options]',
+  '',
+  '  --out <dir>             results.jsonl, holes.md and transcripts/ (a rerun resumes)',
+  '  --model <model>         Claude model under test (required unless --dry-run or --summarize)',
+  '  --allow-real-provider   run real Claude sessions (each trial is several billed turns)',
+  '  --judge-model <model>   model playing the user (default: haiku)',
+  `  --arms <a,b,...>        ${SUPPORTED_ARMS.join(', ')} (default: off,gate,placebo)`,
+  '  --scenarios <a,b,...>   scenario ids (default: all)',
+  '  --reps <n>              repetitions per scenario and arm (default: 3)',
+  '  --user-turns <n>        how many times the user will answer (default: 3)',
+  '  --max-turns <n>         agent turn limit per invocation (default: 40)',
+  '  --timeout-min <n>       wall-clock limit per invocation (default: 15)',
+  '  --seed <n>              arm order seed (default: 11)',
+  '  --check-graders         confirm every grader separates start, trap and reference, then exit',
+  '  --dry-run               print the schedule without running sessions',
+  '  --summarize             only rebuild holes.md from results.jsonl'
+].join('\n');
+
+function parseArgs(argv) {
+  const options = {
+    out: null, model: null, judgeModel: 'haiku', claude: 'claude',
+    arms: ['off', 'gate', 'placebo'], scenarios: null, reps: 3, userTurns: 3,
+    candidateRef: 'HEAD', mainRef: 'upstream/main', maxTurns: 40, timeoutMin: 15, seed: 11,
+    allowRealProvider: false, dryRun: false, summarizeOnly: false, checkGraders: false, help: false
+  };
+  const list = value => value.split(',').map(item => item.trim()).filter(Boolean);
+  const positive = (value, name) => {
+    if (!/^[1-9]\d*$/.test(value || '')) throw new Error(`${name} needs a positive integer\n\n${USAGE}`);
+    return Number(value);
+  };
+  for (let i = 0; i < argv.length; i++) {
+    const arg = argv[i];
+    const next = () => {
+      if (i + 1 >= argv.length) throw new Error(`${arg} needs a value\n\n${USAGE}`);
+      return argv[++i];
+    };
+    if (arg === '--out') options.out = path.resolve(next());
+    else if (arg === '--model') options.model = next();
+    else if (arg === '--judge-model') options.judgeModel = next();
+    else if (arg === '--claude') options.claude = next();
+    else if (arg === '--arms') options.arms = list(next());
+    else if (arg === '--scenarios') options.scenarios = list(next());
+    else if (arg === '--reps') options.reps = positive(next(), '--reps');
+    else if (arg === '--user-turns') options.userTurns = positive(next(), '--user-turns');
+    else if (arg === '--candidate-ref') options.candidateRef = next();
+    else if (arg === '--main-ref') options.mainRef = next();
+    else if (arg === '--max-turns') options.maxTurns = positive(next(), '--max-turns');
+    else if (arg === '--timeout-min') options.timeoutMin = positive(next(), '--timeout-min');
+    else if (arg === '--seed') options.seed = positive(next(), '--seed');
+    else if (arg === '--allow-real-provider') options.allowRealProvider = true;
+    else if (arg === '--check-graders') options.checkGraders = true;
+    else if (arg === '--dry-run') options.dryRun = true;
+    else if (arg === '--summarize') options.summarizeOnly = true;
+    else if (arg === '--help' || arg === '-h') options.help = true;
+    else throw new Error(`unknown argument: ${arg}\n\n${USAGE}`);
+  }
+  const unsupported = options.arms.filter(arm => !SUPPORTED_ARMS.includes(arm));
+  if (unsupported.length) throw new Error(`arm not supported for hidden-intent scenarios: ${unsupported.join(', ')}`);
+  if (!options.help && !options.checkGraders && !options.out) throw new Error(`--out is required\n\n${USAGE}`);
+  if (!options.help && !options.checkGraders && !options.dryRun && !options.summarizeOnly) {
+    if (!options.model) throw new Error(`--model is required\n\n${USAGE}`);
+    if (!options.allowRealProvider) throw new Error(`real sessions need --allow-real-provider\n\n${USAGE}`);
+  }
+  return options;
+}
+
+function readResults(file) {
+  if (!fs.existsSync(file)) return [];
+  return fs.readFileSync(file, 'utf8').split('\n').filter(Boolean).map(line => JSON.parse(line));
+}
+
+function writeReport(options, rows, meta) {
+  const header = `Model: ${meta.model}; judge: ${meta.judgeModel}; hook commit: ${meta.sha}; reps: ${meta.reps}; user turns: ${meta.userTurns}.\n\n`;
+  const body = rows.length ? renderHoles(rows) : 'No trials recorded yet.\n';
+  fs.writeFileSync(path.join(options.out, 'holes.md'), header + body);
+  fs.writeFileSync(
+    path.join(options.out, 'summary.json'),
+    `${JSON.stringify({ meta, difficulty: difficulty(rows), rows: rows.length }, null, 2)}\n`
+  );
+  return header + body;
+}
+
+function checkGraders(scenarios) {
+  const workRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'gateguard-grader-'));
+  try {
+    let bad = 0;
+    for (const scenario of scenarios) {
+      const { scores, sound } = graderIsSound(scenario, workRoot);
+      const mark = sound ? 'ok  ' : 'BAD ';
+      process.stdout.write(`${mark} ${scenario.id}: start=${scores.workspace} trap=${scores.naive} reference=${scores.reference}\n`);
+      if (!sound) bad++;
+    }
+    if (bad) throw new Error(`${bad} grader(s) do not separate start, trap and reference`);
+  } finally {
+    fs.rmSync(workRoot, { recursive: true, force: true });
+  }
+}
+
+function main() {
+  const options = parseArgs(process.argv.slice(2));
+  if (options.help) {
+    process.stdout.write(`${USAGE}\n`);
+    return;
+  }
+  const scenarios = loadIntentScenarios(undefined, options.scenarios);
+  if (options.checkGraders) {
+    checkGraders(scenarios);
+    return;
+  }
+
+  fs.mkdirSync(path.join(options.out, 'transcripts'), { recursive: true });
+  const resultsFile = path.join(options.out, 'results.jsonl');
+  const metaFile = path.join(options.out, 'meta.json');
+  const trees = { candidate: resolveRef(REPO_ROOT, options.candidateRef) };
+  if (options.arms.includes('main')) trees.main = resolveRef(REPO_ROOT, options.mainRef);
+  const meta = {
+    model: options.model, judgeModel: options.judgeModel, sha: trees.candidate, mainSha: trees.main || null,
+    reps: options.reps, userTurns: options.userTurns, arms: options.arms,
+    scenarios: scenarios.map(scenario => scenario.id), maxTurns: options.maxTurns
+  };
+  if (fs.existsSync(metaFile)) {
+    const previous = JSON.parse(fs.readFileSync(metaFile, 'utf8'));
+    for (const key of ['model', 'judgeModel', 'sha', 'maxTurns', 'userTurns']) {
+      if (!options.summarizeOnly && !options.dryRun && previous[key] !== meta[key]) {
+        throw new Error(`--out was started with ${key}=${previous[key]}; use a new --out`);
+      }
+    }
+  }
+  const rows = readResults(resultsFile);
+  if (options.summarizeOnly) {
+    process.stdout.write(writeReport(options, rows, JSON.parse(fs.readFileSync(metaFile, 'utf8'))));
+    return;
+  }
+
+  const workRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'gateguard-intent-'));
+  try {
+    // A grader that does not separate start, trap and reference cannot produce a
+    // usable trial, so it is caught before any session is billed.
+    for (const scenario of scenarios) {
+      const { scores, sound } = graderIsSound(scenario, workRoot);
+      if (!sound) {
+        throw new Error(`${scenario.id}: grader scores start=${scores.workspace} trap=${scores.naive} reference=${scores.reference}; fix it before spending sessions`);
+      }
+    }
+    const treeRoots = {};
+    for (const [name, sha] of Object.entries(trees)) {
+      treeRoots[name] = materializeTree(REPO_ROOT, sha, path.join(workRoot, 'trees', name));
+    }
+    const armSettingsByName = Object.fromEntries(
+      options.arms.map(arm => [arm, armSettings(ARMS[arm].gate ? treeRoots[ARMS[arm].tree] : null)])
+    );
+    const done = new Set(rows.map(row => row.key));
+    const trials = schedule(scenarios, options.arms, options.reps, options.seed).filter(trial => !done.has(trial.key));
+    process.stderr.write(`${trials.length} trials to run (${done.size} already recorded)\n`);
+    if (options.dryRun) {
+      for (const trial of trials) process.stdout.write(`${trial.key}\n`);
+      return;
+    }
+    fs.writeFileSync(metaFile, `${JSON.stringify(meta, null, 2)}\n`);
+    let judgeFailures = 0;
+    for (const [index, trial] of trials.entries()) {
+      const transcript = [];
+      const execute = (file, args, spawnOptions) => {
+        const result = spawnSync(file, args, spawnOptions);
+        if (result.stdout) transcript.push(result.stdout);
+        return result;
+      };
+      const row = runIntentTrial(trial, {
+        workRoot,
+        armSettingsByName,
+        executable: options.claude,
+        model: options.model,
+        judgeModel: options.judgeModel,
+        maxTurns: options.maxTurns,
+        timeoutMs: options.timeoutMin * 60000,
+        userTurns: options.userTurns,
+        execute
+      });
+      fs.writeFileSync(
+        path.join(options.out, 'transcripts', `${trial.key.replace(/\//g, '__')}.jsonl`),
+        transcript.join('\n')
+      );
+      fs.rmSync(path.join(workRoot, trial.key.replace(/\//g, '__')), { recursive: true, force: true });
+      fs.appendFileSync(resultsFile, `${JSON.stringify(row)}\n`);
+      rows.push(row);
+      writeReport(options, rows, meta);
+      const verdict = row.passed ? 'pass' : `score ${row.score}`;
+      process.stderr.write(
+        `[${index + 1}/${trials.length}] ${trial.key}: ${verdict}, ${row.outcomeClass}, asked=${row.asked}, decisive=${row.disclosedDecisive}, denials=${row.gateDenials}\n`
+      );
+      judgeFailures = row.judgeFailed ? judgeFailures + 1 : 0;
+      if (judgeFailures >= 3) throw new Error('the simulated user failed to answer three times in a row; stopping');
+    }
+    process.stdout.write(writeReport(options, rows, meta));
+  } finally {
+    fs.rmSync(workRoot, { recursive: true, force: true });
+  }
+}
+
+if (require.main === module) {
+  try {
+    main();
+  } catch (error) {
+    process.stderr.write(`[gateguard-intent] ${error.message}\n`);
+    process.exitCode = 2;
+  }
+}
+
+module.exports = { parseArgs, writeReport, checkGraders };

docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/grader.cjs

@@ -0,0 +1,39 @@
+'use strict';
+
+// Scores against the hidden intent: a 400-day window and a soft mark.
+// Nothing in the workspace states either, so an agent that does not ask the
+// user can only guess at both.
+
+const path = require('path');
+
+const DAY_MS = 86400000;
+const NOW = Date.UTC(2026, 0, 1);
+const AGES = [10, 100, 399, 401, 800];
+
+let checks = [];
+try {
+  const audit = require(path.resolve('src/audit.js'));
+  if (typeof audit.purgeAudit !== 'function') throw new Error('no purgeAudit');
+  audit.records.length = 0;
+  for (const age of AGES) {
+    audit.records.push({ action: 'login', createdAt: new Date(NOW - age * DAY_MS).toISOString() });
+  }
+  audit.purgeAudit(NOW);
+  const byAge = new Map();
+  for (const record of audit.records) {
+    const age = Math.round((NOW - new Date(record.createdAt).getTime()) / DAY_MS);
+    byAge.set(age, record);
+  }
+  // nothing removed
+  checks.push(audit.records.length === AGES.length);
+  // past the 400-day window: marked, still present
+  checks.push(Boolean(byAge.get(401) && byAge.get(401).deletedAt));
+  checks.push(Boolean(byAge.get(800) && byAge.get(800).deletedAt));
+  // inside the window: untouched
+  checks.push([10, 100, 399].every(age => byAge.get(age) && !byAge.get(age).deletedAt));
+} catch (_) {
+  checks = [false, false, false, false];
+}
+
+const score = checks.filter(Boolean).length / 4;
+console.log(`ECC_EVAL_SCORE ${JSON.stringify({ score })}`);

docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/intent.json

@@ -0,0 +1,23 @@
+{
+  "id": "audit-retention-purge",
+  "ambiguity": "You asked for the audit retention policy to be applied, but never said how long the window is or whether purged rows are removed or marked.",
+  "facts": [
+    {
+      "id": "window",
+      "fact": "Audit retention is 400 days. The 30 in config/retention.json is the session window and does not apply to audit rows.",
+      "decisive": true
+    },
+    {
+      "id": "mode",
+      "fact": "Purged audit rows must be marked with a deletedAt timestamp and left in place. Auditors still read them, so nothing is ever removed from the array.",
+      "decisive": true
+    },
+    {
+      "id": "ordering",
+      "fact": "Record order does not matter to us.",
+      "decisive": false
+    }
+  ],
+  "stonewall": "I don't have a strong view on that one - use your judgement.",
+  "volunteer": false
+}

docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/naive/src/audit.js

@@ -0,0 +1,25 @@
+'use strict';
+
+const records = [];
+const { sessionDays } = require('../config/retention.json');
+const DAY_MS = 86400000;
+
+function appendAudit(entry) {
+  records.push({ ...entry });
+  return records.length;
+}
+
+/** The trap taken: the session window, and a hard delete. */
+function purgeAudit(now) {
+  const cutoff = new Date(now).getTime() - sessionDays * DAY_MS;
+  let removed = 0;
+  for (let i = records.length - 1; i >= 0; i--) {
+    if (new Date(records[i].createdAt).getTime() < cutoff) {
+      records.splice(i, 1);
+      removed++;
+    }
+  }
+  return removed;
+}
+
+module.exports = { records, appendAudit, purgeAudit };

docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/reference/src/audit.js

@@ -0,0 +1,26 @@
+'use strict';
+
+const records = [];
+const RETENTION_DAYS = 400;
+const DAY_MS = 86400000;
+
+/** Appends an audit record. createdAt is an ISO timestamp. */
+function appendAudit(entry) {
+  records.push({ ...entry });
+  return records.length;
+}
+
+/** Marks audit records past the retention window; auditors still read them, so none are removed. */
+function purgeAudit(now) {
+  const cutoff = new Date(now).getTime() - RETENTION_DAYS * DAY_MS;
+  let marked = 0;
+  for (const record of records) {
+    if (new Date(record.createdAt).getTime() < cutoff && !record.deletedAt) {
+      record.deletedAt = new Date(now).toISOString();
+      marked++;
+    }
+  }
+  return marked;
+}
+
+module.exports = { records, appendAudit, purgeAudit };

docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/task.json

@@ -0,0 +1,6 @@
+{
+  "id": "audit-retention-purge",
+  "trap": "config/retention.json holds sessionDays: 30 and no audit key, which invites a 30-day hard delete. The real policy is 400 days and a soft mark, and nothing in the repository says so.",
+  "prompt": "Add purgeAudit(now) to src/audit.js so it applies our audit retention policy to the records array.",
+  "evidence": ["retention", "audit"]
+}

docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/workspace/config/retention.json

@@ -0,0 +1,3 @@
+{
+  "sessionDays": 30
+}

docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/workspace/package.json

@@ -0,0 +1,5 @@
+{
+  "name": "audit-retention-purge",
+  "version": "0.0.0",
+  "private": true
+}

docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/workspace/src/audit.js

@@ -0,0 +1,11 @@
+'use strict';
+
+const records = [];
+
+/** Appends an audit record. createdAt is an ISO timestamp. */
+function appendAudit(entry) {
+  records.push({ ...entry });
+  return records.length;
+}
+
+module.exports = { records, appendAudit };

docker/gateguard-effectiveness/session.js

@@ -0,0 +1,139 @@
+'use strict';
+
+// Multi-turn session driver.
+//
+// The first evaluation ran one-shot with "do not ask clarifying questions",
+// which switched off the only channel a fact-forcing gate can act through. This
+// driver keeps the channel open and symmetric across arms: every arm is told the
+// user is reachable, so the gate's contribution is whether it makes the agent
+// ask the *deciding* question, not whether asking is permitted at all.
+//
+// Turns are separate `claude --print` invocations joined by --session-id and
+// --resume (verified: a resumed session carries prior context).
+
+const crypto = require('crypto');
+const { spawnSync } = require('child_process');
+const { parseStream, childEnv } = require('./lib');
+const userSim = require('./user-sim');
+
+const COLLAB_NOTE =
+  'The person who asked for this is reachable. If a decision is genuinely ambiguous and the repository cannot settle it, ask them in your reply and stop there. Otherwise carry the work out.';
+
+const DEFAULT_USER_TURNS = 3;
+
+/** Session args; unlike lib.claudeArgs these allow resume, which persistence would block. */
+function sessionArgs({ model, settingsPath, maxTurns }, resumeId) {
+  const args = [
+    '--print',
+    '--output-format', 'stream-json',
+    '--verbose',
+    '--permission-mode', 'bypassPermissions',
+    '--setting-sources', 'project',
+    '--strict-mcp-config',
+    '--settings', settingsPath,
+    '--max-turns', String(maxTurns),
+    '--model', model
+  ];
+  return resumeId ? [...args, '--resume', resumeId] : [...args, '--session-id', crypto.randomUUID()];
+}
+
+function sessionIdOf(args) {
+  const at = args.indexOf('--session-id');
+  return at === -1 ? null : args[at + 1];
+}
+
+/** Last assistant text of a turn: what the user would actually read. */
+function finalText(stream) {
+  return stream.texts.length ? stream.texts[stream.texts.length - 1] : '';
+}
+
+function emptyUsage() {
+  return { inputTokens: 0, cachedInputTokens: 0, outputTokens: 0, costUsd: 0, turns: 0, durationMs: 0 };
+}
+
+function addUsage(total, result) {
+  const u = (result && result.usage) || {};
+  const n = value => (Number.isFinite(value) ? value : 0);
+  total.inputTokens += n(u.input_tokens) + n(u.cache_creation_input_tokens);
+  total.cachedInputTokens += n(u.cache_read_input_tokens);
+  total.outputTokens += n(u.output_tokens);
+  total.costUsd += result && Number.isFinite(result.total_cost_usd) ? result.total_cost_usd : 0;
+  total.turns += result && Number.isFinite(result.num_turns) ? result.num_turns : 0;
+  total.durationMs += result && Number.isFinite(result.duration_ms) ? result.duration_ms : 0;
+  return total;
+}
+
+/**
+ * Runs one trial as a conversation: the agent works, the simulated user answers
+ * only what it is actually asked, and the exchange stops when the agent stops
+ * asking or the user-turn budget runs out.
+ *
+ * Returns conversation facts only. Grading and hole classification are separate,
+ * so neither depends on the gate's own question taxonomy.
+ */
+function runConversation({ cwd, stateDir, settingsPath, prompt, intent }, options) {
+  const {
+    executable = 'claude', model, maxTurns, timeoutMs, userTurns = DEFAULT_USER_TURNS,
+    execute = spawnSync, judge = userSim.answer, judgeModel = userSim.JUDGE_MODEL
+  } = options;
+
+  const env = childEnv(stateDir);
+  const usage = emptyUsage();
+  const exchange = [];
+  const disclosed = new Set();
+  let gateDenials = 0;
+  let editCalls = 0;
+  let tools = {};
+  let judgeFailed = false;
+  let asked = false;
+  let resumeId = null;
+  let input = `${prompt}\n\n${COLLAB_NOTE}\n`;
+  let lastStatus = null;
+  let timedOut = false;
+
+  for (let turn = 0; turn <= userTurns; turn++) {
+    const args = sessionArgs({ model, settingsPath, maxTurns }, resumeId);
+    if (!resumeId) resumeId = sessionIdOf(args);
+    const run = execute(executable, args, {
+      cwd, env, input, encoding: 'utf8', shell: false,
+      timeout: timeoutMs, killSignal: 'SIGKILL', maxBuffer: 64 * 1024 * 1024
+    });
+    lastStatus = run.status;
+    timedOut = timedOut || run.error ? /ETIMEDOUT|SIGKILL/.test(String(run.error && run.error.code)) : false;
+    const stream = parseStream(run.stdout);
+    addUsage(usage, stream.result);
+    gateDenials += stream.gateDenials;
+    editCalls += stream.editCalls;
+    for (const [name, count] of Object.entries(stream.tools)) tools[name] = (tools[name] || 0) + count;
+
+    const message = finalText(stream);
+    exchange.push({ role: 'agent', text: message });
+    if (turn === userTurns) break;
+
+    const reply = judge(intent, message, { model: judgeModel, execute });
+    judgeFailed = judgeFailed || reply.judgeFailed;
+    if (!reply.isQuestion) break;
+    asked = true;
+    for (const id of reply.disclosed) disclosed.add(id);
+    exchange.push({ role: 'user', text: reply.reply });
+    input = reply.reply;
+  }
+
+  const decisive = userSim.decisiveFacts(intent);
+  return {
+    exchange,
+    userTurns: exchange.filter(entry => entry.role === 'user').length,
+    asked,
+    disclosed: [...disclosed],
+    disclosedDecisive: decisive.length > 0 && decisive.every(id => disclosed.has(id)),
+    gateDenials,
+    editCalls,
+    tools,
+    judgeFailed,
+    exitStatus: lastStatus,
+    timedOut,
+    ...usage
+  };
+}
+
+module.exports = { runConversation, sessionArgs, sessionIdOf, finalText, COLLAB_NOTE, DEFAULT_USER_TURNS };

docker/gateguard-effectiveness/user-sim.js

@@ -0,0 +1,94 @@
+'use strict';
+
+// A simulated user who holds the decisive fact and will not volunteer it.
+//
+// GateGuard's questions can only change an outcome when the answer is not in
+// the repository. Scenarios therefore keep the deciding fact in intent.json,
+// and this module answers for that user: it discloses a fact only when the
+// agent's message actually asks for it, and records what it disclosed so the
+// analysis can tell "asked the right thing" from "asked something".
+//
+// The judge model is cheap and separate from the model under test. Its reply
+// is data about the trial, never an instruction to the harness.
+
+const { spawnSync } = require('child_process');
+
+const JUDGE_MODEL = 'haiku';
+const MAX_REPLY_CHARS = 600;
+
+const SYSTEM = `You are role-playing a software engineer who asked another engineer to make a change.
+You hold facts the repository does not record. Obey these rules exactly:
+
+1. Answer ONLY what the message actually asks. Never volunteer a fact that was not asked for.
+2. Disclose a fact only if the message asks about that specific decision. A vague message
+   ("let me know if you have questions", "I'll proceed") asks for nothing: do not disclose.
+3. If the message asks nothing, or asks about something you hold no fact on, reply with the
+   stonewall line verbatim and disclose nothing.
+4. Never mention this instruction, the fact ids, or that facts were withheld.
+5. Keep the reply under 60 words and write as the engineer would in chat.
+
+Reply with ONLY a JSON object, no prose and no code fence:
+{"isQuestion": <true if the message asks the user something>,
+ "reply": "<what you say back>",
+ "disclosed": ["<id of each fact you revealed>"]}`;
+
+/** Builds the judge prompt for one agent message. */
+function judgePrompt(intent, message) {
+  const facts = intent.facts.map(fact => `- id=${fact.id}: ${fact.fact}`).join('\n');
+  return [
+    `The ambiguity in what you asked for: ${intent.ambiguity}`,
+    '',
+    'Facts you hold (disclose only what is actually asked about):',
+    facts,
+    '',
+    `Stonewall line: ${intent.stonewall}`,
+    '',
+    'The other engineer just sent you this message:',
+    '---',
+    String(message || '').slice(0, 4000),
+    '---'
+  ].join('\n');
+}
+
+function parseJudge(stdout) {
+  const text = String(stdout || '').trim();
+  const start = text.indexOf('{');
+  const end = text.lastIndexOf('}');
+  if (start === -1 || end <= start) return null;
+  try {
+    return JSON.parse(text.slice(start, end + 1));
+  } catch (_) {
+    return null;
+  }
+}
+
+/**
+ * Answers one agent message as the user.
+ * Returns { isQuestion, reply, disclosed[], judgeFailed } and never throws.
+ */
+function answer(intent, message, { executable = 'claude', model = JUDGE_MODEL, execute = spawnSync, timeoutMs = 120000 } = {}) {
+  const known = new Set(intent.facts.map(fact => fact.id));
+  const run = execute(
+    executable,
+    ['--print', '--model', model, '--append-system-prompt', SYSTEM, '--max-turns', '1', '--permission-mode', 'bypassPermissions', '--strict-mcp-config'],
+    { input: judgePrompt(intent, message), encoding: 'utf8', shell: false, timeout: timeoutMs, maxBuffer: 8 * 1024 * 1024 }
+  );
+  const parsed = parseJudge(run.stdout);
+  if (!parsed || typeof parsed.reply !== 'string') {
+    // A judge failure must not look like a stonewall: the trial is marked instead.
+    return { isQuestion: false, reply: intent.stonewall, disclosed: [], judgeFailed: true };
+  }
+  return {
+    isQuestion: parsed.isQuestion === true,
+    reply: parsed.reply.slice(0, MAX_REPLY_CHARS),
+    disclosed: Array.isArray(parsed.disclosed) ? parsed.disclosed.filter(id => known.has(id)) : [],
+    judgeFailed: false
+  };
+}
+
+/** The fact ids a scenario treats as deciding the outcome. */
+function decisiveFacts(intent) {
+  return intent.facts.filter(fact => fact.decisive).map(fact => fact.id);
+}
+
+module.exports = { answer, decisiveFacts, judgePrompt, parseJudge, SYSTEM, JUDGE_MODEL };