From 7afde35e4936598266717885b49c39e96523da1a Mon Sep 17 00:00:00 2001 From: montjeffrey Date: Fri, 2 Oct 2026 09:53:08 -0400 Subject: [PATCH] feat(gateguard): measure question effectiveness against a hidden user intent The existing harness cannot measure what it set out to measure. Every trap hides its deciding fact in the repository, which agents reliably discover, and trials run one-shot under "do not ask clarifying questions", so a fact-forcing gate is evaluated with no user to answer it. Its scenarios also declare the question the gate ought to raise, chosen by reading the gate's own taxonomy, so the suite passes by construction and can never find a missing question. Add a second runner that measures effectiveness instead of coverage: - scenarios-intent/ keeps the deciding fact in intent.json, as a choice between plausible conventions the repository cannot settle, so an agent that does not ask can only guess and the ungated arm cannot sit at the ceiling. - user-sim.js plays that user with a separate cheap model: it answers the question in front of it, never volunteers, and reports which fact ids it disclosed, so asking something and asking the deciding thing are told apart. An unparseable judge is a recorded failure, not a silent stonewall. - session.js runs multi-turn sessions joined by --session-id and --resume, and tells every arm the user is reachable, so the comparison is symmetric and the gate's contribution is whether it makes the agent ask the deciding question. - coverage.js classifies each trial from behaviour alone into working, lucky, or a follow-through, targeting, silence or coverage hole, and reports the questions the gate actually raised rather than asserting declared ones. - run-intent.js refuses minus-target, which needs targets these scenarios deliberately lack, and checks every grader separates the start, the trap and the reference before a session is billed. A scenario counts as carrying no information only when nothing varies across arms. Trap strength is reported separately: the ungated arm failing is what a working trap looks like, so it must not make a scenario unusable. --- docker/gateguard-effectiveness/README.md | 20 ++ docker/gateguard-effectiveness/coverage.js | 172 ++++++++++ docker/gateguard-effectiveness/intent-eval.js | 128 ++++++++ docker/gateguard-effectiveness/run-intent.js | 231 ++++++++++++++ .../audit-retention-purge/grader.cjs | 39 +++ .../audit-retention-purge/intent.json | 23 ++ .../audit-retention-purge/naive/src/audit.js | 25 ++ .../reference/src/audit.js | 26 ++ .../audit-retention-purge/task.json | 6 + .../workspace/config/retention.json | 3 + .../workspace/package.json | 5 + .../workspace/src/audit.js | 11 + docker/gateguard-effectiveness/session.js | 139 ++++++++ docker/gateguard-effectiveness/user-sim.js | 94 ++++++ tests/docker/gateguard-intent.test.js | 297 ++++++++++++++++++ 15 files changed, 1219 insertions(+) create mode 100644 docker/gateguard-effectiveness/coverage.js create mode 100644 docker/gateguard-effectiveness/intent-eval.js create mode 100644 docker/gateguard-effectiveness/run-intent.js create mode 100644 docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/grader.cjs create mode 100644 docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/intent.json create mode 100644 docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/naive/src/audit.js create mode 100644 docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/reference/src/audit.js create mode 100644 docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/task.json create mode 100644 docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/workspace/config/retention.json create mode 100644 docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/workspace/package.json create mode 100644 docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/workspace/src/audit.js create mode 100644 docker/gateguard-effectiveness/session.js create mode 100644 docker/gateguard-effectiveness/user-sim.js create mode 100644 tests/docker/gateguard-intent.test.js diff --git a/docker/gateguard-effectiveness/README.md b/docker/gateguard-effectiveness/README.md index 6b7f9c298..3aa3770cb 100644 --- a/docker/gateguard-effectiveness/README.md +++ b/docker/gateguard-effectiveness/README.md @@ -4,7 +4,27 @@ Development-only harness that runs real Claude sessions on trapped coding tasks with and without GateGuard's questions. Design, arms, measures and results: [docs/gateguard/question-effectiveness.md](../../docs/gateguard/question-effectiveness.md). +Two runners, measuring different things. + +`run.js` — cost baseline. One-shot autonomous sessions over `scenarios/`, where +each trap hides its deciding fact in the repository. The pilot returned no +outcome difference between any arm, so treat this as a measure of the gate's +turn and token cost, not of its effect. + ```bash node docker/gateguard-effectiveness/run.js --out gg-eff --dry-run node docker/gateguard-effectiveness/run.js --out gg-eff --model --allow-real-provider ``` + +`run-intent.js` — effectiveness. Multi-turn sessions over `scenarios-intent/`, +where the deciding fact exists only in a simulated user's hidden intent, so an +agent that does not ask can only guess. Scenarios are authored blind to the +gate's question taxonomy, and the report classifies each trial into a hole worth +fixing. + +```bash +node docker/gateguard-effectiveness/run-intent.js --check-graders +node docker/gateguard-effectiveness/run-intent.js --out gg-intent --arms off,gate --dry-run +node docker/gateguard-effectiveness/run-intent.js --out gg-intent --model --allow-real-provider +node docker/gateguard-effectiveness/run-intent.js --out gg-intent --summarize +``` diff --git a/docker/gateguard-effectiveness/coverage.js b/docker/gateguard-effectiveness/coverage.js new file mode 100644 index 000000000..7fedb62c6 --- /dev/null +++ b/docker/gateguard-effectiveness/coverage.js @@ -0,0 +1,172 @@ +'use strict'; + +// Hole finding. +// +// The first evaluation asserted that the gate asks each scenario's declared +// targetQuestions. Those targets were chosen by reading the gate's own +// taxonomy, so the check passed by construction and could never find a missing +// question. Nothing here reads the taxonomy. Every class below is computed from +// what the trial *did*: whether the gate fired, whether the agent asked the +// user, whether the deciding fact came out, and whether the outcome was right. +// +// The questions a scenario provokes are therefore discovered and reported, not +// declared and asserted. + +/** Outcome classes. The four ending in -hole are the improvable cases. */ +const CLASSES = Object.freeze({ + working: 'the deciding fact was obtained and the outcome is right', + lucky: 'right outcome without ever asking: the scenario does not force the question', + 'follow-through-hole': 'the deciding fact was obtained and the outcome is still wrong', + 'targeting-hole': 'the agent asked, but not for the deciding fact', + 'silence-hole': 'the gate fired and the agent still did not ask', + 'coverage-hole': 'a deciding ambiguity the gate never engaged, and the outcome is wrong' +}); + +/** Classifies one trial from its behaviour alone. */ +function classify(row) { + const gateFired = Number(row.gateDenials) > 0; + if (row.disclosedDecisive) return row.passed ? 'working' : 'follow-through-hole'; + if (row.asked) return 'targeting-hole'; + if (row.passed) return 'lucky'; + return gateFired ? 'silence-hole' : 'coverage-hole'; +} + +function tally(rows) { + const counts = {}; + for (const key of Object.keys(CLASSES)) counts[key] = 0; + for (const row of rows) counts[classify(row)]++; + return counts; +} + +/** + * Trap strength: how often the ungated arm passes. + * + * A trap the ungated arm always beats is not a trap. This is reported for its + * own sake and does NOT decide whether a scenario is usable, because the + * ungated arm failing is what a working trap looks like. + */ +function difficulty(rows, { baselineArm = 'off' } = {}) { + const byScenario = {}; + for (const row of rows.filter(row => row.arm === baselineArm)) { + (byScenario[row.scenario] ||= []).push(row); + } + return Object.entries(byScenario) + .map(([scenario, rs]) => { + const passed = rs.filter(row => row.passed).length; + const rate = passed / rs.length; + return { scenario, n: rs.length, passed, rate, label: rate === 1 ? 'ceiling' : 'traps' }; + }) + .sort((a, b) => a.scenario.localeCompare(b.scenario)); +} + +/** + * Whether a scenario separates any arm from any other, judged across every arm. + * + * Only a scenario where nothing varies is uninformative: every trial passing + * (the trap does not bite) or every trial failing (nothing reaches it). Judging + * this from the ungated arm alone would discard exactly the scenarios that + * work, since there the ungated arm is meant to fail. + */ +function informative(rows) { + const byScenario = {}; + for (const row of rows) (byScenario[row.scenario] ||= []).push(row); + return Object.entries(byScenario) + .map(([scenario, rs]) => { + const passed = rs.filter(row => row.passed).length; + const label = passed === rs.length ? 'ceiling' : passed === 0 ? 'floor' : 'informative'; + return { scenario, n: rs.length, passed, label }; + }) + .sort((a, b) => a.scenario.localeCompare(b.scenario)); +} + +/** Questions the gate actually raised per scenario, discovered from the runs. */ +function discoveredQuestions(rows) { + const byScenario = {}; + for (const row of rows) { + const set = (byScenario[row.scenario] ||= new Set()); + for (const question of row.questionsAsked || []) set.add(question); + } + return Object.entries(byScenario) + .map(([scenario, set]) => ({ scenario, questions: [...set].sort() })) + .sort((a, b) => a.scenario.localeCompare(b.scenario)); +} + +/** + * Scenarios where the gate asked nothing at all, across every gated arm. + * These are the strongest candidates for a missing question. + */ +function unengaged(rows, { baselineArm = 'off' } = {}) { + const gated = rows.filter(row => row.arm !== baselineArm); + const byScenario = {}; + for (const row of gated) { + const entry = (byScenario[row.scenario] ||= { fired: 0, n: 0 }); + entry.n++; + if (Number(row.gateDenials) > 0) entry.fired++; + } + return Object.entries(byScenario) + .filter(([, entry]) => entry.fired === 0) + .map(([scenario]) => scenario) + .sort(); +} + +function pct(value) { + return `${Math.round(value * 100)}%`; +} + +function renderHoles(rows, { baselineArm = 'off' } = {}) { + const arms = [...new Set(rows.map(row => row.arm))].sort(); + const classes = Object.keys(CLASSES); + const lines = []; + + const strength = difficulty(rows, { baselineArm }); + const verdicts = informative(rows); + + lines.push('### Trap strength and usability', ''); + lines.push('| Scenario | Ungated passed | Rate | Trap | Separates arms |', '| --- | ---: | ---: | --- | --- |'); + const verdictOf = Object.fromEntries(verdicts.map(row => [row.scenario, row.label])); + for (const row of strength) { + lines.push(`| ${row.scenario} | ${row.passed}/${row.n} | ${pct(row.rate)} | ${row.label} | ${verdictOf[row.scenario]} |`); + } + const dead = verdicts.filter(row => row.label !== 'informative'); + lines.push(''); + lines.push( + dead.length + ? `${dead.length} of ${verdicts.length} scenarios carry no information (${dead.map(row => `${row.scenario}: ${row.label}`).join('; ')}). Arm comparisons below exclude them.` + : `All ${verdicts.length} scenarios separate at least one arm from another.` + ); + + const usableScenarios = new Set(verdicts.filter(row => row.label === 'informative').map(row => row.scenario)); + const usable = rows.filter(row => usableScenarios.has(row.scenario)); + + lines.push('', '### Outcome classes', ''); + lines.push(`| Arm | ${classes.join(' | ')} |`, `| --- | ${classes.map(() => '---:').join(' | ')} |`); + for (const arm of arms) { + const counts = tally(usable.filter(row => row.arm === arm)); + lines.push(`| ${arm} | ${classes.map(key => counts[key]).join(' | ')} |`); + } + + lines.push('', '### Holes to work on', ''); + const holes = classes.filter(key => key.endsWith('-hole')); + const totals = tally(usable); + const present = holes.filter(key => totals[key] > 0); + if (!present.length) { + lines.push('No holes on the informative scenarios in this run.'); + } else { + for (const key of present) lines.push(`- **${key}** (${totals[key]}): ${CLASSES[key]}`); + } + + const missing = unengaged(rows, { baselineArm }); + if (missing.length) { + lines.push('', `Gate never fired on: ${missing.join(', ')} — candidates for a question the taxonomy lacks.`); + } + + lines.push('', '### Questions the gate actually raised', ''); + lines.push('| Scenario | Observed questions |', '| --- | --- |'); + for (const row of discoveredQuestions(rows)) { + lines.push(`| ${row.scenario} | ${row.questions.length ? row.questions.join(', ') : '_none_'} |`); + } + + return `${lines.join('\n')}\n`; +} + +module.exports = { CLASSES, classify, tally, difficulty, informative, discoveredQuestions, unengaged, renderHoles }; diff --git a/docker/gateguard-effectiveness/intent-eval.js b/docker/gateguard-effectiveness/intent-eval.js new file mode 100644 index 000000000..b20546a5e --- /dev/null +++ b/docker/gateguard-effectiveness/intent-eval.js @@ -0,0 +1,128 @@ +'use strict'; + +// Loader and trial runner for hidden-intent scenarios. +// +// Deliberately absent from task.json: targetQuestions. The first generation of +// scenarios declared which question the gate ought to raise, chosen by reading +// the gate's own taxonomy, and the suite then asserted the gate raised it. That +// is a tautology, so it never found a missing question. Here the scenario +// declares only the trap and the user's hidden intent; which questions fire is +// observed from the run and classified in coverage.js. + +const fs = require('fs'); +const path = require('path'); +const { prepareWorkspace, commitWorkspace, grade, readMetrics } = require('./lib'); +const { ARMS, TRIAL_FILE, armSettings, trialPatch } = require('./arms'); +const { runConversation } = require('./session'); +const { classify } = require('./coverage'); + +const SCENARIO_DIR = path.join(__dirname, 'scenarios-intent'); + +// minus-target needs declared target questions, which these scenarios do not have. +const SUPPORTED_ARMS = Object.freeze(['off', 'gate', 'placebo', 'main']); + +function loadIntentScenarios(dir = SCENARIO_DIR, ids = null) { + const scenarios = fs + .readdirSync(dir, { withFileTypes: true }) + .filter(entry => entry.isDirectory()) + .map(entry => { + const root = path.join(dir, entry.name); + const spec = JSON.parse(fs.readFileSync(path.join(root, 'task.json'), 'utf8')); + const intent = JSON.parse(fs.readFileSync(path.join(root, 'intent.json'), 'utf8')); + if (spec.id !== entry.name) throw new Error(`scenario ${entry.name} declares id ${spec.id}`); + if (spec.targetQuestions) { + throw new Error(`scenario ${spec.id} declares targetQuestions; hidden-intent scenarios are authored blind to the taxonomy`); + } + for (const key of ['trap', 'prompt']) { + if (typeof spec[key] !== 'string' || !spec[key]) throw new Error(`scenario ${spec.id} is missing ${key}`); + } + if (!Array.isArray(spec.evidence) || !spec.evidence.length) throw new Error(`scenario ${spec.id} has no evidence`); + if (typeof intent.ambiguity !== 'string' || !intent.ambiguity) throw new Error(`scenario ${spec.id} intent has no ambiguity`); + if (typeof intent.stonewall !== 'string' || !intent.stonewall) throw new Error(`scenario ${spec.id} intent has no stonewall`); + if (!Array.isArray(intent.facts) || !intent.facts.some(fact => fact.decisive)) { + throw new Error(`scenario ${spec.id} intent has no decisive fact`); + } + for (const fact of intent.facts) { + if (!fact.id || typeof fact.fact !== 'string' || !fact.fact) throw new Error(`scenario ${spec.id} has a malformed fact`); + } + return { + ...spec, + root, + intent, + evidence: spec.evidence.map(source => new RegExp(source, 'i')), + grader: fs.readFileSync(path.join(root, 'grader.cjs'), 'utf8') + }; + }) + .sort((a, b) => a.id.localeCompare(b.id)); + if (!ids) return scenarios; + const unknown = ids.filter(id => !scenarios.some(scenario => scenario.id === id)); + if (unknown.length) throw new Error(`unknown scenario: ${unknown.join(', ')}`); + return scenarios.filter(scenario => ids.includes(scenario.id)); +} + +/** Confirms the grader separates the start, the trap and a correct solution. */ +function graderIsSound(scenario, workRoot) { + const scores = {}; + for (const variant of ['workspace', 'reference', 'naive']) { + const dest = path.join(workRoot, `check-${scenario.id}-${variant}`); + fs.rmSync(dest, { recursive: true, force: true }); + prepareWorkspace(scenario, dest, variant === 'workspace' ? null : variant); + scores[variant] = grade(scenario, dest).score; + } + return { scores, sound: scores.reference === 1 && scores.workspace < 1 && scores.naive < 1 }; +} + +/** One trial: fresh workspace, a conversation with the simulated user, then the hidden grader. */ +function runIntentTrial(trial, { workRoot, armSettingsByName, executable, model, judgeModel, maxTurns, timeoutMs, userTurns, execute, judge }) { + if (!SUPPORTED_ARMS.includes(trial.arm)) throw new Error(`arm ${trial.arm} is not supported for hidden-intent scenarios`); + const dir = path.join(workRoot, trial.key.replace(/\//g, '__')); + fs.rmSync(dir, { recursive: true, force: true }); + const cwd = prepareWorkspace(trial.scenario, path.join(dir, 'repo')); + const git = commitWorkspace(cwd); + const stateDir = path.join(dir, 'state'); + fs.mkdirSync(stateDir, { recursive: true }); + const settingsPath = path.join(dir, 'settings.json'); + fs.writeFileSync(settingsPath, `${JSON.stringify(armSettingsByName[trial.arm], null, 2)}\n`); + fs.writeFileSync(path.join(dir, TRIAL_FILE), `${JSON.stringify({ stateDir, patch: trialPatch(trial.arm, trial.scenario) })}\n`); + + const conversation = runConversation( + { cwd, stateDir, settingsPath, prompt: trial.scenario.prompt, intent: trial.scenario.intent }, + { executable, model, judgeModel, maxTurns, timeoutMs, userTurns, execute, judge } + ); + const metrics = readMetrics(stateDir); + const denials = metrics.filter(event => event.decision === 'deny' || event.decision === 'routine-deny'); + const outcome = grade(trial.scenario, cwd); + + const row = { + key: trial.key, + scenario: trial.scenario.id, + arm: trial.arm, + rep: trial.rep, + git, + gated: ARMS[trial.arm].gate, + passed: outcome.passed, + score: outcome.score, + asked: conversation.asked, + userTurns: conversation.userTurns, + disclosed: conversation.disclosed, + disclosedDecisive: conversation.disclosedDecisive, + judgeFailed: conversation.judgeFailed, + gateDenials: conversation.gateDenials, + questionsAsked: [...new Set(denials.flatMap(event => event.questions || []))].sort(), + editCalls: conversation.editCalls, + tools: conversation.tools, + exitStatus: conversation.exitStatus, + timedOut: conversation.timedOut, + inputTokens: conversation.inputTokens, + cachedInputTokens: conversation.cachedInputTokens, + outputTokens: conversation.outputTokens, + costUsd: conversation.costUsd, + turns: conversation.turns, + durationMs: conversation.durationMs, + exchange: conversation.exchange + }; + row.outcomeClass = classify(row); + return row; +} + +module.exports = { loadIntentScenarios, graderIsSound, runIntentTrial, SCENARIO_DIR, SUPPORTED_ARMS }; diff --git a/docker/gateguard-effectiveness/run-intent.js b/docker/gateguard-effectiveness/run-intent.js new file mode 100644 index 000000000..c6e3202a3 --- /dev/null +++ b/docker/gateguard-effectiveness/run-intent.js @@ -0,0 +1,231 @@ +#!/usr/bin/env node +'use strict'; + +// Hidden-intent effectiveness run. +// +// Separate from run.js on purpose: run.js measures the gate's cost in one-shot +// autonomous sessions and remains the cost baseline. This runner measures +// whether the gate's questions obtain a fact the repository does not hold, and +// reports the holes rather than asserting coverage. + +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { spawnSync } = require('child_process'); +const { schedule } = require('./lib'); +const { ARMS, armSettings, resolveRef, materializeTree } = require('./arms'); +const { loadIntentScenarios, graderIsSound, runIntentTrial, SUPPORTED_ARMS } = require('./intent-eval'); +const { renderHoles, difficulty } = require('./coverage'); + +const REPO_ROOT = path.resolve(__dirname, '..', '..'); + +const USAGE = [ + 'Usage: node docker/gateguard-effectiveness/run-intent.js --out --model --allow-real-provider [options]', + '', + ' --out results.jsonl, holes.md and transcripts/ (a rerun resumes)', + ' --model Claude model under test (required unless --dry-run or --summarize)', + ' --allow-real-provider run real Claude sessions (each trial is several billed turns)', + ' --judge-model model playing the user (default: haiku)', + ` --arms ${SUPPORTED_ARMS.join(', ')} (default: off,gate,placebo)`, + ' --scenarios scenario ids (default: all)', + ' --reps repetitions per scenario and arm (default: 3)', + ' --user-turns how many times the user will answer (default: 3)', + ' --max-turns agent turn limit per invocation (default: 40)', + ' --timeout-min wall-clock limit per invocation (default: 15)', + ' --seed arm order seed (default: 11)', + ' --check-graders confirm every grader separates start, trap and reference, then exit', + ' --dry-run print the schedule without running sessions', + ' --summarize only rebuild holes.md from results.jsonl' +].join('\n'); + +function parseArgs(argv) { + const options = { + out: null, model: null, judgeModel: 'haiku', claude: 'claude', + arms: ['off', 'gate', 'placebo'], scenarios: null, reps: 3, userTurns: 3, + candidateRef: 'HEAD', mainRef: 'upstream/main', maxTurns: 40, timeoutMin: 15, seed: 11, + allowRealProvider: false, dryRun: false, summarizeOnly: false, checkGraders: false, help: false + }; + const list = value => value.split(',').map(item => item.trim()).filter(Boolean); + const positive = (value, name) => { + if (!/^[1-9]\d*$/.test(value || '')) throw new Error(`${name} needs a positive integer\n\n${USAGE}`); + return Number(value); + }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + const next = () => { + if (i + 1 >= argv.length) throw new Error(`${arg} needs a value\n\n${USAGE}`); + return argv[++i]; + }; + if (arg === '--out') options.out = path.resolve(next()); + else if (arg === '--model') options.model = next(); + else if (arg === '--judge-model') options.judgeModel = next(); + else if (arg === '--claude') options.claude = next(); + else if (arg === '--arms') options.arms = list(next()); + else if (arg === '--scenarios') options.scenarios = list(next()); + else if (arg === '--reps') options.reps = positive(next(), '--reps'); + else if (arg === '--user-turns') options.userTurns = positive(next(), '--user-turns'); + else if (arg === '--candidate-ref') options.candidateRef = next(); + else if (arg === '--main-ref') options.mainRef = next(); + else if (arg === '--max-turns') options.maxTurns = positive(next(), '--max-turns'); + else if (arg === '--timeout-min') options.timeoutMin = positive(next(), '--timeout-min'); + else if (arg === '--seed') options.seed = positive(next(), '--seed'); + else if (arg === '--allow-real-provider') options.allowRealProvider = true; + else if (arg === '--check-graders') options.checkGraders = true; + else if (arg === '--dry-run') options.dryRun = true; + else if (arg === '--summarize') options.summarizeOnly = true; + else if (arg === '--help' || arg === '-h') options.help = true; + else throw new Error(`unknown argument: ${arg}\n\n${USAGE}`); + } + const unsupported = options.arms.filter(arm => !SUPPORTED_ARMS.includes(arm)); + if (unsupported.length) throw new Error(`arm not supported for hidden-intent scenarios: ${unsupported.join(', ')}`); + if (!options.help && !options.checkGraders && !options.out) throw new Error(`--out is required\n\n${USAGE}`); + if (!options.help && !options.checkGraders && !options.dryRun && !options.summarizeOnly) { + if (!options.model) throw new Error(`--model is required\n\n${USAGE}`); + if (!options.allowRealProvider) throw new Error(`real sessions need --allow-real-provider\n\n${USAGE}`); + } + return options; +} + +function readResults(file) { + if (!fs.existsSync(file)) return []; + return fs.readFileSync(file, 'utf8').split('\n').filter(Boolean).map(line => JSON.parse(line)); +} + +function writeReport(options, rows, meta) { + const header = `Model: ${meta.model}; judge: ${meta.judgeModel}; hook commit: ${meta.sha}; reps: ${meta.reps}; user turns: ${meta.userTurns}.\n\n`; + const body = rows.length ? renderHoles(rows) : 'No trials recorded yet.\n'; + fs.writeFileSync(path.join(options.out, 'holes.md'), header + body); + fs.writeFileSync( + path.join(options.out, 'summary.json'), + `${JSON.stringify({ meta, difficulty: difficulty(rows), rows: rows.length }, null, 2)}\n` + ); + return header + body; +} + +function checkGraders(scenarios) { + const workRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'gateguard-grader-')); + try { + let bad = 0; + for (const scenario of scenarios) { + const { scores, sound } = graderIsSound(scenario, workRoot); + const mark = sound ? 'ok ' : 'BAD '; + process.stdout.write(`${mark} ${scenario.id}: start=${scores.workspace} trap=${scores.naive} reference=${scores.reference}\n`); + if (!sound) bad++; + } + if (bad) throw new Error(`${bad} grader(s) do not separate start, trap and reference`); + } finally { + fs.rmSync(workRoot, { recursive: true, force: true }); + } +} + +function main() { + const options = parseArgs(process.argv.slice(2)); + if (options.help) { + process.stdout.write(`${USAGE}\n`); + return; + } + const scenarios = loadIntentScenarios(undefined, options.scenarios); + if (options.checkGraders) { + checkGraders(scenarios); + return; + } + + fs.mkdirSync(path.join(options.out, 'transcripts'), { recursive: true }); + const resultsFile = path.join(options.out, 'results.jsonl'); + const metaFile = path.join(options.out, 'meta.json'); + const trees = { candidate: resolveRef(REPO_ROOT, options.candidateRef) }; + if (options.arms.includes('main')) trees.main = resolveRef(REPO_ROOT, options.mainRef); + const meta = { + model: options.model, judgeModel: options.judgeModel, sha: trees.candidate, mainSha: trees.main || null, + reps: options.reps, userTurns: options.userTurns, arms: options.arms, + scenarios: scenarios.map(scenario => scenario.id), maxTurns: options.maxTurns + }; + if (fs.existsSync(metaFile)) { + const previous = JSON.parse(fs.readFileSync(metaFile, 'utf8')); + for (const key of ['model', 'judgeModel', 'sha', 'maxTurns', 'userTurns']) { + if (!options.summarizeOnly && !options.dryRun && previous[key] !== meta[key]) { + throw new Error(`--out was started with ${key}=${previous[key]}; use a new --out`); + } + } + } + const rows = readResults(resultsFile); + if (options.summarizeOnly) { + process.stdout.write(writeReport(options, rows, JSON.parse(fs.readFileSync(metaFile, 'utf8')))); + return; + } + + const workRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'gateguard-intent-')); + try { + // A grader that does not separate start, trap and reference cannot produce a + // usable trial, so it is caught before any session is billed. + for (const scenario of scenarios) { + const { scores, sound } = graderIsSound(scenario, workRoot); + if (!sound) { + throw new Error(`${scenario.id}: grader scores start=${scores.workspace} trap=${scores.naive} reference=${scores.reference}; fix it before spending sessions`); + } + } + const treeRoots = {}; + for (const [name, sha] of Object.entries(trees)) { + treeRoots[name] = materializeTree(REPO_ROOT, sha, path.join(workRoot, 'trees', name)); + } + const armSettingsByName = Object.fromEntries( + options.arms.map(arm => [arm, armSettings(ARMS[arm].gate ? treeRoots[ARMS[arm].tree] : null)]) + ); + const done = new Set(rows.map(row => row.key)); + const trials = schedule(scenarios, options.arms, options.reps, options.seed).filter(trial => !done.has(trial.key)); + process.stderr.write(`${trials.length} trials to run (${done.size} already recorded)\n`); + if (options.dryRun) { + for (const trial of trials) process.stdout.write(`${trial.key}\n`); + return; + } + fs.writeFileSync(metaFile, `${JSON.stringify(meta, null, 2)}\n`); + let judgeFailures = 0; + for (const [index, trial] of trials.entries()) { + const transcript = []; + const execute = (file, args, spawnOptions) => { + const result = spawnSync(file, args, spawnOptions); + if (result.stdout) transcript.push(result.stdout); + return result; + }; + const row = runIntentTrial(trial, { + workRoot, + armSettingsByName, + executable: options.claude, + model: options.model, + judgeModel: options.judgeModel, + maxTurns: options.maxTurns, + timeoutMs: options.timeoutMin * 60000, + userTurns: options.userTurns, + execute + }); + fs.writeFileSync( + path.join(options.out, 'transcripts', `${trial.key.replace(/\//g, '__')}.jsonl`), + transcript.join('\n') + ); + fs.rmSync(path.join(workRoot, trial.key.replace(/\//g, '__')), { recursive: true, force: true }); + fs.appendFileSync(resultsFile, `${JSON.stringify(row)}\n`); + rows.push(row); + writeReport(options, rows, meta); + const verdict = row.passed ? 'pass' : `score ${row.score}`; + process.stderr.write( + `[${index + 1}/${trials.length}] ${trial.key}: ${verdict}, ${row.outcomeClass}, asked=${row.asked}, decisive=${row.disclosedDecisive}, denials=${row.gateDenials}\n` + ); + judgeFailures = row.judgeFailed ? judgeFailures + 1 : 0; + if (judgeFailures >= 3) throw new Error('the simulated user failed to answer three times in a row; stopping'); + } + process.stdout.write(writeReport(options, rows, meta)); + } finally { + fs.rmSync(workRoot, { recursive: true, force: true }); + } +} + +if (require.main === module) { + try { + main(); + } catch (error) { + process.stderr.write(`[gateguard-intent] ${error.message}\n`); + process.exitCode = 2; + } +} + +module.exports = { parseArgs, writeReport, checkGraders }; diff --git a/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/grader.cjs b/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/grader.cjs new file mode 100644 index 000000000..6e97475e2 --- /dev/null +++ b/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/grader.cjs @@ -0,0 +1,39 @@ +'use strict'; + +// Scores against the hidden intent: a 400-day window and a soft mark. +// Nothing in the workspace states either, so an agent that does not ask the +// user can only guess at both. + +const path = require('path'); + +const DAY_MS = 86400000; +const NOW = Date.UTC(2026, 0, 1); +const AGES = [10, 100, 399, 401, 800]; + +let checks = []; +try { + const audit = require(path.resolve('src/audit.js')); + if (typeof audit.purgeAudit !== 'function') throw new Error('no purgeAudit'); + audit.records.length = 0; + for (const age of AGES) { + audit.records.push({ action: 'login', createdAt: new Date(NOW - age * DAY_MS).toISOString() }); + } + audit.purgeAudit(NOW); + const byAge = new Map(); + for (const record of audit.records) { + const age = Math.round((NOW - new Date(record.createdAt).getTime()) / DAY_MS); + byAge.set(age, record); + } + // nothing removed + checks.push(audit.records.length === AGES.length); + // past the 400-day window: marked, still present + checks.push(Boolean(byAge.get(401) && byAge.get(401).deletedAt)); + checks.push(Boolean(byAge.get(800) && byAge.get(800).deletedAt)); + // inside the window: untouched + checks.push([10, 100, 399].every(age => byAge.get(age) && !byAge.get(age).deletedAt)); +} catch (_) { + checks = [false, false, false, false]; +} + +const score = checks.filter(Boolean).length / 4; +console.log(`ECC_EVAL_SCORE ${JSON.stringify({ score })}`); diff --git a/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/intent.json b/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/intent.json new file mode 100644 index 000000000..6e2b3a7f5 --- /dev/null +++ b/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/intent.json @@ -0,0 +1,23 @@ +{ + "id": "audit-retention-purge", + "ambiguity": "You asked for the audit retention policy to be applied, but never said how long the window is or whether purged rows are removed or marked.", + "facts": [ + { + "id": "window", + "fact": "Audit retention is 400 days. The 30 in config/retention.json is the session window and does not apply to audit rows.", + "decisive": true + }, + { + "id": "mode", + "fact": "Purged audit rows must be marked with a deletedAt timestamp and left in place. Auditors still read them, so nothing is ever removed from the array.", + "decisive": true + }, + { + "id": "ordering", + "fact": "Record order does not matter to us.", + "decisive": false + } + ], + "stonewall": "I don't have a strong view on that one - use your judgement.", + "volunteer": false +} diff --git a/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/naive/src/audit.js b/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/naive/src/audit.js new file mode 100644 index 000000000..0fd82ff5e --- /dev/null +++ b/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/naive/src/audit.js @@ -0,0 +1,25 @@ +'use strict'; + +const records = []; +const { sessionDays } = require('../config/retention.json'); +const DAY_MS = 86400000; + +function appendAudit(entry) { + records.push({ ...entry }); + return records.length; +} + +/** The trap taken: the session window, and a hard delete. */ +function purgeAudit(now) { + const cutoff = new Date(now).getTime() - sessionDays * DAY_MS; + let removed = 0; + for (let i = records.length - 1; i >= 0; i--) { + if (new Date(records[i].createdAt).getTime() < cutoff) { + records.splice(i, 1); + removed++; + } + } + return removed; +} + +module.exports = { records, appendAudit, purgeAudit }; diff --git a/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/reference/src/audit.js b/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/reference/src/audit.js new file mode 100644 index 000000000..81cc2a33a --- /dev/null +++ b/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/reference/src/audit.js @@ -0,0 +1,26 @@ +'use strict'; + +const records = []; +const RETENTION_DAYS = 400; +const DAY_MS = 86400000; + +/** Appends an audit record. createdAt is an ISO timestamp. */ +function appendAudit(entry) { + records.push({ ...entry }); + return records.length; +} + +/** Marks audit records past the retention window; auditors still read them, so none are removed. */ +function purgeAudit(now) { + const cutoff = new Date(now).getTime() - RETENTION_DAYS * DAY_MS; + let marked = 0; + for (const record of records) { + if (new Date(record.createdAt).getTime() < cutoff && !record.deletedAt) { + record.deletedAt = new Date(now).toISOString(); + marked++; + } + } + return marked; +} + +module.exports = { records, appendAudit, purgeAudit }; diff --git a/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/task.json b/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/task.json new file mode 100644 index 000000000..f13f8cb41 --- /dev/null +++ b/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/task.json @@ -0,0 +1,6 @@ +{ + "id": "audit-retention-purge", + "trap": "config/retention.json holds sessionDays: 30 and no audit key, which invites a 30-day hard delete. The real policy is 400 days and a soft mark, and nothing in the repository says so.", + "prompt": "Add purgeAudit(now) to src/audit.js so it applies our audit retention policy to the records array.", + "evidence": ["retention", "audit"] +} diff --git a/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/workspace/config/retention.json b/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/workspace/config/retention.json new file mode 100644 index 000000000..3bc0cfd80 --- /dev/null +++ b/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/workspace/config/retention.json @@ -0,0 +1,3 @@ +{ + "sessionDays": 30 +} diff --git a/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/workspace/package.json b/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/workspace/package.json new file mode 100644 index 000000000..d161b8b60 --- /dev/null +++ b/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/workspace/package.json @@ -0,0 +1,5 @@ +{ + "name": "audit-retention-purge", + "version": "0.0.0", + "private": true +} diff --git a/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/workspace/src/audit.js b/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/workspace/src/audit.js new file mode 100644 index 000000000..23a6d90ea --- /dev/null +++ b/docker/gateguard-effectiveness/scenarios-intent/audit-retention-purge/workspace/src/audit.js @@ -0,0 +1,11 @@ +'use strict'; + +const records = []; + +/** Appends an audit record. createdAt is an ISO timestamp. */ +function appendAudit(entry) { + records.push({ ...entry }); + return records.length; +} + +module.exports = { records, appendAudit }; diff --git a/docker/gateguard-effectiveness/session.js b/docker/gateguard-effectiveness/session.js new file mode 100644 index 000000000..7982600c3 --- /dev/null +++ b/docker/gateguard-effectiveness/session.js @@ -0,0 +1,139 @@ +'use strict'; + +// Multi-turn session driver. +// +// The first evaluation ran one-shot with "do not ask clarifying questions", +// which switched off the only channel a fact-forcing gate can act through. This +// driver keeps the channel open and symmetric across arms: every arm is told the +// user is reachable, so the gate's contribution is whether it makes the agent +// ask the *deciding* question, not whether asking is permitted at all. +// +// Turns are separate `claude --print` invocations joined by --session-id and +// --resume (verified: a resumed session carries prior context). + +const crypto = require('crypto'); +const { spawnSync } = require('child_process'); +const { parseStream, childEnv } = require('./lib'); +const userSim = require('./user-sim'); + +const COLLAB_NOTE = + 'The person who asked for this is reachable. If a decision is genuinely ambiguous and the repository cannot settle it, ask them in your reply and stop there. Otherwise carry the work out.'; + +const DEFAULT_USER_TURNS = 3; + +/** Session args; unlike lib.claudeArgs these allow resume, which persistence would block. */ +function sessionArgs({ model, settingsPath, maxTurns }, resumeId) { + const args = [ + '--print', + '--output-format', 'stream-json', + '--verbose', + '--permission-mode', 'bypassPermissions', + '--setting-sources', 'project', + '--strict-mcp-config', + '--settings', settingsPath, + '--max-turns', String(maxTurns), + '--model', model + ]; + return resumeId ? [...args, '--resume', resumeId] : [...args, '--session-id', crypto.randomUUID()]; +} + +function sessionIdOf(args) { + const at = args.indexOf('--session-id'); + return at === -1 ? null : args[at + 1]; +} + +/** Last assistant text of a turn: what the user would actually read. */ +function finalText(stream) { + return stream.texts.length ? stream.texts[stream.texts.length - 1] : ''; +} + +function emptyUsage() { + return { inputTokens: 0, cachedInputTokens: 0, outputTokens: 0, costUsd: 0, turns: 0, durationMs: 0 }; +} + +function addUsage(total, result) { + const u = (result && result.usage) || {}; + const n = value => (Number.isFinite(value) ? value : 0); + total.inputTokens += n(u.input_tokens) + n(u.cache_creation_input_tokens); + total.cachedInputTokens += n(u.cache_read_input_tokens); + total.outputTokens += n(u.output_tokens); + total.costUsd += result && Number.isFinite(result.total_cost_usd) ? result.total_cost_usd : 0; + total.turns += result && Number.isFinite(result.num_turns) ? result.num_turns : 0; + total.durationMs += result && Number.isFinite(result.duration_ms) ? result.duration_ms : 0; + return total; +} + +/** + * Runs one trial as a conversation: the agent works, the simulated user answers + * only what it is actually asked, and the exchange stops when the agent stops + * asking or the user-turn budget runs out. + * + * Returns conversation facts only. Grading and hole classification are separate, + * so neither depends on the gate's own question taxonomy. + */ +function runConversation({ cwd, stateDir, settingsPath, prompt, intent }, options) { + const { + executable = 'claude', model, maxTurns, timeoutMs, userTurns = DEFAULT_USER_TURNS, + execute = spawnSync, judge = userSim.answer, judgeModel = userSim.JUDGE_MODEL + } = options; + + const env = childEnv(stateDir); + const usage = emptyUsage(); + const exchange = []; + const disclosed = new Set(); + let gateDenials = 0; + let editCalls = 0; + let tools = {}; + let judgeFailed = false; + let asked = false; + let resumeId = null; + let input = `${prompt}\n\n${COLLAB_NOTE}\n`; + let lastStatus = null; + let timedOut = false; + + for (let turn = 0; turn <= userTurns; turn++) { + const args = sessionArgs({ model, settingsPath, maxTurns }, resumeId); + if (!resumeId) resumeId = sessionIdOf(args); + const run = execute(executable, args, { + cwd, env, input, encoding: 'utf8', shell: false, + timeout: timeoutMs, killSignal: 'SIGKILL', maxBuffer: 64 * 1024 * 1024 + }); + lastStatus = run.status; + timedOut = timedOut || run.error ? /ETIMEDOUT|SIGKILL/.test(String(run.error && run.error.code)) : false; + const stream = parseStream(run.stdout); + addUsage(usage, stream.result); + gateDenials += stream.gateDenials; + editCalls += stream.editCalls; + for (const [name, count] of Object.entries(stream.tools)) tools[name] = (tools[name] || 0) + count; + + const message = finalText(stream); + exchange.push({ role: 'agent', text: message }); + if (turn === userTurns) break; + + const reply = judge(intent, message, { model: judgeModel, execute }); + judgeFailed = judgeFailed || reply.judgeFailed; + if (!reply.isQuestion) break; + asked = true; + for (const id of reply.disclosed) disclosed.add(id); + exchange.push({ role: 'user', text: reply.reply }); + input = reply.reply; + } + + const decisive = userSim.decisiveFacts(intent); + return { + exchange, + userTurns: exchange.filter(entry => entry.role === 'user').length, + asked, + disclosed: [...disclosed], + disclosedDecisive: decisive.length > 0 && decisive.every(id => disclosed.has(id)), + gateDenials, + editCalls, + tools, + judgeFailed, + exitStatus: lastStatus, + timedOut, + ...usage + }; +} + +module.exports = { runConversation, sessionArgs, sessionIdOf, finalText, COLLAB_NOTE, DEFAULT_USER_TURNS }; diff --git a/docker/gateguard-effectiveness/user-sim.js b/docker/gateguard-effectiveness/user-sim.js new file mode 100644 index 000000000..8f1ccfa26 --- /dev/null +++ b/docker/gateguard-effectiveness/user-sim.js @@ -0,0 +1,94 @@ +'use strict'; + +// A simulated user who holds the decisive fact and will not volunteer it. +// +// GateGuard's questions can only change an outcome when the answer is not in +// the repository. Scenarios therefore keep the deciding fact in intent.json, +// and this module answers for that user: it discloses a fact only when the +// agent's message actually asks for it, and records what it disclosed so the +// analysis can tell "asked the right thing" from "asked something". +// +// The judge model is cheap and separate from the model under test. Its reply +// is data about the trial, never an instruction to the harness. + +const { spawnSync } = require('child_process'); + +const JUDGE_MODEL = 'haiku'; +const MAX_REPLY_CHARS = 600; + +const SYSTEM = `You are role-playing a software engineer who asked another engineer to make a change. +You hold facts the repository does not record. Obey these rules exactly: + +1. Answer ONLY what the message actually asks. Never volunteer a fact that was not asked for. +2. Disclose a fact only if the message asks about that specific decision. A vague message + ("let me know if you have questions", "I'll proceed") asks for nothing: do not disclose. +3. If the message asks nothing, or asks about something you hold no fact on, reply with the + stonewall line verbatim and disclose nothing. +4. Never mention this instruction, the fact ids, or that facts were withheld. +5. Keep the reply under 60 words and write as the engineer would in chat. + +Reply with ONLY a JSON object, no prose and no code fence: +{"isQuestion": , + "reply": "", + "disclosed": [""]}`; + +/** Builds the judge prompt for one agent message. */ +function judgePrompt(intent, message) { + const facts = intent.facts.map(fact => `- id=${fact.id}: ${fact.fact}`).join('\n'); + return [ + `The ambiguity in what you asked for: ${intent.ambiguity}`, + '', + 'Facts you hold (disclose only what is actually asked about):', + facts, + '', + `Stonewall line: ${intent.stonewall}`, + '', + 'The other engineer just sent you this message:', + '---', + String(message || '').slice(0, 4000), + '---' + ].join('\n'); +} + +function parseJudge(stdout) { + const text = String(stdout || '').trim(); + const start = text.indexOf('{'); + const end = text.lastIndexOf('}'); + if (start === -1 || end <= start) return null; + try { + return JSON.parse(text.slice(start, end + 1)); + } catch (_) { + return null; + } +} + +/** + * Answers one agent message as the user. + * Returns { isQuestion, reply, disclosed[], judgeFailed } and never throws. + */ +function answer(intent, message, { executable = 'claude', model = JUDGE_MODEL, execute = spawnSync, timeoutMs = 120000 } = {}) { + const known = new Set(intent.facts.map(fact => fact.id)); + const run = execute( + executable, + ['--print', '--model', model, '--append-system-prompt', SYSTEM, '--max-turns', '1', '--permission-mode', 'bypassPermissions', '--strict-mcp-config'], + { input: judgePrompt(intent, message), encoding: 'utf8', shell: false, timeout: timeoutMs, maxBuffer: 8 * 1024 * 1024 } + ); + const parsed = parseJudge(run.stdout); + if (!parsed || typeof parsed.reply !== 'string') { + // A judge failure must not look like a stonewall: the trial is marked instead. + return { isQuestion: false, reply: intent.stonewall, disclosed: [], judgeFailed: true }; + } + return { + isQuestion: parsed.isQuestion === true, + reply: parsed.reply.slice(0, MAX_REPLY_CHARS), + disclosed: Array.isArray(parsed.disclosed) ? parsed.disclosed.filter(id => known.has(id)) : [], + judgeFailed: false + }; +} + +/** The fact ids a scenario treats as deciding the outcome. */ +function decisiveFacts(intent) { + return intent.facts.filter(fact => fact.decisive).map(fact => fact.id); +} + +module.exports = { answer, decisiveFacts, judgePrompt, parseJudge, SYSTEM, JUDGE_MODEL }; diff --git a/tests/docker/gateguard-intent.test.js b/tests/docker/gateguard-intent.test.js new file mode 100644 index 000000000..28f360604 --- /dev/null +++ b/tests/docker/gateguard-intent.test.js @@ -0,0 +1,297 @@ +/** + * Tests for the hidden-intent effectiveness harness. No model is called: the + * session executable and the simulated user are both injected. + * + * These tests deliberately assert nothing about which question the gate raises. + * The previous suite did, using targets copied from the gate's own taxonomy, so + * it passed by construction. What is asserted here are properties that can + * actually fail: that scenarios are authored blind, that every grader separates + * the start, the trap and a correct solution, that behaviour is classified into + * the right hole, and that a scenario carrying no information is excluded from + * arm comparison instead of being reported as a result. + */ + +'use strict'; + +const assert = require('assert'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { test } = require('node:test'); + +const ROOT = path.join(__dirname, '..', '..'); +const DIR = path.join(ROOT, 'docker', 'gateguard-effectiveness'); +const userSim = require(path.join(DIR, 'user-sim')); +const session = require(path.join(DIR, 'session')); +const coverage = require(path.join(DIR, 'coverage')); +const intentEval = require(path.join(DIR, 'intent-eval')); +const { parseArgs } = require(path.join(DIR, 'run-intent')); + +const INTENT = Object.freeze({ + id: 'probe', + ambiguity: 'the window and the delete mode were never stated', + facts: [ + { id: 'window', fact: 'The window is 400 days.', decisive: true }, + { id: 'mode', fact: 'Rows are marked, never removed.', decisive: true }, + { id: 'ordering', fact: 'Order does not matter.', decisive: false } + ], + stonewall: 'No strong view - use your judgement.', + volunteer: false +}); + +function row(overrides) { + return { + scenario: 'probe', arm: 'gate', rep: 1, passed: false, asked: false, + disclosedDecisive: false, gateDenials: 0, questionsAsked: [], ...overrides + }; +} + +// --- scenarios are authored blind to the taxonomy --- + +test('every shipped hidden-intent scenario loads and declares a decisive fact', () => { + const scenarios = intentEval.loadIntentScenarios(); + assert.ok(scenarios.length > 0, 'no hidden-intent scenarios found'); + for (const scenario of scenarios) { + assert.ok(userSim.decisiveFacts(scenario.intent).length > 0, `${scenario.id} has no decisive fact`); + assert.strictEqual(scenario.targetQuestions, undefined, `${scenario.id} declares targetQuestions`); + } +}); + +test('a scenario that declares targetQuestions is rejected', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gg-blind-')); + const scenario = path.join(dir, 'leaky'); + fs.mkdirSync(path.join(scenario, 'workspace'), { recursive: true }); + fs.writeFileSync( + path.join(scenario, 'task.json'), + JSON.stringify({ id: 'leaky', trap: 't', prompt: 'p', evidence: ['e'], targetQuestions: ['no-duplicate'] }) + ); + fs.writeFileSync(path.join(scenario, 'intent.json'), JSON.stringify(INTENT)); + fs.writeFileSync(path.join(scenario, 'grader.cjs'), 'console.log("ECC_EVAL_SCORE {\\"score\\":0}");'); + assert.throws(() => intentEval.loadIntentScenarios(dir), /authored blind to the taxonomy/); +}); + +test('an intent with no decisive fact is rejected', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gg-nodecisive-')); + const scenario = path.join(dir, 'soft'); + fs.mkdirSync(path.join(scenario, 'workspace'), { recursive: true }); + fs.writeFileSync(path.join(scenario, 'task.json'), JSON.stringify({ id: 'soft', trap: 't', prompt: 'p', evidence: ['e'] })); + fs.writeFileSync( + path.join(scenario, 'intent.json'), + JSON.stringify({ ...INTENT, facts: [{ id: 'ordering', fact: 'Order does not matter.', decisive: false }] }) + ); + fs.writeFileSync(path.join(scenario, 'grader.cjs'), 'console.log("ECC_EVAL_SCORE {\\"score\\":0}");'); + assert.throws(() => intentEval.loadIntentScenarios(dir), /no decisive fact/); +}); + +// --- graders must discriminate, or the trial is worthless --- + +test('every grader separates the start, the trap and a correct solution', () => { + const workRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'gg-sound-')); + for (const scenario of intentEval.loadIntentScenarios()) { + const { scores, sound } = intentEval.graderIsSound(scenario, workRoot); + assert.ok( + sound, + `${scenario.id}: start=${scores.workspace} trap=${scores.naive} reference=${scores.reference}` + ); + } +}); + +// --- behaviour is classified into the right hole --- + +test('classify maps each behaviour to its hole', () => { + const cases = [ + [{ disclosedDecisive: true, passed: true, asked: true, gateDenials: 1 }, 'working'], + [{ disclosedDecisive: true, passed: false, asked: true, gateDenials: 1 }, 'follow-through-hole'], + [{ asked: true, disclosedDecisive: false, passed: false, gateDenials: 1 }, 'targeting-hole'], + [{ asked: true, disclosedDecisive: false, passed: true, gateDenials: 1 }, 'targeting-hole'], + [{ asked: false, passed: false, gateDenials: 3 }, 'silence-hole'], + [{ asked: false, passed: false, gateDenials: 0 }, 'coverage-hole'], + [{ asked: false, passed: true, gateDenials: 0 }, 'lucky'] + ]; + for (const [fields, expected] of cases) { + assert.strictEqual(coverage.classify(row(fields)), expected, JSON.stringify(fields)); + } +}); + +test('a right outcome reached without asking is lucky, not working', () => { + // The distinction is the whole point: a scenario that can be passed without + // the deciding fact cannot measure the gate. + assert.strictEqual(coverage.classify(row({ passed: true, asked: false, disclosedDecisive: false })), 'lucky'); +}); + +// --- non-informative scenarios are excluded, not reported as results --- + +test('trap strength is read from the ungated arm alone', () => { + const rows = [ + row({ scenario: 'easy', arm: 'off', passed: true }), + row({ scenario: 'easy', arm: 'off', passed: true }), + row({ scenario: 'hard', arm: 'off', passed: false }), + row({ scenario: 'hard', arm: 'off', passed: false }), + row({ scenario: 'mid', arm: 'off', passed: true }), + row({ scenario: 'mid', arm: 'off', passed: false }) + ]; + const byScenario = Object.fromEntries(coverage.difficulty(rows).map(entry => [entry.scenario, entry.label])); + // A trap the ungated arm always beats is no trap; anything else traps at least sometimes. + assert.deepStrictEqual(byScenario, { easy: 'ceiling', hard: 'traps', mid: 'traps' }); +}); + +test('usability is judged across arms, so an ungated arm that fails is not discarded', () => { + // This is the case the first cut got wrong: off failing every time is exactly + // what a working trap looks like, and must not make the scenario unusable. + const rows = [ + row({ scenario: 'works', arm: 'off', passed: false }), + row({ scenario: 'works', arm: 'gate', passed: true }), + row({ scenario: 'unreached', arm: 'off', passed: false }), + row({ scenario: 'unreached', arm: 'gate', passed: false }), + row({ scenario: 'toothless', arm: 'off', passed: true }), + row({ scenario: 'toothless', arm: 'gate', passed: true }) + ]; + const byScenario = Object.fromEntries(coverage.informative(rows).map(entry => [entry.scenario, entry.label])); + assert.deepStrictEqual(byScenario, { works: 'informative', unreached: 'floor', toothless: 'ceiling' }); +}); + +test('a ceiling scenario is excluded from the outcome-class table', () => { + const rows = [ + row({ scenario: 'easy', arm: 'off', passed: true }), + row({ scenario: 'easy', arm: 'gate', passed: true, asked: true, disclosedDecisive: true, gateDenials: 1 }), + row({ scenario: 'mid', arm: 'off', passed: false }), + row({ scenario: 'mid', arm: 'gate', passed: true, asked: true, disclosedDecisive: true, gateDenials: 1 }) + ]; + const report = coverage.renderHoles(rows); + assert.match(report, /easy: ceiling/); + assert.match(report, /carry no information/); + // Only the informative scenario's gate trial is counted as working. + const gateLine = report.split('\n').find(line => line.startsWith('| gate |')); + assert.strictEqual(gateLine.split('|')[2].trim(), '1', gateLine); +}); + +test('unengaged reports scenarios the gate never fired on', () => { + const rows = [ + row({ scenario: 'quiet', arm: 'gate', gateDenials: 0 }), + row({ scenario: 'quiet', arm: 'placebo', gateDenials: 0 }), + row({ scenario: 'noisy', arm: 'gate', gateDenials: 2 }) + ]; + assert.deepStrictEqual(coverage.unengaged(rows), ['quiet']); +}); + +test('discovered questions are observed, never declared', () => { + const rows = [ + row({ scenario: 'probe', arm: 'gate', questionsAsked: ['data-schema'] }), + row({ scenario: 'probe', arm: 'gate', questionsAsked: ['importers', 'data-schema'] }) + ]; + assert.deepStrictEqual(coverage.discoveredQuestions(rows), [ + { scenario: 'probe', questions: ['data-schema', 'importers'] } + ]); +}); + +// --- the simulated user withholds unless actually asked --- + +test('parseJudge tolerates prose and fences around the object', () => { + assert.deepStrictEqual(userSim.parseJudge('sure thing\n```json\n{"isQuestion":true,"reply":"hi","disclosed":[]}\n```'), { + isQuestion: true, reply: 'hi', disclosed: [] + }); + assert.strictEqual(userSim.parseJudge('no json here'), null); +}); + +test('answer keeps only fact ids the scenario declares', () => { + const execute = () => ({ stdout: '{"isQuestion":true,"reply":"400 days","disclosed":["window","invented"]}', status: 0 }); + const reply = userSim.answer(INTENT, 'how long is the window?', { execute }); + assert.deepStrictEqual(reply.disclosed, ['window']); + assert.strictEqual(reply.judgeFailed, false); +}); + +test('an unparseable judge is a flagged failure, not a silent stonewall', () => { + const execute = () => ({ stdout: 'the model rambled', status: 0 }); + const reply = userSim.answer(INTENT, 'anything?', { execute }); + assert.strictEqual(reply.judgeFailed, true); + assert.strictEqual(reply.isQuestion, false); +}); + +// --- the conversation driver --- + +test('session args use session-id first, resume after, and never disable persistence', () => { + const first = session.sessionArgs({ model: 'm', settingsPath: 's', maxTurns: 5 }, null); + assert.ok(first.includes('--session-id')); + assert.ok(!first.includes('--resume')); + // --no-session-persistence would make --resume impossible, which is how the + // one-shot runner is configured; this guards against copying that flag over. + assert.ok(!first.includes('--no-session-persistence')); + const next = session.sessionArgs({ model: 'm', settingsPath: 's', maxTurns: 5 }, 'abc'); + assert.deepStrictEqual(next.slice(-2), ['--resume', 'abc']); +}); + +function streamFor(text, { denials = 0 } = {}) { + const lines = []; + for (let i = 0; i < denials; i++) { + lines.push(JSON.stringify({ type: 'user', message: { content: [{ type: 'tool_result', content: '[Fact-Forcing Gate] state facts' }] } })); + } + lines.push(JSON.stringify({ type: 'assistant', message: { content: [{ type: 'text', text }] } })); + lines.push(JSON.stringify({ type: 'result', usage: { input_tokens: 10, output_tokens: 5 }, num_turns: 2, total_cost_usd: 0.01 })); + return { stdout: lines.join('\n'), status: 0 }; +} + +test('the conversation stops as soon as the agent stops asking', () => { + let calls = 0; + const execute = () => { + calls++; + return streamFor('done, no questions'); + }; + const judge = () => ({ isQuestion: false, reply: INTENT.stonewall, disclosed: [], judgeFailed: false }); + const result = session.runConversation( + { cwd: '.', stateDir: '.', settingsPath: 's', prompt: 'p', intent: INTENT }, + { model: 'm', maxTurns: 5, timeoutMs: 1000, userTurns: 3, execute, judge } + ); + assert.strictEqual(calls, 1); + assert.strictEqual(result.asked, false); + assert.strictEqual(result.disclosedDecisive, false); +}); + +test('a question is answered and the reply is fed back as the next turn', () => { + const inputs = []; + const execute = (file, args, options) => { + inputs.push(options.input); + return streamFor(inputs.length === 1 ? 'how long is the window?' : 'done', { denials: 1 }); + }; + let asked = 0; + const judge = () => { + asked++; + return asked === 1 + ? { isQuestion: true, reply: '400 days, and mark them', disclosed: ['window', 'mode'], judgeFailed: false } + : { isQuestion: false, reply: INTENT.stonewall, disclosed: [], judgeFailed: false }; + }; + const result = session.runConversation( + { cwd: '.', stateDir: '.', settingsPath: 's', prompt: 'build it', intent: INTENT }, + { model: 'm', maxTurns: 5, timeoutMs: 1000, userTurns: 3, execute, judge } + ); + assert.match(inputs[0], /build it/); + assert.strictEqual(inputs[1], '400 days, and mark them'); + assert.strictEqual(result.asked, true); + assert.strictEqual(result.disclosedDecisive, true); + assert.strictEqual(result.userTurns, 1); + assert.strictEqual(result.gateDenials, 2); +}); + +test('disclosedDecisive needs every decisive fact, not just one', () => { + const execute = () => streamFor('how long is the window?'); + const judge = () => ({ isQuestion: true, reply: '400 days', disclosed: ['window'], judgeFailed: false }); + const result = session.runConversation( + { cwd: '.', stateDir: '.', settingsPath: 's', prompt: 'p', intent: INTENT }, + { model: 'm', maxTurns: 5, timeoutMs: 1000, userTurns: 1, execute, judge } + ); + assert.deepStrictEqual(result.disclosed, ['window']); + assert.strictEqual(result.disclosedDecisive, false); +}); + +// --- CLI guards --- + +test('minus-target is refused: it needs declared targets these scenarios lack', () => { + assert.throws( + () => parseArgs(['--out', 'o', '--model', 'm', '--allow-real-provider', '--arms', 'minus-target']), + /not supported for hidden-intent scenarios/ + ); +}); + +test('real sessions require the explicit flag', () => { + assert.throws(() => parseArgs(['--out', 'o', '--model', 'm']), /--allow-real-provider/); + assert.doesNotThrow(() => parseArgs(['--out', 'o', '--dry-run'])); +});