From 64d155ae1d59f38d3402f23f2224e8e637f97059 Mon Sep 17 00:00:00 2001 From: montjeffrey Date: Sun, 11 Oct 2026 10:00:47 -0400 Subject: [PATCH 1/2] test(gateguard): pin grader decoys, pilot retries and the hookless-trial abort Three #3481 review findings still hold on main 8ce348fb: - csv-export-delimiter's grader checks only the dialect (CRLF, semicolons, quoting), so a constant '"x";"y"' export or one emitting the column names scores 1. Add decoy-constant and decoy-keys overlays and pin that each scores below 1 and that graderIsSound reports them; a scenario whose decoy passes is not sound. - The pilot runner (run.js) marks errored keys done, so they are never retried, and armSummary counts provider-error and timed-out rows in trials, passes and means. Pin the summary on valid rows only, and drive run.js end to end: a provider error is retried on resume, keeps its own row and transcript, and the summary reports it apart. - Disabling run-intent.js's abort on an unobserved hook left every test passing. Drive both runners end to end with a gated trial whose hook never runs: each must stop and record nothing for it. fake-claude.js gains FAKE_CLAUDE_SKIP_HOOK and FAKE_CLAUDE_PROVIDER_ERROR. The intent end-to-end setup moves into syntheticIntentRunner, shared with the existing resume test. Against main: 2 intent failures (the decoy cases) and 2 pilot failures (summary, retry). The two abort tests pass on main, since both aborts exist; each fails when its runner's abort is disabled (if (false && !row.hookObserved)). --- docker/gateguard-effectiveness/fake-claude.js | 8 +- .../decoy-constant/src/export.js | 8 ++ .../decoy-keys/src/export.js | 8 ++ tests/docker/gateguard-effectiveness.test.js | 76 +++++++++++ tests/docker/gateguard-intent.test.js | 125 ++++++++++++------ 5 files changed, 187 insertions(+), 38 deletions(-) create mode 100644 docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/decoy-constant/src/export.js create mode 100644 docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/decoy-keys/src/export.js diff --git a/docker/gateguard-effectiveness/fake-claude.js b/docker/gateguard-effectiveness/fake-claude.js index 41979941f..60ef40433 100644 --- a/docker/gateguard-effectiveness/fake-claude.js +++ b/docker/gateguard-effectiveness/fake-claude.js @@ -39,9 +39,11 @@ const input = fs.existsSync(target) const toolName = 'old_string' in input ? 'Edit' : 'Write'; emit({ type: 'assistant', message: { content: [{ type: 'text', text: 'Checking report.js, invoice.js and csv.js first.' }, { type: 'tool_use', id: 't1', name: toolName, input }] } }); +// FAKE_CLAUDE_SKIP_HOOK=1 edits without running the hook (a broken hook setup); +// FAKE_CLAUDE_PROVIDER_ERROR=1 ends the session in a non-authentication provider error. let denial = ''; const group = (settings.hooks && settings.hooks.PreToolUse || []).find(entry => new RegExp(`^(${entry.matcher})$`).test(toolName)); -if (group) { +if (group && process.env.FAKE_CLAUDE_SKIP_HOOK !== '1') { const env = { ...process.env, CLAUDE_PROJECT_DIR: process.cwd() }; const payload = { session_id: 'fake', cwd: process.cwd(), hook_event_name: 'PreToolUse', tool_name: toolName, tool_input: input, tool_use_id: 't1' }; const result = spawnSync(group.hooks[0].command, { shell: true, input: JSON.stringify(payload), env, encoding: 'utf8' }); @@ -50,4 +52,8 @@ if (group) { } emit({ type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 't1', is_error: Boolean(denial), content: denial || 'ok' }] } }); copyTree(overlay, process.cwd()); +if (process.env.FAKE_CLAUDE_PROVIDER_ERROR === '1') { + emit({ type: 'result', is_error: true, result: 'API Error: 529 overloaded', num_turns: 1, duration_ms: 100, total_cost_usd: 0, usage: {}, modelUsage: {} }); + process.exit(0); +} emit({ type: 'result', is_error: false, result: 'done', num_turns: 3, duration_ms: 1200, total_cost_usd: 0.01, usage: { input_tokens: 100, cache_creation_input_tokens: 50, cache_read_input_tokens: 400, output_tokens: 80 }, modelUsage: { 'fake-model': {} } }); diff --git a/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/decoy-constant/src/export.js b/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/decoy-constant/src/export.js new file mode 100644 index 000000000..bd6ab13da --- /dev/null +++ b/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/decoy-constant/src/export.js @@ -0,0 +1,8 @@ +'use strict'; + +/** Decoy: a constant in the right dialect that ignores its input. */ +function exportCsv() { + return '"x";"y"\r\n"x";"y"\r\n'; +} + +module.exports = { exportCsv }; diff --git a/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/decoy-keys/src/export.js b/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/decoy-keys/src/export.js new file mode 100644 index 000000000..9fe5c9cce --- /dev/null +++ b/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/decoy-keys/src/export.js @@ -0,0 +1,8 @@ +'use strict'; + +/** Decoy: the right dialect around the wrong fields, the column names instead of the values. */ +function exportCsv(rows) { + return rows.map(row => Object.keys(row).map(key => `"${key}"`).join(';')).join('\r\n') + '\r\n'; +} + +module.exports = { exportCsv }; diff --git a/tests/docker/gateguard-effectiveness.test.js b/tests/docker/gateguard-effectiveness.test.js index 143844b0e..c18663116 100644 --- a/tests/docker/gateguard-effectiveness.test.js +++ b/tests/docker/gateguard-effectiveness.test.js @@ -317,7 +317,83 @@ if (gnuTarDir) { }); } +test('pilot summaries count valid trials only and report provider errors apart', () => { + const row = (arm, rep, passed, extra = {}) => ({ + key: `s/${arm}/${rep}`, arm, scenario: 's', rep, question: 'q', passed, score: passed ? 1 : 0, evidenceRecall: 0, gateDenials: 0, + inputTokens: 1, cachedInputTokens: 0, outputTokens: 1, costUsd: null, turns: 1, wallMs: 1, providerError: false, timedOut: false, ...extra + }); + const rows = [ + row('gate', 1, true), + row('gate', 2, false, { providerError: true }), + row('gate', 2, true), + row('gate', 3, false, { timedOut: true }), + row('off', 1, true) + ]; + const gate = lib.summarize(rows, ['gate', 'off']).arms.find(arm => arm.arm === 'gate'); + assert.strictEqual(gate.trials, 2, 'a provider error or timeout is not a trial'); + assert.strictEqual(gate.passes, 2); + assert.strictEqual(gate.meanScore, 1); + assert.strictEqual(gate.providerErrors, 2); + assert.deepStrictEqual([...lib.completedTrialKeys(rows)].sort(), ['s/gate/1', 's/gate/2', 's/off/1']); +}); + +/** The pilot runner driven end to end with fake-claude.js as the provider. */ +function syntheticPilotRunner(root, scenario, arms) { + const out = path.join(root, 'out'); + const preload = path.join(root, 'preload.js'); + fs.writeFileSync(preload, ` + const cp = require('child_process'); + const originalSpawn = cp.spawnSync; + cp.spawnSync = (file, args, options) => file === 'ecc-fake-claude' + ? originalSpawn(process.execPath, [${JSON.stringify(path.join(DIR, 'fake-claude.js'))}, ...args], options) + : originalSpawn(file, args, options); + `); + const args = ['--require', preload, path.join(DIR, 'run.js'), '--out', out, '--model', 'fake', '--claude', 'ecc-fake-claude', '--allow-real-provider', '--arms', arms, '--reps', '1', '--scenarios', scenario.id]; + const env = { ...process.env, FAKE_CLAUDE_OVERLAY: path.join(scenario.root, 'reference') }; + const run = extraEnv => spawnSync(process.execPath, args, { env: { ...env, ...extraEnv }, encoding: 'utf8', timeout: 60000 }); + const results = () => { + const file = path.join(out, 'results.jsonl'); + return fs.existsSync(file) ? fs.readFileSync(file, 'utf8').trim().split('\n').filter(Boolean).map(line => JSON.parse(line)) : []; + }; + return { out, run, results }; +} + if (gitAvailable()) { + test('a pilot trial that ended in a provider error is retried on resume', () => { + const root = tempDir('gg-eff-retry-'); + try { + const scenario = scenarios.find(item => item.id === 'importers-date-shape'); + const { out, run, results } = syntheticPilotRunner(root, scenario, 'off'); + const failed = run({ FAKE_CLAUDE_PROVIDER_ERROR: '1' }); + assert.strictEqual(failed.status, 0, failed.stderr); + assert.deepStrictEqual(results().map(row => row.providerError), [true]); + const retried = run({}); + assert.strictEqual(retried.status, 0, retried.stderr); + assert.ok(retried.stderr.includes('1 trials to run'), retried.stderr); + const rows = results(); + assert.deepStrictEqual(rows.map(row => row.providerError), [true, false], 'the failed attempt stays for audit'); + assert.strictEqual(rows[1].passed, true); + assert.notStrictEqual(rows[0].transcript, rows[1].transcript, 'each attempt keeps its own transcript'); + const { summary } = JSON.parse(fs.readFileSync(path.join(out, 'summary.json'), 'utf8')); + const off = summary.arms.find(arm => arm.arm === 'off'); + assert.deepStrictEqual([off.trials, off.passes, off.providerErrors], [1, 1, 1]); + const again = run({}); + assert.ok(again.stderr.includes('0 trials to run'), again.stderr); + } finally { fs.rmSync(root, { recursive: true, force: true }); } + }); + + test('the pilot runner stops before recording a gated trial whose hook never ran', () => { + const root = tempDir('gg-eff-hookless-'); + try { + const scenario = scenarios.find(item => item.id === 'importers-date-shape'); + const { run, results } = syntheticPilotRunner(root, scenario, 'gate'); + const result = run({ FAKE_CLAUDE_SKIP_HOOK: '1' }); + assert.notStrictEqual(result.status, 0, 'a hookless gated trial must stop the run'); + assert.match(result.stderr, /the gate did not run although the agent edited files/); + assert.deepStrictEqual(results(), [], 'the hookless trial is not recorded as data'); + } finally { fs.rmSync(root, { recursive: true, force: true }); } + }); + test('actual fake-provider CLI resumes identity and resists a metadata publication symlink race', () => { const root = tempDir('gg-eff-cli-'); const out = path.join(root, 'out'); diff --git a/tests/docker/gateguard-intent.test.js b/tests/docker/gateguard-intent.test.js index 1982ebb64..a0ea86f99 100644 --- a/tests/docker/gateguard-intent.test.js +++ b/tests/docker/gateguard-intent.test.js @@ -177,6 +177,33 @@ test('every grader separates the start, the trap and a correct solution', () => } }); +test('the CSV grader rejects the right dialect around the wrong values', () => { + const lib = require(path.join(DIR, 'lib')); + const workRoot = tempDir('gg-decoy-'); + const csv = intentEval.loadIntentScenarios().find(scenario => scenario.id === 'csv-export-delimiter'); + const decoys = fs.readdirSync(csv.root).filter(name => name.startsWith('decoy-')).sort(); + assert.deepStrictEqual(decoys, ['decoy-constant', 'decoy-keys']); + for (const decoy of decoys) { + const dest = path.join(workRoot, decoy); + lib.prepareWorkspace(csv, dest, decoy); + assert.ok(lib.grade(csv, dest).score < 1, `${decoy} must not pass`); + } + const { scores, sound } = intentEval.graderIsSound(csv, workRoot); + assert.ok(sound, JSON.stringify(scores)); + for (const decoy of decoys) assert.ok(scores[decoy] < 1, `graderIsSound grades ${decoy}: ${scores[decoy]}`); +}); + +test('a grader that passes one of its decoys is not sound', () => { + const workRoot = tempDir('gg-decoy-pass-'); + const csv = intentEval.loadIntentScenarios().find(scenario => scenario.id === 'csv-export-delimiter'); + const root = path.join(workRoot, 'scenario'); + fs.cpSync(csv.root, root, { recursive: true }); + fs.cpSync(path.join(root, 'reference'), path.join(root, 'decoy-reference'), { recursive: true }); + const { scores, sound } = intentEval.graderIsSound({ ...csv, root }, workRoot); + assert.strictEqual(scores['decoy-reference'], 1); + assert.strictEqual(sound, false); +}); + // --- behaviour is classified into the right hole --- test('classify maps each behaviour to its hole', () => { @@ -515,50 +542,74 @@ test('session cleanup preserves concurrent and pre-existing folders under custom } }); -test('actual runner freezes arm-patch source and refuses changed-input resume before dispatch', () => { - const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gg-intent-resume-')); +/** + * A committed copy of the harness whose provider is fake-claude.js, so the actual + * runner can be driven end to end. `fakeEnv` reaches the fake provider only. + */ +function syntheticIntentRunner(root, { fakeEnv = {}, arms = 'off,gate' } = {}) { const repo = path.join(root, 'repo'); const copied = path.join(repo, 'docker', 'gateguard-effectiveness'); const out = path.join(root, 'out'); const preload = path.join(root, 'preload.js'); const calls = path.join(root, 'calls.log'); const config = path.join(root, 'claude-config'); + fs.mkdirSync(repo); + fs.cpSync(path.join(ROOT, 'scripts'), path.join(repo, 'scripts'), { recursive: true }); + fs.cpSync(DIR, copied, { recursive: true }); + fs.mkdirSync(path.join(repo, 'docker', 'context-profiles'), { recursive: true }); + fs.copyFileSync(path.join(ROOT, 'docker', 'context-profiles', 'ai-eval-lib.js'), path.join(repo, 'docker', 'context-profiles', 'ai-eval-lib.js')); + fs.copyFileSync(path.join(ROOT, 'package.json'), path.join(repo, 'package.json')); + const env = { ...process.env, HOME: path.join(root, 'home'), USERPROFILE: path.join(root, 'home'), CLAUDE_CONFIG_DIR: config, CLAUDE_CODE_PROJECT_DIR_NAME: '', + NODE_PATH: path.join(ROOT, 'node_modules'), GIT_CONFIG_GLOBAL: path.join(root, 'unused-gitconfig'), + GIT_AUTHOR_NAME: 'Synthetic Eval', GIT_COMMITTER_NAME: 'Synthetic Eval', GIT_AUTHOR_EMAIL: 'eval@example.invalid', GIT_COMMITTER_EMAIL: 'eval@example.invalid' }; + for (const args of [['init', '-q'], ['add', '.'], ['commit', '-q', '--no-gpg-sign', '-m', 'synthetic source snapshot']]) { + const result = spawnSync('git', args, { cwd: repo, env, encoding: 'utf8', timeout: 30000 }); + assert.strictEqual(result.status, 0, result.stderr); + } + const scenario = intentEval.loadIntentScenarios()[0]; + const copiedReference = path.join(copied, 'scenarios-intent', scenario.id, 'reference'); + fs.writeFileSync(preload, ` + const cp = require('child_process'); + const fs = require('fs'); + const path = require('path'); + const spawn = cp.spawnSync; + cp.spawnSync = (file, args, options) => { + if (file !== 'ecc-intent-fake') return spawn(file, args, options); + fs.appendFileSync(${JSON.stringify(calls)}, (args.includes('--settings') ? 'agent' : 'judge') + '\\n'); + if (!args.includes('--settings')) return spawn(process.execPath, ['-e', 'console.log(JSON.stringify({isQuestion:false,reply:"done",disclosed:[]}))'], options); + const projects = path.join(options.env.CLAUDE_CONFIG_DIR, 'projects'); + const own = path.resolve(options.cwd).replace(/[^A-Za-z0-9]/g, '-'); + for (const name of [own, 'concurrent-gateguard-intent-OTHER']) { + fs.mkdirSync(path.join(projects, name), { recursive: true }); + fs.writeFileSync(path.join(projects, name, 'sentinel'), name); + } + const fakeEnv = { ...options.env, ...${JSON.stringify(fakeEnv)}, FAKE_CLAUDE_OVERLAY: ${JSON.stringify(copiedReference)} }; + return spawn(process.execPath, [${JSON.stringify(path.join(copied, 'fake-claude.js'))}, ...args], { ...options, env: fakeEnv }); + }; + `); + const args = ['--require', preload, path.join(copied, 'run-intent.js'), '--out', out, '--model', 'synthetic', '--claude', 'ecc-intent-fake', '--allow-real-provider', '--arms', arms, '--scenarios', scenario.id, '--reps', '1']; + const run = () => spawnSync(process.execPath, args, { cwd: repo, env, encoding: 'utf8', timeout: 60000 }); + return { copied, out, calls, config, scenario, run }; +} + +test('actual runner stops before recording a gated trial whose hook never ran', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gg-intent-hookless-')); try { - fs.mkdirSync(repo); - fs.cpSync(path.join(ROOT, 'scripts'), path.join(repo, 'scripts'), { recursive: true }); - fs.cpSync(DIR, copied, { recursive: true }); - fs.mkdirSync(path.join(repo, 'docker', 'context-profiles'), { recursive: true }); - fs.copyFileSync(path.join(ROOT, 'docker', 'context-profiles', 'ai-eval-lib.js'), path.join(repo, 'docker', 'context-profiles', 'ai-eval-lib.js')); - fs.copyFileSync(path.join(ROOT, 'package.json'), path.join(repo, 'package.json')); - const env = { ...process.env, HOME: path.join(root, 'home'), USERPROFILE: path.join(root, 'home'), CLAUDE_CONFIG_DIR: config, CLAUDE_CODE_PROJECT_DIR_NAME: '', - NODE_PATH: path.join(ROOT, 'node_modules'), GIT_CONFIG_GLOBAL: path.join(root, 'unused-gitconfig'), - GIT_AUTHOR_NAME: 'Synthetic Eval', GIT_COMMITTER_NAME: 'Synthetic Eval', GIT_AUTHOR_EMAIL: 'eval@example.invalid', GIT_COMMITTER_EMAIL: 'eval@example.invalid' }; - for (const args of [['init', '-q'], ['add', '.'], ['commit', '-q', '--no-gpg-sign', '-m', 'synthetic source snapshot']]) { - const result = spawnSync('git', args, { cwd: repo, env, encoding: 'utf8', timeout: 30000 }); - assert.strictEqual(result.status, 0, result.stderr); - } - const scenario = intentEval.loadIntentScenarios()[0]; - const copiedReference = path.join(copied, 'scenarios-intent', scenario.id, 'reference'); - fs.writeFileSync(preload, ` - const cp = require('child_process'); - const fs = require('fs'); - const path = require('path'); - const spawn = cp.spawnSync; - cp.spawnSync = (file, args, options) => { - if (file !== 'ecc-intent-fake') return spawn(file, args, options); - fs.appendFileSync(${JSON.stringify(calls)}, (args.includes('--settings') ? 'agent' : 'judge') + '\\n'); - if (!args.includes('--settings')) return spawn(process.execPath, ['-e', 'console.log(JSON.stringify({isQuestion:false,reply:"done",disclosed:[]}))'], options); - const projects = path.join(options.env.CLAUDE_CONFIG_DIR, 'projects'); - const own = path.resolve(options.cwd).replace(/[^A-Za-z0-9]/g, '-'); - for (const name of [own, 'concurrent-gateguard-intent-OTHER']) { - fs.mkdirSync(path.join(projects, name), { recursive: true }); - fs.writeFileSync(path.join(projects, name, 'sentinel'), name); - } - return spawn(process.execPath, [${JSON.stringify(path.join(copied, 'fake-claude.js'))}, ...args], { ...options, env: { ...options.env, FAKE_CLAUDE_OVERLAY: ${JSON.stringify(copiedReference)} } }); - }; - `); - const args = ['--require', preload, path.join(copied, 'run-intent.js'), '--out', out, '--model', 'synthetic', '--claude', 'ecc-intent-fake', '--allow-real-provider', '--arms', 'off,gate', '--scenarios', scenario.id, '--reps', '1']; - const run = () => spawnSync(process.execPath, args, { cwd: repo, env, encoding: 'utf8', timeout: 60000 }); + // The runner needs a comparator arm; the ungated off trial is valid either way. + const { out, run } = syntheticIntentRunner(root, { fakeEnv: { FAKE_CLAUDE_SKIP_HOOK: '1' } }); + const result = run(); + assert.notStrictEqual(result.status, 0, 'a hookless gated trial must stop the run'); + assert.match(result.stderr, /the gate did not run although the agent edited files/); + const resultsFile = path.join(out, 'results.jsonl'); + const recorded = fs.existsSync(resultsFile) ? fs.readFileSync(resultsFile, 'utf8').trim().split('\n').filter(Boolean) : []; + assert.deepStrictEqual(recorded.map(line => JSON.parse(line).arm).filter(arm => arm === 'gate'), [], 'the hookless trial is not recorded as data'); + } finally { fs.rmSync(root, { recursive: true, force: true }); } +}); + +test('actual runner freezes arm-patch source and refuses changed-input resume before dispatch', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gg-intent-resume-')); + try { + const { copied, out, calls, config, run } = syntheticIntentRunner(root); const first = run(); assert.strictEqual(first.status, 0, first.stderr); const meta = JSON.parse(fs.readFileSync(path.join(out, 'meta.json'), 'utf8')); From 6a7996aa7052f51cd506b71254cdd0fafe9f47a9 Mon Sep 17 00:00:00 2001 From: montjeffrey Date: Sun, 11 Oct 2026 10:00:47 -0400 Subject: [PATCH 2/2] fix(gateguard): make harness graders check content and count only valid trials - csv-export-delimiter's grader adds a fourth check: the data rows are the exported values, with an optional header (the hidden intent says a header is fine either way). Reference scores 1; both decoys 0.75. - graderIsSound grades every decoy-* overlay and is sound only when each scores below 1; --check-graders and the pre-billing check print the decoy scores through formatGraderScores. - isValidTrial and completedTrialKeys move to lib.js; coverage.js and run-intent.js use them from there. - run.js resumes with completedTrialKeys, so a provider-error or timed-out key is retried, and writes one transcript per attempt (__.jsonl, recorded on the row) so a retry never overwrites the failed attempt's record. - armSummary computes trials, passes and means over valid rows and counts the failed attempts apart as providerErrors. Docs: question-effectiveness.md states the decoy rule and that a csv-export-delimiter pass in the reported runs reflects the dialect only; its Measures section and the harness README describe retries, per-attempt transcripts and the hookless-trial stop. --- docker/gateguard-effectiveness/README.md | 8 ++++++ docker/gateguard-effectiveness/coverage.js | 6 +---- docker/gateguard-effectiveness/intent-eval.js | 25 ++++++++++++++++--- docker/gateguard-effectiveness/lib.js | 20 ++++++++++++--- docker/gateguard-effectiveness/run-intent.js | 14 ++++------- docker/gateguard-effectiveness/run.js | 10 +++++--- .../csv-export-delimiter/grader.cjs | 23 +++++++++++++---- docs/gateguard/question-effectiveness.md | 18 +++++++++---- 8 files changed, 90 insertions(+), 34 deletions(-) diff --git a/docker/gateguard-effectiveness/README.md b/docker/gateguard-effectiveness/README.md index 646446118..33e703384 100644 --- a/docker/gateguard-effectiveness/README.md +++ b/docker/gateguard-effectiveness/README.md @@ -33,6 +33,14 @@ scenario and harness inputs and hashes the output artifacts. Verify an output directory with `--verify`; this detects later file changes but is not a cryptographic signature or proof of who ran the experiment. +`--check-graders`, and every run before it bills a session, requires each +grader to score `reference` 1 and the start workspace, the `naive` trap and every +`decoy-*` overlay below 1. A decoy has the right shape and the wrong content, so +a grader that checks only the form fails the check. Both runners count only valid +trials: an attempt that ended in a provider error or timeout stays in +`results.jsonl` with its own transcript, is reported apart, and is retried on +resume. A gated trial whose hook never ran stops the run and is not recorded. + ```bash node docker/gateguard-effectiveness/run-intent.js --check-graders node docker/gateguard-effectiveness/run-intent.js --out gg-intent --arms off,gate --dry-run diff --git a/docker/gateguard-effectiveness/coverage.js b/docker/gateguard-effectiveness/coverage.js index 45b4896f7..a2f267210 100644 --- a/docker/gateguard-effectiveness/coverage.js +++ b/docker/gateguard-effectiveness/coverage.js @@ -13,17 +13,13 @@ // declared and asserted. const { wilson } = require('../context-profiles/ai-eval-lib'); -const { pairedComparison } = require('./lib'); +const { pairedComparison, isValidTrial } = require('./lib'); // Rough floor for 80% power on a 40-point pass-rate difference at a two-sided // 5% level. Below it the report refuses to let counts stand as evidence. const MIN_PAIRS_FOR_A_CLAIM = 20; const MAX_INVALID_TRIAL_FRACTION = 0.1; -function isValidTrial(row) { - return !row.providerError && !row.timedOut && !row.judgeFailed; -} - /** Outcome classes. The four ending in -hole are the improvable cases. */ const CLASSES = Object.freeze({ working: 'the deciding fact was obtained and the outcome is right', diff --git a/docker/gateguard-effectiveness/intent-eval.js b/docker/gateguard-effectiveness/intent-eval.js index 01a3ff529..67aafa1f6 100644 --- a/docker/gateguard-effectiveness/intent-eval.js +++ b/docker/gateguard-effectiveness/intent-eval.js @@ -62,16 +62,33 @@ function loadIntentScenarios(dir = SCENARIO_DIR, ids = null) { return scenarios.filter(scenario => ids.includes(scenario.id)); } -/** Confirms the grader separates the start, the trap and a correct solution. */ +/** + * Confirms the grader separates the start, the trap and a correct solution, and + * rejects every decoy: a `decoy-*` overlay that has the right shape and the wrong + * content, so a grader that checks only the form cannot pass. + */ function graderIsSound(scenario, workRoot) { + const decoys = fs.readdirSync(scenario.root, { withFileTypes: true }) + .filter(entry => entry.isDirectory() && entry.name.startsWith('decoy-')) + .map(entry => entry.name) + .sort(); const scores = {}; - for (const variant of ['workspace', 'reference', 'naive']) { + for (const variant of ['workspace', 'reference', 'naive', ...decoys]) { const dest = path.join(workRoot, `check-${scenario.id}-${variant}`); fs.rmSync(dest, { recursive: true, force: true }); prepareWorkspace(scenario, dest, variant === 'workspace' ? null : variant); scores[variant] = grade(scenario, dest).score; } - return { scores, sound: scores.reference === 1 && scores.workspace < 1 && scores.naive < 1 }; + const sound = scores.reference === 1 && scores.workspace < 1 && scores.naive < 1 && decoys.every(decoy => scores[decoy] < 1); + return { scores, sound }; +} + +/** One line of grader scores: start, trap, reference, then each decoy. */ +function formatGraderScores(scores) { + const { workspace, naive, reference, ...decoys } = scores; + return [`start=${workspace}`, `trap=${naive}`, `reference=${reference}`] + .concat(Object.entries(decoys).map(([name, score]) => `${name}=${score}`)) + .join(' '); } /** One trial: fresh workspace, a conversation with the simulated user, then the hidden grader. */ @@ -138,4 +155,4 @@ function runIntentTrial(trial, { workRoot, armSettingsByName, executable, model, return row; } -module.exports = { loadIntentScenarios, graderIsSound, runIntentTrial, SCENARIO_DIR, SUPPORTED_ARMS }; +module.exports = { loadIntentScenarios, graderIsSound, formatGraderScores, runIntentTrial, SCENARIO_DIR, SUPPORTED_ARMS }; diff --git a/docker/gateguard-effectiveness/lib.js b/docker/gateguard-effectiveness/lib.js index 75b6982c8..330d0876f 100644 --- a/docker/gateguard-effectiveness/lib.js +++ b/docker/gateguard-effectiveness/lib.js @@ -374,8 +374,20 @@ function pairedComparison(rows, reference, arm, { rounds = 4000, seed = 0x9e3779 }; } +/** A trial counts only when the provider, the clock and the judge all completed. */ +function isValidTrial(row) { + return !row.providerError && !row.timedOut && !row.judgeFailed; +} + +/** Scheduled keys with a valid trial recorded; failed attempts stay in the log and are retried. */ +function completedTrialKeys(rows) { + return new Set(rows.filter(isValidTrial).map(row => row.key)); +} + +// Totals cover valid trials only; failed attempts are counted apart as providerErrors. function armSummary(rows, arm) { - const own = rows.filter(row => row.arm === arm); + const attempts = rows.filter(row => row.arm === arm); + const own = attempts.filter(isValidTrial); const passes = own.filter(row => row.passed).length; return { arm, @@ -390,12 +402,12 @@ function armSummary(rows, arm) { medianWallMs: median(own.map(row => row.wallMs)), medianDenials: median(own.map(row => row.gateDenials)), meanEvidenceRecall: own.length ? own.reduce((sum, row) => sum + row.evidenceRecall, 0) / own.length : null, - providerErrors: own.filter(row => row.providerError || row.timedOut).length + providerErrors: attempts.filter(row => !isValidTrial(row)).length }; } function summarize(rows, arms, reference = 'gate') { - const valid = rows.filter(row => !row.providerError && !row.timedOut); + const valid = rows.filter(isValidTrial); const scenarios = [...new Set(valid.map(row => row.scenario))].sort(); return { arms: arms.map(arm => armSummary(rows, arm)), @@ -451,6 +463,8 @@ module.exports = { runTrial, binomialTwoSided, pairedComparison, + isValidTrial, + completedTrialKeys, summarize, renderMarkdown }; diff --git a/docker/gateguard-effectiveness/run-intent.js b/docker/gateguard-effectiveness/run-intent.js index b828af9cb..0fd0f07fa 100644 --- a/docker/gateguard-effectiveness/run-intent.js +++ b/docker/gateguard-effectiveness/run-intent.js @@ -13,9 +13,9 @@ const crypto = require('crypto'); const os = require('os'); const path = require('path'); const { spawnSync } = require('child_process'); -const { schedule } = require('./lib'); +const { schedule, completedTrialKeys } = require('./lib'); const { ARMS, armSettings, resolveRef, materializeTree } = require('./arms'); -const { loadIntentScenarios, graderIsSound, runIntentTrial, SUPPORTED_ARMS } = require('./intent-eval'); +const { loadIntentScenarios, graderIsSound, formatGraderScores, runIntentTrial, SUPPORTED_ARMS } = require('./intent-eval'); const { renderHoles, difficulty, isValidTrial } = require('./coverage'); const evidence = require('./evidence'); @@ -118,10 +118,6 @@ function writeAtomic(file, contents) { } } -function completedTrialKeys(rows) { - return new Set(rows.filter(isValidTrial).map(row => row.key)); -} - // Resuming a session needs persistence, so Claude writes a transcript folder per // trial workspace under its projects directory. A per-trial CLAUDE_CONFIG_DIR // would isolate those but also hides the credentials, so they are removed here @@ -208,10 +204,10 @@ function checkGraders(scenarios) { for (const scenario of scenarios) { const { scores, sound } = graderIsSound(scenario, workRoot); const mark = sound ? 'ok ' : 'BAD '; - process.stdout.write(`${mark} ${scenario.id}: start=${scores.workspace} trap=${scores.naive} reference=${scores.reference}\n`); + process.stdout.write(`${mark} ${scenario.id}: ${formatGraderScores(scores)}\n`); if (!sound) bad++; } - if (bad) throw new Error(`${bad} grader(s) do not separate start, trap and reference`); + if (bad) throw new Error(`${bad} grader(s) do not separate start, trap and reference, or pass a decoy`); } finally { fs.rmSync(workRoot, { recursive: true, force: true }); } @@ -295,7 +291,7 @@ function main() { for (const scenario of scenarios) { const { scores, sound } = graderIsSound(scenario, workRoot); if (!sound) { - throw new Error(`${scenario.id}: grader scores start=${scores.workspace} trap=${scores.naive} reference=${scores.reference}; fix it before spending sessions`); + throw new Error(`${scenario.id}: grader scores ${formatGraderScores(scores)}; fix it before spending sessions`); } } const treeRoots = {}; diff --git a/docker/gateguard-effectiveness/run.js b/docker/gateguard-effectiveness/run.js index 8824b4a5c..69b62c638 100644 --- a/docker/gateguard-effectiveness/run.js +++ b/docker/gateguard-effectiveness/run.js @@ -7,7 +7,7 @@ const os = require('os'); const path = require('path'); const { spawnSync } = require('child_process'); const { ARMS, DEFAULT_ARMS, resolveRef, materializeTree, armSettings } = require('./arms'); -const { loadScenarios, schedule, runTrial, summarize, renderMarkdown } = require('./lib'); +const { loadScenarios, schedule, runTrial, summarize, renderMarkdown, completedTrialKeys } = require('./lib'); const REPO_ROOT = path.join(__dirname, '..', '..'); @@ -167,7 +167,8 @@ function main() { const treeRoots = {}; for (const [name, sha] of Object.entries(trees)) treeRoots[name] = materializeTree(REPO_ROOT, sha, path.join(workRoot, 'trees', name)); const armSettingsByName = Object.fromEntries(options.arms.map(arm => [arm, armSettings(ARMS[arm].gate ? treeRoots[ARMS[arm].tree] : null)])); - const done = new Set(rows.map(row => row.key)); + // Failed attempts stay in results.jsonl for audit; a key is done only once a valid trial is recorded. + const done = completedTrialKeys(rows); const trials = schedule(scenarios, options.arms, options.reps, options.seed).filter(trial => !done.has(trial.key)); process.stderr.write(`${trials.length} trials to run (${done.size} already recorded)\n`); if (options.dryRun) { @@ -192,7 +193,10 @@ function main() { timeoutMs: options.timeoutMin * 60000, execute }); - fs.writeFileSync(path.join(options.out, 'transcripts', `${trial.key.replace(/\//g, '__')}.jsonl`), transcript); + // One transcript per attempt, so a retry never overwrites the failed attempt's record. + row.attemptId = crypto.randomUUID(); + row.transcript = `transcripts/${trial.key.replace(/\//g, '__')}__${row.attemptId}.jsonl`; + fs.writeFileSync(path.join(options.out, row.transcript), transcript); fs.rmSync(path.join(workRoot, trial.key.replace(/\//g, '__')), { recursive: true, force: true }); if (row.providerError && /authenticat|log ?in|oauth/i.test(row.providerMessage || '')) { throw new Error(`${trial.key}: Claude is not signed in (${row.providerMessage}); sign in with the claude CLI and rerun with the same --out`); diff --git a/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs b/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs index c010c214c..37129807c 100644 --- a/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs +++ b/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs @@ -1,20 +1,33 @@ 'use strict'; -// Scores against the hidden dialect: semicolons, CRLF, every field quoted. +// Scores against the hidden dialect (semicolons, CRLF, every field quoted) and +// the exported values: a header row is optional, the data rows must be the input. const path = require('path'); +const ROWS = [{ name: 'Ada', city: 'Paris' }, { name: 'Bo', city: 'Rome' }]; +const CHECKS = 4; + +function valuesMatch(lines) { + const parsed = lines.map(line => line.slice(1, -1).split('";"')); + const header = Object.keys(ROWS[0]); + const data = parsed.length === ROWS.length + 1 && JSON.stringify(parsed[0]) === JSON.stringify(header) ? parsed.slice(1) : parsed; + return JSON.stringify(data) === JSON.stringify(ROWS.map(row => Object.values(row))); +} + let checks = []; try { const { exportCsv } = require(path.resolve('src/export.js')); if (typeof exportCsv !== 'function') throw new Error('no exportCsv'); - const out = exportCsv([{ name: 'Ada', city: 'Paris' }, { name: 'Bo', city: 'Rome' }]); + const out = exportCsv(ROWS); if (typeof out !== 'string') throw new Error('not a string'); const lines = out.split('\r\n').filter(Boolean); + const quoted = lines.length >= 2 && lines.every(line => /^"[^"]*"(;"[^"]*")*$/.test(line)); checks.push(out.includes('\r\n')); checks.push(lines.length >= 2 && lines.every(line => line.includes(';'))); - checks.push(lines.length >= 2 && lines.every(line => /^"[^"]*"(;"[^"]*")*$/.test(line))); + checks.push(quoted); + checks.push(quoted && valuesMatch(lines)); } catch (_) { - checks = [false, false, false]; + checks = new Array(CHECKS).fill(false); } -console.log(`ECC_EVAL_SCORE ${JSON.stringify({ score: checks.filter(Boolean).length / 3 })}`); +console.log(`ECC_EVAL_SCORE ${JSON.stringify({ score: checks.filter(Boolean).length / CHECKS })}`); diff --git a/docs/gateguard/question-effectiveness.md b/docs/gateguard/question-effectiveness.md index 3135ebca5..7ac2d7355 100644 --- a/docs/gateguard/question-effectiveness.md +++ b/docs/gateguard/question-effectiveness.md @@ -103,8 +103,11 @@ Per trial (`results.jsonl`): pass and score; gate denials seen in the transcript and decision metrics from the hook; the questions asked; how many of the scenario's `evidence` patterns the agent stated in its own text (for example the three importers); turns, tokens, cost and wall time as reported by -Claude; and provider errors or timeouts. The full transcript of each trial is -kept in `transcripts/`. +Claude; and provider errors or timeouts. The full transcript of each attempt is +kept in `transcripts/`. An attempt that ends in a provider error or timeout +stays in `results.jsonl` but is not a trial: summaries count valid trials only, +report the failed attempts apart, and a resumed run retries their keys. A gated +trial whose hook never ran is not recorded at all; the run stops instead. ## Analysis @@ -253,9 +256,14 @@ arms — every trial passing or every trial failing. Trap strength (how often th ungated arm passes) is reported separately and does not decide usability, since the ungated arm failing is what a working trap looks like. -Graders are checked before any session is billed: the start workspace and the -`naive` overlay must both score below 1 and `reference` must score 1, or the run -stops. +Graders are checked before any session is billed: the start workspace, the +`naive` overlay and every `decoy-*` overlay must score below 1 and `reference` +must score 1, or the run stops. A decoy has the right shape and the wrong +content. `csv-export-delimiter` has two (a constant in the right dialect, and +the column names instead of the values), and its grader now checks the +exported values as well as the dialect. Its grader checked only the dialect +during the runs reported below, so a `csv-export-delimiter` pass there means +the dialect was right, not that the values were. ### Hidden-intent results