diff --git a/CHANGELOG.md b/CHANGELOG.md index 66e4994d6..9b279b0e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ - Add opt-in `GATEGUARD_FACT_FORCE_MAX_DENIALS`, a per-session denial cap: once that many first-touch denials have been issued, further new paths pass through (counted in `cap_allows`). Unset keeps the current behaviour; a malformed value is reported once on stderr and leaves the gate uncapped (#2755, thanks @SulimanAbdulrazzaq; closes #2608). - GateGuard loads its target-classification, transcript, search-matching and change-profile modules only on the calls that use them, and parses only the searches that name the target, so shell commands and edits of already checked files stay within a few milliseconds of the previous hook. - Add a development-only harness, `docker/gateguard-effectiveness/`, that runs real Claude sessions on six trapped coding tasks with no gate, the gate, the gate without the question meant to catch each trap, and a placebo gate, and grades the results with hidden graders (see `docs/gateguard/question-effectiveness.md`). -- Report that harness's pilot: 72 sessions in which every arm passed every trial, so it measured the gate's cost (about 45% more turns) and nothing about its effect. Add a second runner, `run-intent.js`, that measures effectiveness instead: the deciding fact exists only in a simulated user's hidden intent, sessions are multi-turn so the gate's question can actually be answered, scenarios are authored blind to the gate's question taxonomy, and each trial is classified into a hole worth fixing rather than checked against a declared target. +- Report that harness's pilot: 72 sessions in which every arm passed every trial, so it measured the gate's cost (about 45% more turns) and nothing about its effect. Add a second runner, `run-intent.js`, for exploratory effectiveness measurements: the deciding fact exists only in a simulated user's hidden intent and sessions are multi-turn. Scenario files omit declared gate-question targets, but the authors knew the taxonomy and this is not blind authorship. - Add opt-in `GATEGUARD_METRICS` decision metrics (codes only, never paths, commands or content) and `scripts/gateguard-report.js` to summarise them. - Sensitive targets (`.env*`, keys and certificates, `credentials*`, `secrets.*`, `auth`/`security`/`payment(s)`/`billing`/`migrations` path segments, `.github/workflows/`) and files with more than one hard link get none of these allowances, judged on the path as written and its real location, and are gated in subagents too. `GATEGUARD_EXEMPT_GLOBS` still takes precedence. - Newly denied: a subagent's first touch of a sensitive or hard-linked file, and the first NotebookEdit of each notebook. Every other rule only removes denials and never returns an `allow` decision. See `skills/gateguard/SKILL.md`. diff --git a/docker/gateguard-effectiveness/README.md b/docker/gateguard-effectiveness/README.md index 3aa3770cb..55d7462a8 100644 --- a/docker/gateguard-effectiveness/README.md +++ b/docker/gateguard-effectiveness/README.md @@ -18,13 +18,22 @@ node docker/gateguard-effectiveness/run.js --out gg-eff --model --allow- `run-intent.js` — effectiveness. Multi-turn sessions over `scenarios-intent/`, where the deciding fact exists only in a simulated user's hidden intent, so an -agent that does not ask can only guess. Scenarios are authored blind to the -gate's question taxonomy, and the report classifies each trial into a hole worth -fixing. +agent that does not ask can only guess. Scenario files omit declared gate +question targets, but this does not establish blind or independent authorship: +the authors knew the taxonomy, and the external-contract question was added +after inspecting this corpus. Treat those results as exploratory. The report +classifies behavior into descriptive hole categories. + +Each run writes `meta.json`, raw `results.jsonl`, transcripts, `summary.json`, +`holes.md`, and `evidence.json`. The evidence manifest fingerprints the frozen +scenario and harness inputs and hashes the output artifacts. Verify an output +directory with `--verify`; this detects later file changes but is not a +cryptographic signature or proof of who ran the experiment. ```bash node docker/gateguard-effectiveness/run-intent.js --check-graders node docker/gateguard-effectiveness/run-intent.js --out gg-intent --arms off,gate --dry-run node docker/gateguard-effectiveness/run-intent.js --out gg-intent --model --allow-real-provider node docker/gateguard-effectiveness/run-intent.js --out gg-intent --summarize +node docker/gateguard-effectiveness/run-intent.js --out gg-intent --verify ``` diff --git a/docker/gateguard-effectiveness/coverage.js b/docker/gateguard-effectiveness/coverage.js index 10a49a789..45b4896f7 100644 --- a/docker/gateguard-effectiveness/coverage.js +++ b/docker/gateguard-effectiveness/coverage.js @@ -82,7 +82,8 @@ function difficulty(rows, { baselineArm = 'off' } = {}) { } /** - * Whether a scenario separates any arm from any other, judged across every arm. + * Descriptive outcome range across all arms. This is post-treatment information + * and must not determine which scenarios enter the primary arm comparison. * * Only a scenario where nothing varies is uninformative: every trial passing * (the trap does not bite) or every trial failing (nothing reaches it). Judging @@ -182,7 +183,7 @@ function renderEffect(rows, { referenceArm = 'gate', baselineArm = 'off' } = {}) return `${lines.join('\n')}\n`; } -function renderHoles(rows, { baselineArm = 'off' } = {}) { +function renderHoles(rows, { baselineArm = 'off', expectedTrialKeys = null } = {}) { const invalid = rows.filter(row => !isValidTrial(row)); const invalidFraction = rows.length ? invalid.length / rows.length : 0; const validity = rows.length @@ -197,6 +198,19 @@ function renderHoles(rows, { baselineArm = 'off' } = {}) { ].join('\n'); } rows = rows.filter(isValidTrial); + if (expectedTrialKeys) { + const validKeys = new Set(rows.map(row => row.key)); + const missing = expectedTrialKeys.filter(key => !validKeys.has(key)); + if (missing.length) { + return [ + '### Run validity', + '', + `${validity}`, + `INCOMPLETE: ${validKeys.size}/${expectedTrialKeys.length} scheduled trials have valid results. Primary arm comparisons are suppressed until the fixed schedule is complete.`, + '' + ].join('\n'); + } + } const arms = [...new Set(rows.map(row => row.arm))].sort(); const classes = Object.keys(CLASSES); const lines = []; @@ -215,16 +229,17 @@ function renderHoles(rows, { baselineArm = 'off' } = {}) { lines.push(''); lines.push( dead.length - ? `${dead.length} of ${verdicts.length} scenarios carry no information (${dead.map(row => `${row.scenario}: ${row.label}`).join('; ')}). Arm comparisons below exclude them.` + ? `${dead.length} of ${verdicts.length} scenarios show no outcome variation (${dead.map(row => `${row.scenario}: ${row.label}`).join('; ')}). This is descriptive only; all valid predeclared scenarios remain in the primary arm comparison.` : `All ${verdicts.length} scenarios separate at least one arm from another.` ); const usableScenarios = new Set(verdicts.filter(row => row.label === 'informative').map(row => row.scenario)); const usable = rows.filter(row => usableScenarios.has(row.scenario)); - lines.push('', renderEffect(usable.length ? usable : rows, { baselineArm }).trimEnd()); + lines.push('', renderEffect(rows, { baselineArm }).trimEnd()); lines.push('', '### Outcome classes', ''); + lines.push('Descriptive mechanism counts below include only scenarios with outcome variation; they are not the primary comparison.', ''); lines.push(`| Arm | ${classes.join(' | ')} |`, `| --- | ${classes.map(() => '---:').join(' | ')} |`); for (const arm of arms) { const counts = tally(usable.filter(row => row.arm === arm)); diff --git a/docker/gateguard-effectiveness/evidence.js b/docker/gateguard-effectiveness/evidence.js new file mode 100644 index 000000000..fde4be95e --- /dev/null +++ b/docker/gateguard-effectiveness/evidence.js @@ -0,0 +1,106 @@ +'use strict'; + +const crypto = require('crypto'); +const fs = require('fs'); +const path = require('path'); + +const SCHEMA_VERSION = 1; + +function sha256(content) { + return crypto.createHash('sha256').update(content).digest('hex'); +} + +function hashFile(file) { + return sha256(fs.readFileSync(file)); +} + +function hashDirectory(dir) { + const entries = []; + function visit(current, relative = '') { + for (const entry of fs.readdirSync(current, { withFileTypes: true }).sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0))) { + const child = path.join(current, entry.name); + const name = path.posix.join(relative.split(path.sep).join('/'), entry.name); + if (entry.isDirectory()) visit(child, name); + else if (entry.isFile()) entries.push({ path: name, sha256: hashFile(child) }); + else throw new Error(`unsupported evidence input: ${child}`); + } + } + visit(dir); + return sha256(JSON.stringify(entries)); +} + +function validateRows(rows, trials) { + const scheduled = new Map(trials.map(trial => [trial.key, trial])); + const validKeys = new Set(); + for (const row of rows) { + if (!row || typeof row.key !== 'string' || !scheduled.has(row.key)) { + throw new Error(`result row has an unknown scheduled key: ${row && row.key}`); + } + const trial = scheduled.get(row.key); + if (row.scenario !== trial.scenario.id || row.arm !== trial.arm || row.rep !== trial.rep) { + throw new Error(`result row does not match its scheduled key: ${row.key}`); + } + if (!row.providerError && !row.timedOut && !row.judgeFailed) { + if (validKeys.has(row.key)) throw new Error(`duplicate valid result row: ${row.key}`); + validKeys.add(row.key); + } + } + return validKeys; +} + +function buildManifest({ meta, configuration, scenarioFingerprints, sourceFingerprints, outDir, rows = [] }) { + const files = {}; + for (const name of ['meta.json', 'results.jsonl', 'holes.md', 'summary.json']) { + const file = path.join(outDir, name); + if (fs.existsSync(file)) files[name] = { bytes: fs.statSync(file).size, sha256: hashFile(file) }; + } + const transcriptsDir = path.join(outDir, 'transcripts'); + if (fs.existsSync(transcriptsDir)) { + for (const name of fs.readdirSync(transcriptsDir).sort()) { + const file = path.join(transcriptsDir, name); + if (fs.statSync(file).isFile()) files[`transcripts/${name}`] = { bytes: fs.statSync(file).size, sha256: hashFile(file) }; + } + } + return { + schemaVersion: SCHEMA_VERSION, + generatedAt: new Date().toISOString(), + meta, + configuration, + scenarioFingerprints, + sourceFingerprints, + integrity: { rowTranscriptLinks: rows.length > 0 && rows.every(row => Boolean(row.transcript && row.transcriptSha256)) }, + runtime: { node: process.version, platform: process.platform, arch: process.arch }, + files + }; +} + +function verifyManifest(outDir) { + const manifestPath = path.join(outDir, 'evidence.json'); + const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')); + if (manifest.schemaVersion !== SCHEMA_VERSION || !manifest.files || typeof manifest.files !== 'object') { + throw new Error('unsupported or malformed evidence manifest'); + } + for (const [name, expected] of Object.entries(manifest.files)) { + const file = path.resolve(outDir, name); + if (!file.startsWith(`${path.resolve(outDir)}${path.sep}`)) throw new Error(`invalid evidence path: ${name}`); + if (!fs.existsSync(file) || !fs.statSync(file).isFile()) throw new Error(`missing evidence artifact: ${name}`); + const actual = hashFile(file); + if (actual !== expected.sha256 || fs.statSync(file).size !== expected.bytes) { + throw new Error(`evidence artifact checksum mismatch: ${name}`); + } + } + if (manifest.integrity && manifest.integrity.rowTranscriptLinks) { + const resultFile = path.join(outDir, 'results.jsonl'); + if (!fs.existsSync(resultFile)) throw new Error('results.jsonl is missing from a transcript-linked bundle'); + const rows = fs.readFileSync(resultFile, 'utf8').split('\n').filter(Boolean).map(line => JSON.parse(line)); + for (const row of rows) { + const transcript = manifest.files[row.transcript]; + if (!transcript || transcript.sha256 !== row.transcriptSha256) { + throw new Error(`result transcript link is missing or mismatched: ${row.key}`); + } + } + } + return manifest; +} + +module.exports = { SCHEMA_VERSION, hashFile, hashDirectory, validateRows, buildManifest, verifyManifest }; diff --git a/docker/gateguard-effectiveness/intent-eval.js b/docker/gateguard-effectiveness/intent-eval.js index 86d9751f7..01a3ff529 100644 --- a/docker/gateguard-effectiveness/intent-eval.js +++ b/docker/gateguard-effectiveness/intent-eval.js @@ -31,7 +31,7 @@ function loadIntentScenarios(dir = SCENARIO_DIR, ids = null) { const intent = JSON.parse(fs.readFileSync(path.join(root, 'intent.json'), 'utf8')); if (spec.id !== entry.name) throw new Error(`scenario ${entry.name} declares id ${spec.id}`); if (spec.targetQuestions) { - throw new Error(`scenario ${spec.id} declares targetQuestions; hidden-intent scenarios are authored blind to the taxonomy`); + throw new Error(`scenario ${spec.id} declares targetQuestions; hidden-intent scenarios must not encode gate-taxonomy targets`); } for (const key of ['trap', 'prompt']) { if (typeof spec[key] !== 'string' || !spec[key]) throw new Error(`scenario ${spec.id} is missing ${key}`); diff --git a/docker/gateguard-effectiveness/run-intent.js b/docker/gateguard-effectiveness/run-intent.js index e1f0720d7..4bdfe90f8 100644 --- a/docker/gateguard-effectiveness/run-intent.js +++ b/docker/gateguard-effectiveness/run-intent.js @@ -9,6 +9,7 @@ // reports the holes rather than asserting coverage. const fs = require('fs'); +const crypto = require('crypto'); const os = require('os'); const path = require('path'); const { spawnSync } = require('child_process'); @@ -16,6 +17,7 @@ const { schedule } = require('./lib'); const { ARMS, armSettings, resolveRef, materializeTree } = require('./arms'); const { loadIntentScenarios, graderIsSound, runIntentTrial, SUPPORTED_ARMS } = require('./intent-eval'); const { renderHoles, difficulty, isValidTrial } = require('./coverage'); +const evidence = require('./evidence'); const REPO_ROOT = path.resolve(__dirname, '..', '..'); @@ -35,7 +37,8 @@ const USAGE = [ ' --seed arm order seed (default: 11)', ' --check-graders confirm every grader separates start, trap and reference, then exit', ' --dry-run print the schedule without running sessions', - ' --summarize only rebuild holes.md from results.jsonl' + ' --summarize rebuild the report from results.jsonl', + ' --verify verify evidence.json checksums and exit' ].join('\n'); function parseArgs(argv) { @@ -43,7 +46,7 @@ function parseArgs(argv) { out: null, model: null, judgeModel: 'haiku', claude: 'claude', arms: ['off', 'gate', 'placebo'], scenarios: null, reps: 3, userTurns: 3, candidateRef: 'HEAD', mainRef: 'upstream/main', maxTurns: 40, timeoutMin: 15, seed: 11, - allowRealProvider: false, dryRun: false, summarizeOnly: false, checkGraders: false, help: false + allowRealProvider: false, dryRun: false, summarizeOnly: false, verifyOnly: false, checkGraders: false, help: false }; const list = value => value.split(',').map(item => item.trim()).filter(Boolean); const positive = (value, name) => { @@ -73,13 +76,18 @@ function parseArgs(argv) { else if (arg === '--check-graders') options.checkGraders = true; else if (arg === '--dry-run') options.dryRun = true; else if (arg === '--summarize') options.summarizeOnly = true; + else if (arg === '--verify') options.verifyOnly = true; else if (arg === '--help' || arg === '-h') options.help = true; else throw new Error(`unknown argument: ${arg}\n\n${USAGE}`); } const unsupported = options.arms.filter(arm => !SUPPORTED_ARMS.includes(arm)); if (unsupported.length) throw new Error(`arm not supported for hidden-intent scenarios: ${unsupported.join(', ')}`); + if (!options.verifyOnly && !options.help && !options.checkGraders && + (!options.arms.includes('gate') || options.arms.length < 2)) { + throw new Error('the effectiveness comparison needs the gate arm and at least one comparator arm'); + } if (!options.help && !options.checkGraders && !options.out) throw new Error(`--out is required\n\n${USAGE}`); - if (!options.help && !options.checkGraders && !options.dryRun && !options.summarizeOnly) { + if (!options.help && !options.checkGraders && !options.dryRun && !options.summarizeOnly && !options.verifyOnly) { if (!options.model) throw new Error(`--model is required\n\n${USAGE}`); if (!options.allowRealProvider) throw new Error(`real sessions need --allow-real-provider\n\n${USAGE}`); } @@ -88,7 +96,25 @@ function parseArgs(argv) { function readResults(file) { if (!fs.existsSync(file)) return []; - return fs.readFileSync(file, 'utf8').split('\n').filter(Boolean).map(line => JSON.parse(line)); + return fs.readFileSync(file, 'utf8').split('\n').flatMap((line, index) => { + if (!line) return []; + try { + return [JSON.parse(line)]; + } catch (error) { + throw new Error(`results.jsonl line ${index + 1} is malformed; preserve the file for recovery (${error.message})`); + } + }); +} + +function writeAtomic(file, contents) { + const temporary = path.join(path.dirname(file), `.${path.basename(file)}.${crypto.randomUUID()}.tmp`); + try { + fs.writeFileSync(temporary, contents, { flag: 'wx' }); + fs.renameSync(temporary, file); + } catch (error) { + if (fs.existsSync(temporary)) fs.rmSync(temporary, { force: true }); + throw error; + } } function completedTrialKeys(rows) { @@ -123,18 +149,39 @@ function removeTrialSessions(before) { function writeReport(options, rows, meta) { const header = `Model: ${meta.model}; judge: ${meta.judgeModel}; hook commit: ${meta.sha}; reps: ${meta.reps}; user turns: ${meta.userTurns}.\n\n`; - const body = rows.length ? renderHoles(rows) : 'No trials recorded yet.\n'; - fs.writeFileSync(path.join(options.out, 'holes.md'), header + body); - fs.writeFileSync( + const configuration = meta.configuration; + const expectedTrials = configuration + ? schedule(configuration.scenarios.map(id => ({ id })), configuration.arms, configuration.reps, configuration.seed) + : []; + const validKeys = new Set(rows.filter(isValidTrial).map(row => row.key)); + const body = configuration + ? renderHoles(rows, { expectedTrialKeys: expectedTrials.map(trial => trial.key) }) + : rows.length ? renderHoles(rows) : 'No trials recorded yet.\n'; + writeAtomic(path.join(options.out, 'holes.md'), header + body); + writeAtomic( path.join(options.out, 'summary.json'), `${JSON.stringify({ meta, difficulty: difficulty(rows.filter(isValidTrial)), rows: rows.length, + expectedTrials: expectedTrials.length || null, + validScheduledTrials: validKeys.size, + complete: expectedTrials.length ? validKeys.size === expectedTrials.length : null, validRows: rows.filter(isValidTrial).length, invalidRows: rows.filter(row => !isValidTrial(row)).length }, null, 2)}\n` ); + writeAtomic( + path.join(options.out, 'evidence.json'), + `${JSON.stringify(evidence.buildManifest({ + meta, + configuration: meta.configuration, + scenarioFingerprints: meta.scenarioFingerprints, + sourceFingerprints: meta.sourceFingerprints, + outDir: options.out, + rows + }), null, 2)}\n` + ); return header + body; } @@ -160,6 +207,21 @@ function main() { process.stdout.write(`${USAGE}\n`); return; } + if (options.verifyOnly) { + const manifest = evidence.verifyManifest(options.out); + process.stdout.write(`Evidence verified: ${Object.keys(manifest.files).length} artifact(s), schema ${manifest.schemaVersion}.\n`); + return; + } + const savedMetaFile = options.out && path.join(options.out, 'meta.json'); + if (options.summarizeOnly && savedMetaFile && fs.existsSync(savedMetaFile)) { + const saved = JSON.parse(fs.readFileSync(savedMetaFile, 'utf8')); + const config = saved.configuration; + if (config) Object.assign(options, { + model: config.model, judgeModel: config.judgeModel, arms: config.arms, + scenarios: config.scenarios, reps: config.reps, userTurns: config.userTurns, + maxTurns: config.maxTurns, timeoutMin: config.timeoutMin, seed: config.seed + }); + } const scenarios = loadIntentScenarios(undefined, options.scenarios); if (options.checkGraders) { checkGraders(scenarios); @@ -171,22 +233,42 @@ function main() { const metaFile = path.join(options.out, 'meta.json'); const trees = { candidate: resolveRef(REPO_ROOT, options.candidateRef) }; if (options.arms.includes('main')) trees.main = resolveRef(REPO_ROOT, options.mainRef); + const configuration = { + model: options.model, judgeModel: options.judgeModel, candidateSha: trees.candidate, mainSha: trees.main || null, + reps: options.reps, userTurns: options.userTurns, arms: options.arms, scenarios: scenarios.map(scenario => scenario.id), + maxTurns: options.maxTurns, timeoutMin: options.timeoutMin, seed: options.seed + }; const meta = { model: options.model, judgeModel: options.judgeModel, sha: trees.candidate, mainSha: trees.main || null, reps: options.reps, userTurns: options.userTurns, arms: options.arms, - scenarios: scenarios.map(scenario => scenario.id), maxTurns: options.maxTurns + scenarios: scenarios.map(scenario => scenario.id), maxTurns: options.maxTurns, timeoutMin: options.timeoutMin, + seed: options.seed, configuration, + scenarioFingerprints: Object.fromEntries(scenarios.map(scenario => [scenario.id, evidence.hashDirectory(scenario.root)])), + sourceFingerprints: Object.fromEntries([ + 'run-intent.js', 'coverage.js', 'intent-eval.js', 'lib.js', 'session.js', 'user-sim.js', 'arms.js', 'evidence.js', + '../context-profiles/ai-eval-lib.js' + ].map(name => [name, evidence.hashFile(path.resolve(__dirname, name))])) }; if (fs.existsSync(metaFile)) { const previous = JSON.parse(fs.readFileSync(metaFile, 'utf8')); - for (const key of ['model', 'judgeModel', 'sha', 'maxTurns', 'userTurns']) { - if (!options.summarizeOnly && !options.dryRun && previous[key] !== meta[key]) { - throw new Error(`--out was started with ${key}=${previous[key]}; use a new --out`); - } + if (!options.summarizeOnly && !options.dryRun && ( + JSON.stringify(previous.configuration) !== JSON.stringify(configuration) || + JSON.stringify(previous.scenarioFingerprints) !== JSON.stringify(meta.scenarioFingerprints) || + JSON.stringify(previous.sourceFingerprints) !== JSON.stringify(meta.sourceFingerprints) + )) { + throw new Error('--out was started with a different experiment configuration; use a new --out'); } } const rows = readResults(resultsFile); + const planned = schedule(scenarios, options.arms, options.reps, options.seed); + evidence.validateRows(rows, planned); if (options.summarizeOnly) { - process.stdout.write(writeReport(options, rows, JSON.parse(fs.readFileSync(metaFile, 'utf8')))); + const previous = JSON.parse(fs.readFileSync(metaFile, 'utf8')); + if (JSON.stringify(previous.scenarioFingerprints) !== JSON.stringify(meta.scenarioFingerprints) || + JSON.stringify(previous.sourceFingerprints) !== JSON.stringify(meta.sourceFingerprints)) { + throw new Error('--summarize cannot use changed scenario or harness inputs; preserve the original run inputs or use a new --out'); + } + process.stdout.write(writeReport(options, rows, previous)); return; } @@ -210,13 +292,13 @@ function main() { // Failed attempts remain in results.jsonl for audit, but the scheduled key // is complete only after a valid trial has been recorded. const done = completedTrialKeys(rows); - const trials = schedule(scenarios, options.arms, options.reps, options.seed).filter(trial => !done.has(trial.key)); + const trials = planned.filter(trial => !done.has(trial.key)); process.stderr.write(`${trials.length} trials to run (${done.size} already recorded)\n`); if (options.dryRun) { for (const trial of trials) process.stdout.write(`${trial.key}\n`); return; } - fs.writeFileSync(metaFile, `${JSON.stringify(meta, null, 2)}\n`); + writeAtomic(metaFile, `${JSON.stringify(meta, null, 2)}\n`); let judgeFailures = 0; for (const [index, trial] of trials.entries()) { const sessionsBefore = listSessions(); @@ -237,10 +319,15 @@ function main() { userTurns: options.userTurns, execute }); + const attemptId = crypto.randomUUID(); + const transcriptName = `${trial.key.replace(/\//g, '__')}__${attemptId}.jsonl`; + row.attemptId = attemptId; + row.transcript = `transcripts/${transcriptName}`; fs.writeFileSync( - path.join(options.out, 'transcripts', `${trial.key.replace(/\//g, '__')}.jsonl`), + path.join(options.out, row.transcript), transcript.join('\n') ); + row.transcriptSha256 = evidence.hashFile(path.join(options.out, row.transcript)); fs.rmSync(path.join(workRoot, trial.key.replace(/\//g, '__')), { recursive: true, force: true }); removeTrialSessions(sessionsBefore); // An unauthenticated or hookless run must not be recorded: graded as data it @@ -276,4 +363,4 @@ if (require.main === module) { } } -module.exports = { parseArgs, writeReport, checkGraders, completedTrialKeys }; +module.exports = { parseArgs, writeReport, checkGraders, completedTrialKeys, writeAtomic }; diff --git a/docs/gateguard/question-effectiveness.md b/docs/gateguard/question-effectiveness.md index f6278b9e3..f7ab9259a 100644 --- a/docs/gateguard/question-effectiveness.md +++ b/docs/gateguard/question-effectiveness.md @@ -224,9 +224,14 @@ recorded as a judge failure rather than silently read as a stonewall. reachable, so the gate's contribution is whether it makes the agent ask the deciding question — not whether asking was permitted. -**Scenarios are authored blind.** `task.json` must not declare -`targetQuestions`; the loader rejects it. Which questions a scenario provokes is -observed from the run and reported. +**Scenario files do not encode gate targets.** `task.json` must not declare +`targetQuestions`; the loader rejects it, and questions a scenario provokes are +observed from the run. This is a file-format guard, not evidence of blind or +independent authorship: the authors knew the taxonomy, and the later +`external-contract` question was added after inspecting this corpus. Results +for that question are exploratory. A confirmatory corpus needs independent +authors who have not seen the taxonomy or interim results, with scenarios frozen +before evaluation. ### Hole classes @@ -254,20 +259,28 @@ stops. ### Hidden-intent results -Six blind-authored scenarios, `off` and `gate`, four repetitions, 48 trials, +Six scenarios authored by someone familiar with the taxonomy, `off` and `gate`, four repetitions, 48 trials, sonnet under test and haiku as the user. No provider errors, timeouts or judge failures, and the hook was observed in every gated trial. $7.14. -**The scenarios isolate the mechanism.** Obtaining the deciding fact predicts -the outcome almost perfectly, and identically in both arms: +The numeric runs in this section are historical aggregates: their row-level +results and transcripts are not in this repository, so they cannot be +independently recomputed here. Some were originally summarized after excluding +floor/ceiling scenarios. The current runner instead includes every valid +predeclared scenario/repetition pair in the primary comparison; do not treat +these old aggregate tables as output from the current analysis protocol. + +**Disclosure was associated with passing in this corpus.** In this initial run, +28/28 trials that disclosed the deciding facts passed, while 3/20 that did not +disclose them also passed: | Arm | got the fact, passed | did not, passed | | --- | ---: | ---: | | `gate` | 14/14 | 1/10 | | `off` | 14/14 | 2/10 | -So asking the right question is necessary and sufficient here. That is what -makes the next table a measurement rather than noise. +This small observational comparison is consistent with the intended mechanism; +it does not establish that disclosure is necessary, sufficient, or causal. **The gate does not change whether the agent asks.** Paired by scenario and repetition, exact McNemar: @@ -279,9 +292,10 @@ repetition, exact McNemar: | Passed | 15/24 (43-79%) | 16/24 (47-82%) | 4 vs 5 | 1.000 | On the five scenarios that separate any arm, the paired pass-rate difference is -5 points in `off`'s favour with a 95% interval of -25% to +35%. At 20 pairs the -design has about 80% power for a 40-point difference, so a large effect is ruled -out; a 15-to-20-point one is not, and would need roughly 160 pairs. +5 points in `off`'s favour with a 95% interval of -25% to +35%. This interval +does not exclude a large benefit. Under the stated assumptions, 20 pairs has +about 80% power to detect a 40-point difference; it does not imply that a +non-significant result rules out that effect. **The cost is consistent.** `gate` spends 8.3 turns and $0.176 per trial against 5.9 and $0.121, about 41% more turns and 45% more cost, with mean score slightly @@ -350,9 +364,9 @@ does not reject at the 5% level; on eleven discordant pairs the percentile interval is the less conservative of the two, and the exact test is the one to read. -Asking remains necessary: no trial in either arm passed without the deciding -fact, 0 of 8 for `gate` and 0 of 14 for `off`. With the fact, `gate` passed 15 -of 16 and `off` 8 of 10. +In this run, no trial passed without the deciding fact (0 of 8 for `gate`, 0 of +14 for `off`). With the fact, `gate` passed 15 of 16 and `off` 8 of 10. This +observed association is not proof that asking is necessary or sufficient. The contrast between regimes is the clearest part, because the gated arm barely moved while the ungated arm halved: @@ -435,19 +449,20 @@ against 22/48 (p = 0.230), obtained the deciding fact 27/48 against 21/48 (p = 0.327). Both point the same way as the primary and neither reaches significance. -**The mechanism is intact and is the one durable finding across every run.** No -trial in either arm passed without the deciding fact - 0 of 21 for `gate`, 0 of -27 for `off` - and with the fact, `gate` passed 25 of 27 and `off` 18 of 21. -Obtaining the fact is necessary and very nearly sufficient; the gate simply does -not reliably cause it. +**Observed disclosure remained strongly associated with passing.** No trial in +either arm passed without the deciding fact - 0 of 21 for `gate`, 0 of 27 for +`off` - and with the fact, `gate` passed 25 of 27 and `off` 18 of 21. This is +consistent with the proposed mechanism, but scenario-level observational data +do not establish necessity, sufficiency, or causal mediation. The gate did not +reliably cause disclosure in this run. **The cost is the most reliable number in the evaluation.** `gate` spent 8.2 turns and $0.151 per trial against 6.0 and $0.108, about 37% more turns and 40% more cost. That has held within a few points across every run, flagged and unflagged, null and suggestive. -**The effect shrank on replication,** which is what a near-threshold result -usually does: +The point estimate was smaller in the later run; these runs are not pooled, and +this difference alone does not establish a trend: | Run | `gate` | `off` | Difference | Discordant | Exact p | | --- | ---: | ---: | ---: | --- | ---: | @@ -471,7 +486,7 @@ every deciding fact was an external contract, so an `external-contract` question was added for code changes whose profile touches data and the 48-pair design was rerun. 96 trials, none invalid, $11.25. This run is exploratory, not confirmatory: the question was designed after seeing which questions these -scenarios needed, so the corpus is no longer blind to it. +scenarios needed, so this corpus is not independent of that design choice. The headline looks like a win and is not one. `gate` 23 of 48 against `off` 13 of 48, paired difference 21 points, exact McNemar p = 0.021. But the comparison @@ -510,26 +525,27 @@ different difficulty, and whether the question fires depends on what the agent did, so it is a selection effect rather than a measurement. The question is kept, because it asks something the gate genuinely never asked -and the corpus says that kind of answer decides the outcome. What is not claimed -is that adding it improved anything measurable here. A fair test needs scenarios -authored blind to it. +and the corpus suggests that kind of answer matters. What is not claimed is +that adding it improved anything measurable here. A fair test needs scenarios +written by independent authors who have not seen that question or these results. **Standing conclusion.** Across 336 billed trials the gate's benefit is not established, its direction has been positive in the unflagged runs, and its cost is a consistent 35 to 45% in turns and spend. "Not confirmed" is not "no effect", and a 15-point improvement would be worth having; but it has not been demonstrated, and nothing here licenses claiming it has. The evidence that does -hold is narrower and more useful than a verdict on the gate: the deciding fact -decides the outcome, and the gate's questions are aimed at code structure while -every deciding fact in this corpus is an external contract. +hold is narrower than a verdict on the gate: disclosure of the deciding fact +was strongly associated with passing, and the gate's questions were aimed at +code structure while the deciding facts in this corpus were external contracts. ### Reproduce ```bash node docker/gateguard-effectiveness/run-intent.js --check-graders -node docker/gateguard-effectiveness/run-intent.js --out gg-intent --arms off,gate --dry-run -node docker/gateguard-effectiveness/run-intent.js --out gg-intent --model --allow-real-provider -node docker/gateguard-effectiveness/run-intent.js --out gg-intent --summarize +node docker/gateguard-effectiveness/run-intent.js --out gg-confirmatory --candidate-ref --arms off,gate --scenarios audit-retention-purge,csv-export-delimiter,retry-idempotency,soft-limit-overage,timezone-daily-rollup,username-collation --reps 8 --user-turns 3 --max-turns 40 --timeout-min 15 --seed 11 --dry-run +node docker/gateguard-effectiveness/run-intent.js --out gg-confirmatory --candidate-ref --arms off,gate --scenarios audit-retention-purge,csv-export-delimiter,retry-idempotency,soft-limit-overage,timezone-daily-rollup,username-collation --reps 8 --user-turns 3 --max-turns 40 --timeout-min 15 --seed 11 --model --judge-model haiku --allow-real-provider +node docker/gateguard-effectiveness/run-intent.js --out gg-confirmatory --summarize +node docker/gateguard-effectiveness/run-intent.js --out gg-confirmatory --verify node --test tests/docker/gateguard-intent.test.js ``` @@ -541,8 +557,17 @@ per user turn. - The simulated user is a model, so disclosure is not perfectly reproducible; repetitions, not a single trial, carry the estimate. -- One scenario is not a corpus. The blind-authoring rule is only as good as the - independence of whoever writes the next ones. +- No external independent author has produced a frozen confirmatory corpus yet. + Absence of `targetQuestions` prevents explicit labels leaking through the + task file, but does not make authorship blind. +- Historical trial rows and transcript bundles are not committed, so the + historical aggregate claims cannot be independently re-rendered from this + repository. New runs write `evidence.json` with input fingerprints and + artifact checksums. `--verify` detects later file changes; it is not a + signature or proof of provenance. +- Primary arm comparisons use every valid predeclared scenario/repetition pair. + Floor/ceiling classifications and hole counts are descriptive secondary + summaries and must not select the primary comparison subset. - Telling every arm that the user is reachable raises asking across the board, which is the right comparison but not the shipped default. diff --git a/tests/docker/gateguard-intent.test.js b/tests/docker/gateguard-intent.test.js index 7e8f6e53a..00cad5264 100644 --- a/tests/docker/gateguard-intent.test.js +++ b/tests/docker/gateguard-intent.test.js @@ -5,10 +5,10 @@ * These tests deliberately assert nothing about which question the gate raises. * The previous suite did, using targets copied from the gate's own taxonomy, so * it passed by construction. What is asserted here are properties that can - * actually fail: that scenarios are authored blind, that every grader separates + * actually fail: that scenarios do not encode target questions, that every grader separates * the start, the trap and a correct solution, that behaviour is classified into - * the right hole, and that a scenario carrying no information is excluded from - * arm comparison instead of being reported as a result. + * the right hole, and that non-informative outcome classes are kept descriptive + * while every valid predeclared scenario remains in the primary comparison. */ 'use strict'; @@ -24,7 +24,8 @@ const userSim = require(path.join(DIR, 'user-sim')); const session = require(path.join(DIR, 'session')); const coverage = require(path.join(DIR, 'coverage')); const intentEval = require(path.join(DIR, 'intent-eval')); -const { parseArgs, completedTrialKeys } = require(path.join(DIR, 'run-intent')); +const { parseArgs, completedTrialKeys, writeAtomic } = require(path.join(DIR, 'run-intent')); +const evidence = require(path.join(DIR, 'evidence')); let passed = 0; let failed = 0; @@ -64,9 +65,9 @@ function row(overrides) { }; } -// --- scenarios are authored blind to the taxonomy --- +// --- scenario files do not encode taxonomy targets --- -test('every shipped hidden-intent scenario loads and declares a decisive fact', () => { +test('every shipped hidden-intent scenario has one decisive fact and no declared target questions', () => { const scenarios = intentEval.loadIntentScenarios(); assert.ok(scenarios.length > 0, 'no hidden-intent scenarios found'); for (const scenario of scenarios) { @@ -75,7 +76,7 @@ test('every shipped hidden-intent scenario loads and declares a decisive fact', } }); -test('a scenario that declares targetQuestions is rejected', () => { +test('a scenario that declares targetQuestions is rejected without asserting author blindness', () => { const dir = tempDir('gg-blind-'); const scenario = path.join(dir, 'leaky'); fs.mkdirSync(path.join(scenario, 'workspace'), { recursive: true }); @@ -85,7 +86,7 @@ test('a scenario that declares targetQuestions is rejected', () => { ); fs.writeFileSync(path.join(scenario, 'intent.json'), JSON.stringify(INTENT)); fs.writeFileSync(path.join(scenario, 'grader.cjs'), 'console.log("ECC_EVAL_SCORE {\\"score\\":0}");'); - assert.throws(() => intentEval.loadIntentScenarios(dir), /authored blind to the taxonomy/); + assert.throws(() => intentEval.loadIntentScenarios(dir), /must not encode gate-taxonomy targets/); }); test('an intent with no decisive fact is rejected', () => { @@ -121,6 +122,40 @@ test('invalid trial attempts do not complete their scheduled key', () => { assert.deepStrictEqual([...completedTrialKeys(rows)], ['s/4/gate']); }); +test('report files are replaced atomically without leaving temporary files', () => { + const dir = tempDir('gg-atomic-'); + const file = path.join(dir, 'report.json'); + writeAtomic(file, 'first'); + writeAtomic(file, 'second'); + assert.strictEqual(fs.readFileSync(file, 'utf8'), 'second'); + assert.deepStrictEqual(fs.readdirSync(dir), ['report.json']); +}); + +test('evidence validation rejects unknown, mismatched and duplicate valid scheduled rows', () => { + const trial = { key: 'probe/gate/1', scenario: { id: 'probe' }, arm: 'gate', rep: 1 }; + assert.deepStrictEqual([...evidence.validateRows([{ key: trial.key, scenario: 'probe', arm: 'gate', rep: 1 }], [trial])], [trial.key]); + assert.throws(() => evidence.validateRows([{ key: 'other/gate/1' }], [trial]), /unknown scheduled key/); + assert.throws(() => evidence.validateRows([{ key: trial.key, scenario: 'other', arm: 'gate', rep: 1 }], [trial]), /does not match/); + const valid = { key: trial.key, scenario: 'probe', arm: 'gate', rep: 1 }; + assert.throws(() => evidence.validateRows([valid, valid], [trial]), /duplicate valid result/); +}); + +test('evidence manifest detects missing or modified artifacts', () => { + const out = tempDir('gg-evidence-'); + fs.mkdirSync(path.join(out, 'transcripts')); + fs.writeFileSync(path.join(out, 'transcripts', 'trial.jsonl'), 'trial output\n'); + const row = { + key: 'probe/gate/1', transcript: 'transcripts/trial.jsonl', + transcriptSha256: evidence.hashFile(path.join(out, 'transcripts', 'trial.jsonl')) + }; + fs.writeFileSync(path.join(out, 'results.jsonl'), `${JSON.stringify(row)}\n`); + const manifest = evidence.buildManifest({ meta: {}, configuration: {}, scenarioFingerprints: {}, sourceFingerprints: {}, outDir: out, rows: [row] }); + fs.writeFileSync(path.join(out, 'evidence.json'), JSON.stringify(manifest)); + assert.strictEqual(Object.keys(evidence.verifyManifest(out).files).length, 2); + fs.appendFileSync(path.join(out, 'results.jsonl'), 'tampered\n'); + assert.throws(() => evidence.verifyManifest(out), /checksum mismatch/); +}); + test('a gated shell edit without hook output is not observed', () => { const { hookWasObserved } = require(path.join(DIR, 'lib')); assert.strictEqual(hookWasObserved({ gated: true, editCalls: 0, shellCalls: 1, metrics: [], gateDenials: 0 }), false); @@ -167,7 +202,7 @@ test('a right outcome reached without asking is lucky, not working', () => { assert.strictEqual(coverage.classify(row({ passed: true, asked: false, disclosedDecisive: false })), 'lucky'); }); -// --- non-informative scenarios are excluded, not reported as results --- +// --- outcome variation is descriptive, never a primary-selection rule --- test('trap strength is read from the ungated arm alone', () => { const rows = [ @@ -198,7 +233,7 @@ test('usability is judged across arms, so an ungated arm that fails is not disca assert.deepStrictEqual(byScenario, { works: 'informative', unreached: 'floor', toothless: 'ceiling' }); }); -test('a ceiling scenario is excluded from the outcome-class table', () => { +test('a ceiling scenario remains in the primary comparison but not mechanism counts', () => { const rows = [ row({ scenario: 'easy', arm: 'off', passed: true }), row({ scenario: 'easy', arm: 'gate', passed: true, asked: true, disclosedDecisive: true, gateDenials: 1 }), @@ -207,7 +242,10 @@ test('a ceiling scenario is excluded from the outcome-class table', () => { ]; const report = coverage.renderHoles(rows); assert.match(report, /easy: ceiling/); - assert.match(report, /carry no information/); + assert.match(report, /descriptive only/); + const effectTable = report.slice(report.indexOf('### Effect and cost'), report.indexOf('### Outcome classes')); + assert.match(effectTable, /\| gate \| 2\/2 \|/); + assert.match(effectTable, /\| off \| 2 \|/); // Only the informative scenario's gate trial is counted as working. Scope the // lookup to the outcome-class table: the effect table also has a `gate` row. const classTable = report.slice(report.indexOf('### Outcome classes')); @@ -215,6 +253,13 @@ test('a ceiling scenario is excluded from the outcome-class table', () => { assert.strictEqual(gateLine.split('|')[2].trim(), '1', gateLine); }); +test('partial schedules suppress primary comparisons until every planned trial is valid', () => { + const rows = [row({ key: 'probe/gate/1', scenario: 'probe', arm: 'gate', rep: 1, passed: true })]; + const report = coverage.renderHoles(rows, { expectedTrialKeys: ['probe/gate/1', 'probe/off/1'] }); + assert.match(report, /INCOMPLETE: 1\/2 scheduled trials have valid results/); + assert.doesNotMatch(report, /### Effect and cost/); +}); + test('invalid attempts are excluded and counted in the report', () => { const rows = []; for (let rep = 1; rep <= 5; rep++) { @@ -468,6 +513,8 @@ test('minus-target is refused: it needs declared targets these scenarios lack', test('real sessions require the explicit flag', () => { assert.throws(() => parseArgs(['--out', 'o', '--model', 'm']), /--allow-real-provider/); assert.doesNotThrow(() => parseArgs(['--out', 'o', '--dry-run'])); + assert.throws(() => parseArgs(['--out', 'o', '--arms', 'off', '--dry-run']), /needs the gate arm/); + assert.strictEqual(parseArgs(['--out', 'o', '--verify']).verifyOnly, true); }); console.log(`\nPassed: ${passed}`);