diff --git a/docker/gateguard-effectiveness/README.md b/docker/gateguard-effectiveness/README.md index 46b3213b5..e4aed748a 100644 --- a/docker/gateguard-effectiveness/README.md +++ b/docker/gateguard-effectiveness/README.md @@ -38,8 +38,16 @@ node docker/gateguard-effectiveness/run-intent.js --out gg-intent --summarize node docker/gateguard-effectiveness/run-intent.js --out gg-intent --verify ``` -Runs sharing an output directory are serialized by `.run.lock`; concurrent workers +The one-shot `run.js` runner serializes an output directory with `.run.lock`; concurrent workers must use separate `--out` directories. Normal completion and errors release the lock. After a killed process, confirm that its worker has stopped before removing the stale lock and resuming. Metadata uses private exclusive temporary creation and atomic rename so a destination symlink cannot redirect metadata writes. + +Hidden-intent trials retain the selected Claude configuration directory and pass +`--claude` to both the agent and simulated user. Session cleanup removes only new +project folders matching workspaces observed inside that trial's temporary root; +pre-existing folders and concurrent runs remain. Shared project-name overrides +remain untouched because they do not establish exclusive trial ownership. +`arm-patch.js` is included in frozen harness fingerprints, so changes to ablations +require a new output directory rather than continuing an existing experiment. diff --git a/docker/gateguard-effectiveness/lib.js b/docker/gateguard-effectiveness/lib.js index 5ea37e4b8..75b6982c8 100644 --- a/docker/gateguard-effectiveness/lib.js +++ b/docker/gateguard-effectiveness/lib.js @@ -11,7 +11,7 @@ const { ARMS, TRIAL_FILE, trialPatch } = require('./arms'); const SCENARIO_DIR = path.join(__dirname, 'scenarios'); const DOT_PREFIX = 'dot-'; const AUTONOMY_NOTE = 'Work autonomously in this repository and do not ask clarifying questions.'; -const PASSTHROUGH_ENV = new Set(['CLAUDE_CODE_OAUTH_TOKEN', 'CLAUDE_CODE_GIT_BASH_PATH']); +const PASSTHROUGH_ENV = new Set(['CLAUDE_CODE_OAUTH_TOKEN', 'CLAUDE_CODE_GIT_BASH_PATH', 'CLAUDE_CONFIG_DIR']); const SCRUBBED_ENV = /^(CLAUDE_|ECC_|GATEGUARD_)/; // --- scenarios --- diff --git a/docker/gateguard-effectiveness/run-intent.js b/docker/gateguard-effectiveness/run-intent.js index 4bdfe90f8..b828af9cb 100644 --- a/docker/gateguard-effectiveness/run-intent.js +++ b/docker/gateguard-effectiveness/run-intent.js @@ -28,6 +28,7 @@ const USAGE = [ ' --model Claude model under test (required unless --dry-run or --summarize)', ' --allow-real-provider run real Claude sessions (each trial is several billed turns)', ' --judge-model model playing the user (default: haiku)', + ' --claude CLI executable for both agent and simulated user', ` --arms ${SUPPORTED_ARMS.join(', ')} (default: off,gate,placebo)`, ' --scenarios scenario ids (default: all)', ' --reps repetitions per scenario and arm (default: 3)', @@ -126,23 +127,38 @@ function completedTrialKeys(rows) { // would isolate those but also hides the credentials, so they are removed here // instead. Only folders this run created, named after its own temp work root, // are touched. -function sessionsDir() { - const home = process.env.CLAUDE_CONFIG_DIR || path.join(os.homedir(), '.claude'); +function sessionsDir(env = process.env) { + const home = env.CLAUDE_CONFIG_DIR || path.join(env.HOME || env.USERPROFILE || os.homedir(), '.claude'); return path.join(home, 'projects'); } -function listSessions() { +function listSessions(env = process.env) { try { - return new Set(fs.readdirSync(sessionsDir())); + return new Set(fs.readdirSync(sessionsDir(env))); } catch (_) { return new Set(); } } -function removeTrialSessions(before) { - const dir = sessionsDir(); - for (const entry of listSessions()) { - if (before.has(entry) || !entry.includes('gateguard-intent')) continue; +function projectSessionName(workspace) { + const resolved = path.resolve(workspace); + const encoded = resolved.replace(/[^A-Za-z0-9]/g, '-'); + if (encoded.length <= 200) return encoded; + // Match the official Claude Agent SDK0.3.296 project storage key, including + // its long-path suffix. Shared provider project-name overrides are not owned. + let hash = 0; + for (let index = 0; index < resolved.length; index += 1) { + hash = ((hash << 5) - hash + resolved.charCodeAt(index)) | 0; + } + return `${encoded.slice(0, 200)}-${Math.abs(hash).toString(36)}`; +} + +function removeTrialSessions(before, ownedWorkspaces, env = process.env) { + if (env.CLAUDE_CODE_PROJECT_DIR_NAME) return; + const dir = sessionsDir(env); + const ownedNames = new Set([...ownedWorkspaces].map(projectSessionName)); + for (const entry of listSessions(env)) { + if (before.has(entry) || !ownedNames.has(entry)) continue; fs.rmSync(path.join(dir, entry), { recursive: true, force: true }); } } @@ -245,7 +261,7 @@ function main() { seed: options.seed, configuration, scenarioFingerprints: Object.fromEntries(scenarios.map(scenario => [scenario.id, evidence.hashDirectory(scenario.root)])), sourceFingerprints: Object.fromEntries([ - 'run-intent.js', 'coverage.js', 'intent-eval.js', 'lib.js', 'session.js', 'user-sim.js', 'arms.js', 'evidence.js', + 'run-intent.js', 'coverage.js', 'intent-eval.js', 'lib.js', 'session.js', 'user-sim.js', 'arms.js', 'arm-patch.js', 'evidence.js', '../context-profiles/ai-eval-lib.js' ].map(name => [name, evidence.hashFile(path.resolve(__dirname, name))])) }; @@ -303,22 +319,35 @@ function main() { for (const [index, trial] of trials.entries()) { const sessionsBefore = listSessions(); const transcript = []; + const ownedWorkspaces = new Set(); const execute = (file, args, spawnOptions) => { + if (spawnOptions.cwd) { + const cwd = path.resolve(spawnOptions.cwd); + const relative = path.relative(workRoot, cwd); + if (relative && !relative.startsWith(`..${path.sep}`) && relative !== '..' && !path.isAbsolute(relative)) { + ownedWorkspaces.add(cwd); + } + } const result = spawnSync(file, args, spawnOptions); if (result.stdout) transcript.push(result.stdout); return result; }; - const row = runIntentTrial(trial, { - workRoot, - armSettingsByName, - executable: options.claude, - model: options.model, - judgeModel: options.judgeModel, - maxTurns: options.maxTurns, - timeoutMs: options.timeoutMin * 60000, - userTurns: options.userTurns, - execute - }); + let row; + try { + row = runIntentTrial(trial, { + workRoot, + armSettingsByName, + executable: options.claude, + model: options.model, + judgeModel: options.judgeModel, + maxTurns: options.maxTurns, + timeoutMs: options.timeoutMin * 60000, + userTurns: options.userTurns, + execute + }); + } finally { + removeTrialSessions(sessionsBefore, ownedWorkspaces); + } const attemptId = crypto.randomUUID(); const transcriptName = `${trial.key.replace(/\//g, '__')}__${attemptId}.jsonl`; row.attemptId = attemptId; @@ -329,7 +358,6 @@ function main() { ); row.transcriptSha256 = evidence.hashFile(path.join(options.out, row.transcript)); fs.rmSync(path.join(workRoot, trial.key.replace(/\//g, '__')), { recursive: true, force: true }); - removeTrialSessions(sessionsBefore); // An unauthenticated or hookless run must not be recorded: graded as data it // looks like a finding (the first such trial was filed as a coverage hole). if (row.providerError && /authenticat|log ?in|oauth/i.test(row.providerMessage || '')) { @@ -363,4 +391,4 @@ if (require.main === module) { } } -module.exports = { parseArgs, writeReport, checkGraders, completedTrialKeys, writeAtomic }; +module.exports = { parseArgs, writeReport, checkGraders, completedTrialKeys, writeAtomic, sessionsDir, listSessions, projectSessionName, removeTrialSessions }; diff --git a/docker/gateguard-effectiveness/session.js b/docker/gateguard-effectiveness/session.js index ed96dc01e..0dc1e53f5 100644 --- a/docker/gateguard-effectiveness/session.js +++ b/docker/gateguard-effectiveness/session.js @@ -128,7 +128,7 @@ function runConversation({ cwd, stateDir, settingsPath, prompt, intent }, option exchange.push({ role: 'agent', text: message }); if (turn === userTurns) break; - const reply = judge(intent, message, { model: judgeModel, execute }); + const reply = judge(intent, message, { executable, model: judgeModel, execute }); judgeFailed = judgeFailed || reply.judgeFailed; if (!reply.isQuestion) break; asked = true; diff --git a/tests/docker/gateguard-intent.test.js b/tests/docker/gateguard-intent.test.js index 00cad5264..1982ebb64 100644 --- a/tests/docker/gateguard-intent.test.js +++ b/tests/docker/gateguard-intent.test.js @@ -17,6 +17,7 @@ const assert = require('assert'); const fs = require('fs'); const os = require('os'); const path = require('path'); +const { spawnSync } = require('child_process'); const ROOT = path.join(__dirname, '..', '..'); const DIR = path.join(ROOT, 'docker', 'gateguard-effectiveness'); @@ -24,7 +25,7 @@ const userSim = require(path.join(DIR, 'user-sim')); const session = require(path.join(DIR, 'session')); const coverage = require(path.join(DIR, 'coverage')); const intentEval = require(path.join(DIR, 'intent-eval')); -const { parseArgs, completedTrialKeys, writeAtomic } = require(path.join(DIR, 'run-intent')); +const { parseArgs, completedTrialKeys, writeAtomic, sessionsDir, listSessions, projectSessionName, removeTrialSessions } = require(path.join(DIR, 'run-intent')); const evidence = require(path.join(DIR, 'evidence')); let passed = 0; @@ -437,21 +438,170 @@ test('a timeout on an earlier turn is not cleared by a later clean turn', () => assert.strictEqual(result.timedOut, true); }); +test('a selected custom executable runs both agent and simulated user with bare claude absent', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gg-intent-executable-')); + const fake = path.join(dir, 'fake.js'); + const log = path.join(dir, 'calls.jsonl'); + try { + fs.writeFileSync(fake, ` + const fs = require('fs'); + const args = process.argv.slice(2); + const model = args[args.indexOf('--model') + 1]; + const agent = args.includes('--output-format'); + fs.appendFileSync(${JSON.stringify(log)}, JSON.stringify({ agent, model }) + '\\n'); + if (agent) { + const text = args.includes('--resume') ? 'done' : 'how long is the window?'; + console.log(JSON.stringify({ type: 'assistant', message: { content: [{ type: 'text', text }] } })); + console.log(JSON.stringify({ type: 'result', is_error: false, num_turns: 1, usage: {} })); + } else { + const text = fs.readFileSync(0, 'utf8'); + const isQuestion = text.includes('how long is the window?'); + console.log(JSON.stringify({ isQuestion, reply: isQuestion ? '400 days' : 'done', disclosed: isQuestion ? ['window'] : [] })); + } + `); + const execute = (file, args, options) => spawnSync(file, [fake, ...args], { + ...options, env: { ...options.env, PATH: '', HOME: dir, USERPROFILE: dir } + }); + const missing = spawnSync('claude', ['--version'], { env: { ...process.env, PATH: '' }, encoding: 'utf8' }); + assert.ok(missing.error && missing.error.code === 'ENOENT', 'bare claude is unavailable'); + const result = session.runConversation( + { cwd: dir, stateDir: dir, settingsPath: path.join(dir, 'settings.json'), prompt: 'p', intent: INTENT }, + { executable: process.execPath, model: 'selected-agent', judgeModel: 'selected-judge', maxTurns: 2, timeoutMs: 10000, userTurns: 2, execute } + ); + assert.strictEqual(result.judgeFailed, false); + assert.strictEqual(result.providerError, false); + assert.strictEqual(result.userTurns, 1); + assert.deepStrictEqual(fs.readFileSync(log, 'utf8').trim().split('\n').map(line => JSON.parse(line)), [ + { agent: true, model: 'selected-agent' }, { agent: false, model: 'selected-judge' }, + { agent: true, model: 'selected-agent' }, { agent: false, model: 'selected-judge' } + ]); + } finally { fs.rmSync(dir, { recursive: true, force: true }); } +}); + +test('session cleanup preserves concurrent and pre-existing folders under custom config and HOME', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gg-intent-cleanup-')); + const saved = { HOME: process.env.HOME, CLAUDE_CONFIG_DIR: process.env.CLAUDE_CONFIG_DIR }; + try { + for (const env of [{ HOME: path.join(root, 'home') }, { HOME: path.join(root, 'home'), CLAUDE_CONFIG_DIR: path.join(root, 'config') }]) { + process.env.HOME = env.HOME; + if (env.CLAUDE_CONFIG_DIR) process.env.CLAUDE_CONFIG_DIR = env.CLAUDE_CONFIG_DIR; + else delete process.env.CLAUDE_CONFIG_DIR; + const dir = sessionsDir(env); + fs.mkdirSync(dir, { recursive: true }); + const workRoot = path.join(root, 'gateguard-intent-OWNED'); + const workspace = path.join(workRoot, 'date__gate__1', 'repo'); + const own = projectSessionName(workspace); + const preexisting = projectSessionName(path.join(workRoot, 'date__off__1', 'repo')); + const concurrent = projectSessionName(path.join(root, 'gateguard-intent-OTHER', 'date__gate__1', 'repo')); + const make = name => { fs.mkdirSync(path.join(dir, name)); fs.writeFileSync(path.join(dir, name, 'sentinel'), name); }; + make(preexisting); + const before = listSessions(env); + for (const name of [own, concurrent, `${own}-other`]) make(name); + removeTrialSessions(before, new Set([workspace, path.join(workRoot, 'date__off__1', 'repo')]), env); + assert.ok(!fs.existsSync(path.join(dir, own)), 'exact owned folder removed'); + for (const name of [preexisting, concurrent, `${own}-other`]) { + assert.strictEqual(fs.readFileSync(path.join(dir, name, 'sentinel'), 'utf8'), name); + } + make(own); + removeTrialSessions(before, new Set([workspace]), { ...env, CLAUDE_CODE_PROJECT_DIR_NAME: own }); + assert.strictEqual(fs.readFileSync(path.join(dir, own, 'sentinel'), 'utf8'), own, 'shared provider name override stays untouched'); + } + } finally { + for (const [key, value] of Object.entries(saved)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('actual runner freezes arm-patch source and refuses changed-input resume before dispatch', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gg-intent-resume-')); + const repo = path.join(root, 'repo'); + const copied = path.join(repo, 'docker', 'gateguard-effectiveness'); + const out = path.join(root, 'out'); + const preload = path.join(root, 'preload.js'); + const calls = path.join(root, 'calls.log'); + const config = path.join(root, 'claude-config'); + try { + fs.mkdirSync(repo); + fs.cpSync(path.join(ROOT, 'scripts'), path.join(repo, 'scripts'), { recursive: true }); + fs.cpSync(DIR, copied, { recursive: true }); + fs.mkdirSync(path.join(repo, 'docker', 'context-profiles'), { recursive: true }); + fs.copyFileSync(path.join(ROOT, 'docker', 'context-profiles', 'ai-eval-lib.js'), path.join(repo, 'docker', 'context-profiles', 'ai-eval-lib.js')); + fs.copyFileSync(path.join(ROOT, 'package.json'), path.join(repo, 'package.json')); + const env = { ...process.env, HOME: path.join(root, 'home'), USERPROFILE: path.join(root, 'home'), CLAUDE_CONFIG_DIR: config, CLAUDE_CODE_PROJECT_DIR_NAME: '', + NODE_PATH: path.join(ROOT, 'node_modules'), GIT_CONFIG_GLOBAL: path.join(root, 'unused-gitconfig'), + GIT_AUTHOR_NAME: 'Synthetic Eval', GIT_COMMITTER_NAME: 'Synthetic Eval', GIT_AUTHOR_EMAIL: 'eval@example.invalid', GIT_COMMITTER_EMAIL: 'eval@example.invalid' }; + for (const args of [['init', '-q'], ['add', '.'], ['commit', '-q', '--no-gpg-sign', '-m', 'synthetic source snapshot']]) { + const result = spawnSync('git', args, { cwd: repo, env, encoding: 'utf8', timeout: 30000 }); + assert.strictEqual(result.status, 0, result.stderr); + } + const scenario = intentEval.loadIntentScenarios()[0]; + const copiedReference = path.join(copied, 'scenarios-intent', scenario.id, 'reference'); + fs.writeFileSync(preload, ` + const cp = require('child_process'); + const fs = require('fs'); + const path = require('path'); + const spawn = cp.spawnSync; + cp.spawnSync = (file, args, options) => { + if (file !== 'ecc-intent-fake') return spawn(file, args, options); + fs.appendFileSync(${JSON.stringify(calls)}, (args.includes('--settings') ? 'agent' : 'judge') + '\\n'); + if (!args.includes('--settings')) return spawn(process.execPath, ['-e', 'console.log(JSON.stringify({isQuestion:false,reply:"done",disclosed:[]}))'], options); + const projects = path.join(options.env.CLAUDE_CONFIG_DIR, 'projects'); + const own = path.resolve(options.cwd).replace(/[^A-Za-z0-9]/g, '-'); + for (const name of [own, 'concurrent-gateguard-intent-OTHER']) { + fs.mkdirSync(path.join(projects, name), { recursive: true }); + fs.writeFileSync(path.join(projects, name, 'sentinel'), name); + } + return spawn(process.execPath, [${JSON.stringify(path.join(copied, 'fake-claude.js'))}, ...args], { ...options, env: { ...options.env, FAKE_CLAUDE_OVERLAY: ${JSON.stringify(copiedReference)} } }); + }; + `); + const args = ['--require', preload, path.join(copied, 'run-intent.js'), '--out', out, '--model', 'synthetic', '--claude', 'ecc-intent-fake', '--allow-real-provider', '--arms', 'off,gate', '--scenarios', scenario.id, '--reps', '1']; + const run = () => spawnSync(process.execPath, args, { cwd: repo, env, encoding: 'utf8', timeout: 60000 }); + const first = run(); + assert.strictEqual(first.status, 0, first.stderr); + const meta = JSON.parse(fs.readFileSync(path.join(out, 'meta.json'), 'utf8')); + const patch = path.join(copied, 'arm-patch.js'); + assert.strictEqual(meta.sourceFingerprints['arm-patch.js'], evidence.hashFile(patch)); + assert.deepStrictEqual(fs.readdirSync(path.join(config, 'projects')), ['concurrent-gateguard-intent-OTHER'], 'concurrent folder survives exact cleanup'); + const results = fs.readFileSync(path.join(out, 'results.jsonl'), 'utf8'); + const callsBefore = fs.readFileSync(calls, 'utf8'); + const resumed = run(); + assert.strictEqual(resumed.status, 0, resumed.stderr); + assert.ok(resumed.stderr.includes('0 trials to run')); + assert.strictEqual(fs.readFileSync(calls, 'utf8'), callsBefore); + fs.appendFileSync(patch, '\n// Changed ablation implementation must invalidate resume.\n'); + const changed = run(); + assert.notStrictEqual(changed.status, 0); + assert.match(changed.stderr, /different experiment configuration/); + assert.strictEqual(fs.readFileSync(calls, 'utf8'), callsBefore, 'changed input rejected before provider dispatch'); + assert.strictEqual(fs.readFileSync(path.join(out, 'results.jsonl'), 'utf8'), results); + } finally { fs.rmSync(root, { recursive: true, force: true }); } +}); + test('the config directory is never isolated, because that hides the credentials', () => { // A per-trial CLAUDE_CONFIG_DIR looks like the right way to stop persisted // sessions piling up, but it makes every turn return "Not logged in", and the // trial is then graded as a coverage hole. Guard against reintroducing it. - let seen = 'unset'; - const execute = (file, args, options) => { - seen = options.env.CLAUDE_CONFIG_DIR; - return streamFor('done'); - }; - const judge = () => ({ isQuestion: false, reply: INTENT.stonewall, disclosed: [], judgeFailed: false }); - session.runConversation( - { cwd: '.', stateDir: '.', settingsPath: 's', prompt: 'p', intent: INTENT }, - { model: 'm', maxTurns: 5, timeoutMs: 1000, userTurns: 1, execute, judge } - ); - assert.strictEqual(seen, process.env.CLAUDE_CONFIG_DIR); + const previous = process.env.CLAUDE_CONFIG_DIR; + process.env.CLAUDE_CONFIG_DIR = path.join(os.tmpdir(), 'explicit-claude-config-fixture'); + try { + let seen = 'unset'; + const execute = (file, args, options) => { + seen = options.env.CLAUDE_CONFIG_DIR; + return streamFor('done'); + }; + const judge = () => ({ isQuestion: false, reply: INTENT.stonewall, disclosed: [], judgeFailed: false }); + session.runConversation( + { cwd: '.', stateDir: '.', settingsPath: 's', prompt: 'p', intent: INTENT }, + { model: 'm', maxTurns: 5, timeoutMs: 1000, userTurns: 1, execute, judge } + ); + assert.strictEqual(seen, process.env.CLAUDE_CONFIG_DIR); + } finally { + if (previous === undefined) delete process.env.CLAUDE_CONFIG_DIR; + else process.env.CLAUDE_CONFIG_DIR = previous; + } }); test('a missing or errored result is a provider error, not a trial', () => {