fix(eval): preserve session ownership and frozen intent inputs

docker/gateguard-effectiveness/README.md

@@ -38,8 +38,16 @@ node docker/gateguard-effectiveness/run-intent.js --out gg-intent --summarize
 node docker/gateguard-effectiveness/run-intent.js --out gg-intent --verify
 ```
 
-Runs sharing an output directory are serialized by `.run.lock`; concurrent workers
+The one-shot `run.js` runner serializes an output directory with `.run.lock`; concurrent workers
 must use separate `--out` directories. Normal completion and errors release the
 lock. After a killed process, confirm that its worker has stopped before removing
 the stale lock and resuming. Metadata uses private exclusive temporary creation
 and atomic rename so a destination symlink cannot redirect metadata writes.
+
+Hidden-intent trials retain the selected Claude configuration directory and pass
+`--claude` to both the agent and simulated user. Session cleanup removes only new
+project folders matching workspaces observed inside that trial's temporary root;
+pre-existing folders and concurrent runs remain. Shared project-name overrides
+remain untouched because they do not establish exclusive trial ownership.
+`arm-patch.js` is included in frozen harness fingerprints, so changes to ablations
+require a new output directory rather than continuing an existing experiment.

docker/gateguard-effectiveness/lib.js

@@ -11,7 +11,7 @@ const { ARMS, TRIAL_FILE, trialPatch } = require('./arms');
 const SCENARIO_DIR = path.join(__dirname, 'scenarios');
 const DOT_PREFIX = 'dot-';
 const AUTONOMY_NOTE = 'Work autonomously in this repository and do not ask clarifying questions.';
-const PASSTHROUGH_ENV = new Set(['CLAUDE_CODE_OAUTH_TOKEN', 'CLAUDE_CODE_GIT_BASH_PATH']);
+const PASSTHROUGH_ENV = new Set(['CLAUDE_CODE_OAUTH_TOKEN', 'CLAUDE_CODE_GIT_BASH_PATH', 'CLAUDE_CONFIG_DIR']);
 const SCRUBBED_ENV = /^(CLAUDE_|ECC_|GATEGUARD_)/;
 
 // --- scenarios ---

docker/gateguard-effectiveness/run-intent.js

@@ -28,6 +28,7 @@ const USAGE = [
   '  --model <model>         Claude model under test (required unless --dry-run or --summarize)',
   '  --allow-real-provider   run real Claude sessions (each trial is several billed turns)',
   '  --judge-model <model>   model playing the user (default: haiku)',
+  '  --claude <executable>   CLI executable for both agent and simulated user',
   `  --arms <a,b,...>        ${SUPPORTED_ARMS.join(', ')} (default: off,gate,placebo)`,
   '  --scenarios <a,b,...>   scenario ids (default: all)',
   '  --reps <n>              repetitions per scenario and arm (default: 3)',
@@ -126,23 +127,38 @@ function completedTrialKeys(rows) {
 // would isolate those but also hides the credentials, so they are removed here
 // instead. Only folders this run created, named after its own temp work root,
 // are touched.
-function sessionsDir() {
-  const home = process.env.CLAUDE_CONFIG_DIR || path.join(os.homedir(), '.claude');
+function sessionsDir(env = process.env) {
+  const home = env.CLAUDE_CONFIG_DIR || path.join(env.HOME || env.USERPROFILE || os.homedir(), '.claude');
   return path.join(home, 'projects');
 }
 
-function listSessions() {
+function listSessions(env = process.env) {
   try {
-    return new Set(fs.readdirSync(sessionsDir()));
+    return new Set(fs.readdirSync(sessionsDir(env)));
   } catch (_) {
     return new Set();
   }
 }
 
-function removeTrialSessions(before) {
-  const dir = sessionsDir();
-  for (const entry of listSessions()) {
-    if (before.has(entry) || !entry.includes('gateguard-intent')) continue;
+function projectSessionName(workspace) {
+  const resolved = path.resolve(workspace);
+  const encoded = resolved.replace(/[^A-Za-z0-9]/g, '-');
+  if (encoded.length <= 200) return encoded;
+  // Match the official Claude Agent SDK0.3.296 project storage key, including
+  // its long-path suffix. Shared provider project-name overrides are not owned.
+  let hash = 0;
+  for (let index = 0; index < resolved.length; index += 1) {
+    hash = ((hash << 5) - hash + resolved.charCodeAt(index)) | 0;
+  }
+  return `${encoded.slice(0, 200)}-${Math.abs(hash).toString(36)}`;
+}
+
+function removeTrialSessions(before, ownedWorkspaces, env = process.env) {
+  if (env.CLAUDE_CODE_PROJECT_DIR_NAME) return;
+  const dir = sessionsDir(env);
+  const ownedNames = new Set([...ownedWorkspaces].map(projectSessionName));
+  for (const entry of listSessions(env)) {
+    if (before.has(entry) || !ownedNames.has(entry)) continue;
     fs.rmSync(path.join(dir, entry), { recursive: true, force: true });
   }
 }
@@ -245,7 +261,7 @@ function main() {
     seed: options.seed, configuration,
     scenarioFingerprints: Object.fromEntries(scenarios.map(scenario => [scenario.id, evidence.hashDirectory(scenario.root)])),
     sourceFingerprints: Object.fromEntries([
-      'run-intent.js', 'coverage.js', 'intent-eval.js', 'lib.js', 'session.js', 'user-sim.js', 'arms.js', 'evidence.js',
+      'run-intent.js', 'coverage.js', 'intent-eval.js', 'lib.js', 'session.js', 'user-sim.js', 'arms.js', 'arm-patch.js', 'evidence.js',
       '../context-profiles/ai-eval-lib.js'
     ].map(name => [name, evidence.hashFile(path.resolve(__dirname, name))]))
   };
@@ -303,22 +319,35 @@ function main() {
     for (const [index, trial] of trials.entries()) {
       const sessionsBefore = listSessions();
       const transcript = [];
+      const ownedWorkspaces = new Set();
       const execute = (file, args, spawnOptions) => {
+        if (spawnOptions.cwd) {
+          const cwd = path.resolve(spawnOptions.cwd);
+          const relative = path.relative(workRoot, cwd);
+          if (relative && !relative.startsWith(`..${path.sep}`) && relative !== '..' && !path.isAbsolute(relative)) {
+            ownedWorkspaces.add(cwd);
+          }
+        }
         const result = spawnSync(file, args, spawnOptions);
         if (result.stdout) transcript.push(result.stdout);
         return result;
       };
-      const row = runIntentTrial(trial, {
-        workRoot,
-        armSettingsByName,
-        executable: options.claude,
-        model: options.model,
-        judgeModel: options.judgeModel,
-        maxTurns: options.maxTurns,
-        timeoutMs: options.timeoutMin * 60000,
-        userTurns: options.userTurns,
-        execute
-      });
+      let row;
+      try {
+        row = runIntentTrial(trial, {
+          workRoot,
+          armSettingsByName,
+          executable: options.claude,
+          model: options.model,
+          judgeModel: options.judgeModel,
+          maxTurns: options.maxTurns,
+          timeoutMs: options.timeoutMin * 60000,
+          userTurns: options.userTurns,
+          execute
+        });
+      } finally {
+        removeTrialSessions(sessionsBefore, ownedWorkspaces);
+      }
       const attemptId = crypto.randomUUID();
       const transcriptName = `${trial.key.replace(/\//g, '__')}__${attemptId}.jsonl`;
       row.attemptId = attemptId;
@@ -329,7 +358,6 @@ function main() {
       );
       row.transcriptSha256 = evidence.hashFile(path.join(options.out, row.transcript));
       fs.rmSync(path.join(workRoot, trial.key.replace(/\//g, '__')), { recursive: true, force: true });
-      removeTrialSessions(sessionsBefore);
       // An unauthenticated or hookless run must not be recorded: graded as data it
       // looks like a finding (the first such trial was filed as a coverage hole).
       if (row.providerError && /authenticat|log ?in|oauth/i.test(row.providerMessage || '')) {
@@ -363,4 +391,4 @@ if (require.main === module) {
   }
 }
 
-module.exports = { parseArgs, writeReport, checkGraders, completedTrialKeys, writeAtomic };
+module.exports = { parseArgs, writeReport, checkGraders, completedTrialKeys, writeAtomic, sessionsDir, listSessions, projectSessionName, removeTrialSessions };

docker/gateguard-effectiveness/session.js

@@ -128,7 +128,7 @@ function runConversation({ cwd, stateDir, settingsPath, prompt, intent }, option
     exchange.push({ role: 'agent', text: message });
     if (turn === userTurns) break;
 
-    const reply = judge(intent, message, { model: judgeModel, execute });
+    const reply = judge(intent, message, { executable, model: judgeModel, execute });
     judgeFailed = judgeFailed || reply.judgeFailed;
     if (!reply.isQuestion) break;
     asked = true;

tests/docker/gateguard-intent.test.js

@@ -17,6 +17,7 @@ const assert = require('assert');
 const fs = require('fs');
 const os = require('os');
 const path = require('path');
+const { spawnSync } = require('child_process');
 
 const ROOT = path.join(__dirname, '..', '..');
 const DIR = path.join(ROOT, 'docker', 'gateguard-effectiveness');
@@ -24,7 +25,7 @@ const userSim = require(path.join(DIR, 'user-sim'));
 const session = require(path.join(DIR, 'session'));
 const coverage = require(path.join(DIR, 'coverage'));
 const intentEval = require(path.join(DIR, 'intent-eval'));
-const { parseArgs, completedTrialKeys, writeAtomic } = require(path.join(DIR, 'run-intent'));
+const { parseArgs, completedTrialKeys, writeAtomic, sessionsDir, listSessions, projectSessionName, removeTrialSessions } = require(path.join(DIR, 'run-intent'));
 const evidence = require(path.join(DIR, 'evidence'));
 
 let passed = 0;
@@ -437,21 +438,170 @@ test('a timeout on an earlier turn is not cleared by a later clean turn', () =>
   assert.strictEqual(result.timedOut, true);
 });
 
+test('a selected custom executable runs both agent and simulated user with bare claude absent', () => {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gg-intent-executable-'));
+  const fake = path.join(dir, 'fake.js');
+  const log = path.join(dir, 'calls.jsonl');
+  try {
+    fs.writeFileSync(fake, `
+      const fs = require('fs');
+      const args = process.argv.slice(2);
+      const model = args[args.indexOf('--model') + 1];
+      const agent = args.includes('--output-format');
+      fs.appendFileSync(${JSON.stringify(log)}, JSON.stringify({ agent, model }) + '\\n');
+      if (agent) {
+        const text = args.includes('--resume') ? 'done' : 'how long is the window?';
+        console.log(JSON.stringify({ type: 'assistant', message: { content: [{ type: 'text', text }] } }));
+        console.log(JSON.stringify({ type: 'result', is_error: false, num_turns: 1, usage: {} }));
+      } else {
+        const text = fs.readFileSync(0, 'utf8');
+        const isQuestion = text.includes('how long is the window?');
+        console.log(JSON.stringify({ isQuestion, reply: isQuestion ? '400 days' : 'done', disclosed: isQuestion ? ['window'] : [] }));
+      }
+    `);
+    const execute = (file, args, options) => spawnSync(file, [fake, ...args], {
+      ...options, env: { ...options.env, PATH: '', HOME: dir, USERPROFILE: dir }
+    });
+    const missing = spawnSync('claude', ['--version'], { env: { ...process.env, PATH: '' }, encoding: 'utf8' });
+    assert.ok(missing.error && missing.error.code === 'ENOENT', 'bare claude is unavailable');
+    const result = session.runConversation(
+      { cwd: dir, stateDir: dir, settingsPath: path.join(dir, 'settings.json'), prompt: 'p', intent: INTENT },
+      { executable: process.execPath, model: 'selected-agent', judgeModel: 'selected-judge', maxTurns: 2, timeoutMs: 10000, userTurns: 2, execute }
+    );
+    assert.strictEqual(result.judgeFailed, false);
+    assert.strictEqual(result.providerError, false);
+    assert.strictEqual(result.userTurns, 1);
+    assert.deepStrictEqual(fs.readFileSync(log, 'utf8').trim().split('\n').map(line => JSON.parse(line)), [
+      { agent: true, model: 'selected-agent' }, { agent: false, model: 'selected-judge' },
+      { agent: true, model: 'selected-agent' }, { agent: false, model: 'selected-judge' }
+    ]);
+  } finally { fs.rmSync(dir, { recursive: true, force: true }); }
+});
+
+test('session cleanup preserves concurrent and pre-existing folders under custom config and HOME', () => {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gg-intent-cleanup-'));
+  const saved = { HOME: process.env.HOME, CLAUDE_CONFIG_DIR: process.env.CLAUDE_CONFIG_DIR };
+  try {
+    for (const env of [{ HOME: path.join(root, 'home') }, { HOME: path.join(root, 'home'), CLAUDE_CONFIG_DIR: path.join(root, 'config') }]) {
+      process.env.HOME = env.HOME;
+      if (env.CLAUDE_CONFIG_DIR) process.env.CLAUDE_CONFIG_DIR = env.CLAUDE_CONFIG_DIR;
+      else delete process.env.CLAUDE_CONFIG_DIR;
+      const dir = sessionsDir(env);
+      fs.mkdirSync(dir, { recursive: true });
+      const workRoot = path.join(root, 'gateguard-intent-OWNED');
+      const workspace = path.join(workRoot, 'date__gate__1', 'repo');
+      const own = projectSessionName(workspace);
+      const preexisting = projectSessionName(path.join(workRoot, 'date__off__1', 'repo'));
+      const concurrent = projectSessionName(path.join(root, 'gateguard-intent-OTHER', 'date__gate__1', 'repo'));
+      const make = name => { fs.mkdirSync(path.join(dir, name)); fs.writeFileSync(path.join(dir, name, 'sentinel'), name); };
+      make(preexisting);
+      const before = listSessions(env);
+      for (const name of [own, concurrent, `${own}-other`]) make(name);
+      removeTrialSessions(before, new Set([workspace, path.join(workRoot, 'date__off__1', 'repo')]), env);
+      assert.ok(!fs.existsSync(path.join(dir, own)), 'exact owned folder removed');
+      for (const name of [preexisting, concurrent, `${own}-other`]) {
+        assert.strictEqual(fs.readFileSync(path.join(dir, name, 'sentinel'), 'utf8'), name);
+      }
+      make(own);
+      removeTrialSessions(before, new Set([workspace]), { ...env, CLAUDE_CODE_PROJECT_DIR_NAME: own });
+      assert.strictEqual(fs.readFileSync(path.join(dir, own, 'sentinel'), 'utf8'), own, 'shared provider name override stays untouched');
+    }
+  } finally {
+    for (const [key, value] of Object.entries(saved)) {
+      if (value === undefined) delete process.env[key];
+      else process.env[key] = value;
+    }
+    fs.rmSync(root, { recursive: true, force: true });
+  }
+});
+
+test('actual runner freezes arm-patch source and refuses changed-input resume before dispatch', () => {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gg-intent-resume-'));
+  const repo = path.join(root, 'repo');
+  const copied = path.join(repo, 'docker', 'gateguard-effectiveness');
+  const out = path.join(root, 'out');
+  const preload = path.join(root, 'preload.js');
+  const calls = path.join(root, 'calls.log');
+  const config = path.join(root, 'claude-config');
+  try {
+    fs.mkdirSync(repo);
+    fs.cpSync(path.join(ROOT, 'scripts'), path.join(repo, 'scripts'), { recursive: true });
+    fs.cpSync(DIR, copied, { recursive: true });
+    fs.mkdirSync(path.join(repo, 'docker', 'context-profiles'), { recursive: true });
+    fs.copyFileSync(path.join(ROOT, 'docker', 'context-profiles', 'ai-eval-lib.js'), path.join(repo, 'docker', 'context-profiles', 'ai-eval-lib.js'));
+    fs.copyFileSync(path.join(ROOT, 'package.json'), path.join(repo, 'package.json'));
+    const env = { ...process.env, HOME: path.join(root, 'home'), USERPROFILE: path.join(root, 'home'), CLAUDE_CONFIG_DIR: config, CLAUDE_CODE_PROJECT_DIR_NAME: '',
+      NODE_PATH: path.join(ROOT, 'node_modules'), GIT_CONFIG_GLOBAL: path.join(root, 'unused-gitconfig'),
+      GIT_AUTHOR_NAME: 'Synthetic Eval', GIT_COMMITTER_NAME: 'Synthetic Eval', GIT_AUTHOR_EMAIL: 'eval@example.invalid', GIT_COMMITTER_EMAIL: 'eval@example.invalid' };
+    for (const args of [['init', '-q'], ['add', '.'], ['commit', '-q', '--no-gpg-sign', '-m', 'synthetic source snapshot']]) {
+      const result = spawnSync('git', args, { cwd: repo, env, encoding: 'utf8', timeout: 30000 });
+      assert.strictEqual(result.status, 0, result.stderr);
+    }
+    const scenario = intentEval.loadIntentScenarios()[0];
+    const copiedReference = path.join(copied, 'scenarios-intent', scenario.id, 'reference');
+    fs.writeFileSync(preload, `
+      const cp = require('child_process');
+      const fs = require('fs');
+      const path = require('path');
+      const spawn = cp.spawnSync;
+      cp.spawnSync = (file, args, options) => {
+        if (file !== 'ecc-intent-fake') return spawn(file, args, options);
+        fs.appendFileSync(${JSON.stringify(calls)}, (args.includes('--settings') ? 'agent' : 'judge') + '\\n');
+        if (!args.includes('--settings')) return spawn(process.execPath, ['-e', 'console.log(JSON.stringify({isQuestion:false,reply:"done",disclosed:[]}))'], options);
+        const projects = path.join(options.env.CLAUDE_CONFIG_DIR, 'projects');
+        const own = path.resolve(options.cwd).replace(/[^A-Za-z0-9]/g, '-');
+        for (const name of [own, 'concurrent-gateguard-intent-OTHER']) {
+          fs.mkdirSync(path.join(projects, name), { recursive: true });
+          fs.writeFileSync(path.join(projects, name, 'sentinel'), name);
+        }
+        return spawn(process.execPath, [${JSON.stringify(path.join(copied, 'fake-claude.js'))}, ...args], { ...options, env: { ...options.env, FAKE_CLAUDE_OVERLAY: ${JSON.stringify(copiedReference)} } });
+      };
+    `);
+    const args = ['--require', preload, path.join(copied, 'run-intent.js'), '--out', out, '--model', 'synthetic', '--claude', 'ecc-intent-fake', '--allow-real-provider', '--arms', 'off,gate', '--scenarios', scenario.id, '--reps', '1'];
+    const run = () => spawnSync(process.execPath, args, { cwd: repo, env, encoding: 'utf8', timeout: 60000 });
+    const first = run();
+    assert.strictEqual(first.status, 0, first.stderr);
+    const meta = JSON.parse(fs.readFileSync(path.join(out, 'meta.json'), 'utf8'));
+    const patch = path.join(copied, 'arm-patch.js');
+    assert.strictEqual(meta.sourceFingerprints['arm-patch.js'], evidence.hashFile(patch));
+    assert.deepStrictEqual(fs.readdirSync(path.join(config, 'projects')), ['concurrent-gateguard-intent-OTHER'], 'concurrent folder survives exact cleanup');
+    const results = fs.readFileSync(path.join(out, 'results.jsonl'), 'utf8');
+    const callsBefore = fs.readFileSync(calls, 'utf8');
+    const resumed = run();
+    assert.strictEqual(resumed.status, 0, resumed.stderr);
+    assert.ok(resumed.stderr.includes('0 trials to run'));
+    assert.strictEqual(fs.readFileSync(calls, 'utf8'), callsBefore);
+    fs.appendFileSync(patch, '\n// Changed ablation implementation must invalidate resume.\n');
+    const changed = run();
+    assert.notStrictEqual(changed.status, 0);
+    assert.match(changed.stderr, /different experiment configuration/);
+    assert.strictEqual(fs.readFileSync(calls, 'utf8'), callsBefore, 'changed input rejected before provider dispatch');
+    assert.strictEqual(fs.readFileSync(path.join(out, 'results.jsonl'), 'utf8'), results);
+  } finally { fs.rmSync(root, { recursive: true, force: true }); }
+});
+
 test('the config directory is never isolated, because that hides the credentials', () => {
   // A per-trial CLAUDE_CONFIG_DIR looks like the right way to stop persisted
   // sessions piling up, but it makes every turn return "Not logged in", and the
   // trial is then graded as a coverage hole. Guard against reintroducing it.
-  let seen = 'unset';
-  const execute = (file, args, options) => {
-    seen = options.env.CLAUDE_CONFIG_DIR;
-    return streamFor('done');
-  };
-  const judge = () => ({ isQuestion: false, reply: INTENT.stonewall, disclosed: [], judgeFailed: false });
-  session.runConversation(
-    { cwd: '.', stateDir: '.', settingsPath: 's', prompt: 'p', intent: INTENT },
-    { model: 'm', maxTurns: 5, timeoutMs: 1000, userTurns: 1, execute, judge }
-  );
-  assert.strictEqual(seen, process.env.CLAUDE_CONFIG_DIR);
+  const previous = process.env.CLAUDE_CONFIG_DIR;
+  process.env.CLAUDE_CONFIG_DIR = path.join(os.tmpdir(), 'explicit-claude-config-fixture');
+  try {
+    let seen = 'unset';
+    const execute = (file, args, options) => {
+      seen = options.env.CLAUDE_CONFIG_DIR;
+      return streamFor('done');
+    };
+    const judge = () => ({ isQuestion: false, reply: INTENT.stonewall, disclosed: [], judgeFailed: false });
+    session.runConversation(
+      { cwd: '.', stateDir: '.', settingsPath: 's', prompt: 'p', intent: INTENT },
+      { model: 'm', maxTurns: 5, timeoutMs: 1000, userTurns: 1, execute, judge }
+    );
+    assert.strictEqual(seen, process.env.CLAUDE_CONFIG_DIR);
+  } finally {
+    if (previous === undefined) delete process.env.CLAUDE_CONFIG_DIR;
+    else process.env.CLAUDE_CONFIG_DIR = previous;
+  }
 });
 
 test('a missing or errored result is a provider error, not a trial', () => {