fix(profiles): bind routing receipts and harden review paths

docker/context-profiles/ai-eval-lib.js

@@ -255,8 +255,10 @@ function createAuthLease(authHome) {
           if (!after.equals(original)) {
             JSON.parse(after.toString('utf8'));
             const temp = `${source}.${process.pid}.tmp`;
-            fs.writeFileSync(temp, after, { flag: 'wx', mode: 0o600 });
-            fs.renameSync(temp, source);
+            try {
+              fs.writeFileSync(temp, after, { flag: 'wx', mode: 0o600 });
+              fs.renameSync(temp, source);
+            } finally { fs.rmSync(temp, { force: true }); }
           }
         } catch { /* An unreadable refresh keeps the previous login; the next call reports any auth failure. */ }
         fs.rmSync(leased, { force: true });
@@ -282,7 +284,7 @@ function readClaudeKeychainToken() {
   return token;
 }
 
-function createClaudeProvider({ allowRealProvider = false, executable, model,
+function createClaudeProvider({ allowRealProvider = false, allowCredentialedTools = false, executable, model,
   apiKey = process.env.ANTHROPIC_API_KEY, oauthToken = process.env.CLAUDE_CODE_OAUTH_TOKEN,
   tokenSource = readClaudeKeychainToken, persistSessions = false, execute = spawnSync } = {}) {
   if (allowRealProvider !== true) throw new Error('Real provider requires explicit opt-in');
@@ -301,6 +303,9 @@ function createClaudeProvider({ allowRealProvider = false, executable, model,
   const provider = request => {
     if (fingerprintExecutable(binary.path).digest !== pin.executableDigest) fail('source-drift');
     const selection = request.phase === 'selection';
+    if (!selection && !allowCredentialedTools) {
+      throw new Error('Claude task tools can read provider credentials; explicit credentialed-tool opt-in is required');
+    }
     // Selection is tool-free and read-only; task execution may edit and run commands in the workspace.
     // Claude has no cwd-write sandbox flag, so containment relies on the isolated home and temp workspace.
     const args = ['--print', '--output-format', 'json',
@@ -682,8 +687,9 @@ function outcomeTrial(item, arm, repeat, repoRoot, execute, cwd, environment, ta
       if (harvest) harvest(arm, `${item.id}--step${index + 1}`, repeat, environment);
       if (result.status !== 'completed') {
         // A failed ticket ends the chain; remaining tickets are unscored.
-        for (let rest = index; rest < item.steps.length; rest++) {
-          steps.push({ score: 0, ...(metrics ? metricsSince(metrics, start) : {}) });
+        steps.push({ score: 0, ...(metrics ? metricsSince(metrics, start) : {}) });
+        for (let rest = index + 1; rest < item.steps.length; rest++) {
+          steps.push({ score: 0, ...(metrics ? metricsSince(metrics, metrics.length) : {}) });
         }
         break;
       }
@@ -748,7 +754,8 @@ function createHarvester(artifactDir, envs) {
 }
 
 function runEvaluation({ repoRoot = DEFAULT_REPO_ROOT, corpus = loadCorpus(), registration,
-  repeats = 1, provider, family, allowRealProvider = false, executable, model, effort, authHome, environments,
+  repeats = 1, provider, family, allowRealProvider = false, allowCredentialedTools = false,
+  executable, model, effort, authHome, environments,
   arms = undefined, artifactDir = null, maxCalls = 300, deadlineMs = 3600000, callTimeoutMs = 300000 } = {}) {
   if (!provider && !allowRealProvider) throw new Error('Evaluation requires an injected provider or explicit opt-in');
   if (!bounded(maxCalls, 1, 2000) || !bounded(deadlineMs, 1, 8 * 3600000)
@@ -756,6 +763,9 @@ function runEvaluation({ repoRoot = DEFAULT_REPO_ROOT, corpus = loadCorpus(), re
   if (!provider && !registration) throw new Error('Real evaluation requires prior registration');
   const resolvedFamily = provider ? (family || 'codex') : resolveFamily(family, executable);
   if (resolvedFamily === 'claude' && effort !== undefined) throw new Error('Reasoning effort applies only to the Codex provider');
+  if (!provider && resolvedFamily === 'claude' && !allowCredentialedTools) {
+    throw new Error('Claude task tools can read provider credentials; explicit credentialed-tool opt-in is required');
+  }
   const pin = preregister({ repoRoot, corpus, repeats, model, executable, effort, arms });
   if (!provider && resolvedFamily === 'codex' && pin.arms.includes('ecc-legacy')) {
     throw new Error('Codex real evaluation requires --arms without ecc-legacy; the pinned legacy skills arm is Claude-only');
@@ -763,7 +773,8 @@ function runEvaluation({ repoRoot = DEFAULT_REPO_ROOT, corpus = loadCorpus(), re
   if (registration && !isDeepStrictEqual(registration, pin)) throw new Error('Registration pin mismatch');
   const injected = Boolean(provider);
   const liveProvider = provider || (resolvedFamily === 'claude'
-    ? createClaudeProvider({ allowRealProvider, executable, model, persistSessions: Boolean(artifactDir) })
+    ? createClaudeProvider({ allowRealProvider, allowCredentialedTools, executable, model,
+      persistSessions: Boolean(artifactDir) })
     : createCodexProvider({ allowRealProvider, executable, model, effort, authHome }));
   const state = { calls: 0, metrics: [], maxCalls, callTimeoutMs, family: resolvedFamily,
     deadline: Date.now() + deadlineMs, provider: liveProvider,

docker/context-profiles/ai-eval.js

@@ -5,7 +5,7 @@ const { preregister, runEvaluation, loadCorpus } = require('./ai-eval-lib');
 
 function main(argv = process.argv.slice(2), injected = {}) {
   const flags = new Map();
-  const switches = new Set(['--plan', '--allow-real-provider', '--help']);
+  const switches = new Set(['--plan', '--allow-real-provider', '--allow-credentialed-tools', '--help']);
   const values = new Set(['--registration', '--model', '--executable', '--provider', '--auth-home', '--effort', '--repeats', '--max-calls', '--deadline-ms', '--artifact-dir', '--corpus', '--call-timeout-ms', '--arms']);
   for (let i = 0; i < argv.length; i++) {
     const flag = argv[i];
@@ -14,10 +14,13 @@ function main(argv = process.argv.slice(2), injected = {}) {
     flags.set(flag, switches.has(flag) ? true : argv[++i]);
   }
   if (flags.has('--help')) {
-    return { usage: 'ai-eval.js --plan [--corpus FILE] [--arms a,b] [--repeats N] [--model MODEL --executable ABSOLUTE_PATH [--provider claude|codex] [--effort LEVEL]] | --allow-real-provider --registration FILE --model MODEL --executable ABSOLUTE_PATH [--provider claude|codex] [--effort LEVEL (Codex only)] [--auth-home ABSOLUTE_DIR (Codex only)] [--corpus FILE] [--arms a,b] [--repeats N] [--max-calls N] [--deadline-ms N] [--call-timeout-ms N]. Claude auth: CLAUDE_CODE_OAUTH_TOKEN, ANTHROPIC_API_KEY, or the macOS Keychain login.' };
+    return { usage: 'ai-eval.js --plan [--corpus FILE] [--arms a,b] [--repeats N] [--model MODEL --executable ABSOLUTE_PATH [--provider claude|codex] [--effort LEVEL]] | --allow-real-provider --registration FILE --model MODEL --executable ABSOLUTE_PATH [--provider claude|codex] [--allow-credentialed-tools (Claude only)] [--effort LEVEL (Codex only)] [--auth-home ABSOLUTE_DIR (Codex only)] [--corpus FILE] [--arms a,b] [--repeats N] [--max-calls N] [--deadline-ms N] [--call-timeout-ms N]. Claude auth: CLAUDE_CODE_OAUTH_TOKEN, ANTHROPIC_API_KEY, or the macOS Keychain login.' };
   }
   if (flags.get('--provider') !== undefined && !['claude', 'codex'].includes(flags.get('--provider'))) throw new Error('Provider must be claude or codex');
   if (flags.get('--provider') === 'claude' && flags.has('--effort')) throw new Error('Reasoning effort applies only to the Codex provider');
+  if (flags.has('--allow-credentialed-tools') && (!flags.has('--allow-real-provider') || flags.get('--provider') !== 'claude')) {
+    throw new Error('Credentialed-tool opt-in requires a real Claude evaluation');
+  }
   const repeats = flags.has('--repeats') ? Number(flags.get('--repeats')) : 1;
   const corpus = flags.has('--corpus') ? loadCorpus(flags.get('--corpus')) : undefined;
   const arms = flags.has('--arms') ? flags.get('--arms').split(',').map(a => a.trim()).filter(Boolean) : undefined;
@@ -30,6 +33,7 @@ function main(argv = process.argv.slice(2), injected = {}) {
   if (!flags.has('--registration')) throw new Error('Evaluation requires a preregistration file');
   const registration = JSON.parse(fs.readFileSync(flags.get('--registration'), 'utf8'));
   return runEvaluation({ ...injected, registration, repeats, allowRealProvider: flags.has('--allow-real-provider'),
+    allowCredentialedTools: flags.has('--allow-credentialed-tools'),
     executable: flags.get('--executable'), model: flags.get('--model'), family: flags.get('--provider'), effort: flags.get('--effort'), authHome: flags.get('--auth-home'),
     artifactDir: flags.get('--artifact-dir'), ...(corpus ? { corpus } : {}), ...(arms ? { arms } : {}),
     ...(flags.has('--max-calls') ? { maxCalls: Number(flags.get('--max-calls')) } : {}),

docker/context-profiles/complex-eval/DESIGN.md

@@ -136,7 +136,7 @@ node docker/context-profiles/ai-eval.js --plan \
   > registration.json
 
 # 5. Run (requires your own Claude subscription login or API key).
-node docker/context-profiles/ai-eval.js --allow-real-provider \
+node docker/context-profiles/ai-eval.js --allow-real-provider --allow-credentialed-tools \
   --registration registration.json \
   --corpus docker/context-profiles/complex-corpus.json \
   --provider claude --model <model> --executable /absolute/path/to/claude \
@@ -144,6 +144,11 @@ node docker/context-profiles/ai-eval.js --allow-real-provider \
   --artifact-dir /absolute/path/for/transcripts > report.json
 ```
 
+Claude task tools inherit the provider credential through the CLI process and can read it. Use
+`--allow-credentialed-tools` only with a trusted local corpus and credential. Without that
+explicit flag, real Claude task evaluation stops before a provider call; selection-only calls
+remain tool-free. This development evaluator does not provide a credential isolation boundary.
+
 The registration digest binds the exact corpus, evaluator source, model, and
 executable; the run refuses to start if any of them drift, and aborts if the
 tree changes mid-run. `--artifact-dir` retains per-trial session transcripts