fix: distinguish auth failures from ordinary provider discussion

docker/gateguard-effectiveness/lib.js

@@ -227,6 +227,13 @@ function parseStream(stdout) {
   return parsed;
 }
 
+/** Recognize sign-in failure diagnostics, not ordinary authentication discussion. */
+function isAuthenticationFailure(message) {
+  if (typeof message !== 'string') return false;
+  return /^\s*(?:(?:API\s+)?Error:\s*)?(?:Not\s+(?:logged|signed)\s+in|Authentication\s+(?:is\s+)?(?:failed|required)|OAuth(?:\s+(?:token|authentication))?\s+(?:failed|expired|invalid|required)|(?:Login|Sign-in)\s+(?:failed|required)|Invalid\s+API\s+key)(?=\s*(?:$|[.:;!()\u00b7-])|[\r\n])/i.test(message)
+    || /^\s*(?:Please|Must)\s+(?:run\s+\/login|log\s+in|sign\s+in)\b/i.test(message);
+}
+
 function readMetrics(stateDir) {
   const file = path.join(stateDir, 'metrics.jsonl');
   if (!fs.existsSync(file)) return [];
@@ -461,6 +468,7 @@ module.exports = {
   childEnv,
   claudeArgs,
   parseStream,
+  isAuthenticationFailure,
   readMetrics,
   hookWasObserved,
   evidenceRecall,

docker/gateguard-effectiveness/run-intent.js

@@ -13,7 +13,7 @@ const crypto = require('crypto');
 const os = require('os');
 const path = require('path');
 const { spawnSync } = require('child_process');
-const { schedule, completedTrialKeys } = require('./lib');
+const { schedule, completedTrialKeys, isAuthenticationFailure } = require('./lib');
 const { ARMS, armSettings, resolveRef, materializeTree } = require('./arms');
 const { loadIntentScenarios, graderIsSound, formatGraderScores, runIntentTrial, SUPPORTED_ARMS } = require('./intent-eval');
 const { renderHoles, difficulty, isValidTrial } = require('./coverage');
@@ -356,7 +356,7 @@ function main() {
       fs.rmSync(path.join(workRoot, trial.key.replace(/\//g, '__')), { recursive: true, force: true });
       // An unauthenticated or hookless run must not be recorded: graded as data it
       // looks like a finding (the first such trial was filed as a coverage hole).
-      if (row.providerError && /authenticat|log ?in|oauth/i.test(row.providerMessage || '')) {
+      if (row.providerError && isAuthenticationFailure(row.providerMessage)) {
         throw new Error(`${trial.key}: Claude is not signed in (${row.providerMessage}); sign in with the claude CLI and rerun with the same --out`);
       }
       if (!row.hookObserved) {

docker/gateguard-effectiveness/run.js

@@ -7,7 +7,7 @@ const os = require('os');
 const path = require('path');
 const { spawnSync } = require('child_process');
 const { ARMS, DEFAULT_ARMS, resolveRef, materializeTree, armSettings } = require('./arms');
-const { loadScenarios, schedule, runTrial, summarize, renderMarkdown, completedTrialKeys } = require('./lib');
+const { loadScenarios, schedule, runTrial, summarize, renderMarkdown, completedTrialKeys, isAuthenticationFailure } = require('./lib');
 
 const REPO_ROOT = path.join(__dirname, '..', '..');
 
@@ -202,7 +202,7 @@ function main() {
         };
         fs.writeFileSync(path.join(options.out, row.transcript), transcript);
         fs.rmSync(path.join(workRoot, trial.key.replace(/\//g, '__')), { recursive: true, force: true });
-        if (row.providerError && /authenticat|log ?in|oauth/i.test(row.providerMessage || '')) {
+        if (row.providerError && isAuthenticationFailure(row.providerMessage)) {
           throw new Error(`${trial.key}: Claude is not signed in (${row.providerMessage}); sign in with the claude CLI and rerun with the same --out`);
         }
         if (!row.hookObserved) throw new Error(`${trial.key}: the gate did not run although the agent edited files; check the hook setup before spending more sessions`);