'use strict';

// Development-only evaluator; it lives under docker/ so the npm package never ships it.
// see docs/gateguard/question-effectiveness.md
const fs = require('fs');
const path = require('path');
const { spawnSync } = require('child_process');
const { runScoredCheck, wilson } = require('../context-profiles/ai-eval-lib');
const { ARMS, TRIAL_FILE, trialPatch } = require('./arms');

const SCENARIO_DIR = path.join(__dirname, 'scenarios');
const DOT_PREFIX = 'dot-';
const AUTONOMY_NOTE = 'Work autonomously in this repository and do not ask clarifying questions.';
const PASSTHROUGH_ENV = new Set(['CLAUDE_CODE_OAUTH_TOKEN', 'CLAUDE_CODE_GIT_BASH_PATH', 'CLAUDE_CONFIG_DIR']);
const SCRUBBED_ENV = /^(CLAUDE_|ECC_|GATEGUARD_)/;

// --- scenarios ---

function loadScenarios(dir = SCENARIO_DIR, ids = null) {
  const scenarios = fs
    .readdirSync(dir, { withFileTypes: true })
    .filter(entry => entry.isDirectory())
    .map(entry => {
      const root = path.join(dir, entry.name);
      const spec = JSON.parse(fs.readFileSync(path.join(root, 'task.json'), 'utf8'));
      if (spec.id !== entry.name) throw new Error(`scenario ${entry.name} declares id ${spec.id}`);
      for (const key of ['question', 'trap', 'prompt']) {
        if (typeof spec[key] !== 'string' || !spec[key]) throw new Error(`scenario ${spec.id} is missing ${key}`);
      }
      for (const key of ['evidence', 'targetQuestions']) {
        if (!Array.isArray(spec[key]) || spec[key].length === 0) throw new Error(`scenario ${spec.id} has no ${key}`);
      }
      return {
        ...spec,
        root,
        evidence: spec.evidence.map(source => new RegExp(source, 'i')),
        grader: fs.readFileSync(path.join(root, 'grader.cjs'), 'utf8')
      };
    })
    .sort((a, b) => a.id.localeCompare(b.id));
  if (!ids) return scenarios;
  const unknown = ids.filter(id => !scenarios.some(scenario => scenario.id === id));
  if (unknown.length) throw new Error(`unknown scenario: ${unknown.join(', ')}`);
  return scenarios.filter(scenario => ids.includes(scenario.id));
}

function copyTree(from, to) {
  for (const entry of fs.readdirSync(from, { withFileTypes: true })) {
    const name = entry.name.startsWith(DOT_PREFIX) ? `.${entry.name.slice(DOT_PREFIX.length)}` : entry.name;
    const source = path.join(from, entry.name);
    const target = path.join(to, name);
    if (entry.isDirectory()) {
      fs.mkdirSync(target, { recursive: true });
      copyTree(source, target);
    } else if (entry.isFile()) {
      fs.copyFileSync(source, target);
    }
  }
}

/** Writes a scenario's starting workspace into dest, optionally with its reference or naive overlay. */
function prepareWorkspace(scenario, dest, overlay = null) {
  fs.mkdirSync(dest, { recursive: true });
  copyTree(path.join(scenario.root, 'workspace'), dest);
  if (overlay) copyTree(path.join(scenario.root, overlay), dest);
  return dest;
}

/** Commits the starting workspace so `git status` and `git diff` work as in a real checkout. */
function commitWorkspace(cwd) {
  const env = { ...process.env, GIT_AUTHOR_NAME: 'eval', GIT_AUTHOR_EMAIL: 'eval@example.invalid', GIT_COMMITTER_NAME: 'eval', GIT_COMMITTER_EMAIL: 'eval@example.invalid' };
  for (const args of [['init', '-q'], ['add', '-A'], ['commit', '-q', '--no-gpg-sign', '-m', 'start']]) {
    const result = spawnSync('git', args, { cwd, env, encoding: 'utf8', shell: false, timeout: 30000 });
    if (result.status !== 0 || result.error) return false;
  }
  return true;
}

/** Graders always exit 0 and print a score; only a full score passes, and a crashed grader scores 0. */
function grade(scenario, cwd) {
  const { passed, score } = runScoredCheck(cwd, scenario.grader, 20000);
  return { passed: passed && score === 1, score: passed ? score : 0 };
}

function listFiles(dir, base = dir, out = []) {
  for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
    const file = path.join(dir, entry.name);
    if (entry.isDirectory()) listFiles(file, base, out);
    else if (entry.isFile()) out.push(path.relative(base, file));
  }
  return out.sort();
}

function changedHunk(before, after) {
  const a = before.split('\n');
  const b = after.split('\n');
  let head = 0;
  while (head < a.length && head < b.length && a[head] === b[head]) head++;
  let tail = 0;
  while (tail < a.length - head && tail < b.length - head && a[a.length - 1 - tail] === b[b.length - 1 - tail]) tail++;
  return { old_string: a.slice(head, a.length - tail).join('\n'), new_string: b.slice(head, b.length - tail).join('\n') };
}

/** Questions a gate asks for the first edit of every file the reference solution changes. */
function questionsForReference(scenario, hookFile) {
  const workspace = fs.realpathSync(fs.mkdtempSync(path.join(require('os').tmpdir(), 'gg-route-')));
  const stateDir = fs.mkdtempSync(path.join(require('os').tmpdir(), 'gg-route-state-'));
  const saved = { GATEGUARD_STATE_DIR: process.env.GATEGUARD_STATE_DIR, GATEGUARD_METRICS: process.env.GATEGUARD_METRICS, CLAUDE_PROJECT_DIR: process.env.CLAUDE_PROJECT_DIR };
  try {
    prepareWorkspace(scenario, workspace);
    Object.assign(process.env, { GATEGUARD_STATE_DIR: stateDir, GATEGUARD_METRICS: '1', CLAUDE_PROJECT_DIR: workspace });
    const hookDir = path.dirname(path.dirname(hookFile));
    for (const key of Object.keys(require.cache)) if (key.startsWith(hookDir)) delete require.cache[key];
    const hook = require(hookFile);
    const reference = path.join(scenario.root, 'reference');
    for (const relative of listFiles(reference)) {
      const target = path.join(workspace, ...relative.split(path.sep).map(part => (part.startsWith(DOT_PREFIX) ? `.${part.slice(DOT_PREFIX.length)}` : part)));
      const next = fs.readFileSync(path.join(reference, relative), 'utf8');
      const call = fs.existsSync(target)
        ? { tool_name: 'Edit', tool_input: { file_path: target, ...changedHunk(fs.readFileSync(target, 'utf8'), next) } }
        : { tool_name: 'Write', tool_input: { file_path: target, content: next } };
      hook.run(JSON.stringify({ session_id: `route-${scenario.id}`, cwd: workspace, hook_event_name: 'PreToolUse', ...call }));
    }
    return readMetrics(stateDir).map(event => ({ cls: event.class, decision: event.decision, questions: event.questions || [] }));
  } finally {
    for (const [key, value] of Object.entries(saved)) {
      if (value === undefined) delete process.env[key];
      else process.env[key] = value;
    }
    fs.rmSync(workspace, { recursive: true, force: true });
    fs.rmSync(stateDir, { recursive: true, force: true });
  }
}

// --- trials ---

function xorshift(seed) {
  let state = seed >>> 0 || 1;
  return n => {
    state ^= state << 13;
    state >>>= 0;
    state ^= state >>> 17;
    state ^= state << 5;
    state >>>= 0;
    return Math.floor((state / 4294967296) * n);
  };
}

/** Every (rep, scenario, arm) trial; arms run in a shuffled order inside each rep and scenario. */
function schedule(scenarios, arms, reps, seed = 11) {
  const random = xorshift(seed);
  const trials = [];
  for (let rep = 1; rep <= reps; rep++) {
    for (const scenario of scenarios) {
      const order = arms.slice();
      for (let i = order.length - 1; i > 0; i--) {
        const j = random(i + 1);
        [order[i], order[j]] = [order[j], order[i]];
      }
      for (const arm of order) trials.push({ key: `${scenario.id}/${arm}/${rep}`, scenario, arm, rep });
    }
  }
  return trials;
}

function childEnv(stateDir) {
  const env = {};
  for (const [key, value] of Object.entries(process.env)) {
    if (!SCRUBBED_ENV.test(key) || PASSTHROUGH_ENV.has(key)) env[key] = value;
  }
  env.GATEGUARD_STATE_DIR = stateDir;
  env.GATEGUARD_METRICS = '1';
  env.DISABLE_NON_ESSENTIAL_MODEL_CALLS = '1';
  return env;
}

function claudeArgs({ model, settingsPath, maxTurns }) {
  return [
    '--print',
    '--output-format', 'stream-json',
    '--verbose',
    '--no-session-persistence',
    '--permission-mode', 'bypassPermissions',
    '--setting-sources', 'project',
    '--strict-mcp-config',
    '--settings', settingsPath,
    '--max-turns', String(maxTurns),
    '--model', model
  ];
}

const GATE_MARKER = /\[Fact-Forcing Gate\]/;

function toolResultText(content) {
  if (typeof content === 'string') return content;
  if (!Array.isArray(content)) return '';
  return content.map(block => (block && typeof block.text === 'string' ? block.text : '')).join('\n');
}

/** Reads a stream-json transcript: final result, assistant text and tool calls. */
function parseStream(stdout) {
  const parsed = { result: null, texts: [], tools: {}, editCalls: 0, shellCalls: 0, gateDenials: 0 };
  for (const line of String(stdout || '').split('\n')) {
    if (!line.trim()) continue;
    let event;
    try {
      event = JSON.parse(line);
    } catch (_) {
      continue;
    }
    if (event && event.type === 'result') parsed.result = event;
    if (event && event.type === 'user' && event.message && Array.isArray(event.message.content)) {
      for (const block of event.message.content) {
        if (block && block.type === 'tool_result' && GATE_MARKER.test(toolResultText(block.content))) parsed.gateDenials++;
      }
    }
    const content = event && event.type === 'assistant' && event.message && Array.isArray(event.message.content) ? event.message.content : [];
    for (const block of content) {
      if (block.type === 'text' && typeof block.text === 'string') parsed.texts.push(block.text);
      if (block.type === 'tool_use') {
        parsed.tools[block.name] = (parsed.tools[block.name] || 0) + 1;
        if (['Edit', 'Write', 'MultiEdit', 'NotebookEdit'].includes(block.name)) parsed.editCalls++;
        if (['Bash', 'PowerShell'].includes(block.name)) parsed.shellCalls++;
      }
    }
  }
  return parsed;
}

/** Recognize sign-in failure diagnostics, not ordinary authentication discussion. */
function isAuthenticationFailure(message) {
  if (typeof message !== 'string') return false;
  return /^\s*(?:(?:API\s+)?Error:\s*)?(?:Not\s+(?:logged|signed)\s+in|Authentication\s+(?:is\s+)?(?:failed|required)|OAuth(?:\s+(?:token|authentication))?\s+(?:failed|expired|invalid|required)|(?:Login|Sign-in)\s+(?:failed|required)|Invalid\s+API\s+key)(?=\s*(?:$|[.:;!()\u00b7-])|[\r\n])/i.test(message)
    || /^\s*(?:Please|Must)\s+(?:run\s+\/login|log\s+in|sign\s+in)\b/i.test(message);
}

function readMetrics(stateDir) {
  const file = path.join(stateDir, 'metrics.jsonl');
  if (!fs.existsSync(file)) return [];
  return fs
    .readFileSync(file, 'utf8')
    .split('\n')
    .filter(Boolean)
    .map(line => {
      try {
        return JSON.parse(line);
      } catch (_) {
        return null;
      }
    })
    .filter(Boolean);
}

function hookWasObserved({ gated, editCalls, shellCalls, metrics, gateDenials }) {
  return !gated || editCalls + shellCalls === 0 || metrics.length > 0 || gateDenials > 0;
}

/** Share of a scenario's evidence patterns that the agent's own text states. */
function evidenceRecall(scenario, texts) {
  const text = texts.join('\n');
  return scenario.evidence.filter(pattern => pattern.test(text)).length / scenario.evidence.length;
}

function usageOf(result) {
  const u = (result && result.usage) || {};
  const n = value => (Number.isFinite(value) ? value : 0);
  return {
    inputTokens: n(u.input_tokens) + n(u.cache_creation_input_tokens),
    cachedInputTokens: n(u.cache_read_input_tokens),
    outputTokens: n(u.output_tokens),
    costUsd: result && Number.isFinite(result.total_cost_usd) ? result.total_cost_usd : null,
    turns: result && Number.isFinite(result.num_turns) ? result.num_turns : null,
    durationMs: result && Number.isFinite(result.duration_ms) ? result.duration_ms : null,
    models: result && result.modelUsage ? Object.keys(result.modelUsage) : []
  };
}

/** One trial: fresh workspace and state, one headless session, then the hidden grader. */
function runTrial(trial, { workRoot, armSettingsByName, executable, model, maxTurns, timeoutMs, execute = spawnSync }) {
  const dir = path.join(workRoot, trial.key.replace(/\//g, '__'));
  fs.rmSync(dir, { recursive: true, force: true });
  const cwd = prepareWorkspace(trial.scenario, path.join(dir, 'repo'));
  const git = commitWorkspace(cwd);
  const stateDir = path.join(dir, 'state');
  fs.mkdirSync(stateDir, { recursive: true });
  const settingsPath = path.join(dir, 'settings.json');
  fs.writeFileSync(settingsPath, `${JSON.stringify(armSettingsByName[trial.arm], null, 2)}\n`);
  fs.writeFileSync(path.join(dir, TRIAL_FILE), `${JSON.stringify({ stateDir, patch: trialPatch(trial.arm, trial.scenario) })}\n`);
  const started = Date.now();
  const run = execute(executable, claudeArgs({ model, settingsPath, maxTurns }), {
    cwd,
    env: childEnv(stateDir),
    input: `${trial.scenario.prompt}\n\n${AUTONOMY_NOTE}\n`,
    encoding: 'utf8',
    shell: false,
    timeout: timeoutMs,
    killSignal: 'SIGKILL',
    maxBuffer: 64 * 1024 * 1024
  });
  const stream = parseStream(run.stdout);
  const metrics = readMetrics(stateDir);
  const outcome = grade(trial.scenario, cwd);
  const gated = ARMS[trial.arm].gate;
  const denials = metrics.filter(event => event.decision === 'deny' || event.decision === 'routine-deny');
  return {
    key: trial.key,
    scenario: trial.scenario.id,
    question: trial.scenario.question,
    arm: trial.arm,
    rep: trial.rep,
    git,
    exitStatus: run.status,
    timedOut: Boolean(run.error && run.error.code === 'ETIMEDOUT'),
    providerError: !stream.result || stream.result.is_error === true,
    providerMessage: stream.result && stream.result.is_error === true
      ? String(stream.result.result || '').slice(0, 200)
      : !stream.result
        ? String([...stream.texts].reverse().find(text => text.trim() !== '') || '').slice(0, 200) || null
        : null,
    wallMs: Date.now() - started,
    passed: outcome.passed,
    score: outcome.score,
    evidenceRecall: evidenceRecall(trial.scenario, stream.texts),
    tools: stream.tools,
    editCalls: stream.editCalls,
    denials: denials.length,
    gateDenials: stream.gateDenials,
    questionsAsked: [...new Set(denials.flatMap(event => event.questions || []))].sort(),
    hookObserved: hookWasObserved({
      gated,
      editCalls: stream.editCalls,
      shellCalls: stream.shellCalls,
      metrics,
      gateDenials: stream.gateDenials
    }),
    ...usageOf(stream.result)
  };
}

// --- analysis ---

function median(values) {
  const sorted = values.filter(Number.isFinite).sort((a, b) => a - b);
  if (sorted.length === 0) return null;
  const mid = Math.floor(sorted.length / 2);
  return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2;
}

function binomialTwoSided(k, n) {
  if (n === 0) return 1;
  let tail = 0;
  const low = Math.min(k, n - k);
  let term = Math.pow(0.5, n);
  for (let i = 0; i <= low; i++) {
    tail += term;
    term *= (n - i) / (i + 1);
  }
  return Math.min(1, 2 * tail);
}

/** Paired comparison of an arm against the reference arm over trials sharing scenario and rep. */
function pairedComparison(rows, reference, arm, { rounds = 4000, seed = 0x9e3779b9 } = {}) {
  const byKey = new Map(rows.filter(row => row.arm === reference).map(row => [`${row.scenario}/${row.rep}`, row]));
  const pairs = rows.filter(row => row.arm === arm && byKey.has(`${row.scenario}/${row.rep}`)).map(row => [byKey.get(`${row.scenario}/${row.rep}`), row]);
  const diffs = pairs.map(([ref, other]) => Number(other.passed) - Number(ref.passed));
  const onlyReference = pairs.filter(([ref, other]) => ref.passed && !other.passed).length;
  const onlyArm = pairs.filter(([ref, other]) => !ref.passed && other.passed).length;
  const random = xorshift(seed);
  const means = [];
  for (let i = 0; i < rounds && diffs.length; i++) {
    let sum = 0;
    for (let j = 0; j < diffs.length; j++) sum += diffs[random(diffs.length)];
    means.push(sum / diffs.length);
  }
  means.sort((a, b) => a - b);
  return {
    arm,
    reference,
    pairs: pairs.length,
    passRateDifference: diffs.length ? diffs.reduce((a, b) => a + b, 0) / diffs.length : null,
    interval: means.length ? [means[Math.floor(rounds * 0.025)], means[Math.ceil(rounds * 0.975) - 1]] : null,
    onlyReference,
    onlyArm,
    mcnemarP: binomialTwoSided(Math.min(onlyReference, onlyArm), onlyReference + onlyArm)
  };
}

/** A trial counts only when the provider, the clock and the judge all completed. */
function isValidTrial(row) {
  return !row.providerError && !row.timedOut && !row.judgeFailed;
}

/** Scheduled keys with a valid trial recorded; failed attempts stay in the log and are retried. */
function completedTrialKeys(rows) {
  return new Set(rows.filter(isValidTrial).map(row => row.key));
}

// Totals cover valid trials only; provider failures and timeouts are counted apart.
function armSummary(rows, arm) {
  const attempts = rows.filter(row => row.arm === arm);
  const own = attempts.filter(isValidTrial);
  const passes = own.filter(row => row.passed).length;
  return {
    arm,
    trials: own.length,
    passes,
    passInterval: wilson(passes, own.length),
    meanScore: own.length ? own.reduce((sum, row) => sum + row.score, 0) / own.length : null,
    medianTokens: median(own.map(row => row.inputTokens + row.cachedInputTokens + row.outputTokens)),
    medianOutputTokens: median(own.map(row => row.outputTokens)),
    medianCostUsd: median(own.map(row => row.costUsd)),
    medianTurns: median(own.map(row => row.turns)),
    medianWallMs: median(own.map(row => row.wallMs)),
    medianDenials: median(own.map(row => row.gateDenials)),
    meanEvidenceRecall: own.length ? own.reduce((sum, row) => sum + row.evidenceRecall, 0) / own.length : null,
    providerErrors: attempts.filter(row => row.providerError || row.timedOut).length
  };
}

function summarize(rows, arms, reference = 'gate') {
  const valid = rows.filter(isValidTrial);
  const scenarios = [...new Set(valid.map(row => row.scenario))].sort();
  return {
    arms: arms.map(arm => armSummary(rows, arm)),
    comparisons: arms.filter(arm => arm !== reference).map(arm => pairedComparison(valid, reference, arm)),
    byScenario: scenarios.map(scenario => ({
      scenario,
      question: (valid.find(row => row.scenario === scenario) || {}).question,
      arms: Object.fromEntries(arms.map(arm => {
        const own = valid.filter(row => row.scenario === scenario && row.arm === arm);
        return [arm, { passes: own.filter(row => row.passed).length, trials: own.length, evidence: own.length ? own.reduce((s, r) => s + r.evidenceRecall, 0) / own.length : null }];
      }))
    }))
  };
}

function renderMarkdown(summary, meta = {}) {
  const pct = value => (value === null ? 'n/a' : `${Math.round(value * 100)}%`);
  const num = value => (value === null ? 'n/a' : String(Math.round(value * 100) / 100));
  const lines = [];
  if (meta.model) lines.push(`Model: ${meta.model}; hook commit: ${meta.sha || 'n/a'}; reps: ${meta.reps || 'n/a'}.`, '');
  lines.push('| Arm | Trials | Passed (95% interval) | Mean score | Evidence stated | Median denials | Median turns | Median tokens | Median cost (USD) | Errors |');
  lines.push('| --- | ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |');
  for (const a of summary.arms) {
    lines.push(`| ${a.arm} | ${a.trials} | ${a.passes}/${a.trials} (${pct(a.passInterval[0])}–${pct(a.passInterval[1])}) | ${num(a.meanScore)} | ${pct(a.meanEvidenceRecall)} | ${num(a.medianDenials)} | ${num(a.medianTurns)} | ${num(a.medianTokens)} | ${num(a.medianCostUsd)} | ${a.providerErrors} |`);
  }
  lines.push('', '| Arm vs gate | Pairs | Pass-rate difference (95% interval) | Gate only | Arm only | McNemar p |', '| --- | ---: | --- | ---: | ---: | ---: |');
  for (const c of summary.comparisons) {
    const interval = c.interval ? `${pct(c.interval[0])} to ${pct(c.interval[1])}` : 'n/a';
    lines.push(`| ${c.arm} | ${c.pairs} | ${pct(c.passRateDifference)} (${interval}) | ${c.onlyReference} | ${c.onlyArm} | ${num(c.mcnemarP)} |`);
  }
  const arms = summary.arms.map(a => a.arm);
  lines.push('', `| Scenario | Question | ${arms.join(' | ')} |`, `| --- | --- | ${arms.map(() => '---:').join(' | ')} |`);
  for (const s of summary.byScenario) {
    lines.push(`| ${s.scenario} | ${s.question} | ${arms.map(arm => `${s.arms[arm].passes}/${s.arms[arm].trials}`).join(' | ')} |`);
  }
  return `${lines.join('\n')}\n`;
}

module.exports = {
  loadScenarios,
  prepareWorkspace,
  commitWorkspace,
  grade,
  changedHunk,
  questionsForReference,
  schedule,
  childEnv,
  claudeArgs,
  parseStream,
  isAuthenticationFailure,
  readMetrics,
  hookWasObserved,
  evidenceRecall,
  runTrial,
  binomialTwoSided,
  pairedComparison,
  isValidTrial,
  completedTrialKeys,
  summarize,
  renderMarkdown
};