refactor(eval): split the context-profile evaluator run loop

runEvaluation is now plan, execute, report: option validation and family
resolution, registration pinning, provider and state creation, install
preparation, the selection and trial loops, and the report object each sit in
their own function. outcomeTrial takes one trial object instead of ten
positionals and splits into single-step and stepped paths. The provider
adapter separates the guarded provider call from output parsing, and the
complex corpus validator checks one task at a time.

Report keys, error messages, and call ordering are unchanged. The file is in
the implementation digest list, so registration digests move as they do for
any edit here. Clears the four complexity warnings and the max-params warning
on this file.
This commit is contained in:
Affaan Mustafa authored and Affaan Mustafa committed 2026-10-19 22:18:32 -04:00
1 parent d53359a974
commit 35f57c80e6
1 file changed
+194 -82
+194 -82
View File
@@ -75,6 +75,37 @@ function validateCorpusIds(cases) {
// Complex corpora hold a few realistic multi-file tasks with scored hidden graders. Sample gates // Complex corpora hold a few realistic multi-file tasks with scored hidden graders. Sample gates
// are descriptive at this size, so the distinct-task minimum relaxes to the corpus itself. // are descriptive at this size, so the distinct-task minimum relaxes to the corpus itself.
function invalidComplexStep(step) {
return typeof step.query !== 'string' || !bounded(Buffer.byteLength(step.query), 1, 8192)
|| typeof step.check !== 'string' || !bounded(Buffer.byteLength(step.check), 1, 65536)
|| (step.checkTimeoutMs !== undefined && !bounded(step.checkTimeoutMs, 1, 120000))
|| (step.manualIds !== undefined && (!Array.isArray(step.manualIds) || step.manualIds.length > 3));
}
function validateComplexTaskFiles(task) {
const files = Object.entries(task.files || {});
if (!Array.isArray(task.manualIds) || task.manualIds.length > 3 || !bounded(files.length, 1, 24)
|| files.some(([file, content]) => !safeRelative(file) || typeof content !== 'string' || Buffer.byteLength(content) > 65536)) {
throw new Error('Invalid corpus task');
}
}
function validateComplexTask(task) {
if (!/^[a-z][a-z0-9-]{0,63}$/.test(task.id)) throw new Error('Invalid corpus case');
if (task.steps === undefined
&& (typeof task.query !== 'string' || !bounded(Buffer.byteLength(task.query), 1, 8192))) throw new Error('Invalid corpus case');
validateComplexTaskFiles(task);
if (task.steps !== undefined) {
// Stepped (chained) task: sequential tickets graded in one accumulating workspace.
if (!Array.isArray(task.steps) || !bounded(task.steps.length, 2, 8) || task.steps.some(invalidComplexStep)) {
throw new Error('Invalid corpus task');
}
} else if (typeof task.check !== 'string' || !bounded(Buffer.byteLength(task.check), 1, 65536)
|| (task.checkTimeoutMs !== undefined && !bounded(task.checkTimeoutMs, 1, 120000))) {
throw new Error('Invalid corpus task');
}
}
function validateComplexCorpus(corpus) { function validateComplexCorpus(corpus) {
if (!Array.isArray(corpus.selection) || !Array.isArray(corpus.tasks) if (!Array.isArray(corpus.selection) || !Array.isArray(corpus.tasks)
|| !bounded(corpus.selection.length, 0, 50) || !bounded(corpus.tasks.length, 1, 10) || !bounded(corpus.selection.length, 0, 50) || !bounded(corpus.tasks.length, 1, 10)
@@ -83,29 +114,7 @@ function validateComplexCorpus(corpus) {
} }
validateCorpusIds(corpus.selection); validateCorpusIds(corpus.selection);
if (new Set(corpus.tasks.map(c => c.id)).size !== corpus.tasks.length) throw new Error('Duplicate corpus ID'); if (new Set(corpus.tasks.map(c => c.id)).size !== corpus.tasks.length) throw new Error('Duplicate corpus ID');
for (const task of corpus.tasks) { for (const task of corpus.tasks) validateComplexTask(task);
if (!/^[a-z][a-z0-9-]{0,63}$/.test(task.id)) throw new Error('Invalid corpus case');
if (task.steps === undefined
&& (typeof task.query !== 'string' || !bounded(Buffer.byteLength(task.query), 1, 8192))) throw new Error('Invalid corpus case');
const files = Object.entries(task.files || {});
if (!Array.isArray(task.manualIds) || task.manualIds.length > 3 || !bounded(files.length, 1, 24)
|| files.some(([file, content]) => !safeRelative(file) || typeof content !== 'string' || Buffer.byteLength(content) > 65536)) {
throw new Error('Invalid corpus task');
}
if (task.steps !== undefined) {
// Stepped (chained) task: sequential tickets graded in one accumulating workspace.
if (!Array.isArray(task.steps) || !bounded(task.steps.length, 2, 8)
|| task.steps.some(step => typeof step.query !== 'string' || !bounded(Buffer.byteLength(step.query), 1, 8192)
|| typeof step.check !== 'string' || !bounded(Buffer.byteLength(step.check), 1, 65536)
|| (step.checkTimeoutMs !== undefined && !bounded(step.checkTimeoutMs, 1, 120000))
|| (step.manualIds !== undefined && (!Array.isArray(step.manualIds) || step.manualIds.length > 3)))) {
throw new Error('Invalid corpus task');
}
} else if (typeof task.check !== 'string' || !bounded(Buffer.byteLength(task.check), 1, 65536)
|| (task.checkTimeoutMs !== undefined && !bounded(task.checkTimeoutMs, 1, 120000))) {
throw new Error('Invalid corpus task');
}
}
} }
function sourceSnapshot(repoRoot) { function sourceSnapshot(repoRoot) {
@@ -601,8 +610,25 @@ function launchEnvironment(launch) {
TMPDIR: launch.home, LANG: 'C.UTF-8' }; TMPDIR: launch.home, LANG: 'C.UTF-8' };
} }
function executeAdapter(state, cwd, environment) { // One provider call. A throwing provider still records its elapsed time and
return (_command, args, options) => { // restores the environment; only source drift propagates as itself.
function callProvider(state, environment, request, started) {
try {
return state.provider(request);
} catch (error) {
state.metrics.push({ phase: request.phase, elapsedMs: Date.now() - started, usage: null });
if (error?.code === 'source-drift') throw error;
return fail('provider-failed');
} finally { environment.restore(); }
}
function parseProviderOutput(state, raw) {
const parsed = state.family === 'claude' ? parseClaudeJson(raw?.stdout) : parseCodexJsonl(raw?.stdout);
const succeeded = raw?.status === 0 && !raw?.error;
return { parsed, succeeded };
}
function adapterCall(state, cwd, environment, args, options) {
if (state.calls >= state.maxCalls) fail('call-budget'); if (state.calls >= state.maxCalls) fail('call-budget');
state.assertCurrent(); state.assertCurrent();
environment.verify(); environment.verify();
@@ -611,26 +637,23 @@ function executeAdapter(state, cwd, environment) {
const phase = options.phase || (args.includes('read-only') ? 'selection' : 'task'); const phase = options.phase || (args.includes('read-only') ? 'selection' : 'task');
state.calls++; state.calls++;
const started = Date.now(); const started = Date.now();
let raw;
// Coding tasks outgrow the launcher's interactive default, so the evaluator's own call bound governs them. // Coding tasks outgrow the launcher's interactive default, so the evaluator's own call bound governs them.
const timeoutMs = Math.min(phase === 'task' ? state.callTimeoutMs : options.timeout, state.callTimeoutMs, remaining); const timeoutMs = Math.min(phase === 'task' ? state.callTimeoutMs : options.timeout, state.callTimeoutMs, remaining);
const env = options.env || launchEnvironment(environment.launch); const env = options.env || launchEnvironment(environment.launch);
try { const raw = callProvider(state, environment,
raw = state.provider({ phase, input: options.input, cwd, env, timeoutMs, maxBuffer: 1024 * 1024 }); { phase, input: options.input, cwd, env, timeoutMs, maxBuffer: 1024 * 1024 }, started);
} catch (error) {
state.metrics.push({ phase, elapsedMs: Date.now() - started, usage: null });
if (error?.code === 'source-drift') throw error;
fail('provider-failed');
} finally { environment.restore(); }
const elapsedMs = Date.now() - started; const elapsedMs = Date.now() - started;
const parsed = state.family === 'claude' ? parseClaudeJson(raw?.stdout) : parseCodexJsonl(raw?.stdout); const { parsed, succeeded } = parseProviderOutput(state, raw);
state.metrics.push({ phase, elapsedMs, usage: parsed.valid && raw?.status === 0 && !raw?.error ? parsed.usage : null }); state.metrics.push({ phase, elapsedMs, usage: parsed.valid && succeeded ? parsed.usage : null });
if (Date.now() >= state.deadline || elapsedMs > timeoutMs) fail('deadline'); if (Date.now() >= state.deadline || elapsedMs > timeoutMs) fail('deadline');
state.assertCurrent(); state.assertCurrent();
if (raw?.status !== 0 || raw?.error) fail('provider-failed'); if (!succeeded) fail('provider-failed');
if (!parsed.valid) fail(parsed.error ? 'provider-failed' : 'invalid-jsonl'); if (!parsed.valid) fail(parsed.error ? 'provider-failed' : 'invalid-jsonl');
return { status: 0, stdout: parsed.text }; return { status: 0, stdout: parsed.text };
}; }
function executeAdapter(state, cwd, environment) {
return (_command, args, options) => adapterCall(state, cwd, environment, args, options);
} }
function selectionProbe(item, repoRoot, execute, environment, target) { function selectionProbe(item, repoRoot, execute, environment, target) {
@@ -661,48 +684,63 @@ function selectionProbe(item, repoRoot, execute, environment, target) {
// Full relies on native discovery of the whole install; the Lean arms receive ECC-selected skill bodies; // Full relies on native discovery of the whole install; the Lean arms receive ECC-selected skill bodies;
// ecc-legacy runs bare against the pinned pre-scoping skill library; Baseline runs the bare task query. // ecc-legacy runs bare against the pinned pre-scoping skill library; Baseline runs the bare task query.
// Stepped tasks run each ticket in the same accumulating workspace, grading after every step. // Stepped tasks run each ticket in the same accumulating workspace, grading after every step.
function outcomeTrial(item, arm, repeat, repoRoot, execute, cwd, environment, target, harvest, metrics = null) { // `trial` carries repoRoot, execute, cwd, environment, target, harvest (or null)
const launchStep = (query, manualIds) => { // and metrics (or null) for one task trial.
function launchTrialStep(item, arm, trial, query, manualIds) {
const task = { sessionId: 'ecc-eval', taskId: item.id, revision: 1, phase: 'evaluate', query }; const task = { sessionId: 'ecc-eval', taskId: item.id, revision: 1, phase: 'evaluate', query };
return launchTaskContext({ repoRoot, execute, nativeEnvironment: environment.launch, target, return launchTaskContext({ repoRoot: trial.repoRoot, execute: trial.execute,
nativeEnvironment: trial.environment.launch, target: trial.target,
bare: arm === 'baseline' || arm === 'ecc-legacy', bare: arm === 'baseline' || arm === 'ecc-legacy',
task: { ...task, ...(arm === 'manual-lean' && manualIds?.length ? { explicitIds: manualIds } : {}) }, task: { ...task, ...(arm === 'manual-lean' && manualIds?.length ? { explicitIds: manualIds } : {}) },
profileId: arm === 'full' ? 'full@1' : 'lean@1', selectionMode: arm === 'auto-lean' ? 'auto' : 'manual' }); profileId: arm === 'full' ? 'full@1' : 'lean@1', selectionMode: arm === 'auto-lean' ? 'auto' : 'manual' });
}; }
try {
if (!item.steps) { function stepMetrics(metrics, start) {
const result = launchStep(item.query, item.manualIds); return metrics ? metricsSince(metrics, start) : {};
}
function singleStepTrial(item, arm, repeat, trial) {
const { cwd, environment, harvest } = trial;
const result = launchTrialStep(item, arm, trial, item.query, item.manualIds);
if (harvest) harvest(arm, item.id, repeat, environment); if (harvest) harvest(arm, item.id, repeat, environment);
const verdict = runScoredCheck(cwd, item.check, item.checkTimeoutMs); const verdict = runScoredCheck(cwd, item.check, item.checkTimeoutMs);
const passed = result.status === 'completed' && verdict.passed && verdict.score >= 0.999; const passed = result.status === 'completed' && verdict.passed && verdict.score >= 0.999;
return { id: item.id, arm, repeat, passed, score: result.status === 'completed' ? verdict.score : 0, return { id: item.id, arm, repeat, passed, score: result.status === 'completed' ? verdict.score : 0,
selectedIds: result.selection.selectedIds, failure: passed ? null : 'hidden-check' }; selectedIds: result.selection.selectedIds, failure: passed ? null : 'hidden-check' };
} }
function steppedTrial(item, arm, repeat, trial) {
const { cwd, environment, harvest, metrics } = trial;
const steps = []; const steps = [];
const selectedIds = []; const selectedIds = [];
for (let index = 0; index < item.steps.length; index++) { for (let index = 0; index < item.steps.length; index++) {
const step = item.steps[index]; const step = item.steps[index];
const start = metrics ? metrics.length : 0; const start = metrics ? metrics.length : 0;
const result = launchStep(step.query, step.manualIds || item.manualIds); const result = launchTrialStep(item, arm, trial, step.query, step.manualIds || item.manualIds);
if (harvest) harvest(arm, `${item.id}--step${index + 1}`, repeat, environment); if (harvest) harvest(arm, `${item.id}--step${index + 1}`, repeat, environment);
if (result.status !== 'completed') { if (result.status !== 'completed') {
// A failed ticket ends the chain; remaining tickets are unscored. // A failed ticket ends the chain; remaining tickets are unscored.
steps.push({ score: 0, ...(metrics ? metricsSince(metrics, start) : {}) }); steps.push({ score: 0, ...stepMetrics(metrics, start) });
for (let rest = index + 1; rest < item.steps.length; rest++) { for (let rest = index + 1; rest < item.steps.length; rest++) {
steps.push({ score: 0, ...(metrics ? metricsSince(metrics, metrics.length) : {}) }); steps.push({ score: 0, ...stepMetrics(metrics, metrics ? metrics.length : 0) });
} }
break; break;
} }
selectedIds.push(...result.selection.selectedIds); selectedIds.push(...result.selection.selectedIds);
const verdict = runScoredCheck(cwd, step.check, step.checkTimeoutMs, index + 1); const verdict = runScoredCheck(cwd, step.check, step.checkTimeoutMs, index + 1);
steps.push({ score: verdict.passed ? verdict.score : 0, ...(metrics ? metricsSince(metrics, start) : {}) }); steps.push({ score: verdict.passed ? verdict.score : 0, ...stepMetrics(metrics, start) });
} }
const score = steps.reduce((sum, step) => sum + step.score, 0) / item.steps.length; const score = steps.reduce((sum, step) => sum + step.score, 0) / item.steps.length;
const passed = steps.length === item.steps.length && steps.every(step => step.score >= 0.999); const passed = steps.length === item.steps.length && steps.every(step => step.score >= 0.999);
return { id: item.id, arm, repeat, passed, score, selectedIds: [...new Set(selectedIds)], steps, return { id: item.id, arm, repeat, passed, score, selectedIds: [...new Set(selectedIds)], steps,
failure: passed ? null : 'hidden-check' }; failure: passed ? null : 'hidden-check' };
}
function outcomeTrial(item, arm, repeat, trial) {
try {
return item.steps ? steppedTrial(item, arm, repeat, trial) : singleStepTrial(item, arm, repeat, trial);
} catch (error) { } catch (error) {
if (harvest) harvest(arm, item.id, repeat, environment); if (trial.harvest) trial.harvest(arm, item.id, repeat, trial.environment);
return { id: item.id, arm, repeat, passed: false, score: 0, selectedIds: [], failure: failureCode(error) }; return { id: item.id, arm, repeat, passed: false, score: 0, selectedIds: [], failure: failureCode(error) };
} }
} }
@@ -753,10 +791,10 @@ function createHarvester(artifactDir, envs) {
}; };
} }
function runEvaluation({ repoRoot = DEFAULT_REPO_ROOT, corpus = loadCorpus(), registration, // Validate the caller's bounds and opt-ins and settle on the provider family.
repeats = 1, provider, family, allowRealProvider = false, allowCredentialedTools = false, function resolveEvaluationFamily(options) {
executable, model, effort, authHome, environments, const { provider, allowRealProvider, maxCalls, deadlineMs, callTimeoutMs, registration,
arms = undefined, artifactDir = null, maxCalls = 300, deadlineMs = 3600000, callTimeoutMs = 300000 } = {}) { family, executable, effort, allowCredentialedTools } = options;
if (!provider && !allowRealProvider) throw new Error('Evaluation requires an injected provider or explicit opt-in'); if (!provider && !allowRealProvider) throw new Error('Evaluation requires an injected provider or explicit opt-in');
if (!bounded(maxCalls, 1, 2000) || !bounded(deadlineMs, 1, 8 * 3600000) if (!bounded(maxCalls, 1, 2000) || !bounded(deadlineMs, 1, 8 * 3600000)
|| !bounded(callTimeoutMs, 1, 600000)) throw new Error('Invalid call or deadline bound'); || !bounded(callTimeoutMs, 1, 600000)) throw new Error('Invalid call or deadline bound');
@@ -766,40 +804,65 @@ function runEvaluation({ repoRoot = DEFAULT_REPO_ROOT, corpus = loadCorpus(), re
if (!provider && resolvedFamily === 'claude' && !allowCredentialedTools) { if (!provider && resolvedFamily === 'claude' && !allowCredentialedTools) {
throw new Error('Claude task tools can read provider credentials; explicit credentialed-tool opt-in is required'); throw new Error('Claude task tools can read provider credentials; explicit credentialed-tool opt-in is required');
} }
return resolvedFamily;
}
function pinEvaluation(options, resolvedFamily) {
const { repoRoot, corpus, repeats, model, executable, effort, arms, provider, registration } = options;
const pin = preregister({ repoRoot, corpus, repeats, model, executable, effort, arms }); const pin = preregister({ repoRoot, corpus, repeats, model, executable, effort, arms });
if (!provider && resolvedFamily === 'codex' && pin.arms.includes('ecc-legacy')) { if (!provider && resolvedFamily === 'codex' && pin.arms.includes('ecc-legacy')) {
throw new Error('Codex real evaluation requires --arms without ecc-legacy; the pinned legacy skills arm is Claude-only'); throw new Error('Codex real evaluation requires --arms without ecc-legacy; the pinned legacy skills arm is Claude-only');
} }
if (registration && !isDeepStrictEqual(registration, pin)) throw new Error('Registration pin mismatch'); if (registration && !isDeepStrictEqual(registration, pin)) throw new Error('Registration pin mismatch');
const injected = Boolean(provider); return pin;
const liveProvider = provider || (resolvedFamily === 'claude' }
function createLiveProvider(resolvedFamily, options) {
const { allowRealProvider, allowCredentialedTools, executable, model, effort, authHome, artifactDir } = options;
return resolvedFamily === 'claude'
? createClaudeProvider({ allowRealProvider, allowCredentialedTools, executable, model, ? createClaudeProvider({ allowRealProvider, allowCredentialedTools, executable, model,
persistSessions: Boolean(artifactDir) }) persistSessions: Boolean(artifactDir) })
: createCodexProvider({ allowRealProvider, executable, model, effort, authHome })); : createCodexProvider({ allowRealProvider, executable, model, effort, authHome });
const state = { calls: 0, metrics: [], maxCalls, callTimeoutMs, family: resolvedFamily, }
function createEvaluationState(options, plan) {
const { repoRoot, corpus, maxCalls, callTimeoutMs, deadlineMs } = options;
const { pin, resolvedFamily, liveProvider } = plan;
return { calls: 0, metrics: [], maxCalls, callTimeoutMs, family: resolvedFamily,
deadline: Date.now() + deadlineMs, provider: liveProvider, deadline: Date.now() + deadlineMs, provider: liveProvider,
assertCurrent() { assertCurrent() {
if (digestObject(corpus) !== pin.corpusDigest || sourceSnapshot(repoRoot).sourceDigest !== pin.sourceDigest) fail('source-drift'); if (digestObject(corpus) !== pin.corpusDigest || sourceSnapshot(repoRoot).sourceDigest !== pin.sourceDigest) fail('source-drift');
} }; } };
const temp = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-ai-eval-'))); }
const selection = [];
const outcomes = []; function planEvaluation(options) {
let installs = null; const resolvedFamily = resolveEvaluationFamily(options);
let harvester = null; const pin = pinEvaluation(options, resolvedFamily);
try { const injected = Boolean(options.provider);
const installRoot = path.join(temp, 'installs'); const liveProvider = options.provider || createLiveProvider(resolvedFamily, options);
fs.mkdirSync(installRoot, { mode: 0o700 }); const plan = { resolvedFamily, pin, injected, liveProvider };
const envs = environments || (injected ? syntheticEnvironments(installRoot) plan.state = createEvaluationState(options, plan);
: resolvedFamily === 'claude' return plan;
? prepareClaudeEnvironments({ repoRoot, executable, root: installRoot, }
function prepareEvaluationEnvironments(options, plan, installRoot) {
const { repoRoot, executable } = options;
const { injected, resolvedFamily, pin } = plan;
if (injected) return syntheticEnvironments(installRoot);
if (resolvedFamily !== 'claude') return prepareEnvironments({ repoRoot, executable, root: installRoot });
return prepareClaudeEnvironments({ repoRoot, executable, root: installRoot,
...(pin.arms.includes('ecc-legacy') ...(pin.arms.includes('ecc-legacy')
? { legacySource: exportLegacySource({ repoRoot, destination: path.join(installRoot, 'legacy-source') }) } ? { legacySource: exportLegacySource({ repoRoot, destination: path.join(installRoot, 'legacy-source') }) }
: {}) }) : {}) });
: prepareEnvironments({ repoRoot, executable, root: installRoot })); }
installs = Object.fromEntries(Object.entries(envs).map(([name, env]) => [name,
function describeInstalls(envs) {
return Object.fromEntries(Object.entries(envs).map(([name, env]) => [name,
{ profileId: env.profileId, skills: env.skills, ...(env.sourceSha ? { sourceSha: env.sourceSha } : {}) }])); { profileId: env.profileId, skills: env.skills, ...(env.sourceSha ? { sourceSha: env.sourceSha } : {}) }]));
harvester = artifactDir && resolvedFamily === 'claude' && !injected ? createHarvester(artifactDir, envs) : null; }
const harvest = harvester ? (arm, id, repeat, env) => harvester.record(arm, id, repeat, env) : null;
function runSelectionProbes(run, selection) {
const { corpus, repoRoot, state, temp, envs, resolvedFamily } = run;
for (const item of corpus.selection) { for (const item of corpus.selection) {
const cwd = path.join(temp, `${item.id}--selection`); const cwd = path.join(temp, `${item.id}--selection`);
fs.mkdirSync(cwd); fs.mkdirSync(cwd);
@@ -807,6 +870,10 @@ function runEvaluation({ repoRoot = DEFAULT_REPO_ROOT, corpus = loadCorpus(), re
selection.push({ ...selectionProbe(item, repoRoot, executeAdapter(state, cwd, envs.lean), envs.lean, resolvedFamily), selection.push({ ...selectionProbe(item, repoRoot, executeAdapter(state, cwd, envs.lean), envs.lean, resolvedFamily),
...metricsSince(state.metrics, start) }); ...metricsSince(state.metrics, start) });
} }
}
function runOutcomeTrials(run, outcomes) {
const { corpus, pin, repoRoot, state, temp, envs, resolvedFamily, harvest } = run;
for (const scheduled of pin.order) { for (const scheduled of pin.order) {
const item = corpus.tasks.find(c => c.id === scheduled.id); const item = corpus.tasks.find(c => c.id === scheduled.id);
for (const arm of scheduled.arms) { for (const arm of scheduled.arms) {
@@ -815,26 +882,59 @@ function runEvaluation({ repoRoot = DEFAULT_REPO_ROOT, corpus = loadCorpus(), re
fs.mkdirSync(cwd); fs.mkdirSync(cwd);
writeWorkspace(cwd, item.files); writeWorkspace(cwd, item.files);
const start = state.metrics.length; const start = state.metrics.length;
outcomes.push({ ...outcomeTrial(item, arm, scheduled.repeat, repoRoot, const trial = { repoRoot, execute: executeAdapter(state, cwd, environment), cwd, environment,
executeAdapter(state, cwd, environment), cwd, environment, resolvedFamily, harvest, state.metrics), target: resolvedFamily, harvest, metrics: state.metrics };
...metricsSince(state.metrics, start) }); outcomes.push({ ...outcomeTrial(item, arm, scheduled.repeat, trial), ...metricsSince(state.metrics, start) });
fs.rmSync(cwd, { recursive: true, force: true }); fs.rmSync(cwd, { recursive: true, force: true });
} }
} }
}
// Prepare installs in a private temp root, run every selection probe and task
// trial, and always remove the root afterwards.
function executeEvaluation(options, plan) {
const { repoRoot, corpus, environments, artifactDir } = options;
const { resolvedFamily, pin, injected, state } = plan;
const temp = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-ai-eval-')));
const selection = [];
const outcomes = [];
let installs = null;
let harvester = null;
try {
const installRoot = path.join(temp, 'installs');
fs.mkdirSync(installRoot, { mode: 0o700 });
const envs = environments || prepareEvaluationEnvironments(options, plan, installRoot);
installs = describeInstalls(envs);
harvester = artifactDir && resolvedFamily === 'claude' && !injected ? createHarvester(artifactDir, envs) : null;
const harvest = harvester ? (arm, id, repeat, env) => harvester.record(arm, id, repeat, env) : null;
const run = { corpus, pin, repoRoot, state, temp, envs, resolvedFamily, harvest };
runSelectionProbes(run, selection);
runOutcomeTrials(run, outcomes);
if (harvester) harvester.writeIndex(); if (harvester) harvester.writeIndex();
} finally { if (harvester) harvester.writeIndex(); fs.rmSync(temp, { recursive: true, force: true }); } } finally { if (harvester) harvester.writeIndex(); fs.rmSync(temp, { recursive: true, force: true }); }
return { selection, outcomes, installs, harvester };
}
function summarizeSelection(selection) {
const selectionSuccesses = selection.filter(row => row.passed).length;
return { n: selection.length, successes: selectionSuccesses,
categories: [...new Set(selection.map(row => row.category))].map(category => ({ category,
n: selection.filter(row => row.category === category).length,
successes: selection.filter(row => row.category === category && row.passed).length })),
interval: wilson(selectionSuccesses, selection.length), method: 'wilson-95-descriptive-purposive-sample' };
}
function buildEvaluationReport(options, plan, run) {
const { maxCalls, deadlineMs, callTimeoutMs } = options;
const { pin, injected, liveProvider, state, resolvedFamily } = plan;
const { selection, outcomes, installs, harvester } = run;
const summary = summarize(outcomes, pin.arms); const summary = summarize(outcomes, pin.arms);
const insufficient = summary.distinctTasks < pin.minimumDistinctTasks || selection.length < pin.minimumDistinctTasks; const insufficient = summary.distinctTasks < pin.minimumDistinctTasks || selection.length < pin.minimumDistinctTasks;
const selectionSuccesses = selection.filter(row => row.passed).length;
return { schemaVersion: 'ecc.context-eval.v2', registration: pin, return { schemaVersion: 'ecc.context-eval.v2', registration: pin,
evidence: injected ? 'injected-provider' : resolvedFamily === 'claude' ? 'claude-json' : 'codex-jsonl', installs, evidence: injected ? 'injected-provider' : resolvedFamily === 'claude' ? 'claude-json' : 'codex-jsonl', installs,
authentication: injected ? 'injected' : liveProvider.authentication, credentialsRetained: false, authentication: injected ? 'injected' : liveProvider.authentication, credentialsRetained: false,
calls: state.calls, bounds: { maxCalls, deadlineMs, callTimeoutMs }, selection, outcomes, summary, calls: state.calls, bounds: { maxCalls, deadlineMs, callTimeoutMs }, selection, outcomes, summary,
selectionSummary: { n: selection.length, successes: selectionSuccesses, selectionSummary: summarizeSelection(selection),
categories: [...new Set(selection.map(row => row.category))].map(category => ({ category,
n: selection.filter(row => row.category === category).length,
successes: selection.filter(row => row.category === category && row.passed).length })),
interval: wilson(selectionSuccesses, selection.length), method: 'wilson-95-descriptive-purposive-sample' },
gate: { status: insufficient ? 'insufficient-sample' : injected ? 'synthetic-only' : 'review-required', gate: { status: insufficient ? 'insufficient-sample' : injected ? 'synthetic-only' : 'review-required',
nonInferioritySupported: !insufficient && !injected && summary.pairs.every(p => p.interval[0] >= -pin.nonInferiorityMargin), nonInferioritySupported: !insufficient && !injected && summary.pairs.every(p => p.interval[0] >= -pin.nonInferiorityMargin),
releaseApproved: false }, nativeInvocation: 'unobserved', releaseApproved: false }, nativeInvocation: 'unobserved',
@@ -842,6 +942,18 @@ function runEvaluation({ repoRoot = DEFAULT_REPO_ROOT, corpus = loadCorpus(), re
artifactRetention: harvester ? 'session-jsonl-per-task-trial' : 'none', ...metricsSince(state.metrics, 0) }; artifactRetention: harvester ? 'session-jsonl-per-task-trial' : 'none', ...metricsSince(state.metrics, 0) };
} }
function runEvaluation({ repoRoot = DEFAULT_REPO_ROOT, corpus = loadCorpus(), registration,
repeats = 1, provider, family, allowRealProvider = false, allowCredentialedTools = false,
executable, model, effort, authHome, environments,
arms = undefined, artifactDir = null, maxCalls = 300, deadlineMs = 3600000, callTimeoutMs = 300000 } = {}) {
const options = { repoRoot, corpus, registration, repeats, provider, family, allowRealProvider,
allowCredentialedTools, executable, model, effort, authHome, environments, arms, artifactDir,
maxCalls, deadlineMs, callTimeoutMs };
const plan = planEvaluation(options);
const run = executeEvaluation(options, plan);
return buildEvaluationReport(options, plan, run);
}
module.exports = { loadCorpus, preregister, runEvaluation, parseCodexJsonl, parseClaudeJson, summarize, wilson, module.exports = { loadCorpus, preregister, runEvaluation, parseCodexJsonl, parseClaudeJson, summarize, wilson,
runCheck, runScoredCheck, createAuthLease, createCodexProvider, createClaudeProvider, prepareEnvironments, runCheck, runScoredCheck, createAuthLease, createCodexProvider, createClaudeProvider, prepareEnvironments,
prepareClaudeEnvironments, exportLegacySource, providerFamily, resolveFamily, readClaudeKeychainToken }; prepareClaudeEnvironments, exportLegacySource, providerFamily, resolveFamily, readClaudeKeychainToken };