mirror of
https://github.com/affaan-m/ECC.git
synced 2026-10-26 18:35:05 +00:00
fix(adapters): bound parsing and publish review artifacts safely
This commit is contained in:
10 files changed
+349
-104
No files matched your search
@@ -37,3 +37,9 @@ node docker/gateguard-effectiveness/run-intent.js --out gg-intent --model <model
|
||||
node docker/gateguard-effectiveness/run-intent.js --out gg-intent --summarize
|
||||
node docker/gateguard-effectiveness/run-intent.js --out gg-intent --verify
|
||||
```
|
||||
|
||||
Runs sharing an output directory are serialized by `.run.lock`; concurrent workers
|
||||
must use separate `--out` directories. Normal completion and errors release the
|
||||
lock. After a killed process, confirm that its worker has stopped before removing
|
||||
the stale lock and resuming. Metadata uses private exclusive temporary creation
|
||||
and atomic rename so a destination symlink cannot redirect metadata writes.
|
||||
@@ -2,6 +2,7 @@
|
||||
'use strict';
|
||||
|
||||
const fs = require('fs');
|
||||
const crypto = require('crypto');
|
||||
const os = require('os');
|
||||
const path = require('path');
|
||||
const { spawnSync } = require('child_process');
|
||||
@@ -71,9 +72,55 @@ function parseArgs(argv) {
|
||||
return options;
|
||||
}
|
||||
|
||||
function readOptionalText(file) {
|
||||
let fd;
|
||||
try {
|
||||
fd = fs.openSync(file, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW || 0) | (fs.constants.O_NONBLOCK || 0));
|
||||
} catch (error) {
|
||||
if (error.code === 'ENOENT') return null;
|
||||
throw error;
|
||||
}
|
||||
try {
|
||||
if (!fs.fstatSync(fd).isFile()) throw new Error('Resume metadata/results must be regular files');
|
||||
return fs.readFileSync(fd, 'utf8');
|
||||
} finally { fs.closeSync(fd); }
|
||||
}
|
||||
|
||||
function writeMetadata(file, meta) {
|
||||
const temporary = path.join(path.dirname(file), `.meta-${crypto.randomUUID()}.tmp`);
|
||||
const fd = fs.openSync(temporary, 'wx', 0o600);
|
||||
let published = false;
|
||||
try {
|
||||
try { fs.writeFileSync(fd, `${JSON.stringify(meta, null, 2)}\n`); }
|
||||
finally { fs.closeSync(fd); }
|
||||
// Rename replaces the destination entry rather than following a raced symlink.
|
||||
fs.renameSync(temporary, file);
|
||||
published = true;
|
||||
} finally {
|
||||
if (!published) fs.unlinkSync(temporary);
|
||||
}
|
||||
}
|
||||
|
||||
function withOutputLock(out, work) {
|
||||
const lock = path.join(out, '.run.lock');
|
||||
let fd;
|
||||
try { fd = fs.openSync(lock, 'wx', 0o600); }
|
||||
catch (error) {
|
||||
if (error.code === 'EEXIST') throw new Error('Output is locked; concurrent workers must use separate --out directories. Remove .run.lock only after confirming its worker has stopped.');
|
||||
throw error;
|
||||
}
|
||||
try {
|
||||
fs.writeFileSync(fd, `${process.pid}\n`);
|
||||
return work();
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
fs.unlinkSync(lock);
|
||||
}
|
||||
}
|
||||
|
||||
function readResults(file) {
|
||||
if (!fs.existsSync(file)) return [];
|
||||
return fs.readFileSync(file, 'utf8').split('\n').filter(Boolean).map(line => JSON.parse(line));
|
||||
const text = readOptionalText(file);
|
||||
return text === null ? [] : text.split('\n').filter(Boolean).map(line => JSON.parse(line));
|
||||
}
|
||||
|
||||
function writeSummary(options, rows, meta) {
|
||||
@@ -91,71 +138,75 @@ function main() {
|
||||
return;
|
||||
}
|
||||
fs.mkdirSync(path.join(options.out, 'transcripts'), { recursive: true });
|
||||
const resultsFile = path.join(options.out, 'results.jsonl');
|
||||
const metaFile = path.join(options.out, 'meta.json');
|
||||
const scenarios = loadScenarios(undefined, options.scenarios);
|
||||
const trees = { candidate: resolveRef(REPO_ROOT, options.candidateRef) };
|
||||
if (options.arms.includes('main')) trees.main = resolveRef(REPO_ROOT, options.mainRef);
|
||||
const meta = { model: options.model, sha: trees.candidate, mainSha: trees.main || null, reps: options.reps, arms: options.arms, scenarios: scenarios.map(s => s.id), maxTurns: options.maxTurns };
|
||||
if (fs.existsSync(metaFile)) {
|
||||
const previous = JSON.parse(fs.readFileSync(metaFile, 'utf8'));
|
||||
for (const key of ['model', 'sha', 'mainSha', 'maxTurns']) {
|
||||
if (!options.summarizeOnly && !options.dryRun && previous[key] !== meta[key]) throw new Error(`--out was started with ${key}=${previous[key]}; use a new --out`);
|
||||
return withOutputLock(options.out, () => {
|
||||
const resultsFile = path.join(options.out, 'results.jsonl');
|
||||
const metaFile = path.join(options.out, 'meta.json');
|
||||
const scenarios = loadScenarios(undefined, options.scenarios);
|
||||
const trees = { candidate: resolveRef(REPO_ROOT, options.candidateRef) };
|
||||
if (options.arms.includes('main')) trees.main = resolveRef(REPO_ROOT, options.mainRef);
|
||||
const meta = { model: options.model, sha: trees.candidate, mainSha: trees.main || null, reps: options.reps, arms: options.arms, scenarios: scenarios.map(s => s.id), maxTurns: options.maxTurns };
|
||||
const metadataText = readOptionalText(metaFile);
|
||||
const previous = metadataText === null ? null : JSON.parse(metadataText);
|
||||
if (previous) {
|
||||
for (const key of ['model', 'sha', 'mainSha', 'maxTurns']) {
|
||||
if (!options.summarizeOnly && !options.dryRun && previous[key] !== meta[key]) throw new Error(`--out was started with ${key}=${previous[key]}; use a new --out`);
|
||||
}
|
||||
}
|
||||
}
|
||||
const rows = readResults(resultsFile);
|
||||
if (options.summarizeOnly) {
|
||||
process.stdout.write(renderMarkdown(writeSummary(options, rows, JSON.parse(fs.readFileSync(metaFile, 'utf8'))), meta));
|
||||
return;
|
||||
}
|
||||
|
||||
const workRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'gateguard-effectiveness-'));
|
||||
try {
|
||||
const treeRoots = {};
|
||||
for (const [name, sha] of Object.entries(trees)) treeRoots[name] = materializeTree(REPO_ROOT, sha, path.join(workRoot, 'trees', name));
|
||||
const armSettingsByName = Object.fromEntries(options.arms.map(arm => [arm, armSettings(ARMS[arm].gate ? treeRoots[ARMS[arm].tree] : null)]));
|
||||
const done = new Set(rows.map(row => row.key));
|
||||
const trials = schedule(scenarios, options.arms, options.reps, options.seed).filter(trial => !done.has(trial.key));
|
||||
process.stderr.write(`${trials.length} trials to run (${done.size} already recorded)\n`);
|
||||
if (options.dryRun) {
|
||||
for (const trial of trials) process.stdout.write(`${trial.key}\n`);
|
||||
const rows = readResults(resultsFile);
|
||||
if (options.summarizeOnly) {
|
||||
if (!previous) throw new Error('Missing resume metadata');
|
||||
process.stdout.write(renderMarkdown(writeSummary(options, rows, previous), meta));
|
||||
return;
|
||||
}
|
||||
fs.writeFileSync(metaFile, `${JSON.stringify(meta, null, 2)}\n`);
|
||||
let consecutiveErrors = 0;
|
||||
for (const [index, trial] of trials.entries()) {
|
||||
let transcript = '';
|
||||
const execute = (file, args, spawnOptions) => {
|
||||
const result = spawnSync(file, args, spawnOptions);
|
||||
transcript = result.stdout || '';
|
||||
return result;
|
||||
};
|
||||
const row = runTrial(trial, {
|
||||
workRoot,
|
||||
armSettingsByName,
|
||||
executable: options.claude,
|
||||
model: options.model,
|
||||
maxTurns: options.maxTurns,
|
||||
timeoutMs: options.timeoutMin * 60000,
|
||||
execute
|
||||
});
|
||||
fs.writeFileSync(path.join(options.out, 'transcripts', `${trial.key.replace(/\//g, '__')}.jsonl`), transcript);
|
||||
fs.rmSync(path.join(workRoot, trial.key.replace(/\//g, '__')), { recursive: true, force: true });
|
||||
if (row.providerError && /authenticat|log ?in|oauth/i.test(row.providerMessage || '')) {
|
||||
throw new Error(`${trial.key}: Claude is not signed in (${row.providerMessage}); sign in with the claude CLI and rerun with the same --out`);
|
||||
|
||||
const workRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'gateguard-effectiveness-'));
|
||||
try {
|
||||
const treeRoots = {};
|
||||
for (const [name, sha] of Object.entries(trees)) treeRoots[name] = materializeTree(REPO_ROOT, sha, path.join(workRoot, 'trees', name));
|
||||
const armSettingsByName = Object.fromEntries(options.arms.map(arm => [arm, armSettings(ARMS[arm].gate ? treeRoots[ARMS[arm].tree] : null)]));
|
||||
const done = new Set(rows.map(row => row.key));
|
||||
const trials = schedule(scenarios, options.arms, options.reps, options.seed).filter(trial => !done.has(trial.key));
|
||||
process.stderr.write(`${trials.length} trials to run (${done.size} already recorded)\n`);
|
||||
if (options.dryRun) {
|
||||
for (const trial of trials) process.stdout.write(`${trial.key}\n`);
|
||||
return;
|
||||
}
|
||||
if (!row.hookObserved) throw new Error(`${trial.key}: the gate did not run although the agent edited files; check the hook setup before spending more sessions`);
|
||||
fs.appendFileSync(resultsFile, `${JSON.stringify(row)}\n`);
|
||||
rows.push(row);
|
||||
writeSummary(options, rows, meta);
|
||||
process.stderr.write(`[${index + 1}/${trials.length}] ${trial.key}: ${row.passed ? 'pass' : `score ${row.score}`}, ${row.gateDenials} denials, ${row.turns} turns${row.providerError ? ', provider error' : ''}\n`);
|
||||
consecutiveErrors = row.providerError || row.timedOut ? consecutiveErrors + 1 : 0;
|
||||
if (consecutiveErrors >= 3) throw new Error('three trials in a row ended in a provider error or timeout; stopping');
|
||||
writeMetadata(metaFile, meta);
|
||||
let consecutiveErrors = 0;
|
||||
for (const [index, trial] of trials.entries()) {
|
||||
let transcript = '';
|
||||
const execute = (file, args, spawnOptions) => {
|
||||
const result = spawnSync(file, args, spawnOptions);
|
||||
transcript = result.stdout || '';
|
||||
return result;
|
||||
};
|
||||
const row = runTrial(trial, {
|
||||
workRoot,
|
||||
armSettingsByName,
|
||||
executable: options.claude,
|
||||
model: options.model,
|
||||
maxTurns: options.maxTurns,
|
||||
timeoutMs: options.timeoutMin * 60000,
|
||||
execute
|
||||
});
|
||||
fs.writeFileSync(path.join(options.out, 'transcripts', `${trial.key.replace(/\//g, '__')}.jsonl`), transcript);
|
||||
fs.rmSync(path.join(workRoot, trial.key.replace(/\//g, '__')), { recursive: true, force: true });
|
||||
if (row.providerError && /authenticat|log ?in|oauth/i.test(row.providerMessage || '')) {
|
||||
throw new Error(`${trial.key}: Claude is not signed in (${row.providerMessage}); sign in with the claude CLI and rerun with the same --out`);
|
||||
}
|
||||
if (!row.hookObserved) throw new Error(`${trial.key}: the gate did not run although the agent edited files; check the hook setup before spending more sessions`);
|
||||
fs.appendFileSync(resultsFile, `${JSON.stringify(row)}\n`);
|
||||
rows.push(row);
|
||||
writeSummary(options, rows, meta);
|
||||
process.stderr.write(`[${index + 1}/${trials.length}] ${trial.key}: ${row.passed ? 'pass' : `score ${row.score}`}, ${row.gateDenials} denials, ${row.turns} turns${row.providerError ? ', provider error' : ''}\n`);
|
||||
consecutiveErrors = row.providerError || row.timedOut ? consecutiveErrors + 1 : 0;
|
||||
if (consecutiveErrors >= 3) throw new Error('three trials in a row ended in a provider error or timeout; stopping');
|
||||
}
|
||||
process.stdout.write(renderMarkdown(writeSummary(options, rows, meta), meta));
|
||||
} finally {
|
||||
fs.rmSync(workRoot, { recursive: true, force: true });
|
||||
}
|
||||
process.stdout.write(renderMarkdown(writeSummary(options, rows, meta), meta));
|
||||
} finally {
|
||||
fs.rmSync(workRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
@@ -167,4 +218,4 @@ if (require.main === module) {
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { parseArgs };
|
||||
module.exports = { parseArgs, readOptionalText, writeMetadata, withOutputLock };
|
||||
+35
-12
@@ -89,6 +89,26 @@ Options:
|
||||
`);
|
||||
}
|
||||
|
||||
function isIdenticalExistingAgent(target, markdown) {
|
||||
let fd;
|
||||
try {
|
||||
fd = fs.openSync(target, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW || 0) | (fs.constants.O_NONBLOCK || 0));
|
||||
} catch (error) {
|
||||
if (error.code === 'ENOENT') return false;
|
||||
throw error;
|
||||
}
|
||||
try {
|
||||
if (!fs.fstatSync(fd).isFile()
|
||||
|| (!fs.constants.O_NOFOLLOW && fs.lstatSync(target).isSymbolicLink())
|
||||
|| fs.readFileSync(fd, 'utf8') !== markdown) {
|
||||
throw new Error(`refusing to overwrite existing agent: ${target}`);
|
||||
}
|
||||
return true;
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
}
|
||||
|
||||
function main() {
|
||||
let args;
|
||||
try {
|
||||
@@ -135,6 +155,7 @@ function main() {
|
||||
|
||||
if (args.out) {
|
||||
const dir = path.resolve(args.out);
|
||||
const identicalOutputs = new Set();
|
||||
if (!args.dryRun) {
|
||||
if (fs.existsSync(dir) && fs.lstatSync(dir).isSymbolicLink()) {
|
||||
throw new Error('output directory must not be a symlink');
|
||||
@@ -143,14 +164,7 @@ function main() {
|
||||
// cannot be replaced simply because they share an ECC dispatch name.
|
||||
for (const r of results) {
|
||||
const target = path.join(dir, `${r.id}.md`);
|
||||
let stat;
|
||||
try { stat = fs.lstatSync(target); } catch (error) {
|
||||
if (error.code !== 'ENOENT') throw error;
|
||||
}
|
||||
if (stat && (!stat.isFile() || stat.isSymbolicLink()
|
||||
|| fs.readFileSync(target, 'utf8') !== r.markdown)) {
|
||||
throw new Error(`refusing to overwrite existing agent: ${target}`);
|
||||
}
|
||||
if (isIdenticalExistingAgent(target, r.markdown)) identicalOutputs.add(target);
|
||||
}
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
@@ -158,10 +172,19 @@ function main() {
|
||||
const target = path.join(dir, `${r.id}.md`);
|
||||
if (args.dryRun) {
|
||||
progress(`[dry-run] would write ${target}`);
|
||||
} else if (!fs.existsSync(target)) {
|
||||
// Exclusive creation prevents a file/symlink appearing after preflight
|
||||
// from being overwritten. Identical existing output stays untouched.
|
||||
fs.writeFileSync(target, r.markdown, { flag: 'wx' });
|
||||
} else if (identicalOutputs.has(target)) {
|
||||
// Existing output may have changed while the rest of the batch was checked.
|
||||
if (!isIdenticalExistingAgent(target, r.markdown)) {
|
||||
throw new Error(`existing agent disappeared after preflight: ${target}`);
|
||||
}
|
||||
} else {
|
||||
// Exclusive descriptor creation rejects a raced file or symlink.
|
||||
const fd = fs.openSync(target, 'wx', 0o600);
|
||||
try {
|
||||
fs.writeFileSync(fd, r.markdown);
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
progress(`wrote ${target}`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -133,9 +133,10 @@ function extractTopLevelList(yamlSource, key) {
|
||||
break;
|
||||
}
|
||||
|
||||
const match = line.match(/^\s*-\s+(.+?)\s*$/);
|
||||
if (match) {
|
||||
results.push(unquoteYamlScalar(match[1]));
|
||||
const trimmed = line.trim();
|
||||
if (trimmed.startsWith('-') && trimmed.length > 1 && /\s/.test(trimmed[1])) {
|
||||
const value = trimmed.slice(1).trim();
|
||||
if (value) results.push(unquoteYamlScalar(value));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -109,19 +109,22 @@ function formatToolLine(tools) {
|
||||
}
|
||||
|
||||
function adaptColor(line) {
|
||||
const match = line.match(/^(\s*)color\s*:\s*(.+?)\s*$/);
|
||||
if (!match) {
|
||||
const trimmed = line.trimStart();
|
||||
const colon = trimmed.indexOf(':');
|
||||
if (colon < 0 || trimmed.slice(0, colon).trimEnd() !== 'color') {
|
||||
return null;
|
||||
}
|
||||
|
||||
const value = match[2].replace(/^["']|["']$/g, '');
|
||||
const rawValue = trimmed.slice(colon + 1).trim();
|
||||
if (!rawValue) return null;
|
||||
const indent = line.slice(0, line.length - trimmed.length);
|
||||
const value = rawValue.replace(/^["']|["']$/g, '');
|
||||
if (VALID_COLORS.has(value)) {
|
||||
return { line, changed: false };
|
||||
}
|
||||
|
||||
const remapped = COLOR_MAP.get(value);
|
||||
if (remapped) {
|
||||
return { line: `${match[1]}color: ${remapped}`, changed: true };
|
||||
return { line: `${indent}color: ${remapped}`, changed: true };
|
||||
}
|
||||
|
||||
return { line: null, changed: true };
|
||||
|
||||
@@ -103,6 +103,12 @@ check('exports extractFrontmatter and inspectFrontmatter without running the CLI
|
||||
assert.strictEqual(inspected.values.name, 'demo');
|
||||
});
|
||||
|
||||
check('YAML list extraction handles long whitespace without ambiguous backtracking', () => {
|
||||
const { extractTopLevelList } = require('../../scripts/ci/validate-skill-host-compat');
|
||||
const tabs = '\t'.repeat(250000);
|
||||
assert.deepStrictEqual(extractTopLevelList(`skills:\n-${tabs}\n-${tabs}portable${tabs}\n - "second"\nnext: value\n - ignored`, 'skills'), ['portable', 'second']);
|
||||
});
|
||||
|
||||
check('happy: skill with name/description only, no Codex copy required, exits 0', () => {
|
||||
const root = createRoot();
|
||||
try {
|
||||
|
||||
@@ -16,7 +16,7 @@ const HOOK = path.join(ROOT, 'scripts', 'hooks', 'gateguard-fact-force.js');
|
||||
const lib = require(path.join(DIR, 'lib'));
|
||||
const arms = require(path.join(DIR, 'arms'));
|
||||
const { applyArm, armQuestionIds, PLACEBO_TEXT } = require(path.join(DIR, 'arm-patch'));
|
||||
const { parseArgs } = require(path.join(DIR, 'run'));
|
||||
const { parseArgs, writeMetadata, withOutputLock } = require(path.join(DIR, 'run'));
|
||||
|
||||
let passed = 0;
|
||||
let failed = 0;
|
||||
@@ -228,6 +228,41 @@ test('paired comparisons count discordant pairs and bound the difference', () =>
|
||||
assert.ok(markdown.includes('| off | 6 | -83%'), markdown);
|
||||
});
|
||||
|
||||
test('metadata publication replaces a symlink entry without touching its victim', () => {
|
||||
const out = tempDir('gg-eff-meta-');
|
||||
try {
|
||||
const victim = path.join(out, 'victim');
|
||||
const meta = path.join(out, 'meta.json');
|
||||
fs.writeFileSync(victim, 'VICTIM_BYTES');
|
||||
try { fs.symlinkSync(victim, meta); }
|
||||
catch (error) {
|
||||
if (process.platform !== 'win32') throw error;
|
||||
fs.writeFileSync(meta, 'old');
|
||||
}
|
||||
writeMetadata(meta, { model: 'fake', sha: 'fixture' });
|
||||
assert.strictEqual(fs.readFileSync(victim, 'utf8'), 'VICTIM_BYTES');
|
||||
const fd = fs.openSync(meta, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW || 0));
|
||||
try {
|
||||
assert.strictEqual(fs.fstatSync(fd).isFile(), true);
|
||||
assert.deepStrictEqual(JSON.parse(fs.readFileSync(fd, 'utf8')), { model: 'fake', sha: 'fixture' });
|
||||
} finally { fs.closeSync(fd); }
|
||||
assert.ok(!fs.readdirSync(out).some(file => file.endsWith('.tmp')));
|
||||
} finally { fs.rmSync(out, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('same-output workers are rejected and the lock is released on errors', () => {
|
||||
const out = tempDir('gg-eff-lock-');
|
||||
try {
|
||||
assert.throws(() => withOutputLock(out, () => {
|
||||
assert.throws(() => withOutputLock(out, () => assert.fail('second worker started')), /concurrent workers/);
|
||||
throw new Error('trial failure');
|
||||
}), /trial failure/);
|
||||
assert.deepStrictEqual(fs.readdirSync(out), []);
|
||||
assert.strictEqual(withOutputLock(out, () => 'resumed'), 'resumed');
|
||||
assert.deepStrictEqual(fs.readdirSync(out), []);
|
||||
} finally { fs.rmSync(out, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('arguments require an output folder, a model and the real-provider opt-in', () => {
|
||||
assert.throws(() => parseArgs([]), /--out is required/);
|
||||
assert.throws(() => parseArgs(['--out', 'x']), /--model is required/);
|
||||
@@ -283,6 +318,60 @@ if (gnuTarDir) {
|
||||
}
|
||||
|
||||
if (gitAvailable()) {
|
||||
test('actual fake-provider CLI resumes identity and resists a metadata publication symlink race', () => {
|
||||
const root = tempDir('gg-eff-cli-');
|
||||
const out = path.join(root, 'out');
|
||||
const victim = path.join(root, 'victim');
|
||||
const preload = path.join(root, 'preload.js');
|
||||
const scenario = scenarios.find(item => item.id === 'importers-date-shape');
|
||||
try {
|
||||
fs.writeFileSync(victim, 'VICTIM_BYTES');
|
||||
fs.writeFileSync(preload, `
|
||||
const cp = require('child_process');
|
||||
const originalSpawn = cp.spawnSync;
|
||||
cp.spawnSync = (file, args, options) => file === 'ecc-fake-claude'
|
||||
? originalSpawn(process.execPath, [${JSON.stringify(path.join(DIR, 'fake-claude.js'))}, ...args], options)
|
||||
: originalSpawn(file, args, options);
|
||||
if (process.env.ECC_TEST_META_RACE === '1') {
|
||||
const fs = require('fs');
|
||||
const rename = fs.renameSync;
|
||||
fs.renameSync = (from, to) => {
|
||||
if (String(to).endsWith('meta.json')) {
|
||||
fs.unlinkSync(to);
|
||||
fs.symlinkSync(${JSON.stringify(victim)}, to);
|
||||
}
|
||||
return rename(from, to);
|
||||
};
|
||||
}
|
||||
`);
|
||||
const args = ['--require', preload, path.join(DIR, 'run.js'), '--out', out, '--model', 'fake', '--claude', 'ecc-fake-claude', '--allow-real-provider', '--arms', 'off', '--reps', '1', '--scenarios', scenario.id];
|
||||
const env = { ...process.env, FAKE_CLAUDE_OVERLAY: path.join(scenario.root, 'reference') };
|
||||
const run = extraEnv => spawnSync(process.execPath, args, { env: { ...env, ...extraEnv }, encoding: 'utf8', timeout: 60000 });
|
||||
const first = run({});
|
||||
assert.strictEqual(first.status, 0, first.stderr);
|
||||
const metaPath = path.join(out, 'meta.json');
|
||||
const metadata = JSON.parse(fs.readFileSync(metaPath, 'utf8'));
|
||||
assert.strictEqual(metadata.model, 'fake');
|
||||
assert.strictEqual(metadata.sha, arms.resolveRef(ROOT, 'HEAD'));
|
||||
const results = fs.readFileSync(path.join(out, 'results.jsonl'), 'utf8');
|
||||
assert.strictEqual(results.trim().split('\n').length, 1);
|
||||
assert.strictEqual(JSON.parse(results.trim()).passed, true);
|
||||
const second = run({});
|
||||
assert.strictEqual(second.status, 0, second.stderr);
|
||||
assert.ok(second.stderr.includes('0 trials to run'));
|
||||
assert.strictEqual(fs.readFileSync(path.join(out, 'results.jsonl'), 'utf8'), results);
|
||||
assert.deepStrictEqual(JSON.parse(fs.readFileSync(metaPath, 'utf8')), metadata);
|
||||
if (process.platform !== 'win32') {
|
||||
const raced = run({ ECC_TEST_META_RACE: '1' });
|
||||
assert.strictEqual(raced.status, 0, raced.stderr);
|
||||
assert.strictEqual(fs.readFileSync(victim, 'utf8'), 'VICTIM_BYTES');
|
||||
assert.deepStrictEqual(JSON.parse(fs.readFileSync(metaPath, 'utf8')), metadata);
|
||||
assert.strictEqual(fs.readFileSync(path.join(out, 'results.jsonl'), 'utf8'), results);
|
||||
}
|
||||
assert.ok(!fs.readdirSync(out).includes('.run.lock'));
|
||||
} finally { fs.rmSync(root, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('a trial wires the arm hook, counts the denial and grades the result (test double for Claude)', () => {
|
||||
const work = tempDir('gg-eff-trial-');
|
||||
const saved = process.env.FAKE_CLAUDE_OVERLAY;
|
||||
|
||||
@@ -74,6 +74,53 @@ function main() {
|
||||
assert.deepStrictEqual(fs.readdirSync(dir), ["planner.md"]);
|
||||
}],
|
||||
|
||||
["--out rejects a file created at exclusive-open time without overwriting bytes", () => {
|
||||
const root = tmpdir();
|
||||
const dir = path.join(root, "out");
|
||||
const preload = path.join(root, "race.js");
|
||||
fs.mkdirSync(dir);
|
||||
fs.writeFileSync(preload, `
|
||||
const fs = require('fs');
|
||||
const open = fs.openSync;
|
||||
let injected = false;
|
||||
fs.openSync = function(file, flags, mode) {
|
||||
if (!injected && flags === 'wx' && String(file).endsWith('.md')) {
|
||||
injected = true;
|
||||
const fd = open(file, 'wx', 0o600);
|
||||
try { fs.writeFileSync(fd, 'RACED_OPERATOR_BYTES'); }
|
||||
finally { fs.closeSync(fd); }
|
||||
}
|
||||
return open(file, flags, mode);
|
||||
};
|
||||
`);
|
||||
try {
|
||||
assert.throws(() => execFileSync(NODE, ["--require", preload, CLI, "--out", dir], { encoding: "utf8", stdio: "pipe" }), /EEXIST/);
|
||||
const files = fs.readdirSync(dir);
|
||||
assert.strictEqual(files.length, 1);
|
||||
assert.strictEqual(fs.readFileSync(path.join(dir, files[0]), "utf8"), "RACED_OPERATOR_BYTES");
|
||||
} finally { fs.rmSync(root, { recursive: true, force: true }); }
|
||||
}],
|
||||
|
||||
["--out fails closed when existing output changes after batch preflight", () => {
|
||||
const root = tmpdir();
|
||||
const dir = path.join(root, "out");
|
||||
const preload = path.join(root, "change.js");
|
||||
runCli(["--out", dir]);
|
||||
const target = path.join(dir, fs.readdirSync(dir).sort()[0]);
|
||||
fs.writeFileSync(preload, `
|
||||
const fs = require('fs');
|
||||
const mkdir = fs.mkdirSync;
|
||||
fs.mkdirSync = function(file, options) {
|
||||
if (String(file) === ${JSON.stringify(dir)}) fs.writeFileSync(${JSON.stringify(target)}, 'CHANGED_AFTER_PREFLIGHT');
|
||||
return mkdir(file, options);
|
||||
};
|
||||
`);
|
||||
try {
|
||||
assert.throws(() => execFileSync(NODE, ["--require", preload, CLI, "--out", dir], { encoding: "utf8", stdio: "pipe" }), /refusing to overwrite existing agent/);
|
||||
assert.strictEqual(fs.readFileSync(target, "utf8"), "CHANGED_AFTER_PREFLIGHT");
|
||||
} finally { fs.rmSync(root, { recursive: true, force: true }); }
|
||||
}],
|
||||
|
||||
["--out repeats identical output without changing bytes", () => {
|
||||
const dir = tmpdir();
|
||||
runCli(["--out", dir]);
|
||||
|
||||
@@ -285,29 +285,34 @@ test('writeReport writes the whole content via the descriptor and cleans up when
|
||||
test('writeReport replaces a symlink and a world-readable file with a private regular file (real fs)', () => {
|
||||
const posix = process.platform !== 'win32';
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'astra-report-'));
|
||||
const victim = path.join(dir, 'victim.txt');
|
||||
fs.writeFileSync(victim, 'keep me');
|
||||
const linked = path.join(dir, 'linked.json');
|
||||
let canSymlink = true;
|
||||
try {
|
||||
fs.symlinkSync(victim, linked);
|
||||
} catch {
|
||||
canSymlink = false; // Windows without symlink privilege: replacement is still checked below
|
||||
}
|
||||
const shared = path.join(dir, 'shared.json');
|
||||
fs.writeFileSync(shared, 'old', { mode: 0o644 });
|
||||
const victim = path.join(dir, 'victim.txt');
|
||||
fs.writeFileSync(victim, 'keep me');
|
||||
const linked = path.join(dir, 'linked.json');
|
||||
let canSymlink = true;
|
||||
try {
|
||||
fs.symlinkSync(victim, linked);
|
||||
} catch {
|
||||
canSymlink = false; // Windows without symlink privilege: replacement is still checked below
|
||||
}
|
||||
const shared = path.join(dir, 'shared.json');
|
||||
fs.writeFileSync(shared, 'old', { mode: 0o644 });
|
||||
|
||||
if (canSymlink) cli.writeReport(linked, '{"a":1}');
|
||||
cli.writeReport(shared, '{"b":2}');
|
||||
if (canSymlink) cli.writeReport(linked, '{"a":1}');
|
||||
cli.writeReport(shared, '{"b":2}');
|
||||
|
||||
if (canSymlink) {
|
||||
assert.strictEqual(fs.readFileSync(victim, 'utf8'), 'keep me', 'symlink target untouched');
|
||||
assert.strictEqual(fs.lstatSync(linked).isSymbolicLink(), false, 'symlink replaced by a regular file');
|
||||
assert.strictEqual(fs.readFileSync(linked, 'utf8'), '{"a":1}');
|
||||
}
|
||||
assert.strictEqual(fs.readFileSync(shared, 'utf8'), '{"b":2}');
|
||||
if (posix) assert.strictEqual((fs.statSync(shared).mode & 0o777), 0o600, 'existing 0644 file becomes 0600');
|
||||
assert.deepStrictEqual(fs.readdirSync(dir).filter((name) => name.endsWith('.tmp')), [], 'no temp files left');
|
||||
if (canSymlink) {
|
||||
assert.strictEqual(fs.readFileSync(victim, 'utf8'), 'keep me', 'symlink target untouched');
|
||||
const fd = fs.openSync(linked, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW || 0));
|
||||
try {
|
||||
assert.strictEqual(fs.fstatSync(fd).isFile(), true, 'published report is regular');
|
||||
assert.strictEqual(fs.readFileSync(fd, 'utf8'), '{"a":1}');
|
||||
} finally { fs.closeSync(fd); }
|
||||
}
|
||||
assert.strictEqual(fs.readFileSync(shared, 'utf8'), '{"b":2}');
|
||||
if (posix) assert.strictEqual((fs.statSync(shared).mode & 0o777), 0o600, 'existing 0644 file becomes 0600');
|
||||
assert.deepStrictEqual(fs.readdirSync(dir).filter((name) => name.endsWith('.tmp')), [], 'no temp files left');
|
||||
} finally { fs.rmSync(dir, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('CLI exits 2 with usage on a bad argument', () => {
|
||||
|
||||
@@ -203,6 +203,20 @@ function runTests() {
|
||||
}
|
||||
})) passed++; else failed++;
|
||||
|
||||
if (test('adapts color with long whitespace through the actual CLI', () => {
|
||||
const tempDir = createTempDir();
|
||||
try {
|
||||
const tabs = '\t'.repeat(250000);
|
||||
writeAgent(tempDir, 'color.md', ['---', 'name: color', 'tools: Read', `color:${tabs}teal${tabs}`, '---', 'BODY'].join('\n'));
|
||||
const result = run([tempDir]);
|
||||
assert.strictEqual(result.code, 0, result.stderr);
|
||||
assert.ok(readAgent(tempDir, 'color.md').includes('color: cyan'));
|
||||
const before = readAgent(tempDir, 'color.md');
|
||||
assert.strictEqual(run([tempDir]).code, 0);
|
||||
assert.strictEqual(readAgent(tempDir, 'color.md'), before);
|
||||
} finally { cleanupTempDir(tempDir); }
|
||||
})) passed++; else failed++;
|
||||
|
||||
if (test('leaves an already-compatible agent untouched', () => {
|
||||
const tempDir = createTempDir();
|
||||
const agentsDir = path.join(tempDir, '.qwen', 'agents');
|
||||
|
||||
Reference in new issue
Block a user