mirror of
https://github.com/affaan-m/ECC.git
synced 2026-10-26 18:35:05 +00:00
Affaan Mustafa
9471ef4724
The runner executed 422 files one process at a time, which is most of the 12 to 40 minutes a CI lane spends in "Run tests". ECC_TEST_JOBS above 1 now runs files through a pool of that many processes; output is still buffered and printed per file, so logs read the same. The default stays 1 and that path is the original synchronous spawnSync loop, so the VM-driven runner test and the piped-output tests are unchanged. ECC_TEST_ONLY is the mirror of ECC_TEST_SKIP: a comma-separated list of path substrings, and only matching files run. CI uses it to give the package-manager lanes just the suites that read the package-manager variable instead of the whole suite. Two real-process tests cover the pool (grouped output, counted failure, summary last) and the ONLY filter.
285 lines
14 KiB
JavaScript
285 lines
14 KiB
JavaScript
'use strict';
|
|
|
|
const assert = require('assert');
|
|
const fs = require('fs');
|
|
const os = require('os');
|
|
const path = require('path');
|
|
const vm = require('vm');
|
|
const { spawnSync } = require('child_process');
|
|
|
|
const source = fs.readFileSync(path.join(__dirname, '..', 'run-all.js'), 'utf8');
|
|
|
|
function run(result, filename = 'sample.test.js', actions = true) {
|
|
const logs = [];
|
|
const exit = {};
|
|
let status;
|
|
let spawns = 0;
|
|
const fakeProcess = {
|
|
env: actions ? { GITHUB_ACTIONS: 'true' } : {},
|
|
exit(code) { status = code; throw exit; },
|
|
set exitCode(code) { status = code; },
|
|
};
|
|
const fakeFs = {
|
|
readdirSync: () => [{
|
|
name: filename,
|
|
isDirectory: () => false,
|
|
isFile: () => true,
|
|
}],
|
|
existsSync: () => true,
|
|
};
|
|
try {
|
|
vm.runInNewContext(source, {
|
|
__dirname: path.resolve('/virtual/tests'),
|
|
process: fakeProcess,
|
|
console: { log: (...args) => logs.push(args.join(' ')) },
|
|
require(name) {
|
|
if (name === 'fs') return fakeFs;
|
|
if (name === 'path') return path;
|
|
if (name === 'child_process') return {
|
|
spawnSync() { spawns += 1; return result; },
|
|
};
|
|
throw new Error(`Unexpected dependency: ${name}`);
|
|
},
|
|
});
|
|
} catch (error) {
|
|
if (error !== exit) throw error;
|
|
}
|
|
assert.strictEqual(spawns, 1);
|
|
return { status, logs, annotations: logs.filter(line => line.startsWith('::error ')) };
|
|
}
|
|
|
|
const tests = [
|
|
['named suite summaries retain their actual totals', () => {
|
|
const result = run({ status: 0, stdout: 'eval-harness package: Results: Passed: 4, Failed: 0' });
|
|
assert.strictEqual(result.status, 0);
|
|
assert.ok(result.logs.some(line => /Passed:\s+4\s/.test(line)));
|
|
}],
|
|
['named suite failure summaries cannot be lost with a successful exit', () => {
|
|
const result = run({ status: 0, stdout: 'capsule: Results: Passed: 3, Failed: 1' });
|
|
assert.strictEqual(result.status, 1);
|
|
assert.match(result.annotations[0], /reported 1 failed/);
|
|
}],
|
|
['outer summary wins over a nested runner failure summary', () => {
|
|
const result = run({ status: 0, stdout: 'Passed: 0, Failed: 1\nPASS verifies nested failure\nResults: Passed: 4, Failed: 0' });
|
|
assert.strictEqual(result.status, 0);
|
|
assert.ok(result.logs.some(line => /Passed:\s+4\s/.test(line)));
|
|
}],
|
|
['outer failure cannot be hidden by an earlier nested success', () => {
|
|
const result = run({ status: 0, stdout: 'Passed: 99, Failed: 0\nPassed: 2\nFailed: 1' });
|
|
assert.strictEqual(result.status, 1);
|
|
assert.ok(result.logs.some(line => /Passed:\s+2\s/.test(line)));
|
|
assert.match(result.annotations[0], /reported 1 failed/);
|
|
}],
|
|
['diagnostic mentions of counts are not owned summaries', () => {
|
|
const result = run({ status: 0, stdout: 'Assertion fixture: Passed: 99, Failed: 7\nPassed: 3\nFailed: 0', stderr: 'Error fixture expected "Failed: 8"' });
|
|
assert.strictEqual(result.status, 0);
|
|
assert.ok(result.logs.some(line => /Passed:\s+3\s/.test(line)));
|
|
}],
|
|
['stdout summary owns counts even when stderr quotes a nested summary', () => {
|
|
const result = run({ status: 0, stdout: 'Passed: 3, Failed: 0', stderr: 'Passed: 0, Failed: 8' });
|
|
assert.strictEqual(result.status, 0);
|
|
assert.ok(result.logs.some(line => /Passed:\s+3\s/.test(line)));
|
|
}],
|
|
['nonzero exit overrides a zero-failure summary', () => {
|
|
const result = run({ status: 1, stdout: 'Passed: 2, Failed: 0', stderr: 'Error: late crash' });
|
|
assert.strictEqual(result.status, 1);
|
|
assert.strictEqual(result.annotations.length, 1);
|
|
assert.match(result.annotations[0], /file=tests\/sample.test.js/);
|
|
assert.match(result.annotations[0], /status 1.*Error: late crash/);
|
|
assert.ok(result.logs.includes('Error: late crash'));
|
|
}],
|
|
['startup errors always count as failures and annotate their cause', () => {
|
|
const result = run({ status: null, stdout: 'Failed: 0', error: new Error('spawn node ENOENT') });
|
|
assert.strictEqual(result.status, 1);
|
|
assert.strictEqual(result.annotations.length, 1);
|
|
assert.match(result.annotations[0], /failed to start.*spawn node ENOENT/);
|
|
}],
|
|
['annotation properties and messages escape workflow command characters', () => {
|
|
const result = run({ status: null, error: new Error('100% broken\r\nnext line') }, 'sample%,:.test.js');
|
|
assert.strictEqual(result.annotations.length, 1);
|
|
assert.ok(result.annotations[0].includes('file=tests/sample%25%2C%3A.test.js'));
|
|
assert.ok(result.annotations[0].includes('100%25 broken%0D%0Anext line'));
|
|
assert.ok(!result.annotations[0].includes('\n'));
|
|
assert.ok(!result.annotations[0].includes('\r'));
|
|
}],
|
|
['failure summaries annotate concise context even with a successful exit', () => {
|
|
const output = `${'routine log\n'.repeat(100)}FAIL regression example\nPassed: 2, Failed: 1`;
|
|
const result = run({ status: 0, stdout: output });
|
|
assert.strictEqual(result.status, 1);
|
|
assert.strictEqual(result.annotations.length, 1);
|
|
assert.match(result.annotations[0], /FAIL regression example/);
|
|
assert.ok(result.annotations[0].length < 1500);
|
|
assert.ok(!result.annotations[0].includes('routine log'));
|
|
assert.ok(result.logs.includes(output));
|
|
assert.ok(result.logs.some(line => /Failed:\s+1\s/.test(line)));
|
|
}],
|
|
['signals fail even when no summary was printed', () => {
|
|
const result = run({ status: null, signal: 'SIGTERM' });
|
|
assert.strictEqual(result.status, 1);
|
|
assert.match(result.annotations[0], /SIGTERM/);
|
|
}],
|
|
['passing error-handling cases cannot hide the actual failure', () => {
|
|
const output = `${'PASS handles Error conditions\n'.repeat(5)}FAIL actual regression\n AssertionError: mismatch\nFailed: 1`;
|
|
const result = run({ status: 1, stdout: output });
|
|
assert.match(result.annotations[0], /FAIL actual regression/);
|
|
assert.match(result.annotations[0], /AssertionError: mismatch/);
|
|
assert.ok(!result.annotations[0].includes('PASS handles'));
|
|
}],
|
|
['successful files without counts do not invent test totals', () => {
|
|
const result = run({ status: 0, stdout: 'suite complete' });
|
|
assert.strictEqual(result.status, 0);
|
|
assert.ok(result.logs.some(line => /Passed:\s+0\s/.test(line)));
|
|
assert.ok(result.logs.some(line => /Files: 1 passed, 0 failed/.test(line)));
|
|
}],
|
|
['healthy suites preserve successful totals and emit no annotation', () => {
|
|
const result = run({ status: 0, stdout: 'Passed: 3, Failed: 0' });
|
|
assert.strictEqual(result.status, 0);
|
|
assert.deepStrictEqual(result.annotations, []);
|
|
assert.ok(result.logs.some(line => /Passed:\s+3\s/.test(line)));
|
|
}],
|
|
['local failures retain console diagnostics without workflow annotations', () => {
|
|
const result = run({ status: 1, stderr: 'Error: local failure' }, 'sample.test.js', false);
|
|
assert.strictEqual(result.status, 1);
|
|
assert.deepStrictEqual(result.annotations, []);
|
|
assert.ok(result.logs.includes('Error: local failure'));
|
|
}],
|
|
...[0, 1].map(fixtureFailures => [`piped runner flushes the full tail and summary with exit ${fixtureFailures}`, () => {
|
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-runner-pipe-'));
|
|
try {
|
|
const testsDir = path.join(root, 'tests');
|
|
fs.mkdirSync(testsDir);
|
|
const runner = path.join(testsDir, 'run-all.js');
|
|
fs.writeFileSync(runner, source);
|
|
// Keep fixture stdout below the runner's default 1 MiB spawnSync ceiling,
|
|
// but above a pipe buffer so the runner must drain pending async writes.
|
|
const payloadBytes = 256 * 1024;
|
|
fs.writeFileSync(path.join(testsDir, 'large.test.js'), [
|
|
`console.log('x'.repeat(${payloadBytes}));`,
|
|
"console.log('FIXTURE_FULL_TAIL');",
|
|
`console.log('Passed: 1, Failed: ${fixtureFailures}');`,
|
|
`process.exitCode = ${fixtureFailures};`,
|
|
].join('\n'));
|
|
const env = { ...process.env, PATH: `${path.dirname(process.execPath)}${path.delimiter}${process.env.PATH || ''}` };
|
|
delete env.ECC_TEST_SKIP;
|
|
delete env.ECC_TEST_ALLOW_FAILURES;
|
|
const result = spawnSync(process.execPath, [runner], {
|
|
encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'], env,
|
|
maxBuffer: 512 * 1024, timeout: 15000,
|
|
});
|
|
assert.ifError(result.error);
|
|
assert.strictEqual(result.status, fixtureFailures, result.stderr);
|
|
assert.ok(result.stdout.includes('x'.repeat(payloadBytes)), 'full bounded fixture payload must reach the pipe');
|
|
assert.match(result.stdout, /FIXTURE_FULL_TAIL/);
|
|
assert.match(result.stdout, /Final Results/);
|
|
assert.ok(result.stdout.trimEnd().endsWith(`Passed: 1, Failed: ${fixtureFailures}`));
|
|
assert.match(result.stdout, new RegExp(`Files: ${fixtureFailures ? '0 passed, 1 failed' : '1 passed, 0 failed'}`));
|
|
} finally {
|
|
fs.rmSync(root, { recursive: true, force: true });
|
|
}
|
|
}]),
|
|
['worker pool runs every file, keeps per-file output grouped, and keeps the summary last', () => {
|
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-runner-pool-'));
|
|
try {
|
|
const testsDir = path.join(root, 'tests');
|
|
fs.mkdirSync(testsDir);
|
|
fs.writeFileSync(path.join(testsDir, 'run-all.js'), source);
|
|
for (const [name, failures] of [['a', 0], ['b', 1], ['c', 0]]) {
|
|
fs.writeFileSync(path.join(testsDir, `${name}.test.js`), [
|
|
`console.log('BEGIN ${name}');`,
|
|
`console.log('Passed: 2, Failed: ${failures}');`,
|
|
`console.log('END ${name}');`,
|
|
].join('\n'));
|
|
}
|
|
const env = { ...process.env, ECC_TEST_JOBS: '3', PATH: `${path.dirname(process.execPath)}${path.delimiter}${process.env.PATH || ''}` };
|
|
delete env.ECC_TEST_SKIP;
|
|
delete env.ECC_TEST_ONLY;
|
|
const result = spawnSync(process.execPath, [path.join(testsDir, 'run-all.js')], { encoding: 'utf8', env, timeout: 15000 });
|
|
assert.ifError(result.error);
|
|
assert.strictEqual(result.status, 1, result.stderr);
|
|
assert.match(result.stdout, /Running 3 files with 3 parallel jobs/);
|
|
for (const name of ['a', 'b', 'c']) {
|
|
assert.match(result.stdout, new RegExp(`━━━ Running ${name}\\.test\\.js ━━━\\n(?:.*\\n)*?BEGIN ${name}\\n[^━]*END ${name}`), `output for ${name} stays grouped`);
|
|
}
|
|
assert.match(result.stdout, /✗ b\.test\.js reported 1 failed tests/);
|
|
assert.match(result.stdout, /Files: 2 passed, 1 failed/);
|
|
assert.ok(result.stdout.trimEnd().endsWith('Passed: 6, Failed: 1'));
|
|
} finally {
|
|
fs.rmSync(root, { recursive: true, force: true });
|
|
}
|
|
}],
|
|
['ECC_TEST_SERIAL files run one at a time after the pool drains', () => {
|
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-runner-serial-'));
|
|
try {
|
|
const testsDir = path.join(root, 'tests');
|
|
fs.mkdirSync(testsDir);
|
|
fs.writeFileSync(path.join(testsDir, 'run-all.js'), source);
|
|
const marker = path.join(root, 'serial-marker');
|
|
// Pool files sleep briefly; the serial file asserts nothing else is still running by
|
|
// checking that both pool files already wrote their markers.
|
|
for (const name of ['pool-a', 'pool-b']) {
|
|
fs.writeFileSync(path.join(testsDir, `${name}.test.js`), [
|
|
"setTimeout(() => {",
|
|
` require('fs').writeFileSync(${JSON.stringify(path.join(root, `${name}.done`))}, '1');`,
|
|
" console.log('Passed: 1, Failed: 0');",
|
|
'}, 150);',
|
|
].join('\n'));
|
|
}
|
|
fs.writeFileSync(path.join(testsDir, 'opencode-dist.test.js'), [
|
|
"const fs = require('fs');",
|
|
`const done = ['pool-a', 'pool-b'].filter(n => fs.existsSync(${JSON.stringify(root)} + '/' + n + '.done')).length;`,
|
|
`fs.writeFileSync(${JSON.stringify(marker)}, String(done));`,
|
|
"console.log(done === 2 ? 'Passed: 1, Failed: 0' : 'Passed: 0, Failed: 1');",
|
|
'process.exitCode = done === 2 ? 0 : 1;',
|
|
].join('\n'));
|
|
const env = { ...process.env, ECC_TEST_JOBS: '3', ECC_TEST_SERIAL: 'opencode-dist', PATH: `${path.dirname(process.execPath)}${path.delimiter}${process.env.PATH || ''}` };
|
|
delete env.ECC_TEST_SKIP;
|
|
delete env.ECC_TEST_ONLY;
|
|
const result = spawnSync(process.execPath, [path.join(testsDir, 'run-all.js')], { encoding: 'utf8', env, timeout: 15000 });
|
|
assert.ifError(result.error);
|
|
assert.strictEqual(result.status, 0, result.stdout);
|
|
assert.match(result.stdout, /serial after the pool: opencode-dist/);
|
|
assert.match(result.stdout, /Running 1 serial files after the pool/);
|
|
assert.strictEqual(fs.readFileSync(marker, 'utf8'), '2');
|
|
assert.match(result.stdout, /Files: 3 passed, 0 failed/);
|
|
} finally {
|
|
fs.rmSync(root, { recursive: true, force: true });
|
|
}
|
|
}],
|
|
['ECC_TEST_ONLY keeps just the files whose path contains a pattern', () => {
|
|
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-runner-only-'));
|
|
try {
|
|
const testsDir = path.join(root, 'tests');
|
|
fs.mkdirSync(path.join(testsDir, 'lib'), { recursive: true });
|
|
fs.mkdirSync(path.join(testsDir, 'hooks'), { recursive: true });
|
|
fs.writeFileSync(path.join(testsDir, 'run-all.js'), source);
|
|
fs.writeFileSync(path.join(testsDir, 'lib', 'package-manager.test.js'), "console.log('Passed: 1, Failed: 0');");
|
|
fs.writeFileSync(path.join(testsDir, 'hooks', 'hooks.test.js'), "console.log('Passed: 1, Failed: 0');");
|
|
fs.writeFileSync(path.join(testsDir, 'lib', 'other.test.js'), "console.log('Passed: 0, Failed: 1'); process.exitCode = 1;");
|
|
const env = { ...process.env, ECC_TEST_ONLY: 'package-manager, hooks/hooks.test', PATH: `${path.dirname(process.execPath)}${path.delimiter}${process.env.PATH || ''}` };
|
|
delete env.ECC_TEST_SKIP;
|
|
delete env.ECC_TEST_JOBS;
|
|
const result = spawnSync(process.execPath, [path.join(testsDir, 'run-all.js')], { encoding: 'utf8', env, timeout: 15000 });
|
|
assert.ifError(result.error);
|
|
assert.strictEqual(result.status, 0, result.stdout);
|
|
assert.match(result.stdout, /Only patterns: package-manager, hooks\/hooks\.test/);
|
|
assert.ok(!result.stdout.includes('other.test.js'), 'unmatched file must not run');
|
|
assert.match(result.stdout, /Files: 2 passed, 0 failed/);
|
|
} finally {
|
|
fs.rmSync(root, { recursive: true, force: true });
|
|
}
|
|
}],
|
|
];
|
|
|
|
let failed = 0;
|
|
for (const [name, test] of tests) {
|
|
try {
|
|
test();
|
|
console.log(`PASS ${name}`);
|
|
} catch (error) {
|
|
failed += 1;
|
|
console.error(`FAIL ${name}\n${error.stack || error.message}`);
|
|
}
|
|
}
|
|
console.log(`Passed: ${tests.length - failed}, Failed: ${failed}`);
|
|
process.exitCode = failed ? 1 : 0;
|