From 9471ef4724ddcdf8f3de302f08c1ee38c8ef7721 Mon Sep 17 00:00:00 2001 From: Affaan Mustafa Date: Sun, 18 Oct 2026 21:20:00 -0400 Subject: [PATCH] test(runner): add a worker pool and an ECC_TEST_ONLY filter to run-all The runner executed 422 files one process at a time, which is most of the 12 to 40 minutes a CI lane spends in "Run tests". ECC_TEST_JOBS above 1 now runs files through a pool of that many processes; output is still buffered and printed per file, so logs read the same. The default stays 1 and that path is the original synchronous spawnSync loop, so the VM-driven runner test and the piped-output tests are unchanged. ECC_TEST_ONLY is the mirror of ECC_TEST_SKIP: a comma-separated list of path substrings, and only matching files run. CI uses it to give the package-manager lanes just the suites that read the package-manager variable instead of the whole suite. Two real-process tests cover the pool (grouped output, counted failure, summary last) and the ONLY filter. --- CLAUDE.md | 6 ++ tests/ci/run-all.test.js | 91 ++++++++++++++++++++++ tests/run-all.js | 161 ++++++++++++++++++++++++++++++--------- 3 files changed, 224 insertions(+), 34 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index d5649ac50..f514243e4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -21,6 +21,12 @@ This is a **Claude Code plugin** - a collection of production-ready agents, skil # Run all tests node tests/run-all.js +# Run files through a 4-process pool, or only files whose path contains a pattern +ECC_TEST_JOBS=4 node tests/run-all.js +ECC_TEST_ONLY=package-manager,hooks/hooks.test node tests/run-all.js +# Suites that rebuild or back up .opencode/dist, or assert a time budget, must run after the pool, one at a time +ECC_TEST_JOBS=4 ECC_TEST_SERIAL=$(grep -rlE --include='*.test.js' 'build-opencode|\.backup-|dist\.backup|time budget' tests | sed 's#^tests/##' | paste -sd, -) node tests/run-all.js + # Run individual test files node tests/lib/utils.test.js node tests/lib/package-manager.test.js diff --git a/tests/ci/run-all.test.js b/tests/ci/run-all.test.js index 32e016977..69008181b 100644 --- a/tests/ci/run-all.test.js +++ b/tests/ci/run-all.test.js @@ -177,6 +177,97 @@ const tests = [ fs.rmSync(root, { recursive: true, force: true }); } }]), + ['worker pool runs every file, keeps per-file output grouped, and keeps the summary last', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-runner-pool-')); + try { + const testsDir = path.join(root, 'tests'); + fs.mkdirSync(testsDir); + fs.writeFileSync(path.join(testsDir, 'run-all.js'), source); + for (const [name, failures] of [['a', 0], ['b', 1], ['c', 0]]) { + fs.writeFileSync(path.join(testsDir, `${name}.test.js`), [ + `console.log('BEGIN ${name}');`, + `console.log('Passed: 2, Failed: ${failures}');`, + `console.log('END ${name}');`, + ].join('\n')); + } + const env = { ...process.env, ECC_TEST_JOBS: '3', PATH: `${path.dirname(process.execPath)}${path.delimiter}${process.env.PATH || ''}` }; + delete env.ECC_TEST_SKIP; + delete env.ECC_TEST_ONLY; + const result = spawnSync(process.execPath, [path.join(testsDir, 'run-all.js')], { encoding: 'utf8', env, timeout: 15000 }); + assert.ifError(result.error); + assert.strictEqual(result.status, 1, result.stderr); + assert.match(result.stdout, /Running 3 files with 3 parallel jobs/); + for (const name of ['a', 'b', 'c']) { + assert.match(result.stdout, new RegExp(`━━━ Running ${name}\\.test\\.js ━━━\\n(?:.*\\n)*?BEGIN ${name}\\n[^━]*END ${name}`), `output for ${name} stays grouped`); + } + assert.match(result.stdout, /✗ b\.test\.js reported 1 failed tests/); + assert.match(result.stdout, /Files: 2 passed, 1 failed/); + assert.ok(result.stdout.trimEnd().endsWith('Passed: 6, Failed: 1')); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }], + ['ECC_TEST_SERIAL files run one at a time after the pool drains', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-runner-serial-')); + try { + const testsDir = path.join(root, 'tests'); + fs.mkdirSync(testsDir); + fs.writeFileSync(path.join(testsDir, 'run-all.js'), source); + const marker = path.join(root, 'serial-marker'); + // Pool files sleep briefly; the serial file asserts nothing else is still running by + // checking that both pool files already wrote their markers. + for (const name of ['pool-a', 'pool-b']) { + fs.writeFileSync(path.join(testsDir, `${name}.test.js`), [ + "setTimeout(() => {", + ` require('fs').writeFileSync(${JSON.stringify(path.join(root, `${name}.done`))}, '1');`, + " console.log('Passed: 1, Failed: 0');", + '}, 150);', + ].join('\n')); + } + fs.writeFileSync(path.join(testsDir, 'opencode-dist.test.js'), [ + "const fs = require('fs');", + `const done = ['pool-a', 'pool-b'].filter(n => fs.existsSync(${JSON.stringify(root)} + '/' + n + '.done')).length;`, + `fs.writeFileSync(${JSON.stringify(marker)}, String(done));`, + "console.log(done === 2 ? 'Passed: 1, Failed: 0' : 'Passed: 0, Failed: 1');", + 'process.exitCode = done === 2 ? 0 : 1;', + ].join('\n')); + const env = { ...process.env, ECC_TEST_JOBS: '3', ECC_TEST_SERIAL: 'opencode-dist', PATH: `${path.dirname(process.execPath)}${path.delimiter}${process.env.PATH || ''}` }; + delete env.ECC_TEST_SKIP; + delete env.ECC_TEST_ONLY; + const result = spawnSync(process.execPath, [path.join(testsDir, 'run-all.js')], { encoding: 'utf8', env, timeout: 15000 }); + assert.ifError(result.error); + assert.strictEqual(result.status, 0, result.stdout); + assert.match(result.stdout, /serial after the pool: opencode-dist/); + assert.match(result.stdout, /Running 1 serial files after the pool/); + assert.strictEqual(fs.readFileSync(marker, 'utf8'), '2'); + assert.match(result.stdout, /Files: 3 passed, 0 failed/); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }], + ['ECC_TEST_ONLY keeps just the files whose path contains a pattern', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-runner-only-')); + try { + const testsDir = path.join(root, 'tests'); + fs.mkdirSync(path.join(testsDir, 'lib'), { recursive: true }); + fs.mkdirSync(path.join(testsDir, 'hooks'), { recursive: true }); + fs.writeFileSync(path.join(testsDir, 'run-all.js'), source); + fs.writeFileSync(path.join(testsDir, 'lib', 'package-manager.test.js'), "console.log('Passed: 1, Failed: 0');"); + fs.writeFileSync(path.join(testsDir, 'hooks', 'hooks.test.js'), "console.log('Passed: 1, Failed: 0');"); + fs.writeFileSync(path.join(testsDir, 'lib', 'other.test.js'), "console.log('Passed: 0, Failed: 1'); process.exitCode = 1;"); + const env = { ...process.env, ECC_TEST_ONLY: 'package-manager, hooks/hooks.test', PATH: `${path.dirname(process.execPath)}${path.delimiter}${process.env.PATH || ''}` }; + delete env.ECC_TEST_SKIP; + delete env.ECC_TEST_JOBS; + const result = spawnSync(process.execPath, [path.join(testsDir, 'run-all.js')], { encoding: 'utf8', env, timeout: 15000 }); + assert.ifError(result.error); + assert.strictEqual(result.status, 0, result.stdout); + assert.match(result.stdout, /Only patterns: package-manager, hooks\/hooks\.test/); + assert.ok(!result.stdout.includes('other.test.js'), 'unmatched file must not run'); + assert.match(result.stdout, /Files: 2 passed, 0 failed/); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }], ]; let failed = 0; diff --git a/tests/run-all.js b/tests/run-all.js index d03efc67c..5e436c4fa 100644 --- a/tests/run-all.js +++ b/tests/run-all.js @@ -1,6 +1,6 @@ #!/usr/bin/env node -const { spawnSync } = require('child_process'); +const { spawn, spawnSync } = require('child_process'); const path = require('path'); const fs = require('fs'); @@ -8,10 +8,23 @@ const testsDir = __dirname; const repoRoot = path.resolve(testsDir, '..'); const TEST_GLOB = 'tests/**/*.test.js'; const filter = (((process.argv || [])[2]) || '').split(path.sep).join('/'); -const skipPatterns = (process.env.ECC_TEST_SKIP || '') +const splitPatterns = value => (value || '') .split(',') .map(s => s.trim()) .filter(Boolean); +// ECC_TEST_SKIP drops files whose path contains any pattern; ECC_TEST_ONLY keeps +// just the files whose path contains one. CI's package-manager lanes use ONLY to +// run the handful of suites that read CLAUDE_CODE_PACKAGE_MANAGER. +const skipPatterns = splitPatterns(process.env.ECC_TEST_SKIP); +const onlyPatterns = splitPatterns(process.env.ECC_TEST_ONLY); +// ECC_TEST_SERIAL names suites that mutate the shared checkout (for example +// the opencode dist build and the installers that copy from it). The pool +// holds them back and runs them one at a time after every other file. +const serialPatterns = splitPatterns(process.env.ECC_TEST_SERIAL); +// ECC_TEST_JOBS above 1 runs files through a worker pool of that many +// processes. The default is 1: the original synchronous loop, unchanged. +// Output stays grouped per file whatever the pool size. +const jobs = Math.max(1, Math.floor(Number(process.env.ECC_TEST_JOBS)) || 1); function matchesTestGlob(relativePath) { const normalized = relativePath.split(path.sep).join('/'); @@ -21,6 +34,9 @@ function matchesTestGlob(relativePath) { if (skipPatterns.some(pattern => normalized.includes(pattern))) { return false; } + if (onlyPatterns.length > 0 && !onlyPatterns.some(pattern => normalized.includes(pattern))) { + return false; + } if (typeof path.matchesGlob === 'function') { return path.matchesGlob(normalized, TEST_GLOB); } @@ -114,6 +130,14 @@ if (skipPatterns.length > 0) { console.log(`Skipped patterns: ${skipPatterns.join(', ')}`); console.log(); } +if (onlyPatterns.length > 0) { + console.log(`Only patterns: ${onlyPatterns.join(', ')}`); + console.log(); +} +if (jobs > 1) { + console.log(`Running ${testFiles.length} files with ${jobs} parallel jobs${serialPatterns.length ? `, serial after the pool: ${serialPatterns.join(', ')}` : ''}`); + console.log(); +} if (testFiles.length === 0) { console.log(`✗ No test files matched ${TEST_GLOB}${filter ? ` with filter "${filter}"` : ''}`); @@ -126,31 +150,54 @@ let filesPassed = 0; let filesFailed = 0; const failedFiles = []; -for (const testFile of testFiles) { - const testPath = path.join(testsDir, testFile); - const displayPath = testFile.split(path.sep).join('/'); - - if (!fs.existsSync(testPath)) { - console.log(`WARNING Skipping ${displayPath} (file not found)`); - continue; - } - - console.log(`\n━━━ Running ${displayPath} ━━━`); - +function childEnvFor() { const childEnv = { ...process.env }; - for (const key of ['GIT_DIR', 'GIT_WORK_TREE', 'GIT_INDEX_FILE', 'GIT_COMMON_DIR', 'GIT_PREFIX']) { + // Git context would leak into fixtures; ONLY and JOBS are runner controls and + // must not reach suites that spawn the runner themselves. + for (const key of ['GIT_DIR', 'GIT_WORK_TREE', 'GIT_INDEX_FILE', 'GIT_COMMON_DIR', 'GIT_PREFIX', 'ECC_TEST_ONLY', 'ECC_TEST_JOBS', 'ECC_TEST_SERIAL']) { delete childEnv[key]; } + return childEnv; +} - const result = spawnSync('node', [testPath], { +function runTestFileSync(testFile) { + return spawnSync('node', [path.join(testsDir, testFile)], { encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'], - env: childEnv + env: childEnvFor() }); +} +function runTestFile(testFile) { + const testPath = path.join(testsDir, testFile); + const childEnv = childEnvFor(); + return new Promise(resolve => { + let stdout = ''; + let stderr = ''; + let child; + try { + child = spawn('node', [testPath], { stdio: ['pipe', 'pipe', 'pipe'], env: childEnv }); + } catch (error) { + resolve({ error, status: null, signal: null, stdout, stderr }); + return; + } + // spawnSync with piped stdio and no input hands the child a closed stdin; + // spawn leaves it open, and a suite that reads stdin would wait forever. + child.stdin.end(); + child.stdout.setEncoding('utf8'); + child.stderr.setEncoding('utf8'); + child.stdout.on('data', chunk => { stdout += chunk; }); + child.stderr.on('data', chunk => { stderr += chunk; }); + child.on('error', error => resolve({ error, status: null, signal: null, stdout, stderr })); + child.on('close', (status, signal) => resolve({ error: null, status, signal, stdout, stderr })); + }); +} + +function reportTestFile(displayPath, result) { const stdout = result.stdout || ''; const stderr = result.stderr || ''; + console.log(`\n━━━ Running ${displayPath} ━━━`); if (stdout) console.log(stdout); if (stderr) console.log(stderr); @@ -188,25 +235,71 @@ for (const testFile of testFiles) { } } -const totalTests = totalPassed + totalFailed; +function pendingTestFiles() { + return testFiles.filter(testFile => { + if (fs.existsSync(path.join(testsDir, testFile))) return true; + console.log(`WARNING Skipping ${testFile.split(path.sep).join('/')} (file not found)`); + return false; + }); +} -console.log('\n╔' + '═'.repeat(BOX_W) + '╗'); -console.log(boxLine(' Final Results')); -console.log('╠' + '═'.repeat(BOX_W) + '╣'); -console.log(boxLine(` Total Tests: ${String(totalTests).padStart(4)}`)); -console.log(boxLine(` Passed: ${String(totalPassed).padStart(4)} ✓`)); -console.log(boxLine(` Failed: ${String(totalFailed).padStart(4)} ${totalFailed > 0 ? '✗' : ' '}`)); -console.log(boxLine(` Files: ${filesPassed} passed, ${filesFailed} failed`)); -if (failedFiles.length > 0) { - console.log('╠' + '═'.repeat(BOX_W) + '╣'); - for (const name of failedFiles.slice(0, 10)) { - console.log(boxLine(` ✗ ${name}`.slice(0, BOX_W))); - } - if (failedFiles.length > 10) { - console.log(boxLine(` ... and ${failedFiles.length - 10} more`)); +function runAllSync() { + for (const testFile of pendingTestFiles()) { + reportTestFile(testFile.split(path.sep).join('/'), runTestFileSync(testFile)); } } -console.log('╚' + '═'.repeat(BOX_W) + '╝'); -console.log(`\nPassed: ${totalPassed}, Failed: ${totalFailed}`); -process.exitCode = totalFailed > 0 ? 1 : 0; +function isSerial(testFile) { + const normalized = testFile.split(path.sep).join('/'); + return serialPatterns.some(pattern => normalized.includes(pattern)); +} + +async function runAllParallel() { + const pending = pendingTestFiles(); + const queue = pending.filter(testFile => !isSerial(testFile)); + const serial = pending.filter(isSerial); + const worker = async () => { + while (queue.length > 0) { + const testFile = queue.shift(); + const result = await runTestFile(testFile); + reportTestFile(testFile.split(path.sep).join('/'), result); + } + }; + await Promise.all(Array.from({ length: Math.min(jobs, queue.length) }, worker)); + if (serial.length > 0) console.log(`\nRunning ${serial.length} serial files after the pool`); + for (const testFile of serial) { + reportTestFile(testFile.split(path.sep).join('/'), await runTestFile(testFile)); + } +} + +function finish() { + const totalTests = totalPassed + totalFailed; + + console.log('\n╔' + '═'.repeat(BOX_W) + '╗'); + console.log(boxLine(' Final Results')); + console.log('╠' + '═'.repeat(BOX_W) + '╣'); + console.log(boxLine(` Total Tests: ${String(totalTests).padStart(4)}`)); + console.log(boxLine(` Passed: ${String(totalPassed).padStart(4)} ✓`)); + console.log(boxLine(` Failed: ${String(totalFailed).padStart(4)} ${totalFailed > 0 ? '✗' : ' '}`)); + console.log(boxLine(` Files: ${filesPassed} passed, ${filesFailed} failed`)); + if (failedFiles.length > 0) { + console.log('╠' + '═'.repeat(BOX_W) + '╣'); + for (const name of failedFiles.slice(0, 10)) { + console.log(boxLine(` ✗ ${name}`.slice(0, BOX_W))); + } + if (failedFiles.length > 10) { + console.log(boxLine(` ... and ${failedFiles.length - 10} more`)); + } + } + console.log('╚' + '═'.repeat(BOX_W) + '╝'); + console.log(`\nPassed: ${totalPassed}, Failed: ${totalFailed}`); + + process.exitCode = totalFailed > 0 ? 1 : 0; +} + +if (jobs > 1) { + runAllParallel().then(finish); +} else { + runAllSync(); + finish(); +}