From d216de92079163d6ffc9f73a2f0a24391191fcbf Mon Sep 17 00:00:00 2001 From: wellkilo Date: Mon, 7 Sep 2026 14:16:41 +0800 Subject: [PATCH 1/3] fix(audit): recognize Python consumer test suites --- scripts/harness-audit.js | 16 +- scripts/lib/harness-audit-consumer.js | 237 ++++++++++++++++++++++++++ tests/scripts/harness-audit.test.js | 120 +++++++++++++ 3 files changed, 370 insertions(+), 3 deletions(-) create mode 100644 scripts/lib/harness-audit-consumer.js diff --git a/scripts/harness-audit.js b/scripts/harness-audit.js index a523eca9e..2ce394c38 100644 --- a/scripts/harness-audit.js +++ b/scripts/harness-audit.js @@ -3,6 +3,11 @@ const fs = require('fs'); const os = require('os'); const path = require('path'); +const { + countPythonTestFiles, + gitignoreIgnoresEnvFiles, + hasPythonTestSuite, +} = require('./lib/harness-audit-consumer'); const CATEGORIES = [ 'Tool Coverage', @@ -823,6 +828,7 @@ function getConsumerChecks(rootDir) { const gitignore = safeRead(rootDir, '.gitignore'); const projectHooks = safeRead(rootDir, '.claude/settings.json'); const pluginInstall = findPluginInstall(rootDir); + const pythonTestCount = countPythonTestFiles(rootDir); return [ { @@ -876,7 +882,10 @@ function getConsumerChecks(rootDir) { scopes: ['repo'], path: 'tests/', description: 'The project has an automated test entrypoint', - pass: typeof packageJson?.scripts?.test === 'string' || countFiles(rootDir, 'tests', '.test.js') > 0 || hasFileWithExtension(rootDir, '.', ['.spec.js', '.spec.ts', '.test.ts']), + pass: typeof packageJson?.scripts?.test === 'string' || + countFiles(rootDir, 'tests', '.test.js') > 0 || + hasFileWithExtension(rootDir, '.', ['.spec.js', '.spec.ts', '.test.ts']) || + hasPythonTestSuite(rootDir, pythonTestCount), fix: 'Add a test script or checked-in tests so harness recommendations can be verified automatically.', }, { @@ -906,7 +915,8 @@ function getConsumerChecks(rootDir) { scopes: ['repo'], path: 'evals/', description: 'The project has evals or multiple automated tests', - pass: countFiles(rootDir, 'evals', null) > 0 || countFiles(rootDir, 'tests', '.test.js') >= 3, + pass: countFiles(rootDir, 'evals', null) > 0 || + countFiles(rootDir, 'tests', '.test.js') + pythonTestCount >= 3, fix: 'Add eval fixtures or at least a few focused automated tests for critical flows.', }, { @@ -926,7 +936,7 @@ function getConsumerChecks(rootDir) { scopes: ['repo'], path: '.gitignore', description: 'The project ignores common secret env files', - pass: gitignore.includes('.env'), + pass: gitignoreIgnoresEnvFiles(gitignore), fix: 'Ignore .env-style files in .gitignore so secrets do not land in the repo.', }, { diff --git a/scripts/lib/harness-audit-consumer.js b/scripts/lib/harness-audit-consumer.js new file mode 100644 index 000000000..8d212a59c --- /dev/null +++ b/scripts/lib/harness-audit-consumer.js @@ -0,0 +1,237 @@ +'use strict'; + +const fs = require('fs'); +const path = require('path'); +const TOML = require('@iarna/toml'); + +const IGNORED_SCAN_DIRS = new Set([ + '.git', + '.nox', + '.pytest_cache', + '.tox', + '.venv', + '__pycache__', + 'build', + 'dist', + 'env', + 'node_modules', + 'venv', + 'vendor', +]); + +function safeRead(filePath) { + try { + return fs.readFileSync(filePath, 'utf8'); + } catch (_error) { + return ''; + } +} + +function walkFiles(rootDir, relativeRoots, predicate) { + const projectRoot = fs.realpathSync(path.resolve(rootDir)); + const files = new Set(); + + for (const relativeRoot of relativeRoots) { + const scanRoot = path.resolve(projectRoot, relativeRoot); + const relative = path.relative(projectRoot, scanRoot); + const segments = relative.split(path.sep).filter(Boolean); + if ( + relative === '..' || + relative.startsWith(`..${path.sep}`) || + path.isAbsolute(relative) || + segments.some(segment => IGNORED_SCAN_DIRS.has(segment)) || + !fs.existsSync(scanRoot) + ) { + continue; + } + try { + const realRoot = fs.realpathSync(scanRoot); + const realRelative = path.relative(projectRoot, realRoot); + if ( + realRelative === '..' || + realRelative.startsWith(`..${path.sep}`) || + path.isAbsolute(realRelative) + ) { + continue; + } + } catch (_error) { + continue; + } + + const stack = [scanRoot]; + while (stack.length > 0) { + const current = stack.pop(); + let entries; + try { + entries = fs.readdirSync(current, { withFileTypes: true }); + } catch (_error) { + continue; + } + + for (const entry of entries) { + const nextPath = path.join(current, entry.name); + if (entry.isDirectory()) { + if (!IGNORED_SCAN_DIRS.has(entry.name)) { + stack.push(nextPath); + } + } else if (entry.isFile() && predicate(entry.name, nextPath)) { + files.add(nextPath); + } + } + } + } + + return files; +} + +function parseIniTestPaths(text, sectionName) { + if (!text) return []; + const sectionPattern = new RegExp( + `^\\s*\\[${sectionName.replace(/[.*+?^${}()|[\\]\\]/g, '\\$&')}\\]\\s*$`, + 'i' + ); + const lines = text.split(/\r?\n/); + let inSection = false; + let collecting = false; + const values = []; + + for (const line of lines) { + if (/^\s*\[.*]\s*$/.test(line)) { + inSection = sectionPattern.test(line); + collecting = false; + continue; + } + if (!inSection || /^\s*[#;]/.test(line)) continue; + + const match = line.match(/^\s*testpaths\s*=\s*(.*)$/i); + if (match) { + collecting = true; + values.push(match[1]); + continue; + } + if (collecting && /^\s+\S/.test(line)) { + values.push(line.trim()); + } else if (line.trim()) { + collecting = false; + } + } + + return values + .flatMap(value => value.split(/\s+/)) + .map(value => value.replace(/^['"]|['"]$/g, '').trim()) + .filter(Boolean); +} + +function getPytestTestPaths(rootDir) { + const paths = []; + const pyproject = safeRead(path.join(rootDir, 'pyproject.toml')); + if (pyproject) { + try { + const configured = TOML.parse(pyproject)?.tool?.pytest?.ini_options?.testpaths; + if (Array.isArray(configured)) paths.push(...configured); + else if (typeof configured === 'string') paths.push(...configured.split(/\s+/)); + } catch (_error) { + // Malformed project metadata is not evidence of a configured test path. + } + } + + paths.push(...parseIniTestPaths(safeRead(path.join(rootDir, 'pytest.ini')), 'pytest')); + paths.push(...parseIniTestPaths(safeRead(path.join(rootDir, 'tox.ini')), 'pytest')); + paths.push(...parseIniTestPaths(safeRead(path.join(rootDir, 'setup.cfg')), 'tool:pytest')); + + return [...new Set(paths.filter(value => typeof value === 'string' && value.trim()))]; +} + +function isPythonTestFile(fileName) { + return /^(test_.*|.*_test)\.py$/i.test(fileName); +} + +function countPythonTestFiles(rootDir) { + const roots = ['tests', ...getPytestTestPaths(rootDir)]; + return walkFiles(rootDir, roots, isPythonTestFile).size; +} + +function hasPythonTestSuite(rootDir, pythonTestCount = countPythonTestFiles(rootDir)) { + const pyproject = safeRead(path.join(rootDir, 'pyproject.toml')); + const setupCfg = safeRead(path.join(rootDir, 'setup.cfg')); + const hasConfig = ( + /^\s*\[tool\.pytest\.ini_options]\s*$/m.test(pyproject) + || /^\s*\[tool:pytest]\s*$/m.test(setupCfg) + || fs.existsSync(path.join(rootDir, 'pytest.ini')) + || fs.existsSync(path.join(rootDir, 'tox.ini')) + ); + return hasConfig || + pythonTestCount > 0 || + walkFiles(rootDir, ['.'], fileName => fileName === 'conftest.py').size > 0; +} + +function globToRegex(pattern) { + let output = ''; + for (let index = 0; index < pattern.length; index += 1) { + const char = pattern[index]; + if (char === '*' && pattern[index + 1] === '*') { + output += '.*'; + index += 1; + } else if (char === '*') { + output += '[^/]*'; + } else if (char === '?') { + output += '[^/]'; + } else { + output += char.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + } + } + return output; +} + +function matchesGitignorePattern(candidate, rawPattern) { + const directoryOnly = rawPattern.endsWith('/'); + const anchored = rawPattern.startsWith('/'); + const pattern = rawPattern.replace(/^\//, '').replace(/\/$/, ''); + if (!pattern) return false; + + const body = globToRegex(pattern); + const prefix = anchored || pattern.includes('/') ? '^' : '(^|.*/)'; + const suffix = directoryOnly ? '(/.*)?$' : '$'; + return new RegExp(`${prefix}${body}${suffix}`).test(candidate); +} + +function gitignoreIgnoresEnvFiles(text) { + if (typeof text !== 'string' || !text.trim()) return false; + const candidates = [ + '.env', + '.env.local', + '.env.production', + '.env/example', + 'config/.env', + 'secrets.env', + ]; + const states = new Map(candidates.map(candidate => [candidate, false])); + + for (const rawLine of text.split(/\r?\n/)) { + let line = rawLine.trim(); + if (!line || line.startsWith('#')) continue; + + let negated = false; + if (line.startsWith('!')) { + negated = true; + line = line.slice(1); + } else if (line.startsWith('\\#') || line.startsWith('\\!')) { + line = line.slice(1); + } + if (!line) continue; + + for (const candidate of candidates) { + if (matchesGitignorePattern(candidate, line)) { + states.set(candidate, !negated); + } + } + } + + return [...states.values()].some(Boolean); +} + +module.exports = { + countPythonTestFiles, + gitignoreIgnoresEnvFiles, + hasPythonTestSuite, +}; diff --git a/tests/scripts/harness-audit.test.js b/tests/scripts/harness-audit.test.js index c0087475d..a0ece0aee 100644 --- a/tests/scripts/harness-audit.test.js +++ b/tests/scripts/harness-audit.test.js @@ -10,6 +10,7 @@ const { execFileSync, spawnSync } = require('child_process'); const SCRIPT = path.join(__dirname, '..', '..', 'scripts', 'harness-audit.js'); const { parseArgs, findPluginInstall, compareVersionDesc } = require(SCRIPT); +const { countPythonTestFiles } = require('../../scripts/lib/harness-audit-consumer'); function createTempDir(prefix) { return fs.mkdtempSync(path.join(os.tmpdir(), prefix)); @@ -425,6 +426,125 @@ function runTests() { } })) passed++; else failed++; + if (test('combines Python and JavaScript tests for eval coverage', () => { + const homeDir = createTempDir('harness-audit-python-home-'); + const projectRoot = createTempDir('harness-audit-python-project-'); + + try { + fs.mkdirSync(path.join(projectRoot, 'qa'), { recursive: true }); + fs.mkdirSync(path.join(projectRoot, 'integration'), { recursive: true }); + fs.mkdirSync(path.join(projectRoot, 'tests'), { recursive: true }); + fs.mkdirSync(path.join(projectRoot, '.venv', 'tests'), { recursive: true }); + fs.mkdirSync(path.join(projectRoot, 'node_modules', 'pkg', 'tests'), { recursive: true }); + fs.writeFileSync( + path.join(projectRoot, 'pyproject.toml'), + '[tool.pytest.ini_options]\ntestpaths = ["qa", "integration"]\n' + ); + fs.writeFileSync(path.join(projectRoot, 'qa', 'test_alpha.py'), 'def test_alpha(): pass\n'); + fs.writeFileSync(path.join(projectRoot, 'qa', 'beta_test.py'), 'def test_beta(): pass\n'); + fs.writeFileSync(path.join(projectRoot, 'tests', 'gamma.test.js'), 'test placeholder\n'); + fs.writeFileSync(path.join(projectRoot, '.venv', 'tests', 'test_vendor.py'), ''); + fs.writeFileSync(path.join(projectRoot, 'node_modules', 'pkg', 'tests', 'test_vendor.py'), ''); + + assert.strictEqual(countPythonTestFiles(projectRoot), 2); + const parsed = JSON.parse(run(['repo', '--format', 'json'], { cwd: projectRoot, homeDir })); + assert.ok(parsed.checks.some(check => check.id === 'consumer-test-suite' && check.pass)); + assert.ok(parsed.checks.some(check => check.id === 'consumer-eval-coverage' && check.pass)); + } finally { + cleanup(homeDir); + cleanup(projectRoot); + } + })) passed++; else failed++; + + if (test('recognizes standard pytest suite markers without JavaScript tests', () => { + const homeDir = createTempDir('harness-audit-python-markers-home-'); + const fixtures = [ + ['pytest.ini', '[pytest]\n'], + ['tox.ini', '[tox]\nenvlist = py\n'], + ['setup.cfg', '[tool:pytest]\n'], + ['quality/conftest.py', ''], + ['tests/test_smoke.py', 'def test_smoke(): pass\n'], + ]; + + try { + for (const [relativePath, contents] of fixtures) { + const projectRoot = createTempDir('harness-audit-python-marker-'); + try { + fs.mkdirSync(path.dirname(path.join(projectRoot, relativePath)), { recursive: true }); + fs.writeFileSync(path.join(projectRoot, relativePath), contents); + const parsed = JSON.parse(run(['repo', '--format', 'json'], { cwd: projectRoot, homeDir })); + assert.ok( + parsed.checks.some(check => check.id === 'consumer-test-suite' && check.pass), + `${relativePath} should identify a Python test suite` + ); + } finally { + cleanup(projectRoot); + } + } + } finally { + cleanup(homeDir); + } + })) passed++; else failed++; + + if (test('counts tests from INI testpaths without scanning ignored dependency roots', () => { + const homeDir = createTempDir('harness-audit-python-ini-home-'); + const projectRoot = createTempDir('harness-audit-python-ini-project-'); + + try { + fs.mkdirSync(path.join(projectRoot, 'quality'), { recursive: true }); + fs.mkdirSync(path.join(projectRoot, '.venv', 'tests'), { recursive: true }); + fs.writeFileSync( + path.join(projectRoot, 'pytest.ini'), + '[pytest]\ntestpaths =\n quality\n .venv/tests\n' + ); + for (const name of ['test_one.py', 'test_two.py', 'three_test.py']) { + fs.writeFileSync(path.join(projectRoot, 'quality', name), ''); + } + fs.writeFileSync(path.join(projectRoot, '.venv', 'tests', 'test_vendor.py'), ''); + + assert.strictEqual(countPythonTestFiles(projectRoot), 3); + const parsed = JSON.parse(run(['repo', '--format', 'json'], { cwd: projectRoot, homeDir })); + assert.ok(parsed.checks.some(check => check.id === 'consumer-test-suite' && check.pass)); + assert.ok(parsed.checks.some(check => check.id === 'consumer-eval-coverage' && check.pass)); + } finally { + cleanup(homeDir); + cleanup(projectRoot); + } + })) passed++; else failed++; + + if (test('secret hygiene requires an active gitignore rule for env files', () => { + const homeDir = createTempDir('harness-audit-gitignore-home-'); + const cases = [ + ['# Do not commit .env files\nnode_modules/\n', false], + ['environment.json\n.envexample\n', false], + ['.env.example\n', false], + ['.env\n', true], + ['.env.local\n', true], + ['.env.*\n', true], + ['*.env\n', true], + ['.env/\n', true], + ['config/.env\n', true], + ['.env\n!.env\n', false], + ['.env*\n!.env.example\n', true], + ]; + + try { + for (const [gitignore, expected] of cases) { + const projectRoot = createTempDir('harness-audit-gitignore-project-'); + try { + fs.writeFileSync(path.join(projectRoot, '.gitignore'), gitignore); + const parsed = JSON.parse(run(['repo', '--format', 'json'], { cwd: projectRoot, homeDir })); + const check = parsed.checks.find(candidate => candidate.id === 'consumer-secret-hygiene'); + assert.strictEqual(check.pass, expected, JSON.stringify({ gitignore, expected })); + } finally { + cleanup(projectRoot); + } + } + } finally { + cleanup(homeDir); + } + })) passed++; else failed++; + if (test('prints no top actions when consumer checks all pass', () => { const homeDir = createTempDir('harness-audit-passing-home-'); const projectRoot = createTempDir('harness-audit-passing-project-'); From 720796ed48b7e6e8f16e9c3d029ab3e595e88d37 Mon Sep 17 00:00:00 2001 From: wellkilo Date: Mon, 7 Sep 2026 15:45:40 +0800 Subject: [PATCH 2/3] fix(audit): align pytest discovery semantics --- scripts/lib/harness-audit-consumer.js | 378 +++++++++++++++++++------- tests/scripts/harness-audit.test.js | 177 +++++++++++- 2 files changed, 460 insertions(+), 95 deletions(-) diff --git a/scripts/lib/harness-audit-consumer.js b/scripts/lib/harness-audit-consumer.js index 8d212a59c..cafe5cff9 100644 --- a/scripts/lib/harness-audit-consumer.js +++ b/scripts/lib/harness-audit-consumer.js @@ -1,20 +1,27 @@ 'use strict'; const fs = require('fs'); +const os = require('os'); const path = require('path'); +const { spawnSync } = require('child_process'); const TOML = require('@iarna/toml'); const IGNORED_SCAN_DIRS = new Set([ '.git', + '.eggs', + '.mypy_cache', '.nox', '.pytest_cache', + '.ruff_cache', '.tox', '.venv', '__pycache__', + '__pypackages__', 'build', 'dist', 'env', 'node_modules', + 'site-packages', 'venv', 'vendor', ]); @@ -29,9 +36,25 @@ function safeRead(filePath) { function walkFiles(rootDir, relativeRoots, predicate) { const projectRoot = fs.realpathSync(path.resolve(rootDir)); - const files = new Set(); - for (const relativeRoot of relativeRoots) { + function visit(current) { + let entries; + try { + entries = fs.readdirSync(current, { withFileTypes: true }); + } catch (_error) { + return []; + } + + return entries.flatMap(entry => { + const nextPath = path.join(current, entry.name); + if (entry.isDirectory()) { + return IGNORED_SCAN_DIRS.has(entry.name) ? [] : visit(nextPath); + } + return entry.isFile() && predicate(entry.name, nextPath) ? [nextPath] : []; + }); + } + + const files = relativeRoots.flatMap(relativeRoot => { const scanRoot = path.resolve(projectRoot, relativeRoot); const relative = path.relative(projectRoot, scanRoot); const segments = relative.split(path.sep).filter(Boolean); @@ -42,7 +65,7 @@ function walkFiles(rootDir, relativeRoots, predicate) { segments.some(segment => IGNORED_SCAN_DIRS.has(segment)) || !fs.existsSync(scanRoot) ) { - continue; + return []; } try { const realRoot = fs.realpathSync(scanRoot); @@ -52,39 +75,21 @@ function walkFiles(rootDir, relativeRoots, predicate) { realRelative.startsWith(`..${path.sep}`) || path.isAbsolute(realRelative) ) { - continue; + return []; } } catch (_error) { - continue; + return []; } - - const stack = [scanRoot]; - while (stack.length > 0) { - const current = stack.pop(); - let entries; - try { - entries = fs.readdirSync(current, { withFileTypes: true }); - } catch (_error) { - continue; - } - - for (const entry of entries) { - const nextPath = path.join(current, entry.name); - if (entry.isDirectory()) { - if (!IGNORED_SCAN_DIRS.has(entry.name)) { - stack.push(nextPath); - } - } else if (entry.isFile() && predicate(entry.name, nextPath)) { - files.add(nextPath); - } - } + if (fs.statSync(scanRoot).isFile()) { + return predicate(path.basename(scanRoot), scanRoot) ? [scanRoot] : []; } - } + return visit(scanRoot); + }); - return files; + return new Set(files); } -function parseIniTestPaths(text, sectionName) { +function parseIniOption(text, sectionName, optionName) { if (!text) return []; const sectionPattern = new RegExp( `^\\s*\\[${sectionName.replace(/[.*+?^${}()|[\\]\\]/g, '\\$&')}\\]\\s*$`, @@ -93,7 +98,7 @@ function parseIniTestPaths(text, sectionName) { const lines = text.split(/\r?\n/); let inSection = false; let collecting = false; - const values = []; + let values = []; for (const line of lines) { if (/^\s*\[.*]\s*$/.test(line)) { @@ -103,66 +108,243 @@ function parseIniTestPaths(text, sectionName) { } if (!inSection || /^\s*[#;]/.test(line)) continue; - const match = line.match(/^\s*testpaths\s*=\s*(.*)$/i); + const match = line.match(new RegExp(`^\\s*${optionName}\\s*=\\s*(.*)$`, 'i')); if (match) { collecting = true; - values.push(match[1]); + values = [...values, match[1]]; continue; } if (collecting && /^\s+\S/.test(line)) { - values.push(line.trim()); + values = [...values, line.trim()]; } else if (line.trim()) { collecting = false; } } return values - .flatMap(value => value.split(/\s+/)) + .flatMap(value => value.match(/"[^"]*"|'[^']*'|\S+/g) || []) .map(value => value.replace(/^['"]|['"]$/g, '').trim()) .filter(Boolean); } -function getPytestTestPaths(rootDir) { - const paths = []; - const pyproject = safeRead(path.join(rootDir, 'pyproject.toml')); - if (pyproject) { +function normalizeTomlList(value) { + if (Array.isArray(value)) { + return value.filter(item => typeof item === 'string' && item.trim()); + } + return typeof value === 'string' ? value.split(/\s+/) : []; +} + +function getActivePytestConfig(rootDir) { + for (const fileName of ['pytest.toml', '.pytest.toml']) { + const configPath = path.join(rootDir, fileName); + if (!fs.existsSync(configPath)) continue; + const text = safeRead(configPath); try { - const configured = TOML.parse(pyproject)?.tool?.pytest?.ini_options?.testpaths; - if (Array.isArray(configured)) paths.push(...configured); - else if (typeof configured === 'string') paths.push(...configured.split(/\s+/)); + const config = TOML.parse(text)?.pytest || {}; + return { + paths: normalizeTomlList(config.testpaths), + pythonFiles: normalizeTomlList(config.python_files), + }; } catch (_error) { - // Malformed project metadata is not evidence of a configured test path. + return null; } } - paths.push(...parseIniTestPaths(safeRead(path.join(rootDir, 'pytest.ini')), 'pytest')); - paths.push(...parseIniTestPaths(safeRead(path.join(rootDir, 'tox.ini')), 'pytest')); - paths.push(...parseIniTestPaths(safeRead(path.join(rootDir, 'setup.cfg')), 'tool:pytest')); + for (const fileName of ['pytest.ini', '.pytest.ini']) { + const configPath = path.join(rootDir, fileName); + if (!fs.existsSync(configPath)) continue; + const text = safeRead(configPath); + if (!text.trim() || /^\s*\[pytest]\s*$/m.test(text)) { + return { + paths: parseIniOption(text, 'pytest', 'testpaths'), + pythonFiles: parseIniOption(text, 'pytest', 'python_files'), + }; + } + } - return [...new Set(paths.filter(value => typeof value === 'string' && value.trim()))]; + const pyproject = safeRead(path.join(rootDir, 'pyproject.toml')); + if (/^\s*\[tool\.pytest(?:\.ini_options)?]\s*$/m.test(pyproject)) { + try { + const pytest = TOML.parse(pyproject)?.tool?.pytest || {}; + const config = pytest.ini_options || pytest; + return { + paths: normalizeTomlList(config.testpaths), + pythonFiles: normalizeTomlList(config.python_files), + }; + } catch (_error) { + return null; + } + } + + const tox = safeRead(path.join(rootDir, 'tox.ini')); + if (/^\s*\[pytest]\s*$/m.test(tox)) { + return { + paths: parseIniOption(tox, 'pytest', 'testpaths'), + pythonFiles: parseIniOption(tox, 'pytest', 'python_files'), + }; + } + + const setupCfg = safeRead(path.join(rootDir, 'setup.cfg')); + if (/^\s*\[tool:pytest]\s*$/m.test(setupCfg)) { + return { + paths: parseIniOption(setupCfg, 'tool:pytest', 'testpaths'), + pythonFiles: parseIniOption(setupCfg, 'tool:pytest', 'python_files'), + }; + } + + return null; } -function isPythonTestFile(fileName) { - return /^(test_.*|.*_test)\.py$/i.test(fileName); +function hasGlobSyntax(value) { + return /[*?[]/.test(value); +} + +function getGlobScanRoot(pattern) { + const segments = pattern.replace(/\\/g, '/').split('/'); + const staticSegments = segments.slice(0, segments.findIndex(hasGlobSyntax)); + return staticSegments.join('/') || '.'; +} + +function matchesGlobPath(relativePath, pattern) { + const candidate = relativePath.split(path.sep).join('/'); + const normalized = pattern.replace(/\\/g, '/').replace(/^\.\//, '').replace(/\/$/, ''); + let matcher; + try { + matcher = new RegExp(`^${globToRegex(normalized)}$`); + } catch (_error) { + return false; + } + const segments = candidate.split('/'); + return segments.some((_, index) => matcher.test(segments.slice(0, index + 1).join('/'))); +} + +function globMatchesAnyPath(rootDir, scanRoot, pattern) { + const projectRoot = fs.realpathSync(rootDir); + const absoluteRoot = path.resolve(projectRoot, scanRoot); + if (!fs.existsSync(absoluteRoot)) return false; + try { + const realRoot = fs.realpathSync(absoluteRoot); + const relative = path.relative(projectRoot, realRoot); + if ( + relative === '..' || + relative.startsWith(`..${path.sep}`) || + path.isAbsolute(relative) + ) { + return false; + } + } catch (_error) { + return false; + } + + function visit(current) { + const relative = path.relative(projectRoot, current); + if (relative && matchesGlobPath(relative, pattern)) return true; + + let entries; + try { + entries = fs.readdirSync(current, { withFileTypes: true }); + } catch (_error) { + return false; + } + return entries.some(entry => { + if (entry.isDirectory() && IGNORED_SCAN_DIRS.has(entry.name)) return false; + return visit(path.join(current, entry.name)); + }); + } + + return visit(absoluteRoot); } function countPythonTestFiles(rootDir) { - const roots = ['tests', ...getPytestTestPaths(rootDir)]; - return walkFiles(rootDir, roots, isPythonTestFile).size; + const config = getActivePytestConfig(rootDir); + const configured = config?.paths || []; + const patterns = config?.pythonFiles?.length > 0 + ? config.pythonFiles + : ['test_*.py', '*_test.py']; + const isPythonTestFile = fileName => patterns.some(pattern => + matchesGlobPath(fileName, pattern) + ); + let roots = configured.length > 0 ? configured : ['.']; + const directRoots = roots.filter(value => !hasGlobSyntax(value)); + let files = walkFiles(rootDir, directRoots, isPythonTestFile); + let configuredPathMatched = directRoots.some(relativeRoot => { + const candidate = path.resolve(fs.realpathSync(rootDir), relativeRoot); + const relative = path.relative(fs.realpathSync(rootDir), candidate); + return relative !== '..' && + !relative.startsWith(`..${path.sep}`) && + !path.isAbsolute(relative) && + fs.existsSync(candidate); + }); + + for (const pattern of roots.filter(hasGlobSyntax)) { + const scanRoot = getGlobScanRoot(pattern); + const normalizedPattern = pattern.replace(/\\/g, '/').replace(/\/$/, ''); + const recursiveRootMatch = normalizedPattern.endsWith('/**') && + fs.existsSync(path.resolve(fs.realpathSync(rootDir), scanRoot)); + const matching = walkFiles( + rootDir, + [scanRoot], + (fileName, filePath) => isPythonTestFile(fileName) && + matchesGlobPath(path.relative(fs.realpathSync(rootDir), filePath), pattern) + ); + const matchingPaths = walkFiles( + rootDir, + [scanRoot], + (_fileName, filePath) => matchesGlobPath( + path.relative(fs.realpathSync(rootDir), filePath), + pattern + ) + ); + configuredPathMatched = configuredPathMatched || + recursiveRootMatch || + matchingPaths.size > 0 || + globMatchesAnyPath(rootDir, scanRoot, pattern); + files = new Set([...files, ...matching]); + } + if (configured.length > 0 && !configuredPathMatched) { + roots = ['.']; + files = walkFiles(rootDir, roots, isPythonTestFile); + } + + return files.size; +} + +function toxInvokesPytest(text) { + if (!text) return false; + const lines = text.split(/\r?\n/); + let inTestEnv = false; + let collectingCommands = false; + + for (const line of lines) { + const section = line.match(/^\s*\[([^\]]+)]\s*$/); + if (section) { + inTestEnv = /^testenv(?::|$)/i.test(section[1]); + collectingCommands = false; + continue; + } + if (!inTestEnv || /^\s*[#;]/.test(line)) continue; + + const command = line.match(/^\s*commands\s*=\s*(.*)$/i); + if (command) { + collectingCommands = true; + if (command[1].trim().split(/\s+/).some(token => token === 'pytest')) return true; + continue; + } + if (collectingCommands && /^\s+\S/.test(line)) { + if (line.trim().split(/\s+/).some(token => token === 'pytest')) return true; + } else if (line.trim()) { + collectingCommands = false; + } + } + + return false; } function hasPythonTestSuite(rootDir, pythonTestCount = countPythonTestFiles(rootDir)) { - const pyproject = safeRead(path.join(rootDir, 'pyproject.toml')); - const setupCfg = safeRead(path.join(rootDir, 'setup.cfg')); - const hasConfig = ( - /^\s*\[tool\.pytest\.ini_options]\s*$/m.test(pyproject) - || /^\s*\[tool:pytest]\s*$/m.test(setupCfg) - || fs.existsSync(path.join(rootDir, 'pytest.ini')) - || fs.existsSync(path.join(rootDir, 'tox.ini')) - ); - return hasConfig || + return Boolean(getActivePytestConfig(rootDir)) || pythonTestCount > 0 || - walkFiles(rootDir, ['.'], fileName => fileName === 'conftest.py').size > 0; + walkFiles(rootDir, ['.'], fileName => fileName === 'conftest.py').size > 0 || + toxInvokesPytest(safeRead(path.join(rootDir, 'tox.ini'))); } function globToRegex(pattern) { @@ -170,12 +352,28 @@ function globToRegex(pattern) { for (let index = 0; index < pattern.length; index += 1) { const char = pattern[index]; if (char === '*' && pattern[index + 1] === '*') { - output += '.*'; - index += 1; + if (pattern[index + 2] === '/') { + output += '(?:.*/)?'; + index += 2; + } else { + output += '.*'; + index += 1; + } } else if (char === '*') { output += '[^/]*'; } else if (char === '?') { output += '[^/]'; + } else if (char === '[') { + const closing = pattern.indexOf(']', index + 1); + if (closing > index + 1) { + let content = pattern.slice(index + 1, closing); + const negated = content.startsWith('!') || content.startsWith('^'); + if (negated) content = content.slice(1); + output += `[${negated ? '^' : ''}${content.replace(/\\/g, '\\\\')}]`; + index = closing; + } else { + output += '\\['; + } } else { output += char.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); } @@ -183,18 +381,6 @@ function globToRegex(pattern) { return output; } -function matchesGitignorePattern(candidate, rawPattern) { - const directoryOnly = rawPattern.endsWith('/'); - const anchored = rawPattern.startsWith('/'); - const pattern = rawPattern.replace(/^\//, '').replace(/\/$/, ''); - if (!pattern) return false; - - const body = globToRegex(pattern); - const prefix = anchored || pattern.includes('/') ? '^' : '(^|.*/)'; - const suffix = directoryOnly ? '(/.*)?$' : '$'; - return new RegExp(`${prefix}${body}${suffix}`).test(candidate); -} - function gitignoreIgnoresEnvFiles(text) { if (typeof text !== 'string' || !text.trim()) return false; const candidates = [ @@ -205,29 +391,35 @@ function gitignoreIgnoresEnvFiles(text) { 'config/.env', 'secrets.env', ]; - const states = new Map(candidates.map(candidate => [candidate, false])); + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-gitignore-')); + const repoRoot = path.join(tempRoot, 'repo'); + const templateRoot = path.join(tempRoot, 'template'); - for (const rawLine of text.split(/\r?\n/)) { - let line = rawLine.trim(); - if (!line || line.startsWith('#')) continue; + try { + fs.mkdirSync(repoRoot); + fs.mkdirSync(templateRoot); + fs.writeFileSync(path.join(repoRoot, '.gitignore'), text); + const init = spawnSync('git', ['init', '--quiet', `--template=${templateRoot}`, repoRoot], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 5000, + }); + if (init.error || init.status !== 0) return false; - let negated = false; - if (line.startsWith('!')) { - negated = true; - line = line.slice(1); - } else if (line.startsWith('\\#') || line.startsWith('\\!')) { - line = line.slice(1); - } - if (!line) continue; - - for (const candidate of candidates) { - if (matchesGitignorePattern(candidate, line)) { - states.set(candidate, !negated); + const result = spawnSync( + 'git', + ['-C', repoRoot, '-c', `core.excludesFile=${os.devNull}`, 'check-ignore', '--no-index', '--stdin'], + { + input: `${candidates.join('\n')}\n`, + encoding: 'utf8', + stdio: ['pipe', 'pipe', 'pipe'], + timeout: 5000, } - } + ); + return !result.error && result.status === 0 && Boolean(result.stdout.trim()); + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }); } - - return [...states.values()].some(Boolean); } module.exports = { diff --git a/tests/scripts/harness-audit.test.js b/tests/scripts/harness-audit.test.js index a0ece0aee..f92500cad 100644 --- a/tests/scripts/harness-audit.test.js +++ b/tests/scripts/harness-audit.test.js @@ -459,8 +459,11 @@ function runTests() { if (test('recognizes standard pytest suite markers without JavaScript tests', () => { const homeDir = createTempDir('harness-audit-python-markers-home-'); const fixtures = [ - ['pytest.ini', '[pytest]\n'], - ['tox.ini', '[tox]\nenvlist = py\n'], + ['pytest.toml', ''], + ['.pytest.toml', ''], + ['pytest.ini', ''], + ['.pytest.ini', ''], + ['tox.ini', '[testenv]\ncommands = pytest\n'], ['setup.cfg', '[tool:pytest]\n'], ['quality/conftest.py', ''], ['tests/test_smoke.py', 'def test_smoke(): pass\n'], @@ -486,6 +489,127 @@ function runTests() { } })) passed++; else failed++; + if (test('supports spaced testpaths, custom python_files, and empty-path fallback', () => { + const homeDir = createTempDir('harness-audit-python-options-home-'); + const projectRoot = createTempDir('harness-audit-python-options-project-'); + + try { + fs.mkdirSync(path.join(projectRoot, 'quality suite'), { recursive: true }); + fs.mkdirSync(path.join(projectRoot, 'package'), { recursive: true }); + fs.writeFileSync( + path.join(projectRoot, '.pytest.ini'), + '[pytest]\ntestpaths = "quality suite"\npython_files = check_*.py\n' + ); + fs.writeFileSync(path.join(projectRoot, 'quality suite', 'check_one.py'), ''); + fs.writeFileSync(path.join(projectRoot, 'quality suite', 'test_default.py'), ''); + assert.strictEqual(countPythonTestFiles(projectRoot), 1); + + fs.writeFileSync( + path.join(projectRoot, '.pytest.ini'), + '[pytest]\ntestpaths = missing/**\npython_files = check_*.py\n' + ); + fs.writeFileSync(path.join(projectRoot, 'package', 'check_fallback.py'), ''); + assert.strictEqual(countPythonTestFiles(projectRoot), 2); + + fs.writeFileSync(path.join(projectRoot, 'pytest.toml'), '[pytest]\ntestpaths = ["package"]\npython_files = ["test_*.py"]\n'); + assert.strictEqual(countPythonTestFiles(projectRoot), 0); + + fs.mkdirSync(path.join(projectRoot, 'quality', 'empty'), { recursive: true }); + fs.writeFileSync( + path.join(projectRoot, 'pytest.toml'), + '[pytest]\ntestpaths = ["quality/*"]\n' + ); + assert.strictEqual(countPythonTestFiles(projectRoot), 0); + } finally { + cleanup(homeDir); + cleanup(projectRoot); + } + })) passed++; else failed++; + + if (test('ignores non-string pytest TOML list entries without crashing', () => { + const homeDir = createTempDir('harness-audit-python-invalid-home-'); + const projectRoot = createTempDir('harness-audit-python-invalid-project-'); + + try { + fs.mkdirSync(path.join(projectRoot, 'tests'), { recursive: true }); + fs.writeFileSync( + path.join(projectRoot, 'pytest.toml'), + '[pytest]\ntestpaths = ["tests", 42]\npython_files = ["test_*.py", false]\n' + ); + fs.writeFileSync(path.join(projectRoot, 'tests', 'test_valid.py'), ''); + + assert.strictEqual(countPythonTestFiles(projectRoot), 1); + assert.doesNotThrow(() => run(['repo', '--format', 'json'], { cwd: projectRoot, homeDir })); + } finally { + cleanup(homeDir); + cleanup(projectRoot); + } + })) passed++; else failed++; + + if (test('supports testpaths that point directly to a test file', () => { + const projectRoot = createTempDir('harness-audit-python-file-path-'); + + try { + fs.mkdirSync(path.join(projectRoot, 'quality'), { recursive: true }); + fs.writeFileSync(path.join(projectRoot, 'quality', 'test_exact.py'), ''); + fs.writeFileSync( + path.join(projectRoot, 'pytest.toml'), + '[pytest]\ntestpaths = ["quality/test_exact.py"]\n' + ); + assert.strictEqual(countPythonTestFiles(projectRoot), 1); + } finally { + cleanup(projectRoot); + } + })) passed++; else failed++; + + if (test('uses pytest default discovery and active config precedence', () => { + const homeDir = createTempDir('harness-audit-python-default-home-'); + const projectRoot = createTempDir('harness-audit-python-default-project-'); + + try { + fs.mkdirSync(path.join(projectRoot, 'package'), { recursive: true }); + for (const name of ['test_one.py', 'test_two.py', 'three_test.py']) { + fs.writeFileSync(path.join(projectRoot, 'package', name), ''); + } + assert.strictEqual(countPythonTestFiles(projectRoot), 3); + + fs.mkdirSync(path.join(projectRoot, 'active'), { recursive: true }); + fs.mkdirSync(path.join(projectRoot, 'inactive'), { recursive: true }); + fs.writeFileSync(path.join(projectRoot, 'pytest.ini'), '[pytest]\ntestpaths = active\n'); + fs.writeFileSync( + path.join(projectRoot, 'pyproject.toml'), + '[tool.pytest.ini_options]\ntestpaths = ["inactive"]\n' + ); + fs.writeFileSync(path.join(projectRoot, 'active', 'test_active.py'), ''); + fs.writeFileSync(path.join(projectRoot, 'inactive', 'test_inactive.py'), ''); + assert.strictEqual(countPythonTestFiles(projectRoot), 1); + + const parsed = JSON.parse(run(['repo', '--format', 'json'], { cwd: projectRoot, homeDir })); + assert.ok(parsed.checks.some(check => check.id === 'consumer-test-suite' && check.pass)); + assert.ok(parsed.checks.some(check => check.id === 'consumer-eval-coverage' && !check.pass)); + } finally { + cleanup(homeDir); + cleanup(projectRoot); + } + })) passed++; else failed++; + + if (test('does not treat a lint-only tox configuration as a test suite', () => { + const homeDir = createTempDir('harness-audit-tox-home-'); + const projectRoot = createTempDir('harness-audit-tox-project-'); + + try { + fs.writeFileSync( + path.join(projectRoot, 'tox.ini'), + '[tox]\nenvlist = lint\n[testenv:lint]\ndeps = pytest\ncommands = ruff check .\n' + ); + const parsed = JSON.parse(run(['repo', '--format', 'json'], { cwd: projectRoot, homeDir })); + assert.ok(parsed.checks.some(check => check.id === 'consumer-test-suite' && !check.pass)); + } finally { + cleanup(homeDir); + cleanup(projectRoot); + } + })) passed++; else failed++; + if (test('counts tests from INI testpaths without scanning ignored dependency roots', () => { const homeDir = createTempDir('harness-audit-python-ini-home-'); const projectRoot = createTempDir('harness-audit-python-ini-project-'); @@ -512,11 +636,56 @@ function runTests() { } })) passed++; else failed++; + if (test('expands recursive glob patterns in pytest testpaths', () => { + const homeDir = createTempDir('harness-audit-python-glob-home-'); + const projectRoot = createTempDir('harness-audit-python-glob-project-'); + + try { + fs.mkdirSync(path.join(projectRoot, 'quality', 'unit'), { recursive: true }); + fs.writeFileSync( + path.join(projectRoot, 'pyproject.toml'), + '[tool.pytest.ini_options]\ntestpaths = ["quality/**"]\n' + ); + for (const name of ['test_one.py', 'test_two.py', 'three_test.py']) { + fs.writeFileSync(path.join(projectRoot, 'quality', 'unit', name), ''); + } + + assert.strictEqual(countPythonTestFiles(projectRoot), 3); + const parsed = JSON.parse(run(['repo', '--format', 'json'], { cwd: projectRoot, homeDir })); + assert.ok(parsed.checks.some(check => check.id === 'consumer-eval-coverage' && check.pass)); + } finally { + cleanup(homeDir); + cleanup(projectRoot); + } + })) passed++; else failed++; + + if (test('does not fall back when a recursive testpath matches an empty directory', () => { + const projectRoot = createTempDir('harness-audit-python-empty-glob-'); + + try { + fs.mkdirSync(path.join(projectRoot, 'quality'), { recursive: true }); + fs.mkdirSync(path.join(projectRoot, 'package'), { recursive: true }); + fs.writeFileSync( + path.join(projectRoot, 'pytest.toml'), + '[pytest]\ntestpaths = ["quality/**"]\n' + ); + for (const name of ['test_one.py', 'test_two.py', 'three_test.py']) { + fs.writeFileSync(path.join(projectRoot, 'package', name), ''); + } + + assert.strictEqual(countPythonTestFiles(projectRoot), 0); + } finally { + cleanup(projectRoot); + } + })) passed++; else failed++; + if (test('secret hygiene requires an active gitignore rule for env files', () => { const homeDir = createTempDir('harness-audit-gitignore-home-'); const cases = [ ['# Do not commit .env files\nnode_modules/\n', false], ['environment.json\n.envexample\n', false], + ['[z-a]env\n', false], + ['[z-a]env\n', false], ['.env.example\n', false], ['.env\n', true], ['.env.local\n', true], @@ -524,6 +693,10 @@ function runTests() { ['*.env\n', true], ['.env/\n', true], ['config/.env\n', true], + ['[.]env\n', true], + ['**/secrets.env\n', true], + ['secrets/\n!secrets/.env\n', false], + [' [.]env\n', false], ['.env\n!.env\n', false], ['.env*\n!.env.example\n', true], ]; From c9dd92037d257ea76bc47634d600369127bd9a13 Mon Sep 17 00:00:00 2001 From: wellkilo Date: Mon, 7 Sep 2026 16:04:42 +0800 Subject: [PATCH 3/3] fix(audit): mirror pytest discovery rules --- scripts/lib/harness-audit-consumer.js | 37 +++++++++++++++++---------- tests/scripts/harness-audit.test.js | 9 +++++-- 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/scripts/lib/harness-audit-consumer.js b/scripts/lib/harness-audit-consumer.js index cafe5cff9..0bb51516d 100644 --- a/scripts/lib/harness-audit-consumer.js +++ b/scripts/lib/harness-audit-consumer.js @@ -154,12 +154,10 @@ function getActivePytestConfig(rootDir) { const configPath = path.join(rootDir, fileName); if (!fs.existsSync(configPath)) continue; const text = safeRead(configPath); - if (!text.trim() || /^\s*\[pytest]\s*$/m.test(text)) { - return { - paths: parseIniOption(text, 'pytest', 'testpaths'), - pythonFiles: parseIniOption(text, 'pytest', 'python_files'), - }; - } + return { + paths: parseIniOption(text, 'pytest', 'testpaths'), + pythonFiles: parseIniOption(text, 'pytest', 'python_files'), + }; } const pyproject = safeRead(path.join(rootDir, 'pyproject.toml')); @@ -391,24 +389,30 @@ function gitignoreIgnoresEnvFiles(text) { 'config/.env', 'secrets.env', ]; - const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-gitignore-')); - const repoRoot = path.join(tempRoot, 'repo'); - const templateRoot = path.join(tempRoot, 'template'); + let tempRoot; try { + tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-gitignore-')); + const repoRoot = path.join(tempRoot, 'repo'); + const templateRoot = path.join(tempRoot, 'template'); + const globalExcludesPath = path.join(tempRoot, 'global-excludes'); fs.mkdirSync(repoRoot); fs.mkdirSync(templateRoot); + fs.writeFileSync(globalExcludesPath, ''); fs.writeFileSync(path.join(repoRoot, '.gitignore'), text); const init = spawnSync('git', ['init', '--quiet', `--template=${templateRoot}`, repoRoot], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], timeout: 5000, }); - if (init.error || init.status !== 0) return false; + if (init.error || init.status !== 0) { + process.stderr.write('[harness-audit] Unable to initialize isolated gitignore check\n'); + return false; + } const result = spawnSync( 'git', - ['-C', repoRoot, '-c', `core.excludesFile=${os.devNull}`, 'check-ignore', '--no-index', '--stdin'], + ['-C', repoRoot, '-c', `core.excludesFile=${globalExcludesPath}`, 'check-ignore', '--no-index', '--stdin'], { input: `${candidates.join('\n')}\n`, encoding: 'utf8', @@ -416,9 +420,16 @@ function gitignoreIgnoresEnvFiles(text) { timeout: 5000, } ); - return !result.error && result.status === 0 && Boolean(result.stdout.trim()); + if (result.error || ![0, 1].includes(result.status)) { + process.stderr.write('[harness-audit] Unable to evaluate gitignore rules with git\n'); + return false; + } + return result.status === 0 && Boolean(result.stdout.trim()); + } catch (_error) { + process.stderr.write('[harness-audit] Unable to prepare isolated gitignore check\n'); + return false; } finally { - fs.rmSync(tempRoot, { recursive: true, force: true }); + if (tempRoot) fs.rmSync(tempRoot, { recursive: true, force: true }); } } diff --git a/tests/scripts/harness-audit.test.js b/tests/scripts/harness-audit.test.js index f92500cad..f0fb56c1b 100644 --- a/tests/scripts/harness-audit.test.js +++ b/tests/scripts/harness-audit.test.js @@ -583,10 +583,16 @@ function runTests() { fs.writeFileSync(path.join(projectRoot, 'active', 'test_active.py'), ''); fs.writeFileSync(path.join(projectRoot, 'inactive', 'test_inactive.py'), ''); assert.strictEqual(countPythonTestFiles(projectRoot), 1); - const parsed = JSON.parse(run(['repo', '--format', 'json'], { cwd: projectRoot, homeDir })); assert.ok(parsed.checks.some(check => check.id === 'consumer-test-suite' && check.pass)); assert.ok(parsed.checks.some(check => check.id === 'consumer-eval-coverage' && !check.pass)); + + fs.writeFileSync(path.join(projectRoot, 'pytest.ini'), '[invalid]\nvalue = true\n'); + assert.strictEqual( + countPythonTestFiles(projectRoot), + 5, + 'Existing pytest.ini must win by presence and use default root discovery' + ); } finally { cleanup(homeDir); cleanup(projectRoot); @@ -685,7 +691,6 @@ function runTests() { ['# Do not commit .env files\nnode_modules/\n', false], ['environment.json\n.envexample\n', false], ['[z-a]env\n', false], - ['[z-a]env\n', false], ['.env.example\n', false], ['.env\n', true], ['.env.local\n', true],