diff --git a/scripts/harness-audit.js b/scripts/harness-audit.js index 58b1f1716..5393b05cf 100644 --- a/scripts/harness-audit.js +++ b/scripts/harness-audit.js @@ -3,6 +3,11 @@ const fs = require('fs'); const os = require('os'); const path = require('path'); +const { + countPythonTestFiles, + gitignoreIgnoresEnvFiles, + hasPythonTestSuite, +} = require('./lib/harness-audit-consumer'); const CATEGORIES = [ 'Tool Coverage', @@ -826,6 +831,7 @@ function getConsumerChecks(rootDir) { const gitignore = safeRead(rootDir, '.gitignore'); const projectHooks = safeRead(rootDir, '.claude/settings.json'); const pluginInstall = findPluginInstall(rootDir); + const pythonTestCount = countPythonTestFiles(rootDir); return [ { @@ -879,7 +885,10 @@ function getConsumerChecks(rootDir) { scopes: ['repo'], path: 'tests/', description: 'The project has an automated test entrypoint', - pass: typeof packageJson?.scripts?.test === 'string' || countFiles(rootDir, 'tests', '.test.js') > 0 || hasFileWithExtension(rootDir, '.', ['.spec.js', '.spec.ts', '.test.ts']), + pass: typeof packageJson?.scripts?.test === 'string' || + countFiles(rootDir, 'tests', '.test.js') > 0 || + hasFileWithExtension(rootDir, '.', ['.spec.js', '.spec.ts', '.test.ts']) || + hasPythonTestSuite(rootDir, pythonTestCount), fix: 'Add a test script or checked-in tests so harness recommendations can be verified automatically.', }, { @@ -909,7 +918,8 @@ function getConsumerChecks(rootDir) { scopes: ['repo'], path: 'evals/', description: 'The project has evals or multiple automated tests', - pass: countFiles(rootDir, 'evals', null) > 0 || countFiles(rootDir, 'tests', '.test.js') >= 3, + pass: countFiles(rootDir, 'evals', null) > 0 || + countFiles(rootDir, 'tests', '.test.js') + pythonTestCount >= 3, fix: 'Add eval fixtures or at least a few focused automated tests for critical flows.', }, { @@ -929,7 +939,7 @@ function getConsumerChecks(rootDir) { scopes: ['repo'], path: '.gitignore', description: 'The project ignores common secret env files', - pass: gitignore.includes('.env'), + pass: gitignoreIgnoresEnvFiles(gitignore), fix: 'Ignore .env-style files in .gitignore so secrets do not land in the repo.', }, { diff --git a/scripts/lib/harness-audit-consumer.js b/scripts/lib/harness-audit-consumer.js new file mode 100644 index 000000000..0bb51516d --- /dev/null +++ b/scripts/lib/harness-audit-consumer.js @@ -0,0 +1,440 @@ +'use strict'; + +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { spawnSync } = require('child_process'); +const TOML = require('@iarna/toml'); + +const IGNORED_SCAN_DIRS = new Set([ + '.git', + '.eggs', + '.mypy_cache', + '.nox', + '.pytest_cache', + '.ruff_cache', + '.tox', + '.venv', + '__pycache__', + '__pypackages__', + 'build', + 'dist', + 'env', + 'node_modules', + 'site-packages', + 'venv', + 'vendor', +]); + +function safeRead(filePath) { + try { + return fs.readFileSync(filePath, 'utf8'); + } catch (_error) { + return ''; + } +} + +function walkFiles(rootDir, relativeRoots, predicate) { + const projectRoot = fs.realpathSync(path.resolve(rootDir)); + + function visit(current) { + let entries; + try { + entries = fs.readdirSync(current, { withFileTypes: true }); + } catch (_error) { + return []; + } + + return entries.flatMap(entry => { + const nextPath = path.join(current, entry.name); + if (entry.isDirectory()) { + return IGNORED_SCAN_DIRS.has(entry.name) ? [] : visit(nextPath); + } + return entry.isFile() && predicate(entry.name, nextPath) ? [nextPath] : []; + }); + } + + const files = relativeRoots.flatMap(relativeRoot => { + const scanRoot = path.resolve(projectRoot, relativeRoot); + const relative = path.relative(projectRoot, scanRoot); + const segments = relative.split(path.sep).filter(Boolean); + if ( + relative === '..' || + relative.startsWith(`..${path.sep}`) || + path.isAbsolute(relative) || + segments.some(segment => IGNORED_SCAN_DIRS.has(segment)) || + !fs.existsSync(scanRoot) + ) { + return []; + } + try { + const realRoot = fs.realpathSync(scanRoot); + const realRelative = path.relative(projectRoot, realRoot); + if ( + realRelative === '..' || + realRelative.startsWith(`..${path.sep}`) || + path.isAbsolute(realRelative) + ) { + return []; + } + } catch (_error) { + return []; + } + if (fs.statSync(scanRoot).isFile()) { + return predicate(path.basename(scanRoot), scanRoot) ? [scanRoot] : []; + } + return visit(scanRoot); + }); + + return new Set(files); +} + +function parseIniOption(text, sectionName, optionName) { + if (!text) return []; + const sectionPattern = new RegExp( + `^\\s*\\[${sectionName.replace(/[.*+?^${}()|[\\]\\]/g, '\\$&')}\\]\\s*$`, + 'i' + ); + const lines = text.split(/\r?\n/); + let inSection = false; + let collecting = false; + let values = []; + + for (const line of lines) { + if (/^\s*\[.*]\s*$/.test(line)) { + inSection = sectionPattern.test(line); + collecting = false; + continue; + } + if (!inSection || /^\s*[#;]/.test(line)) continue; + + const match = line.match(new RegExp(`^\\s*${optionName}\\s*=\\s*(.*)$`, 'i')); + if (match) { + collecting = true; + values = [...values, match[1]]; + continue; + } + if (collecting && /^\s+\S/.test(line)) { + values = [...values, line.trim()]; + } else if (line.trim()) { + collecting = false; + } + } + + return values + .flatMap(value => value.match(/"[^"]*"|'[^']*'|\S+/g) || []) + .map(value => value.replace(/^['"]|['"]$/g, '').trim()) + .filter(Boolean); +} + +function normalizeTomlList(value) { + if (Array.isArray(value)) { + return value.filter(item => typeof item === 'string' && item.trim()); + } + return typeof value === 'string' ? value.split(/\s+/) : []; +} + +function getActivePytestConfig(rootDir) { + for (const fileName of ['pytest.toml', '.pytest.toml']) { + const configPath = path.join(rootDir, fileName); + if (!fs.existsSync(configPath)) continue; + const text = safeRead(configPath); + try { + const config = TOML.parse(text)?.pytest || {}; + return { + paths: normalizeTomlList(config.testpaths), + pythonFiles: normalizeTomlList(config.python_files), + }; + } catch (_error) { + return null; + } + } + + for (const fileName of ['pytest.ini', '.pytest.ini']) { + const configPath = path.join(rootDir, fileName); + if (!fs.existsSync(configPath)) continue; + const text = safeRead(configPath); + return { + paths: parseIniOption(text, 'pytest', 'testpaths'), + pythonFiles: parseIniOption(text, 'pytest', 'python_files'), + }; + } + + const pyproject = safeRead(path.join(rootDir, 'pyproject.toml')); + if (/^\s*\[tool\.pytest(?:\.ini_options)?]\s*$/m.test(pyproject)) { + try { + const pytest = TOML.parse(pyproject)?.tool?.pytest || {}; + const config = pytest.ini_options || pytest; + return { + paths: normalizeTomlList(config.testpaths), + pythonFiles: normalizeTomlList(config.python_files), + }; + } catch (_error) { + return null; + } + } + + const tox = safeRead(path.join(rootDir, 'tox.ini')); + if (/^\s*\[pytest]\s*$/m.test(tox)) { + return { + paths: parseIniOption(tox, 'pytest', 'testpaths'), + pythonFiles: parseIniOption(tox, 'pytest', 'python_files'), + }; + } + + const setupCfg = safeRead(path.join(rootDir, 'setup.cfg')); + if (/^\s*\[tool:pytest]\s*$/m.test(setupCfg)) { + return { + paths: parseIniOption(setupCfg, 'tool:pytest', 'testpaths'), + pythonFiles: parseIniOption(setupCfg, 'tool:pytest', 'python_files'), + }; + } + + return null; +} + +function hasGlobSyntax(value) { + return /[*?[]/.test(value); +} + +function getGlobScanRoot(pattern) { + const segments = pattern.replace(/\\/g, '/').split('/'); + const staticSegments = segments.slice(0, segments.findIndex(hasGlobSyntax)); + return staticSegments.join('/') || '.'; +} + +function matchesGlobPath(relativePath, pattern) { + const candidate = relativePath.split(path.sep).join('/'); + const normalized = pattern.replace(/\\/g, '/').replace(/^\.\//, '').replace(/\/$/, ''); + let matcher; + try { + matcher = new RegExp(`^${globToRegex(normalized)}$`); + } catch (_error) { + return false; + } + const segments = candidate.split('/'); + return segments.some((_, index) => matcher.test(segments.slice(0, index + 1).join('/'))); +} + +function globMatchesAnyPath(rootDir, scanRoot, pattern) { + const projectRoot = fs.realpathSync(rootDir); + const absoluteRoot = path.resolve(projectRoot, scanRoot); + if (!fs.existsSync(absoluteRoot)) return false; + try { + const realRoot = fs.realpathSync(absoluteRoot); + const relative = path.relative(projectRoot, realRoot); + if ( + relative === '..' || + relative.startsWith(`..${path.sep}`) || + path.isAbsolute(relative) + ) { + return false; + } + } catch (_error) { + return false; + } + + function visit(current) { + const relative = path.relative(projectRoot, current); + if (relative && matchesGlobPath(relative, pattern)) return true; + + let entries; + try { + entries = fs.readdirSync(current, { withFileTypes: true }); + } catch (_error) { + return false; + } + return entries.some(entry => { + if (entry.isDirectory() && IGNORED_SCAN_DIRS.has(entry.name)) return false; + return visit(path.join(current, entry.name)); + }); + } + + return visit(absoluteRoot); +} + +function countPythonTestFiles(rootDir) { + const config = getActivePytestConfig(rootDir); + const configured = config?.paths || []; + const patterns = config?.pythonFiles?.length > 0 + ? config.pythonFiles + : ['test_*.py', '*_test.py']; + const isPythonTestFile = fileName => patterns.some(pattern => + matchesGlobPath(fileName, pattern) + ); + let roots = configured.length > 0 ? configured : ['.']; + const directRoots = roots.filter(value => !hasGlobSyntax(value)); + let files = walkFiles(rootDir, directRoots, isPythonTestFile); + let configuredPathMatched = directRoots.some(relativeRoot => { + const candidate = path.resolve(fs.realpathSync(rootDir), relativeRoot); + const relative = path.relative(fs.realpathSync(rootDir), candidate); + return relative !== '..' && + !relative.startsWith(`..${path.sep}`) && + !path.isAbsolute(relative) && + fs.existsSync(candidate); + }); + + for (const pattern of roots.filter(hasGlobSyntax)) { + const scanRoot = getGlobScanRoot(pattern); + const normalizedPattern = pattern.replace(/\\/g, '/').replace(/\/$/, ''); + const recursiveRootMatch = normalizedPattern.endsWith('/**') && + fs.existsSync(path.resolve(fs.realpathSync(rootDir), scanRoot)); + const matching = walkFiles( + rootDir, + [scanRoot], + (fileName, filePath) => isPythonTestFile(fileName) && + matchesGlobPath(path.relative(fs.realpathSync(rootDir), filePath), pattern) + ); + const matchingPaths = walkFiles( + rootDir, + [scanRoot], + (_fileName, filePath) => matchesGlobPath( + path.relative(fs.realpathSync(rootDir), filePath), + pattern + ) + ); + configuredPathMatched = configuredPathMatched || + recursiveRootMatch || + matchingPaths.size > 0 || + globMatchesAnyPath(rootDir, scanRoot, pattern); + files = new Set([...files, ...matching]); + } + if (configured.length > 0 && !configuredPathMatched) { + roots = ['.']; + files = walkFiles(rootDir, roots, isPythonTestFile); + } + + return files.size; +} + +function toxInvokesPytest(text) { + if (!text) return false; + const lines = text.split(/\r?\n/); + let inTestEnv = false; + let collectingCommands = false; + + for (const line of lines) { + const section = line.match(/^\s*\[([^\]]+)]\s*$/); + if (section) { + inTestEnv = /^testenv(?::|$)/i.test(section[1]); + collectingCommands = false; + continue; + } + if (!inTestEnv || /^\s*[#;]/.test(line)) continue; + + const command = line.match(/^\s*commands\s*=\s*(.*)$/i); + if (command) { + collectingCommands = true; + if (command[1].trim().split(/\s+/).some(token => token === 'pytest')) return true; + continue; + } + if (collectingCommands && /^\s+\S/.test(line)) { + if (line.trim().split(/\s+/).some(token => token === 'pytest')) return true; + } else if (line.trim()) { + collectingCommands = false; + } + } + + return false; +} + +function hasPythonTestSuite(rootDir, pythonTestCount = countPythonTestFiles(rootDir)) { + return Boolean(getActivePytestConfig(rootDir)) || + pythonTestCount > 0 || + walkFiles(rootDir, ['.'], fileName => fileName === 'conftest.py').size > 0 || + toxInvokesPytest(safeRead(path.join(rootDir, 'tox.ini'))); +} + +function globToRegex(pattern) { + let output = ''; + for (let index = 0; index < pattern.length; index += 1) { + const char = pattern[index]; + if (char === '*' && pattern[index + 1] === '*') { + if (pattern[index + 2] === '/') { + output += '(?:.*/)?'; + index += 2; + } else { + output += '.*'; + index += 1; + } + } else if (char === '*') { + output += '[^/]*'; + } else if (char === '?') { + output += '[^/]'; + } else if (char === '[') { + const closing = pattern.indexOf(']', index + 1); + if (closing > index + 1) { + let content = pattern.slice(index + 1, closing); + const negated = content.startsWith('!') || content.startsWith('^'); + if (negated) content = content.slice(1); + output += `[${negated ? '^' : ''}${content.replace(/\\/g, '\\\\')}]`; + index = closing; + } else { + output += '\\['; + } + } else { + output += char.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + } + } + return output; +} + +function gitignoreIgnoresEnvFiles(text) { + if (typeof text !== 'string' || !text.trim()) return false; + const candidates = [ + '.env', + '.env.local', + '.env.production', + '.env/example', + 'config/.env', + 'secrets.env', + ]; + let tempRoot; + + try { + tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-gitignore-')); + const repoRoot = path.join(tempRoot, 'repo'); + const templateRoot = path.join(tempRoot, 'template'); + const globalExcludesPath = path.join(tempRoot, 'global-excludes'); + fs.mkdirSync(repoRoot); + fs.mkdirSync(templateRoot); + fs.writeFileSync(globalExcludesPath, ''); + fs.writeFileSync(path.join(repoRoot, '.gitignore'), text); + const init = spawnSync('git', ['init', '--quiet', `--template=${templateRoot}`, repoRoot], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 5000, + }); + if (init.error || init.status !== 0) { + process.stderr.write('[harness-audit] Unable to initialize isolated gitignore check\n'); + return false; + } + + const result = spawnSync( + 'git', + ['-C', repoRoot, '-c', `core.excludesFile=${globalExcludesPath}`, 'check-ignore', '--no-index', '--stdin'], + { + input: `${candidates.join('\n')}\n`, + encoding: 'utf8', + stdio: ['pipe', 'pipe', 'pipe'], + timeout: 5000, + } + ); + if (result.error || ![0, 1].includes(result.status)) { + process.stderr.write('[harness-audit] Unable to evaluate gitignore rules with git\n'); + return false; + } + return result.status === 0 && Boolean(result.stdout.trim()); + } catch (_error) { + process.stderr.write('[harness-audit] Unable to prepare isolated gitignore check\n'); + return false; + } finally { + if (tempRoot) fs.rmSync(tempRoot, { recursive: true, force: true }); + } +} + +module.exports = { + countPythonTestFiles, + gitignoreIgnoresEnvFiles, + hasPythonTestSuite, +}; diff --git a/tests/scripts/harness-audit.test.js b/tests/scripts/harness-audit.test.js index c0087475d..f0fb56c1b 100644 --- a/tests/scripts/harness-audit.test.js +++ b/tests/scripts/harness-audit.test.js @@ -10,6 +10,7 @@ const { execFileSync, spawnSync } = require('child_process'); const SCRIPT = path.join(__dirname, '..', '..', 'scripts', 'harness-audit.js'); const { parseArgs, findPluginInstall, compareVersionDesc } = require(SCRIPT); +const { countPythonTestFiles } = require('../../scripts/lib/harness-audit-consumer'); function createTempDir(prefix) { return fs.mkdtempSync(path.join(os.tmpdir(), prefix)); @@ -425,6 +426,303 @@ function runTests() { } })) passed++; else failed++; + if (test('combines Python and JavaScript tests for eval coverage', () => { + const homeDir = createTempDir('harness-audit-python-home-'); + const projectRoot = createTempDir('harness-audit-python-project-'); + + try { + fs.mkdirSync(path.join(projectRoot, 'qa'), { recursive: true }); + fs.mkdirSync(path.join(projectRoot, 'integration'), { recursive: true }); + fs.mkdirSync(path.join(projectRoot, 'tests'), { recursive: true }); + fs.mkdirSync(path.join(projectRoot, '.venv', 'tests'), { recursive: true }); + fs.mkdirSync(path.join(projectRoot, 'node_modules', 'pkg', 'tests'), { recursive: true }); + fs.writeFileSync( + path.join(projectRoot, 'pyproject.toml'), + '[tool.pytest.ini_options]\ntestpaths = ["qa", "integration"]\n' + ); + fs.writeFileSync(path.join(projectRoot, 'qa', 'test_alpha.py'), 'def test_alpha(): pass\n'); + fs.writeFileSync(path.join(projectRoot, 'qa', 'beta_test.py'), 'def test_beta(): pass\n'); + fs.writeFileSync(path.join(projectRoot, 'tests', 'gamma.test.js'), 'test placeholder\n'); + fs.writeFileSync(path.join(projectRoot, '.venv', 'tests', 'test_vendor.py'), ''); + fs.writeFileSync(path.join(projectRoot, 'node_modules', 'pkg', 'tests', 'test_vendor.py'), ''); + + assert.strictEqual(countPythonTestFiles(projectRoot), 2); + const parsed = JSON.parse(run(['repo', '--format', 'json'], { cwd: projectRoot, homeDir })); + assert.ok(parsed.checks.some(check => check.id === 'consumer-test-suite' && check.pass)); + assert.ok(parsed.checks.some(check => check.id === 'consumer-eval-coverage' && check.pass)); + } finally { + cleanup(homeDir); + cleanup(projectRoot); + } + })) passed++; else failed++; + + if (test('recognizes standard pytest suite markers without JavaScript tests', () => { + const homeDir = createTempDir('harness-audit-python-markers-home-'); + const fixtures = [ + ['pytest.toml', ''], + ['.pytest.toml', ''], + ['pytest.ini', ''], + ['.pytest.ini', ''], + ['tox.ini', '[testenv]\ncommands = pytest\n'], + ['setup.cfg', '[tool:pytest]\n'], + ['quality/conftest.py', ''], + ['tests/test_smoke.py', 'def test_smoke(): pass\n'], + ]; + + try { + for (const [relativePath, contents] of fixtures) { + const projectRoot = createTempDir('harness-audit-python-marker-'); + try { + fs.mkdirSync(path.dirname(path.join(projectRoot, relativePath)), { recursive: true }); + fs.writeFileSync(path.join(projectRoot, relativePath), contents); + const parsed = JSON.parse(run(['repo', '--format', 'json'], { cwd: projectRoot, homeDir })); + assert.ok( + parsed.checks.some(check => check.id === 'consumer-test-suite' && check.pass), + `${relativePath} should identify a Python test suite` + ); + } finally { + cleanup(projectRoot); + } + } + } finally { + cleanup(homeDir); + } + })) passed++; else failed++; + + if (test('supports spaced testpaths, custom python_files, and empty-path fallback', () => { + const homeDir = createTempDir('harness-audit-python-options-home-'); + const projectRoot = createTempDir('harness-audit-python-options-project-'); + + try { + fs.mkdirSync(path.join(projectRoot, 'quality suite'), { recursive: true }); + fs.mkdirSync(path.join(projectRoot, 'package'), { recursive: true }); + fs.writeFileSync( + path.join(projectRoot, '.pytest.ini'), + '[pytest]\ntestpaths = "quality suite"\npython_files = check_*.py\n' + ); + fs.writeFileSync(path.join(projectRoot, 'quality suite', 'check_one.py'), ''); + fs.writeFileSync(path.join(projectRoot, 'quality suite', 'test_default.py'), ''); + assert.strictEqual(countPythonTestFiles(projectRoot), 1); + + fs.writeFileSync( + path.join(projectRoot, '.pytest.ini'), + '[pytest]\ntestpaths = missing/**\npython_files = check_*.py\n' + ); + fs.writeFileSync(path.join(projectRoot, 'package', 'check_fallback.py'), ''); + assert.strictEqual(countPythonTestFiles(projectRoot), 2); + + fs.writeFileSync(path.join(projectRoot, 'pytest.toml'), '[pytest]\ntestpaths = ["package"]\npython_files = ["test_*.py"]\n'); + assert.strictEqual(countPythonTestFiles(projectRoot), 0); + + fs.mkdirSync(path.join(projectRoot, 'quality', 'empty'), { recursive: true }); + fs.writeFileSync( + path.join(projectRoot, 'pytest.toml'), + '[pytest]\ntestpaths = ["quality/*"]\n' + ); + assert.strictEqual(countPythonTestFiles(projectRoot), 0); + } finally { + cleanup(homeDir); + cleanup(projectRoot); + } + })) passed++; else failed++; + + if (test('ignores non-string pytest TOML list entries without crashing', () => { + const homeDir = createTempDir('harness-audit-python-invalid-home-'); + const projectRoot = createTempDir('harness-audit-python-invalid-project-'); + + try { + fs.mkdirSync(path.join(projectRoot, 'tests'), { recursive: true }); + fs.writeFileSync( + path.join(projectRoot, 'pytest.toml'), + '[pytest]\ntestpaths = ["tests", 42]\npython_files = ["test_*.py", false]\n' + ); + fs.writeFileSync(path.join(projectRoot, 'tests', 'test_valid.py'), ''); + + assert.strictEqual(countPythonTestFiles(projectRoot), 1); + assert.doesNotThrow(() => run(['repo', '--format', 'json'], { cwd: projectRoot, homeDir })); + } finally { + cleanup(homeDir); + cleanup(projectRoot); + } + })) passed++; else failed++; + + if (test('supports testpaths that point directly to a test file', () => { + const projectRoot = createTempDir('harness-audit-python-file-path-'); + + try { + fs.mkdirSync(path.join(projectRoot, 'quality'), { recursive: true }); + fs.writeFileSync(path.join(projectRoot, 'quality', 'test_exact.py'), ''); + fs.writeFileSync( + path.join(projectRoot, 'pytest.toml'), + '[pytest]\ntestpaths = ["quality/test_exact.py"]\n' + ); + assert.strictEqual(countPythonTestFiles(projectRoot), 1); + } finally { + cleanup(projectRoot); + } + })) passed++; else failed++; + + if (test('uses pytest default discovery and active config precedence', () => { + const homeDir = createTempDir('harness-audit-python-default-home-'); + const projectRoot = createTempDir('harness-audit-python-default-project-'); + + try { + fs.mkdirSync(path.join(projectRoot, 'package'), { recursive: true }); + for (const name of ['test_one.py', 'test_two.py', 'three_test.py']) { + fs.writeFileSync(path.join(projectRoot, 'package', name), ''); + } + assert.strictEqual(countPythonTestFiles(projectRoot), 3); + + fs.mkdirSync(path.join(projectRoot, 'active'), { recursive: true }); + fs.mkdirSync(path.join(projectRoot, 'inactive'), { recursive: true }); + fs.writeFileSync(path.join(projectRoot, 'pytest.ini'), '[pytest]\ntestpaths = active\n'); + fs.writeFileSync( + path.join(projectRoot, 'pyproject.toml'), + '[tool.pytest.ini_options]\ntestpaths = ["inactive"]\n' + ); + fs.writeFileSync(path.join(projectRoot, 'active', 'test_active.py'), ''); + fs.writeFileSync(path.join(projectRoot, 'inactive', 'test_inactive.py'), ''); + assert.strictEqual(countPythonTestFiles(projectRoot), 1); + const parsed = JSON.parse(run(['repo', '--format', 'json'], { cwd: projectRoot, homeDir })); + assert.ok(parsed.checks.some(check => check.id === 'consumer-test-suite' && check.pass)); + assert.ok(parsed.checks.some(check => check.id === 'consumer-eval-coverage' && !check.pass)); + + fs.writeFileSync(path.join(projectRoot, 'pytest.ini'), '[invalid]\nvalue = true\n'); + assert.strictEqual( + countPythonTestFiles(projectRoot), + 5, + 'Existing pytest.ini must win by presence and use default root discovery' + ); + } finally { + cleanup(homeDir); + cleanup(projectRoot); + } + })) passed++; else failed++; + + if (test('does not treat a lint-only tox configuration as a test suite', () => { + const homeDir = createTempDir('harness-audit-tox-home-'); + const projectRoot = createTempDir('harness-audit-tox-project-'); + + try { + fs.writeFileSync( + path.join(projectRoot, 'tox.ini'), + '[tox]\nenvlist = lint\n[testenv:lint]\ndeps = pytest\ncommands = ruff check .\n' + ); + const parsed = JSON.parse(run(['repo', '--format', 'json'], { cwd: projectRoot, homeDir })); + assert.ok(parsed.checks.some(check => check.id === 'consumer-test-suite' && !check.pass)); + } finally { + cleanup(homeDir); + cleanup(projectRoot); + } + })) passed++; else failed++; + + if (test('counts tests from INI testpaths without scanning ignored dependency roots', () => { + const homeDir = createTempDir('harness-audit-python-ini-home-'); + const projectRoot = createTempDir('harness-audit-python-ini-project-'); + + try { + fs.mkdirSync(path.join(projectRoot, 'quality'), { recursive: true }); + fs.mkdirSync(path.join(projectRoot, '.venv', 'tests'), { recursive: true }); + fs.writeFileSync( + path.join(projectRoot, 'pytest.ini'), + '[pytest]\ntestpaths =\n quality\n .venv/tests\n' + ); + for (const name of ['test_one.py', 'test_two.py', 'three_test.py']) { + fs.writeFileSync(path.join(projectRoot, 'quality', name), ''); + } + fs.writeFileSync(path.join(projectRoot, '.venv', 'tests', 'test_vendor.py'), ''); + + assert.strictEqual(countPythonTestFiles(projectRoot), 3); + const parsed = JSON.parse(run(['repo', '--format', 'json'], { cwd: projectRoot, homeDir })); + assert.ok(parsed.checks.some(check => check.id === 'consumer-test-suite' && check.pass)); + assert.ok(parsed.checks.some(check => check.id === 'consumer-eval-coverage' && check.pass)); + } finally { + cleanup(homeDir); + cleanup(projectRoot); + } + })) passed++; else failed++; + + if (test('expands recursive glob patterns in pytest testpaths', () => { + const homeDir = createTempDir('harness-audit-python-glob-home-'); + const projectRoot = createTempDir('harness-audit-python-glob-project-'); + + try { + fs.mkdirSync(path.join(projectRoot, 'quality', 'unit'), { recursive: true }); + fs.writeFileSync( + path.join(projectRoot, 'pyproject.toml'), + '[tool.pytest.ini_options]\ntestpaths = ["quality/**"]\n' + ); + for (const name of ['test_one.py', 'test_two.py', 'three_test.py']) { + fs.writeFileSync(path.join(projectRoot, 'quality', 'unit', name), ''); + } + + assert.strictEqual(countPythonTestFiles(projectRoot), 3); + const parsed = JSON.parse(run(['repo', '--format', 'json'], { cwd: projectRoot, homeDir })); + assert.ok(parsed.checks.some(check => check.id === 'consumer-eval-coverage' && check.pass)); + } finally { + cleanup(homeDir); + cleanup(projectRoot); + } + })) passed++; else failed++; + + if (test('does not fall back when a recursive testpath matches an empty directory', () => { + const projectRoot = createTempDir('harness-audit-python-empty-glob-'); + + try { + fs.mkdirSync(path.join(projectRoot, 'quality'), { recursive: true }); + fs.mkdirSync(path.join(projectRoot, 'package'), { recursive: true }); + fs.writeFileSync( + path.join(projectRoot, 'pytest.toml'), + '[pytest]\ntestpaths = ["quality/**"]\n' + ); + for (const name of ['test_one.py', 'test_two.py', 'three_test.py']) { + fs.writeFileSync(path.join(projectRoot, 'package', name), ''); + } + + assert.strictEqual(countPythonTestFiles(projectRoot), 0); + } finally { + cleanup(projectRoot); + } + })) passed++; else failed++; + + if (test('secret hygiene requires an active gitignore rule for env files', () => { + const homeDir = createTempDir('harness-audit-gitignore-home-'); + const cases = [ + ['# Do not commit .env files\nnode_modules/\n', false], + ['environment.json\n.envexample\n', false], + ['[z-a]env\n', false], + ['.env.example\n', false], + ['.env\n', true], + ['.env.local\n', true], + ['.env.*\n', true], + ['*.env\n', true], + ['.env/\n', true], + ['config/.env\n', true], + ['[.]env\n', true], + ['**/secrets.env\n', true], + ['secrets/\n!secrets/.env\n', false], + [' [.]env\n', false], + ['.env\n!.env\n', false], + ['.env*\n!.env.example\n', true], + ]; + + try { + for (const [gitignore, expected] of cases) { + const projectRoot = createTempDir('harness-audit-gitignore-project-'); + try { + fs.writeFileSync(path.join(projectRoot, '.gitignore'), gitignore); + const parsed = JSON.parse(run(['repo', '--format', 'json'], { cwd: projectRoot, homeDir })); + const check = parsed.checks.find(candidate => candidate.id === 'consumer-secret-hygiene'); + assert.strictEqual(check.pass, expected, JSON.stringify({ gitignore, expected })); + } finally { + cleanup(projectRoot); + } + } + } finally { + cleanup(homeDir); + } + })) passed++; else failed++; + if (test('prints no top actions when consumer checks all pass', () => { const homeDir = createTempDir('harness-audit-passing-home-'); const projectRoot = createTempDir('harness-audit-passing-project-');