From cd878ad57d3e27e835d0db533c9603b24ae8607a Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 30 Oct 2026 21:23:33 +0800 Subject: [PATCH] refactor(ci): unify lane ownership and guard Python tests (#8150) * refactor(ci): unify lane ownership and guard Python tests * fix(ci): reject entirely skipped bridge subtests --- docs/testing.md | 14 +- scripts/ci/plan.mjs | 52 +++---- scripts/ci/required.mjs | 22 +-- scripts/ci/scopes.mjs | 92 +++++++++--- scripts/run-python-tests.py | 100 ++++++++++--- .../boundary/filesystem/ciChangeScope.test.ts | 8 +- .../filesystem/ciRequiredLanes.test.ts | 135 ++++++++++++++++++ .../boundary/filesystem/ciTestReports.test.ts | 46 +++++- 8 files changed, 370 insertions(+), 99 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index a40bbcbcb4..362e136636 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -857,9 +857,10 @@ bridges, analysis-efficiency inputs, or the Python test runner select a separate required Python job. That job installs locked dependencies and prepares the compiled runtime before the suite, because analysis-efficiency checks import built process modules. Bridge tests are discovered from the source tree. The -mitmproxy, native, and pwndbg families must each keep a test module, and a -newly added failing test fails the job. Portable test shards and documentation -checks remain independent of static results, so they can still provide +mitmproxy, native, and pwndbg families must each contain a test module and +execute at least one non-skipped case or subtest. A parent case with only skipped +subtests does not satisfy this requirement. A newly added failing test fails the job. +Portable test shards and documentation checks remain independent of static results, so they can still provide diagnostic feedback. The required gate rejects skipped selected lanes when static checks fail; on the full tier it also requires the deep static job. Passing revisions retain all selected coverage. This ordering reduces peak @@ -1577,9 +1578,10 @@ initial proposal measures only leaf producer-test omissions; it does not promise the savings of a broader affected-test planner. `scripts/ci/plan.mjs` selects lanes using the ownership table in -`scripts/ci/scopes.mjs`. Provider implementation changes run their real-provider -verifiers and the Linux x64 installed-package/Inspector baseline. The `apple` -scope owns the Apple artifact lane; the `package` scope owns the multi-host +`scripts/ci/scopes.mjs`. Each lane record owns its path and test selection +rules and required fast/full jobs; workflow contract tests verify the mapped jobs. +Provider implementation changes run their real-provider verifiers and the Linux +x64 installed-package/Inspector baseline. The `apple` scope owns the Apple artifact lane; the `package` scope owns the multi-host package matrix and the Windows curated lane, including Windows native controls, Inspector, setup, configuration and packaging entrypoints. Runtime sources under `src/` or `bridge/` without an owner, and the authored skill, run the Linux suite diff --git a/scripts/ci/plan.mjs b/scripts/ci/plan.mjs index 4480d17d65..99be6414d7 100644 --- a/scripts/ci/plan.mjs +++ b/scripts/ci/plan.mjs @@ -4,19 +4,11 @@ import { matchesGlob } from "node:path"; import { CI_FLAGS as flags, MANUAL_VERIFIERS, - SCOPES, + CI_LANES, WINDOWS_EXECUTABLE_SELECTION_INPUTS, } from "./scopes.mjs"; import { planTestSelection } from "./test-selection.mjs"; -// Test files select the host/runtime lanes that execute their focused suites. -const HOST_TEST_OWNERS = [ - "apple", - "package", - "sqlite", - "packet_capture", - "signal_recording", -]; // Actionlint validates workflow syntax only. The semantic suite asserts these // workflows' release, publication and self-hosted runner safeguards. const GUARDED_WORKFLOWS = new Set([ @@ -155,11 +147,11 @@ function classifyPackage(base, head) { const scriptOwners = simple && entries.every((entry) => - Object.values(SCOPES).some((patterns) => matchesAny(entry, patterns)), + Object.values(CI_LANES).some(({ paths }) => matchesAny(entry, paths)), ) - ? Object.entries(SCOPES) - .filter(([, patterns]) => - entries.some((entry) => matchesAny(entry, patterns)), + ? Object.entries(CI_LANES) + .filter(([, { paths }]) => + entries.some((entry) => matchesAny(entry, paths)), ) .map(([owner]) => owner) : []; @@ -225,16 +217,12 @@ if (tier === "reused") { file, WINDOWS_EXECUTABLE_SELECTION_INPUTS, ); - if (matchesAny(file, SCOPES.reverse_tools)) selected.reverse_tools = true; - if (matchesAny(file, SCOPES.python)) selected.python = true; - // Both live capture lanes exercise Linux resource admission. - if ( - matchesAny(file, ["src/process/LinuxResourceLimits*"]) || - file === "scripts/ci/require-provider-tests.mjs" - ) { - selected.packet_capture = true; - selected.signal_recording = true; - } + for (const [ + owner, + { paths, earlyAll, earlyInputs = [] }, + ] of Object.entries(CI_LANES)) + if (matchesAny(file, earlyAll ? paths : earlyInputs)) + selected[owner] = true; if ( /^(README(?:_[^/]+)?\.md|AGENTS\.md|CONTRIBUTING\.md|CHANGELOG\.md|SECURITY\.md|native\/windows\/README\.md|\.github\/pull_request_template\.md)$/.test( file, @@ -248,13 +236,13 @@ if (tier === "reused") { file.startsWith("docs/") && (/\.(?:[cm]?[jt]s|[jt]sx)$/u.test(file) || file.endsWith("/package.json")); - selected.website ||= matchesAny(file, SCOPES.website); + selected.website ||= matchesAny(file, CI_LANES.website.paths); continue; } if (/^(src|tests)\/.*\.(?:test|spec)\.[cm]?[jt]sx?$/.test(file)) { selected.code = true; - for (const owner of HOST_TEST_OWNERS) - selected[owner] ||= matchesAny(file, SCOPES[owner]); + for (const [owner, { paths, testInputs }] of Object.entries(CI_LANES)) + if (testInputs && matchesAny(file, paths)) selected[owner] = true; continue; } if (matchesAny(file, MANUAL_VERIFIERS)) { @@ -274,19 +262,15 @@ if (tier === "reused") { if (tier !== "full" && SHARED_BOUNDARY.test(file)) { selected.code = true; selected.runtime = true; - const owners = Object.entries(SCOPES) - .filter(([, patterns]) => - patterns.some((pattern) => matchesGlob(file, pattern)), - ) + const owners = Object.entries(CI_LANES) + .filter(([, { paths }]) => matchesAny(file, paths)) .map(([owner]) => owner); if (owners.length === 1) selected[owners[0]] = true; reasons.push(`Shared boundary on fast tier: ${file}`); continue; } - const owners = Object.entries(SCOPES) - .filter(([, patterns]) => - patterns.some((pattern) => matchesGlob(file, pattern)), - ) + const owners = Object.entries(CI_LANES) + .filter(([, { paths }]) => matchesAny(file, paths)) .map(([owner]) => owner); for (const owner of owners) selected[owner] = true; if ( diff --git a/scripts/ci/required.mjs b/scripts/ci/required.mjs index f43491186a..c0df9e2191 100644 --- a/scripts/ci/required.mjs +++ b/scripts/ci/required.mjs @@ -1,4 +1,4 @@ -import { CI_FLAGS } from "./scopes.mjs"; +import { CI_FLAGS, CI_LANES } from "./scopes.mjs"; import { recordQualification } from "./qualification.mjs"; const jobs = JSON.parse(process.env.CI_JOBS); @@ -30,24 +30,8 @@ if (outputs.tier === "full" && selectedScopes.length !== CI_FLAGS.length) throw new Error("Full qualification requires every canonical CI scope"); const required = reused ? ["changes"] : ["changes", "static", "test"]; if (outputs.tier === "full") required.push("static-deep"); -const lanes = { - code: [ - "build", - "test-shard", - ...(outputs.tier === "fast" ? ["test-source"] : []), - ], - reverse_tools: ["reverse-tools-quality"], - runtime: ["windows-native-build", "package-e2e"], - package: ["windows-curated"], - sqlite: ["sqlite-runtime"], - apple: ["apple-artifacts"], - fixtures: ["conformance-fixtures"], - readiness: ["reconstruction-readiness"], - docs: ["docs"], - workflows: ["static"], - python: [outputs.tier === "full" ? "static-deep" : "python"], -}; -for (const scope of selectedScopes) required.push(...(lanes[scope] ?? [scope])); +for (const scope of selectedScopes) + required.push(...CI_LANES[scope].required[outputs.tier]); const failed = [...new Set(required)].filter( (name) => jobs[name]?.result !== "success", ); diff --git a/scripts/ci/scopes.mjs b/scripts/ci/scopes.mjs index fb148959ac..c447a91a6a 100644 --- a/scripts/ci/scopes.mjs +++ b/scripts/ci/scopes.mjs @@ -8,10 +8,16 @@ export const WINDOWS_EXECUTABLE_SELECTION_INPUTS = [ "tests/boundary/process/windowsExecutableSelection.test.ts", ]; +// Inputs whose changes must select both real capture lanes, even at shared boundaries. +const CAPTURE_SHARED_INPUTS = [ + "src/process/LinuxResourceLimits*", + "scripts/ci/require-provider-tests.mjs", +]; + // Ownership of the native-host, real-provider and website verification lanes. // Shared build inputs are classified once in plan.mjs rather than repeated in // every workflow. -export const SCOPES = { +const PATH_SCOPES = { python: [ "bridge/**/*.py", "tests/support/analysis-efficiency/**", @@ -35,9 +41,8 @@ export const SCOPES = { "src/composition/captures.ts", "src/cli/captureCommands.ts", "src/server/registerCaptureTools.ts", - "src/process/LinuxResourceLimits*", + ...CAPTURE_SHARED_INPUTS, "scripts/verify-packet-capture.mjs", - "scripts/ci/require-provider-tests.mjs", ".github/workflows/real-packet-capture.yml", ], signal_recording: [ @@ -51,9 +56,8 @@ export const SCOPES = { "src/composition/signals.ts", "src/cli/signalCommands.ts", "src/server/registerSignalTools.ts", - "src/process/LinuxResourceLimits*", + ...CAPTURE_SHARED_INPUTS, "scripts/verify-signal-recording.mjs", - "scripts/ci/require-provider-tests.mjs", ".github/workflows/real-signal-recording.yml", ], readiness: [ @@ -609,19 +613,73 @@ export const SCOPES = { ], }; -export const CI_FLAGS = [ - ...new Set([ - "code", - "runtime", - "package", - "apple", - "fixtures", - "readiness", - "docs", - "workflows", - ...Object.keys(SCOPES), +// Each lane owns its path inputs, selection policy, and required jobs. The +// workflow stays explicit; contract tests bind it to this registry. +const sameJobs = (paths, jobs, selection = {}) => ({ + paths, + required: { fast: jobs, full: jobs }, + ...selection, +}); +export const CI_LANES = { + code: { + paths: [], + required: { + fast: ["build", "test-shard", "test-source"], + full: ["build", "test-shard"], + }, + }, + runtime: sameJobs([], ["windows-native-build", "package-e2e"]), + package: sameJobs(PATH_SCOPES.package, ["windows-curated"], { + testInputs: true, + }), + apple: sameJobs(PATH_SCOPES.apple, ["apple-artifacts"], { testInputs: true }), + fixtures: sameJobs([], ["conformance-fixtures"]), + readiness: sameJobs(PATH_SCOPES.readiness, ["reconstruction-readiness"]), + docs: sameJobs([], ["docs"]), + workflows: sameJobs([], ["static"]), + python: { + paths: PATH_SCOPES.python, + earlyAll: true, + required: { fast: ["python"], full: ["static-deep"] }, + }, + packet_capture: sameJobs(PATH_SCOPES.packet_capture, ["packet_capture"], { + testInputs: true, + earlyInputs: CAPTURE_SHARED_INPUTS, + }), + signal_recording: sameJobs( + PATH_SCOPES.signal_recording, + ["signal_recording"], + { testInputs: true, earlyInputs: CAPTURE_SHARED_INPUTS }, + ), + reverse_tools: sameJobs( + PATH_SCOPES.reverse_tools, + ["reverse-tools-quality"], + { earlyAll: true }, + ), + sqlite: sameJobs(PATH_SCOPES.sqlite, ["sqlite-runtime"], { + testInputs: true, + }), + browser: sameJobs(PATH_SCOPES.browser, ["browser"]), + binary_layout: sameJobs(PATH_SCOPES.binary_layout, ["binary_layout"]), + evm: sameJobs(PATH_SCOPES.evm, ["evm"]), + apk_signature: sameJobs(PATH_SCOPES.apk_signature, ["apk_signature"]), + javascript_recovery: sameJobs(PATH_SCOPES.javascript_recovery, [ + "javascript_recovery", ]), -]; + recorded_crash: sameJobs(PATH_SCOPES.recorded_crash, ["recorded_crash"]), + web_network: sameJobs(PATH_SCOPES.web_network, ["web_network"]), + web_runtime: sameJobs(PATH_SCOPES.web_runtime, ["web_runtime"]), + web_source_map: sameJobs(PATH_SCOPES.web_source_map, ["web_source_map"]), + agentenv: sameJobs(PATH_SCOPES.agentenv, ["agentenv"]), + nativeaot: sameJobs(PATH_SCOPES.nativeaot, ["nativeaot"]), + ghidra_bridge: sameJobs(PATH_SCOPES.ghidra_bridge, ["ghidra_bridge"]), + website: sameJobs(PATH_SCOPES.website, ["website"]), +}; +// An input owner omitted from the registry must never silently lose a lane. +for (const [name, paths] of Object.entries(PATH_SCOPES)) + if (!Object.hasOwn(CI_LANES, name) || CI_LANES[name].paths !== paths) + throw new Error(`CI lane registry does not own input scope: ${name}`); +export const CI_FLAGS = Object.keys(CI_LANES); // Verifier entrypoints that only manual, licensed, or self-hosted workflows // run. No selected CI lane executes them, so their edits need only the static diff --git a/scripts/run-python-tests.py b/scripts/run-python-tests.py index c80ddd9ae5..81cc015e92 100644 --- a/scripts/run-python-tests.py +++ b/scripts/run-python-tests.py @@ -4,10 +4,9 @@ from pathlib import Path import json import subprocess import sys +import unittest root = Path(__file__).resolve().parent.parent -# The previous explicit command named these families. Discovery still runs -# every bridge test, and each named family must keep a test module. REQUIRED_BRIDGE_FAMILIES = ("mitmproxy", "native", "pwndbg") @@ -16,35 +15,100 @@ def fail(code, **details): raise SystemExit(1) -def family_tests(family): - return [ - path - for path in root.joinpath("bridge", family).rglob("test_*.py") - if path.is_file() +class BridgeResult(unittest.TextTestResult): + """Record non-skipped test and subtest outcomes for required families.""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.completed = set() + self.active_test = None + self.has_subtests = False + + def record(self, test): + parts = test.id().split(".") + if len(parts) > 2 and parts[0] == "bridge": + if parts[1] in REQUIRED_BRIDGE_FAMILIES: + self.completed.add(parts[1]) + + def startTest(self, test): + self.active_test = test + self.has_subtests = False + super().startTest(test) + + def stopTest(self, test): + super().stopTest(test) + self.active_test = None + + def addSuccess(self, test): + # A parent with only skipped subtests also receives addSuccess. + if not self.has_subtests: + self.record(test) + super().addSuccess(test) + + def addSubTest(self, test, subtest, outcome): + self.has_subtests = True + if outcome is None: + self.record(test) + super().addSubTest(test, subtest, outcome) + + def addSkip(self, test, reason): + if self.active_test is not None and test is not self.active_test: + self.has_subtests = True + super().addSkip(test, reason) + + def addExpectedFailure(self, test, err): + self.record(test) + super().addExpectedFailure(test, err) + + +def run_bridge(modules): + # The script directory is sys.path[0]; imports must resolve from the checkout. + sys.path.insert(0, str(root)) + suite = unittest.defaultTestLoader.loadTestsFromNames(modules) + result = unittest.TextTestRunner(verbosity=2, resultclass=BridgeResult).run(suite) + missing = [ + family + for family in REQUIRED_BRIDGE_FAMILIES + if family not in result.completed ] + if missing: + fail("no_executed_bridge_tests", families=missing) + if not result.wasSuccessful(): + raise SystemExit(1) +bridge_root = root / "bridge" +bridge_tests = sorted( + path for path in bridge_root.rglob("test_*.py") if path.is_file() +) +families = {path.relative_to(bridge_root).parts[0] for path in bridge_tests} missing_families = [ - family for family in REQUIRED_BRIDGE_FAMILIES if not family_tests(family) + family for family in REQUIRED_BRIDGE_FAMILIES if family not in families ] if missing_families: fail("missing_bridge_family", families=missing_families) -bridge_tests = sorted(root.joinpath("bridge").rglob("test_*.py")) -if not bridge_tests: - fail("missing_bridge_tests") if not list(root.joinpath("tests/support/analysis-efficiency").glob("test_*.py")): fail("missing_analysis_efficiency_tests") modules = [ ".".join(path.relative_to(root).with_suffix("").parts) for path in bridge_tests ] -commands = [ - [sys.executable, "-m", "unittest", "-v", *modules], +if sys.argv[1:] == ["--bridge"]: + run_bridge(modules) + raise SystemExit(0) +if sys.argv[1:]: + fail("invalid_python_test_arguments") +bridge_result = subprocess.run( + [sys.executable, __file__, "--bridge"], cwd=root, check=False +) +if bridge_result.returncode: + sys.exit(bridge_result.returncode) +result = subprocess.run( [ sys.executable, "-m", "unittest", "discover", "-s", "tests/support/analysis-efficiency", "-p", "test_*.py", "-v", ], -] -for command in commands: - result = subprocess.run(command, cwd=root, check=False) - if result.returncode: - sys.exit(result.returncode) + cwd=root, + check=False, +) +if result.returncode: + sys.exit(result.returncode) diff --git a/tests/boundary/filesystem/ciChangeScope.test.ts b/tests/boundary/filesystem/ciChangeScope.test.ts index 40c16b158f..10cb2dd45f 100644 --- a/tests/boundary/filesystem/ciChangeScope.test.ts +++ b/tests/boundary/filesystem/ciChangeScope.test.ts @@ -1612,7 +1612,7 @@ const workflowFile = async (name: string): Promise => const scopesSchema = z.object({ CI_FLAGS: z.array(z.string()), - SCOPES: z.record(z.string(), z.array(z.string())), + CI_LANES: z.record(z.string(), z.object({ paths: z.array(z.string()) })), MANUAL_VERIFIERS: z.array(z.string()), }); @@ -1634,7 +1634,7 @@ const stepRuns = z.object({ }); it("owns every explicit native-host test path with the lane that runs it", async () => { - const { SCOPES } = await loadScopes(); + const { CI_LANES } = await loadScopes(); const ci = stepRuns.parse(await workflowFile("ci.yml")); const listed = (job: string) => (ci.jobs[job]?.steps ?? []) @@ -1643,7 +1643,9 @@ it("owns every explicit native-host test path with the lane that runs it", async const unowned = (job: string, scope: string) => listed(job).filter( (path) => - !(SCOPES[scope] ?? []).some((pattern) => matchesGlob(path, pattern)), + !(CI_LANES[scope]?.paths ?? []).some((pattern) => + matchesGlob(path, pattern), + ), ); expect(listed("apple-artifacts").length).toBeGreaterThan(0); expect(listed("windows-curated").length).toBeGreaterThan(0); diff --git a/tests/boundary/filesystem/ciRequiredLanes.test.ts b/tests/boundary/filesystem/ciRequiredLanes.test.ts index 83e73a24e8..f0282a37a4 100644 --- a/tests/boundary/filesystem/ciRequiredLanes.test.ts +++ b/tests/boundary/filesystem/ciRequiredLanes.test.ts @@ -729,3 +729,138 @@ it("fails fast only for ordinary PR matrices and rejects partial portable feedba }); } }); + +it("binds each canonical lane to its explicit fast and full workflow jobs", async () => { + const { jobs, needs } = await requiredNeeds(); + const { CI_LANES } = await import( + new URL("../../../scripts/ci/scopes.mjs", import.meta.url).href + ); + // Independent contract: adding or changing a lane requires a workflow and + // gate decision, rather than silently assuming that its job has the same name. + const expected: Record = { + code: { + fast: ["build", "test-shard", "test-source"], + full: ["build", "test-shard"], + }, + runtime: { + fast: ["windows-native-build", "package-e2e"], + full: ["windows-native-build", "package-e2e"], + }, + package: { fast: ["windows-curated"], full: ["windows-curated"] }, + apple: { fast: ["apple-artifacts"], full: ["apple-artifacts"] }, + fixtures: { + fast: ["conformance-fixtures"], + full: ["conformance-fixtures"], + }, + readiness: { + fast: ["reconstruction-readiness"], + full: ["reconstruction-readiness"], + }, + docs: { fast: ["docs"], full: ["docs"] }, + workflows: { fast: ["static"], full: ["static"] }, + python: { fast: ["python"], full: ["static-deep"] }, + reverse_tools: { + fast: ["reverse-tools-quality"], + full: ["reverse-tools-quality"], + }, + sqlite: { fast: ["sqlite-runtime"], full: ["sqlite-runtime"] }, + ...Object.fromEntries( + [ + "packet_capture", + "signal_recording", + "browser", + "binary_layout", + "evm", + "apk_signature", + "javascript_recovery", + "recorded_crash", + "web_network", + "web_runtime", + "web_source_map", + "agentenv", + "nativeaot", + "ghidra_bridge", + "website", + ].map((name) => [name, { fast: [name], full: [name] }]), + ), + }; + expect(Object.keys(CI_LANES).sort()).toEqual(Object.keys(expected).sort()); + for (const [scope, required] of Object.entries(expected)) { + expect(CI_LANES[scope].required, scope).toEqual(required); + for (const name of new Set([...required.fast, ...required.full])) { + expect(needs, `${scope}: ${name}`).toContain(name); + expect(jobs[name], `${scope}: ${name}`).toBeDefined(); + // Static and deep-static are shared gates; other mapped jobs are admitted + // by their scope output in the workflow itself. + if (name !== "static" && name !== "static-deep") + expect(String(jobs[name]?.if), `${scope}: ${name}`).toContain( + `needs.changes.outputs.${scope}`, + ); + } + } + + const executeGate = (context: Record) => + executeWorkflowFixture( + process.execPath, + [ + fileURLToPath( + new URL("../../../scripts/ci/required.mjs", import.meta.url), + ), + ], + { + env: { + ...process.env, + CI_JOBS: JSON.stringify(context), + CI_QUALIFICATION_DIRECTORY: "", + }, + }, + ); + const flags = Object.keys(expected); + for (const [scope, required] of Object.entries(expected)) { + const context: Record = Object.fromEntries( + needs.map((name) => [name, { result: "skipped" }]), + ); + context.changes = { + result: "success", + outputs: { + ...Object.fromEntries(flags.map((name) => [name, "false"])), + tier: "fast", + [scope]: "true", + }, + }; + context.static = { result: "success" }; + context.test = { result: "success" }; + for (const name of required.fast) context[name] = { result: "success" }; + await expect(executeGate(context), scope).resolves.toMatchObject({ + stderr: "", + }); + for (const name of required.fast) { + await expect( + executeGate({ ...context, [name]: { result: "skipped" } }), + `${scope}: ${name}`, + ).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining(name), + }); + } + } + const fullContext: Record = Object.fromEntries( + needs.map((name) => [name, { result: "success" }]), + ); + fullContext.changes = { + result: "success", + outputs: { + ...Object.fromEntries(flags.map((name) => [name, "true"])), + tier: "full", + }, + }; + await expect(executeGate(fullContext)).resolves.toMatchObject({ stderr: "" }); + for (const name of new Set( + Object.values(expected).flatMap(({ full }) => full), + )) { + await expect( + executeGate({ ...fullContext, [name]: { result: "skipped" } }), + name, + ).rejects.toMatchObject({ code: 1, stderr: expect.stringContaining(name) }); + } +}); diff --git a/tests/boundary/filesystem/ciTestReports.test.ts b/tests/boundary/filesystem/ciTestReports.test.ts index fc49c99a8c..23ed977b65 100644 --- a/tests/boundary/filesystem/ciTestReports.test.ts +++ b/tests/boundary/filesystem/ciTestReports.test.ts @@ -733,7 +733,7 @@ it("records failed omissions from full reports without treating partial or unhan ).rejects.toMatchObject({ code: "ENOENT" }); }); -it("discovers Python bridge tests and fails on a newly added failing producer test", async () => { +it("requires executed Python cases in every bridge family", async () => { const root = await createTestTempDirectory("rea-python-inventory-"); await mkdir(join(root, "scripts"), { recursive: true }); await mkdir(join(root, "bridge", "example"), { recursive: true }); @@ -764,6 +764,46 @@ it("discovers Python bridge tests and fails on a newly added failing producer te expect(discovered.stderr).toContain("bridge.native.test_case"); expect(discovered.stderr).toContain("bridge.pwndbg.test_case"); expect(discovered.stderr).toContain("OK"); + const native = join(root, "bridge", "native", "test_case.py"); + for (const content of [ + "# A present file can contain no tests.\n", + "import unittest\nclass Example(unittest.TestCase):\n @unittest.skip('unavailable')\n def test_case(self): pass\n", + "import unittest\nclass Example(unittest.TestCase):\n def test_case(self):\n for x in range(2):\n with self.subTest(x=x): self.skipTest('unavailable')\n", + "import unittest\nclass Example(unittest.TestCase):\n def test_case(self):\n with self.subTest(outer=1):\n with self.subTest(inner=2): self.skipTest('unavailable')\n", + ]) { + await writeFile(native, content); + await expect( + execute("python3", command, { cwd: root }), + ).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining('"code": "no_executed_bridge_tests"'), + }); + } + await writeFile( + native, + "import unittest\nclass Example(unittest.TestCase):\n def test_case(self): pass\n @unittest.skip('optional')\n def test_skipped(self): pass\n", + ); + expect((await execute("python3", command, { cwd: root })).stderr).toContain( + "skipped 'optional'", + ); + await writeFile( + native, + "import unittest\nclass Example(unittest.TestCase):\n def test_case(self):\n for x in range(2):\n with self.subTest(x=x):\n if x == 0: self.skipTest('optional')\n self.assertEqual(x, 1)\n", + ); + expect((await execute("python3", command, { cwd: root })).stderr).toContain( + "skipped 'optional'", + ); + await writeFile( + native, + "import unittest\nclass Example(unittest.TestCase):\n def test_case(self):\n with self.subTest(value=1): self.fail('subtest failed')\n", + ); + await expect( + execute("python3", command, { cwd: root }), + ).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining("subtest failed"), + }); + await writeFile(native, passing); await writeFile( join(root, "bridge", "example", "test_second.py"), "import unittest\nclass Example(unittest.TestCase):\n def test_case(self): self.fail('new test executed')\n", @@ -780,7 +820,9 @@ it("discovers Python bridge tests and fails on a newly added failing producer te (error: { code?: unknown; stderr?: unknown }) => error, ); expect(missingFamily).toMatchObject({ code: 1 }); - expect(JSON.parse(String(missingFamily?.stderr))).toEqual({ + expect( + JSON.parse(String(missingFamily?.stderr).trim().split("\n").at(-1)!), + ).toEqual({ code: "missing_bridge_family", families: ["mitmproxy"], });