import { execFile } from "node:child_process"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; import { expect, it } from "vitest"; import { parseTestSelection, planTestSelection, testShards, } from "../../scripts/ci/test-selection.mjs"; const execute = promisify(execFile); const plan = (files: readonly string[], sources: Record) => planTestSelection({ files, event: "pull_request", tier: "fast", code: true, snapshot: { list: () => Object.keys(sources), read: (paths) => new Map(paths.map((path) => [path, sources[path] ?? ""])), }, }); const selectedFiles = (result: ReturnType) => (result.mode === "selected" ? result.files : []).toSorted(); it("follows test and verifier imports and directory scans, not production importers", () => { const sources = { "src/domain/Value.ts": "", "src/application/UsesValue.ts": 'import "../domain/Value.js";', "src/application/UsesValue.test.ts": 'import "./UsesValue.js";', "src/domain/Value.test.ts": 'import { value } from "./Value.js";', "src/domain/Registry.test.ts": "readdirSync(here);", "src/domain/Unrelated.test.ts": 'import "./Unrelated.js";', "tests/boundary/filesystem/verifier.test.ts": 'const m = await import("../../../scripts/verify-thing.mjs");', }; expect(selectedFiles(plan(["src/domain/Value.ts"], sources))).toEqual([ "src/domain/Registry.test.ts", "src/domain/Value.test.ts", ]); expect(selectedFiles(plan(["scripts/verify-thing.mjs"], sources))).toEqual([ "tests/boundary/filesystem/verifier.test.ts", ]); }); it("selects a leaf adapter's tests by its name and directory across test projects", () => { const sources = { "src/jeb/JebMcpDeadline.ts": "", "tests/boundary/providers/jebMcpTimeout.test.ts": "", "tests/boundary/mcp/jebMcp.test.ts": "", "tests/acceptance/providers/jebConfigurationIsolation.test.ts": "", "tests/boundary/providers/adb/adbBugreport.test.ts": "", "tests/boundary/providers/ghidraJava.test.ts": "", }; expect(selectedFiles(plan(["src/jeb/JebMcpDeadline.ts"], sources))).toEqual([ "tests/acceptance/providers/jebConfigurationIsolation.test.ts", "tests/boundary/mcp/jebMcp.test.ts", "tests/boundary/providers/jebMcpTimeout.test.ts", ]); expect(selectedFiles(plan(["src/adb/AdbProvider.ts"], sources))).toEqual([ "tests/boundary/providers/adb/adbBugreport.test.ts", ]); }); it("selects tests that start the built runtime beside a changed presentation owner", () => { const sources = { "src/server/Core.ts": "", "src/server/recovery.ts": "", "src/domain/Value.ts": "", "tests/support/launch.ts": 'spawn("scripts/rea.mjs", ["mcp"]);', "tests/fixtures/client.ts": 'import "../support/launch.js";', // Starts the built server and imports a sibling of the changed owner. "tests/boundary/mcp/direct.test.ts": 'import "../../../src/server/recovery.js"; resolve("dist/main.js");', "tests/boundary/mcp/helper.test.ts": 'import "../../fixtures/client.js"; import "../../../src/server/recovery.js";', // Starts it but imports nothing from that owner, or imports it in process. "tests/boundary/mcp/unrelated.test.ts": 'import "../../fixtures/client.js"; import "../../../src/domain/Value.js";', "tests/boundary/mcp/inProcess.test.ts": 'import "../../../src/server/recovery.js";', // A package's own dist/ subpath is not the built runtime. "tests/boundary/mcp/ajv.test.ts": 'import { Ajv2020 } from "ajv/dist/2020.js"; import "../../../src/server/recovery.js";', }; expect(selectedFiles(plan(["src/server/Core.ts"], sources))).toEqual([ "tests/boundary/mcp/direct.test.ts", "tests/boundary/mcp/helper.test.ts", ]); // Other owners keep import tracing only. expect(plan(["src/domain/Value.ts"], sources)).toMatchObject({ reason: "Affected tests: 1 imports", }); }); it("lets checker-only inputs and prose select nothing by themselves", () => { const sources = { "src/domain/Value.test.ts": "", "tests/boundary/filesystem/docs.test.ts": 'read("docs/guide.md");', "tests/boundary/filesystem/mermaid.test.ts": 'read("docs/architecture/architecture.mermaid");', "tests/boundary/filesystem/licenses.test.ts": 'read("licenses/npm-dependencies.txt"); read("LICENSE");', "tests/boundary/filesystem/issueTemplate.test.ts": 'read(".github/ISSUE_TEMPLATE/bug_report.yml");', }; // A declaration, guard/layer baseline, blame-ignore list or schema summary adds no tests… expect( selectedFiles( plan( [ "src/domain/Value.test.ts", "scripts/lib/helper.d.mts", "scripts/lib/module-layers.baseline.json", ".git-blame-ignore-revs", "scripts/verify/baselines/some-guard.json", "scripts/verify/advertised-schemas/tools/tool.json", ], sources, ), ), ).toEqual(["src/domain/Value.test.ts"]); // …and prose, non-code docs assets, and license inventories select the tests that read them by path. expect(selectedFiles(plan(["docs/guide.md"], sources))).toEqual([ "tests/boundary/filesystem/docs.test.ts", ]); expect( selectedFiles(plan(["docs/architecture/architecture.mermaid"], sources)), ).toEqual(["tests/boundary/filesystem/mermaid.test.ts"]); expect( selectedFiles(plan(["licenses/npm-dependencies.txt"], sources)), ).toEqual(["tests/boundary/filesystem/licenses.test.ts"]); expect(selectedFiles(plan(["LICENSE"], sources))).toEqual([ "tests/boundary/filesystem/licenses.test.ts", ]); expect( selectedFiles(plan([".github/ISSUE_TEMPLATE/bug_report.yml"], sources)), ).toEqual(["tests/boundary/filesystem/issueTemplate.test.ts"]); // Authored code under docs/ stays untraced so it falls back to the full suite. expect(plan(["docs/.vitepress/config.ts"], sources)).toMatchObject({ mode: "full", reason: "Input is not traced to tests: docs/.vitepress/config.ts", }); expect(plan(["scripts/verify/baselines/x.json"], sources)).toMatchObject({ mode: "full", }); }); it("selects tests under shared-boundary directories and declared fixture owners, failing closed on missing owners", () => { const sources = { "src/process/OwnedCommand.test.ts": "", "tests/boundary/cli/cliParityGoldens.test.ts": "", "tests/boundary/cli/cliContractSchemaParity.test.ts": "", }; expect( selectedFiles(plan(["src/process/OwnedCommand.test.ts"], sources)), ).toEqual(["src/process/OwnedCommand.test.ts"]); expect(plan(["src/process/OwnedCommand.ts"], sources)).toMatchObject({ mode: "full", reason: "Shared boundary: src/process/OwnedCommand.ts", }); expect( selectedFiles( plan(["tests/fixtures/cli-parity/error-unknown-command.txt"], sources), ), ).toEqual(["tests/boundary/cli/cliParityGoldens.test.ts"]); expect( selectedFiles( plan( ["tests/boundary/cli/cliContractSchemaParity.uncomparable.json"], sources, ), ), ).toEqual(["tests/boundary/cli/cliContractSchemaParity.test.ts"]); // If an explicit fixture owner is missing or renamed, selection fails closed. expect( plan(["tests/fixtures/cli-parity/error-unknown-command.txt"], { "src/process/OwnedCommand.test.ts": "", }), ).toMatchObject({ mode: "full", reason: "Missing test owner tests/boundary/cli/cliParityGoldens.test.ts for tests/fixtures/cli-parity/error-unknown-command.txt", }); }); it("traces shared script modules to their tests and falls back for other consumers", () => { const sources = { "scripts/lib/shared.mjs": "", "scripts/lib/wrapper.mjs": 'export * from "./shared.mjs";', "scripts/verify-thing.mjs": 'import "./lib/wrapper.mjs";', "scripts/lib/unused.mjs": "", "tests/boundary/filesystem/shared.test.ts": 'import "../../../scripts/lib/shared.mjs";', "tests/boundary/filesystem/verifier.test.ts": 'await import("../../../scripts/verify-thing.mjs");', "tests/boundary/filesystem/other.test.ts": "", "package.json": '{"scripts":{"verify:thing":"node scripts/verify-thing.mjs"}}', }; // Through another shared module and a verifier to the tests that import them. expect(selectedFiles(plan(["scripts/lib/shared.mjs"], sources))).toEqual([ "tests/boundary/filesystem/shared.test.ts", "tests/boundary/filesystem/verifier.test.ts", ]); // A build script, configuration or package script that loads it is untraced. expect( plan(["scripts/lib/shared.mjs"], { ...sources, "scripts/build.mjs": 'import { x } from "./lib/wrapper.mjs";', }), ).toMatchObject({ mode: "full", reason: "Script consumer is not traced: scripts/build.mjs uses scripts/lib/wrapper.mjs", }); expect( plan(["scripts/lib/shared.mjs"], { ...sources, "package.json": '{"scripts":{"x":"node scripts/lib/shared.mjs"}}', }), ).toMatchObject({ mode: "full", reason: "Script consumer is not traced: package.json uses scripts/lib/shared.mjs", }); // A module no test reaches runs everything, even beside a selected change. for (const files of [ ["scripts/lib/unused.mjs"], ["scripts/lib/unused.mjs", "tests/boundary/filesystem/other.test.ts"], ]) expect(plan(files, sources)).toMatchObject({ mode: "full", reason: "No test reaches shared script module: scripts/lib/unused.mjs", }); // Verifier source loads are module edges, and a test that starts a reached // verifier by path is selected without importing it. const loaded = { "scripts/lib/startup.mts": "", "scripts/lib/lifecycle.mjs": 'await importVerifierSource(new URL("./startup.mts", import.meta.url).href, import.meta.url);', "scripts/verify-runtime.mjs": 'import "./lib/lifecycle.mjs";', "tests/boundary/filesystem/startup.test.ts": 'import "../../../scripts/lib/startup.mts";', "tests/boundary/filesystem/lifecycle.test.ts": 'import "../../../scripts/lib/lifecycle.mjs";', "tests/boundary/filesystem/runtimeVerifier.test.ts": 'spawn(process.execPath, ["scripts/verify-runtime.mjs"]);', }; expect(selectedFiles(plan(["scripts/lib/startup.mts"], loaded))).toEqual([ "tests/boundary/filesystem/lifecycle.test.ts", "tests/boundary/filesystem/runtimeVerifier.test.ts", "tests/boundary/filesystem/startup.test.ts", ]); // Static checker JSON/Stryker configs, hyphen-prefixed script names in // package.json (such as verify-shared.mjs vs scripts/lib/shared.mjs), and // path-string tables in scopes.mjs do not force a full-suite fallback. expect( selectedFiles( plan( [ "scripts/lib/shared.mjs", "tests/support/analysis-efficiency/test_eval.py", ], { ...sources, "package.json": '{"scripts":{"verify:shared":"node scripts/verify-shared.mjs"}}', "tsconfig.scripts.json": '{"include":["scripts/lib/shared.mjs"]}', "stryker.config.mjs": 'import "./scripts/lib/shared.mjs";', "scripts/check-doc-facts.mjs": 'import "./lib/shared.mjs";', "scripts/ci/scopes.mjs": 'export const S = ["scripts/lib/shared.mjs"];', }, ), ), ).toEqual([ "tests/boundary/filesystem/shared.test.ts", "tests/boundary/filesystem/verifier.test.ts", ]); expect( selectedFiles( plan(["tests/fixtures/card.json"], { "tests/fixtures/card.json": '{"ok":true}', "tests/boundary/mcp/card.test.ts": 'new URL("../../fixtures/card.json", import.meta.url);', "tests/boundary/mcp/unrelated.test.ts": 'import "../../../src/a.js";', }), ), ).toEqual(["tests/boundary/mcp/card.test.ts"]); expect( plan(["tests/fixtures/unreferenced.bin"], { "tests/fixtures/unreferenced.bin": "raw", "tests/boundary/mcp/unrelated.test.ts": 'import "../../../src/a.js";', }), ).toMatchObject({ mode: "full", reason: "Input is not traced to tests: tests/fixtures/unreferenced.bin", }); expect( plan(["scripts/lib/shared.mjs"], { ...sources, "scripts/ci/scopes.mjs": 'import "../lib/shared.mjs";', }), ).toMatchObject({ mode: "full", reason: "Script consumer is not traced: scripts/ci/scopes.mjs uses scripts/lib/shared.mjs", }); // Other scripts remain untraced inputs. expect(plan(["scripts/build.mjs"], sources)).toMatchObject({ mode: "full", reason: "Input is not traced to tests: scripts/build.mjs", }); }); it("traces JavaScript fixture modules transitively through JS and TS helpers", () => { const sources = { "tests/fixtures/fakeHopperReplies.mjs": "export const fakeHopperReply = () => ({});", "tests/fixtures/fakeHopper.mjs": 'import { fakeHopperReply } from "./fakeHopperReplies.mjs"; export { fakeHopperReply };', "tests/boundary/providers/hopper/hopperClient.fixture.ts": 'export const bridge = new URL("../../../fixtures/fakeHopper.mjs", import.meta.url);', "tests/boundary/providers/hopper/fakeHopperReplyContracts.test.ts": 'import { fakeHopperReply } from "../../../fixtures/fakeHopperReplies.mjs"; void fakeHopperReply;', "tests/boundary/providers/hopper/hopperSocket.test.ts": 'import { bridge } from "./hopperClient.fixture.js"; void bridge;', "tests/acceptance/providers/hopperStartupRecovery.test.ts": 'const path = "tests/fixtures/fakeHopper.mjs"; void path;', "src/unrelated.test.ts": "", }; expect(plan(["tests/fixtures/fakeHopperReplies.mjs"], sources)).toMatchObject( { mode: "selected", files: [ "tests/acceptance/providers/hopperStartupRecovery.test.ts", "tests/boundary/providers/hopper/fakeHopperReplyContracts.test.ts", "tests/boundary/providers/hopper/hopperSocket.test.ts", ], }, ); }); it("parses only well-formed plans and derives the selected shard count", () => { const selected = { version: 3, mode: "selected", reason: "Affected tests: 2 changed", files: ["src/a.test.ts", "tests/b.test.ts"], shards: 2, }; expect( testShards("fast", parseTestSelection(JSON.stringify(selected))), ).toEqual(["1/2", "2/2"]); const maxSelectedFiles = Object.fromEntries( Array.from({ length: 400 }, (_, i) => [ `src/feature-${String(i)}.test.ts`, "", ]), ); expect(plan(Object.keys(maxSelectedFiles), maxSelectedFiles)).toMatchObject({ mode: "selected", shards: 8, }); const overCapFiles = { ...maxSelectedFiles, "src/feature-400.test.ts": "", }; expect(plan(Object.keys(overCapFiles), overCapFiles)).toMatchObject({ mode: "full", reason: "401 affected test files exceed the 400-file selection cap", }); const full = { version: 3, mode: "full", reason: "Shared" }; expect(testShards("fast", parseTestSelection(full))).toHaveLength(12); expect(testShards("full", parseTestSelection(full))).toHaveLength(12); for (const value of [ "", "null", { ...selected, version: 2 }, { ...selected, mode: "candidate" }, { ...selected, files: [] }, { ...selected, files: ["src/a.ts"] }, { ...selected, files: ["/abs/a.test.ts"] }, { ...selected, files: ["src/a.test.ts", "src/a.test.ts"] }, // More shards than files, or than one selection uses. { ...selected, shards: 3 }, { ...selected, shards: 0 }, { ...selected, files: Array.from({ length: 9 }, (_, i) => `src/${i}.test.ts`), shards: 9, }, { ...full, shards: 1 }, ]) expect(() => parseTestSelection(value)).toThrow( /Invalid portable test selection plan|JSON/u, ); }); it("runs only a shard the plan defines", async () => { const group = fileURLToPath( new URL("../../scripts/ci/test-group.mjs", import.meta.url), ); const selection = JSON.stringify({ version: 3, mode: "selected", reason: "Affected tests: 1 changed", files: ["src/a.test.ts"], shards: 1, }); for (const [shard, environment] of [ ["2/2", selection], ["1/2", selection], ["1/1", JSON.stringify({ version: 3, mode: "full", reason: "x" })], ["1/1", ""], ] as const) await expect( execute(process.execPath, [group, "selected", shard], { env: { ...process.env, CI_TEST_SELECTION: environment }, }), ).rejects.toMatchObject({ code: 1 }); await expect( execute(process.execPath, [group, "selected", "1/1", "--shard=1/2"], { env: { ...process.env, CI_TEST_SELECTION: selection }, }), ).rejects.toMatchObject({ stderr: expect.stringContaining("already sharded"), }); });