fix(ci): preserve all cases in selected test shards (#12425)

* fix(ci): keep test-name filters out of selected shards

* fix(ci): reject parsed name filters in selected test shards

* fix(ci): reject parsed tag filters in selected test shards

---------

Co-authored-by: ShuaixinHuang <ShuaixinHuang@users.noreply.github.com>
Co-authored-by: morluto <76467478+morluto@users.noreply.github.com>
This commit is contained in:
authored and GitHub committed 2026-11-19 20:09:12 +00:00
1 parent e79e57552f
commit 2206984be2
3 files changed
+92 -2

No files matched your search

+1 -1
View File
@@ -28,7 +28,7 @@ if (group === "selected") {
throw new Error("Selected test groups are already sharded");
runner = [
resolve("scripts/run-exact-tests.mjs"),
"run",
"ci-selected",
`--shard=${shard}`,
...rest,
...plan.files,
+9 -1
View File
@@ -5,7 +5,15 @@ import { createVitest, parseCLI } from "vitest/node";
// CLI file filters are substring matches. Select discovered specifications by
// identity instead, retaining each project's include/exclude and runner settings.
const { filter, options } = parseCLI(["vitest", ...process.argv.slice(2)]);
const args = process.argv.slice(2);
const ciSelected = args[0] === "ci-selected";
if (ciSelected) args[0] = "run";
const { filter, options } = parseCLI(["vitest", ...args]);
if (
ciSelected &&
(options.testNamePattern !== undefined || options.tagsFilter !== undefined)
)
throw new Error("Selected test groups cannot filter cases by name or tag");
const vitest = await createVitest({ ...options, watch: false });
const signalExitCodes = { SIGINT: 130, SIGTERM: 143 };
let shutdownSignal;
+82
View File
@@ -1,4 +1,6 @@
import { execFile } from "node:child_process";
import { readFile } from "node:fs/promises";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
@@ -9,9 +11,21 @@ import {
planTestSelection,
testShards,
} from "../../scripts/ci/test-selection.mjs";
import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js";
const execute = promisify(execFile);
const selectedEnvironment = (file: string) => ({
...process.env,
CI_TEST_SELECTION: JSON.stringify({
version: 3,
mode: "selected",
reason: "Affected tests: 1 changed",
files: [file],
shards: 1,
}),
});
const plan = (files: readonly string[], sources: Record<string, string>) =>
planTestSelection({
files,
@@ -480,3 +494,71 @@ it("runs only a shard the plan defines", async () => {
stderr: expect.stringContaining("already sharded"),
});
});
it.each([
["-t", "one case"],
["-tX", "unused value"],
["--testNamePattern=one case"],
["--testNamePattern.only=one case"],
["--test-name-pattern", "one case"],
["--tagsFilter=long-running"],
["--tags-filter", "long-running"],
])("rejects a case filter in a selected shard: %j", async (...filter) => {
const group = fileURLToPath(
new URL("../../scripts/ci/test-group.mjs", import.meta.url),
);
await expect(
execute(process.execPath, [group, "selected", "1/1", ...filter], {
env: selectedEnvironment("tests/evaluation/reasonMarker.test.ts"),
}),
).rejects.toMatchObject({
stderr: expect.stringContaining(
"Selected test groups cannot filter cases by name or tag",
),
});
});
it("runs every selected case and still permits developer name and tag filters", async () => {
const group = fileURLToPath(
new URL("../../scripts/ci/test-group.mjs", import.meta.url),
);
const exact = fileURLToPath(
new URL("../../scripts/run-exact-tests.mjs", import.meta.url),
);
const file = "tests/evaluation/reasonMarker.test.ts";
const root = await createTestTempDirectory("rea-selected-cases-");
const selectedReport = join(root, "selected.json");
await execute(
process.execPath,
[
group,
"selected",
"1/1",
"--reporter=json",
`--outputFile.json=${selectedReport}`,
],
{ env: selectedEnvironment(file) },
);
const selected = JSON.parse(await readFile(selectedReport, "utf8")) as {
numPassedTests: number;
};
expect(selected).toMatchObject({ numPendingTests: 0, numFailedTests: 0 });
expect(selected.numPassedTests).toBeGreaterThan(1);
const devReport = join(root, "developer.json");
await execute(process.execPath, [
exact,
"run",
"-t",
"counts an unmarked slice",
"--tagsFilter=!long-running",
"--reporter=json",
`--outputFile.json=${devReport}`,
file,
]);
const dev: unknown = JSON.parse(await readFile(devReport, "utf8"));
expect(dev).toMatchObject({
numPassedTests: 1,
numPendingTests: selected.numPassedTests - 1,
});
});