ci: trace script modules and built-server consumers in PR test selection, and balance selected shards (#11516)

* ci: trace shared script modules to the tests that import them

A pull request that changed any scripts/lib module ran the complete
suite. Shared script modules now pass a change through to the tests,
verifiers and other shared modules that import them, like test helpers.
Another consumer (a build or test-runner script, root configuration,
package metadata, a hook or a workflow) that imports the module or names
it by file name still selects the complete suite.

The tracing core is exported as traceAffectedTests so a bounded local
gate can reuse it.

Fixes #11381

* ci: follow verifier source loads and named verifiers when tracing script modules

- Module URLs (new URL("./x.mts", import.meta.url), as loaded by
  importVerifierSource) are import edges.
- A test that names a verifier or shared module the change reaches is
  selected, not only one that names a changed path.
- A changed shared module that no test reaches runs the complete suite
  even when other changes select tests.

* ci: select tests that start the built server beside a changed presentation owner (#11519)

* ci: select tests that start the built server beside a changed presentation owner

A test that starts the built CLI or MCP server reaches server, contract,
entrypoint and CLI modules through the build, not an import, so a change
there selected none of them (#11146 failed on main this way). Such a
test is now selected when it, or a helper it imports, starts
scripts/rea.mjs or dist/, and it imports a module from the changed
owner's directory.

Fixes #11366

* ci: recognise only the repository's built entry points as runtime launchers

A package subpath such as ajv/dist/2020.js matched the dist/ pattern, so
in-process tests importing Ajv counted as built-runtime launchers.

* ci: balance selected test shards by recorded duration (#11522)

A selected PR run split its files across groups round-robin, so a few
slow files could land together. The plan now records the selected files
and a shard count (version 3), and each job runs one --shard of the
selection through the exact-test runner with the timing history main
already publishes, so the CI sequencer balances shards by per-file
duration (the median for files without a timing). Without history,
Vitest's own deterministic sharding applies.

Fixes #11382
This commit is contained in:
morluto authored and GitHub committed 2026-11-16 06:14:14 +08:00
1 parent 336275259c
commit 062b3835f9
11 files changed
+374 -118

No files matched your search

+1
View File
@@ -367,6 +367,7 @@ jobs:
env:
MATRIX_SHARD: ${{ matrix.shard }}
CI_TEST_SELECTION: ${{ needs.changes.outputs.test_selection }}
REA_CI_TEST_HISTORY: .cache/ci-test-history/test-results.json
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
if: ${{ !cancelled() }}
with:
+3 -2
View File
@@ -346,9 +346,10 @@ export function planChanges(request, repository) {
event,
tier,
code: selected.code,
// The checked-out revision is the one whose tests run.
// The checked-out revision is the one whose tests run. Script and
// configuration files are listed so shared script modules can be traced.
snapshot: {
list: () => repository.listFiles("HEAD", ["src", "tests"]),
list: () => repository.listFiles("HEAD", []),
read: (paths) => repository.readFiles("HEAD", paths),
},
});
+10 -6
View File
@@ -17,23 +17,27 @@ if (
let runner;
let environment = process.env;
if (group === "selected") {
// `selected <k/n>` runs group k of the plan's selected test files, by exact
// identity, in whichever projects own them.
// `selected <k/n>` runs shard k of the plan's selected test files, by exact
// identity, in whichever projects own them. With REA_CI_TEST_HISTORY the CI
// sequencer balances the shards by main's recorded per-file durations.
const [shard, ...rest] = arguments_;
const plan = parseTestSelection(process.env.CI_TEST_SELECTION ?? "");
if (plan.mode !== "selected" || !testShards("fast", plan).includes(shard))
throw new Error(`No selected test group ${shard ?? ""} in this plan`);
if (rest.some((argument) => /^--shard(?:=|$)/u.test(argument)))
throw new Error("Selected test groups are already sharded");
const index = Number(shard.split("/")[0]) - 1;
runner = [
resolve("scripts/run-exact-tests.mjs"),
"run",
`--shard=${shard}`,
...rest,
...plan.groups[index],
...plan.files,
];
// Each group writes its own blob report for the merge.
environment = { ...process.env, VITEST_BLOB_LABEL: `selected-${index + 1}` };
// Each shard writes its own blob report for the merge.
environment = {
...process.env,
VITEST_BLOB_LABEL: `selected-${shard.split("/")[0]}`,
};
} else {
const projects = {
source: SOURCE_TEST_PROJECTS,
+2 -2
View File
@@ -63,7 +63,7 @@ const selected = selection?.mode === "selected";
const expected = !verifyInventory
? []
: selected
? selection.groups.flat()
? selection.files
: execFileSync("git", ["ls-files", "-z", "--", "src", "tests"], {
encoding: "utf8",
maxBuffer: 8 * 1024 * 1024,
@@ -95,7 +95,7 @@ const summary = [
"",
...(selection
? [
`Portable tests: ${selected ? `${selection.groups.flat().length} selected files` : "complete suite"}. ${selection.reason}.`,
`Portable tests: ${selected ? `${selection.files.length} selected files` : "complete suite"}. ${selection.reason}.`,
"",
]
: []),
+23 -8
View File
@@ -1,25 +1,40 @@
export type TestSelectionPlan =
| {
readonly version: 2;
readonly version: 3;
readonly mode: "full";
readonly reason: string;
readonly groups: readonly [];
}
| {
readonly version: 2;
readonly version: 3;
readonly mode: "selected";
readonly reason: string;
readonly groups: readonly (readonly string[])[];
readonly files: readonly string[];
readonly shards: number;
};
export type TestSelectionSnapshot = {
readonly list: () => readonly string[];
readonly read: (paths: readonly string[]) => ReadonlyMap<string, string>;
};
export type TestSelectionReason =
| "changed"
| "imports"
| "built"
| "beside"
| "owner"
| "names";
export function traceAffectedTests(request: {
readonly files: readonly string[];
readonly snapshot: TestSelectionSnapshot;
}): {
readonly selected: ReadonlyMap<string, TestSelectionReason>;
readonly untraced: readonly string[];
};
export function planTestSelection(request: {
readonly files: readonly string[];
readonly event: string;
readonly tier: string;
readonly code: boolean;
readonly snapshot: {
readonly list: () => readonly string[];
readonly read: (paths: readonly string[]) => ReadonlyMap<string, string>;
};
readonly snapshot: TestSelectionSnapshot;
}): TestSelectionPlan;
export function parseTestSelection(value: unknown): TestSelectionPlan;
export function testShards(tier: string, plan: TestSelectionPlan): string[];
+195 -69
View File
@@ -10,9 +10,11 @@ import { SHARED_BOUNDARY } from "./scopes.mjs";
/** More selected files than this run the complete suite instead. */
const SELECTION_CAP = 150;
/** Selected files per test job, and the most jobs one selection uses. */
const FILES_PER_GROUP = 40;
const MAX_GROUPS = 4;
// Selected files per test job, and the most jobs one selection uses. Each job
// runs one `--shard` of the selection; the CI sequencer balances the shards by
// main's recorded per-file durations when that history is available.
const FILES_PER_SHARD = 40;
const MAX_SHARDS = 4;
// Leaf adapters whose tests also reach their sources through the built CLI,
// MCP server or a fake provider process rather than an import. An adapter
@@ -39,6 +41,16 @@ const ownedTests = (path) =>
);
const isTest = (path) => path.endsWith(".test.ts");
// Owners of what the built CLI and MCP server present: tool contracts,
// server registration and presentation, entrypoints and CLI commands.
const isPresentationOwner = (path) =>
!isTest(path) &&
/^src\/(?:server|contracts|main|cli)\/|^src\/(?:main|cli)\.ts$/u.test(path);
// A test that starts the built CLI or MCP server instead of importing it:
// the CLI launcher or this repository's built entry points. Other `dist/`
// paths (package subpaths such as `ajv/dist/2020.js`, fixture apps) are not.
const LAUNCHES_BUILT_RUNTIME =
/scripts\/rea\.mjs|\bdist\/(?:main|cli)\.js|["'`]dist["'`]\s*,\s*["'`](?:main|cli)\.js/u;
// Inputs that only a non-test check reads: declarations (typecheck), guard
// baselines (architecture guards, static job) and the advertised-schema
// summaries (build job). They select no tests.
@@ -46,24 +58,55 @@ const isCheckerInput = (path) =>
/\.d\.m?ts$/u.test(path) ||
/^scripts\/verify\/baselines\/[^/]+\.json$/u.test(path) ||
/^scripts\/verify\/advertised-schemas\/tools\/[^/]+\.json$/u.test(path);
// Verifier scripts are imported by their tests like source modules. Other
// scripts (build, test runners, CI) shape every run and are not traced.
// Verifier scripts are imported by their tests like source modules.
const isVerifier = (path) =>
/^scripts\/(verify-[^/]+|verify\/.+)\.mjs$/u.test(path);
// Shared script modules. A change is traced through the tests, verifiers and
// other shared modules that import it; any other consumer (build, test
// runners, configuration) shapes every run, so it selects the full suite.
const isScriptLibrary = (path) =>
!isCheckerInput(path) && /^scripts\/lib\/.+\.m?[jt]s$/u.test(path);
// Other scripts (build, test runners, CI) shape every run and are not traced.
const isModule = (path) =>
!isCheckerInput(path) &&
(/^(src|tests)\/.+\.m?ts$/u.test(path) ||
/^scripts\/(verify-[^/]+|verify\/.+)\.mjs$/u.test(path));
isVerifier(path) ||
isScriptLibrary(path));
// Test-only modules pass a change through to the tests that use them.
// Production importers never widen the selection transitively.
const isHelper = (path) =>
!isTest(path) &&
(/^tests\/.+\.m?ts$/u.test(path) ||
/^src\/.+\.fixture[^/]*\.ts$/u.test(path));
/^src\/.+\.fixture[^/]*\.ts$/u.test(path) ||
isVerifier(path) ||
isScriptLibrary(path));
// Files outside src/ and tests/ that can run or load a shared script module:
// other scripts, root configuration, package metadata, hooks and workflows.
const isScriptConsumer = (path) =>
!isHelper(path) &&
!isCheckerInput(path) &&
(/^scripts\/.+\.(?:m?[jt]s|cjs)$/u.test(path) ||
(/^[^/]+\.(?:m?[jt]s|cjs|json)$/u.test(path) &&
path !== "package-lock.json") ||
/^\.github\/.+\.ya?ml$/u.test(path) ||
/^\.husky\/[^/]+$/u.test(path));
// Prose that tests may read by path. Other non-module inputs (fixtures, data,
// scripts, workflows, package metadata) are not traced, so they run everything.
const isProse = (path) => path.endsWith(".md") && !/^(src|tests)\//u.test(path);
/** Why a changed path cannot be traced to tests, if it cannot. */
const untracedInput = (path) =>
SHARED_BOUNDARY.test(path)
? `Shared boundary: ${path}`
: !isModule(path) && !isProse(path) && !isCheckerInput(path)
? `Input is not traced to tests: ${path}`
: undefined;
// Static and literal dynamic imports, mocks, and module URLs such as the
// `new URL("./x.mts", import.meta.url)` that `importVerifierSource` loads
// (scripts/lib/import-specifiers.mjs owns the exact parse; the planning job
// runs before dependencies are installed, so it cannot load that parser).
const SPECIFIER =
/(?:\bfrom\s*|\bimport\s*\(\s*|\bimport\s+|\bvi\.mock\(\s*)["'](\.{1,2}\/[^"']+)["']/gu;
/(?:\bfrom\s*|\bimport\s*\(\s*|\bimport\s+|\bvi\.mock\(\s*|\bnew\s+URL\(\s*)["'](\.{1,2}\/[^"']+)["']/gu;
// TypeScript imports name the emitted `.js`/`.mjs`; scripts are imported as is.
const importsOf = (path, source) =>
[...source.matchAll(SPECIFIER)].flatMap(([, specifier]) => {
@@ -71,25 +114,17 @@ const importsOf = (path, source) =>
return [target, target.replace(/\.(m?)js$/u, ".$1ts")];
});
const full = (reason) => ({ version: 2, mode: "full", reason, groups: [] });
/**
* Plan the portable tests one CI run executes.
* Trace the tests a change affects.
*
* `snapshot` lists the tracked `src/` and `tests/` paths of the revision under
* test and reads them; it is consulted only when a subset is possible.
* `snapshot` lists the tracked paths of the revision under test and reads
* them. `selected` maps each affected test to the first reason that selected
* it: `changed`, `imports`, `built`, `beside`, `owner` or `names`. `untraced` names
* inputs whose consumers cannot be traced to tests; CI runs the complete suite
* for them, while a bounded local gate runs the traced selection.
*/
export function planTestSelection({ files, event, tier, code, snapshot }) {
// `ci:full` selects the full tier, so it also lands here.
if (event !== "pull_request" || tier !== "fast")
return full("Main, ci:full and qualification runs execute every test");
if (!code) return full("No portable tests run for this change");
if (!files.length) return full("No changed paths available");
for (const path of files) {
if (SHARED_BOUNDARY.test(path)) return full(`Shared boundary: ${path}`);
if (!isModule(path) && !isProse(path) && !isCheckerInput(path))
return full(`Input is not traced to tests: ${path}`);
}
export function traceAffectedTests({ files, snapshot }) {
const untraced = files.flatMap((path) => untracedInput(path) ?? []);
const tracked = snapshot.list();
const trackedSet = new Set(tracked);
@@ -112,27 +147,102 @@ export function planTestSelection({ files, event, tier, code, snapshot }) {
);
for (const path of files) if (isTest(path)) select(path, "changed");
// Close over test-only helpers, then take the tests that import a change.
const affected = new Set(changedModules);
for (let grew = true; grew;) {
grew = false;
for (const helper of helpers)
if (
!affected.has(helper) &&
imports.get(helper).some((target) => affected.has(target))
) {
affected.add(helper);
grew = true;
}
}
// A seed and every helper that imports it, directly or through another.
const throughHelpers = (seed) => {
const reached = new Set(seed);
for (let grew = true; grew;) {
grew = false;
for (const helper of helpers)
if (
!reached.has(helper) &&
imports.get(helper).some((target) => reached.has(target))
) {
reached.add(helper);
grew = true;
}
}
return reached;
};
// Close over test-only helpers, verifiers and shared script modules, then
// take the tests that import a change.
const affected = throughHelpers(changedModules);
for (const test of tests)
if (imports.get(test).some((target) => affected.has(target)))
select(test, "imports");
// Script paths a test can start or read by name: changed files, plus the
// verifiers and shared modules a change reaches.
const named = (reached) => [
...new Set([
...files,
...[...reached].filter((path) => path.startsWith("scripts/")),
]),
];
const names = (test, paths) =>
paths.some((path) => (sources.get(test) ?? "").includes(path));
// A changed shared module that no test reaches has no coverage in a
// selection, even when other changes select tests.
for (const library of changedModules.filter(isScriptLibrary)) {
const reached = throughHelpers([library]);
const paths = named(reached);
if (
!tests.some(
(test) =>
imports.get(test).some((target) => reached.has(target)) ||
names(test, paths),
)
)
untraced.push(`No test reaches shared script module: ${library}`);
}
// A shared script module that another script, configuration file, hook or
// workflow imports or names by file name can shape every run.
const libraries = [...affected].filter(isScriptLibrary);
if (libraries.length) {
const consumers = tracked.filter(isScriptConsumer);
const consumerSources = snapshot.read(consumers);
for (const consumer of consumers) {
const source = consumerSources.get(consumer) ?? "";
const targets = new Set(importsOf(consumer, source));
const library = libraries.find(
(path) => targets.has(path) || source.includes(posix.basename(path)),
);
if (library !== undefined)
untraced.push(
`Script consumer is not traced: ${consumer} uses ${library}`,
);
}
}
// Tests that start the built CLI or MCP server reach a presentation owner
// through the build rather than an import. Bound them to the tests that
// also import a module from a changed owner's directory.
const changedOwners = new Set(
changedModules
.filter(isPresentationOwner)
.map((path) => posix.dirname(path)),
);
if (changedOwners.size) {
const launchers = throughHelpers(
helpers.filter((path) =>
LAUNCHES_BUILT_RUNTIME.test(sources.get(path) ?? ""),
),
);
for (const test of tests) {
const targets = imports.get(test);
if (
(LAUNCHES_BUILT_RUNTIME.test(sources.get(test) ?? "") ||
targets.some((target) => launchers.has(target))) &&
targets.some((target) => changedOwners.has(posix.dirname(target)))
)
select(test, "built");
}
}
for (const path of changedModules) {
// Same-stem tests, and same-directory tests that scan their directory's
// sources by path instead of importing them.
const stem = path.replace(/\.m?ts$/u, "");
const stem = path.replace(/\.m?[jt]s$/u, "");
for (const test of tests)
if (
posix.dirname(test) === posix.dirname(path) &&
@@ -146,11 +256,33 @@ export function planTestSelection({ files, event, tier, code, snapshot }) {
if (patterns.some((glob) => matchesGlob(test, glob)))
select(test, "owner");
}
// Tests that name a changed path read it directly.
for (const path of files)
for (const test of tests)
if ((sources.get(test) ?? "").includes(path)) select(test, "names");
// Tests that name a changed path, or a verifier or shared module the change
// reaches, read or start it directly.
const reachedPaths = named(affected);
for (const test of tests)
if (names(test, reachedPaths)) select(test, "names");
return { selected, untraced };
}
const full = (reason) => ({ version: 3, mode: "full", reason });
/**
* Plan the portable tests one CI run executes.
*
* `snapshot` lists the tracked paths of the revision under test and reads
* them; it is consulted only when a subset is possible.
*/
export function planTestSelection({ files, event, tier, code, snapshot }) {
// `ci:full` selects the full tier, so it also lands here.
if (event !== "pull_request" || tier !== "fast")
return full("Main, ci:full and qualification runs execute every test");
if (!code) return full("No portable tests run for this change");
if (!files.length) return full("No changed paths available");
// Decide from the paths alone before reading the snapshot.
const untracedPath = files.map(untracedInput).find(Boolean);
if (untracedPath !== undefined) return full(untracedPath);
const { selected, untraced } = traceAffectedTests({ files, snapshot });
if (untraced.length) return full(untraced[0]);
const paths = [...selected.keys()].sort();
if (!paths.length)
return full("No test imports, names or sits beside a changed path");
@@ -164,46 +296,42 @@ export function planTestSelection({ files, event, tier, code, snapshot }) {
} catch {
return full(`No single test project runs ${path}`);
}
const count = Math.min(MAX_GROUPS, Math.ceil(paths.length / FILES_PER_GROUP));
const groups = Array.from({ length: count }, () => []);
paths.forEach((path, index) => groups[index % count].push(path));
const reasons = Object.groupBy([...selected.values()], (reason) => reason);
return {
version: 2,
version: 3,
mode: "selected",
reason: `Affected tests: ${Object.entries(reasons)
.map(([reason, entries]) => `${entries.length} ${reason}`)
.join(", ")}`,
groups,
files: paths,
shards: Math.min(MAX_SHARDS, Math.ceil(paths.length / FILES_PER_SHARD)),
};
}
/** Parse a plan produced by `planTestSelection`; anything else fails closed. */
export function parseTestSelection(value) {
const plan = typeof value === "string" ? JSON.parse(value) : value;
const files = plan?.files;
const valid =
plan?.version === 2 &&
plan?.version === 3 &&
typeof plan.reason === "string" &&
Array.isArray(plan.groups) &&
(plan.mode === "full"
? plan.groups.length === 0
? files === undefined && plan.shards === undefined
: plan.mode === "selected" &&
plan.groups.length >= 1 &&
plan.groups.length <= MAX_GROUPS &&
plan.groups.every(
(group) =>
Array.isArray(group) &&
group.length > 0 &&
group.every(
(path) =>
typeof path === "string" &&
isTest(path) &&
/^(src|tests)\//u.test(path) &&
!path.split("/").some((part) => ["", ".", ".."].includes(part)),
),
Array.isArray(files) &&
files.length > 0 &&
files.length <= SELECTION_CAP &&
new Set(files).size === files.length &&
files.every(
(path) =>
typeof path === "string" &&
isTest(path) &&
/^(src|tests)\//u.test(path) &&
!path.split("/").some((part) => ["", ".", ".."].includes(part)),
) &&
new Set(plan.groups.flat()).size === plan.groups.flat().length &&
plan.groups.flat().length <= SELECTION_CAP);
Number.isInteger(plan.shards) &&
plan.shards >= 1 &&
plan.shards <= Math.min(MAX_SHARDS, files.length));
if (!valid) throw new Error("Invalid portable test selection plan");
return plan;
}
@@ -213,12 +341,10 @@ export function parseTestSelection(value) {
// slowest shard sets the critical path: at eight shards it ran 24% above the
// mean shard. Smaller shards bound that excess at a few percent more runner
// time. Full qualification retains six complete shards with coverage. A
// selection runs one job per group.
// selection runs its own shard count.
const shardsOf = (count) =>
Array.from({ length: count }, (_, index) => `${index + 1}/${count}`);
/** The `test-shard` matrix labels for a tier and its test selection. */
export const testShards = (tier, plan) =>
shardsOf(
plan.mode === "selected" ? plan.groups.length : tier === "full" ? 6 : 12,
);
shardsOf(plan.mode === "selected" ? plan.shards : tier === "full" ? 6 : 12);
@@ -458,7 +458,7 @@ it("selects the tests a pull request affects and runs everything when it cannot
"tests/conformance/named.test.ts",
];
expect(await runPlanScript(directory, base, head)).toMatchObject({
testSelection: { version: 2, mode: "selected", groups: [selected] },
testSelection: { version: 3, mode: "selected", files: selected, shards: 1 },
testShards: ["1/1"],
});
const output = await readFile(join(directory, "scope-output"), "utf8");
@@ -469,7 +469,7 @@ it("selects the tests a pull request affects and runs everything when it cannot
expect(
await runPlanScript(directory, base, head, "pull_request", true),
).toMatchObject({
testSelection: { mode: "full", groups: [] },
testSelection: { version: 3, mode: "full" },
testShards: ["1/6", "2/6", "3/6", "4/6", "5/6", "6/6"],
});
expect(await classify(directory, base, head, "push")).toMatchObject({
+1 -2
View File
@@ -406,10 +406,9 @@ it("merges source and built reports exactly once using the workflow's shard matr
outputs: {
test_shards: JSON.stringify(
testShards("fast", {
version: 2,
version: 3,
mode: "full",
reason: "Complete fast suite",
groups: [],
}),
),
},
@@ -807,6 +807,7 @@ it("runs a selected PR subset in planner-owned groups and still gates it", async
expect(selected?.env).toEqual({
MATRIX_SHARD: "${{ matrix.shard }}",
CI_TEST_SELECTION: "${{ needs.changes.outputs.test_selection }}",
REA_CI_TEST_HISTORY: ".cache/ci-test-history/test-results.json",
});
const flags = await scopeFlags();
@@ -670,10 +670,11 @@ it("verifies exactly the selected subset and never publishes it as timing histor
}
await workflowGit(root, ["add", "src", "tests"]);
const selection = {
version: 2,
version: 3,
mode: "selected",
reason: "Affected tests: 2 changed",
groups: [["src/a.test.ts"], ["tests/b.test.ts"]],
files: ["src/a.test.ts", "tests/b.test.ts"],
shards: 2,
};
const run = async (names: readonly string[], plan: unknown = selection) => {
await writeFile(
@@ -707,7 +708,7 @@ it("verifies exactly the selected subset and never publishes it as timing histor
stderr: expect.stringContaining("Incomplete selected test inventory"),
});
// A full plan keeps the complete tracked inventory and its history.
const full = { version: 2, mode: "full", reason: "Shared", groups: [] };
const full = { version: 3, mode: "full", reason: "Shared" };
await expect(
run(["src/a.test.ts", "tests/b.test.ts"], full),
).rejects.toMatchObject({
@@ -719,11 +720,12 @@ it("verifies exactly the selected subset and never publishes it as timing histor
).resolves.toContain("tests/unselected.test.ts");
// An unreadable plan fails closed.
for (const plan of [
{ ...selection, version: 1 },
{ ...selection, groups: [] },
{ ...selection, groups: [["../escape.test.ts"]] },
{ ...selection, groups: [["src/a.test.ts"], ["src/a.test.ts"]] },
{ ...full, groups: [["src/a.test.ts"]] },
{ ...selection, version: 2 },
{ ...selection, files: [] },
{ ...selection, files: ["../escape.test.ts"] },
{ ...selection, files: ["src/a.test.ts", "src/a.test.ts"] },
{ ...selection, shards: 3 },
{ ...full, files: ["src/a.test.ts"] },
])
await expect(run(["src/a.test.ts"], plan)).rejects.toMatchObject({
stderr: expect.stringContaining("Invalid portable test selection plan"),
+126 -19
View File
@@ -26,7 +26,7 @@ const plan = (files: readonly string[], sources: Record<string, string>) =>
});
const selectedFiles = (result: ReturnType<typeof plan>) =>
result.groups.flat().toSorted();
(result.mode === "selected" ? result.files : []).toSorted();
it("follows test and verifier imports and directory scans, not production importers", () => {
const sources = {
@@ -67,6 +67,37 @@ it("selects a leaf adapter's tests by its name and directory across test project
]);
});
it("selects tests that start the built runtime beside a changed presentation owner", () => {
const sources = {
"src/server/Core.ts": "",
"src/server/recovery.ts": "",
"src/domain/Value.ts": "",
"tests/support/launch.ts": 'spawn("scripts/rea.mjs", ["mcp"]);',
"tests/fixtures/client.ts": 'import "../support/launch.js";',
// Starts the built server and imports a sibling of the changed owner.
"tests/boundary/mcp/direct.test.ts":
'import "../../../src/server/recovery.js"; resolve("dist/main.js");',
"tests/boundary/mcp/helper.test.ts":
'import "../../fixtures/client.js"; import "../../../src/server/recovery.js";',
// Starts it but imports nothing from that owner, or imports it in process.
"tests/boundary/mcp/unrelated.test.ts":
'import "../../fixtures/client.js"; import "../../../src/domain/Value.js";',
"tests/boundary/mcp/inProcess.test.ts":
'import "../../../src/server/recovery.js";',
// A package's own dist/ subpath is not the built runtime.
"tests/boundary/mcp/ajv.test.ts":
'import { Ajv2020 } from "ajv/dist/2020.js"; import "../../../src/server/recovery.js";',
};
expect(selectedFiles(plan(["src/server/Core.ts"], sources))).toEqual([
"tests/boundary/mcp/direct.test.ts",
"tests/boundary/mcp/helper.test.ts",
]);
// Other owners keep import tracing only.
expect(plan(["src/domain/Value.ts"], sources)).toMatchObject({
reason: "Affected tests: 1 imports",
});
});
it("lets checker-only inputs and prose select nothing by themselves", () => {
const sources = {
"src/domain/Value.test.ts": "",
@@ -93,60 +124,136 @@ it("lets checker-only inputs and prose select nothing by themselves", () => {
expect(plan(["scripts/verify/baselines/x.json"], sources)).toMatchObject({
mode: "full",
});
expect(plan(["scripts/lib/shared.mjs"], sources)).toMatchObject({
});
it("traces shared script modules to their tests and falls back for other consumers", () => {
const sources = {
"scripts/lib/shared.mjs": "",
"scripts/lib/wrapper.mjs": 'export * from "./shared.mjs";',
"scripts/verify-thing.mjs": 'import "./lib/wrapper.mjs";',
"scripts/lib/unused.mjs": "",
"tests/boundary/filesystem/shared.test.ts":
'import "../../../scripts/lib/shared.mjs";',
"tests/boundary/filesystem/verifier.test.ts":
'await import("../../../scripts/verify-thing.mjs");',
"tests/boundary/filesystem/other.test.ts": "",
"package.json":
'{"scripts":{"verify:thing":"node scripts/verify-thing.mjs"}}',
};
// Through another shared module and a verifier to the tests that import them.
expect(selectedFiles(plan(["scripts/lib/shared.mjs"], sources))).toEqual([
"tests/boundary/filesystem/shared.test.ts",
"tests/boundary/filesystem/verifier.test.ts",
]);
// A build script, configuration or package script that loads it is untraced.
expect(
plan(["scripts/lib/shared.mjs"], {
...sources,
"scripts/build.mjs": 'import { x } from "./lib/wrapper.mjs";',
}),
).toMatchObject({
mode: "full",
reason: "Input is not traced to tests: scripts/lib/shared.mjs",
reason:
"Script consumer is not traced: scripts/build.mjs uses scripts/lib/wrapper.mjs",
});
expect(
plan(["scripts/lib/shared.mjs"], {
...sources,
"package.json": '{"scripts":{"x":"node scripts/lib/shared.mjs"}}',
}),
).toMatchObject({
mode: "full",
reason:
"Script consumer is not traced: package.json uses scripts/lib/shared.mjs",
});
// A module no test reaches runs everything, even beside a selected change.
for (const files of [
["scripts/lib/unused.mjs"],
["scripts/lib/unused.mjs", "tests/boundary/filesystem/other.test.ts"],
])
expect(plan(files, sources)).toMatchObject({
mode: "full",
reason: "No test reaches shared script module: scripts/lib/unused.mjs",
});
// Verifier source loads are module edges, and a test that starts a reached
// verifier by path is selected without importing it.
const loaded = {
"scripts/lib/startup.mts": "",
"scripts/lib/lifecycle.mjs":
'await importVerifierSource(new URL("./startup.mts", import.meta.url).href, import.meta.url);',
"scripts/verify-runtime.mjs": 'import "./lib/lifecycle.mjs";',
"tests/boundary/filesystem/startup.test.ts":
'import "../../../scripts/lib/startup.mts";',
"tests/boundary/filesystem/lifecycle.test.ts":
'import "../../../scripts/lib/lifecycle.mjs";',
"tests/boundary/filesystem/runtimeVerifier.test.ts":
'spawn(process.execPath, ["scripts/verify-runtime.mjs"]);',
};
expect(selectedFiles(plan(["scripts/lib/startup.mts"], loaded))).toEqual([
"tests/boundary/filesystem/lifecycle.test.ts",
"tests/boundary/filesystem/runtimeVerifier.test.ts",
"tests/boundary/filesystem/startup.test.ts",
]);
// Other scripts remain untraced inputs.
expect(plan(["scripts/build.mjs"], sources)).toMatchObject({
mode: "full",
reason: "Input is not traced to tests: scripts/build.mjs",
});
});
it("parses only well-formed plans and derives one shard per selected group", () => {
it("parses only well-formed plans and derives the selected shard count", () => {
const selected = {
version: 2,
version: 3,
mode: "selected",
reason: "Affected tests: 2 changed",
groups: [["src/a.test.ts"], ["tests/b.test.ts"]],
files: ["src/a.test.ts", "tests/b.test.ts"],
shards: 2,
};
expect(
testShards("fast", parseTestSelection(JSON.stringify(selected))),
).toEqual(["1/2", "2/2"]);
const full = { version: 2, mode: "full", reason: "Shared", groups: [] };
const full = { version: 3, mode: "full", reason: "Shared" };
expect(testShards("fast", parseTestSelection(full))).toHaveLength(12);
expect(testShards("full", parseTestSelection(full))).toHaveLength(6);
for (const value of [
"",
"null",
{ ...selected, version: 2 },
{ ...selected, mode: "candidate" },
{ ...selected, groups: [[]] },
{ ...selected, groups: [["src/a.ts"]] },
{ ...selected, groups: [["/abs/a.test.ts"]] },
{ ...selected, groups: [["src/a.test.ts"], ["src/a.test.ts"]] },
{ ...selected, files: [] },
{ ...selected, files: ["src/a.ts"] },
{ ...selected, files: ["/abs/a.test.ts"] },
{ ...selected, files: ["src/a.test.ts", "src/a.test.ts"] },
// More shards than files, or than one selection uses.
{ ...selected, shards: 3 },
{ ...selected, shards: 0 },
{
...selected,
groups: Array.from({ length: 5 }, (_, i) => [`src/${i}.test.ts`]),
files: Array.from({ length: 5 }, (_, i) => `src/${i}.test.ts`),
shards: 5,
},
{ ...full, shards: 1 },
])
expect(() => parseTestSelection(value)).toThrow(
/Invalid portable test selection plan|JSON/u,
);
});
it("runs only a group the plan defines", async () => {
it("runs only a shard the plan defines", async () => {
const group = fileURLToPath(
new URL("../../scripts/ci/test-group.mjs", import.meta.url),
);
const selection = JSON.stringify({
version: 2,
version: 3,
mode: "selected",
reason: "Affected tests: 1 changed",
groups: [["src/a.test.ts"]],
files: ["src/a.test.ts"],
shards: 1,
});
for (const [shard, environment] of [
["2/2", selection],
["1/2", selection],
[
"1/1",
JSON.stringify({ version: 2, mode: "full", reason: "x", groups: [] }),
],
["1/1", JSON.stringify({ version: 3, mode: "full", reason: "x" })],
["1/1", ""],
] as const)
await expect(