mirror of
https://github.com/morluto/rea.git
synced 2026-11-16 11:43:20 +00:00
ci: trace script modules and built-server consumers in PR test selection, and balance selected shards (#11516)
* ci: trace shared script modules to the tests that import them A pull request that changed any scripts/lib module ran the complete suite. Shared script modules now pass a change through to the tests, verifiers and other shared modules that import them, like test helpers. Another consumer (a build or test-runner script, root configuration, package metadata, a hook or a workflow) that imports the module or names it by file name still selects the complete suite. The tracing core is exported as traceAffectedTests so a bounded local gate can reuse it. Fixes #11381 * ci: follow verifier source loads and named verifiers when tracing script modules - Module URLs (new URL("./x.mts", import.meta.url), as loaded by importVerifierSource) are import edges. - A test that names a verifier or shared module the change reaches is selected, not only one that names a changed path. - A changed shared module that no test reaches runs the complete suite even when other changes select tests. * ci: select tests that start the built server beside a changed presentation owner (#11519) * ci: select tests that start the built server beside a changed presentation owner A test that starts the built CLI or MCP server reaches server, contract, entrypoint and CLI modules through the build, not an import, so a change there selected none of them (#11146 failed on main this way). Such a test is now selected when it, or a helper it imports, starts scripts/rea.mjs or dist/, and it imports a module from the changed owner's directory. Fixes #11366 * ci: recognise only the repository's built entry points as runtime launchers A package subpath such as ajv/dist/2020.js matched the dist/ pattern, so in-process tests importing Ajv counted as built-runtime launchers. * ci: balance selected test shards by recorded duration (#11522) A selected PR run split its files across groups round-robin, so a few slow files could land together. The plan now records the selected files and a shard count (version 3), and each job runs one --shard of the selection through the exact-test runner with the timing history main already publishes, so the CI sequencer balances shards by per-file duration (the median for files without a timing). Without history, Vitest's own deterministic sharding applies. Fixes #11382
This commit is contained in:
1 parent
336275259c
commit
062b3835f9
11 files changed
+374
-118
No files matched your search
@@ -367,6 +367,7 @@ jobs:
|
||||
env:
|
||||
MATRIX_SHARD: ${{ matrix.shard }}
|
||||
CI_TEST_SELECTION: ${{ needs.changes.outputs.test_selection }}
|
||||
REA_CI_TEST_HISTORY: .cache/ci-test-history/test-results.json
|
||||
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||
if: ${{ !cancelled() }}
|
||||
with:
|
||||
|
||||
@@ -346,9 +346,10 @@ export function planChanges(request, repository) {
|
||||
event,
|
||||
tier,
|
||||
code: selected.code,
|
||||
// The checked-out revision is the one whose tests run.
|
||||
// The checked-out revision is the one whose tests run. Script and
|
||||
// configuration files are listed so shared script modules can be traced.
|
||||
snapshot: {
|
||||
list: () => repository.listFiles("HEAD", ["src", "tests"]),
|
||||
list: () => repository.listFiles("HEAD", []),
|
||||
read: (paths) => repository.readFiles("HEAD", paths),
|
||||
},
|
||||
});
|
||||
|
||||
@@ -17,23 +17,27 @@ if (
|
||||
let runner;
|
||||
let environment = process.env;
|
||||
if (group === "selected") {
|
||||
// `selected <k/n>` runs group k of the plan's selected test files, by exact
|
||||
// identity, in whichever projects own them.
|
||||
// `selected <k/n>` runs shard k of the plan's selected test files, by exact
|
||||
// identity, in whichever projects own them. With REA_CI_TEST_HISTORY the CI
|
||||
// sequencer balances the shards by main's recorded per-file durations.
|
||||
const [shard, ...rest] = arguments_;
|
||||
const plan = parseTestSelection(process.env.CI_TEST_SELECTION ?? "");
|
||||
if (plan.mode !== "selected" || !testShards("fast", plan).includes(shard))
|
||||
throw new Error(`No selected test group ${shard ?? ""} in this plan`);
|
||||
if (rest.some((argument) => /^--shard(?:=|$)/u.test(argument)))
|
||||
throw new Error("Selected test groups are already sharded");
|
||||
const index = Number(shard.split("/")[0]) - 1;
|
||||
runner = [
|
||||
resolve("scripts/run-exact-tests.mjs"),
|
||||
"run",
|
||||
`--shard=${shard}`,
|
||||
...rest,
|
||||
...plan.groups[index],
|
||||
...plan.files,
|
||||
];
|
||||
// Each group writes its own blob report for the merge.
|
||||
environment = { ...process.env, VITEST_BLOB_LABEL: `selected-${index + 1}` };
|
||||
// Each shard writes its own blob report for the merge.
|
||||
environment = {
|
||||
...process.env,
|
||||
VITEST_BLOB_LABEL: `selected-${shard.split("/")[0]}`,
|
||||
};
|
||||
} else {
|
||||
const projects = {
|
||||
source: SOURCE_TEST_PROJECTS,
|
||||
|
||||
@@ -63,7 +63,7 @@ const selected = selection?.mode === "selected";
|
||||
const expected = !verifyInventory
|
||||
? []
|
||||
: selected
|
||||
? selection.groups.flat()
|
||||
? selection.files
|
||||
: execFileSync("git", ["ls-files", "-z", "--", "src", "tests"], {
|
||||
encoding: "utf8",
|
||||
maxBuffer: 8 * 1024 * 1024,
|
||||
@@ -95,7 +95,7 @@ const summary = [
|
||||
"",
|
||||
...(selection
|
||||
? [
|
||||
`Portable tests: ${selected ? `${selection.groups.flat().length} selected files` : "complete suite"}. ${selection.reason}.`,
|
||||
`Portable tests: ${selected ? `${selection.files.length} selected files` : "complete suite"}. ${selection.reason}.`,
|
||||
"",
|
||||
]
|
||||
: []),
|
||||
|
||||
@@ -1,25 +1,40 @@
|
||||
export type TestSelectionPlan =
|
||||
| {
|
||||
readonly version: 2;
|
||||
readonly version: 3;
|
||||
readonly mode: "full";
|
||||
readonly reason: string;
|
||||
readonly groups: readonly [];
|
||||
}
|
||||
| {
|
||||
readonly version: 2;
|
||||
readonly version: 3;
|
||||
readonly mode: "selected";
|
||||
readonly reason: string;
|
||||
readonly groups: readonly (readonly string[])[];
|
||||
readonly files: readonly string[];
|
||||
readonly shards: number;
|
||||
};
|
||||
export type TestSelectionSnapshot = {
|
||||
readonly list: () => readonly string[];
|
||||
readonly read: (paths: readonly string[]) => ReadonlyMap<string, string>;
|
||||
};
|
||||
export type TestSelectionReason =
|
||||
| "changed"
|
||||
| "imports"
|
||||
| "built"
|
||||
| "beside"
|
||||
| "owner"
|
||||
| "names";
|
||||
export function traceAffectedTests(request: {
|
||||
readonly files: readonly string[];
|
||||
readonly snapshot: TestSelectionSnapshot;
|
||||
}): {
|
||||
readonly selected: ReadonlyMap<string, TestSelectionReason>;
|
||||
readonly untraced: readonly string[];
|
||||
};
|
||||
export function planTestSelection(request: {
|
||||
readonly files: readonly string[];
|
||||
readonly event: string;
|
||||
readonly tier: string;
|
||||
readonly code: boolean;
|
||||
readonly snapshot: {
|
||||
readonly list: () => readonly string[];
|
||||
readonly read: (paths: readonly string[]) => ReadonlyMap<string, string>;
|
||||
};
|
||||
readonly snapshot: TestSelectionSnapshot;
|
||||
}): TestSelectionPlan;
|
||||
export function parseTestSelection(value: unknown): TestSelectionPlan;
|
||||
export function testShards(tier: string, plan: TestSelectionPlan): string[];
|
||||
+195
-69
@@ -10,9 +10,11 @@ import { SHARED_BOUNDARY } from "./scopes.mjs";
|
||||
|
||||
/** More selected files than this run the complete suite instead. */
|
||||
const SELECTION_CAP = 150;
|
||||
/** Selected files per test job, and the most jobs one selection uses. */
|
||||
const FILES_PER_GROUP = 40;
|
||||
const MAX_GROUPS = 4;
|
||||
// Selected files per test job, and the most jobs one selection uses. Each job
|
||||
// runs one `--shard` of the selection; the CI sequencer balances the shards by
|
||||
// main's recorded per-file durations when that history is available.
|
||||
const FILES_PER_SHARD = 40;
|
||||
const MAX_SHARDS = 4;
|
||||
|
||||
// Leaf adapters whose tests also reach their sources through the built CLI,
|
||||
// MCP server or a fake provider process rather than an import. An adapter
|
||||
@@ -39,6 +41,16 @@ const ownedTests = (path) =>
|
||||
);
|
||||
|
||||
const isTest = (path) => path.endsWith(".test.ts");
|
||||
// Owners of what the built CLI and MCP server present: tool contracts,
|
||||
// server registration and presentation, entrypoints and CLI commands.
|
||||
const isPresentationOwner = (path) =>
|
||||
!isTest(path) &&
|
||||
/^src\/(?:server|contracts|main|cli)\/|^src\/(?:main|cli)\.ts$/u.test(path);
|
||||
// A test that starts the built CLI or MCP server instead of importing it:
|
||||
// the CLI launcher or this repository's built entry points. Other `dist/`
|
||||
// paths (package subpaths such as `ajv/dist/2020.js`, fixture apps) are not.
|
||||
const LAUNCHES_BUILT_RUNTIME =
|
||||
/scripts\/rea\.mjs|\bdist\/(?:main|cli)\.js|["'`]dist["'`]\s*,\s*["'`](?:main|cli)\.js/u;
|
||||
// Inputs that only a non-test check reads: declarations (typecheck), guard
|
||||
// baselines (architecture guards, static job) and the advertised-schema
|
||||
// summaries (build job). They select no tests.
|
||||
@@ -46,24 +58,55 @@ const isCheckerInput = (path) =>
|
||||
/\.d\.m?ts$/u.test(path) ||
|
||||
/^scripts\/verify\/baselines\/[^/]+\.json$/u.test(path) ||
|
||||
/^scripts\/verify\/advertised-schemas\/tools\/[^/]+\.json$/u.test(path);
|
||||
// Verifier scripts are imported by their tests like source modules. Other
|
||||
// scripts (build, test runners, CI) shape every run and are not traced.
|
||||
// Verifier scripts are imported by their tests like source modules.
|
||||
const isVerifier = (path) =>
|
||||
/^scripts\/(verify-[^/]+|verify\/.+)\.mjs$/u.test(path);
|
||||
// Shared script modules. A change is traced through the tests, verifiers and
|
||||
// other shared modules that import it; any other consumer (build, test
|
||||
// runners, configuration) shapes every run, so it selects the full suite.
|
||||
const isScriptLibrary = (path) =>
|
||||
!isCheckerInput(path) && /^scripts\/lib\/.+\.m?[jt]s$/u.test(path);
|
||||
// Other scripts (build, test runners, CI) shape every run and are not traced.
|
||||
const isModule = (path) =>
|
||||
!isCheckerInput(path) &&
|
||||
(/^(src|tests)\/.+\.m?ts$/u.test(path) ||
|
||||
/^scripts\/(verify-[^/]+|verify\/.+)\.mjs$/u.test(path));
|
||||
isVerifier(path) ||
|
||||
isScriptLibrary(path));
|
||||
// Test-only modules pass a change through to the tests that use them.
|
||||
// Production importers never widen the selection transitively.
|
||||
const isHelper = (path) =>
|
||||
!isTest(path) &&
|
||||
(/^tests\/.+\.m?ts$/u.test(path) ||
|
||||
/^src\/.+\.fixture[^/]*\.ts$/u.test(path));
|
||||
/^src\/.+\.fixture[^/]*\.ts$/u.test(path) ||
|
||||
isVerifier(path) ||
|
||||
isScriptLibrary(path));
|
||||
// Files outside src/ and tests/ that can run or load a shared script module:
|
||||
// other scripts, root configuration, package metadata, hooks and workflows.
|
||||
const isScriptConsumer = (path) =>
|
||||
!isHelper(path) &&
|
||||
!isCheckerInput(path) &&
|
||||
(/^scripts\/.+\.(?:m?[jt]s|cjs)$/u.test(path) ||
|
||||
(/^[^/]+\.(?:m?[jt]s|cjs|json)$/u.test(path) &&
|
||||
path !== "package-lock.json") ||
|
||||
/^\.github\/.+\.ya?ml$/u.test(path) ||
|
||||
/^\.husky\/[^/]+$/u.test(path));
|
||||
// Prose that tests may read by path. Other non-module inputs (fixtures, data,
|
||||
// scripts, workflows, package metadata) are not traced, so they run everything.
|
||||
const isProse = (path) => path.endsWith(".md") && !/^(src|tests)\//u.test(path);
|
||||
/** Why a changed path cannot be traced to tests, if it cannot. */
|
||||
const untracedInput = (path) =>
|
||||
SHARED_BOUNDARY.test(path)
|
||||
? `Shared boundary: ${path}`
|
||||
: !isModule(path) && !isProse(path) && !isCheckerInput(path)
|
||||
? `Input is not traced to tests: ${path}`
|
||||
: undefined;
|
||||
|
||||
// Static and literal dynamic imports, mocks, and module URLs such as the
|
||||
// `new URL("./x.mts", import.meta.url)` that `importVerifierSource` loads
|
||||
// (scripts/lib/import-specifiers.mjs owns the exact parse; the planning job
|
||||
// runs before dependencies are installed, so it cannot load that parser).
|
||||
const SPECIFIER =
|
||||
/(?:\bfrom\s*|\bimport\s*\(\s*|\bimport\s+|\bvi\.mock\(\s*)["'](\.{1,2}\/[^"']+)["']/gu;
|
||||
/(?:\bfrom\s*|\bimport\s*\(\s*|\bimport\s+|\bvi\.mock\(\s*|\bnew\s+URL\(\s*)["'](\.{1,2}\/[^"']+)["']/gu;
|
||||
// TypeScript imports name the emitted `.js`/`.mjs`; scripts are imported as is.
|
||||
const importsOf = (path, source) =>
|
||||
[...source.matchAll(SPECIFIER)].flatMap(([, specifier]) => {
|
||||
@@ -71,25 +114,17 @@ const importsOf = (path, source) =>
|
||||
return [target, target.replace(/\.(m?)js$/u, ".$1ts")];
|
||||
});
|
||||
|
||||
const full = (reason) => ({ version: 2, mode: "full", reason, groups: [] });
|
||||
|
||||
/**
|
||||
* Plan the portable tests one CI run executes.
|
||||
* Trace the tests a change affects.
|
||||
*
|
||||
* `snapshot` lists the tracked `src/` and `tests/` paths of the revision under
|
||||
* test and reads them; it is consulted only when a subset is possible.
|
||||
* `snapshot` lists the tracked paths of the revision under test and reads
|
||||
* them. `selected` maps each affected test to the first reason that selected
|
||||
* it: `changed`, `imports`, `built`, `beside`, `owner` or `names`. `untraced` names
|
||||
* inputs whose consumers cannot be traced to tests; CI runs the complete suite
|
||||
* for them, while a bounded local gate runs the traced selection.
|
||||
*/
|
||||
export function planTestSelection({ files, event, tier, code, snapshot }) {
|
||||
// `ci:full` selects the full tier, so it also lands here.
|
||||
if (event !== "pull_request" || tier !== "fast")
|
||||
return full("Main, ci:full and qualification runs execute every test");
|
||||
if (!code) return full("No portable tests run for this change");
|
||||
if (!files.length) return full("No changed paths available");
|
||||
for (const path of files) {
|
||||
if (SHARED_BOUNDARY.test(path)) return full(`Shared boundary: ${path}`);
|
||||
if (!isModule(path) && !isProse(path) && !isCheckerInput(path))
|
||||
return full(`Input is not traced to tests: ${path}`);
|
||||
}
|
||||
export function traceAffectedTests({ files, snapshot }) {
|
||||
const untraced = files.flatMap((path) => untracedInput(path) ?? []);
|
||||
|
||||
const tracked = snapshot.list();
|
||||
const trackedSet = new Set(tracked);
|
||||
@@ -112,27 +147,102 @@ export function planTestSelection({ files, event, tier, code, snapshot }) {
|
||||
);
|
||||
for (const path of files) if (isTest(path)) select(path, "changed");
|
||||
|
||||
// Close over test-only helpers, then take the tests that import a change.
|
||||
const affected = new Set(changedModules);
|
||||
for (let grew = true; grew;) {
|
||||
grew = false;
|
||||
for (const helper of helpers)
|
||||
if (
|
||||
!affected.has(helper) &&
|
||||
imports.get(helper).some((target) => affected.has(target))
|
||||
) {
|
||||
affected.add(helper);
|
||||
grew = true;
|
||||
}
|
||||
}
|
||||
// A seed and every helper that imports it, directly or through another.
|
||||
const throughHelpers = (seed) => {
|
||||
const reached = new Set(seed);
|
||||
for (let grew = true; grew;) {
|
||||
grew = false;
|
||||
for (const helper of helpers)
|
||||
if (
|
||||
!reached.has(helper) &&
|
||||
imports.get(helper).some((target) => reached.has(target))
|
||||
) {
|
||||
reached.add(helper);
|
||||
grew = true;
|
||||
}
|
||||
}
|
||||
return reached;
|
||||
};
|
||||
// Close over test-only helpers, verifiers and shared script modules, then
|
||||
// take the tests that import a change.
|
||||
const affected = throughHelpers(changedModules);
|
||||
for (const test of tests)
|
||||
if (imports.get(test).some((target) => affected.has(target)))
|
||||
select(test, "imports");
|
||||
// Script paths a test can start or read by name: changed files, plus the
|
||||
// verifiers and shared modules a change reaches.
|
||||
const named = (reached) => [
|
||||
...new Set([
|
||||
...files,
|
||||
...[...reached].filter((path) => path.startsWith("scripts/")),
|
||||
]),
|
||||
];
|
||||
const names = (test, paths) =>
|
||||
paths.some((path) => (sources.get(test) ?? "").includes(path));
|
||||
|
||||
// A changed shared module that no test reaches has no coverage in a
|
||||
// selection, even when other changes select tests.
|
||||
for (const library of changedModules.filter(isScriptLibrary)) {
|
||||
const reached = throughHelpers([library]);
|
||||
const paths = named(reached);
|
||||
if (
|
||||
!tests.some(
|
||||
(test) =>
|
||||
imports.get(test).some((target) => reached.has(target)) ||
|
||||
names(test, paths),
|
||||
)
|
||||
)
|
||||
untraced.push(`No test reaches shared script module: ${library}`);
|
||||
}
|
||||
|
||||
// A shared script module that another script, configuration file, hook or
|
||||
// workflow imports or names by file name can shape every run.
|
||||
const libraries = [...affected].filter(isScriptLibrary);
|
||||
if (libraries.length) {
|
||||
const consumers = tracked.filter(isScriptConsumer);
|
||||
const consumerSources = snapshot.read(consumers);
|
||||
for (const consumer of consumers) {
|
||||
const source = consumerSources.get(consumer) ?? "";
|
||||
const targets = new Set(importsOf(consumer, source));
|
||||
const library = libraries.find(
|
||||
(path) => targets.has(path) || source.includes(posix.basename(path)),
|
||||
);
|
||||
if (library !== undefined)
|
||||
untraced.push(
|
||||
`Script consumer is not traced: ${consumer} uses ${library}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Tests that start the built CLI or MCP server reach a presentation owner
|
||||
// through the build rather than an import. Bound them to the tests that
|
||||
// also import a module from a changed owner's directory.
|
||||
const changedOwners = new Set(
|
||||
changedModules
|
||||
.filter(isPresentationOwner)
|
||||
.map((path) => posix.dirname(path)),
|
||||
);
|
||||
if (changedOwners.size) {
|
||||
const launchers = throughHelpers(
|
||||
helpers.filter((path) =>
|
||||
LAUNCHES_BUILT_RUNTIME.test(sources.get(path) ?? ""),
|
||||
),
|
||||
);
|
||||
for (const test of tests) {
|
||||
const targets = imports.get(test);
|
||||
if (
|
||||
(LAUNCHES_BUILT_RUNTIME.test(sources.get(test) ?? "") ||
|
||||
targets.some((target) => launchers.has(target))) &&
|
||||
targets.some((target) => changedOwners.has(posix.dirname(target)))
|
||||
)
|
||||
select(test, "built");
|
||||
}
|
||||
}
|
||||
|
||||
for (const path of changedModules) {
|
||||
// Same-stem tests, and same-directory tests that scan their directory's
|
||||
// sources by path instead of importing them.
|
||||
const stem = path.replace(/\.m?ts$/u, "");
|
||||
const stem = path.replace(/\.m?[jt]s$/u, "");
|
||||
for (const test of tests)
|
||||
if (
|
||||
posix.dirname(test) === posix.dirname(path) &&
|
||||
@@ -146,11 +256,33 @@ export function planTestSelection({ files, event, tier, code, snapshot }) {
|
||||
if (patterns.some((glob) => matchesGlob(test, glob)))
|
||||
select(test, "owner");
|
||||
}
|
||||
// Tests that name a changed path read it directly.
|
||||
for (const path of files)
|
||||
for (const test of tests)
|
||||
if ((sources.get(test) ?? "").includes(path)) select(test, "names");
|
||||
// Tests that name a changed path, or a verifier or shared module the change
|
||||
// reaches, read or start it directly.
|
||||
const reachedPaths = named(affected);
|
||||
for (const test of tests)
|
||||
if (names(test, reachedPaths)) select(test, "names");
|
||||
return { selected, untraced };
|
||||
}
|
||||
|
||||
const full = (reason) => ({ version: 3, mode: "full", reason });
|
||||
|
||||
/**
|
||||
* Plan the portable tests one CI run executes.
|
||||
*
|
||||
* `snapshot` lists the tracked paths of the revision under test and reads
|
||||
* them; it is consulted only when a subset is possible.
|
||||
*/
|
||||
export function planTestSelection({ files, event, tier, code, snapshot }) {
|
||||
// `ci:full` selects the full tier, so it also lands here.
|
||||
if (event !== "pull_request" || tier !== "fast")
|
||||
return full("Main, ci:full and qualification runs execute every test");
|
||||
if (!code) return full("No portable tests run for this change");
|
||||
if (!files.length) return full("No changed paths available");
|
||||
// Decide from the paths alone before reading the snapshot.
|
||||
const untracedPath = files.map(untracedInput).find(Boolean);
|
||||
if (untracedPath !== undefined) return full(untracedPath);
|
||||
const { selected, untraced } = traceAffectedTests({ files, snapshot });
|
||||
if (untraced.length) return full(untraced[0]);
|
||||
const paths = [...selected.keys()].sort();
|
||||
if (!paths.length)
|
||||
return full("No test imports, names or sits beside a changed path");
|
||||
@@ -164,46 +296,42 @@ export function planTestSelection({ files, event, tier, code, snapshot }) {
|
||||
} catch {
|
||||
return full(`No single test project runs ${path}`);
|
||||
}
|
||||
const count = Math.min(MAX_GROUPS, Math.ceil(paths.length / FILES_PER_GROUP));
|
||||
const groups = Array.from({ length: count }, () => []);
|
||||
paths.forEach((path, index) => groups[index % count].push(path));
|
||||
const reasons = Object.groupBy([...selected.values()], (reason) => reason);
|
||||
return {
|
||||
version: 2,
|
||||
version: 3,
|
||||
mode: "selected",
|
||||
reason: `Affected tests: ${Object.entries(reasons)
|
||||
.map(([reason, entries]) => `${entries.length} ${reason}`)
|
||||
.join(", ")}`,
|
||||
groups,
|
||||
files: paths,
|
||||
shards: Math.min(MAX_SHARDS, Math.ceil(paths.length / FILES_PER_SHARD)),
|
||||
};
|
||||
}
|
||||
|
||||
/** Parse a plan produced by `planTestSelection`; anything else fails closed. */
|
||||
export function parseTestSelection(value) {
|
||||
const plan = typeof value === "string" ? JSON.parse(value) : value;
|
||||
const files = plan?.files;
|
||||
const valid =
|
||||
plan?.version === 2 &&
|
||||
plan?.version === 3 &&
|
||||
typeof plan.reason === "string" &&
|
||||
Array.isArray(plan.groups) &&
|
||||
(plan.mode === "full"
|
||||
? plan.groups.length === 0
|
||||
? files === undefined && plan.shards === undefined
|
||||
: plan.mode === "selected" &&
|
||||
plan.groups.length >= 1 &&
|
||||
plan.groups.length <= MAX_GROUPS &&
|
||||
plan.groups.every(
|
||||
(group) =>
|
||||
Array.isArray(group) &&
|
||||
group.length > 0 &&
|
||||
group.every(
|
||||
(path) =>
|
||||
typeof path === "string" &&
|
||||
isTest(path) &&
|
||||
/^(src|tests)\//u.test(path) &&
|
||||
!path.split("/").some((part) => ["", ".", ".."].includes(part)),
|
||||
),
|
||||
Array.isArray(files) &&
|
||||
files.length > 0 &&
|
||||
files.length <= SELECTION_CAP &&
|
||||
new Set(files).size === files.length &&
|
||||
files.every(
|
||||
(path) =>
|
||||
typeof path === "string" &&
|
||||
isTest(path) &&
|
||||
/^(src|tests)\//u.test(path) &&
|
||||
!path.split("/").some((part) => ["", ".", ".."].includes(part)),
|
||||
) &&
|
||||
new Set(plan.groups.flat()).size === plan.groups.flat().length &&
|
||||
plan.groups.flat().length <= SELECTION_CAP);
|
||||
Number.isInteger(plan.shards) &&
|
||||
plan.shards >= 1 &&
|
||||
plan.shards <= Math.min(MAX_SHARDS, files.length));
|
||||
if (!valid) throw new Error("Invalid portable test selection plan");
|
||||
return plan;
|
||||
}
|
||||
@@ -213,12 +341,10 @@ export function parseTestSelection(value) {
|
||||
// slowest shard sets the critical path: at eight shards it ran 24% above the
|
||||
// mean shard. Smaller shards bound that excess at a few percent more runner
|
||||
// time. Full qualification retains six complete shards with coverage. A
|
||||
// selection runs one job per group.
|
||||
// selection runs its own shard count.
|
||||
const shardsOf = (count) =>
|
||||
Array.from({ length: count }, (_, index) => `${index + 1}/${count}`);
|
||||
|
||||
/** The `test-shard` matrix labels for a tier and its test selection. */
|
||||
export const testShards = (tier, plan) =>
|
||||
shardsOf(
|
||||
plan.mode === "selected" ? plan.groups.length : tier === "full" ? 6 : 12,
|
||||
);
|
||||
shardsOf(plan.mode === "selected" ? plan.shards : tier === "full" ? 6 : 12);
|
||||
@@ -458,7 +458,7 @@ it("selects the tests a pull request affects and runs everything when it cannot
|
||||
"tests/conformance/named.test.ts",
|
||||
];
|
||||
expect(await runPlanScript(directory, base, head)).toMatchObject({
|
||||
testSelection: { version: 2, mode: "selected", groups: [selected] },
|
||||
testSelection: { version: 3, mode: "selected", files: selected, shards: 1 },
|
||||
testShards: ["1/1"],
|
||||
});
|
||||
const output = await readFile(join(directory, "scope-output"), "utf8");
|
||||
@@ -469,7 +469,7 @@ it("selects the tests a pull request affects and runs everything when it cannot
|
||||
expect(
|
||||
await runPlanScript(directory, base, head, "pull_request", true),
|
||||
).toMatchObject({
|
||||
testSelection: { mode: "full", groups: [] },
|
||||
testSelection: { version: 3, mode: "full" },
|
||||
testShards: ["1/6", "2/6", "3/6", "4/6", "5/6", "6/6"],
|
||||
});
|
||||
expect(await classify(directory, base, head, "push")).toMatchObject({
|
||||
|
||||
@@ -406,10 +406,9 @@ it("merges source and built reports exactly once using the workflow's shard matr
|
||||
outputs: {
|
||||
test_shards: JSON.stringify(
|
||||
testShards("fast", {
|
||||
version: 2,
|
||||
version: 3,
|
||||
mode: "full",
|
||||
reason: "Complete fast suite",
|
||||
groups: [],
|
||||
}),
|
||||
),
|
||||
},
|
||||
|
||||
@@ -807,6 +807,7 @@ it("runs a selected PR subset in planner-owned groups and still gates it", async
|
||||
expect(selected?.env).toEqual({
|
||||
MATRIX_SHARD: "${{ matrix.shard }}",
|
||||
CI_TEST_SELECTION: "${{ needs.changes.outputs.test_selection }}",
|
||||
REA_CI_TEST_HISTORY: ".cache/ci-test-history/test-results.json",
|
||||
});
|
||||
|
||||
const flags = await scopeFlags();
|
||||
|
||||
@@ -670,10 +670,11 @@ it("verifies exactly the selected subset and never publishes it as timing histor
|
||||
}
|
||||
await workflowGit(root, ["add", "src", "tests"]);
|
||||
const selection = {
|
||||
version: 2,
|
||||
version: 3,
|
||||
mode: "selected",
|
||||
reason: "Affected tests: 2 changed",
|
||||
groups: [["src/a.test.ts"], ["tests/b.test.ts"]],
|
||||
files: ["src/a.test.ts", "tests/b.test.ts"],
|
||||
shards: 2,
|
||||
};
|
||||
const run = async (names: readonly string[], plan: unknown = selection) => {
|
||||
await writeFile(
|
||||
@@ -707,7 +708,7 @@ it("verifies exactly the selected subset and never publishes it as timing histor
|
||||
stderr: expect.stringContaining("Incomplete selected test inventory"),
|
||||
});
|
||||
// A full plan keeps the complete tracked inventory and its history.
|
||||
const full = { version: 2, mode: "full", reason: "Shared", groups: [] };
|
||||
const full = { version: 3, mode: "full", reason: "Shared" };
|
||||
await expect(
|
||||
run(["src/a.test.ts", "tests/b.test.ts"], full),
|
||||
).rejects.toMatchObject({
|
||||
@@ -719,11 +720,12 @@ it("verifies exactly the selected subset and never publishes it as timing histor
|
||||
).resolves.toContain("tests/unselected.test.ts");
|
||||
// An unreadable plan fails closed.
|
||||
for (const plan of [
|
||||
{ ...selection, version: 1 },
|
||||
{ ...selection, groups: [] },
|
||||
{ ...selection, groups: [["../escape.test.ts"]] },
|
||||
{ ...selection, groups: [["src/a.test.ts"], ["src/a.test.ts"]] },
|
||||
{ ...full, groups: [["src/a.test.ts"]] },
|
||||
{ ...selection, version: 2 },
|
||||
{ ...selection, files: [] },
|
||||
{ ...selection, files: ["../escape.test.ts"] },
|
||||
{ ...selection, files: ["src/a.test.ts", "src/a.test.ts"] },
|
||||
{ ...selection, shards: 3 },
|
||||
{ ...full, files: ["src/a.test.ts"] },
|
||||
])
|
||||
await expect(run(["src/a.test.ts"], plan)).rejects.toMatchObject({
|
||||
stderr: expect.stringContaining("Invalid portable test selection plan"),
|
||||
|
||||
@@ -26,7 +26,7 @@ const plan = (files: readonly string[], sources: Record<string, string>) =>
|
||||
});
|
||||
|
||||
const selectedFiles = (result: ReturnType<typeof plan>) =>
|
||||
result.groups.flat().toSorted();
|
||||
(result.mode === "selected" ? result.files : []).toSorted();
|
||||
|
||||
it("follows test and verifier imports and directory scans, not production importers", () => {
|
||||
const sources = {
|
||||
@@ -67,6 +67,37 @@ it("selects a leaf adapter's tests by its name and directory across test project
|
||||
]);
|
||||
});
|
||||
|
||||
it("selects tests that start the built runtime beside a changed presentation owner", () => {
|
||||
const sources = {
|
||||
"src/server/Core.ts": "",
|
||||
"src/server/recovery.ts": "",
|
||||
"src/domain/Value.ts": "",
|
||||
"tests/support/launch.ts": 'spawn("scripts/rea.mjs", ["mcp"]);',
|
||||
"tests/fixtures/client.ts": 'import "../support/launch.js";',
|
||||
// Starts the built server and imports a sibling of the changed owner.
|
||||
"tests/boundary/mcp/direct.test.ts":
|
||||
'import "../../../src/server/recovery.js"; resolve("dist/main.js");',
|
||||
"tests/boundary/mcp/helper.test.ts":
|
||||
'import "../../fixtures/client.js"; import "../../../src/server/recovery.js";',
|
||||
// Starts it but imports nothing from that owner, or imports it in process.
|
||||
"tests/boundary/mcp/unrelated.test.ts":
|
||||
'import "../../fixtures/client.js"; import "../../../src/domain/Value.js";',
|
||||
"tests/boundary/mcp/inProcess.test.ts":
|
||||
'import "../../../src/server/recovery.js";',
|
||||
// A package's own dist/ subpath is not the built runtime.
|
||||
"tests/boundary/mcp/ajv.test.ts":
|
||||
'import { Ajv2020 } from "ajv/dist/2020.js"; import "../../../src/server/recovery.js";',
|
||||
};
|
||||
expect(selectedFiles(plan(["src/server/Core.ts"], sources))).toEqual([
|
||||
"tests/boundary/mcp/direct.test.ts",
|
||||
"tests/boundary/mcp/helper.test.ts",
|
||||
]);
|
||||
// Other owners keep import tracing only.
|
||||
expect(plan(["src/domain/Value.ts"], sources)).toMatchObject({
|
||||
reason: "Affected tests: 1 imports",
|
||||
});
|
||||
});
|
||||
|
||||
it("lets checker-only inputs and prose select nothing by themselves", () => {
|
||||
const sources = {
|
||||
"src/domain/Value.test.ts": "",
|
||||
@@ -93,60 +124,136 @@ it("lets checker-only inputs and prose select nothing by themselves", () => {
|
||||
expect(plan(["scripts/verify/baselines/x.json"], sources)).toMatchObject({
|
||||
mode: "full",
|
||||
});
|
||||
expect(plan(["scripts/lib/shared.mjs"], sources)).toMatchObject({
|
||||
});
|
||||
|
||||
it("traces shared script modules to their tests and falls back for other consumers", () => {
|
||||
const sources = {
|
||||
"scripts/lib/shared.mjs": "",
|
||||
"scripts/lib/wrapper.mjs": 'export * from "./shared.mjs";',
|
||||
"scripts/verify-thing.mjs": 'import "./lib/wrapper.mjs";',
|
||||
"scripts/lib/unused.mjs": "",
|
||||
"tests/boundary/filesystem/shared.test.ts":
|
||||
'import "../../../scripts/lib/shared.mjs";',
|
||||
"tests/boundary/filesystem/verifier.test.ts":
|
||||
'await import("../../../scripts/verify-thing.mjs");',
|
||||
"tests/boundary/filesystem/other.test.ts": "",
|
||||
"package.json":
|
||||
'{"scripts":{"verify:thing":"node scripts/verify-thing.mjs"}}',
|
||||
};
|
||||
// Through another shared module and a verifier to the tests that import them.
|
||||
expect(selectedFiles(plan(["scripts/lib/shared.mjs"], sources))).toEqual([
|
||||
"tests/boundary/filesystem/shared.test.ts",
|
||||
"tests/boundary/filesystem/verifier.test.ts",
|
||||
]);
|
||||
// A build script, configuration or package script that loads it is untraced.
|
||||
expect(
|
||||
plan(["scripts/lib/shared.mjs"], {
|
||||
...sources,
|
||||
"scripts/build.mjs": 'import { x } from "./lib/wrapper.mjs";',
|
||||
}),
|
||||
).toMatchObject({
|
||||
mode: "full",
|
||||
reason: "Input is not traced to tests: scripts/lib/shared.mjs",
|
||||
reason:
|
||||
"Script consumer is not traced: scripts/build.mjs uses scripts/lib/wrapper.mjs",
|
||||
});
|
||||
expect(
|
||||
plan(["scripts/lib/shared.mjs"], {
|
||||
...sources,
|
||||
"package.json": '{"scripts":{"x":"node scripts/lib/shared.mjs"}}',
|
||||
}),
|
||||
).toMatchObject({
|
||||
mode: "full",
|
||||
reason:
|
||||
"Script consumer is not traced: package.json uses scripts/lib/shared.mjs",
|
||||
});
|
||||
// A module no test reaches runs everything, even beside a selected change.
|
||||
for (const files of [
|
||||
["scripts/lib/unused.mjs"],
|
||||
["scripts/lib/unused.mjs", "tests/boundary/filesystem/other.test.ts"],
|
||||
])
|
||||
expect(plan(files, sources)).toMatchObject({
|
||||
mode: "full",
|
||||
reason: "No test reaches shared script module: scripts/lib/unused.mjs",
|
||||
});
|
||||
// Verifier source loads are module edges, and a test that starts a reached
|
||||
// verifier by path is selected without importing it.
|
||||
const loaded = {
|
||||
"scripts/lib/startup.mts": "",
|
||||
"scripts/lib/lifecycle.mjs":
|
||||
'await importVerifierSource(new URL("./startup.mts", import.meta.url).href, import.meta.url);',
|
||||
"scripts/verify-runtime.mjs": 'import "./lib/lifecycle.mjs";',
|
||||
"tests/boundary/filesystem/startup.test.ts":
|
||||
'import "../../../scripts/lib/startup.mts";',
|
||||
"tests/boundary/filesystem/lifecycle.test.ts":
|
||||
'import "../../../scripts/lib/lifecycle.mjs";',
|
||||
"tests/boundary/filesystem/runtimeVerifier.test.ts":
|
||||
'spawn(process.execPath, ["scripts/verify-runtime.mjs"]);',
|
||||
};
|
||||
expect(selectedFiles(plan(["scripts/lib/startup.mts"], loaded))).toEqual([
|
||||
"tests/boundary/filesystem/lifecycle.test.ts",
|
||||
"tests/boundary/filesystem/runtimeVerifier.test.ts",
|
||||
"tests/boundary/filesystem/startup.test.ts",
|
||||
]);
|
||||
// Other scripts remain untraced inputs.
|
||||
expect(plan(["scripts/build.mjs"], sources)).toMatchObject({
|
||||
mode: "full",
|
||||
reason: "Input is not traced to tests: scripts/build.mjs",
|
||||
});
|
||||
});
|
||||
|
||||
it("parses only well-formed plans and derives one shard per selected group", () => {
|
||||
it("parses only well-formed plans and derives the selected shard count", () => {
|
||||
const selected = {
|
||||
version: 2,
|
||||
version: 3,
|
||||
mode: "selected",
|
||||
reason: "Affected tests: 2 changed",
|
||||
groups: [["src/a.test.ts"], ["tests/b.test.ts"]],
|
||||
files: ["src/a.test.ts", "tests/b.test.ts"],
|
||||
shards: 2,
|
||||
};
|
||||
expect(
|
||||
testShards("fast", parseTestSelection(JSON.stringify(selected))),
|
||||
).toEqual(["1/2", "2/2"]);
|
||||
const full = { version: 2, mode: "full", reason: "Shared", groups: [] };
|
||||
const full = { version: 3, mode: "full", reason: "Shared" };
|
||||
expect(testShards("fast", parseTestSelection(full))).toHaveLength(12);
|
||||
expect(testShards("full", parseTestSelection(full))).toHaveLength(6);
|
||||
for (const value of [
|
||||
"",
|
||||
"null",
|
||||
{ ...selected, version: 2 },
|
||||
{ ...selected, mode: "candidate" },
|
||||
{ ...selected, groups: [[]] },
|
||||
{ ...selected, groups: [["src/a.ts"]] },
|
||||
{ ...selected, groups: [["/abs/a.test.ts"]] },
|
||||
{ ...selected, groups: [["src/a.test.ts"], ["src/a.test.ts"]] },
|
||||
{ ...selected, files: [] },
|
||||
{ ...selected, files: ["src/a.ts"] },
|
||||
{ ...selected, files: ["/abs/a.test.ts"] },
|
||||
{ ...selected, files: ["src/a.test.ts", "src/a.test.ts"] },
|
||||
// More shards than files, or than one selection uses.
|
||||
{ ...selected, shards: 3 },
|
||||
{ ...selected, shards: 0 },
|
||||
{
|
||||
...selected,
|
||||
groups: Array.from({ length: 5 }, (_, i) => [`src/${i}.test.ts`]),
|
||||
files: Array.from({ length: 5 }, (_, i) => `src/${i}.test.ts`),
|
||||
shards: 5,
|
||||
},
|
||||
{ ...full, shards: 1 },
|
||||
])
|
||||
expect(() => parseTestSelection(value)).toThrow(
|
||||
/Invalid portable test selection plan|JSON/u,
|
||||
);
|
||||
});
|
||||
|
||||
it("runs only a group the plan defines", async () => {
|
||||
it("runs only a shard the plan defines", async () => {
|
||||
const group = fileURLToPath(
|
||||
new URL("../../scripts/ci/test-group.mjs", import.meta.url),
|
||||
);
|
||||
const selection = JSON.stringify({
|
||||
version: 2,
|
||||
version: 3,
|
||||
mode: "selected",
|
||||
reason: "Affected tests: 1 changed",
|
||||
groups: [["src/a.test.ts"]],
|
||||
files: ["src/a.test.ts"],
|
||||
shards: 1,
|
||||
});
|
||||
for (const [shard, environment] of [
|
||||
["2/2", selection],
|
||||
["1/2", selection],
|
||||
[
|
||||
"1/1",
|
||||
JSON.stringify({ version: 2, mode: "full", reason: "x", groups: [] }),
|
||||
],
|
||||
["1/1", JSON.stringify({ version: 3, mode: "full", reason: "x" })],
|
||||
["1/1", ""],
|
||||
] as const)
|
||||
await expect(
|
||||
|
||||
Reference in new issue
Block a user