From 062b3835f9bead17456d24f0ca103b8e8df17ca3 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Mon, 16 Nov 2026 06:14:14 +0800 Subject: [PATCH] ci: trace script modules and built-server consumers in PR test selection, and balance selected shards (#11516) * ci: trace shared script modules to the tests that import them A pull request that changed any scripts/lib module ran the complete suite. Shared script modules now pass a change through to the tests, verifiers and other shared modules that import them, like test helpers. Another consumer (a build or test-runner script, root configuration, package metadata, a hook or a workflow) that imports the module or names it by file name still selects the complete suite. The tracing core is exported as traceAffectedTests so a bounded local gate can reuse it. Fixes #11381 * ci: follow verifier source loads and named verifiers when tracing script modules - Module URLs (new URL("./x.mts", import.meta.url), as loaded by importVerifierSource) are import edges. - A test that names a verifier or shared module the change reaches is selected, not only one that names a changed path. - A changed shared module that no test reaches runs the complete suite even when other changes select tests. * ci: select tests that start the built server beside a changed presentation owner (#11519) * ci: select tests that start the built server beside a changed presentation owner A test that starts the built CLI or MCP server reaches server, contract, entrypoint and CLI modules through the build, not an import, so a change there selected none of them (#11146 failed on main this way). Such a test is now selected when it, or a helper it imports, starts scripts/rea.mjs or dist/, and it imports a module from the changed owner's directory. Fixes #11366 * ci: recognise only the repository's built entry points as runtime launchers A package subpath such as ajv/dist/2020.js matched the dist/ pattern, so in-process tests importing Ajv counted as built-runtime launchers. * ci: balance selected test shards by recorded duration (#11522) A selected PR run split its files across groups round-robin, so a few slow files could land together. The plan now records the selected files and a shard count (version 3), and each job runs one --shard of the selection through the exact-test runner with the timing history main already publishes, so the CI sequencer balances shards by per-file duration (the median for files without a timing). Without history, Vitest's own deterministic sharding applies. Fixes #11382 --- .github/workflows/ci.yml | 1 + scripts/ci/change-plan.mjs | 5 +- scripts/ci/test-group.mjs | 16 +- scripts/ci/test-report.mjs | 4 +- scripts/ci/test-selection.d.mts | 31 +- scripts/ci/test-selection.mjs | 264 +++++++++++++----- .../boundary/filesystem/ciChangeScope.test.ts | 4 +- tests/boundary/filesystem/ciFeedback.test.ts | 3 +- .../filesystem/ciRequiredLanes.test.ts | 1 + .../boundary/filesystem/ciTestReports.test.ts | 18 +- tests/evaluation/ciTestSelection.test.ts | 145 ++++++++-- 11 files changed, 374 insertions(+), 118 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8e70db3771..ecad1fded4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -367,6 +367,7 @@ jobs: env: MATRIX_SHARD: ${{ matrix.shard }} CI_TEST_SELECTION: ${{ needs.changes.outputs.test_selection }} + REA_CI_TEST_HISTORY: .cache/ci-test-history/test-results.json - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 if: ${{ !cancelled() }} with: diff --git a/scripts/ci/change-plan.mjs b/scripts/ci/change-plan.mjs index 151233dd02..112db3889f 100644 --- a/scripts/ci/change-plan.mjs +++ b/scripts/ci/change-plan.mjs @@ -346,9 +346,10 @@ export function planChanges(request, repository) { event, tier, code: selected.code, - // The checked-out revision is the one whose tests run. + // The checked-out revision is the one whose tests run. Script and + // configuration files are listed so shared script modules can be traced. snapshot: { - list: () => repository.listFiles("HEAD", ["src", "tests"]), + list: () => repository.listFiles("HEAD", []), read: (paths) => repository.readFiles("HEAD", paths), }, }); diff --git a/scripts/ci/test-group.mjs b/scripts/ci/test-group.mjs index 5d8c8f70a3..a50dbcdd6a 100644 --- a/scripts/ci/test-group.mjs +++ b/scripts/ci/test-group.mjs @@ -17,23 +17,27 @@ if ( let runner; let environment = process.env; if (group === "selected") { - // `selected ` runs group k of the plan's selected test files, by exact - // identity, in whichever projects own them. + // `selected ` runs shard k of the plan's selected test files, by exact + // identity, in whichever projects own them. With REA_CI_TEST_HISTORY the CI + // sequencer balances the shards by main's recorded per-file durations. const [shard, ...rest] = arguments_; const plan = parseTestSelection(process.env.CI_TEST_SELECTION ?? ""); if (plan.mode !== "selected" || !testShards("fast", plan).includes(shard)) throw new Error(`No selected test group ${shard ?? ""} in this plan`); if (rest.some((argument) => /^--shard(?:=|$)/u.test(argument))) throw new Error("Selected test groups are already sharded"); - const index = Number(shard.split("/")[0]) - 1; runner = [ resolve("scripts/run-exact-tests.mjs"), "run", + `--shard=${shard}`, ...rest, - ...plan.groups[index], + ...plan.files, ]; - // Each group writes its own blob report for the merge. - environment = { ...process.env, VITEST_BLOB_LABEL: `selected-${index + 1}` }; + // Each shard writes its own blob report for the merge. + environment = { + ...process.env, + VITEST_BLOB_LABEL: `selected-${shard.split("/")[0]}`, + }; } else { const projects = { source: SOURCE_TEST_PROJECTS, diff --git a/scripts/ci/test-report.mjs b/scripts/ci/test-report.mjs index 0ae3cabc61..8824b8800d 100644 --- a/scripts/ci/test-report.mjs +++ b/scripts/ci/test-report.mjs @@ -63,7 +63,7 @@ const selected = selection?.mode === "selected"; const expected = !verifyInventory ? [] : selected - ? selection.groups.flat() + ? selection.files : execFileSync("git", ["ls-files", "-z", "--", "src", "tests"], { encoding: "utf8", maxBuffer: 8 * 1024 * 1024, @@ -95,7 +95,7 @@ const summary = [ "", ...(selection ? [ - `Portable tests: ${selected ? `${selection.groups.flat().length} selected files` : "complete suite"}. ${selection.reason}.`, + `Portable tests: ${selected ? `${selection.files.length} selected files` : "complete suite"}. ${selection.reason}.`, "", ] : []), diff --git a/scripts/ci/test-selection.d.mts b/scripts/ci/test-selection.d.mts index 0ef16bf3ae..686b58daf6 100644 --- a/scripts/ci/test-selection.d.mts +++ b/scripts/ci/test-selection.d.mts @@ -1,25 +1,40 @@ export type TestSelectionPlan = | { - readonly version: 2; + readonly version: 3; readonly mode: "full"; readonly reason: string; - readonly groups: readonly []; } | { - readonly version: 2; + readonly version: 3; readonly mode: "selected"; readonly reason: string; - readonly groups: readonly (readonly string[])[]; + readonly files: readonly string[]; + readonly shards: number; }; +export type TestSelectionSnapshot = { + readonly list: () => readonly string[]; + readonly read: (paths: readonly string[]) => ReadonlyMap; +}; +export type TestSelectionReason = + | "changed" + | "imports" + | "built" + | "beside" + | "owner" + | "names"; +export function traceAffectedTests(request: { + readonly files: readonly string[]; + readonly snapshot: TestSelectionSnapshot; +}): { + readonly selected: ReadonlyMap; + readonly untraced: readonly string[]; +}; export function planTestSelection(request: { readonly files: readonly string[]; readonly event: string; readonly tier: string; readonly code: boolean; - readonly snapshot: { - readonly list: () => readonly string[]; - readonly read: (paths: readonly string[]) => ReadonlyMap; - }; + readonly snapshot: TestSelectionSnapshot; }): TestSelectionPlan; export function parseTestSelection(value: unknown): TestSelectionPlan; export function testShards(tier: string, plan: TestSelectionPlan): string[]; diff --git a/scripts/ci/test-selection.mjs b/scripts/ci/test-selection.mjs index fbd963d85a..9359ab1900 100644 --- a/scripts/ci/test-selection.mjs +++ b/scripts/ci/test-selection.mjs @@ -10,9 +10,11 @@ import { SHARED_BOUNDARY } from "./scopes.mjs"; /** More selected files than this run the complete suite instead. */ const SELECTION_CAP = 150; -/** Selected files per test job, and the most jobs one selection uses. */ -const FILES_PER_GROUP = 40; -const MAX_GROUPS = 4; +// Selected files per test job, and the most jobs one selection uses. Each job +// runs one `--shard` of the selection; the CI sequencer balances the shards by +// main's recorded per-file durations when that history is available. +const FILES_PER_SHARD = 40; +const MAX_SHARDS = 4; // Leaf adapters whose tests also reach their sources through the built CLI, // MCP server or a fake provider process rather than an import. An adapter @@ -39,6 +41,16 @@ const ownedTests = (path) => ); const isTest = (path) => path.endsWith(".test.ts"); +// Owners of what the built CLI and MCP server present: tool contracts, +// server registration and presentation, entrypoints and CLI commands. +const isPresentationOwner = (path) => + !isTest(path) && + /^src\/(?:server|contracts|main|cli)\/|^src\/(?:main|cli)\.ts$/u.test(path); +// A test that starts the built CLI or MCP server instead of importing it: +// the CLI launcher or this repository's built entry points. Other `dist/` +// paths (package subpaths such as `ajv/dist/2020.js`, fixture apps) are not. +const LAUNCHES_BUILT_RUNTIME = + /scripts\/rea\.mjs|\bdist\/(?:main|cli)\.js|["'`]dist["'`]\s*,\s*["'`](?:main|cli)\.js/u; // Inputs that only a non-test check reads: declarations (typecheck), guard // baselines (architecture guards, static job) and the advertised-schema // summaries (build job). They select no tests. @@ -46,24 +58,55 @@ const isCheckerInput = (path) => /\.d\.m?ts$/u.test(path) || /^scripts\/verify\/baselines\/[^/]+\.json$/u.test(path) || /^scripts\/verify\/advertised-schemas\/tools\/[^/]+\.json$/u.test(path); -// Verifier scripts are imported by their tests like source modules. Other -// scripts (build, test runners, CI) shape every run and are not traced. +// Verifier scripts are imported by their tests like source modules. +const isVerifier = (path) => + /^scripts\/(verify-[^/]+|verify\/.+)\.mjs$/u.test(path); +// Shared script modules. A change is traced through the tests, verifiers and +// other shared modules that import it; any other consumer (build, test +// runners, configuration) shapes every run, so it selects the full suite. +const isScriptLibrary = (path) => + !isCheckerInput(path) && /^scripts\/lib\/.+\.m?[jt]s$/u.test(path); +// Other scripts (build, test runners, CI) shape every run and are not traced. const isModule = (path) => !isCheckerInput(path) && (/^(src|tests)\/.+\.m?ts$/u.test(path) || - /^scripts\/(verify-[^/]+|verify\/.+)\.mjs$/u.test(path)); + isVerifier(path) || + isScriptLibrary(path)); // Test-only modules pass a change through to the tests that use them. // Production importers never widen the selection transitively. const isHelper = (path) => !isTest(path) && (/^tests\/.+\.m?ts$/u.test(path) || - /^src\/.+\.fixture[^/]*\.ts$/u.test(path)); + /^src\/.+\.fixture[^/]*\.ts$/u.test(path) || + isVerifier(path) || + isScriptLibrary(path)); +// Files outside src/ and tests/ that can run or load a shared script module: +// other scripts, root configuration, package metadata, hooks and workflows. +const isScriptConsumer = (path) => + !isHelper(path) && + !isCheckerInput(path) && + (/^scripts\/.+\.(?:m?[jt]s|cjs)$/u.test(path) || + (/^[^/]+\.(?:m?[jt]s|cjs|json)$/u.test(path) && + path !== "package-lock.json") || + /^\.github\/.+\.ya?ml$/u.test(path) || + /^\.husky\/[^/]+$/u.test(path)); // Prose that tests may read by path. Other non-module inputs (fixtures, data, // scripts, workflows, package metadata) are not traced, so they run everything. const isProse = (path) => path.endsWith(".md") && !/^(src|tests)\//u.test(path); +/** Why a changed path cannot be traced to tests, if it cannot. */ +const untracedInput = (path) => + SHARED_BOUNDARY.test(path) + ? `Shared boundary: ${path}` + : !isModule(path) && !isProse(path) && !isCheckerInput(path) + ? `Input is not traced to tests: ${path}` + : undefined; +// Static and literal dynamic imports, mocks, and module URLs such as the +// `new URL("./x.mts", import.meta.url)` that `importVerifierSource` loads +// (scripts/lib/import-specifiers.mjs owns the exact parse; the planning job +// runs before dependencies are installed, so it cannot load that parser). const SPECIFIER = - /(?:\bfrom\s*|\bimport\s*\(\s*|\bimport\s+|\bvi\.mock\(\s*)["'](\.{1,2}\/[^"']+)["']/gu; + /(?:\bfrom\s*|\bimport\s*\(\s*|\bimport\s+|\bvi\.mock\(\s*|\bnew\s+URL\(\s*)["'](\.{1,2}\/[^"']+)["']/gu; // TypeScript imports name the emitted `.js`/`.mjs`; scripts are imported as is. const importsOf = (path, source) => [...source.matchAll(SPECIFIER)].flatMap(([, specifier]) => { @@ -71,25 +114,17 @@ const importsOf = (path, source) => return [target, target.replace(/\.(m?)js$/u, ".$1ts")]; }); -const full = (reason) => ({ version: 2, mode: "full", reason, groups: [] }); - /** - * Plan the portable tests one CI run executes. + * Trace the tests a change affects. * - * `snapshot` lists the tracked `src/` and `tests/` paths of the revision under - * test and reads them; it is consulted only when a subset is possible. + * `snapshot` lists the tracked paths of the revision under test and reads + * them. `selected` maps each affected test to the first reason that selected + * it: `changed`, `imports`, `built`, `beside`, `owner` or `names`. `untraced` names + * inputs whose consumers cannot be traced to tests; CI runs the complete suite + * for them, while a bounded local gate runs the traced selection. */ -export function planTestSelection({ files, event, tier, code, snapshot }) { - // `ci:full` selects the full tier, so it also lands here. - if (event !== "pull_request" || tier !== "fast") - return full("Main, ci:full and qualification runs execute every test"); - if (!code) return full("No portable tests run for this change"); - if (!files.length) return full("No changed paths available"); - for (const path of files) { - if (SHARED_BOUNDARY.test(path)) return full(`Shared boundary: ${path}`); - if (!isModule(path) && !isProse(path) && !isCheckerInput(path)) - return full(`Input is not traced to tests: ${path}`); - } +export function traceAffectedTests({ files, snapshot }) { + const untraced = files.flatMap((path) => untracedInput(path) ?? []); const tracked = snapshot.list(); const trackedSet = new Set(tracked); @@ -112,27 +147,102 @@ export function planTestSelection({ files, event, tier, code, snapshot }) { ); for (const path of files) if (isTest(path)) select(path, "changed"); - // Close over test-only helpers, then take the tests that import a change. - const affected = new Set(changedModules); - for (let grew = true; grew;) { - grew = false; - for (const helper of helpers) - if ( - !affected.has(helper) && - imports.get(helper).some((target) => affected.has(target)) - ) { - affected.add(helper); - grew = true; - } - } + // A seed and every helper that imports it, directly or through another. + const throughHelpers = (seed) => { + const reached = new Set(seed); + for (let grew = true; grew;) { + grew = false; + for (const helper of helpers) + if ( + !reached.has(helper) && + imports.get(helper).some((target) => reached.has(target)) + ) { + reached.add(helper); + grew = true; + } + } + return reached; + }; + // Close over test-only helpers, verifiers and shared script modules, then + // take the tests that import a change. + const affected = throughHelpers(changedModules); for (const test of tests) if (imports.get(test).some((target) => affected.has(target))) select(test, "imports"); + // Script paths a test can start or read by name: changed files, plus the + // verifiers and shared modules a change reaches. + const named = (reached) => [ + ...new Set([ + ...files, + ...[...reached].filter((path) => path.startsWith("scripts/")), + ]), + ]; + const names = (test, paths) => + paths.some((path) => (sources.get(test) ?? "").includes(path)); + + // A changed shared module that no test reaches has no coverage in a + // selection, even when other changes select tests. + for (const library of changedModules.filter(isScriptLibrary)) { + const reached = throughHelpers([library]); + const paths = named(reached); + if ( + !tests.some( + (test) => + imports.get(test).some((target) => reached.has(target)) || + names(test, paths), + ) + ) + untraced.push(`No test reaches shared script module: ${library}`); + } + + // A shared script module that another script, configuration file, hook or + // workflow imports or names by file name can shape every run. + const libraries = [...affected].filter(isScriptLibrary); + if (libraries.length) { + const consumers = tracked.filter(isScriptConsumer); + const consumerSources = snapshot.read(consumers); + for (const consumer of consumers) { + const source = consumerSources.get(consumer) ?? ""; + const targets = new Set(importsOf(consumer, source)); + const library = libraries.find( + (path) => targets.has(path) || source.includes(posix.basename(path)), + ); + if (library !== undefined) + untraced.push( + `Script consumer is not traced: ${consumer} uses ${library}`, + ); + } + } + + // Tests that start the built CLI or MCP server reach a presentation owner + // through the build rather than an import. Bound them to the tests that + // also import a module from a changed owner's directory. + const changedOwners = new Set( + changedModules + .filter(isPresentationOwner) + .map((path) => posix.dirname(path)), + ); + if (changedOwners.size) { + const launchers = throughHelpers( + helpers.filter((path) => + LAUNCHES_BUILT_RUNTIME.test(sources.get(path) ?? ""), + ), + ); + for (const test of tests) { + const targets = imports.get(test); + if ( + (LAUNCHES_BUILT_RUNTIME.test(sources.get(test) ?? "") || + targets.some((target) => launchers.has(target))) && + targets.some((target) => changedOwners.has(posix.dirname(target))) + ) + select(test, "built"); + } + } for (const path of changedModules) { // Same-stem tests, and same-directory tests that scan their directory's // sources by path instead of importing them. - const stem = path.replace(/\.m?ts$/u, ""); + const stem = path.replace(/\.m?[jt]s$/u, ""); for (const test of tests) if ( posix.dirname(test) === posix.dirname(path) && @@ -146,11 +256,33 @@ export function planTestSelection({ files, event, tier, code, snapshot }) { if (patterns.some((glob) => matchesGlob(test, glob))) select(test, "owner"); } - // Tests that name a changed path read it directly. - for (const path of files) - for (const test of tests) - if ((sources.get(test) ?? "").includes(path)) select(test, "names"); + // Tests that name a changed path, or a verifier or shared module the change + // reaches, read or start it directly. + const reachedPaths = named(affected); + for (const test of tests) + if (names(test, reachedPaths)) select(test, "names"); + return { selected, untraced }; +} +const full = (reason) => ({ version: 3, mode: "full", reason }); + +/** + * Plan the portable tests one CI run executes. + * + * `snapshot` lists the tracked paths of the revision under test and reads + * them; it is consulted only when a subset is possible. + */ +export function planTestSelection({ files, event, tier, code, snapshot }) { + // `ci:full` selects the full tier, so it also lands here. + if (event !== "pull_request" || tier !== "fast") + return full("Main, ci:full and qualification runs execute every test"); + if (!code) return full("No portable tests run for this change"); + if (!files.length) return full("No changed paths available"); + // Decide from the paths alone before reading the snapshot. + const untracedPath = files.map(untracedInput).find(Boolean); + if (untracedPath !== undefined) return full(untracedPath); + const { selected, untraced } = traceAffectedTests({ files, snapshot }); + if (untraced.length) return full(untraced[0]); const paths = [...selected.keys()].sort(); if (!paths.length) return full("No test imports, names or sits beside a changed path"); @@ -164,46 +296,42 @@ export function planTestSelection({ files, event, tier, code, snapshot }) { } catch { return full(`No single test project runs ${path}`); } - const count = Math.min(MAX_GROUPS, Math.ceil(paths.length / FILES_PER_GROUP)); - const groups = Array.from({ length: count }, () => []); - paths.forEach((path, index) => groups[index % count].push(path)); const reasons = Object.groupBy([...selected.values()], (reason) => reason); return { - version: 2, + version: 3, mode: "selected", reason: `Affected tests: ${Object.entries(reasons) .map(([reason, entries]) => `${entries.length} ${reason}`) .join(", ")}`, - groups, + files: paths, + shards: Math.min(MAX_SHARDS, Math.ceil(paths.length / FILES_PER_SHARD)), }; } /** Parse a plan produced by `planTestSelection`; anything else fails closed. */ export function parseTestSelection(value) { const plan = typeof value === "string" ? JSON.parse(value) : value; + const files = plan?.files; const valid = - plan?.version === 2 && + plan?.version === 3 && typeof plan.reason === "string" && - Array.isArray(plan.groups) && (plan.mode === "full" - ? plan.groups.length === 0 + ? files === undefined && plan.shards === undefined : plan.mode === "selected" && - plan.groups.length >= 1 && - plan.groups.length <= MAX_GROUPS && - plan.groups.every( - (group) => - Array.isArray(group) && - group.length > 0 && - group.every( - (path) => - typeof path === "string" && - isTest(path) && - /^(src|tests)\//u.test(path) && - !path.split("/").some((part) => ["", ".", ".."].includes(part)), - ), + Array.isArray(files) && + files.length > 0 && + files.length <= SELECTION_CAP && + new Set(files).size === files.length && + files.every( + (path) => + typeof path === "string" && + isTest(path) && + /^(src|tests)\//u.test(path) && + !path.split("/").some((part) => ["", ".", ".."].includes(part)), ) && - new Set(plan.groups.flat()).size === plan.groups.flat().length && - plan.groups.flat().length <= SELECTION_CAP); + Number.isInteger(plan.shards) && + plan.shards >= 1 && + plan.shards <= Math.min(MAX_SHARDS, files.length)); if (!valid) throw new Error("Invalid portable test selection plan"); return plan; } @@ -213,12 +341,10 @@ export function parseTestSelection(value) { // slowest shard sets the critical path: at eight shards it ran 24% above the // mean shard. Smaller shards bound that excess at a few percent more runner // time. Full qualification retains six complete shards with coverage. A -// selection runs one job per group. +// selection runs its own shard count. const shardsOf = (count) => Array.from({ length: count }, (_, index) => `${index + 1}/${count}`); /** The `test-shard` matrix labels for a tier and its test selection. */ export const testShards = (tier, plan) => - shardsOf( - plan.mode === "selected" ? plan.groups.length : tier === "full" ? 6 : 12, - ); + shardsOf(plan.mode === "selected" ? plan.shards : tier === "full" ? 6 : 12); diff --git a/tests/boundary/filesystem/ciChangeScope.test.ts b/tests/boundary/filesystem/ciChangeScope.test.ts index 198b6f528f..d86edae4f7 100644 --- a/tests/boundary/filesystem/ciChangeScope.test.ts +++ b/tests/boundary/filesystem/ciChangeScope.test.ts @@ -458,7 +458,7 @@ it("selects the tests a pull request affects and runs everything when it cannot "tests/conformance/named.test.ts", ]; expect(await runPlanScript(directory, base, head)).toMatchObject({ - testSelection: { version: 2, mode: "selected", groups: [selected] }, + testSelection: { version: 3, mode: "selected", files: selected, shards: 1 }, testShards: ["1/1"], }); const output = await readFile(join(directory, "scope-output"), "utf8"); @@ -469,7 +469,7 @@ it("selects the tests a pull request affects and runs everything when it cannot expect( await runPlanScript(directory, base, head, "pull_request", true), ).toMatchObject({ - testSelection: { mode: "full", groups: [] }, + testSelection: { version: 3, mode: "full" }, testShards: ["1/6", "2/6", "3/6", "4/6", "5/6", "6/6"], }); expect(await classify(directory, base, head, "push")).toMatchObject({ diff --git a/tests/boundary/filesystem/ciFeedback.test.ts b/tests/boundary/filesystem/ciFeedback.test.ts index bdd507d5b7..5c63bb3efc 100644 --- a/tests/boundary/filesystem/ciFeedback.test.ts +++ b/tests/boundary/filesystem/ciFeedback.test.ts @@ -406,10 +406,9 @@ it("merges source and built reports exactly once using the workflow's shard matr outputs: { test_shards: JSON.stringify( testShards("fast", { - version: 2, + version: 3, mode: "full", reason: "Complete fast suite", - groups: [], }), ), }, diff --git a/tests/boundary/filesystem/ciRequiredLanes.test.ts b/tests/boundary/filesystem/ciRequiredLanes.test.ts index 69314d4467..619d62bc0f 100644 --- a/tests/boundary/filesystem/ciRequiredLanes.test.ts +++ b/tests/boundary/filesystem/ciRequiredLanes.test.ts @@ -807,6 +807,7 @@ it("runs a selected PR subset in planner-owned groups and still gates it", async expect(selected?.env).toEqual({ MATRIX_SHARD: "${{ matrix.shard }}", CI_TEST_SELECTION: "${{ needs.changes.outputs.test_selection }}", + REA_CI_TEST_HISTORY: ".cache/ci-test-history/test-results.json", }); const flags = await scopeFlags(); diff --git a/tests/boundary/filesystem/ciTestReports.test.ts b/tests/boundary/filesystem/ciTestReports.test.ts index b1d1f5d920..3d6bdc4337 100644 --- a/tests/boundary/filesystem/ciTestReports.test.ts +++ b/tests/boundary/filesystem/ciTestReports.test.ts @@ -670,10 +670,11 @@ it("verifies exactly the selected subset and never publishes it as timing histor } await workflowGit(root, ["add", "src", "tests"]); const selection = { - version: 2, + version: 3, mode: "selected", reason: "Affected tests: 2 changed", - groups: [["src/a.test.ts"], ["tests/b.test.ts"]], + files: ["src/a.test.ts", "tests/b.test.ts"], + shards: 2, }; const run = async (names: readonly string[], plan: unknown = selection) => { await writeFile( @@ -707,7 +708,7 @@ it("verifies exactly the selected subset and never publishes it as timing histor stderr: expect.stringContaining("Incomplete selected test inventory"), }); // A full plan keeps the complete tracked inventory and its history. - const full = { version: 2, mode: "full", reason: "Shared", groups: [] }; + const full = { version: 3, mode: "full", reason: "Shared" }; await expect( run(["src/a.test.ts", "tests/b.test.ts"], full), ).rejects.toMatchObject({ @@ -719,11 +720,12 @@ it("verifies exactly the selected subset and never publishes it as timing histor ).resolves.toContain("tests/unselected.test.ts"); // An unreadable plan fails closed. for (const plan of [ - { ...selection, version: 1 }, - { ...selection, groups: [] }, - { ...selection, groups: [["../escape.test.ts"]] }, - { ...selection, groups: [["src/a.test.ts"], ["src/a.test.ts"]] }, - { ...full, groups: [["src/a.test.ts"]] }, + { ...selection, version: 2 }, + { ...selection, files: [] }, + { ...selection, files: ["../escape.test.ts"] }, + { ...selection, files: ["src/a.test.ts", "src/a.test.ts"] }, + { ...selection, shards: 3 }, + { ...full, files: ["src/a.test.ts"] }, ]) await expect(run(["src/a.test.ts"], plan)).rejects.toMatchObject({ stderr: expect.stringContaining("Invalid portable test selection plan"), diff --git a/tests/evaluation/ciTestSelection.test.ts b/tests/evaluation/ciTestSelection.test.ts index bfe8baa287..eec26a8922 100644 --- a/tests/evaluation/ciTestSelection.test.ts +++ b/tests/evaluation/ciTestSelection.test.ts @@ -26,7 +26,7 @@ const plan = (files: readonly string[], sources: Record) => }); const selectedFiles = (result: ReturnType) => - result.groups.flat().toSorted(); + (result.mode === "selected" ? result.files : []).toSorted(); it("follows test and verifier imports and directory scans, not production importers", () => { const sources = { @@ -67,6 +67,37 @@ it("selects a leaf adapter's tests by its name and directory across test project ]); }); +it("selects tests that start the built runtime beside a changed presentation owner", () => { + const sources = { + "src/server/Core.ts": "", + "src/server/recovery.ts": "", + "src/domain/Value.ts": "", + "tests/support/launch.ts": 'spawn("scripts/rea.mjs", ["mcp"]);', + "tests/fixtures/client.ts": 'import "../support/launch.js";', + // Starts the built server and imports a sibling of the changed owner. + "tests/boundary/mcp/direct.test.ts": + 'import "../../../src/server/recovery.js"; resolve("dist/main.js");', + "tests/boundary/mcp/helper.test.ts": + 'import "../../fixtures/client.js"; import "../../../src/server/recovery.js";', + // Starts it but imports nothing from that owner, or imports it in process. + "tests/boundary/mcp/unrelated.test.ts": + 'import "../../fixtures/client.js"; import "../../../src/domain/Value.js";', + "tests/boundary/mcp/inProcess.test.ts": + 'import "../../../src/server/recovery.js";', + // A package's own dist/ subpath is not the built runtime. + "tests/boundary/mcp/ajv.test.ts": + 'import { Ajv2020 } from "ajv/dist/2020.js"; import "../../../src/server/recovery.js";', + }; + expect(selectedFiles(plan(["src/server/Core.ts"], sources))).toEqual([ + "tests/boundary/mcp/direct.test.ts", + "tests/boundary/mcp/helper.test.ts", + ]); + // Other owners keep import tracing only. + expect(plan(["src/domain/Value.ts"], sources)).toMatchObject({ + reason: "Affected tests: 1 imports", + }); +}); + it("lets checker-only inputs and prose select nothing by themselves", () => { const sources = { "src/domain/Value.test.ts": "", @@ -93,60 +124,136 @@ it("lets checker-only inputs and prose select nothing by themselves", () => { expect(plan(["scripts/verify/baselines/x.json"], sources)).toMatchObject({ mode: "full", }); - expect(plan(["scripts/lib/shared.mjs"], sources)).toMatchObject({ +}); + +it("traces shared script modules to their tests and falls back for other consumers", () => { + const sources = { + "scripts/lib/shared.mjs": "", + "scripts/lib/wrapper.mjs": 'export * from "./shared.mjs";', + "scripts/verify-thing.mjs": 'import "./lib/wrapper.mjs";', + "scripts/lib/unused.mjs": "", + "tests/boundary/filesystem/shared.test.ts": + 'import "../../../scripts/lib/shared.mjs";', + "tests/boundary/filesystem/verifier.test.ts": + 'await import("../../../scripts/verify-thing.mjs");', + "tests/boundary/filesystem/other.test.ts": "", + "package.json": + '{"scripts":{"verify:thing":"node scripts/verify-thing.mjs"}}', + }; + // Through another shared module and a verifier to the tests that import them. + expect(selectedFiles(plan(["scripts/lib/shared.mjs"], sources))).toEqual([ + "tests/boundary/filesystem/shared.test.ts", + "tests/boundary/filesystem/verifier.test.ts", + ]); + // A build script, configuration or package script that loads it is untraced. + expect( + plan(["scripts/lib/shared.mjs"], { + ...sources, + "scripts/build.mjs": 'import { x } from "./lib/wrapper.mjs";', + }), + ).toMatchObject({ mode: "full", - reason: "Input is not traced to tests: scripts/lib/shared.mjs", + reason: + "Script consumer is not traced: scripts/build.mjs uses scripts/lib/wrapper.mjs", + }); + expect( + plan(["scripts/lib/shared.mjs"], { + ...sources, + "package.json": '{"scripts":{"x":"node scripts/lib/shared.mjs"}}', + }), + ).toMatchObject({ + mode: "full", + reason: + "Script consumer is not traced: package.json uses scripts/lib/shared.mjs", + }); + // A module no test reaches runs everything, even beside a selected change. + for (const files of [ + ["scripts/lib/unused.mjs"], + ["scripts/lib/unused.mjs", "tests/boundary/filesystem/other.test.ts"], + ]) + expect(plan(files, sources)).toMatchObject({ + mode: "full", + reason: "No test reaches shared script module: scripts/lib/unused.mjs", + }); + // Verifier source loads are module edges, and a test that starts a reached + // verifier by path is selected without importing it. + const loaded = { + "scripts/lib/startup.mts": "", + "scripts/lib/lifecycle.mjs": + 'await importVerifierSource(new URL("./startup.mts", import.meta.url).href, import.meta.url);', + "scripts/verify-runtime.mjs": 'import "./lib/lifecycle.mjs";', + "tests/boundary/filesystem/startup.test.ts": + 'import "../../../scripts/lib/startup.mts";', + "tests/boundary/filesystem/lifecycle.test.ts": + 'import "../../../scripts/lib/lifecycle.mjs";', + "tests/boundary/filesystem/runtimeVerifier.test.ts": + 'spawn(process.execPath, ["scripts/verify-runtime.mjs"]);', + }; + expect(selectedFiles(plan(["scripts/lib/startup.mts"], loaded))).toEqual([ + "tests/boundary/filesystem/lifecycle.test.ts", + "tests/boundary/filesystem/runtimeVerifier.test.ts", + "tests/boundary/filesystem/startup.test.ts", + ]); + // Other scripts remain untraced inputs. + expect(plan(["scripts/build.mjs"], sources)).toMatchObject({ + mode: "full", + reason: "Input is not traced to tests: scripts/build.mjs", }); }); -it("parses only well-formed plans and derives one shard per selected group", () => { +it("parses only well-formed plans and derives the selected shard count", () => { const selected = { - version: 2, + version: 3, mode: "selected", reason: "Affected tests: 2 changed", - groups: [["src/a.test.ts"], ["tests/b.test.ts"]], + files: ["src/a.test.ts", "tests/b.test.ts"], + shards: 2, }; expect( testShards("fast", parseTestSelection(JSON.stringify(selected))), ).toEqual(["1/2", "2/2"]); - const full = { version: 2, mode: "full", reason: "Shared", groups: [] }; + const full = { version: 3, mode: "full", reason: "Shared" }; expect(testShards("fast", parseTestSelection(full))).toHaveLength(12); expect(testShards("full", parseTestSelection(full))).toHaveLength(6); for (const value of [ "", "null", + { ...selected, version: 2 }, { ...selected, mode: "candidate" }, - { ...selected, groups: [[]] }, - { ...selected, groups: [["src/a.ts"]] }, - { ...selected, groups: [["/abs/a.test.ts"]] }, - { ...selected, groups: [["src/a.test.ts"], ["src/a.test.ts"]] }, + { ...selected, files: [] }, + { ...selected, files: ["src/a.ts"] }, + { ...selected, files: ["/abs/a.test.ts"] }, + { ...selected, files: ["src/a.test.ts", "src/a.test.ts"] }, + // More shards than files, or than one selection uses. + { ...selected, shards: 3 }, + { ...selected, shards: 0 }, { ...selected, - groups: Array.from({ length: 5 }, (_, i) => [`src/${i}.test.ts`]), + files: Array.from({ length: 5 }, (_, i) => `src/${i}.test.ts`), + shards: 5, }, + { ...full, shards: 1 }, ]) expect(() => parseTestSelection(value)).toThrow( /Invalid portable test selection plan|JSON/u, ); }); -it("runs only a group the plan defines", async () => { +it("runs only a shard the plan defines", async () => { const group = fileURLToPath( new URL("../../scripts/ci/test-group.mjs", import.meta.url), ); const selection = JSON.stringify({ - version: 2, + version: 3, mode: "selected", reason: "Affected tests: 1 changed", - groups: [["src/a.test.ts"]], + files: ["src/a.test.ts"], + shards: 1, }); for (const [shard, environment] of [ ["2/2", selection], ["1/2", selection], - [ - "1/1", - JSON.stringify({ version: 2, mode: "full", reason: "x", groups: [] }), - ], + ["1/1", JSON.stringify({ version: 3, mode: "full", reason: "x" })], ["1/1", ""], ] as const) await expect(