ci: run only the affected portable tests on pull requests (#11346)

* ci: run only the affected portable tests on pull requests

Pull-request CI now runs static checks, the build and the portable tests a change affects, selected from test imports (through test-only helpers), same-stem and directory-scanning tests, leaf-owner boundary tests and tests that name a changed path. Shared boundaries, untraced inputs, empty selections and selections above 150 files fall back to the complete fast suite. Main pushes, schedules, releases and ci:full keep the complete suite. The portable report verifies the exact selected inventory and the CI required gate accepts a selected plan only for a fast pull request with code changes.

* ci: own leaf-adapter tests by name and keep subset summaries diagnostic
This commit is contained in:
morluto authored and GitHub committed 2026-11-16 03:37:15 +08:00
1 parent 97112f14a1
commit ff6d1d8142
18 files changed
+901 -577

No files matched your search

+9
View File
@@ -15,6 +15,14 @@ export interface ChangePlanRepository {
readonly isAncestor: (ancestor: string, head: string) => boolean;
readonly changedFiles: (base: string, head: string) => readonly string[];
readonly readFile: (revision: string, path: string) => string;
readonly listFiles: (
revision: string,
directories: readonly string[],
) => readonly string[];
readonly readFiles: (
revision: string,
paths: readonly string[],
) => ReadonlyMap<string, string>;
}
export type ChangePlan = Readonly<Record<string, unknown>> & {
@@ -29,6 +37,7 @@ export type ChangePlan = Readonly<Record<string, unknown>> & {
readonly files: readonly string[];
readonly reasons: readonly string[];
readonly testSelection: TestSelectionPlan;
readonly testShards: readonly string[];
};
export function planChanges(
+50 -13
View File
@@ -5,9 +5,10 @@ import {
CI_FLAGS as flags,
MANUAL_VERIFIERS,
CI_LANES,
SHARED_BOUNDARY,
WINDOWS_EXECUTABLE_SELECTION_INPUTS,
} from "./scopes.mjs";
import { planTestSelection } from "./test-selection.mjs";
import { planTestSelection, testShards } from "./test-selection.mjs";
// Actionlint validates workflow syntax only. The semantic suite asserts these
// workflows' release, publication and self-hosted runner safeguards.
@@ -58,11 +59,6 @@ const matchesAny = (file, patterns) =>
export function planChanges(request, repository) {
const selected = Object.fromEntries(flags.map((flag) => [flag, false]));
const reasons = [];
// Shared process, contract, CLI, and CI-script edits are not a specific provider.
// Fast qualification keeps the portable suite and the Linux x64 runtime
// baseline. Full qualification still selects every lane.
const SHARED_BOUNDARY =
/^(src\/process\/|src\/filesystem\/|src\/contracts\/tool|src\/server\/createServer|src\/cli\.ts|scripts\/ci\/)/;
const full = (reason) => {
if (tier === "full") {
for (const flag of flags) selected[flag] = true;
@@ -269,11 +265,7 @@ export function planChanges(request, repository) {
selected.readiness = true;
}
if (file.startsWith("tests/")) return;
if (
/^(src\/process\/|src\/filesystem\/|src\/contracts\/tool|src\/server\/createServer|src\/cli\.ts|scripts\/ci\/)/.test(
file,
)
) {
if (SHARED_BOUNDARY.test(file)) {
full(`Shared boundary: ${file}`);
} else if (owners.length) {
if (file.startsWith("src/") || file.startsWith("bridge/"))
@@ -343,8 +335,26 @@ export function planChanges(request, repository) {
]
: [linux],
};
const testSelection = planTestSelection(files, event, tier);
return { ...selected, tier, matrix, files, reasons, testSelection };
const testSelection = planTestSelection({
files,
event,
tier,
code: selected.code,
// The checked-out revision is the one whose tests run.
snapshot: {
list: () => repository.listFiles("HEAD", ["src", "tests"]),
read: (paths) => repository.readFiles("HEAD", paths),
},
});
return {
...selected,
tier,
matrix,
files,
reasons,
testSelection,
testShards: testShards(tier, testSelection),
};
}
/** Answer the plan's Git questions from the repository in `cwd`. */
@@ -370,5 +380,32 @@ export const gitRepository = (cwd = process.cwd()) => {
.split("\0")
.filter(Boolean),
readFile: (revision, path) => git("show", `${revision}:${path}`),
listFiles: (revision, directories) =>
git("ls-tree", "-r", "-z", "--name-only", revision, "--", ...directories)
.split("\0")
.filter(Boolean),
// One batch process reads every requested blob.
readFiles: (revision, paths) => {
const output = execFileSync("git", ["cat-file", "--batch"], {
cwd,
input: paths.map((path) => `${revision}:${path}\n`).join(""),
maxBuffer: 512 * 1024 * 1024,
});
const contents = new Map();
let offset = 0;
for (const path of paths) {
const end = output.indexOf(10, offset);
const [, type, size] = output.toString("utf8", offset, end).split(" ");
if (type !== "blob")
throw new Error(`Git could not read ${revision}:${path}`);
const start = end + 1;
contents.set(
path,
output.toString("utf8", start, start + Number(size)),
);
offset = start + Number(size) + 1;
}
return contents;
},
};
};
+4 -2
View File
@@ -14,7 +14,7 @@ const plan = planChanges(
},
gitRepository(),
);
const { tier, matrix, reasons, testSelection } = plan;
const { tier, matrix, reasons, testSelection, testShards } = plan;
const selected = Object.fromEntries(flags.map((flag) => [flag, plan[flag]]));
if (process.env.GITHUB_OUTPUT)
appendFileSync(
@@ -24,6 +24,8 @@ if (process.env.GITHUB_OUTPUT)
tier,
matrix: JSON.stringify(matrix),
test_selection: JSON.stringify(testSelection),
test_mode: testSelection.mode,
test_shards: JSON.stringify(testShards),
})
.map(([key, value]) => `${key}=${value}\n`)
.join(""),
@@ -31,6 +33,6 @@ if (process.env.GITHUB_OUTPUT)
if (process.env.GITHUB_STEP_SUMMARY)
appendFileSync(
process.env.GITHUB_STEP_SUMMARY,
`### Selected CI lanes (${tier})\n\n${flags.filter((flag) => selected[flag]).join(", ") || "Formatting only"}\n\n${reasons.join("\n\n")}\n`,
`### Selected CI lanes (${tier})\n\n${flags.filter((flag) => selected[flag]).join(", ") || "Formatting only"}\n\n${reasons.join("\n\n")}\n\nPortable tests (${testSelection.mode}): ${testSelection.reason}\n`,
);
console.log(JSON.stringify(plan));
+16 -1
View File
@@ -28,10 +28,25 @@ if (
);
if (outputs.tier === "full" && selectedScopes.length !== CI_FLAGS.length)
throw new Error("Full qualification requires every canonical CI scope");
// Only a fast pull-request plan may run a selected test subset.
const testMode = outputs.test_mode;
if (
!["full", "selected"].includes(testMode) ||
(testMode === "selected" &&
(outputs.tier !== "fast" ||
process.env.GITHUB_EVENT_NAME !== "pull_request" ||
outputs.code !== "true"))
)
throw new Error(
"CI classification requires a valid test mode: full, or selected on a fast pull request",
);
const required = reused ? ["changes"] : ["changes", "static", "test"];
if (outputs.tier === "full") required.push("static-deep");
for (const scope of selectedScopes)
required.push(...CI_LANES[scope].required[outputs.tier]);
required.push(
...((testMode === "selected" && CI_LANES[scope].selectedTests) ||
CI_LANES[scope].required[outputs.tier]),
);
const failed = [...new Set(required)].filter(
(name) => jobs[name]?.result !== "success",
);
+9
View File
@@ -8,6 +8,13 @@ export const WINDOWS_EXECUTABLE_SELECTION_INPUTS = [
"tests/boundary/process/windowsExecutableSelection.test.ts",
];
// Shared process, contract, CLI, and CI-script edits are not a specific provider.
// Fast qualification keeps the portable suite and the Linux x64 runtime
// baseline, and pull requests run the complete portable suite rather than a
// selected subset. Full qualification still selects every lane.
export const SHARED_BOUNDARY =
/^(src\/process\/|src\/filesystem\/|src\/contracts\/tool|src\/server\/createServer|src\/cli\.ts|scripts\/ci\/)/;
// Inputs whose changes must select both real capture lanes, even at shared boundaries.
const CAPTURE_SHARED_INPUTS = [
"src/process/LinuxResourceLimits*",
@@ -679,6 +686,8 @@ export const CI_LANES = {
fast: ["build", "test-shard", "test-source"],
full: ["build", "test-shard"],
},
// A selected PR subset runs its source tests in the test-shard groups.
selectedTests: ["build", "test-shard"],
},
runtime: sameJobs([], ["windows-native-build", "package-e2e"]),
package: sameJobs(PATH_SCOPES.package, ["windows-curated"], {
+38 -10
View File
@@ -5,17 +5,48 @@ import {
BUILT_TEST_PROJECTS,
SOURCE_TEST_PROJECTS,
} from "../lib/test-projects.mjs";
import { parseTestSelection, testShards } from "./test-selection.mjs";
const [group, ...arguments_] = process.argv.slice(2);
const projects = { source: SOURCE_TEST_PROJECTS, built: BUILT_TEST_PROJECTS }[
group
];
if (!projects) throw new Error("Expected source or built test group");
// The group owns project selection; callers can request reporters or shards.
// The group owns project and file selection; callers can request reporters
// or (for project groups) shards.
if (
arguments_.some((argument) => /^--(?:project|exclude)(?:=|$)/u.test(argument))
)
throw new Error("Test groups cannot override their project inventory");
let runner;
let environment = process.env;
if (group === "selected") {
// `selected <k/n>` runs group k of the plan's selected test files, by exact
// identity, in whichever projects own them.
const [shard, ...rest] = arguments_;
const plan = parseTestSelection(process.env.CI_TEST_SELECTION ?? "");
if (plan.mode !== "selected" || !testShards("fast", plan).includes(shard))
throw new Error(`No selected test group ${shard ?? ""} in this plan`);
if (rest.some((argument) => /^--shard(?:=|$)/u.test(argument)))
throw new Error("Selected test groups are already sharded");
const index = Number(shard.split("/")[0]) - 1;
runner = [
resolve("scripts/run-exact-tests.mjs"),
"run",
...rest,
...plan.groups[index],
];
// Each group writes its own blob report for the merge.
environment = { ...process.env, VITEST_BLOB_LABEL: `selected-${index + 1}` };
} else {
const projects = {
source: SOURCE_TEST_PROJECTS,
built: BUILT_TEST_PROJECTS,
}[group];
if (!projects) throw new Error("Expected source, built or selected tests");
runner = [
resolve("node_modules/vitest/vitest.mjs"),
"run",
...projects.flatMap(({ name }) => ["--project", name]),
...arguments_,
];
}
const child = spawn(
process.execPath,
[
@@ -24,12 +55,9 @@ const child = spawn(
...(process.platform === "win32"
? []
: [resolve("scripts/run-exclusive.mjs"), "--unlocked", process.execPath]),
resolve("node_modules/vitest/vitest.mjs"),
"run",
...projects.flatMap(({ name }) => ["--project", name]),
...arguments_,
...runner,
],
{ stdio: "inherit" },
{ stdio: "inherit", env: environment },
);
let shutdownSignal;
for (const signal of ["SIGINT", "SIGTERM"])
+19 -41
View File
@@ -4,16 +4,13 @@ import { isAbsolute, join, relative, resolve, sep } from "node:path";
import { z } from "zod";
import { compareTestSelection } from "./test-selection.mjs";
import { parseTestSelection } from "./test-selection.mjs";
const root = process.cwd();
const historyDirectory = join(root, ".cache/ci-test-history");
const historyPath = join(historyDirectory, "test-results.json");
const selectionDirectory = join(root, ".cache/ci-test-selection");
const selectionPath = join(selectionDirectory, "report.json");
// Never let a failed or partial rerun leave a previous successful history file.
await rm(historyPath, { force: true });
await rm(selectionPath, { force: true });
const report = z
.object({
version: z.literal(1),
@@ -56,40 +53,23 @@ const passed =
process.env.MERGE_RESULT === "success" &&
report.success;
// Successful shards and merged reports must account for the complete portable
// inventory. Coverage thresholds alone cannot prove that every file ran.
// inventory, or exactly the selected subset. Coverage thresholds alone cannot
// prove that every file ran.
const verifyInventory = process.env.CI_VERIFY_TEST_INVENTORY === "true";
const expected =
verifyInventory || process.env.CI_TEST_SELECTION
? execFileSync("git", ["ls-files", "-z", "--", "src", "tests"], {
const selection = process.env.CI_TEST_SELECTION
? parseTestSelection(process.env.CI_TEST_SELECTION)
: undefined;
const selected = selection?.mode === "selected";
const expected = !verifyInventory
? []
: selected
? selection.groups.flat()
: execFileSync("git", ["ls-files", "-z", "--", "src", "tests"], {
encoding: "utf8",
maxBuffer: 8 * 1024 * 1024,
})
.split("\0")
.filter((path) => path.endsWith(".test.ts"))
: [];
let selection;
if (process.env.CI_TEST_SELECTION) {
const comparison = compareTestSelection(
JSON.parse(process.env.CI_TEST_SELECTION),
expected,
files,
);
selection = {
...comparison,
complete:
comparison.complete &&
report.unhandledErrors === 0 &&
["passed", "failed"].includes(report.reason) &&
groupsCompleted &&
["success", "failure"].includes(process.env.MERGE_RESULT),
executionSha: process.env.GITHUB_SHA ?? null,
prHeadSha: process.env.CI_PR_HEAD_SHA ?? null,
runId: process.env.GITHUB_RUN_ID ?? null,
runAttempt: process.env.GITHUB_RUN_ATTEMPT ?? null,
};
await mkdir(selectionDirectory, { recursive: true });
await writeFile(selectionPath, JSON.stringify(selection));
}
.filter((path) => path.endsWith(".test.ts"));
if (passed && verifyInventory) {
const actual = files.map(({ name }) => name);
if (actual.length === 0)
@@ -100,11 +80,13 @@ if (passed && verifyInventory) {
const unexpected = actual.filter((path) => !expectedSet.has(path));
if (missing.length || unexpected.length || actualSet.size !== actual.length)
throw new Error(
`Incomplete portable test inventory: ${JSON.stringify({ missing, unexpected, duplicateFiles: actual.length - actualSet.size })}`,
`Incomplete ${selected ? "selected" : "portable"} test inventory: ${JSON.stringify({ missing, unexpected, duplicateFiles: actual.length - actualSet.size })}`,
);
if (files.every(({ state }) => state === "skipped"))
throw new Error("Every portable test module was skipped");
}
// A selected subset is not a timing baseline for the complete suite.
const publishesHistory = passed && !selected;
const skippedFiles = files.filter(({ state }) => state === "skipped");
const summary = [
groupsCompleted
@@ -113,11 +95,7 @@ const summary = [
"",
...(selection
? [
"### Observational PR test selection",
"",
`Plan: ${selection.plan.mode}. Proposed files: ${selection.selected.length}; omitted: ${selection.omitted.length}.`,
`Full-run comparison: ${selection.complete ? "complete" : "incomplete"}; failures outside proposed selection: ${selection.missedFailures.length}.`,
"This observational report cannot enable omissions or qualify a revision. Skipped, missing, and interrupted observations do not establish parity.",
`Portable tests: ${selected ? `${selection.groups.flat().length} selected files` : "complete suite"}. ${selection.reason}.`,
"",
]
: []),
@@ -129,7 +107,7 @@ const summary = [
...(skippedFiles.length ? [""] : []),
...skippedFiles.map(({ name }) => `- ${name.replaceAll("|", "\\|")}`),
"",
passed
publishesHistory
? "Successful timing history is available."
: "This report is diagnostic only and is not a successful timing baseline.",
"",
@@ -150,7 +128,7 @@ if (process.env.GITHUB_STEP_SUMMARY)
await appendFile(process.env.GITHUB_STEP_SUMMARY, summary);
else console.log(summary);
if (passed) {
if (publishesHistory) {
await mkdir(historyDirectory, { recursive: true });
await writeFile(
historyPath,
+25 -35
View File
@@ -1,35 +1,25 @@
export interface TestSelectionPlan {
readonly version: 1;
readonly mode: "full" | "candidate";
readonly owners: readonly string[];
readonly changedTests: readonly string[];
readonly changedPaths: readonly string[];
readonly reason: string;
}
export function planTestSelection(
files: readonly string[],
event: string,
tier: string,
): TestSelectionPlan;
export function compareTestSelection(
plan: unknown,
inventory: readonly string[],
results: readonly {
readonly name: string;
readonly durationMs: number;
readonly state?: string;
}[],
): {
readonly version: 1;
readonly observational: true;
readonly plan: TestSelectionPlan;
readonly selected: readonly string[];
readonly omitted: readonly string[];
readonly missing: readonly string[];
readonly unexpected: readonly string[];
readonly duplicateFiles: number;
readonly unobserved: readonly string[];
readonly missedFailures: readonly string[];
readonly complete: boolean;
readonly omittedDurationMs: number;
};
export type TestSelectionPlan =
| {
readonly version: 2;
readonly mode: "full";
readonly reason: string;
readonly groups: readonly [];
}
| {
readonly version: 2;
readonly mode: "selected";
readonly reason: string;
readonly groups: readonly (readonly string[])[];
};
export function planTestSelection(request: {
readonly files: readonly string[];
readonly event: string;
readonly tier: string;
readonly code: boolean;
readonly snapshot: {
readonly list: () => readonly string[];
readonly read: (paths: readonly string[]) => ReadonlyMap<string, string>;
};
}): TestSelectionPlan;
export function parseTestSelection(value: unknown): TestSelectionPlan;
export function testShards(tier: string, plan: TestSelectionPlan): string[];
+213 -140
View File
@@ -1,151 +1,224 @@
import { matchesGlob } from "node:path";
import { matchesGlob, posix } from "node:path";
// Candidate ownership only: these leaf adapters and their producer tests can be
// compared with full-suite outcomes. This table never changes test execution.
// Shared application/domain code and dynamic CLI/MCP consumers are not leaf owners.
const OWNERS = {
adb: ["src/adb/**", "tests/boundary/providers/adb/**"],
apktool: ["src/apktool/**"],
dotnet: ["src/dotnet/**"],
evm: ["src/evm/**", "tests/boundary/evm/**"],
firmware: ["src/firmware/**", "tests/boundary/firmware/**"],
flutter: ["src/flutter/**"],
ghidra: ["src/ghidra/**", "tests/boundary/providers/ghidra/**"],
hopper: ["src/hopper/**", "tests/boundary/providers/hopper/**"],
ida: ["src/ida/**", "tests/boundary/providers/ida/**"],
jeb: ["src/jeb/**"],
pcap: ["src/pcap/**", "tests/boundary/pcap/**"],
sqlite: ["src/sqlite/**", "tests/boundary/sqlite/**"],
wasm: ["src/wasm/**"],
};
const ownersOf = (path) =>
Object.entries(OWNERS)
.filter(([, patterns]) => patterns.some((glob) => matchesGlob(path, glob)))
.map(([owner]) => owner);
const validPath = (path) =>
typeof path === "string" &&
/^(src|tests)\//.test(path) &&
!/[\\\0\r\n]/.test(path) &&
!path.split("/").some((part) => ["", ".", ".."].includes(part));
const full = (reason, changedPaths = []) => ({
version: 1,
mode: "full",
owners: [],
changedTests: [],
changedPaths,
reason,
});
import { projectForTest } from "../lib/test-projects.mjs";
import { SHARED_BOUNDARY } from "./scopes.mjs";
/** Plan an observational PR subset from the classifier's existing Git diff. */
export function planTestSelection(files, event, tier) {
// Pull-request feedback runs the tests a change most directly affects. Main
// pushes, schedules, releases and `ci:full` keep the complete suite, so a
// consumer the selection misses fails on main right after it lands.
// Selection falls back to the complete suite whenever it cannot decide.
/** More selected files than this run the complete suite instead. */
const SELECTION_CAP = 150;
/** Selected files per test job, and the most jobs one selection uses. */
const FILES_PER_GROUP = 40;
const MAX_GROUPS = 4;
// Leaf adapters whose tests also reach their sources through the built CLI,
// MCP server or a fake provider process rather than an import. An adapter
// owns `src/<name>/` and every test named for it (`<name>*.test.ts`) or
// filed under a `<name>/` directory anywhere in `tests/`.
const OWNERS = [
"adb",
"apktool",
"dotnet",
"evm",
"firmware",
"flutter",
"ghidra",
"hopper",
"ida",
"jeb",
"pcap",
"sqlite",
"wasm",
];
const ownedTests = (path) =>
OWNERS.filter((owner) => path.startsWith(`src/${owner}/`)).flatMap(
(owner) => [`tests/**/${owner}/**/*.test.ts`, `tests/**/${owner}*.test.ts`],
);
const isTest = (path) => path.endsWith(".test.ts");
// Inputs that only a non-test check reads: declarations (typecheck), guard
// baselines (architecture guards, static job) and the advertised-schema
// summaries (build job). They select no tests.
const isCheckerInput = (path) =>
/\.d\.m?ts$/u.test(path) ||
/^scripts\/verify\/baselines\/[^/]+\.json$/u.test(path) ||
/^scripts\/verify\/advertised-schemas\/tools\/[^/]+\.json$/u.test(path);
// Verifier scripts are imported by their tests like source modules. Other
// scripts (build, test runners, CI) shape every run and are not traced.
const isModule = (path) =>
!isCheckerInput(path) &&
(/^(src|tests)\/.+\.m?ts$/u.test(path) ||
/^scripts\/(verify-[^/]+|verify\/.+)\.mjs$/u.test(path));
// Test-only modules pass a change through to the tests that use them.
// Production importers never widen the selection transitively.
const isHelper = (path) =>
!isTest(path) &&
(/^tests\/.+\.m?ts$/u.test(path) ||
/^src\/.+\.fixture[^/]*\.ts$/u.test(path));
// Prose that tests may read by path. Other non-module inputs (fixtures, data,
// scripts, workflows, package metadata) are not traced, so they run everything.
const isProse = (path) => path.endsWith(".md") && !/^(src|tests)\//u.test(path);
const SPECIFIER =
/(?:\bfrom\s*|\bimport\s*\(\s*|\bimport\s+|\bvi\.mock\(\s*)["'](\.{1,2}\/[^"']+)["']/gu;
// TypeScript imports name the emitted `.js`/`.mjs`; scripts are imported as is.
const importsOf = (path, source) =>
[...source.matchAll(SPECIFIER)].flatMap(([, specifier]) => {
const target = posix.normalize(posix.join(posix.dirname(path), specifier));
return [target, target.replace(/\.(m?)js$/u, ".$1ts")];
});
const full = (reason) => ({ version: 2, mode: "full", reason, groups: [] });
/**
* Plan the portable tests one CI run executes.
*
* `snapshot` lists the tracked `src/` and `tests/` paths of the revision under
* test and reads them; it is consulted only when a subset is possible.
*/
export function planTestSelection({ files, event, tier, code, snapshot }) {
// `ci:full` selects the full tier, so it also lands here.
if (event !== "pull_request" || tier !== "fast")
return full("Only ordinary PRs are candidates", files);
return full("Main, ci:full and qualification runs execute every test");
if (!code) return full("No portable tests run for this change");
if (!files.length) return full("No changed paths available");
const owners = new Set();
const changedTests = [];
for (const path of files) {
if (!validPath(path))
return full(`Shared or unknown input: ${path}`, files);
const matches = ownersOf(path);
if (!matches.length) return full(`Shared or unknown input: ${path}`, files);
for (const owner of matches) owners.add(owner);
if (path.endsWith(".test.ts")) changedTests.push(path);
if (SHARED_BOUNDARY.test(path)) return full(`Shared boundary: ${path}`);
if (!isModule(path) && !isProse(path) && !isCheckerInput(path))
return full(`Input is not traced to tests: ${path}`);
}
const tracked = snapshot.list();
const trackedSet = new Set(tracked);
const tests = tracked.filter(isTest);
const helpers = tracked.filter(isHelper);
const sources = snapshot.read([...tests, ...helpers]);
const imports = new Map(
[...tests, ...helpers].map((path) => [
path,
importsOf(path, sources.get(path) ?? ""),
]),
);
const selected = new Map();
const select = (path, reason) => {
if (trackedSet.has(path) && !selected.has(path)) selected.set(path, reason);
};
const changedModules = files.filter(
(path) => isModule(path) && !isTest(path),
);
for (const path of files) if (isTest(path)) select(path, "changed");
// Close over test-only helpers, then take the tests that import a change.
const affected = new Set(changedModules);
for (let grew = true; grew;) {
grew = false;
for (const helper of helpers)
if (
!affected.has(helper) &&
imports.get(helper).some((target) => affected.has(target))
) {
affected.add(helper);
grew = true;
}
}
for (const test of tests)
if (imports.get(test).some((target) => affected.has(target)))
select(test, "imports");
for (const path of changedModules) {
// Same-stem tests, and same-directory tests that scan their directory's
// sources by path instead of importing them.
const stem = path.replace(/\.m?ts$/u, "");
for (const test of tests)
if (
posix.dirname(test) === posix.dirname(path) &&
(test.startsWith(`${stem}.`) ||
/\breaddir(?:Sync)?\s*\(/u.test(sources.get(test) ?? ""))
)
select(test, "beside");
// Leaf adapters' boundary tests.
const patterns = ownedTests(path);
for (const test of tests)
if (patterns.some((glob) => matchesGlob(test, glob)))
select(test, "owner");
}
// Tests that name a changed path read it directly.
for (const path of files)
for (const test of tests)
if ((sources.get(test) ?? "").includes(path)) select(test, "names");
const paths = [...selected.keys()].sort();
if (!paths.length)
return full("No test imports, names or sits beside a changed path");
if (paths.length > SELECTION_CAP)
return full(
`${paths.length} affected test files exceed the ${SELECTION_CAP}-file selection cap`,
);
for (const path of paths)
try {
projectForTest(path);
} catch {
return full(`No single test project runs ${path}`);
}
const count = Math.min(MAX_GROUPS, Math.ceil(paths.length / FILES_PER_GROUP));
const groups = Array.from({ length: count }, () => []);
paths.forEach((path, index) => groups[index % count].push(path));
const reasons = Object.groupBy([...selected.values()], (reason) => reason);
return {
version: 1,
mode: "candidate",
owners: [...owners].sort(),
changedTests: [...new Set(changedTests)].sort(),
changedPaths: files,
reason:
"Observe affected leaf owners; retain integration and unowned tests",
version: 2,
mode: "selected",
reason: `Affected tests: ${Object.entries(reasons)
.map(([reason, entries]) => `${entries.length} ${reason}`)
.join(", ")}`,
groups,
};
}
/** Compare a proposal with the full run; missing observations never prove parity. */
export function compareTestSelection(plan, inventory, results) {
if (
plan?.version !== 1 ||
!["full", "candidate"].includes(plan.mode) ||
!Array.isArray(plan.owners) ||
plan.owners.some((owner) => !Object.hasOwn(OWNERS, owner)) ||
!Array.isArray(plan.changedTests) ||
!Array.isArray(plan.changedPaths) ||
plan.changedPaths.some((path) => typeof path !== "string") ||
plan.changedTests.some(
(path) => !validPath(path) || !path.endsWith(".test.ts"),
) ||
typeof plan.reason !== "string" ||
(plan.mode === "candidate" && !plan.owners.length) ||
(plan.mode === "full" && (plan.owners.length || plan.changedTests.length))
)
throw new Error("Invalid observational test selection plan");
const expected = new Set(inventory);
if (!expected.size || expected.size !== inventory.length)
throw new Error("Test selection requires a nonempty unique inventory");
const ownersByFile = new Map(
plan.mode === "candidate"
? inventory.map((path) => [path, ownersOf(path)])
: [],
);
// Removed/renamed tests have no current executable identity. Keep all tests.
const removed = plan.changedTests.filter((path) => !expected.has(path));
const missingOwners = plan.owners.filter(
(owner) =>
![...ownersByFile.values()].some((owners) => owners.includes(owner)),
);
const effective = removed.length
? full(
`Changed tests absent from current inventory: ${removed.join(", ")}`,
plan.changedPaths,
)
: missingOwners.length
? full(
`Owners without executable tests: ${missingOwners.join(", ")}`,
plan.changedPaths,
)
: plan;
const selected = [];
const omitted = [];
for (const path of inventory.toSorted()) {
const owners = ownersByFile.get(path) ?? [];
const keep =
effective.mode === "full" ||
!owners.length ||
effective.changedTests.includes(path) ||
owners.some((owner) => effective.owners.includes(owner));
(keep ? selected : omitted).push(path);
}
const actual = new Set(results.map((file) => file.name));
const missing = inventory.filter((path) => !actual.has(path));
const unexpected = [...actual].filter((path) => !expected.has(path));
const duplicateFiles = results.length - actual.size;
const unobserved = results
.filter((file) => !["passed", "failed"].includes(file.state))
.map((file) => file.name);
const omittedSet = new Set(omitted);
const missedFailures = results
.filter((file) => omittedSet.has(file.name) && file.state === "failed")
.map((file) => file.name);
return {
version: 1,
observational: true,
plan: effective,
selected,
omitted,
missing,
unexpected,
duplicateFiles,
unobserved,
missedFailures,
complete:
!missing.length &&
!unexpected.length &&
!duplicateFiles &&
!unobserved.length,
// Phase totals are scheduling weights, not an estimate of wall-time savings.
omittedDurationMs: results
.filter((file) => omittedSet.has(file.name))
.reduce((sum, file) => sum + file.durationMs, 0),
};
/** Parse a plan produced by `planTestSelection`; anything else fails closed. */
export function parseTestSelection(value) {
const plan = typeof value === "string" ? JSON.parse(value) : value;
const valid =
plan?.version === 2 &&
typeof plan.reason === "string" &&
Array.isArray(plan.groups) &&
(plan.mode === "full"
? plan.groups.length === 0
: plan.mode === "selected" &&
plan.groups.length >= 1 &&
plan.groups.length <= MAX_GROUPS &&
plan.groups.every(
(group) =>
Array.isArray(group) &&
group.length > 0 &&
group.every(
(path) =>
typeof path === "string" &&
isTest(path) &&
/^(src|tests)\//u.test(path) &&
!path.split("/").some((part) => ["", ".", ".."].includes(part)),
),
) &&
new Set(plan.groups.flat()).size === plan.groups.flat().length &&
plan.groups.flat().length <= SELECTION_CAP);
if (!valid) throw new Error("Invalid portable test selection plan");
return plan;
}
// Fast feedback uses one source group and twelve built groups. Shards are
// balanced by recorded duration, but hosted runner speed varies, so the
// slowest shard sets the critical path: at eight shards it ran 24% above the
// mean shard. Smaller shards bound that excess at a few percent more runner
// time. Full qualification retains six complete shards with coverage. A
// selection runs one job per group.
const shardsOf = (count) =>
Array.from({ length: count }, (_, index) => `${index + 1}/${count}`);
/** The `test-shard` matrix labels for a tier and its test selection. */
export const testShards = (tier, plan) =>
shardsOf(
plan.mode === "selected" ? plan.groups.length : tier === "full" ? 6 : 12,
);