mirror of
https://github.com/morluto/rea.git
synced 2026-11-16 19:53:50 +00:00
ci: run only the affected portable tests on pull requests (#11346)
* ci: run only the affected portable tests on pull requests Pull-request CI now runs static checks, the build and the portable tests a change affects, selected from test imports (through test-only helpers), same-stem and directory-scanning tests, leaf-owner boundary tests and tests that name a changed path. Shared boundaries, untraced inputs, empty selections and selections above 150 files fall back to the complete fast suite. Main pushes, schedules, releases and ci:full keep the complete suite. The portable report verifies the exact selected inventory and the CI required gate accepts a selected plan only for a fast pull request with code changes. * ci: own leaf-adapter tests by name and keep subset summaries diagnostic
This commit is contained in:
1 parent
97112f14a1
commit
ff6d1d8142
18 files changed
+901
-577
No files matched your search
@@ -15,6 +15,14 @@ export interface ChangePlanRepository {
|
||||
readonly isAncestor: (ancestor: string, head: string) => boolean;
|
||||
readonly changedFiles: (base: string, head: string) => readonly string[];
|
||||
readonly readFile: (revision: string, path: string) => string;
|
||||
readonly listFiles: (
|
||||
revision: string,
|
||||
directories: readonly string[],
|
||||
) => readonly string[];
|
||||
readonly readFiles: (
|
||||
revision: string,
|
||||
paths: readonly string[],
|
||||
) => ReadonlyMap<string, string>;
|
||||
}
|
||||
|
||||
export type ChangePlan = Readonly<Record<string, unknown>> & {
|
||||
@@ -29,6 +37,7 @@ export type ChangePlan = Readonly<Record<string, unknown>> & {
|
||||
readonly files: readonly string[];
|
||||
readonly reasons: readonly string[];
|
||||
readonly testSelection: TestSelectionPlan;
|
||||
readonly testShards: readonly string[];
|
||||
};
|
||||
|
||||
export function planChanges(
|
||||
|
||||
+50
-13
@@ -5,9 +5,10 @@ import {
|
||||
CI_FLAGS as flags,
|
||||
MANUAL_VERIFIERS,
|
||||
CI_LANES,
|
||||
SHARED_BOUNDARY,
|
||||
WINDOWS_EXECUTABLE_SELECTION_INPUTS,
|
||||
} from "./scopes.mjs";
|
||||
import { planTestSelection } from "./test-selection.mjs";
|
||||
import { planTestSelection, testShards } from "./test-selection.mjs";
|
||||
|
||||
// Actionlint validates workflow syntax only. The semantic suite asserts these
|
||||
// workflows' release, publication and self-hosted runner safeguards.
|
||||
@@ -58,11 +59,6 @@ const matchesAny = (file, patterns) =>
|
||||
export function planChanges(request, repository) {
|
||||
const selected = Object.fromEntries(flags.map((flag) => [flag, false]));
|
||||
const reasons = [];
|
||||
// Shared process, contract, CLI, and CI-script edits are not a specific provider.
|
||||
// Fast qualification keeps the portable suite and the Linux x64 runtime
|
||||
// baseline. Full qualification still selects every lane.
|
||||
const SHARED_BOUNDARY =
|
||||
/^(src\/process\/|src\/filesystem\/|src\/contracts\/tool|src\/server\/createServer|src\/cli\.ts|scripts\/ci\/)/;
|
||||
const full = (reason) => {
|
||||
if (tier === "full") {
|
||||
for (const flag of flags) selected[flag] = true;
|
||||
@@ -269,11 +265,7 @@ export function planChanges(request, repository) {
|
||||
selected.readiness = true;
|
||||
}
|
||||
if (file.startsWith("tests/")) return;
|
||||
if (
|
||||
/^(src\/process\/|src\/filesystem\/|src\/contracts\/tool|src\/server\/createServer|src\/cli\.ts|scripts\/ci\/)/.test(
|
||||
file,
|
||||
)
|
||||
) {
|
||||
if (SHARED_BOUNDARY.test(file)) {
|
||||
full(`Shared boundary: ${file}`);
|
||||
} else if (owners.length) {
|
||||
if (file.startsWith("src/") || file.startsWith("bridge/"))
|
||||
@@ -343,8 +335,26 @@ export function planChanges(request, repository) {
|
||||
]
|
||||
: [linux],
|
||||
};
|
||||
const testSelection = planTestSelection(files, event, tier);
|
||||
return { ...selected, tier, matrix, files, reasons, testSelection };
|
||||
const testSelection = planTestSelection({
|
||||
files,
|
||||
event,
|
||||
tier,
|
||||
code: selected.code,
|
||||
// The checked-out revision is the one whose tests run.
|
||||
snapshot: {
|
||||
list: () => repository.listFiles("HEAD", ["src", "tests"]),
|
||||
read: (paths) => repository.readFiles("HEAD", paths),
|
||||
},
|
||||
});
|
||||
return {
|
||||
...selected,
|
||||
tier,
|
||||
matrix,
|
||||
files,
|
||||
reasons,
|
||||
testSelection,
|
||||
testShards: testShards(tier, testSelection),
|
||||
};
|
||||
}
|
||||
|
||||
/** Answer the plan's Git questions from the repository in `cwd`. */
|
||||
@@ -370,5 +380,32 @@ export const gitRepository = (cwd = process.cwd()) => {
|
||||
.split("\0")
|
||||
.filter(Boolean),
|
||||
readFile: (revision, path) => git("show", `${revision}:${path}`),
|
||||
listFiles: (revision, directories) =>
|
||||
git("ls-tree", "-r", "-z", "--name-only", revision, "--", ...directories)
|
||||
.split("\0")
|
||||
.filter(Boolean),
|
||||
// One batch process reads every requested blob.
|
||||
readFiles: (revision, paths) => {
|
||||
const output = execFileSync("git", ["cat-file", "--batch"], {
|
||||
cwd,
|
||||
input: paths.map((path) => `${revision}:${path}\n`).join(""),
|
||||
maxBuffer: 512 * 1024 * 1024,
|
||||
});
|
||||
const contents = new Map();
|
||||
let offset = 0;
|
||||
for (const path of paths) {
|
||||
const end = output.indexOf(10, offset);
|
||||
const [, type, size] = output.toString("utf8", offset, end).split(" ");
|
||||
if (type !== "blob")
|
||||
throw new Error(`Git could not read ${revision}:${path}`);
|
||||
const start = end + 1;
|
||||
contents.set(
|
||||
path,
|
||||
output.toString("utf8", start, start + Number(size)),
|
||||
);
|
||||
offset = start + Number(size) + 1;
|
||||
}
|
||||
return contents;
|
||||
},
|
||||
};
|
||||
};
|
||||
+4
-2
@@ -14,7 +14,7 @@ const plan = planChanges(
|
||||
},
|
||||
gitRepository(),
|
||||
);
|
||||
const { tier, matrix, reasons, testSelection } = plan;
|
||||
const { tier, matrix, reasons, testSelection, testShards } = plan;
|
||||
const selected = Object.fromEntries(flags.map((flag) => [flag, plan[flag]]));
|
||||
if (process.env.GITHUB_OUTPUT)
|
||||
appendFileSync(
|
||||
@@ -24,6 +24,8 @@ if (process.env.GITHUB_OUTPUT)
|
||||
tier,
|
||||
matrix: JSON.stringify(matrix),
|
||||
test_selection: JSON.stringify(testSelection),
|
||||
test_mode: testSelection.mode,
|
||||
test_shards: JSON.stringify(testShards),
|
||||
})
|
||||
.map(([key, value]) => `${key}=${value}\n`)
|
||||
.join(""),
|
||||
@@ -31,6 +33,6 @@ if (process.env.GITHUB_OUTPUT)
|
||||
if (process.env.GITHUB_STEP_SUMMARY)
|
||||
appendFileSync(
|
||||
process.env.GITHUB_STEP_SUMMARY,
|
||||
`### Selected CI lanes (${tier})\n\n${flags.filter((flag) => selected[flag]).join(", ") || "Formatting only"}\n\n${reasons.join("\n\n")}\n`,
|
||||
`### Selected CI lanes (${tier})\n\n${flags.filter((flag) => selected[flag]).join(", ") || "Formatting only"}\n\n${reasons.join("\n\n")}\n\nPortable tests (${testSelection.mode}): ${testSelection.reason}\n`,
|
||||
);
|
||||
console.log(JSON.stringify(plan));
|
||||
+16
-1
@@ -28,10 +28,25 @@ if (
|
||||
);
|
||||
if (outputs.tier === "full" && selectedScopes.length !== CI_FLAGS.length)
|
||||
throw new Error("Full qualification requires every canonical CI scope");
|
||||
// Only a fast pull-request plan may run a selected test subset.
|
||||
const testMode = outputs.test_mode;
|
||||
if (
|
||||
!["full", "selected"].includes(testMode) ||
|
||||
(testMode === "selected" &&
|
||||
(outputs.tier !== "fast" ||
|
||||
process.env.GITHUB_EVENT_NAME !== "pull_request" ||
|
||||
outputs.code !== "true"))
|
||||
)
|
||||
throw new Error(
|
||||
"CI classification requires a valid test mode: full, or selected on a fast pull request",
|
||||
);
|
||||
const required = reused ? ["changes"] : ["changes", "static", "test"];
|
||||
if (outputs.tier === "full") required.push("static-deep");
|
||||
for (const scope of selectedScopes)
|
||||
required.push(...CI_LANES[scope].required[outputs.tier]);
|
||||
required.push(
|
||||
...((testMode === "selected" && CI_LANES[scope].selectedTests) ||
|
||||
CI_LANES[scope].required[outputs.tier]),
|
||||
);
|
||||
const failed = [...new Set(required)].filter(
|
||||
(name) => jobs[name]?.result !== "success",
|
||||
);
|
||||
|
||||
@@ -8,6 +8,13 @@ export const WINDOWS_EXECUTABLE_SELECTION_INPUTS = [
|
||||
"tests/boundary/process/windowsExecutableSelection.test.ts",
|
||||
];
|
||||
|
||||
// Shared process, contract, CLI, and CI-script edits are not a specific provider.
|
||||
// Fast qualification keeps the portable suite and the Linux x64 runtime
|
||||
// baseline, and pull requests run the complete portable suite rather than a
|
||||
// selected subset. Full qualification still selects every lane.
|
||||
export const SHARED_BOUNDARY =
|
||||
/^(src\/process\/|src\/filesystem\/|src\/contracts\/tool|src\/server\/createServer|src\/cli\.ts|scripts\/ci\/)/;
|
||||
|
||||
// Inputs whose changes must select both real capture lanes, even at shared boundaries.
|
||||
const CAPTURE_SHARED_INPUTS = [
|
||||
"src/process/LinuxResourceLimits*",
|
||||
@@ -679,6 +686,8 @@ export const CI_LANES = {
|
||||
fast: ["build", "test-shard", "test-source"],
|
||||
full: ["build", "test-shard"],
|
||||
},
|
||||
// A selected PR subset runs its source tests in the test-shard groups.
|
||||
selectedTests: ["build", "test-shard"],
|
||||
},
|
||||
runtime: sameJobs([], ["windows-native-build", "package-e2e"]),
|
||||
package: sameJobs(PATH_SCOPES.package, ["windows-curated"], {
|
||||
|
||||
+38
-10
@@ -5,17 +5,48 @@ import {
|
||||
BUILT_TEST_PROJECTS,
|
||||
SOURCE_TEST_PROJECTS,
|
||||
} from "../lib/test-projects.mjs";
|
||||
import { parseTestSelection, testShards } from "./test-selection.mjs";
|
||||
|
||||
const [group, ...arguments_] = process.argv.slice(2);
|
||||
const projects = { source: SOURCE_TEST_PROJECTS, built: BUILT_TEST_PROJECTS }[
|
||||
group
|
||||
];
|
||||
if (!projects) throw new Error("Expected source or built test group");
|
||||
// The group owns project selection; callers can request reporters or shards.
|
||||
// The group owns project and file selection; callers can request reporters
|
||||
// or (for project groups) shards.
|
||||
if (
|
||||
arguments_.some((argument) => /^--(?:project|exclude)(?:=|$)/u.test(argument))
|
||||
)
|
||||
throw new Error("Test groups cannot override their project inventory");
|
||||
let runner;
|
||||
let environment = process.env;
|
||||
if (group === "selected") {
|
||||
// `selected <k/n>` runs group k of the plan's selected test files, by exact
|
||||
// identity, in whichever projects own them.
|
||||
const [shard, ...rest] = arguments_;
|
||||
const plan = parseTestSelection(process.env.CI_TEST_SELECTION ?? "");
|
||||
if (plan.mode !== "selected" || !testShards("fast", plan).includes(shard))
|
||||
throw new Error(`No selected test group ${shard ?? ""} in this plan`);
|
||||
if (rest.some((argument) => /^--shard(?:=|$)/u.test(argument)))
|
||||
throw new Error("Selected test groups are already sharded");
|
||||
const index = Number(shard.split("/")[0]) - 1;
|
||||
runner = [
|
||||
resolve("scripts/run-exact-tests.mjs"),
|
||||
"run",
|
||||
...rest,
|
||||
...plan.groups[index],
|
||||
];
|
||||
// Each group writes its own blob report for the merge.
|
||||
environment = { ...process.env, VITEST_BLOB_LABEL: `selected-${index + 1}` };
|
||||
} else {
|
||||
const projects = {
|
||||
source: SOURCE_TEST_PROJECTS,
|
||||
built: BUILT_TEST_PROJECTS,
|
||||
}[group];
|
||||
if (!projects) throw new Error("Expected source, built or selected tests");
|
||||
runner = [
|
||||
resolve("node_modules/vitest/vitest.mjs"),
|
||||
"run",
|
||||
...projects.flatMap(({ name }) => ["--project", name]),
|
||||
...arguments_,
|
||||
];
|
||||
}
|
||||
const child = spawn(
|
||||
process.execPath,
|
||||
[
|
||||
@@ -24,12 +55,9 @@ const child = spawn(
|
||||
...(process.platform === "win32"
|
||||
? []
|
||||
: [resolve("scripts/run-exclusive.mjs"), "--unlocked", process.execPath]),
|
||||
resolve("node_modules/vitest/vitest.mjs"),
|
||||
"run",
|
||||
...projects.flatMap(({ name }) => ["--project", name]),
|
||||
...arguments_,
|
||||
...runner,
|
||||
],
|
||||
{ stdio: "inherit" },
|
||||
{ stdio: "inherit", env: environment },
|
||||
);
|
||||
let shutdownSignal;
|
||||
for (const signal of ["SIGINT", "SIGTERM"])
|
||||
|
||||
+19
-41
@@ -4,16 +4,13 @@ import { isAbsolute, join, relative, resolve, sep } from "node:path";
|
||||
|
||||
import { z } from "zod";
|
||||
|
||||
import { compareTestSelection } from "./test-selection.mjs";
|
||||
import { parseTestSelection } from "./test-selection.mjs";
|
||||
|
||||
const root = process.cwd();
|
||||
const historyDirectory = join(root, ".cache/ci-test-history");
|
||||
const historyPath = join(historyDirectory, "test-results.json");
|
||||
const selectionDirectory = join(root, ".cache/ci-test-selection");
|
||||
const selectionPath = join(selectionDirectory, "report.json");
|
||||
// Never let a failed or partial rerun leave a previous successful history file.
|
||||
await rm(historyPath, { force: true });
|
||||
await rm(selectionPath, { force: true });
|
||||
const report = z
|
||||
.object({
|
||||
version: z.literal(1),
|
||||
@@ -56,40 +53,23 @@ const passed =
|
||||
process.env.MERGE_RESULT === "success" &&
|
||||
report.success;
|
||||
// Successful shards and merged reports must account for the complete portable
|
||||
// inventory. Coverage thresholds alone cannot prove that every file ran.
|
||||
// inventory, or exactly the selected subset. Coverage thresholds alone cannot
|
||||
// prove that every file ran.
|
||||
const verifyInventory = process.env.CI_VERIFY_TEST_INVENTORY === "true";
|
||||
const expected =
|
||||
verifyInventory || process.env.CI_TEST_SELECTION
|
||||
? execFileSync("git", ["ls-files", "-z", "--", "src", "tests"], {
|
||||
const selection = process.env.CI_TEST_SELECTION
|
||||
? parseTestSelection(process.env.CI_TEST_SELECTION)
|
||||
: undefined;
|
||||
const selected = selection?.mode === "selected";
|
||||
const expected = !verifyInventory
|
||||
? []
|
||||
: selected
|
||||
? selection.groups.flat()
|
||||
: execFileSync("git", ["ls-files", "-z", "--", "src", "tests"], {
|
||||
encoding: "utf8",
|
||||
maxBuffer: 8 * 1024 * 1024,
|
||||
})
|
||||
.split("\0")
|
||||
.filter((path) => path.endsWith(".test.ts"))
|
||||
: [];
|
||||
let selection;
|
||||
if (process.env.CI_TEST_SELECTION) {
|
||||
const comparison = compareTestSelection(
|
||||
JSON.parse(process.env.CI_TEST_SELECTION),
|
||||
expected,
|
||||
files,
|
||||
);
|
||||
selection = {
|
||||
...comparison,
|
||||
complete:
|
||||
comparison.complete &&
|
||||
report.unhandledErrors === 0 &&
|
||||
["passed", "failed"].includes(report.reason) &&
|
||||
groupsCompleted &&
|
||||
["success", "failure"].includes(process.env.MERGE_RESULT),
|
||||
executionSha: process.env.GITHUB_SHA ?? null,
|
||||
prHeadSha: process.env.CI_PR_HEAD_SHA ?? null,
|
||||
runId: process.env.GITHUB_RUN_ID ?? null,
|
||||
runAttempt: process.env.GITHUB_RUN_ATTEMPT ?? null,
|
||||
};
|
||||
await mkdir(selectionDirectory, { recursive: true });
|
||||
await writeFile(selectionPath, JSON.stringify(selection));
|
||||
}
|
||||
.filter((path) => path.endsWith(".test.ts"));
|
||||
if (passed && verifyInventory) {
|
||||
const actual = files.map(({ name }) => name);
|
||||
if (actual.length === 0)
|
||||
@@ -100,11 +80,13 @@ if (passed && verifyInventory) {
|
||||
const unexpected = actual.filter((path) => !expectedSet.has(path));
|
||||
if (missing.length || unexpected.length || actualSet.size !== actual.length)
|
||||
throw new Error(
|
||||
`Incomplete portable test inventory: ${JSON.stringify({ missing, unexpected, duplicateFiles: actual.length - actualSet.size })}`,
|
||||
`Incomplete ${selected ? "selected" : "portable"} test inventory: ${JSON.stringify({ missing, unexpected, duplicateFiles: actual.length - actualSet.size })}`,
|
||||
);
|
||||
if (files.every(({ state }) => state === "skipped"))
|
||||
throw new Error("Every portable test module was skipped");
|
||||
}
|
||||
// A selected subset is not a timing baseline for the complete suite.
|
||||
const publishesHistory = passed && !selected;
|
||||
const skippedFiles = files.filter(({ state }) => state === "skipped");
|
||||
const summary = [
|
||||
groupsCompleted
|
||||
@@ -113,11 +95,7 @@ const summary = [
|
||||
"",
|
||||
...(selection
|
||||
? [
|
||||
"### Observational PR test selection",
|
||||
"",
|
||||
`Plan: ${selection.plan.mode}. Proposed files: ${selection.selected.length}; omitted: ${selection.omitted.length}.`,
|
||||
`Full-run comparison: ${selection.complete ? "complete" : "incomplete"}; failures outside proposed selection: ${selection.missedFailures.length}.`,
|
||||
"This observational report cannot enable omissions or qualify a revision. Skipped, missing, and interrupted observations do not establish parity.",
|
||||
`Portable tests: ${selected ? `${selection.groups.flat().length} selected files` : "complete suite"}. ${selection.reason}.`,
|
||||
"",
|
||||
]
|
||||
: []),
|
||||
@@ -129,7 +107,7 @@ const summary = [
|
||||
...(skippedFiles.length ? [""] : []),
|
||||
...skippedFiles.map(({ name }) => `- ${name.replaceAll("|", "\\|")}`),
|
||||
"",
|
||||
passed
|
||||
publishesHistory
|
||||
? "Successful timing history is available."
|
||||
: "This report is diagnostic only and is not a successful timing baseline.",
|
||||
"",
|
||||
@@ -150,7 +128,7 @@ if (process.env.GITHUB_STEP_SUMMARY)
|
||||
await appendFile(process.env.GITHUB_STEP_SUMMARY, summary);
|
||||
else console.log(summary);
|
||||
|
||||
if (passed) {
|
||||
if (publishesHistory) {
|
||||
await mkdir(historyDirectory, { recursive: true });
|
||||
await writeFile(
|
||||
historyPath,
|
||||
|
||||
@@ -1,35 +1,25 @@
|
||||
export interface TestSelectionPlan {
|
||||
readonly version: 1;
|
||||
readonly mode: "full" | "candidate";
|
||||
readonly owners: readonly string[];
|
||||
readonly changedTests: readonly string[];
|
||||
readonly changedPaths: readonly string[];
|
||||
readonly reason: string;
|
||||
}
|
||||
export function planTestSelection(
|
||||
files: readonly string[],
|
||||
event: string,
|
||||
tier: string,
|
||||
): TestSelectionPlan;
|
||||
export function compareTestSelection(
|
||||
plan: unknown,
|
||||
inventory: readonly string[],
|
||||
results: readonly {
|
||||
readonly name: string;
|
||||
readonly durationMs: number;
|
||||
readonly state?: string;
|
||||
}[],
|
||||
): {
|
||||
readonly version: 1;
|
||||
readonly observational: true;
|
||||
readonly plan: TestSelectionPlan;
|
||||
readonly selected: readonly string[];
|
||||
readonly omitted: readonly string[];
|
||||
readonly missing: readonly string[];
|
||||
readonly unexpected: readonly string[];
|
||||
readonly duplicateFiles: number;
|
||||
readonly unobserved: readonly string[];
|
||||
readonly missedFailures: readonly string[];
|
||||
readonly complete: boolean;
|
||||
readonly omittedDurationMs: number;
|
||||
};
|
||||
export type TestSelectionPlan =
|
||||
| {
|
||||
readonly version: 2;
|
||||
readonly mode: "full";
|
||||
readonly reason: string;
|
||||
readonly groups: readonly [];
|
||||
}
|
||||
| {
|
||||
readonly version: 2;
|
||||
readonly mode: "selected";
|
||||
readonly reason: string;
|
||||
readonly groups: readonly (readonly string[])[];
|
||||
};
|
||||
export function planTestSelection(request: {
|
||||
readonly files: readonly string[];
|
||||
readonly event: string;
|
||||
readonly tier: string;
|
||||
readonly code: boolean;
|
||||
readonly snapshot: {
|
||||
readonly list: () => readonly string[];
|
||||
readonly read: (paths: readonly string[]) => ReadonlyMap<string, string>;
|
||||
};
|
||||
}): TestSelectionPlan;
|
||||
export function parseTestSelection(value: unknown): TestSelectionPlan;
|
||||
export function testShards(tier: string, plan: TestSelectionPlan): string[];
|
||||
+213
-140
@@ -1,151 +1,224 @@
|
||||
import { matchesGlob } from "node:path";
|
||||
import { matchesGlob, posix } from "node:path";
|
||||
|
||||
// Candidate ownership only: these leaf adapters and their producer tests can be
|
||||
// compared with full-suite outcomes. This table never changes test execution.
|
||||
// Shared application/domain code and dynamic CLI/MCP consumers are not leaf owners.
|
||||
const OWNERS = {
|
||||
adb: ["src/adb/**", "tests/boundary/providers/adb/**"],
|
||||
apktool: ["src/apktool/**"],
|
||||
dotnet: ["src/dotnet/**"],
|
||||
evm: ["src/evm/**", "tests/boundary/evm/**"],
|
||||
firmware: ["src/firmware/**", "tests/boundary/firmware/**"],
|
||||
flutter: ["src/flutter/**"],
|
||||
ghidra: ["src/ghidra/**", "tests/boundary/providers/ghidra/**"],
|
||||
hopper: ["src/hopper/**", "tests/boundary/providers/hopper/**"],
|
||||
ida: ["src/ida/**", "tests/boundary/providers/ida/**"],
|
||||
jeb: ["src/jeb/**"],
|
||||
pcap: ["src/pcap/**", "tests/boundary/pcap/**"],
|
||||
sqlite: ["src/sqlite/**", "tests/boundary/sqlite/**"],
|
||||
wasm: ["src/wasm/**"],
|
||||
};
|
||||
const ownersOf = (path) =>
|
||||
Object.entries(OWNERS)
|
||||
.filter(([, patterns]) => patterns.some((glob) => matchesGlob(path, glob)))
|
||||
.map(([owner]) => owner);
|
||||
const validPath = (path) =>
|
||||
typeof path === "string" &&
|
||||
/^(src|tests)\//.test(path) &&
|
||||
!/[\\\0\r\n]/.test(path) &&
|
||||
!path.split("/").some((part) => ["", ".", ".."].includes(part));
|
||||
const full = (reason, changedPaths = []) => ({
|
||||
version: 1,
|
||||
mode: "full",
|
||||
owners: [],
|
||||
changedTests: [],
|
||||
changedPaths,
|
||||
reason,
|
||||
});
|
||||
import { projectForTest } from "../lib/test-projects.mjs";
|
||||
import { SHARED_BOUNDARY } from "./scopes.mjs";
|
||||
|
||||
/** Plan an observational PR subset from the classifier's existing Git diff. */
|
||||
export function planTestSelection(files, event, tier) {
|
||||
// Pull-request feedback runs the tests a change most directly affects. Main
|
||||
// pushes, schedules, releases and `ci:full` keep the complete suite, so a
|
||||
// consumer the selection misses fails on main right after it lands.
|
||||
// Selection falls back to the complete suite whenever it cannot decide.
|
||||
|
||||
/** More selected files than this run the complete suite instead. */
|
||||
const SELECTION_CAP = 150;
|
||||
/** Selected files per test job, and the most jobs one selection uses. */
|
||||
const FILES_PER_GROUP = 40;
|
||||
const MAX_GROUPS = 4;
|
||||
|
||||
// Leaf adapters whose tests also reach their sources through the built CLI,
|
||||
// MCP server or a fake provider process rather than an import. An adapter
|
||||
// owns `src/<name>/` and every test named for it (`<name>*.test.ts`) or
|
||||
// filed under a `<name>/` directory anywhere in `tests/`.
|
||||
const OWNERS = [
|
||||
"adb",
|
||||
"apktool",
|
||||
"dotnet",
|
||||
"evm",
|
||||
"firmware",
|
||||
"flutter",
|
||||
"ghidra",
|
||||
"hopper",
|
||||
"ida",
|
||||
"jeb",
|
||||
"pcap",
|
||||
"sqlite",
|
||||
"wasm",
|
||||
];
|
||||
const ownedTests = (path) =>
|
||||
OWNERS.filter((owner) => path.startsWith(`src/${owner}/`)).flatMap(
|
||||
(owner) => [`tests/**/${owner}/**/*.test.ts`, `tests/**/${owner}*.test.ts`],
|
||||
);
|
||||
|
||||
const isTest = (path) => path.endsWith(".test.ts");
|
||||
// Inputs that only a non-test check reads: declarations (typecheck), guard
|
||||
// baselines (architecture guards, static job) and the advertised-schema
|
||||
// summaries (build job). They select no tests.
|
||||
const isCheckerInput = (path) =>
|
||||
/\.d\.m?ts$/u.test(path) ||
|
||||
/^scripts\/verify\/baselines\/[^/]+\.json$/u.test(path) ||
|
||||
/^scripts\/verify\/advertised-schemas\/tools\/[^/]+\.json$/u.test(path);
|
||||
// Verifier scripts are imported by their tests like source modules. Other
|
||||
// scripts (build, test runners, CI) shape every run and are not traced.
|
||||
const isModule = (path) =>
|
||||
!isCheckerInput(path) &&
|
||||
(/^(src|tests)\/.+\.m?ts$/u.test(path) ||
|
||||
/^scripts\/(verify-[^/]+|verify\/.+)\.mjs$/u.test(path));
|
||||
// Test-only modules pass a change through to the tests that use them.
|
||||
// Production importers never widen the selection transitively.
|
||||
const isHelper = (path) =>
|
||||
!isTest(path) &&
|
||||
(/^tests\/.+\.m?ts$/u.test(path) ||
|
||||
/^src\/.+\.fixture[^/]*\.ts$/u.test(path));
|
||||
// Prose that tests may read by path. Other non-module inputs (fixtures, data,
|
||||
// scripts, workflows, package metadata) are not traced, so they run everything.
|
||||
const isProse = (path) => path.endsWith(".md") && !/^(src|tests)\//u.test(path);
|
||||
|
||||
const SPECIFIER =
|
||||
/(?:\bfrom\s*|\bimport\s*\(\s*|\bimport\s+|\bvi\.mock\(\s*)["'](\.{1,2}\/[^"']+)["']/gu;
|
||||
// TypeScript imports name the emitted `.js`/`.mjs`; scripts are imported as is.
|
||||
const importsOf = (path, source) =>
|
||||
[...source.matchAll(SPECIFIER)].flatMap(([, specifier]) => {
|
||||
const target = posix.normalize(posix.join(posix.dirname(path), specifier));
|
||||
return [target, target.replace(/\.(m?)js$/u, ".$1ts")];
|
||||
});
|
||||
|
||||
const full = (reason) => ({ version: 2, mode: "full", reason, groups: [] });
|
||||
|
||||
/**
|
||||
* Plan the portable tests one CI run executes.
|
||||
*
|
||||
* `snapshot` lists the tracked `src/` and `tests/` paths of the revision under
|
||||
* test and reads them; it is consulted only when a subset is possible.
|
||||
*/
|
||||
export function planTestSelection({ files, event, tier, code, snapshot }) {
|
||||
// `ci:full` selects the full tier, so it also lands here.
|
||||
if (event !== "pull_request" || tier !== "fast")
|
||||
return full("Only ordinary PRs are candidates", files);
|
||||
return full("Main, ci:full and qualification runs execute every test");
|
||||
if (!code) return full("No portable tests run for this change");
|
||||
if (!files.length) return full("No changed paths available");
|
||||
const owners = new Set();
|
||||
const changedTests = [];
|
||||
for (const path of files) {
|
||||
if (!validPath(path))
|
||||
return full(`Shared or unknown input: ${path}`, files);
|
||||
const matches = ownersOf(path);
|
||||
if (!matches.length) return full(`Shared or unknown input: ${path}`, files);
|
||||
for (const owner of matches) owners.add(owner);
|
||||
if (path.endsWith(".test.ts")) changedTests.push(path);
|
||||
if (SHARED_BOUNDARY.test(path)) return full(`Shared boundary: ${path}`);
|
||||
if (!isModule(path) && !isProse(path) && !isCheckerInput(path))
|
||||
return full(`Input is not traced to tests: ${path}`);
|
||||
}
|
||||
|
||||
const tracked = snapshot.list();
|
||||
const trackedSet = new Set(tracked);
|
||||
const tests = tracked.filter(isTest);
|
||||
const helpers = tracked.filter(isHelper);
|
||||
const sources = snapshot.read([...tests, ...helpers]);
|
||||
const imports = new Map(
|
||||
[...tests, ...helpers].map((path) => [
|
||||
path,
|
||||
importsOf(path, sources.get(path) ?? ""),
|
||||
]),
|
||||
);
|
||||
|
||||
const selected = new Map();
|
||||
const select = (path, reason) => {
|
||||
if (trackedSet.has(path) && !selected.has(path)) selected.set(path, reason);
|
||||
};
|
||||
const changedModules = files.filter(
|
||||
(path) => isModule(path) && !isTest(path),
|
||||
);
|
||||
for (const path of files) if (isTest(path)) select(path, "changed");
|
||||
|
||||
// Close over test-only helpers, then take the tests that import a change.
|
||||
const affected = new Set(changedModules);
|
||||
for (let grew = true; grew;) {
|
||||
grew = false;
|
||||
for (const helper of helpers)
|
||||
if (
|
||||
!affected.has(helper) &&
|
||||
imports.get(helper).some((target) => affected.has(target))
|
||||
) {
|
||||
affected.add(helper);
|
||||
grew = true;
|
||||
}
|
||||
}
|
||||
for (const test of tests)
|
||||
if (imports.get(test).some((target) => affected.has(target)))
|
||||
select(test, "imports");
|
||||
|
||||
for (const path of changedModules) {
|
||||
// Same-stem tests, and same-directory tests that scan their directory's
|
||||
// sources by path instead of importing them.
|
||||
const stem = path.replace(/\.m?ts$/u, "");
|
||||
for (const test of tests)
|
||||
if (
|
||||
posix.dirname(test) === posix.dirname(path) &&
|
||||
(test.startsWith(`${stem}.`) ||
|
||||
/\breaddir(?:Sync)?\s*\(/u.test(sources.get(test) ?? ""))
|
||||
)
|
||||
select(test, "beside");
|
||||
// Leaf adapters' boundary tests.
|
||||
const patterns = ownedTests(path);
|
||||
for (const test of tests)
|
||||
if (patterns.some((glob) => matchesGlob(test, glob)))
|
||||
select(test, "owner");
|
||||
}
|
||||
// Tests that name a changed path read it directly.
|
||||
for (const path of files)
|
||||
for (const test of tests)
|
||||
if ((sources.get(test) ?? "").includes(path)) select(test, "names");
|
||||
|
||||
const paths = [...selected.keys()].sort();
|
||||
if (!paths.length)
|
||||
return full("No test imports, names or sits beside a changed path");
|
||||
if (paths.length > SELECTION_CAP)
|
||||
return full(
|
||||
`${paths.length} affected test files exceed the ${SELECTION_CAP}-file selection cap`,
|
||||
);
|
||||
for (const path of paths)
|
||||
try {
|
||||
projectForTest(path);
|
||||
} catch {
|
||||
return full(`No single test project runs ${path}`);
|
||||
}
|
||||
const count = Math.min(MAX_GROUPS, Math.ceil(paths.length / FILES_PER_GROUP));
|
||||
const groups = Array.from({ length: count }, () => []);
|
||||
paths.forEach((path, index) => groups[index % count].push(path));
|
||||
const reasons = Object.groupBy([...selected.values()], (reason) => reason);
|
||||
return {
|
||||
version: 1,
|
||||
mode: "candidate",
|
||||
owners: [...owners].sort(),
|
||||
changedTests: [...new Set(changedTests)].sort(),
|
||||
changedPaths: files,
|
||||
reason:
|
||||
"Observe affected leaf owners; retain integration and unowned tests",
|
||||
version: 2,
|
||||
mode: "selected",
|
||||
reason: `Affected tests: ${Object.entries(reasons)
|
||||
.map(([reason, entries]) => `${entries.length} ${reason}`)
|
||||
.join(", ")}`,
|
||||
groups,
|
||||
};
|
||||
}
|
||||
|
||||
/** Compare a proposal with the full run; missing observations never prove parity. */
|
||||
export function compareTestSelection(plan, inventory, results) {
|
||||
if (
|
||||
plan?.version !== 1 ||
|
||||
!["full", "candidate"].includes(plan.mode) ||
|
||||
!Array.isArray(plan.owners) ||
|
||||
plan.owners.some((owner) => !Object.hasOwn(OWNERS, owner)) ||
|
||||
!Array.isArray(plan.changedTests) ||
|
||||
!Array.isArray(plan.changedPaths) ||
|
||||
plan.changedPaths.some((path) => typeof path !== "string") ||
|
||||
plan.changedTests.some(
|
||||
(path) => !validPath(path) || !path.endsWith(".test.ts"),
|
||||
) ||
|
||||
typeof plan.reason !== "string" ||
|
||||
(plan.mode === "candidate" && !plan.owners.length) ||
|
||||
(plan.mode === "full" && (plan.owners.length || plan.changedTests.length))
|
||||
)
|
||||
throw new Error("Invalid observational test selection plan");
|
||||
const expected = new Set(inventory);
|
||||
if (!expected.size || expected.size !== inventory.length)
|
||||
throw new Error("Test selection requires a nonempty unique inventory");
|
||||
const ownersByFile = new Map(
|
||||
plan.mode === "candidate"
|
||||
? inventory.map((path) => [path, ownersOf(path)])
|
||||
: [],
|
||||
);
|
||||
// Removed/renamed tests have no current executable identity. Keep all tests.
|
||||
const removed = plan.changedTests.filter((path) => !expected.has(path));
|
||||
const missingOwners = plan.owners.filter(
|
||||
(owner) =>
|
||||
![...ownersByFile.values()].some((owners) => owners.includes(owner)),
|
||||
);
|
||||
const effective = removed.length
|
||||
? full(
|
||||
`Changed tests absent from current inventory: ${removed.join(", ")}`,
|
||||
plan.changedPaths,
|
||||
)
|
||||
: missingOwners.length
|
||||
? full(
|
||||
`Owners without executable tests: ${missingOwners.join(", ")}`,
|
||||
plan.changedPaths,
|
||||
)
|
||||
: plan;
|
||||
const selected = [];
|
||||
const omitted = [];
|
||||
for (const path of inventory.toSorted()) {
|
||||
const owners = ownersByFile.get(path) ?? [];
|
||||
const keep =
|
||||
effective.mode === "full" ||
|
||||
!owners.length ||
|
||||
effective.changedTests.includes(path) ||
|
||||
owners.some((owner) => effective.owners.includes(owner));
|
||||
(keep ? selected : omitted).push(path);
|
||||
}
|
||||
const actual = new Set(results.map((file) => file.name));
|
||||
const missing = inventory.filter((path) => !actual.has(path));
|
||||
const unexpected = [...actual].filter((path) => !expected.has(path));
|
||||
const duplicateFiles = results.length - actual.size;
|
||||
const unobserved = results
|
||||
.filter((file) => !["passed", "failed"].includes(file.state))
|
||||
.map((file) => file.name);
|
||||
const omittedSet = new Set(omitted);
|
||||
const missedFailures = results
|
||||
.filter((file) => omittedSet.has(file.name) && file.state === "failed")
|
||||
.map((file) => file.name);
|
||||
return {
|
||||
version: 1,
|
||||
observational: true,
|
||||
plan: effective,
|
||||
selected,
|
||||
omitted,
|
||||
missing,
|
||||
unexpected,
|
||||
duplicateFiles,
|
||||
unobserved,
|
||||
missedFailures,
|
||||
complete:
|
||||
!missing.length &&
|
||||
!unexpected.length &&
|
||||
!duplicateFiles &&
|
||||
!unobserved.length,
|
||||
// Phase totals are scheduling weights, not an estimate of wall-time savings.
|
||||
omittedDurationMs: results
|
||||
.filter((file) => omittedSet.has(file.name))
|
||||
.reduce((sum, file) => sum + file.durationMs, 0),
|
||||
};
|
||||
/** Parse a plan produced by `planTestSelection`; anything else fails closed. */
|
||||
export function parseTestSelection(value) {
|
||||
const plan = typeof value === "string" ? JSON.parse(value) : value;
|
||||
const valid =
|
||||
plan?.version === 2 &&
|
||||
typeof plan.reason === "string" &&
|
||||
Array.isArray(plan.groups) &&
|
||||
(plan.mode === "full"
|
||||
? plan.groups.length === 0
|
||||
: plan.mode === "selected" &&
|
||||
plan.groups.length >= 1 &&
|
||||
plan.groups.length <= MAX_GROUPS &&
|
||||
plan.groups.every(
|
||||
(group) =>
|
||||
Array.isArray(group) &&
|
||||
group.length > 0 &&
|
||||
group.every(
|
||||
(path) =>
|
||||
typeof path === "string" &&
|
||||
isTest(path) &&
|
||||
/^(src|tests)\//u.test(path) &&
|
||||
!path.split("/").some((part) => ["", ".", ".."].includes(part)),
|
||||
),
|
||||
) &&
|
||||
new Set(plan.groups.flat()).size === plan.groups.flat().length &&
|
||||
plan.groups.flat().length <= SELECTION_CAP);
|
||||
if (!valid) throw new Error("Invalid portable test selection plan");
|
||||
return plan;
|
||||
}
|
||||
|
||||
// Fast feedback uses one source group and twelve built groups. Shards are
|
||||
// balanced by recorded duration, but hosted runner speed varies, so the
|
||||
// slowest shard sets the critical path: at eight shards it ran 24% above the
|
||||
// mean shard. Smaller shards bound that excess at a few percent more runner
|
||||
// time. Full qualification retains six complete shards with coverage. A
|
||||
// selection runs one job per group.
|
||||
const shardsOf = (count) =>
|
||||
Array.from({ length: count }, (_, index) => `${index + 1}/${count}`);
|
||||
|
||||
/** The `test-shard` matrix labels for a tier and its test selection. */
|
||||
export const testShards = (tier, plan) =>
|
||||
shardsOf(
|
||||
plan.mode === "selected" ? plan.groups.length : tier === "full" ? 6 : 12,
|
||||
);
|
||||
Reference in new issue
Block a user