ci: run only the affected portable tests on pull requests (#11346)

* ci: run only the affected portable tests on pull requests

Pull-request CI now runs static checks, the build and the portable tests a change affects, selected from test imports (through test-only helpers), same-stem and directory-scanning tests, leaf-owner boundary tests and tests that name a changed path. Shared boundaries, untraced inputs, empty selections and selections above 150 files fall back to the complete fast suite. Main pushes, schedules, releases and ci:full keep the complete suite. The portable report verifies the exact selected inventory and the CI required gate accepts a selected plan only for a fast pull request with code changes.

* ci: own leaf-adapter tests by name and keep subset summaries diagnostic
This commit is contained in:
morluto authored and GitHub committed 2026-11-16 03:37:15 +08:00
1 parent 97112f14a1
commit ff6d1d8142
18 files changed
+901 -577

No files matched your search

+17 -13
View File
@@ -87,6 +87,8 @@ jobs:
website: ${{ steps.plan.outputs.website }}
matrix: ${{ steps.plan.outputs.matrix }}
test_selection: ${{ steps.plan.outputs.test_selection }}
test_mode: ${{ steps.plan.outputs.test_mode }}
test_shards: ${{ steps.plan.outputs.test_shards }}
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
with:
@@ -294,7 +296,8 @@ jobs:
test-source:
name: Source tests
needs: changes
if: needs.changes.outputs.code == 'true' && needs.changes.outputs.tier == 'fast'
# A selected PR subset runs its source tests in the selected test jobs.
if: needs.changes.outputs.code == 'true' && needs.changes.outputs.tier == 'fast' && needs.changes.outputs.test_mode == 'full'
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
@@ -330,14 +333,10 @@ jobs:
# every shard's diagnostics and coverage. Passing PRs run the full suite.
fail-fast: ${{ github.event_name == 'pull_request' && needs.changes.outputs.tier == 'fast' }}
matrix:
# Fast feedback uses one source group and twelve built groups. Shards
# are balanced by recorded duration, but hosted runner speed varies,
# so the slowest shard sets the critical path: at eight shards it ran
# 24% above the mean shard. Smaller shards bound that excess at a few
# percent more runner time. Classification, build, and report merge
# sit outside these jobs. Full qualification retains six complete
# shards with coverage.
shard: ${{ fromJSON(needs.changes.outputs.tier == 'fast' && '["1/12","2/12","3/12","4/12","5/12","6/12","7/12","8/12","9/12","10/12","11/12","12/12"]' || '["1/6","2/6","3/6","4/6","5/6","6/6"]') }}
# scripts/ci/test-selection.mjs owns the shard counts: twelve fast
# built shards, six full-qualification shards with coverage, or one
# job per group of a selected PR subset.
shard: ${{ fromJSON(needs.changes.outputs.test_shards) }}
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
@@ -357,11 +356,17 @@ jobs:
env:
REA_CI_TEST_HISTORY: .cache/ci-test-history/test-results.json
- name: Run built-dependent tests for fast feedback
if: needs.changes.outputs.tier == 'fast'
if: needs.changes.outputs.tier == 'fast' && needs.changes.outputs.test_mode == 'full'
run: npm run test:built -- --reporter=default --reporter=blob --shard=${{ matrix.shard }}
env:
VITEST_BLOB_LABEL: built
REA_CI_TEST_HISTORY: .cache/ci-test-history/test-results.json
- name: Run the tests this pull request affects
if: needs.changes.outputs.test_mode == 'selected'
run: node scripts/ci/test-group.mjs selected "${MATRIX_SHARD}" --reporter=default --reporter=blob
env:
MATRIX_SHARD: ${{ matrix.shard }}
CI_TEST_SELECTION: ${{ needs.changes.outputs.test_selection }}
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
if: ${{ !cancelled() }}
with:
@@ -608,7 +613,7 @@ jobs:
if: needs.changes.outputs.code == 'true' && !cancelled()
env:
SHARD_RESULT: ${{ needs.test-shard.result }}
SOURCE_REQUIRED: ${{ needs.changes.outputs.tier == 'fast' }}
SOURCE_REQUIRED: ${{ needs.changes.outputs.tier == 'fast' && needs.changes.outputs.test_mode == 'full' }}
SOURCE_RESULT: ${{ needs.test-source.result }}
MERGE_RESULT: ${{ needs.changes.outputs.tier == 'full' && steps.merge.outcome || steps.merge-fast.outcome }}
CI_VERIFY_TEST_INVENTORY: "true"
@@ -624,7 +629,6 @@ jobs:
test-results.json
test-timings.json
coverage/
.cache/ci-test-selection/report.json
include-hidden-files: true
if-no-files-found: error
retention-days: 30
@@ -653,7 +657,7 @@ jobs:
env:
CODE_REQUIRED: ${{ needs.changes.outputs.code }}
SHARD_RESULT: ${{ needs.test-shard.result }}
SOURCE_REQUIRED: ${{ needs.changes.outputs.tier == 'fast' }}
SOURCE_REQUIRED: ${{ needs.changes.outputs.tier == 'fast' && needs.changes.outputs.test_mode == 'full' }}
SOURCE_RESULT: ${{ needs.test-source.result }}
run: |
if [ "${CODE_REQUIRED}" = "true" ]; then
+56
View File
@@ -1299,6 +1299,62 @@ and workspace rather than reporting successful removal. The controlled
`packageUpdateRegistryFailure` regression needs built process modules, uses no
public registry, and checks both the primary failure and live descendant absence.
## Pull-request test selection
CI on a pull request runs static checks, the build and the portable tests the
change affects. The complete portable suite runs on every push to `main`, on
the scheduled and manual qualification runs, in release qualification, and on a
PR labelled `ci:full`. Those complete runs catch a consumer that selection
missed, after it has merged. Add `ci:full` to a PR when its effect reaches
further than its imports show, such as runtime registration, generated files,
subprocesses or bridges. Adding the label starts full qualification at once,
and later pushes stay on the full tier while the label remains.
`scripts/ci/test-selection.mjs` owns the selection. It reads the checked-out
revision and selects:
- changed test files;
- tests that import a changed module, directly or through test-only helpers
(non-test modules under `tests/` and `src/**/*.fixture*.ts`). Selection never
follows production importers transitively;
- tests beside a changed module that share its stem, or that scan their
directory with `readdir`;
- the boundary tests of a changed leaf adapter (the owner table in the same
file), which reach their sources through the built CLI, the MCP server or a
fake provider process;
- tests whose text names a changed path.
Verifier scripts (`scripts/verify-*.mjs`, `scripts/verify/**/*.mjs`) are traced
like source modules. Changed Markdown outside `src/` and `tests/` selects only
the tests that name it. Declarations, guard baselines
(`scripts/verify/baselines/*.json`) and advertised-schema summaries select no
tests, because the static and build jobs check them.
The PR falls back to the complete fast suite when it cannot decide:
- a shared boundary (`src/process/`, `src/filesystem/`, `src/contracts/tool*`,
`src/server/createServer*`, `src/cli.ts`, `scripts/ci/`) changed;
- any other changed input is not traced, such as workflows, package metadata,
build and test-runner scripts, `scripts/lib/`, fixtures or data;
- no test is selected;
- more than 150 test files are selected;
- a selected file has no single owning test project.
A selection runs as one `Test shard (k/n)` job per group of up to 40 files, at
most four jobs, using exact file identity (`scripts/ci/test-group.mjs selected`).
The separate `Source tests` job does not run, because the groups include the
selected source tests. The `Classify changes` summary gives the mode and its
reason.
The required checks still report on every PR and fail closed:
- `Portable tests` checks that the merged report contains exactly the selected
files: none missing, none unplanned, no duplicates.
- `CI required` accepts a selected plan only for a fast `pull_request` run with
code changes, and then requires `build` and every selected test group.
- A missing or malformed plan fails classification, the test group and the gate.
- A selected run never publishes timing history.
## Real-toolchain verification lanes
See [Real-toolchain verification lanes](../testing-lanes.md#real-toolchain-verification-lanes).
+10 -30
View File
@@ -631,30 +631,13 @@ generated-document validation, including the small onboarding example consistenc
suite, without the source-test shards or native package
lanes. Classification compares the PR head with its merge base, so later
base-branch changes do not expand that scope. Static and portable-test aggregate jobs
remain present and fail if classification fails. Every code change retains the
complete portable Linux suite; individual test files are not selected by
imports or filenames.
remain present and fail if classification fails.
Ordinary PRs also produce an **observational** test-selection plan. The candidate
leaf-adapter ownership table lives in `scripts/ci/test-selection.mjs`, separately
from provider-job scopes. It proposes keeping affected owners plus every unowned
test, including all CLI/MCP, acceptance, composition, conformance, and process
tests. Shared inputs, unknown paths, missing owner tests, and removed tests fall
back to the complete inventory. Renames account for both paths. Main, scheduled,
manual, and `ci:full` runs always propose the full suite.
The source group and every built shard together still execute the complete suite. The existing report merge
compares the proposal with actual module outcomes and retains
`.cache/ci-test-selection/report.json` in `test-results`, including selected and
omitted paths, changed input paths, missed failures, unobserved files, scheduling weights, PR head and
executed merge SHA. This adds no test reruns or runner jobs. Missing, duplicate,
skipped, interrupted, or unhandled results cannot establish a complete comparison;
even a complete passing comparison is not proof of ownership completeness.
Before enabling omissions, review representative full-run comparisons and
cross-owner consumers, including failures, and measure completed CI feedback and
runner demand. There is no automatic promotion or switch to skip tests. The
initial proposal measures only leaf producer-test omissions; it does not promise
the savings of a broader affected-test planner.
Pull requests with code changes run only the portable tests the change
affects; main pushes, scheduled and manual runs, release qualification and
`ci:full` run the complete suite. See
[pull-request test selection](development/testing.md#pull-request-test-selection)
for the selection rules, fallbacks and gate.
`scripts/ci/plan.mjs` selects lanes using the ownership table in
`scripts/ci/scopes.mjs`. Each lane record owns its path and test selection
@@ -739,13 +722,14 @@ push: changes from failed or cancelled intermediate runs remain in scope.
Missing, untrusted, or non-ancestor history selects the fast Linux x64
baseline: portable checks and the Linux runtime lane. The lookup is
bounded to the last 100 successful push runs; lookup failures broaden selection.
Every code change still selects the source group and every fast built shard, without coverage
Pull requests run their selected tests, or the complete suite when selection
falls back; main pushes run the source group and every fast built shard, without coverage
instrumentation in this fast tier. Ordinary PR matrices cancel remaining
shards after the first failure; passing PRs execute the complete suite. Main,
shards after the first failure. Main,
`ci:full`, scheduled, manual, and release qualification retain every shard's
results after a failure. Available failed-shard reports remain diagnostic only:
failed, cancelled, skipped, or missing selected shards cannot pass the gate or
publish successful timing history. This deliberately avoids relying on an
publish successful timing history. Complete-suite runs do not rely on an
import graph for generated files, filesystem inputs, subprocesses, or bridges.
The portable report rejects an entirely skipped module inventory and lists
fully skipped modules in its summary. A mixed passed/skipped inventory is
@@ -930,10 +914,6 @@ Main warms the new cache namespace after merge; a cold cache falls back to the
normal install. This prevents each PR from saving another branch-scoped copy of
the same download cache. It does not remove runner assignment waits.
Portable owner-based test selection remains observational. Its reports compare
proposed omissions with the full run; incomplete or skipped observations do not
establish that fewer tests would preserve regression detection.
The automatic browser, binary-layout, EVM, JavaScript-recovery, packet-capture,
signal-recording, recorded-crash, web and environment-bundle provider lanes download
`rea-build-cache` from their own run after the build and static jobs succeed. Their existing build commands still
+9
View File
@@ -15,6 +15,14 @@ export interface ChangePlanRepository {
readonly isAncestor: (ancestor: string, head: string) => boolean;
readonly changedFiles: (base: string, head: string) => readonly string[];
readonly readFile: (revision: string, path: string) => string;
readonly listFiles: (
revision: string,
directories: readonly string[],
) => readonly string[];
readonly readFiles: (
revision: string,
paths: readonly string[],
) => ReadonlyMap<string, string>;
}
export type ChangePlan = Readonly<Record<string, unknown>> & {
@@ -29,6 +37,7 @@ export type ChangePlan = Readonly<Record<string, unknown>> & {
readonly files: readonly string[];
readonly reasons: readonly string[];
readonly testSelection: TestSelectionPlan;
readonly testShards: readonly string[];
};
export function planChanges(
+50 -13
View File
@@ -5,9 +5,10 @@ import {
CI_FLAGS as flags,
MANUAL_VERIFIERS,
CI_LANES,
SHARED_BOUNDARY,
WINDOWS_EXECUTABLE_SELECTION_INPUTS,
} from "./scopes.mjs";
import { planTestSelection } from "./test-selection.mjs";
import { planTestSelection, testShards } from "./test-selection.mjs";
// Actionlint validates workflow syntax only. The semantic suite asserts these
// workflows' release, publication and self-hosted runner safeguards.
@@ -58,11 +59,6 @@ const matchesAny = (file, patterns) =>
export function planChanges(request, repository) {
const selected = Object.fromEntries(flags.map((flag) => [flag, false]));
const reasons = [];
// Shared process, contract, CLI, and CI-script edits are not a specific provider.
// Fast qualification keeps the portable suite and the Linux x64 runtime
// baseline. Full qualification still selects every lane.
const SHARED_BOUNDARY =
/^(src\/process\/|src\/filesystem\/|src\/contracts\/tool|src\/server\/createServer|src\/cli\.ts|scripts\/ci\/)/;
const full = (reason) => {
if (tier === "full") {
for (const flag of flags) selected[flag] = true;
@@ -269,11 +265,7 @@ export function planChanges(request, repository) {
selected.readiness = true;
}
if (file.startsWith("tests/")) return;
if (
/^(src\/process\/|src\/filesystem\/|src\/contracts\/tool|src\/server\/createServer|src\/cli\.ts|scripts\/ci\/)/.test(
file,
)
) {
if (SHARED_BOUNDARY.test(file)) {
full(`Shared boundary: ${file}`);
} else if (owners.length) {
if (file.startsWith("src/") || file.startsWith("bridge/"))
@@ -343,8 +335,26 @@ export function planChanges(request, repository) {
]
: [linux],
};
const testSelection = planTestSelection(files, event, tier);
return { ...selected, tier, matrix, files, reasons, testSelection };
const testSelection = planTestSelection({
files,
event,
tier,
code: selected.code,
// The checked-out revision is the one whose tests run.
snapshot: {
list: () => repository.listFiles("HEAD", ["src", "tests"]),
read: (paths) => repository.readFiles("HEAD", paths),
},
});
return {
...selected,
tier,
matrix,
files,
reasons,
testSelection,
testShards: testShards(tier, testSelection),
};
}
/** Answer the plan's Git questions from the repository in `cwd`. */
@@ -370,5 +380,32 @@ export const gitRepository = (cwd = process.cwd()) => {
.split("\0")
.filter(Boolean),
readFile: (revision, path) => git("show", `${revision}:${path}`),
listFiles: (revision, directories) =>
git("ls-tree", "-r", "-z", "--name-only", revision, "--", ...directories)
.split("\0")
.filter(Boolean),
// One batch process reads every requested blob.
readFiles: (revision, paths) => {
const output = execFileSync("git", ["cat-file", "--batch"], {
cwd,
input: paths.map((path) => `${revision}:${path}\n`).join(""),
maxBuffer: 512 * 1024 * 1024,
});
const contents = new Map();
let offset = 0;
for (const path of paths) {
const end = output.indexOf(10, offset);
const [, type, size] = output.toString("utf8", offset, end).split(" ");
if (type !== "blob")
throw new Error(`Git could not read ${revision}:${path}`);
const start = end + 1;
contents.set(
path,
output.toString("utf8", start, start + Number(size)),
);
offset = start + Number(size) + 1;
}
return contents;
},
};
};
+4 -2
View File
@@ -14,7 +14,7 @@ const plan = planChanges(
},
gitRepository(),
);
const { tier, matrix, reasons, testSelection } = plan;
const { tier, matrix, reasons, testSelection, testShards } = plan;
const selected = Object.fromEntries(flags.map((flag) => [flag, plan[flag]]));
if (process.env.GITHUB_OUTPUT)
appendFileSync(
@@ -24,6 +24,8 @@ if (process.env.GITHUB_OUTPUT)
tier,
matrix: JSON.stringify(matrix),
test_selection: JSON.stringify(testSelection),
test_mode: testSelection.mode,
test_shards: JSON.stringify(testShards),
})
.map(([key, value]) => `${key}=${value}\n`)
.join(""),
@@ -31,6 +33,6 @@ if (process.env.GITHUB_OUTPUT)
if (process.env.GITHUB_STEP_SUMMARY)
appendFileSync(
process.env.GITHUB_STEP_SUMMARY,
`### Selected CI lanes (${tier})\n\n${flags.filter((flag) => selected[flag]).join(", ") || "Formatting only"}\n\n${reasons.join("\n\n")}\n`,
`### Selected CI lanes (${tier})\n\n${flags.filter((flag) => selected[flag]).join(", ") || "Formatting only"}\n\n${reasons.join("\n\n")}\n\nPortable tests (${testSelection.mode}): ${testSelection.reason}\n`,
);
console.log(JSON.stringify(plan));
+16 -1
View File
@@ -28,10 +28,25 @@ if (
);
if (outputs.tier === "full" && selectedScopes.length !== CI_FLAGS.length)
throw new Error("Full qualification requires every canonical CI scope");
// Only a fast pull-request plan may run a selected test subset.
const testMode = outputs.test_mode;
if (
!["full", "selected"].includes(testMode) ||
(testMode === "selected" &&
(outputs.tier !== "fast" ||
process.env.GITHUB_EVENT_NAME !== "pull_request" ||
outputs.code !== "true"))
)
throw new Error(
"CI classification requires a valid test mode: full, or selected on a fast pull request",
);
const required = reused ? ["changes"] : ["changes", "static", "test"];
if (outputs.tier === "full") required.push("static-deep");
for (const scope of selectedScopes)
required.push(...CI_LANES[scope].required[outputs.tier]);
required.push(
...((testMode === "selected" && CI_LANES[scope].selectedTests) ||
CI_LANES[scope].required[outputs.tier]),
);
const failed = [...new Set(required)].filter(
(name) => jobs[name]?.result !== "success",
);
+9
View File
@@ -8,6 +8,13 @@ export const WINDOWS_EXECUTABLE_SELECTION_INPUTS = [
"tests/boundary/process/windowsExecutableSelection.test.ts",
];
// Shared process, contract, CLI, and CI-script edits are not a specific provider.
// Fast qualification keeps the portable suite and the Linux x64 runtime
// baseline, and pull requests run the complete portable suite rather than a
// selected subset. Full qualification still selects every lane.
export const SHARED_BOUNDARY =
/^(src\/process\/|src\/filesystem\/|src\/contracts\/tool|src\/server\/createServer|src\/cli\.ts|scripts\/ci\/)/;
// Inputs whose changes must select both real capture lanes, even at shared boundaries.
const CAPTURE_SHARED_INPUTS = [
"src/process/LinuxResourceLimits*",
@@ -679,6 +686,8 @@ export const CI_LANES = {
fast: ["build", "test-shard", "test-source"],
full: ["build", "test-shard"],
},
// A selected PR subset runs its source tests in the test-shard groups.
selectedTests: ["build", "test-shard"],
},
runtime: sameJobs([], ["windows-native-build", "package-e2e"]),
package: sameJobs(PATH_SCOPES.package, ["windows-curated"], {
+38 -10
View File
@@ -5,17 +5,48 @@ import {
BUILT_TEST_PROJECTS,
SOURCE_TEST_PROJECTS,
} from "../lib/test-projects.mjs";
import { parseTestSelection, testShards } from "./test-selection.mjs";
const [group, ...arguments_] = process.argv.slice(2);
const projects = { source: SOURCE_TEST_PROJECTS, built: BUILT_TEST_PROJECTS }[
group
];
if (!projects) throw new Error("Expected source or built test group");
// The group owns project selection; callers can request reporters or shards.
// The group owns project and file selection; callers can request reporters
// or (for project groups) shards.
if (
arguments_.some((argument) => /^--(?:project|exclude)(?:=|$)/u.test(argument))
)
throw new Error("Test groups cannot override their project inventory");
let runner;
let environment = process.env;
if (group === "selected") {
// `selected <k/n>` runs group k of the plan's selected test files, by exact
// identity, in whichever projects own them.
const [shard, ...rest] = arguments_;
const plan = parseTestSelection(process.env.CI_TEST_SELECTION ?? "");
if (plan.mode !== "selected" || !testShards("fast", plan).includes(shard))
throw new Error(`No selected test group ${shard ?? ""} in this plan`);
if (rest.some((argument) => /^--shard(?:=|$)/u.test(argument)))
throw new Error("Selected test groups are already sharded");
const index = Number(shard.split("/")[0]) - 1;
runner = [
resolve("scripts/run-exact-tests.mjs"),
"run",
...rest,
...plan.groups[index],
];
// Each group writes its own blob report for the merge.
environment = { ...process.env, VITEST_BLOB_LABEL: `selected-${index + 1}` };
} else {
const projects = {
source: SOURCE_TEST_PROJECTS,
built: BUILT_TEST_PROJECTS,
}[group];
if (!projects) throw new Error("Expected source, built or selected tests");
runner = [
resolve("node_modules/vitest/vitest.mjs"),
"run",
...projects.flatMap(({ name }) => ["--project", name]),
...arguments_,
];
}
const child = spawn(
process.execPath,
[
@@ -24,12 +55,9 @@ const child = spawn(
...(process.platform === "win32"
? []
: [resolve("scripts/run-exclusive.mjs"), "--unlocked", process.execPath]),
resolve("node_modules/vitest/vitest.mjs"),
"run",
...projects.flatMap(({ name }) => ["--project", name]),
...arguments_,
...runner,
],
{ stdio: "inherit" },
{ stdio: "inherit", env: environment },
);
let shutdownSignal;
for (const signal of ["SIGINT", "SIGTERM"])
+19 -41
View File
@@ -4,16 +4,13 @@ import { isAbsolute, join, relative, resolve, sep } from "node:path";
import { z } from "zod";
import { compareTestSelection } from "./test-selection.mjs";
import { parseTestSelection } from "./test-selection.mjs";
const root = process.cwd();
const historyDirectory = join(root, ".cache/ci-test-history");
const historyPath = join(historyDirectory, "test-results.json");
const selectionDirectory = join(root, ".cache/ci-test-selection");
const selectionPath = join(selectionDirectory, "report.json");
// Never let a failed or partial rerun leave a previous successful history file.
await rm(historyPath, { force: true });
await rm(selectionPath, { force: true });
const report = z
.object({
version: z.literal(1),
@@ -56,40 +53,23 @@ const passed =
process.env.MERGE_RESULT === "success" &&
report.success;
// Successful shards and merged reports must account for the complete portable
// inventory. Coverage thresholds alone cannot prove that every file ran.
// inventory, or exactly the selected subset. Coverage thresholds alone cannot
// prove that every file ran.
const verifyInventory = process.env.CI_VERIFY_TEST_INVENTORY === "true";
const expected =
verifyInventory || process.env.CI_TEST_SELECTION
? execFileSync("git", ["ls-files", "-z", "--", "src", "tests"], {
const selection = process.env.CI_TEST_SELECTION
? parseTestSelection(process.env.CI_TEST_SELECTION)
: undefined;
const selected = selection?.mode === "selected";
const expected = !verifyInventory
? []
: selected
? selection.groups.flat()
: execFileSync("git", ["ls-files", "-z", "--", "src", "tests"], {
encoding: "utf8",
maxBuffer: 8 * 1024 * 1024,
})
.split("\0")
.filter((path) => path.endsWith(".test.ts"))
: [];
let selection;
if (process.env.CI_TEST_SELECTION) {
const comparison = compareTestSelection(
JSON.parse(process.env.CI_TEST_SELECTION),
expected,
files,
);
selection = {
...comparison,
complete:
comparison.complete &&
report.unhandledErrors === 0 &&
["passed", "failed"].includes(report.reason) &&
groupsCompleted &&
["success", "failure"].includes(process.env.MERGE_RESULT),
executionSha: process.env.GITHUB_SHA ?? null,
prHeadSha: process.env.CI_PR_HEAD_SHA ?? null,
runId: process.env.GITHUB_RUN_ID ?? null,
runAttempt: process.env.GITHUB_RUN_ATTEMPT ?? null,
};
await mkdir(selectionDirectory, { recursive: true });
await writeFile(selectionPath, JSON.stringify(selection));
}
.filter((path) => path.endsWith(".test.ts"));
if (passed && verifyInventory) {
const actual = files.map(({ name }) => name);
if (actual.length === 0)
@@ -100,11 +80,13 @@ if (passed && verifyInventory) {
const unexpected = actual.filter((path) => !expectedSet.has(path));
if (missing.length || unexpected.length || actualSet.size !== actual.length)
throw new Error(
`Incomplete portable test inventory: ${JSON.stringify({ missing, unexpected, duplicateFiles: actual.length - actualSet.size })}`,
`Incomplete ${selected ? "selected" : "portable"} test inventory: ${JSON.stringify({ missing, unexpected, duplicateFiles: actual.length - actualSet.size })}`,
);
if (files.every(({ state }) => state === "skipped"))
throw new Error("Every portable test module was skipped");
}
// A selected subset is not a timing baseline for the complete suite.
const publishesHistory = passed && !selected;
const skippedFiles = files.filter(({ state }) => state === "skipped");
const summary = [
groupsCompleted
@@ -113,11 +95,7 @@ const summary = [
"",
...(selection
? [
"### Observational PR test selection",
"",
`Plan: ${selection.plan.mode}. Proposed files: ${selection.selected.length}; omitted: ${selection.omitted.length}.`,
`Full-run comparison: ${selection.complete ? "complete" : "incomplete"}; failures outside proposed selection: ${selection.missedFailures.length}.`,
"This observational report cannot enable omissions or qualify a revision. Skipped, missing, and interrupted observations do not establish parity.",
`Portable tests: ${selected ? `${selection.groups.flat().length} selected files` : "complete suite"}. ${selection.reason}.`,
"",
]
: []),
@@ -129,7 +107,7 @@ const summary = [
...(skippedFiles.length ? [""] : []),
...skippedFiles.map(({ name }) => `- ${name.replaceAll("|", "\\|")}`),
"",
passed
publishesHistory
? "Successful timing history is available."
: "This report is diagnostic only and is not a successful timing baseline.",
"",
@@ -150,7 +128,7 @@ if (process.env.GITHUB_STEP_SUMMARY)
await appendFile(process.env.GITHUB_STEP_SUMMARY, summary);
else console.log(summary);
if (passed) {
if (publishesHistory) {
await mkdir(historyDirectory, { recursive: true });
await writeFile(
historyPath,
+25 -35
View File
@@ -1,35 +1,25 @@
export interface TestSelectionPlan {
readonly version: 1;
readonly mode: "full" | "candidate";
readonly owners: readonly string[];
readonly changedTests: readonly string[];
readonly changedPaths: readonly string[];
readonly reason: string;
}
export function planTestSelection(
files: readonly string[],
event: string,
tier: string,
): TestSelectionPlan;
export function compareTestSelection(
plan: unknown,
inventory: readonly string[],
results: readonly {
readonly name: string;
readonly durationMs: number;
readonly state?: string;
}[],
): {
readonly version: 1;
readonly observational: true;
readonly plan: TestSelectionPlan;
readonly selected: readonly string[];
readonly omitted: readonly string[];
readonly missing: readonly string[];
readonly unexpected: readonly string[];
readonly duplicateFiles: number;
readonly unobserved: readonly string[];
readonly missedFailures: readonly string[];
readonly complete: boolean;
readonly omittedDurationMs: number;
};
export type TestSelectionPlan =
| {
readonly version: 2;
readonly mode: "full";
readonly reason: string;
readonly groups: readonly [];
}
| {
readonly version: 2;
readonly mode: "selected";
readonly reason: string;
readonly groups: readonly (readonly string[])[];
};
export function planTestSelection(request: {
readonly files: readonly string[];
readonly event: string;
readonly tier: string;
readonly code: boolean;
readonly snapshot: {
readonly list: () => readonly string[];
readonly read: (paths: readonly string[]) => ReadonlyMap<string, string>;
};
}): TestSelectionPlan;
export function parseTestSelection(value: unknown): TestSelectionPlan;
export function testShards(tier: string, plan: TestSelectionPlan): string[];
+213 -140
View File
@@ -1,151 +1,224 @@
import { matchesGlob } from "node:path";
import { matchesGlob, posix } from "node:path";
// Candidate ownership only: these leaf adapters and their producer tests can be
// compared with full-suite outcomes. This table never changes test execution.
// Shared application/domain code and dynamic CLI/MCP consumers are not leaf owners.
const OWNERS = {
adb: ["src/adb/**", "tests/boundary/providers/adb/**"],
apktool: ["src/apktool/**"],
dotnet: ["src/dotnet/**"],
evm: ["src/evm/**", "tests/boundary/evm/**"],
firmware: ["src/firmware/**", "tests/boundary/firmware/**"],
flutter: ["src/flutter/**"],
ghidra: ["src/ghidra/**", "tests/boundary/providers/ghidra/**"],
hopper: ["src/hopper/**", "tests/boundary/providers/hopper/**"],
ida: ["src/ida/**", "tests/boundary/providers/ida/**"],
jeb: ["src/jeb/**"],
pcap: ["src/pcap/**", "tests/boundary/pcap/**"],
sqlite: ["src/sqlite/**", "tests/boundary/sqlite/**"],
wasm: ["src/wasm/**"],
};
const ownersOf = (path) =>
Object.entries(OWNERS)
.filter(([, patterns]) => patterns.some((glob) => matchesGlob(path, glob)))
.map(([owner]) => owner);
const validPath = (path) =>
typeof path === "string" &&
/^(src|tests)\//.test(path) &&
!/[\\\0\r\n]/.test(path) &&
!path.split("/").some((part) => ["", ".", ".."].includes(part));
const full = (reason, changedPaths = []) => ({
version: 1,
mode: "full",
owners: [],
changedTests: [],
changedPaths,
reason,
});
import { projectForTest } from "../lib/test-projects.mjs";
import { SHARED_BOUNDARY } from "./scopes.mjs";
/** Plan an observational PR subset from the classifier's existing Git diff. */
export function planTestSelection(files, event, tier) {
// Pull-request feedback runs the tests a change most directly affects. Main
// pushes, schedules, releases and `ci:full` keep the complete suite, so a
// consumer the selection misses fails on main right after it lands.
// Selection falls back to the complete suite whenever it cannot decide.
/** More selected files than this run the complete suite instead. */
const SELECTION_CAP = 150;
/** Selected files per test job, and the most jobs one selection uses. */
const FILES_PER_GROUP = 40;
const MAX_GROUPS = 4;
// Leaf adapters whose tests also reach their sources through the built CLI,
// MCP server or a fake provider process rather than an import. An adapter
// owns `src/<name>/` and every test named for it (`<name>*.test.ts`) or
// filed under a `<name>/` directory anywhere in `tests/`.
const OWNERS = [
"adb",
"apktool",
"dotnet",
"evm",
"firmware",
"flutter",
"ghidra",
"hopper",
"ida",
"jeb",
"pcap",
"sqlite",
"wasm",
];
const ownedTests = (path) =>
OWNERS.filter((owner) => path.startsWith(`src/${owner}/`)).flatMap(
(owner) => [`tests/**/${owner}/**/*.test.ts`, `tests/**/${owner}*.test.ts`],
);
const isTest = (path) => path.endsWith(".test.ts");
// Inputs that only a non-test check reads: declarations (typecheck), guard
// baselines (architecture guards, static job) and the advertised-schema
// summaries (build job). They select no tests.
const isCheckerInput = (path) =>
/\.d\.m?ts$/u.test(path) ||
/^scripts\/verify\/baselines\/[^/]+\.json$/u.test(path) ||
/^scripts\/verify\/advertised-schemas\/tools\/[^/]+\.json$/u.test(path);
// Verifier scripts are imported by their tests like source modules. Other
// scripts (build, test runners, CI) shape every run and are not traced.
const isModule = (path) =>
!isCheckerInput(path) &&
(/^(src|tests)\/.+\.m?ts$/u.test(path) ||
/^scripts\/(verify-[^/]+|verify\/.+)\.mjs$/u.test(path));
// Test-only modules pass a change through to the tests that use them.
// Production importers never widen the selection transitively.
const isHelper = (path) =>
!isTest(path) &&
(/^tests\/.+\.m?ts$/u.test(path) ||
/^src\/.+\.fixture[^/]*\.ts$/u.test(path));
// Prose that tests may read by path. Other non-module inputs (fixtures, data,
// scripts, workflows, package metadata) are not traced, so they run everything.
const isProse = (path) => path.endsWith(".md") && !/^(src|tests)\//u.test(path);
const SPECIFIER =
/(?:\bfrom\s*|\bimport\s*\(\s*|\bimport\s+|\bvi\.mock\(\s*)["'](\.{1,2}\/[^"']+)["']/gu;
// TypeScript imports name the emitted `.js`/`.mjs`; scripts are imported as is.
const importsOf = (path, source) =>
[...source.matchAll(SPECIFIER)].flatMap(([, specifier]) => {
const target = posix.normalize(posix.join(posix.dirname(path), specifier));
return [target, target.replace(/\.(m?)js$/u, ".$1ts")];
});
const full = (reason) => ({ version: 2, mode: "full", reason, groups: [] });
/**
* Plan the portable tests one CI run executes.
*
* `snapshot` lists the tracked `src/` and `tests/` paths of the revision under
* test and reads them; it is consulted only when a subset is possible.
*/
export function planTestSelection({ files, event, tier, code, snapshot }) {
// `ci:full` selects the full tier, so it also lands here.
if (event !== "pull_request" || tier !== "fast")
return full("Only ordinary PRs are candidates", files);
return full("Main, ci:full and qualification runs execute every test");
if (!code) return full("No portable tests run for this change");
if (!files.length) return full("No changed paths available");
const owners = new Set();
const changedTests = [];
for (const path of files) {
if (!validPath(path))
return full(`Shared or unknown input: ${path}`, files);
const matches = ownersOf(path);
if (!matches.length) return full(`Shared or unknown input: ${path}`, files);
for (const owner of matches) owners.add(owner);
if (path.endsWith(".test.ts")) changedTests.push(path);
if (SHARED_BOUNDARY.test(path)) return full(`Shared boundary: ${path}`);
if (!isModule(path) && !isProse(path) && !isCheckerInput(path))
return full(`Input is not traced to tests: ${path}`);
}
const tracked = snapshot.list();
const trackedSet = new Set(tracked);
const tests = tracked.filter(isTest);
const helpers = tracked.filter(isHelper);
const sources = snapshot.read([...tests, ...helpers]);
const imports = new Map(
[...tests, ...helpers].map((path) => [
path,
importsOf(path, sources.get(path) ?? ""),
]),
);
const selected = new Map();
const select = (path, reason) => {
if (trackedSet.has(path) && !selected.has(path)) selected.set(path, reason);
};
const changedModules = files.filter(
(path) => isModule(path) && !isTest(path),
);
for (const path of files) if (isTest(path)) select(path, "changed");
// Close over test-only helpers, then take the tests that import a change.
const affected = new Set(changedModules);
for (let grew = true; grew;) {
grew = false;
for (const helper of helpers)
if (
!affected.has(helper) &&
imports.get(helper).some((target) => affected.has(target))
) {
affected.add(helper);
grew = true;
}
}
for (const test of tests)
if (imports.get(test).some((target) => affected.has(target)))
select(test, "imports");
for (const path of changedModules) {
// Same-stem tests, and same-directory tests that scan their directory's
// sources by path instead of importing them.
const stem = path.replace(/\.m?ts$/u, "");
for (const test of tests)
if (
posix.dirname(test) === posix.dirname(path) &&
(test.startsWith(`${stem}.`) ||
/\breaddir(?:Sync)?\s*\(/u.test(sources.get(test) ?? ""))
)
select(test, "beside");
// Leaf adapters' boundary tests.
const patterns = ownedTests(path);
for (const test of tests)
if (patterns.some((glob) => matchesGlob(test, glob)))
select(test, "owner");
}
// Tests that name a changed path read it directly.
for (const path of files)
for (const test of tests)
if ((sources.get(test) ?? "").includes(path)) select(test, "names");
const paths = [...selected.keys()].sort();
if (!paths.length)
return full("No test imports, names or sits beside a changed path");
if (paths.length > SELECTION_CAP)
return full(
`${paths.length} affected test files exceed the ${SELECTION_CAP}-file selection cap`,
);
for (const path of paths)
try {
projectForTest(path);
} catch {
return full(`No single test project runs ${path}`);
}
const count = Math.min(MAX_GROUPS, Math.ceil(paths.length / FILES_PER_GROUP));
const groups = Array.from({ length: count }, () => []);
paths.forEach((path, index) => groups[index % count].push(path));
const reasons = Object.groupBy([...selected.values()], (reason) => reason);
return {
version: 1,
mode: "candidate",
owners: [...owners].sort(),
changedTests: [...new Set(changedTests)].sort(),
changedPaths: files,
reason:
"Observe affected leaf owners; retain integration and unowned tests",
version: 2,
mode: "selected",
reason: `Affected tests: ${Object.entries(reasons)
.map(([reason, entries]) => `${entries.length} ${reason}`)
.join(", ")}`,
groups,
};
}
/** Compare a proposal with the full run; missing observations never prove parity. */
export function compareTestSelection(plan, inventory, results) {
if (
plan?.version !== 1 ||
!["full", "candidate"].includes(plan.mode) ||
!Array.isArray(plan.owners) ||
plan.owners.some((owner) => !Object.hasOwn(OWNERS, owner)) ||
!Array.isArray(plan.changedTests) ||
!Array.isArray(plan.changedPaths) ||
plan.changedPaths.some((path) => typeof path !== "string") ||
plan.changedTests.some(
(path) => !validPath(path) || !path.endsWith(".test.ts"),
) ||
typeof plan.reason !== "string" ||
(plan.mode === "candidate" && !plan.owners.length) ||
(plan.mode === "full" && (plan.owners.length || plan.changedTests.length))
)
throw new Error("Invalid observational test selection plan");
const expected = new Set(inventory);
if (!expected.size || expected.size !== inventory.length)
throw new Error("Test selection requires a nonempty unique inventory");
const ownersByFile = new Map(
plan.mode === "candidate"
? inventory.map((path) => [path, ownersOf(path)])
: [],
);
// Removed/renamed tests have no current executable identity. Keep all tests.
const removed = plan.changedTests.filter((path) => !expected.has(path));
const missingOwners = plan.owners.filter(
(owner) =>
![...ownersByFile.values()].some((owners) => owners.includes(owner)),
);
const effective = removed.length
? full(
`Changed tests absent from current inventory: ${removed.join(", ")}`,
plan.changedPaths,
)
: missingOwners.length
? full(
`Owners without executable tests: ${missingOwners.join(", ")}`,
plan.changedPaths,
)
: plan;
const selected = [];
const omitted = [];
for (const path of inventory.toSorted()) {
const owners = ownersByFile.get(path) ?? [];
const keep =
effective.mode === "full" ||
!owners.length ||
effective.changedTests.includes(path) ||
owners.some((owner) => effective.owners.includes(owner));
(keep ? selected : omitted).push(path);
}
const actual = new Set(results.map((file) => file.name));
const missing = inventory.filter((path) => !actual.has(path));
const unexpected = [...actual].filter((path) => !expected.has(path));
const duplicateFiles = results.length - actual.size;
const unobserved = results
.filter((file) => !["passed", "failed"].includes(file.state))
.map((file) => file.name);
const omittedSet = new Set(omitted);
const missedFailures = results
.filter((file) => omittedSet.has(file.name) && file.state === "failed")
.map((file) => file.name);
return {
version: 1,
observational: true,
plan: effective,
selected,
omitted,
missing,
unexpected,
duplicateFiles,
unobserved,
missedFailures,
complete:
!missing.length &&
!unexpected.length &&
!duplicateFiles &&
!unobserved.length,
// Phase totals are scheduling weights, not an estimate of wall-time savings.
omittedDurationMs: results
.filter((file) => omittedSet.has(file.name))
.reduce((sum, file) => sum + file.durationMs, 0),
};
/** Parse a plan produced by `planTestSelection`; anything else fails closed. */
export function parseTestSelection(value) {
const plan = typeof value === "string" ? JSON.parse(value) : value;
const valid =
plan?.version === 2 &&
typeof plan.reason === "string" &&
Array.isArray(plan.groups) &&
(plan.mode === "full"
? plan.groups.length === 0
: plan.mode === "selected" &&
plan.groups.length >= 1 &&
plan.groups.length <= MAX_GROUPS &&
plan.groups.every(
(group) =>
Array.isArray(group) &&
group.length > 0 &&
group.every(
(path) =>
typeof path === "string" &&
isTest(path) &&
/^(src|tests)\//u.test(path) &&
!path.split("/").some((part) => ["", ".", ".."].includes(part)),
),
) &&
new Set(plan.groups.flat()).size === plan.groups.flat().length &&
plan.groups.flat().length <= SELECTION_CAP);
if (!valid) throw new Error("Invalid portable test selection plan");
return plan;
}
// Fast feedback uses one source group and twelve built groups. Shards are
// balanced by recorded duration, but hosted runner speed varies, so the
// slowest shard sets the critical path: at eight shards it ran 24% above the
// mean shard. Smaller shards bound that excess at a few percent more runner
// time. Full qualification retains six complete shards with coverage. A
// selection runs one job per group.
const shardsOf = (count) =>
Array.from({ length: count }, (_, index) => `${index + 1}/${count}`);
/** The `test-shard` matrix labels for a tier and its test selection. */
export const testShards = (tier, plan) =>
shardsOf(
plan.mode === "selected" ? plan.groups.length : tier === "full" ? 6 : 12,
);
+107 -10
View File
@@ -168,6 +168,9 @@ function classifyCommit(
mergeBase: (base) => base,
isAncestor: () => true,
changedFiles: () => paths,
// No test reaches these synthetic paths; selection falls back to all.
listFiles: () => [],
readFiles: () => new Map(),
readFile: (revision, path) => {
if (path === "package.json" && packageJson !== undefined)
return revision === PATH_BASE
@@ -376,21 +379,114 @@ it("keeps unknown CI scripts on the fast Linux baseline", async () => {
for (const flag of CI_FLAGS) expect(full[flag]).toBe(true);
});
it("emits an observational owner proposal without changing ordinary or full portable execution", async () => {
const { directory, base } = await fixture();
const head = await change(directory, ["src/evm/Example.ts"]);
expect(await classify(directory, base, head)).toMatchObject({
code: true,
testSelection: { mode: "candidate", owners: ["evm"] },
it("selects the tests a pull request affects and runs everything when it cannot decide", async () => {
const { directory } = await fixture();
// The entrypoint's output file is not part of the change.
await mkdir(join(directory, ".git", "info"), { recursive: true });
await writeFile(join(directory, ".git", "info", "exclude"), "scope-output\n");
const write = async (path: string, content: string) => {
await mkdir(dirname(join(directory, path)), { recursive: true });
await writeFile(join(directory, path), content);
};
const imports = (specifier: string) =>
`import { value } from "${specifier}";\nvoid value;\n`;
await write("src/evm/Example.ts", "export const value = 1;\n");
await write("src/evm/Example.test.ts", imports("./Example.js"));
await write("src/evm/Other.ts", "export const value = 2;\n");
await write(
"tests/support/evmHelper.ts",
imports("../../src/evm/Example.js"),
);
// Through a test-only helper, through the leaf owner, and by name.
await write(
"tests/composition/helped.test.ts",
imports("../support/evmHelper.js"),
);
await write("tests/boundary/evm/cli.test.ts", "// spawns the built CLI\n");
await write(
"tests/conformance/named.test.ts",
'readFileSync("src/evm/Example.ts");\n',
);
await write(
"tests/composition/unrelated.test.ts",
imports("../../src/provider.js"),
);
const base = await commit(directory);
const head = await change(
directory,
["src/evm/Example.ts"],
"export const value = 3;\n",
);
const selected = [
"src/evm/Example.test.ts",
"tests/boundary/evm/cli.test.ts",
"tests/composition/helped.test.ts",
"tests/conformance/named.test.ts",
];
expect(await runPlanScript(directory, base, head)).toMatchObject({
testSelection: { version: 2, mode: "selected", groups: [selected] },
testShards: ["1/1"],
});
const output = await readFile(join(directory, "scope-output"), "utf8");
expect(output).toContain("test_mode=selected\n");
expect(output).toContain('test_shards=["1/1"]\n');
// ci:full, main pushes and shared or untraced inputs run the complete suite.
expect(
await runPlanScript(directory, base, head, "pull_request", true),
).toMatchObject({
code: true,
testSelection: { mode: "full", owners: [] },
testSelection: { mode: "full", groups: [] },
testShards: ["1/6", "2/6", "3/6", "4/6", "5/6", "6/6"],
});
expect(await classify(directory, base, head, "push")).toMatchObject({
testSelection: { mode: "full" },
testShards: expect.arrayContaining(["12/12"]),
});
let previous = head;
for (const path of [
"src/process/Shared.ts",
"scripts/build-something.mjs",
"tests/fixtures/data.bin",
]) {
const parent = previous;
previous = await change(directory, [path]);
expect(await classify(directory, parent, previous)).toMatchObject({
testSelection: { mode: "full", reason: expect.stringContaining(path) },
});
}
// A leaf owner's boundary tests cover its other modules.
const unreached = await change(
directory,
["src/evm/Other.ts"],
"export const value = 4;\n",
);
expect(await classify(directory, previous, unreached)).toMatchObject({
testSelection: { mode: "selected" },
});
const isolated = await change(directory, ["src/unowned/Lonely.ts"]);
expect(await classify(directory, unreached, isolated)).toMatchObject({
testSelection: {
mode: "full",
reason: "No test imports, names or sits beside a changed path",
},
});
// Above the cap, selection gives way to the complete suite.
const many = Array.from(
{ length: 151 },
(_, index) => `tests/composition/many${String(index)}.test.ts`,
);
const wide = await change(directory, many);
expect(await classify(directory, isolated, wide)).toMatchObject({
testSelection: {
mode: "full",
reason: "151 affected test files exceed the 150-file selection cap",
},
});
const capped = await change(directory, many.slice(0, 81), "// edited\n");
expect(await classify(directory, wide, capped)).toMatchObject({
testSelection: { mode: "selected" },
testShards: ["1/3", "2/3", "3/3"],
});
const output = await readFile(join(directory, "scope-output"), "utf8");
expect(output).toContain('test_selection={"version":1,"mode":"full"');
});
it("passes the workflow environment through the entrypoint unchanged", async () => {
@@ -1468,6 +1564,7 @@ it.each(["evm", "browser"])(
outputs: {
...Object.fromEntries(CI_FLAGS.map((flag) => [flag, "false"])),
tier: "fast",
test_mode: "full",
[scope]: "true",
},
},
+17 -1
View File
@@ -12,6 +12,7 @@ import {
parseStaticTaskContext,
recordStaticTaskContext,
} from "../../../scripts/ci/static-task-context.mjs";
import { testShards } from "../../../scripts/ci/test-selection.mjs";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
const execute = promisify(execFile);
@@ -121,6 +122,7 @@ it.each(["fast", "full"] as const)(
flags.map((flag) => [flag, tier === "full" ? "true" : "false"]),
),
tier,
test_mode: "full",
code: "true",
reverse_tools: "true",
},
@@ -265,6 +267,7 @@ it("starts source feedback before build and requires both groups without waiving
flags.map((flag) => [flag, tier === "full" ? "true" : "false"]),
),
tier,
test_mode: "full",
code: "true",
};
const context: Record<
@@ -398,7 +401,20 @@ it("merges source and built reports exactly once using the workflow's shard matr
matrix.slice(3, -2),
{
fromJSON: JSON.parse,
needs: { changes: { outputs: { tier: "fast" } } },
needs: {
changes: {
outputs: {
test_shards: JSON.stringify(
testShards("fast", {
version: 2,
mode: "full",
reason: "Complete fast suite",
groups: [],
}),
),
},
},
},
},
{ timeout: 1_000 },
),
@@ -259,7 +259,10 @@ async function requiredRun(
: "success",
};
}
results.changes = { result: "success", outputs: changes };
results.changes = {
result: "success",
outputs: { test_mode: "full", ...changes },
};
const receiptDirectory = join(directory, ".cache", "receipt");
const output = join(directory, "github-output");
await writeFile(output, "");
@@ -593,7 +593,9 @@ it("keeps fast feedback separate from complete qualification and release proof",
const shards = ci.jobs["test-shard"]?.steps ?? [];
const fast = shards.find((step) => step.run?.includes("npm run test:built"));
const full = shards.find((step) => step.run?.includes("test:ci:shard:run"));
expect(fast?.if).toBe("needs.changes.outputs.tier == 'fast'");
expect(fast?.if).toBe(
"needs.changes.outputs.tier == 'fast' && needs.changes.outputs.test_mode == 'full'",
);
expect(fast?.run).toContain("--reporter=blob");
expect(fast?.run).not.toContain("--coverage");
expect(full?.if).toBe("needs.changes.outputs.tier == 'full'");
@@ -640,6 +642,7 @@ it.each([
(await scopeFlags()).map((flag) => [flag, "false"]),
),
tier: "fast",
test_mode: "full",
code: "true",
[scope]: selected,
},
@@ -673,6 +676,7 @@ it("rejects incomplete or malformed classification even when available jobs succ
const valid = {
...Object.fromEntries(flags.map((flag) => [flag, "false"])),
tier: "fast",
test_mode: "full",
};
const run = (outputs: unknown) =>
executeWorkflowFixture(
@@ -750,6 +754,7 @@ it("fails fast only for ordinary PR matrices and rejects partial portable feedba
outputs: {
...Object.fromEntries(flags.map((flag) => [flag, "false"])),
tier: "fast",
test_mode: "full",
code: "true",
},
},
@@ -775,6 +780,105 @@ it("fails fast only for ordinary PR matrices and rejects partial portable feedba
}
});
it("runs a selected PR subset in planner-owned groups and still gates it", async () => {
const { jobs } = await workflow();
// The planner owns every shard list, including one job per selected group.
expect(jobs["test-shard"]?.strategy).toMatchObject({
matrix: { shard: "${{ fromJSON(needs.changes.outputs.test_shards) }}" },
});
expect(jobs["test-source"]).toMatchObject({
if: expect.stringContaining("needs.changes.outputs.test_mode == 'full'"),
});
const selected = z
.array(
z.object({
if: z.string().optional(),
run: z.string().optional(),
env: z.record(z.string(), z.string()).optional(),
}),
)
.parse(jobs["test-shard"]?.steps)
.find(
(step) => step.if === "needs.changes.outputs.test_mode == 'selected'",
);
expect(selected?.run).toBe(
'node scripts/ci/test-group.mjs selected "${MATRIX_SHARD}" --reporter=default --reporter=blob',
);
expect(selected?.env).toEqual({
MATRIX_SHARD: "${{ matrix.shard }}",
CI_TEST_SELECTION: "${{ needs.changes.outputs.test_selection }}",
});
const flags = await scopeFlags();
const gate = (
outputs: Record<string, unknown>,
results: Record<string, string>,
event = "pull_request",
) =>
executeWorkflowFixture(
process.execPath,
[
fileURLToPath(
new URL("../../../scripts/ci/required.mjs", import.meta.url),
),
],
{
env: {
...process.env,
GITHUB_EVENT_NAME: event,
CI_QUALIFICATION_DIRECTORY: "",
CI_JOBS: JSON.stringify({
changes: {
result: "success",
outputs: {
...Object.fromEntries(flags.map((flag) => [flag, "false"])),
tier: "fast",
code: "true",
test_mode: "selected",
...outputs,
},
},
static: { result: "success" },
test: { result: "success" },
...Object.fromEntries(
Object.entries(results).map(([name, result]) => [
name,
{ result },
]),
),
}),
},
},
);
const ran = { build: "success", "test-shard": "success" };
// A selected subset skips the separate source job.
await expect(gate({}, ran)).resolves.toMatchObject({ stderr: "" });
for (const [name, result] of [
["test-shard", "skipped"],
["test-shard", "failure"],
["build", "cancelled"],
] as const)
await expect(gate({}, { ...ran, [name]: result })).rejects.toMatchObject({
code: 1,
stderr: expect.stringContaining(name),
});
// The complete fast suite still requires its source job.
await expect(gate({ test_mode: "full" }, ran)).rejects.toMatchObject({
stderr: expect.stringContaining("test-source"),
});
// Only a fast pull-request plan with code changes may select tests.
for (const [outputs, event] of [
[{ test_mode: undefined }, "pull_request"],
[{ test_mode: "candidate" }, "pull_request"],
[{}, "push"],
[{ code: "false" }, "pull_request"],
] as const)
await expect(gate(outputs, ran, event)).rejects.toMatchObject({
code: 1,
stderr: expect.stringContaining("valid test mode"),
});
});
it("binds each canonical lane to its explicit fast and full workflow jobs", async () => {
const { jobs, needs } = await requiredNeeds();
const { CI_LANES } = await import(
@@ -870,6 +974,7 @@ it("binds each canonical lane to its explicit fast and full workflow jobs", asyn
outputs: {
...Object.fromEntries(flags.map((name) => [name, "false"])),
tier: "fast",
test_mode: "full",
[scope]: "true",
},
};
@@ -897,6 +1002,7 @@ it("binds each canonical lane to its explicit fast and full workflow jobs", asyn
outputs: {
...Object.fromEntries(flags.map((name) => [name, "true"])),
tier: "full",
test_mode: "full",
},
};
await expect(executeGate(fullContext)).resolves.toMatchObject({ stderr: "" });
+58 -61
View File
@@ -9,8 +9,6 @@ import { expect, it } from "vitest";
import { parse } from "yaml";
import { z } from "zod";
import { planTestSelection } from "../../../scripts/ci/test-selection.mjs";
import { parseTestDurations } from "../../ciTestSequencer.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import { workflowGit } from "../../support/workflowGit.js";
@@ -102,9 +100,6 @@ it("merges available failed-shard diagnostics and skips unavailable shard report
}
expect(steps[upload]?.with?.path).toContain("test-timings.json");
expect(steps[upload]?.with).toMatchObject({ "include-hidden-files": true });
expect(steps[upload]?.with?.path).toContain(
".cache/ci-test-selection/report.json",
);
const summary = steps.find((step) =>
step.run?.includes("scripts/ci/test-report.mjs"),
);
@@ -212,8 +207,9 @@ it("collects either fast test group while keeping full reports tied to full shar
);
expect(summary?.env?.SHARD_RESULT).toBe("${{ needs.test-shard.result }}");
expect(summary?.env?.SOURCE_RESULT).toBe("${{ needs.test-source.result }}");
// A selected PR subset runs its source tests in the selected test jobs.
expect(summary?.env?.SOURCE_REQUIRED).toBe(
"${{ needs.changes.outputs.tier == 'fast' }}",
"${{ needs.changes.outputs.tier == 'fast' && needs.changes.outputs.test_mode == 'full' }}",
);
});
@@ -660,77 +656,78 @@ it("recovers complete module costs from real blob reports without executing test
).rejects.toMatchObject({ code: "ENOENT" });
});
it("records failed omissions from full reports without treating partial or unhandled outcomes as parity", async () => {
it("verifies exactly the selected subset and never publishes it as timing history", async () => {
const root = await createTestTempDirectory("rea-ci-selection-report-");
await workflowGit(root, ["init"]);
const paths = [
"src/sqlite/provider.test.ts",
"src/evm/provider.test.ts",
"tests/boundary/mcp/workflow.test.ts",
"src/a.test.ts",
"tests/b.test.ts",
"tests/unselected.test.ts",
];
for (const path of paths) {
await mkdir(join(root, path, ".."), { recursive: true });
await writeFile(join(root, path), "// fixture\n");
}
await workflowGit(root, ["add", "src", "tests"]);
const timing = {
version: 1,
success: false,
reason: "failed",
unhandledErrors: 0,
files: paths.map((name, index) => ({
name,
durationMs: 10,
state: index === 1 ? "failed" : "passed",
})),
const selection = {
version: 2,
mode: "selected",
reason: "Affected tests: 2 changed",
groups: [["src/a.test.ts"], ["tests/b.test.ts"]],
};
const run = async (report: unknown, source = "failure") => {
await writeFile(join(root, "test-timings.json"), JSON.stringify(report));
await executeTimingReport(root, {
SHARD_RESULT: "failure",
SOURCE_REQUIRED: "true",
SOURCE_RESULT: source,
MERGE_RESULT: "failure",
const run = async (names: readonly string[], plan: unknown = selection) => {
await writeFile(
join(root, "test-timings.json"),
JSON.stringify({
version: 1,
success: true,
files: names.map((name) => ({ name, durationMs: 1, state: "passed" })),
}),
);
return executeTimingReport(root, {
CI_VERIFY_TEST_INVENTORY: "true",
CI_TEST_SELECTION: JSON.stringify(
planTestSelection(["src/sqlite/Provider.ts"], "pull_request", "fast"),
),
GITHUB_SHA: "merge-sha",
CI_PR_HEAD_SHA: "pr-head-sha",
CI_TEST_SELECTION: JSON.stringify(plan),
});
return JSON.parse(
await readFile(
join(root, ".cache/ci-test-selection/report.json"),
"utf8",
),
) as unknown;
};
expect(await run(timing)).toMatchObject({
observational: true,
complete: true,
selected: [paths[0], paths[2]],
omitted: [paths[1]],
missedFailures: [paths[1]],
executionSha: "merge-sha",
prHeadSha: "pr-head-sha",
});
for (const report of [
{ ...timing, files: timing.files.slice(0, 1) },
{ ...timing, unhandledErrors: 1 },
{ ...timing, reason: "interrupted" },
])
expect(await run(report)).toMatchObject({ complete: false });
for (const source of ["skipped", "cancelled", ""])
expect(await run(timing, source)).toMatchObject({ complete: false });
await expect(run({ ...timing, files: "invalid" })).rejects.toMatchObject({
code: 1,
});
await expect(
readFile(join(root, ".cache/ci-test-selection/report.json")),
).rejects.toMatchObject({ code: "ENOENT" });
const passed = await run(["src/a.test.ts", "tests/b.test.ts"]);
expect(passed.summary).toContain("Portable tests: 2 selected files.");
expect(passed.summary).toContain("is not a successful timing baseline");
expect(passed.summary).not.toContain("Successful timing history");
// A subset is not a complete-suite timing baseline.
await expect(
readFile(join(root, ".cache/ci-test-history/test-results.json")),
).rejects.toMatchObject({ code: "ENOENT" });
// Missing, unplanned (even if tracked) and duplicate files fail.
for (const names of [
["src/a.test.ts"],
["src/a.test.ts", "tests/b.test.ts", "tests/unselected.test.ts"],
["src/a.test.ts", "src/a.test.ts", "tests/b.test.ts"],
])
await expect(run(names)).rejects.toMatchObject({
stderr: expect.stringContaining("Incomplete selected test inventory"),
});
// A full plan keeps the complete tracked inventory and its history.
const full = { version: 2, mode: "full", reason: "Shared", groups: [] };
await expect(
run(["src/a.test.ts", "tests/b.test.ts"], full),
).rejects.toMatchObject({
stderr: expect.stringContaining("Incomplete portable test inventory"),
});
await run(paths, full);
await expect(
readFile(join(root, ".cache/ci-test-history/test-results.json"), "utf8"),
).resolves.toContain("tests/unselected.test.ts");
// An unreadable plan fails closed.
for (const plan of [
{ ...selection, version: 1 },
{ ...selection, groups: [] },
{ ...selection, groups: [["../escape.test.ts"]] },
{ ...selection, groups: [["src/a.test.ts"], ["src/a.test.ts"]] },
{ ...full, groups: [["src/a.test.ts"]] },
])
await expect(run(["src/a.test.ts"], plan)).rejects.toMatchObject({
stderr: expect.stringContaining("Invalid portable test selection plan"),
});
});
it("requires executed Python analysis-efficiency cases while allowing optional skips", async () => {
+142 -218
View File
@@ -1,240 +1,164 @@
import { execFile } from "node:child_process";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { expect, it } from "vitest";
import {
compareTestSelection,
parseTestSelection,
planTestSelection,
testShards,
} from "../../scripts/ci/test-selection.mjs";
const sqliteSource = "src/sqlite/SqliteDatabaseProvider.test.ts";
const sqliteBoundary = "tests/boundary/sqlite/sqliteDatabaseProvider.test.ts";
const adbSource = "src/adb/AdbProvider.test.ts";
const adbBoundary =
"tests/boundary/providers/adb/adbBugreportPublication.test.ts";
const cliBoundary = "tests/boundary/cli/apktoolCli.test.ts";
const mcpBoundary = "tests/boundary/mcp/apktoolMcp.test.ts";
const processBoundary =
"tests/boundary/process/processCaptureLifecycle.test.ts";
const acceptance = "tests/acceptance/tools/localToolContracts.test.ts";
const composition = "tests/composition/directAnalysisCleanup.test.ts";
const evaluation = "tests/evaluation/testDiscovery.test.ts";
const execute = promisify(execFile);
const passed = (name: string) => ({ name, durationMs: 10, state: "passed" });
it("unions affected leaf owners, keeps changed tests and retains unowned integration tests", () => {
const plan = planTestSelection(
["src/sqlite/SqliteDatabaseProvider.ts", adbBoundary],
"pull_request",
"fast",
);
const inventory = [
sqliteSource,
sqliteBoundary,
adbSource,
adbBoundary,
cliBoundary,
mcpBoundary,
processBoundary,
acceptance,
composition,
evaluation,
];
const report = compareTestSelection(plan, inventory, inventory.map(passed));
expect(plan).toMatchObject({
mode: "candidate",
owners: ["adb", "sqlite"],
const plan = (files: readonly string[], sources: Record<string, string>) =>
planTestSelection({
files,
event: "pull_request",
tier: "fast",
code: true,
snapshot: {
list: () => Object.keys(sources),
read: (paths) =>
new Map(paths.map((path) => [path, sources[path] ?? ""])),
},
});
expect(report.selected).toEqual(
expect.arrayContaining([
sqliteSource,
sqliteBoundary,
adbSource,
adbBoundary,
cliBoundary,
mcpBoundary,
processBoundary,
acceptance,
composition,
evaluation,
]),
);
expect(report.omitted).toEqual([]);
const selectedFiles = (result: ReturnType<typeof plan>) =>
result.groups.flat().toSorted();
it("follows test and verifier imports and directory scans, not production importers", () => {
const sources = {
"src/domain/Value.ts": "",
"src/application/UsesValue.ts": 'import "../domain/Value.js";',
"src/application/UsesValue.test.ts": 'import "./UsesValue.js";',
"src/domain/Value.test.ts": 'import { value } from "./Value.js";',
"src/domain/Registry.test.ts": "readdirSync(here);",
"src/domain/Unrelated.test.ts": 'import "./Unrelated.js";',
"tests/boundary/filesystem/verifier.test.ts":
'const m = await import("../../../scripts/verify-thing.mjs");',
};
expect(selectedFiles(plan(["src/domain/Value.ts"], sources))).toEqual([
"src/domain/Registry.test.ts",
"src/domain/Value.test.ts",
]);
expect(selectedFiles(plan(["scripts/verify-thing.mjs"], sources))).toEqual([
"tests/boundary/filesystem/verifier.test.ts",
]);
});
it("omits only unrelated owned tests while retaining dynamic boundary suites", () => {
const plan = planTestSelection(
["src/sqlite/SqliteDatabaseProvider.ts"],
"pull_request",
"fast",
);
const inventory = [
sqliteSource,
sqliteBoundary,
adbSource,
adbBoundary,
cliBoundary,
mcpBoundary,
processBoundary,
acceptance,
composition,
evaluation,
];
const report = compareTestSelection(plan, inventory, inventory.map(passed));
expect(report.selected).toEqual(
expect.arrayContaining([
sqliteSource,
sqliteBoundary,
cliBoundary,
mcpBoundary,
processBoundary,
acceptance,
composition,
evaluation,
]),
);
expect(report.omitted).toEqual([adbSource, adbBoundary]);
expect(report.complete).toBe(true);
it("selects a leaf adapter's tests by its name and directory across test projects", () => {
const sources = {
"src/jeb/JebMcpDeadline.ts": "",
"tests/boundary/providers/jebMcpTimeout.test.ts": "",
"tests/boundary/mcp/jebMcp.test.ts": "",
"tests/acceptance/providers/jebConfigurationIsolation.test.ts": "",
"tests/boundary/providers/adb/adbBugreport.test.ts": "",
"tests/boundary/providers/ghidraJava.test.ts": "",
};
expect(selectedFiles(plan(["src/jeb/JebMcpDeadline.ts"], sources))).toEqual([
"tests/acceptance/providers/jebConfigurationIsolation.test.ts",
"tests/boundary/mcp/jebMcp.test.ts",
"tests/boundary/providers/jebMcpTimeout.test.ts",
]);
expect(selectedFiles(plan(["src/adb/AdbProvider.ts"], sources))).toEqual([
"tests/boundary/providers/adb/adbBugreport.test.ts",
]);
});
it.each([
["unowned shared source", "src/application/Workflow.ts"],
["fixture input", "tests/conformance/c/fixture.c"],
["configuration", "vitest.config.ts"],
["documentation", "README.md"],
["CLI implementation", "src/cli/apktoolCommands.ts"],
["MCP implementation", "src/server/contractToolBindings.ts"],
["process ownership", "src/process/ProcessOwnership.ts"],
["acceptance test", acceptance],
["composition test", composition],
])("uses full fallback for %s changes", (_description, path) => {
expect(planTestSelection([path], "pull_request", "fast")).toMatchObject({
it("lets checker-only inputs and prose select nothing by themselves", () => {
const sources = {
"src/domain/Value.test.ts": "",
"tests/boundary/filesystem/docs.test.ts": 'read("docs/guide.md");',
};
// A declaration, guard baseline or schema summary adds no tests…
expect(
selectedFiles(
plan(
[
"src/domain/Value.test.ts",
"scripts/lib/helper.d.mts",
"scripts/verify/baselines/some-guard.json",
"scripts/verify/advertised-schemas/tools/tool.json",
],
sources,
),
),
).toEqual(["src/domain/Value.test.ts"]);
// …and prose selects the tests that read it by path.
expect(selectedFiles(plan(["docs/guide.md"], sources))).toEqual([
"tests/boundary/filesystem/docs.test.ts",
]);
expect(plan(["scripts/verify/baselines/x.json"], sources)).toMatchObject({
mode: "full",
owners: [],
changedTests: [],
});
expect(plan(["scripts/lib/shared.mjs"], sources)).toMatchObject({
mode: "full",
reason: "Input is not traced to tests: scripts/lib/shared.mjs",
});
});
it("uses full fallback outside ordinary fast pull requests or without diff paths", () => {
it("parses only well-formed plans and derives one shard per selected group", () => {
const selected = {
version: 2,
mode: "selected",
reason: "Affected tests: 2 changed",
groups: [["src/a.test.ts"], ["tests/b.test.ts"]],
};
expect(
planTestSelection(["src/sqlite/SqliteDatabaseProvider.ts"], "push", "fast")
.mode,
).toBe("full");
expect(
planTestSelection(
["src/sqlite/SqliteDatabaseProvider.ts"],
"pull_request",
"full",
).mode,
).toBe("full");
expect(planTestSelection([], "pull_request", "fast").mode).toBe("full");
testShards("fast", parseTestSelection(JSON.stringify(selected))),
).toEqual(["1/2", "2/2"]);
const full = { version: 2, mode: "full", reason: "Shared", groups: [] };
expect(testShards("fast", parseTestSelection(full))).toHaveLength(12);
expect(testShards("full", parseTestSelection(full))).toHaveLength(6);
for (const value of [
"",
"null",
{ ...selected, mode: "candidate" },
{ ...selected, groups: [[]] },
{ ...selected, groups: [["src/a.ts"]] },
{ ...selected, groups: [["/abs/a.test.ts"]] },
{ ...selected, groups: [["src/a.test.ts"], ["src/a.test.ts"]] },
{
...selected,
groups: Array.from({ length: 5 }, (_, i) => [`src/${i}.test.ts`]),
},
])
expect(() => parseTestSelection(value)).toThrow(
/Invalid portable test selection plan|JSON/u,
);
});
it("keeps new changed tests and falls back when a changed test was removed", () => {
const addedTest = "src/sqlite/NewDatabaseRule.test.ts";
const candidate = planTestSelection([addedTest], "pull_request", "fast");
expect(candidate.changedTests).toEqual([addedTest]);
const inventory = [addedTest, adbSource];
expect(
compareTestSelection(candidate, inventory, inventory.map(passed)).selected,
).toContain(addedTest);
const removedTest = "src/sqlite/RemovedDatabaseRule.test.ts";
const report = compareTestSelection(
planTestSelection([removedTest], "pull_request", "fast"),
[sqliteSource, adbSource],
[passed(sqliteSource), passed(adbSource)],
it("runs only a group the plan defines", async () => {
const group = fileURLToPath(
new URL("../../scripts/ci/test-group.mjs", import.meta.url),
);
expect(report.plan.mode).toBe("full");
expect(report.selected).toEqual([adbSource, sqliteSource]);
expect(report.omitted).toEqual([]);
});
it("marks incomplete inventories and nonterminal outcomes as incomplete", () => {
const plan = planTestSelection(
["src/sqlite/SqliteDatabaseProvider.ts"],
"pull_request",
"fast",
);
const inventory = [sqliteSource, adbSource];
const completeResults = inventory.map(passed);
expect(
compareTestSelection(plan, inventory, [passed(sqliteSource)]).complete,
).toBe(false);
expect(
compareTestSelection(plan, inventory, [
...completeResults,
passed("tests/evaluation/unexpected.test.ts"),
]),
).toMatchObject({
complete: false,
unexpected: ["tests/evaluation/unexpected.test.ts"],
const selection = JSON.stringify({
version: 2,
mode: "selected",
reason: "Affected tests: 1 changed",
groups: [["src/a.test.ts"]],
});
for (const [shard, environment] of [
["2/2", selection],
["1/2", selection],
[
"1/1",
JSON.stringify({ version: 2, mode: "full", reason: "x", groups: [] }),
],
["1/1", ""],
] as const)
await expect(
execute(process.execPath, [group, "selected", shard], {
env: { ...process.env, CI_TEST_SELECTION: environment },
}),
).rejects.toMatchObject({ code: 1 });
await expect(
execute(process.execPath, [group, "selected", "1/1", "--shard=1/2"], {
env: { ...process.env, CI_TEST_SELECTION: selection },
}),
).rejects.toMatchObject({
stderr: expect.stringContaining("already sharded"),
});
expect(
compareTestSelection(plan, inventory, [
...completeResults,
passed(adbSource),
]),
).toMatchObject({ complete: false, duplicateFiles: 1 });
for (const state of ["pending", "skipped"])
expect(
compareTestSelection(plan, inventory, [
passed(sqliteSource),
{ name: adbSource, durationMs: 0, state },
]),
).toMatchObject({ complete: false, unobserved: [adbSource] });
});
it("reports a failure from an omitted test, including collection or module failures", () => {
const plan = planTestSelection(
["src/sqlite/SqliteDatabaseProvider.ts"],
"pull_request",
"fast",
);
const report = compareTestSelection(
plan,
[sqliteSource, adbSource],
[passed(sqliteSource), { name: adbSource, durationMs: 1, state: "failed" }],
);
expect(report.omitted).toContain(adbSource);
expect(report.missedFailures).toEqual([adbSource]);
});
it("rejects plans that name unowned owners or malformed changed tests", () => {
const plan = planTestSelection(
["src/sqlite/SqliteDatabaseProvider.ts"],
"pull_request",
"fast",
);
expect(() =>
compareTestSelection(
{ ...plan, owners: ["unknown"] },
[sqliteSource],
[passed(sqliteSource)],
),
).toThrow("Invalid observational test selection plan");
expect(() =>
compareTestSelection(
{ ...plan, changedTests: ["src/sqlite/not-a-test.ts"] },
[sqliteSource],
[passed(sqliteSource)],
),
).toThrow("Invalid observational test selection plan");
});
it("uses full fallback when a changed owner has no executable tests", () => {
const report = compareTestSelection(
planTestSelection(
["src/sqlite/SqliteDatabaseProvider.ts"],
"pull_request",
"fast",
),
[adbSource],
[passed(adbSource)],
);
expect(report.plan.mode).toBe("full");
expect(report.omitted).toEqual([]);
});