mirror of
https://github.com/morluto/rea.git
synced 2026-11-16 11:43:20 +00:00
ci: run only the affected portable tests on pull requests (#11346)
* ci: run only the affected portable tests on pull requests Pull-request CI now runs static checks, the build and the portable tests a change affects, selected from test imports (through test-only helpers), same-stem and directory-scanning tests, leaf-owner boundary tests and tests that name a changed path. Shared boundaries, untraced inputs, empty selections and selections above 150 files fall back to the complete fast suite. Main pushes, schedules, releases and ci:full keep the complete suite. The portable report verifies the exact selected inventory and the CI required gate accepts a selected plan only for a fast pull request with code changes. * ci: own leaf-adapter tests by name and keep subset summaries diagnostic
This commit is contained in:
1 parent
97112f14a1
commit
ff6d1d8142
18 files changed
+901
-577
No files matched your search
+17
-13
@@ -87,6 +87,8 @@ jobs:
|
||||
website: ${{ steps.plan.outputs.website }}
|
||||
matrix: ${{ steps.plan.outputs.matrix }}
|
||||
test_selection: ${{ steps.plan.outputs.test_selection }}
|
||||
test_mode: ${{ steps.plan.outputs.test_mode }}
|
||||
test_shards: ${{ steps.plan.outputs.test_shards }}
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
|
||||
with:
|
||||
@@ -294,7 +296,8 @@ jobs:
|
||||
test-source:
|
||||
name: Source tests
|
||||
needs: changes
|
||||
if: needs.changes.outputs.code == 'true' && needs.changes.outputs.tier == 'fast'
|
||||
# A selected PR subset runs its source tests in the selected test jobs.
|
||||
if: needs.changes.outputs.code == 'true' && needs.changes.outputs.tier == 'fast' && needs.changes.outputs.test_mode == 'full'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
@@ -330,14 +333,10 @@ jobs:
|
||||
# every shard's diagnostics and coverage. Passing PRs run the full suite.
|
||||
fail-fast: ${{ github.event_name == 'pull_request' && needs.changes.outputs.tier == 'fast' }}
|
||||
matrix:
|
||||
# Fast feedback uses one source group and twelve built groups. Shards
|
||||
# are balanced by recorded duration, but hosted runner speed varies,
|
||||
# so the slowest shard sets the critical path: at eight shards it ran
|
||||
# 24% above the mean shard. Smaller shards bound that excess at a few
|
||||
# percent more runner time. Classification, build, and report merge
|
||||
# sit outside these jobs. Full qualification retains six complete
|
||||
# shards with coverage.
|
||||
shard: ${{ fromJSON(needs.changes.outputs.tier == 'fast' && '["1/12","2/12","3/12","4/12","5/12","6/12","7/12","8/12","9/12","10/12","11/12","12/12"]' || '["1/6","2/6","3/6","4/6","5/6","6/6"]') }}
|
||||
# scripts/ci/test-selection.mjs owns the shard counts: twelve fast
|
||||
# built shards, six full-qualification shards with coverage, or one
|
||||
# job per group of a selected PR subset.
|
||||
shard: ${{ fromJSON(needs.changes.outputs.test_shards) }}
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
|
||||
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
|
||||
@@ -357,11 +356,17 @@ jobs:
|
||||
env:
|
||||
REA_CI_TEST_HISTORY: .cache/ci-test-history/test-results.json
|
||||
- name: Run built-dependent tests for fast feedback
|
||||
if: needs.changes.outputs.tier == 'fast'
|
||||
if: needs.changes.outputs.tier == 'fast' && needs.changes.outputs.test_mode == 'full'
|
||||
run: npm run test:built -- --reporter=default --reporter=blob --shard=${{ matrix.shard }}
|
||||
env:
|
||||
VITEST_BLOB_LABEL: built
|
||||
REA_CI_TEST_HISTORY: .cache/ci-test-history/test-results.json
|
||||
- name: Run the tests this pull request affects
|
||||
if: needs.changes.outputs.test_mode == 'selected'
|
||||
run: node scripts/ci/test-group.mjs selected "${MATRIX_SHARD}" --reporter=default --reporter=blob
|
||||
env:
|
||||
MATRIX_SHARD: ${{ matrix.shard }}
|
||||
CI_TEST_SELECTION: ${{ needs.changes.outputs.test_selection }}
|
||||
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||
if: ${{ !cancelled() }}
|
||||
with:
|
||||
@@ -608,7 +613,7 @@ jobs:
|
||||
if: needs.changes.outputs.code == 'true' && !cancelled()
|
||||
env:
|
||||
SHARD_RESULT: ${{ needs.test-shard.result }}
|
||||
SOURCE_REQUIRED: ${{ needs.changes.outputs.tier == 'fast' }}
|
||||
SOURCE_REQUIRED: ${{ needs.changes.outputs.tier == 'fast' && needs.changes.outputs.test_mode == 'full' }}
|
||||
SOURCE_RESULT: ${{ needs.test-source.result }}
|
||||
MERGE_RESULT: ${{ needs.changes.outputs.tier == 'full' && steps.merge.outcome || steps.merge-fast.outcome }}
|
||||
CI_VERIFY_TEST_INVENTORY: "true"
|
||||
@@ -624,7 +629,6 @@ jobs:
|
||||
test-results.json
|
||||
test-timings.json
|
||||
coverage/
|
||||
.cache/ci-test-selection/report.json
|
||||
include-hidden-files: true
|
||||
if-no-files-found: error
|
||||
retention-days: 30
|
||||
@@ -653,7 +657,7 @@ jobs:
|
||||
env:
|
||||
CODE_REQUIRED: ${{ needs.changes.outputs.code }}
|
||||
SHARD_RESULT: ${{ needs.test-shard.result }}
|
||||
SOURCE_REQUIRED: ${{ needs.changes.outputs.tier == 'fast' }}
|
||||
SOURCE_REQUIRED: ${{ needs.changes.outputs.tier == 'fast' && needs.changes.outputs.test_mode == 'full' }}
|
||||
SOURCE_RESULT: ${{ needs.test-source.result }}
|
||||
run: |
|
||||
if [ "${CODE_REQUIRED}" = "true" ]; then
|
||||
|
||||
@@ -1299,6 +1299,62 @@ and workspace rather than reporting successful removal. The controlled
|
||||
`packageUpdateRegistryFailure` regression needs built process modules, uses no
|
||||
public registry, and checks both the primary failure and live descendant absence.
|
||||
|
||||
## Pull-request test selection
|
||||
|
||||
CI on a pull request runs static checks, the build and the portable tests the
|
||||
change affects. The complete portable suite runs on every push to `main`, on
|
||||
the scheduled and manual qualification runs, in release qualification, and on a
|
||||
PR labelled `ci:full`. Those complete runs catch a consumer that selection
|
||||
missed, after it has merged. Add `ci:full` to a PR when its effect reaches
|
||||
further than its imports show, such as runtime registration, generated files,
|
||||
subprocesses or bridges. Adding the label starts full qualification at once,
|
||||
and later pushes stay on the full tier while the label remains.
|
||||
|
||||
`scripts/ci/test-selection.mjs` owns the selection. It reads the checked-out
|
||||
revision and selects:
|
||||
|
||||
- changed test files;
|
||||
- tests that import a changed module, directly or through test-only helpers
|
||||
(non-test modules under `tests/` and `src/**/*.fixture*.ts`). Selection never
|
||||
follows production importers transitively;
|
||||
- tests beside a changed module that share its stem, or that scan their
|
||||
directory with `readdir`;
|
||||
- the boundary tests of a changed leaf adapter (the owner table in the same
|
||||
file), which reach their sources through the built CLI, the MCP server or a
|
||||
fake provider process;
|
||||
- tests whose text names a changed path.
|
||||
|
||||
Verifier scripts (`scripts/verify-*.mjs`, `scripts/verify/**/*.mjs`) are traced
|
||||
like source modules. Changed Markdown outside `src/` and `tests/` selects only
|
||||
the tests that name it. Declarations, guard baselines
|
||||
(`scripts/verify/baselines/*.json`) and advertised-schema summaries select no
|
||||
tests, because the static and build jobs check them.
|
||||
|
||||
The PR falls back to the complete fast suite when it cannot decide:
|
||||
|
||||
- a shared boundary (`src/process/`, `src/filesystem/`, `src/contracts/tool*`,
|
||||
`src/server/createServer*`, `src/cli.ts`, `scripts/ci/`) changed;
|
||||
- any other changed input is not traced, such as workflows, package metadata,
|
||||
build and test-runner scripts, `scripts/lib/`, fixtures or data;
|
||||
- no test is selected;
|
||||
- more than 150 test files are selected;
|
||||
- a selected file has no single owning test project.
|
||||
|
||||
A selection runs as one `Test shard (k/n)` job per group of up to 40 files, at
|
||||
most four jobs, using exact file identity (`scripts/ci/test-group.mjs selected`).
|
||||
The separate `Source tests` job does not run, because the groups include the
|
||||
selected source tests. The `Classify changes` summary gives the mode and its
|
||||
reason.
|
||||
|
||||
The required checks still report on every PR and fail closed:
|
||||
|
||||
- `Portable tests` checks that the merged report contains exactly the selected
|
||||
files: none missing, none unplanned, no duplicates.
|
||||
- `CI required` accepts a selected plan only for a fast `pull_request` run with
|
||||
code changes, and then requires `build` and every selected test group.
|
||||
- A missing or malformed plan fails classification, the test group and the gate.
|
||||
- A selected run never publishes timing history.
|
||||
|
||||
## Real-toolchain verification lanes
|
||||
|
||||
See [Real-toolchain verification lanes](../testing-lanes.md#real-toolchain-verification-lanes).
|
||||
|
||||
+10
-30
@@ -631,30 +631,13 @@ generated-document validation, including the small onboarding example consistenc
|
||||
suite, without the source-test shards or native package
|
||||
lanes. Classification compares the PR head with its merge base, so later
|
||||
base-branch changes do not expand that scope. Static and portable-test aggregate jobs
|
||||
remain present and fail if classification fails. Every code change retains the
|
||||
complete portable Linux suite; individual test files are not selected by
|
||||
imports or filenames.
|
||||
remain present and fail if classification fails.
|
||||
|
||||
Ordinary PRs also produce an **observational** test-selection plan. The candidate
|
||||
leaf-adapter ownership table lives in `scripts/ci/test-selection.mjs`, separately
|
||||
from provider-job scopes. It proposes keeping affected owners plus every unowned
|
||||
test, including all CLI/MCP, acceptance, composition, conformance, and process
|
||||
tests. Shared inputs, unknown paths, missing owner tests, and removed tests fall
|
||||
back to the complete inventory. Renames account for both paths. Main, scheduled,
|
||||
manual, and `ci:full` runs always propose the full suite.
|
||||
|
||||
The source group and every built shard together still execute the complete suite. The existing report merge
|
||||
compares the proposal with actual module outcomes and retains
|
||||
`.cache/ci-test-selection/report.json` in `test-results`, including selected and
|
||||
omitted paths, changed input paths, missed failures, unobserved files, scheduling weights, PR head and
|
||||
executed merge SHA. This adds no test reruns or runner jobs. Missing, duplicate,
|
||||
skipped, interrupted, or unhandled results cannot establish a complete comparison;
|
||||
even a complete passing comparison is not proof of ownership completeness.
|
||||
Before enabling omissions, review representative full-run comparisons and
|
||||
cross-owner consumers, including failures, and measure completed CI feedback and
|
||||
runner demand. There is no automatic promotion or switch to skip tests. The
|
||||
initial proposal measures only leaf producer-test omissions; it does not promise
|
||||
the savings of a broader affected-test planner.
|
||||
Pull requests with code changes run only the portable tests the change
|
||||
affects; main pushes, scheduled and manual runs, release qualification and
|
||||
`ci:full` run the complete suite. See
|
||||
[pull-request test selection](development/testing.md#pull-request-test-selection)
|
||||
for the selection rules, fallbacks and gate.
|
||||
|
||||
`scripts/ci/plan.mjs` selects lanes using the ownership table in
|
||||
`scripts/ci/scopes.mjs`. Each lane record owns its path and test selection
|
||||
@@ -739,13 +722,14 @@ push: changes from failed or cancelled intermediate runs remain in scope.
|
||||
Missing, untrusted, or non-ancestor history selects the fast Linux x64
|
||||
baseline: portable checks and the Linux runtime lane. The lookup is
|
||||
bounded to the last 100 successful push runs; lookup failures broaden selection.
|
||||
Every code change still selects the source group and every fast built shard, without coverage
|
||||
Pull requests run their selected tests, or the complete suite when selection
|
||||
falls back; main pushes run the source group and every fast built shard, without coverage
|
||||
instrumentation in this fast tier. Ordinary PR matrices cancel remaining
|
||||
shards after the first failure; passing PRs execute the complete suite. Main,
|
||||
shards after the first failure. Main,
|
||||
`ci:full`, scheduled, manual, and release qualification retain every shard's
|
||||
results after a failure. Available failed-shard reports remain diagnostic only:
|
||||
failed, cancelled, skipped, or missing selected shards cannot pass the gate or
|
||||
publish successful timing history. This deliberately avoids relying on an
|
||||
publish successful timing history. Complete-suite runs do not rely on an
|
||||
import graph for generated files, filesystem inputs, subprocesses, or bridges.
|
||||
The portable report rejects an entirely skipped module inventory and lists
|
||||
fully skipped modules in its summary. A mixed passed/skipped inventory is
|
||||
@@ -930,10 +914,6 @@ Main warms the new cache namespace after merge; a cold cache falls back to the
|
||||
normal install. This prevents each PR from saving another branch-scoped copy of
|
||||
the same download cache. It does not remove runner assignment waits.
|
||||
|
||||
Portable owner-based test selection remains observational. Its reports compare
|
||||
proposed omissions with the full run; incomplete or skipped observations do not
|
||||
establish that fewer tests would preserve regression detection.
|
||||
|
||||
The automatic browser, binary-layout, EVM, JavaScript-recovery, packet-capture,
|
||||
signal-recording, recorded-crash, web and environment-bundle provider lanes download
|
||||
`rea-build-cache` from their own run after the build and static jobs succeed. Their existing build commands still
|
||||
|
||||
@@ -15,6 +15,14 @@ export interface ChangePlanRepository {
|
||||
readonly isAncestor: (ancestor: string, head: string) => boolean;
|
||||
readonly changedFiles: (base: string, head: string) => readonly string[];
|
||||
readonly readFile: (revision: string, path: string) => string;
|
||||
readonly listFiles: (
|
||||
revision: string,
|
||||
directories: readonly string[],
|
||||
) => readonly string[];
|
||||
readonly readFiles: (
|
||||
revision: string,
|
||||
paths: readonly string[],
|
||||
) => ReadonlyMap<string, string>;
|
||||
}
|
||||
|
||||
export type ChangePlan = Readonly<Record<string, unknown>> & {
|
||||
@@ -29,6 +37,7 @@ export type ChangePlan = Readonly<Record<string, unknown>> & {
|
||||
readonly files: readonly string[];
|
||||
readonly reasons: readonly string[];
|
||||
readonly testSelection: TestSelectionPlan;
|
||||
readonly testShards: readonly string[];
|
||||
};
|
||||
|
||||
export function planChanges(
|
||||
|
||||
+50
-13
@@ -5,9 +5,10 @@ import {
|
||||
CI_FLAGS as flags,
|
||||
MANUAL_VERIFIERS,
|
||||
CI_LANES,
|
||||
SHARED_BOUNDARY,
|
||||
WINDOWS_EXECUTABLE_SELECTION_INPUTS,
|
||||
} from "./scopes.mjs";
|
||||
import { planTestSelection } from "./test-selection.mjs";
|
||||
import { planTestSelection, testShards } from "./test-selection.mjs";
|
||||
|
||||
// Actionlint validates workflow syntax only. The semantic suite asserts these
|
||||
// workflows' release, publication and self-hosted runner safeguards.
|
||||
@@ -58,11 +59,6 @@ const matchesAny = (file, patterns) =>
|
||||
export function planChanges(request, repository) {
|
||||
const selected = Object.fromEntries(flags.map((flag) => [flag, false]));
|
||||
const reasons = [];
|
||||
// Shared process, contract, CLI, and CI-script edits are not a specific provider.
|
||||
// Fast qualification keeps the portable suite and the Linux x64 runtime
|
||||
// baseline. Full qualification still selects every lane.
|
||||
const SHARED_BOUNDARY =
|
||||
/^(src\/process\/|src\/filesystem\/|src\/contracts\/tool|src\/server\/createServer|src\/cli\.ts|scripts\/ci\/)/;
|
||||
const full = (reason) => {
|
||||
if (tier === "full") {
|
||||
for (const flag of flags) selected[flag] = true;
|
||||
@@ -269,11 +265,7 @@ export function planChanges(request, repository) {
|
||||
selected.readiness = true;
|
||||
}
|
||||
if (file.startsWith("tests/")) return;
|
||||
if (
|
||||
/^(src\/process\/|src\/filesystem\/|src\/contracts\/tool|src\/server\/createServer|src\/cli\.ts|scripts\/ci\/)/.test(
|
||||
file,
|
||||
)
|
||||
) {
|
||||
if (SHARED_BOUNDARY.test(file)) {
|
||||
full(`Shared boundary: ${file}`);
|
||||
} else if (owners.length) {
|
||||
if (file.startsWith("src/") || file.startsWith("bridge/"))
|
||||
@@ -343,8 +335,26 @@ export function planChanges(request, repository) {
|
||||
]
|
||||
: [linux],
|
||||
};
|
||||
const testSelection = planTestSelection(files, event, tier);
|
||||
return { ...selected, tier, matrix, files, reasons, testSelection };
|
||||
const testSelection = planTestSelection({
|
||||
files,
|
||||
event,
|
||||
tier,
|
||||
code: selected.code,
|
||||
// The checked-out revision is the one whose tests run.
|
||||
snapshot: {
|
||||
list: () => repository.listFiles("HEAD", ["src", "tests"]),
|
||||
read: (paths) => repository.readFiles("HEAD", paths),
|
||||
},
|
||||
});
|
||||
return {
|
||||
...selected,
|
||||
tier,
|
||||
matrix,
|
||||
files,
|
||||
reasons,
|
||||
testSelection,
|
||||
testShards: testShards(tier, testSelection),
|
||||
};
|
||||
}
|
||||
|
||||
/** Answer the plan's Git questions from the repository in `cwd`. */
|
||||
@@ -370,5 +380,32 @@ export const gitRepository = (cwd = process.cwd()) => {
|
||||
.split("\0")
|
||||
.filter(Boolean),
|
||||
readFile: (revision, path) => git("show", `${revision}:${path}`),
|
||||
listFiles: (revision, directories) =>
|
||||
git("ls-tree", "-r", "-z", "--name-only", revision, "--", ...directories)
|
||||
.split("\0")
|
||||
.filter(Boolean),
|
||||
// One batch process reads every requested blob.
|
||||
readFiles: (revision, paths) => {
|
||||
const output = execFileSync("git", ["cat-file", "--batch"], {
|
||||
cwd,
|
||||
input: paths.map((path) => `${revision}:${path}\n`).join(""),
|
||||
maxBuffer: 512 * 1024 * 1024,
|
||||
});
|
||||
const contents = new Map();
|
||||
let offset = 0;
|
||||
for (const path of paths) {
|
||||
const end = output.indexOf(10, offset);
|
||||
const [, type, size] = output.toString("utf8", offset, end).split(" ");
|
||||
if (type !== "blob")
|
||||
throw new Error(`Git could not read ${revision}:${path}`);
|
||||
const start = end + 1;
|
||||
contents.set(
|
||||
path,
|
||||
output.toString("utf8", start, start + Number(size)),
|
||||
);
|
||||
offset = start + Number(size) + 1;
|
||||
}
|
||||
return contents;
|
||||
},
|
||||
};
|
||||
};
|
||||
+4
-2
@@ -14,7 +14,7 @@ const plan = planChanges(
|
||||
},
|
||||
gitRepository(),
|
||||
);
|
||||
const { tier, matrix, reasons, testSelection } = plan;
|
||||
const { tier, matrix, reasons, testSelection, testShards } = plan;
|
||||
const selected = Object.fromEntries(flags.map((flag) => [flag, plan[flag]]));
|
||||
if (process.env.GITHUB_OUTPUT)
|
||||
appendFileSync(
|
||||
@@ -24,6 +24,8 @@ if (process.env.GITHUB_OUTPUT)
|
||||
tier,
|
||||
matrix: JSON.stringify(matrix),
|
||||
test_selection: JSON.stringify(testSelection),
|
||||
test_mode: testSelection.mode,
|
||||
test_shards: JSON.stringify(testShards),
|
||||
})
|
||||
.map(([key, value]) => `${key}=${value}\n`)
|
||||
.join(""),
|
||||
@@ -31,6 +33,6 @@ if (process.env.GITHUB_OUTPUT)
|
||||
if (process.env.GITHUB_STEP_SUMMARY)
|
||||
appendFileSync(
|
||||
process.env.GITHUB_STEP_SUMMARY,
|
||||
`### Selected CI lanes (${tier})\n\n${flags.filter((flag) => selected[flag]).join(", ") || "Formatting only"}\n\n${reasons.join("\n\n")}\n`,
|
||||
`### Selected CI lanes (${tier})\n\n${flags.filter((flag) => selected[flag]).join(", ") || "Formatting only"}\n\n${reasons.join("\n\n")}\n\nPortable tests (${testSelection.mode}): ${testSelection.reason}\n`,
|
||||
);
|
||||
console.log(JSON.stringify(plan));
|
||||
+16
-1
@@ -28,10 +28,25 @@ if (
|
||||
);
|
||||
if (outputs.tier === "full" && selectedScopes.length !== CI_FLAGS.length)
|
||||
throw new Error("Full qualification requires every canonical CI scope");
|
||||
// Only a fast pull-request plan may run a selected test subset.
|
||||
const testMode = outputs.test_mode;
|
||||
if (
|
||||
!["full", "selected"].includes(testMode) ||
|
||||
(testMode === "selected" &&
|
||||
(outputs.tier !== "fast" ||
|
||||
process.env.GITHUB_EVENT_NAME !== "pull_request" ||
|
||||
outputs.code !== "true"))
|
||||
)
|
||||
throw new Error(
|
||||
"CI classification requires a valid test mode: full, or selected on a fast pull request",
|
||||
);
|
||||
const required = reused ? ["changes"] : ["changes", "static", "test"];
|
||||
if (outputs.tier === "full") required.push("static-deep");
|
||||
for (const scope of selectedScopes)
|
||||
required.push(...CI_LANES[scope].required[outputs.tier]);
|
||||
required.push(
|
||||
...((testMode === "selected" && CI_LANES[scope].selectedTests) ||
|
||||
CI_LANES[scope].required[outputs.tier]),
|
||||
);
|
||||
const failed = [...new Set(required)].filter(
|
||||
(name) => jobs[name]?.result !== "success",
|
||||
);
|
||||
|
||||
@@ -8,6 +8,13 @@ export const WINDOWS_EXECUTABLE_SELECTION_INPUTS = [
|
||||
"tests/boundary/process/windowsExecutableSelection.test.ts",
|
||||
];
|
||||
|
||||
// Shared process, contract, CLI, and CI-script edits are not a specific provider.
|
||||
// Fast qualification keeps the portable suite and the Linux x64 runtime
|
||||
// baseline, and pull requests run the complete portable suite rather than a
|
||||
// selected subset. Full qualification still selects every lane.
|
||||
export const SHARED_BOUNDARY =
|
||||
/^(src\/process\/|src\/filesystem\/|src\/contracts\/tool|src\/server\/createServer|src\/cli\.ts|scripts\/ci\/)/;
|
||||
|
||||
// Inputs whose changes must select both real capture lanes, even at shared boundaries.
|
||||
const CAPTURE_SHARED_INPUTS = [
|
||||
"src/process/LinuxResourceLimits*",
|
||||
@@ -679,6 +686,8 @@ export const CI_LANES = {
|
||||
fast: ["build", "test-shard", "test-source"],
|
||||
full: ["build", "test-shard"],
|
||||
},
|
||||
// A selected PR subset runs its source tests in the test-shard groups.
|
||||
selectedTests: ["build", "test-shard"],
|
||||
},
|
||||
runtime: sameJobs([], ["windows-native-build", "package-e2e"]),
|
||||
package: sameJobs(PATH_SCOPES.package, ["windows-curated"], {
|
||||
|
||||
+38
-10
@@ -5,17 +5,48 @@ import {
|
||||
BUILT_TEST_PROJECTS,
|
||||
SOURCE_TEST_PROJECTS,
|
||||
} from "../lib/test-projects.mjs";
|
||||
import { parseTestSelection, testShards } from "./test-selection.mjs";
|
||||
|
||||
const [group, ...arguments_] = process.argv.slice(2);
|
||||
const projects = { source: SOURCE_TEST_PROJECTS, built: BUILT_TEST_PROJECTS }[
|
||||
group
|
||||
];
|
||||
if (!projects) throw new Error("Expected source or built test group");
|
||||
// The group owns project selection; callers can request reporters or shards.
|
||||
// The group owns project and file selection; callers can request reporters
|
||||
// or (for project groups) shards.
|
||||
if (
|
||||
arguments_.some((argument) => /^--(?:project|exclude)(?:=|$)/u.test(argument))
|
||||
)
|
||||
throw new Error("Test groups cannot override their project inventory");
|
||||
let runner;
|
||||
let environment = process.env;
|
||||
if (group === "selected") {
|
||||
// `selected <k/n>` runs group k of the plan's selected test files, by exact
|
||||
// identity, in whichever projects own them.
|
||||
const [shard, ...rest] = arguments_;
|
||||
const plan = parseTestSelection(process.env.CI_TEST_SELECTION ?? "");
|
||||
if (plan.mode !== "selected" || !testShards("fast", plan).includes(shard))
|
||||
throw new Error(`No selected test group ${shard ?? ""} in this plan`);
|
||||
if (rest.some((argument) => /^--shard(?:=|$)/u.test(argument)))
|
||||
throw new Error("Selected test groups are already sharded");
|
||||
const index = Number(shard.split("/")[0]) - 1;
|
||||
runner = [
|
||||
resolve("scripts/run-exact-tests.mjs"),
|
||||
"run",
|
||||
...rest,
|
||||
...plan.groups[index],
|
||||
];
|
||||
// Each group writes its own blob report for the merge.
|
||||
environment = { ...process.env, VITEST_BLOB_LABEL: `selected-${index + 1}` };
|
||||
} else {
|
||||
const projects = {
|
||||
source: SOURCE_TEST_PROJECTS,
|
||||
built: BUILT_TEST_PROJECTS,
|
||||
}[group];
|
||||
if (!projects) throw new Error("Expected source, built or selected tests");
|
||||
runner = [
|
||||
resolve("node_modules/vitest/vitest.mjs"),
|
||||
"run",
|
||||
...projects.flatMap(({ name }) => ["--project", name]),
|
||||
...arguments_,
|
||||
];
|
||||
}
|
||||
const child = spawn(
|
||||
process.execPath,
|
||||
[
|
||||
@@ -24,12 +55,9 @@ const child = spawn(
|
||||
...(process.platform === "win32"
|
||||
? []
|
||||
: [resolve("scripts/run-exclusive.mjs"), "--unlocked", process.execPath]),
|
||||
resolve("node_modules/vitest/vitest.mjs"),
|
||||
"run",
|
||||
...projects.flatMap(({ name }) => ["--project", name]),
|
||||
...arguments_,
|
||||
...runner,
|
||||
],
|
||||
{ stdio: "inherit" },
|
||||
{ stdio: "inherit", env: environment },
|
||||
);
|
||||
let shutdownSignal;
|
||||
for (const signal of ["SIGINT", "SIGTERM"])
|
||||
|
||||
+19
-41
@@ -4,16 +4,13 @@ import { isAbsolute, join, relative, resolve, sep } from "node:path";
|
||||
|
||||
import { z } from "zod";
|
||||
|
||||
import { compareTestSelection } from "./test-selection.mjs";
|
||||
import { parseTestSelection } from "./test-selection.mjs";
|
||||
|
||||
const root = process.cwd();
|
||||
const historyDirectory = join(root, ".cache/ci-test-history");
|
||||
const historyPath = join(historyDirectory, "test-results.json");
|
||||
const selectionDirectory = join(root, ".cache/ci-test-selection");
|
||||
const selectionPath = join(selectionDirectory, "report.json");
|
||||
// Never let a failed or partial rerun leave a previous successful history file.
|
||||
await rm(historyPath, { force: true });
|
||||
await rm(selectionPath, { force: true });
|
||||
const report = z
|
||||
.object({
|
||||
version: z.literal(1),
|
||||
@@ -56,40 +53,23 @@ const passed =
|
||||
process.env.MERGE_RESULT === "success" &&
|
||||
report.success;
|
||||
// Successful shards and merged reports must account for the complete portable
|
||||
// inventory. Coverage thresholds alone cannot prove that every file ran.
|
||||
// inventory, or exactly the selected subset. Coverage thresholds alone cannot
|
||||
// prove that every file ran.
|
||||
const verifyInventory = process.env.CI_VERIFY_TEST_INVENTORY === "true";
|
||||
const expected =
|
||||
verifyInventory || process.env.CI_TEST_SELECTION
|
||||
? execFileSync("git", ["ls-files", "-z", "--", "src", "tests"], {
|
||||
const selection = process.env.CI_TEST_SELECTION
|
||||
? parseTestSelection(process.env.CI_TEST_SELECTION)
|
||||
: undefined;
|
||||
const selected = selection?.mode === "selected";
|
||||
const expected = !verifyInventory
|
||||
? []
|
||||
: selected
|
||||
? selection.groups.flat()
|
||||
: execFileSync("git", ["ls-files", "-z", "--", "src", "tests"], {
|
||||
encoding: "utf8",
|
||||
maxBuffer: 8 * 1024 * 1024,
|
||||
})
|
||||
.split("\0")
|
||||
.filter((path) => path.endsWith(".test.ts"))
|
||||
: [];
|
||||
let selection;
|
||||
if (process.env.CI_TEST_SELECTION) {
|
||||
const comparison = compareTestSelection(
|
||||
JSON.parse(process.env.CI_TEST_SELECTION),
|
||||
expected,
|
||||
files,
|
||||
);
|
||||
selection = {
|
||||
...comparison,
|
||||
complete:
|
||||
comparison.complete &&
|
||||
report.unhandledErrors === 0 &&
|
||||
["passed", "failed"].includes(report.reason) &&
|
||||
groupsCompleted &&
|
||||
["success", "failure"].includes(process.env.MERGE_RESULT),
|
||||
executionSha: process.env.GITHUB_SHA ?? null,
|
||||
prHeadSha: process.env.CI_PR_HEAD_SHA ?? null,
|
||||
runId: process.env.GITHUB_RUN_ID ?? null,
|
||||
runAttempt: process.env.GITHUB_RUN_ATTEMPT ?? null,
|
||||
};
|
||||
await mkdir(selectionDirectory, { recursive: true });
|
||||
await writeFile(selectionPath, JSON.stringify(selection));
|
||||
}
|
||||
.filter((path) => path.endsWith(".test.ts"));
|
||||
if (passed && verifyInventory) {
|
||||
const actual = files.map(({ name }) => name);
|
||||
if (actual.length === 0)
|
||||
@@ -100,11 +80,13 @@ if (passed && verifyInventory) {
|
||||
const unexpected = actual.filter((path) => !expectedSet.has(path));
|
||||
if (missing.length || unexpected.length || actualSet.size !== actual.length)
|
||||
throw new Error(
|
||||
`Incomplete portable test inventory: ${JSON.stringify({ missing, unexpected, duplicateFiles: actual.length - actualSet.size })}`,
|
||||
`Incomplete ${selected ? "selected" : "portable"} test inventory: ${JSON.stringify({ missing, unexpected, duplicateFiles: actual.length - actualSet.size })}`,
|
||||
);
|
||||
if (files.every(({ state }) => state === "skipped"))
|
||||
throw new Error("Every portable test module was skipped");
|
||||
}
|
||||
// A selected subset is not a timing baseline for the complete suite.
|
||||
const publishesHistory = passed && !selected;
|
||||
const skippedFiles = files.filter(({ state }) => state === "skipped");
|
||||
const summary = [
|
||||
groupsCompleted
|
||||
@@ -113,11 +95,7 @@ const summary = [
|
||||
"",
|
||||
...(selection
|
||||
? [
|
||||
"### Observational PR test selection",
|
||||
"",
|
||||
`Plan: ${selection.plan.mode}. Proposed files: ${selection.selected.length}; omitted: ${selection.omitted.length}.`,
|
||||
`Full-run comparison: ${selection.complete ? "complete" : "incomplete"}; failures outside proposed selection: ${selection.missedFailures.length}.`,
|
||||
"This observational report cannot enable omissions or qualify a revision. Skipped, missing, and interrupted observations do not establish parity.",
|
||||
`Portable tests: ${selected ? `${selection.groups.flat().length} selected files` : "complete suite"}. ${selection.reason}.`,
|
||||
"",
|
||||
]
|
||||
: []),
|
||||
@@ -129,7 +107,7 @@ const summary = [
|
||||
...(skippedFiles.length ? [""] : []),
|
||||
...skippedFiles.map(({ name }) => `- ${name.replaceAll("|", "\\|")}`),
|
||||
"",
|
||||
passed
|
||||
publishesHistory
|
||||
? "Successful timing history is available."
|
||||
: "This report is diagnostic only and is not a successful timing baseline.",
|
||||
"",
|
||||
@@ -150,7 +128,7 @@ if (process.env.GITHUB_STEP_SUMMARY)
|
||||
await appendFile(process.env.GITHUB_STEP_SUMMARY, summary);
|
||||
else console.log(summary);
|
||||
|
||||
if (passed) {
|
||||
if (publishesHistory) {
|
||||
await mkdir(historyDirectory, { recursive: true });
|
||||
await writeFile(
|
||||
historyPath,
|
||||
|
||||
@@ -1,35 +1,25 @@
|
||||
export interface TestSelectionPlan {
|
||||
readonly version: 1;
|
||||
readonly mode: "full" | "candidate";
|
||||
readonly owners: readonly string[];
|
||||
readonly changedTests: readonly string[];
|
||||
readonly changedPaths: readonly string[];
|
||||
readonly reason: string;
|
||||
}
|
||||
export function planTestSelection(
|
||||
files: readonly string[],
|
||||
event: string,
|
||||
tier: string,
|
||||
): TestSelectionPlan;
|
||||
export function compareTestSelection(
|
||||
plan: unknown,
|
||||
inventory: readonly string[],
|
||||
results: readonly {
|
||||
readonly name: string;
|
||||
readonly durationMs: number;
|
||||
readonly state?: string;
|
||||
}[],
|
||||
): {
|
||||
readonly version: 1;
|
||||
readonly observational: true;
|
||||
readonly plan: TestSelectionPlan;
|
||||
readonly selected: readonly string[];
|
||||
readonly omitted: readonly string[];
|
||||
readonly missing: readonly string[];
|
||||
readonly unexpected: readonly string[];
|
||||
readonly duplicateFiles: number;
|
||||
readonly unobserved: readonly string[];
|
||||
readonly missedFailures: readonly string[];
|
||||
readonly complete: boolean;
|
||||
readonly omittedDurationMs: number;
|
||||
};
|
||||
export type TestSelectionPlan =
|
||||
| {
|
||||
readonly version: 2;
|
||||
readonly mode: "full";
|
||||
readonly reason: string;
|
||||
readonly groups: readonly [];
|
||||
}
|
||||
| {
|
||||
readonly version: 2;
|
||||
readonly mode: "selected";
|
||||
readonly reason: string;
|
||||
readonly groups: readonly (readonly string[])[];
|
||||
};
|
||||
export function planTestSelection(request: {
|
||||
readonly files: readonly string[];
|
||||
readonly event: string;
|
||||
readonly tier: string;
|
||||
readonly code: boolean;
|
||||
readonly snapshot: {
|
||||
readonly list: () => readonly string[];
|
||||
readonly read: (paths: readonly string[]) => ReadonlyMap<string, string>;
|
||||
};
|
||||
}): TestSelectionPlan;
|
||||
export function parseTestSelection(value: unknown): TestSelectionPlan;
|
||||
export function testShards(tier: string, plan: TestSelectionPlan): string[];
|
||||
+213
-140
@@ -1,151 +1,224 @@
|
||||
import { matchesGlob } from "node:path";
|
||||
import { matchesGlob, posix } from "node:path";
|
||||
|
||||
// Candidate ownership only: these leaf adapters and their producer tests can be
|
||||
// compared with full-suite outcomes. This table never changes test execution.
|
||||
// Shared application/domain code and dynamic CLI/MCP consumers are not leaf owners.
|
||||
const OWNERS = {
|
||||
adb: ["src/adb/**", "tests/boundary/providers/adb/**"],
|
||||
apktool: ["src/apktool/**"],
|
||||
dotnet: ["src/dotnet/**"],
|
||||
evm: ["src/evm/**", "tests/boundary/evm/**"],
|
||||
firmware: ["src/firmware/**", "tests/boundary/firmware/**"],
|
||||
flutter: ["src/flutter/**"],
|
||||
ghidra: ["src/ghidra/**", "tests/boundary/providers/ghidra/**"],
|
||||
hopper: ["src/hopper/**", "tests/boundary/providers/hopper/**"],
|
||||
ida: ["src/ida/**", "tests/boundary/providers/ida/**"],
|
||||
jeb: ["src/jeb/**"],
|
||||
pcap: ["src/pcap/**", "tests/boundary/pcap/**"],
|
||||
sqlite: ["src/sqlite/**", "tests/boundary/sqlite/**"],
|
||||
wasm: ["src/wasm/**"],
|
||||
};
|
||||
const ownersOf = (path) =>
|
||||
Object.entries(OWNERS)
|
||||
.filter(([, patterns]) => patterns.some((glob) => matchesGlob(path, glob)))
|
||||
.map(([owner]) => owner);
|
||||
const validPath = (path) =>
|
||||
typeof path === "string" &&
|
||||
/^(src|tests)\//.test(path) &&
|
||||
!/[\\\0\r\n]/.test(path) &&
|
||||
!path.split("/").some((part) => ["", ".", ".."].includes(part));
|
||||
const full = (reason, changedPaths = []) => ({
|
||||
version: 1,
|
||||
mode: "full",
|
||||
owners: [],
|
||||
changedTests: [],
|
||||
changedPaths,
|
||||
reason,
|
||||
});
|
||||
import { projectForTest } from "../lib/test-projects.mjs";
|
||||
import { SHARED_BOUNDARY } from "./scopes.mjs";
|
||||
|
||||
/** Plan an observational PR subset from the classifier's existing Git diff. */
|
||||
export function planTestSelection(files, event, tier) {
|
||||
// Pull-request feedback runs the tests a change most directly affects. Main
|
||||
// pushes, schedules, releases and `ci:full` keep the complete suite, so a
|
||||
// consumer the selection misses fails on main right after it lands.
|
||||
// Selection falls back to the complete suite whenever it cannot decide.
|
||||
|
||||
/** More selected files than this run the complete suite instead. */
|
||||
const SELECTION_CAP = 150;
|
||||
/** Selected files per test job, and the most jobs one selection uses. */
|
||||
const FILES_PER_GROUP = 40;
|
||||
const MAX_GROUPS = 4;
|
||||
|
||||
// Leaf adapters whose tests also reach their sources through the built CLI,
|
||||
// MCP server or a fake provider process rather than an import. An adapter
|
||||
// owns `src/<name>/` and every test named for it (`<name>*.test.ts`) or
|
||||
// filed under a `<name>/` directory anywhere in `tests/`.
|
||||
const OWNERS = [
|
||||
"adb",
|
||||
"apktool",
|
||||
"dotnet",
|
||||
"evm",
|
||||
"firmware",
|
||||
"flutter",
|
||||
"ghidra",
|
||||
"hopper",
|
||||
"ida",
|
||||
"jeb",
|
||||
"pcap",
|
||||
"sqlite",
|
||||
"wasm",
|
||||
];
|
||||
const ownedTests = (path) =>
|
||||
OWNERS.filter((owner) => path.startsWith(`src/${owner}/`)).flatMap(
|
||||
(owner) => [`tests/**/${owner}/**/*.test.ts`, `tests/**/${owner}*.test.ts`],
|
||||
);
|
||||
|
||||
const isTest = (path) => path.endsWith(".test.ts");
|
||||
// Inputs that only a non-test check reads: declarations (typecheck), guard
|
||||
// baselines (architecture guards, static job) and the advertised-schema
|
||||
// summaries (build job). They select no tests.
|
||||
const isCheckerInput = (path) =>
|
||||
/\.d\.m?ts$/u.test(path) ||
|
||||
/^scripts\/verify\/baselines\/[^/]+\.json$/u.test(path) ||
|
||||
/^scripts\/verify\/advertised-schemas\/tools\/[^/]+\.json$/u.test(path);
|
||||
// Verifier scripts are imported by their tests like source modules. Other
|
||||
// scripts (build, test runners, CI) shape every run and are not traced.
|
||||
const isModule = (path) =>
|
||||
!isCheckerInput(path) &&
|
||||
(/^(src|tests)\/.+\.m?ts$/u.test(path) ||
|
||||
/^scripts\/(verify-[^/]+|verify\/.+)\.mjs$/u.test(path));
|
||||
// Test-only modules pass a change through to the tests that use them.
|
||||
// Production importers never widen the selection transitively.
|
||||
const isHelper = (path) =>
|
||||
!isTest(path) &&
|
||||
(/^tests\/.+\.m?ts$/u.test(path) ||
|
||||
/^src\/.+\.fixture[^/]*\.ts$/u.test(path));
|
||||
// Prose that tests may read by path. Other non-module inputs (fixtures, data,
|
||||
// scripts, workflows, package metadata) are not traced, so they run everything.
|
||||
const isProse = (path) => path.endsWith(".md") && !/^(src|tests)\//u.test(path);
|
||||
|
||||
const SPECIFIER =
|
||||
/(?:\bfrom\s*|\bimport\s*\(\s*|\bimport\s+|\bvi\.mock\(\s*)["'](\.{1,2}\/[^"']+)["']/gu;
|
||||
// TypeScript imports name the emitted `.js`/`.mjs`; scripts are imported as is.
|
||||
const importsOf = (path, source) =>
|
||||
[...source.matchAll(SPECIFIER)].flatMap(([, specifier]) => {
|
||||
const target = posix.normalize(posix.join(posix.dirname(path), specifier));
|
||||
return [target, target.replace(/\.(m?)js$/u, ".$1ts")];
|
||||
});
|
||||
|
||||
const full = (reason) => ({ version: 2, mode: "full", reason, groups: [] });
|
||||
|
||||
/**
|
||||
* Plan the portable tests one CI run executes.
|
||||
*
|
||||
* `snapshot` lists the tracked `src/` and `tests/` paths of the revision under
|
||||
* test and reads them; it is consulted only when a subset is possible.
|
||||
*/
|
||||
export function planTestSelection({ files, event, tier, code, snapshot }) {
|
||||
// `ci:full` selects the full tier, so it also lands here.
|
||||
if (event !== "pull_request" || tier !== "fast")
|
||||
return full("Only ordinary PRs are candidates", files);
|
||||
return full("Main, ci:full and qualification runs execute every test");
|
||||
if (!code) return full("No portable tests run for this change");
|
||||
if (!files.length) return full("No changed paths available");
|
||||
const owners = new Set();
|
||||
const changedTests = [];
|
||||
for (const path of files) {
|
||||
if (!validPath(path))
|
||||
return full(`Shared or unknown input: ${path}`, files);
|
||||
const matches = ownersOf(path);
|
||||
if (!matches.length) return full(`Shared or unknown input: ${path}`, files);
|
||||
for (const owner of matches) owners.add(owner);
|
||||
if (path.endsWith(".test.ts")) changedTests.push(path);
|
||||
if (SHARED_BOUNDARY.test(path)) return full(`Shared boundary: ${path}`);
|
||||
if (!isModule(path) && !isProse(path) && !isCheckerInput(path))
|
||||
return full(`Input is not traced to tests: ${path}`);
|
||||
}
|
||||
|
||||
const tracked = snapshot.list();
|
||||
const trackedSet = new Set(tracked);
|
||||
const tests = tracked.filter(isTest);
|
||||
const helpers = tracked.filter(isHelper);
|
||||
const sources = snapshot.read([...tests, ...helpers]);
|
||||
const imports = new Map(
|
||||
[...tests, ...helpers].map((path) => [
|
||||
path,
|
||||
importsOf(path, sources.get(path) ?? ""),
|
||||
]),
|
||||
);
|
||||
|
||||
const selected = new Map();
|
||||
const select = (path, reason) => {
|
||||
if (trackedSet.has(path) && !selected.has(path)) selected.set(path, reason);
|
||||
};
|
||||
const changedModules = files.filter(
|
||||
(path) => isModule(path) && !isTest(path),
|
||||
);
|
||||
for (const path of files) if (isTest(path)) select(path, "changed");
|
||||
|
||||
// Close over test-only helpers, then take the tests that import a change.
|
||||
const affected = new Set(changedModules);
|
||||
for (let grew = true; grew;) {
|
||||
grew = false;
|
||||
for (const helper of helpers)
|
||||
if (
|
||||
!affected.has(helper) &&
|
||||
imports.get(helper).some((target) => affected.has(target))
|
||||
) {
|
||||
affected.add(helper);
|
||||
grew = true;
|
||||
}
|
||||
}
|
||||
for (const test of tests)
|
||||
if (imports.get(test).some((target) => affected.has(target)))
|
||||
select(test, "imports");
|
||||
|
||||
for (const path of changedModules) {
|
||||
// Same-stem tests, and same-directory tests that scan their directory's
|
||||
// sources by path instead of importing them.
|
||||
const stem = path.replace(/\.m?ts$/u, "");
|
||||
for (const test of tests)
|
||||
if (
|
||||
posix.dirname(test) === posix.dirname(path) &&
|
||||
(test.startsWith(`${stem}.`) ||
|
||||
/\breaddir(?:Sync)?\s*\(/u.test(sources.get(test) ?? ""))
|
||||
)
|
||||
select(test, "beside");
|
||||
// Leaf adapters' boundary tests.
|
||||
const patterns = ownedTests(path);
|
||||
for (const test of tests)
|
||||
if (patterns.some((glob) => matchesGlob(test, glob)))
|
||||
select(test, "owner");
|
||||
}
|
||||
// Tests that name a changed path read it directly.
|
||||
for (const path of files)
|
||||
for (const test of tests)
|
||||
if ((sources.get(test) ?? "").includes(path)) select(test, "names");
|
||||
|
||||
const paths = [...selected.keys()].sort();
|
||||
if (!paths.length)
|
||||
return full("No test imports, names or sits beside a changed path");
|
||||
if (paths.length > SELECTION_CAP)
|
||||
return full(
|
||||
`${paths.length} affected test files exceed the ${SELECTION_CAP}-file selection cap`,
|
||||
);
|
||||
for (const path of paths)
|
||||
try {
|
||||
projectForTest(path);
|
||||
} catch {
|
||||
return full(`No single test project runs ${path}`);
|
||||
}
|
||||
const count = Math.min(MAX_GROUPS, Math.ceil(paths.length / FILES_PER_GROUP));
|
||||
const groups = Array.from({ length: count }, () => []);
|
||||
paths.forEach((path, index) => groups[index % count].push(path));
|
||||
const reasons = Object.groupBy([...selected.values()], (reason) => reason);
|
||||
return {
|
||||
version: 1,
|
||||
mode: "candidate",
|
||||
owners: [...owners].sort(),
|
||||
changedTests: [...new Set(changedTests)].sort(),
|
||||
changedPaths: files,
|
||||
reason:
|
||||
"Observe affected leaf owners; retain integration and unowned tests",
|
||||
version: 2,
|
||||
mode: "selected",
|
||||
reason: `Affected tests: ${Object.entries(reasons)
|
||||
.map(([reason, entries]) => `${entries.length} ${reason}`)
|
||||
.join(", ")}`,
|
||||
groups,
|
||||
};
|
||||
}
|
||||
|
||||
/** Compare a proposal with the full run; missing observations never prove parity. */
|
||||
export function compareTestSelection(plan, inventory, results) {
|
||||
if (
|
||||
plan?.version !== 1 ||
|
||||
!["full", "candidate"].includes(plan.mode) ||
|
||||
!Array.isArray(plan.owners) ||
|
||||
plan.owners.some((owner) => !Object.hasOwn(OWNERS, owner)) ||
|
||||
!Array.isArray(plan.changedTests) ||
|
||||
!Array.isArray(plan.changedPaths) ||
|
||||
plan.changedPaths.some((path) => typeof path !== "string") ||
|
||||
plan.changedTests.some(
|
||||
(path) => !validPath(path) || !path.endsWith(".test.ts"),
|
||||
) ||
|
||||
typeof plan.reason !== "string" ||
|
||||
(plan.mode === "candidate" && !plan.owners.length) ||
|
||||
(plan.mode === "full" && (plan.owners.length || plan.changedTests.length))
|
||||
)
|
||||
throw new Error("Invalid observational test selection plan");
|
||||
const expected = new Set(inventory);
|
||||
if (!expected.size || expected.size !== inventory.length)
|
||||
throw new Error("Test selection requires a nonempty unique inventory");
|
||||
const ownersByFile = new Map(
|
||||
plan.mode === "candidate"
|
||||
? inventory.map((path) => [path, ownersOf(path)])
|
||||
: [],
|
||||
);
|
||||
// Removed/renamed tests have no current executable identity. Keep all tests.
|
||||
const removed = plan.changedTests.filter((path) => !expected.has(path));
|
||||
const missingOwners = plan.owners.filter(
|
||||
(owner) =>
|
||||
![...ownersByFile.values()].some((owners) => owners.includes(owner)),
|
||||
);
|
||||
const effective = removed.length
|
||||
? full(
|
||||
`Changed tests absent from current inventory: ${removed.join(", ")}`,
|
||||
plan.changedPaths,
|
||||
)
|
||||
: missingOwners.length
|
||||
? full(
|
||||
`Owners without executable tests: ${missingOwners.join(", ")}`,
|
||||
plan.changedPaths,
|
||||
)
|
||||
: plan;
|
||||
const selected = [];
|
||||
const omitted = [];
|
||||
for (const path of inventory.toSorted()) {
|
||||
const owners = ownersByFile.get(path) ?? [];
|
||||
const keep =
|
||||
effective.mode === "full" ||
|
||||
!owners.length ||
|
||||
effective.changedTests.includes(path) ||
|
||||
owners.some((owner) => effective.owners.includes(owner));
|
||||
(keep ? selected : omitted).push(path);
|
||||
}
|
||||
const actual = new Set(results.map((file) => file.name));
|
||||
const missing = inventory.filter((path) => !actual.has(path));
|
||||
const unexpected = [...actual].filter((path) => !expected.has(path));
|
||||
const duplicateFiles = results.length - actual.size;
|
||||
const unobserved = results
|
||||
.filter((file) => !["passed", "failed"].includes(file.state))
|
||||
.map((file) => file.name);
|
||||
const omittedSet = new Set(omitted);
|
||||
const missedFailures = results
|
||||
.filter((file) => omittedSet.has(file.name) && file.state === "failed")
|
||||
.map((file) => file.name);
|
||||
return {
|
||||
version: 1,
|
||||
observational: true,
|
||||
plan: effective,
|
||||
selected,
|
||||
omitted,
|
||||
missing,
|
||||
unexpected,
|
||||
duplicateFiles,
|
||||
unobserved,
|
||||
missedFailures,
|
||||
complete:
|
||||
!missing.length &&
|
||||
!unexpected.length &&
|
||||
!duplicateFiles &&
|
||||
!unobserved.length,
|
||||
// Phase totals are scheduling weights, not an estimate of wall-time savings.
|
||||
omittedDurationMs: results
|
||||
.filter((file) => omittedSet.has(file.name))
|
||||
.reduce((sum, file) => sum + file.durationMs, 0),
|
||||
};
|
||||
/** Parse a plan produced by `planTestSelection`; anything else fails closed. */
|
||||
export function parseTestSelection(value) {
|
||||
const plan = typeof value === "string" ? JSON.parse(value) : value;
|
||||
const valid =
|
||||
plan?.version === 2 &&
|
||||
typeof plan.reason === "string" &&
|
||||
Array.isArray(plan.groups) &&
|
||||
(plan.mode === "full"
|
||||
? plan.groups.length === 0
|
||||
: plan.mode === "selected" &&
|
||||
plan.groups.length >= 1 &&
|
||||
plan.groups.length <= MAX_GROUPS &&
|
||||
plan.groups.every(
|
||||
(group) =>
|
||||
Array.isArray(group) &&
|
||||
group.length > 0 &&
|
||||
group.every(
|
||||
(path) =>
|
||||
typeof path === "string" &&
|
||||
isTest(path) &&
|
||||
/^(src|tests)\//u.test(path) &&
|
||||
!path.split("/").some((part) => ["", ".", ".."].includes(part)),
|
||||
),
|
||||
) &&
|
||||
new Set(plan.groups.flat()).size === plan.groups.flat().length &&
|
||||
plan.groups.flat().length <= SELECTION_CAP);
|
||||
if (!valid) throw new Error("Invalid portable test selection plan");
|
||||
return plan;
|
||||
}
|
||||
|
||||
// Fast feedback uses one source group and twelve built groups. Shards are
|
||||
// balanced by recorded duration, but hosted runner speed varies, so the
|
||||
// slowest shard sets the critical path: at eight shards it ran 24% above the
|
||||
// mean shard. Smaller shards bound that excess at a few percent more runner
|
||||
// time. Full qualification retains six complete shards with coverage. A
|
||||
// selection runs one job per group.
|
||||
const shardsOf = (count) =>
|
||||
Array.from({ length: count }, (_, index) => `${index + 1}/${count}`);
|
||||
|
||||
/** The `test-shard` matrix labels for a tier and its test selection. */
|
||||
export const testShards = (tier, plan) =>
|
||||
shardsOf(
|
||||
plan.mode === "selected" ? plan.groups.length : tier === "full" ? 6 : 12,
|
||||
);
|
||||
@@ -168,6 +168,9 @@ function classifyCommit(
|
||||
mergeBase: (base) => base,
|
||||
isAncestor: () => true,
|
||||
changedFiles: () => paths,
|
||||
// No test reaches these synthetic paths; selection falls back to all.
|
||||
listFiles: () => [],
|
||||
readFiles: () => new Map(),
|
||||
readFile: (revision, path) => {
|
||||
if (path === "package.json" && packageJson !== undefined)
|
||||
return revision === PATH_BASE
|
||||
@@ -376,21 +379,114 @@ it("keeps unknown CI scripts on the fast Linux baseline", async () => {
|
||||
for (const flag of CI_FLAGS) expect(full[flag]).toBe(true);
|
||||
});
|
||||
|
||||
it("emits an observational owner proposal without changing ordinary or full portable execution", async () => {
|
||||
const { directory, base } = await fixture();
|
||||
const head = await change(directory, ["src/evm/Example.ts"]);
|
||||
expect(await classify(directory, base, head)).toMatchObject({
|
||||
code: true,
|
||||
testSelection: { mode: "candidate", owners: ["evm"] },
|
||||
it("selects the tests a pull request affects and runs everything when it cannot decide", async () => {
|
||||
const { directory } = await fixture();
|
||||
// The entrypoint's output file is not part of the change.
|
||||
await mkdir(join(directory, ".git", "info"), { recursive: true });
|
||||
await writeFile(join(directory, ".git", "info", "exclude"), "scope-output\n");
|
||||
const write = async (path: string, content: string) => {
|
||||
await mkdir(dirname(join(directory, path)), { recursive: true });
|
||||
await writeFile(join(directory, path), content);
|
||||
};
|
||||
const imports = (specifier: string) =>
|
||||
`import { value } from "${specifier}";\nvoid value;\n`;
|
||||
await write("src/evm/Example.ts", "export const value = 1;\n");
|
||||
await write("src/evm/Example.test.ts", imports("./Example.js"));
|
||||
await write("src/evm/Other.ts", "export const value = 2;\n");
|
||||
await write(
|
||||
"tests/support/evmHelper.ts",
|
||||
imports("../../src/evm/Example.js"),
|
||||
);
|
||||
// Through a test-only helper, through the leaf owner, and by name.
|
||||
await write(
|
||||
"tests/composition/helped.test.ts",
|
||||
imports("../support/evmHelper.js"),
|
||||
);
|
||||
await write("tests/boundary/evm/cli.test.ts", "// spawns the built CLI\n");
|
||||
await write(
|
||||
"tests/conformance/named.test.ts",
|
||||
'readFileSync("src/evm/Example.ts");\n',
|
||||
);
|
||||
await write(
|
||||
"tests/composition/unrelated.test.ts",
|
||||
imports("../../src/provider.js"),
|
||||
);
|
||||
const base = await commit(directory);
|
||||
const head = await change(
|
||||
directory,
|
||||
["src/evm/Example.ts"],
|
||||
"export const value = 3;\n",
|
||||
);
|
||||
const selected = [
|
||||
"src/evm/Example.test.ts",
|
||||
"tests/boundary/evm/cli.test.ts",
|
||||
"tests/composition/helped.test.ts",
|
||||
"tests/conformance/named.test.ts",
|
||||
];
|
||||
expect(await runPlanScript(directory, base, head)).toMatchObject({
|
||||
testSelection: { version: 2, mode: "selected", groups: [selected] },
|
||||
testShards: ["1/1"],
|
||||
});
|
||||
const output = await readFile(join(directory, "scope-output"), "utf8");
|
||||
expect(output).toContain("test_mode=selected\n");
|
||||
expect(output).toContain('test_shards=["1/1"]\n');
|
||||
|
||||
// ci:full, main pushes and shared or untraced inputs run the complete suite.
|
||||
expect(
|
||||
await runPlanScript(directory, base, head, "pull_request", true),
|
||||
).toMatchObject({
|
||||
code: true,
|
||||
testSelection: { mode: "full", owners: [] },
|
||||
testSelection: { mode: "full", groups: [] },
|
||||
testShards: ["1/6", "2/6", "3/6", "4/6", "5/6", "6/6"],
|
||||
});
|
||||
expect(await classify(directory, base, head, "push")).toMatchObject({
|
||||
testSelection: { mode: "full" },
|
||||
testShards: expect.arrayContaining(["12/12"]),
|
||||
});
|
||||
let previous = head;
|
||||
for (const path of [
|
||||
"src/process/Shared.ts",
|
||||
"scripts/build-something.mjs",
|
||||
"tests/fixtures/data.bin",
|
||||
]) {
|
||||
const parent = previous;
|
||||
previous = await change(directory, [path]);
|
||||
expect(await classify(directory, parent, previous)).toMatchObject({
|
||||
testSelection: { mode: "full", reason: expect.stringContaining(path) },
|
||||
});
|
||||
}
|
||||
// A leaf owner's boundary tests cover its other modules.
|
||||
const unreached = await change(
|
||||
directory,
|
||||
["src/evm/Other.ts"],
|
||||
"export const value = 4;\n",
|
||||
);
|
||||
expect(await classify(directory, previous, unreached)).toMatchObject({
|
||||
testSelection: { mode: "selected" },
|
||||
});
|
||||
const isolated = await change(directory, ["src/unowned/Lonely.ts"]);
|
||||
expect(await classify(directory, unreached, isolated)).toMatchObject({
|
||||
testSelection: {
|
||||
mode: "full",
|
||||
reason: "No test imports, names or sits beside a changed path",
|
||||
},
|
||||
});
|
||||
// Above the cap, selection gives way to the complete suite.
|
||||
const many = Array.from(
|
||||
{ length: 151 },
|
||||
(_, index) => `tests/composition/many${String(index)}.test.ts`,
|
||||
);
|
||||
const wide = await change(directory, many);
|
||||
expect(await classify(directory, isolated, wide)).toMatchObject({
|
||||
testSelection: {
|
||||
mode: "full",
|
||||
reason: "151 affected test files exceed the 150-file selection cap",
|
||||
},
|
||||
});
|
||||
const capped = await change(directory, many.slice(0, 81), "// edited\n");
|
||||
expect(await classify(directory, wide, capped)).toMatchObject({
|
||||
testSelection: { mode: "selected" },
|
||||
testShards: ["1/3", "2/3", "3/3"],
|
||||
});
|
||||
const output = await readFile(join(directory, "scope-output"), "utf8");
|
||||
expect(output).toContain('test_selection={"version":1,"mode":"full"');
|
||||
});
|
||||
|
||||
it("passes the workflow environment through the entrypoint unchanged", async () => {
|
||||
@@ -1468,6 +1564,7 @@ it.each(["evm", "browser"])(
|
||||
outputs: {
|
||||
...Object.fromEntries(CI_FLAGS.map((flag) => [flag, "false"])),
|
||||
tier: "fast",
|
||||
test_mode: "full",
|
||||
[scope]: "true",
|
||||
},
|
||||
},
|
||||
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
parseStaticTaskContext,
|
||||
recordStaticTaskContext,
|
||||
} from "../../../scripts/ci/static-task-context.mjs";
|
||||
import { testShards } from "../../../scripts/ci/test-selection.mjs";
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
|
||||
const execute = promisify(execFile);
|
||||
@@ -121,6 +122,7 @@ it.each(["fast", "full"] as const)(
|
||||
flags.map((flag) => [flag, tier === "full" ? "true" : "false"]),
|
||||
),
|
||||
tier,
|
||||
test_mode: "full",
|
||||
code: "true",
|
||||
reverse_tools: "true",
|
||||
},
|
||||
@@ -265,6 +267,7 @@ it("starts source feedback before build and requires both groups without waiving
|
||||
flags.map((flag) => [flag, tier === "full" ? "true" : "false"]),
|
||||
),
|
||||
tier,
|
||||
test_mode: "full",
|
||||
code: "true",
|
||||
};
|
||||
const context: Record<
|
||||
@@ -398,7 +401,20 @@ it("merges source and built reports exactly once using the workflow's shard matr
|
||||
matrix.slice(3, -2),
|
||||
{
|
||||
fromJSON: JSON.parse,
|
||||
needs: { changes: { outputs: { tier: "fast" } } },
|
||||
needs: {
|
||||
changes: {
|
||||
outputs: {
|
||||
test_shards: JSON.stringify(
|
||||
testShards("fast", {
|
||||
version: 2,
|
||||
mode: "full",
|
||||
reason: "Complete fast suite",
|
||||
groups: [],
|
||||
}),
|
||||
),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{ timeout: 1_000 },
|
||||
),
|
||||
|
||||
@@ -259,7 +259,10 @@ async function requiredRun(
|
||||
: "success",
|
||||
};
|
||||
}
|
||||
results.changes = { result: "success", outputs: changes };
|
||||
results.changes = {
|
||||
result: "success",
|
||||
outputs: { test_mode: "full", ...changes },
|
||||
};
|
||||
const receiptDirectory = join(directory, ".cache", "receipt");
|
||||
const output = join(directory, "github-output");
|
||||
await writeFile(output, "");
|
||||
|
||||
@@ -593,7 +593,9 @@ it("keeps fast feedback separate from complete qualification and release proof",
|
||||
const shards = ci.jobs["test-shard"]?.steps ?? [];
|
||||
const fast = shards.find((step) => step.run?.includes("npm run test:built"));
|
||||
const full = shards.find((step) => step.run?.includes("test:ci:shard:run"));
|
||||
expect(fast?.if).toBe("needs.changes.outputs.tier == 'fast'");
|
||||
expect(fast?.if).toBe(
|
||||
"needs.changes.outputs.tier == 'fast' && needs.changes.outputs.test_mode == 'full'",
|
||||
);
|
||||
expect(fast?.run).toContain("--reporter=blob");
|
||||
expect(fast?.run).not.toContain("--coverage");
|
||||
expect(full?.if).toBe("needs.changes.outputs.tier == 'full'");
|
||||
@@ -640,6 +642,7 @@ it.each([
|
||||
(await scopeFlags()).map((flag) => [flag, "false"]),
|
||||
),
|
||||
tier: "fast",
|
||||
test_mode: "full",
|
||||
code: "true",
|
||||
[scope]: selected,
|
||||
},
|
||||
@@ -673,6 +676,7 @@ it("rejects incomplete or malformed classification even when available jobs succ
|
||||
const valid = {
|
||||
...Object.fromEntries(flags.map((flag) => [flag, "false"])),
|
||||
tier: "fast",
|
||||
test_mode: "full",
|
||||
};
|
||||
const run = (outputs: unknown) =>
|
||||
executeWorkflowFixture(
|
||||
@@ -750,6 +754,7 @@ it("fails fast only for ordinary PR matrices and rejects partial portable feedba
|
||||
outputs: {
|
||||
...Object.fromEntries(flags.map((flag) => [flag, "false"])),
|
||||
tier: "fast",
|
||||
test_mode: "full",
|
||||
code: "true",
|
||||
},
|
||||
},
|
||||
@@ -775,6 +780,105 @@ it("fails fast only for ordinary PR matrices and rejects partial portable feedba
|
||||
}
|
||||
});
|
||||
|
||||
it("runs a selected PR subset in planner-owned groups and still gates it", async () => {
|
||||
const { jobs } = await workflow();
|
||||
// The planner owns every shard list, including one job per selected group.
|
||||
expect(jobs["test-shard"]?.strategy).toMatchObject({
|
||||
matrix: { shard: "${{ fromJSON(needs.changes.outputs.test_shards) }}" },
|
||||
});
|
||||
expect(jobs["test-source"]).toMatchObject({
|
||||
if: expect.stringContaining("needs.changes.outputs.test_mode == 'full'"),
|
||||
});
|
||||
const selected = z
|
||||
.array(
|
||||
z.object({
|
||||
if: z.string().optional(),
|
||||
run: z.string().optional(),
|
||||
env: z.record(z.string(), z.string()).optional(),
|
||||
}),
|
||||
)
|
||||
.parse(jobs["test-shard"]?.steps)
|
||||
.find(
|
||||
(step) => step.if === "needs.changes.outputs.test_mode == 'selected'",
|
||||
);
|
||||
expect(selected?.run).toBe(
|
||||
'node scripts/ci/test-group.mjs selected "${MATRIX_SHARD}" --reporter=default --reporter=blob',
|
||||
);
|
||||
expect(selected?.env).toEqual({
|
||||
MATRIX_SHARD: "${{ matrix.shard }}",
|
||||
CI_TEST_SELECTION: "${{ needs.changes.outputs.test_selection }}",
|
||||
});
|
||||
|
||||
const flags = await scopeFlags();
|
||||
const gate = (
|
||||
outputs: Record<string, unknown>,
|
||||
results: Record<string, string>,
|
||||
event = "pull_request",
|
||||
) =>
|
||||
executeWorkflowFixture(
|
||||
process.execPath,
|
||||
[
|
||||
fileURLToPath(
|
||||
new URL("../../../scripts/ci/required.mjs", import.meta.url),
|
||||
),
|
||||
],
|
||||
{
|
||||
env: {
|
||||
...process.env,
|
||||
GITHUB_EVENT_NAME: event,
|
||||
CI_QUALIFICATION_DIRECTORY: "",
|
||||
CI_JOBS: JSON.stringify({
|
||||
changes: {
|
||||
result: "success",
|
||||
outputs: {
|
||||
...Object.fromEntries(flags.map((flag) => [flag, "false"])),
|
||||
tier: "fast",
|
||||
code: "true",
|
||||
test_mode: "selected",
|
||||
...outputs,
|
||||
},
|
||||
},
|
||||
static: { result: "success" },
|
||||
test: { result: "success" },
|
||||
...Object.fromEntries(
|
||||
Object.entries(results).map(([name, result]) => [
|
||||
name,
|
||||
{ result },
|
||||
]),
|
||||
),
|
||||
}),
|
||||
},
|
||||
},
|
||||
);
|
||||
const ran = { build: "success", "test-shard": "success" };
|
||||
// A selected subset skips the separate source job.
|
||||
await expect(gate({}, ran)).resolves.toMatchObject({ stderr: "" });
|
||||
for (const [name, result] of [
|
||||
["test-shard", "skipped"],
|
||||
["test-shard", "failure"],
|
||||
["build", "cancelled"],
|
||||
] as const)
|
||||
await expect(gate({}, { ...ran, [name]: result })).rejects.toMatchObject({
|
||||
code: 1,
|
||||
stderr: expect.stringContaining(name),
|
||||
});
|
||||
// The complete fast suite still requires its source job.
|
||||
await expect(gate({ test_mode: "full" }, ran)).rejects.toMatchObject({
|
||||
stderr: expect.stringContaining("test-source"),
|
||||
});
|
||||
// Only a fast pull-request plan with code changes may select tests.
|
||||
for (const [outputs, event] of [
|
||||
[{ test_mode: undefined }, "pull_request"],
|
||||
[{ test_mode: "candidate" }, "pull_request"],
|
||||
[{}, "push"],
|
||||
[{ code: "false" }, "pull_request"],
|
||||
] as const)
|
||||
await expect(gate(outputs, ran, event)).rejects.toMatchObject({
|
||||
code: 1,
|
||||
stderr: expect.stringContaining("valid test mode"),
|
||||
});
|
||||
});
|
||||
|
||||
it("binds each canonical lane to its explicit fast and full workflow jobs", async () => {
|
||||
const { jobs, needs } = await requiredNeeds();
|
||||
const { CI_LANES } = await import(
|
||||
@@ -870,6 +974,7 @@ it("binds each canonical lane to its explicit fast and full workflow jobs", asyn
|
||||
outputs: {
|
||||
...Object.fromEntries(flags.map((name) => [name, "false"])),
|
||||
tier: "fast",
|
||||
test_mode: "full",
|
||||
[scope]: "true",
|
||||
},
|
||||
};
|
||||
@@ -897,6 +1002,7 @@ it("binds each canonical lane to its explicit fast and full workflow jobs", asyn
|
||||
outputs: {
|
||||
...Object.fromEntries(flags.map((name) => [name, "true"])),
|
||||
tier: "full",
|
||||
test_mode: "full",
|
||||
},
|
||||
};
|
||||
await expect(executeGate(fullContext)).resolves.toMatchObject({ stderr: "" });
|
||||
|
||||
@@ -9,8 +9,6 @@ import { expect, it } from "vitest";
|
||||
import { parse } from "yaml";
|
||||
import { z } from "zod";
|
||||
|
||||
import { planTestSelection } from "../../../scripts/ci/test-selection.mjs";
|
||||
|
||||
import { parseTestDurations } from "../../ciTestSequencer.js";
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
import { workflowGit } from "../../support/workflowGit.js";
|
||||
@@ -102,9 +100,6 @@ it("merges available failed-shard diagnostics and skips unavailable shard report
|
||||
}
|
||||
expect(steps[upload]?.with?.path).toContain("test-timings.json");
|
||||
expect(steps[upload]?.with).toMatchObject({ "include-hidden-files": true });
|
||||
expect(steps[upload]?.with?.path).toContain(
|
||||
".cache/ci-test-selection/report.json",
|
||||
);
|
||||
const summary = steps.find((step) =>
|
||||
step.run?.includes("scripts/ci/test-report.mjs"),
|
||||
);
|
||||
@@ -212,8 +207,9 @@ it("collects either fast test group while keeping full reports tied to full shar
|
||||
);
|
||||
expect(summary?.env?.SHARD_RESULT).toBe("${{ needs.test-shard.result }}");
|
||||
expect(summary?.env?.SOURCE_RESULT).toBe("${{ needs.test-source.result }}");
|
||||
// A selected PR subset runs its source tests in the selected test jobs.
|
||||
expect(summary?.env?.SOURCE_REQUIRED).toBe(
|
||||
"${{ needs.changes.outputs.tier == 'fast' }}",
|
||||
"${{ needs.changes.outputs.tier == 'fast' && needs.changes.outputs.test_mode == 'full' }}",
|
||||
);
|
||||
});
|
||||
|
||||
@@ -660,77 +656,78 @@ it("recovers complete module costs from real blob reports without executing test
|
||||
).rejects.toMatchObject({ code: "ENOENT" });
|
||||
});
|
||||
|
||||
it("records failed omissions from full reports without treating partial or unhandled outcomes as parity", async () => {
|
||||
it("verifies exactly the selected subset and never publishes it as timing history", async () => {
|
||||
const root = await createTestTempDirectory("rea-ci-selection-report-");
|
||||
await workflowGit(root, ["init"]);
|
||||
const paths = [
|
||||
"src/sqlite/provider.test.ts",
|
||||
"src/evm/provider.test.ts",
|
||||
"tests/boundary/mcp/workflow.test.ts",
|
||||
"src/a.test.ts",
|
||||
"tests/b.test.ts",
|
||||
"tests/unselected.test.ts",
|
||||
];
|
||||
for (const path of paths) {
|
||||
await mkdir(join(root, path, ".."), { recursive: true });
|
||||
await writeFile(join(root, path), "// fixture\n");
|
||||
}
|
||||
await workflowGit(root, ["add", "src", "tests"]);
|
||||
const timing = {
|
||||
version: 1,
|
||||
success: false,
|
||||
reason: "failed",
|
||||
unhandledErrors: 0,
|
||||
files: paths.map((name, index) => ({
|
||||
name,
|
||||
durationMs: 10,
|
||||
state: index === 1 ? "failed" : "passed",
|
||||
})),
|
||||
const selection = {
|
||||
version: 2,
|
||||
mode: "selected",
|
||||
reason: "Affected tests: 2 changed",
|
||||
groups: [["src/a.test.ts"], ["tests/b.test.ts"]],
|
||||
};
|
||||
const run = async (report: unknown, source = "failure") => {
|
||||
await writeFile(join(root, "test-timings.json"), JSON.stringify(report));
|
||||
await executeTimingReport(root, {
|
||||
SHARD_RESULT: "failure",
|
||||
SOURCE_REQUIRED: "true",
|
||||
SOURCE_RESULT: source,
|
||||
MERGE_RESULT: "failure",
|
||||
const run = async (names: readonly string[], plan: unknown = selection) => {
|
||||
await writeFile(
|
||||
join(root, "test-timings.json"),
|
||||
JSON.stringify({
|
||||
version: 1,
|
||||
success: true,
|
||||
files: names.map((name) => ({ name, durationMs: 1, state: "passed" })),
|
||||
}),
|
||||
);
|
||||
return executeTimingReport(root, {
|
||||
CI_VERIFY_TEST_INVENTORY: "true",
|
||||
CI_TEST_SELECTION: JSON.stringify(
|
||||
planTestSelection(["src/sqlite/Provider.ts"], "pull_request", "fast"),
|
||||
),
|
||||
GITHUB_SHA: "merge-sha",
|
||||
CI_PR_HEAD_SHA: "pr-head-sha",
|
||||
CI_TEST_SELECTION: JSON.stringify(plan),
|
||||
});
|
||||
return JSON.parse(
|
||||
await readFile(
|
||||
join(root, ".cache/ci-test-selection/report.json"),
|
||||
"utf8",
|
||||
),
|
||||
) as unknown;
|
||||
};
|
||||
expect(await run(timing)).toMatchObject({
|
||||
observational: true,
|
||||
complete: true,
|
||||
selected: [paths[0], paths[2]],
|
||||
omitted: [paths[1]],
|
||||
missedFailures: [paths[1]],
|
||||
executionSha: "merge-sha",
|
||||
prHeadSha: "pr-head-sha",
|
||||
});
|
||||
for (const report of [
|
||||
{ ...timing, files: timing.files.slice(0, 1) },
|
||||
{ ...timing, unhandledErrors: 1 },
|
||||
{ ...timing, reason: "interrupted" },
|
||||
])
|
||||
expect(await run(report)).toMatchObject({ complete: false });
|
||||
for (const source of ["skipped", "cancelled", ""])
|
||||
expect(await run(timing, source)).toMatchObject({ complete: false });
|
||||
await expect(run({ ...timing, files: "invalid" })).rejects.toMatchObject({
|
||||
code: 1,
|
||||
});
|
||||
await expect(
|
||||
readFile(join(root, ".cache/ci-test-selection/report.json")),
|
||||
).rejects.toMatchObject({ code: "ENOENT" });
|
||||
const passed = await run(["src/a.test.ts", "tests/b.test.ts"]);
|
||||
expect(passed.summary).toContain("Portable tests: 2 selected files.");
|
||||
expect(passed.summary).toContain("is not a successful timing baseline");
|
||||
expect(passed.summary).not.toContain("Successful timing history");
|
||||
// A subset is not a complete-suite timing baseline.
|
||||
await expect(
|
||||
readFile(join(root, ".cache/ci-test-history/test-results.json")),
|
||||
).rejects.toMatchObject({ code: "ENOENT" });
|
||||
// Missing, unplanned (even if tracked) and duplicate files fail.
|
||||
for (const names of [
|
||||
["src/a.test.ts"],
|
||||
["src/a.test.ts", "tests/b.test.ts", "tests/unselected.test.ts"],
|
||||
["src/a.test.ts", "src/a.test.ts", "tests/b.test.ts"],
|
||||
])
|
||||
await expect(run(names)).rejects.toMatchObject({
|
||||
stderr: expect.stringContaining("Incomplete selected test inventory"),
|
||||
});
|
||||
// A full plan keeps the complete tracked inventory and its history.
|
||||
const full = { version: 2, mode: "full", reason: "Shared", groups: [] };
|
||||
await expect(
|
||||
run(["src/a.test.ts", "tests/b.test.ts"], full),
|
||||
).rejects.toMatchObject({
|
||||
stderr: expect.stringContaining("Incomplete portable test inventory"),
|
||||
});
|
||||
await run(paths, full);
|
||||
await expect(
|
||||
readFile(join(root, ".cache/ci-test-history/test-results.json"), "utf8"),
|
||||
).resolves.toContain("tests/unselected.test.ts");
|
||||
// An unreadable plan fails closed.
|
||||
for (const plan of [
|
||||
{ ...selection, version: 1 },
|
||||
{ ...selection, groups: [] },
|
||||
{ ...selection, groups: [["../escape.test.ts"]] },
|
||||
{ ...selection, groups: [["src/a.test.ts"], ["src/a.test.ts"]] },
|
||||
{ ...full, groups: [["src/a.test.ts"]] },
|
||||
])
|
||||
await expect(run(["src/a.test.ts"], plan)).rejects.toMatchObject({
|
||||
stderr: expect.stringContaining("Invalid portable test selection plan"),
|
||||
});
|
||||
});
|
||||
|
||||
it("requires executed Python analysis-efficiency cases while allowing optional skips", async () => {
|
||||
|
||||
@@ -1,240 +1,164 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
import { expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
compareTestSelection,
|
||||
parseTestSelection,
|
||||
planTestSelection,
|
||||
testShards,
|
||||
} from "../../scripts/ci/test-selection.mjs";
|
||||
|
||||
const sqliteSource = "src/sqlite/SqliteDatabaseProvider.test.ts";
|
||||
const sqliteBoundary = "tests/boundary/sqlite/sqliteDatabaseProvider.test.ts";
|
||||
const adbSource = "src/adb/AdbProvider.test.ts";
|
||||
const adbBoundary =
|
||||
"tests/boundary/providers/adb/adbBugreportPublication.test.ts";
|
||||
const cliBoundary = "tests/boundary/cli/apktoolCli.test.ts";
|
||||
const mcpBoundary = "tests/boundary/mcp/apktoolMcp.test.ts";
|
||||
const processBoundary =
|
||||
"tests/boundary/process/processCaptureLifecycle.test.ts";
|
||||
const acceptance = "tests/acceptance/tools/localToolContracts.test.ts";
|
||||
const composition = "tests/composition/directAnalysisCleanup.test.ts";
|
||||
const evaluation = "tests/evaluation/testDiscovery.test.ts";
|
||||
const execute = promisify(execFile);
|
||||
|
||||
const passed = (name: string) => ({ name, durationMs: 10, state: "passed" });
|
||||
|
||||
it("unions affected leaf owners, keeps changed tests and retains unowned integration tests", () => {
|
||||
const plan = planTestSelection(
|
||||
["src/sqlite/SqliteDatabaseProvider.ts", adbBoundary],
|
||||
"pull_request",
|
||||
"fast",
|
||||
);
|
||||
const inventory = [
|
||||
sqliteSource,
|
||||
sqliteBoundary,
|
||||
adbSource,
|
||||
adbBoundary,
|
||||
cliBoundary,
|
||||
mcpBoundary,
|
||||
processBoundary,
|
||||
acceptance,
|
||||
composition,
|
||||
evaluation,
|
||||
];
|
||||
const report = compareTestSelection(plan, inventory, inventory.map(passed));
|
||||
|
||||
expect(plan).toMatchObject({
|
||||
mode: "candidate",
|
||||
owners: ["adb", "sqlite"],
|
||||
const plan = (files: readonly string[], sources: Record<string, string>) =>
|
||||
planTestSelection({
|
||||
files,
|
||||
event: "pull_request",
|
||||
tier: "fast",
|
||||
code: true,
|
||||
snapshot: {
|
||||
list: () => Object.keys(sources),
|
||||
read: (paths) =>
|
||||
new Map(paths.map((path) => [path, sources[path] ?? ""])),
|
||||
},
|
||||
});
|
||||
expect(report.selected).toEqual(
|
||||
expect.arrayContaining([
|
||||
sqliteSource,
|
||||
sqliteBoundary,
|
||||
adbSource,
|
||||
adbBoundary,
|
||||
cliBoundary,
|
||||
mcpBoundary,
|
||||
processBoundary,
|
||||
acceptance,
|
||||
composition,
|
||||
evaluation,
|
||||
]),
|
||||
);
|
||||
expect(report.omitted).toEqual([]);
|
||||
|
||||
const selectedFiles = (result: ReturnType<typeof plan>) =>
|
||||
result.groups.flat().toSorted();
|
||||
|
||||
it("follows test and verifier imports and directory scans, not production importers", () => {
|
||||
const sources = {
|
||||
"src/domain/Value.ts": "",
|
||||
"src/application/UsesValue.ts": 'import "../domain/Value.js";',
|
||||
"src/application/UsesValue.test.ts": 'import "./UsesValue.js";',
|
||||
"src/domain/Value.test.ts": 'import { value } from "./Value.js";',
|
||||
"src/domain/Registry.test.ts": "readdirSync(here);",
|
||||
"src/domain/Unrelated.test.ts": 'import "./Unrelated.js";',
|
||||
"tests/boundary/filesystem/verifier.test.ts":
|
||||
'const m = await import("../../../scripts/verify-thing.mjs");',
|
||||
};
|
||||
expect(selectedFiles(plan(["src/domain/Value.ts"], sources))).toEqual([
|
||||
"src/domain/Registry.test.ts",
|
||||
"src/domain/Value.test.ts",
|
||||
]);
|
||||
expect(selectedFiles(plan(["scripts/verify-thing.mjs"], sources))).toEqual([
|
||||
"tests/boundary/filesystem/verifier.test.ts",
|
||||
]);
|
||||
});
|
||||
|
||||
it("omits only unrelated owned tests while retaining dynamic boundary suites", () => {
|
||||
const plan = planTestSelection(
|
||||
["src/sqlite/SqliteDatabaseProvider.ts"],
|
||||
"pull_request",
|
||||
"fast",
|
||||
);
|
||||
const inventory = [
|
||||
sqliteSource,
|
||||
sqliteBoundary,
|
||||
adbSource,
|
||||
adbBoundary,
|
||||
cliBoundary,
|
||||
mcpBoundary,
|
||||
processBoundary,
|
||||
acceptance,
|
||||
composition,
|
||||
evaluation,
|
||||
];
|
||||
const report = compareTestSelection(plan, inventory, inventory.map(passed));
|
||||
|
||||
expect(report.selected).toEqual(
|
||||
expect.arrayContaining([
|
||||
sqliteSource,
|
||||
sqliteBoundary,
|
||||
cliBoundary,
|
||||
mcpBoundary,
|
||||
processBoundary,
|
||||
acceptance,
|
||||
composition,
|
||||
evaluation,
|
||||
]),
|
||||
);
|
||||
expect(report.omitted).toEqual([adbSource, adbBoundary]);
|
||||
expect(report.complete).toBe(true);
|
||||
it("selects a leaf adapter's tests by its name and directory across test projects", () => {
|
||||
const sources = {
|
||||
"src/jeb/JebMcpDeadline.ts": "",
|
||||
"tests/boundary/providers/jebMcpTimeout.test.ts": "",
|
||||
"tests/boundary/mcp/jebMcp.test.ts": "",
|
||||
"tests/acceptance/providers/jebConfigurationIsolation.test.ts": "",
|
||||
"tests/boundary/providers/adb/adbBugreport.test.ts": "",
|
||||
"tests/boundary/providers/ghidraJava.test.ts": "",
|
||||
};
|
||||
expect(selectedFiles(plan(["src/jeb/JebMcpDeadline.ts"], sources))).toEqual([
|
||||
"tests/acceptance/providers/jebConfigurationIsolation.test.ts",
|
||||
"tests/boundary/mcp/jebMcp.test.ts",
|
||||
"tests/boundary/providers/jebMcpTimeout.test.ts",
|
||||
]);
|
||||
expect(selectedFiles(plan(["src/adb/AdbProvider.ts"], sources))).toEqual([
|
||||
"tests/boundary/providers/adb/adbBugreport.test.ts",
|
||||
]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["unowned shared source", "src/application/Workflow.ts"],
|
||||
["fixture input", "tests/conformance/c/fixture.c"],
|
||||
["configuration", "vitest.config.ts"],
|
||||
["documentation", "README.md"],
|
||||
["CLI implementation", "src/cli/apktoolCommands.ts"],
|
||||
["MCP implementation", "src/server/contractToolBindings.ts"],
|
||||
["process ownership", "src/process/ProcessOwnership.ts"],
|
||||
["acceptance test", acceptance],
|
||||
["composition test", composition],
|
||||
])("uses full fallback for %s changes", (_description, path) => {
|
||||
expect(planTestSelection([path], "pull_request", "fast")).toMatchObject({
|
||||
it("lets checker-only inputs and prose select nothing by themselves", () => {
|
||||
const sources = {
|
||||
"src/domain/Value.test.ts": "",
|
||||
"tests/boundary/filesystem/docs.test.ts": 'read("docs/guide.md");',
|
||||
};
|
||||
// A declaration, guard baseline or schema summary adds no tests…
|
||||
expect(
|
||||
selectedFiles(
|
||||
plan(
|
||||
[
|
||||
"src/domain/Value.test.ts",
|
||||
"scripts/lib/helper.d.mts",
|
||||
"scripts/verify/baselines/some-guard.json",
|
||||
"scripts/verify/advertised-schemas/tools/tool.json",
|
||||
],
|
||||
sources,
|
||||
),
|
||||
),
|
||||
).toEqual(["src/domain/Value.test.ts"]);
|
||||
// …and prose selects the tests that read it by path.
|
||||
expect(selectedFiles(plan(["docs/guide.md"], sources))).toEqual([
|
||||
"tests/boundary/filesystem/docs.test.ts",
|
||||
]);
|
||||
expect(plan(["scripts/verify/baselines/x.json"], sources)).toMatchObject({
|
||||
mode: "full",
|
||||
owners: [],
|
||||
changedTests: [],
|
||||
});
|
||||
expect(plan(["scripts/lib/shared.mjs"], sources)).toMatchObject({
|
||||
mode: "full",
|
||||
reason: "Input is not traced to tests: scripts/lib/shared.mjs",
|
||||
});
|
||||
});
|
||||
|
||||
it("uses full fallback outside ordinary fast pull requests or without diff paths", () => {
|
||||
it("parses only well-formed plans and derives one shard per selected group", () => {
|
||||
const selected = {
|
||||
version: 2,
|
||||
mode: "selected",
|
||||
reason: "Affected tests: 2 changed",
|
||||
groups: [["src/a.test.ts"], ["tests/b.test.ts"]],
|
||||
};
|
||||
expect(
|
||||
planTestSelection(["src/sqlite/SqliteDatabaseProvider.ts"], "push", "fast")
|
||||
.mode,
|
||||
).toBe("full");
|
||||
expect(
|
||||
planTestSelection(
|
||||
["src/sqlite/SqliteDatabaseProvider.ts"],
|
||||
"pull_request",
|
||||
"full",
|
||||
).mode,
|
||||
).toBe("full");
|
||||
expect(planTestSelection([], "pull_request", "fast").mode).toBe("full");
|
||||
testShards("fast", parseTestSelection(JSON.stringify(selected))),
|
||||
).toEqual(["1/2", "2/2"]);
|
||||
const full = { version: 2, mode: "full", reason: "Shared", groups: [] };
|
||||
expect(testShards("fast", parseTestSelection(full))).toHaveLength(12);
|
||||
expect(testShards("full", parseTestSelection(full))).toHaveLength(6);
|
||||
for (const value of [
|
||||
"",
|
||||
"null",
|
||||
{ ...selected, mode: "candidate" },
|
||||
{ ...selected, groups: [[]] },
|
||||
{ ...selected, groups: [["src/a.ts"]] },
|
||||
{ ...selected, groups: [["/abs/a.test.ts"]] },
|
||||
{ ...selected, groups: [["src/a.test.ts"], ["src/a.test.ts"]] },
|
||||
{
|
||||
...selected,
|
||||
groups: Array.from({ length: 5 }, (_, i) => [`src/${i}.test.ts`]),
|
||||
},
|
||||
])
|
||||
expect(() => parseTestSelection(value)).toThrow(
|
||||
/Invalid portable test selection plan|JSON/u,
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps new changed tests and falls back when a changed test was removed", () => {
|
||||
const addedTest = "src/sqlite/NewDatabaseRule.test.ts";
|
||||
const candidate = planTestSelection([addedTest], "pull_request", "fast");
|
||||
expect(candidate.changedTests).toEqual([addedTest]);
|
||||
const inventory = [addedTest, adbSource];
|
||||
expect(
|
||||
compareTestSelection(candidate, inventory, inventory.map(passed)).selected,
|
||||
).toContain(addedTest);
|
||||
|
||||
const removedTest = "src/sqlite/RemovedDatabaseRule.test.ts";
|
||||
const report = compareTestSelection(
|
||||
planTestSelection([removedTest], "pull_request", "fast"),
|
||||
[sqliteSource, adbSource],
|
||||
[passed(sqliteSource), passed(adbSource)],
|
||||
it("runs only a group the plan defines", async () => {
|
||||
const group = fileURLToPath(
|
||||
new URL("../../scripts/ci/test-group.mjs", import.meta.url),
|
||||
);
|
||||
expect(report.plan.mode).toBe("full");
|
||||
expect(report.selected).toEqual([adbSource, sqliteSource]);
|
||||
expect(report.omitted).toEqual([]);
|
||||
});
|
||||
|
||||
it("marks incomplete inventories and nonterminal outcomes as incomplete", () => {
|
||||
const plan = planTestSelection(
|
||||
["src/sqlite/SqliteDatabaseProvider.ts"],
|
||||
"pull_request",
|
||||
"fast",
|
||||
);
|
||||
const inventory = [sqliteSource, adbSource];
|
||||
const completeResults = inventory.map(passed);
|
||||
|
||||
expect(
|
||||
compareTestSelection(plan, inventory, [passed(sqliteSource)]).complete,
|
||||
).toBe(false);
|
||||
expect(
|
||||
compareTestSelection(plan, inventory, [
|
||||
...completeResults,
|
||||
passed("tests/evaluation/unexpected.test.ts"),
|
||||
]),
|
||||
).toMatchObject({
|
||||
complete: false,
|
||||
unexpected: ["tests/evaluation/unexpected.test.ts"],
|
||||
const selection = JSON.stringify({
|
||||
version: 2,
|
||||
mode: "selected",
|
||||
reason: "Affected tests: 1 changed",
|
||||
groups: [["src/a.test.ts"]],
|
||||
});
|
||||
for (const [shard, environment] of [
|
||||
["2/2", selection],
|
||||
["1/2", selection],
|
||||
[
|
||||
"1/1",
|
||||
JSON.stringify({ version: 2, mode: "full", reason: "x", groups: [] }),
|
||||
],
|
||||
["1/1", ""],
|
||||
] as const)
|
||||
await expect(
|
||||
execute(process.execPath, [group, "selected", shard], {
|
||||
env: { ...process.env, CI_TEST_SELECTION: environment },
|
||||
}),
|
||||
).rejects.toMatchObject({ code: 1 });
|
||||
await expect(
|
||||
execute(process.execPath, [group, "selected", "1/1", "--shard=1/2"], {
|
||||
env: { ...process.env, CI_TEST_SELECTION: selection },
|
||||
}),
|
||||
).rejects.toMatchObject({
|
||||
stderr: expect.stringContaining("already sharded"),
|
||||
});
|
||||
expect(
|
||||
compareTestSelection(plan, inventory, [
|
||||
...completeResults,
|
||||
passed(adbSource),
|
||||
]),
|
||||
).toMatchObject({ complete: false, duplicateFiles: 1 });
|
||||
for (const state of ["pending", "skipped"])
|
||||
expect(
|
||||
compareTestSelection(plan, inventory, [
|
||||
passed(sqliteSource),
|
||||
{ name: adbSource, durationMs: 0, state },
|
||||
]),
|
||||
).toMatchObject({ complete: false, unobserved: [adbSource] });
|
||||
});
|
||||
|
||||
it("reports a failure from an omitted test, including collection or module failures", () => {
|
||||
const plan = planTestSelection(
|
||||
["src/sqlite/SqliteDatabaseProvider.ts"],
|
||||
"pull_request",
|
||||
"fast",
|
||||
);
|
||||
const report = compareTestSelection(
|
||||
plan,
|
||||
[sqliteSource, adbSource],
|
||||
[passed(sqliteSource), { name: adbSource, durationMs: 1, state: "failed" }],
|
||||
);
|
||||
|
||||
expect(report.omitted).toContain(adbSource);
|
||||
expect(report.missedFailures).toEqual([adbSource]);
|
||||
});
|
||||
|
||||
it("rejects plans that name unowned owners or malformed changed tests", () => {
|
||||
const plan = planTestSelection(
|
||||
["src/sqlite/SqliteDatabaseProvider.ts"],
|
||||
"pull_request",
|
||||
"fast",
|
||||
);
|
||||
expect(() =>
|
||||
compareTestSelection(
|
||||
{ ...plan, owners: ["unknown"] },
|
||||
[sqliteSource],
|
||||
[passed(sqliteSource)],
|
||||
),
|
||||
).toThrow("Invalid observational test selection plan");
|
||||
expect(() =>
|
||||
compareTestSelection(
|
||||
{ ...plan, changedTests: ["src/sqlite/not-a-test.ts"] },
|
||||
[sqliteSource],
|
||||
[passed(sqliteSource)],
|
||||
),
|
||||
).toThrow("Invalid observational test selection plan");
|
||||
});
|
||||
|
||||
it("uses full fallback when a changed owner has no executable tests", () => {
|
||||
const report = compareTestSelection(
|
||||
planTestSelection(
|
||||
["src/sqlite/SqliteDatabaseProvider.ts"],
|
||||
"pull_request",
|
||||
"fast",
|
||||
),
|
||||
[adbSource],
|
||||
[passed(adbSource)],
|
||||
);
|
||||
|
||||
expect(report.plan.mode).toBe("full");
|
||||
expect(report.omitted).toEqual([]);
|
||||
});
|
||||
Reference in new issue
Block a user