diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8cd3bbd9d7..8e70db3771 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,6 +87,8 @@ jobs: website: ${{ steps.plan.outputs.website }} matrix: ${{ steps.plan.outputs.matrix }} test_selection: ${{ steps.plan.outputs.test_selection }} + test_mode: ${{ steps.plan.outputs.test_mode }} + test_shards: ${{ steps.plan.outputs.test_shards }} steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: @@ -294,7 +296,8 @@ jobs: test-source: name: Source tests needs: changes - if: needs.changes.outputs.code == 'true' && needs.changes.outputs.tier == 'fast' + # A selected PR subset runs its source tests in the selected test jobs. + if: needs.changes.outputs.code == 'true' && needs.changes.outputs.tier == 'fast' && needs.changes.outputs.test_mode == 'full' runs-on: ubuntu-latest timeout-minutes: 10 steps: @@ -330,14 +333,10 @@ jobs: # every shard's diagnostics and coverage. Passing PRs run the full suite. fail-fast: ${{ github.event_name == 'pull_request' && needs.changes.outputs.tier == 'fast' }} matrix: - # Fast feedback uses one source group and twelve built groups. Shards - # are balanced by recorded duration, but hosted runner speed varies, - # so the slowest shard sets the critical path: at eight shards it ran - # 24% above the mean shard. Smaller shards bound that excess at a few - # percent more runner time. Classification, build, and report merge - # sit outside these jobs. Full qualification retains six complete - # shards with coverage. - shard: ${{ fromJSON(needs.changes.outputs.tier == 'fast' && '["1/12","2/12","3/12","4/12","5/12","6/12","7/12","8/12","9/12","10/12","11/12","12/12"]' || '["1/6","2/6","3/6","4/6","5/6","6/6"]') }} + # scripts/ci/test-selection.mjs owns the shard counts: twelve fast + # built shards, six full-qualification shards with coverage, or one + # job per group of a selected PR subset. + shard: ${{ fromJSON(needs.changes.outputs.test_shards) }} steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 @@ -357,11 +356,17 @@ jobs: env: REA_CI_TEST_HISTORY: .cache/ci-test-history/test-results.json - name: Run built-dependent tests for fast feedback - if: needs.changes.outputs.tier == 'fast' + if: needs.changes.outputs.tier == 'fast' && needs.changes.outputs.test_mode == 'full' run: npm run test:built -- --reporter=default --reporter=blob --shard=${{ matrix.shard }} env: VITEST_BLOB_LABEL: built REA_CI_TEST_HISTORY: .cache/ci-test-history/test-results.json + - name: Run the tests this pull request affects + if: needs.changes.outputs.test_mode == 'selected' + run: node scripts/ci/test-group.mjs selected "${MATRIX_SHARD}" --reporter=default --reporter=blob + env: + MATRIX_SHARD: ${{ matrix.shard }} + CI_TEST_SELECTION: ${{ needs.changes.outputs.test_selection }} - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 if: ${{ !cancelled() }} with: @@ -608,7 +613,7 @@ jobs: if: needs.changes.outputs.code == 'true' && !cancelled() env: SHARD_RESULT: ${{ needs.test-shard.result }} - SOURCE_REQUIRED: ${{ needs.changes.outputs.tier == 'fast' }} + SOURCE_REQUIRED: ${{ needs.changes.outputs.tier == 'fast' && needs.changes.outputs.test_mode == 'full' }} SOURCE_RESULT: ${{ needs.test-source.result }} MERGE_RESULT: ${{ needs.changes.outputs.tier == 'full' && steps.merge.outcome || steps.merge-fast.outcome }} CI_VERIFY_TEST_INVENTORY: "true" @@ -624,7 +629,6 @@ jobs: test-results.json test-timings.json coverage/ - .cache/ci-test-selection/report.json include-hidden-files: true if-no-files-found: error retention-days: 30 @@ -653,7 +657,7 @@ jobs: env: CODE_REQUIRED: ${{ needs.changes.outputs.code }} SHARD_RESULT: ${{ needs.test-shard.result }} - SOURCE_REQUIRED: ${{ needs.changes.outputs.tier == 'fast' }} + SOURCE_REQUIRED: ${{ needs.changes.outputs.tier == 'fast' && needs.changes.outputs.test_mode == 'full' }} SOURCE_RESULT: ${{ needs.test-source.result }} run: | if [ "${CODE_REQUIRED}" = "true" ]; then diff --git a/docs/development/testing.md b/docs/development/testing.md index 6837d20d4f..317b12fd49 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -1299,6 +1299,62 @@ and workspace rather than reporting successful removal. The controlled `packageUpdateRegistryFailure` regression needs built process modules, uses no public registry, and checks both the primary failure and live descendant absence. +## Pull-request test selection + +CI on a pull request runs static checks, the build and the portable tests the +change affects. The complete portable suite runs on every push to `main`, on +the scheduled and manual qualification runs, in release qualification, and on a +PR labelled `ci:full`. Those complete runs catch a consumer that selection +missed, after it has merged. Add `ci:full` to a PR when its effect reaches +further than its imports show, such as runtime registration, generated files, +subprocesses or bridges. Adding the label starts full qualification at once, +and later pushes stay on the full tier while the label remains. + +`scripts/ci/test-selection.mjs` owns the selection. It reads the checked-out +revision and selects: + +- changed test files; +- tests that import a changed module, directly or through test-only helpers + (non-test modules under `tests/` and `src/**/*.fixture*.ts`). Selection never + follows production importers transitively; +- tests beside a changed module that share its stem, or that scan their + directory with `readdir`; +- the boundary tests of a changed leaf adapter (the owner table in the same + file), which reach their sources through the built CLI, the MCP server or a + fake provider process; +- tests whose text names a changed path. + +Verifier scripts (`scripts/verify-*.mjs`, `scripts/verify/**/*.mjs`) are traced +like source modules. Changed Markdown outside `src/` and `tests/` selects only +the tests that name it. Declarations, guard baselines +(`scripts/verify/baselines/*.json`) and advertised-schema summaries select no +tests, because the static and build jobs check them. + +The PR falls back to the complete fast suite when it cannot decide: + +- a shared boundary (`src/process/`, `src/filesystem/`, `src/contracts/tool*`, + `src/server/createServer*`, `src/cli.ts`, `scripts/ci/`) changed; +- any other changed input is not traced, such as workflows, package metadata, + build and test-runner scripts, `scripts/lib/`, fixtures or data; +- no test is selected; +- more than 150 test files are selected; +- a selected file has no single owning test project. + +A selection runs as one `Test shard (k/n)` job per group of up to 40 files, at +most four jobs, using exact file identity (`scripts/ci/test-group.mjs selected`). +The separate `Source tests` job does not run, because the groups include the +selected source tests. The `Classify changes` summary gives the mode and its +reason. + +The required checks still report on every PR and fail closed: + +- `Portable tests` checks that the merged report contains exactly the selected + files: none missing, none unplanned, no duplicates. +- `CI required` accepts a selected plan only for a fast `pull_request` run with + code changes, and then requires `build` and every selected test group. +- A missing or malformed plan fails classification, the test group and the gate. +- A selected run never publishes timing history. + ## Real-toolchain verification lanes See [Real-toolchain verification lanes](../testing-lanes.md#real-toolchain-verification-lanes). diff --git a/docs/testing-lanes.md b/docs/testing-lanes.md index 81ab6d99d0..22064a17dc 100644 --- a/docs/testing-lanes.md +++ b/docs/testing-lanes.md @@ -631,30 +631,13 @@ generated-document validation, including the small onboarding example consistenc suite, without the source-test shards or native package lanes. Classification compares the PR head with its merge base, so later base-branch changes do not expand that scope. Static and portable-test aggregate jobs -remain present and fail if classification fails. Every code change retains the -complete portable Linux suite; individual test files are not selected by -imports or filenames. +remain present and fail if classification fails. -Ordinary PRs also produce an **observational** test-selection plan. The candidate -leaf-adapter ownership table lives in `scripts/ci/test-selection.mjs`, separately -from provider-job scopes. It proposes keeping affected owners plus every unowned -test, including all CLI/MCP, acceptance, composition, conformance, and process -tests. Shared inputs, unknown paths, missing owner tests, and removed tests fall -back to the complete inventory. Renames account for both paths. Main, scheduled, -manual, and `ci:full` runs always propose the full suite. - -The source group and every built shard together still execute the complete suite. The existing report merge -compares the proposal with actual module outcomes and retains -`.cache/ci-test-selection/report.json` in `test-results`, including selected and -omitted paths, changed input paths, missed failures, unobserved files, scheduling weights, PR head and -executed merge SHA. This adds no test reruns or runner jobs. Missing, duplicate, -skipped, interrupted, or unhandled results cannot establish a complete comparison; -even a complete passing comparison is not proof of ownership completeness. -Before enabling omissions, review representative full-run comparisons and -cross-owner consumers, including failures, and measure completed CI feedback and -runner demand. There is no automatic promotion or switch to skip tests. The -initial proposal measures only leaf producer-test omissions; it does not promise -the savings of a broader affected-test planner. +Pull requests with code changes run only the portable tests the change +affects; main pushes, scheduled and manual runs, release qualification and +`ci:full` run the complete suite. See +[pull-request test selection](development/testing.md#pull-request-test-selection) +for the selection rules, fallbacks and gate. `scripts/ci/plan.mjs` selects lanes using the ownership table in `scripts/ci/scopes.mjs`. Each lane record owns its path and test selection @@ -739,13 +722,14 @@ push: changes from failed or cancelled intermediate runs remain in scope. Missing, untrusted, or non-ancestor history selects the fast Linux x64 baseline: portable checks and the Linux runtime lane. The lookup is bounded to the last 100 successful push runs; lookup failures broaden selection. -Every code change still selects the source group and every fast built shard, without coverage +Pull requests run their selected tests, or the complete suite when selection +falls back; main pushes run the source group and every fast built shard, without coverage instrumentation in this fast tier. Ordinary PR matrices cancel remaining -shards after the first failure; passing PRs execute the complete suite. Main, +shards after the first failure. Main, `ci:full`, scheduled, manual, and release qualification retain every shard's results after a failure. Available failed-shard reports remain diagnostic only: failed, cancelled, skipped, or missing selected shards cannot pass the gate or -publish successful timing history. This deliberately avoids relying on an +publish successful timing history. Complete-suite runs do not rely on an import graph for generated files, filesystem inputs, subprocesses, or bridges. The portable report rejects an entirely skipped module inventory and lists fully skipped modules in its summary. A mixed passed/skipped inventory is @@ -930,10 +914,6 @@ Main warms the new cache namespace after merge; a cold cache falls back to the normal install. This prevents each PR from saving another branch-scoped copy of the same download cache. It does not remove runner assignment waits. -Portable owner-based test selection remains observational. Its reports compare -proposed omissions with the full run; incomplete or skipped observations do not -establish that fewer tests would preserve regression detection. - The automatic browser, binary-layout, EVM, JavaScript-recovery, packet-capture, signal-recording, recorded-crash, web and environment-bundle provider lanes download `rea-build-cache` from their own run after the build and static jobs succeed. Their existing build commands still diff --git a/scripts/ci/change-plan.d.mts b/scripts/ci/change-plan.d.mts index c3d2f48a76..55e2788947 100644 --- a/scripts/ci/change-plan.d.mts +++ b/scripts/ci/change-plan.d.mts @@ -15,6 +15,14 @@ export interface ChangePlanRepository { readonly isAncestor: (ancestor: string, head: string) => boolean; readonly changedFiles: (base: string, head: string) => readonly string[]; readonly readFile: (revision: string, path: string) => string; + readonly listFiles: ( + revision: string, + directories: readonly string[], + ) => readonly string[]; + readonly readFiles: ( + revision: string, + paths: readonly string[], + ) => ReadonlyMap; } export type ChangePlan = Readonly> & { @@ -29,6 +37,7 @@ export type ChangePlan = Readonly> & { readonly files: readonly string[]; readonly reasons: readonly string[]; readonly testSelection: TestSelectionPlan; + readonly testShards: readonly string[]; }; export function planChanges( diff --git a/scripts/ci/change-plan.mjs b/scripts/ci/change-plan.mjs index 393c352d70..e696cebb31 100644 --- a/scripts/ci/change-plan.mjs +++ b/scripts/ci/change-plan.mjs @@ -5,9 +5,10 @@ import { CI_FLAGS as flags, MANUAL_VERIFIERS, CI_LANES, + SHARED_BOUNDARY, WINDOWS_EXECUTABLE_SELECTION_INPUTS, } from "./scopes.mjs"; -import { planTestSelection } from "./test-selection.mjs"; +import { planTestSelection, testShards } from "./test-selection.mjs"; // Actionlint validates workflow syntax only. The semantic suite asserts these // workflows' release, publication and self-hosted runner safeguards. @@ -58,11 +59,6 @@ const matchesAny = (file, patterns) => export function planChanges(request, repository) { const selected = Object.fromEntries(flags.map((flag) => [flag, false])); const reasons = []; - // Shared process, contract, CLI, and CI-script edits are not a specific provider. - // Fast qualification keeps the portable suite and the Linux x64 runtime - // baseline. Full qualification still selects every lane. - const SHARED_BOUNDARY = - /^(src\/process\/|src\/filesystem\/|src\/contracts\/tool|src\/server\/createServer|src\/cli\.ts|scripts\/ci\/)/; const full = (reason) => { if (tier === "full") { for (const flag of flags) selected[flag] = true; @@ -269,11 +265,7 @@ export function planChanges(request, repository) { selected.readiness = true; } if (file.startsWith("tests/")) return; - if ( - /^(src\/process\/|src\/filesystem\/|src\/contracts\/tool|src\/server\/createServer|src\/cli\.ts|scripts\/ci\/)/.test( - file, - ) - ) { + if (SHARED_BOUNDARY.test(file)) { full(`Shared boundary: ${file}`); } else if (owners.length) { if (file.startsWith("src/") || file.startsWith("bridge/")) @@ -343,8 +335,26 @@ export function planChanges(request, repository) { ] : [linux], }; - const testSelection = planTestSelection(files, event, tier); - return { ...selected, tier, matrix, files, reasons, testSelection }; + const testSelection = planTestSelection({ + files, + event, + tier, + code: selected.code, + // The checked-out revision is the one whose tests run. + snapshot: { + list: () => repository.listFiles("HEAD", ["src", "tests"]), + read: (paths) => repository.readFiles("HEAD", paths), + }, + }); + return { + ...selected, + tier, + matrix, + files, + reasons, + testSelection, + testShards: testShards(tier, testSelection), + }; } /** Answer the plan's Git questions from the repository in `cwd`. */ @@ -370,5 +380,32 @@ export const gitRepository = (cwd = process.cwd()) => { .split("\0") .filter(Boolean), readFile: (revision, path) => git("show", `${revision}:${path}`), + listFiles: (revision, directories) => + git("ls-tree", "-r", "-z", "--name-only", revision, "--", ...directories) + .split("\0") + .filter(Boolean), + // One batch process reads every requested blob. + readFiles: (revision, paths) => { + const output = execFileSync("git", ["cat-file", "--batch"], { + cwd, + input: paths.map((path) => `${revision}:${path}\n`).join(""), + maxBuffer: 512 * 1024 * 1024, + }); + const contents = new Map(); + let offset = 0; + for (const path of paths) { + const end = output.indexOf(10, offset); + const [, type, size] = output.toString("utf8", offset, end).split(" "); + if (type !== "blob") + throw new Error(`Git could not read ${revision}:${path}`); + const start = end + 1; + contents.set( + path, + output.toString("utf8", start, start + Number(size)), + ); + offset = start + Number(size) + 1; + } + return contents; + }, }; }; diff --git a/scripts/ci/plan.mjs b/scripts/ci/plan.mjs index b2128a509c..973decca56 100644 --- a/scripts/ci/plan.mjs +++ b/scripts/ci/plan.mjs @@ -14,7 +14,7 @@ const plan = planChanges( }, gitRepository(), ); -const { tier, matrix, reasons, testSelection } = plan; +const { tier, matrix, reasons, testSelection, testShards } = plan; const selected = Object.fromEntries(flags.map((flag) => [flag, plan[flag]])); if (process.env.GITHUB_OUTPUT) appendFileSync( @@ -24,6 +24,8 @@ if (process.env.GITHUB_OUTPUT) tier, matrix: JSON.stringify(matrix), test_selection: JSON.stringify(testSelection), + test_mode: testSelection.mode, + test_shards: JSON.stringify(testShards), }) .map(([key, value]) => `${key}=${value}\n`) .join(""), @@ -31,6 +33,6 @@ if (process.env.GITHUB_OUTPUT) if (process.env.GITHUB_STEP_SUMMARY) appendFileSync( process.env.GITHUB_STEP_SUMMARY, - `### Selected CI lanes (${tier})\n\n${flags.filter((flag) => selected[flag]).join(", ") || "Formatting only"}\n\n${reasons.join("\n\n")}\n`, + `### Selected CI lanes (${tier})\n\n${flags.filter((flag) => selected[flag]).join(", ") || "Formatting only"}\n\n${reasons.join("\n\n")}\n\nPortable tests (${testSelection.mode}): ${testSelection.reason}\n`, ); console.log(JSON.stringify(plan)); diff --git a/scripts/ci/required.mjs b/scripts/ci/required.mjs index c0df9e2191..a7c9af3a02 100644 --- a/scripts/ci/required.mjs +++ b/scripts/ci/required.mjs @@ -28,10 +28,25 @@ if ( ); if (outputs.tier === "full" && selectedScopes.length !== CI_FLAGS.length) throw new Error("Full qualification requires every canonical CI scope"); +// Only a fast pull-request plan may run a selected test subset. +const testMode = outputs.test_mode; +if ( + !["full", "selected"].includes(testMode) || + (testMode === "selected" && + (outputs.tier !== "fast" || + process.env.GITHUB_EVENT_NAME !== "pull_request" || + outputs.code !== "true")) +) + throw new Error( + "CI classification requires a valid test mode: full, or selected on a fast pull request", + ); const required = reused ? ["changes"] : ["changes", "static", "test"]; if (outputs.tier === "full") required.push("static-deep"); for (const scope of selectedScopes) - required.push(...CI_LANES[scope].required[outputs.tier]); + required.push( + ...((testMode === "selected" && CI_LANES[scope].selectedTests) || + CI_LANES[scope].required[outputs.tier]), + ); const failed = [...new Set(required)].filter( (name) => jobs[name]?.result !== "success", ); diff --git a/scripts/ci/scopes.mjs b/scripts/ci/scopes.mjs index c74b6f2097..4e3c07c7b1 100644 --- a/scripts/ci/scopes.mjs +++ b/scripts/ci/scopes.mjs @@ -8,6 +8,13 @@ export const WINDOWS_EXECUTABLE_SELECTION_INPUTS = [ "tests/boundary/process/windowsExecutableSelection.test.ts", ]; +// Shared process, contract, CLI, and CI-script edits are not a specific provider. +// Fast qualification keeps the portable suite and the Linux x64 runtime +// baseline, and pull requests run the complete portable suite rather than a +// selected subset. Full qualification still selects every lane. +export const SHARED_BOUNDARY = + /^(src\/process\/|src\/filesystem\/|src\/contracts\/tool|src\/server\/createServer|src\/cli\.ts|scripts\/ci\/)/; + // Inputs whose changes must select both real capture lanes, even at shared boundaries. const CAPTURE_SHARED_INPUTS = [ "src/process/LinuxResourceLimits*", @@ -679,6 +686,8 @@ export const CI_LANES = { fast: ["build", "test-shard", "test-source"], full: ["build", "test-shard"], }, + // A selected PR subset runs its source tests in the test-shard groups. + selectedTests: ["build", "test-shard"], }, runtime: sameJobs([], ["windows-native-build", "package-e2e"]), package: sameJobs(PATH_SCOPES.package, ["windows-curated"], { diff --git a/scripts/ci/test-group.mjs b/scripts/ci/test-group.mjs index d69ea008f3..5d8c8f70a3 100644 --- a/scripts/ci/test-group.mjs +++ b/scripts/ci/test-group.mjs @@ -5,17 +5,48 @@ import { BUILT_TEST_PROJECTS, SOURCE_TEST_PROJECTS, } from "../lib/test-projects.mjs"; +import { parseTestSelection, testShards } from "./test-selection.mjs"; const [group, ...arguments_] = process.argv.slice(2); -const projects = { source: SOURCE_TEST_PROJECTS, built: BUILT_TEST_PROJECTS }[ - group -]; -if (!projects) throw new Error("Expected source or built test group"); -// The group owns project selection; callers can request reporters or shards. +// The group owns project and file selection; callers can request reporters +// or (for project groups) shards. if ( arguments_.some((argument) => /^--(?:project|exclude)(?:=|$)/u.test(argument)) ) throw new Error("Test groups cannot override their project inventory"); +let runner; +let environment = process.env; +if (group === "selected") { + // `selected ` runs group k of the plan's selected test files, by exact + // identity, in whichever projects own them. + const [shard, ...rest] = arguments_; + const plan = parseTestSelection(process.env.CI_TEST_SELECTION ?? ""); + if (plan.mode !== "selected" || !testShards("fast", plan).includes(shard)) + throw new Error(`No selected test group ${shard ?? ""} in this plan`); + if (rest.some((argument) => /^--shard(?:=|$)/u.test(argument))) + throw new Error("Selected test groups are already sharded"); + const index = Number(shard.split("/")[0]) - 1; + runner = [ + resolve("scripts/run-exact-tests.mjs"), + "run", + ...rest, + ...plan.groups[index], + ]; + // Each group writes its own blob report for the merge. + environment = { ...process.env, VITEST_BLOB_LABEL: `selected-${index + 1}` }; +} else { + const projects = { + source: SOURCE_TEST_PROJECTS, + built: BUILT_TEST_PROJECTS, + }[group]; + if (!projects) throw new Error("Expected source, built or selected tests"); + runner = [ + resolve("node_modules/vitest/vitest.mjs"), + "run", + ...projects.flatMap(({ name }) => ["--project", name]), + ...arguments_, + ]; +} const child = spawn( process.execPath, [ @@ -24,12 +55,9 @@ const child = spawn( ...(process.platform === "win32" ? [] : [resolve("scripts/run-exclusive.mjs"), "--unlocked", process.execPath]), - resolve("node_modules/vitest/vitest.mjs"), - "run", - ...projects.flatMap(({ name }) => ["--project", name]), - ...arguments_, + ...runner, ], - { stdio: "inherit" }, + { stdio: "inherit", env: environment }, ); let shutdownSignal; for (const signal of ["SIGINT", "SIGTERM"]) diff --git a/scripts/ci/test-report.mjs b/scripts/ci/test-report.mjs index 4dff45ea13..0ae3cabc61 100644 --- a/scripts/ci/test-report.mjs +++ b/scripts/ci/test-report.mjs @@ -4,16 +4,13 @@ import { isAbsolute, join, relative, resolve, sep } from "node:path"; import { z } from "zod"; -import { compareTestSelection } from "./test-selection.mjs"; +import { parseTestSelection } from "./test-selection.mjs"; const root = process.cwd(); const historyDirectory = join(root, ".cache/ci-test-history"); const historyPath = join(historyDirectory, "test-results.json"); -const selectionDirectory = join(root, ".cache/ci-test-selection"); -const selectionPath = join(selectionDirectory, "report.json"); // Never let a failed or partial rerun leave a previous successful history file. await rm(historyPath, { force: true }); -await rm(selectionPath, { force: true }); const report = z .object({ version: z.literal(1), @@ -56,40 +53,23 @@ const passed = process.env.MERGE_RESULT === "success" && report.success; // Successful shards and merged reports must account for the complete portable -// inventory. Coverage thresholds alone cannot prove that every file ran. +// inventory, or exactly the selected subset. Coverage thresholds alone cannot +// prove that every file ran. const verifyInventory = process.env.CI_VERIFY_TEST_INVENTORY === "true"; -const expected = - verifyInventory || process.env.CI_TEST_SELECTION - ? execFileSync("git", ["ls-files", "-z", "--", "src", "tests"], { +const selection = process.env.CI_TEST_SELECTION + ? parseTestSelection(process.env.CI_TEST_SELECTION) + : undefined; +const selected = selection?.mode === "selected"; +const expected = !verifyInventory + ? [] + : selected + ? selection.groups.flat() + : execFileSync("git", ["ls-files", "-z", "--", "src", "tests"], { encoding: "utf8", maxBuffer: 8 * 1024 * 1024, }) .split("\0") - .filter((path) => path.endsWith(".test.ts")) - : []; -let selection; -if (process.env.CI_TEST_SELECTION) { - const comparison = compareTestSelection( - JSON.parse(process.env.CI_TEST_SELECTION), - expected, - files, - ); - selection = { - ...comparison, - complete: - comparison.complete && - report.unhandledErrors === 0 && - ["passed", "failed"].includes(report.reason) && - groupsCompleted && - ["success", "failure"].includes(process.env.MERGE_RESULT), - executionSha: process.env.GITHUB_SHA ?? null, - prHeadSha: process.env.CI_PR_HEAD_SHA ?? null, - runId: process.env.GITHUB_RUN_ID ?? null, - runAttempt: process.env.GITHUB_RUN_ATTEMPT ?? null, - }; - await mkdir(selectionDirectory, { recursive: true }); - await writeFile(selectionPath, JSON.stringify(selection)); -} + .filter((path) => path.endsWith(".test.ts")); if (passed && verifyInventory) { const actual = files.map(({ name }) => name); if (actual.length === 0) @@ -100,11 +80,13 @@ if (passed && verifyInventory) { const unexpected = actual.filter((path) => !expectedSet.has(path)); if (missing.length || unexpected.length || actualSet.size !== actual.length) throw new Error( - `Incomplete portable test inventory: ${JSON.stringify({ missing, unexpected, duplicateFiles: actual.length - actualSet.size })}`, + `Incomplete ${selected ? "selected" : "portable"} test inventory: ${JSON.stringify({ missing, unexpected, duplicateFiles: actual.length - actualSet.size })}`, ); if (files.every(({ state }) => state === "skipped")) throw new Error("Every portable test module was skipped"); } +// A selected subset is not a timing baseline for the complete suite. +const publishesHistory = passed && !selected; const skippedFiles = files.filter(({ state }) => state === "skipped"); const summary = [ groupsCompleted @@ -113,11 +95,7 @@ const summary = [ "", ...(selection ? [ - "### Observational PR test selection", - "", - `Plan: ${selection.plan.mode}. Proposed files: ${selection.selected.length}; omitted: ${selection.omitted.length}.`, - `Full-run comparison: ${selection.complete ? "complete" : "incomplete"}; failures outside proposed selection: ${selection.missedFailures.length}.`, - "This observational report cannot enable omissions or qualify a revision. Skipped, missing, and interrupted observations do not establish parity.", + `Portable tests: ${selected ? `${selection.groups.flat().length} selected files` : "complete suite"}. ${selection.reason}.`, "", ] : []), @@ -129,7 +107,7 @@ const summary = [ ...(skippedFiles.length ? [""] : []), ...skippedFiles.map(({ name }) => `- ${name.replaceAll("|", "\\|")}`), "", - passed + publishesHistory ? "Successful timing history is available." : "This report is diagnostic only and is not a successful timing baseline.", "", @@ -150,7 +128,7 @@ if (process.env.GITHUB_STEP_SUMMARY) await appendFile(process.env.GITHUB_STEP_SUMMARY, summary); else console.log(summary); -if (passed) { +if (publishesHistory) { await mkdir(historyDirectory, { recursive: true }); await writeFile( historyPath, diff --git a/scripts/ci/test-selection.d.mts b/scripts/ci/test-selection.d.mts index c194143984..0ef16bf3ae 100644 --- a/scripts/ci/test-selection.d.mts +++ b/scripts/ci/test-selection.d.mts @@ -1,35 +1,25 @@ -export interface TestSelectionPlan { - readonly version: 1; - readonly mode: "full" | "candidate"; - readonly owners: readonly string[]; - readonly changedTests: readonly string[]; - readonly changedPaths: readonly string[]; - readonly reason: string; -} -export function planTestSelection( - files: readonly string[], - event: string, - tier: string, -): TestSelectionPlan; -export function compareTestSelection( - plan: unknown, - inventory: readonly string[], - results: readonly { - readonly name: string; - readonly durationMs: number; - readonly state?: string; - }[], -): { - readonly version: 1; - readonly observational: true; - readonly plan: TestSelectionPlan; - readonly selected: readonly string[]; - readonly omitted: readonly string[]; - readonly missing: readonly string[]; - readonly unexpected: readonly string[]; - readonly duplicateFiles: number; - readonly unobserved: readonly string[]; - readonly missedFailures: readonly string[]; - readonly complete: boolean; - readonly omittedDurationMs: number; -}; +export type TestSelectionPlan = + | { + readonly version: 2; + readonly mode: "full"; + readonly reason: string; + readonly groups: readonly []; + } + | { + readonly version: 2; + readonly mode: "selected"; + readonly reason: string; + readonly groups: readonly (readonly string[])[]; + }; +export function planTestSelection(request: { + readonly files: readonly string[]; + readonly event: string; + readonly tier: string; + readonly code: boolean; + readonly snapshot: { + readonly list: () => readonly string[]; + readonly read: (paths: readonly string[]) => ReadonlyMap; + }; +}): TestSelectionPlan; +export function parseTestSelection(value: unknown): TestSelectionPlan; +export function testShards(tier: string, plan: TestSelectionPlan): string[]; diff --git a/scripts/ci/test-selection.mjs b/scripts/ci/test-selection.mjs index 8165e3658c..fbd963d85a 100644 --- a/scripts/ci/test-selection.mjs +++ b/scripts/ci/test-selection.mjs @@ -1,151 +1,224 @@ -import { matchesGlob } from "node:path"; +import { matchesGlob, posix } from "node:path"; -// Candidate ownership only: these leaf adapters and their producer tests can be -// compared with full-suite outcomes. This table never changes test execution. -// Shared application/domain code and dynamic CLI/MCP consumers are not leaf owners. -const OWNERS = { - adb: ["src/adb/**", "tests/boundary/providers/adb/**"], - apktool: ["src/apktool/**"], - dotnet: ["src/dotnet/**"], - evm: ["src/evm/**", "tests/boundary/evm/**"], - firmware: ["src/firmware/**", "tests/boundary/firmware/**"], - flutter: ["src/flutter/**"], - ghidra: ["src/ghidra/**", "tests/boundary/providers/ghidra/**"], - hopper: ["src/hopper/**", "tests/boundary/providers/hopper/**"], - ida: ["src/ida/**", "tests/boundary/providers/ida/**"], - jeb: ["src/jeb/**"], - pcap: ["src/pcap/**", "tests/boundary/pcap/**"], - sqlite: ["src/sqlite/**", "tests/boundary/sqlite/**"], - wasm: ["src/wasm/**"], -}; -const ownersOf = (path) => - Object.entries(OWNERS) - .filter(([, patterns]) => patterns.some((glob) => matchesGlob(path, glob))) - .map(([owner]) => owner); -const validPath = (path) => - typeof path === "string" && - /^(src|tests)\//.test(path) && - !/[\\\0\r\n]/.test(path) && - !path.split("/").some((part) => ["", ".", ".."].includes(part)); -const full = (reason, changedPaths = []) => ({ - version: 1, - mode: "full", - owners: [], - changedTests: [], - changedPaths, - reason, -}); +import { projectForTest } from "../lib/test-projects.mjs"; +import { SHARED_BOUNDARY } from "./scopes.mjs"; -/** Plan an observational PR subset from the classifier's existing Git diff. */ -export function planTestSelection(files, event, tier) { +// Pull-request feedback runs the tests a change most directly affects. Main +// pushes, schedules, releases and `ci:full` keep the complete suite, so a +// consumer the selection misses fails on main right after it lands. +// Selection falls back to the complete suite whenever it cannot decide. + +/** More selected files than this run the complete suite instead. */ +const SELECTION_CAP = 150; +/** Selected files per test job, and the most jobs one selection uses. */ +const FILES_PER_GROUP = 40; +const MAX_GROUPS = 4; + +// Leaf adapters whose tests also reach their sources through the built CLI, +// MCP server or a fake provider process rather than an import. An adapter +// owns `src//` and every test named for it (`*.test.ts`) or +// filed under a `/` directory anywhere in `tests/`. +const OWNERS = [ + "adb", + "apktool", + "dotnet", + "evm", + "firmware", + "flutter", + "ghidra", + "hopper", + "ida", + "jeb", + "pcap", + "sqlite", + "wasm", +]; +const ownedTests = (path) => + OWNERS.filter((owner) => path.startsWith(`src/${owner}/`)).flatMap( + (owner) => [`tests/**/${owner}/**/*.test.ts`, `tests/**/${owner}*.test.ts`], + ); + +const isTest = (path) => path.endsWith(".test.ts"); +// Inputs that only a non-test check reads: declarations (typecheck), guard +// baselines (architecture guards, static job) and the advertised-schema +// summaries (build job). They select no tests. +const isCheckerInput = (path) => + /\.d\.m?ts$/u.test(path) || + /^scripts\/verify\/baselines\/[^/]+\.json$/u.test(path) || + /^scripts\/verify\/advertised-schemas\/tools\/[^/]+\.json$/u.test(path); +// Verifier scripts are imported by their tests like source modules. Other +// scripts (build, test runners, CI) shape every run and are not traced. +const isModule = (path) => + !isCheckerInput(path) && + (/^(src|tests)\/.+\.m?ts$/u.test(path) || + /^scripts\/(verify-[^/]+|verify\/.+)\.mjs$/u.test(path)); +// Test-only modules pass a change through to the tests that use them. +// Production importers never widen the selection transitively. +const isHelper = (path) => + !isTest(path) && + (/^tests\/.+\.m?ts$/u.test(path) || + /^src\/.+\.fixture[^/]*\.ts$/u.test(path)); +// Prose that tests may read by path. Other non-module inputs (fixtures, data, +// scripts, workflows, package metadata) are not traced, so they run everything. +const isProse = (path) => path.endsWith(".md") && !/^(src|tests)\//u.test(path); + +const SPECIFIER = + /(?:\bfrom\s*|\bimport\s*\(\s*|\bimport\s+|\bvi\.mock\(\s*)["'](\.{1,2}\/[^"']+)["']/gu; +// TypeScript imports name the emitted `.js`/`.mjs`; scripts are imported as is. +const importsOf = (path, source) => + [...source.matchAll(SPECIFIER)].flatMap(([, specifier]) => { + const target = posix.normalize(posix.join(posix.dirname(path), specifier)); + return [target, target.replace(/\.(m?)js$/u, ".$1ts")]; + }); + +const full = (reason) => ({ version: 2, mode: "full", reason, groups: [] }); + +/** + * Plan the portable tests one CI run executes. + * + * `snapshot` lists the tracked `src/` and `tests/` paths of the revision under + * test and reads them; it is consulted only when a subset is possible. + */ +export function planTestSelection({ files, event, tier, code, snapshot }) { + // `ci:full` selects the full tier, so it also lands here. if (event !== "pull_request" || tier !== "fast") - return full("Only ordinary PRs are candidates", files); + return full("Main, ci:full and qualification runs execute every test"); + if (!code) return full("No portable tests run for this change"); if (!files.length) return full("No changed paths available"); - const owners = new Set(); - const changedTests = []; for (const path of files) { - if (!validPath(path)) - return full(`Shared or unknown input: ${path}`, files); - const matches = ownersOf(path); - if (!matches.length) return full(`Shared or unknown input: ${path}`, files); - for (const owner of matches) owners.add(owner); - if (path.endsWith(".test.ts")) changedTests.push(path); + if (SHARED_BOUNDARY.test(path)) return full(`Shared boundary: ${path}`); + if (!isModule(path) && !isProse(path) && !isCheckerInput(path)) + return full(`Input is not traced to tests: ${path}`); } + + const tracked = snapshot.list(); + const trackedSet = new Set(tracked); + const tests = tracked.filter(isTest); + const helpers = tracked.filter(isHelper); + const sources = snapshot.read([...tests, ...helpers]); + const imports = new Map( + [...tests, ...helpers].map((path) => [ + path, + importsOf(path, sources.get(path) ?? ""), + ]), + ); + + const selected = new Map(); + const select = (path, reason) => { + if (trackedSet.has(path) && !selected.has(path)) selected.set(path, reason); + }; + const changedModules = files.filter( + (path) => isModule(path) && !isTest(path), + ); + for (const path of files) if (isTest(path)) select(path, "changed"); + + // Close over test-only helpers, then take the tests that import a change. + const affected = new Set(changedModules); + for (let grew = true; grew;) { + grew = false; + for (const helper of helpers) + if ( + !affected.has(helper) && + imports.get(helper).some((target) => affected.has(target)) + ) { + affected.add(helper); + grew = true; + } + } + for (const test of tests) + if (imports.get(test).some((target) => affected.has(target))) + select(test, "imports"); + + for (const path of changedModules) { + // Same-stem tests, and same-directory tests that scan their directory's + // sources by path instead of importing them. + const stem = path.replace(/\.m?ts$/u, ""); + for (const test of tests) + if ( + posix.dirname(test) === posix.dirname(path) && + (test.startsWith(`${stem}.`) || + /\breaddir(?:Sync)?\s*\(/u.test(sources.get(test) ?? "")) + ) + select(test, "beside"); + // Leaf adapters' boundary tests. + const patterns = ownedTests(path); + for (const test of tests) + if (patterns.some((glob) => matchesGlob(test, glob))) + select(test, "owner"); + } + // Tests that name a changed path read it directly. + for (const path of files) + for (const test of tests) + if ((sources.get(test) ?? "").includes(path)) select(test, "names"); + + const paths = [...selected.keys()].sort(); + if (!paths.length) + return full("No test imports, names or sits beside a changed path"); + if (paths.length > SELECTION_CAP) + return full( + `${paths.length} affected test files exceed the ${SELECTION_CAP}-file selection cap`, + ); + for (const path of paths) + try { + projectForTest(path); + } catch { + return full(`No single test project runs ${path}`); + } + const count = Math.min(MAX_GROUPS, Math.ceil(paths.length / FILES_PER_GROUP)); + const groups = Array.from({ length: count }, () => []); + paths.forEach((path, index) => groups[index % count].push(path)); + const reasons = Object.groupBy([...selected.values()], (reason) => reason); return { - version: 1, - mode: "candidate", - owners: [...owners].sort(), - changedTests: [...new Set(changedTests)].sort(), - changedPaths: files, - reason: - "Observe affected leaf owners; retain integration and unowned tests", + version: 2, + mode: "selected", + reason: `Affected tests: ${Object.entries(reasons) + .map(([reason, entries]) => `${entries.length} ${reason}`) + .join(", ")}`, + groups, }; } -/** Compare a proposal with the full run; missing observations never prove parity. */ -export function compareTestSelection(plan, inventory, results) { - if ( - plan?.version !== 1 || - !["full", "candidate"].includes(plan.mode) || - !Array.isArray(plan.owners) || - plan.owners.some((owner) => !Object.hasOwn(OWNERS, owner)) || - !Array.isArray(plan.changedTests) || - !Array.isArray(plan.changedPaths) || - plan.changedPaths.some((path) => typeof path !== "string") || - plan.changedTests.some( - (path) => !validPath(path) || !path.endsWith(".test.ts"), - ) || - typeof plan.reason !== "string" || - (plan.mode === "candidate" && !plan.owners.length) || - (plan.mode === "full" && (plan.owners.length || plan.changedTests.length)) - ) - throw new Error("Invalid observational test selection plan"); - const expected = new Set(inventory); - if (!expected.size || expected.size !== inventory.length) - throw new Error("Test selection requires a nonempty unique inventory"); - const ownersByFile = new Map( - plan.mode === "candidate" - ? inventory.map((path) => [path, ownersOf(path)]) - : [], - ); - // Removed/renamed tests have no current executable identity. Keep all tests. - const removed = plan.changedTests.filter((path) => !expected.has(path)); - const missingOwners = plan.owners.filter( - (owner) => - ![...ownersByFile.values()].some((owners) => owners.includes(owner)), - ); - const effective = removed.length - ? full( - `Changed tests absent from current inventory: ${removed.join(", ")}`, - plan.changedPaths, - ) - : missingOwners.length - ? full( - `Owners without executable tests: ${missingOwners.join(", ")}`, - plan.changedPaths, - ) - : plan; - const selected = []; - const omitted = []; - for (const path of inventory.toSorted()) { - const owners = ownersByFile.get(path) ?? []; - const keep = - effective.mode === "full" || - !owners.length || - effective.changedTests.includes(path) || - owners.some((owner) => effective.owners.includes(owner)); - (keep ? selected : omitted).push(path); - } - const actual = new Set(results.map((file) => file.name)); - const missing = inventory.filter((path) => !actual.has(path)); - const unexpected = [...actual].filter((path) => !expected.has(path)); - const duplicateFiles = results.length - actual.size; - const unobserved = results - .filter((file) => !["passed", "failed"].includes(file.state)) - .map((file) => file.name); - const omittedSet = new Set(omitted); - const missedFailures = results - .filter((file) => omittedSet.has(file.name) && file.state === "failed") - .map((file) => file.name); - return { - version: 1, - observational: true, - plan: effective, - selected, - omitted, - missing, - unexpected, - duplicateFiles, - unobserved, - missedFailures, - complete: - !missing.length && - !unexpected.length && - !duplicateFiles && - !unobserved.length, - // Phase totals are scheduling weights, not an estimate of wall-time savings. - omittedDurationMs: results - .filter((file) => omittedSet.has(file.name)) - .reduce((sum, file) => sum + file.durationMs, 0), - }; +/** Parse a plan produced by `planTestSelection`; anything else fails closed. */ +export function parseTestSelection(value) { + const plan = typeof value === "string" ? JSON.parse(value) : value; + const valid = + plan?.version === 2 && + typeof plan.reason === "string" && + Array.isArray(plan.groups) && + (plan.mode === "full" + ? plan.groups.length === 0 + : plan.mode === "selected" && + plan.groups.length >= 1 && + plan.groups.length <= MAX_GROUPS && + plan.groups.every( + (group) => + Array.isArray(group) && + group.length > 0 && + group.every( + (path) => + typeof path === "string" && + isTest(path) && + /^(src|tests)\//u.test(path) && + !path.split("/").some((part) => ["", ".", ".."].includes(part)), + ), + ) && + new Set(plan.groups.flat()).size === plan.groups.flat().length && + plan.groups.flat().length <= SELECTION_CAP); + if (!valid) throw new Error("Invalid portable test selection plan"); + return plan; } + +// Fast feedback uses one source group and twelve built groups. Shards are +// balanced by recorded duration, but hosted runner speed varies, so the +// slowest shard sets the critical path: at eight shards it ran 24% above the +// mean shard. Smaller shards bound that excess at a few percent more runner +// time. Full qualification retains six complete shards with coverage. A +// selection runs one job per group. +const shardsOf = (count) => + Array.from({ length: count }, (_, index) => `${index + 1}/${count}`); + +/** The `test-shard` matrix labels for a tier and its test selection. */ +export const testShards = (tier, plan) => + shardsOf( + plan.mode === "selected" ? plan.groups.length : tier === "full" ? 6 : 12, + ); diff --git a/tests/boundary/filesystem/ciChangeScope.test.ts b/tests/boundary/filesystem/ciChangeScope.test.ts index 407f8b19fe..f8de9015d3 100644 --- a/tests/boundary/filesystem/ciChangeScope.test.ts +++ b/tests/boundary/filesystem/ciChangeScope.test.ts @@ -168,6 +168,9 @@ function classifyCommit( mergeBase: (base) => base, isAncestor: () => true, changedFiles: () => paths, + // No test reaches these synthetic paths; selection falls back to all. + listFiles: () => [], + readFiles: () => new Map(), readFile: (revision, path) => { if (path === "package.json" && packageJson !== undefined) return revision === PATH_BASE @@ -376,21 +379,114 @@ it("keeps unknown CI scripts on the fast Linux baseline", async () => { for (const flag of CI_FLAGS) expect(full[flag]).toBe(true); }); -it("emits an observational owner proposal without changing ordinary or full portable execution", async () => { - const { directory, base } = await fixture(); - const head = await change(directory, ["src/evm/Example.ts"]); - expect(await classify(directory, base, head)).toMatchObject({ - code: true, - testSelection: { mode: "candidate", owners: ["evm"] }, +it("selects the tests a pull request affects and runs everything when it cannot decide", async () => { + const { directory } = await fixture(); + // The entrypoint's output file is not part of the change. + await mkdir(join(directory, ".git", "info"), { recursive: true }); + await writeFile(join(directory, ".git", "info", "exclude"), "scope-output\n"); + const write = async (path: string, content: string) => { + await mkdir(dirname(join(directory, path)), { recursive: true }); + await writeFile(join(directory, path), content); + }; + const imports = (specifier: string) => + `import { value } from "${specifier}";\nvoid value;\n`; + await write("src/evm/Example.ts", "export const value = 1;\n"); + await write("src/evm/Example.test.ts", imports("./Example.js")); + await write("src/evm/Other.ts", "export const value = 2;\n"); + await write( + "tests/support/evmHelper.ts", + imports("../../src/evm/Example.js"), + ); + // Through a test-only helper, through the leaf owner, and by name. + await write( + "tests/composition/helped.test.ts", + imports("../support/evmHelper.js"), + ); + await write("tests/boundary/evm/cli.test.ts", "// spawns the built CLI\n"); + await write( + "tests/conformance/named.test.ts", + 'readFileSync("src/evm/Example.ts");\n', + ); + await write( + "tests/composition/unrelated.test.ts", + imports("../../src/provider.js"), + ); + const base = await commit(directory); + const head = await change( + directory, + ["src/evm/Example.ts"], + "export const value = 3;\n", + ); + const selected = [ + "src/evm/Example.test.ts", + "tests/boundary/evm/cli.test.ts", + "tests/composition/helped.test.ts", + "tests/conformance/named.test.ts", + ]; + expect(await runPlanScript(directory, base, head)).toMatchObject({ + testSelection: { version: 2, mode: "selected", groups: [selected] }, + testShards: ["1/1"], }); + const output = await readFile(join(directory, "scope-output"), "utf8"); + expect(output).toContain("test_mode=selected\n"); + expect(output).toContain('test_shards=["1/1"]\n'); + + // ci:full, main pushes and shared or untraced inputs run the complete suite. expect( await runPlanScript(directory, base, head, "pull_request", true), ).toMatchObject({ - code: true, - testSelection: { mode: "full", owners: [] }, + testSelection: { mode: "full", groups: [] }, + testShards: ["1/6", "2/6", "3/6", "4/6", "5/6", "6/6"], + }); + expect(await classify(directory, base, head, "push")).toMatchObject({ + testSelection: { mode: "full" }, + testShards: expect.arrayContaining(["12/12"]), + }); + let previous = head; + for (const path of [ + "src/process/Shared.ts", + "scripts/build-something.mjs", + "tests/fixtures/data.bin", + ]) { + const parent = previous; + previous = await change(directory, [path]); + expect(await classify(directory, parent, previous)).toMatchObject({ + testSelection: { mode: "full", reason: expect.stringContaining(path) }, + }); + } + // A leaf owner's boundary tests cover its other modules. + const unreached = await change( + directory, + ["src/evm/Other.ts"], + "export const value = 4;\n", + ); + expect(await classify(directory, previous, unreached)).toMatchObject({ + testSelection: { mode: "selected" }, + }); + const isolated = await change(directory, ["src/unowned/Lonely.ts"]); + expect(await classify(directory, unreached, isolated)).toMatchObject({ + testSelection: { + mode: "full", + reason: "No test imports, names or sits beside a changed path", + }, + }); + // Above the cap, selection gives way to the complete suite. + const many = Array.from( + { length: 151 }, + (_, index) => `tests/composition/many${String(index)}.test.ts`, + ); + const wide = await change(directory, many); + expect(await classify(directory, isolated, wide)).toMatchObject({ + testSelection: { + mode: "full", + reason: "151 affected test files exceed the 150-file selection cap", + }, + }); + const capped = await change(directory, many.slice(0, 81), "// edited\n"); + expect(await classify(directory, wide, capped)).toMatchObject({ + testSelection: { mode: "selected" }, + testShards: ["1/3", "2/3", "3/3"], }); - const output = await readFile(join(directory, "scope-output"), "utf8"); - expect(output).toContain('test_selection={"version":1,"mode":"full"'); }); it("passes the workflow environment through the entrypoint unchanged", async () => { @@ -1468,6 +1564,7 @@ it.each(["evm", "browser"])( outputs: { ...Object.fromEntries(CI_FLAGS.map((flag) => [flag, "false"])), tier: "fast", + test_mode: "full", [scope]: "true", }, }, diff --git a/tests/boundary/filesystem/ciFeedback.test.ts b/tests/boundary/filesystem/ciFeedback.test.ts index 48530fd75f..bdd507d5b7 100644 --- a/tests/boundary/filesystem/ciFeedback.test.ts +++ b/tests/boundary/filesystem/ciFeedback.test.ts @@ -12,6 +12,7 @@ import { parseStaticTaskContext, recordStaticTaskContext, } from "../../../scripts/ci/static-task-context.mjs"; +import { testShards } from "../../../scripts/ci/test-selection.mjs"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; const execute = promisify(execFile); @@ -121,6 +122,7 @@ it.each(["fast", "full"] as const)( flags.map((flag) => [flag, tier === "full" ? "true" : "false"]), ), tier, + test_mode: "full", code: "true", reverse_tools: "true", }, @@ -265,6 +267,7 @@ it("starts source feedback before build and requires both groups without waiving flags.map((flag) => [flag, tier === "full" ? "true" : "false"]), ), tier, + test_mode: "full", code: "true", }; const context: Record< @@ -398,7 +401,20 @@ it("merges source and built reports exactly once using the workflow's shard matr matrix.slice(3, -2), { fromJSON: JSON.parse, - needs: { changes: { outputs: { tier: "fast" } } }, + needs: { + changes: { + outputs: { + test_shards: JSON.stringify( + testShards("fast", { + version: 2, + mode: "full", + reason: "Complete fast suite", + groups: [], + }), + ), + }, + }, + }, }, { timeout: 1_000 }, ), diff --git a/tests/boundary/filesystem/ciQualification.test.ts b/tests/boundary/filesystem/ciQualification.test.ts index 673b09d34c..a8347c7a08 100644 --- a/tests/boundary/filesystem/ciQualification.test.ts +++ b/tests/boundary/filesystem/ciQualification.test.ts @@ -259,7 +259,10 @@ async function requiredRun( : "success", }; } - results.changes = { result: "success", outputs: changes }; + results.changes = { + result: "success", + outputs: { test_mode: "full", ...changes }, + }; const receiptDirectory = join(directory, ".cache", "receipt"); const output = join(directory, "github-output"); await writeFile(output, ""); diff --git a/tests/boundary/filesystem/ciRequiredLanes.test.ts b/tests/boundary/filesystem/ciRequiredLanes.test.ts index c693dd7687..69314d4467 100644 --- a/tests/boundary/filesystem/ciRequiredLanes.test.ts +++ b/tests/boundary/filesystem/ciRequiredLanes.test.ts @@ -593,7 +593,9 @@ it("keeps fast feedback separate from complete qualification and release proof", const shards = ci.jobs["test-shard"]?.steps ?? []; const fast = shards.find((step) => step.run?.includes("npm run test:built")); const full = shards.find((step) => step.run?.includes("test:ci:shard:run")); - expect(fast?.if).toBe("needs.changes.outputs.tier == 'fast'"); + expect(fast?.if).toBe( + "needs.changes.outputs.tier == 'fast' && needs.changes.outputs.test_mode == 'full'", + ); expect(fast?.run).toContain("--reporter=blob"); expect(fast?.run).not.toContain("--coverage"); expect(full?.if).toBe("needs.changes.outputs.tier == 'full'"); @@ -640,6 +642,7 @@ it.each([ (await scopeFlags()).map((flag) => [flag, "false"]), ), tier: "fast", + test_mode: "full", code: "true", [scope]: selected, }, @@ -673,6 +676,7 @@ it("rejects incomplete or malformed classification even when available jobs succ const valid = { ...Object.fromEntries(flags.map((flag) => [flag, "false"])), tier: "fast", + test_mode: "full", }; const run = (outputs: unknown) => executeWorkflowFixture( @@ -750,6 +754,7 @@ it("fails fast only for ordinary PR matrices and rejects partial portable feedba outputs: { ...Object.fromEntries(flags.map((flag) => [flag, "false"])), tier: "fast", + test_mode: "full", code: "true", }, }, @@ -775,6 +780,105 @@ it("fails fast only for ordinary PR matrices and rejects partial portable feedba } }); +it("runs a selected PR subset in planner-owned groups and still gates it", async () => { + const { jobs } = await workflow(); + // The planner owns every shard list, including one job per selected group. + expect(jobs["test-shard"]?.strategy).toMatchObject({ + matrix: { shard: "${{ fromJSON(needs.changes.outputs.test_shards) }}" }, + }); + expect(jobs["test-source"]).toMatchObject({ + if: expect.stringContaining("needs.changes.outputs.test_mode == 'full'"), + }); + const selected = z + .array( + z.object({ + if: z.string().optional(), + run: z.string().optional(), + env: z.record(z.string(), z.string()).optional(), + }), + ) + .parse(jobs["test-shard"]?.steps) + .find( + (step) => step.if === "needs.changes.outputs.test_mode == 'selected'", + ); + expect(selected?.run).toBe( + 'node scripts/ci/test-group.mjs selected "${MATRIX_SHARD}" --reporter=default --reporter=blob', + ); + expect(selected?.env).toEqual({ + MATRIX_SHARD: "${{ matrix.shard }}", + CI_TEST_SELECTION: "${{ needs.changes.outputs.test_selection }}", + }); + + const flags = await scopeFlags(); + const gate = ( + outputs: Record, + results: Record, + event = "pull_request", + ) => + executeWorkflowFixture( + process.execPath, + [ + fileURLToPath( + new URL("../../../scripts/ci/required.mjs", import.meta.url), + ), + ], + { + env: { + ...process.env, + GITHUB_EVENT_NAME: event, + CI_QUALIFICATION_DIRECTORY: "", + CI_JOBS: JSON.stringify({ + changes: { + result: "success", + outputs: { + ...Object.fromEntries(flags.map((flag) => [flag, "false"])), + tier: "fast", + code: "true", + test_mode: "selected", + ...outputs, + }, + }, + static: { result: "success" }, + test: { result: "success" }, + ...Object.fromEntries( + Object.entries(results).map(([name, result]) => [ + name, + { result }, + ]), + ), + }), + }, + }, + ); + const ran = { build: "success", "test-shard": "success" }; + // A selected subset skips the separate source job. + await expect(gate({}, ran)).resolves.toMatchObject({ stderr: "" }); + for (const [name, result] of [ + ["test-shard", "skipped"], + ["test-shard", "failure"], + ["build", "cancelled"], + ] as const) + await expect(gate({}, { ...ran, [name]: result })).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining(name), + }); + // The complete fast suite still requires its source job. + await expect(gate({ test_mode: "full" }, ran)).rejects.toMatchObject({ + stderr: expect.stringContaining("test-source"), + }); + // Only a fast pull-request plan with code changes may select tests. + for (const [outputs, event] of [ + [{ test_mode: undefined }, "pull_request"], + [{ test_mode: "candidate" }, "pull_request"], + [{}, "push"], + [{ code: "false" }, "pull_request"], + ] as const) + await expect(gate(outputs, ran, event)).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining("valid test mode"), + }); +}); + it("binds each canonical lane to its explicit fast and full workflow jobs", async () => { const { jobs, needs } = await requiredNeeds(); const { CI_LANES } = await import( @@ -870,6 +974,7 @@ it("binds each canonical lane to its explicit fast and full workflow jobs", asyn outputs: { ...Object.fromEntries(flags.map((name) => [name, "false"])), tier: "fast", + test_mode: "full", [scope]: "true", }, }; @@ -897,6 +1002,7 @@ it("binds each canonical lane to its explicit fast and full workflow jobs", asyn outputs: { ...Object.fromEntries(flags.map((name) => [name, "true"])), tier: "full", + test_mode: "full", }, }; await expect(executeGate(fullContext)).resolves.toMatchObject({ stderr: "" }); diff --git a/tests/boundary/filesystem/ciTestReports.test.ts b/tests/boundary/filesystem/ciTestReports.test.ts index 1a4e8e171a..b1d1f5d920 100644 --- a/tests/boundary/filesystem/ciTestReports.test.ts +++ b/tests/boundary/filesystem/ciTestReports.test.ts @@ -9,8 +9,6 @@ import { expect, it } from "vitest"; import { parse } from "yaml"; import { z } from "zod"; -import { planTestSelection } from "../../../scripts/ci/test-selection.mjs"; - import { parseTestDurations } from "../../ciTestSequencer.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { workflowGit } from "../../support/workflowGit.js"; @@ -102,9 +100,6 @@ it("merges available failed-shard diagnostics and skips unavailable shard report } expect(steps[upload]?.with?.path).toContain("test-timings.json"); expect(steps[upload]?.with).toMatchObject({ "include-hidden-files": true }); - expect(steps[upload]?.with?.path).toContain( - ".cache/ci-test-selection/report.json", - ); const summary = steps.find((step) => step.run?.includes("scripts/ci/test-report.mjs"), ); @@ -212,8 +207,9 @@ it("collects either fast test group while keeping full reports tied to full shar ); expect(summary?.env?.SHARD_RESULT).toBe("${{ needs.test-shard.result }}"); expect(summary?.env?.SOURCE_RESULT).toBe("${{ needs.test-source.result }}"); + // A selected PR subset runs its source tests in the selected test jobs. expect(summary?.env?.SOURCE_REQUIRED).toBe( - "${{ needs.changes.outputs.tier == 'fast' }}", + "${{ needs.changes.outputs.tier == 'fast' && needs.changes.outputs.test_mode == 'full' }}", ); }); @@ -660,77 +656,78 @@ it("recovers complete module costs from real blob reports without executing test ).rejects.toMatchObject({ code: "ENOENT" }); }); -it("records failed omissions from full reports without treating partial or unhandled outcomes as parity", async () => { +it("verifies exactly the selected subset and never publishes it as timing history", async () => { const root = await createTestTempDirectory("rea-ci-selection-report-"); await workflowGit(root, ["init"]); const paths = [ - "src/sqlite/provider.test.ts", - "src/evm/provider.test.ts", - "tests/boundary/mcp/workflow.test.ts", + "src/a.test.ts", + "tests/b.test.ts", + "tests/unselected.test.ts", ]; for (const path of paths) { await mkdir(join(root, path, ".."), { recursive: true }); await writeFile(join(root, path), "// fixture\n"); } await workflowGit(root, ["add", "src", "tests"]); - const timing = { - version: 1, - success: false, - reason: "failed", - unhandledErrors: 0, - files: paths.map((name, index) => ({ - name, - durationMs: 10, - state: index === 1 ? "failed" : "passed", - })), + const selection = { + version: 2, + mode: "selected", + reason: "Affected tests: 2 changed", + groups: [["src/a.test.ts"], ["tests/b.test.ts"]], }; - const run = async (report: unknown, source = "failure") => { - await writeFile(join(root, "test-timings.json"), JSON.stringify(report)); - await executeTimingReport(root, { - SHARD_RESULT: "failure", - SOURCE_REQUIRED: "true", - SOURCE_RESULT: source, - MERGE_RESULT: "failure", + const run = async (names: readonly string[], plan: unknown = selection) => { + await writeFile( + join(root, "test-timings.json"), + JSON.stringify({ + version: 1, + success: true, + files: names.map((name) => ({ name, durationMs: 1, state: "passed" })), + }), + ); + return executeTimingReport(root, { CI_VERIFY_TEST_INVENTORY: "true", - CI_TEST_SELECTION: JSON.stringify( - planTestSelection(["src/sqlite/Provider.ts"], "pull_request", "fast"), - ), - GITHUB_SHA: "merge-sha", - CI_PR_HEAD_SHA: "pr-head-sha", + CI_TEST_SELECTION: JSON.stringify(plan), }); - return JSON.parse( - await readFile( - join(root, ".cache/ci-test-selection/report.json"), - "utf8", - ), - ) as unknown; }; - expect(await run(timing)).toMatchObject({ - observational: true, - complete: true, - selected: [paths[0], paths[2]], - omitted: [paths[1]], - missedFailures: [paths[1]], - executionSha: "merge-sha", - prHeadSha: "pr-head-sha", - }); - for (const report of [ - { ...timing, files: timing.files.slice(0, 1) }, - { ...timing, unhandledErrors: 1 }, - { ...timing, reason: "interrupted" }, - ]) - expect(await run(report)).toMatchObject({ complete: false }); - for (const source of ["skipped", "cancelled", ""]) - expect(await run(timing, source)).toMatchObject({ complete: false }); - await expect(run({ ...timing, files: "invalid" })).rejects.toMatchObject({ - code: 1, - }); - await expect( - readFile(join(root, ".cache/ci-test-selection/report.json")), - ).rejects.toMatchObject({ code: "ENOENT" }); + const passed = await run(["src/a.test.ts", "tests/b.test.ts"]); + expect(passed.summary).toContain("Portable tests: 2 selected files."); + expect(passed.summary).toContain("is not a successful timing baseline"); + expect(passed.summary).not.toContain("Successful timing history"); + // A subset is not a complete-suite timing baseline. await expect( readFile(join(root, ".cache/ci-test-history/test-results.json")), ).rejects.toMatchObject({ code: "ENOENT" }); + // Missing, unplanned (even if tracked) and duplicate files fail. + for (const names of [ + ["src/a.test.ts"], + ["src/a.test.ts", "tests/b.test.ts", "tests/unselected.test.ts"], + ["src/a.test.ts", "src/a.test.ts", "tests/b.test.ts"], + ]) + await expect(run(names)).rejects.toMatchObject({ + stderr: expect.stringContaining("Incomplete selected test inventory"), + }); + // A full plan keeps the complete tracked inventory and its history. + const full = { version: 2, mode: "full", reason: "Shared", groups: [] }; + await expect( + run(["src/a.test.ts", "tests/b.test.ts"], full), + ).rejects.toMatchObject({ + stderr: expect.stringContaining("Incomplete portable test inventory"), + }); + await run(paths, full); + await expect( + readFile(join(root, ".cache/ci-test-history/test-results.json"), "utf8"), + ).resolves.toContain("tests/unselected.test.ts"); + // An unreadable plan fails closed. + for (const plan of [ + { ...selection, version: 1 }, + { ...selection, groups: [] }, + { ...selection, groups: [["../escape.test.ts"]] }, + { ...selection, groups: [["src/a.test.ts"], ["src/a.test.ts"]] }, + { ...full, groups: [["src/a.test.ts"]] }, + ]) + await expect(run(["src/a.test.ts"], plan)).rejects.toMatchObject({ + stderr: expect.stringContaining("Invalid portable test selection plan"), + }); }); it("requires executed Python analysis-efficiency cases while allowing optional skips", async () => { diff --git a/tests/evaluation/ciTestSelection.test.ts b/tests/evaluation/ciTestSelection.test.ts index bf32fcf45a..bfe8baa287 100644 --- a/tests/evaluation/ciTestSelection.test.ts +++ b/tests/evaluation/ciTestSelection.test.ts @@ -1,240 +1,164 @@ +import { execFile } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + import { expect, it } from "vitest"; import { - compareTestSelection, + parseTestSelection, planTestSelection, + testShards, } from "../../scripts/ci/test-selection.mjs"; -const sqliteSource = "src/sqlite/SqliteDatabaseProvider.test.ts"; -const sqliteBoundary = "tests/boundary/sqlite/sqliteDatabaseProvider.test.ts"; -const adbSource = "src/adb/AdbProvider.test.ts"; -const adbBoundary = - "tests/boundary/providers/adb/adbBugreportPublication.test.ts"; -const cliBoundary = "tests/boundary/cli/apktoolCli.test.ts"; -const mcpBoundary = "tests/boundary/mcp/apktoolMcp.test.ts"; -const processBoundary = - "tests/boundary/process/processCaptureLifecycle.test.ts"; -const acceptance = "tests/acceptance/tools/localToolContracts.test.ts"; -const composition = "tests/composition/directAnalysisCleanup.test.ts"; -const evaluation = "tests/evaluation/testDiscovery.test.ts"; +const execute = promisify(execFile); -const passed = (name: string) => ({ name, durationMs: 10, state: "passed" }); - -it("unions affected leaf owners, keeps changed tests and retains unowned integration tests", () => { - const plan = planTestSelection( - ["src/sqlite/SqliteDatabaseProvider.ts", adbBoundary], - "pull_request", - "fast", - ); - const inventory = [ - sqliteSource, - sqliteBoundary, - adbSource, - adbBoundary, - cliBoundary, - mcpBoundary, - processBoundary, - acceptance, - composition, - evaluation, - ]; - const report = compareTestSelection(plan, inventory, inventory.map(passed)); - - expect(plan).toMatchObject({ - mode: "candidate", - owners: ["adb", "sqlite"], +const plan = (files: readonly string[], sources: Record) => + planTestSelection({ + files, + event: "pull_request", + tier: "fast", + code: true, + snapshot: { + list: () => Object.keys(sources), + read: (paths) => + new Map(paths.map((path) => [path, sources[path] ?? ""])), + }, }); - expect(report.selected).toEqual( - expect.arrayContaining([ - sqliteSource, - sqliteBoundary, - adbSource, - adbBoundary, - cliBoundary, - mcpBoundary, - processBoundary, - acceptance, - composition, - evaluation, - ]), - ); - expect(report.omitted).toEqual([]); + +const selectedFiles = (result: ReturnType) => + result.groups.flat().toSorted(); + +it("follows test and verifier imports and directory scans, not production importers", () => { + const sources = { + "src/domain/Value.ts": "", + "src/application/UsesValue.ts": 'import "../domain/Value.js";', + "src/application/UsesValue.test.ts": 'import "./UsesValue.js";', + "src/domain/Value.test.ts": 'import { value } from "./Value.js";', + "src/domain/Registry.test.ts": "readdirSync(here);", + "src/domain/Unrelated.test.ts": 'import "./Unrelated.js";', + "tests/boundary/filesystem/verifier.test.ts": + 'const m = await import("../../../scripts/verify-thing.mjs");', + }; + expect(selectedFiles(plan(["src/domain/Value.ts"], sources))).toEqual([ + "src/domain/Registry.test.ts", + "src/domain/Value.test.ts", + ]); + expect(selectedFiles(plan(["scripts/verify-thing.mjs"], sources))).toEqual([ + "tests/boundary/filesystem/verifier.test.ts", + ]); }); -it("omits only unrelated owned tests while retaining dynamic boundary suites", () => { - const plan = planTestSelection( - ["src/sqlite/SqliteDatabaseProvider.ts"], - "pull_request", - "fast", - ); - const inventory = [ - sqliteSource, - sqliteBoundary, - adbSource, - adbBoundary, - cliBoundary, - mcpBoundary, - processBoundary, - acceptance, - composition, - evaluation, - ]; - const report = compareTestSelection(plan, inventory, inventory.map(passed)); - - expect(report.selected).toEqual( - expect.arrayContaining([ - sqliteSource, - sqliteBoundary, - cliBoundary, - mcpBoundary, - processBoundary, - acceptance, - composition, - evaluation, - ]), - ); - expect(report.omitted).toEqual([adbSource, adbBoundary]); - expect(report.complete).toBe(true); +it("selects a leaf adapter's tests by its name and directory across test projects", () => { + const sources = { + "src/jeb/JebMcpDeadline.ts": "", + "tests/boundary/providers/jebMcpTimeout.test.ts": "", + "tests/boundary/mcp/jebMcp.test.ts": "", + "tests/acceptance/providers/jebConfigurationIsolation.test.ts": "", + "tests/boundary/providers/adb/adbBugreport.test.ts": "", + "tests/boundary/providers/ghidraJava.test.ts": "", + }; + expect(selectedFiles(plan(["src/jeb/JebMcpDeadline.ts"], sources))).toEqual([ + "tests/acceptance/providers/jebConfigurationIsolation.test.ts", + "tests/boundary/mcp/jebMcp.test.ts", + "tests/boundary/providers/jebMcpTimeout.test.ts", + ]); + expect(selectedFiles(plan(["src/adb/AdbProvider.ts"], sources))).toEqual([ + "tests/boundary/providers/adb/adbBugreport.test.ts", + ]); }); -it.each([ - ["unowned shared source", "src/application/Workflow.ts"], - ["fixture input", "tests/conformance/c/fixture.c"], - ["configuration", "vitest.config.ts"], - ["documentation", "README.md"], - ["CLI implementation", "src/cli/apktoolCommands.ts"], - ["MCP implementation", "src/server/contractToolBindings.ts"], - ["process ownership", "src/process/ProcessOwnership.ts"], - ["acceptance test", acceptance], - ["composition test", composition], -])("uses full fallback for %s changes", (_description, path) => { - expect(planTestSelection([path], "pull_request", "fast")).toMatchObject({ +it("lets checker-only inputs and prose select nothing by themselves", () => { + const sources = { + "src/domain/Value.test.ts": "", + "tests/boundary/filesystem/docs.test.ts": 'read("docs/guide.md");', + }; + // A declaration, guard baseline or schema summary adds no tests… + expect( + selectedFiles( + plan( + [ + "src/domain/Value.test.ts", + "scripts/lib/helper.d.mts", + "scripts/verify/baselines/some-guard.json", + "scripts/verify/advertised-schemas/tools/tool.json", + ], + sources, + ), + ), + ).toEqual(["src/domain/Value.test.ts"]); + // …and prose selects the tests that read it by path. + expect(selectedFiles(plan(["docs/guide.md"], sources))).toEqual([ + "tests/boundary/filesystem/docs.test.ts", + ]); + expect(plan(["scripts/verify/baselines/x.json"], sources)).toMatchObject({ mode: "full", - owners: [], - changedTests: [], + }); + expect(plan(["scripts/lib/shared.mjs"], sources)).toMatchObject({ + mode: "full", + reason: "Input is not traced to tests: scripts/lib/shared.mjs", }); }); -it("uses full fallback outside ordinary fast pull requests or without diff paths", () => { +it("parses only well-formed plans and derives one shard per selected group", () => { + const selected = { + version: 2, + mode: "selected", + reason: "Affected tests: 2 changed", + groups: [["src/a.test.ts"], ["tests/b.test.ts"]], + }; expect( - planTestSelection(["src/sqlite/SqliteDatabaseProvider.ts"], "push", "fast") - .mode, - ).toBe("full"); - expect( - planTestSelection( - ["src/sqlite/SqliteDatabaseProvider.ts"], - "pull_request", - "full", - ).mode, - ).toBe("full"); - expect(planTestSelection([], "pull_request", "fast").mode).toBe("full"); + testShards("fast", parseTestSelection(JSON.stringify(selected))), + ).toEqual(["1/2", "2/2"]); + const full = { version: 2, mode: "full", reason: "Shared", groups: [] }; + expect(testShards("fast", parseTestSelection(full))).toHaveLength(12); + expect(testShards("full", parseTestSelection(full))).toHaveLength(6); + for (const value of [ + "", + "null", + { ...selected, mode: "candidate" }, + { ...selected, groups: [[]] }, + { ...selected, groups: [["src/a.ts"]] }, + { ...selected, groups: [["/abs/a.test.ts"]] }, + { ...selected, groups: [["src/a.test.ts"], ["src/a.test.ts"]] }, + { + ...selected, + groups: Array.from({ length: 5 }, (_, i) => [`src/${i}.test.ts`]), + }, + ]) + expect(() => parseTestSelection(value)).toThrow( + /Invalid portable test selection plan|JSON/u, + ); }); -it("keeps new changed tests and falls back when a changed test was removed", () => { - const addedTest = "src/sqlite/NewDatabaseRule.test.ts"; - const candidate = planTestSelection([addedTest], "pull_request", "fast"); - expect(candidate.changedTests).toEqual([addedTest]); - const inventory = [addedTest, adbSource]; - expect( - compareTestSelection(candidate, inventory, inventory.map(passed)).selected, - ).toContain(addedTest); - - const removedTest = "src/sqlite/RemovedDatabaseRule.test.ts"; - const report = compareTestSelection( - planTestSelection([removedTest], "pull_request", "fast"), - [sqliteSource, adbSource], - [passed(sqliteSource), passed(adbSource)], +it("runs only a group the plan defines", async () => { + const group = fileURLToPath( + new URL("../../scripts/ci/test-group.mjs", import.meta.url), ); - expect(report.plan.mode).toBe("full"); - expect(report.selected).toEqual([adbSource, sqliteSource]); - expect(report.omitted).toEqual([]); -}); - -it("marks incomplete inventories and nonterminal outcomes as incomplete", () => { - const plan = planTestSelection( - ["src/sqlite/SqliteDatabaseProvider.ts"], - "pull_request", - "fast", - ); - const inventory = [sqliteSource, adbSource]; - const completeResults = inventory.map(passed); - - expect( - compareTestSelection(plan, inventory, [passed(sqliteSource)]).complete, - ).toBe(false); - expect( - compareTestSelection(plan, inventory, [ - ...completeResults, - passed("tests/evaluation/unexpected.test.ts"), - ]), - ).toMatchObject({ - complete: false, - unexpected: ["tests/evaluation/unexpected.test.ts"], + const selection = JSON.stringify({ + version: 2, + mode: "selected", + reason: "Affected tests: 1 changed", + groups: [["src/a.test.ts"]], + }); + for (const [shard, environment] of [ + ["2/2", selection], + ["1/2", selection], + [ + "1/1", + JSON.stringify({ version: 2, mode: "full", reason: "x", groups: [] }), + ], + ["1/1", ""], + ] as const) + await expect( + execute(process.execPath, [group, "selected", shard], { + env: { ...process.env, CI_TEST_SELECTION: environment }, + }), + ).rejects.toMatchObject({ code: 1 }); + await expect( + execute(process.execPath, [group, "selected", "1/1", "--shard=1/2"], { + env: { ...process.env, CI_TEST_SELECTION: selection }, + }), + ).rejects.toMatchObject({ + stderr: expect.stringContaining("already sharded"), }); - expect( - compareTestSelection(plan, inventory, [ - ...completeResults, - passed(adbSource), - ]), - ).toMatchObject({ complete: false, duplicateFiles: 1 }); - for (const state of ["pending", "skipped"]) - expect( - compareTestSelection(plan, inventory, [ - passed(sqliteSource), - { name: adbSource, durationMs: 0, state }, - ]), - ).toMatchObject({ complete: false, unobserved: [adbSource] }); -}); - -it("reports a failure from an omitted test, including collection or module failures", () => { - const plan = planTestSelection( - ["src/sqlite/SqliteDatabaseProvider.ts"], - "pull_request", - "fast", - ); - const report = compareTestSelection( - plan, - [sqliteSource, adbSource], - [passed(sqliteSource), { name: adbSource, durationMs: 1, state: "failed" }], - ); - - expect(report.omitted).toContain(adbSource); - expect(report.missedFailures).toEqual([adbSource]); -}); - -it("rejects plans that name unowned owners or malformed changed tests", () => { - const plan = planTestSelection( - ["src/sqlite/SqliteDatabaseProvider.ts"], - "pull_request", - "fast", - ); - expect(() => - compareTestSelection( - { ...plan, owners: ["unknown"] }, - [sqliteSource], - [passed(sqliteSource)], - ), - ).toThrow("Invalid observational test selection plan"); - expect(() => - compareTestSelection( - { ...plan, changedTests: ["src/sqlite/not-a-test.ts"] }, - [sqliteSource], - [passed(sqliteSource)], - ), - ).toThrow("Invalid observational test selection plan"); -}); - -it("uses full fallback when a changed owner has no executable tests", () => { - const report = compareTestSelection( - planTestSelection( - ["src/sqlite/SqliteDatabaseProvider.ts"], - "pull_request", - "fast", - ), - [adbSource], - [passed(adbSource)], - ); - - expect(report.plan.mode).toBe("full"); - expect(report.omitted).toEqual([]); });