mirror of
https://github.com/morluto/rea.git
synced 2026-11-16 19:53:50 +00:00
* ci: trace shared script modules to the tests that import them A pull request that changed any scripts/lib module ran the complete suite. Shared script modules now pass a change through to the tests, verifiers and other shared modules that import them, like test helpers. Another consumer (a build or test-runner script, root configuration, package metadata, a hook or a workflow) that imports the module or names it by file name still selects the complete suite. The tracing core is exported as traceAffectedTests so a bounded local gate can reuse it. Fixes #11381 * ci: follow verifier source loads and named verifiers when tracing script modules - Module URLs (new URL("./x.mts", import.meta.url), as loaded by importVerifierSource) are import edges. - A test that names a verifier or shared module the change reaches is selected, not only one that names a changed path. - A changed shared module that no test reaches runs the complete suite even when other changes select tests. * ci: select tests that start the built server beside a changed presentation owner (#11519) * ci: select tests that start the built server beside a changed presentation owner A test that starts the built CLI or MCP server reaches server, contract, entrypoint and CLI modules through the build, not an import, so a change there selected none of them (#11146 failed on main this way). Such a test is now selected when it, or a helper it imports, starts scripts/rea.mjs or dist/, and it imports a module from the changed owner's directory. Fixes #11366 * ci: recognise only the repository's built entry points as runtime launchers A package subpath such as ajv/dist/2020.js matched the dist/ pattern, so in-process tests importing Ajv counted as built-runtime launchers. * ci: balance selected test shards by recorded duration (#11522) A selected PR run split its files across groups round-robin, so a few slow files could land together. The plan now records the selected files and a shard count (version 3), and each job runs one --shard of the selection through the exact-test runner with the timing history main already publishes, so the CI sequencer balances shards by per-file duration (the median for files without a timing). Without history, Vitest's own deterministic sharding applies. Fixes #11382
272 lines
10 KiB
TypeScript
272 lines
10 KiB
TypeScript
import { execFile } from "node:child_process";
|
|
import { fileURLToPath } from "node:url";
|
|
import { promisify } from "node:util";
|
|
|
|
import { expect, it } from "vitest";
|
|
|
|
import {
|
|
parseTestSelection,
|
|
planTestSelection,
|
|
testShards,
|
|
} from "../../scripts/ci/test-selection.mjs";
|
|
|
|
const execute = promisify(execFile);
|
|
|
|
const plan = (files: readonly string[], sources: Record<string, string>) =>
|
|
planTestSelection({
|
|
files,
|
|
event: "pull_request",
|
|
tier: "fast",
|
|
code: true,
|
|
snapshot: {
|
|
list: () => Object.keys(sources),
|
|
read: (paths) =>
|
|
new Map(paths.map((path) => [path, sources[path] ?? ""])),
|
|
},
|
|
});
|
|
|
|
const selectedFiles = (result: ReturnType<typeof plan>) =>
|
|
(result.mode === "selected" ? result.files : []).toSorted();
|
|
|
|
it("follows test and verifier imports and directory scans, not production importers", () => {
|
|
const sources = {
|
|
"src/domain/Value.ts": "",
|
|
"src/application/UsesValue.ts": 'import "../domain/Value.js";',
|
|
"src/application/UsesValue.test.ts": 'import "./UsesValue.js";',
|
|
"src/domain/Value.test.ts": 'import { value } from "./Value.js";',
|
|
"src/domain/Registry.test.ts": "readdirSync(here);",
|
|
"src/domain/Unrelated.test.ts": 'import "./Unrelated.js";',
|
|
"tests/boundary/filesystem/verifier.test.ts":
|
|
'const m = await import("../../../scripts/verify-thing.mjs");',
|
|
};
|
|
expect(selectedFiles(plan(["src/domain/Value.ts"], sources))).toEqual([
|
|
"src/domain/Registry.test.ts",
|
|
"src/domain/Value.test.ts",
|
|
]);
|
|
expect(selectedFiles(plan(["scripts/verify-thing.mjs"], sources))).toEqual([
|
|
"tests/boundary/filesystem/verifier.test.ts",
|
|
]);
|
|
});
|
|
|
|
it("selects a leaf adapter's tests by its name and directory across test projects", () => {
|
|
const sources = {
|
|
"src/jeb/JebMcpDeadline.ts": "",
|
|
"tests/boundary/providers/jebMcpTimeout.test.ts": "",
|
|
"tests/boundary/mcp/jebMcp.test.ts": "",
|
|
"tests/acceptance/providers/jebConfigurationIsolation.test.ts": "",
|
|
"tests/boundary/providers/adb/adbBugreport.test.ts": "",
|
|
"tests/boundary/providers/ghidraJava.test.ts": "",
|
|
};
|
|
expect(selectedFiles(plan(["src/jeb/JebMcpDeadline.ts"], sources))).toEqual([
|
|
"tests/acceptance/providers/jebConfigurationIsolation.test.ts",
|
|
"tests/boundary/mcp/jebMcp.test.ts",
|
|
"tests/boundary/providers/jebMcpTimeout.test.ts",
|
|
]);
|
|
expect(selectedFiles(plan(["src/adb/AdbProvider.ts"], sources))).toEqual([
|
|
"tests/boundary/providers/adb/adbBugreport.test.ts",
|
|
]);
|
|
});
|
|
|
|
it("selects tests that start the built runtime beside a changed presentation owner", () => {
|
|
const sources = {
|
|
"src/server/Core.ts": "",
|
|
"src/server/recovery.ts": "",
|
|
"src/domain/Value.ts": "",
|
|
"tests/support/launch.ts": 'spawn("scripts/rea.mjs", ["mcp"]);',
|
|
"tests/fixtures/client.ts": 'import "../support/launch.js";',
|
|
// Starts the built server and imports a sibling of the changed owner.
|
|
"tests/boundary/mcp/direct.test.ts":
|
|
'import "../../../src/server/recovery.js"; resolve("dist/main.js");',
|
|
"tests/boundary/mcp/helper.test.ts":
|
|
'import "../../fixtures/client.js"; import "../../../src/server/recovery.js";',
|
|
// Starts it but imports nothing from that owner, or imports it in process.
|
|
"tests/boundary/mcp/unrelated.test.ts":
|
|
'import "../../fixtures/client.js"; import "../../../src/domain/Value.js";',
|
|
"tests/boundary/mcp/inProcess.test.ts":
|
|
'import "../../../src/server/recovery.js";',
|
|
// A package's own dist/ subpath is not the built runtime.
|
|
"tests/boundary/mcp/ajv.test.ts":
|
|
'import { Ajv2020 } from "ajv/dist/2020.js"; import "../../../src/server/recovery.js";',
|
|
};
|
|
expect(selectedFiles(plan(["src/server/Core.ts"], sources))).toEqual([
|
|
"tests/boundary/mcp/direct.test.ts",
|
|
"tests/boundary/mcp/helper.test.ts",
|
|
]);
|
|
// Other owners keep import tracing only.
|
|
expect(plan(["src/domain/Value.ts"], sources)).toMatchObject({
|
|
reason: "Affected tests: 1 imports",
|
|
});
|
|
});
|
|
|
|
it("lets checker-only inputs and prose select nothing by themselves", () => {
|
|
const sources = {
|
|
"src/domain/Value.test.ts": "",
|
|
"tests/boundary/filesystem/docs.test.ts": 'read("docs/guide.md");',
|
|
};
|
|
// A declaration, guard baseline or schema summary adds no tests…
|
|
expect(
|
|
selectedFiles(
|
|
plan(
|
|
[
|
|
"src/domain/Value.test.ts",
|
|
"scripts/lib/helper.d.mts",
|
|
"scripts/verify/baselines/some-guard.json",
|
|
"scripts/verify/advertised-schemas/tools/tool.json",
|
|
],
|
|
sources,
|
|
),
|
|
),
|
|
).toEqual(["src/domain/Value.test.ts"]);
|
|
// …and prose selects the tests that read it by path.
|
|
expect(selectedFiles(plan(["docs/guide.md"], sources))).toEqual([
|
|
"tests/boundary/filesystem/docs.test.ts",
|
|
]);
|
|
expect(plan(["scripts/verify/baselines/x.json"], sources)).toMatchObject({
|
|
mode: "full",
|
|
});
|
|
});
|
|
|
|
it("traces shared script modules to their tests and falls back for other consumers", () => {
|
|
const sources = {
|
|
"scripts/lib/shared.mjs": "",
|
|
"scripts/lib/wrapper.mjs": 'export * from "./shared.mjs";',
|
|
"scripts/verify-thing.mjs": 'import "./lib/wrapper.mjs";',
|
|
"scripts/lib/unused.mjs": "",
|
|
"tests/boundary/filesystem/shared.test.ts":
|
|
'import "../../../scripts/lib/shared.mjs";',
|
|
"tests/boundary/filesystem/verifier.test.ts":
|
|
'await import("../../../scripts/verify-thing.mjs");',
|
|
"tests/boundary/filesystem/other.test.ts": "",
|
|
"package.json":
|
|
'{"scripts":{"verify:thing":"node scripts/verify-thing.mjs"}}',
|
|
};
|
|
// Through another shared module and a verifier to the tests that import them.
|
|
expect(selectedFiles(plan(["scripts/lib/shared.mjs"], sources))).toEqual([
|
|
"tests/boundary/filesystem/shared.test.ts",
|
|
"tests/boundary/filesystem/verifier.test.ts",
|
|
]);
|
|
// A build script, configuration or package script that loads it is untraced.
|
|
expect(
|
|
plan(["scripts/lib/shared.mjs"], {
|
|
...sources,
|
|
"scripts/build.mjs": 'import { x } from "./lib/wrapper.mjs";',
|
|
}),
|
|
).toMatchObject({
|
|
mode: "full",
|
|
reason:
|
|
"Script consumer is not traced: scripts/build.mjs uses scripts/lib/wrapper.mjs",
|
|
});
|
|
expect(
|
|
plan(["scripts/lib/shared.mjs"], {
|
|
...sources,
|
|
"package.json": '{"scripts":{"x":"node scripts/lib/shared.mjs"}}',
|
|
}),
|
|
).toMatchObject({
|
|
mode: "full",
|
|
reason:
|
|
"Script consumer is not traced: package.json uses scripts/lib/shared.mjs",
|
|
});
|
|
// A module no test reaches runs everything, even beside a selected change.
|
|
for (const files of [
|
|
["scripts/lib/unused.mjs"],
|
|
["scripts/lib/unused.mjs", "tests/boundary/filesystem/other.test.ts"],
|
|
])
|
|
expect(plan(files, sources)).toMatchObject({
|
|
mode: "full",
|
|
reason: "No test reaches shared script module: scripts/lib/unused.mjs",
|
|
});
|
|
// Verifier source loads are module edges, and a test that starts a reached
|
|
// verifier by path is selected without importing it.
|
|
const loaded = {
|
|
"scripts/lib/startup.mts": "",
|
|
"scripts/lib/lifecycle.mjs":
|
|
'await importVerifierSource(new URL("./startup.mts", import.meta.url).href, import.meta.url);',
|
|
"scripts/verify-runtime.mjs": 'import "./lib/lifecycle.mjs";',
|
|
"tests/boundary/filesystem/startup.test.ts":
|
|
'import "../../../scripts/lib/startup.mts";',
|
|
"tests/boundary/filesystem/lifecycle.test.ts":
|
|
'import "../../../scripts/lib/lifecycle.mjs";',
|
|
"tests/boundary/filesystem/runtimeVerifier.test.ts":
|
|
'spawn(process.execPath, ["scripts/verify-runtime.mjs"]);',
|
|
};
|
|
expect(selectedFiles(plan(["scripts/lib/startup.mts"], loaded))).toEqual([
|
|
"tests/boundary/filesystem/lifecycle.test.ts",
|
|
"tests/boundary/filesystem/runtimeVerifier.test.ts",
|
|
"tests/boundary/filesystem/startup.test.ts",
|
|
]);
|
|
// Other scripts remain untraced inputs.
|
|
expect(plan(["scripts/build.mjs"], sources)).toMatchObject({
|
|
mode: "full",
|
|
reason: "Input is not traced to tests: scripts/build.mjs",
|
|
});
|
|
});
|
|
|
|
it("parses only well-formed plans and derives the selected shard count", () => {
|
|
const selected = {
|
|
version: 3,
|
|
mode: "selected",
|
|
reason: "Affected tests: 2 changed",
|
|
files: ["src/a.test.ts", "tests/b.test.ts"],
|
|
shards: 2,
|
|
};
|
|
expect(
|
|
testShards("fast", parseTestSelection(JSON.stringify(selected))),
|
|
).toEqual(["1/2", "2/2"]);
|
|
const full = { version: 3, mode: "full", reason: "Shared" };
|
|
expect(testShards("fast", parseTestSelection(full))).toHaveLength(12);
|
|
expect(testShards("full", parseTestSelection(full))).toHaveLength(6);
|
|
for (const value of [
|
|
"",
|
|
"null",
|
|
{ ...selected, version: 2 },
|
|
{ ...selected, mode: "candidate" },
|
|
{ ...selected, files: [] },
|
|
{ ...selected, files: ["src/a.ts"] },
|
|
{ ...selected, files: ["/abs/a.test.ts"] },
|
|
{ ...selected, files: ["src/a.test.ts", "src/a.test.ts"] },
|
|
// More shards than files, or than one selection uses.
|
|
{ ...selected, shards: 3 },
|
|
{ ...selected, shards: 0 },
|
|
{
|
|
...selected,
|
|
files: Array.from({ length: 5 }, (_, i) => `src/${i}.test.ts`),
|
|
shards: 5,
|
|
},
|
|
{ ...full, shards: 1 },
|
|
])
|
|
expect(() => parseTestSelection(value)).toThrow(
|
|
/Invalid portable test selection plan|JSON/u,
|
|
);
|
|
});
|
|
|
|
it("runs only a shard the plan defines", async () => {
|
|
const group = fileURLToPath(
|
|
new URL("../../scripts/ci/test-group.mjs", import.meta.url),
|
|
);
|
|
const selection = JSON.stringify({
|
|
version: 3,
|
|
mode: "selected",
|
|
reason: "Affected tests: 1 changed",
|
|
files: ["src/a.test.ts"],
|
|
shards: 1,
|
|
});
|
|
for (const [shard, environment] of [
|
|
["2/2", selection],
|
|
["1/2", selection],
|
|
["1/1", JSON.stringify({ version: 3, mode: "full", reason: "x" })],
|
|
["1/1", ""],
|
|
] as const)
|
|
await expect(
|
|
execute(process.execPath, [group, "selected", shard], {
|
|
env: { ...process.env, CI_TEST_SELECTION: environment },
|
|
}),
|
|
).rejects.toMatchObject({ code: 1 });
|
|
await expect(
|
|
execute(process.execPath, [group, "selected", "1/1", "--shard=1/2"], {
|
|
env: { ...process.env, CI_TEST_SELECTION: selection },
|
|
}),
|
|
).rejects.toMatchObject({
|
|
stderr: expect.stringContaining("already sharded"),
|
|
});
|
|
});
|