diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f55667dce7..66952cdd97 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -250,7 +250,7 @@ jobs: retention-days: 7 reverse-tools-quality: - name: Reverse tools coverage and mutation + name: Reverse tools quality needs: [changes, static] if: needs.changes.outputs.reverse_tools == 'true' runs-on: ubuntu-latest @@ -266,8 +266,25 @@ jobs: - run: npm ci - name: Run reverse tools tests with coverage run: npm run test:coverage:reverse-tools - - name: Report reverse tools mutation results + - name: Preflight both reverse tools mutation configurations + if: needs.changes.outputs.tier == 'fast' + run: | + node scripts/prepare-reverse-tools-mutation.mjs + npx --no-install stryker run stryker.config.mjs --dryRunOnly + npx --no-install stryker run stryker.semantics.config.mjs --dryRunOnly + - name: Report complete reverse tools mutation results + if: needs.changes.outputs.tier == 'full' run: npm run test:mutation:reverse-tools + - name: Retain available reverse tools diagnostics + if: ${{ !cancelled() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: reverse-tools-quality-${{ needs.changes.outputs.tier }}-${{ github.run_attempt }} + path: | + coverage/reverse-tools/ + reports/mutation/ + if-no-files-found: warn + retention-days: 7 build: name: Build package diff --git a/docs/testing.md b/docs/testing.md index 042248a26f..ca5fc5a6d4 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -280,6 +280,25 @@ focused suite also exercises affected MCP routes and the Python Cutter bridge. Contract, integration, and real-provider lanes complement these focused tests; simulated providers do not establish real-provider behavior. +Fast CI retains fresh reverse-tools coverage and preflights both canonical +Stryker configurations with `--dryRunOnly`. These preflights resolve the owned +targets, instrument the source, and execute the real initial tests. Invalid +configuration, target resolution, and initial test failures still fail the job. +Preflights execute no mutants and produce no mutation-outcome report. Mutant +outcome feedback therefore arrives during full-tier qualification, which keeps +both complete passes and their configured targets, checkers and execution +limits. The standalone mutation command also retains both complete passes; +use `ci:full` to request full qualification before merging a PR. + +The two passes write separate JSON and HTML reports under `reports/mutation/`: +`reverse-tools` and `reverse-tools-semantics`. CI retains available coverage and +mutation diagnostics for seven days, including after failure, with tier and run +attempt in the artifact name. Canceled runs skip the upload. Fast artifacts +contain focused coverage; full artifacts additionally contain mutation reports +from completed passes. Partial diagnostics do not establish qualification, and +a failed, canceled, skipped or missing selected quality job fails the required +gate in either tier. + Fast CI starts the source-only projects after classification, independently of build and static checks, and runs five built-dependent shards after the build. The six test jobs still select every portable test. Full qualification keeps six diff --git a/stryker.config.mjs b/stryker.config.mjs index 30a04e5236..0fe7f37455 100644 --- a/stryker.config.mjs +++ b/stryker.config.mjs @@ -22,6 +22,8 @@ const config = { concurrency: 2, timeoutMS: 30000, reporters: ["clear-text", "progress", "html", "json"], + jsonReporter: { fileName: "reports/mutation/reverse-tools.json" }, + htmlReporter: { fileName: "reports/mutation/reverse-tools.html" }, }; export default { diff --git a/stryker.semantics.config.mjs b/stryker.semantics.config.mjs index 84d6d649a0..81c58dc40e 100644 --- a/stryker.semantics.config.mjs +++ b/stryker.semantics.config.mjs @@ -18,6 +18,8 @@ const config = { concurrency: 2, timeoutMS: 30000, reporters: ["clear-text", "progress", "html", "json"], + jsonReporter: { fileName: "reports/mutation/reverse-tools-semantics.json" }, + htmlReporter: { fileName: "reports/mutation/reverse-tools-semantics.html" }, }; export default { diff --git a/tests/boundary/filesystem/ciFeedback.test.ts b/tests/boundary/filesystem/ciFeedback.test.ts index c61cb0d92b..8545a64b7a 100644 --- a/tests/boundary/filesystem/ciFeedback.test.ts +++ b/tests/boundary/filesystem/ciFeedback.test.ts @@ -38,6 +38,9 @@ const workflow = async () => z .object({ run: z.string().optional(), + if: z.string().optional(), + uses: z.string().optional(), + with: z.record(z.string(), z.unknown()).optional(), env: z.record(z.string(), z.string()).optional(), }) .passthrough(), @@ -56,6 +59,162 @@ const workflow = async () => ), ).jobs; +const evaluateCondition = ( + condition: string, + context: Record, +): boolean => + z + .boolean() + .parse( + runInNewContext( + condition.startsWith("${{") ? condition.slice(3, -2) : condition, + context, + { timeout: 1_000 }, + ), + ); + +it.each(["fast", "full"] as const)( + "runs actual mutation preflights only in fast feedback and requires quality in %s", + async (tier) => { + const jobs = await workflow(); + const steps = jobs["reverse-tools-quality"]?.steps ?? []; + const commands = steps + .filter( + (step) => + step.if === undefined || + evaluateCondition(step.if, { + needs: { changes: { outputs: { tier } } }, + cancelled: () => false, + success: () => true, + failure: () => false, + always: () => true, + }), + ) + .flatMap((step) => step.run?.trim().split("\n") ?? []) + .map((command) => command.trim()); + const mutationCommands = commands.filter( + (command) => command.includes("mutation") || command.includes("stryker"), + ); + expect(mutationCommands).toEqual( + tier === "fast" + ? [ + "node scripts/prepare-reverse-tools-mutation.mjs", + "npx --no-install stryker run stryker.config.mjs --dryRunOnly", + "npx --no-install stryker run stryker.semantics.config.mjs --dryRunOnly", + ] + : ["npm run test:mutation:reverse-tools"], + ); + const coverage = commands.indexOf("npm run test:coverage:reverse-tools"); + expect(coverage).toBeGreaterThanOrEqual(0); + expect(coverage).toBeLessThan(commands.indexOf(mutationCommands[0]!)); + + const packageJson = z + .object({ scripts: z.record(z.string(), z.string()) }) + .parse( + JSON.parse( + await readFile( + new URL("../../../package.json", import.meta.url), + "utf8", + ), + ), + ); + const fullCommand = z + .string() + .parse(packageJson.scripts["test:mutation:reverse-tools"]); + expect(fullCommand.split("&&").map((part) => part.trim())).toEqual([ + "node scripts/prepare-reverse-tools-mutation.mjs", + "stryker run stryker.config.mjs", + "stryker run stryker.semantics.config.mjs", + ]); + + const flags = await scopeFlags(); + const context: Record< + string, + { result: string; outputs?: Record } + > = Object.fromEntries( + Object.keys(jobs).map((name) => [name, { result: "success" }]), + ); + context.changes = { + result: "success", + outputs: { + ...Object.fromEntries( + flags.map((flag) => [flag, tier === "full" ? "true" : "false"]), + ), + tier, + code: "true", + reverse_tools: "true", + }, + }; + for (const result of [ + "success", + "failure", + "cancelled", + "skipped", + "missing", + ]) { + const needs = { ...context }; + if (result === "missing") delete needs["reverse-tools-quality"]; + else needs["reverse-tools-quality"] = { result }; + const execution = execute( + process.execPath, + [ + fileURLToPath( + new URL("../../../scripts/ci/required.mjs", import.meta.url), + ), + ], + { + env: { + ...process.env, + CI_JOBS: JSON.stringify(needs), + CI_QUALIFICATION_DIRECTORY: "", + }, + }, + ); + if (result === "success") + await expect(execution).resolves.toMatchObject({ stderr: "" }); + else + await expect(execution).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining("reverse-tools-quality"), + }); + } + }, +); + +it("retains available quality diagnostics after failure without keeping canceled uploads alive", async () => { + const steps = (await workflow())["reverse-tools-quality"]?.steps ?? []; + const upload = steps.find((step) => + step.uses?.startsWith("actions/upload-artifact@"), + ); + const condition = z.string().parse(upload?.if); + for (const cancelled of [false, true]) + expect( + evaluateCondition(condition, { + cancelled: () => cancelled, + success: () => false, + failure: () => true, + always: () => true, + }), + ).toBe(!cancelled); + const options = upload?.with ?? {}; + expect(z.string().parse(options.name)).toContain( + "${{ needs.changes.outputs.tier }}", + ); + expect(z.string().parse(options.name)).toContain("${{ github.run_attempt }}"); + expect( + z + .string() + .parse(options.path) + .trim() + .split("\n") + .map((path) => path.trim()), + ).toEqual( + expect.arrayContaining(["coverage/reverse-tools/", "reports/mutation/"]), + ); + expect(options["retention-days"]).toBe(7); + expect(options["if-no-files-found"]).toBe("warn"); +}); + it("starts source feedback before build and requires both groups without waiving missing jobs", async () => { const jobs = await workflow(); expect( diff --git a/tests/boundary/filesystem/reverseToolsMutationConfig.test.ts b/tests/boundary/filesystem/reverseToolsMutationConfig.test.ts index d6941c06cb..113fb910d1 100644 --- a/tests/boundary/filesystem/reverseToolsMutationConfig.test.ts +++ b/tests/boundary/filesystem/reverseToolsMutationConfig.test.ts @@ -1,9 +1,10 @@ import { execFileSync } from "node:child_process"; import { writeFile } from "node:fs/promises"; -import { join } from "node:path"; +import { join, resolve } from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { describe, expect, it } from "vitest"; +import { z } from "zod"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; @@ -16,24 +17,45 @@ const runNode = (source: string, ...arguments_: string[]): string => ); describe("reverse-tools mutation configuration", () => { - it("loads nonempty targets from both actual Stryker configs", () => { + it("loads nonempty targets and distinct report destinations from both actual Stryker configs", () => { const result = runNode( `const [mainUrl, semanticsUrl] = process.argv.slice(1); const main = await import(mainUrl); const semantics = await import(semanticsUrl); - process.stdout.write(JSON.stringify({ - main: main.default.mutate.length, - semantics: semantics.default.mutate.length, - }));`, + process.stdout.write(JSON.stringify([main.default, semantics.default].map(config => ({ + targets: config.mutate.length, + json: config.jsonReporter?.fileName, + html: config.htmlReporter?.fileName, + dryRunOnly: config.dryRunOnly ?? false, + reporters: config.reporters, + }))));`, pathToFileURL(join(root, "stryker.config.mjs")).href, pathToFileURL(join(root, "stryker.semantics.config.mjs")).href, ); - const targets = JSON.parse(result) as { - main: number; - semantics: number; - }; - expect(targets.main).toBeGreaterThan(0); - expect(targets.semantics).toBeGreaterThan(0); + const configs = z + .array( + z.object({ + targets: z.number().int().positive(), + json: z.string().regex(/^reports\/mutation\/.+\.json$/u), + html: z.string().regex(/^reports\/mutation\/.+\.html$/u), + dryRunOnly: z.literal(false), + reporters: z.array(z.string()), + }), + ) + .length(2) + .parse(JSON.parse(result)); + expect( + new Set( + configs.flatMap(({ json, html }) => [ + resolve(root, json), + resolve(root, html), + ]), + ).size, + ).toBe(4); + for (const config of configs) + expect(config.reporters).toEqual( + expect.arrayContaining(["json", "html"]), + ); }); it("fails closed when an owned mutation method is missing", async () => {