perf(ci): preflight mutation configurations during fast feedback

Keep full qualification and standalone mutation execution intact, retain distinct reports, and preserve fail-closed quality gates.

Related: #3281

Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com>

.github/workflows/ci.yml

@@ -250,7 +250,7 @@ jobs:
           retention-days: 7
 
   reverse-tools-quality:
-    name: Reverse tools coverage and mutation
+    name: Reverse tools quality
     needs: [changes, static]
     if: needs.changes.outputs.reverse_tools == 'true'
     runs-on: ubuntu-latest
@@ -266,8 +266,25 @@ jobs:
       - run: npm ci
       - name: Run reverse tools tests with coverage
         run: npm run test:coverage:reverse-tools
-      - name: Report reverse tools mutation results
+      - name: Preflight both reverse tools mutation configurations
+        if: needs.changes.outputs.tier == 'fast'
+        run: |
+          node scripts/prepare-reverse-tools-mutation.mjs
+          npx --no-install stryker run stryker.config.mjs --dryRunOnly
+          npx --no-install stryker run stryker.semantics.config.mjs --dryRunOnly
+      - name: Report complete reverse tools mutation results
+        if: needs.changes.outputs.tier == 'full'
         run: npm run test:mutation:reverse-tools
+      - name: Retain available reverse tools diagnostics
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: reverse-tools-quality-${{ needs.changes.outputs.tier }}-${{ github.run_attempt }}
+          path: |
+            coverage/reverse-tools/
+            reports/mutation/
+          if-no-files-found: warn
+          retention-days: 7
 
   build:
     name: Build package

docs/testing.md

@@ -280,6 +280,25 @@ focused suite also exercises affected MCP routes and the Python Cutter bridge.
 Contract, integration, and real-provider lanes complement these focused tests;
 simulated providers do not establish real-provider behavior.
 
+Fast CI retains fresh reverse-tools coverage and preflights both canonical
+Stryker configurations with `--dryRunOnly`. These preflights resolve the owned
+targets, instrument the source, and execute the real initial tests. Invalid
+configuration, target resolution, and initial test failures still fail the job.
+Preflights execute no mutants and produce no mutation-outcome report. Mutant
+outcome feedback therefore arrives during full-tier qualification, which keeps
+both complete passes and their configured targets, checkers and execution
+limits. The standalone mutation command also retains both complete passes;
+use `ci:full` to request full qualification before merging a PR.
+
+The two passes write separate JSON and HTML reports under `reports/mutation/`:
+`reverse-tools` and `reverse-tools-semantics`. CI retains available coverage and
+mutation diagnostics for seven days, including after failure, with tier and run
+attempt in the artifact name. Canceled runs skip the upload. Fast artifacts
+contain focused coverage; full artifacts additionally contain mutation reports
+from completed passes. Partial diagnostics do not establish qualification, and
+a failed, canceled, skipped or missing selected quality job fails the required
+gate in either tier.
+
 Fast CI starts the source-only projects after classification, independently of
 build and static checks, and runs five built-dependent shards after the build.
 The six test jobs still select every portable test. Full qualification keeps six

stryker.config.mjs

@@ -22,6 +22,8 @@ const config = {
   concurrency: 2,
   timeoutMS: 30000,
   reporters: ["clear-text", "progress", "html", "json"],
+  jsonReporter: { fileName: "reports/mutation/reverse-tools.json" },
+  htmlReporter: { fileName: "reports/mutation/reverse-tools.html" },
 };
 
 export default {

stryker.semantics.config.mjs

@@ -18,6 +18,8 @@ const config = {
   concurrency: 2,
   timeoutMS: 30000,
   reporters: ["clear-text", "progress", "html", "json"],
+  jsonReporter: { fileName: "reports/mutation/reverse-tools-semantics.json" },
+  htmlReporter: { fileName: "reports/mutation/reverse-tools-semantics.html" },
 };
 
 export default {

tests/boundary/filesystem/ciFeedback.test.ts

@@ -38,6 +38,9 @@ const workflow = async () =>
                 z
                   .object({
                     run: z.string().optional(),
+                    if: z.string().optional(),
+                    uses: z.string().optional(),
+                    with: z.record(z.string(), z.unknown()).optional(),
                     env: z.record(z.string(), z.string()).optional(),
                   })
                   .passthrough(),
@@ -56,6 +59,162 @@ const workflow = async () =>
       ),
     ).jobs;
 
+const evaluateCondition = (
+  condition: string,
+  context: Record<string, unknown>,
+): boolean =>
+  z
+    .boolean()
+    .parse(
+      runInNewContext(
+        condition.startsWith("${{") ? condition.slice(3, -2) : condition,
+        context,
+        { timeout: 1_000 },
+      ),
+    );
+
+it.each(["fast", "full"] as const)(
+  "runs actual mutation preflights only in fast feedback and requires quality in %s",
+  async (tier) => {
+    const jobs = await workflow();
+    const steps = jobs["reverse-tools-quality"]?.steps ?? [];
+    const commands = steps
+      .filter(
+        (step) =>
+          step.if === undefined ||
+          evaluateCondition(step.if, {
+            needs: { changes: { outputs: { tier } } },
+            cancelled: () => false,
+            success: () => true,
+            failure: () => false,
+            always: () => true,
+          }),
+      )
+      .flatMap((step) => step.run?.trim().split("\n") ?? [])
+      .map((command) => command.trim());
+    const mutationCommands = commands.filter(
+      (command) => command.includes("mutation") || command.includes("stryker"),
+    );
+    expect(mutationCommands).toEqual(
+      tier === "fast"
+        ? [
+            "node scripts/prepare-reverse-tools-mutation.mjs",
+            "npx --no-install stryker run stryker.config.mjs --dryRunOnly",
+            "npx --no-install stryker run stryker.semantics.config.mjs --dryRunOnly",
+          ]
+        : ["npm run test:mutation:reverse-tools"],
+    );
+    const coverage = commands.indexOf("npm run test:coverage:reverse-tools");
+    expect(coverage).toBeGreaterThanOrEqual(0);
+    expect(coverage).toBeLessThan(commands.indexOf(mutationCommands[0]!));
+
+    const packageJson = z
+      .object({ scripts: z.record(z.string(), z.string()) })
+      .parse(
+        JSON.parse(
+          await readFile(
+            new URL("../../../package.json", import.meta.url),
+            "utf8",
+          ),
+        ),
+      );
+    const fullCommand = z
+      .string()
+      .parse(packageJson.scripts["test:mutation:reverse-tools"]);
+    expect(fullCommand.split("&&").map((part) => part.trim())).toEqual([
+      "node scripts/prepare-reverse-tools-mutation.mjs",
+      "stryker run stryker.config.mjs",
+      "stryker run stryker.semantics.config.mjs",
+    ]);
+
+    const flags = await scopeFlags();
+    const context: Record<
+      string,
+      { result: string; outputs?: Record<string, string> }
+    > = Object.fromEntries(
+      Object.keys(jobs).map((name) => [name, { result: "success" }]),
+    );
+    context.changes = {
+      result: "success",
+      outputs: {
+        ...Object.fromEntries(
+          flags.map((flag) => [flag, tier === "full" ? "true" : "false"]),
+        ),
+        tier,
+        code: "true",
+        reverse_tools: "true",
+      },
+    };
+    for (const result of [
+      "success",
+      "failure",
+      "cancelled",
+      "skipped",
+      "missing",
+    ]) {
+      const needs = { ...context };
+      if (result === "missing") delete needs["reverse-tools-quality"];
+      else needs["reverse-tools-quality"] = { result };
+      const execution = execute(
+        process.execPath,
+        [
+          fileURLToPath(
+            new URL("../../../scripts/ci/required.mjs", import.meta.url),
+          ),
+        ],
+        {
+          env: {
+            ...process.env,
+            CI_JOBS: JSON.stringify(needs),
+            CI_QUALIFICATION_DIRECTORY: "",
+          },
+        },
+      );
+      if (result === "success")
+        await expect(execution).resolves.toMatchObject({ stderr: "" });
+      else
+        await expect(execution).rejects.toMatchObject({
+          code: 1,
+          stderr: expect.stringContaining("reverse-tools-quality"),
+        });
+    }
+  },
+);
+
+it("retains available quality diagnostics after failure without keeping canceled uploads alive", async () => {
+  const steps = (await workflow())["reverse-tools-quality"]?.steps ?? [];
+  const upload = steps.find((step) =>
+    step.uses?.startsWith("actions/upload-artifact@"),
+  );
+  const condition = z.string().parse(upload?.if);
+  for (const cancelled of [false, true])
+    expect(
+      evaluateCondition(condition, {
+        cancelled: () => cancelled,
+        success: () => false,
+        failure: () => true,
+        always: () => true,
+      }),
+    ).toBe(!cancelled);
+  const options = upload?.with ?? {};
+  expect(z.string().parse(options.name)).toContain(
+    "${{ needs.changes.outputs.tier }}",
+  );
+  expect(z.string().parse(options.name)).toContain("${{ github.run_attempt }}");
+  expect(
+    z
+      .string()
+      .parse(options.path)
+      .trim()
+      .split("\n")
+      .map((path) => path.trim()),
+  ).toEqual(
+    expect.arrayContaining(["coverage/reverse-tools/", "reports/mutation/"]),
+  );
+  expect(options["retention-days"]).toBe(7);
+  expect(options["if-no-files-found"]).toBe("warn");
+});
+
 it("starts source feedback before build and requires both groups without waiving missing jobs", async () => {
   const jobs = await workflow();
   expect(

tests/boundary/filesystem/reverseToolsMutationConfig.test.ts

@@ -1,9 +1,10 @@
 import { execFileSync } from "node:child_process";
 import { writeFile } from "node:fs/promises";
-import { join } from "node:path";
+import { join, resolve } from "node:path";
 import { fileURLToPath, pathToFileURL } from "node:url";
 
 import { describe, expect, it } from "vitest";
+import { z } from "zod";
 
 import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
 
@@ -16,24 +17,45 @@ const runNode = (source: string, ...arguments_: string[]): string =>
   );
 
 describe("reverse-tools mutation configuration", () => {
-  it("loads nonempty targets from both actual Stryker configs", () => {
+  it("loads nonempty targets and distinct report destinations from both actual Stryker configs", () => {
     const result = runNode(
       `const [mainUrl, semanticsUrl] = process.argv.slice(1);
        const main = await import(mainUrl);
        const semantics = await import(semanticsUrl);
-       process.stdout.write(JSON.stringify({
-         main: main.default.mutate.length,
-         semantics: semantics.default.mutate.length,
-       }));`,
+       process.stdout.write(JSON.stringify([main.default, semantics.default].map(config => ({
+         targets: config.mutate.length,
+         json: config.jsonReporter?.fileName,
+         html: config.htmlReporter?.fileName,
+         dryRunOnly: config.dryRunOnly ?? false,
+         reporters: config.reporters,
+       }))));`,
       pathToFileURL(join(root, "stryker.config.mjs")).href,
       pathToFileURL(join(root, "stryker.semantics.config.mjs")).href,
     );
-    const targets = JSON.parse(result) as {
-      main: number;
-      semantics: number;
-    };
-    expect(targets.main).toBeGreaterThan(0);
-    expect(targets.semantics).toBeGreaterThan(0);
+    const configs = z
+      .array(
+        z.object({
+          targets: z.number().int().positive(),
+          json: z.string().regex(/^reports\/mutation\/.+\.json$/u),
+          html: z.string().regex(/^reports\/mutation\/.+\.html$/u),
+          dryRunOnly: z.literal(false),
+          reporters: z.array(z.string()),
+        }),
+      )
+      .length(2)
+      .parse(JSON.parse(result));
+    expect(
+      new Set(
+        configs.flatMap(({ json, html }) => [
+          resolve(root, json),
+          resolve(root, html),
+        ]),
+      ).size,
+    ).toBe(4);
+    for (const config of configs)
+      expect(config.reporters).toEqual(
+        expect.arrayContaining(["json", "html"]),
+      );
   });
 
   it("fails closed when an owned mutation method is missing", async () => {