diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 66952cdd97..bacfcf7fc7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -267,14 +267,10 @@ jobs: - name: Run reverse tools tests with coverage run: npm run test:coverage:reverse-tools - name: Preflight both reverse tools mutation configurations - if: needs.changes.outputs.tier == 'fast' run: | node scripts/prepare-reverse-tools-mutation.mjs npx --no-install stryker run stryker.config.mjs --dryRunOnly npx --no-install stryker run stryker.semantics.config.mjs --dryRunOnly - - name: Report complete reverse tools mutation results - if: needs.changes.outputs.tier == 'full' - run: npm run test:mutation:reverse-tools - name: Retain available reverse tools diagnostics if: ${{ !cancelled() }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 diff --git a/.github/workflows/mutation.yml b/.github/workflows/mutation.yml new file mode 100644 index 0000000000..a16cd1e01a --- /dev/null +++ b/.github/workflows/mutation.yml @@ -0,0 +1,62 @@ +name: Mutation sweep + +on: + schedule: + - cron: "47 4 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: mutation-${{ github.ref }} + cancel-in-progress: false + +env: + HUSKY: "0" + +jobs: + mutation: + name: Mutation sweep (${{ matrix.suite }}) + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + fail-fast: false + max-parallel: 2 + matrix: + include: + - suite: reverse-tools + config: stryker.config.mjs + - suite: reverse-tools-semantics + config: stryker.semantics.config.mjs + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version-file: .nvmrc + - name: Cache npm package downloads + uses: ./.github/actions/npm-cache + - run: npm ci + - run: npm run test:prepare + - name: Run the complete configuration + env: + MUTATION_CONFIG: ${{ matrix.config }} + run: | + node scripts/prepare-reverse-tools-mutation.mjs + npx --no-install stryker run "${MUTATION_CONFIG}" + - name: Require both reports from the completed sweep + env: + REPORT_BASE: reports/mutation/${{ matrix.suite }} + run: | + test -s "${REPORT_BASE}.json" + test -s "${REPORT_BASE}.html" + - name: Retain available mutation diagnostics + if: ${{ !cancelled() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: mutation-${{ matrix.suite }}-${{ github.run_attempt }} + path: | + reports/mutation/${{ matrix.suite }}.json + reports/mutation/${{ matrix.suite }}.html + if-no-files-found: warn + retention-days: 7 diff --git a/docs/releasing.md b/docs/releasing.md index 5974232964..60fdb23cfd 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -10,6 +10,9 @@ SHA returned by Release Please. Before either registry is published, the release run calls full CI qualification at that exact revision, including coverage, portable providers, and every native package host. Later main commits cannot change or cancel that proof. +The required reverse-tools lane runs fresh coverage and both mutation +preflights. Complete mutation sweeps run separately on a schedule or manual +dispatch and do not block publication; see [the testing guide](testing.md). Ordinary main pushes and closed, unmerged PRs cannot publish packages. Check both publication jobs and the published-package canary; a merged PR or GitHub tag alone does not establish that publication finished. Retry failed jobs in diff --git a/docs/testing.md b/docs/testing.md index ca5fc5a6d4..d5a4cf4bc9 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -280,24 +280,32 @@ focused suite also exercises affected MCP routes and the Python Cutter bridge. Contract, integration, and real-provider lanes complement these focused tests; simulated providers do not establish real-provider behavior. -Fast CI retains fresh reverse-tools coverage and preflights both canonical +Required CI retains fresh reverse-tools coverage and preflights both canonical Stryker configurations with `--dryRunOnly`. These preflights resolve the owned targets, instrument the source, and execute the real initial tests. Invalid configuration, target resolution, and initial test failures still fail the job. -Preflights execute no mutants and produce no mutation-outcome report. Mutant -outcome feedback therefore arrives during full-tier qualification, which keeps -both complete passes and their configured targets, checkers and execution -limits. The standalone mutation command also retains both complete passes; -use `ci:full` to request full qualification before merging a PR. +Preflights execute no mutants and produce no mutation-outcome report. Both fast +feedback and full qualification, including exact-revision release qualification, +require coverage and both preflights. A failed, canceled, skipped or missing +selected quality job still fails the required gate. + +The separate **Mutation sweep** workflow runs both complete configurations daily +and on manual dispatch, in two isolated jobs. It is a maintenance check outside +the PR and release publication gates. `ci:full` requests full regression, +coverage and provider qualification; it does not request a complete mutation +sweep. Use the Mutation sweep workflow to request that result. Targets, +checkers, concurrency, deadlines and the existing non-blocking mutation-score +threshold are unchanged. `npm run test:mutation:reverse-tools` also retains both +complete passes for local callers. The two passes write separate JSON and HTML reports under `reports/mutation/`: -`reverse-tools` and `reverse-tools-semantics`. CI retains available coverage and -mutation diagnostics for seven days, including after failure, with tier and run -attempt in the artifact name. Canceled runs skip the upload. Fast artifacts -contain focused coverage; full artifacts additionally contain mutation reports -from completed passes. Partial diagnostics do not establish qualification, and -a failed, canceled, skipped or missing selected quality job fails the required -gate in either tier. +`reverse-tools` and `reverse-tools-semantics`. Required CI retains focused +coverage; the sweep retains each configuration's JSON/HTML reports separately. +Both retain available diagnostics for seven days after failure and skip uploads +on cancellation. Artifact names identify the tier or configuration and run +attempt. A successful sweep requires both jobs and their nonempty reports; +partial diagnostics and preflights do not establish complete mutation outcomes +or qualify the release revision for mutation coverage. Fast CI starts the source-only projects after classification, independently of build and static checks, and runs five built-dependent shards after the build. diff --git a/scripts/ci/plan.mjs b/scripts/ci/plan.mjs index 0d17d7bbd6..fdfc3518c5 100644 --- a/scripts/ci/plan.mjs +++ b/scripts/ci/plan.mjs @@ -16,6 +16,7 @@ const GUARDED_WORKFLOWS = new Set([ ".github/workflows/real-ghidra-windows.yml", ".github/workflows/sqlite-runtime.yml", ".github/workflows/ci-timings.yml", + ".github/workflows/mutation.yml", ]); // Sources compiled by build-conformance-fixtures.mjs. The readiness journey // analyzes the built C fixture as well as its own applications. diff --git a/scripts/ci/scopes.mjs b/scripts/ci/scopes.mjs index 01cb8e329d..7ea5063ce4 100644 --- a/scripts/ci/scopes.mjs +++ b/scripts/ci/scopes.mjs @@ -3,6 +3,7 @@ // every workflow. export const SCOPES = { reverse_tools: [ + ".github/workflows/mutation.yml", "src/application/reverse/**", "src/application/binary/BinarySession.ts", "src/cutter/**", @@ -23,6 +24,7 @@ export const SCOPES = { "tests/composition/analysis-sessions/binarySession.test.ts", "tests/boundary/filesystem/directAnalysisSnapshotReplay.test.ts", "tests/boundary/filesystem/reverseToolsMutationConfig.test.ts", + "tests/boundary/filesystem/mutationWorkflow.test.ts", "src/server/registerRizinDebugTools.ts", "tests/boundary/providers/CutterBridge*.test.ts", "tests/boundary/providers/cutterBridgeProducer.test.ts", diff --git a/tests/boundary/filesystem/ciChangeScope.test.ts b/tests/boundary/filesystem/ciChangeScope.test.ts index 4139583fb8..13625bc176 100644 --- a/tests/boundary/filesystem/ciChangeScope.test.ts +++ b/tests/boundary/filesystem/ciChangeScope.test.ts @@ -338,68 +338,6 @@ it.each(reverseToolsInputs)( }, ); -it("requires the reverse-tool gate when its scope is selected", async () => { - const workflow = parse( - await readFile( - new URL("../../../.github/workflows/ci.yml", import.meta.url), - "utf8", - ), - ); - expect(workflow.jobs["reverse-tools-quality"].if).toBe( - "needs.changes.outputs.reverse_tools == 'true'", - ); - expect(workflow.jobs.required.needs).toContain("reverse-tools-quality"); - const { CI_FLAGS } = await loadScopes(); - const outputs = { - ...Object.fromEntries(CI_FLAGS.map((flag) => [flag, "false"])), - tier: "fast", - reverse_tools: "true", - }; - const command = new URL("../../../scripts/ci/required.mjs", import.meta.url) - .pathname; - await expect( - execFileAsync(process.execPath, [command], { - env: { - ...process.env, - CI_JOBS: JSON.stringify({ - changes: { result: "success", outputs }, - static: { result: "success" }, - test: { result: "success" }, - "reverse-tools-quality": { result: "skipped" }, - }), - }, - }), - ).rejects.toMatchObject({ code: 1 }); - await expect( - execFileAsync(process.execPath, [command], { - env: { - ...process.env, - CI_JOBS: JSON.stringify({ - changes: { result: "success", outputs }, - static: { result: "success" }, - test: { result: "success" }, - "reverse-tools-quality": { result: "failure" }, - }), - }, - }), - ).rejects.toMatchObject({ code: 1 }); - await expect( - execFileAsync(process.execPath, [command], { - env: { - ...process.env, - CI_JOBS: JSON.stringify({ - changes: { result: "success", outputs }, - static: { result: "success" }, - test: { result: "success" }, - "reverse-tools-quality": { result: "success" }, - }), - }, - }), - ).resolves.toMatchObject({ - stdout: expect.stringContaining("Every selected CI lane passed"), - }); -}); - it.each([ "src/README.md", "README.md.ts", @@ -958,6 +896,10 @@ it("rejects skipped selected work while allowing unselected lanes to skip", asyn }); it.each([ + [ + ".github/workflows/mutation.yml", + { workflows: true, code: true, reverse_tools: true, package: false }, + ], [ ".github/workflows/sqlite-runtime.yml", { workflows: true, code: true, package: false, website: false }, diff --git a/tests/boundary/filesystem/ciFeedback.test.ts b/tests/boundary/filesystem/ciFeedback.test.ts index 8545a64b7a..e813b0d7c9 100644 --- a/tests/boundary/filesystem/ciFeedback.test.ts +++ b/tests/boundary/filesystem/ciFeedback.test.ts @@ -74,9 +74,13 @@ const evaluateCondition = ( ); it.each(["fast", "full"] as const)( - "runs actual mutation preflights only in fast feedback and requires quality in %s", + "requires coverage and both actual mutation preflights in %s qualification", async (tier) => { const jobs = await workflow(); + expect(jobs["reverse-tools-quality"]?.if).toBe( + "needs.changes.outputs.reverse_tools == 'true'", + ); + expect(jobs.required?.needs).toContain("reverse-tools-quality"); const steps = jobs["reverse-tools-quality"]?.steps ?? []; const commands = steps .filter( @@ -95,38 +99,15 @@ it.each(["fast", "full"] as const)( const mutationCommands = commands.filter( (command) => command.includes("mutation") || command.includes("stryker"), ); - expect(mutationCommands).toEqual( - tier === "fast" - ? [ - "node scripts/prepare-reverse-tools-mutation.mjs", - "npx --no-install stryker run stryker.config.mjs --dryRunOnly", - "npx --no-install stryker run stryker.semantics.config.mjs --dryRunOnly", - ] - : ["npm run test:mutation:reverse-tools"], - ); + expect(mutationCommands).toEqual([ + "node scripts/prepare-reverse-tools-mutation.mjs", + "npx --no-install stryker run stryker.config.mjs --dryRunOnly", + "npx --no-install stryker run stryker.semantics.config.mjs --dryRunOnly", + ]); const coverage = commands.indexOf("npm run test:coverage:reverse-tools"); expect(coverage).toBeGreaterThanOrEqual(0); expect(coverage).toBeLessThan(commands.indexOf(mutationCommands[0]!)); - const packageJson = z - .object({ scripts: z.record(z.string(), z.string()) }) - .parse( - JSON.parse( - await readFile( - new URL("../../../package.json", import.meta.url), - "utf8", - ), - ), - ); - const fullCommand = z - .string() - .parse(packageJson.scripts["test:mutation:reverse-tools"]); - expect(fullCommand.split("&&").map((part) => part.trim())).toEqual([ - "node scripts/prepare-reverse-tools-mutation.mjs", - "stryker run stryker.config.mjs", - "stryker run stryker.semantics.config.mjs", - ]); - const flags = await scopeFlags(); const context: Record< string, diff --git a/tests/boundary/filesystem/mutationWorkflow.test.ts b/tests/boundary/filesystem/mutationWorkflow.test.ts new file mode 100644 index 0000000000..4369b67c65 --- /dev/null +++ b/tests/boundary/filesystem/mutationWorkflow.test.ts @@ -0,0 +1,145 @@ +import { execFile } from "node:child_process"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { expect, it } from "vitest"; +import { parse } from "yaml"; +import { z } from "zod"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const execute = promisify(execFile); +const stepSchema = z + .object({ + name: z.string().optional(), + run: z.string().optional(), + if: z.string().optional(), + uses: z.string().optional(), + env: z.record(z.string(), z.string()).optional(), + with: z.record(z.string(), z.unknown()).optional(), + }) + .passthrough(); +const readWorkflow = async () => + z + .object({ + on: z.record(z.string(), z.unknown()), + permissions: z.object({ contents: z.literal("read") }).strict(), + concurrency: z.object({ "cancel-in-progress": z.literal(false) }), + jobs: z.object({ + mutation: z + .object({ + strategy: z.object({ + "fail-fast": z.literal(false), + "max-parallel": z.literal(2), + matrix: z.object({ + include: z.array( + z.object({ suite: z.string(), config: z.string() }), + ), + }), + }), + steps: z.array(stepSchema), + }) + .passthrough(), + }), + }) + .parse( + parse( + await readFile( + new URL("../../../.github/workflows/mutation.yml", import.meta.url), + "utf8", + ), + ), + ); + +it("runs both complete canonical sweeps independently of PR and release gates", async () => { + const workflow = await readWorkflow(); + expect(Object.keys(workflow.on).sort()).toEqual([ + "schedule", + "workflow_dispatch", + ]); + const job = workflow.jobs.mutation; + expect(job["continue-on-error"]).toBeUndefined(); + expect(job.if).toBeUndefined(); + expect(job.strategy.matrix.include).toEqual([ + { suite: "reverse-tools", config: "stryker.config.mjs" }, + { + suite: "reverse-tools-semantics", + config: "stryker.semantics.config.mjs", + }, + ]); + const sweep = job.steps.find( + (step) => step.env?.MUTATION_CONFIG !== undefined, + ); + expect(sweep?.env?.MUTATION_CONFIG).toBe("${{ matrix.config }}"); + expect(sweep?.run?.trim().split("\n")).toEqual([ + "node scripts/prepare-reverse-tools-mutation.mjs", + 'npx --no-install stryker run "${MUTATION_CONFIG}"', + ]); + expect(job.steps.some((step) => step.run === "npm run test:prepare")).toBe( + true, + ); + const upload = job.steps.find((step) => + step.uses?.startsWith("actions/upload-artifact@"), + ); + expect(upload?.if).toBe("${{ !cancelled() }}"); + expect(upload?.with?.name).toBe( + "mutation-${{ matrix.suite }}-${{ github.run_attempt }}", + ); + expect(upload?.with?.["retention-days"]).toBe(7); + const paths = z.string().parse(upload?.with?.path).trim().split("\n"); + for (const { suite, config } of job.strategy.matrix.include) { + const canonical = z + .object({ + default: z.object({ + jsonReporter: z.object({ fileName: z.string() }), + htmlReporter: z.object({ fileName: z.string() }), + }), + }) + .parse( + await import(new URL(`../../../${config}`, import.meta.url).href), + ).default; + expect( + paths.map((path) => path.replace("${{ matrix.suite }}", suite)), + ).toEqual([ + canonical.jsonReporter.fileName, + canonical.htmlReporter.fileName, + ]); + } + for (const step of job.steps) + expect(step["continue-on-error"]).toBeUndefined(); +}); + +it("fails a successful sweep when either required report is absent or empty", async () => { + const steps = (await readWorkflow()).jobs.mutation.steps; + const guard = steps.find((step) => step.env?.REPORT_BASE !== undefined); + expect(guard?.if).toBeUndefined(); + const command = z.string().parse(guard?.run); + const guardIndex = steps.indexOf(guard!); + expect(guardIndex).toBeGreaterThan( + steps.findIndex((step) => step.env?.MUTATION_CONFIG), + ); + expect(guardIndex).toBeLessThan( + steps.findIndex((step) => + step.uses?.startsWith("actions/upload-artifact@"), + ), + ); + for (const state of ["missing", "empty-json", "empty-html", "complete"]) { + const directory = await createTestTempDirectory("rea-mutation-reports-"); + const base = join(directory, "reports", "reverse-tools"); + await mkdir(join(directory, "reports")); + if (state !== "missing") { + await writeFile(`${base}.json`, state === "empty-json" ? "" : "{}"); + await writeFile( + `${base}.html`, + state === "empty-html" ? "" : "", + ); + } + const result = execute("bash", ["-eu", "-c", command], { + cwd: directory, + env: { ...process.env, REPORT_BASE: base }, + }); + if (state === "complete") + await expect(result).resolves.toMatchObject({ stderr: "" }); + else await expect(result).rejects.toMatchObject({ code: 1 }); + } +});