import { chmod, mkdir, readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { expect, it } from "vitest"; import { parse } from "yaml"; import { z } from "zod"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { executeWorkflowFixture, workflowGit, } from "../../support/workflowGit.js"; const qualificationScript = fileURLToPath( new URL("../../../scripts/ci/qualification.mjs", import.meta.url), ); const requiredScript = fileURLToPath( new URL("../../../scripts/ci/required.mjs", import.meta.url), ); const now = Date.now(); const validSha = "a".repeat(40); const repository = "example/rea"; function artifact(overrides: Record = {}) { return { id: 42, name: "ci-full-qualification-v1", expired: false, created_at: new Date(now - 60_000).toISOString(), workflow_run: { id: 7, repository_id: 1, head_repository_id: 1, head_branch: "main", head_sha: validSha, }, ...overrides, }; } function run(overrides: Record = {}) { return { id: 7, status: "completed", conclusion: "success", path: ".github/workflows/ci.yml@main", event: "schedule", created_at: new Date(now - 120_000).toISOString(), head_sha: validSha, head_branch: "main", head_repository: { full_name: repository }, ...overrides, }; } async function admissionFixture( candidate: Record = artifact(), candidateRun: Record = run(), options: { apiError?: boolean; malformed?: boolean; force?: boolean } = {}, ) { const directory = await createTestTempDirectory("rea-ci-qualification-"); const bin = join(directory, "bin"); await mkdir(bin); const fixturePath = join(directory, "api.json"); await writeFile( fixturePath, options.malformed ? "not-json" : JSON.stringify({ artifacts: [candidate], run: candidateRun }), ); const gh = join(bin, "gh"); await writeFile( gh, [ "#!/bin/sh", 'if [ "$GH_API_ERROR" = "true" ]; then exit 1; fi', 'case "$*" in', ' *actions/artifacts*) cat "$GH_FIXTURE_RESPONSE" ;;', " *actions/runs/7*) node -e 'const x=require(process.env.GH_FIXTURE_RESPONSE); process.stdout.write(JSON.stringify(x.run))' ;;", " *) exit 2 ;;", "esac", "", ].join("\n"), ); await chmod(gh, 0o755); const output = join(directory, "github-output"); await writeFile(output, ""); const env = { ...process.env, PATH: `${bin}:${process.env.PATH ?? ""}`, GH_FIXTURE_RESPONSE: fixturePath, GH_API_ERROR: String(options.apiError ?? false), CI_REPOSITORY: repository, CI_REPOSITORY_ID: "1", CI_DEFAULT_BRANCH: "main", HEAD_SHA: validSha, GITHUB_RUN_ID: "100", GITHUB_OUTPUT: output, ...(options.force ? { CI_FORCE_FULL: "true" } : {}), }; return { directory, output, env }; } async function decide( candidate?: Record, candidateRun?: Record, options?: { apiError?: boolean; malformed?: boolean; force?: boolean }, ) { const fixture = await admissionFixture(candidate, candidateRun, options); const result = await executeWorkflowFixture( process.execPath, [qualificationScript], { cwd: fixture.directory, env: fixture.env, }, ); return { ...fixture, decision: z .record(z.string(), z.unknown()) .parse(JSON.parse(result.stdout)), outputText: await readFile(fixture.output, "utf8"), }; } it("reuses only a recent trusted full qualification for the exact default-branch SHA", async () => { const { decision, outputText } = await decide(); expect(decision).toMatchObject({ run_full: false, run_id: 7 }); expect(Object.keys(decision).sort()).toEqual([ "reason", "run_full", "run_id", ]); expect(outputText).toContain("run_full=false\n"); expect(outputText).toContain("run_id=7\n"); }); const rejectedCandidates: Array< [string, () => Record, () => Record] > = [ [ "old", () => artifact({ created_at: new Date(now - 24 * 60 * 60 * 1000 - 1).toISOString(), }), run, ], [ "future", () => artifact({ created_at: new Date(now + 60_000).toISOString() }), run, ], ["expired", () => artifact({ expired: true }), run], [ "foreign repository", () => artifact({ workflow_run: { ...artifact().workflow_run, repository_id: 2 }, }), run, ], [ "foreign branch", () => artifact({ workflow_run: { ...artifact().workflow_run, head_branch: "other" }, }), run, ], [ "foreign SHA", () => artifact({ workflow_run: { ...artifact().workflow_run, head_sha: "b".repeat(40) }, }), run, ], [ "untrusted workflow", artifact, () => run({ path: ".github/workflows/release.yml" }), ], ["failed run", artifact, () => run({ conclusion: "failure" })], [ "old producer rerun with a fresh artifact", artifact, () => run({ created_at: new Date(now - 25 * 60 * 60 * 1000).toISOString() }), ], ["wrong workflow event", artifact, () => run({ event: "push" })], ]; it.each(rejectedCandidates)( "runs full qualification for a %s candidate", async (_name, makeArtifact, makeRun) => { const { decision } = await decide(makeArtifact(), makeRun()); expect(decision.run_full).toBe(true); }, ); it.each([ ["API error", { apiError: true }], ["malformed query response", { malformed: true }], ] as const)("fails closed on %s", async (_name, options) => { const { decision } = await decide(undefined, undefined, options); expect(decision.run_full).toBe(true); }); it("lets the daily force-full setting bypass the GitHub API", async () => { const { decision } = await decide(undefined, undefined, { force: true, apiError: true, }); expect(decision).toMatchObject({ run_full: true }); expect(decision.reason).toContain("Forced"); }); async function workflowContract() { const workflowPath = new URL( "../../../.github/workflows/ci.yml", import.meta.url, ); const parsed = parse(await readFile(workflowPath, "utf8")); return z .object({ jobs: z.record( z.string(), z.object({ needs: z.unknown().optional() }).passthrough(), ), }) .parse(parsed).jobs; } async function requiredRun( changes: Record, overrides: { failed?: string; runAttempt?: string; event?: string } = {}, ) { const directory = await createTestTempDirectory("rea-ci-receipt-"); await workflowGit(directory, ["init", "--initial-branch=main"]); await writeFile(join(directory, "fixture.txt"), "qualified source\n"); await workflowGit(directory, ["add", "."]); await workflowGit(directory, ["commit", "-m", "fixture source"]); const head = await workflowGit(directory, ["rev-parse", "HEAD"]); const jobs = await workflowContract(); const results: Record< string, { result: string; outputs?: Record } > = {}; for (const name of z.array(z.string()).parse(jobs.required?.needs)) { results[name] = { result: name === overrides.failed ? "failure" : changes.tier === "reused" ? "skipped" : "success", }; } results.changes = { result: "success", outputs: changes }; const receiptDirectory = join(directory, ".cache", "receipt"); const output = join(directory, "github-output"); await writeFile(output, ""); const env = { ...process.env, CI_JOBS: JSON.stringify(results), CI_QUALIFICATION_DIRECTORY: receiptDirectory, GITHUB_EVENT_NAME: overrides.event ?? "schedule", GITHUB_SHA: head, GITHUB_RUN_ID: "100", GITHUB_RUN_ATTEMPT: overrides.runAttempt ?? "1", GITHUB_REPOSITORY: repository, GITHUB_OUTPUT: output, }; const result = executeWorkflowFixture(process.execPath, [requiredScript], { cwd: directory, env, }); return { directory, head, output, result }; } it("writes a full receipt only after every canonical flag and workflow need pass", async () => { const scopesPath = new URL("../../../scripts/ci/scopes.mjs", import.meta.url); const { CI_FLAGS } = await import(scopesPath.href); const changes = Object.fromEntries( CI_FLAGS.map((flag: string) => [flag, "true"]), ); changes.tier = "full"; const { directory, head, output, result } = await requiredRun(changes); await expect(result).resolves.toMatchObject({ stderr: "" }); const receiptPath = join(directory, ".cache", "receipt", "receipt.json"); const receipt = z .object({ sourceSha: z.string(), runId: z.number(), runAttempt: z.number(), }) .parse(JSON.parse(await readFile(receiptPath, "utf8"))); expect(receipt).toMatchObject({ sourceSha: head, runId: 100, runAttempt: 1 }); expect(await readFile(output, "utf8")).toContain( `receipt_path=${receiptPath}\n`, ); expect(await readFile(output, "utf8")).toContain( "receipt_name=ci-full-qualification-v1\n", ); }); it.each([ ["failed selected lane", { failed: "sqlite-runtime" }], ["partial full scope", {}], ] as const)( "does not issue a receipt after %s", async (caseName, overrides) => { const scopesPath = new URL( "../../../scripts/ci/scopes.mjs", import.meta.url, ); const { CI_FLAGS } = await import(scopesPath.href); const changes = Object.fromEntries( CI_FLAGS.map((flag: string) => [flag, "true"]), ); changes.tier = "full"; if (caseName === "partial full scope") changes.sqlite = "false"; const outcome = await requiredRun(changes, overrides); await expect(outcome.result).rejects.toMatchObject({ code: 1 }); await expect( readFile(join(outcome.directory, ".cache", "receipt", "receipt.json")), ).rejects.toMatchObject({ code: "ENOENT" }); expect(await readFile(outcome.output, "utf8")).not.toContain("receipt_"); }, ); it("never writes a fresh receipt for a reused scheduled qualification", async () => { const scopesPath = new URL("../../../scripts/ci/scopes.mjs", import.meta.url); const { CI_FLAGS } = await import(scopesPath.href); const changes = Object.fromEntries( CI_FLAGS.map((flag: string) => [flag, "false"]), ); changes.tier = "reused"; changes.qualification_run_id = "7"; const outcome = await requiredRun(changes); const result = await outcome.result; await expect( readFile(join(outcome.directory, ".cache", "receipt", "receipt.json")), ).rejects.toMatchObject({ code: "ENOENT" }); expect(result.stdout).toContain("Qualification reused from run 7"); expect(await readFile(outcome.output, "utf8")).not.toContain("receipt_"); }); it.each([ ["missing receipt run", { qualification_run_id: "" }, "schedule"], ["selected validation lane", { sqlite: "true" }, "schedule"], ["manual dispatch", {}, "workflow_dispatch"], ] as const)( "rejects qualification reuse with %s", async (_name, override, event) => { const { CI_FLAGS } = await import( new URL("../../../scripts/ci/scopes.mjs", import.meta.url).href ); const changes: Record = { ...Object.fromEntries(CI_FLAGS.map((flag: string) => [flag, "false"])), tier: "reused", qualification_run_id: "7", ...override, }; const outcome = await requiredRun(changes, { event }); await expect(outcome.result).rejects.toMatchObject({ code: 1, stderr: expect.stringContaining("scheduled admission receipt"), }); expect(await readFile(outcome.output, "utf8")).not.toContain("receipt_"); }, );