diff --git a/docs/testing.md b/docs/testing.md index 22b5c0c772..6dddda64b3 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -874,7 +874,10 @@ summary. Representative Git-diff cases live in `tests/boundary/filesystem/ciChangeScope.test.ts`. `CI required` always runs and checks classification and every selected lane, -rejecting failed, cancelled, or unexpectedly skipped work. Branch protection +rejecting failed, cancelled, or unexpectedly skipped work. It rejects missing or +unknown tiers and requires every canonical scope to be explicitly `true` or +`false`; incomplete classification cannot silently omit verification lanes. +Branch protection should require this aggregate, `Static checks`, `Portable tests`, and `Conventional PR title`. Title validation runs separately on PR opening, reopening, commits, and edits; correcting a title refreshes only this lightweight diff --git a/scripts/ci/required.mjs b/scripts/ci/required.mjs index afe52c2b81..01907de376 100644 --- a/scripts/ci/required.mjs +++ b/scripts/ci/required.mjs @@ -2,22 +2,31 @@ import { CI_FLAGS } from "./scopes.mjs"; import { recordQualification } from "./qualification.mjs"; const jobs = JSON.parse(process.env.CI_JOBS); -const outputs = jobs.changes.outputs; +const outputs = jobs?.changes?.outputs; +if (!["fast", "full", "reused"].includes(outputs?.tier)) + throw new Error( + "CI classification requires a valid tier: fast, full, or reused", + ); +const invalidFlags = CI_FLAGS.filter( + (flag) => outputs[flag] !== "true" && outputs[flag] !== "false", +); +if (invalidFlags.length) + throw new Error( + `CI classification requires explicit true/false scope outputs: ${invalidFlags.join(", ")}`, + ); +const selectedScopes = CI_FLAGS.filter((flag) => outputs[flag] === "true"); const reused = outputs.tier === "reused"; if ( reused && (process.env.GITHUB_EVENT_NAME !== "schedule" || !/^[1-9]\d*$/.test(outputs.qualification_run_id ?? "") || !Number.isSafeInteger(Number(outputs.qualification_run_id)) || - CI_FLAGS.some((flag) => outputs[flag] !== "false")) + selectedScopes.length !== 0) ) throw new Error( "Reused qualification requires a scheduled admission receipt and no selected lanes", ); -if ( - outputs.tier === "full" && - CI_FLAGS.some((flag) => outputs[flag] !== "true") -) +if (outputs.tier === "full" && selectedScopes.length !== CI_FLAGS.length) throw new Error("Full qualification requires every canonical CI scope"); const required = reused ? ["changes"] : ["changes", "static", "test"]; const lanes = { @@ -31,9 +40,7 @@ const lanes = { docs: ["docs"], workflows: ["static"], }; -for (const [scope, value] of Object.entries(outputs)) { - if (value === "true") required.push(...(lanes[scope] ?? [scope])); -} +for (const scope of selectedScopes) required.push(...(lanes[scope] ?? [scope])); const failed = [...new Set(required)].filter( (name) => jobs[name]?.result !== "success", ); diff --git a/tests/boundary/filesystem/ciChangeScope.test.ts b/tests/boundary/filesystem/ciChangeScope.test.ts index 8ed397e845..759f5c367f 100644 --- a/tests/boundary/filesystem/ciChangeScope.test.ts +++ b/tests/boundary/filesystem/ciChangeScope.test.ts @@ -662,8 +662,16 @@ it("rejects skipped selected work while allowing unselected lanes to skip", asyn const command = fileURLToPath( new URL("../../../scripts/ci/required.mjs", import.meta.url), ); + const { CI_FLAGS } = await loadScopes(); const jobs = { - changes: { result: "success", outputs: { evm: "true" } }, + changes: { + result: "success", + outputs: { + ...Object.fromEntries(CI_FLAGS.map((flag) => [flag, "false"])), + tier: "fast", + evm: "true", + }, + }, static: { result: "success" }, test: { result: "success" }, evm: { result: "success" }, diff --git a/tests/boundary/filesystem/ciRequiredLanes.test.ts b/tests/boundary/filesystem/ciRequiredLanes.test.ts index 1bac771a21..e460bb52b5 100644 --- a/tests/boundary/filesystem/ciRequiredLanes.test.ts +++ b/tests/boundary/filesystem/ciRequiredLanes.test.ts @@ -26,6 +26,15 @@ const workflow = async (filename = "ci.yml") => ), ); +const scopeFlags = async () => + z + .object({ CI_FLAGS: z.array(z.string()) }) + .parse( + await import( + new URL("../../../scripts/ci/scopes.mjs", import.meta.url).href + ), + ).CI_FLAGS; + it("binds SQLite compatibility jobs to the build's prepared runtime artifact and root layout", async () => { expect((await workflow()).jobs["sqlite-runtime"]?.if).toBe( "needs.changes.outputs.sqlite == 'true'", @@ -57,13 +66,7 @@ it("binds SQLite compatibility jobs to the build's prepared runtime artifact and it("exports every canonical scope with read authority for admission and keeps telemetry outside the gate", async () => { const { jobs } = await workflow(); - const { CI_FLAGS } = z - .object({ CI_FLAGS: z.array(z.string()) }) - .parse( - await import( - new URL("../../../scripts/ci/scopes.mjs", import.meta.url).href - ), - ); + const CI_FLAGS = await scopeFlags(); const outputs = z.record(z.string(), z.string()).parse(jobs.changes?.outputs); expect(jobs.changes?.permissions).toEqual({ contents: "read", @@ -294,7 +297,14 @@ it.each([ ); needs.changes = { result: "success", - outputs: { code: "true", [scope]: selected }, + outputs: { + ...Object.fromEntries( + (await scopeFlags()).map((flag) => [flag, "false"]), + ), + tier: "fast", + code: "true", + [scope]: selected, + }, }; if (result === "missing") delete needs[lane]; else needs[lane] = { result }; @@ -319,3 +329,69 @@ it.each([ } }, ); + +it("rejects incomplete or malformed classification even when available jobs succeeded", async () => { + const flags = await scopeFlags(); + const valid = { + ...Object.fromEntries(flags.map((flag) => [flag, "false"])), + tier: "fast", + }; + const run = (outputs: unknown) => + executeWorkflowFixture( + process.execPath, + [ + fileURLToPath( + new URL("../../../scripts/ci/required.mjs", import.meta.url), + ), + ], + { + env: { + ...process.env, + CI_JOBS: JSON.stringify({ + changes: { result: "success", outputs }, + static: { result: "success" }, + test: { result: "success" }, + }), + CI_QUALIFICATION_DIRECTORY: "", + }, + }, + ); + await expect(run(valid)).resolves.toMatchObject({ stderr: "" }); + // Metadata is not a scope, even when its string happens to be "true". + await expect( + run({ ...valid, qualification_run_id: "true" }), + ).resolves.toMatchObject({ stderr: "" }); + for (const outputs of [ + undefined, + null, + {}, + { ...valid, tier: undefined }, + { ...valid, tier: "" }, + { ...valid, tier: "unknown" }, + ]) { + await expect(run(outputs)).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining("valid tier"), + }); + } + for (const flag of flags) { + await expect(run({ ...valid, [flag]: undefined })).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining(`scope outputs: ${flag}`), + }); + } + for (const value of ["", "TRUE", "yes", true, false, 0, null, [], {}]) { + await expect(run({ ...valid, sqlite: value })).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining("scope outputs: sqlite"), + }); + } + for (const tier of ["full", "reused"]) { + await expect( + run({ ...valid, tier, sqlite: undefined }), + ).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining("scope outputs: sqlite"), + }); + } +});