From 2092f90b7e0e4c05b473850d059cc1e37914b418 Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Sat, 17 Oct 2026 17:44:11 +0800 Subject: [PATCH] perf(ci): upload only current portable build cache entries Stage only the current portable Turbo task archives, metadata and available manifests for same-run artifact consumers. Share task identity/hash validation with cross-run keys and fail publication on incomplete preparation, while retaining existing cache authority and consumer build/prepack verification. Validation: 252 focused tests, full check:ci, docs:check, check:fast, actionlint, real Turbo output equivalence and invalidation, restored CLI/MCP scenarios and full Linux package verification passed. Fault injection detected historical-entry inclusion. Exact-head hosted CI and available review passed; measurement limits are recorded in the PR. Refs #4744. --- .../actions/portable-build-cache/action.yml | 14 +- .github/workflows/ci.yml | 7 +- docs/testing.md | 7 +- scripts/ci/portable-build-cache.mjs | 86 +++++++++ .../filesystem/ciPortableBuildCache.test.ts | 176 ++++++++++++++++++ .../filesystem/ciRequiredLanes.test.ts | 39 ++++ 6 files changed, 313 insertions(+), 16 deletions(-) create mode 100644 scripts/ci/portable-build-cache.mjs create mode 100644 tests/boundary/filesystem/ciPortableBuildCache.test.ts diff --git a/.github/actions/portable-build-cache/action.yml b/.github/actions/portable-build-cache/action.yml index 799b9476be..deec2c4cbb 100644 --- a/.github/actions/portable-build-cache/action.yml +++ b/.github/actions/portable-build-cache/action.yml @@ -24,19 +24,7 @@ runs: BUILD_CACHE_PREFIX: rea-portable-build-v2-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.npmrc', '.nvmrc', 'package-lock.json') }} run: | npm run --silent test:prepare -- --dry=json | - node --input-type=module -e ' - import { createHash } from "node:crypto"; - import { readFileSync } from "node:fs"; - const { tasks } = JSON.parse(readFileSync(0, "utf8")); - const hashes = tasks.map(({ taskId, hash }) => { - if (typeof hash !== "string") - throw new Error(`Build task ${taskId} must have an eager Turbo hash`); - return `${taskId}:${hash}`; - }).sort(); - const hash = createHash("sha256").update(hashes.join("\n")).digest("hex"); - console.log(`key=${process.env.BUILD_CACHE_PREFIX}-${hash}`); - console.log(`prefix=${process.env.BUILD_CACHE_PREFIX}-`); - ' >> "$GITHUB_OUTPUT" + node scripts/ci/portable-build-cache.mjs key >> "$GITHUB_OUTPUT" - name: Restore portable build outputs if: inputs.build_cache_artifact == '' && inputs.save != 'true' uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 82bf893175..60459cdeda 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -357,10 +357,15 @@ jobs: # test:prepare caches only portable JS, skill, and catalog outputs. # Native package jobs still run their build and prepack checks; Turbo # restores matching outputs or rebuilds when their inputs differ. + - name: Stage current portable build task cache + shell: bash + run: | + npm run --silent test:prepare -- --dry=json | + node scripts/ci/portable-build-cache.mjs snapshot - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: rea-build-cache - path: .turbo/cache/ + path: .cache/ci-build-cache/ include-hidden-files: true if-no-files-found: error retention-days: 7 diff --git a/docs/testing.md b/docs/testing.md index 0708237017..aa73ede80c 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -503,7 +503,7 @@ costs. Its queue, execution, and failures are independent of the required gate and publication. CPU time and peak process-family memory are not measured by this API. The build job saves portable Turbo outputs across runs. Eight Linux real-system -lanes restore this cache without waiting for the build job. The shared action +lanes reuse its current-run artifact; standalone dispatches restore the cross-run cache. The shared action derives its key from `test:prepare --dry=json` task hashes, runner OS and architecture, `.npmrc`, `.nvmrc`, and the lockfile; test-only changes can reuse compiled outputs. Every build and verification command still runs, so Turbo checks task inputs and @@ -513,7 +513,10 @@ that generator's entry. Native outputs are not stored in this cache. Native package jobs, the Windows curated lane, and the Apple artifact lane reuse the same workflow's Turbo cache for portable compiled JavaScript, skills, and -catalogs. The Windows lane relies on the Linux static job for typechecking. They retain the normal build and prepack +catalogs. The producer stages only the completed `test:prepare` task hashes into +`.cache/ci-build-cache/`, preserving their archives, metadata, and available manifests. +Missing required entries fail publication; older cross-run entries stay in the +original cache. The Windows lane relies on the Linux static job for typechecking. They retain the normal build and prepack commands, which restore matching outputs or rebuild on a cache miss. Native artifacts, dependency installs, package archives, and runtime verification remain host-local. Keep platform-dependent outputs out of this shared cache. diff --git a/scripts/ci/portable-build-cache.mjs b/scripts/ci/portable-build-cache.mjs new file mode 100644 index 0000000000..9a029d9e70 --- /dev/null +++ b/scripts/ci/portable-build-cache.mjs @@ -0,0 +1,86 @@ +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { copyFile, lstat, mkdir, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { z } from "zod"; + +const output = ".cache/ci-build-cache"; + +function readTasks() { + const { tasks } = z + .object({ + tasks: z + .array( + z.object({ + taskId: z.string().min(1), + // The pinned Turbo version uses eager 64-bit hexadecimal hashes. + hash: z.string().regex(/^[0-9a-f]{16}$/u), + }), + ) + .min(1), + }) + .parse(JSON.parse(readFileSync(0, "utf8"))); + const identities = new Set(); + for (const { taskId } of tasks) { + if (identities.has(taskId)) + throw new Error(`Duplicate portable build task: ${taskId}`); + identities.add(taskId); + } + return tasks; +} + +async function snapshot(tasks) { + await mkdir(output, { recursive: true }); + let files = 0; + let bytes = 0; + try { + for (const hash of new Set(tasks.map((task) => task.hash))) { + for (const suffix of [".tar.zst", "-meta.json", "-manifest.json"]) { + const name = `${hash}${suffix}`; + const source = join(".turbo/cache", name); + const entry = await lstat(source).catch((error) => { + // Manifests are not present in every supported local cache entry. + if (suffix === "-manifest.json" && error.code === "ENOENT") + return undefined; + throw error; + }); + if (!entry) continue; + if (!entry.isFile()) + throw new Error( + `Portable build cache entry must be a regular file: ${source}`, + ); + await copyFile(source, join(output, name)); + files++; + bytes += entry.size; + } + } + } catch (error) { + await rm(output, { recursive: true, force: true }); + throw error; + } + console.log( + JSON.stringify({ tasks: tasks.length, files, bytes, path: output }), + ); +} + +const [mode, ...extra] = process.argv.slice(2); +if (extra.length || !["key", "snapshot"].includes(mode)) + throw new Error( + "Usage: portable-build-cache.mjs key|snapshot < turbo-dry-plan.json", + ); +// Remove any earlier snapshot before parsing: a failed second invocation must +// never leave an old successful artifact eligible for upload. +if (mode === "snapshot") await rm(output, { recursive: true, force: true }); +const tasks = readTasks(); +if (mode === "key") { + const prefix = z + .string() + .min(1) + .regex(/^[^\r\n]+$/u) + .parse(process.env.BUILD_CACHE_PREFIX); + const hashes = tasks.map(({ taskId, hash }) => `${taskId}:${hash}`).sort(); + const digest = createHash("sha256").update(hashes.join("\n")).digest("hex"); + console.log(`key=${prefix}-${digest}\nprefix=${prefix}-`); +} else { + await snapshot(tasks); +} diff --git a/tests/boundary/filesystem/ciPortableBuildCache.test.ts b/tests/boundary/filesystem/ciPortableBuildCache.test.ts new file mode 100644 index 0000000000..a677d2aa0e --- /dev/null +++ b/tests/boundary/filesystem/ciPortableBuildCache.test.ts @@ -0,0 +1,176 @@ +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { + mkdir, + readFile, + readdir, + rm, + symlink, + writeFile, +} from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { expect, it } from "vitest"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const script = fileURLToPath( + new URL("../../../scripts/ci/portable-build-cache.mjs", import.meta.url), +); +const first = { taskId: "compile", hash: "0123456789abcdef" }; +const second = { taskId: "artifacts:skills", hash: "fedcba9876543210" }; +const plan = { tasks: [first, second] }; +const prefix = "rea-portable-build-v2-Linux-X64-toolchain"; +const output = ".cache/ci-build-cache"; + +function run(root: string, mode: "key" | "snapshot", input: unknown = plan) { + const result = spawnSync(process.execPath, [script, mode], { + cwd: root, + input: typeof input === "string" ? input : JSON.stringify(input), + encoding: "utf8", + env: { ...process.env, BUILD_CACHE_PREFIX: prefix }, + }); + expect(result.error).toBeUndefined(); + return result; +} + +async function fixture() { + const root = await createTestTempDirectory("rea-ci-portable-cache-"); + await mkdir(join(root, ".turbo/cache"), { recursive: true }); + return root; +} + +async function entry(root: string, hash: string, manifest = true) { + for (const suffix of [ + ".tar.zst", + "-meta.json", + ...(manifest ? ["-manifest.json"] : []), + ]) + await writeFile( + join(root, ".turbo/cache", `${hash}${suffix}`), + `${hash}${suffix}\n`, + ); +} + +async function inventory(root: string, path: string) { + const files = (await readdir(join(root, path))).sort(); + return Promise.all( + files.map(async (name) => [ + name, + await readFile(join(root, path, name), "hex"), + ]), + ); +} + +it("publishes only current task companions without changing the source cache or retaining old snapshots", async () => { + const root = await fixture(); + await entry(root, first.hash); + await entry(root, second.hash, false); + await entry(root, "aaaaaaaaaaaaaaaa"); + const before = await inventory(root, ".turbo/cache"); + const result = run(root, "snapshot"); + expect(result.status, result.stderr).toBe(0); + const current = before.filter( + ([name]) => name?.startsWith(first.hash) || name?.startsWith(second.hash), + ); + expect(await inventory(root, output)).toEqual(current); + expect(JSON.parse(result.stdout)).toMatchObject({ + tasks: 2, + files: 5, + path: output, + }); + expect(run(root, "snapshot", { tasks: [second] }).status).toBe(0); + expect(await inventory(root, output)).toEqual( + before.filter(([name]) => name?.startsWith(second.hash)), + ); + expect(await inventory(root, ".turbo/cache")).toEqual(before); +}); + +it("retains existing cross-run keys independent of plan order and invalidates changed task hashes", async () => { + const root = await fixture(); + const digest = createHash("sha256") + .update( + plan.tasks + .map(({ taskId, hash }) => `${taskId}:${hash}`) + .sort() + .join("\n"), + ) + .digest("hex"); + const expected = `key=${prefix}-${digest}\nprefix=${prefix}-\n`; + const result = run(root, "key"); + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toBe(expected); + expect(run(root, "key", { tasks: [second, first] }).stdout).toBe(expected); + const changed = run(root, "key", { + tasks: [first, { ...second, hash: "aaaaaaaaaaaaaaaa" }], + }); + expect(changed.status, changed.stderr).toBe(0); + expect(changed.stdout).not.toBe(expected); +}); + +it("rejects malformed or ambiguous plans in both commands and removes stale publication output", async () => { + const root = await fixture(); + for (const invalid of [ + "not JSON", + {}, + { tasks: [] }, + { tasks: [{ taskId: "compile" }] }, + { tasks: [{ ...first, hash: null }] }, + { tasks: [{ ...first, hash: "../outside" }] }, + { tasks: [first, { ...first, hash: second.hash }] }, + ]) { + await mkdir(join(root, output), { recursive: true }); + await writeFile(join(root, output, "old-success"), "stale"); + for (const mode of ["key", "snapshot"] as const) { + const result = run(root, mode, invalid); + expect(result.status, result.stderr).not.toBe(0); + expect(result.stdout).toBe(""); + } + await expect(readdir(join(root, output))).rejects.toMatchObject({ + code: "ENOENT", + }); + } +}); + +it("fails publication on a cold cache, missing companions or non-file entries, including after a partial copy", async () => { + const root = await fixture(); + expect(run(root, "snapshot").status).not.toBe(0); + await expect(readdir(join(root, output))).rejects.toMatchObject({ + code: "ENOENT", + }); + await entry(root, first.hash); + for (const suffix of [".tar.zst", "-meta.json", "-manifest.json"]) { + await entry(root, second.hash); + const path = join(root, ".turbo/cache", `${second.hash}${suffix}`); + await rm(path); + if (suffix === "-manifest.json") await mkdir(path); + const result = run(root, "snapshot"); + expect(result.status, result.stderr).not.toBe(0); + await expect(readdir(join(root, output))).rejects.toMatchObject({ + code: "ENOENT", + }); + await rm(path, { recursive: true, force: true }); + } +}); + +it.skipIf(process.platform === "win32")( + "rejects symlinked archives and optional manifests without copying their targets", + async () => { + const root = await fixture(); + const outside = join(root, "outside"); + await writeFile(outside, "must not publish"); + for (const suffix of [".tar.zst", "-manifest.json"]) { + await entry(root, first.hash); + const path = join(root, ".turbo/cache", `${first.hash}${suffix}`); + await rm(path); + await symlink(outside, path); + const result = run(root, "snapshot", { tasks: [first] }); + expect(result.status, result.stderr).not.toBe(0); + expect(result.stderr).toContain("must be a regular file"); + await expect(readdir(join(root, output))).rejects.toMatchObject({ + code: "ENOENT", + }); + expect(await readFile(outside, "utf8")).toBe("must not publish"); + await rm(path); + } + }, +); diff --git a/tests/boundary/filesystem/ciRequiredLanes.test.ts b/tests/boundary/filesystem/ciRequiredLanes.test.ts index 0bfd64a023..139ebfe7b9 100644 --- a/tests/boundary/filesystem/ciRequiredLanes.test.ts +++ b/tests/boundary/filesystem/ciRequiredLanes.test.ts @@ -175,6 +175,41 @@ it("holds native and provider fan-out behind static validation while portable fe it("binds provider build reuse to the current run while keeping standalone dispatches independent", async () => { const { jobs } = await workflow(); + const buildSteps = z + .array( + z.object({ + run: z.string().optional(), + uses: z.string().optional(), + shell: z.string().optional(), + if: z.string().optional(), + with: z.record(z.string(), z.unknown()).optional(), + "continue-on-error": z.boolean().optional(), + }), + ) + .parse(jobs.build?.steps); + const prepare = buildSteps.findIndex( + (step) => step.run === "npm run test:prepare", + ); + const stage = buildSteps.findIndex((step) => + step.run?.includes("portable-build-cache.mjs snapshot"), + ); + const upload = buildSteps.findIndex( + (step) => step.with?.name === "rea-build-cache", + ); + expect(prepare).toBeGreaterThanOrEqual(0); + expect(stage).toBeGreaterThan(prepare); + expect(upload).toBeGreaterThan(stage); + expect(buildSteps[stage]?.run).toContain( + "npm run --silent test:prepare -- --dry=json |", + ); + expect(buildSteps[stage]?.if).toBeUndefined(); + expect(buildSteps[stage]?.shell).toBe("bash"); + expect(buildSteps[stage]?.["continue-on-error"]).not.toBe(true); + expect(buildSteps[upload]?.if).toBeUndefined(); + expect(buildSteps[upload]?.with).toMatchObject({ + path: ".cache/ci-build-cache/", + "if-no-files-found": "error", + }); const consumers = [ "browser", "binary_layout", @@ -252,6 +287,7 @@ it("binds provider build reuse to the current run while keeping standalone dispa if: z.string().optional(), with: z.record(z.string(), z.unknown()).optional(), "continue-on-error": z.boolean().optional(), + run: z.string().optional(), }), ), }), @@ -276,6 +312,9 @@ it("binds provider build reuse to the current run while keeping standalone dispa }); expect(download?.if).toBe("inputs.build_cache_artifact != ''"); expect(download?.["continue-on-error"]).not.toBe(true); + expect(action.runs.steps.find((step) => step.id === "plan")?.run).toContain( + 'node scripts/ci/portable-build-cache.mjs key >> "$GITHUB_OUTPUT"', + ); for (const step of action.runs.steps.filter( (step) => step.id === "plan" || step.uses?.startsWith("actions/cache"), ))