mirror of
https://github.com/morluto/rea.git
synced 2026-10-26 17:35:11 +00:00
fix(ci): run cache retention from its trusted workflow revision
This commit is contained in:
1 parent
5518363aa6
commit
60956c8c56
2 files changed
+249
-25
No files matched your search
@@ -18,10 +18,11 @@ concurrency:
|
||||
jobs:
|
||||
retain:
|
||||
name: Retain Turbo snapshots and npm download caches
|
||||
# Checkout and run only a same-repository default-branch commit. Pull
|
||||
# requests, including forks, do not reach these steps. Static and build
|
||||
# save Turbo snapshots before later lanes finish, so a failed, timed-out,
|
||||
# or cancelled run still retains those snapshots.
|
||||
# Restrict the triggering-source checkout to same-repository default-branch
|
||||
# CI events. The helper runs from this trusted workflow revision; triggering
|
||||
# source is used only for cache identity and is never executed. Static and
|
||||
# build save snapshots before later lanes finish, so failed, timed-out, or
|
||||
# cancelled runs still retain those snapshots.
|
||||
if: >-
|
||||
(github.event.workflow_run.conclusion == 'success' ||
|
||||
github.event.workflow_run.conclusion == 'failure' ||
|
||||
@@ -35,23 +36,29 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
|
||||
with:
|
||||
ref: ${{ github.workflow_sha }}
|
||||
path: trusted-retention
|
||||
persist-credentials: false
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
|
||||
with:
|
||||
ref: ${{ github.event.workflow_run.head_sha }}
|
||||
path: triggering-source
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
|
||||
with:
|
||||
node-version-file: .nvmrc
|
||||
node-version-file: trusted-retention/.nvmrc
|
||||
- name: Retain current Turbo snapshots and npm download caches
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
CI_REPOSITORY: ${{ github.repository }}
|
||||
CACHE_PREFIX: rea-static-v1-${{ runner.os }}-${{ runner.arch }}-
|
||||
CACHE_KEEP_KEY: rea-static-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.nvmrc', '.npmrc', 'package-lock.json') }}-${{ github.event.workflow_run.head_sha }}
|
||||
CACHE_KEEP_KEY: rea-static-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('triggering-source/.nvmrc', 'triggering-source/.npmrc', 'triggering-source/package-lock.json') }}-${{ github.event.workflow_run.head_sha }}
|
||||
CACHE_DEEP_PREFIX: rea-static-deep-v1-${{ runner.os }}-${{ runner.arch }}-
|
||||
CACHE_DEEP_KEEP_KEY: rea-static-deep-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.nvmrc', '.npmrc', 'package-lock.json') }}-${{ github.event.workflow_run.head_sha }}
|
||||
CACHE_DEEP_KEEP_KEY: rea-static-deep-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('triggering-source/.nvmrc', 'triggering-source/.npmrc', 'triggering-source/package-lock.json') }}-${{ github.event.workflow_run.head_sha }}
|
||||
CACHE_KNIP_PREFIX: knip-meta-v1-${{ runner.os }}-
|
||||
CACHE_KNIP_KEEP_KEY: knip-meta-v1-${{ runner.os }}-${{ hashFiles('knip.json', 'tsconfig.json', 'package.json', 'package-lock.json') }}-${{ github.event.workflow_run.head_sha }}
|
||||
CACHE_KNIP_KEEP_KEY: knip-meta-v1-${{ runner.os }}-${{ hashFiles('triggering-source/knip.json', 'triggering-source/tsconfig.json', 'triggering-source/package.json', 'triggering-source/package-lock.json') }}-${{ github.event.workflow_run.head_sha }}
|
||||
CACHE_PORTABLE_PREFIX: rea-portable-build-v2-${{ runner.os }}-${{ runner.arch }}-
|
||||
DEDUPE_NPM_DOWNLOADS: "true"
|
||||
run: node scripts/ci/cache-retention.mjs
|
||||
run: node trusted-retention/scripts/ci/cache-retention.mjs
|
||||
@@ -1,5 +1,14 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { chmod, readFile, readdir, writeFile } from "node:fs/promises";
|
||||
import { createHash } from "node:crypto";
|
||||
import {
|
||||
access,
|
||||
chmod,
|
||||
copyFile,
|
||||
mkdir,
|
||||
readFile,
|
||||
readdir,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import { delimiter, join } from "node:path";
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
@@ -61,6 +70,185 @@ const readWorkflows = async () => {
|
||||
);
|
||||
};
|
||||
|
||||
function aggregateHashFilesContent(contents: Buffer[]) {
|
||||
const aggregate = createHash("sha256");
|
||||
for (const content of contents)
|
||||
aggregate.update(createHash("sha256").update(content).digest());
|
||||
return aggregate.digest("hex");
|
||||
}
|
||||
|
||||
async function verifyPrefixedHashFiles(
|
||||
retentionEnv: Record<string, string>,
|
||||
runnerWorkspace: string,
|
||||
legacyRoot: string,
|
||||
) {
|
||||
const keyExpressions = [
|
||||
retentionEnv.CACHE_KEEP_KEY,
|
||||
retentionEnv.CACHE_DEEP_KEEP_KEY,
|
||||
retentionEnv.CACHE_KNIP_KEEP_KEY,
|
||||
];
|
||||
let targetHash: string | undefined;
|
||||
for (const expression of keyExpressions) {
|
||||
if (expression === undefined)
|
||||
throw new Error("Missing retention cache key");
|
||||
const paths = expression.match(/hashFiles\(([^)]*)\)/u)?.[1];
|
||||
expect(paths).toBeDefined();
|
||||
const targetPaths = paths!.split(", ").map((path) => path.slice(1, -1));
|
||||
expect(
|
||||
targetPaths.every((path) => path.startsWith("triggering-source/")),
|
||||
).toBe(true);
|
||||
const fileNames = targetPaths.map((path) =>
|
||||
path.slice("triggering-source/".length),
|
||||
);
|
||||
const nestedHash = aggregateHashFilesContent(
|
||||
await Promise.all(
|
||||
targetPaths.map((path) => readFile(join(runnerWorkspace, path))),
|
||||
),
|
||||
);
|
||||
const flatHash = aggregateHashFilesContent(
|
||||
await Promise.all(
|
||||
fileNames.map((name) => readFile(join(legacyRoot, name))),
|
||||
),
|
||||
);
|
||||
expect(nestedHash).toBe(flatHash);
|
||||
if (expression === retentionEnv.CACHE_KEEP_KEY) targetHash = nestedHash;
|
||||
}
|
||||
expect(targetHash).toBeDefined();
|
||||
return targetHash!;
|
||||
}
|
||||
|
||||
async function exerciseTrustedRetentionWithLegacyTarget(
|
||||
retentionRun: unknown,
|
||||
retentionEnv: Record<string, string>,
|
||||
) {
|
||||
const runnerWorkspace = await createTestTempDirectory(
|
||||
"rea-ci-retention-trusted-checkout-",
|
||||
);
|
||||
const legacyTarget = join(runnerWorkspace, "triggering-source");
|
||||
const legacyRoot = join(runnerWorkspace, "legacy-root");
|
||||
const trustedSource = join(runnerWorkspace, "trusted-retention");
|
||||
const inputNames = [
|
||||
".nvmrc",
|
||||
".npmrc",
|
||||
"package-lock.json",
|
||||
"knip.json",
|
||||
"tsconfig.json",
|
||||
"package.json",
|
||||
];
|
||||
const targetSha = "0123456789abcdef0123456789abcdef01234567";
|
||||
await mkdir(legacyTarget, { recursive: true });
|
||||
await mkdir(legacyRoot, { recursive: true });
|
||||
for (const name of inputNames) {
|
||||
const contents = `target-source:${name}\n`;
|
||||
await writeFile(join(legacyTarget, name), contents);
|
||||
await writeFile(join(legacyRoot, name), contents);
|
||||
}
|
||||
const trustedHelperPath = join(
|
||||
trustedSource,
|
||||
"scripts/ci/cache-retention.mjs",
|
||||
);
|
||||
await mkdir(join(trustedSource, "scripts/ci"), { recursive: true });
|
||||
await copyFile(
|
||||
join(repository, "scripts/ci/cache-retention.mjs"),
|
||||
trustedHelperPath,
|
||||
);
|
||||
await expect(
|
||||
access(join(legacyTarget, "scripts/ci/cache-retention.mjs")),
|
||||
).rejects.toMatchObject({ code: "ENOENT" });
|
||||
await expect(access(trustedHelperPath)).resolves.toBeUndefined();
|
||||
|
||||
// Actions hashFiles hashes file bytes, then the ordered per-file digests.
|
||||
// A common checkout-directory prefix preserves both file contents and order.
|
||||
const targetHash = await verifyPrefixedHashFiles(
|
||||
retentionEnv,
|
||||
runnerWorkspace,
|
||||
legacyRoot,
|
||||
);
|
||||
|
||||
const prefix = "rea-static-v1-Linux-X64-";
|
||||
const keepKey = `${prefix}${targetHash}-${targetSha}`;
|
||||
const cacheFixture = {
|
||||
actions_caches: [
|
||||
{
|
||||
id: 1,
|
||||
key: `${prefix}old`,
|
||||
created_at: "2026-10-01T00:00:00Z",
|
||||
last_accessed_at: "2026-10-01T00:00:00Z",
|
||||
size_in_bytes: 10,
|
||||
version: "a".repeat(64),
|
||||
ref: "refs/heads/main",
|
||||
},
|
||||
{
|
||||
id: 2,
|
||||
key: `${prefix}mid`,
|
||||
created_at: "2026-10-15T00:00:00Z",
|
||||
last_accessed_at: "2026-10-15T00:00:00Z",
|
||||
size_in_bytes: 20,
|
||||
version: "a".repeat(64),
|
||||
ref: "refs/heads/main",
|
||||
},
|
||||
{
|
||||
id: 3,
|
||||
key: `${prefix}new`,
|
||||
created_at: "2026-10-20T00:00:00Z",
|
||||
last_accessed_at: "2026-10-20T00:00:00Z",
|
||||
size_in_bytes: 30,
|
||||
version: "a".repeat(64),
|
||||
ref: "refs/heads/main",
|
||||
},
|
||||
{
|
||||
id: 4,
|
||||
key: keepKey,
|
||||
created_at: "2026-09-30T00:00:00Z",
|
||||
last_accessed_at: "2026-09-30T00:00:00Z",
|
||||
size_in_bytes: 40,
|
||||
version: "a".repeat(64),
|
||||
ref: "refs/heads/main",
|
||||
},
|
||||
],
|
||||
};
|
||||
const ghPath = join(runnerWorkspace, "bin/gh");
|
||||
const ghLog = join(runnerWorkspace, "fake-gh.log");
|
||||
const cacheInput = join(runnerWorkspace, "cache-list.json");
|
||||
await mkdir(join(runnerWorkspace, "bin"), { recursive: true });
|
||||
await writeFile(cacheInput, JSON.stringify(cacheFixture));
|
||||
await writeFile(
|
||||
ghPath,
|
||||
`#!${process.execPath}\nconst fs = require("node:fs");\nconst [command, option, method, endpoint] = process.argv.slice(2);\nif (command !== "api" || option !== "--method") process.exit(2);\nfs.appendFileSync(process.env.FAKE_GH_LOG, method + " " + endpoint + "\\n");\nif (method === "GET") process.stdout.write(fs.readFileSync(process.env.FAKE_GH_CACHE_INPUT, "utf8"));\nelse if (method !== "DELETE") process.exit(2);\n`,
|
||||
);
|
||||
await chmod(ghPath, 0o755);
|
||||
expect(retentionRun).toBe(
|
||||
"node trusted-retention/scripts/ci/cache-retention.mjs",
|
||||
);
|
||||
const helperPath = join(
|
||||
runnerWorkspace,
|
||||
String(retentionRun).replace(/^node /u, ""),
|
||||
);
|
||||
const command = execFileSync(process.execPath, [helperPath], {
|
||||
cwd: runnerWorkspace,
|
||||
env: {
|
||||
PATH: join(runnerWorkspace, "bin"),
|
||||
CI_REPOSITORY: "morluto/rea",
|
||||
CACHE_PREFIX: prefix,
|
||||
CACHE_KEEP_KEY: keepKey,
|
||||
FAKE_GH_LOG: ghLog,
|
||||
FAKE_GH_CACHE_INPUT: cacheInput,
|
||||
},
|
||||
encoding: "utf8",
|
||||
});
|
||||
const requests = await readFile(ghLog, "utf8");
|
||||
expect(JSON.parse(command)).toEqual({
|
||||
dryRun: false,
|
||||
families: [{ prefix, deleteCount: 1, deleteBytes: 10 }],
|
||||
npmDuplicateDeleteCount: 0,
|
||||
npmDuplicateDeleteBytes: 0,
|
||||
});
|
||||
expect(requests.trim().split("\n")).toEqual([
|
||||
"GET repos/morluto/rea/actions/caches?per_page=100&page=1&key=rea-static-v1-Linux-X64-",
|
||||
"DELETE repos/morluto/rea/actions/caches/1",
|
||||
]);
|
||||
}
|
||||
|
||||
describe("hosted npm download-cache policy", () => {
|
||||
it("only allows shared writes for trusted original default-branch events", async () => {
|
||||
const action = await readAction();
|
||||
@@ -345,29 +533,58 @@ it("deletes superseded caches only from the trusted follow-up workflow", async (
|
||||
expect(job?.if).not.toContain("pull_request");
|
||||
const steps = job?.steps ?? [];
|
||||
expect(steps.some((step) => step.run === "npm ci")).toBe(false);
|
||||
const checkout = steps[0];
|
||||
expect(checkout?.uses).toBe(
|
||||
"actions/checkout@11d5960a326750d5838078e36cf38b85af677262",
|
||||
const checkouts = steps.filter(
|
||||
(step) =>
|
||||
step.uses === "actions/checkout@11d5960a326750d5838078e36cf38b85af677262",
|
||||
);
|
||||
expect(checkout?.with).toMatchObject({
|
||||
ref: "${{ github.event.workflow_run.head_sha }}",
|
||||
const trustedCheckout = checkouts[0];
|
||||
expect(trustedCheckout?.with).toMatchObject({
|
||||
ref: "${{ github.workflow_sha }}",
|
||||
path: "trusted-retention",
|
||||
"persist-credentials": false,
|
||||
});
|
||||
const retention = steps.find(
|
||||
(step) => step.run === "node scripts/ci/cache-retention.mjs",
|
||||
const targetCheckout = checkouts[1];
|
||||
expect(targetCheckout?.with).toMatchObject({
|
||||
ref: "${{ github.event.workflow_run.head_sha }}",
|
||||
path: "triggering-source",
|
||||
"persist-credentials": false,
|
||||
});
|
||||
expect(checkouts).toHaveLength(2);
|
||||
expect(steps.some((step) => step.run === "npm ci")).toBe(false);
|
||||
const setupNode = steps.find(
|
||||
(step) =>
|
||||
step.uses ===
|
||||
"actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020",
|
||||
);
|
||||
expect(setupNode?.with).toMatchObject({
|
||||
"node-version-file": "trusted-retention/.nvmrc",
|
||||
});
|
||||
const retention = steps.find(
|
||||
(step) =>
|
||||
step.run === "node trusted-retention/scripts/ci/cache-retention.mjs",
|
||||
);
|
||||
expect(retention).toBeDefined();
|
||||
const retentionEnv = z.record(z.string(), z.string()).parse(retention?.env);
|
||||
expect(retentionEnv.CACHE_PREFIX).toBe(
|
||||
"rea-static-v1-${{ runner.os }}-${{ runner.arch }}-",
|
||||
);
|
||||
expect(retentionEnv.CACHE_KEEP_KEY).toBe(
|
||||
"rea-static-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.nvmrc', '.npmrc', 'package-lock.json') }}-${{ github.event.workflow_run.head_sha }}",
|
||||
"rea-static-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('triggering-source/.nvmrc', 'triggering-source/.npmrc', 'triggering-source/package-lock.json') }}-${{ github.event.workflow_run.head_sha }}",
|
||||
);
|
||||
expect(retentionEnv.CACHE_DEEP_KEEP_KEY).toContain(
|
||||
"hashFiles('triggering-source/.nvmrc', 'triggering-source/.npmrc', 'triggering-source/package-lock.json')",
|
||||
);
|
||||
expect(retentionEnv.CACHE_KNIP_KEEP_KEY).toContain(
|
||||
"hashFiles('triggering-source/knip.json', 'triggering-source/tsconfig.json', 'triggering-source/package.json', 'triggering-source/package-lock.json')",
|
||||
);
|
||||
expect(retentionEnv.CACHE_PORTABLE_PREFIX).toBe(
|
||||
"rea-portable-build-v2-${{ runner.os }}-${{ runner.arch }}-",
|
||||
);
|
||||
expect(retentionEnv.DEDUPE_NPM_DOWNLOADS).toBe("true");
|
||||
expect(retentionEnv.GH_TOKEN).toBe("${{ github.token }}");
|
||||
|
||||
// Exercise the selected trusted helper against a legacy target tree.
|
||||
await exerciseTrustedRetentionWithLegacyTarget(retention?.run, retentionEnv);
|
||||
const ci = (await readWorkflows()).find(({ file }) => file === "ci.yml");
|
||||
expect(JSON.stringify(ci?.workflow)).not.toContain('"actions":"write"');
|
||||
const deepSteps = ci?.workflow.jobs?.["static-deep"]?.steps ?? [];
|
||||
@@ -382,18 +599,18 @@ it("deletes superseded caches only from the trusted follow-up workflow", async (
|
||||
const savedOnThisRun = "${{ github.event.workflow_run.head_sha }}";
|
||||
const deepPrefix = "rea-static-deep-v1-${{ runner.os }}-${{ runner.arch }}-";
|
||||
const knipPrefix = "knip-meta-v1-${{ runner.os }}-";
|
||||
const targetSourceKey = (id: string) =>
|
||||
publishedKey(id)
|
||||
.replaceAll("${{ github.sha }}", savedOnThisRun)
|
||||
.replaceAll("hashFiles('", "hashFiles('triggering-source/")
|
||||
.replaceAll(", '", ", 'triggering-source/");
|
||||
expect(retentionEnv.CACHE_DEEP_PREFIX).toBe(deepPrefix);
|
||||
expect(retentionEnv.CACHE_DEEP_KEEP_KEY).toBe(
|
||||
publishedKey("static-cache").replaceAll(
|
||||
"${{ github.sha }}",
|
||||
savedOnThisRun,
|
||||
),
|
||||
targetSourceKey("static-cache"),
|
||||
);
|
||||
expect(publishedKey("static-cache").startsWith(deepPrefix)).toBe(true);
|
||||
expect(retentionEnv.CACHE_KNIP_PREFIX).toBe(knipPrefix);
|
||||
expect(retentionEnv.CACHE_KNIP_KEEP_KEY).toBe(
|
||||
publishedKey("knip-cache").replaceAll("${{ github.sha }}", savedOnThisRun),
|
||||
);
|
||||
expect(retentionEnv.CACHE_KNIP_KEEP_KEY).toBe(targetSourceKey("knip-cache"));
|
||||
expect(publishedKey("knip-cache").startsWith(knipPrefix)).toBe(true);
|
||||
});
|
||||
|
||||
|
||||
Reference in new issue
Block a user