mirror of
https://github.com/morluto/rea.git
synced 2026-11-20 13:45:30 +00:00
* fix(ci): bind timing history to the verified workflow source * test(ci): reject timing history without valid commit lineage --------- Co-authored-by: ShuaixinHuang <ShuaixinHuang@users.noreply.github.com> Co-authored-by: morluto <76467478+morluto@users.noreply.github.com>
424 lines
13 KiB
TypeScript
424 lines
13 KiB
TypeScript
import { chmod, mkdir, readFile, writeFile } from "node:fs/promises";
|
|
import { delimiter, dirname, join } from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
|
|
import { expect, it } from "vitest";
|
|
|
|
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
|
import { executeWorkflowFixture } from "../../support/workflowGit.js";
|
|
|
|
const script = fileURLToPath(
|
|
new URL("../../../scripts/ci/test-history.mjs", import.meta.url),
|
|
);
|
|
const repository = "example/rea";
|
|
const repositoryId = 1;
|
|
const now = Date.now();
|
|
const headSha = "a".repeat(40);
|
|
|
|
function artifact(id: number, overrides: Record<string, unknown> = {}) {
|
|
return {
|
|
id: id + 100,
|
|
name: "test-timings-full",
|
|
expired: false,
|
|
created_at: new Date(now - 60_000).toISOString(),
|
|
workflow_run: {
|
|
id,
|
|
repository_id: repositoryId,
|
|
head_repository_id: repositoryId,
|
|
head_branch: "main",
|
|
head_sha: headSha,
|
|
},
|
|
...overrides,
|
|
};
|
|
}
|
|
|
|
function run(id: number, overrides: Record<string, unknown> = {}) {
|
|
return {
|
|
id,
|
|
status: "completed",
|
|
conclusion: "success",
|
|
path: ".github/workflows/ci.yml@main",
|
|
event: "schedule",
|
|
created_at: new Date(now - 120_000).toISOString(),
|
|
head_branch: "main",
|
|
head_repository: { full_name: repository },
|
|
head_sha: headSha,
|
|
...overrides,
|
|
};
|
|
}
|
|
|
|
async function invoke(
|
|
artifacts: unknown[],
|
|
runs: Record<number, unknown>,
|
|
options: {
|
|
apiError?: boolean;
|
|
repository?: string;
|
|
repositoryId?: string;
|
|
tier?: string;
|
|
successfulRuns?: unknown[];
|
|
} = {},
|
|
) {
|
|
const directory = await createTestTempDirectory("rea-ci-test-history-");
|
|
const fakeBin = join(directory, "bin");
|
|
await mkdir(fakeBin);
|
|
const fixture = join(directory, "api.json");
|
|
const calls = join(directory, "api-calls");
|
|
const output = join(directory, "github-output");
|
|
await writeFile(
|
|
fixture,
|
|
JSON.stringify({
|
|
artifacts,
|
|
runs,
|
|
successfulRuns: options.successfulRuns ?? [],
|
|
}),
|
|
);
|
|
await writeFile(output, "");
|
|
await writeFile(calls, "");
|
|
const fakeGh = join(fakeBin, "gh");
|
|
await writeFile(
|
|
fakeGh,
|
|
[
|
|
"#!/bin/sh",
|
|
'printf \'%s\\n\' "$*" >> "$GH_API_CALLS"',
|
|
'if [ "$GH_API_ERROR" = "true" ]; then exit 1; fi',
|
|
'if [ "$4" = "repos/example/rea/actions/artifacts" ]; then',
|
|
' node -e \'const fs=require("node:fs"); const x=JSON.parse(fs.readFileSync(process.env.GH_FIXTURE_RESPONSE,"utf8")); process.stdout.write(JSON.stringify({artifacts:x.artifacts}));\'',
|
|
'elif [ "$4" = "repos/example/rea/actions/workflows/ci.yml/runs" ]; then',
|
|
' node -e \'const fs=require("node:fs"); const x=JSON.parse(fs.readFileSync(process.env.GH_FIXTURE_RESPONSE,"utf8")); process.stdout.write(JSON.stringify({workflow_runs:x.successfulRuns}));\'',
|
|
"else",
|
|
' run_id="${4##*/}"',
|
|
' node -e \'const fs=require("node:fs"); const x=JSON.parse(fs.readFileSync(process.env.GH_FIXTURE_RESPONSE,"utf8")); const run=x.runs[Number(process.argv[1])]; if(run===null){console.error("gh: Not Found (HTTP 404)"); process.exit(1);} process.stdout.write(JSON.stringify(run ?? {}));\' "$run_id"',
|
|
"fi",
|
|
"",
|
|
].join("\n"),
|
|
);
|
|
await chmod(fakeGh, 0o755);
|
|
const node24 = dirname(process.execPath);
|
|
const result = await executeWorkflowFixture(process.execPath, [script], {
|
|
cwd: directory,
|
|
env: {
|
|
...process.env,
|
|
PATH: [fakeBin, node24, process.env.PATH ?? ""].join(delimiter),
|
|
GH_FIXTURE_RESPONSE: fixture,
|
|
GH_API_CALLS: calls,
|
|
GH_API_ERROR: String(options.apiError ?? false),
|
|
GH_TOKEN: "fixture-token",
|
|
CI_REPOSITORY: options.repository ?? repository,
|
|
CI_REPOSITORY_ID: options.repositoryId ?? String(repositoryId),
|
|
CI_DEFAULT_BRANCH: "main",
|
|
CI_TIMING_TIER: options.tier ?? "",
|
|
GITHUB_OUTPUT: output,
|
|
},
|
|
});
|
|
return {
|
|
result,
|
|
output: await readFile(output, "utf8"),
|
|
calls: await readFile(calls, "utf8"),
|
|
};
|
|
}
|
|
|
|
it("selects the latest trusted full run that actually published timing history", async () => {
|
|
const { result, output, calls } = await invoke(
|
|
[
|
|
artifact(15),
|
|
artifact(14),
|
|
artifact(13, {
|
|
workflow_run: { ...artifact(13).workflow_run, head_repository_id: 2 },
|
|
}),
|
|
artifact(12, {
|
|
workflow_run: { ...artifact(12).workflow_run, head_branch: "feature" },
|
|
}),
|
|
artifact(11),
|
|
],
|
|
{
|
|
15: run(15, { conclusion: "failure" }),
|
|
14: run(14, { path: ".github/workflows/release.yml" }),
|
|
11: run(11, { event: "workflow_dispatch" }),
|
|
},
|
|
{ tier: "full" },
|
|
);
|
|
|
|
expect(JSON.parse(result.stdout)).toEqual({
|
|
run_id: "11",
|
|
artifact_name: "test-timings-full",
|
|
});
|
|
expect(output).toBe("run-id=11\nartifact-name=test-timings-full\n");
|
|
expect(calls).toContain("name=test-timings-full");
|
|
expect(calls).toContain("per_page=100");
|
|
expect(calls).not.toContain("actions/runs/13");
|
|
expect(calls.match(/actions\/runs\//gu)).toHaveLength(3);
|
|
expect(calls).not.toContain("actions/workflows");
|
|
});
|
|
|
|
it("rejects expired, old, and pull-request candidates", async () => {
|
|
const { result, output } = await invoke(
|
|
[
|
|
artifact(15, { expired: true }),
|
|
artifact(13, {
|
|
created_at: new Date(now - 31 * 24 * 60 * 60 * 1000).toISOString(),
|
|
}),
|
|
artifact(12),
|
|
],
|
|
{
|
|
15: run(15),
|
|
13: run(13),
|
|
12: run(12, { event: "pull_request" }),
|
|
},
|
|
{ tier: "full" },
|
|
);
|
|
|
|
expect(JSON.parse(result.stdout)).toEqual({
|
|
run_id: "",
|
|
artifact_name: "",
|
|
});
|
|
expect(output).toBe("run-id=\nartifact-name=\n");
|
|
});
|
|
|
|
it("writes an empty run id when GitHub history is unavailable", async () => {
|
|
const { result, output } = await invoke(
|
|
[],
|
|
{},
|
|
{
|
|
apiError: true,
|
|
tier: "full",
|
|
},
|
|
);
|
|
expect(JSON.parse(result.stdout)).toEqual({
|
|
run_id: "",
|
|
artifact_name: "",
|
|
});
|
|
expect(output).toBe("run-id=\nartifact-name=\n");
|
|
});
|
|
|
|
it("bounds run lookups to five artifacts even when inventory has more", async () => {
|
|
const ids = [25, 24, 23, 22, 21, 20, 19];
|
|
const { result, calls } = await invoke(
|
|
ids.map((id) => artifact(id)),
|
|
Object.fromEntries(
|
|
ids.map((id, index) => [
|
|
id,
|
|
run(id, index < 5 ? { conclusion: "failure" } : {}),
|
|
]),
|
|
),
|
|
{ tier: "full" },
|
|
);
|
|
|
|
expect(JSON.parse(result.stdout)).toEqual({
|
|
run_id: "",
|
|
artifact_name: "",
|
|
});
|
|
expect(calls.match(/actions\/runs\//gu)).toHaveLength(5);
|
|
});
|
|
|
|
it.each([
|
|
{ repository: "owner/repo?per_page=100", repositoryId: "1" },
|
|
{ repository: "example/rea", repositoryId: "not-an-id" },
|
|
])(
|
|
"returns empty history without querying for an invalid repository identity: %j",
|
|
async (identity) => {
|
|
const { result, output, calls } = await invoke(
|
|
[],
|
|
{},
|
|
{ ...identity, tier: "full" },
|
|
);
|
|
expect(JSON.parse(result.stdout)).toEqual({
|
|
run_id: "",
|
|
artifact_name: "",
|
|
});
|
|
expect(output).toBe("run-id=\nartifact-name=\n");
|
|
expect(calls).toBe("");
|
|
},
|
|
);
|
|
|
|
it("selects fast history without reading the full-tier artifact", async () => {
|
|
const { result, calls } = await invoke(
|
|
[
|
|
artifact(32, {
|
|
created_at: new Date(now - 30_000).toISOString(),
|
|
}),
|
|
artifact(31, {
|
|
name: "test-timings-fast",
|
|
created_at: new Date(now - 90_000).toISOString(),
|
|
}),
|
|
artifact(30, {
|
|
name: "test-timings-fast",
|
|
created_at: new Date(now - 20_000).toISOString(),
|
|
workflow_run: {
|
|
...artifact(30).workflow_run,
|
|
head_branch: "feature",
|
|
},
|
|
}),
|
|
],
|
|
{
|
|
32: run(32, { event: "schedule" }),
|
|
31: run(31, { event: "push" }),
|
|
},
|
|
{ tier: "fast" },
|
|
);
|
|
|
|
expect(JSON.parse(result.stdout)).toEqual({
|
|
run_id: "31",
|
|
artifact_name: "test-timings-fast",
|
|
});
|
|
expect(calls).toContain("name=test-timings-fast");
|
|
expect(calls).not.toContain("name=test-timings-full");
|
|
expect(calls.match(/actions\/runs\//gu)).toHaveLength(1);
|
|
});
|
|
|
|
it("ignores pull-request fast history instead of borrowing another artifact", async () => {
|
|
const { result, calls } = await invoke(
|
|
[artifact(41, { name: "test-timings-fast" }), artifact(40)],
|
|
{
|
|
41: run(41, { event: "pull_request" }),
|
|
40: run(40, { event: "workflow_dispatch" }),
|
|
},
|
|
{ tier: "fast" },
|
|
);
|
|
|
|
expect(JSON.parse(result.stdout)).toEqual({
|
|
run_id: "",
|
|
artifact_name: "",
|
|
});
|
|
expect(calls).toContain("name=test-timings-fast");
|
|
expect(calls).not.toContain("name=test-timings-full");
|
|
expect(calls.match(/actions\/runs\//gu)).toHaveLength(1);
|
|
});
|
|
|
|
it("leaves fast history empty when only full-tier history was published", async () => {
|
|
const { result, output, calls } = await invoke(
|
|
[artifact(70)],
|
|
{ 70: run(70, { event: "schedule" }) },
|
|
{ tier: "fast" },
|
|
);
|
|
|
|
expect(JSON.parse(result.stdout)).toEqual({
|
|
run_id: "",
|
|
artifact_name: "",
|
|
});
|
|
expect(output).toBe("run-id=\nartifact-name=\n");
|
|
expect(calls).toContain("name=test-timings-fast");
|
|
expect(calls).not.toContain("name=test-timings-full");
|
|
expect(calls).not.toMatch(/actions\/runs\//u);
|
|
});
|
|
|
|
it("does not query when the tier has no timing artifact", async () => {
|
|
const { result, output, calls } = await invoke([artifact(71)], {
|
|
71: run(71, { event: "schedule" }),
|
|
});
|
|
|
|
expect(JSON.parse(result.stdout)).toEqual({
|
|
run_id: "",
|
|
artifact_name: "",
|
|
});
|
|
expect(output).toBe("run-id=\nartifact-name=\n");
|
|
expect(calls).toBe("");
|
|
});
|
|
|
|
it("selects full-tier history without using a newer fast artifact", async () => {
|
|
const { result } = await invoke(
|
|
[
|
|
artifact(61, {
|
|
name: "test-timings-fast",
|
|
created_at: new Date(now - 10_000).toISOString(),
|
|
}),
|
|
artifact(60, {
|
|
name: "test-timings-full",
|
|
created_at: new Date(now - 80_000).toISOString(),
|
|
}),
|
|
],
|
|
{
|
|
61: run(61, { event: "push" }),
|
|
60: run(60, { event: "schedule" }),
|
|
},
|
|
{ tier: "full" },
|
|
);
|
|
|
|
expect(JSON.parse(result.stdout)).toEqual({
|
|
run_id: "60",
|
|
artifact_name: "test-timings-full",
|
|
});
|
|
});
|
|
|
|
it("uses the next valid timing run when one listed run disappears", async () => {
|
|
const { result, calls } = await invoke(
|
|
[
|
|
artifact(81, { created_at: new Date(now - 30_000).toISOString() }),
|
|
artifact(80),
|
|
artifact(79),
|
|
],
|
|
{ 81: null, 80: run(80), 79: run(79) },
|
|
{ tier: "full" },
|
|
);
|
|
expect(JSON.parse(result.stdout)).toEqual({
|
|
run_id: "80",
|
|
artifact_name: "test-timings-full",
|
|
});
|
|
expect(calls).toContain("actions/runs/81");
|
|
expect(calls).toContain("actions/runs/80");
|
|
expect(calls).not.toContain("actions/runs/79");
|
|
});
|
|
|
|
it("finds an older successful timing artifact beyond five unsuccessful runs", async () => {
|
|
const newest = [107, 106, 105, 104, 103];
|
|
const { result, calls } = await invoke(
|
|
[...newest, 102, 101, 100].map((id) =>
|
|
artifact(id, {
|
|
created_at: new Date(now - (108 - id) * 1000).toISOString(),
|
|
}),
|
|
),
|
|
Object.fromEntries(
|
|
newest.map((id) => [id, run(id, { conclusion: "failure" })]),
|
|
),
|
|
{
|
|
tier: "full",
|
|
successfulRuns: [
|
|
run(999), // A successful run without this timing artifact.
|
|
run(102, { head_repository: { full_name: "fork/rea" } }),
|
|
run(100),
|
|
run(101),
|
|
],
|
|
},
|
|
);
|
|
expect(JSON.parse(result.stdout)).toEqual({
|
|
run_id: "101",
|
|
artifact_name: "test-timings-full",
|
|
});
|
|
expect(calls.match(/actions\/runs\//gu)).toHaveLength(5);
|
|
expect(calls.match(/actions\/workflows\/ci\.yml\/runs/gu)).toHaveLength(1);
|
|
expect(calls).toContain("branch=main");
|
|
expect(calls).toContain("status=success");
|
|
});
|
|
|
|
it.each([
|
|
["the commit SHAs differ", headSha, "b".repeat(40)],
|
|
["both commit SHAs are omitted", undefined, undefined],
|
|
["both commit SHAs are malformed", "invalid-sha", "invalid-sha"],
|
|
])("skips newer timing history when %s", async (_, artifactSha, runSha) => {
|
|
const unverifiedArtifact = artifact(51, {
|
|
created_at: new Date(now - 30_000).toISOString(),
|
|
workflow_run: { ...artifact(51).workflow_run, head_sha: artifactSha },
|
|
});
|
|
const unverifiedRun = run(51, { head_sha: runSha });
|
|
const [fallback, empty] = await Promise.all([
|
|
invoke(
|
|
[unverifiedArtifact, artifact(50)],
|
|
{ 51: unverifiedRun, 50: run(50) },
|
|
{
|
|
tier: "full",
|
|
},
|
|
),
|
|
invoke([unverifiedArtifact], { 51: unverifiedRun }, { tier: "full" }),
|
|
]);
|
|
|
|
expect(JSON.parse(fallback.result.stdout)).toEqual({
|
|
run_id: "50",
|
|
artifact_name: "test-timings-full",
|
|
});
|
|
expect(fallback.output).toBe("run-id=50\nartifact-name=test-timings-full\n");
|
|
expect(JSON.parse(empty.result.stdout)).toEqual({
|
|
run_id: "",
|
|
artifact_name: "",
|
|
});
|
|
expect(empty.output).toBe("run-id=\nartifact-name=\n");
|
|
});
|