fix(ci): expose failed test modules in portable diagnostics (#12069)

* fix(ci): expose failed test modules in portable diagnostics

* test(ci): verify failed paths render as literal text

---------

Co-authored-by: ShuaixinHuang <197200901+ShuaixinHuang@users.noreply.github.com>
This commit is contained in:
ShuaixinHuangandShuaixinHuang authored and GitHub committed 2026-11-18 10:54:41 -05:00
1 parent 884fda99f0
commit 644cb8243d
3 files changed
+77 -3

No files matched your search

+3 -1
View File
@@ -1178,7 +1178,9 @@ Source and built blobs have separate labels to prevent merge collisions.
Each test job emits a blob report; the merge job produces JUnit and JSON diagnostics, a module timing
report, plus aggregate coverage during full qualification. It retains available
reports even when a shard fails, uploads available diagnostics, and writes the
slowest files to the workflow summary. Partial reports are marked diagnostic
slowest files to the workflow summary. When a merged run fails, the summary
and job log also list up to 20 reported failed test files and the count of
unhandled Vitest errors when reported. Partial reports are marked diagnostic
only. Failed, cancelled, or missing selected shards still fail the required gate,
and aggregate coverage thresholds remain enforced in full qualification.
Only a successful complete merge on the default branch publishes shard history,
+28 -2
View File
@@ -88,6 +88,28 @@ if (passed && verifyInventory) {
// A selected subset is not a timing baseline for the complete suite.
const publishesHistory = passed && !selected;
const skippedFiles = files.filter(({ state }) => state === "skipped");
const failedFiles = files
.filter(({ state }) => state === "failed")
.map(({ name }) => name)
.sort();
const failureListLimit = 20;
const failureDetails =
!passed || failedFiles.length > 0
? [
`Failed test modules reported: ${failedFiles.length}.`,
...failedFiles
.slice(0, failureListLimit)
.map((name) => `- ${JSON.stringify(name)}`),
...(failedFiles.length > failureListLimit
? [
`… ${failedFiles.length - failureListLimit} more in the test report.`,
]
: []),
...(report.unhandledErrors === undefined
? []
: [`Unhandled Vitest errors reported: ${report.unhandledErrors}.`]),
]
: [];
const summary = [
groupsCompleted
? "## Merged test diagnostics"
@@ -103,6 +125,9 @@ const summary = [
...(process.env.SOURCE_REQUIRED === "true"
? [`Source result: ${process.env.SOURCE_RESULT ?? "unknown"}.`]
: []),
...(failureDetails.length > 0
? ["", "```text", ...failureDetails, "```", ""]
: []),
`Skipped test modules: ${skippedFiles.length}. Their skip reasons and required coverage are not established by this report.`,
...(skippedFiles.length ? [""] : []),
...skippedFiles.map(({ name }) => `- ${name.replaceAll("|", "\\|")}`),
@@ -124,9 +149,10 @@ const summary = [
),
"",
].join("\n");
if (process.env.GITHUB_STEP_SUMMARY)
if (process.env.GITHUB_STEP_SUMMARY) {
await appendFile(process.env.GITHUB_STEP_SUMMARY, summary);
else console.log(summary);
if (failureDetails.length > 0) console.log(failureDetails.join("\n"));
} else console.log(summary);
if (publishesHistory) {
await mkdir(historyDirectory, { recursive: true });
@@ -649,6 +649,14 @@ it("recovers complete module costs from real blob reports without executing test
unhandledErrors: 0,
files: [{ state: "failed" }],
});
const failedRun = await executeTimingReport(root, {
SHARD_RESULT: "failure",
MERGE_RESULT: "failure",
});
expect(failedRun.summary).toContain("Failed test modules reported: 1.");
expect(failedRun.summary).toContain('- "sample.test.js"');
expect(failedRun.stdout).toContain("Failed test modules reported: 1.");
expect(failedRun.stdout).toContain('- "sample.test.js"');
// Even a caller accidentally marking the merge successful cannot publish this report.
await executeTimingReport(root);
await expect(
@@ -656,6 +664,44 @@ it("recovers complete module costs from real blob reports without executing test
).rejects.toMatchObject({ code: "ENOENT" });
});
it("caps the failed file list and limits partial diagnostics to reported modules", async () => {
const root = await createTestTempDirectory("rea-ci-failed-sample-");
await writeFile(
join(root, "test-timings.json"),
JSON.stringify({
version: 1,
success: false,
unhandledErrors: 2,
files: [
...Array.from({ length: 21 }, (_, index) => ({
name: `tests/z${String(20 - index).padStart(2, "0")}.test.ts`,
durationMs: 1,
state: "failed",
})),
{ name: "tests/<tag>.test.ts", durationMs: 1, state: "failed" },
{ name: "tests/passed.test.ts", durationMs: 1, state: "passed" },
],
}),
);
const { summary, stdout } = await executeTimingReport(root, {
SHARD_RESULT: "skipped",
SOURCE_REQUIRED: "true",
SOURCE_RESULT: "failure",
MERGE_RESULT: "failure",
});
expect(summary).toContain("Partial test diagnostics");
expect(summary).toContain(
'```text\nFailed test modules reported: 22.\n- "tests/<tag>.test.ts"',
);
expect(stdout).toContain("Failed test modules reported: 22.");
expect(stdout).toContain('- "tests/<tag>.test.ts"');
expect(stdout).toContain('- "tests/z18.test.ts"');
expect(stdout).not.toContain("z19.test.ts");
expect(stdout).not.toContain("passed.test.ts");
expect(stdout).toContain("2 more in the test report.");
expect(stdout).toContain("Unhandled Vitest errors reported: 2.");
});
it("verifies exactly the selected subset and never publishes it as timing history", async () => {
const root = await createTestTempDirectory("rea-ci-selection-report-");
await workflowGit(root, ["init"]);