diff --git a/docs/development/testing.md b/docs/development/testing.md index 9e81909e08..36cd185071 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -1178,7 +1178,9 @@ Source and built blobs have separate labels to prevent merge collisions. Each test job emits a blob report; the merge job produces JUnit and JSON diagnostics, a module timing report, plus aggregate coverage during full qualification. It retains available reports even when a shard fails, uploads available diagnostics, and writes the -slowest files to the workflow summary. Partial reports are marked diagnostic +slowest files to the workflow summary. When a merged run fails, the summary +and job log also list up to 20 reported failed test files and the count of +unhandled Vitest errors when reported. Partial reports are marked diagnostic only. Failed, cancelled, or missing selected shards still fail the required gate, and aggregate coverage thresholds remain enforced in full qualification. Only a successful complete merge on the default branch publishes shard history, diff --git a/scripts/ci/test-report.mjs b/scripts/ci/test-report.mjs index 8824b8800d..e52a1453ff 100644 --- a/scripts/ci/test-report.mjs +++ b/scripts/ci/test-report.mjs @@ -88,6 +88,28 @@ if (passed && verifyInventory) { // A selected subset is not a timing baseline for the complete suite. const publishesHistory = passed && !selected; const skippedFiles = files.filter(({ state }) => state === "skipped"); +const failedFiles = files + .filter(({ state }) => state === "failed") + .map(({ name }) => name) + .sort(); +const failureListLimit = 20; +const failureDetails = + !passed || failedFiles.length > 0 + ? [ + `Failed test modules reported: ${failedFiles.length}.`, + ...failedFiles + .slice(0, failureListLimit) + .map((name) => `- ${JSON.stringify(name)}`), + ...(failedFiles.length > failureListLimit + ? [ + `… ${failedFiles.length - failureListLimit} more in the test report.`, + ] + : []), + ...(report.unhandledErrors === undefined + ? [] + : [`Unhandled Vitest errors reported: ${report.unhandledErrors}.`]), + ] + : []; const summary = [ groupsCompleted ? "## Merged test diagnostics" @@ -103,6 +125,9 @@ const summary = [ ...(process.env.SOURCE_REQUIRED === "true" ? [`Source result: ${process.env.SOURCE_RESULT ?? "unknown"}.`] : []), + ...(failureDetails.length > 0 + ? ["", "```text", ...failureDetails, "```", ""] + : []), `Skipped test modules: ${skippedFiles.length}. Their skip reasons and required coverage are not established by this report.`, ...(skippedFiles.length ? [""] : []), ...skippedFiles.map(({ name }) => `- ${name.replaceAll("|", "\\|")}`), @@ -124,9 +149,10 @@ const summary = [ ), "", ].join("\n"); -if (process.env.GITHUB_STEP_SUMMARY) +if (process.env.GITHUB_STEP_SUMMARY) { await appendFile(process.env.GITHUB_STEP_SUMMARY, summary); -else console.log(summary); + if (failureDetails.length > 0) console.log(failureDetails.join("\n")); +} else console.log(summary); if (publishesHistory) { await mkdir(historyDirectory, { recursive: true }); diff --git a/tests/boundary/filesystem/ciTestReports.test.ts b/tests/boundary/filesystem/ciTestReports.test.ts index 3d6bdc4337..d5e6335548 100644 --- a/tests/boundary/filesystem/ciTestReports.test.ts +++ b/tests/boundary/filesystem/ciTestReports.test.ts @@ -649,6 +649,14 @@ it("recovers complete module costs from real blob reports without executing test unhandledErrors: 0, files: [{ state: "failed" }], }); + const failedRun = await executeTimingReport(root, { + SHARD_RESULT: "failure", + MERGE_RESULT: "failure", + }); + expect(failedRun.summary).toContain("Failed test modules reported: 1."); + expect(failedRun.summary).toContain('- "sample.test.js"'); + expect(failedRun.stdout).toContain("Failed test modules reported: 1."); + expect(failedRun.stdout).toContain('- "sample.test.js"'); // Even a caller accidentally marking the merge successful cannot publish this report. await executeTimingReport(root); await expect( @@ -656,6 +664,44 @@ it("recovers complete module costs from real blob reports without executing test ).rejects.toMatchObject({ code: "ENOENT" }); }); +it("caps the failed file list and limits partial diagnostics to reported modules", async () => { + const root = await createTestTempDirectory("rea-ci-failed-sample-"); + await writeFile( + join(root, "test-timings.json"), + JSON.stringify({ + version: 1, + success: false, + unhandledErrors: 2, + files: [ + ...Array.from({ length: 21 }, (_, index) => ({ + name: `tests/z${String(20 - index).padStart(2, "0")}.test.ts`, + durationMs: 1, + state: "failed", + })), + { name: "tests/.test.ts", durationMs: 1, state: "failed" }, + { name: "tests/passed.test.ts", durationMs: 1, state: "passed" }, + ], + }), + ); + const { summary, stdout } = await executeTimingReport(root, { + SHARD_RESULT: "skipped", + SOURCE_REQUIRED: "true", + SOURCE_RESULT: "failure", + MERGE_RESULT: "failure", + }); + expect(summary).toContain("Partial test diagnostics"); + expect(summary).toContain( + '```text\nFailed test modules reported: 22.\n- "tests/.test.ts"', + ); + expect(stdout).toContain("Failed test modules reported: 22."); + expect(stdout).toContain('- "tests/.test.ts"'); + expect(stdout).toContain('- "tests/z18.test.ts"'); + expect(stdout).not.toContain("z19.test.ts"); + expect(stdout).not.toContain("passed.test.ts"); + expect(stdout).toContain("2 more in the test report."); + expect(stdout).toContain("Unhandled Vitest errors reported: 2."); +}); + it("verifies exactly the selected subset and never publishes it as timing history", async () => { const root = await createTestTempDirectory("rea-ci-selection-report-"); await workflowGit(root, ["init"]);