From ce5258043c47483d2c8db9a3c0b8f369542bed73 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 21 Nov 2026 08:49:29 -0500 Subject: [PATCH] test(docs): isolate README drift cases and remove duplicate checks --- scripts/lib/docs-facts.d.mts | 2 + scripts/lib/docs-facts.mjs | 22 +++-- .../filesystem/documentationFacts.test.ts | 87 +++++++------------ .../filesystem/productCatalog.test.ts | 1 - tests/conformance/readmeConsistency.test.ts | 37 ++++---- 5 files changed, 65 insertions(+), 84 deletions(-) diff --git a/scripts/lib/docs-facts.d.mts b/scripts/lib/docs-facts.d.mts index c2465c2aaa..83803b3b69 100644 --- a/scripts/lib/docs-facts.d.mts +++ b/scripts/lib/docs-facts.d.mts @@ -15,6 +15,8 @@ export function literalToolCountIssues( root: string, ): Promise; +export function readmeFactIssues(root: string): Promise; + export function documentationFactIssues( root: string, catalog: ProductCatalog, diff --git a/scripts/lib/docs-facts.mjs b/scripts/lib/docs-facts.mjs index 877c1ef661..ef6ff653e0 100644 --- a/scripts/lib/docs-facts.mjs +++ b/scripts/lib/docs-facts.mjs @@ -161,14 +161,9 @@ export const literalToolCountIssues = async (root) => { return issues; }; -/** Return every caller-visible documentation mismatch against canonical facts. */ -export const documentationFactIssues = async (root, catalog) => { - const issues = [ - ...(await skillReferenceIssues( - join(root, "skills/reverse-engineer-anything"), - )), - ...(await literalToolCountIssues(root)), - ]; +/** Check the canonical README links and their translated destinations. */ +export const readmeFactIssues = async (root) => { + const issues = []; const translatedFolder = join(root, "docs/readme"); let translatedNames = []; try { @@ -230,7 +225,18 @@ export const documentationFactIssues = async (root, catalog) => { ) !== JSON.stringify(canonicalLinks) ) issues.push(`${path}: documentation links differ from README.md`); + return issues; +}; +/** Return every caller-visible documentation mismatch against canonical facts. */ +export const documentationFactIssues = async (root, catalog) => { + const issues = [ + ...(await skillReferenceIssues( + join(root, "skills/reverse-engineer-anything"), + )), + ...(await literalToolCountIssues(root)), + ...(await readmeFactIssues(root)), + ]; const installationPath = "docs/getting-started/installation.md"; const installation = await readFile(join(root, installationPath), "utf8"); for (const client of catalog.setup_clients) diff --git a/tests/boundary/filesystem/documentationFacts.test.ts b/tests/boundary/filesystem/documentationFacts.test.ts index 1d6b6887db..df2266d7a7 100644 --- a/tests/boundary/filesystem/documentationFacts.test.ts +++ b/tests/boundary/filesystem/documentationFacts.test.ts @@ -1,4 +1,4 @@ -import { copyFile, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; import { dirname, join, matchesGlob } from "node:path"; import { fileURLToPath } from "node:url"; @@ -8,57 +8,14 @@ import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { LIVING_PROSE_ROOTS, - documentationFactIssues, + readmeFactIssues, isLivingProsePath, literalToolCountIssues, } from "../../../scripts/lib/docs-facts.mjs"; -import type { ProductCatalog } from "../../../scripts/lib/product-catalog.mjs"; const root = fileURLToPath(new URL("../../..", import.meta.url)); const temporaryRoots: string[] = []; -const README_FACT_FIXTURE_PATHS = [ - "README.md", - "docs/readme/README_zh.md", - "docs/readme/README_ja.md", - "docs/readme/README_ko.md", - "docs/readme/README_ar.md", - "website/locales/languages.json", - "docs/getting-started/installation.md", - ".agents/skills/reverse-engineer-anything/references/connection-and-recovery.md", - "AGENTS.md", - ".github/pull_request_template.md", -] as const; - -const README_LINK_DRIFT_MUTATIONS = [ - [ - "(../getting-started/installation.md#supported-agents)", - "(../getting-started/installation.md#missing-agents)", - ], - ["https://rea.tools/zh-CN/guides/", "https://rea.tools/zh-CN/missing/"], - ["https://rea.tools/zh-CN/guides/", "https://rea.tools/zh-TW/guides/"], - [ - "https://rea.tools/zh-CN/guides/", - "https://rea.tools/zh-CN/guides/?other=1", - ], - [ - 'src="../assets/rea-hopper-analysis.png"', - 'src="../assets/missing-hopper.png"', - ], - ['href="https://discord.gg/GkcryMnJDM"', 'href="https://discord.gg/missing"'], -] as const; - -const seedReadmeFactFixture = async (directory: string) => { - for (const path of README_FACT_FIXTURE_PATHS) { - const destination = join(directory, path); - await mkdir(dirname(destination), { recursive: true }); - await copyFile(join(root, path), destination); - } - await mkdir(join(directory, "skills/reverse-engineer-anything"), { - recursive: true, - }); -}; - afterEach(async () => { await Promise.all( temporaryRoots @@ -125,25 +82,41 @@ describe("translated README facts", () => { it("keeps translated readmes linked to canonical setup and tool details", async () => { const directory = await createTestTempDirectory("rea-readme-facts-"); temporaryRoots.push(directory); - await seedReadmeFactFixture(directory); const translatedPath = join(directory, "docs/readme/README_zh.md"); - const translated = await readFile(translatedPath, "utf8"); - const catalog: ProductCatalog = JSON.parse( - await readFile(join(root, "docs/public/product-catalog.json"), "utf8"), + const links = [ + "[Agents](docs/getting-started/installation.md#supported-agents)", + "[MCP](docs/getting-started/installation.md#mcp-registry)", + "[Tools](docs/reference/mcp-contracts.md#generated-catalog)", + "[Guide](https://rea.tools/guides/)", + '', + 'Community', + ].join("\n"); + const translated = links + .replaceAll("docs/", "../") + .replace("rea.tools/guides/", "rea.tools/zh-CN/guides/"); + await mkdir(dirname(translatedPath), { recursive: true }); + await mkdir(join(directory, "website/locales"), { recursive: true }); + await writeFile(join(directory, "README.md"), links); + await writeFile(translatedPath, translated); + await writeFile( + join(directory, "website/locales/languages.json"), + JSON.stringify([{ code: "zh-CN", readme: "docs/readme/README_zh.md" }]), ); - await expect(documentationFactIssues(directory, catalog)).resolves.toEqual( - [], - ); - for (const [original, changed] of README_LINK_DRIFT_MUTATIONS) { - expect(translated).toContain(original); + await expect(readmeFactIssues(directory)).resolves.toEqual([]); + for (const [original, changed] of [ + ["#supported-agents", "#missing-agents"], + ["/zh-CN/guides/", "/zh-CN/missing/"], + ["/zh-CN/guides/", "/zh-TW/guides/"], + ["/zh-CN/guides/", "/zh-CN/guides/?other=1"], + ["figure.svg", "missing.svg"], + ["example.org/community", "example.org/missing"], + ] as const) { await writeFile( translatedPath, translated.replace(original, changed), "utf8", ); - await expect( - documentationFactIssues(directory, catalog), - ).resolves.toEqual([ + await expect(readmeFactIssues(directory)).resolves.toEqual([ "docs/readme/README_zh.md: documentation links differ from README.md", ]); } diff --git a/tests/boundary/filesystem/productCatalog.test.ts b/tests/boundary/filesystem/productCatalog.test.ts index 61a586c3fd..4244b50c05 100644 --- a/tests/boundary/filesystem/productCatalog.test.ts +++ b/tests/boundary/filesystem/productCatalog.test.ts @@ -121,7 +121,6 @@ describe("canonical product catalog", () => { expect(await serializeProductCatalog(catalog)).toBe( await readFile("docs/public/product-catalog.json", "utf8"), ); - await expect(documentationFactIssues(root, catalog)).resolves.toEqual([]); }); }); diff --git a/tests/conformance/readmeConsistency.test.ts b/tests/conformance/readmeConsistency.test.ts index 083d18eddd..9ef5fa56c6 100644 --- a/tests/conformance/readmeConsistency.test.ts +++ b/tests/conformance/readmeConsistency.test.ts @@ -1,19 +1,21 @@ -import { readFile, readdir, writeFile } from "node:fs/promises"; +import { readFile, writeFile } from "node:fs/promises"; import { resolve } from "node:path"; +import MarkdownIt from "markdown-it"; import { describe, expect, it } from "vitest"; import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; import { processTest } from "../support/process/processFixture.js"; -const jsonExamples = (content: string): unknown[] => - [...content.matchAll(/```json\s*([\s\S]*?)```/gu)].map((match): unknown => - JSON.parse(match[1] ?? ""), - ); +const markdown = new MarkdownIt({ html: true }); -const shellExamples = (content: string): string[] => - [...content.matchAll(/```bash\s*([\s\S]*?)```/gu)].map((match) => - (match[1] ?? "").trim(), - ); +const examples = (content: string, languages: readonly string[]): string[] => + markdown + .parse(content, {}) + .flatMap((token) => + token.type === "fence" && languages.includes(token.info.trim()) + ? [token.content.trim()] + : [], + ); describe("onboarding documentation product facts", () => { it("documents manual MCP registration with an absolute runtime and entry script", async () => { @@ -21,7 +23,9 @@ describe("onboarding documentation product facts", () => { resolve("docs/getting-started/installation.md"), "utf8", ); - expect(jsonExamples(content)).toContainEqual( + expect( + examples(content, ["json"]).map((value): unknown => JSON.parse(value)), + ).toContainEqual( expect.objectContaining({ mcpServers: expect.objectContaining({ rea: expect.objectContaining({ @@ -40,16 +44,13 @@ describe("onboarding documentation product facts", () => { const translatedReadmes = inventory .filter((language) => language.code !== "en") .map((language) => language.readme); - expect(translatedReadmes).toContain("docs/readme/README_it.md"); - expect(translatedReadmes).toContain("docs/readme/README_el.md"); - const editions = (await readdir(resolve("docs/readme"))) - .filter((name) => /^README_[A-Za-z0-9-]+\.md$/u.test(name)) - .map((name) => `docs/readme/${name}`); - expect([...translatedReadmes].sort()).toEqual(editions.sort()); - const english = shellExamples(await readFile(resolve("README.md"), "utf8")); + const english = examples(await readFile(resolve("README.md"), "utf8"), [ + "bash", + "sh", + ]); for (const path of translatedReadmes) expect( - shellExamples(await readFile(resolve(path), "utf8")), + examples(await readFile(resolve(path), "utf8"), ["bash", "sh"]), path, ).toEqual(english); });