From eef8f44003e6bf781baf4f885f84d03b31ea6c3a Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Tue, 3 Nov 2026 09:33:41 +0800 Subject: [PATCH] feat(website): publish agent discovery and Markdown responses --- .github/workflows/website-check.yml | 4 + .github/workflows/website-pages.yml | 44 +++++- scripts/ci/scopes.mjs | 4 +- scripts/prepare-website.py | 3 + scripts/test-website-agents.py | 109 +++++++++++++++ scripts/website.py | 3 +- scripts/website_agents.py | 116 ++++++++++++++++ .../boundary/filesystem/ciChangeScope.test.ts | 9 +- website/README.md | 58 +++++++- website/public/_headers | 11 ++ website/requirements.txt | 3 + website/worker.mjs | 103 ++++++++++++++ website/worker.test.mjs | 127 ++++++++++++++++++ website/wrangler.toml | 5 +- 14 files changed, 591 insertions(+), 8 deletions(-) create mode 100644 scripts/test-website-agents.py create mode 100644 scripts/website_agents.py create mode 100644 website/public/_headers create mode 100644 website/worker.mjs create mode 100644 website/worker.test.mjs diff --git a/.github/workflows/website-check.yml b/.github/workflows/website-check.yml index dc17b4b969..a7106ac7c2 100644 --- a/.github/workflows/website-check.yml +++ b/.github/workflows/website-check.yml @@ -21,6 +21,10 @@ jobs: python-version: "3.12" cache: pip cache-dependency-path: website/requirements.txt + - name: Set up Node.js for the website Worker checks + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version-file: .nvmrc - name: Install sharing-image dependencies run: | sudo apt-get update diff --git a/.github/workflows/website-pages.yml b/.github/workflows/website-pages.yml index 491b028a65..9bfa7b630e 100644 --- a/.github/workflows/website-pages.yml +++ b/.github/workflows/website-pages.yml @@ -54,6 +54,10 @@ jobs: python-version: "3.12" cache: pip cache-dependency-path: website/requirements.txt + - name: Set up Node.js for the website Worker checks + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version-file: .nvmrc - name: Install sharing-image dependencies run: | sudo apt-get update @@ -75,10 +79,21 @@ jobs: PY - name: Configure Pages uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5.0.0 + # upload-pages-artifact excludes dot directories, including .well-known. + - name: Package the website including agent discovery + run: | + tar --dereference --hard-dereference --directory website/_site \ + --exclude=.git --exclude=.github -cf "$RUNNER_TEMP/artifact.tar" . + tar -tf "$RUNNER_TEMP/artifact.tar" \ + ./.well-known/ard.json ./.well-known/ai-catalog.json \ + ./.well-known/agent-skills/index.json > /dev/null - name: Upload the website - uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b # v4.0.0 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - path: website/_site + name: github-pages + path: ${{ runner.temp }}/artifact.tar + retention-days: 1 + if-no-files-found: error deploy: needs: prepare @@ -130,12 +145,13 @@ jobs: PUBLISH_TARGET: ${{ inputs.target }} run: | python3 -u - <<'PY' + import hashlib import json import os import sys import time from urllib.error import URLError - from urllib.parse import urljoin + from urllib.parse import urljoin, urlsplit from urllib.request import Request, urlopen expected = os.environ["GITHUB_SHA"] @@ -172,6 +188,28 @@ jobs: time.sleep(5) if pending: sys.exit("Could not verify published revision on: " + ", ".join(pending)) + for name, base in hosts.items(): + with urlopen(urljoin(base, ".well-known/agent-skills/index.json"), timeout=15) as response: + skill, = json.load(response)["skills"] + # Fetch from each selected host, including the /rea/ mirror. + artifact_url = urljoin(base, urlsplit(skill["url"]).path.lstrip("/")) + with urlopen(artifact_url, timeout=15) as response: + digest = "sha256:" + hashlib.sha256(response.read()).hexdigest() + if digest != skill["digest"]: + sys.exit(name + " serves a skill archive with the wrong digest") + for manifest in ("ard.json", "ai-catalog.json"): + with urlopen(urljoin(base, ".well-known/" + manifest), timeout=15) as response: + if not json.load(response).get("entries"): + sys.exit(name + " serves an empty discovery manifest") + if name == "Cloudflare": + request = Request(base, headers={"Accept": "text/markdown"}) + with urlopen(request, timeout=15) as response: + vary = {value.strip().lower() for value in response.headers.get("Vary", "").split(",")} + if response.headers.get_content_type() != "text/markdown" or "accept" not in vary: + sys.exit("Cloudflare did not negotiate Markdown with Vary: Accept") + if 'rel="ard"' not in response.headers.get("Link", ""): + sys.exit("Cloudflare is missing its discovery Link header") + print(name + " serves the discovery manifests and verified skill archive") with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as summary: links = " and ".join(f"[{name}]({url})" for name, url in hosts.items()) summary.write(f"Published commit `{expected}` to {links}.\n") diff --git a/scripts/ci/scopes.mjs b/scripts/ci/scopes.mjs index 7f5e4d3839..03bcf518d3 100644 --- a/scripts/ci/scopes.mjs +++ b/scripts/ci/scopes.mjs @@ -704,7 +704,9 @@ export const CI_LANES = { agentenv: sameJobs(PATH_SCOPES.agentenv, ["agentenv"]), nativeaot: sameJobs(PATH_SCOPES.nativeaot, ["nativeaot"]), ghidra_bridge: sameJobs(PATH_SCOPES.ghidra_bridge, ["ghidra_bridge"]), - website: sameJobs(PATH_SCOPES.website, ["website"]), + website: sameJobs(PATH_SCOPES.website, ["website"], { + earlyInputs: [".agents/skills/reverse-engineer-anything/**"], + }), }; // An input owner omitted from the registry must never silently lose a lane. for (const [name, paths] of Object.entries(PATH_SCOPES)) diff --git a/scripts/prepare-website.py b/scripts/prepare-website.py index cd284d3e47..33881a25e1 100644 --- a/scripts/prepare-website.py +++ b/scripts/prepare-website.py @@ -7,6 +7,7 @@ import xml.etree.ElementTree as ET from zipfile import ZIP_DEFLATED, ZipFile, ZipInfo from website import PageMetadata, page_url +from website_agents import prepare_discovery, prepare_markdown EXAMPLE_FILES = ( @@ -81,6 +82,8 @@ def prepare_assets(site): prepare_sharing_image(site) prepare_sitemap(site) prepare_notes_example(site) + prepare_discovery(Path(__file__).resolve().parent.parent, site) + prepare_markdown(site) if __name__ == "__main__": diff --git a/scripts/test-website-agents.py b/scripts/test-website-agents.py new file mode 100644 index 0000000000..d5349e1b1a --- /dev/null +++ b/scripts/test-website-agents.py @@ -0,0 +1,109 @@ +"""Check the public build artifacts and Markdown content contract.""" + +import hashlib +import json +from pathlib import Path +import tempfile +import unittest +from zipfile import ZipFile + +from bs4 import BeautifulSoup +import yaml + +from website_agents import prepare_discovery, prepare_markdown + + +ROOT = Path(__file__).resolve().parent.parent + + +class AgentPublishingChecks(unittest.TestCase): + def test_published_skill_digest_covers_every_reference_and_catalogs_resolve(self): + site = ROOT / "website/_site" + index = json.loads((site / ".well-known/agent-skills/index.json").read_text()) + self.assertEqual(index["$schema"], "https://schemas.agentskills.io/discovery/0.2.0/schema.json") + skill, = index["skills"] + self.assertEqual(skill["name"], "reverse-engineer-anything") + self.assertEqual(skill["type"], "archive") + artifact = site / skill["url"].removeprefix("https://rea.tools/") + self.assertEqual(skill["digest"], "sha256:" + hashlib.sha256(artifact.read_bytes()).hexdigest()) + source = ROOT / ".agents/skills/reverse-engineer-anything" + with ZipFile(artifact) as archive: + expected = {f"{source.name}/{p.relative_to(source).as_posix()}": p.read_bytes() + for p in source.rglob("*.md")} + self.assertEqual(set(archive.namelist()), set(expected)) + for name, data in expected.items(): + self.assertEqual(archive.read(name), data) + manifest = archive.read(f"{source.name}/SKILL.md").decode() + metadata = yaml.safe_load(manifest[4:].split("\n---\n", 1)[0]) + self.assertEqual(skill["description"], metadata["description"]) + catalog = json.loads((site / ".well-known/ai-catalog.json").read_text()) + self.assertEqual(catalog["specVersion"], "1.0") + entry, = catalog["entries"] + self.assertEqual(entry["identifier"], "urn:air:rea.tools:skill:reverse-engineer-anything") + self.assertEqual(entry["url"], skill["url"]) + self.assertNotIn("data", entry) + ard = json.loads((site / ".well-known/ard.json").read_text()) + resource, = ard["entries"] + self.assertEqual(resource["url"], skill["url"]) + self.assertTrue(2 <= len(resource["representativeQueries"]) <= 5) + + def test_reference_changes_invalidate_digest_and_yaml_scalars_are_parsed(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / ".agents/skills/reverse-engineer-anything" + source.mkdir(parents=True) + (source / "SKILL.md").write_text( + '---\nname: reverse-engineer-anything\ndescription: >-\n Inspect programs:\n preserve evidence.\n---\n# REA\n' + ) + (source / "references").mkdir() + reference = source / "references/evidence.md" + reference.write_text("First revision") + site = root / "site" + def build(): + prepare_discovery(root, site) + return json.loads((site / ".well-known/agent-skills/index.json").read_text())["skills"][0] + before = build() + self.assertEqual(before["description"], "Inspect programs: preserve evidence.") + self.assertEqual(build(), before) + reference.write_text("Second revision") + self.assertNotEqual(build()["digest"], before["digest"]) + + def test_markdown_preserves_content_and_resources_without_publishing_excluded_pages(self): + with tempfile.TemporaryDirectory() as directory: + site = Path(directory) + page = site / "zh-CN/guides/test/index.html" + page.parent.mkdir(parents=True) + page.write_text('''REA: test +
+

Inspect a program

+
  1. Readinput.exeKeep evidence.
+

Another guide

+ 
const value = a < b ? "x_y" : "&";\n  keepIndent();
+
Evidence

Hidden evidence remains available.

+ Investigation diagram +
''') + fixture = site / "examples/notes-web/index.html" + fixture.parent.mkdir(parents=True) + fixture.write_text("
Target application
") + fallback = site / "zh-CN/fallback/index.html" + fallback.parent.mkdir(parents=True) + fallback.write_text('
English fallback notice.
') + prepare_markdown(site) + markdown = page.with_suffix(".md").read_text() + self.assertIn('# Inspect a program', markdown) + self.assertIn('const value = a < b ? "x_y" : "&";\n keepIndent();', markdown) + self.assertIn("Hidden evidence remains available.", markdown) + self.assertIn("Shortened display paths.", markdown) + self.assertNotIn("Read`input.exe`", markdown) + self.assertIn("https://rea.tools/zh-CN/guides/other/?q=a%2Fb#code", markdown) + self.assertIn("https://rea.tools/assets/diagram.svg", markdown) + for text in ("Outside navigation", "Contents navigation", "Copy code", "doNotPublish"): + self.assertNotIn(text, markdown) + self.assertFalse(fixture.with_suffix(".md").exists()) + self.assertFalse(fallback.with_suffix(".md").exists()) + html = BeautifulSoup(page.read_text(), "html.parser") + self.assertEqual(html.find("link", rel="alternate")["href"], "https://rea.tools/zh-CN/guides/test/index.md") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/website.py b/scripts/website.py index 4891347ff9..b6b3a39f19 100644 --- a/scripts/website.py +++ b/scripts/website.py @@ -327,8 +327,9 @@ def main(): refresh_contributors(root) elif args.command == "check": sync_pages(root, check=True) - for name in ("prepare-website.py", "verify-website.py", "test-website.py"): + for name in ("prepare-website.py", "verify-website.py", "test-website.py", "test-website-agents.py"): run_script(root, name) + subprocess.run(["node", "--test", "website/worker.test.mjs"], cwd=root, check=True) elif args.command == "links": run_script(root, "audit-website-links.py", ["--report", args.report] if args.report else []) else: diff --git a/scripts/website_agents.py b/scripts/website_agents.py new file mode 100644 index 0000000000..74a3e0a5d5 --- /dev/null +++ b/scripts/website_agents.py @@ -0,0 +1,116 @@ +"""Build public agent discovery and Markdown from their maintained sources.""" + +import hashlib +import json +from urllib.parse import urljoin +from zipfile import ZIP_DEFLATED, ZipFile, ZipInfo + +from bs4 import BeautifulSoup +from markdownify import MarkdownConverter +import yaml + +from website import NOTES_FIXTURES, PageMetadata, SITE_ORIGIN, page_url + + +def write_json(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + + +def prepare_discovery(root, site): + """Publish the complete skill, including its relative reference documents.""" + source = root / ".agents/skills/reverse-engineer-anything" + skill = (source / "SKILL.md").read_text(encoding="utf-8") + if not skill.startswith("---\n") or "\n---\n" not in skill[4:]: + raise ValueError("Public REA skill needs YAML frontmatter") + metadata = yaml.safe_load(skill[4:].split("\n---\n", 1)[0]) + if not isinstance(metadata, dict): + raise ValueError("Public REA skill needs YAML metadata") + name, description = metadata["name"], metadata["description"] + if name != source.name or not isinstance(description, str) or not description.strip(): + raise ValueError("Invalid public REA skill metadata") + discovery = site / ".well-known/agent-skills" + discovery.mkdir(parents=True, exist_ok=True) + artifact = discovery / f"{name}.zip" + with ZipFile(artifact, "w") as archive: + for path in [source / "SKILL.md", *sorted(source.glob("references/**/*.md"))]: + relative = path.relative_to(source) + entry = ZipInfo(f"{name}/{relative.as_posix()}", (1980, 1, 1, 0, 0, 0)) + entry.create_system = 3 + entry.external_attr = 0o100644 << 16 + archive.writestr(entry, path.read_bytes(), compress_type=ZIP_DEFLATED) + artifact_url = f"{SITE_ORIGIN}/.well-known/agent-skills/{artifact.name}" + write_json(discovery / "index.json", { + "$schema": "https://schemas.agentskills.io/discovery/0.2.0/schema.json", + "skills": [{ + "name": name, "type": "archive", "description": description, + "url": artifact_url, + "digest": "sha256:" + hashlib.sha256(artifact.read_bytes()).hexdigest(), + }], + }) + entry = { + "identifier": f"urn:air:rea.tools:skill:{name}", + "displayName": "REA reverse engineering skill", + "description": description, + "type": "application/zip", + "url": artifact_url, + } + write_json(site / ".well-known/ai-catalog.json", { + "specVersion": "1.0", + "host": {"displayName": "REA", "documentationUrl": f"{SITE_ORIGIN}/get-started/"}, + "entries": [entry], + }) + write_json(site / ".well-known/ard.json", {"entries": [{ + **entry, + "representativeQueries": [ + "Explain how a feature works in a shipped executable using REA.", + "Trace an Electron application's behavior back to its packaged JavaScript.", + "Build an AgentEnv bundle from a web application capture.", + ], + }]}) + print("Prepared agent skill archive, discovery index, AI Catalog and ARD manifest.") + + +def prepare_markdown(site): + """Convert rendered reading pages, keeping code and resolving resource URLs.""" + converter = MarkdownConverter(heading_style="ATX", bullets="-", escape_underscores=False) + count = 0 + for path in sorted(site.rglob("*.html")): + if path.relative_to(site).as_posix() in NOTES_FIXTURES: + continue + source = path.read_text(encoding="utf-8") + metadata = PageMetadata() + metadata.feed(source) + # Drafts and English fallbacks keep their HTML indexing policy intact. + if metadata.noindex: + continue + soup = BeautifulSoup(source, "html.parser") + main = soup.find("main") + if main is None: + continue + # Preserve fallback notices and closed disclosures; remove interface chrome. + for node in main.select('script, style, nav, button, svg, .home-page-nav, [aria-hidden="true"]'): + if node.parent is not None: + node.decompose() + # These spans are separate lines in the shared flow component's CSS. + for node in main.select("span.flow-label, span.flow-node, span.flow-note"): + node.name = "div" + canonical = page_url(site, path) + for node in main.select("[href], [src]"): + for attribute in ("href", "src"): + if node.has_attr(attribute): + node[attribute] = urljoin(canonical, node[attribute]) + title = metadata.titles[0] if metadata.titles else "REA" + header = yaml.safe_dump({"title": title, "url": canonical}, allow_unicode=True, sort_keys=False) + markdown = converter.convert_soup(main).strip() + path.with_suffix(".md").write_text(f"---\n{header}---\n\n{markdown}\n", encoding="utf-8") + alternate = page_url(site, path.with_suffix(".md")) + links = ( + f'\n' + f'\n' + f'\n' + f'\n' + ) + path.write_text(source.replace("", links + "", 1), encoding="utf-8") + count += 1 + print(f"Prepared Markdown for {count} reading pages.") diff --git a/tests/boundary/filesystem/ciChangeScope.test.ts b/tests/boundary/filesystem/ciChangeScope.test.ts index 351223eb05..e5710daa25 100644 --- a/tests/boundary/filesystem/ciChangeScope.test.ts +++ b/tests/boundary/filesystem/ciChangeScope.test.ts @@ -879,7 +879,14 @@ it.each([ ["install.sh", { code: true, runtime: true, package: true, evm: false }], [ ".agents/skills/reverse-engineer-anything/SKILL.md", - { code: true, runtime: true, docs: true, package: false, evm: false }, + { + code: true, + runtime: true, + docs: true, + website: true, + package: false, + evm: false, + }, ], [ "scripts/verify-real-ghidra.mjs", diff --git a/website/README.md b/website/README.md index 47ea63ca14..d8cd2d5f41 100644 --- a/website/README.md +++ b/website/README.md @@ -486,10 +486,66 @@ The mask SVG is drawn from the seventh decoded mask. It selects zero-based position 21 and compares its character code with 55 (`7`). Keep the figure, solver constants and evidence notes aligned when changing this case. +## Agent discovery and Markdown + +`scripts/website_agents.py` runs during asset preparation, after translations. +It generates these public files in ignored `_site/`: + +- `.well-known/agent-skills/index.json`: the draft v0.2.0 skills discovery index. + Its archive contains the maintained + `.agents/skills/reverse-engineer-anything/SKILL.md` and Markdown references. + The name and description come from YAML frontmatter; SHA-256 covers the + complete, reproducible ZIP, so reference changes invalidate the digest too. +- `.well-known/ard.json`: an ARD manifest with representative analyst queries. +- `.well-known/ai-catalog.json`: an AI Catalog 1.0 document listing that same + downloadable skill. These advertise installation instructions for the local + tool, without claiming a hosted analysis endpoint. +- `index.md` beside each indexable rendered reading page, including translations. + Markdown retains code, evidence disclosures and image descriptions, with + absolute resource links. Drafts, labeled English fallbacks and target example + applications retain their HTML and indexing policy. + +The [skills discovery draft](https://github.com/cloudflare/agent-skills-discovery-rfc), +[ARD proposal](https://agenticresourcediscovery.org/spec/) and +[AI Catalog proposal](https://ai-catalog.io/) are evolving formats. ARD v0.91 +uses `ard.json` and `rel="ard"`; AI Catalog uses its separate `ai-catalog.json` +envelope. Both entries point to the same ZIP with the registered +`application/zip` media type. + +Cloudflare's Worker selects Markdown when an explicit `Accept: text/markdown` +has higher quality than HTML. HTML wins ties and remains the default. Successful +reading-page responses include `Vary: Accept` and an alternate Markdown Link; +Markdown uses `Content-Type: text/markdown; charset=utf-8`. Static `_headers` +advertises discovery resources, sets discovery CORS to `*`, and supplies MIME +types. Build-generated HTML `` elements also advertise these resources. + +GitHub Pages and the Python preview serve the same files and HTML links, with +direct Markdown URLs such as `guides/javascript/index.md`. They do not execute +the Worker or apply Cloudflare's `_headers`: content negotiation, custom MIME +overrides and discovery CORS headers are Cloudflare behavior. The publish +workflow packages `.well-known` explicitly because `upload-pages-artifact` +excludes dot directories by default. + +`python3 scripts/website.py check` includes artifact/content regressions and +Worker request-contract tests. To check the actual local Cloudflare runtime +after preparing assets, use the repository's pinned Node.js and Wrangler: + +```sh +npx wrangler@4.149.0 dev --cwd website --env production --local +curl -i -H 'Accept: text/markdown' http://localhost:8787/guides/javascript/ +curl -I http://localhost:8787/.well-known/agent-skills/index.json +``` + +The `production` environment here skips the preview build and uses the already +verified artifact; `--local` runs on your machine. Check HTML and Markdown, +GET and HEAD, discovery CORS, the downloaded ZIP digest, and unknown routes. +Production remains published only by the manual website workflow. + ## Cloudflare Workers `wrangler.toml` serves `website/_site/` through Workers Static Assets. There is -no Worker script or frontend framework. Before a preview or deployment, Wrangler +one small Worker (`worker.mjs`) for Markdown content negotiation and no frontend +framework. Before a preview or deployment, Wrangler runs the Python scripts to build the language editions, prepare the example ZIP and verify the website. Node.js and Python 3 must be available; the commands below use a pinned Wrangler version without adding it to REA's package dependencies. diff --git a/website/public/_headers b/website/public/_headers new file mode 100644 index 0000000000..2065dd5df3 --- /dev/null +++ b/website/public/_headers @@ -0,0 +1,11 @@ +/* + Link: ; rel="ard", ; rel="ai-catalog", ; rel="related"; type="application/json"; title="Agent skills index" + +/.well-known/* + Access-Control-Allow-Origin: * + +/.well-known/ai-catalog.json + Content-Type: application/ai-catalog+json + +/*.md + Content-Type: text/markdown; charset=utf-8 diff --git a/website/requirements.txt b/website/requirements.txt index e6099d25dc..67e1541cfd 100644 --- a/website/requirements.txt +++ b/website/requirements.txt @@ -2,3 +2,6 @@ CairoSVG==2.9.1 # Standard gettext PO/POT catalogs; no translation service is used by builds. polib==1.2.0 +# Build Markdown reading pages and parse the maintained skill's YAML metadata. +markdownify==1.2.3 +PyYAML==6.0.3 diff --git a/website/worker.mjs b/website/worker.mjs new file mode 100644 index 0000000000..b87bf8acfb --- /dev/null +++ b/website/worker.mjs @@ -0,0 +1,103 @@ +// Negotiate prebuilt Markdown while keeping the static asset service's routing. +function prefersMarkdown(accept) { + const ranges = (accept ?? "") + .toLowerCase() + .split(",") + .map((range) => { + const [type, ...parameters] = range.trim().split(";"); + const quality = parameters + .map((value) => value.trim()) + .find((value) => value.startsWith("q=")); + const q = quality === undefined ? 1 : Number(quality.slice(2)); + return { + type: type.trim(), + q: Number.isFinite(q) && q >= 0 && q <= 1 ? q : 0, + }; + }); + const quality = (type) => { + for (const match of [type, "text/*", "*/*"]) { + const matches = ranges.filter((range) => range.type === match); + if (matches.length) return Math.max(...matches.map((range) => range.q)); + } + return 0; + }; + // Wildcards alone never opt a browser into Markdown; HTML wins ties. + return ( + ranges.some((range) => range.type === "text/markdown") && + quality("text/markdown") > 0 && + quality("text/markdown") > quality("text/html") + ); +} + +function advertiseMarkdown(response, path) { + const vary = new Set( + (response.headers.get("Vary") ?? "") + .split(",") + .map((s) => s.trim()) + .filter(Boolean), + ); + if (![...vary].some((s) => s.toLowerCase() === "accept" || s === "*")) + vary.add("Accept"); + response.headers.set("Vary", [...vary].join(", ")); + response.headers.append( + "Link", + `<${path}>; rel="alternate"; type="text/markdown"`, + ); + return response; +} + +export default { + async fetch(request, env) { + const useMarkdown = prefersMarkdown(request.headers.get("Accept")); + // HTML validators describe a different representation. Probe the route first, + // then let the asset service evaluate validators/ranges against the Markdown. + const probe = new Request(request); + if (useMarkdown) { + for (const name of [ + "If-None-Match", + "If-Modified-Since", + "If-Match", + "If-Unmodified-Since", + "If-Range", + "Range", + ]) + probe.headers.delete(name); + } + const original = await env.ASSETS.fetch(probe); + if ( + ![200, 206, 304].includes(original.status) || + !["GET", "HEAD"].includes(request.method) || + !original.headers.get("Content-Type")?.startsWith("text/html") + ) + return original; + + const url = new URL(request.url); + if (!url.pathname.endsWith("/") && !url.pathname.endsWith(".html")) + return original; + const markdown = new URL(url); + markdown.pathname = url.pathname.endsWith("/") + ? `${url.pathname}index.md` + : url.pathname.slice(0, -5) + ".md"; + const candidate = await env.ASSETS.fetch( + useMarkdown + ? new Request(markdown, request) + : new Request(markdown, { method: "HEAD" }), + ); + if (!candidate.ok && ![304, 412, 416].includes(candidate.status)) { + await candidate.body?.cancel(); + return original; + } + const response = new Response( + useMarkdown ? candidate.body : original.body, + useMarkdown ? candidate : original, + ); + if (useMarkdown) { + await original.body?.cancel(); + if (candidate.status < 400) + response.headers.set("Content-Type", "text/markdown; charset=utf-8"); + } else { + await candidate.body?.cancel(); + } + return advertiseMarkdown(response, markdown.pathname); + }, +}; diff --git a/website/worker.test.mjs b/website/worker.test.mjs new file mode 100644 index 0000000000..490e8205a9 --- /dev/null +++ b/website/worker.test.mjs @@ -0,0 +1,127 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import worker from "./worker.mjs"; + +function assets({ markdownStatus = 200 } = {}) { + return { + ASSETS: { + async fetch(request) { + const url = new URL(request.url); + const markdown = url.pathname.endsWith("index.md"); + if (url.pathname === "/missing/") + return new Response("Missing", { status: 404 }); + if (url.pathname === "/guide") + return Response.redirect(`${url.origin}/guide/`, 307); + const headers = { + "Content-Type": markdown ? "text/plain" : "text/html", + ETag: markdown ? '"markdown"' : '"html"', + Link: '; rel="ard"', + }; + const status = + request.headers.get("If-None-Match") === headers.ETag + ? 304 + : markdown + ? markdownStatus + : 200; + return new Response( + request.method === "HEAD" || status === 304 + ? null + : markdown + ? "# Guide" + : "

Guide

", + { headers, status }, + ); + }, + }, + }; +} + +test("HTML is the default and explicit Accept preferences select Markdown", async () => { + for (const [accept, markdown] of [ + [null, false], + ["*/*", false], + ["text/html", false], + ["text/markdown", true], + ["text/markdown;q=0", false], + ["text/markdown;q=0.5,text/html", false], + ["text/markdown,text/html;q=0.5", true], + ["text/markdown,text/html", false], + ["TEXT/MARKDOWN; q=1,text/*;q=0.5", true], + ["text/markdown;q=bad", false], + ["text/markdown;q=0,*/*;q=1", false], + ]) { + const response = await worker.fetch( + new Request("https://rea.tools/guide/", { + headers: accept === null ? {} : { Accept: accept }, + }), + assets(), + ); + assert.equal( + response.headers.get("Content-Type"), + markdown ? "text/markdown; charset=utf-8" : "text/html", + accept, + ); + assert.equal( + await response.text(), + markdown ? "# Guide" : "

Guide

", + accept, + ); + assert.match(response.headers.get("Vary"), /Accept/); + assert.match(response.headers.get("Link"), /rel="ard"/); + assert.match( + response.headers.get("Link"), + /<\/guide\/index.md>; rel="alternate"/, + ); + } +}); + +test("HEAD and validators use the selected representation", async () => { + for (const method of ["GET", "HEAD"]) { + const response = await worker.fetch( + new Request("https://rea.tools/guide/", { + method, + headers: { Accept: "text/markdown", "If-None-Match": '"html"' }, + }), + assets(), + ); + assert.equal(response.status, 200); + assert.equal(response.headers.get("ETag"), '"markdown"'); + assert.equal(await response.text(), method === "HEAD" ? "" : "# Guide"); + } + for (const [accept, tag] of [ + ["text/html", '"html"'], + ["text/markdown", '"markdown"'], + ]) { + const response = await worker.fetch( + new Request("https://rea.tools/guide/", { + headers: { Accept: accept, "If-None-Match": tag }, + }), + assets(), + ); + assert.equal(response.status, 304); + assert.equal(await response.text(), ""); + assert.match(response.headers.get("Vary"), /Accept/); + } +}); + +test("missing Markdown keeps HTML, and asset redirects and missing routes survive", async () => { + const html = await worker.fetch( + new Request("https://rea.tools/guide/", { + headers: { Accept: "text/markdown" }, + }), + assets({ markdownStatus: 404 }), + ); + assert.equal(await html.text(), "

Guide

"); + for (const [path, status] of [ + ["/guide", 307], + ["/missing/", 404], + ]) { + const response = await worker.fetch( + new Request(`https://rea.tools${path}`, { + headers: { Accept: "text/markdown" }, + }), + assets(), + ); + assert.equal(response.status, status); + } +}); diff --git a/website/wrangler.toml b/website/wrangler.toml index cb3f21b386..7c64886b30 100644 --- a/website/wrangler.toml +++ b/website/wrangler.toml @@ -1,18 +1,21 @@ # Serve only the public website. The default deployment is a workers.dev preview. name = "rea-website-preview" +main = "worker.mjs" compatibility_date = "2026-10-08" workers_dev = true preview_urls = false [assets] directory = "./_site" +binding = "ASSETS" +run_worker_first = ["/", "/*/", "/*.html"] html_handling = "auto-trailing-slash" not_found_handling = "none" [build] cwd = ".." command = "python3 scripts/website.py check" -watch_dir = ["./public", "./locales", "./templates"] +watch_dir = ["./public", "./locales", "./templates", "../.agents/skills/reverse-engineer-anything"] # Public HTTPS depends on zone activation and certificate readiness. [env.production]