"""Exercise automatic route discovery and mistakes that break search metadata."""
import contextlib
import errno
from html import unescape
import importlib.util
import io
import json
from pathlib import Path
import shutil
import subprocess
import sys
import tempfile
from textwrap import dedent
from threading import Thread
import unittest
from unittest.mock import patch
from http.server import ThreadingHTTPServer
from urllib.error import HTTPError
from urllib.parse import urljoin, urlsplit
from urllib.request import build_opener, ProxyHandler, urlopen
import xml.etree.ElementTree as ET
import polib
from website_i18n import Document, build_translations, checked_catalog, extract_catalogs, languages, prose_units, reading_pages, read_runtime
ROOT = Path(__file__).resolve().parent.parent
def load_script(name):
"""Load a repository script without executing its command-line entrypoint."""
spec = importlib.util.spec_from_file_location(name, ROOT / "scripts" / f"{name}.py")
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
PREPARE = load_script("prepare-website")
VERIFY = load_script("verify-website")
WEBSITE = load_script("website")
AUDIT = load_script("audit-website-links")
class WebsiteLinkAuditChecks(unittest.TestCase):
"""Check GitHub transport and URL classification without live network access."""
def test_github_json_is_decoded_as_utf8_under_a_legacy_locale(self):
# Exercise real subprocess pipe decoding with UTF-8 mode disabled. Only
# the external gh executable and the default locale are substituted.
program = dedent('''\
import importlib.util
import json
from pathlib import Path
import subprocess
import sys
from unittest.mock import patch
scripts = Path(sys.argv[1])
sys.path.insert(0, str(scripts))
spec = importlib.util.spec_from_file_location("audit", scripts / "audit-website-links.py")
audit = importlib.util.module_from_spec(spec)
spec.loader.exec_module(audit)
payload = json.dumps({"title": "Release \\u2603"}, ensure_ascii=False).encode("utf-8")
run = subprocess.run
def gh(command, **options):
assert command == ["gh", "api", "repos/example/project"]
return run([sys.executable, "-c", "import sys; sys.stdout.buffer.write(" + repr(payload) + ")"], **options)
with patch("locale.getencoding", return_value="cp936"), patch.object(audit.subprocess, "run", side_effect=gh):
print(json.dumps(audit.github_api("repos/example/project")))
''')
result = subprocess.run(
[sys.executable, "-X", "utf8=0", "-c", program, str(ROOT / "scripts")],
capture_output=True, encoding="utf-8", timeout=20,
)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(json.loads(result.stdout), {"title": "Release \u2603"})
def test_raw_and_browser_links_resolve_the_same_ref_and_encoded_path(self):
# Prefix lookalikes must not win, and tags use the same ref/path split.
references = {
"repos/example/project/git/matching-refs/heads/topic": [
{"ref": "refs/heads/topic"},
{"ref": "refs/heads/topic/nested"},
{"ref": "refs/heads/topic/nestedness"},
],
"repos/example/project/git/matching-refs/tags/topic": [
{"ref": "refs/tags/topic/nested/v1"},
],
"repos/example/project/git/matching-refs/heads/topic%2Fnested": [
{"ref": "refs/heads/topic/nested"},
{"ref": "refs/heads/topic/nestedness"},
],
"repos/example/project/git/matching-refs/tags/topic%2Fnested": [
{"ref": "refs/tags/topic/nested/v1"},
],
}
cases = (
("topic/nested/guide%20%23%3F.md", "guide%20%23%3F.md", "topic%2Fnested"),
("topic/nested/v1/README.md", "README.md", "topic%2Fnested%2Fv1"),
("topic%2Fnested/README.md", "README.md", "topic%2Fnested"),
("refs/heads/topic/nested/v1/README.md", "v1/README.md", "refs%2Fheads%2Ftopic%2Fnested"),
("refs/tags/topic/nested/v1/README.md", "README.md", "refs%2Ftags%2Ftopic%2Fnested%2Fv1"),
)
for suffix, path, ref in cases:
endpoint = f"repos/example/project/contents/{path}?ref={ref}"
responses = {**references, endpoint: {"name": "README.md"}}
def github_api(target):
if target not in responses:
raise ValueError("gh: Not Found (HTTP 404)")
return responses[target]
for base in (
"https://raw.githubusercontent.com/example/project/",
"https://github.com/example/project/blob/",
"https://github.com/example/project/tree/",
):
with self.subTest(url=base + suffix), patch.object(AUDIT, "github_api", side_effect=github_api):
report = AUDIT.audit_url((base + suffix, ["fixture.html"]))
self.assertEqual(report["status"], "ok")
self.assertEqual(report["endpoint"], endpoint)
def test_repeated_link_checks_do_not_mutate_cached_github_refs(self):
heads = [{"ref": "refs/heads/topic"}]
tags = [{"ref": "refs/tags/topic/nested"}]
ref_endpoint = "repos/example/project/git/matching-refs/"
responses = {
ref_endpoint + "heads/topic": heads,
ref_endpoint + "tags/topic": tags,
}
with patch.object(AUDIT, "github_api", side_effect=responses.__getitem__):
for file in ("README.md", "guide.md", "examples/a.md"):
url = urlsplit(f"https://github.com/example/project/blob/topic/nested/{file}")
self.assertEqual(
AUDIT.github_target(url).endpoint,
f"repos/example/project/contents/{file}?ref=topic%2Fnested",
)
self.assertEqual(heads, [{"ref": "refs/heads/topic"}])
self.assertEqual(tags, [{"ref": "refs/tags/topic/nested"}])
def test_plain_refs_and_pinned_commits_keep_their_contents_target(self):
for ref in ("main", "a" * 40):
responses = {
"repos/example/project/git/matching-refs/heads/main": [{"ref": "refs/heads/main"}],
"repos/example/project/git/matching-refs/tags/main": [],
f"repos/example/project/contents/README.md?ref={ref}": {"name": "README.md"},
}
for base in ("https://raw.githubusercontent.com/example/project/", "https://github.com/example/project/blob/"):
with self.subTest(ref=ref, base=base), patch.object(AUDIT, "github_api", side_effect=responses.__getitem__):
report = AUDIT.audit_url((base + ref + "/README.md", ["fixture.html"]))
self.assertEqual(report["status"], "ok")
self.assertEqual(report["endpoint"], f"repos/example/project/contents/README.md?ref={ref}")
def test_raw_github_urls_distinguish_repository_files_from_other_routes(self):
ref = "a" * 40
responses = {
f"repos/example/project/contents/scripts?ref={ref}": [{"name": "tool.py", "type": "file"}],
f"repos/example/project/contents/vendor/library?ref={ref}": {
"type": "file", "submodule_git_url": "https://github.com/example/library.git",
},
"repos/example/project/git/matching-refs/heads/releases": [{"ref": "refs/heads/releases"}],
"repos/example/project/git/matching-refs/tags/releases": [],
"repos/example/project/contents/download/packages/tool.zip?ref=releases": {"type": "file"},
"repos/example/project/releases/tags/v1": {"assets": [{"name": "tool.zip"}]},
}
cases = (
(f"https://raw.githubusercontent.com/example/project/{ref}/scripts", "broken"),
(f"https://github.com/example/project/tree/{ref}/scripts", "ok"),
(f"https://raw.githubusercontent.com/example/project/{ref}/vendor/library", "broken"),
(f"https://github.com/example/project/tree/{ref}/vendor/library", "ok"),
("https://raw.githubusercontent.com/example/project/releases/download/packages/tool.zip", "ok"),
("https://github.com/example/project/releases/download/v1/tool.zip", "ok"),
("https://github.com/example/project/releases/download/v1/missing.zip", "broken"),
("https://raw.githubusercontent.com/example/project", "broken"),
)
with patch.object(AUDIT, "github_api", side_effect=responses.__getitem__):
for url, expected in cases:
with self.subTest(url=url):
self.assertEqual(AUDIT.audit_url((url, ["fixture.html"]))["status"], expected)
def test_malformed_external_links_do_not_hide_other_audit_results(self):
invalid = (
"https://[::1/guide",
"https://github.com:bad/example/project",
"https:///no-host",
"http://bad host.invalid/guide",
"http://example.invalid/not escaped",
)
valid = "https://github.com/example/project"
html = "website/public/index.html"
markdown = "website/guide.md"
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
page = root / html
page.parent.mkdir(parents=True)
page.write_text("".join(f'Link ' for url in (*invalid, valid, "https://rea.tools/")))
(root / markdown).write_text(f"[Guide]({invalid[0]})")
opener = build_opener(ProxyHandler({}))
with (
patch.object(AUDIT, "github_api", return_value={"name": "project"}) as probe,
patch.object(AUDIT, "urlopen", opener.open),
):
reports = {
report["url"]: report
for item in AUDIT.external_links(root).items()
if (report := AUDIT.audit_url(item)) is not None
}
probe.assert_called_once_with("repos/example/project")
self.assertEqual(set(reports), {*invalid, valid})
self.assertEqual(reports[valid]["status"], "ok")
for index, url in enumerate(invalid):
with self.subTest(url=url):
self.assertEqual(reports[url]["status"], "broken")
self.assertEqual(reports[url]["sources"], sorted([html, markdown] if index == 0 else [html]))
self.assertIn("Invalid URL", reports[url]["reason"])
def test_markdown_destinations_reach_both_link_checks(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
(root / "website").mkdir()
(root / "docs").mkdir()
(root / "docs/releases(v1).md").write_text("# Release notes\n")
source = root / "website/guide.md"
source.write_text(dedent('''\
[Release](../docs/releases(v1).md#release-notes)
[Upstream][ref]
[Malformed](https:/missing-authority)
[IPv6](https://[::1/guide)
`[Code example](https://example.invalid/ignore-inline)`
```md
[Code example](../missing.md)
[Code example](https://example.invalid/ignore-fenced)
```
[ref]: HTTPS://example.invalid/release(v1)
'''))
self.assertCountEqual(AUDIT.external_links(root), [
"HTTPS://example.invalid/release(v1)", "https:/missing-authority",
"https://[::1/guide", "https://example.invalid/docs(1)",
"HTTPS://example.invalid/figure(1).png",
])
errors = VERIFY.check_repository_links(root)
self.assertEqual(len(errors), 2)
for malformed in ("https:/missing-authority", "https://[::1/guide"):
self.assertTrue(any(f"website/guide.md: invalid URL '{malformed}'" in error for error in errors))
def test_missing_contents_and_unavailable_github_probes_remain_distinct(self):
cases = (
(ValueError("gh: Not Found (HTTP 404)"), "broken"),
(ValueError("gh: Forbidden (HTTP 403)"), "unknown"),
(subprocess.TimeoutExpired(["gh", "api"], 20), "unknown"),
)
for failure, status in cases:
for base in ("https://raw.githubusercontent.com/example/project/", "https://github.com/example/project/blob/"):
with self.subTest(failure=str(failure), base=base), patch.object(AUDIT, "github_api", side_effect=failure):
report = AUDIT.audit_url((base + "a" * 40 + "/missing.md", ["fixture.html"]))
self.assertEqual(report["status"], status)
self.assertEqual(report["reason"], str(failure))
class WebsiteSearchChecks(unittest.TestCase):
"""Use authored pages and temporary fixtures to check the publishing boundary."""
def test_sitemap_tracks_added_moved_removed_and_nonindexable_pages(self):
with tempfile.TemporaryDirectory() as directory:
site = Path(directory)
def page(name, head="", body=""):
path = site / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(f"{head}{body}")
return path
page("index.html")
nested = page("guides/new/index.html")
page(
"examples/fixture/index.html",
' ',
)
page("draft.html", ' ')
page("code & notes.html")
page("visible/index.html", body=' ')
(site / "asset.json").write_text("{}")
def urls():
with contextlib.redirect_stdout(io.StringIO()):
PREPARE.prepare_sitemap(site)
sitemap = ET.parse(site / "sitemap.xml")
return {node.text for node in sitemap.findall(".//{*}loc")}
self.assertEqual(
urls(),
{
"https://rea.tools/",
"https://rea.tools/guides/new/",
"https://rea.tools/code%20%26%20notes.html",
"https://rea.tools/visible/",
},
)
nested.rename(site / "moved.html")
self.assertIn("https://rea.tools/moved.html", urls())
self.assertNotIn("https://rea.tools/guides/new/", urls())
(site / "moved.html").unlink()
self.assertNotIn("https://rea.tools/moved.html", urls())
def test_verifier_rejects_wrong_canonicals_stale_sitemaps_and_missing_previews(
self,
):
cases = (
(
"index.html",
'rel="canonical" href="https://rea.tools/"',
'rel="canonical" href="https://morluto.github.io/rea/"',
"head canonical",
),
(
"faq/index.html",
'hreflang="zh-CN" href="https://rea.tools/zh-CN/faq/"',
'hreflang="zh-CN" href="https://[::1/faq/"',
"invalid URL",
),
(
"showcase/aegis/index.html",
'rel="canonical" href="https://rea.tools/showcase/aegis/"',
'rel="canonical" href="https://rea.tools/"',
"head canonical",
),
(
"examples/notes-electron/index.html",
'content="noindex"',
'content="index"',
"must have a head noindex",
),
(
"sitemap.xml",
"https://rea.tools/faq/",
"https://rea.tools/deleted/",
"each indexable page once",
),
(
"faq/index.html",
'name="twitter:card" content="summary_large_image"',
'name="twitter:card" content="summary"',
"twitter:card",
),
)
for name, old, new, message in cases:
with self.subTest(name=name), tempfile.TemporaryDirectory() as directory:
site = Path(directory) / "public"
shutil.copytree(ROOT / "website/_site", site)
path = site / name
source = path.read_text()
self.assertIn(old, source)
path.write_text(source.replace(old, new, 1))
self.assertTrue(
any(message in error for error in VERIFY.check_site(site)[3])
)
with tempfile.TemporaryDirectory() as directory:
site = Path(directory) / "public"
shutil.copytree(ROOT / "website/_site", site)
(site / "assets/social-card.png").unlink()
self.assertIn(
"Social preview PNG is missing; run scripts/prepare-website.py",
VERIFY.check_site(site)[3],
)
class WebsiteMaintenanceChecks(unittest.TestCase):
"""Exercise contributor edits and the static preview through real consumers."""
def test_electron_guide_renderer_matches_download(self):
public = ROOT / "website/public"
source = (public / "guides/javascript/index.html").read_text(encoding="utf-8")
document = Document(source)
blocks = [node for node in document.nodes if node.attrs.get("id") == "notes-renderer-source"]
self.assertEqual(len(blocks), 1)
self.assertEqual(blocks[0].tag, "code")
displayed = unescape(source[blocks[0].open_end:blocks[0].end_start]) + "\n"
self.assertEqual(displayed, (public / "examples/notes-electron/renderer.js").read_text(encoding="utf-8"))
def test_contributors_render_from_snapshot_and_reject_drift_without_rewriting(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
for name in ("public", "templates"):
shutil.copytree(ROOT / "website" / name, root / "website" / name)
page = root / "website/public/contributors/index.html"
snapshot = root / "website/public/contributors/people.json"
data = json.loads(snapshot.read_text())
people = data["contributors"][:3]
data["contributors"] = list(reversed(people))
data["updated"] = "2026-10-15"
snapshot.write_text(json.dumps(data))
original = page.read_text()
with self.assertRaisesRegex(ValueError, "Shared HTML is stale"):
WEBSITE.sync_pages(root, check=True)
self.assertEqual(page.read_text(), original)
WEBSITE.sync_pages(root)
source = page.read_text()
doc = Document(source)
cards = [node for node in doc.nodes if "data-contributor" in node.attrs]
self.assertEqual([node.attrs["data-contributor"] for node in cards], sorted(
(person["login"] for person in people), key=str.casefold,
))
links = [node.attrs["href"] for node in doc.nodes if node.attrs.get("class") == "contributor-card"]
self.assertCountEqual(links, [person["html_url"] for person in people])
self.assertIn('2026-10-15 ', source)
self.assertEqual(WEBSITE.sync_pages(root), [])
translated_units = [unit[1] for unit in prose_units(source, "contributors/index.html")]
for person in people:
self.assertFalse(any(person["login"] in unit for unit in translated_units))
def test_contributor_refresh_collects_all_pages_and_preserves_snapshot_on_failure(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
snapshot = root / "website/public/contributors/people.json"
snapshot.parent.mkdir(parents=True)
original = (ROOT / "website/public/contributors/people.json").read_text()
snapshot.write_text(original)
people = json.loads(original)["contributors"][:3]
response = WEBSITE.subprocess.CompletedProcess([], 0, stdout=json.dumps(people[:2]) + "\n" + json.dumps(people[2:]))
with patch.object(WEBSITE.subprocess, "run", return_value=response) as api:
WEBSITE.refresh_contributors(root)
self.assertIn("--paginate", api.call_args.args[0])
self.assertEqual(json.loads(snapshot.read_text())["contributors"], people)
self.assertEqual(set(snapshot.parent.iterdir()), {snapshot})
good = snapshot.read_text()
for output in (json.dumps(people) + "\n[", json.dumps(people + people), "[]", '{"message":"API error"}'):
with self.subTest(output=output), patch.object(WEBSITE.subprocess, "run", return_value=WEBSITE.subprocess.CompletedProcess([], 0, stdout=output)):
with self.assertRaises(ValueError):
WEBSITE.refresh_contributors(root)
self.assertEqual(snapshot.read_text(), good)
with patch.object(WEBSITE.subprocess, "run", side_effect=WEBSITE.subprocess.CalledProcessError(1, ["gh"])):
with self.assertRaises(WEBSITE.subprocess.CalledProcessError):
WEBSITE.refresh_contributors(root)
self.assertEqual(snapshot.read_text(), good)
def test_contributor_refresh_preserves_snapshot_and_cleans_staging_on_save_failure(self):
people = json.loads((ROOT / "website/public/contributors/people.json").read_text())["contributors"][:2]
response = WEBSITE.subprocess.CompletedProcess([], 0, stdout=json.dumps(people))
write_text = Path.write_text
def partial_write(path, text, *args, **kwargs):
write_text(path, text[:20], *args, **kwargs)
raise OSError(errno.ENOSPC, "injected partial write", str(path))
failures = (
("partial write", patch.object(
Path, "write_text", autospec=True, side_effect=partial_write,
)),
("replace", patch.object(
Path, "replace", autospec=True,
side_effect=PermissionError(errno.EACCES, "injected replace failure"),
)),
)
for name, failure in failures:
with self.subTest(failure=name), tempfile.TemporaryDirectory() as directory:
root = Path(directory)
snapshot = root / "website/public/contributors/people.json"
snapshot.parent.mkdir(parents=True)
snapshot.write_bytes(json.dumps({
"repository": "morluto/rea", "updated": "2026-10-14", "contributors": people[:1],
}).encode("utf-8"))
before = snapshot.read_bytes()
with patch.object(WEBSITE.subprocess, "run", return_value=response), failure as save:
with self.assertRaises(OSError):
WEBSITE.refresh_contributors(root)
self.assertEqual(save.call_count, 1)
self.assertEqual(snapshot.read_bytes(), before)
self.assertEqual(set(snapshot.parent.iterdir()), {snapshot})
def test_contributor_snapshot_rejects_malformed_identity_and_unsafe_links(self):
original = json.loads((ROOT / "website/public/contributors/people.json").read_text())
for field, value in (("id", True), ("id", 0), ("login", ""), ("type", "unknown"),
("html_url", "javascript:alert(1)"), ("html_url", "https://github.com.evil.test/person"),
("avatar_url", "https://token@avatars.githubusercontent.com/u/1")):
with self.subTest(field=field, value=value):
data = {**original, "contributors": [{**original["contributors"][0], field: value}]}
with self.assertRaises(ValueError):
WEBSITE.checked_contributors(data)
def test_shared_edits_preserve_body_and_catch_drift_before_rewriting(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
for name in ("public", "templates"):
shutil.copytree(ROOT / "website" / name, root / "website" / name)
page = root / "website/public/blog/touhou-reconstruction/index.html"
original = page.read_text()
body = original.split('", 1)[0]
template = root / "website/templates/header.html.tmpl"
template.write_text(template.read_text().replace("Reverse Engineer Anything", "REA test navigation"))
with self.assertRaisesRegex(ValueError, "Shared HTML is stale"):
WEBSITE.sync_pages(root, check=True)
self.assertEqual(page.read_text(), original)
WEBSITE.sync_pages(root)
updated = page.read_text()
self.assertIn("REA test navigation", updated)
self.assertEqual(updated.split('", 1)[0], body)
self.assertEqual(WEBSITE.sync_pages(root), [])
page.write_text(updated.replace(' ', '', 1))
with self.assertRaisesRegex(ValueError, "needs one header region"):
WEBSITE.sync_pages(root)
def test_new_page_escapes_metadata_stays_a_draft_and_joins_sitemap_when_ready(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
for name in ("public", "templates"):
shutil.copytree(ROOT / "website" / name, root / "website" / name)
site = root / "website/public"
WEBSITE.sync_pages(root)
shutil.copyfile(ROOT / "website/_site/assets/social-card.png", site / "assets/social-card.png")
PREPARE.prepare_notes_example(site)
page = WEBSITE.new_page(root, "blog/new-example", 'A & "result"', 'Check "quotes" & .')
source = page.read_text()
self.assertIn('href="../../blog/" aria-current="page"', source)
self.assertIn('href="../../assets/styles.css"', source)
self.assertIn("A <test> & "result"", source)
with self.assertRaisesRegex(ValueError, "Page already exists"):
WEBSITE.new_page(root, "blog/new-example", "Another page", "Description")
with self.assertRaises(ValueError):
WEBSITE.new_page(root, "../outside", "Title", "Description")
WEBSITE.sync_pages(root, check=True)
PREPARE.prepare_sitemap(site)
self.assertNotIn("https://rea.tools/blog/new-example/", (site / "sitemap.xml").read_text())
page.write_text(source.replace(' \n', ''))
PREPARE.prepare_sitemap(site)
self.assertIn("https://rea.tools/blog/new-example/", (site / "sitemap.xml").read_text())
self.assertEqual(VERIFY.check_site(site)[3], [])
def test_links_in_prompts_fragments_and_copy_controls_are_verified(self):
with tempfile.TemporaryDirectory() as directory:
site = Path(directory) / "public"
shutil.copytree(ROOT / "website/_site", site)
page = site / "faq/index.html"
original = page.read_text()
cases = (
('Broken ', "missing local target"),
('https://rea.tools/guides/native/#deleted', "missing HTML fragment"),
('https://morluto.github.io/rea/examples/notes-web/', "obsolete website URL"),
('Copy ', "missing copy target"),
('Malformed ', "invalid URL"),
)
for inserted, message in cases:
with self.subTest(message=message):
page.write_text(original.replace(" ", inserted + " ", 1))
self.assertTrue(any(message in error for error in VERIFY.check_site(site)[3]))
def test_repository_links_preserve_targets_and_containment_through_aliased_roots(self):
with tempfile.TemporaryDirectory() as directory:
container = Path(directory)
root = container / "repository"
(root / "website/public").mkdir(parents=True)
(root / "docs").mkdir()
(root / "docs/guide.md").write_text("# Guide\n## Set up REA\n")
(container / "outside.md").write_text("# Outside\n")
page = root / "website/README.md"
page.write_text("[Guide](../docs/guide.md#set-up-rea)\n")
(root / "website/.venv").mkdir()
(root / "website/.venv/vendor.md").write_text("[File](missing.md)")
self.assertEqual(VERIFY.check_repository_links(root), [])
alias = container / "repository-alias"
alias.symlink_to(root.resolve(), target_is_directory=True)
self.assertEqual(VERIFY.check_repository_links(alias), [])
page.write_text(
"[Guide](../docs/guide.md#removed)\n"
"[File](../docs/removed.md)\n"
"[Escape](../../outside.md)\n"
)
for boundary in (root, alias):
with self.subTest(boundary=boundary):
self.assertCountEqual(VERIFY.check_repository_links(boundary), [
"website/README.md: missing Markdown fragment: ../docs/guide.md#removed",
"website/README.md: missing repository target: ../docs/removed.md",
"website/README.md: missing repository target: ../../outside.md",
])
def test_site_links_preserve_containment_through_aliased_roots(self):
with tempfile.TemporaryDirectory() as directory:
site = (Path(directory) / "site").resolve()
site.mkdir()
page = site / "index.html"
page.write_text('Fixture Fixture Guide ')
(site / "guide.html").write_text('Guide Answer ')
(site / "sitemap.xml").write_text(' ')
(site / "robots.txt").write_text("User-agent: *\nAllow: /\nSitemap: https://rea.tools/sitemap.xml\n")
alias = Path(directory) / "site-alias"
alias.symlink_to(site, target_is_directory=True)
# The tiny link fixture intentionally omits the social preview.
expected = ["Social preview PNG is missing; run scripts/prepare-website.py"]
self.assertEqual(VERIFY.check_site(site)[3], expected)
self.assertEqual(VERIFY.check_site(alias)[3], expected)
(site.parent / "outside.html").write_text("Outside
")
content = page.read_text()
for reference in ("../outside.html", "https://rea.tools/../outside.html"):
page.write_text(content.replace("guide.html#answer", reference))
for boundary in (site, alias):
with self.subTest(boundary=boundary, reference=reference):
self.assertIn(f"index.html: reference leaves the website: {reference}", VERIFY.check_site(boundary)[3])
def test_preview_preserves_pages_prefix_for_assets_and_directory_redirects(self):
with tempfile.TemporaryDirectory() as directory:
site = Path(directory)
(site / "guide").mkdir()
(site / "guide/index.html").write_text("Guide ")
(site / "asset.css").write_text("body { color: black; }")
comment = b"\xff\x80\r\n"
archive = bytes.fromhex("504b0506" + "00" * 16) + len(comment).to_bytes(2, "little") + comment
(site / "download.zip").write_bytes(archive)
for prefix in ("/", "/rea/"):
with self.subTest(prefix=prefix), ThreadingHTTPServer(("127.0.0.1", 0), WEBSITE.preview_handler(site, prefix)) as server:
worker = Thread(target=server.serve_forever, daemon=True)
worker.start()
try:
base = f"http://127.0.0.1:{server.server_port}"
with urlopen(base + prefix + "guide?test=1") as response:
self.assertEqual(response.url, base + prefix + "guide/?test=1")
self.assertEqual(response.read(), b"Guide ")
with urlopen(base + prefix + "asset.css") as response:
self.assertIn(b"color: black", response.read())
with urlopen(base + prefix + "download.zip") as response:
self.assertEqual(response.read(), archive)
with self.assertRaises(HTTPError) as error:
urlopen(base + prefix + "missing")
with contextlib.closing(error.exception) as response:
self.assertEqual(response.code, 404)
if prefix != "/":
with self.assertRaises(HTTPError) as error:
urlopen(base + "/guide/")
with contextlib.closing(error.exception) as response:
self.assertEqual(response.code, 404)
finally:
server.shutdown()
worker.join()
class WebsiteTranslationChecks(unittest.TestCase):
"""Exercise source edits, translated evidence and language editions at their consumers."""
def test_translation_keeps_malformed_references_for_the_verifier(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
shutil.copytree(ROOT / "website/templates", root / "website/templates")
locales = root / "website/locales"
locales.mkdir()
inventory = [item for item in languages(ROOT) if item["code"] in {"en", "zh-CN", "ar"}]
(locales / "languages.json").write_text(json.dumps(inventory))
page = WEBSITE.new_page(root, "guide", "Fixture", "Fixture guide.")
bad_links = ("https://[::1/guide", "https://[::1/figure", "https://[::1/script")
source = page.read_text().replace(' \n', "")
source = source.replace("", f'Broken ')
page.write_text(source.replace("
REA ka ta'aruf ", f''))
WEBSITE.sync_pages(root)
extract_catalogs(root)
catalog_path = locales / "zh-CN/messages.po"
catalog = polib.pofile(catalog_path)
for entry in catalog:
entry.msgstr = entry.msgid
catalog.save(catalog_path)
output = root / "website/_site"
build_translations(root, output)
diagnostics = VERIFY.check_site(output)[3]
for published in ("guide/index.html", "zh-CN/guide/index.html", "ar/guide/index.html"):
for link in bad_links:
with self.subTest(published=published, link=link):
self.assertTrue(any(f"{published}: invalid URL '{link}'" in error for error in diagnostics))
def test_language_switcher_preserves_encoded_paths_under_both_site_prefixes(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
shutil.copytree(ROOT / "website/templates", root / "website/templates")
locales = root / "website/locales"
locales.mkdir()
shutil.copyfile(ROOT / "website/locales/languages.json", locales / "languages.json")
route = "guides/Notes #1?%23 café"
path = WEBSITE.new_page(root, route, "Fixture", "Fixture guide.")
path.write_text(path.read_text().replace(' \n', ""))
WEBSITE.sync_pages(root)
nodes = Document(path.read_text()).nodes
canonical, = [node.attrs["href"] for node in nodes if node.attrs.get("rel") == "canonical"]
picker = next(node for node in nodes if node.attrs.get("data-language") == "zh-CN")
alternate = next(node for node in nodes if node.attrs.get("hreflang") == "zh-CN")
encoded = "guides/Notes%20%231%3F%2523%20caf%C3%A9/"
self.assertEqual(canonical, "https://rea.tools/" + encoded)
self.assertEqual(alternate.attrs["href"], "https://rea.tools/zh-CN/" + encoded)
for prefix in ("/", "/rea/"):
with self.subTest(prefix=prefix):
base = "https://example.com" + prefix
self.assertEqual(urljoin(base + encoded, picker.attrs["href"]), base + "zh-CN/" + encoded)
def test_language_inventory_preserves_script_identity(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
(root / "website/locales").mkdir(parents=True)
for code in ("ur", "ur-PK", "ur-Latn", "ur-Latn-PK", "hi-Latn"):
inventory = [
{"code": "en", "name": "English", "readme": "README.md", "dir": "ltr"},
{"code": code, "name": "Fixture", "readme": f"docs/readme/README_{code}.md", "dir": "ltr"},
]
path = root / "website/locales/languages.json"
path.write_text(json.dumps(inventory))
with self.subTest(code=code):
self.assertEqual(languages(root), inventory)
for code in ("ur-latn", "ur-LATN", "ur-PK-Latn", "ur/Latn"):
inventory[1]["code"] = code
path.write_text(json.dumps(inventory))
with self.subTest(code=code), self.assertRaisesRegex(ValueError, "Invalid or duplicate language"):
languages(root)
def test_completed_script_tag_pages_pass_search_metadata(self):
with tempfile.TemporaryDirectory() as directory:
# check_site canonicalizes its root before collecting page keys.
site = Path(directory).resolve()
(site / "assets").mkdir()
shutil.copyfile(ROOT / "website/_site/assets/social-card.png", site / "assets/social-card.png")
shutil.copyfile(ROOT / "website/public/robots.txt", site / "robots.txt")
pages = {}
for code in ("ur-Latn", "ur-Latn-PK"):
path = site / code / "index.html"
path.parent.mkdir()
url = f"https://rea.tools/{code}/"
source = dedent(f'''\