mirror of
https://github.com/morluto/rea.git
synced 2026-11-22 22:45:47 +00:00
455 lines
21 KiB
Python
455 lines
21 KiB
Python
"""Maintain shared HTML and run the static website's contributor commands."""
|
|
|
|
import argparse
|
|
from datetime import date
|
|
from functools import partial
|
|
from html import escape
|
|
from html.parser import HTMLParser
|
|
from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
|
|
import json
|
|
import re
|
|
import shutil
|
|
from pathlib import Path, PurePosixPath
|
|
from string import Template
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from textwrap import indent
|
|
from urllib.parse import quote, urlsplit
|
|
|
|
|
|
SITE_ORIGIN = "https://rea.tools"
|
|
NOTES_FIXTURES = {"examples/notes-electron/index.html", "examples/notes-web/index.html"}
|
|
NAVIGATION = (
|
|
("Get started", "get-started/"),
|
|
("Showcases", "showcase/"),
|
|
("Blog", "blog/"),
|
|
("Guides", "guides/"),
|
|
)
|
|
|
|
|
|
# Sources are shared with the figures on the corresponding reading pages.
|
|
DEFAULT_IMAGE_ALT = "Blue REA lettering beside disassembly, hex data and an agent workflow."
|
|
SOCIAL_FIGURES = {
|
|
"showcase/": "docs/assets/rea-showcases.svg",
|
|
"showcase/aegis/": "website/artwork/aegis.svg",
|
|
"showcase/ctf/": "website/public/assets/figures/ctf-mask-equation.svg",
|
|
"showcase/dx-ball/": "website/public/assets/figures/dx-ball-sound-pan-investigation.svg",
|
|
"showcase/notion/": "website/artwork/notion.svg",
|
|
"showcase/th04/": "website/public/assets/figures/th04-bullet-ring.svg",
|
|
"guides/": "website/public/assets/figures/rea-investigation-flow.svg",
|
|
"guides/browser/": "website/artwork/browser-export.svg",
|
|
"guides/javascript/": "website/artwork/electron-export.svg",
|
|
"guides/native/": "website/public/assets/figures/dx-ball-sound-pan-investigation.svg",
|
|
"guides/rl-environments/": "website/artwork/environments.svg",
|
|
"blog/touhou-reconstruction/": "website/public/assets/figures/th04-bullet-ring.svg",
|
|
}
|
|
|
|
|
|
def social_image_path(route):
|
|
source = SOCIAL_FIGURES.get(route)
|
|
return f"assets/share/{Path(source).stem}.png" if source else "assets/social-card.png"
|
|
|
|
|
|
def checked_contributors(data):
|
|
"""Validate the public snapshot once, before rendering or replacing it."""
|
|
if not isinstance(data, dict) or data.get("repository") != "morluto/rea":
|
|
raise ValueError("Contributors must come from morluto/rea")
|
|
if not isinstance(data.get("updated"), str):
|
|
raise ValueError("Contributors need an ISO update date")
|
|
date.fromisoformat(data["updated"])
|
|
if not re.fullmatch(r"[0-9a-f]{40}", str(data.get("revision", ""))):
|
|
raise ValueError("Contributors need the source commit SHA")
|
|
people = data.get("contributors")
|
|
if not isinstance(people, list) or not people:
|
|
raise ValueError("Contributors must be a nonempty list")
|
|
ids, logins = set(), set()
|
|
for person in people:
|
|
if not isinstance(person, dict):
|
|
raise ValueError("Invalid contributor record")
|
|
identity, login = person.get("id"), person.get("login")
|
|
if type(identity) is not int or identity <= 0 or identity in ids:
|
|
raise ValueError("Contributor IDs must be unique positive integers")
|
|
if not isinstance(login, str) or not login.strip() or login.casefold() in logins:
|
|
raise ValueError("Contributor usernames must be nonempty and unique")
|
|
if type(person.get("commits")) is not int or person["commits"] < 1:
|
|
raise ValueError(f"Invalid commit count: {login}")
|
|
if person.get("type") not in ("User", "Bot"):
|
|
raise ValueError(f"Unknown contributor type: {login}")
|
|
for key, host in (("html_url", "github.com"), ("avatar_url", "avatars.githubusercontent.com")):
|
|
value = person.get(key)
|
|
if not isinstance(value, str):
|
|
raise ValueError(f"Missing {key}: {login}")
|
|
url = urlsplit(value)
|
|
if url.scheme != "https" or url.netloc != host or not url.path:
|
|
raise ValueError(f"Invalid public {key}: {login}")
|
|
ids.add(identity)
|
|
logins.add(login.casefold())
|
|
return data
|
|
|
|
|
|
def ranked_contributors(data):
|
|
return sorted(data["contributors"], key=lambda person: (-person["commits"], person["login"].casefold()))
|
|
|
|
|
|
def contributor_block(root):
|
|
"""Render equal-sized profile links from the checked GitHub snapshot."""
|
|
path = root / "website/public/contributors/people.json"
|
|
data = checked_contributors(json.loads(path.read_text(encoding="utf-8")))
|
|
cards = []
|
|
for person in ranked_contributors(data):
|
|
login = escape(person["login"], quote=True)
|
|
profile = escape(person["html_url"], quote=True)
|
|
avatar = escape(person["avatar_url"], quote=True)
|
|
cards.append(
|
|
f' <li data-contributor="{login}"><a class="contributor-card" href="{profile}">\n'
|
|
f' <img src="{avatar}" alt="" width="56" height="56" loading="lazy" decoding="async" referrerpolicy="no-referrer" />\n'
|
|
f' <span dir="ltr">@{login}</span>\n'
|
|
' </a></li>'
|
|
)
|
|
template = root / "website/templates/contributors.html.tmpl"
|
|
return indent(Template(template.read_text(encoding="utf-8")).substitute(
|
|
cards="\n".join(cards),
|
|
), " ")
|
|
|
|
|
|
def github_json(*arguments):
|
|
result = subprocess.run(["gh", "api", *arguments], capture_output=True, text=True, check=True)
|
|
return json.loads(result.stdout)
|
|
|
|
|
|
def collect_contributors():
|
|
"""Count each GitHub profile once per commit on a fixed revision of main."""
|
|
revision = github_json("repos/morluto/rea/git/ref/heads/main")["object"]["sha"]
|
|
commits = subprocess.run(
|
|
["gh", "api", "--paginate", f"repos/morluto/rea/commits?sha={revision}&per_page=100",
|
|
"--jq", ".[] | {sha, author: (.author | if . then {id,login,type,html_url,avatar_url} else null end)} | @json"],
|
|
capture_output=True, text=True, check=True,
|
|
)
|
|
primary = {}
|
|
for line in commits.stdout.splitlines():
|
|
commit = json.loads(line)
|
|
if commit["sha"] in primary:
|
|
raise ValueError("Duplicate commit in GitHub commit history")
|
|
primary[commit["sha"]] = commit["author"]
|
|
|
|
query = """query($revision: GitObjectID!, $endCursor: String) {
|
|
repository(owner: "morluto", name: "rea") {
|
|
object(oid: $revision) { ... on Commit {
|
|
history(first: 100, after: $endCursor) {
|
|
pageInfo { hasNextPage endCursor }
|
|
nodes { oid authors(first: 100) { totalCount nodes {
|
|
user { id: databaseId login type: __typename html_url: url avatar_url: avatarUrl }
|
|
} } }
|
|
}
|
|
} }
|
|
}
|
|
}"""
|
|
pages = github_json("graphql", "--paginate", "--slurp", "-F", f"revision={revision}", "-f", f"query={query}")
|
|
people = {}
|
|
for page in pages:
|
|
for commit in page["data"]["repository"]["object"]["history"]["nodes"]:
|
|
sha, authors = commit["oid"], commit["authors"]
|
|
if sha not in primary or authors["totalCount"] != len(authors["nodes"]):
|
|
raise ValueError(f"Incomplete GitHub commit attribution at {sha}")
|
|
profiles = {node["user"]["id"]: node["user"] for node in authors["nodes"] if node["user"]}
|
|
author = primary.pop(sha)
|
|
if author:
|
|
profiles[author["id"]] = author
|
|
for identity, profile in profiles.items():
|
|
person = people.setdefault(identity, {**profile, "commits": 0})
|
|
if author and identity == author["id"]:
|
|
person["type"] = author["type"]
|
|
person["commits"] += 1
|
|
if primary:
|
|
raise ValueError("Incomplete GitHub commit history")
|
|
data = checked_contributors({
|
|
"repository": "morluto/rea", "revision": revision, "updated": date.today().isoformat(),
|
|
"contributors": list(people.values()),
|
|
})
|
|
data["contributors"] = ranked_contributors(data)
|
|
return data
|
|
|
|
|
|
def refresh_contributors(root):
|
|
"""Replace the snapshot only after both sources have been collected and checked."""
|
|
data = collect_contributors()
|
|
path = root / "website/public/contributors/people.json"
|
|
with tempfile.TemporaryDirectory(dir=path.parent, prefix=f".{path.name}-") as directory:
|
|
staged = Path(directory) / path.name
|
|
staged.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
from website_contributor_grid import write_avatar_grid
|
|
grid = Path(directory) / "avatars.png"
|
|
write_avatar_grid(data["contributors"], grid)
|
|
destination = path.with_name(grid.name)
|
|
previous = Path(directory) / "previous.png"
|
|
if destination.exists():
|
|
shutil.copyfile(destination, previous)
|
|
grid.replace(destination)
|
|
try:
|
|
staged.replace(path)
|
|
except OSError:
|
|
if previous.exists():
|
|
previous.replace(destination)
|
|
else:
|
|
destination.unlink()
|
|
raise
|
|
print(f"Refreshed {len(data['contributors'])} contributors.")
|
|
|
|
|
|
def authored_markdown(root):
|
|
"""Discover website documentation while excluding local environments and vendors."""
|
|
site = root / "website"
|
|
return [
|
|
path for path in sorted(site.rglob("*.md"))
|
|
if not any(part.startswith(".") or part in ("node_modules", "__pycache__")
|
|
for part in path.relative_to(site).parts)
|
|
and "_site" not in path.relative_to(site).parts
|
|
]
|
|
|
|
|
|
class PageMetadata(HTMLParser):
|
|
"""Read the authored title, description and indexing policy from the head."""
|
|
|
|
def __init__(self):
|
|
super().__init__()
|
|
self.in_head = False
|
|
self.in_title = False
|
|
self.titles = []
|
|
self.descriptions = []
|
|
self.noindex = False
|
|
|
|
def handle_starttag(self, tag, attrs):
|
|
if tag == "head":
|
|
self.in_head = True
|
|
if not self.in_head:
|
|
return
|
|
if tag == "title":
|
|
self.in_title = True
|
|
self.titles.append("")
|
|
if tag == "meta":
|
|
attributes = dict(attrs)
|
|
name = (attributes.get("name") or "").lower()
|
|
content = attributes.get("content") or ""
|
|
if name == "description":
|
|
self.descriptions.append(content)
|
|
if name in ("robots", "googlebot"):
|
|
self.noindex |= bool(
|
|
set(content.lower().replace(",", " ").split()) & {"noindex", "none"}
|
|
)
|
|
|
|
def handle_endtag(self, tag):
|
|
if tag == "head":
|
|
self.in_head = False
|
|
if tag == "title":
|
|
self.in_title = False
|
|
|
|
def handle_data(self, data):
|
|
if self.in_title:
|
|
self.titles[-1] += data
|
|
|
|
|
|
def page_route(site, path):
|
|
"""Map an HTML file to its encoded public route relative to the site root."""
|
|
route = path.relative_to(site).as_posix()
|
|
if path.name == "index.html":
|
|
route = route.removesuffix("index.html")
|
|
return quote(route, safe="/")
|
|
|
|
|
|
def page_url(site, path):
|
|
"""Return the canonical public URL of an authored HTML file."""
|
|
return f"{SITE_ORIGIN}/{page_route(site, path)}"
|
|
|
|
|
|
def shared_blocks(root, path, source):
|
|
"""Render shared regions while keeping the page's body and metadata authored."""
|
|
site = root / "website/public"
|
|
metadata = PageMetadata()
|
|
metadata.feed(source)
|
|
for name, values in (("title", metadata.titles), ("description", metadata.descriptions)):
|
|
if len(values) != 1 or not values[0].strip():
|
|
raise ValueError(f"{path.relative_to(site)}: needs one nonempty head {name}")
|
|
depth = len(path.relative_to(site).parent.parts)
|
|
base = "../" * depth or "./"
|
|
route = page_route(site, path)
|
|
links = []
|
|
for label, destination in NAVIGATION:
|
|
attributes = ' class="nav-start"' if destination == "get-started/" else ""
|
|
if route.startswith(destination):
|
|
attributes += ' aria-current="page"'
|
|
links.append(f' <a href="{base}{destination}"{attributes}>{label}</a>')
|
|
values = {
|
|
"base": base,
|
|
"navigation": "\n" + "\n".join(links) + "\n ",
|
|
"canonical": escape(page_url(site, path), quote=True),
|
|
"origin": SITE_ORIGIN,
|
|
"image": f"{SITE_ORIGIN}/{social_image_path(route)}",
|
|
"image_alt": escape(metadata.titles[0], quote=True) if route in SOCIAL_FIGURES else DEFAULT_IMAGE_ALT,
|
|
"og_type": "article" if route.startswith("blog/") and route != "blog/" else "website",
|
|
"title": escape(metadata.titles[0].strip(), quote=True),
|
|
"description": escape(metadata.descriptions[0].strip(), quote=True),
|
|
}
|
|
from website_i18n import language_markup
|
|
selector, alternates = language_markup(root, path.relative_to(site).as_posix())
|
|
values["language_picker"] = indent(selector, " ") if not metadata.noindex else ""
|
|
values["alternates"] = alternates if not metadata.noindex else ""
|
|
templates = root / "website/templates"
|
|
blocks = {
|
|
name: indent(Template((templates / f"{name}.html.tmpl").read_text(encoding="utf-8")).substitute(values), " ")
|
|
for name in ("sharing", "theme", "header", "footer")
|
|
}
|
|
if route == "contributors/":
|
|
blocks["contributors"] = contributor_block(root)
|
|
return blocks
|
|
|
|
|
|
def sync_pages(root, check=False):
|
|
"""Update shared regions, or reject drift without changing any authored file."""
|
|
site = root / "website/public"
|
|
changes = []
|
|
for path in sorted(site.rglob("*.html")):
|
|
if path.relative_to(site).as_posix() in NOTES_FIXTURES:
|
|
continue
|
|
original = path.read_text(encoding="utf-8")
|
|
source = original
|
|
for name, block in shared_blocks(root, path, source).items():
|
|
start = f" <!-- website:{name}:start -->\n"
|
|
end = f" <!-- website:{name}:end -->"
|
|
if source.count(start) != 1 or source.count(end) != 1:
|
|
raise ValueError(f"{path.relative_to(site)}: needs one {name} region; use new-page for a new reading page")
|
|
before, _, remaining = source.partition(start)
|
|
_, found, after = remaining.partition(end)
|
|
if not found:
|
|
raise ValueError(f"{path.relative_to(site)}: {name} region is out of order")
|
|
source = before + start + block + end + after
|
|
if source != original:
|
|
changes.append((path, source))
|
|
if check and changes:
|
|
names = ", ".join(str(path.relative_to(site)) for path, _ in changes)
|
|
raise ValueError(f"Shared HTML is stale: {names}. Run python3 scripts/website.py sync and commit the updated pages.")
|
|
if not check:
|
|
for path, source in changes:
|
|
path.write_text(source, encoding="utf-8")
|
|
print(f"{'Checked' if check else 'Synced'} shared HTML ({len(changes)} pages changed).")
|
|
return [path for path, _ in changes]
|
|
|
|
|
|
def new_page(root, route, title, description):
|
|
"""Create a draft reading page with correct depth, metadata and shared regions."""
|
|
parts = PurePosixPath(route)
|
|
if not route or parts.is_absolute() or ".." in parts.parts or "\\" in route or parts.suffix:
|
|
raise ValueError("Use a relative directory route, such as blog/my-article.")
|
|
site = root / "website/public"
|
|
path = site.joinpath(*parts.parts, "index.html")
|
|
if not path.resolve().is_relative_to(site.resolve()):
|
|
raise ValueError("The new page must stay inside website/public.")
|
|
if path.exists():
|
|
raise ValueError(f"Page already exists: {path.relative_to(site)}")
|
|
base = "../" * len(parts.parts) or "./"
|
|
source = Template((root / "website/templates/page.html").read_text(encoding="utf-8")).substitute(
|
|
title=escape(title, quote=True), description=escape(description, quote=True), base=base,
|
|
)
|
|
# Render before writing, so invalid input cannot leave an incomplete page.
|
|
blocks = shared_blocks(root, path, source)
|
|
for name, block in blocks.items():
|
|
source = source.replace(f" <!-- website:{name}:start -->\n", f" <!-- website:{name}:start -->\n{block}")
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(source, encoding="utf-8")
|
|
print(f"Created {path.relative_to(root)}. Write the page, add its index link, then remove the Draft label and noindex directive when ready.")
|
|
return path
|
|
|
|
|
|
def run_script(root, name, arguments=()):
|
|
"""Use the active Python environment for the existing preparation and checks."""
|
|
subprocess.run([sys.executable, str(root / "scripts" / name), *arguments], cwd=root, check=True)
|
|
|
|
|
|
def preview_handler(site, prefix):
|
|
"""Serve one static directory at root or a Pages prefix, preserving redirects."""
|
|
class Handler(SimpleHTTPRequestHandler):
|
|
def send_head(self):
|
|
if not urlsplit(self.path).path.startswith(prefix):
|
|
self.send_error(404)
|
|
return None
|
|
return super().send_head()
|
|
|
|
def translate_path(self, path):
|
|
return super().translate_path("/" + path[len(prefix):])
|
|
|
|
return partial(Handler, directory=str(site))
|
|
|
|
|
|
def main():
|
|
"""Expose one small command for synchronizing, checking and previewing the site."""
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
commands = parser.add_subparsers(dest="command", required=True)
|
|
sync = commands.add_parser("sync", help="Update shared HTML after editing templates or page metadata")
|
|
sync.add_argument("--check", action="store_true", help="Report drift without rewriting pages")
|
|
commands.add_parser("check", help="Check shared HTML, prepare assets, verify links and run regression tests")
|
|
commands.add_parser("contributors", help="Refresh the public contributor snapshot using GitHub CLI")
|
|
links = commands.add_parser("links", help="Audit external targets using gh and HTTP; separate from offline checks")
|
|
links.add_argument("--report", help="Write the full external-link JSON report")
|
|
create = commands.add_parser("new-page", help="Create a draft reading page")
|
|
create.add_argument("route")
|
|
create.add_argument("--title", required=True)
|
|
create.add_argument("--description", required=True)
|
|
translations = commands.add_parser("translations", help="Maintain gettext catalogs")
|
|
translations.add_argument("action", choices=("extract", "status", "check"))
|
|
translations.add_argument("--language", help="Check one language, such as zh-CN")
|
|
translations.add_argument("--strict", action="store_true", help="Fail on missing or fuzzy translations")
|
|
serve = commands.add_parser("serve", help="Prepare and preview the static site")
|
|
serve.add_argument("--port", type=int, default=4173)
|
|
serve.add_argument("--base-path", choices=("/", "/rea/"), default="/")
|
|
args = parser.parse_args()
|
|
root = Path(__file__).resolve().parent.parent
|
|
try:
|
|
if args.command == "translations":
|
|
from website_i18n import checked_catalog, extract_catalogs, languages
|
|
template = extract_catalogs(root, check=args.action != "extract")
|
|
if args.action != "extract":
|
|
inventory = languages(root)[1:]
|
|
if args.language:
|
|
inventory = [item for item in inventory if item["code"] == args.language]
|
|
if not inventory:
|
|
raise ValueError(f"Unknown translation language: {args.language}")
|
|
for language in inventory:
|
|
messages, missing = checked_catalog(root, language, template, strict=args.strict)
|
|
print(f"{language['code']}: {len(messages)}/{len(template)} translated; {len(missing)} missing or fuzzy")
|
|
elif args.command == "new-page":
|
|
new_page(root, args.route, args.title, args.description)
|
|
elif args.command == "sync":
|
|
sync_pages(root, check=args.check)
|
|
elif args.command == "contributors":
|
|
refresh_contributors(root)
|
|
elif args.command == "check":
|
|
sync_pages(root, check=True)
|
|
for name in ("prepare-website.py", "verify-website.py", "test-website.py", "test-website-agents.py"):
|
|
run_script(root, name)
|
|
subprocess.run(["node", "--test", "website/worker.test.mjs"], cwd=root, check=True)
|
|
elif args.command == "links":
|
|
run_script(root, "audit-website-links.py", ["--report", args.report] if args.report else [])
|
|
else:
|
|
sync_pages(root)
|
|
run_script(root, "prepare-website.py")
|
|
prefix = args.base_path
|
|
|
|
handler = preview_handler(root / "website/_site", prefix)
|
|
with ThreadingHTTPServer(("127.0.0.1", args.port), handler) as server:
|
|
print(f"Preview: http://127.0.0.1:{server.server_port}{prefix}", flush=True)
|
|
server.serve_forever()
|
|
except (OSError, ValueError, KeyError) as error:
|
|
print(str(error), file=sys.stderr)
|
|
return 1
|
|
except subprocess.CalledProcessError as error:
|
|
if error.stderr:
|
|
print(error.stderr, file=sys.stderr)
|
|
return error.returncode
|
|
except KeyboardInterrupt:
|
|
return 0
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|