docs: publish translated docs in twelve languages and the tool that maintains them (#3280)

scripts/docs/build.sh

@@ -1,18 +1,26 @@
 #!/usr/bin/env bash
 #
-# Build the v2 documentation site for this checkout into `site/`.
+# Build the v2 documentation site for this checkout into `site/`: the English
+# site at the root, then one translated site per language in
+# i18n/languages.yml under `site/<code>/`.
 #
-# Zensical runs no MkDocs plugins or hooks, so the build is three steps:
-# materialise the API reference pages and the concrete config, build the
-# site strictly (plus the order-independence and cross-reference checks
+# Zensical runs no MkDocs plugins or hooks, so the English build is three
+# steps: materialise the API reference pages and the concrete config, build
+# the site strictly (plus the order-independence and cross-reference checks
 # Zensical doesn't do itself), then generate llms.txt and the per-page
-# markdown renditions. This script is the single owner of that recipe, dependency
-# sync included — CI (shared.yml, docs-preview.yml) and scripts/build-docs.sh
-# all call it. The toolchain detection in docs-preview.yml and build-docs.sh
-# keys on this file's path and expects the site under site/.
+# markdown renditions. A language site is lighter: the translation tool stages
+# a docs tree (English pages overlaid with that language's translations),
+# build_config.py writes its config, and Zensical builds it (non-strict: a
+# dead link or anchor in a translation is a warning, counted at the end)
+# straight into site/<code>/ — no API reference (it links the English one),
+# so no render-order or cross-reference checks. This script is the single
+# owner of that recipe, dependency sync included — CI (shared.yml,
+# docs-preview.yml) and scripts/build-docs.sh all call it. The toolchain
+# detection in docs-preview.yml and build-docs.sh keys on this file's path and
+# expects the site under site/.
 #
 # Usage:
-#   scripts/docs/build.sh
+#   scripts/docs/build.sh       (DOCS_LANGUAGES=en-only skips the language sites)
 #
 set -euo pipefail
 
@@ -26,8 +34,8 @@ uv sync --frozen --group docs
 # Zensical's incremental cache is unsound: a warm rebuild where only some
 # pages re-render silently drops cross-references to cache-hit pages, and
 # HTML for since-deleted pages lingers in site/. Build cold so the output
-# (and the checks below) are deterministic.
-rm -rf .cache site
+# (and the checks below) are deterministic. Staged language trees likewise.
+rm -rf .cache site .build/i18n
 
 uv run --frozen --no-sync python scripts/docs/build_config.py
 uv run --frozen --no-sync zensical build -f mkdocs.gen.yml --strict
@@ -44,3 +52,24 @@ uv run --frozen --no-sync python scripts/docs/check_render_order.py
 uv run --frozen --no-sync python scripts/docs/check_crossrefs.py --site-dir site
 
 uv run --frozen --no-sync python scripts/docs/llms_txt.py --site-dir site
+
+# Language sites build after English: `zensical build` clears its site_dir, so
+# the English build (site_dir site/) would wipe every site/<code>/, while a
+# language build (site_dir site/<code>/) leaves its parent alone. All the
+# language trees are staged in one pass, which reads the English pages once.
+languages=""
+if [[ "${DOCS_LANGUAGES:-}" != "en-only" ]]; then
+    languages="$(PYTHONPATH=scripts/docs uv run --frozen --no-sync python -c \
+        'import build_config; print(*(language.code for language in build_config.load_registry().languages))')"
+    uv run --frozen --no-sync python scripts/docs/translations.py stage
+fi
+
+for lang in $languages; do
+    echo "=== Building language site: ${lang} ==="
+    uv run --frozen --no-sync python scripts/docs/build_config.py --lang "$lang"
+    rm -rf .cache
+    log=".build/i18n/${lang}/build.log"
+    uv run --frozen --no-sync zensical build -f "mkdocs.${lang}.gen.yml" 2>&1 | tee "$log"
+    # Zensical reports each dead link/anchor as a "Warning:" diagnostic on stderr.
+    echo "${lang}: $(grep -c 'Warning:' "$log" || true) warnings"
+done

scripts/docs/build_config.py

@@ -6,42 +6,145 @@ as explicit entries. This script regenerates the `docs/api/` tree (via
 gen_ref_pages) and writes `mkdocs.gen.yml` with the real API nav spliced
 in — that generated file is what `zensical build`/`serve` consumes.
 
+With `--lang CODE` it writes `mkdocs.CODE.gen.yml` for one translated site
+instead: built from the tree `scripts/docs/translations.py stage` assembled
+under `.build/i18n/CODE/docs/` into `site/CODE/`, with no API reference of its
+own (its nav entry links the English one) and nav titles taken from the staged
+pages (the headings `stage` recorded beside the tree). Every config, English
+included, carries the language switcher (`extra.alternate`) built from
+`i18n/languages.yml`, which this module also loads for the translation tool.
+
 Usage:
-    python scripts/docs/build_config.py
+    python scripts/docs/build_config.py [--lang CODE]
 """
 
 from __future__ import annotations
 
+import argparse
+import json
 import posixpath
 import re
+from dataclasses import dataclass
 from pathlib import Path
+from typing import Any
 
 # Both scripts live in this directory, which Python puts on sys.path[0] when
 # `build_config.py` is run directly (its documented invocation).
 import gen_ref_pages
 import yaml
+from gen_ref_pages import NavItem
 
 ROOT = Path(__file__).parent.parent.parent
+LANGUAGES_FILE = "i18n/languages.yml"
 
-# A scheme-prefixed nav value (https:, mailto:, ...) is an external link, not
-# a page path (same classifier as llms_txt.py; a `://` test would misread
-# scheme-only URIs as pages).
-_EXTERNAL = re.compile(r"[a-zA-Z][a-zA-Z0-9+.-]*:")
+# A language site carries no API reference; its nav entry links the English
+# one (a sibling site one level up), which opens on the first package's index.
+API_REFERENCE_URL = "../api/mcp/"
+
+# A nav value with a URL scheme (https:, mailto:, ...), a leading `/`, or a
+# leading `../` (out of this site) is a link, not a page under docs_dir.
+_LINK = re.compile(r"[a-zA-Z][a-zA-Z0-9+.-]*:|/|\.\./")
 
 
-def _nav_pages(nav: list) -> set[str]:
-    """Collect every local page reference in the nav (external links excluded)."""
-    pages: set[str] = set()
+@dataclass(frozen=True)
+class Language:
+    """One translated site from `i18n/languages.yml`."""
+
+    code: str
+    name: str
+    theme: str
+    hreflang: str
+
+
+@dataclass(frozen=True)
+class Registry:
+    """The parsed `i18n/languages.yml`."""
+
+    model: str
+    exclude: list[str]
+    languages: list[Language]
+
+
+def load_registry(root: Path = ROOT) -> Registry:
+    """Parse `i18n/languages.yml` under repository `root`.
+
+    Raises:
+        ValueError: The file is missing, unparsable, or not the shape of `Registry`.
+    """
+    try:
+        raw = yaml.safe_load((root / LANGUAGES_FILE).read_text(encoding="utf-8"))
+        languages = [Language(**entry) for entry in raw["languages"]]
+        return Registry(str(raw["model"]), [str(pattern) for pattern in raw["exclude"]], languages)
+    except (OSError, yaml.YAMLError, TypeError, KeyError) as exc:
+        raise ValueError(f"{LANGUAGES_FILE}: {exc!r}") from exc
+
+
+def staged_docs_dir(code: str, root: Path = ROOT) -> Path:
+    """Where `translations.py stage` assembles a language's docs tree before its site is built."""
+    return root / ".build" / "i18n" / code / "docs"
+
+
+def staged_titles_file(code: str, root: Path = ROOT) -> Path:
+    """Where `translations.py stage` records each staged page's `#` heading text, keyed by page path."""
+    return root / ".build" / "i18n" / code / "titles.json"
+
+
+def nav_page_paths(nav: list[NavItem]) -> list[str]:
+    """Every local page path in the nav, depth first in nav order (link entries excluded)."""
+    paths: list[str] = []
     for entry in nav:
         value = next(iter(entry.values())) if isinstance(entry, dict) else entry
         if isinstance(value, list):
-            pages |= _nav_pages(value)
-        elif not _EXTERNAL.match(value):
-            pages.add(value)
-    return pages
+            paths.extend(nav_page_paths(value))
+        elif not _LINK.match(value):
+            paths.append(value)
+    return paths
 
 
-def _validate_nav(nav: list, docs_dir: Path) -> None:
+def language_nav(nav: list[NavItem], titles: dict[str, str]) -> list[NavItem]:
+    """The nav of a language site: every title taken from the staged pages (`titles` maps page path to H1).
+
+    Page labels are dropped, so Zensical titles each page from its (translated)
+    H1, and a section is titled with the H1 of the index page that leads it, so
+    the sidebar cannot drift from the pages. A link entry, and a section that
+    does not lead with a titled page, keeps its English label.
+    """
+    entries: list[NavItem] = []
+    for entry in nav:
+        if isinstance(entry, str):
+            entries.append(entry)
+            continue
+        ((label, value),) = entry.items()
+        if isinstance(value, list):
+            title = titles.get(value[0]) if value and isinstance(value[0], str) else None
+            entries.append({title or label: language_nav(value, titles)})
+        else:
+            entries.append({label: value} if _LINK.match(value) else value)
+    return entries
+
+
+def alternate(languages: list[Language], lang: str | None = None) -> list[dict[str, str]]:
+    """The `extra.alternate` switcher of the English site, or with `lang` of that language site.
+
+    Each label leads with the site's code (`ja - 日本語`). Links are relative
+    to the site being built (English one level up from a language site), so
+    the theme's `url` filter makes them page-relative and a mirror keeps working.
+    """
+    up = "" if lang is None else "../"
+    entries = [{"name": "en - English", "link": up or "./", "lang": "en"}]
+    entries += [{"name": f"{o.code} - {o.name}", "link": f"{up}{o.code}/", "lang": o.hreflang} for o in languages]
+    return entries
+
+
+def _api_entry(nav: list[NavItem]) -> dict[str, str | list[NavItem]]:
+    """The `mkdocs.yml` placeholder entry the API reference is spliced into."""
+    for entry in nav:
+        if isinstance(entry, dict) and "API Reference" in entry:
+            return entry
+    raise SystemExit("build_config: no 'API Reference' entry found in mkdocs.yml nav")
+
+
+def _validate_nav(nav: list[NavItem], docs_dir: Path) -> None:
     """Fail on nav/page drift in either direction.
 
     Zensical (0.0.48) ships a nav entry for a nonexistent page as a broken
@@ -52,39 +155,85 @@ def _validate_nav(nav: list, docs_dir: Path) -> None:
     exempt from the orphan check: its nav is spliced in from the same
     generator that writes the files, so it cannot drift.
     """
-    pages = _nav_pages(nav)
+    pages = set(nav_page_paths(nav))
     # Containment before existence: `docs_dir / page` would happily resolve
-    # an absolute value or a `../` escape against the wrong root.
-    if escaping := sorted(p for p in pages if p.startswith("/") or posixpath.normpath(p).startswith("..")):
-        raise SystemExit(f"build_config: nav references pages outside docs/: {escaping}")
+    # a `../` escape against the wrong root.
+    if escaping := sorted(page for page in pages if posixpath.normpath(page).startswith("..")):
+        raise SystemExit(f"build_config: nav references pages outside {docs_dir}: {escaping}")
     if missing := sorted(page for page in pages if not (docs_dir / page).is_file()):
-        raise SystemExit(f"build_config: nav references pages that don't exist under docs/: {missing}")
+        raise SystemExit(f"build_config: nav references pages that don't exist under {docs_dir}: {missing}")
     # Dot-directories (e.g. `.overrides` theme files) are not pages: the site
     # builder ignores them, so the orphan check must too.
     relative = (page.relative_to(docs_dir) for page in docs_dir.rglob("*.md"))
     on_disk = {page.as_posix() for page in relative if not any(part.startswith(".") for part in page.parts)}
     if orphaned := sorted(page for page in on_disk - pages if not page.startswith("api/")):
-        raise SystemExit(f"build_config: pages under docs/ that no nav entry reaches: {orphaned}")
+        raise SystemExit(f"build_config: pages under {docs_dir} that no nav entry reaches: {orphaned}")
 
 
-def build_config() -> None:
-    config = yaml.safe_load((ROOT / "mkdocs.yml").read_text(encoding="utf-8"))
+def build_config(lang: str | None = None, root: Path = ROOT) -> Path:
+    """Write the English config, or with `lang` that language site's config; returns the file written.
 
-    api_nav = gen_ref_pages.generate()
-    if not api_nav:
-        raise SystemExit("build_config: gen_ref_pages produced no API pages — did the src/ layout move?")
-    for entry in config["nav"]:
-        if isinstance(entry, dict) and "API Reference" in entry:
-            entry["API Reference"] = api_nav
-            break
+    `root` is the repository the config is read from and written to (a
+    scratch tree in tests); the English API reference is always generated
+    from this checkout's `src/`.
+    """
+    config: dict[str, Any] = yaml.safe_load((root / "mkdocs.yml").read_text(encoding="utf-8"))
+    try:
+        # No registry yet means English is the only site there is.
+        no_languages = lang is None and not (root / LANGUAGES_FILE).is_file()
+        languages = [] if no_languages else load_registry(root).languages
+    except ValueError as exc:
+        raise SystemExit(f"build_config: {exc}") from exc
+    if languages:  # a switcher listing English alone is noise
+        config.setdefault("extra", {})["alternate"] = alternate(languages, lang)
+
+    if lang is None:
+        api_nav: list[NavItem] = gen_ref_pages.generate()
+        if not api_nav:
+            raise SystemExit("build_config: gen_ref_pages produced no API pages — did the src/ layout move?")
+        _api_entry(config["nav"])["API Reference"] = api_nav
+        docs_dir = root / "docs"
+        output = root / "mkdocs.gen.yml"
     else:
-        raise SystemExit("build_config: no 'API Reference' entry found in mkdocs.yml nav")
+        language = next((candidate for candidate in languages if candidate.code == lang), None)
+        if language is None:
+            raise SystemExit(f"build_config: unknown language {lang!r} (see {LANGUAGES_FILE})")
+        docs_dir, titles_file = staged_docs_dir(lang, root), staged_titles_file(lang, root)
+        try:  # written last by `stage`, so its presence means the tree beside it is complete
+            titles: dict[str, str] = json.loads(titles_file.read_text(encoding="utf-8"))
+        except OSError as exc:
+            raise SystemExit(
+                f"build_config: cannot read {titles_file} (run translations.py stage --lang {lang})"
+            ) from exc
+        _api_entry(config["nav"])["API Reference"] = API_REFERENCE_URL
+        config["nav"] = language_nav(config["nav"], titles)
+        # No API reference on a language site, so no mkdocstrings pass either.
+        plugins: list[str | dict[str, Any]] = config["plugins"]
+        config["plugins"] = [p for p in plugins if (next(iter(p)) if isinstance(p, dict) else p) != "mkdocstrings"]
+        # A stored translation can name a `docs_src` file an English change has since renamed:
+        # that block renders empty under the outdated notice instead of stopping the build.
+        extensions: list[str | dict[str, Any]] = config["markdown_extensions"]
+        for extension in extensions:
+            if isinstance(extension, dict) and extension.get("pymdownx.snippets"):
+                extension["pymdownx.snippets"]["check_paths"] = False
+        config["theme"]["language"] = language.theme
+        # Zensical resolves docs_dir/site_dir against the config file and
+        # rejects absolute paths.
+        config["docs_dir"] = docs_dir.relative_to(root).as_posix()
+        config["site_dir"] = f"site/{lang}"
+        config["site_url"] = config["site_url"].rstrip("/") + f"/{lang}/"
+        output = root / f"mkdocs.{lang}.gen.yml"
 
-    _validate_nav(config["nav"], ROOT / "docs")
-
-    output = ROOT / "mkdocs.gen.yml"
+    _validate_nav(config["nav"], docs_dir)
     output.write_text(yaml.safe_dump(config, sort_keys=False, allow_unicode=True), encoding="utf-8")
+    return output
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--lang", metavar="CODE", help="write the config of this language site instead of English")
+    build_config(parser.parse_args().lang)
 
 
 if __name__ == "__main__":
-    build_config()
+    main()

scripts/docs/gen_ref_pages.py

@@ -14,13 +14,14 @@ from __future__ import annotations
 
 import shutil
 from pathlib import Path
+from typing import TypeAlias
 
 import griffe
 
 # A MkDocs/Zensical nav is a list of entries, each either `{title: url}` for a
 # page or `{title: [children]}` for a section (a bare `url` string attaches
 # a section index page, courtesy of the `navigation.indexes` feature).
-NavItem = "str | dict[str, str | list[NavItem]]"
+NavItem: TypeAlias = "str | dict[str, str | list[NavItem]]"
 
 ROOT = Path(__file__).parent.parent.parent
 API_DIR = ROOT / "docs" / "api"

scripts/docs/llms_txt.py

@@ -77,9 +77,9 @@ _EXTERNAL = re.compile(r"[a-zA-Z][a-zA-Z0-9+.-]*:")
 # rendition), never under-validated; and span pairing is bounded by blank
 # lines rather than full block structure.
 _FENCE = re.compile(r"^[ \t]*(`{3,}|~{3,})")
-_CODE_SPAN = re.compile(r"(?s)(?<!`)(`+)(?!`)((?:(?!\n[ \t]*\n).)+?)(?<!`)\1(?!`)")
+CODE_SPAN = re.compile(r"(?s)(?<!`)(`+)(?!`)((?:(?!\n[ \t]*\n).)+?)(?<!`)\1(?!`)")
 # A leading YAML frontmatter block, as MkDocs/Zensical parse it (mkdocs.utils.meta).
-_FRONTMATTER = re.compile(r"\A---[ \t]*\n(?P<block>.*?)^(?:---|\.\.\.)[ \t]*(?:\n|\Z)", flags=re.MULTILINE | re.DOTALL)
+FRONT_MATTER = re.compile(r"\A---[ \t]*\n(?P<block>.*?)^(?:---|\.\.\.)[ \t]*(?:\n|\Z)", flags=re.MULTILINE | re.DOTALL)
 
 
 class _BuildError(Exception):
@@ -108,7 +108,7 @@ def _split_frontmatter(text: str) -> tuple[dict[str, Any], str]:
     isn't a YAML mapping is page content, not frontmatter; an empty block is
     frontmatter with no meta.
     """
-    match = _FRONTMATTER.match(text)
+    match = FRONT_MATTER.match(text)
     if match is None:
         return {}, text
     try:
@@ -225,7 +225,7 @@ def _code_intervals(markdown: str) -> list[tuple[int, int]]:
     previous_end = 0
     for fence_start, fence_end in [*fences, (len(markdown), len(markdown))]:
         segment = markdown[previous_end:fence_start]
-        for pattern in (_CODE_SPAN, _HTML_COMMENT):
+        for pattern in (CODE_SPAN, _HTML_COMMENT):
             intervals += [(previous_end + m.start(), previous_end + m.end()) for m in pattern.finditer(segment)]
         previous_end = fence_end
     return intervals

scripts/serve-docs.sh

@@ -1,10 +1,13 @@
 #!/usr/bin/env bash
 #
-# Serve the v2 documentation locally with live reload.
+# Serve the v2 documentation locally with live reload (the English site only).
 #
 # Regenerates the API reference and the concrete Zensical config, then serves
 # it. Re-run the script to pick up changes to `src/` (the API reference) or the
-# nav; edits to prose pages under `docs/` are picked up by live reload.
+# nav; edits to prose pages under `docs/` are picked up by live reload. The
+# translated sites are never served here, so the language switcher's other
+# entries 404 locally; preview them with `scripts/docs/build.sh` and any static
+# file server over site/.
 #
 # Usage:
 #   scripts/serve-docs.sh [<extra zensical serve args>...]