Make catalog PDF links host-relative and add a hostname rewrite script.
Some checks are pending
offline / test (push) Waiting to run

New builds write path-relative PDF annotations so the tree can be served
from docs.verae-time.net (or any host at the site root) without a rebuild.
scripts/rewrite-docs-host.py retargets an already-built site when absolute
URLs are preferred. Live service doors stay on their own hostnames.
This commit is contained in:
George Lambert 2026-09-11 23:00:12 -04:00
parent 85b5b53ea9
commit 20155de96b
4 changed files with 246 additions and 40 deletions

View file

@ -7,6 +7,7 @@ defaults to PDF links; index-md.html is the Markdown/HTML index.
from __future__ import annotations from __future__ import annotations
import os
import shutil import shutil
import subprocess import subprocess
from concurrent.futures import ThreadPoolExecutor, as_completed from concurrent.futures import ThreadPoolExecutor, as_completed
@ -17,6 +18,11 @@ from urllib.parse import unquote
ROOT = Path(__file__).resolve().parents[1] ROOT = Path(__file__).resolve().parents[1]
SITE = ROOT / "site" SITE = ROOT / "site"
CSS = ROOT / "scripts" / "docs-print.css" CSS = ROOT / "scripts" / "docs-print.css"
# Display label only. Catalog file links are relative so the tree can move hosts.
DOCS_PUBLIC_HOST = os.environ.get("DOCS_PUBLIC_HOST", "zapier.georgelambert.org")
# If set (e.g. https://docs.verae-time.net), PDF annotations become absolute.
# Empty (default) writes path-relative URIs that work on any hostname at site root.
DOCS_PUBLIC_URL = os.environ.get("DOCS_PUBLIC_URL", "").rstrip("/")
SECTIONS = [ SECTIONS = [
( (
@ -180,7 +186,7 @@ def render_md(src: Path, html_dest: Path, pdf_dest: Path, title: str) -> str:
header.write_text(f"<style>{css_text}</style>\n", encoding="utf-8") header.write_text(f"<style>{css_text}</style>\n", encoding="utf-8")
banner.write_text( banner.write_text(
f'<div class="doc-banner">' f'<div class="doc-banner">'
f'<nav class="site"><a href="/">zapier.georgelambert.org</a>' f'<nav class="site"><a href="/">{DOCS_PUBLIC_HOST}</a>'
f' · <a href="/index-md.html">Markdown indexes</a></nav>' f' · <a href="/index-md.html">Markdown indexes</a></nav>'
f'<div class="kicker">Verae Time × Zapier</div>' f'<div class="kicker">Verae Time × Zapier</div>'
f"<h1>{title}</h1>" f"<h1>{title}</h1>"
@ -318,45 +324,68 @@ def _web_target(site: Path, rel: str, index: dict[str, list[str]] | None = None)
return rel return rel
_DOCS_ORIGINS = (
"https://zapier.georgelambert.org/",
"http://zapier.georgelambert.org/",
)
def _href_for(pdf: Path, site: Path, target: str, frag: str = "") -> str:
target = (target or "index.html").lstrip("/") or "index.html"
dest = site / target
if DOCS_PUBLIC_URL:
return DOCS_PUBLIC_URL + "/" + target + frag
rel = os.path.relpath(dest.resolve(), pdf.parent.resolve())
return Path(rel).as_posix() + frag
def rewrite_pdf_uris(site: Path) -> None: def rewrite_pdf_uris(site: Path) -> None:
"""Turn WeasyPrint file:///… annotations into https://zapier.georgelambert.org/…""" """Turn WeasyPrint file:///… annotations into host-relative catalog paths.
Default is a path relative to the PDF (works on any hostname at the site
root). Set DOCS_PUBLIC_URL to emit absolute URLs instead.
"""
try: try:
from pypdf import PdfReader, PdfWriter from pypdf import PdfReader, PdfWriter
from pypdf.generic import NameObject, create_string_object from pypdf.generic import NameObject, create_string_object
except ImportError: except ImportError:
print("pypdf missing; PDF URI rewrite skipped") print("pypdf missing; PDF URI rewrite skipped")
return return
web = "https://zapier.georgelambert.org/" site = site.resolve()
prefix = site.resolve().as_uri().rstrip("/") + "/" prefix = site.as_uri().rstrip("/") + "/"
origins = list(_DOCS_ORIGINS)
if DOCS_PUBLIC_URL:
origins.append(DOCS_PUBLIC_URL.rstrip("/") + "/")
index = _file_index(site) index = _file_index(site)
def resolve(raw: str) -> str: def resolve(pdf: Path, raw: str) -> str:
frag = "" frag = ""
if "#" in raw: if "#" in raw:
raw, frag = raw.split("#", 1) raw, frag = raw.split("#", 1)
frag = "#" + frag frag = "#" + frag
mapped = raw target = None
if raw.startswith(prefix): if raw.startswith(prefix):
mapped = web + _web_target(site, raw[len(prefix) :], index) target = _web_target(site, raw[len(prefix) :], index)
elif raw.startswith("file:"): elif raw.startswith("file:"):
name = unquote(raw.rsplit("/", 1)[-1] if "/" in raw else raw) name = unquote(raw.rsplit("/", 1)[-1] if "/" in raw else raw)
if not name or name in {".", "/", "file:"}: if not name or name in {".", "/", "file:"}:
mapped = web target = "index.html"
else: else:
hit = _web_target(site, name, index) hit = _web_target(site, name, index)
if (site / hit).exists(): if (site / hit).exists():
mapped = web + hit target = hit
elif "index-md" in raw: elif "index-md" in raw:
mapped = web + "index-md.html" target = "index-md.html"
else: else:
mapped = web target = "index.html"
elif raw.startswith(web): else:
hit = _web_target(site, raw[len(web) :], index) for origin in origins:
if (site / hit).is_file() or (site / hit).is_dir(): if raw.startswith(origin):
mapped = web + hit target = _web_target(site, raw[len(origin) :], index)
if mapped != raw: break
return mapped + frag if target is None:
return raw + frag return raw + frag
return _href_for(pdf, site, target, frag)
n_pdf = 0 n_pdf = 0
n_fix = 0 n_fix = 0
@ -379,7 +408,7 @@ def rewrite_pdf_uris(site: Path) -> None:
if not uri: if not uri:
continue continue
raw = str(uri) raw = str(uri)
new = resolve(raw) new = resolve(pdf, raw)
if new != raw: if new != raw:
action[NameObject("/URI")] = create_string_object(new) action[NameObject("/URI")] = create_string_object(new)
changed = True changed = True
@ -391,7 +420,8 @@ def rewrite_pdf_uris(site: Path) -> None:
writer.write(fh) writer.write(fh)
tmp.replace(pdf) tmp.replace(pdf)
n_pdf += 1 n_pdf += 1
print(f"rewrote {n_fix} PDF URIs in {n_pdf} files → {web}") mode = DOCS_PUBLIC_URL + "/" if DOCS_PUBLIC_URL else "relative paths"
print(f"rewrote {n_fix} PDF URIs in {n_pdf} files → {mode}")
def convert_all_markdown(copied: list[tuple[Path, Path, str]]) -> list[str]: def convert_all_markdown(copied: list[tuple[Path, Path, str]]) -> list[str]:
@ -489,7 +519,7 @@ def page_shell(now: str, body: str, switch_to_md: bool, title: str) -> str:
<a class="skip" href="#main">Skip to content</a> <a class="skip" href="#main">Skip to content</a>
<header> <header>
{switch} {switch}
<div class="kicker">zapier.georgelambert.org</div> <div class="kicker">{DOCS_PUBLIC_HOST}</div>
<h1>Verae Time × Zapier documentation</h1> <h1>Verae Time × Zapier documentation</h1>
<p>Project reference: setup, architecture, NATS, module APIs, user guide, and operator console. <p>Project reference: setup, architecture, NATS, module APIs, user guide, and operator console.
Catalog defaults to <strong>colored PDFs</strong>. Generated {now}.</p> Catalog defaults to <strong>colored PDFs</strong>. Generated {now}.</p>
@ -849,7 +879,7 @@ def main() -> None:
"Zapier /zapier/v1, jobs, archive, chain (MOCK_VERAE in lab)."), "Zapier /zapier/v1, jobs, archive, chain (MOCK_VERAE in lab)."),
("fleet.png", "https://fleet.zapier.georgelambert.org/", "Operator console", ("fleet.png", "https://fleet.zapier.georgelambert.org/", "Operator console",
"Fleet replicas, Trace, Docs. Pause/stop honored by keep."), "Fleet replicas, Trace, Docs. Pause/stop honored by keep."),
("docs.png", "https://zapier.georgelambert.org/", "This catalog", ("docs.png", "/", "This catalog",
"Colored PDF documentation site."), "Colored PDF documentation site."),
("git.png", "https://git.georgelambert.org/", "Forgejo", ("git.png", "https://git.georgelambert.org/", "Forgejo",
"Independent module repos (SSH 2223)."), "Independent module repos (SSH 2223)."),

View file

@ -5,7 +5,6 @@ from __future__ import annotations
import sys import sys
from html.parser import HTMLParser from html.parser import HTMLParser
from pathlib import Path from pathlib import Path
from urllib.parse import unquote, urljoin, urlparse
ROOT = Path(__file__).resolve().parents[1] ROOT = Path(__file__).resolve().parents[1]
SITE = ROOT / "site" SITE = ROOT / "site"
@ -24,23 +23,46 @@ class Anchors(HTMLParser):
self.hrefs.append(d["href"] or "") self.hrefs.append(d["href"] or "")
def _exists(dest: Path) -> bool:
if dest.is_file():
return True
if dest.is_dir() and any(
(dest / n).exists() for n in ("index.html", "index.pdf", "README.html", "README.pdf")
):
return True
return False
def local_ok(url: str) -> bool: def local_ok(url: str) -> bool:
if url.startswith(WEB): if url.startswith(WEB) or url.startswith("http://zapier.georgelambert.org/"):
rel = url[len(WEB) :].split("#", 1)[0] rel = url.split("://", 1)[-1].split("/", 1)[-1].split("#", 1)[0]
dest = SITE / rel return _exists(SITE / rel)
if dest.is_file():
return True
if dest.is_dir() and any(
(dest / n).exists() for n in ("index.html", "index.pdf", "README.html", "README.pdf")
):
return True
# yaml/svg/png on homepage
return False
if url.startswith("file:"): if url.startswith("file:"):
return False return False
return True # http(s) off-site; homepage checker uses GET separately return True # http(s) off-site; homepage checker uses GET separately
def pdf_dest_ok(pdf: Path, raw: str) -> bool:
raw = raw.split("#", 1)[0]
if raw.startswith("file:"):
return False
if raw.startswith("mailto:") or raw.startswith("data:"):
return True
if raw.startswith("http://") or raw.startswith("https://"):
if raw.startswith(WEB) or raw.startswith("http://zapier.georgelambert.org/"):
return local_ok(raw)
return True
if raw.startswith("/"):
dest = SITE / raw.lstrip("/")
else:
dest = (pdf.parent / raw).resolve()
try:
dest.relative_to(SITE.resolve())
except ValueError:
return False
return _exists(dest)
def main() -> int: def main() -> int:
idx = SITE / "index.html" idx = SITE / "index.html"
if not idx.exists(): if not idx.exists():
@ -91,12 +113,10 @@ def main() -> int:
n_file += 1 n_file += 1
broken.append(f"file URI {pdf.relative_to(SITE)} -> {raw[:120]}") broken.append(f"file URI {pdf.relative_to(SITE)} -> {raw[:120]}")
continue continue
if raw.startswith(WEB): if "/research/" in raw:
path = raw.split("#", 1)[0] continue
if "/research/" in path: if not pdf_dest_ok(pdf, raw):
continue broken.append(f"pdf missing {pdf.relative_to(SITE)} -> {raw}")
if not local_ok(path):
broken.append(f"pdf missing {pdf.relative_to(SITE)} -> {raw}")
print(f"file:// leftover {n_file}") print(f"file:// leftover {n_file}")
print(f"broken {len(broken)}") print(f"broken {len(broken)}")

View file

@ -1,5 +1,11 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# Build and rsync docs site to zapier.georgelambert.org on NS1. # Build and rsync docs site to zapier.georgelambert.org on NS1.
#
# Catalog file links are relative (any hostname at the site root).
# To republish under another name without a full rebuild:
# python3 scripts/rewrite-docs-host.py --site ./site \
# --from https://zapier.georgelambert.org --to https://docs.verae-time.net
# Optional absolute PDF URLs at build time: DOCS_PUBLIC_URL=https://docs.verae-time.net
set -euo pipefail set -euo pipefail
ROOT="$(cd "$(dirname "$0")/.." && pwd)" ROOT="$(cd "$(dirname "$0")/.." && pwd)"
HOST="${DEPLOY_HOST:-marchon@70.88.205.138}" HOST="${DEPLOY_HOST:-marchon@70.88.205.138}"

150
scripts/rewrite-docs-host.py Executable file
View file

@ -0,0 +1,150 @@
#!/usr/bin/env python3
"""Rewrite catalog hostnames in a built ./site tree (HTML + PDF annotations).
Catalog HTML links are already relative. PDF annotations used to be baked as
https://zapier.georgelambert.org/ this script retargets those (and any HTML
that still names a docs origin) when the tree is hosted on another name, e.g.
docs.verae-time.net.
It does **not** rewrite live service doors (portal/api/iam/) or Forgejo.
Examples:
# Lab catalog → future docs host (keep absolute URLs)
python3 scripts/rewrite-docs-host.py \\
--site ./site \\
--from https://zapier.georgelambert.org \\
--to https://docs.verae-time.net
# Drop the host entirely (path-relative PDF links; any hostname at site root)
python3 scripts/rewrite-docs-host.py --site ./site --relative
python3 scripts/rewrite-docs-host.py --site ./site --from --to --dry-run
"""
from __future__ import annotations
import argparse
import os
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def _norm_origin(url: str) -> str:
url = url.strip()
if not url:
return ""
if "://" not in url:
url = "https://" + url
return url.rstrip("/") + "/"
def rewrite_html(site: Path, src: str, dst: str, dry: bool) -> int:
n = 0
src_bare = src.rstrip("/")
dst_bare = dst.rstrip("/") or "/"
for path in site.rglob("*"):
if not path.is_file() or path.suffix.lower() not in {".html", ".htm", ".xml", ".svg", ".js", ".css", ".md"}:
continue
text = path.read_text(encoding="utf-8", errors="replace")
if src not in text and src_bare not in text:
continue
new = text.replace(src, dst).replace(src_bare, dst_bare)
if new == text:
continue
n += 1
if dry:
print(f" html {path.relative_to(site)}")
else:
path.write_text(new, encoding="utf-8")
return n
def rewrite_pdfs(site: Path, src: str, dst: str | None, dry: bool) -> int:
try:
from pypdf import PdfReader, PdfWriter
from pypdf.generic import NameObject, create_string_object
except ImportError:
print("pypdf missing; pip install pypdf", file=sys.stderr)
return 0
n_pdf = 0
n_uri = 0
src_http = src.replace("https://", "http://", 1) if src.startswith("https://") else src
for pdf in site.rglob("*.pdf"):
try:
reader = PdfReader(str(pdf))
except Exception:
continue
changed = False
for page in reader.pages:
annots = page.get("/Annots")
if not annots:
continue
for annot in annots:
obj = annot.get_object()
action = obj.get("/A")
if not action:
continue
uri = action.get("/URI")
if not uri:
continue
raw = str(uri)
new = raw
if raw.startswith(src) or raw.startswith(src_http):
rest = raw[len(src) :] if raw.startswith(src) else raw[len(src_http) :]
if dst:
new = dst + rest
else:
dest = site / rest.split("#", 1)[0]
frag = "#" + rest.split("#", 1)[1] if "#" in rest else ""
if not dest.exists():
dest = site / "index.html"
rel = os.path.relpath(dest.resolve(), pdf.parent.resolve())
new = Path(rel).as_posix() + frag
if new != raw:
action[NameObject("/URI")] = create_string_object(new)
changed = True
n_uri += 1
if changed:
n_pdf += 1
if dry:
print(f" pdf {pdf.relative_to(site)}")
else:
writer = PdfWriter(clone_from=reader)
tmp = pdf.with_suffix(".pdf.tmp")
with tmp.open("wb") as fh:
writer.write(fh)
tmp.replace(pdf)
print(f"pdf annotations {n_uri} in {n_pdf} files")
return n_pdf
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--site", type=Path, default=ROOT / "site", help="built catalog directory")
ap.add_argument("--from", dest="src", default="https://zapier.georgelambert.org", help="current docs origin")
g = ap.add_mutually_exclusive_group(required=True)
g.add_argument("--to", help="new docs origin, e.g. https://docs.verae-time.net")
g.add_argument("--relative", action="store_true", help="convert docs-origin URLs to path-relative links")
ap.add_argument("--dry-run", action="store_true")
args = ap.parse_args()
site = args.site.resolve()
if not site.is_dir():
print(f"no site at {site}", file=sys.stderr)
return 1
src = _norm_origin(args.src)
dst = None if args.relative else _norm_origin(args.to)
print(f"{src}{dst or 'relative paths'} ({site})")
# HTML: root-relative `/…` so nested pages still hit the site root on any host.
html_dst = "/" if dst is None else dst
n_html = rewrite_html(site, src, html_dst, args.dry_run)
print(f"html/text files {n_html}")
rewrite_pdfs(site, src, dst, args.dry_run)
if args.dry_run:
print("dry-run; nothing written")
return 0
if __name__ == "__main__":
sys.exit(main())