Fix catalog book links and rewrite PDF destinations to live HTTPS.
Some checks are pending
offline / test (push) Waiting to run

Module and model book cards now point at generated index.html/index.pdf
instead of directory URLs that 404. WeasyPrint file:// annotations are
mapped to https://zapier.georgelambert.org/ paths that exist. Developer
module, architecture, and research snapshot links target published files.
This commit is contained in:
George Lambert 2026-09-11 22:50:00 -04:00
parent 666eca79a1
commit 85b5b53ea9
26 changed files with 2253 additions and 35 deletions

View file

@ -12,6 +12,7 @@ import subprocess
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timezone
from pathlib import Path
from urllib.parse import unquote
ROOT = Path(__file__).resolve().parents[1]
SITE = ROOT / "site"
@ -219,6 +220,180 @@ def render_md(src: Path, html_dest: Path, pdf_dest: Path, title: str) -> str:
return ""
def _file_index(site: Path) -> dict[str, list[str]]:
idx: dict[str, list[str]] = {}
for p in site.rglob("*"):
if p.is_file():
rel = p.relative_to(site).as_posix()
idx.setdefault(p.name, []).append(rel)
return idx
_PREFERRED_PREFIXES = (
"overview/",
"packages/overview/",
"packages/verae-ops/",
"packages/verae-fleet/docs/",
"user-docs/",
"packages/zapier-user-docs/",
"docs/02-architecture/",
"docs/modules/",
"docs-master/",
"packages/zapier-decisions/",
"docs/modules-pdf/",
"docs/models-pdf/",
"sphinx/",
)
def _pick_hit(hits: list[str]) -> str:
for prefix in _PREFERRED_PREFIXES:
for h in hits:
if h.startswith(prefix):
return h
return hits[0]
def _web_target(site: Path, rel: str, index: dict[str, list[str]] | None = None) -> str:
"""Map a site-relative path to a file that actually exists."""
rel = unquote(rel).split("#", 1)[0].lstrip("/")
if rel.endswith(".html.pdf"):
rel = rel[:-4]
aliases = {
"index-md.pdf": "index-md.html",
"index-md.md": "index-md.html",
"docs/architecture/overview.pdf": "docs/02-architecture/overview.pdf",
"docs/architecture/nats-subjects.pdf": "docs/02-architecture/nats-subjects.pdf",
"docs/architecture/nats-gateway.pdf": "docs/02-architecture/nats-gateway.pdf",
"docs/architecture/composition.pdf": "docs/02-architecture/composition.pdf",
"docs/architecture/fleet.pdf": "docs/02-architecture/fleet.pdf",
"docs/sphinx/_build/html/index.pdf": "sphinx/index.html",
"docs/sphinx/_build/html/index.html": "sphinx/index.html",
}
rel = aliases.get(rel, rel)
candidates = [rel]
if rel.endswith(".pdf"):
candidates.append(rel[:-4] + ".html")
candidates.append(rel[:-4] + ".md")
if rel.endswith(".md"):
candidates.append(rel[:-3] + ".pdf")
candidates.append(rel[:-3] + ".html")
stem = Path(rel).name
if not stem or stem in {".", "/"}:
return rel or "index.html"
stem_pdf = Path(stem).with_suffix(".pdf").as_posix()
stem_html = Path(stem).with_suffix(".html").as_posix()
for folder in (
"overview",
"packages/overview",
"packages/verae-ops",
"packages/verae-fleet/docs",
"user-docs",
"packages/zapier-user-docs",
"docs/02-architecture",
"docs/modules-pdf",
"docs/models-pdf",
"packages/zapier-decisions",
"docs-master",
"sphinx",
):
candidates.append(f"{folder}/{stem_pdf}")
candidates.append(f"{folder}/{stem_html}")
for c in candidates:
if (site / c).is_file():
return c
if (site / c).is_dir():
for name in ("index.html", "index.pdf", "README.pdf", "README.html"):
if (site / c / name).is_file():
return f"{c.rstrip('/')}/{name}"
if index:
names = [stem, stem_pdf, stem_html]
if stem.endswith(".md"):
names.append(Path(stem).with_suffix(".pdf").as_posix())
names.append(Path(stem).with_suffix(".html").as_posix())
for name in names:
hits = index.get(name) or []
if hits:
return _pick_hit(hits)
return rel
def rewrite_pdf_uris(site: Path) -> None:
"""Turn WeasyPrint file:///… annotations into https://zapier.georgelambert.org/…"""
try:
from pypdf import PdfReader, PdfWriter
from pypdf.generic import NameObject, create_string_object
except ImportError:
print("pypdf missing; PDF URI rewrite skipped")
return
web = "https://zapier.georgelambert.org/"
prefix = site.resolve().as_uri().rstrip("/") + "/"
index = _file_index(site)
def resolve(raw: str) -> str:
frag = ""
if "#" in raw:
raw, frag = raw.split("#", 1)
frag = "#" + frag
mapped = raw
if raw.startswith(prefix):
mapped = web + _web_target(site, raw[len(prefix) :], index)
elif raw.startswith("file:"):
name = unquote(raw.rsplit("/", 1)[-1] if "/" in raw else raw)
if not name or name in {".", "/", "file:"}:
mapped = web
else:
hit = _web_target(site, name, index)
if (site / hit).exists():
mapped = web + hit
elif "index-md" in raw:
mapped = web + "index-md.html"
else:
mapped = web
elif raw.startswith(web):
hit = _web_target(site, raw[len(web) :], index)
if (site / hit).is_file() or (site / hit).is_dir():
mapped = web + hit
if mapped != raw:
return mapped + frag
return raw + frag
n_pdf = 0
n_fix = 0
for pdf in site.rglob("*.pdf"):
try:
reader = PdfReader(str(pdf))
except Exception:
continue
changed = False
for page in reader.pages:
annots = page.get("/Annots")
if not annots:
continue
for annot in annots:
obj = annot.get_object()
action = obj.get("/A")
if not action:
continue
uri = action.get("/URI")
if not uri:
continue
raw = str(uri)
new = resolve(raw)
if new != raw:
action[NameObject("/URI")] = create_string_object(new)
changed = True
n_fix += 1
if changed:
writer = PdfWriter(clone_from=reader)
tmp = pdf.with_suffix(".pdf.tmp")
with tmp.open("wb") as fh:
writer.write(fh)
tmp.replace(pdf)
n_pdf += 1
print(f"rewrote {n_fix} PDF URIs in {n_pdf} files → {web}")
def convert_all_markdown(copied: list[tuple[Path, Path, str]]) -> list[str]:
errors = []
n = len(copied)
@ -403,14 +578,20 @@ def main() -> None:
SITE / "packages" / pkg / "docs",
ignore=shutil.ignore_patterns("node_modules"),
)
copy_tree(pkg_root / "services", SITE / "packages" / pkg / "services")
elif pkg == "verae-nats-process":
for p in pkg_root.glob("*.md"):
copy_tree(p, SITE / "packages" / pkg / p.name)
else:
for name in ("README.md", "SUMMARY.md", "NATS.md"):
extras = ("README.md", "SUMMARY.md", "NATS.md")
if pkg == "zapier-decisions":
extras = extras + ("LOG.md", "TODO.md")
for name in extras:
p = pkg_root / name
if p.exists():
copy_tree(p, SITE / "packages" / pkg / name)
if pkg == "zapier-decisions":
copy_tree(pkg_root / "decisions", SITE / "packages" / pkg / "decisions")
ui_docs = ROOT / "packages" / "ui-docs"
for name in ("WALKTHROUGH.md", "REPORT.md", "UI-REVIEW.pdf"):
p = ui_docs / name
@ -432,6 +613,7 @@ def main() -> None:
SITE / "packages" / "zappier" / "docs",
ignore=shutil.ignore_patterns("screenshots", "walkthrough", "superpowers"),
)
copy_tree(ROOT / "packages" / "zappier" / "openapi.yaml", SITE / "packages" / "zappier" / "openapi.yaml")
for name in ("README.md", "TODO.md", "OPEN.md"):
src = ROOT / name
if not src.exists() and name == "OPEN.md":
@ -469,12 +651,40 @@ def main() -> None:
if diagrams.exists():
copy_tree(diagrams, SITE / "research" / "docs" / "diagrams")
def write_book_index(folder: Path, title: str, patterns: tuple[str, ...]) -> None:
if not folder.is_dir():
return
files: list[Path] = []
for pat in patterns:
files.extend(p for p in folder.rglob(pat) if p.is_file())
skip = {"index.md", "index.html", "index.pdf"}
files = sorted({p for p in files if p.name not in skip})
lines = [f"# {title}", "", f"{len(files)} files in this book.", ""]
for p in files:
rel = p.relative_to(folder).as_posix()
lines.append(f"- [{rel}]({rel})")
(folder / "index.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
write_book_index(SITE / "docs" / "modules-pdf", "Module PDFs (book)", ("*.pdf",))
write_book_index(SITE / "docs" / "models-pdf", "Model PDFs (book)", ("*.pdf",))
write_book_index(SITE / "packages" / "ui-docs" / "screenshots", "UI screenshots", ("*.png", "*.jpg", "*.webp"))
write_book_index(SITE / "packages" / "overview" / "diagrams", "Overview diagrams", ("*.svg", "*.png"))
write_book_index(SITE / "overview" / "diagrams", "Overview diagrams", ("*.svg", "*.png"))
write_book_index(SITE / "packages" / "zapier-decisions" / "decisions", "Decisions", ("*.md", "*.pdf"))
write_book_index(SITE / "packages" / "verae-fleet" / "services", "Fleet service JSON", ("*.json",))
write_book_index(SITE / "docs" / "00-sources" / "docs" / "diagrams", "Research diagrams", ("*.svg", "*.png"))
# Render every published .md from the site copy so relative images resolve.
to_render: list[tuple[Path, Path, str]] = []
seen: set[Path] = set()
for md in SITE.rglob("*.md"):
if not md.is_file():
continue
if "research" in md.parts:
continue
# Sphinx HTML/PDF are copied from _build; don't WeasyPrint the rst sources.
if "sphinx" in md.parts:
continue
key = md.resolve()
if key in seen:
continue
@ -496,6 +706,7 @@ def main() -> None:
labeled.append((src, dest, title_map.get(key, title_map.get(src.relative_to(ROOT).as_posix() if src.is_relative_to(ROOT) else key, title))))
errors = convert_all_markdown(labeled)
rewrite_pdf_uris(SITE)
# Also write CONSOLE.pdf next to the markdown in the fleet package (local console)
console_md = ROOT / "packages" / "verae-fleet" / "docs" / "CONSOLE.md"
@ -591,8 +802,8 @@ def main() -> None:
("packages/ui-docs/UI-REVIEW.pdf", "UI review (screenshots + live doors)"),
("overview/README.pdf", "System overview"),
("overview/INDEX.pdf", "Documentation index"),
("docs/modules-pdf/", "Module PDFs (book)"),
("docs/models-pdf/", "Model PDFs (book)"),
("docs/modules-pdf/index.pdf", "Module PDFs (book)"),
("docs/models-pdf/index.pdf", "Model PDFs (book)"),
("sphinx/index.html", "Sphinx HTML"),
("sphinx/verae-zapier-modules.pdf", "Sphinx LaTeX PDF"),
]
@ -605,8 +816,10 @@ def main() -> None:
("packages/verae-fleet/docs/CONSOLE.html", "Operator console"),
("overview/README.html", "System overview"),
("overview/INDEX.html", "Documentation index"),
("docs/modules/", "All module markdown"),
("docs/models/", "All model markdown"),
("docs/modules/README.html", "All module markdown"),
("docs/models/README.html", "All model markdown"),
("docs/modules-pdf/index.html", "Module PDFs (book)"),
("docs/models-pdf/index.html", "Model PDFs (book)"),
("sphinx/index.html", "Sphinx HTML"),
("sphinx/verae-zapier-modules.pdf", "Sphinx LaTeX PDF"),
]
@ -676,12 +889,12 @@ def main() -> None:
)
def extra_section(items: list[tuple[str, str]]) -> str:
lis = "".join(
f'<li><a href="{h}">{lab}</a></li>'
for h, lab in items
if (SITE / h).exists() or h.endswith("/") or h.startswith("sphinx")
)
return f"<section><h2>Catalogs</h2><ul>{lis}</ul></section>"
lis = []
for h, lab in items:
dest = SITE / h
if dest.exists() or (h.startswith("sphinx") and (SITE / "sphinx").exists()):
lis.append(f'<li><a href="{h}">{lab}</a></li>')
return f"<section><h2>Catalogs</h2><ul>{''.join(lis)}</ul></section>"
rest_pdf = f"""
{cards("pdf")}