#!/usr/bin/env python3
"""Build static documentation site into ./site for zapier.georgelambert.org.
Markdown is rendered to colored HTML *and* PDF. The home page (index.html)
defaults to PDF links; index-md.html is the Markdown/HTML index.
"""
from __future__ import annotations
import shutil
import subprocess
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timezone
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SITE = ROOT / "site"
CSS = ROOT / "scripts" / "docs-print.css"
SECTIONS = [
(
"Start here",
[
("packages/verae-ops/GETTING-STARTED.md", "Bring the system online (install, first-up, monitor)"),
("packages/overview/README.md", "What the system is (plain language)"),
("packages/verae-fleet/docs/CONSOLE.md", "Operator console (loopback Fleet · Trace · Docs)"),
],
),
(
"Architecture",
[
("docs/02-architecture/composition.md", "Zappier + middleware composition"),
("docs/02-architecture/overview.md", "Middleware overview"),
("docs/02-architecture/nats-gateway.md", "NATS on NS1"),
("docs/02-architecture/nats-subjects.md", "NATS subjects"),
("docs/02-architecture/archive-nats.md", "Archive NATS, bloom, multi-receipt"),
("docs/02-architecture/modules-and-nats.md", "Module catalog and NATS addresses"),
("docs/02-architecture/tree-nodes.md", "Tree nodes and bulk Merkle summaries"),
("docs/02-architecture/fleet.md", "Fleet replica floors and monitor"),
("packages/overview/03-nats-cluster.md", "NATS.IO 3-server cluster"),
("packages/overview/04-uptime.md", "Uptime / replica floors"),
("packages/overview/05-network-failures.md", "Local network failures"),
("packages/overview/06-address-routing.md", "Address routing"),
("packages/overview/08-diagrams.md", "Architectural diagrams"),
],
),
(
"Per-module README (public PDFs)",
[
("packages/zappier/README.md", "zappier-edge (billing + users)"),
("packages/verae-zapier-middleware/README.md", "verae-middleware"),
("packages/verae-zapier/README.md", "verae-zapier-app"),
("packages/verae-activate/README.md", "verae-activate"),
("packages/verae-request-splitter/README.md", "verae-request-splitter"),
("packages/verae-archive-worm/README.md", "verae-archive-worm"),
("packages/verae-archive-aggregator/README.md", "verae-archive-aggregator"),
("packages/verae-tree-node/README.md", "verae-tree-node"),
("packages/verae-fleet/README.md", "verae-fleet"),
("packages/verae-zapier-simulator/README.md", "verae-zapier-simulator"),
("packages/zapier-user-docs/README.md", "zapier-user-docs"),
("packages/docs-master/README.md", "zapier-docs-master"),
("packages/verae-nats-process/README.md", "verae-nats-process"),
],
),
(
"Billing, users, portal (zappier-edge)",
[
("packages/zappier/README.md", "zappier-edge README (the billing/user platform)"),
("packages/zappier/docs/USER-MANUAL.md", "Operations & usage"),
("packages/zappier/docs/USER-MANAGEMENT.md", "Admin users, pricing, customer types"),
("packages/zappier/docs/CUSTOMER-PORTAL.md", "Signup, 2FA, API keys, reloads"),
("packages/zappier/docs/ACCOUNTING.md", "Invoices, PO billing, reports"),
("packages/zappier/docs/DEVELOPER.md", "Developer reference"),
],
),
(
"Install (detail)",
[
("packages/verae-ops/GETTING-STARTED.md", "Bring online (start here)"),
("packages/verae-ops/02-docker.md", "Docker Compose"),
("packages/verae-ops/03-proxmox.md", "Proxmox"),
("packages/verae-ops/05-dedicated-hardware.md", "Dedicated hardware"),
("packages/verae-ops/06-linking-services.md", "Linking services"),
],
),
(
"Samples",
[
("packages/zappier/zapier-app/README.md", "Sample Zapier app (metered zappier API)"),
("packages/verae-activate/README.md", "Activate-now Add Numbers"),
("packages/verae-zapier-simulator/README.md", "Hop-trace simulator"),
],
),
(
"User guide (signup → lookup)",
[
("packages/zapier-user-docs/README.md", "User guide index"),
("packages/zapier-user-docs/02-signup-zappier-portal.md", "Sign up"),
("packages/zapier-user-docs/04-register-a-hash.md", "Register a SHA-256"),
("packages/zapier-user-docs/06-lookup-central-chain.md", "Central chain lookup"),
("packages/zapier-user-docs/09-lookup-tree-nodes.md", "Tree-node lookup"),
],
),
(
"Product",
[
("docs/01-product/features-a-m.md", "Features a–m"),
("docs/01-product/api-gap-analysis.md", "Verae OpenAPI gaps"),
("docs/01-product/billing-and-keys.md", "Billing and API keys"),
("docs/00-sources/workspace-brief.md", "Original brief"),
("docs/00-sources/provenance.md", "Provenance"),
],
),
(
"API specs",
[
("docs/api/middleware-openapi.yaml", "Middleware OpenAPI"),
("packages/zappier/openapi.yaml", "Zappier OpenAPI"),
("docs/00-sources/veraetime-openapi.yaml", "Verae Timestamping OpenAPI snapshot"),
],
),
]
def rel(p: Path) -> str:
return p.relative_to(ROOT).as_posix()
def copy_tree(src: Path, dest: Path, ignore=None) -> None:
if not src.exists():
return
dest.parent.mkdir(parents=True, exist_ok=True)
if src.is_file():
shutil.copy2(src, dest)
return
shutil.copytree(src, dest, dirs_exist_ok=True, ignore=ignore)
def site_href_for(path: str, src: Path) -> tuple[str, str]:
"""Return (pdf_or_native_href, md_or_html_href) relative to SITE."""
suffix = src.suffix.lower()
if suffix == ".md":
pdf = Path(path).with_suffix(".pdf").as_posix()
html = Path(path).with_suffix(".html").as_posix()
return pdf, html
return path, path
def render_md(src: Path, html_dest: Path, pdf_dest: Path, title: str) -> str:
"""Pandoc markdown → colored HTML + WeasyPrint PDF. Returns '' on success."""
html_dest.parent.mkdir(parents=True, exist_ok=True)
pdf_dest.parent.mkdir(parents=True, exist_ok=True)
header = html_dest.with_suffix(".hdr.html")
banner = html_dest.with_suffix(".ban.html")
rel_src = src.relative_to(ROOT).as_posix() if src.is_relative_to(ROOT) else src.name
css_text = CSS.read_text(encoding="utf-8")
header.write_text(f"\n", encoding="utf-8")
banner.write_text(
f'
\n",
encoding="utf-8",
)
resource = str(src.parent)
r = subprocess.run(
[
"pandoc",
str(src),
"-o",
str(html_dest),
"--standalone",
f"--resource-path={resource}",
"--syntax-highlighting=breezedark",
f"--metadata=title={title}",
f"--include-in-header={header}",
f"--include-before-body={banner}",
f"--lua-filter={ROOT / 'scripts' / 'pdf-links.lua'}",
],
capture_output=True,
text=True,
)
header.unlink(missing_ok=True)
banner.unlink(missing_ok=True)
if r.returncode != 0:
return f"pandoc {src}: {r.stderr[-400:]}"
w = subprocess.run(
["weasyprint", str(html_dest), str(pdf_dest)],
capture_output=True,
text=True,
)
if w.returncode != 0:
return f"weasyprint {src}: {w.stderr[-400:]}"
return ""
def convert_all_markdown(copied: list[tuple[Path, Path, str]]) -> list[str]:
errors = []
n = len(copied)
print(f"rendering {n} markdown files to HTML + PDF…")
with ThreadPoolExecutor(max_workers=6) as pool:
futs = {
pool.submit(render_md, src, html, pdf, title): src
for src, html, pdf, title in (
(
src,
dest.with_suffix(".html"),
dest.with_suffix(".pdf"),
title,
)
for src, dest, title in copied
)
}
done = 0
for fut in as_completed(futs):
err = fut.result()
done += 1
if err:
errors.append(err)
print(f" [{done}/{n}] FAIL {futs[fut].name}")
elif done % 25 == 0 or done == n:
print(f" [{done}/{n}]")
return errors
def format_switch(to_md: bool) -> str:
if to_md:
href, label = "/index-md.html", "Markdown indexes"
else:
href, label = "/", "PDF catalog"
return (
f'{label} '
)
def page_shell(now: str, body: str, switch_to_md: bool, title: str) -> str:
switch = format_switch(switch_to_md)
return f"""
{title}
{switch}
zapier.georgelambert.org
Verae Time × Zapier documentation
Project reference: setup, architecture, NATS, module APIs, user guide, and operator console.
Catalog defaults to colored PDFs . Generated {now}.
{body}
"""
def main() -> None:
if SITE.exists():
shutil.rmtree(SITE)
SITE.mkdir()
copy_tree(ROOT / "docs", SITE / "docs", ignore=shutil.ignore_patterns("_build", ".DS_Store"))
sphinx = ROOT / "docs" / "sphinx" / "_build" / "html"
if sphinx.exists():
copy_tree(sphinx, SITE / "sphinx")
latex_pdf = ROOT / "docs" / "sphinx" / "_build" / "latex" / "verae-zapier-modules.pdf"
if latex_pdf.exists():
copy_tree(latex_pdf, SITE / "sphinx" / "verae-zapier-modules.pdf")
for pkg in (
"verae-activate",
"verae-zapier",
"verae-zapier-middleware",
"zappier",
"verae-request-splitter",
"verae-archive-worm",
"verae-archive-aggregator",
"verae-tree-node",
"verae-zapier-simulator",
"zapier-user-docs",
"verae-fleet",
"overview",
"verae-nats-process",
"docs-master",
"verae-ops",
):
pkg_root = ROOT / "packages" / pkg
if pkg in {"zapier-user-docs", "overview", "docs-master", "verae-ops"}:
copy_tree(
pkg_root,
SITE / "packages" / pkg,
ignore=shutil.ignore_patterns("node_modules", ".git", "src", "test", "data"),
)
elif pkg == "verae-fleet":
for name in ("README.md", "SUMMARY.md", "NATS.md", "SERVICES.md"):
p = pkg_root / name
if p.exists():
copy_tree(p, SITE / "packages" / pkg / name)
copy_tree(
pkg_root / "docs",
SITE / "packages" / pkg / "docs",
ignore=shutil.ignore_patterns("node_modules"),
)
elif pkg == "verae-nats-process":
for p in pkg_root.glob("*.md"):
copy_tree(p, SITE / "packages" / pkg / p.name)
else:
for name in ("README.md", "SUMMARY.md", "NATS.md"):
p = pkg_root / name
if p.exists():
copy_tree(p, SITE / "packages" / pkg / name)
zap_app = ROOT / "packages" / "zappier" / "zapier-app" / "README.md"
if zap_app.exists():
copy_tree(zap_app, SITE / "packages" / "zappier" / "zapier-app" / "README.md")
zdocs = ROOT / "packages" / "zappier" / "docs"
if zdocs.exists():
copy_tree(
zdocs,
SITE / "packages" / "zappier" / "docs",
ignore=shutil.ignore_patterns("screenshots", "walkthrough", "superpowers"),
)
for name in ("README.md", "TODO.md", "OPEN.md"):
src = ROOT / name
if not src.exists() and name == "OPEN.md":
src = ROOT / "docs" / "OPEN.md"
if src.exists():
copy_tree(src, SITE / src.name if src.parent == ROOT else SITE / "docs" / "OPEN.md")
copy_tree(ROOT / "docs" / "WORK-LOG.md", SITE / "docs" / "WORK-LOG.md")
copy_tree(ROOT / "packages" / "docs-master", SITE / "docs-master")
copy_tree(ROOT / "packages" / "overview", SITE / "overview")
copy_tree(ROOT / "packages" / "zapier-user-docs", SITE / "user-docs", ignore=shutil.ignore_patterns("node_modules"))
copy_tree(
ROOT / "packages" / "verae-fleet" / "docs",
SITE / "packages" / "verae-fleet" / "docs",
ignore=shutil.ignore_patterns("node_modules"),
)
sim_pub = ROOT / "packages" / "verae-zapier-simulator" / "public" / "index.html"
if sim_pub.exists():
copy_tree(sim_pub, SITE / "simulator" / "index.html")
r = ROOT / "research" / "zapier"
if r.exists():
for fname in (
"README.md",
"getting-started.md",
"PLATFORM-REFERENCE.md",
"FUNCTIONS-REFERENCE.md",
"MCP-REFERENCE.md",
"LOGIN.md",
"docs/zapier-billing.md",
):
p = r / fname
if p.exists():
copy_tree(p, SITE / "research" / fname)
diagrams = r / "docs" / "diagrams"
if diagrams.exists():
copy_tree(diagrams, SITE / "research" / "docs" / "diagrams")
# Render every published .md from the site copy so relative images resolve.
to_render: list[tuple[Path, Path, str]] = []
seen: set[Path] = set()
for md in SITE.rglob("*.md"):
if not md.is_file():
continue
key = md.resolve()
if key in seen:
continue
seen.add(key)
to_render.append((md, md, md.stem.replace("-", " ")))
# Prefer curated titles for catalog entries
title_map = {}
for _sec, links in SECTIONS:
for path, label in links:
title_map[path] = label
labeled = []
for src, dest, title in to_render:
try:
key = dest.relative_to(SITE).as_posix()
except ValueError:
key = dest.name
# packages/overview/... also listed as packages/overview
labeled.append((src, dest, title_map.get(key, title_map.get(src.relative_to(ROOT).as_posix() if src.is_relative_to(ROOT) else key, title))))
errors = convert_all_markdown(labeled)
# Also write CONSOLE.pdf next to the markdown in the fleet package (local console)
console_md = ROOT / "packages" / "verae-fleet" / "docs" / "CONSOLE.md"
if console_md.exists():
err = render_md(
console_md,
ROOT / "packages" / "verae-fleet" / "docs" / "CONSOLE.html",
ROOT / "packages" / "verae-fleet" / "docs" / "CONSOLE.pdf",
"Operator console",
)
if err:
errors.append(err)
now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
def cards(mode: str) -> str:
out = []
for title, links in SECTIONS:
lis = []
for path, label in links:
src = ROOT / path
if not src.exists() and path == "OPEN.md":
src = ROOT / "docs" / "OPEN.md"
path = "docs/OPEN.md"
if not src.exists():
continue
pdf_href, html_href = site_href_for(path, src)
if mode == "pdf":
href = pdf_href if (SITE / pdf_href).exists() or src.suffix.lower() in {".pdf", ".yaml", ".yml"} else html_href
badge = "PDF" if href.endswith(".pdf") else src.suffix.lstrip(".").upper() or "FILE"
else:
href = html_href if (SITE / html_href).exists() else path
badge = "MD" if src.suffix.lower() == ".md" else src.suffix.lstrip(".").upper() or "FILE"
lis.append(
f'{label} '
f'{badge} '
f'{path} '
)
out.append(f"")
return "".join(out)
git_repos = [
("master-zapier-plan-draft", "Monorepo (this workspace)"),
("zappier-edge", "Metered HTTPS edge"),
("verae-middleware", "Zapier-facing HTTP + NATS workers"),
("verae-zapier-app", "Full Zapier Platform app"),
("verae-activate", "Activate-now Zapier app"),
("verae-request-splitter", "Hash vs attachment splitter"),
("verae-archive-worm", "Bloom-filtered WORM archive node"),
("verae-archive-aggregator", "Archive reply aggregator"),
("zapier-docs-master", "Master summaries and NATS contracts"),
("verae-tree-node", "Merkle leaf proofs (bulk summaries)"),
("verae-zapier-simulator", "Zapier interface + trace console"),
("zapier-user-docs", "Signup-to-usage user guide"),
("verae-fleet", "Service catalog, replica floors, monitor"),
("overview", "High-level system overview"),
("verae-nats-process", "Template for a new NATS address"),
("verae-ops", "Install, Docker, Proxmox, VMs, dedicated hardware"),
]
git_lis = "".join(
f'{name} '
f'{desc} '
for name, desc in git_repos
)
extra_pdf = [
("docs-master/README.pdf", "Master summaries (docs-master)"),
("docs-master/MESSAGE-FLOWS.pdf", "Numbered message flows"),
("docs-master/modules-and-nats.pdf", "NATS address table"),
("user-docs/README.pdf", "User guide index"),
("packages/verae-zapier-simulator/README.pdf", "Simulator README"),
("packages/verae-fleet/docs/CONSOLE.pdf", "Operator console"),
("overview/README.pdf", "System overview"),
("overview/INDEX.pdf", "Documentation index"),
("docs/modules-pdf/", "Module PDFs (book)"),
("docs/models-pdf/", "Model PDFs (book)"),
("sphinx/index.html", "Sphinx HTML"),
("sphinx/verae-zapier-modules.pdf", "Sphinx LaTeX PDF"),
]
extra_md = [
("docs-master/README.html", "Master summaries (docs-master)"),
("docs-master/MESSAGE-FLOWS.html", "Numbered message flows"),
("docs-master/modules-and-nats.html", "NATS address table"),
("user-docs/README.html", "User guide index"),
("packages/verae-zapier-simulator/README.html", "Simulator README"),
("packages/verae-fleet/docs/CONSOLE.html", "Operator console"),
("overview/README.html", "System overview"),
("overview/INDEX.html", "Documentation index"),
("docs/modules/", "All module markdown"),
("docs/models/", "All model markdown"),
("sphinx/index.html", "Sphinx HTML"),
("sphinx/verae-zapier-modules.pdf", "Sphinx LaTeX PDF"),
]
def extra_section(items: list[tuple[str, str]]) -> str:
lis = "".join(
f'{lab} '
for h, lab in items
if (SITE / h).exists() or h.endswith("/") or h.startswith("sphinx")
)
return f""
rest_pdf = f"""
{cards("pdf")}
Git repositories (Forgejo)
Clone: ssh://git@git.georgelambert.org:2223/marchon/<name>.git (SSH port 2223). Branches main and master.
{extra_section(extra_pdf)}
"""
rest_md = f"""
{cards("md")}
Git repositories (Forgejo)
Clone: ssh://git@git.georgelambert.org:2223/marchon/<name>.git (SSH port 2223).
Master module docs (HTML / Markdown)
{extra_section(extra_md)}
"""
(SITE / "index.html").write_text(
page_shell(now, rest_pdf, switch_to_md=True, title="Verae Time × Zapier documentation"),
encoding="utf-8",
)
(SITE / "index-md.html").write_text(
page_shell(now, rest_md, switch_to_md=False, title="Verae Time × Zapier — Markdown indexes"),
encoding="utf-8",
)
nfiles = sum(1 for _ in SITE.rglob("*") if _.is_file())
npdf = sum(1 for _ in SITE.rglob("*.pdf") if _.is_file())
print(f"site built at {SITE} ({nfiles} files, {npdf} PDFs)")
if errors:
print(f"{len(errors)} render errors:")
for e in errors[:20]:
print(" ", e[:300])
if __name__ == "__main__":
main()