#!/usr/bin/env python3 """Build static documentation site into ./site for zapier.georgelambert.org. Markdown is rendered to colored HTML *and* PDF. The home page (index.html) defaults to PDF links; index-md.html is the Markdown/HTML index. """ from __future__ import annotations import shutil import subprocess from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime, timezone from pathlib import Path ROOT = Path(__file__).resolve().parents[1] SITE = ROOT / "site" CSS = ROOT / "scripts" / "docs-print.css" SECTIONS = [ ( "Start here", [ ("packages/verae-ops/GETTING-STARTED.md", "Bring the system online (install, first-up, monitor)"), ("packages/overview/README.md", "What the system is (plain language)"), ("packages/verae-fleet/docs/CONSOLE.md", "Operator console (loopback Fleet · Trace · Docs)"), ], ), ( "Architecture", [ ("docs/02-architecture/composition.md", "Zappier + middleware composition"), ("docs/02-architecture/overview.md", "Middleware overview"), ("docs/02-architecture/nats-gateway.md", "NATS on NS1"), ("docs/02-architecture/nats-subjects.md", "NATS subjects"), ("docs/02-architecture/archive-nats.md", "Archive NATS, bloom, multi-receipt"), ("docs/02-architecture/modules-and-nats.md", "Module catalog and NATS addresses"), ("docs/02-architecture/tree-nodes.md", "Tree nodes and bulk Merkle summaries"), ("docs/02-architecture/fleet.md", "Fleet replica floors and monitor"), ("packages/overview/03-nats-cluster.md", "NATS.IO 3-server cluster"), ("packages/overview/04-uptime.md", "Uptime / replica floors"), ("packages/overview/05-network-failures.md", "Local network failures"), ("packages/overview/06-address-routing.md", "Address routing"), ("packages/overview/08-diagrams.md", "Architectural diagrams"), ], ), ( "Per-module README (public PDFs)", [ ("packages/zappier/README.md", "zappier-edge (billing + users)"), ("packages/verae-zapier-middleware/README.md", "verae-middleware"), ("packages/verae-zapier/README.md", "verae-zapier-app"), ("packages/verae-activate/README.md", "verae-activate"), ("packages/verae-request-splitter/README.md", "verae-request-splitter"), ("packages/verae-archive-worm/README.md", "verae-archive-worm"), ("packages/verae-archive-aggregator/README.md", "verae-archive-aggregator"), ("packages/verae-tree-node/README.md", "verae-tree-node"), ("packages/verae-fleet/README.md", "verae-fleet"), ("packages/verae-zapier-simulator/README.md", "verae-zapier-simulator"), ("packages/zapier-user-docs/README.md", "zapier-user-docs"), ("packages/docs-master/README.md", "zapier-docs-master"), ("packages/verae-nats-process/README.md", "verae-nats-process"), ], ), ( "Billing, users, portal (zappier-edge)", [ ("packages/zappier/README.md", "zappier-edge README (the billing/user platform)"), ("packages/zappier/docs/USER-MANUAL.md", "Operations & usage"), ("packages/zappier/docs/USER-MANAGEMENT.md", "Admin users, pricing, customer types"), ("packages/zappier/docs/CUSTOMER-PORTAL.md", "Signup, 2FA, API keys, reloads"), ("packages/zappier/docs/ACCOUNTING.md", "Invoices, PO billing, reports"), ("packages/zappier/docs/DEVELOPER.md", "Developer reference"), ], ), ( "Install (detail)", [ ("packages/verae-ops/GETTING-STARTED.md", "Bring online (start here)"), ("packages/verae-ops/02-docker.md", "Docker Compose"), ("packages/verae-ops/03-proxmox.md", "Proxmox"), ("packages/verae-ops/05-dedicated-hardware.md", "Dedicated hardware"), ("packages/verae-ops/06-linking-services.md", "Linking services"), ], ), ( "Samples", [ ("packages/zappier/zapier-app/README.md", "Sample Zapier app (metered zappier API)"), ("packages/verae-activate/README.md", "Activate-now Add Numbers"), ("packages/verae-zapier-simulator/README.md", "Hop-trace simulator"), ], ), ( "User guide (signup → lookup)", [ ("packages/zapier-user-docs/README.md", "User guide index"), ("packages/zapier-user-docs/02-signup-zappier-portal.md", "Sign up"), ("packages/zapier-user-docs/04-register-a-hash.md", "Register a SHA-256"), ("packages/zapier-user-docs/06-lookup-central-chain.md", "Central chain lookup"), ("packages/zapier-user-docs/09-lookup-tree-nodes.md", "Tree-node lookup"), ], ), ( "Product", [ ("docs/01-product/features-a-m.md", "Features a–m"), ("docs/01-product/api-gap-analysis.md", "Verae OpenAPI gaps"), ("docs/01-product/billing-and-keys.md", "Billing and API keys"), ("docs/00-sources/workspace-brief.md", "Original brief"), ("docs/00-sources/provenance.md", "Provenance"), ], ), ( "API specs", [ ("docs/api/middleware-openapi.yaml", "Middleware OpenAPI"), ("packages/zappier/openapi.yaml", "Zappier OpenAPI"), ("docs/00-sources/veraetime-openapi.yaml", "Verae Timestamping OpenAPI snapshot"), ], ), ] def rel(p: Path) -> str: return p.relative_to(ROOT).as_posix() def copy_tree(src: Path, dest: Path, ignore=None) -> None: if not src.exists(): return dest.parent.mkdir(parents=True, exist_ok=True) if src.is_file(): shutil.copy2(src, dest) return shutil.copytree(src, dest, dirs_exist_ok=True, ignore=ignore) def site_href_for(path: str, src: Path) -> tuple[str, str]: """Return (pdf_or_native_href, md_or_html_href) relative to SITE.""" suffix = src.suffix.lower() if suffix == ".md": pdf = Path(path).with_suffix(".pdf").as_posix() html = Path(path).with_suffix(".html").as_posix() return pdf, html return path, path def render_md(src: Path, html_dest: Path, pdf_dest: Path, title: str) -> str: """Pandoc markdown → colored HTML + WeasyPrint PDF. Returns '' on success.""" html_dest.parent.mkdir(parents=True, exist_ok=True) pdf_dest.parent.mkdir(parents=True, exist_ok=True) header = html_dest.with_suffix(".hdr.html") banner = html_dest.with_suffix(".ban.html") rel_src = src.relative_to(ROOT).as_posix() if src.is_relative_to(ROOT) else src.name css_text = CSS.read_text(encoding="utf-8") header.write_text(f"\n", encoding="utf-8") banner.write_text( f'
' f'' f'
Verae Time × Zapier
' f"

{title}

" f'
{rel_src}
' f"
\n", encoding="utf-8", ) resource = str(src.parent) r = subprocess.run( [ "pandoc", str(src), "-o", str(html_dest), "--standalone", f"--resource-path={resource}", "--syntax-highlighting=breezedark", f"--metadata=title={title}", f"--include-in-header={header}", f"--include-before-body={banner}", f"--lua-filter={ROOT / 'scripts' / 'pdf-links.lua'}", ], capture_output=True, text=True, ) header.unlink(missing_ok=True) banner.unlink(missing_ok=True) if r.returncode != 0: return f"pandoc {src}: {r.stderr[-400:]}" w = subprocess.run( ["weasyprint", str(html_dest), str(pdf_dest)], capture_output=True, text=True, ) if w.returncode != 0: return f"weasyprint {src}: {w.stderr[-400:]}" return "" def convert_all_markdown(copied: list[tuple[Path, Path, str]]) -> list[str]: errors = [] n = len(copied) print(f"rendering {n} markdown files to HTML + PDF…") with ThreadPoolExecutor(max_workers=6) as pool: futs = { pool.submit(render_md, src, html, pdf, title): src for src, html, pdf, title in ( ( src, dest.with_suffix(".html"), dest.with_suffix(".pdf"), title, ) for src, dest, title in copied ) } done = 0 for fut in as_completed(futs): err = fut.result() done += 1 if err: errors.append(err) print(f" [{done}/{n}] FAIL {futs[fut].name}") elif done % 25 == 0 or done == n: print(f" [{done}/{n}]") return errors def format_switch(to_md: bool) -> str: if to_md: href, label = "/index-md.html", "Markdown indexes" else: href, label = "/", "PDF catalog" return ( f'{label}' ) def page_shell(now: str, body: str, switch_to_md: bool, title: str) -> str: switch = format_switch(switch_to_md) return f""" {title}
{switch}
zapier.georgelambert.org

Verae Time × Zapier documentation

Project reference: setup, architecture, NATS, module APIs, user guide, and operator console. Catalog defaults to colored PDFs. Generated {now}.

{body}
""" def main() -> None: if SITE.exists(): shutil.rmtree(SITE) SITE.mkdir() copy_tree(ROOT / "docs", SITE / "docs", ignore=shutil.ignore_patterns("_build", ".DS_Store")) sphinx = ROOT / "docs" / "sphinx" / "_build" / "html" if sphinx.exists(): copy_tree(sphinx, SITE / "sphinx") latex_pdf = ROOT / "docs" / "sphinx" / "_build" / "latex" / "verae-zapier-modules.pdf" if latex_pdf.exists(): copy_tree(latex_pdf, SITE / "sphinx" / "verae-zapier-modules.pdf") for pkg in ( "verae-activate", "verae-zapier", "verae-zapier-middleware", "zappier", "verae-request-splitter", "verae-archive-worm", "verae-archive-aggregator", "verae-tree-node", "verae-zapier-simulator", "zapier-user-docs", "verae-fleet", "overview", "verae-nats-process", "docs-master", "verae-ops", ): pkg_root = ROOT / "packages" / pkg if pkg in {"zapier-user-docs", "overview", "docs-master", "verae-ops"}: copy_tree( pkg_root, SITE / "packages" / pkg, ignore=shutil.ignore_patterns("node_modules", ".git", "src", "test", "data"), ) elif pkg == "verae-fleet": for name in ("README.md", "SUMMARY.md", "NATS.md", "SERVICES.md"): p = pkg_root / name if p.exists(): copy_tree(p, SITE / "packages" / pkg / name) copy_tree( pkg_root / "docs", SITE / "packages" / pkg / "docs", ignore=shutil.ignore_patterns("node_modules"), ) elif pkg == "verae-nats-process": for p in pkg_root.glob("*.md"): copy_tree(p, SITE / "packages" / pkg / p.name) else: for name in ("README.md", "SUMMARY.md", "NATS.md"): p = pkg_root / name if p.exists(): copy_tree(p, SITE / "packages" / pkg / name) zap_app = ROOT / "packages" / "zappier" / "zapier-app" / "README.md" if zap_app.exists(): copy_tree(zap_app, SITE / "packages" / "zappier" / "zapier-app" / "README.md") zdocs = ROOT / "packages" / "zappier" / "docs" if zdocs.exists(): copy_tree( zdocs, SITE / "packages" / "zappier" / "docs", ignore=shutil.ignore_patterns("screenshots", "walkthrough", "superpowers"), ) for name in ("README.md", "TODO.md", "OPEN.md"): src = ROOT / name if not src.exists() and name == "OPEN.md": src = ROOT / "docs" / "OPEN.md" if src.exists(): copy_tree(src, SITE / src.name if src.parent == ROOT else SITE / "docs" / "OPEN.md") copy_tree(ROOT / "docs" / "WORK-LOG.md", SITE / "docs" / "WORK-LOG.md") copy_tree(ROOT / "packages" / "docs-master", SITE / "docs-master") copy_tree(ROOT / "packages" / "overview", SITE / "overview") copy_tree(ROOT / "packages" / "zapier-user-docs", SITE / "user-docs", ignore=shutil.ignore_patterns("node_modules")) copy_tree( ROOT / "packages" / "verae-fleet" / "docs", SITE / "packages" / "verae-fleet" / "docs", ignore=shutil.ignore_patterns("node_modules"), ) sim_pub = ROOT / "packages" / "verae-zapier-simulator" / "public" / "index.html" if sim_pub.exists(): copy_tree(sim_pub, SITE / "simulator" / "index.html") r = ROOT / "research" / "zapier" if r.exists(): for fname in ( "README.md", "getting-started.md", "PLATFORM-REFERENCE.md", "FUNCTIONS-REFERENCE.md", "MCP-REFERENCE.md", "LOGIN.md", "docs/zapier-billing.md", ): p = r / fname if p.exists(): copy_tree(p, SITE / "research" / fname) diagrams = r / "docs" / "diagrams" if diagrams.exists(): copy_tree(diagrams, SITE / "research" / "docs" / "diagrams") # Render every published .md from the site copy so relative images resolve. to_render: list[tuple[Path, Path, str]] = [] seen: set[Path] = set() for md in SITE.rglob("*.md"): if not md.is_file(): continue key = md.resolve() if key in seen: continue seen.add(key) to_render.append((md, md, md.stem.replace("-", " "))) # Prefer curated titles for catalog entries title_map = {} for _sec, links in SECTIONS: for path, label in links: title_map[path] = label labeled = [] for src, dest, title in to_render: try: key = dest.relative_to(SITE).as_posix() except ValueError: key = dest.name # packages/overview/... also listed as packages/overview labeled.append((src, dest, title_map.get(key, title_map.get(src.relative_to(ROOT).as_posix() if src.is_relative_to(ROOT) else key, title)))) errors = convert_all_markdown(labeled) # Also write CONSOLE.pdf next to the markdown in the fleet package (local console) console_md = ROOT / "packages" / "verae-fleet" / "docs" / "CONSOLE.md" if console_md.exists(): err = render_md( console_md, ROOT / "packages" / "verae-fleet" / "docs" / "CONSOLE.html", ROOT / "packages" / "verae-fleet" / "docs" / "CONSOLE.pdf", "Operator console", ) if err: errors.append(err) now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC") def cards(mode: str) -> str: out = [] for title, links in SECTIONS: lis = [] for path, label in links: src = ROOT / path if not src.exists() and path == "OPEN.md": src = ROOT / "docs" / "OPEN.md" path = "docs/OPEN.md" if not src.exists(): continue pdf_href, html_href = site_href_for(path, src) if mode == "pdf": href = pdf_href if (SITE / pdf_href).exists() or src.suffix.lower() in {".pdf", ".yaml", ".yml"} else html_href badge = "PDF" if href.endswith(".pdf") else src.suffix.lstrip(".").upper() or "FILE" else: href = html_href if (SITE / html_href).exists() else path badge = "MD" if src.suffix.lower() == ".md" else src.suffix.lstrip(".").upper() or "FILE" lis.append( f'
  • {label}' f'{badge}' f'{path}
  • ' ) out.append(f"

    {title}

    ") return "".join(out) git_repos = [ ("master-zapier-plan-draft", "Monorepo (this workspace)"), ("zappier-edge", "Metered HTTPS edge"), ("verae-middleware", "Zapier-facing HTTP + NATS workers"), ("verae-zapier-app", "Full Zapier Platform app"), ("verae-activate", "Activate-now Zapier app"), ("verae-request-splitter", "Hash vs attachment splitter"), ("verae-archive-worm", "Bloom-filtered WORM archive node"), ("verae-archive-aggregator", "Archive reply aggregator"), ("zapier-docs-master", "Master summaries and NATS contracts"), ("verae-tree-node", "Merkle leaf proofs (bulk summaries)"), ("verae-zapier-simulator", "Zapier interface + trace console"), ("zapier-user-docs", "Signup-to-usage user guide"), ("verae-fleet", "Service catalog, replica floors, monitor"), ("overview", "High-level system overview"), ("verae-nats-process", "Template for a new NATS address"), ("verae-ops", "Install, Docker, Proxmox, VMs, dedicated hardware"), ] git_lis = "".join( f'
  • {name} ' f'{desc}
  • ' for name, desc in git_repos ) extra_pdf = [ ("docs-master/README.pdf", "Master summaries (docs-master)"), ("docs-master/MESSAGE-FLOWS.pdf", "Numbered message flows"), ("docs-master/modules-and-nats.pdf", "NATS address table"), ("user-docs/README.pdf", "User guide index"), ("packages/verae-zapier-simulator/README.pdf", "Simulator README"), ("packages/verae-fleet/docs/CONSOLE.pdf", "Operator console"), ("overview/README.pdf", "System overview"), ("overview/INDEX.pdf", "Documentation index"), ("docs/modules-pdf/", "Module PDFs (book)"), ("docs/models-pdf/", "Model PDFs (book)"), ("sphinx/index.html", "Sphinx HTML"), ("sphinx/verae-zapier-modules.pdf", "Sphinx LaTeX PDF"), ] extra_md = [ ("docs-master/README.html", "Master summaries (docs-master)"), ("docs-master/MESSAGE-FLOWS.html", "Numbered message flows"), ("docs-master/modules-and-nats.html", "NATS address table"), ("user-docs/README.html", "User guide index"), ("packages/verae-zapier-simulator/README.html", "Simulator README"), ("packages/verae-fleet/docs/CONSOLE.html", "Operator console"), ("overview/README.html", "System overview"), ("overview/INDEX.html", "Documentation index"), ("docs/modules/", "All module markdown"), ("docs/models/", "All model markdown"), ("sphinx/index.html", "Sphinx HTML"), ("sphinx/verae-zapier-modules.pdf", "Sphinx LaTeX PDF"), ] def extra_section(items: list[tuple[str, str]]) -> str: lis = "".join( f'
  • {lab}
  • ' for h, lab in items if (SITE / h).exists() or h.endswith("/") or h.startswith("sphinx") ) return f"

    Catalogs

    " rest_pdf = f""" {cards("pdf")}

    Git repositories (Forgejo)

    Clone: ssh://git@git.georgelambert.org:2223/marchon/<name>.git (SSH port 2223). Branches main and master.

    Master module docs

    {extra_section(extra_pdf)} """ rest_md = f""" {cards("md")}

    Git repositories (Forgejo)

    Clone: ssh://git@git.georgelambert.org:2223/marchon/<name>.git (SSH port 2223).

    Master module docs (HTML / Markdown)

    {extra_section(extra_md)} """ (SITE / "index.html").write_text( page_shell(now, rest_pdf, switch_to_md=True, title="Verae Time × Zapier documentation"), encoding="utf-8", ) (SITE / "index-md.html").write_text( page_shell(now, rest_md, switch_to_md=False, title="Verae Time × Zapier — Markdown indexes"), encoding="utf-8", ) nfiles = sum(1 for _ in SITE.rglob("*") if _.is_file()) npdf = sum(1 for _ in SITE.rglob("*.pdf") if _.is_file()) print(f"site built at {SITE} ({nfiles} files, {npdf} PDFs)") if errors: print(f"{len(errors)} render errors:") for e in errors[:20]: print(" ", e[:300]) if __name__ == "__main__": main()