master-zapier-plan-draft/scripts/build-docs-site.py
George Lambert fef7590c27
Some checks are pending
offline / test (push) Waiting to run
Add a NATS cluster message-speed bench (throughput and delay).
Client LXC 510 against nats-a/b/c at several core and JetStream r=3 loads.
2026-09-12 00:56:48 -04:00

1029 lines
46 KiB
Python
Executable file
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
"""Build static documentation site into ./site for zapier.georgelambert.org.
Markdown is rendered to colored HTML *and* PDF. The home page (index.html)
defaults to PDF links; index-md.html is the Markdown/HTML index.
"""
from __future__ import annotations
import os
import shutil
import subprocess
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timezone
from pathlib import Path
from urllib.parse import unquote
ROOT = Path(__file__).resolve().parents[1]
SITE = ROOT / "site"
CSS = ROOT / "scripts" / "docs-print.css"
# Display label only. Catalog file links are relative so the tree can move hosts.
DOCS_PUBLIC_HOST = os.environ.get("DOCS_PUBLIC_HOST", "zapier.georgelambert.org")
# If set (e.g. https://docs.verae-time.net), PDF annotations become absolute.
# Empty (default) writes path-relative URIs that work on any hostname at site root.
DOCS_PUBLIC_URL = os.environ.get("DOCS_PUBLIC_URL", "").rstrip("/")
SECTIONS = [
(
"Start here",
[
("packages/verae-ops/GETTING-STARTED.md", "Bring the system online (install, first-up, monitor)"),
("packages/overview/README.md", "What the system is (plain language)"),
("packages/verae-fleet/docs/CONSOLE.md", "Operator console (Fleet · Trace · Docs on :3850)"),
],
),
(
"Architecture",
[
("docs/02-architecture/composition.md", "Zappier + middleware composition"),
("docs/02-architecture/overview.md", "Middleware overview"),
("docs/02-architecture/nats-gateway.md", "NATS on NS1"),
("docs/02-architecture/nats-subjects.md", "NATS subjects"),
("docs/02-architecture/archive-nats.md", "Archive NATS, bloom, multi-receipt"),
("docs/02-architecture/modules-and-nats.md", "Module catalog and NATS addresses"),
("docs/02-architecture/tree-nodes.md", "Tree nodes and bulk Merkle summaries"),
("docs/02-architecture/fleet.md", "Fleet replica floors and monitor"),
("packages/overview/03-nats-cluster.md", "NATS.IO 3-server cluster"),
("packages/overview/04-uptime.md", "Uptime / replica floors"),
("packages/overview/05-network-failures.md", "Local network failures"),
("packages/overview/06-address-routing.md", "Address routing"),
("packages/overview/08-diagrams.md", "Architectural diagrams"),
("packages/overview/10-docs-sync.md", "Docs sync: portal, IAM, keep, bootstrap"),
("packages/overview/11-hostnames.md", "Service → machine and DNS host names"),
],
),
(
"Per-module README (public PDFs)",
[
("packages/zappier/README.md", "zappier-edge (billing + users)"),
("packages/verae-zapier-middleware/README.md", "verae-middleware"),
("packages/verae-zapier/README.md", "verae-zapier-app"),
("packages/verae-activate/README.md", "verae-activate"),
("packages/verae-request-splitter/README.md", "verae-request-splitter"),
("packages/verae-archive-worm/README.md", "verae-archive-worm"),
("packages/verae-archive-aggregator/README.md", "verae-archive-aggregator"),
("packages/verae-tree-node/README.md", "verae-tree-node"),
("packages/verae-fleet/README.md", "verae-fleet"),
("packages/verae-zapier-simulator/README.md", "verae-zapier-simulator"),
("packages/zapier-user-docs/README.md", "zapier-user-docs"),
("packages/docs-master/README.md", "zapier-docs-master"),
("packages/verae-nats-process/README.md", "verae-nats-process"),
("packages/verae-staff-iam/README.md", "verae-staff-iam"),
("packages/verae-staff-session/README.md", "verae-staff-session"),
("packages/verae-staff-ui/README.md", "verae-staff-ui"),
("packages/verae-keep/README.md", "verae-keep"),
("packages/verae-bootstrap/README.md", "verae-bootstrap (clone + deps per server type)"),
("packages/verae-uptime/README.md", "Off-box HTTPS door watch"),
("packages/verae-backup/README.md", "Backup SQLite / JetStream / worm+tree"),
("packages/verae-deploy/README.md", "Tagged deploy on the target OS"),
("packages/verae-proxmox/README.md", "Proxmox LXC worker on NS1 vmbr1"),
("packages/verae-nats-cluster/README.md", "Proxmox 3-node NATS JetStream cluster"),
],
),
(
"Billing, users, portal (zappier-edge)",
[
("packages/zappier/README.md", "zappier-edge README (the billing/user platform)"),
("packages/zappier/docs/USER-MANUAL.md", "Operations & usage"),
("packages/zappier/docs/USER-MANAGEMENT.md", "Admin users, pricing, customer types"),
("packages/zappier/docs/CUSTOMER-PORTAL.md", "Signup, 2FA, API keys, reloads"),
("packages/zappier/docs/ACCOUNTING.md", "Invoices, PO billing, reports"),
("packages/zappier/docs/DEVELOPER.md", "Developer reference"),
("packages/zappier-account-balance/README.md", "Account balance (NATS SoT)"),
("packages/zappier-customer-service/README.md", "Customer-service credits API"),
("packages/zappier-sales-pricing/README.md", "Sales per-customer pricing API"),
("packages/zappier-accounting-export/README.md", "QuickBooks / accounting export"),
("packages/verae-access-authz/README.md", "Access-plane authorization"),
("packages/verae-access-web/README.md", "Direct web access plane"),
("packages/verae-access-api/README.md", "Direct customer API plane"),
("packages/verae-access-leaf/README.md", "S2S NATS leaf plane"),
("packages/verae-access-zapier/README.md", "Zapier-only access plane"),
("packages/verae-access-staff/README.md", "Staff access plane"),
("packages/zappier-identity/README.md", "veraeUserId identity mailbox"),
("packages/verae-jobs-events/README.md", "jobs.events mailbox"),
("packages/verae-nats-accounts/README.md", "NATS leaf/internal policy"),
("packages/zapier-decisions/README.md", "Decisions log"),
("packages/ui-docs/README.md", "UI walkthrough, screenshots, review PDF"),
],
),
(
"Install (detail)",
[
("packages/verae-ops/GETTING-STARTED.md", "Bring online (start here)"),
("packages/verae-ops/02-docker.md", "Docker Compose"),
("packages/verae-ops/03-proxmox.md", "Proxmox"),
("packages/verae-ops/05-dedicated-hardware.md", "Dedicated hardware"),
("packages/verae-ops/06-linking-services.md", "Linking services"),
],
),
(
"Samples",
[
("packages/zappier/zapier-app/README.md", "Sample Zapier app (metered zappier API)"),
("packages/verae-activate/README.md", "Activate-now Add Numbers"),
("packages/verae-zapier-simulator/README.md", "Hop-trace simulator"),
],
),
(
"User guide (signup → lookup)",
[
("packages/zapier-user-docs/README.md", "User guide index"),
("packages/zapier-user-docs/02-signup-zappier-portal.md", "Sign up"),
("packages/zapier-user-docs/04-register-a-hash.md", "Register a SHA-256"),
("packages/zapier-user-docs/06-lookup-central-chain.md", "Central chain lookup"),
("packages/zapier-user-docs/09-lookup-tree-nodes.md", "Tree-node lookup"),
],
),
(
"Product",
[
("docs/01-product/features-a-m.md", "Features am"),
("docs/01-product/api-gap-analysis.md", "Verae OpenAPI gaps"),
("docs/01-product/billing-and-keys.md", "Billing and API keys"),
("docs/00-sources/workspace-brief.md", "Original brief"),
("docs/00-sources/provenance.md", "Provenance"),
],
),
(
"API specs",
[
("docs/api/middleware-openapi.yaml", "Middleware OpenAPI"),
("packages/zappier/openapi.yaml", "Zappier OpenAPI"),
("docs/00-sources/veraetime-openapi.yaml", "Verae Timestamping OpenAPI snapshot"),
],
),
]
def rel(p: Path) -> str:
return p.relative_to(ROOT).as_posix()
def copy_tree(src: Path, dest: Path, ignore=None) -> None:
if not src.exists():
return
dest.parent.mkdir(parents=True, exist_ok=True)
if src.is_file():
shutil.copy2(src, dest)
return
shutil.copytree(src, dest, dirs_exist_ok=True, ignore=ignore)
def site_href_for(path: str, src: Path) -> tuple[str, str]:
"""Return (pdf_or_native_href, md_or_html_href) relative to SITE."""
suffix = src.suffix.lower()
if suffix == ".md":
pdf = Path(path).with_suffix(".pdf").as_posix()
html = Path(path).with_suffix(".html").as_posix()
return pdf, html
return path, path
def render_md(src: Path, html_dest: Path, pdf_dest: Path, title: str) -> str:
"""Pandoc markdown → colored HTML + WeasyPrint PDF. Returns '' on success."""
html_dest.parent.mkdir(parents=True, exist_ok=True)
pdf_dest.parent.mkdir(parents=True, exist_ok=True)
header = html_dest.with_suffix(".hdr.html")
banner = html_dest.with_suffix(".ban.html")
rel_src = src.relative_to(ROOT).as_posix() if src.is_relative_to(ROOT) else src.name
css_text = CSS.read_text(encoding="utf-8")
header.write_text(f"<style>{css_text}</style>\n", encoding="utf-8")
banner.write_text(
f'<div class="doc-banner">'
f'<nav class="site"><a href="/">{DOCS_PUBLIC_HOST}</a>'
f' · <a href="/index-md.html">Markdown indexes</a></nav>'
f'<div class="kicker">Verae Time × Zapier</div>'
f"<h1>{title}</h1>"
f'<div class="source-path">{rel_src}</div>'
f"</div>\n",
encoding="utf-8",
)
resource = str(src.parent)
r = subprocess.run(
[
"pandoc",
str(src),
"-o",
str(html_dest),
"--standalone",
f"--resource-path={resource}",
"--syntax-highlighting=breezedark",
f"--metadata=title={title}",
f"--include-in-header={header}",
f"--include-before-body={banner}",
f"--lua-filter={ROOT / 'scripts' / 'pdf-links.lua'}",
],
capture_output=True,
text=True,
)
header.unlink(missing_ok=True)
banner.unlink(missing_ok=True)
if r.returncode != 0:
return f"pandoc {src}: {r.stderr[-400:]}"
w = subprocess.run(
["weasyprint", str(html_dest), str(pdf_dest)],
capture_output=True,
text=True,
)
if w.returncode != 0:
return f"weasyprint {src}: {w.stderr[-400:]}"
return ""
def _file_index(site: Path) -> dict[str, list[str]]:
idx: dict[str, list[str]] = {}
for p in site.rglob("*"):
if p.is_file():
rel = p.relative_to(site).as_posix()
idx.setdefault(p.name, []).append(rel)
return idx
_PREFERRED_PREFIXES = (
"overview/",
"packages/overview/",
"packages/verae-ops/",
"packages/verae-fleet/docs/",
"user-docs/",
"packages/zapier-user-docs/",
"docs/02-architecture/",
"docs/modules/",
"docs-master/",
"packages/zapier-decisions/",
"docs/modules-pdf/",
"docs/models-pdf/",
"sphinx/",
)
def _pick_hit(hits: list[str]) -> str:
for prefix in _PREFERRED_PREFIXES:
for h in hits:
if h.startswith(prefix):
return h
return hits[0]
def _web_target(site: Path, rel: str, index: dict[str, list[str]] | None = None) -> str:
"""Map a site-relative path to a file that actually exists."""
rel = unquote(rel).split("#", 1)[0].lstrip("/")
if rel.endswith(".html.pdf"):
rel = rel[:-4]
aliases = {
"index-md.pdf": "index-md.html",
"index-md.md": "index-md.html",
"docs/architecture/overview.pdf": "docs/02-architecture/overview.pdf",
"docs/architecture/nats-subjects.pdf": "docs/02-architecture/nats-subjects.pdf",
"docs/architecture/nats-gateway.pdf": "docs/02-architecture/nats-gateway.pdf",
"docs/architecture/composition.pdf": "docs/02-architecture/composition.pdf",
"docs/architecture/fleet.pdf": "docs/02-architecture/fleet.pdf",
"docs/sphinx/_build/html/index.pdf": "sphinx/index.html",
"docs/sphinx/_build/html/index.html": "sphinx/index.html",
}
rel = aliases.get(rel, rel)
candidates = [rel]
if rel.endswith(".pdf"):
candidates.append(rel[:-4] + ".html")
candidates.append(rel[:-4] + ".md")
if rel.endswith(".md"):
candidates.append(rel[:-3] + ".pdf")
candidates.append(rel[:-3] + ".html")
stem = Path(rel).name
if not stem or stem in {".", "/"}:
return rel or "index.html"
stem_pdf = Path(stem).with_suffix(".pdf").as_posix()
stem_html = Path(stem).with_suffix(".html").as_posix()
for folder in (
"overview",
"packages/overview",
"packages/verae-ops",
"packages/verae-fleet/docs",
"user-docs",
"packages/zapier-user-docs",
"docs/02-architecture",
"docs/modules-pdf",
"docs/models-pdf",
"packages/zapier-decisions",
"docs-master",
"sphinx",
):
candidates.append(f"{folder}/{stem_pdf}")
candidates.append(f"{folder}/{stem_html}")
for c in candidates:
if (site / c).is_file():
return c
if (site / c).is_dir():
for name in ("index.html", "index.pdf", "README.pdf", "README.html"):
if (site / c / name).is_file():
return f"{c.rstrip('/')}/{name}"
if index:
names = [stem, stem_pdf, stem_html]
if stem.endswith(".md"):
names.append(Path(stem).with_suffix(".pdf").as_posix())
names.append(Path(stem).with_suffix(".html").as_posix())
for name in names:
hits = index.get(name) or []
if hits:
return _pick_hit(hits)
return rel
_DOCS_ORIGINS = (
"https://zapier.georgelambert.org/",
"http://zapier.georgelambert.org/",
)
def _href_for(pdf: Path, site: Path, target: str, frag: str = "") -> str:
target = (target or "index.html").lstrip("/") or "index.html"
dest = site / target
if DOCS_PUBLIC_URL:
return DOCS_PUBLIC_URL + "/" + target + frag
rel = os.path.relpath(dest.resolve(), pdf.parent.resolve())
return Path(rel).as_posix() + frag
def rewrite_pdf_uris(site: Path) -> None:
"""Turn WeasyPrint file:///… annotations into host-relative catalog paths.
Default is a path relative to the PDF (works on any hostname at the site
root). Set DOCS_PUBLIC_URL to emit absolute URLs instead.
"""
try:
from pypdf import PdfReader, PdfWriter
from pypdf.generic import NameObject, create_string_object
except ImportError:
print("pypdf missing; PDF URI rewrite skipped")
return
site = site.resolve()
prefix = site.as_uri().rstrip("/") + "/"
origins = list(_DOCS_ORIGINS)
if DOCS_PUBLIC_URL:
origins.append(DOCS_PUBLIC_URL.rstrip("/") + "/")
index = _file_index(site)
def resolve(pdf: Path, raw: str) -> str:
frag = ""
if "#" in raw:
raw, frag = raw.split("#", 1)
frag = "#" + frag
target = None
if raw.startswith(prefix):
target = _web_target(site, raw[len(prefix) :], index)
elif raw.startswith("file:"):
name = unquote(raw.rsplit("/", 1)[-1] if "/" in raw else raw)
if not name or name in {".", "/", "file:"}:
target = "index.html"
else:
hit = _web_target(site, name, index)
if (site / hit).exists():
target = hit
elif "index-md" in raw:
target = "index-md.html"
else:
target = "index.html"
else:
for origin in origins:
if raw.startswith(origin):
target = _web_target(site, raw[len(origin) :], index)
break
if target is None:
return raw + frag
return _href_for(pdf, site, target, frag)
n_pdf = 0
n_fix = 0
for pdf in site.rglob("*.pdf"):
try:
reader = PdfReader(str(pdf))
except Exception:
continue
changed = False
for page in reader.pages:
annots = page.get("/Annots")
if not annots:
continue
for annot in annots:
obj = annot.get_object()
action = obj.get("/A")
if not action:
continue
uri = action.get("/URI")
if not uri:
continue
raw = str(uri)
new = resolve(pdf, raw)
if new != raw:
action[NameObject("/URI")] = create_string_object(new)
changed = True
n_fix += 1
if changed:
writer = PdfWriter(clone_from=reader)
tmp = pdf.with_suffix(".pdf.tmp")
with tmp.open("wb") as fh:
writer.write(fh)
tmp.replace(pdf)
n_pdf += 1
mode = DOCS_PUBLIC_URL + "/" if DOCS_PUBLIC_URL else "relative paths"
print(f"rewrote {n_fix} PDF URIs in {n_pdf} files → {mode}")
def convert_all_markdown(copied: list[tuple[Path, Path, str]]) -> list[str]:
errors = []
n = len(copied)
print(f"rendering {n} markdown files to HTML + PDF…")
with ThreadPoolExecutor(max_workers=6) as pool:
futs = {
pool.submit(render_md, src, html, pdf, title): src
for src, html, pdf, title in (
(
src,
dest.with_suffix(".html"),
dest.with_suffix(".pdf"),
title,
)
for src, dest, title in copied
)
}
done = 0
for fut in as_completed(futs):
err = fut.result()
done += 1
if err:
errors.append(err)
print(f" [{done}/{n}] FAIL {futs[fut].name}")
elif done % 25 == 0 or done == n:
print(f" [{done}/{n}]")
return errors
def format_switch(to_md: bool) -> str:
if to_md:
href, label = "/index-md.html", "Markdown indexes"
else:
href, label = "/", "PDF catalog"
return (
f'<a class="format-switch" href="{href}">{label}</a>'
)
def page_shell(now: str, body: str, switch_to_md: bool, title: str) -> str:
switch = format_switch(switch_to_md)
return f"""<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8"/>
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<title>{title}</title>
<link rel="icon" href="data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 32 32'%3E%3Crect width='32' height='32' rx='8' fill='%234f46e5'/%3E%3C/svg%3E"/>
<style>
:root {{ --ink:#171a26; --muted:#5b6178; --line:#d9dce8; --bg:#f4f5fb; --accent:#4f46e5; --accent-deep:#312e81; --soft:#eef0fe; }}
body {{ margin:0; font:16px/1.5 -apple-system, "SF Pro Text", "Segoe UI", "PingFang SC", sans-serif; color:var(--ink); background:var(--bg); }}
.skip {{ position:absolute; left:-999px; }}
.skip:focus {{ left:1rem; top:1rem; z-index:20; background:#fff; color:var(--accent); padding:.5rem .9rem; border-radius:8px; }}
a:focus-visible, input:focus-visible {{ outline:2px solid #fff; outline-offset:2px; }}
header {{ background:linear-gradient(160deg,#312e81 0%,#4f46e5 60%,#7c74f0 100%); color:#eef0fe; padding:2rem 1.5rem 1.4rem; position:relative; }}
header p {{ max-width:44rem; color:#e4e7ff; }}
.kicker {{ letter-spacing:.12em; text-transform:uppercase; font:700 11px system-ui; opacity:.75; }}
.format-switch {{
position:absolute; top:1.25rem; right:1.5rem;
font:650 13px system-ui, sans-serif; color:#fff; text-decoration:none;
background:rgba(255,255,255,.16); border:1px solid rgba(255,255,255,.35);
padding:.4rem .75rem; border-radius:999px;
}}
.format-switch:hover {{ background:rgba(255,255,255,.28); }}
main {{ max-width:64rem; margin:0 auto; padding:1.5rem 1.25rem 3rem; }}
h1 {{ margin:.4rem 0 .6rem; font-size:1.85rem; }}
h2 {{ border-top:1px solid var(--line); padding-top:.8rem; margin-top:1.8rem; color:var(--accent-deep); }}
ul {{ padding-left:1.2rem; }}
li {{ margin:.35rem 0; }}
a {{ color:var(--accent); }}
.path {{ color:var(--muted); font:12px ui-monospace, Menlo, monospace; margin-left:.4rem; }}
.badge {{ font:700 10px system-ui; letter-spacing:.04em; text-transform:uppercase;
background:var(--soft); color:var(--accent); padding:.1rem .35rem; border-radius:4px; margin-left:.25rem; }}
input {{ width:100%; padding:.55rem .7rem; font:16px system-ui; border:1px solid var(--line); border-radius:6px; }}
.cards {{ display:grid; grid-template-columns:repeat(auto-fill,minmax(240px,1fr)); gap:.75rem; }}
.card {{ background:#fff; border:1px solid var(--line); border-radius:12px; padding:.85rem .95rem;
box-shadow:0 1px 2px rgba(23,26,38,.05), 0 8px 24px rgba(23,26,38,.06); }}
.card h3 {{ margin:0 0 .35rem; font-size:14px; }}
.card p {{ margin:0 0 .7rem; color:var(--muted); font-size:13px; }}
.card a.act {{ display:inline-block; background:var(--accent); color:#fff; text-decoration:none;
border-radius:8px; padding:.28rem .6rem; font:650 12px system-ui; }}
.live {{ display:grid; grid-template-columns:repeat(auto-fill,minmax(280px,1fr)); gap:.9rem; }}
.live article {{ background:#fff; border:1px solid var(--line); border-radius:12px; overflow:hidden;
box-shadow:0 1px 2px rgba(23,26,38,.05), 0 8px 24px rgba(23,26,38,.06); }}
.live img {{ display:block; width:100%; height:140px; object-fit:cover; object-position:top; background:#eef0fe; }}
.live .body {{ padding:.75rem .9rem 1rem; }}
.live h3 {{ margin:0 0 .35rem; font-size:14px; }}
.live p {{ margin:0 0 .65rem; color:var(--muted); font-size:13px; }}
.live a.host {{ display:block; font:12px ui-monospace, Menlo, monospace; color:var(--accent); word-break:break-all; margin-bottom:.55rem; }}
</style>
</head>
<body>
<a class="skip" href="#main">Skip to content</a>
<header>
{switch}
<div class="kicker">{DOCS_PUBLIC_HOST}</div>
<h1>Verae Time × Zapier documentation</h1>
<p>Project reference: setup, architecture, NATS, module APIs, user guide, and operator console.
Catalog defaults to <strong>colored PDFs</strong>. Generated {now}.</p>
</header>
<main id="main">
<p><input id="q" type="search" placeholder="Filter links…" /></p>
{body}
</main>
<script>
const q = document.getElementById('q');
q.addEventListener('input', () => {{
const v = q.value.toLowerCase();
document.querySelectorAll('.card').forEach(card => {{
card.style.display = card.textContent.toLowerCase().includes(v) ? '' : 'none';
}});
}});
</script>
</body>
</html>
"""
def main() -> None:
if SITE.exists():
shutil.rmtree(SITE)
SITE.mkdir()
copy_tree(ROOT / "docs", SITE / "docs", ignore=shutil.ignore_patterns("_build", ".DS_Store"))
sphinx = ROOT / "docs" / "sphinx" / "_build" / "html"
if sphinx.exists():
copy_tree(sphinx, SITE / "sphinx")
latex_pdf = ROOT / "docs" / "sphinx" / "_build" / "latex" / "verae-zapier-modules.pdf"
if latex_pdf.exists():
copy_tree(latex_pdf, SITE / "sphinx" / "verae-zapier-modules.pdf")
for pkg in (
"verae-activate",
"verae-zapier",
"verae-zapier-middleware",
"zappier",
"verae-request-splitter",
"verae-archive-worm",
"verae-archive-aggregator",
"verae-tree-node",
"verae-zapier-simulator",
"zapier-user-docs",
"verae-fleet",
"overview",
"verae-nats-process",
"docs-master",
"verae-ops",
"zappier-account-balance",
"zappier-customer-service",
"zappier-sales-pricing",
"zappier-accounting-export",
"verae-access-authz",
"verae-access-web",
"verae-access-api",
"verae-access-leaf",
"verae-access-zapier",
"verae-access-staff",
"zappier-identity",
"verae-jobs-events",
"verae-nats-accounts",
"zapier-decisions",
"ui-docs",
"verae-staff-session",
"verae-staff-ui",
"verae-staff-iam",
"verae-keep",
"verae-bootstrap",
"verae-uptime",
"verae-backup",
"verae-deploy",
"verae-proxmox",
"verae-nats-cluster",
):
pkg_root = ROOT / "packages" / pkg
if pkg in {"zapier-user-docs", "overview", "docs-master", "verae-ops"}:
copy_tree(
pkg_root,
SITE / "packages" / pkg,
ignore=shutil.ignore_patterns("node_modules", ".git", "src", "test", "data"),
)
elif pkg == "verae-fleet":
for name in ("README.md", "SUMMARY.md", "NATS.md", "SERVICES.md"):
p = pkg_root / name
if p.exists():
copy_tree(p, SITE / "packages" / pkg / name)
copy_tree(
pkg_root / "docs",
SITE / "packages" / pkg / "docs",
ignore=shutil.ignore_patterns("node_modules"),
)
copy_tree(pkg_root / "services", SITE / "packages" / pkg / "services")
elif pkg == "verae-nats-process":
for p in pkg_root.glob("*.md"):
copy_tree(p, SITE / "packages" / pkg / p.name)
else:
extras = ("README.md", "SUMMARY.md", "NATS.md")
if pkg == "zapier-decisions":
extras = extras + ("LOG.md", "TODO.md", "STATUS.md")
if pkg == "verae-bootstrap":
extras = extras + ("HOST-DEPS.md",)
if pkg == "verae-nats-cluster":
extras = extras + ("BENCH.md",)
for name in extras:
p = pkg_root / name
if p.exists():
copy_tree(p, SITE / "packages" / pkg / name)
if pkg == "zapier-decisions":
copy_tree(pkg_root / "decisions", SITE / "packages" / pkg / "decisions")
ui_docs = ROOT / "packages" / "ui-docs"
for name in ("WALKTHROUGH.md", "REPORT.md", "UI-REVIEW.pdf"):
p = ui_docs / name
if p.exists():
copy_tree(p, SITE / "packages" / "ui-docs" / name)
copy_tree(
ui_docs / "screenshots",
SITE / "packages" / "ui-docs" / "screenshots",
ignore=shutil.ignore_patterns(".DS_Store"),
)
zap_app = ROOT / "packages" / "zappier" / "zapier-app" / "README.md"
if zap_app.exists():
copy_tree(zap_app, SITE / "packages" / "zappier" / "zapier-app" / "README.md")
zdocs = ROOT / "packages" / "zappier" / "docs"
if zdocs.exists():
copy_tree(
zdocs,
SITE / "packages" / "zappier" / "docs",
ignore=shutil.ignore_patterns("screenshots", "walkthrough", "superpowers"),
)
copy_tree(ROOT / "packages" / "zappier" / "openapi.yaml", SITE / "packages" / "zappier" / "openapi.yaml")
for name in ("README.md", "TODO.md", "OPEN.md"):
src = ROOT / name
if not src.exists() and name == "OPEN.md":
src = ROOT / "docs" / "OPEN.md"
if src.exists():
copy_tree(src, SITE / src.name if src.parent == ROOT else SITE / "docs" / "OPEN.md")
copy_tree(ROOT / "docs" / "WORK-LOG.md", SITE / "docs" / "WORK-LOG.md")
copy_tree(ROOT / "packages" / "docs-master", SITE / "docs-master")
copy_tree(ROOT / "packages" / "overview", SITE / "overview")
copy_tree(ROOT / "packages" / "zapier-user-docs", SITE / "user-docs", ignore=shutil.ignore_patterns("node_modules"))
copy_tree(
ROOT / "packages" / "verae-fleet" / "docs",
SITE / "packages" / "verae-fleet" / "docs",
ignore=shutil.ignore_patterns("node_modules"),
)
sim_pub = ROOT / "packages" / "verae-zapier-simulator" / "public" / "index.html"
if sim_pub.exists():
copy_tree(sim_pub, SITE / "simulator" / "index.html")
r = ROOT / "research" / "zapier"
if r.exists():
for fname in (
"README.md",
"getting-started.md",
"PLATFORM-REFERENCE.md",
"FUNCTIONS-REFERENCE.md",
"MCP-REFERENCE.md",
"LOGIN.md",
"docs/zapier-billing.md",
):
p = r / fname
if p.exists():
copy_tree(p, SITE / "research" / fname)
diagrams = r / "docs" / "diagrams"
if diagrams.exists():
copy_tree(diagrams, SITE / "research" / "docs" / "diagrams")
def write_book_index(folder: Path, title: str, patterns: tuple[str, ...]) -> None:
if not folder.is_dir():
return
files: list[Path] = []
for pat in patterns:
files.extend(p for p in folder.rglob(pat) if p.is_file())
skip = {"index.md", "index.html", "index.pdf"}
files = sorted({p for p in files if p.name not in skip})
lines = [f"# {title}", "", f"{len(files)} files in this book.", ""]
for p in files:
rel = p.relative_to(folder).as_posix()
lines.append(f"- [{rel}]({rel})")
(folder / "index.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
write_book_index(SITE / "docs" / "modules-pdf", "Module PDFs (book)", ("*.pdf",))
write_book_index(SITE / "docs" / "models-pdf", "Model PDFs (book)", ("*.pdf",))
write_book_index(SITE / "packages" / "ui-docs" / "screenshots", "UI screenshots", ("*.png", "*.jpg", "*.webp"))
write_book_index(SITE / "packages" / "overview" / "diagrams", "Overview diagrams", ("*.svg", "*.png"))
write_book_index(SITE / "overview" / "diagrams", "Overview diagrams", ("*.svg", "*.png"))
write_book_index(SITE / "packages" / "zapier-decisions" / "decisions", "Decisions", ("*.md", "*.pdf"))
write_book_index(SITE / "packages" / "verae-fleet" / "services", "Fleet service JSON", ("*.json",))
write_book_index(SITE / "docs" / "00-sources" / "docs" / "diagrams", "Research diagrams", ("*.svg", "*.png"))
# Render every published .md from the site copy so relative images resolve.
to_render: list[tuple[Path, Path, str]] = []
seen: set[Path] = set()
for md in SITE.rglob("*.md"):
if not md.is_file():
continue
if "research" in md.parts:
continue
# Sphinx HTML/PDF are copied from _build; don't WeasyPrint the rst sources.
if "sphinx" in md.parts:
continue
key = md.resolve()
if key in seen:
continue
seen.add(key)
to_render.append((md, md, md.stem.replace("-", " ")))
# Prefer curated titles for catalog entries
title_map = {}
for _sec, links in SECTIONS:
for path, label in links:
title_map[path] = label
labeled = []
for src, dest, title in to_render:
try:
key = dest.relative_to(SITE).as_posix()
except ValueError:
key = dest.name
# packages/overview/... also listed as packages/overview
labeled.append((src, dest, title_map.get(key, title_map.get(src.relative_to(ROOT).as_posix() if src.is_relative_to(ROOT) else key, title))))
errors = convert_all_markdown(labeled)
rewrite_pdf_uris(SITE)
# Also write CONSOLE.pdf next to the markdown in the fleet package (local console)
console_md = ROOT / "packages" / "verae-fleet" / "docs" / "CONSOLE.md"
if console_md.exists():
err = render_md(
console_md,
ROOT / "packages" / "verae-fleet" / "docs" / "CONSOLE.html",
ROOT / "packages" / "verae-fleet" / "docs" / "CONSOLE.pdf",
"Operator console",
)
if err:
errors.append(err)
now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
def cards(mode: str) -> str:
out = []
for title, links in SECTIONS:
lis = []
for path, label in links:
src = ROOT / path
if not src.exists() and path == "OPEN.md":
src = ROOT / "docs" / "OPEN.md"
path = "docs/OPEN.md"
if not src.exists():
continue
pdf_href, html_href = site_href_for(path, src)
if mode == "pdf":
href = pdf_href if (SITE / pdf_href).exists() or src.suffix.lower() in {".pdf", ".yaml", ".yml"} else html_href
badge = "PDF" if href.endswith(".pdf") else src.suffix.lstrip(".").upper() or "FILE"
else:
href = html_href if (SITE / html_href).exists() else path
badge = "MD" if src.suffix.lower() == ".md" else src.suffix.lstrip(".").upper() or "FILE"
lis.append(
f'<article class="card"><h3>{label}</h3>'
f'<p><span class="badge">{badge}</span><span class="path">{path}</span></p>'
f'<a class="act" href="{href}">Open {badge}</a></article>'
)
out.append(f'<section><h2>{title}</h2><div class="cards">{"".join(lis)}</div></section>')
return "".join(out)
git_repos = [
("master-zapier-plan-draft", "Monorepo (this workspace)"),
("zappier-edge", "Metered HTTPS edge"),
("verae-middleware", "Zapier-facing HTTP + NATS workers"),
("verae-zapier-app", "Full Zapier Platform app"),
("verae-activate", "Activate-now Zapier app"),
("verae-request-splitter", "Hash vs attachment splitter"),
("verae-archive-worm", "Bloom-filtered WORM archive node"),
("verae-archive-aggregator", "Archive reply aggregator"),
("zapier-docs-master", "Master summaries and NATS contracts"),
("verae-tree-node", "Merkle leaf proofs (bulk summaries)"),
("verae-zapier-simulator", "Zapier interface + trace console"),
("zapier-user-docs", "Signup-to-usage user guide"),
("verae-fleet", "Service catalog, replica floors, monitor"),
("overview", "High-level system overview"),
("verae-nats-process", "Template for a new NATS address"),
("verae-ops", "Install, Docker, Proxmox, VMs, dedicated hardware"),
("zappier-account-balance", "NATS account balances"),
("zappier-customer-service", "CS credits"),
("zappier-sales-pricing", "Sales per-customer pricing"),
("zappier-accounting-export", "QuickBooks export"),
("verae-access-authz", "Access-plane authorization"),
("verae-access-web", "Direct web access"),
("verae-access-api", "Direct customer API"),
("verae-access-leaf", "S2S NATS leaf"),
("verae-access-zapier", "Zapier access plane"),
("verae-access-staff", "Staff access plane"),
("zappier-identity", "veraeUserId bind/lookup"),
("verae-jobs-events", "jobs.events mailbox"),
("verae-nats-accounts", "NATS leaf/internal policy"),
("zapier-decisions", "Decisions and action log"),
("UI-Docs", "UI walkthrough and screenshot PDF"),
("verae-staff-session", "Staff cookie login"),
("verae-staff-ui", "Shared staff review HTML"),
("verae-staff-iam", "Staff users, roles, permissions"),
("verae-keep", "Host keep-alive"),
("verae-bootstrap", "Clone + deps per server type"),
("verae-uptime", "Off-box HTTPS door watch"),
("verae-backup", "Backup SQLite / JetStream / worm+tree"),
("verae-deploy", "Tagged deploy on the target OS"),
("verae-proxmox", "Proxmox LXC worker (NS1 vmbr1)"),
("verae-nats-cluster", "Proxmox 3-node NATS JetStream cluster"),
]
git_lis = "".join(
f'<li><a href="https://git.georgelambert.org/marchon/{name}">{name}</a> '
f'<span class="path">{desc}</span></li>'
for name, desc in git_repos
)
extra_pdf = [
("docs-master/README.pdf", "Master summaries (docs-master)"),
("docs-master/CATALOG-HOST.pdf", "Move the catalog to another hostname"),
("docs-master/HOST-DEPS.pdf", "Host OS / Node / NATS installer"),
("docs-master/MESSAGE-FLOWS.pdf", "Numbered message flows"),
("docs-master/modules-and-nats.pdf", "NATS address table"),
("packages/verae-bootstrap/HOST-DEPS.pdf", "Host OS / Node / NATS installer"),
("user-docs/README.pdf", "User guide index"),
("packages/verae-zapier-simulator/README.pdf", "Simulator README"),
("packages/verae-fleet/docs/CONSOLE.pdf", "Operator console"),
("packages/ui-docs/UI-REVIEW.pdf", "UI review (screenshots + live doors)"),
("packages/zapier-decisions/STATUS.pdf", "Current lab status (2026-09-12)"),
("packages/verae-nats-cluster/BENCH.pdf", "NATS cluster throughput and delay (2026-09-12)"),
("overview/README.pdf", "System overview"),
("overview/INDEX.pdf", "Documentation index"),
("docs/modules-pdf/index.pdf", "Module PDFs (book)"),
("docs/models-pdf/index.pdf", "Model PDFs (book)"),
("sphinx/index.html", "Sphinx HTML"),
("sphinx/verae-zapier-modules.pdf", "Sphinx LaTeX PDF"),
]
extra_md = [
("docs-master/README.html", "Master summaries (docs-master)"),
("docs-master/CATALOG-HOST.html", "Move the catalog to another hostname"),
("docs-master/HOST-DEPS.html", "Host OS / Node / NATS installer"),
("packages/zapier-decisions/STATUS.html", "Current lab status"),
("packages/verae-nats-cluster/BENCH.html", "NATS cluster throughput and delay"),
("docs-master/MESSAGE-FLOWS.html", "Numbered message flows"),
("docs-master/modules-and-nats.html", "NATS address table"),
("packages/verae-bootstrap/HOST-DEPS.html", "Host OS / Node / NATS installer"),
("user-docs/README.html", "User guide index"),
("packages/verae-zapier-simulator/README.html", "Simulator README"),
("packages/verae-fleet/docs/CONSOLE.html", "Operator console"),
("overview/README.html", "System overview"),
("overview/INDEX.html", "Documentation index"),
("docs/modules/README.html", "All module markdown"),
("docs/models/README.html", "All model markdown"),
("docs/modules-pdf/index.html", "Module PDFs (book)"),
("docs/models-pdf/index.html", "Model PDFs (book)"),
("sphinx/index.html", "Sphinx HTML"),
("sphinx/verae-zapier-modules.pdf", "Sphinx LaTeX PDF"),
]
LIVE_DOORS = [
("portal.png", "https://portal.zapier.georgelambert.org/portal/", "Customer portal",
"Public signup, sign-in, API key, statement, invoices. Access-web door."),
("api.png", "https://api.zapier.georgelambert.org/health", "Customer API",
"Direct customer API (x-api-key). Not Zapier."),
("leaf.png", "https://leaf.zapier.georgelambert.org/health", "S2S leaf",
"Server-to-server NATS leaf plane. No billing subjects."),
("zap.png", "https://zap.zapier.georgelambert.org/health", "Zapier plane",
"Only Zapier Platform HTTPS. Cloud never talks to NATS."),
("staff.png", "https://staff.zapier.georgelambert.org/", "Staff plane",
"CS / sales / accounting door after IAM."),
("iam.png", "https://iam.zapier.georgelambert.org/", "Staff IAM",
"Named staff users, roles, permissions, sessions."),
("login.png", "https://login.zapier.georgelambert.org/health", "Staff session",
"Shared staff cookie; proxies IAM when STAFF_IAM_URL is set."),
("cs.png", "https://cs.zapier.georgelambert.org/", "Customer service",
"Review prepaid balance and apply credits (USD)."),
("sales.png", "https://sales.zapier.georgelambert.org/", "Sales",
"Per-customer quote, tier, and multiplier."),
("acct.png", "https://acct.zapier.georgelambert.org/", "Accounting",
"QuickBooks IIF and accounting CSV export."),
("mw.png", "https://mw.zapier.georgelambert.org/health", "Middleware",
"Zapier /zapier/v1, jobs, archive, chain (MOCK_VERAE in lab)."),
("fleet.png", "https://fleet.zapier.georgelambert.org/", "Operator console",
"Fleet replicas, Trace, Docs. Pause/stop honored by keep."),
("docs.png", "/", "This catalog",
"Colored PDF documentation site."),
("git.png", "https://git.georgelambert.org/", "Forgejo",
"Independent module repos (SSH 2223)."),
]
def live_section() -> str:
live_dir = ROOT / "packages" / "ui-docs" / "screenshots" / "live"
dest = SITE / "live"
dest.mkdir(parents=True, exist_ok=True)
cards_html = []
for fname, href, title, desc in LIVE_DOORS:
src = live_dir / fname
img = ""
if src.exists():
copy_tree(src, dest / fname)
img = f'<img src="/live/{fname}" alt="{title}"/>'
cards_html.append(
f'<article>{img}<div class="body"><h3>{title}</h3>'
f'<a class="host" href="{href}">{href}</a>'
f"<p>{desc}</p>"
f'<a class="act" href="{href}">Open</a></div></article>'
)
review = ""
pdf = ROOT / "packages" / "ui-docs" / "UI-REVIEW.pdf"
if pdf.exists():
copy_tree(pdf, SITE / "packages" / "ui-docs" / "UI-REVIEW.pdf")
review = (
'<p>Screenshot atlas with usage notes: '
'<a href="/packages/ui-docs/UI-REVIEW.pdf">UI-REVIEW.pdf</a>.</p>'
)
return (
f'<section><h2>Live services (NS1 · HTTPS)</h2>'
f"<p>Each name is on Let&rsquo;s Encrypt via Caddy on "
f"<strong>NS1.GEORGELAMBERT.ORG</strong>. NATS, keep, and archive workers stay on loopback.</p>"
f"{review}"
f'<div class="live">{"".join(cards_html)}</div></section>'
)
def extra_section(items: list[tuple[str, str]]) -> str:
lis = []
for h, lab in items:
dest = SITE / h
if dest.exists() or (h.startswith("sphinx") and (SITE / "sphinx").exists()):
lis.append(f'<li><a href="{h}">{lab}</a></li>')
return f"<section><h2>Catalogs</h2><ul>{''.join(lis)}</ul></section>"
rest_pdf = f"""
{cards("pdf")}
<section>
<h2>Git repositories (Forgejo)</h2>
<p>Clone: <code>ssh://git@git.georgelambert.org:2223/marchon/&lt;name&gt;.git</code> (SSH port 2223). Branches <code>main</code> and <code>master</code>.</p>
<ul>{git_lis}</ul>
</section>
<section>
<h2>Master module docs</h2>
<ul>
<li><a href="docs-master/README.pdf">docs-master README</a><span class="badge">PDF</span></li>
<li><a href="docs-master/MESSAGE-FLOWS.pdf">MESSAGE-FLOWS</a><span class="badge">PDF</span></li>
<li><a href="docs-master/modules-and-nats.pdf">modules-and-nats</a><span class="badge">PDF</span></li>
<li><a href="docs-master/archive-nats.pdf">archive-nats</a><span class="badge">PDF</span></li>
<li><a href="docs-master/composition.pdf">composition</a><span class="badge">PDF</span></li>
<li><a href="user-docs/README.pdf">User guide (signup → tree-node lookup)</a><span class="badge">PDF</span></li>
<li><a href="packages/verae-zapier-simulator/README.pdf">Simulator README</a><span class="badge">PDF</span></li>
<li><a href="overview/README.pdf">System overview</a><span class="badge">PDF</span></li>
<li><a href="overview/INDEX.pdf">Documentation index</a><span class="badge">PDF</span></li>
<li><a href="packages/verae-fleet/docs/CONSOLE.pdf">Operator console</a><span class="badge">PDF</span></li>
<li><a href="overview/diagrams/system.svg">System diagram (SVG)</a></li>
</ul>
</section>
{extra_section(extra_pdf)}
{live_section()}
"""
rest_md = f"""
{cards("md")}
<section>
<h2>Git repositories (Forgejo)</h2>
<p>Clone: <code>ssh://git@git.georgelambert.org:2223/marchon/&lt;name&gt;.git</code> (SSH port 2223).</p>
<ul>{git_lis}</ul>
</section>
<section>
<h2>Master module docs (HTML / Markdown)</h2>
<ul>
<li><a href="docs-master/README.html">docs-master README</a><span class="badge">MD</span></li>
<li><a href="docs-master/MESSAGE-FLOWS.html">MESSAGE-FLOWS</a><span class="badge">MD</span></li>
<li><a href="docs-master/modules-and-nats.html">modules-and-nats</a><span class="badge">MD</span></li>
<li><a href="docs-master/archive-nats.md">archive-nats.md</a><span class="badge">MD</span></li>
<li><a href="docs-master/composition.md">composition.md</a><span class="badge">MD</span></li>
<li><a href="user-docs/README.html">User guide</a><span class="badge">MD</span></li>
<li><a href="packages/verae-zapier-simulator/README.html">Simulator README</a><span class="badge">MD</span></li>
<li><a href="overview/README.html">System overview</a><span class="badge">MD</span></li>
<li><a href="overview/INDEX.html">Documentation index</a><span class="badge">MD</span></li>
<li><a href="packages/verae-fleet/docs/CONSOLE.html">Operator console</a><span class="badge">MD</span></li>
<li><a href="overview/diagrams/system.svg">System diagram (SVG)</a></li>
</ul>
</section>
{extra_section(extra_md)}
{live_section()}
"""
(SITE / "index.html").write_text(
page_shell(now, rest_pdf, switch_to_md=True, title="Verae Time × Zapier documentation"),
encoding="utf-8",
)
(SITE / "index-md.html").write_text(
page_shell(now, rest_md, switch_to_md=False, title="Verae Time × Zapier — Markdown indexes"),
encoding="utf-8",
)
nfiles = sum(1 for _ in SITE.rglob("*") if _.is_file())
npdf = sum(1 for _ in SITE.rglob("*.pdf") if _.is_file())
print(f"site built at {SITE} ({nfiles} files, {npdf} PDFs)")
if errors:
print(f"{len(errors)} render errors:")
for e in errors[:20]:
print(" ", e[:300])
if __name__ == "__main__":
main()