From ce998457950b35554235311366f2175fdba199e5 Mon Sep 17 00:00:00 2001 From: George Lambert Date: Fri, 11 Sep 2026 12:23:23 -0400 Subject: [PATCH] Publish docs site for zapier.georgelambert.org and archive-NATS design Static catalog (index + copied docs/modules/models/sphinx/setup HTML+PDF). Deploy via rsync to NS1 /SSD2/sites/zapier.georgelambert.org with Caddy vhost and Let's Encrypt. Architecture note for job wait, multipart split, hash receipts, and bloom-filtered WORM archive fan-out over NATS. --- .gitignore | 1 + docs/02-architecture/archive-nats.md | 90 ++++++++++ package.json | 2 + scripts/build-docs-site.py | 249 +++++++++++++++++++++++++++ scripts/deploy-docs-site.sh | 12 ++ 5 files changed, 354 insertions(+) create mode 100644 docs/02-architecture/archive-nats.md create mode 100755 scripts/build-docs-site.py create mode 100755 scripts/deploy-docs-site.sh diff --git a/.gitignore b/.gitignore index f57cc77..81949a6 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ packages/zappier/zappier.db packages/verae-zapier-middleware/data/ .grok-session docs/sphinx/_build/ +site/ diff --git a/docs/02-architecture/archive-nats.md b/docs/02-architecture/archive-nats.md new file mode 100644 index 0000000..a50e0b2 --- /dev/null +++ b/docs/02-architecture/archive-nats.md @@ -0,0 +1,90 @@ +# Archive NATS: jobs, multipart split, hash receipts, WORM bloom fan-out + +Zapier never speaks NATS. HTTPS stops at zappier → middleware. Middleware owns jobs, splitting, chain lookup, and archive aggregation. + +## End-to-end + +```text +Zapier --HTTPS--> zappier (meter, x-api-key) + --HTTPS--> middleware /zapier/v1/timestamp[/wait] + 1. split multipart + chain: SHA256 only (+ optional public-meta digest) + archives: public JSON, encrypted private JSON, files + 2. publish jobs.watch → return jobId (202) + 3. waiters subscribe jobs.events + 4. lookup SHA256 on chain (mock or Verae) + already sealed → original receipt, no new seal + later attach records → extra receipts + 5. if includeAttached: + publish verae.archive.query + WORM nodes: bloom miss = silence + bloom hit = reply + aggregate until WAIT_ARCHIVE_MS + 6. jobs.events completed JSON → waiter / REST Hook +``` + +Blockchain stores **hash + time + block + certificate**. Public metadata, encrypted metadata, and file bytes live on **WORM archives**. + +## Splitter + +`splitRequest(body | multipart)`: + +| Field | Destination | +|-------|-------------| +| `data` / `sha256` | Chain register or lookup | +| `publicMetadata` | Archive put (clear) | +| `privateMetadata` | Archive put (ciphertext) | +| `files[]` | Archive put; chain gets content hashes + ids | +| `includeAttached` | Whether wait path queries archives | + +## Hash already registered + +Return original `jobId` and original seal. Do not write a second chain timestamp. + +If later attach jobs exist for that hash, `receipts` is an array: seal first, then attachment receipts in time order. + +## Subjects + +| Subject | Publisher | Subscriber | +|---------|-----------|------------| +| `verae.zapier.jobs.watch` | HTTP edge | job poller | +| `verae.zapier.jobs.events` | poller | waiter, webhook router | +| `verae.archive.put` | splitter | archive that owns the shard | +| `verae.archive.query` | aggregator | **every** archive (not a shared queue group) | +| `verae.archive.reply.` | archive on bloom hit | aggregator | + +Query payload: `{ correlationId, sha256, tenantId, kinds[] }`. +Reply payload: `{ archiveId, sha256, records[] }`. + +Bloom miss → no reply. Aggregator timeout → complete with whatever arrived. + +## WORM archives + +Each process holds append-only records + a bloom of SHA256 keys it stores. False positives OK; false negatives must be rare. Bloom is not an ACL — on hit, still check tenant/share. + +Harness: three mock archives with overlapping hashes. + +## Completed job JSON (wait / hook) + +```json +{ + "jobId": "…", + "status": "completed", + "sha256": "…", + "receipts": [ + { "kind": "seal", "timestamp": "…", "certificate": "…", "blockIndex": 42 }, + { "kind": "metadata-attach", "attachedAt": "…", "publicMetadata": {} } + ], + "files": [{ "id": "…", "sha256": "…", "archiveId": "archive-b" }], + "archivesQueried": true, + "archiveReplies": 2 +} +``` + +Flag off or all blooms miss → `files` empty, extra receipts omitted. + +## Security + +- Zapier never connects to NATS or archives. +- Private metadata only on authenticated archive replies. +- NS1 NATS stays loopback; use `scripts/nats-tunnel.sh`. diff --git a/package.json b/package.json index 471a9a7..c490436 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,8 @@ "docs:modules": "python3 scripts/gen-module-docs.py", "docs:sphinx": "sphinx-build -b html docs/sphinx docs/sphinx/_build/html", "docs:pdf": "python3 scripts/gen-module-pdfs.py", + "docs:site": "python3 scripts/build-docs-site.py", + "docs:deploy": "bash scripts/deploy-docs-site.sh", "test:offline": "bash scripts/test-offline.sh", "nats:tunnel": "bash scripts/nats-tunnel.sh" }, diff --git a/scripts/build-docs-site.py b/scripts/build-docs-site.py new file mode 100755 index 0000000..fe51071 --- /dev/null +++ b/scripts/build-docs-site.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python3 +"""Build static documentation site into ./site for zapier.georgelambert.org.""" + +from __future__ import annotations + +import shutil +import subprocess +from datetime import datetime, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SITE = ROOT / "site" + +SECTIONS = [ + ( + "Start here", + [ + ("README.md", "Workspace README"), + ("docs/04-activate/SETUP-ZAPIER-DEVELOPER.html", "Zapier developer setup (HTML)"), + ("docs/04-activate/SETUP-ZAPIER-DEVELOPER.pdf", "Zapier developer setup (PDF)"), + ("docs/OPEN.md", "Still open"), + ("TODO.md", "Implementation TODO / gates"), + ], + ), + ( + "Architecture", + [ + ("docs/02-architecture/composition.md", "Zappier + middleware composition"), + ("docs/02-architecture/overview.md", "Middleware overview"), + ("docs/02-architecture/nats-gateway.md", "NATS on NS1"), + ("docs/02-architecture/nats-subjects.md", "NATS subjects"), + ("docs/02-architecture/archive-nats.md", "Archive NATS, bloom, multi-receipt"), + ], + ), + ( + "Product", + [ + ("docs/01-product/features-a-m.md", "Features a–m"), + ("docs/01-product/api-gap-analysis.md", "Verae OpenAPI gaps"), + ("docs/01-product/billing-and-keys.md", "Billing and API keys"), + ("docs/00-sources/workspace-brief.md", "Original brief"), + ("docs/00-sources/provenance.md", "Provenance"), + ], + ), + ( + "API specs", + [ + ("docs/api/middleware-openapi.yaml", "Middleware OpenAPI"), + ("packages/zappier/openapi.yaml", "Zappier OpenAPI"), + ("docs/00-sources/veraetime-openapi.yaml", "Verae Timestamping OpenAPI snapshot"), + ], + ), +] + + +def rel(p: Path) -> str: + return p.relative_to(ROOT).as_posix() + + +def copy_tree(src: Path, dest: Path, ignore=None) -> None: + if not src.exists(): + return + dest.parent.mkdir(parents=True, exist_ok=True) + if src.is_file(): + shutil.copy2(src, dest) + return + shutil.copytree(src, dest, dirs_exist_ok=True, ignore=ignore) + + +def pandoc_md(src: Path, dest: Path, title: str) -> None: + dest.parent.mkdir(parents=True, exist_ok=True) + css = """ + body{font:16px/1.5 Georgia,serif;max-width:48rem;margin:2rem auto;padding:0 1rem;color:#122} + a{color:#0b4f8a} code,pre{font-family:ui-monospace,Menlo,monospace;font-size:0.88rem} + pre{background:#1b2833;color:#eef;padding:0.8rem;overflow:auto} + table{border-collapse:collapse} td,th{border:1px solid #ccc;padding:0.35rem 0.5rem} + nav{font:14px system-ui;margin-bottom:1.5rem} + """ + header = dest.with_suffix(".hdr.html") + header.write_text( + f"\n\n", + encoding="utf-8", + ) + subprocess.run( + [ + "pandoc", + str(src), + "-o", + str(dest), + "--standalone", + f"--metadata=title={title}", + f"--include-in-header={header}", + ], + check=False, + capture_output=True, + ) + header.unlink(missing_ok=True) + + +def walk_md_pdf(prefix: Path) -> list[tuple[str, str]]: + items = [] + if not prefix.exists(): + return items + for p in sorted(prefix.rglob("*")): + if p.suffix.lower() in {".md", ".pdf", ".html", ".yaml", ".yml", ".svg"} and p.is_file(): + items.append((rel(p), p.name)) + return items + + +def main() -> None: + if SITE.exists(): + shutil.rmtree(SITE) + SITE.mkdir() + + copy_tree(ROOT / "docs", SITE / "docs", ignore=shutil.ignore_patterns("_build", ".DS_Store")) + sphinx = ROOT / "docs" / "sphinx" / "_build" / "html" + if sphinx.exists(): + copy_tree(sphinx, SITE / "sphinx") + for pkg in ("verae-activate", "verae-zapier", "verae-zapier-middleware", "zappier"): + readme = ROOT / "packages" / pkg / "README.md" + if readme.exists(): + copy_tree(readme, SITE / "packages" / pkg / "README.md") + zdocs = ROOT / "packages" / "zappier" / "docs" + if zdocs.exists(): + copy_tree(zdocs, SITE / "packages" / "zappier" / "docs", ignore=shutil.ignore_patterns("screenshots", "walkthrough", "superpowers")) + for name in ("README.md", "TODO.md", "OPEN.md"): + src = ROOT / name + if not src.exists() and name == "OPEN.md": + src = ROOT / "docs" / "OPEN.md" + if src.exists(): + copy_tree(src, SITE / src.name if src.parent == ROOT else SITE / "docs" / "OPEN.md") + copy_tree(ROOT / "docs" / "WORK-LOG.md", SITE / "docs" / "WORK-LOG.md") + + # research markdown + diagrams only + r = ROOT / "research" / "zapier" + if r.exists(): + for fname in ( + "README.md", + "getting-started.md", + "PLATFORM-REFERENCE.md", + "FUNCTIONS-REFERENCE.md", + "MCP-REFERENCE.md", + "LOGIN.md", + "docs/zapier-billing.md", + ): + p = r / fname + if p.exists(): + copy_tree(p, SITE / "research" / fname) + diagrams = r / "docs" / "diagrams" + if diagrams.exists(): + copy_tree(diagrams, SITE / "research" / "docs" / "diagrams") + + # HTML versions of markdown for the catalog entries + html_pairs = [] + for _title, links in SECTIONS: + for path, label in links: + src = ROOT / path + if src.suffix == ".md" and src.exists(): + dest = SITE / Path(path).with_suffix(".html") + pandoc_md(src, dest, label) + html_pairs.append((path, dest.relative_to(SITE).as_posix(), label)) + + now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + cards = [] + for title, links in SECTIONS: + lis = [] + for path, label in links: + src = ROOT / path + if not src.exists() and path == "OPEN.md": + src = ROOT / "docs" / "OPEN.md" + path = "docs/OPEN.md" + if not src.exists(): + continue + href = path + html = Path(path).with_suffix(".html").as_posix() + if src.suffix == ".md" and (SITE / html).exists(): + href = html + lis.append(f'
  • {label} {path}
    • ') + cards.append(f"

    {title}

      {''.join(lis)}
    ") + + extra = [ + ("docs/modules/README.md", "Module API sheets"), + ("docs/models/README.md", "Data models"), + ("sphinx/index.html", "Sphinx HTML"), + ("docs/04-activate/SETUP-ZAPIER-DEVELOPER.html", "Setup HTML"), + ] + extra_lis = "".join( + f'
  • {lab}
    • ' for h, lab in extra if (SITE / h).exists() or h.startswith("sphinx") + ) + + index = f""" + + + + + Verae Time × Zapier documentation + + + +
    +
    zapier.georgelambert.org
    +

    Verae Time × Zapier documentation

    +

    Project reference: setup checklist, architecture (including NATS job wait and WORM archive fan-out), module API sheets, models, OpenAPI, and research notes. Generated {now}.

    +
    +
    +

    + {''.join(cards)} +
    +

    Catalogs

    + +
    +
    + + + +""" + (SITE / "index.html").write_text(index, encoding="utf-8") + print(f"site built at {SITE} ({sum(1 for _ in SITE.rglob('*') if _.is_file())} files)") + + +if __name__ == "__main__": + main() diff --git a/scripts/deploy-docs-site.sh b/scripts/deploy-docs-site.sh new file mode 100755 index 0000000..2ff948a --- /dev/null +++ b/scripts/deploy-docs-site.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Build and rsync docs site to zapier.georgelambert.org on NS1. +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +HOST="${DEPLOY_HOST:-marchon@70.88.205.138}" +DEST="${DEPLOY_DEST:-/SSD2/sites/zapier.georgelambert.org}" + +python3 "$ROOT/scripts/build-docs-site.py" +ssh "$HOST" "mkdir -p '$DEST'" +rsync -avz --delete "$ROOT/site/" "$HOST:$DEST/" +echo "rsync done → $HOST:$DEST" +echo "https://zapier.georgelambert.org/"