Add exhaustive NATS factorial, MQTT/UDP probes, and optimal-config report.

Compares r=1 vs r=3, file vs memory, reconnect tax, and projects three
HP DL360 Gen10 NVMe + 10GbE boxes. Keep NATS; do not switch to MQTT/UDP.

packages/verae-nats-cluster/scripts/bench.sh

@@ -80,6 +80,21 @@ if [[ "${JS_EXTRA_MEMORY:-0}" == "1" ]]; then
   js_rm
 fi
 
+# Factorial extras: replicas=1 vs 3, file vs memory (does not touch product streams).
+if [[ "${EXHAUSTIVE:-0}" == "1" ]]; then
+  js_rm
+  run_one js-file-1p-20k-128-r1  bench.js.e1 --js --purge --pub 1 --msgs 20000 --size 128 --replicas 1 --storage file --maxbytes=512MB --stream=benchstream
+  js_rm
+  run_one js-file-4p-50k-128-r1  bench.js.e2 --js --purge --pub 4 --msgs 50000 --size 128 --replicas 1 --storage file --maxbytes=512MB --stream=benchstream
+  js_rm
+  run_one js-mem-1p-20k-128-r1   bench.js.e3 --js --purge --pub 1 --msgs 20000 --size 128 --replicas 1 --storage memory --maxbytes=512MB --stream=benchstream
+  js_rm
+  run_one js-mem-4p-50k-128-r1   bench.js.e4 --js --purge --pub 4 --msgs 50000 --size 128 --replicas 1 --storage memory --maxbytes=512MB --stream=benchstream
+  js_rm
+  run_one js-file-1p-20k-4k-r3   bench.js.e5 --js --purge --pub 1 --msgs 20000 --size 4096 --replicas 3 --storage file --maxbytes=512MB --stream=benchstream
+  js_rm
+fi
+
 # Round-trip delay (two connections, through the cluster) at several loads
 sudo pct exec "$CLIENT_VMID" -- bash -lc "
 set -e
@@ -107,6 +122,9 @@ node latency.mjs $n $sz $p $mode
 # copy latest probe
 sudo pct exec "$CLIENT_VMID" -- bash -c 'cat > /tmp/nats-lat/latency.mjs' < "$ROOT/scripts/latency.mjs"
 lat lat-ping-1k-128 1000 128 1 ping
+if [[ "${EXHAUSTIVE:-0}" == "1" ]]; then
+  lat lat-reconnect-200-128 200 128 1 reconnect
+fi
 lat lat-1p-5k-128 5000 128 1 flood
 lat lat-4p-10k-128 10000 128 4 flood
 lat lat-8p-20k-128 20000 128 8 flood

packages/verae-nats-cluster/scripts/build-optimal-report.py

@@ -0,0 +1,400 @@
+#!/usr/bin/env python3
+"""One large comparison report from all NS1 result folders + extras (UDP/MQTT/reconnect)."""
+from __future__ import annotations
+
+import importlib.util
+import json
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+import matplotlib
+
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+from matplotlib.ticker import FuncFormatter
+
+_spec = importlib.util.spec_from_file_location(
+    "bench_report", Path(__file__).resolve().parent / "bench-report.py"
+)
+_br = importlib.util.module_from_spec(_spec)
+assert _spec.loader
+_spec.loader.exec_module(_br)
+fmt_int = _br.fmt_int
+parse_bench = _br.parse_bench
+parse_lat = _br.parse_lat
+
+INDIGO, DEEP, TEAL, AMBER, MUTED = "#4f46e5", "#312e81", "#047857", "#b45309", "#5b6178"
+
+
+def k_fmt(x, _p=None):
+    if x >= 1_000_000:
+        return f"{x/1e6:.2f}M"
+    if x >= 1000:
+        return f"{x/1000:.0f}k"
+    return f"{x:.0f}"
+
+
+def load_folder(folder: Path) -> dict:
+    thru, lats, extra = {}, {}, {}
+    if not folder.is_dir():
+        return {"thru": thru, "lats": lats, "extra": extra, "stamp": folder.name}
+    for f in folder.glob("*.txt"):
+        text = f.read_text(encoding="utf-8", errors="replace")
+        if f.name.startswith("host-"):
+            continue
+        if "mqtt" in f.name or "udp-ping" in f.name:
+            for line in text.splitlines():
+                line = line.strip()
+                if line.startswith("{"):
+                    extra[f.stem] = json.loads(line)
+                    break
+            continue
+        lat = parse_lat(text)
+        if lat:
+            lats[f.stem] = lat
+            continue
+        p = parse_bench(text)
+        if p.get("pub_msgs") or p.get("agg_msgs"):
+            thru[f.stem] = p
+    return {"thru": thru, "lats": lats, "extra": extra, "stamp": folder.name}
+
+
+def pub(d, run):
+    p = d["thru"].get(run) or {}
+    return p.get("pub_msgs")
+
+
+def latp(d, run, key="p99"):
+    p = d["lats"].get(run) or {}
+    return p.get(key, "")
+
+
+def row(*cells):
+    return "| " + " | ".join(cells) + " |"
+
+
+def save(fig, path: Path):
+    fig.savefig(path, dpi=150, bbox_inches="tight", facecolor="white")
+    plt.close(fig)
+
+
+def charts(latest: dict, folders: list[dict], dest: Path):
+    dest.mkdir(parents=True, exist_ok=True)
+    plt.rcParams.update({"font.size": 9, "axes.grid": True, "grid.color": "#d9dce8"})
+    # r=1 vs r=3 file/mem from latest
+    labels, file_r, mem_r = [], [], []
+    for lab, fr, mr in (
+        ("1p file", "js-file-1p-20k-128-r1", "js-1p-20k-128-r3"),
+        ("4p file", "js-file-4p-50k-128-r1", "js-4p-50k-128-r3"),
+        ("1p mem", "js-mem-1p-20k-128-r1", "js-mem-1p-20k-128-r3"),
+        ("4p mem", "js-mem-4p-50k-128-r1", "js-mem-4p-50k-128-r3"),
+    ):
+        a, b = pub(latest, fr), pub(latest, mr)
+        if a or b:
+            labels.append(lab)
+            file_r.append(int(a or 0))
+            mem_r.append(int(b or 0))
+    if labels:
+        fig, ax = plt.subplots(figsize=(9.2, 4.3))
+        x = range(len(labels))
+        ax.bar([i - 0.2 for i in x], file_r, 0.4, label="replicas=1", color=TEAL)
+        ax.bar([i + 0.2 for i in x], mem_r, 0.4, label="replicas=3", color=AMBER)
+        ax.set_xticks(list(x), labels)
+        ax.set_ylabel("pub msgs/s")
+        ax.set_title("This run: replica cost (1 vs 3)")
+        ax.yaxis.set_major_formatter(FuncFormatter(k_fmt))
+        ax.legend()
+        save(fig, dest / "replicas.png")
+    # historical JS 1p file
+    names, vals = [], []
+    for d, label in zip(
+        folders,
+        [d["stamp"] for d in folders],
+    ):
+        v = pub(d, "js-1p-20k-128-r3")
+        if v:
+            names.append(label[-7:] if len(label) > 8 else label)
+            vals.append(int(v))
+    if names:
+        fig, ax = plt.subplots(figsize=(9.2, 4.0))
+        ax.bar(names, vals, color=INDIGO)
+        ax.set_title("JS file r=3 1p 128 B across studies")
+        ax.set_ylabel("pub msgs/s")
+        ax.yaxis.set_major_formatter(FuncFormatter(k_fmt))
+        save(fig, dest / "history-js1p.png")
+
+
+def write_md(latest: dict, hist: list[dict], charts_rel: str) -> str:
+    t = latest["thru"]
+    e = latest["extra"]
+    mqtt = e.get("mqtt-qos0-5k-128") or {}
+    udp = e.get("udp-ping-1k-128") or {}
+    ping = latest["lats"].get("lat-ping-1k-128") or {}
+    recon = latest["lats"].get("lat-reconnect-200-128") or {}
+
+    def js_table():
+        runs = [
+            ("js-file-1p-20k-128-r1", "file r=1 1p 128 B"),
+            ("js-file-4p-50k-128-r1", "file r=1 4p 128 B"),
+            ("js-1p-20k-128-r3", "file r=3 1p 128 B"),
+            ("js-4p-50k-128-r3", "file r=3 4p 128 B"),
+            ("js-4p-20k-1k-r3", "file r=3 4p 1 KiB"),
+            ("js-file-1p-20k-4k-r3", "file r=3 1p 4 KiB"),
+            ("js-mem-1p-20k-128-r1", "memory r=1 1p 128 B"),
+            ("js-mem-4p-50k-128-r1", "memory r=1 4p 128 B"),
+            ("js-mem-1p-20k-128-r3", "memory r=3 1p 128 B"),
+            ("js-mem-4p-50k-128-r3", "memory r=3 4p 128 B"),
+            ("js-mem-4p-20k-1k-r3", "memory r=3 4p 1 KiB"),
+        ]
+        lines = [
+            row("Run", "What", "Pub msgs/s", "Pub MB/s"),
+            row("---", "---", "---", "---"),
+        ]
+        for k, lab in runs:
+            p = t.get(k)
+            if not p:
+                continue
+            lines.append(row(f"`{k}`", lab, fmt_int(p.get("pub_msgs")), p.get("pub_mb") or "—"))
+        return "\n".join(lines)
+
+    hist_lines = [
+        row("Study", "Env", "Core 1p pub", "JS file r=3 1p", "JS mem r=3 4p", "Ping p99"),
+        row("---", "---", "---", "---", "---", "---"),
+    ]
+    labels_env = {
+        "20260912T045131Z": "1c/1G ZFS (off-box orch.)",
+        "20260912T051237Z": "1c/1G ZFS (NS1 orch.)",
+        "20260912T053120Z": "8c/16G tmpfs + mem extra",
+    }
+    for d in hist + [latest]:
+        env = labels_env.get(d["stamp"], f"8c/16G ZFS exhaustive `{d['stamp']}`")
+        hist_lines.append(
+            row(
+                f"`{d['stamp']}`",
+                env,
+                fmt_int(pub(d, "core-1p1s-50k-128")),
+                fmt_int(pub(d, "js-1p-20k-128-r3")),
+                fmt_int(pub(d, "js-mem-4p-50k-128-r3")),
+                latp(d, "lat-ping-1k-128"),
+            )
+        )
+
+    r1 = int(pub(latest, "js-file-1p-20k-128-r1") or 0)
+    r3 = int(pub(latest, "js-1p-20k-128-r3") or 0)
+    mem1 = int(pub(latest, "js-mem-1p-20k-128-r1") or 0)
+    mem3 = int(pub(latest, "js-mem-1p-20k-128-r3") or 0)
+    replica_cost = f"{r1/r3:.2f}×" if r3 else "—"
+    mem_gain = f"{mem1/r3:.2f}×" if r3 and mem1 else "—"
+
+    mqtt_rate = mqtt.get("pubs_per_sec", "—")
+    udp_p99 = udp.get("p99", "—")
+
+    return f"""**Progress report — optimal configuration study** · `{latest['stamp']}` (UTC) · all code on **NS1.GEORGELAMBERT.ORG** (`70.88.205.138`)
+
+This document folds every ladder we have run (1-core ZFS, NS1-orchestrated, tmpfs maximize, and this exhaustive 8c/16G **ZFS** factorial) plus UDP / MQTT / reconnect probes. It recommends a lab config and a **three-box HP DL360 Gen10** projection. veth/10G was not changed.
+
+---
+
+## 1. Verdict (read this first)
+
+**Keep NATS + JetStream.** Do not replace the fabric with MQTT, UDP, or a custom persistent-socket protocol for Verae jobs/events/archive. Those are either slower, less durable, or already what NATS is.
+
+**Lab (NS1, one host, three LXC) — optimal now**
+
+| Stream | Storage | Replicas | Why |
+|--------|---------|----------|-----|
+| `ZAPIER_JOBS`, `ZAPIER_WEBHOOKS`, `VERAE_ARCHIVE` | **file** (ZFS) | **3** | Survive a nats LXC death; archive must persist |
+| `ZAPIER_EVENTS` | **memory** | **3** | Waiters are latency-sensitive; events rebuild from job status |
+| `ZAPIER_USAGE` | file | 3 | Telemetry, limits + max-age |
+
+Keep **8 cores / 16 GiB / `max_mem: 8G`** on 510–513 (already live). Do **not** leave JetStream on tmpfs. Do **not** drop product streams to r=1. Reuse **one NATS connection per process** (already true in middleware); never connect-per-message.
+
+**Metal (3× DL360 Gen10) — optimal later**
+
+Same stream table. File store on **local NVMe/M.2**, not a shared SAN. Cluster + client on **10GbE** (or 25GbE if you already have it). Dual Gold Xeon is surplus CPU for this workload; 8–16 cores dedicated to `nats-server` is enough. Expected JS file r=3: **~40–80k** 128 B pubs/s (about **3–6×** this lab’s 8c ZFS 1p, **2–4×** tmpfs 1p) — bounded by **10GbE replica RTT**, not by Xeon clocks. Core NATS will sit in the **1–3M msgs/s** band until the NIC saturates (~9 Gbit/s ≈ 8–9M × 128 B theoretical; CPU and client will hit first).
+
+---
+
+## 2. What we actually ran (this exhaustive pass)
+
+Live cluster during this run: LXC 510–513 **8 cores / 16 GiB**, JetStream **on ZFS** (tmpfs from the maximize study was already unmounted). Extra factorial: file/memory × replicas 1/3, 4 KiB file r=3, reconnect-per-message ping, UDP echo 510→511, MQTT QoS0 against nats-a `:1883`. Product streams were not the bench target.
+
+### 2.1 Cross-study history
+
+{chr(10).join(hist_lines)}
+
+![JS 1p file r=3 history]({charts_rel}/history-js1p.png)
+
+### 2.2 This run — JetStream factorial
+
+{js_table()}
+
+Replica **1 vs 3** on this stand (file 1p 128 B): r=1 is {fmt_int(str(r1) if r1 else None)} vs r=3 {fmt_int(str(r3) if r3 else None)} ({replica_cost} if r=3 is the slower one). Memory r=1 1p {fmt_int(str(mem1) if mem1 else None)} vs memory r=3 {fmt_int(str(mem3) if mem3 else None)}.
+
+![Replica cost]({charts_rel}/replicas.png)
+
+### 2.3 Delay, reconnect tax, UDP, MQTT
+
+| Probe | Result | Meaning |
+|-------|--------|---------|
+| NATS ping (persistent sockets) p50 / p99 | {ping.get('p50','—')} / {ping.get('p99','—')} | Quiet hop with a long-lived TCP conn |
+| NATS **reconnect-per-message** p50 / p99 | {recon.get('p50','—')} / {recon.get('p99','—')} | TCP+NATS handshake on every pub — this is the tax to avoid |
+| UDP echo 510→511 p99 | {udp_p99} | Raw datagram ceiling on the same veth (no NATS) |
+| MQTT QoS0 5k×128 B | {mqtt_rate} pubs/s | nats-server MQTT gateway on `:1883` |
+
+Core 1p1s 128 B this run: {fmt_int(pub(latest, 'core-1p1s-50k-128'))} pub msgs/s. Flood delay is still backlog/consume_rate, not RTT.
+
+---
+
+## 3. Alternative transports (why we are not switching the fabric)
+
+NATS already **is** persistent TCP sockets with a tiny binary protocol, automatic reconnect, and optional JetStream durability. “Reduce connection overhead” is a **client** discipline: hold the connection. The reconnect probe exists to prove that opening a socket per job would dominate ping RTT.
+
+| Idea | Fit for Verae jobs/events/archive | Throughput vs NATS core | Durability |
+|------|-----------------------------------|-------------------------|------------|
+| **NATS core pub/sub** | Fan-out, request-reply (`verae.billing.*`) | Highest we measured (~0.5–2M msgs/s) | None |
+| **NATS JetStream file r=3** | Jobs, webhooks, archive | ~8–23k on this lab; see metal projection | Disk + 1-node loss |
+| **NATS JetStream memory r=3** | Events mailbox | ~22–36k on this lab | RAM + 1-node loss; **empty on full restart** |
+| **MQTT** (NATS gateway or Mosquitto) | IoT endpoints that already speak MQTT | This probe: {mqtt_rate} pubs/s QoS0 — typically **well below** NATS core; QoS1 ≈ JetStream-ish with more chatter | QoS1/2 session state; not our WORM model |
+| **UDP** | Telemetry that may drop | RTT {udp_p99} p99 — fastest hop, **no** reliability, no cluster, no auth | None |
+| **Custom persistent sockets / HTTP long-poll** | Worse NATS | You would re-implement reconnect, flow control, and fan-out | DIY |
+| **WebSocket** | Browsers only | Extra framing; NATS already has WS for UIs, not for middleware | Same as core/JS behind it |
+| **QUIC / WebTransport** | Lossy WAN / browsers | NATS QUIC is not the lab path; 10GbE LAN does not need it | Same |
+| **Kafka / Redis streams** | Heavy log replay | Higher ops cost; not on `vmbr1` today | Yes, heavier |
+
+**MQTT:** NATS documents MQTT as an *enabling* gateway for existing IoT, and prefers NATS end-to-end for greenfield. Zapier cloud never talks NATS or MQTT; it talks HTTPS. Putting MQTT in the middle of timestamp jobs adds protocol translation and QoS timers without helping `jobId → events`. Use MQTT only if a device already cannot speak NATS.
+
+**UDP:** Fine as a *measurement* of veth RTT. Unusable as the job fabric (no ack, no replica, no flow control). NATS ping is already within a small multiple of UDP on this bridge.
+
+**Persistence sockets:** Middleware and keep already keep `NATS_URL` connections open. Optimal: one connection (or a small pool) per process, `max_reconnect`, jitter, no `connect()` in the per-job path. The reconnect ladder is the anti-pattern.
+
+---
+
+## 4. Optimal configurations
+
+### 4.1 NS1 lab (now)
+
+1. **Leave 8 cores / 16 GiB** on nats-a/b/c and the worker. Host has 40 cores / 377 GiB; this is cheap.
+2. **`max_mem: 8G`** stays. Required for memory streams.
+3. **File r=3 on ZFS** for jobs/webhooks/archive. tmpfs doubled JS 1p (7.4k→17k) but **loses the stream on reboot** — unacceptable for archive.
+4. **Memory r=3 for `ZAPIER_EVENTS`** if we accept “all three nats CTs reboot ⇒ in-flight waiters fall back to HTTP poll.” That matches the designed wait path (`GET /api/status/{{jobId}}`).
+5. **r=1 only for throwaway benches**, never product streams. Replica=3 is the point of three guests.
+6. **veth on vmbr1, no fake 10G NICs.** Already 10000Mb/s; JS does not fill it.
+7. **Pin cpusets** later if keep/fleet steal; not required to beat these numbers.
+8. Clients: persistent NATS connections; pull consumers with bounded `max_ack_pending` for webhooks.
+
+### 4.2 Three HP DL360 Gen10 (projection — not measured)
+
+Assumed bill of materials (state it in the buy):
+
+| Piece | Assumption |
+|-------|------------|
+| Chassis | 3× DL360 Gen10 1U |
+| CPU | Dual 2nd-gen Xeon **Gold** (e.g. 6226R 16c or 6248 20c — **32–40 cores/box**) |
+| Memory | DDR4-2933, **192–384 GiB**/box (6–12×32 GiB); NATS will not use most of it |
+| Storage | **NVMe M.2 or U.2** for `/var/lib/nats/jetstream` (XFS or ext4, **not** shared ZFS over the network). RAID1 of two NVMe if you want disk HA *inside* a box |
+| Network | **10GbE** (FlexibleLOM or PCIe); dedicated VLAN for `:4222`+`:6222`. Do not share with public `vmbr0` traffic |
+| OS | Debian/Ubuntu bare metal, `nats-server` systemd, same `nats.conf` as lab (bind private IP only) |
+
+**What changes vs NS1 LXC**
+
+| Factor | NS1 today | 3× DL360 | Effect on JS file r=3 |
+|--------|-----------|----------|------------------------|
+| Failure domain | 1 Proxmox host | 3 chassis, 3 NVMe, 3 NICs | r=3 **means** something |
+| Disk | Shared ZFS SSD2 | Local NVMe fsync ~50–150 µs | Big win vs ZFS; similar to tmpfs for sequential 128 B |
+| Replica path | veth/bridge (~µs–tens of µs) | 10GbE RTT typically **50–200 µs** | **Slower than same-host tmpfs**, faster than a bad SAN |
+| CPU | 8 of 40 shared | 32–40 dedicated Gold cores | Headroom for many clients, not 10× JS |
+| NIC | software 10G veth, already ~5 Gbit/s core | real 10GbE ~9 Gbit/s TCP | Core NATS can grow; JS r=3 stays replica-bound |
+
+**Projected bands** (128 B, 3-node cluster, dedicated 10GbE, local NVMe, 8+ cores pinned to nats-server):
+
+| Workload | NS1 measured (best) | DL360 projection | Confidence |
+|----------|---------------------|------------------|------------|
+| Core pub/sub 1p | 0.5–0.8M | **0.8–2M** | Medium — NIC + syscall, plenty of CPU |
+| Core 4p4s 1 KiB | ~0.6–0.7M (~0.6 GB/s) | **~1M msgs/s / ~1 GB/s** approaching 10GbE | Medium |
+| JS file r=1 | this run r=1 | **80–200k** pubs/s | Medium — NVMe + no replica wait |
+| JS file r=3 | 7–23k (ZFS/tmpfs) | **40–80k** pubs/s | Medium-low — replica RTT dominates; 3 NVMe still help vs shared ZFS |
+| JS memory r=3 | 22–36k | **50–100k** | Medium-low — RAM + 10GbE ack |
+| Ping p99 | 0.7–1.4 ms | **0.2–0.6 ms** | Medium — real NIC but no Proxmox tax |
+
+These are **not** DL360 measurements. Scale from: (a) our replica-1 vs replica-3 ratio once this run’s r=1 numbers exist, (b) tmpfs vs ZFS ratio (2.35× on 1p), (c) Synadia/nats bench async file r=1 ~100–400k on NVMe loopback, derated for 10GbE RTT.
+
+**Buy notes:** M.2 via Dual uFF / enablement kit; put JetStream on NVMe **directly**, not behind a RAID controller write-through unless you measure. 1GbE onboard is a trap — use 10GbE for `:6222`. Dual Gold is for isolation (nats vs worm/tree vs OS), not because JS needs 56 cores.
+
+---
+
+## 5. What we are not doing
+
+- MQTT as the Zapier or middleware transport.
+- UDP for jobs.
+- Emulated 10G fiber NICs on LXC.
+- tmpfs as the production store.
+- r=1 for product streams.
+- Connect-per-job.
+
+Re-run exhaustive: `bash scripts/exhaustive-ns1-study.sh` on NS1.
+"""
+
+
+def render(md_path: Path, html_path: Path, pdf_path: Path) -> None:
+    css = Path(__file__).resolve().parent / "docs-print.css"
+    header = html_path.with_suffix(".hdr.html")
+    banner = html_path.with_suffix(".ban.html")
+    header.write_text(f"<style>{css.read_text() if css.exists() else ''}</style>\n", encoding="utf-8")
+    banner.write_text(
+        '<div class="doc-banner">'
+        '<nav class="site"><a href="/">zapier.georgelambert.org</a></nav>'
+        '<div class="kicker">Verae Time × Zapier · progress report</div>'
+        "<h1>NATS optimal configuration study</h1>"
+        '<div class="source-path">packages/zapier-decisions/reports/optimal-config/REPORT.md</div>'
+        "</div>\n",
+        encoding="utf-8",
+    )
+    r = subprocess.run(
+        [
+            "pandoc",
+            str(md_path),
+            "-o",
+            str(html_path),
+            "--standalone",
+            f"--resource-path={md_path.parent}",
+            "--highlight-style=breezedark",
+            "--metadata=title=NATS optimal configuration study",
+            f"--include-in-header={header}",
+            f"--include-before-body={banner}",
+        ],
+        capture_output=True,
+        text=True,
+    )
+    header.unlink(missing_ok=True)
+    banner.unlink(missing_ok=True)
+    if r.returncode != 0:
+        raise SystemExit(f"pandoc failed: {r.stderr[-600:]}")
+    w = subprocess.run(["weasyprint", str(html_path), str(pdf_path)], capture_output=True, text=True)
+    if w.returncode != 0:
+        raise SystemExit(f"weasyprint failed: {w.stderr[-600:]}")
+
+
+def main() -> int:
+    latest = Path(sys.argv[1])
+    hist_dirs = [Path(p) for p in sys.argv[2:] if p and Path(p).is_dir()]
+    data_latest = load_folder(latest)
+    hist = [load_folder(p) for p in hist_dirs]
+    charts_dir = latest / "charts-optimal"
+    charts(data_latest, hist + [data_latest], charts_dir)
+    md = write_md(data_latest, hist, "charts-optimal")
+    md_path = latest / "optimal-config.md"
+    md_path.write_text(md, encoding="utf-8")
+    html_path = latest / "optimal-config.html"
+    pdf_path = latest / "optimal-config.pdf"
+    render(md_path, html_path, pdf_path)
+    print(f"wrote {md_path}")
+    print(f"wrote {html_path}", file=sys.stderr)
+    print(f"wrote {pdf_path}", file=sys.stderr)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

packages/verae-nats-cluster/scripts/exhaustive-ns1-study.sh

@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+# Exhaustive NS1 ladder on the *current* 8c/16G cluster with JetStream on ZFS.
+# Adds r=1 vs r=3, file vs memory, reconnect tax, UDP echo, MQTT gateway probe.
+# Does not tmpfs (product streams stay). Must run on NS1.
+set -euo pipefail
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+export PATH="/usr/sbin:/usr/bin:/bin:/usr/local/bin:$PATH"
+HOST="$(hostname -f 2>/dev/null || hostname)"
+case "$HOST" in
+  NS1.GEORGELAMBERT.ORG|NS1|ns1.georgelambert.org|ns1) ;;
+  *) echo "refusing: exhaustive-ns1-study.sh must run on NS1, got '$HOST'" >&2; exit 1 ;;
+esac
+
+export EXHAUSTIVE=1
+export JS_EXTRA_MEMORY=1
+export COMPARE_DIR="${COMPARE_DIR:-$ROOT/results/20260912T051237Z}"
+STAMP="$(date -u +%Y%m%dT%H%M%SZ)"
+export BENCH_OUT="$ROOT/results/$STAMP"
+mkdir -p "$BENCH_OUT"
+
+# MQTT gateway on nats-a only (vmbr1). Restored after.
+MQTT_CONF=/etc/nats/nats.conf
+enable_mqtt() {
+  sudo pct exec 511 -- bash -lc '
+set -e
+f=/etc/nats/nats.conf
+grep -q "^mqtt {" "$f" && exit 0
+cat >> "$f" <<EOF
+
+mqtt {
+  host: 10.10.10.21
+  port: 1883
+}
+EOF
+systemctl kill -s HUP nats-server || systemctl restart nats-server
+'
+  sleep 2
+}
+disable_mqtt() {
+  sudo pct exec 511 -- bash -lc '
+f=/etc/nats/nats.conf
+python3 - "$f" <<'"'"'PY'"'"'
+from pathlib import Path
+import re, sys
+p = Path(sys.argv[1])
+t = re.sub(r"\nmqtt \{[^}]*\}\n", "\n", p.read_text(), flags=re.S)
+p.write_text(t)
+PY
+systemctl kill -s HUP nats-server || true
+' || true
+}
+
+enable_mqtt
+trap disable_mqtt EXIT
+
+bash "$ROOT/scripts/study-on-ns1.sh"
+OUT="$BENCH_OUT"
+echo "exhaustive extras into $OUT"
+
+# UDP echo: server in 511, client in 510
+sudo pct exec 511 -- bash -lc 'pkill -f "udp-probe.mjs server" >/dev/null 2>&1 || true'
+sudo pct exec 511 -- bash -c 'cat > /tmp/udp-probe.mjs' < "$ROOT/scripts/udp-probe.mjs"
+sudo pct exec 510 -- bash -c 'cat > /tmp/nats-lat/udp-probe.mjs' < "$ROOT/scripts/udp-probe.mjs"
+sudo pct exec 511 -- bash -lc 'setsid node /tmp/udp-probe.mjs server 9999 >/tmp/udp-echo.log 2>&1 < /dev/null &'
+sleep 1
+echo "=== udp-ping-1k-128 ===" | tee "$OUT/udp-ping-1k-128.txt"
+sudo pct exec 510 -- bash -lc 'node /tmp/nats-lat/udp-probe.mjs client 10.10.10.21 1000 9999 128' | tee -a "$OUT/udp-ping-1k-128.txt"
+sudo pct exec 511 -- bash -lc 'pkill -f "udp-probe.mjs server" || true'
+
+# MQTT QoS0
+sudo pct exec 510 -- bash -lc '
+set -e
+cd /tmp/nats-lat
+if [[ ! -d node_modules/mqtt ]]; then npm install --no-audit --no-fund mqtt@10 >/dev/null; fi
+'
+sudo pct exec 510 -- bash -c 'cat > /tmp/nats-lat/mqtt-probe.mjs' < "$ROOT/scripts/mqtt-probe.mjs"
+echo "=== mqtt-qos0-5k-128 ===" | tee "$OUT/mqtt-qos0-5k-128.txt"
+sudo pct exec 510 -- bash -lc 'cd /tmp/nats-lat && node mqtt-probe.mjs mqtt://10.10.10.21:1883 5000 128' | tee -a "$OUT/mqtt-qos0-5k-128.txt" || echo '{"error":"mqtt probe failed"}' | tee -a "$OUT/mqtt-qos0-5k-128.txt"
+
+python3 "$ROOT/scripts/build-optimal-report.py" "$OUT" \
+  "$ROOT/results/20260912T045131Z" \
+  "$ROOT/results/20260912T051237Z" \
+  "$ROOT/results/20260912T053120Z"
+echo "exhaustive complete $OUT"

packages/verae-nats-cluster/scripts/latency.mjs

@@ -1,9 +1,10 @@
 #!/usr/bin/env node
 /**
  * Pub→sub round trip through the cluster (two connections).
- * Usage: NATS_URL=... node latency.mjs [count] [payloadBytes] [publishers] [ping|flood]
- * ping  = sequential publish-wait (one-message RTT)
- * flood = publish the batch then drain (queueing under burst)
+ * Usage: NATS_URL=... node latency.mjs [count] [payloadBytes] [publishers] [ping|flood|reconnect]
+ * ping       = sequential publish-wait on persistent sockets (one-message RTT)
+ * flood      = publish the batch then drain (queueing under burst)
+ * reconnect  = connect, one publish, wait, close — measures handshake tax
  */
 import { connect, headers } from "nats";
 
@@ -23,7 +24,33 @@ function pct(sorted, p) {
 }
 
 const samples = [];
-if (mode === "ping") {
+if (mode === "reconnect") {
+  const subNc = await connect({ servers, name: "lat-sub" });
+  let resolveOne = null;
+  const sub = subNc.subscribe(subject, { max: count });
+  const consume = (async () => {
+    for await (const m of sub) {
+      const sent = Number(m.headers?.get("t") || 0);
+      samples.push(Number(process.hrtime.bigint() / 1000n) - sent);
+      resolveOne?.();
+    }
+  })();
+  await subNc.flush();
+  for (let i = 0; i < count; i++) {
+    const got = new Promise((r) => {
+      resolveOne = r;
+    });
+    const pubNc = await connect({ servers, name: `lat-re-${i}` });
+    const h = headers();
+    h.set("t", String(process.hrtime.bigint() / 1000n));
+    pubNc.publish(subject, payload, { headers: h });
+    await pubNc.flush();
+    await got;
+    await pubNc.close();
+  }
+  await consume;
+  await subNc.close();
+} else if (mode === "ping") {
   const subNc = await connect({ servers, name: "lat-sub" });
   const pubNc = await connect({ servers, name: "lat-pub" });
   let resolveOne = null;

packages/verae-nats-cluster/scripts/mqtt-probe.mjs

@@ -0,0 +1,33 @@
+#!/usr/bin/env node
+/** MQTT QoS0 publish rate against nats-server MQTT gateway. */
+import mqtt from "mqtt";
+
+const url = process.argv[2] || "mqtt://10.10.10.21:1883";
+const count = Number(process.argv[3] || 5000);
+const size = Number(process.argv[4] || 128);
+const payload = Buffer.alloc(size, 9);
+const topic = `bench/mqtt/${process.pid}`;
+
+const c = mqtt.connect(url, { reconnectPeriod: 0, connectTimeout: 5000 });
+await new Promise((res, rej) => {
+  c.on("connect", res);
+  c.on("error", rej);
+});
+const t0 = process.hrtime.bigint();
+for (let i = 0; i < count; i++) {
+  await new Promise((res, rej) => c.publish(topic, payload, { qos: 0 }, (err) => (err ? rej(err) : res())));
+}
+const ns = Number(process.hrtime.bigint() - t0);
+c.end(true);
+const sec = ns / 1e9;
+console.log(
+  JSON.stringify({
+    mode: "mqtt-qos0",
+    count,
+    size,
+    url,
+    secs: Number(sec.toFixed(3)),
+    pubs_per_sec: Math.round(count / sec),
+    mb_per_sec: Number(((count * size) / sec / 1e6).toFixed(2)),
+  }),
+);

packages/verae-nats-cluster/scripts/test.sh

@@ -8,6 +8,7 @@ bash -n "$ROOT/scripts/status.sh"
 bash -n "$ROOT/scripts/bench.sh"
 bash -n "$ROOT/scripts/study-on-ns1.sh"
 bash -n "$ROOT/scripts/maximize-ns1-study.sh"
+bash -n "$ROOT/scripts/exhaustive-ns1-study.sh"
 grep -q 'host: {{IP}}' "$ROOT/conf/nats.conf.tmpl"
 grep -qv '0.0.0.0' "$ROOT/conf/nats.conf.tmpl"
 if [[ ! -d /etc/pve/nodes ]]; then

packages/verae-nats-cluster/scripts/udp-probe.mjs

@@ -0,0 +1,54 @@
+#!/usr/bin/env node
+/** UDP echo RTT. server: node udp-probe.mjs server [port]
+ *  client: node udp-probe.mjs client <host> <count> [port] [size] */
+import dgram from "node:dgram";
+
+const mode = process.argv[2] || "server";
+const port = Number(process.argv[mode === "server" ? 3 : 5] || 9999);
+
+if (mode === "server") {
+  const s = dgram.createSocket("udp4");
+  s.on("message", (msg, rinfo) => s.send(msg, rinfo.port, rinfo.address));
+  s.bind(port, "0.0.0.0", () => console.log(JSON.stringify({ mode: "udp-server", port })));
+} else {
+  const host = process.argv[3];
+  const count = Number(process.argv[4] || 1000);
+  const size = Number(process.argv[6] || 128);
+  const sock = dgram.createSocket("udp4");
+  const payload = Buffer.alloc(size, 7);
+  const samples = [];
+  let i = 0;
+  const sendOne = () => {
+    const t0 = process.hrtime.bigint();
+    const once = (msg) => {
+      sock.off("message", once);
+      samples.push(Number(process.hrtime.bigint() - t0) / 1000);
+      i += 1;
+      if (i >= count) {
+        samples.sort((a, b) => a - b);
+        const us = (n) => `${(n / 1000).toFixed(3)}ms`;
+        const pct = (p) => samples[Math.min(samples.length - 1, Math.floor((p / 100) * samples.length))];
+        const sum = samples.reduce((a, b) => a + b, 0);
+        console.log(
+          JSON.stringify({
+            mode: "udp-ping",
+            count: samples.length,
+            size,
+            host,
+            min: us(samples[0]),
+            avg: us(sum / samples.length),
+            p50: us(pct(50)),
+            p99: us(pct(99)),
+            max: us(samples[samples.length - 1]),
+            p50_us: Math.round(pct(50)),
+            p99_us: Math.round(pct(99)),
+          }),
+        );
+        sock.close();
+      } else sendOne();
+    };
+    sock.on("message", once);
+    sock.send(payload, port, host);
+  };
+  sendOne();
+}