From 69e28af4d88b2c6c5fb0e0e792e29e9c3f96f5f6 Mon Sep 17 00:00:00 2001 From: George Lambert Date: Fri, 11 Sep 2026 23:47:42 -0400 Subject: [PATCH] Add a three-node NATS JetStream cluster on distinct Proxmox LXC guests. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit LXC 511–513 (nats-a/b/c on 10.10.10.21–23, vmbr1 only) form cluster verae with replicas=3. Host loopback 127.0.0.1:4222 is unchanged. --- packages/docs-master/README.md | 1 + .../modules/verae-nats-cluster/NATS.md | 3 + .../modules/verae-nats-cluster/SUMMARY.md | 3 + packages/overview/02-modules-and-repos.md | 1 + packages/overview/03-nats-cluster.md | 8 ++- packages/overview/10-docs-sync.md | 1 + packages/overview/11-hostnames.md | 3 + packages/verae-bootstrap/README.md | 1 + packages/verae-bootstrap/host-deps.tsv | 1 + packages/verae-bootstrap/repos.txt | 1 + .../types/nats-cluster/README.md | 9 +++ .../types/nats-cluster/repos.txt | 3 + packages/verae-nats-cluster/NATS.md | 5 ++ packages/verae-nats-cluster/README.md | 33 +++++++++ packages/verae-nats-cluster/SUMMARY.md | 3 + packages/verae-nats-cluster/cluster.env | 19 +++++ .../verae-nats-cluster/conf/nats.conf.tmpl | 19 +++++ packages/verae-nats-cluster/package.json | 11 +++ .../scripts/create-cluster.sh | 69 +++++++++++++++++++ packages/verae-nats-cluster/scripts/lib-ct.sh | 62 +++++++++++++++++ packages/verae-nats-cluster/scripts/status.sh | 22 ++++++ packages/verae-nats-cluster/scripts/test.sh | 50 ++++++++++++++ .../systemd/nats-server.service | 16 +++++ packages/verae-ops/03-proxmox.md | 12 +++- packages/zapier-decisions/LOG.md | 6 ++ scripts/build-docs-site.py | 3 + scripts/push-module-repos.sh | 2 + 27 files changed, 365 insertions(+), 2 deletions(-) create mode 100644 packages/docs-master/modules/verae-nats-cluster/NATS.md create mode 100644 packages/docs-master/modules/verae-nats-cluster/SUMMARY.md create mode 100644 packages/verae-bootstrap/types/nats-cluster/README.md create mode 100644 packages/verae-bootstrap/types/nats-cluster/repos.txt create mode 100644 packages/verae-nats-cluster/NATS.md create mode 100644 packages/verae-nats-cluster/README.md create mode 100644 packages/verae-nats-cluster/SUMMARY.md create mode 100644 packages/verae-nats-cluster/cluster.env create mode 100644 packages/verae-nats-cluster/conf/nats.conf.tmpl create mode 100644 packages/verae-nats-cluster/package.json create mode 100755 packages/verae-nats-cluster/scripts/create-cluster.sh create mode 100755 packages/verae-nats-cluster/scripts/lib-ct.sh create mode 100755 packages/verae-nats-cluster/scripts/status.sh create mode 100755 packages/verae-nats-cluster/scripts/test.sh create mode 100644 packages/verae-nats-cluster/systemd/nats-server.service diff --git a/packages/docs-master/README.md b/packages/docs-master/README.md index 5cdce20..88d3afc 100644 --- a/packages/docs-master/README.md +++ b/packages/docs-master/README.md @@ -36,6 +36,7 @@ Summaries, NATS contracts, and message flows for every Verae Time × Zapier modu | verae-backup | https://git.georgelambert.org/marchon/verae-backup | | verae-deploy | https://git.georgelambert.org/marchon/verae-deploy | | verae-proxmox | https://git.georgelambert.org/marchon/verae-proxmox | +| verae-nats-cluster | https://git.georgelambert.org/marchon/verae-nats-cluster | Clone (SSH port 2223): diff --git a/packages/docs-master/modules/verae-nats-cluster/NATS.md b/packages/docs-master/modules/verae-nats-cluster/NATS.md new file mode 100644 index 0000000..9067ba1 --- /dev/null +++ b/packages/docs-master/modules/verae-nats-cluster/NATS.md @@ -0,0 +1,3 @@ +# NATS — verae-nats-cluster + +This repo runs the cluster. Clients: `nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222`. Routes `:6222`. Monitor loopback `:8222` per guest. Never on `vmbr0`. diff --git a/packages/docs-master/modules/verae-nats-cluster/SUMMARY.md b/packages/docs-master/modules/verae-nats-cluster/SUMMARY.md new file mode 100644 index 0000000..978d54a --- /dev/null +++ b/packages/docs-master/modules/verae-nats-cluster/SUMMARY.md @@ -0,0 +1,3 @@ +# verae-nats-cluster + +Three Proxmox LXC (511–513, `nats-a/b/c` on `10.10.10.21–23`) form a JetStream cluster on `vmbr1` for HA testing before dedicated hardware. diff --git a/packages/overview/02-modules-and-repos.md b/packages/overview/02-modules-and-repos.md index 7625ade..fc08d1e 100644 --- a/packages/overview/02-modules-and-repos.md +++ b/packages/overview/02-modules-and-repos.md @@ -44,6 +44,7 @@ Each runtime piece is its **own git repo** on Forgejo (`git.georgelambert.org`, | **verae-backup** | `packages/verae-backup` | SQLite / JetStream / worm+tree snapshots | | **verae-deploy** | `packages/verae-deploy` | Tagged checkout + npm ci on the target OS | | **verae-proxmox** | `packages/verae-proxmox` | LXC 510 worker on NS1 `vmbr1` | +| **verae-nats-cluster** | `packages/verae-nats-cluster` | LXC 511–513 JetStream cluster on `vmbr1` | Libraries that are **not** separate Forgejo apps today: `verae-chain-client` (inside middleware), job-poller and webhook-deliver (middleware workers, fleet-spawned). diff --git a/packages/overview/03-nats-cluster.md b/packages/overview/03-nats-cluster.md index 4769ba6..9e0d5bb 100644 --- a/packages/overview/03-nats-cluster.md +++ b/packages/overview/03-nats-cluster.md @@ -12,7 +12,13 @@ nats://127.0.0.1:4222,nats://127.0.0.1:4223,nats://127.0.0.1:4224 On a given machine the listener stays on **loopback** (or a private interface). Operators reach it with `scripts/nats-tunnel.sh` / `ssh -L 14222:127.0.0.1:4222`. **Do not bind 4222 on `0.0.0.0` without auth.** -Today’s NS1 box (`NS1.GEORGELAMBERT.ORG`, `70.88.205.138`) already runs JetStream on `127.0.0.1:4222`. The **target** is three clustered nodes so losing one server does not lose the stream. Fleet SSH hosts (`ns1`, later `lan-134`) run **workers**, not extra public NATS listeners. +Today’s NS1 box (`NS1.GEORGELAMBERT.ORG`, `70.88.205.138`) still runs the **lab** JetStream on `127.0.0.1:4222` (keep/fleet/middleware). The **test cluster** is three distinct Proxmox LXC guests on `vmbr1` ([verae-nats-cluster](https://git.georgelambert.org/marchon/verae-nats-cluster)): + +```text +nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222 +``` + +(`nats-a` VMID 511, `nats-b` 512, `nats-c` 513). Cluster routes on `:6222`. Nothing on `vmbr0`. Cut over workers with `NATS_URL` listing all three; then the same configs move to separate hardware. Fleet SSH hosts run **workers**, not extra public NATS listeners. ![NATS cluster](diagrams/nats-cluster.svg) diff --git a/packages/overview/10-docs-sync.md b/packages/overview/10-docs-sync.md index caabefe..fe3883a 100644 --- a/packages/overview/10-docs-sync.md +++ b/packages/overview/10-docs-sync.md @@ -22,6 +22,7 @@ Validated against the module list in `packages/` and Forgejo repos. - `verae-keep` - `verae-bootstrap` (clone + npm per server type; `scripts/host-deps.sh` installs OS/Node/NATS) - `verae-uptime`, `verae-backup`, `verae-deploy`, `verae-proxmox` (LXC 510 on NS1 `vmbr1`) +- `verae-nats-cluster` (LXC 511–513 JetStream on `vmbr1`) ## Move the catalog to another hostname diff --git a/packages/overview/11-hostnames.md b/packages/overview/11-hostnames.md index 479dd39..dd4dbee 100644 --- a/packages/overview/11-hostnames.md +++ b/packages/overview/11-hostnames.md @@ -10,6 +10,9 @@ All of the running Verae Time × Zapier processes on **2026-09-11** live on one | `ns1` (SSH worker profile) | **NS1.GEORGELAMBERT.ORG** | `70.88.205.138` | Same box; SSH spawn unused while all-in-one is local | | `lan-134` | *(none yet)* | `70.88.205.134` | Disabled until SSH works | | `px-worker` | **verae-px-worker** (LXC **510**) | `10.10.10.20` on `vmbr1` | Extra worm/tree; enable after `verae-proxmox` create-worker | +| *(nats-cluster)* | **nats-a** (LXC **511**) | `10.10.10.21` | JetStream node A (`:4222` / `:6222`) | +| *(nats-cluster)* | **nats-b** (LXC **512**) | `10.10.10.22` | JetStream node B | +| *(nats-cluster)* | **nats-c** (LXC **513**) | `10.10.10.23` | JetStream node C | Loopback on that box is `127.0.0.1`. Do **not** publish NATS `:4222`. diff --git a/packages/verae-bootstrap/README.md b/packages/verae-bootstrap/README.md index 557332a..b06ee57 100644 --- a/packages/verae-bootstrap/README.md +++ b/packages/verae-bootstrap/README.md @@ -45,6 +45,7 @@ bash scripts/fetch.sh ns1-all-in-one # one server type | **ns1-archive** | `types/ns1-archive/` | NATS + worm ×3 + tree-node ×3 + poller + webhook + aggregator + keep | | **lan-worker** | `types/lan-worker/` | Extra SSH worker (tree-node, archive-worm). Keep disabled until SSH works (`lan-134`). | | **proxmox-worker** | `types/proxmox-worker/` | LXC 510 on NS1 `vmbr1` (`10.10.10.20`) — extra worm/tree copies | +| **nats-cluster** | `types/nats-cluster/` | LXC 511–513 JetStream cluster on `vmbr1` (`10.10.10.21–23`) | Each type has `repos.txt` (subset of the master list) and `README.md`. diff --git a/packages/verae-bootstrap/host-deps.tsv b/packages/verae-bootstrap/host-deps.tsv index 66f7390..5d0c0ad 100644 --- a/packages/verae-bootstrap/host-deps.tsv +++ b/packages/verae-bootstrap/host-deps.tsv @@ -40,5 +40,6 @@ verae-uptime|20|npm verae-backup|0| verae-deploy|20|npm verae-proxmox|0|ssh +verae-nats-cluster|0|nats_server,ssh zapier-docs-master|0|docs master-zapier-plan-draft|22|npm,native,nats,docs diff --git a/packages/verae-bootstrap/repos.txt b/packages/verae-bootstrap/repos.txt index c4ba5d5..f576db9 100644 --- a/packages/verae-bootstrap/repos.txt +++ b/packages/verae-bootstrap/repos.txt @@ -37,3 +37,4 @@ verae-uptime verae-backup verae-deploy verae-proxmox +verae-nats-cluster diff --git a/packages/verae-bootstrap/types/nats-cluster/README.md b/packages/verae-bootstrap/types/nats-cluster/README.md new file mode 100644 index 0000000..4df2c1f --- /dev/null +++ b/packages/verae-bootstrap/types/nats-cluster/README.md @@ -0,0 +1,9 @@ +# Server type: nats-cluster + +Three Proxmox LXC guests (`nats-a/b/c`) on `vmbr1`. Provision from the hypervisor with [verae-nats-cluster](https://git.georgelambert.org/marchon/verae-nats-cluster) `scripts/create-cluster.sh`. + +Client URL (private only): + +`nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222` + +Host loopback `127.0.0.1:4222` stays the lab bus until you cut over. diff --git a/packages/verae-bootstrap/types/nats-cluster/repos.txt b/packages/verae-bootstrap/types/nats-cluster/repos.txt new file mode 100644 index 0000000..4f80d5f --- /dev/null +++ b/packages/verae-bootstrap/types/nats-cluster/repos.txt @@ -0,0 +1,3 @@ +verae-nats-cluster +verae-nats-accounts +verae-bootstrap diff --git a/packages/verae-nats-cluster/NATS.md b/packages/verae-nats-cluster/NATS.md new file mode 100644 index 0000000..9d51b66 --- /dev/null +++ b/packages/verae-nats-cluster/NATS.md @@ -0,0 +1,5 @@ +# NATS — verae-nats-cluster + +This repo **is** the three-node JetStream cluster. Listeners are the guest private IPs on `vmbr1` (`10.10.10.21–23:4222` and `:6222`). HTTP monitor is loopback `:8222` inside each guest. + +Zapier cloud, browsers, and `vmbr0` never get a NATS socket. diff --git a/packages/verae-nats-cluster/README.md b/packages/verae-nats-cluster/README.md new file mode 100644 index 0000000..75543c3 --- /dev/null +++ b/packages/verae-nats-cluster/README.md @@ -0,0 +1,33 @@ +# verae-nats-cluster + +Three **distinct Proxmox LXC** guests running **NATS JetStream** as a cluster, on the private bridge only. This is the test stand before the same layout moves to separate hardware. + +**Forgejo:** https://git.georgelambert.org/marchon/verae-nats-cluster +**SSH:** `ssh://git@git.georgelambert.org:2223/marchon/verae-nats-cluster.git` + +| Guest | VMID | `vmbr1` IP | Client | Cluster | Monitor | +|-------|------|------------|--------|---------|---------| +| nats-a | 511 | 10.10.10.21 | `:4222` | `:6222` | `127.0.0.1:8222` | +| nats-b | 512 | 10.10.10.22 | `:4222` | `:6222` | `127.0.0.1:8222` | +| nats-c | 513 | 10.10.10.23 | `:4222` | `:6222` | `127.0.0.1:8222` | + +Cluster name: `verae`. Client URL: + +```text +nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222 +``` + +**Not** on `vmbr0`. **Not** `0.0.0.0`. Host lab NATS on `127.0.0.1:4222` is left running so current keep/fleet/middleware stay up until you cut over. + +```bash +# on NS1 (the Proxmox host) +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-nats-cluster.git +cd verae-nats-cluster +bash scripts/create-cluster.sh +bash scripts/status.sh +bash scripts/test.sh +``` + +Cut over a worker later with `NATS_URL=nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222` (for example `px-worker`). Zapier cloud never talks to NATS. + +Hardware move: same three configs, three boxes, private NIC only — change IPs in `cluster.env` and `conf/nats.conf.tmpl`. diff --git a/packages/verae-nats-cluster/SUMMARY.md b/packages/verae-nats-cluster/SUMMARY.md new file mode 100644 index 0000000..724bbeb --- /dev/null +++ b/packages/verae-nats-cluster/SUMMARY.md @@ -0,0 +1,3 @@ +# verae-nats-cluster + +Proxmox LXC 511–513 (`nats-a/b/c`) form a JetStream cluster on `vmbr1` for HA testing before dedicated hardware. diff --git a/packages/verae-nats-cluster/cluster.env b/packages/verae-nats-cluster/cluster.env new file mode 100644 index 0000000..a0df60b --- /dev/null +++ b/packages/verae-nats-cluster/cluster.env @@ -0,0 +1,19 @@ +# Distinct Proxmox LXC guests on NS1 vmbr1. Not the host loopback NATS. +BRIDGE="${BRIDGE:-vmbr1}" +GW="${GW:-10.10.10.1}" +MEMORY="${MEMORY:-1024}" +CORES="${CORES:-1}" +DISK="${DISK:-8}" +STORAGE="${STORAGE:-SSD2}" +TEMPLATE="${TEMPLATE:-/var/lib/vz/template/cache/ubuntu-24.04-standard_24.04-2_amd64.tar.zst}" +DNS="${DNS:-8.8.8.8}" +USER_NAME="${USER_NAME:-marchon}" +NATS_VER="${NATS_VER:-2.10.26}" +CLUSTER_NAME="${CLUSTER_NAME:-verae}" + +# vmid hostname ipv4 +NODES="${NODES:- +511 nats-a 10.10.10.21 +512 nats-b 10.10.10.22 +513 nats-c 10.10.10.23 +}" diff --git a/packages/verae-nats-cluster/conf/nats.conf.tmpl b/packages/verae-nats-cluster/conf/nats.conf.tmpl new file mode 100644 index 0000000..481de1c --- /dev/null +++ b/packages/verae-nats-cluster/conf/nats.conf.tmpl @@ -0,0 +1,19 @@ +# JetStream cluster node. Bind the private guest IP only — never vmbr0 / 0.0.0.0. +server_name: {{NAME}} +host: {{IP}} +port: 4222 +http: 127.0.0.1:8222 + +jetstream { + store_dir: /var/lib/nats/jetstream + max_mem: 256M + max_file: 4G +} + +cluster { + name: {{CLUSTER}} + listen: {{IP}}:6222 + routes: [ +{{ROUTES}} + ] +} diff --git a/packages/verae-nats-cluster/package.json b/packages/verae-nats-cluster/package.json new file mode 100644 index 0000000..90465d3 --- /dev/null +++ b/packages/verae-nats-cluster/package.json @@ -0,0 +1,11 @@ +{ + "name": "verae-nats-cluster", + "version": "0.1.0", + "private": true, + "description": "Three-node NATS JetStream cluster on distinct Proxmox LXC guests (vmbr1 only)", + "scripts": { + "create": "bash scripts/create-cluster.sh", + "test": "bash scripts/test.sh", + "status": "bash scripts/status.sh" + } +} diff --git a/packages/verae-nats-cluster/scripts/create-cluster.sh b/packages/verae-nats-cluster/scripts/create-cluster.sh new file mode 100755 index 0000000..bb42239 --- /dev/null +++ b/packages/verae-nats-cluster/scripts/create-cluster.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Create three distinct Proxmox LXC guests and start a JetStream cluster on vmbr1. +# Does not touch host loopback NATS (127.0.0.1:4222) or vmbr0. +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +# shellcheck disable=SC1091 +. "$ROOT/cluster.env" +# shellcheck disable=SC1091 +. "$ROOT/scripts/lib-ct.sh" +ct_require_proxmox + +mapfile -t rows < <(printf '%s\n' "$NODES" | awk 'NF==3 {print}') +[[ ${#rows[@]} -eq 3 ]] || { echo "need exactly 3 nodes in cluster.env" >&2; exit 1; } + +declare -a VMIDS NAMES IPS +for row in "${rows[@]}"; do + # shellcheck disable=SC2086 + set -- $row + VMIDS+=("$1"); NAMES+=("$2"); IPS+=("$3") +done + +i=0 +for i in 0 1 2; do + ct_ensure "${VMIDS[$i]}" "${NAMES[$i]}" "${IPS[$i]}" + ct_bootstrap_user "${VMIDS[$i]}" +done + +# Install nats-server + conf + systemd on each guest +for i in 0 1 2; do + routes="" + for j in 0 1 2; do + [[ $i -eq $j ]] && continue + routes="${routes} nats-route://${IPS[$j]}:6222"$'\n' + done + tmpconf="$(mktemp)" + NAME="${NAMES[$i]}" IP="${IPS[$i]}" CLUSTER="$CLUSTER_NAME" ROUTES="$routes" \ + python3 - "$ROOT/conf/nats.conf.tmpl" "$tmpconf" <<'PY' +import os, pathlib, sys +t = pathlib.Path(sys.argv[1]).read_text() +out = t.replace("{{NAME}}", os.environ["NAME"]).replace("{{IP}}", os.environ["IP"]).replace("{{CLUSTER}}", os.environ["CLUSTER"]).replace("{{ROUTES}}", os.environ["ROUTES"]) +pathlib.Path(sys.argv[2]).write_text(out) +PY + sudo pct exec "${VMIDS[$i]}" -- bash -c 'cat > /tmp/nats.conf' < "$tmpconf" + sudo pct exec "${VMIDS[$i]}" -- bash -c 'cat > /tmp/nats-server.service' < "$ROOT/systemd/nats-server.service" + rm -f "$tmpconf" + sudo pct exec "${VMIDS[$i]}" -- bash -lc " +set -e +export DEBIAN_FRONTEND=noninteractive +id nats >/dev/null 2>&1 || useradd -r -s /usr/sbin/nologin nats +install -d -m 755 -o nats -g nats /var/lib/nats/jetstream /etc/nats +mv /tmp/nats.conf /etc/nats/nats.conf +chown root:root /etc/nats/nats.conf +chmod 644 /etc/nats/nats.conf +if [[ ! -x /usr/local/bin/nats-server ]]; then + curl -fsSL https://github.com/nats-io/nats-server/releases/download/v${NATS_VER}/nats-server-v${NATS_VER}-linux-amd64.tar.gz -o /tmp/nats.tgz + tar -xzf /tmp/nats.tgz -C /tmp + install -m 0755 /tmp/nats-server-v${NATS_VER}-linux-amd64/nats-server /usr/local/bin/nats-server + rm -rf /tmp/nats.tgz /tmp/nats-server-v${NATS_VER}-linux-amd64 +fi +install -m 644 /tmp/nats-server.service /etc/systemd/system/nats-server.service +systemctl daemon-reload +systemctl enable --now nats-server +" + echo "nats-server ${NAMES[$i]} ${IPS[$i]}:4222 cluster ${IPS[$i]}:6222" +done + +echo "cluster client URL: nats://${IPS[0]}:4222,nats://${IPS[1]}:4222,nats://${IPS[2]}:4222" +echo "lab loopback NATS on the host is unchanged (127.0.0.1:4222)" +echo "next: bash $ROOT/scripts/test.sh" diff --git a/packages/verae-nats-cluster/scripts/lib-ct.sh b/packages/verae-nats-cluster/scripts/lib-ct.sh new file mode 100755 index 0000000..1a4ff8e --- /dev/null +++ b/packages/verae-nats-cluster/scripts/lib-ct.sh @@ -0,0 +1,62 @@ +# shellcheck shell=bash +# Shared LXC bootstrap for NS1 Proxmox. Does not generate SSH keys if one exists. +export PATH="/usr/sbin:/usr/bin:/bin:$PATH" + +ct_require_proxmox() { + if [[ ! -d /etc/pve/nodes ]]; then + echo "not a Proxmox host" >&2 + return 1 + fi + command -v pct >/dev/null || { echo "pct missing" >&2; return 1; } +} + +ct_ensure() { + local vmid="$1" hostname="$2" ip="$3" + if [[ ! -f "$TEMPLATE" ]]; then + echo "missing template $TEMPLATE" >&2 + return 1 + fi + if ! sudo pct status "$vmid" >/dev/null 2>&1; then + echo "pct create $vmid $hostname $ip/24" + sudo pct create "$vmid" "$TEMPLATE" \ + --hostname "$hostname" \ + --memory "$MEMORY" --cores "$CORES" --swap 256 \ + --net0 "name=eth0,bridge=${BRIDGE},ip=${ip}/24,gw=${GW},type=veth" \ + --rootfs "${STORAGE}:${DISK}" \ + --unprivileged 1 --onboot 1 --nameserver "$DNS" \ + --features nesting=1 \ + --ostype ubuntu + else + echo "CT $vmid already exists" + fi + sudo pct start "$vmid" 2>/dev/null || true + local i + for i in $(seq 1 40); do + sudo pct exec "$vmid" -- true 2>/dev/null && return 0 + sleep 2 + done + echo "CT $vmid did not start" >&2 + return 1 +} + +ct_bootstrap_user() { + local vmid="$1" + local pub="" + [[ -f "$HOME/.ssh/id_ed25519.pub" ]] && pub="$(cat "$HOME/.ssh/id_ed25519.pub")" + [[ -z "$pub" && -f "$HOME/.ssh/authorized_keys" ]] && pub="$(head -1 "$HOME/.ssh/authorized_keys")" + [[ -n "$pub" ]] || { echo "no ssh public key" >&2; return 1; } + sudo pct exec "$vmid" -- bash -lc " +set -e +export DEBIAN_FRONTEND=noninteractive +apt-get update -qq +apt-get install -y --no-install-recommends openssh-server sudo curl ca-certificates xz-utils tar +id $USER_NAME >/dev/null 2>&1 || useradd -m -s /bin/bash $USER_NAME +echo '$USER_NAME ALL=(ALL) NOPASSWD:ALL' >/etc/sudoers.d/90-$USER_NAME +chmod 440 /etc/sudoers.d/90-$USER_NAME +install -d -m 700 -o $USER_NAME -g $USER_NAME /home/$USER_NAME/.ssh +grep -qxF '$pub' /home/$USER_NAME/.ssh/authorized_keys 2>/dev/null || echo '$pub' >>/home/$USER_NAME/.ssh/authorized_keys +chown $USER_NAME:$USER_NAME /home/$USER_NAME/.ssh/authorized_keys +chmod 600 /home/$USER_NAME/.ssh/authorized_keys +systemctl enable --now ssh +" +} diff --git a/packages/verae-nats-cluster/scripts/status.sh b/packages/verae-nats-cluster/scripts/status.sh new file mode 100755 index 0000000..1306656 --- /dev/null +++ b/packages/verae-nats-cluster/scripts/status.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +# shellcheck disable=SC1091 +. "$ROOT/cluster.env" +export PATH="/usr/sbin:/usr/bin:/bin:$PATH" +printf '%s\n' "$NODES" | awk 'NF==3 {print}' | while read -r vmid name ip; do + st="$(sudo pct status "$vmid" 2>/dev/null || echo missing)" + js="$(sudo pct exec "$vmid" -- curl -fsS --max-time 2 http://127.0.0.1:8222/varz 2>/dev/null || echo '{}')" + echo "$vmid $name $ip $st" + python3 -c " +import json,sys +try: + d=json.loads(sys.argv[1]) +except Exception: + print(' nats down') + raise SystemExit +print(' server_name', d.get('server_name'), 'cluster', (d.get('cluster') or {}).get('name'), 'routes', len((d.get('cluster') or {}).get('urls') or d.get('connect_urls') or [])) +print(' jetstream', bool(d.get('jetstream')), 'port', d.get('port'), 'host', d.get('host')) +" "$js" 2>/dev/null || echo " nats down" +done +echo "host loopback still: $(ss -lnt | grep '127.0.0.1:4222' && echo up || echo down)" diff --git a/packages/verae-nats-cluster/scripts/test.sh b/packages/verae-nats-cluster/scripts/test.sh new file mode 100755 index 0000000..957916c --- /dev/null +++ b/packages/verae-nats-cluster/scripts/test.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Local syntax check always. Live cluster check when pct is present. +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +bash -n "$ROOT/scripts/lib-ct.sh" +bash -n "$ROOT/scripts/create-cluster.sh" +bash -n "$ROOT/scripts/status.sh" +grep -q 'host: {{IP}}' "$ROOT/conf/nats.conf.tmpl" +grep -qv '0.0.0.0' "$ROOT/conf/nats.conf.tmpl" +if [[ ! -d /etc/pve/nodes ]]; then + echo "OK (syntax; not on Proxmox)" + exit 0 +fi +# shellcheck disable=SC1091 +. "$ROOT/cluster.env" +export PATH="/usr/sbin:/usr/bin:/bin:$PATH" +mapfile -t rows < <(printf '%s\n' "$NODES" | awk 'NF==3 {print}') +ready=0 +for row in "${rows[@]}"; do + # shellcheck disable=SC2086 + set -- $row + vmid=$1 name=$2 ip=$3 + js="$(sudo pct exec "$vmid" -- curl -fsS --max-time 3 http://127.0.0.1:8222/varz 2>/dev/null || true)" + echo "$js" | grep -q '"jetstream"' && ready=$((ready + 1)) || echo "not ready $name" +done +[[ $ready -eq 3 ]] || { echo "cluster not fully up ($ready/3)" >&2; exit 1; } + +# nats CLI on first node +first="$(echo "${rows[0]}" | awk '{print $1}')" +sudo pct exec "$first" -- bash -lc ' +set -e +export DEBIAN_FRONTEND=noninteractive +if [[ ! -x /usr/local/bin/nats ]]; then + apt-get install -y --no-install-recommends unzip >/dev/null + curl -fsSL https://github.com/nats-io/natscli/releases/download/v0.1.6/nats-0.1.6-linux-amd64.zip -o /tmp/natscli.zip + rm -rf /tmp/natscli && mkdir -p /tmp/natscli + unzip -o /tmp/natscli.zip -d /tmp/natscli >/dev/null + BIN=$(find /tmp/natscli /tmp -maxdepth 3 -type f -name nats | head -1) + test -n "$BIN" + install -m 0755 "$BIN" /usr/local/bin/nats +fi +IP=$(hostname -I | awk "{print \$1}") +export NATS_URL=nats://$IP:4222 +export PATH=/usr/local/bin:/usr/bin:/bin +nats stream rm VERAE_PX_TEST --force >/dev/null 2>&1 || true +nats stream add VERAE_PX_TEST --subjects="verae.px.test" --replicas=3 --storage=file --retention=limits --discard=old --max-msgs=-1 --max-bytes=-1 --max-age=1h --dupe-window=2m --defaults +nats pub verae.px.test cluster-ok +nats stream info VERAE_PX_TEST +' +echo "OK live cluster (3/3 + replicas=3 stream)" diff --git a/packages/verae-nats-cluster/systemd/nats-server.service b/packages/verae-nats-cluster/systemd/nats-server.service new file mode 100644 index 0000000..86e5076 --- /dev/null +++ b/packages/verae-nats-cluster/systemd/nats-server.service @@ -0,0 +1,16 @@ +[Unit] +Description=NATS JetStream (Verae cluster node) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=nats +Group=nats +ExecStart=/usr/local/bin/nats-server -c /etc/nats/nats.conf +LimitNOFILE=65536 +Restart=on-failure +RestartSec=2 + +[Install] +WantedBy=multi-user.target diff --git a/packages/verae-ops/03-proxmox.md b/packages/verae-ops/03-proxmox.md index 4817306..491195c 100644 --- a/packages/verae-ops/03-proxmox.md +++ b/packages/verae-ops/03-proxmox.md @@ -34,7 +34,17 @@ On the guest, bootstrap type `proxmox-worker` ([verae-bootstrap](https://git.geo ## NATS for guests -Loopback NATS on the host stays `127.0.0.1:4222`. `verae-nats-private-proxy.service` binds **only** `10.10.10.1:4222`. Do not listen on `vmbr0`. +Loopback NATS on the host stays `127.0.0.1:4222` (current lab bus). `verae-nats-private-proxy.service` binds **only** `10.10.10.1:4222` for the worker CT. + +**Three-node test cluster** (distinct LXC, same chassis): [verae-nats-cluster](https://git.georgelambert.org/marchon/verae-nats-cluster) + +| Guest | VMID | IP | +|-------|------|-----| +| nats-a | 511 | 10.10.10.21 | +| nats-b | 512 | 10.10.10.22 | +| nats-c | 513 | 10.10.10.23 | + +`bash scripts/create-cluster.sh` then `bash scripts/test.sh`. Client URL lists all three. Do not listen on `vmbr0`. Same configs later move to separate hardware. ## Suggested future guests (not created by create-worker.sh) diff --git a/packages/zapier-decisions/LOG.md b/packages/zapier-decisions/LOG.md index 5f36d68..1860670 100644 --- a/packages/zapier-decisions/LOG.md +++ b/packages/zapier-decisions/LOG.md @@ -1,5 +1,11 @@ # Action log +## 2026-09-12 — NATS cluster on three Proxmox LXC + +- New repo **verae-nats-cluster**: LXC 511/512/513 (`nats-a/b/c`, `10.10.10.21–23`) JetStream cluster on `vmbr1`. +- Lab loopback `127.0.0.1:4222` unchanged until cut-over. +- Test: stream replicas=3; same configs later move to distinct hardware. + ## 2026-09-11 — proxmox worker + deploy/monitor/backup - NS1 **is** Proxmox (`vmbr0` public, `vmbr1` `10.10.10.0/24`). New repo **verae-proxmox** creates LXC **510** `verae-px-worker` at `10.10.10.20`. diff --git a/scripts/build-docs-site.py b/scripts/build-docs-site.py index 6af2b91..3d4b134 100755 --- a/scripts/build-docs-site.py +++ b/scripts/build-docs-site.py @@ -78,6 +78,7 @@ SECTIONS = [ ("packages/verae-backup/README.md", "Backup SQLite / JetStream / worm+tree"), ("packages/verae-deploy/README.md", "Tagged deploy on the target OS"), ("packages/verae-proxmox/README.md", "Proxmox LXC worker on NS1 vmbr1"), + ("packages/verae-nats-cluster/README.md", "Proxmox 3-node NATS JetStream cluster"), ], ), ( @@ -598,6 +599,7 @@ def main() -> None: "verae-backup", "verae-deploy", "verae-proxmox", + "verae-nats-cluster", ): pkg_root = ROOT / "packages" / pkg if pkg in {"zapier-user-docs", "overview", "docs-master", "verae-ops"}: @@ -829,6 +831,7 @@ def main() -> None: ("verae-backup", "Backup SQLite / JetStream / worm+tree"), ("verae-deploy", "Tagged deploy on the target OS"), ("verae-proxmox", "Proxmox LXC worker (NS1 vmbr1)"), + ("verae-nats-cluster", "Proxmox 3-node NATS JetStream cluster"), ] git_lis = "".join( f'
  • {name} ' diff --git a/scripts/push-module-repos.sh b/scripts/push-module-repos.sh index 3c2a01c..a69150b 100755 --- a/scripts/push-module-repos.sh +++ b/scripts/push-module-repos.sh @@ -74,6 +74,7 @@ create verae-uptime "Off-box HTTPS health watch for public doors" create verae-backup "Backup/restore SQLite, JetStream, worm/tree, keep, IAM" create verae-deploy "Tagged checkout + npm ci on the target OS" create verae-proxmox "Provision archive-worker LXC on NS1 Proxmox" +create verae-nats-cluster "Three-node NATS JetStream cluster on Proxmox LXC" push_dir "$ROOT/packages/zappier" zappier-edge push_dir "$ROOT/packages/verae-zapier-middleware" verae-middleware @@ -114,5 +115,6 @@ push_dir "$ROOT/packages/verae-uptime" verae-uptime push_dir "$ROOT/packages/verae-backup" verae-backup push_dir "$ROOT/packages/verae-deploy" verae-deploy push_dir "$ROOT/packages/verae-proxmox" verae-proxmox +push_dir "$ROOT/packages/verae-nats-cluster" verae-nats-cluster echo ALL_MODULE_REPOS_PUSHED