From 3eaad04d7b45ada70167e67cd6ba96cebcbfef41 Mon Sep 17 00:00:00 2001 From: George Lambert Date: Fri, 11 Sep 2026 23:59:41 -0400 Subject: [PATCH] Test env NATS cluster cut-over --- NATS.md | 5 +++ README.md | 44 +++++++++++++++++++++++ SUMMARY.md | 3 ++ client.env | 3 ++ cluster.env | 19 ++++++++++ conf/nats.conf.tmpl | 19 ++++++++++ package.json | 11 ++++++ scripts/create-cluster.sh | 69 +++++++++++++++++++++++++++++++++++++ scripts/cutover-ns1.sh | 10 ++++++ scripts/ensure-streams.sh | 34 ++++++++++++++++++ scripts/lib-ct.sh | 62 +++++++++++++++++++++++++++++++++ scripts/status.sh | 22 ++++++++++++ scripts/test.sh | 50 +++++++++++++++++++++++++++ systemd/nats-server.service | 16 +++++++++ 14 files changed, 367 insertions(+) create mode 100644 NATS.md create mode 100644 README.md create mode 100644 SUMMARY.md create mode 100644 client.env create mode 100644 cluster.env create mode 100644 conf/nats.conf.tmpl create mode 100644 package.json create mode 100755 scripts/create-cluster.sh create mode 100755 scripts/cutover-ns1.sh create mode 100755 scripts/ensure-streams.sh create mode 100755 scripts/lib-ct.sh create mode 100755 scripts/status.sh create mode 100755 scripts/test.sh create mode 100644 systemd/nats-server.service diff --git a/NATS.md b/NATS.md new file mode 100644 index 0000000..9d51b66 --- /dev/null +++ b/NATS.md @@ -0,0 +1,5 @@ +# NATS — verae-nats-cluster + +This repo **is** the three-node JetStream cluster. Listeners are the guest private IPs on `vmbr1` (`10.10.10.21–23:4222` and `:6222`). HTTP monitor is loopback `:8222` inside each guest. + +Zapier cloud, browsers, and `vmbr0` never get a NATS socket. diff --git a/README.md b/README.md new file mode 100644 index 0000000..fd0f381 --- /dev/null +++ b/README.md @@ -0,0 +1,44 @@ +# verae-nats-cluster + +Three **distinct Proxmox LXC** guests running **NATS JetStream** as a cluster, on the private bridge only. This is the test stand before the same layout moves to separate hardware. + +**Forgejo:** https://git.georgelambert.org/marchon/verae-nats-cluster +**SSH:** `ssh://git@git.georgelambert.org:2223/marchon/verae-nats-cluster.git` + +| Guest | VMID | `vmbr1` IP | Client | Cluster | Monitor | +|-------|------|------------|--------|---------|---------| +| nats-a | 511 | 10.10.10.21 | `:4222` | `:6222` | `127.0.0.1:8222` | +| nats-b | 512 | 10.10.10.22 | `:4222` | `:6222` | `127.0.0.1:8222` | +| nats-c | 513 | 10.10.10.23 | `:4222` | `:6222` | `127.0.0.1:8222` | + +Cluster name: `verae`. Client URL: + +```text +nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222 +``` + +**Not** on `vmbr0`. **Not** `0.0.0.0`. Host lab NATS on `127.0.0.1:4222` is left running so current keep/fleet/middleware stay up until you cut over. + +```bash +# on NS1 (the Proxmox host) +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-nats-cluster.git +cd verae-nats-cluster +bash scripts/create-cluster.sh +bash scripts/status.sh +bash scripts/test.sh +``` + +Test-env cut-over (keep/fleet/middleware on NS1): + +```bash +# 1) create product streams (replicas=3) +bash scripts/ensure-streams.sh +# 2) fleet overlay nats.url + services/*.json already list the three URLs +# 3) restart fleet and keep with NATS_URL from client.env +``` + +`JETSTREAM_REPLICAS=3` on middleware. Host `127.0.0.1:4222` can stay up unused. Zapier cloud never talks to NATS. + +nkeys/mTLS: `verae-nats-accounts` still has the INTERNAL/LEAF sketch. Do **not** enable accounts on this cluster until every client passes credentials in `NATS_URL`. Private `vmbr1` is the current isolation. + +Hardware move: same three configs, three boxes, private NIC only — change IPs in `cluster.env` and `conf/nats.conf.tmpl`. diff --git a/SUMMARY.md b/SUMMARY.md new file mode 100644 index 0000000..724bbeb --- /dev/null +++ b/SUMMARY.md @@ -0,0 +1,3 @@ +# verae-nats-cluster + +Proxmox LXC 511–513 (`nats-a/b/c`) form a JetStream cluster on `vmbr1` for HA testing before dedicated hardware. diff --git a/client.env b/client.env new file mode 100644 index 0000000..4afa412 --- /dev/null +++ b/client.env @@ -0,0 +1,3 @@ +# Test-environment client URL. Private vmbr1 only. Lab loopback stays until cut-over. +export NATS_URL="${NATS_URL:-nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222}" +export JETSTREAM_REPLICAS="${JETSTREAM_REPLICAS:-3}" diff --git a/cluster.env b/cluster.env new file mode 100644 index 0000000..a0df60b --- /dev/null +++ b/cluster.env @@ -0,0 +1,19 @@ +# Distinct Proxmox LXC guests on NS1 vmbr1. Not the host loopback NATS. +BRIDGE="${BRIDGE:-vmbr1}" +GW="${GW:-10.10.10.1}" +MEMORY="${MEMORY:-1024}" +CORES="${CORES:-1}" +DISK="${DISK:-8}" +STORAGE="${STORAGE:-SSD2}" +TEMPLATE="${TEMPLATE:-/var/lib/vz/template/cache/ubuntu-24.04-standard_24.04-2_amd64.tar.zst}" +DNS="${DNS:-8.8.8.8}" +USER_NAME="${USER_NAME:-marchon}" +NATS_VER="${NATS_VER:-2.10.26}" +CLUSTER_NAME="${CLUSTER_NAME:-verae}" + +# vmid hostname ipv4 +NODES="${NODES:- +511 nats-a 10.10.10.21 +512 nats-b 10.10.10.22 +513 nats-c 10.10.10.23 +}" diff --git a/conf/nats.conf.tmpl b/conf/nats.conf.tmpl new file mode 100644 index 0000000..481de1c --- /dev/null +++ b/conf/nats.conf.tmpl @@ -0,0 +1,19 @@ +# JetStream cluster node. Bind the private guest IP only — never vmbr0 / 0.0.0.0. +server_name: {{NAME}} +host: {{IP}} +port: 4222 +http: 127.0.0.1:8222 + +jetstream { + store_dir: /var/lib/nats/jetstream + max_mem: 256M + max_file: 4G +} + +cluster { + name: {{CLUSTER}} + listen: {{IP}}:6222 + routes: [ +{{ROUTES}} + ] +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..90465d3 --- /dev/null +++ b/package.json @@ -0,0 +1,11 @@ +{ + "name": "verae-nats-cluster", + "version": "0.1.0", + "private": true, + "description": "Three-node NATS JetStream cluster on distinct Proxmox LXC guests (vmbr1 only)", + "scripts": { + "create": "bash scripts/create-cluster.sh", + "test": "bash scripts/test.sh", + "status": "bash scripts/status.sh" + } +} diff --git a/scripts/create-cluster.sh b/scripts/create-cluster.sh new file mode 100755 index 0000000..bb42239 --- /dev/null +++ b/scripts/create-cluster.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Create three distinct Proxmox LXC guests and start a JetStream cluster on vmbr1. +# Does not touch host loopback NATS (127.0.0.1:4222) or vmbr0. +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +# shellcheck disable=SC1091 +. "$ROOT/cluster.env" +# shellcheck disable=SC1091 +. "$ROOT/scripts/lib-ct.sh" +ct_require_proxmox + +mapfile -t rows < <(printf '%s\n' "$NODES" | awk 'NF==3 {print}') +[[ ${#rows[@]} -eq 3 ]] || { echo "need exactly 3 nodes in cluster.env" >&2; exit 1; } + +declare -a VMIDS NAMES IPS +for row in "${rows[@]}"; do + # shellcheck disable=SC2086 + set -- $row + VMIDS+=("$1"); NAMES+=("$2"); IPS+=("$3") +done + +i=0 +for i in 0 1 2; do + ct_ensure "${VMIDS[$i]}" "${NAMES[$i]}" "${IPS[$i]}" + ct_bootstrap_user "${VMIDS[$i]}" +done + +# Install nats-server + conf + systemd on each guest +for i in 0 1 2; do + routes="" + for j in 0 1 2; do + [[ $i -eq $j ]] && continue + routes="${routes} nats-route://${IPS[$j]}:6222"$'\n' + done + tmpconf="$(mktemp)" + NAME="${NAMES[$i]}" IP="${IPS[$i]}" CLUSTER="$CLUSTER_NAME" ROUTES="$routes" \ + python3 - "$ROOT/conf/nats.conf.tmpl" "$tmpconf" <<'PY' +import os, pathlib, sys +t = pathlib.Path(sys.argv[1]).read_text() +out = t.replace("{{NAME}}", os.environ["NAME"]).replace("{{IP}}", os.environ["IP"]).replace("{{CLUSTER}}", os.environ["CLUSTER"]).replace("{{ROUTES}}", os.environ["ROUTES"]) +pathlib.Path(sys.argv[2]).write_text(out) +PY + sudo pct exec "${VMIDS[$i]}" -- bash -c 'cat > /tmp/nats.conf' < "$tmpconf" + sudo pct exec "${VMIDS[$i]}" -- bash -c 'cat > /tmp/nats-server.service' < "$ROOT/systemd/nats-server.service" + rm -f "$tmpconf" + sudo pct exec "${VMIDS[$i]}" -- bash -lc " +set -e +export DEBIAN_FRONTEND=noninteractive +id nats >/dev/null 2>&1 || useradd -r -s /usr/sbin/nologin nats +install -d -m 755 -o nats -g nats /var/lib/nats/jetstream /etc/nats +mv /tmp/nats.conf /etc/nats/nats.conf +chown root:root /etc/nats/nats.conf +chmod 644 /etc/nats/nats.conf +if [[ ! -x /usr/local/bin/nats-server ]]; then + curl -fsSL https://github.com/nats-io/nats-server/releases/download/v${NATS_VER}/nats-server-v${NATS_VER}-linux-amd64.tar.gz -o /tmp/nats.tgz + tar -xzf /tmp/nats.tgz -C /tmp + install -m 0755 /tmp/nats-server-v${NATS_VER}-linux-amd64/nats-server /usr/local/bin/nats-server + rm -rf /tmp/nats.tgz /tmp/nats-server-v${NATS_VER}-linux-amd64 +fi +install -m 644 /tmp/nats-server.service /etc/systemd/system/nats-server.service +systemctl daemon-reload +systemctl enable --now nats-server +" + echo "nats-server ${NAMES[$i]} ${IPS[$i]}:4222 cluster ${IPS[$i]}:6222" +done + +echo "cluster client URL: nats://${IPS[0]}:4222,nats://${IPS[1]}:4222,nats://${IPS[2]}:4222" +echo "lab loopback NATS on the host is unchanged (127.0.0.1:4222)" +echo "next: bash $ROOT/scripts/test.sh" diff --git a/scripts/cutover-ns1.sh b/scripts/cutover-ns1.sh new file mode 100755 index 0000000..6fa073c --- /dev/null +++ b/scripts/cutover-ns1.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# Point NS1 test modules at the 3-node vmbr1 cluster. Does not change MOCK_VERAE or Zapier. +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +# shellcheck disable=SC1091 +. "$ROOT/client.env" +export PATH="/usr/sbin:/usr/bin:/bin:$PATH" +bash "$ROOT/scripts/ensure-streams.sh" +echo "NATS_URL=$NATS_URL" +echo "streams ensured. restart keep + fleet on the host after copying overlay/service JSON." diff --git a/scripts/ensure-streams.sh b/scripts/ensure-streams.sh new file mode 100755 index 0000000..4b661be --- /dev/null +++ b/scripts/ensure-streams.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Create product JetStream streams with replicas=3 on the Proxmox cluster. +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +# shellcheck disable=SC1091 +. "$ROOT/client.env" +export PATH="/usr/sbin:/usr/bin:/bin:/usr/local/bin:$PATH" +VMID="${1:-511}" +sudo pct exec "$VMID" -- bash -lc " +set -e +export DEBIAN_FRONTEND=noninteractive +export PATH=/usr/local/bin:/usr/bin:/bin +export NATS_URL=nats://10.10.10.21:4222 +if [[ ! -x /usr/local/bin/nats ]]; then + apt-get install -y --no-install-recommends unzip >/dev/null + curl -fsSL https://github.com/nats-io/natscli/releases/download/v0.1.6/nats-0.1.6-linux-amd64.zip -o /tmp/natscli.zip + rm -rf /tmp/natscli && mkdir -p /tmp/natscli + unzip -o /tmp/natscli.zip -d /tmp/natscli >/dev/null + BIN=\$(find /tmp/natscli -type f -name nats | head -1) + install -m 0755 \"\$BIN\" /usr/local/bin/nats +fi +add() { + local name=\$1 subj=\$2 + nats stream info \"\$name\" >/dev/null 2>&1 && return 0 + nats stream add \"\$name\" --subjects=\"\$subj\" --replicas=3 --storage=file --retention=limits --discard=old --max-msgs=-1 --max-bytes=-1 --max-age=24h --dupe-window=2m --defaults +} +add ZAPIER_JOBS 'verae.zapier.jobs.watch' +add ZAPIER_EVENTS 'verae.zapier.jobs.events' +add ZAPIER_WEBHOOKS 'verae.zapier.webhooks.deliver' +add ZAPIER_USAGE 'verae.zapier.usage' +add VERAE_ARCHIVE 'verae.archive.>' +nats stream ls +" +echo "streams ready on cluster (replicas=3)" diff --git a/scripts/lib-ct.sh b/scripts/lib-ct.sh new file mode 100755 index 0000000..1a4ff8e --- /dev/null +++ b/scripts/lib-ct.sh @@ -0,0 +1,62 @@ +# shellcheck shell=bash +# Shared LXC bootstrap for NS1 Proxmox. Does not generate SSH keys if one exists. +export PATH="/usr/sbin:/usr/bin:/bin:$PATH" + +ct_require_proxmox() { + if [[ ! -d /etc/pve/nodes ]]; then + echo "not a Proxmox host" >&2 + return 1 + fi + command -v pct >/dev/null || { echo "pct missing" >&2; return 1; } +} + +ct_ensure() { + local vmid="$1" hostname="$2" ip="$3" + if [[ ! -f "$TEMPLATE" ]]; then + echo "missing template $TEMPLATE" >&2 + return 1 + fi + if ! sudo pct status "$vmid" >/dev/null 2>&1; then + echo "pct create $vmid $hostname $ip/24" + sudo pct create "$vmid" "$TEMPLATE" \ + --hostname "$hostname" \ + --memory "$MEMORY" --cores "$CORES" --swap 256 \ + --net0 "name=eth0,bridge=${BRIDGE},ip=${ip}/24,gw=${GW},type=veth" \ + --rootfs "${STORAGE}:${DISK}" \ + --unprivileged 1 --onboot 1 --nameserver "$DNS" \ + --features nesting=1 \ + --ostype ubuntu + else + echo "CT $vmid already exists" + fi + sudo pct start "$vmid" 2>/dev/null || true + local i + for i in $(seq 1 40); do + sudo pct exec "$vmid" -- true 2>/dev/null && return 0 + sleep 2 + done + echo "CT $vmid did not start" >&2 + return 1 +} + +ct_bootstrap_user() { + local vmid="$1" + local pub="" + [[ -f "$HOME/.ssh/id_ed25519.pub" ]] && pub="$(cat "$HOME/.ssh/id_ed25519.pub")" + [[ -z "$pub" && -f "$HOME/.ssh/authorized_keys" ]] && pub="$(head -1 "$HOME/.ssh/authorized_keys")" + [[ -n "$pub" ]] || { echo "no ssh public key" >&2; return 1; } + sudo pct exec "$vmid" -- bash -lc " +set -e +export DEBIAN_FRONTEND=noninteractive +apt-get update -qq +apt-get install -y --no-install-recommends openssh-server sudo curl ca-certificates xz-utils tar +id $USER_NAME >/dev/null 2>&1 || useradd -m -s /bin/bash $USER_NAME +echo '$USER_NAME ALL=(ALL) NOPASSWD:ALL' >/etc/sudoers.d/90-$USER_NAME +chmod 440 /etc/sudoers.d/90-$USER_NAME +install -d -m 700 -o $USER_NAME -g $USER_NAME /home/$USER_NAME/.ssh +grep -qxF '$pub' /home/$USER_NAME/.ssh/authorized_keys 2>/dev/null || echo '$pub' >>/home/$USER_NAME/.ssh/authorized_keys +chown $USER_NAME:$USER_NAME /home/$USER_NAME/.ssh/authorized_keys +chmod 600 /home/$USER_NAME/.ssh/authorized_keys +systemctl enable --now ssh +" +} diff --git a/scripts/status.sh b/scripts/status.sh new file mode 100755 index 0000000..1306656 --- /dev/null +++ b/scripts/status.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +# shellcheck disable=SC1091 +. "$ROOT/cluster.env" +export PATH="/usr/sbin:/usr/bin:/bin:$PATH" +printf '%s\n' "$NODES" | awk 'NF==3 {print}' | while read -r vmid name ip; do + st="$(sudo pct status "$vmid" 2>/dev/null || echo missing)" + js="$(sudo pct exec "$vmid" -- curl -fsS --max-time 2 http://127.0.0.1:8222/varz 2>/dev/null || echo '{}')" + echo "$vmid $name $ip $st" + python3 -c " +import json,sys +try: + d=json.loads(sys.argv[1]) +except Exception: + print(' nats down') + raise SystemExit +print(' server_name', d.get('server_name'), 'cluster', (d.get('cluster') or {}).get('name'), 'routes', len((d.get('cluster') or {}).get('urls') or d.get('connect_urls') or [])) +print(' jetstream', bool(d.get('jetstream')), 'port', d.get('port'), 'host', d.get('host')) +" "$js" 2>/dev/null || echo " nats down" +done +echo "host loopback still: $(ss -lnt | grep '127.0.0.1:4222' && echo up || echo down)" diff --git a/scripts/test.sh b/scripts/test.sh new file mode 100755 index 0000000..957916c --- /dev/null +++ b/scripts/test.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Local syntax check always. Live cluster check when pct is present. +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +bash -n "$ROOT/scripts/lib-ct.sh" +bash -n "$ROOT/scripts/create-cluster.sh" +bash -n "$ROOT/scripts/status.sh" +grep -q 'host: {{IP}}' "$ROOT/conf/nats.conf.tmpl" +grep -qv '0.0.0.0' "$ROOT/conf/nats.conf.tmpl" +if [[ ! -d /etc/pve/nodes ]]; then + echo "OK (syntax; not on Proxmox)" + exit 0 +fi +# shellcheck disable=SC1091 +. "$ROOT/cluster.env" +export PATH="/usr/sbin:/usr/bin:/bin:$PATH" +mapfile -t rows < <(printf '%s\n' "$NODES" | awk 'NF==3 {print}') +ready=0 +for row in "${rows[@]}"; do + # shellcheck disable=SC2086 + set -- $row + vmid=$1 name=$2 ip=$3 + js="$(sudo pct exec "$vmid" -- curl -fsS --max-time 3 http://127.0.0.1:8222/varz 2>/dev/null || true)" + echo "$js" | grep -q '"jetstream"' && ready=$((ready + 1)) || echo "not ready $name" +done +[[ $ready -eq 3 ]] || { echo "cluster not fully up ($ready/3)" >&2; exit 1; } + +# nats CLI on first node +first="$(echo "${rows[0]}" | awk '{print $1}')" +sudo pct exec "$first" -- bash -lc ' +set -e +export DEBIAN_FRONTEND=noninteractive +if [[ ! -x /usr/local/bin/nats ]]; then + apt-get install -y --no-install-recommends unzip >/dev/null + curl -fsSL https://github.com/nats-io/natscli/releases/download/v0.1.6/nats-0.1.6-linux-amd64.zip -o /tmp/natscli.zip + rm -rf /tmp/natscli && mkdir -p /tmp/natscli + unzip -o /tmp/natscli.zip -d /tmp/natscli >/dev/null + BIN=$(find /tmp/natscli /tmp -maxdepth 3 -type f -name nats | head -1) + test -n "$BIN" + install -m 0755 "$BIN" /usr/local/bin/nats +fi +IP=$(hostname -I | awk "{print \$1}") +export NATS_URL=nats://$IP:4222 +export PATH=/usr/local/bin:/usr/bin:/bin +nats stream rm VERAE_PX_TEST --force >/dev/null 2>&1 || true +nats stream add VERAE_PX_TEST --subjects="verae.px.test" --replicas=3 --storage=file --retention=limits --discard=old --max-msgs=-1 --max-bytes=-1 --max-age=1h --dupe-window=2m --defaults +nats pub verae.px.test cluster-ok +nats stream info VERAE_PX_TEST +' +echo "OK live cluster (3/3 + replicas=3 stream)" diff --git a/systemd/nats-server.service b/systemd/nats-server.service new file mode 100644 index 0000000..86e5076 --- /dev/null +++ b/systemd/nats-server.service @@ -0,0 +1,16 @@ +[Unit] +Description=NATS JetStream (Verae cluster node) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=nats +Group=nats +ExecStart=/usr/local/bin/nats-server -c /etc/nats/nats.conf +LimitNOFILE=65536 +Restart=on-failure +RestartSec=2 + +[Install] +WantedBy=multi-user.target