diff --git a/packages/verae-nats-cluster/README.md b/packages/verae-nats-cluster/README.md index a73d1fb..93f5f00 100644 --- a/packages/verae-nats-cluster/README.md +++ b/packages/verae-nats-cluster/README.md @@ -17,7 +17,14 @@ Cluster name: `verae`. Client URL: nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222 ``` -**Not** on `vmbr0`. **Not** `0.0.0.0`. Host lab NATS on `127.0.0.1:4222` is left running so current keep/fleet/middleware stay up until you cut over. +**Cluster routes (`:6222`) and hub leaf (`:7422`) stay on `vmbr1`.** A **token-gated public client edge** on NS1 forwards to that hub: + +```text +nats://@70.88.205.138:4222 +wss://nats.zapier.georgelambert.org # WebSocket (CONNECT with the same token) +``` + +Token file on NS1: `/etc/nats/public-edge.token`. Enable/recreate: `bash scripts/expose-public.sh`. Do **not** publish `:6222` or `:8222`. Internal clients keep using the three `10.10.10.21–23` URLs (no token). ```bash # on NS1 (the Proxmox host) diff --git a/packages/verae-nats-cluster/conf/nats.conf.tmpl b/packages/verae-nats-cluster/conf/nats.conf.tmpl index abae4ae..3b0bfc3 100644 --- a/packages/verae-nats-cluster/conf/nats.conf.tmpl +++ b/packages/verae-nats-cluster/conf/nats.conf.tmpl @@ -17,3 +17,9 @@ cluster { {{ROUTES}} ] } + +# Leaf hub for the public edge on NS1. Private vmbr1 only — not vmbr0. +leafnodes { + host: {{IP}} + port: 7422 +} diff --git a/packages/verae-nats-cluster/conf/public-edge.conf.tmpl b/packages/verae-nats-cluster/conf/public-edge.conf.tmpl new file mode 100644 index 0000000..c8b1e7f --- /dev/null +++ b/packages/verae-nats-cluster/conf/public-edge.conf.tmpl @@ -0,0 +1,24 @@ +# Public client edge on NS1. Binds the public IPv4 only (not 0.0.0.0). +# Forwards to the private 3-node cluster via leafnodes. No JetStream here. +# Cluster routes :6222 and hub leaf :7422 stay on vmbr1. +server_name: nats-public-edge +listen: {{PUBLIC_IP}}:4222 +http: 127.0.0.1:8229 + +authorization { + token: "{{TOKEN}}" +} + +websocket { + listen: 127.0.0.1:14422 + no_tls: true + compression: true +} + +leafnodes { + remotes: [ + { url: "nats://10.10.10.21:7422" } + { url: "nats://10.10.10.22:7422" } + { url: "nats://10.10.10.23:7422" } + ] +} diff --git a/packages/verae-nats-cluster/scripts/expose-public.sh b/packages/verae-nats-cluster/scripts/expose-public.sh new file mode 100755 index 0000000..bd7f297 --- /dev/null +++ b/packages/verae-nats-cluster/scripts/expose-public.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# Public NATS *client* edge on NS1. Does not publish cluster :6222 or monitor :8222. +# Hub stays on vmbr1; this process binds 70.88.205.138:4222 with a token. +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +export PATH="/usr/sbin:/usr/bin:/bin:/usr/local/bin:$PATH" +HOST="$(hostname -f 2>/dev/null || hostname)" +case "$HOST" in + NS1.GEORGELAMBERT.ORG|NS1|ns1.georgelambert.org|ns1) ;; + *) echo "refusing: expose-public.sh must run on NS1, got '$HOST'" >&2; exit 1 ;; +esac + +PUBLIC_IP="${PUBLIC_IP:-70.88.205.138}" +TOKEN_FILE="${TOKEN_FILE:-/etc/nats/public-edge.token}" +# shellcheck disable=SC1091 +. "$ROOT/cluster.env" + +if [[ -f "$TOKEN_FILE" ]]; then + TOKEN="$(cat "$TOKEN_FILE")" +else + TOKEN="$(openssl rand -hex 24)" + sudo install -d -m 755 /etc/nats + echo "$TOKEN" | sudo tee "$TOKEN_FILE" >/dev/null + sudo chmod 600 "$TOKEN_FILE" +fi + +# Enable leaf hub on each cluster node (vmbr1 :7422 only). +mapfile -t rows < <(printf '%s\n' "$NODES" | awk 'NF==3 {print}') +for row in "${rows[@]}"; do + # shellcheck disable=SC2086 + set -- $row + vmid=$1 ip=$3 + sudo pct exec "$vmid" -- bash -lc " +set -e +f=/etc/nats/nats.conf +if ! grep -q '^leafnodes' \"\$f\"; then + cat >> \"\$f\" <&2; exit 1; } + +tmp="$(mktemp)" +sed -e "s/{{PUBLIC_IP}}/$PUBLIC_IP/" -e "s/{{TOKEN}}/$TOKEN/" \ + "$ROOT/conf/public-edge.conf.tmpl" >"$tmp" +sudo install -d -m 755 /etc/nats +sudo install -m 600 "$tmp" /etc/nats/public-edge.conf +rm -f "$tmp" +sed "s|/usr/local/bin/nats-server|$NSBIN|" "$ROOT/systemd/nats-public-edge.service" \ + | sudo tee /etc/systemd/system/nats-public-edge.service >/dev/null +sudo systemctl daemon-reload +sudo systemctl enable --now nats-public-edge.service +sleep 1 +sudo systemctl --no-pager --full status nats-public-edge.service | head -20 +ss -lntp | grep -E "4222|14422|8229" || true + +# Caddy WebSocket TLS front (wss://nats.zapier.georgelambert.org) +CADDY=/etc/caddy/sites/zapier-nats.caddy +sudo tee "$CADDY" >/dev/null <<'C' +# Public NATS WebSocket. Raw TCP is 70.88.205.138:4222 (token). Do not proxy :6222/:7422. +http://nats.zapier.georgelambert.org { + redir https://nats.zapier.georgelambert.org{uri} permanent +} +nats.zapier.georgelambert.org { + encode gzip zstd + reverse_proxy 127.0.0.1:14422 +} +C +sudo systemctl reload caddy || sudo service caddy reload || true + +echo "PUBLIC_NATS_URL=nats://${TOKEN}@${PUBLIC_IP}:4222" +echo "PUBLIC_WS_URL=wss://nats.zapier.georgelambert.org" +echo "token file $TOKEN_FILE" diff --git a/packages/verae-nats-cluster/scripts/test.sh b/packages/verae-nats-cluster/scripts/test.sh index feb50b8..60003bf 100755 --- a/packages/verae-nats-cluster/scripts/test.sh +++ b/packages/verae-nats-cluster/scripts/test.sh @@ -9,6 +9,9 @@ bash -n "$ROOT/scripts/bench.sh" bash -n "$ROOT/scripts/study-on-ns1.sh" bash -n "$ROOT/scripts/maximize-ns1-study.sh" bash -n "$ROOT/scripts/exhaustive-ns1-study.sh" +bash -n "$ROOT/scripts/expose-public.sh" +grep -q 'leafnodes' "$ROOT/conf/nats.conf.tmpl" +grep -q '70.88.205.138\|PUBLIC_IP' "$ROOT/conf/public-edge.conf.tmpl" grep -q 'host: {{IP}}' "$ROOT/conf/nats.conf.tmpl" grep -qv '0.0.0.0' "$ROOT/conf/nats.conf.tmpl" if [[ ! -d /etc/pve/nodes ]]; then diff --git a/packages/verae-nats-cluster/systemd/nats-public-edge.service b/packages/verae-nats-cluster/systemd/nats-public-edge.service new file mode 100644 index 0000000..00d9c38 --- /dev/null +++ b/packages/verae-nats-cluster/systemd/nats-public-edge.service @@ -0,0 +1,14 @@ +[Unit] +Description=NATS public client edge (token) → verae cluster leaf +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +ExecStart=/usr/local/bin/nats-server -c /etc/nats/public-edge.conf +LimitNOFILE=65536 +Restart=on-failure +RestartSec=2 + +[Install] +WantedBy=multi-user.target