Add a three-node NATS JetStream cluster on distinct Proxmox LXC guests.
Some checks are pending
offline / test (push) Waiting to run
Some checks are pending
offline / test (push) Waiting to run
LXC 511–513 (nats-a/b/c on 10.10.10.21–23, vmbr1 only) form cluster verae with replicas=3. Host loopback 127.0.0.1:4222 is unchanged.
This commit is contained in:
parent
7a5e25639e
commit
69e28af4d8
27 changed files with 365 additions and 2 deletions
69
packages/verae-nats-cluster/scripts/create-cluster.sh
Executable file
69
packages/verae-nats-cluster/scripts/create-cluster.sh
Executable file
|
|
@ -0,0 +1,69 @@
|
|||
#!/usr/bin/env bash
|
||||
# Create three distinct Proxmox LXC guests and start a JetStream cluster on vmbr1.
|
||||
# Does not touch host loopback NATS (127.0.0.1:4222) or vmbr0.
|
||||
set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
. "$ROOT/cluster.env"
|
||||
# shellcheck disable=SC1091
|
||||
. "$ROOT/scripts/lib-ct.sh"
|
||||
ct_require_proxmox
|
||||
|
||||
mapfile -t rows < <(printf '%s\n' "$NODES" | awk 'NF==3 {print}')
|
||||
[[ ${#rows[@]} -eq 3 ]] || { echo "need exactly 3 nodes in cluster.env" >&2; exit 1; }
|
||||
|
||||
declare -a VMIDS NAMES IPS
|
||||
for row in "${rows[@]}"; do
|
||||
# shellcheck disable=SC2086
|
||||
set -- $row
|
||||
VMIDS+=("$1"); NAMES+=("$2"); IPS+=("$3")
|
||||
done
|
||||
|
||||
i=0
|
||||
for i in 0 1 2; do
|
||||
ct_ensure "${VMIDS[$i]}" "${NAMES[$i]}" "${IPS[$i]}"
|
||||
ct_bootstrap_user "${VMIDS[$i]}"
|
||||
done
|
||||
|
||||
# Install nats-server + conf + systemd on each guest
|
||||
for i in 0 1 2; do
|
||||
routes=""
|
||||
for j in 0 1 2; do
|
||||
[[ $i -eq $j ]] && continue
|
||||
routes="${routes} nats-route://${IPS[$j]}:6222"$'\n'
|
||||
done
|
||||
tmpconf="$(mktemp)"
|
||||
NAME="${NAMES[$i]}" IP="${IPS[$i]}" CLUSTER="$CLUSTER_NAME" ROUTES="$routes" \
|
||||
python3 - "$ROOT/conf/nats.conf.tmpl" "$tmpconf" <<'PY'
|
||||
import os, pathlib, sys
|
||||
t = pathlib.Path(sys.argv[1]).read_text()
|
||||
out = t.replace("{{NAME}}", os.environ["NAME"]).replace("{{IP}}", os.environ["IP"]).replace("{{CLUSTER}}", os.environ["CLUSTER"]).replace("{{ROUTES}}", os.environ["ROUTES"])
|
||||
pathlib.Path(sys.argv[2]).write_text(out)
|
||||
PY
|
||||
sudo pct exec "${VMIDS[$i]}" -- bash -c 'cat > /tmp/nats.conf' < "$tmpconf"
|
||||
sudo pct exec "${VMIDS[$i]}" -- bash -c 'cat > /tmp/nats-server.service' < "$ROOT/systemd/nats-server.service"
|
||||
rm -f "$tmpconf"
|
||||
sudo pct exec "${VMIDS[$i]}" -- bash -lc "
|
||||
set -e
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
id nats >/dev/null 2>&1 || useradd -r -s /usr/sbin/nologin nats
|
||||
install -d -m 755 -o nats -g nats /var/lib/nats/jetstream /etc/nats
|
||||
mv /tmp/nats.conf /etc/nats/nats.conf
|
||||
chown root:root /etc/nats/nats.conf
|
||||
chmod 644 /etc/nats/nats.conf
|
||||
if [[ ! -x /usr/local/bin/nats-server ]]; then
|
||||
curl -fsSL https://github.com/nats-io/nats-server/releases/download/v${NATS_VER}/nats-server-v${NATS_VER}-linux-amd64.tar.gz -o /tmp/nats.tgz
|
||||
tar -xzf /tmp/nats.tgz -C /tmp
|
||||
install -m 0755 /tmp/nats-server-v${NATS_VER}-linux-amd64/nats-server /usr/local/bin/nats-server
|
||||
rm -rf /tmp/nats.tgz /tmp/nats-server-v${NATS_VER}-linux-amd64
|
||||
fi
|
||||
install -m 644 /tmp/nats-server.service /etc/systemd/system/nats-server.service
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now nats-server
|
||||
"
|
||||
echo "nats-server ${NAMES[$i]} ${IPS[$i]}:4222 cluster ${IPS[$i]}:6222"
|
||||
done
|
||||
|
||||
echo "cluster client URL: nats://${IPS[0]}:4222,nats://${IPS[1]}:4222,nats://${IPS[2]}:4222"
|
||||
echo "lab loopback NATS on the host is unchanged (127.0.0.1:4222)"
|
||||
echo "next: bash $ROOT/scripts/test.sh"
|
||||
62
packages/verae-nats-cluster/scripts/lib-ct.sh
Executable file
62
packages/verae-nats-cluster/scripts/lib-ct.sh
Executable file
|
|
@ -0,0 +1,62 @@
|
|||
# shellcheck shell=bash
|
||||
# Shared LXC bootstrap for NS1 Proxmox. Does not generate SSH keys if one exists.
|
||||
export PATH="/usr/sbin:/usr/bin:/bin:$PATH"
|
||||
|
||||
ct_require_proxmox() {
|
||||
if [[ ! -d /etc/pve/nodes ]]; then
|
||||
echo "not a Proxmox host" >&2
|
||||
return 1
|
||||
fi
|
||||
command -v pct >/dev/null || { echo "pct missing" >&2; return 1; }
|
||||
}
|
||||
|
||||
ct_ensure() {
|
||||
local vmid="$1" hostname="$2" ip="$3"
|
||||
if [[ ! -f "$TEMPLATE" ]]; then
|
||||
echo "missing template $TEMPLATE" >&2
|
||||
return 1
|
||||
fi
|
||||
if ! sudo pct status "$vmid" >/dev/null 2>&1; then
|
||||
echo "pct create $vmid $hostname $ip/24"
|
||||
sudo pct create "$vmid" "$TEMPLATE" \
|
||||
--hostname "$hostname" \
|
||||
--memory "$MEMORY" --cores "$CORES" --swap 256 \
|
||||
--net0 "name=eth0,bridge=${BRIDGE},ip=${ip}/24,gw=${GW},type=veth" \
|
||||
--rootfs "${STORAGE}:${DISK}" \
|
||||
--unprivileged 1 --onboot 1 --nameserver "$DNS" \
|
||||
--features nesting=1 \
|
||||
--ostype ubuntu
|
||||
else
|
||||
echo "CT $vmid already exists"
|
||||
fi
|
||||
sudo pct start "$vmid" 2>/dev/null || true
|
||||
local i
|
||||
for i in $(seq 1 40); do
|
||||
sudo pct exec "$vmid" -- true 2>/dev/null && return 0
|
||||
sleep 2
|
||||
done
|
||||
echo "CT $vmid did not start" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
ct_bootstrap_user() {
|
||||
local vmid="$1"
|
||||
local pub=""
|
||||
[[ -f "$HOME/.ssh/id_ed25519.pub" ]] && pub="$(cat "$HOME/.ssh/id_ed25519.pub")"
|
||||
[[ -z "$pub" && -f "$HOME/.ssh/authorized_keys" ]] && pub="$(head -1 "$HOME/.ssh/authorized_keys")"
|
||||
[[ -n "$pub" ]] || { echo "no ssh public key" >&2; return 1; }
|
||||
sudo pct exec "$vmid" -- bash -lc "
|
||||
set -e
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get update -qq
|
||||
apt-get install -y --no-install-recommends openssh-server sudo curl ca-certificates xz-utils tar
|
||||
id $USER_NAME >/dev/null 2>&1 || useradd -m -s /bin/bash $USER_NAME
|
||||
echo '$USER_NAME ALL=(ALL) NOPASSWD:ALL' >/etc/sudoers.d/90-$USER_NAME
|
||||
chmod 440 /etc/sudoers.d/90-$USER_NAME
|
||||
install -d -m 700 -o $USER_NAME -g $USER_NAME /home/$USER_NAME/.ssh
|
||||
grep -qxF '$pub' /home/$USER_NAME/.ssh/authorized_keys 2>/dev/null || echo '$pub' >>/home/$USER_NAME/.ssh/authorized_keys
|
||||
chown $USER_NAME:$USER_NAME /home/$USER_NAME/.ssh/authorized_keys
|
||||
chmod 600 /home/$USER_NAME/.ssh/authorized_keys
|
||||
systemctl enable --now ssh
|
||||
"
|
||||
}
|
||||
22
packages/verae-nats-cluster/scripts/status.sh
Executable file
22
packages/verae-nats-cluster/scripts/status.sh
Executable file
|
|
@ -0,0 +1,22 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
. "$ROOT/cluster.env"
|
||||
export PATH="/usr/sbin:/usr/bin:/bin:$PATH"
|
||||
printf '%s\n' "$NODES" | awk 'NF==3 {print}' | while read -r vmid name ip; do
|
||||
st="$(sudo pct status "$vmid" 2>/dev/null || echo missing)"
|
||||
js="$(sudo pct exec "$vmid" -- curl -fsS --max-time 2 http://127.0.0.1:8222/varz 2>/dev/null || echo '{}')"
|
||||
echo "$vmid $name $ip $st"
|
||||
python3 -c "
|
||||
import json,sys
|
||||
try:
|
||||
d=json.loads(sys.argv[1])
|
||||
except Exception:
|
||||
print(' nats down')
|
||||
raise SystemExit
|
||||
print(' server_name', d.get('server_name'), 'cluster', (d.get('cluster') or {}).get('name'), 'routes', len((d.get('cluster') or {}).get('urls') or d.get('connect_urls') or []))
|
||||
print(' jetstream', bool(d.get('jetstream')), 'port', d.get('port'), 'host', d.get('host'))
|
||||
" "$js" 2>/dev/null || echo " nats down"
|
||||
done
|
||||
echo "host loopback still: $(ss -lnt | grep '127.0.0.1:4222' && echo up || echo down)"
|
||||
50
packages/verae-nats-cluster/scripts/test.sh
Executable file
50
packages/verae-nats-cluster/scripts/test.sh
Executable file
|
|
@ -0,0 +1,50 @@
|
|||
#!/usr/bin/env bash
|
||||
# Local syntax check always. Live cluster check when pct is present.
|
||||
set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
bash -n "$ROOT/scripts/lib-ct.sh"
|
||||
bash -n "$ROOT/scripts/create-cluster.sh"
|
||||
bash -n "$ROOT/scripts/status.sh"
|
||||
grep -q 'host: {{IP}}' "$ROOT/conf/nats.conf.tmpl"
|
||||
grep -qv '0.0.0.0' "$ROOT/conf/nats.conf.tmpl"
|
||||
if [[ ! -d /etc/pve/nodes ]]; then
|
||||
echo "OK (syntax; not on Proxmox)"
|
||||
exit 0
|
||||
fi
|
||||
# shellcheck disable=SC1091
|
||||
. "$ROOT/cluster.env"
|
||||
export PATH="/usr/sbin:/usr/bin:/bin:$PATH"
|
||||
mapfile -t rows < <(printf '%s\n' "$NODES" | awk 'NF==3 {print}')
|
||||
ready=0
|
||||
for row in "${rows[@]}"; do
|
||||
# shellcheck disable=SC2086
|
||||
set -- $row
|
||||
vmid=$1 name=$2 ip=$3
|
||||
js="$(sudo pct exec "$vmid" -- curl -fsS --max-time 3 http://127.0.0.1:8222/varz 2>/dev/null || true)"
|
||||
echo "$js" | grep -q '"jetstream"' && ready=$((ready + 1)) || echo "not ready $name"
|
||||
done
|
||||
[[ $ready -eq 3 ]] || { echo "cluster not fully up ($ready/3)" >&2; exit 1; }
|
||||
|
||||
# nats CLI on first node
|
||||
first="$(echo "${rows[0]}" | awk '{print $1}')"
|
||||
sudo pct exec "$first" -- bash -lc '
|
||||
set -e
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
if [[ ! -x /usr/local/bin/nats ]]; then
|
||||
apt-get install -y --no-install-recommends unzip >/dev/null
|
||||
curl -fsSL https://github.com/nats-io/natscli/releases/download/v0.1.6/nats-0.1.6-linux-amd64.zip -o /tmp/natscli.zip
|
||||
rm -rf /tmp/natscli && mkdir -p /tmp/natscli
|
||||
unzip -o /tmp/natscli.zip -d /tmp/natscli >/dev/null
|
||||
BIN=$(find /tmp/natscli /tmp -maxdepth 3 -type f -name nats | head -1)
|
||||
test -n "$BIN"
|
||||
install -m 0755 "$BIN" /usr/local/bin/nats
|
||||
fi
|
||||
IP=$(hostname -I | awk "{print \$1}")
|
||||
export NATS_URL=nats://$IP:4222
|
||||
export PATH=/usr/local/bin:/usr/bin:/bin
|
||||
nats stream rm VERAE_PX_TEST --force >/dev/null 2>&1 || true
|
||||
nats stream add VERAE_PX_TEST --subjects="verae.px.test" --replicas=3 --storage=file --retention=limits --discard=old --max-msgs=-1 --max-bytes=-1 --max-age=1h --dupe-window=2m --defaults
|
||||
nats pub verae.px.test cluster-ok
|
||||
nats stream info VERAE_PX_TEST
|
||||
'
|
||||
echo "OK live cluster (3/3 + replicas=3 stream)"
|
||||
Loading…
Add table
Add a link
Reference in a new issue