Snapshot of verae-nats-cluster from zapier monorepo (NATS cluster bench)
This commit is contained in:
commit
8afd8165dd
41 changed files with 1054 additions and 0 deletions
176
scripts/bench-report.py
Executable file
176
scripts/bench-report.py
Executable file
|
|
@ -0,0 +1,176 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Turn nats bench text logs + latency JSON into markdown."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def fmt_int(s: str | None) -> str:
|
||||
if not s:
|
||||
return "—"
|
||||
return f"{int(s):,}"
|
||||
|
||||
|
||||
def parse_bench(text: str) -> dict[str, str]:
|
||||
out: dict[str, str] = {"kind": "throughput"}
|
||||
m = re.search(r"(?m)^\s*Pub stats:\s*([0-9,]+)\s*msgs/sec\s*~\s*([0-9.]+)\s*MB/sec", text)
|
||||
if m:
|
||||
out["pub_msgs"] = m.group(1).replace(",", "")
|
||||
out["pub_mb"] = m.group(2)
|
||||
m = re.search(r"(?m)^\s*Sub stats:\s*([0-9,]+)\s*msgs/sec\s*~\s*([0-9.]+)\s*MB/sec", text)
|
||||
if m:
|
||||
out["sub_msgs"] = m.group(1).replace(",", "")
|
||||
out["sub_mb"] = m.group(2)
|
||||
m = re.search(r"NATS Pub/Sub stats:\s*([0-9,]+)\s*msgs/sec\s*~\s*([0-9.]+)\s*MB/sec", text)
|
||||
if m:
|
||||
out["agg_msgs"] = m.group(1).replace(",", "")
|
||||
out["agg_mb"] = m.group(2)
|
||||
if "JetStream" in text or "--js" in text or "js-" in text:
|
||||
out["mode"] = "jetstream r=3 file"
|
||||
else:
|
||||
out["mode"] = "core pub/sub"
|
||||
# nats 0.1.6 prints min/avg/max as msgs/sec across publishers, not µs delay
|
||||
m = re.search(
|
||||
r"min\s+([0-9,]+)\s*\|\s*avg\s+([0-9,]+)\s*\|\s*max\s+([0-9,]+)\s*\|\s*stddev\s+([0-9,]+)\s*msgs",
|
||||
text,
|
||||
)
|
||||
if m:
|
||||
out["pub_spread"] = f"{m.group(1)}–{m.group(3)} (avg {m.group(2)})"
|
||||
return out
|
||||
|
||||
|
||||
def parse_lat(text: str) -> dict[str, str] | None:
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if line.startswith("{") and "p99_us" in line:
|
||||
d = json.loads(line)
|
||||
mode = d.get("mode") or ""
|
||||
return {
|
||||
"kind": "latency",
|
||||
"count": str(d.get("count", "")),
|
||||
"pubs": str(d.get("pubs", "")),
|
||||
"size": str(d.get("size", "")),
|
||||
"mode": str(mode),
|
||||
"min": d.get("min", ""),
|
||||
"avg": d.get("avg", ""),
|
||||
"p50": d.get("p50", ""),
|
||||
"p90": d.get("p90", ""),
|
||||
"p99": d.get("p99", ""),
|
||||
"max": d.get("max", ""),
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
def lat_mode(run: str, recorded: str) -> str:
|
||||
if recorded in ("ping", "flood"):
|
||||
return recorded
|
||||
if "ping" in run:
|
||||
return "ping"
|
||||
return "flood"
|
||||
|
||||
|
||||
def thru_sort(p: dict[str, str]) -> tuple:
|
||||
return (0 if p.get("mode", "").startswith("core") else 1, p.get("run", ""))
|
||||
|
||||
|
||||
def lat_sort(p: dict[str, str]) -> tuple:
|
||||
mode = lat_mode(p.get("run", ""), p.get("mode", ""))
|
||||
return (0 if mode == "ping" else 1, int(p.get("count") or 0), p.get("run", ""))
|
||||
|
||||
|
||||
def main() -> int:
|
||||
folder = Path(sys.argv[1] if len(sys.argv) > 1 else ".")
|
||||
thru: list[dict[str, str]] = []
|
||||
lats: list[dict[str, str]] = []
|
||||
for f in sorted(folder.glob("*.txt")):
|
||||
text = f.read_text(encoding="utf-8", errors="replace")
|
||||
lat = parse_lat(text)
|
||||
if lat:
|
||||
lat["run"] = f.stem
|
||||
lats.append(lat)
|
||||
continue
|
||||
p = parse_bench(text)
|
||||
if p.get("pub_msgs") or p.get("agg_msgs"):
|
||||
p["run"] = f.stem
|
||||
thru.append(p)
|
||||
|
||||
stamp = folder.name if re.fullmatch(r"\d{8}T\d{6}Z", folder.name) else ""
|
||||
|
||||
print("# NATS cluster message speed")
|
||||
print()
|
||||
if stamp:
|
||||
print(f"Run **`{stamp}`** (UTC). ", end="")
|
||||
print(
|
||||
"Client: LXC **510** `verae-px-worker` (`10.10.10.20`), not a nats-* server. "
|
||||
"Servers: `nats-a/b/c` on `10.10.10.21–23` (`vmbr1` only)."
|
||||
)
|
||||
print()
|
||||
print("Client URL:")
|
||||
print()
|
||||
print("```text")
|
||||
print("nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222")
|
||||
print("```")
|
||||
print()
|
||||
print("## Method")
|
||||
print()
|
||||
print("- **Core NATS** is fire-and-forget pub/sub (`nats bench`). No disk, no replica ack.")
|
||||
print(
|
||||
"- **JetStream** uses **file** storage and **replicas=3** (same as product streams). "
|
||||
"The unique stream `benchstream` is deleted between JS loads."
|
||||
)
|
||||
print("- Throughput is **msgs/sec** from nats CLI **0.1.6** (`--no-progress --csv`). Its min/avg/max are publisher **rate spread**, not delay.")
|
||||
print(
|
||||
"- **Ping** delay: one publisher, sequential publish-then-wait. This is one-message round-trip through the cluster."
|
||||
)
|
||||
print(
|
||||
"- **Flood** delay: N publishers dump the whole batch, then the subscriber drains. "
|
||||
"This is **queueing under burst**, not wire RTT."
|
||||
)
|
||||
print("- Probe: `scripts/latency.mjs` (two connections, header timestamp).")
|
||||
print()
|
||||
print("## Throughput")
|
||||
print()
|
||||
print("| Run | Mode | Aggregate msgs/s | Pub msgs/s | Pub MB/s | Sub msgs/s | Sub MB/s |")
|
||||
print("|-----|------|------------------|------------|----------|------------|----------|")
|
||||
for p in sorted(thru, key=thru_sort):
|
||||
print(
|
||||
f"| `{p['run']}` | {p.get('mode', '')} | {fmt_int(p.get('agg_msgs'))} | "
|
||||
f"{fmt_int(p.get('pub_msgs'))} | {p.get('pub_mb') or '—'} | "
|
||||
f"{fmt_int(p.get('sub_msgs'))} | {p.get('sub_mb') or '—'} |"
|
||||
)
|
||||
print()
|
||||
print("## Round-trip delay")
|
||||
print()
|
||||
print("| Run | Kind | Count | Pubs | Size | min | avg | p50 | p90 | p99 | max |")
|
||||
print("|-----|------|-------|------|------|-----|-----|-----|-----|-----|-----|")
|
||||
for p in sorted(lats, key=lat_sort):
|
||||
kind = lat_mode(p.get("run", ""), p.get("mode", ""))
|
||||
label = "ping (sequential RTT)" if kind == "ping" else "flood (burst queueing)"
|
||||
print(
|
||||
f"| `{p['run']}` | {label} | {p.get('count', '')} | {p.get('pubs', '')} | "
|
||||
f"{p.get('size', '')} B | {p.get('min', '')} | {p.get('avg', '')} | "
|
||||
f"{p.get('p50', '')} | {p.get('p90', '')} | {p.get('p99', '')} | {p.get('max', '')} |"
|
||||
)
|
||||
print()
|
||||
print("## What the numbers mean")
|
||||
print()
|
||||
print(
|
||||
"Product job/event/archive traffic is **JetStream r=3 file**. On this three-LXC stand that is about "
|
||||
"**16k durable 128 B pubs/s** (about **13k** at 1 KiB). Core NATS is an upper bound for "
|
||||
"non-durable fan-out: about **0.7–2.0M msgs/s** aggregate at 128 B, or **~630k msgs/s (~616 MB/s)** at 1 KiB with 4 publishers."
|
||||
)
|
||||
print()
|
||||
print(
|
||||
"A quiet request-reply is **~0.3 ms** average, **p99 < 1 ms**. Flood rows in the **150–500 ms** band "
|
||||
"are the subscriber catching up after a burst, which is what a job-events mailbox sees if publishers outrun consumers."
|
||||
)
|
||||
print()
|
||||
print("Re-run on NS1: `bash scripts/bench.sh`. Raw logs/CSVs are under `results/<utc>/`.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
106
scripts/bench.sh
Executable file
106
scripts/bench.sh
Executable file
|
|
@ -0,0 +1,106 @@
|
|||
#!/usr/bin/env bash
|
||||
# Message throughput and delay ladder against the 3-node vmbr1 cluster.
|
||||
# Prefers a client that is not a nats-* server (px-worker LXC 510).
|
||||
set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
. "$ROOT/client.env"
|
||||
export PATH="/usr/sbin:/usr/bin:/bin:/usr/local/bin:$PATH"
|
||||
STAMP="$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
OUT="${BENCH_OUT:-$ROOT/results/$STAMP}"
|
||||
CLIENT_VMID="${CLIENT_VMID:-510}"
|
||||
mkdir -p "$OUT"
|
||||
|
||||
ensure_nats_cli() {
|
||||
local vmid="$1"
|
||||
sudo pct exec "$vmid" -- bash -lc '
|
||||
set -e
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
export PATH=/usr/local/bin:/usr/bin:/bin
|
||||
if [[ ! -x /usr/local/bin/nats ]]; then
|
||||
apt-get install -y --no-install-recommends unzip curl ca-certificates >/dev/null
|
||||
curl -fsSL https://github.com/nats-io/natscli/releases/download/v0.1.6/nats-0.1.6-linux-amd64.zip -o /tmp/natscli.zip
|
||||
rm -rf /tmp/natscli && mkdir -p /tmp/natscli
|
||||
unzip -o /tmp/natscli.zip -d /tmp/natscli >/dev/null
|
||||
BIN=$(find /tmp/natscli -type f -name nats | head -1)
|
||||
install -m 0755 "$BIN" /usr/local/bin/nats
|
||||
fi
|
||||
nats --version
|
||||
'
|
||||
}
|
||||
|
||||
run_one() {
|
||||
local name="$1"
|
||||
shift
|
||||
echo "=== $name ===" | tee "$OUT/$name.txt"
|
||||
# nats bench writes csv itself when --csv is a path inside the guest
|
||||
sudo pct exec "$CLIENT_VMID" -- bash -lc "
|
||||
export PATH=/usr/local/bin:/usr/bin:/bin
|
||||
export NATS_URL='$NATS_URL'
|
||||
nats bench --no-progress --csv=/tmp/bench.csv $*
|
||||
" | tee -a "$OUT/$name.txt"
|
||||
sudo pct exec "$CLIENT_VMID" -- cat /tmp/bench.csv >"$OUT/$name.csv" || true
|
||||
}
|
||||
|
||||
echo "client LXC $CLIENT_VMID NATS_URL=$NATS_URL out=$OUT"
|
||||
ensure_nats_cli "$CLIENT_VMID"
|
||||
|
||||
# Core NATS pub/sub — increasing publishers (same 128 B payload)
|
||||
run_one core-1p1s-50k-128 bench.core.a --pub 1 --sub 1 --msgs 50000 --size 128
|
||||
run_one core-4p4s-100k-128 bench.core.b --pub 4 --sub 4 --msgs 100000 --size 128
|
||||
run_one core-8p8s-200k-128 bench.core.c --pub 8 --sub 8 --msgs 200000 --size 128
|
||||
run_one core-4p4s-50k-1k bench.core.d --pub 4 --sub 4 --msgs 50000 --size 1024
|
||||
|
||||
js_rm() {
|
||||
sudo pct exec "$CLIENT_VMID" -- bash -lc "
|
||||
export PATH=/usr/local/bin:/usr/bin:/bin
|
||||
export NATS_URL='$NATS_URL'
|
||||
nats stream rm benchstream --force >/dev/null 2>&1 || true
|
||||
"
|
||||
}
|
||||
# JetStream file store, replicas=3 (matches product streams)
|
||||
js_rm
|
||||
run_one js-1p-20k-128-r3 bench.js.a --js --purge --pub 1 --msgs 20000 --size 128 --replicas 3 --storage file --maxbytes=512MB --stream=benchstream
|
||||
js_rm
|
||||
run_one js-4p-50k-128-r3 bench.js.b --js --purge --pub 4 --msgs 50000 --size 128 --replicas 3 --storage file --maxbytes=512MB --stream=benchstream
|
||||
js_rm
|
||||
run_one js-4p-20k-1k-r3 bench.js.c --js --purge --pub 4 --msgs 20000 --size 1024 --replicas 3 --storage file --maxbytes=512MB --stream=benchstream
|
||||
js_rm
|
||||
run_one js-2p2s-20k-128-r3 bench.js.d --js --purge --pub 2 --sub 2 --msgs 20000 --size 128 --replicas 3 --storage file --maxbytes=512MB --pull --stream=benchstream
|
||||
js_rm
|
||||
|
||||
# Round-trip delay (two connections, through the cluster) at several loads
|
||||
sudo pct exec "$CLIENT_VMID" -- bash -lc "
|
||||
set -e
|
||||
export PATH=/usr/local/bin:/usr/bin:/bin
|
||||
export NATS_URL='$NATS_URL'
|
||||
mkdir -p /tmp/nats-lat
|
||||
cd /tmp/nats-lat
|
||||
if [[ ! -d node_modules/nats ]]; then
|
||||
npm init -y >/dev/null
|
||||
npm install --no-audit --no-fund nats@2 >/dev/null
|
||||
fi
|
||||
" >/dev/null
|
||||
sudo pct push "$CLIENT_VMID" "$ROOT/scripts/latency.mjs" /tmp/nats-lat/latency.mjs || \
|
||||
sudo pct exec "$CLIENT_VMID" -- bash -c 'cat > /tmp/nats-lat/latency.mjs' < "$ROOT/scripts/latency.mjs"
|
||||
lat() {
|
||||
local name="$1" n="$2" sz="$3" p="$4" mode="${5:-flood}"
|
||||
echo "=== $name ===" | tee "$OUT/$name.txt"
|
||||
sudo pct exec "$CLIENT_VMID" -- bash -lc "
|
||||
export PATH=/usr/local/bin:/usr/bin:/bin
|
||||
export NATS_URL='$NATS_URL'
|
||||
cd /tmp/nats-lat
|
||||
node latency.mjs $n $sz $p $mode
|
||||
" | tee -a "$OUT/$name.txt"
|
||||
}
|
||||
# copy latest probe
|
||||
sudo pct exec "$CLIENT_VMID" -- bash -c 'cat > /tmp/nats-lat/latency.mjs' < "$ROOT/scripts/latency.mjs"
|
||||
lat lat-ping-1k-128 1000 128 1 ping
|
||||
lat lat-1p-5k-128 5000 128 1 flood
|
||||
lat lat-4p-10k-128 10000 128 4 flood
|
||||
lat lat-8p-20k-128 20000 128 8 flood
|
||||
lat lat-4p-5k-1k 5000 1024 4 flood
|
||||
|
||||
python3 "$ROOT/scripts/bench-report.py" "$OUT" >"$OUT/BENCH.md"
|
||||
cp "$OUT/BENCH.md" "$ROOT/BENCH.md"
|
||||
echo "wrote $OUT/BENCH.md and $ROOT/BENCH.md"
|
||||
69
scripts/create-cluster.sh
Executable file
69
scripts/create-cluster.sh
Executable file
|
|
@ -0,0 +1,69 @@
|
|||
#!/usr/bin/env bash
|
||||
# Create three distinct Proxmox LXC guests and start a JetStream cluster on vmbr1.
|
||||
# Does not touch host loopback NATS (127.0.0.1:4222) or vmbr0.
|
||||
set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
. "$ROOT/cluster.env"
|
||||
# shellcheck disable=SC1091
|
||||
. "$ROOT/scripts/lib-ct.sh"
|
||||
ct_require_proxmox
|
||||
|
||||
mapfile -t rows < <(printf '%s\n' "$NODES" | awk 'NF==3 {print}')
|
||||
[[ ${#rows[@]} -eq 3 ]] || { echo "need exactly 3 nodes in cluster.env" >&2; exit 1; }
|
||||
|
||||
declare -a VMIDS NAMES IPS
|
||||
for row in "${rows[@]}"; do
|
||||
# shellcheck disable=SC2086
|
||||
set -- $row
|
||||
VMIDS+=("$1"); NAMES+=("$2"); IPS+=("$3")
|
||||
done
|
||||
|
||||
i=0
|
||||
for i in 0 1 2; do
|
||||
ct_ensure "${VMIDS[$i]}" "${NAMES[$i]}" "${IPS[$i]}"
|
||||
ct_bootstrap_user "${VMIDS[$i]}"
|
||||
done
|
||||
|
||||
# Install nats-server + conf + systemd on each guest
|
||||
for i in 0 1 2; do
|
||||
routes=""
|
||||
for j in 0 1 2; do
|
||||
[[ $i -eq $j ]] && continue
|
||||
routes="${routes} nats-route://${IPS[$j]}:6222"$'\n'
|
||||
done
|
||||
tmpconf="$(mktemp)"
|
||||
NAME="${NAMES[$i]}" IP="${IPS[$i]}" CLUSTER="$CLUSTER_NAME" ROUTES="$routes" \
|
||||
python3 - "$ROOT/conf/nats.conf.tmpl" "$tmpconf" <<'PY'
|
||||
import os, pathlib, sys
|
||||
t = pathlib.Path(sys.argv[1]).read_text()
|
||||
out = t.replace("{{NAME}}", os.environ["NAME"]).replace("{{IP}}", os.environ["IP"]).replace("{{CLUSTER}}", os.environ["CLUSTER"]).replace("{{ROUTES}}", os.environ["ROUTES"])
|
||||
pathlib.Path(sys.argv[2]).write_text(out)
|
||||
PY
|
||||
sudo pct exec "${VMIDS[$i]}" -- bash -c 'cat > /tmp/nats.conf' < "$tmpconf"
|
||||
sudo pct exec "${VMIDS[$i]}" -- bash -c 'cat > /tmp/nats-server.service' < "$ROOT/systemd/nats-server.service"
|
||||
rm -f "$tmpconf"
|
||||
sudo pct exec "${VMIDS[$i]}" -- bash -lc "
|
||||
set -e
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
id nats >/dev/null 2>&1 || useradd -r -s /usr/sbin/nologin nats
|
||||
install -d -m 755 -o nats -g nats /var/lib/nats/jetstream /etc/nats
|
||||
mv /tmp/nats.conf /etc/nats/nats.conf
|
||||
chown root:root /etc/nats/nats.conf
|
||||
chmod 644 /etc/nats/nats.conf
|
||||
if [[ ! -x /usr/local/bin/nats-server ]]; then
|
||||
curl -fsSL https://github.com/nats-io/nats-server/releases/download/v${NATS_VER}/nats-server-v${NATS_VER}-linux-amd64.tar.gz -o /tmp/nats.tgz
|
||||
tar -xzf /tmp/nats.tgz -C /tmp
|
||||
install -m 0755 /tmp/nats-server-v${NATS_VER}-linux-amd64/nats-server /usr/local/bin/nats-server
|
||||
rm -rf /tmp/nats.tgz /tmp/nats-server-v${NATS_VER}-linux-amd64
|
||||
fi
|
||||
install -m 644 /tmp/nats-server.service /etc/systemd/system/nats-server.service
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now nats-server
|
||||
"
|
||||
echo "nats-server ${NAMES[$i]} ${IPS[$i]}:4222 cluster ${IPS[$i]}:6222"
|
||||
done
|
||||
|
||||
echo "cluster client URL: nats://${IPS[0]}:4222,nats://${IPS[1]}:4222,nats://${IPS[2]}:4222"
|
||||
echo "lab loopback NATS on the host is unchanged (127.0.0.1:4222)"
|
||||
echo "next: bash $ROOT/scripts/test.sh"
|
||||
10
scripts/cutover-ns1.sh
Executable file
10
scripts/cutover-ns1.sh
Executable file
|
|
@ -0,0 +1,10 @@
|
|||
#!/usr/bin/env bash
|
||||
# Point NS1 test modules at the 3-node vmbr1 cluster. Does not change MOCK_VERAE or Zapier.
|
||||
set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
. "$ROOT/client.env"
|
||||
export PATH="/usr/sbin:/usr/bin:/bin:$PATH"
|
||||
bash "$ROOT/scripts/ensure-streams.sh"
|
||||
echo "NATS_URL=$NATS_URL"
|
||||
echo "streams ensured. restart keep + fleet on the host after copying overlay/service JSON."
|
||||
34
scripts/ensure-streams.sh
Executable file
34
scripts/ensure-streams.sh
Executable file
|
|
@ -0,0 +1,34 @@
|
|||
#!/usr/bin/env bash
|
||||
# Create product JetStream streams with replicas=3 on the Proxmox cluster.
|
||||
set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
. "$ROOT/client.env"
|
||||
export PATH="/usr/sbin:/usr/bin:/bin:/usr/local/bin:$PATH"
|
||||
VMID="${1:-511}"
|
||||
sudo pct exec "$VMID" -- bash -lc "
|
||||
set -e
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
export PATH=/usr/local/bin:/usr/bin:/bin
|
||||
export NATS_URL=nats://10.10.10.21:4222
|
||||
if [[ ! -x /usr/local/bin/nats ]]; then
|
||||
apt-get install -y --no-install-recommends unzip >/dev/null
|
||||
curl -fsSL https://github.com/nats-io/natscli/releases/download/v0.1.6/nats-0.1.6-linux-amd64.zip -o /tmp/natscli.zip
|
||||
rm -rf /tmp/natscli && mkdir -p /tmp/natscli
|
||||
unzip -o /tmp/natscli.zip -d /tmp/natscli >/dev/null
|
||||
BIN=\$(find /tmp/natscli -type f -name nats | head -1)
|
||||
install -m 0755 \"\$BIN\" /usr/local/bin/nats
|
||||
fi
|
||||
add() {
|
||||
local name=\$1 subj=\$2
|
||||
nats stream info \"\$name\" >/dev/null 2>&1 && return 0
|
||||
nats stream add \"\$name\" --subjects=\"\$subj\" --replicas=3 --storage=file --retention=limits --discard=old --max-msgs=-1 --max-bytes=-1 --max-age=24h --dupe-window=2m --defaults
|
||||
}
|
||||
add ZAPIER_JOBS 'verae.zapier.jobs.watch'
|
||||
add ZAPIER_EVENTS 'verae.zapier.jobs.events'
|
||||
add ZAPIER_WEBHOOKS 'verae.zapier.webhooks.deliver'
|
||||
add ZAPIER_USAGE 'verae.zapier.usage'
|
||||
add VERAE_ARCHIVE 'verae.archive.>'
|
||||
nats stream ls
|
||||
"
|
||||
echo "streams ready on cluster (replicas=3)"
|
||||
105
scripts/latency.mjs
Normal file
105
scripts/latency.mjs
Normal file
|
|
@ -0,0 +1,105 @@
|
|||
#!/usr/bin/env node
|
||||
/**
|
||||
* Pub→sub round trip through the cluster (two connections).
|
||||
* Usage: NATS_URL=... node latency.mjs [count] [payloadBytes] [publishers] [ping|flood]
|
||||
* ping = sequential publish-wait (one-message RTT)
|
||||
* flood = publish the batch then drain (queueing under burst)
|
||||
*/
|
||||
import { connect, headers } from "nats";
|
||||
|
||||
const url = process.env.NATS_URL || "nats://10.10.10.21:4222";
|
||||
const count = Number(process.argv[2] || 5000);
|
||||
const size = Number(process.argv[3] || 128);
|
||||
const pubs = Number(process.argv[4] || 1);
|
||||
const mode = process.argv[5] || "flood";
|
||||
const servers = url.split(",").map((s) => s.trim());
|
||||
const subject = `bench.lat.${process.pid}`;
|
||||
const payload = new Uint8Array(size);
|
||||
|
||||
function pct(sorted, p) {
|
||||
if (!sorted.length) return 0;
|
||||
const i = Math.min(sorted.length - 1, Math.floor((p / 100) * sorted.length));
|
||||
return sorted[i];
|
||||
}
|
||||
|
||||
const samples = [];
|
||||
if (mode === "ping") {
|
||||
const subNc = await connect({ servers, name: "lat-sub" });
|
||||
const pubNc = await connect({ servers, name: "lat-pub" });
|
||||
let resolveOne = null;
|
||||
const sub = subNc.subscribe(subject, { max: count });
|
||||
const consume = (async () => {
|
||||
for await (const m of sub) {
|
||||
const sent = Number(m.headers?.get("t") || 0);
|
||||
samples.push(Number(process.hrtime.bigint() / 1000n) - sent);
|
||||
resolveOne?.();
|
||||
}
|
||||
})();
|
||||
await subNc.flush();
|
||||
for (let i = 0; i < count; i++) {
|
||||
const got = new Promise((r) => {
|
||||
resolveOne = r;
|
||||
});
|
||||
const h = headers();
|
||||
h.set("t", String(process.hrtime.bigint() / 1000n));
|
||||
pubNc.publish(subject, payload, { headers: h });
|
||||
await got;
|
||||
}
|
||||
await consume;
|
||||
await pubNc.close();
|
||||
await subNc.close();
|
||||
} else {
|
||||
const subNc = await connect({ servers, name: "lat-sub" });
|
||||
const sub = subNc.subscribe(subject, { max: count });
|
||||
const done = (async () => {
|
||||
for await (const m of sub) {
|
||||
const sent = Number(m.headers?.get("t") || 0);
|
||||
if (sent) samples.push(Number(process.hrtime.bigint() / 1000n) - sent);
|
||||
}
|
||||
})();
|
||||
await subNc.flush();
|
||||
const per = Math.ceil(count / pubs);
|
||||
const publishers = [];
|
||||
for (let p = 0; p < pubs; p++) {
|
||||
publishers.push(
|
||||
(async () => {
|
||||
const nc = await connect({ servers, name: `lat-pub-${p}` });
|
||||
const n = p === pubs - 1 ? count - per * (pubs - 1) : per;
|
||||
for (let i = 0; i < n; i++) {
|
||||
const h = headers();
|
||||
h.set("t", String(process.hrtime.bigint() / 1000n));
|
||||
nc.publish(subject, payload, { headers: h });
|
||||
}
|
||||
await nc.flush();
|
||||
await nc.close();
|
||||
})(),
|
||||
);
|
||||
}
|
||||
await Promise.all(publishers);
|
||||
await done;
|
||||
await subNc.close();
|
||||
}
|
||||
|
||||
samples.sort((a, b) => a - b);
|
||||
const sum = samples.reduce((a, b) => a + b, 0);
|
||||
const us = (n) => `${(n / 1000).toFixed(3)}ms`;
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
count: samples.length,
|
||||
pubs,
|
||||
size,
|
||||
mode,
|
||||
min_us: samples[0],
|
||||
avg_us: Math.round(sum / samples.length),
|
||||
p50_us: pct(samples, 50),
|
||||
p90_us: pct(samples, 90),
|
||||
p99_us: pct(samples, 99),
|
||||
max_us: samples[samples.length - 1],
|
||||
min: us(samples[0]),
|
||||
avg: us(sum / samples.length),
|
||||
p50: us(pct(samples, 50)),
|
||||
p90: us(pct(samples, 90)),
|
||||
p99: us(pct(samples, 99)),
|
||||
max: us(samples[samples.length - 1]),
|
||||
}),
|
||||
);
|
||||
62
scripts/lib-ct.sh
Executable file
62
scripts/lib-ct.sh
Executable file
|
|
@ -0,0 +1,62 @@
|
|||
# shellcheck shell=bash
|
||||
# Shared LXC bootstrap for NS1 Proxmox. Does not generate SSH keys if one exists.
|
||||
export PATH="/usr/sbin:/usr/bin:/bin:$PATH"
|
||||
|
||||
ct_require_proxmox() {
|
||||
if [[ ! -d /etc/pve/nodes ]]; then
|
||||
echo "not a Proxmox host" >&2
|
||||
return 1
|
||||
fi
|
||||
command -v pct >/dev/null || { echo "pct missing" >&2; return 1; }
|
||||
}
|
||||
|
||||
ct_ensure() {
|
||||
local vmid="$1" hostname="$2" ip="$3"
|
||||
if [[ ! -f "$TEMPLATE" ]]; then
|
||||
echo "missing template $TEMPLATE" >&2
|
||||
return 1
|
||||
fi
|
||||
if ! sudo pct status "$vmid" >/dev/null 2>&1; then
|
||||
echo "pct create $vmid $hostname $ip/24"
|
||||
sudo pct create "$vmid" "$TEMPLATE" \
|
||||
--hostname "$hostname" \
|
||||
--memory "$MEMORY" --cores "$CORES" --swap 256 \
|
||||
--net0 "name=eth0,bridge=${BRIDGE},ip=${ip}/24,gw=${GW},type=veth" \
|
||||
--rootfs "${STORAGE}:${DISK}" \
|
||||
--unprivileged 1 --onboot 1 --nameserver "$DNS" \
|
||||
--features nesting=1 \
|
||||
--ostype ubuntu
|
||||
else
|
||||
echo "CT $vmid already exists"
|
||||
fi
|
||||
sudo pct start "$vmid" 2>/dev/null || true
|
||||
local i
|
||||
for i in $(seq 1 40); do
|
||||
sudo pct exec "$vmid" -- true 2>/dev/null && return 0
|
||||
sleep 2
|
||||
done
|
||||
echo "CT $vmid did not start" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
ct_bootstrap_user() {
|
||||
local vmid="$1"
|
||||
local pub=""
|
||||
[[ -f "$HOME/.ssh/id_ed25519.pub" ]] && pub="$(cat "$HOME/.ssh/id_ed25519.pub")"
|
||||
[[ -z "$pub" && -f "$HOME/.ssh/authorized_keys" ]] && pub="$(head -1 "$HOME/.ssh/authorized_keys")"
|
||||
[[ -n "$pub" ]] || { echo "no ssh public key" >&2; return 1; }
|
||||
sudo pct exec "$vmid" -- bash -lc "
|
||||
set -e
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get update -qq
|
||||
apt-get install -y --no-install-recommends openssh-server sudo curl ca-certificates xz-utils tar
|
||||
id $USER_NAME >/dev/null 2>&1 || useradd -m -s /bin/bash $USER_NAME
|
||||
echo '$USER_NAME ALL=(ALL) NOPASSWD:ALL' >/etc/sudoers.d/90-$USER_NAME
|
||||
chmod 440 /etc/sudoers.d/90-$USER_NAME
|
||||
install -d -m 700 -o $USER_NAME -g $USER_NAME /home/$USER_NAME/.ssh
|
||||
grep -qxF '$pub' /home/$USER_NAME/.ssh/authorized_keys 2>/dev/null || echo '$pub' >>/home/$USER_NAME/.ssh/authorized_keys
|
||||
chown $USER_NAME:$USER_NAME /home/$USER_NAME/.ssh/authorized_keys
|
||||
chmod 600 /home/$USER_NAME/.ssh/authorized_keys
|
||||
systemctl enable --now ssh
|
||||
"
|
||||
}
|
||||
22
scripts/status.sh
Executable file
22
scripts/status.sh
Executable file
|
|
@ -0,0 +1,22 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
. "$ROOT/cluster.env"
|
||||
export PATH="/usr/sbin:/usr/bin:/bin:$PATH"
|
||||
printf '%s\n' "$NODES" | awk 'NF==3 {print}' | while read -r vmid name ip; do
|
||||
st="$(sudo pct status "$vmid" 2>/dev/null || echo missing)"
|
||||
js="$(sudo pct exec "$vmid" -- curl -fsS --max-time 2 http://127.0.0.1:8222/varz 2>/dev/null || echo '{}')"
|
||||
echo "$vmid $name $ip $st"
|
||||
python3 -c "
|
||||
import json,sys
|
||||
try:
|
||||
d=json.loads(sys.argv[1])
|
||||
except Exception:
|
||||
print(' nats down')
|
||||
raise SystemExit
|
||||
print(' server_name', d.get('server_name'), 'cluster', (d.get('cluster') or {}).get('name'), 'routes', len((d.get('cluster') or {}).get('urls') or d.get('connect_urls') or []))
|
||||
print(' jetstream', bool(d.get('jetstream')), 'port', d.get('port'), 'host', d.get('host'))
|
||||
" "$js" 2>/dev/null || echo " nats down"
|
||||
done
|
||||
echo "host loopback still: $(ss -lnt | grep '127.0.0.1:4222' && echo up || echo down)"
|
||||
51
scripts/test.sh
Executable file
51
scripts/test.sh
Executable file
|
|
@ -0,0 +1,51 @@
|
|||
#!/usr/bin/env bash
|
||||
# Local syntax check always. Live cluster check when pct is present.
|
||||
set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
bash -n "$ROOT/scripts/lib-ct.sh"
|
||||
bash -n "$ROOT/scripts/create-cluster.sh"
|
||||
bash -n "$ROOT/scripts/status.sh"
|
||||
bash -n "$ROOT/scripts/bench.sh"
|
||||
grep -q 'host: {{IP}}' "$ROOT/conf/nats.conf.tmpl"
|
||||
grep -qv '0.0.0.0' "$ROOT/conf/nats.conf.tmpl"
|
||||
if [[ ! -d /etc/pve/nodes ]]; then
|
||||
echo "OK (syntax; not on Proxmox)"
|
||||
exit 0
|
||||
fi
|
||||
# shellcheck disable=SC1091
|
||||
. "$ROOT/cluster.env"
|
||||
export PATH="/usr/sbin:/usr/bin:/bin:$PATH"
|
||||
mapfile -t rows < <(printf '%s\n' "$NODES" | awk 'NF==3 {print}')
|
||||
ready=0
|
||||
for row in "${rows[@]}"; do
|
||||
# shellcheck disable=SC2086
|
||||
set -- $row
|
||||
vmid=$1 name=$2 ip=$3
|
||||
js="$(sudo pct exec "$vmid" -- curl -fsS --max-time 3 http://127.0.0.1:8222/varz 2>/dev/null || true)"
|
||||
echo "$js" | grep -q '"jetstream"' && ready=$((ready + 1)) || echo "not ready $name"
|
||||
done
|
||||
[[ $ready -eq 3 ]] || { echo "cluster not fully up ($ready/3)" >&2; exit 1; }
|
||||
|
||||
# nats CLI on first node
|
||||
first="$(echo "${rows[0]}" | awk '{print $1}')"
|
||||
sudo pct exec "$first" -- bash -lc '
|
||||
set -e
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
if [[ ! -x /usr/local/bin/nats ]]; then
|
||||
apt-get install -y --no-install-recommends unzip >/dev/null
|
||||
curl -fsSL https://github.com/nats-io/natscli/releases/download/v0.1.6/nats-0.1.6-linux-amd64.zip -o /tmp/natscli.zip
|
||||
rm -rf /tmp/natscli && mkdir -p /tmp/natscli
|
||||
unzip -o /tmp/natscli.zip -d /tmp/natscli >/dev/null
|
||||
BIN=$(find /tmp/natscli /tmp -maxdepth 3 -type f -name nats | head -1)
|
||||
test -n "$BIN"
|
||||
install -m 0755 "$BIN" /usr/local/bin/nats
|
||||
fi
|
||||
IP=$(hostname -I | awk "{print \$1}")
|
||||
export NATS_URL=nats://$IP:4222
|
||||
export PATH=/usr/local/bin:/usr/bin:/bin
|
||||
nats stream rm VERAE_PX_TEST --force >/dev/null 2>&1 || true
|
||||
nats stream add VERAE_PX_TEST --subjects="verae.px.test" --replicas=3 --storage=file --retention=limits --discard=old --max-msgs=-1 --max-bytes=-1 --max-age=1h --dupe-window=2m --defaults
|
||||
nats pub verae.px.test cluster-ok
|
||||
nats stream info VERAE_PX_TEST
|
||||
'
|
||||
echo "OK live cluster (3/3 + replicas=3 stream)"
|
||||
Loading…
Add table
Add a link
Reference in a new issue