Measure maximized NATS cluster: 8 cores, 16 GiB, tmpfs JetStream.
Some checks are pending
offline / test (push) Waiting to run
Some checks are pending
offline / test (push) Waiting to run
Same NS1 ladder as 20260912T051237Z. Product streams restored to ZFS; raised cores, RAM, and max_mem stay on 510–513.
This commit is contained in:
parent
c5d286dc6b
commit
a7a7ec86ce
63 changed files with 4298 additions and 11 deletions
|
|
@ -69,6 +69,17 @@ js_rm
|
|||
run_one js-2p2s-20k-128-r3 bench.js.d --js --purge --pub 2 --sub 2 --msgs 20000 --size 128 --replicas 3 --storage file --maxbytes=512MB --pull --stream=benchstream
|
||||
js_rm
|
||||
|
||||
# Optional native memory-store ladder (same replica count). Used by maximize-ns1-study.sh.
|
||||
if [[ "${JS_EXTRA_MEMORY:-0}" == "1" ]]; then
|
||||
js_rm
|
||||
run_one js-mem-1p-20k-128-r3 bench.js.m1 --js --purge --pub 1 --msgs 20000 --size 128 --replicas 3 --storage memory --maxbytes=512MB --stream=benchstream
|
||||
js_rm
|
||||
run_one js-mem-4p-50k-128-r3 bench.js.m2 --js --purge --pub 4 --msgs 50000 --size 128 --replicas 3 --storage memory --maxbytes=512MB --stream=benchstream
|
||||
js_rm
|
||||
run_one js-mem-4p-20k-1k-r3 bench.js.m3 --js --purge --pub 4 --msgs 20000 --size 1024 --replicas 3 --storage memory --maxbytes=512MB --stream=benchstream
|
||||
js_rm
|
||||
fi
|
||||
|
||||
# Round-trip delay (two connections, through the cluster) at several loads
|
||||
sudo pct exec "$CLIENT_VMID" -- bash -lc "
|
||||
set -e
|
||||
|
|
|
|||
|
|
@ -57,6 +57,9 @@ JS_LABELS = {
|
|||
"js-4p-50k-128-r3": "4p 50k×128 B",
|
||||
"js-4p-20k-1k-r3": "4p 20k×1 KiB",
|
||||
"js-2p2s-20k-128-r3": "2p2s pull 20k×128 B",
|
||||
"js-mem-1p-20k-128-r3": "mem 1p 128 B",
|
||||
"js-mem-4p-50k-128-r3": "mem 4p 128 B",
|
||||
"js-mem-4p-20k-1k-r3": "mem 4p 1 KiB",
|
||||
}
|
||||
LAT_LABELS = {
|
||||
"lat-ping-1k-128": "Ping\n1k×128 B",
|
||||
|
|
@ -286,11 +289,117 @@ def figure(name: str, caption: str) -> str:
|
|||
return f"\n\n*{caption}*"
|
||||
|
||||
|
||||
def write_markdown(folder: Path, thru: list[dict[str, str]], lats: list[dict[str, str]]) -> str:
|
||||
def ratio(new: str | None, old: str | None) -> str:
|
||||
if not new or not old:
|
||||
return "—"
|
||||
a, b = float(new), float(old)
|
||||
if b == 0:
|
||||
return "—"
|
||||
return f"{a / b:.2f}×"
|
||||
|
||||
|
||||
def delay_ms_val(p: dict[str, str] | None, key: str) -> str | None:
|
||||
if not p or not p.get(key):
|
||||
return None
|
||||
return str(ms(p[key]))
|
||||
|
||||
|
||||
def delta_table(
|
||||
thru: list[dict[str, str]],
|
||||
lats: list[dict[str, str]],
|
||||
base_thru: list[dict[str, str]],
|
||||
base_lats: list[dict[str, str]],
|
||||
base_stamp: str,
|
||||
) -> str:
|
||||
bt = {p["run"]: p for p in base_thru}
|
||||
nt = {p["run"]: p for p in thru}
|
||||
bl = {p["run"]: p for p in base_lats}
|
||||
nl = {p["run"]: p for p in lats}
|
||||
keys = [
|
||||
("core-1p1s-50k-128", "pub", "Core 1p1s 128 B pub msgs/s"),
|
||||
("core-8p8s-200k-128", "agg", "Core 8p8s 128 B aggregate msgs/s"),
|
||||
("js-1p-20k-128-r3", "pub", "JS file r=3 1p 128 B pub msgs/s"),
|
||||
("js-4p-50k-128-r3", "pub", "JS file r=3 4p 128 B pub msgs/s"),
|
||||
("js-4p-20k-1k-r3", "pub", "JS file r=3 4p 1 KiB pub msgs/s"),
|
||||
("js-mem-1p-20k-128-r3", "pub", "JS memory r=3 1p 128 B pub msgs/s"),
|
||||
("js-mem-4p-50k-128-r3", "pub", "JS memory r=3 4p 128 B pub msgs/s"),
|
||||
]
|
||||
lines = [
|
||||
f"| Metric | Baseline `{base_stamp}` | This run | Ratio |",
|
||||
"|--------|-------------------------|----------|-------|",
|
||||
]
|
||||
for run, kind, label in keys:
|
||||
old, new = bt.get(run), nt.get(run)
|
||||
ok = "pub_msgs" if kind == "pub" else "agg_msgs"
|
||||
ov = old.get(ok) if old else None
|
||||
nv = new.get(ok) if new else None
|
||||
lines.append(f"| {label} | {fmt_int(ov)} | {fmt_int(nv)} | {ratio(nv, ov)} |")
|
||||
old_p, new_p = bl.get("lat-ping-1k-128"), nl.get("lat-ping-1k-128")
|
||||
if old_p or new_p:
|
||||
ov = delay_ms_val(old_p, "p99")
|
||||
nv = delay_ms_val(new_p, "p99")
|
||||
# smaller delay is better — invert ratio label
|
||||
r = "—"
|
||||
if ov and nv and float(nv) != 0:
|
||||
r = f"{float(ov) / float(nv):.2f}× faster" if float(nv) < float(ov) else f"{float(nv) / float(ov):.2f}× slower"
|
||||
lines.append(
|
||||
f"| Ping p99 (ms) | {old_p.get('p99') if old_p else '—'} | {new_p.get('p99') if new_p else '—'} | {r} |"
|
||||
)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def chart_delta(
|
||||
thru: list[dict[str, str]],
|
||||
base_thru: list[dict[str, str]],
|
||||
dest: Path,
|
||||
) -> None:
|
||||
bt = {p["run"]: p for p in base_thru}
|
||||
nt = {p["run"]: p for p in thru}
|
||||
labels = ["Core 1p\n128 B pub", "JS file 1p\n128 B", "JS file 4p\n128 B", "JS mem 1p\n128 B"]
|
||||
keys = ["core-1p1s-50k-128", "js-1p-20k-128-r3", "js-4p-50k-128-r3", "js-mem-1p-20k-128-r3"]
|
||||
old = [int(bt[k]["pub_msgs"]) if k in bt and bt[k].get("pub_msgs") else 0 for k in keys]
|
||||
new = [int(nt[k]["pub_msgs"]) if k in nt and nt[k].get("pub_msgs") else 0 for k in keys]
|
||||
if not any(new):
|
||||
return
|
||||
fig, ax = plt.subplots(figsize=(9.2, 4.4))
|
||||
x = list(range(len(labels)))
|
||||
w = 0.35
|
||||
ax.bar([i - w / 2 for i in x], old, w, label="Baseline 1c/1G/ZFS", color=MUTED)
|
||||
ax.bar([i + w / 2 for i in x], new, w, label="8c/16G/tmpfs (+ mem rows)", color=INDIGO)
|
||||
ax.set_xticks(x, labels)
|
||||
ax.set_yscale("log")
|
||||
ax.set_ylabel("publish messages / second (log)")
|
||||
ax.set_title("Measured delta vs 20260912T051237Z")
|
||||
ax.legend(loc="upper right")
|
||||
ax.set_axisbelow(True)
|
||||
save(fig, dest / "delta-vs-baseline.png")
|
||||
|
||||
|
||||
def write_markdown(
|
||||
folder: Path,
|
||||
thru: list[dict[str, str]],
|
||||
lats: list[dict[str, str]],
|
||||
compare: Path | None = None,
|
||||
) -> str:
|
||||
before = kv_file(folder / "host-before.txt")
|
||||
after = kv_file(folder / "host-after.txt")
|
||||
stamp = folder.name
|
||||
method = (Path(__file__).resolve().parent / "ns1-study-methodology.md").read_text(encoding="utf-8")
|
||||
delta_md = ""
|
||||
base_thru: list[dict[str, str]] = []
|
||||
base_lats: list[dict[str, str]] = []
|
||||
if compare and compare.is_dir():
|
||||
base_thru, base_lats = load_runs(compare)
|
||||
delta_md = (
|
||||
f"## Measured delta vs `{compare.name}`\n\n"
|
||||
"Baseline: 1 core / 1 GiB / JetStream on ZFS. This run: 8 cores / 16 GiB / "
|
||||
"JetStream **tmpfs** (file r=3) plus extra **memory** store rows. veth/10G unchanged.\n\n"
|
||||
+ delta_table(thru, lats, base_thru, base_lats, compare.name)
|
||||
+ "\n"
|
||||
)
|
||||
if (folder / "charts" / "delta-vs-baseline.png").exists():
|
||||
delta_md += "\n" + figure("delta-vs-baseline.png", "Baseline vs maximized publish rates (log)")
|
||||
delta_md += "\n"
|
||||
figs = []
|
||||
charts_dir = folder / "charts"
|
||||
if (charts_dir / "core-throughput.png").exists():
|
||||
|
|
@ -308,11 +417,11 @@ def write_markdown(folder: Path, thru: list[dict[str, str]], lats: list[dict[str
|
|||
js1 = next((p for p in thru if p["run"] == "js-1p-20k-128-r3"), None)
|
||||
core1 = next((p for p in thru if p["run"] == "core-1p1s-50k-128"), None)
|
||||
|
||||
md = f"""**Progress report (second study)** · run `{stamp}` (UTC)
|
||||
md = f"""**Progress report (maximized NS1 study)** · run `{stamp}` (UTC)
|
||||
|
||||
> **Execution provenance.** Every process for this study ran on **NS1.GEORGELAMBERT.ORG** (`70.88.205.138`): the orchestrator (`study-on-ns1.sh`), `nats bench`, `latency.mjs` (inside LXC 510 on this hypervisor), charting (`matplotlib`), and HTML/PDF (`pandoc` + `weasyprint`). The operator laptop did **not** publish, subscribe, draw charts, or render the PDF. Traffic stayed on `vmbr1` from LXC **510** to `nats-a/b/c` (**511–513**).
|
||||
> **Execution provenance.** Every process for this study ran on **NS1.GEORGELAMBERT.ORG** (`70.88.205.138`): `maximize-ns1-study.sh` (cores/RAM/`max_mem`/tmpfs), then `study-on-ns1.sh`, `nats bench`, `latency.mjs` (LXC 510), matplotlib, pandoc, weasyprint. Traffic stayed on `vmbr1`. veth/10G was **not** changed. After the ladder, JetStream was put back on ZFS and product streams were re-created; **8 cores / 16 GiB / max_mem 8G stay**.
|
||||
|
||||
This is a full methodology write-up plus the numbers from that on-host run. The earlier report (`nats-cluster-bench`, run `20260912T045131Z`) used the same cluster but was **orchestrated and rendered off-box**. Use this document when you need “it was all run on 138.”
|
||||
{delta_md}
|
||||
|
||||
---
|
||||
|
||||
|
|
@ -407,8 +516,8 @@ def render(md_path: Path, html_path: Path, pdf_path: Path) -> None:
|
|||
'<div class="doc-banner">'
|
||||
'<nav class="site"><a href="/">zapier.georgelambert.org</a>'
|
||||
' · <a href="/index-md.html">Markdown indexes</a></nav>'
|
||||
'<div class="kicker">Verae Time × Zapier · progress report · run on NS1.GEORGELAMBERT.ORG</div>'
|
||||
"<h1>NATS cluster message speed — NS1 host study</h1>"
|
||||
'<div class="kicker">Verae Time × Zapier · progress report · maximized NS1 study</div>'
|
||||
"<h1>NATS cluster message speed — maximized (RAM disk + 8 cores)</h1>"
|
||||
'<div class="source-path">packages/zapier-decisions/reports/nats-cluster-bench-ns1.md</div>'
|
||||
"</div>\n",
|
||||
encoding="utf-8",
|
||||
|
|
@ -440,9 +549,13 @@ def render(md_path: Path, html_path: Path, pdf_path: Path) -> None:
|
|||
|
||||
def main() -> int:
|
||||
folder = Path(sys.argv[1] if len(sys.argv) > 1 else ".")
|
||||
compare = Path(sys.argv[2]) if len(sys.argv) > 2 and sys.argv[2] else None
|
||||
thru, lats = load_runs(folder)
|
||||
charts(thru, lats, folder / "charts")
|
||||
md = write_markdown(folder, thru, lats)
|
||||
if compare and compare.is_dir():
|
||||
base_thru, _base_lats = load_runs(compare)
|
||||
chart_delta(thru, base_thru, folder / "charts")
|
||||
md = write_markdown(folder, thru, lats, compare if compare and compare.is_dir() else None)
|
||||
md_path = folder / "nats-cluster-bench-ns1.md"
|
||||
md_path.write_text(md, encoding="utf-8")
|
||||
html_path = folder / "nats-cluster-bench-ns1.html"
|
||||
|
|
|
|||
102
packages/verae-nats-cluster/scripts/maximize-ns1-study.sh
Executable file
102
packages/verae-nats-cluster/scripts/maximize-ns1-study.sh
Executable file
|
|
@ -0,0 +1,102 @@
|
|||
#!/usr/bin/env bash
|
||||
# Maximize nats LXC resources + RAM-disk JetStream, run the NS1 study, then
|
||||
# put product streams back on ZFS. Cores/RAM/max_mem stay raised.
|
||||
# Must run on NS1.GEORGELAMBERT.ORG.
|
||||
set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
export PATH="/usr/sbin:/usr/bin:/bin:/usr/local/bin:$PATH"
|
||||
|
||||
HOST="$(hostname -f 2>/dev/null || hostname)"
|
||||
case "$HOST" in
|
||||
NS1.GEORGELAMBERT.ORG|NS1|ns1.georgelambert.org|ns1) ;;
|
||||
*)
|
||||
echo "refusing: maximize-ns1-study.sh must run on NS1, got '$HOST'" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
CORES="${CORES:-8}"
|
||||
MEMORY="${MEMORY:-16384}"
|
||||
TMPFS_SIZE="${TMPFS_SIZE:-8G}"
|
||||
NATS_VMS=(511 512 513)
|
||||
ALL_VMS=(510 511 512 513)
|
||||
|
||||
apply_resources() {
|
||||
local v
|
||||
for v in "${ALL_VMS[@]}"; do
|
||||
echo "pct set $v --cores $CORES --memory $MEMORY"
|
||||
sudo pct set "$v" --cores "$CORES" --memory "$MEMORY"
|
||||
done
|
||||
}
|
||||
|
||||
patch_max_mem() {
|
||||
local v
|
||||
for v in "${NATS_VMS[@]}"; do
|
||||
sudo pct exec "$v" -- sed -i -E 's/max_mem:[[:space:]]*[0-9]+[MmGg]/max_mem: 8G/' /etc/nats/nats.conf
|
||||
sudo pct exec "$v" -- grep -n max_mem /etc/nats/nats.conf
|
||||
done
|
||||
}
|
||||
|
||||
mount_tmpfs() {
|
||||
local v
|
||||
for v in "${NATS_VMS[@]}"; do
|
||||
sudo pct exec "$v" -- bash -lc "
|
||||
set -e
|
||||
systemctl stop nats-server
|
||||
mkdir -p /var/lib/nats/jetstream
|
||||
if ! mountpoint -q /var/lib/nats/jetstream; then
|
||||
mount -t tmpfs -o size=${TMPFS_SIZE} nats-js /var/lib/nats/jetstream
|
||||
fi
|
||||
chown nats:nats /var/lib/nats/jetstream
|
||||
chmod 755 /var/lib/nats/jetstream
|
||||
systemctl start nats-server
|
||||
mount | grep jetstream
|
||||
"
|
||||
done
|
||||
}
|
||||
|
||||
unmount_tmpfs() {
|
||||
local v
|
||||
for v in "${NATS_VMS[@]}"; do
|
||||
sudo pct exec "$v" -- bash -lc '
|
||||
set -e
|
||||
systemctl stop nats-server || true
|
||||
if mountpoint -q /var/lib/nats/jetstream; then
|
||||
umount /var/lib/nats/jetstream
|
||||
fi
|
||||
mkdir -p /var/lib/nats/jetstream
|
||||
chown nats:nats /var/lib/nats/jetstream
|
||||
systemctl start nats-server
|
||||
'
|
||||
done
|
||||
}
|
||||
|
||||
wait_cluster() {
|
||||
local n=0
|
||||
until sudo pct exec 511 -- curl -fsS --max-time 2 http://127.0.0.1:8222/varz >/dev/null 2>&1; do
|
||||
n=$((n + 1))
|
||||
[[ $n -lt 30 ]] || { echo "nats-a varz not up" >&2; return 1; }
|
||||
sleep 1
|
||||
done
|
||||
sleep 2
|
||||
}
|
||||
|
||||
restore_durable() {
|
||||
echo "restoring ZFS JetStream (product streams)"
|
||||
unmount_tmpfs
|
||||
wait_cluster
|
||||
bash "$ROOT/scripts/ensure-streams.sh" || true
|
||||
}
|
||||
|
||||
apply_resources
|
||||
patch_max_mem
|
||||
mount_tmpfs
|
||||
wait_cluster
|
||||
trap restore_durable EXIT
|
||||
|
||||
export JS_EXTRA_MEMORY=1
|
||||
export COMPARE_DIR="${COMPARE_DIR:-$ROOT/results/20260912T051237Z}"
|
||||
# recorded in host-before by appending after dump starts — study script reads pct config live
|
||||
bash "$ROOT/scripts/study-on-ns1.sh"
|
||||
|
||||
echo "maximize study finished; trap will restore ZFS jetstream"
|
||||
|
|
@ -37,6 +37,12 @@ dump_env() {
|
|||
echo "loadavg=$(cat /proc/loadavg)"
|
||||
echo "client_vmid=$CLIENT_VMID"
|
||||
echo "nats_url=$NATS_URL"
|
||||
echo "js_extra_memory=${JS_EXTRA_MEMORY:-0}"
|
||||
echo "compare_dir=${COMPARE_DIR:-}"
|
||||
echo "--- nats 511 max_mem ---"
|
||||
sudo pct exec 511 -- grep max_mem /etc/nats/nats.conf || true
|
||||
echo "--- nats 511 jetstream mount ---"
|
||||
sudo pct exec 511 -- mount | grep jetstream || echo "jetstream on rootfs"
|
||||
echo "--- free ---"
|
||||
free -h
|
||||
echo "--- pct list ---"
|
||||
|
|
@ -86,6 +92,6 @@ echo "NS1 study $STAMP out=$OUT"
|
|||
dump_env before
|
||||
BENCH_OUT="$OUT" CLIENT_VMID="$CLIENT_VMID" bash "$ROOT/scripts/bench.sh"
|
||||
dump_env after
|
||||
python3 "$ROOT/scripts/build-ns1-study-report.py" "$OUT"
|
||||
python3 "$ROOT/scripts/build-ns1-study-report.py" "$OUT" "${COMPARE_DIR:-}"
|
||||
echo "NS1 study complete $OUT"
|
||||
ls -la "$OUT"/nats-cluster-bench-ns1.* "$OUT"/charts 2>/dev/null || ls -la "$OUT"
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ bash -n "$ROOT/scripts/create-cluster.sh"
|
|||
bash -n "$ROOT/scripts/status.sh"
|
||||
bash -n "$ROOT/scripts/bench.sh"
|
||||
bash -n "$ROOT/scripts/study-on-ns1.sh"
|
||||
bash -n "$ROOT/scripts/maximize-ns1-study.sh"
|
||||
grep -q 'host: {{IP}}' "$ROOT/conf/nats.conf.tmpl"
|
||||
grep -qv '0.0.0.0' "$ROOT/conf/nats.conf.tmpl"
|
||||
if [[ ! -d /etc/pve/nodes ]]; then
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue