diff --git a/packages/docs-master/README.md b/packages/docs-master/README.md index 83b52cc..5cdce20 100644 --- a/packages/docs-master/README.md +++ b/packages/docs-master/README.md @@ -32,6 +32,10 @@ Summaries, NATS contracts, and message flows for every Verae Time × Zapier modu | verae-staff-iam | https://git.georgelambert.org/marchon/verae-staff-iam | | verae-keep | https://git.georgelambert.org/marchon/verae-keep | | verae-bootstrap | https://git.georgelambert.org/marchon/verae-bootstrap | +| verae-uptime | https://git.georgelambert.org/marchon/verae-uptime | +| verae-backup | https://git.georgelambert.org/marchon/verae-backup | +| verae-deploy | https://git.georgelambert.org/marchon/verae-deploy | +| verae-proxmox | https://git.georgelambert.org/marchon/verae-proxmox | Clone (SSH port 2223): diff --git a/packages/docs-master/modules/verae-backup/NATS.md b/packages/docs-master/modules/verae-backup/NATS.md new file mode 100644 index 0000000..fe8b997 --- /dev/null +++ b/packages/docs-master/modules/verae-backup/NATS.md @@ -0,0 +1,3 @@ +# NATS — verae-backup + +Copies JetStream store directory if present. No client connection. diff --git a/packages/docs-master/modules/verae-backup/SUMMARY.md b/packages/docs-master/modules/verae-backup/SUMMARY.md new file mode 100644 index 0000000..9d4acf9 --- /dev/null +++ b/packages/docs-master/modules/verae-backup/SUMMARY.md @@ -0,0 +1,3 @@ +# verae-backup + +Nightly tar of SQLite, IAM, keep, worm/tree runtime, optional JetStream dir. Restore is dry-run unless `--apply`. diff --git a/packages/docs-master/modules/verae-deploy/NATS.md b/packages/docs-master/modules/verae-deploy/NATS.md new file mode 100644 index 0000000..880b4e3 --- /dev/null +++ b/packages/docs-master/modules/verae-deploy/NATS.md @@ -0,0 +1,3 @@ +# NATS — verae-deploy + +None during deploy. Archive hosts still need nats-server from host-deps. diff --git a/packages/docs-master/modules/verae-deploy/SUMMARY.md b/packages/docs-master/modules/verae-deploy/SUMMARY.md new file mode 100644 index 0000000..4a6b816 --- /dev/null +++ b/packages/docs-master/modules/verae-deploy/SUMMARY.md @@ -0,0 +1,3 @@ +# verae-deploy + +`host-deps` + git checkout tag/branch + npm ci/rebuild on the target OS. Restart keep/fleet after. diff --git a/packages/docs-master/modules/verae-proxmox/NATS.md b/packages/docs-master/modules/verae-proxmox/NATS.md new file mode 100644 index 0000000..8ad9111 --- /dev/null +++ b/packages/docs-master/modules/verae-proxmox/NATS.md @@ -0,0 +1,3 @@ +# NATS — verae-proxmox + +Guest clients use `nats://10.10.10.1:4222` (socat to host loopback). Never bind NATS on vmbr0. diff --git a/packages/docs-master/modules/verae-proxmox/SUMMARY.md b/packages/docs-master/modules/verae-proxmox/SUMMARY.md new file mode 100644 index 0000000..f8a065a --- /dev/null +++ b/packages/docs-master/modules/verae-proxmox/SUMMARY.md @@ -0,0 +1,3 @@ +# verae-proxmox + +LXC 510 `verae-px-worker` on NS1 `vmbr1` `10.10.10.20`. Extra worm/tree copies. Private NATS proxy `10.10.10.1:4222`. diff --git a/packages/docs-master/modules/verae-uptime/NATS.md b/packages/docs-master/modules/verae-uptime/NATS.md new file mode 100644 index 0000000..282b692 --- /dev/null +++ b/packages/docs-master/modules/verae-uptime/NATS.md @@ -0,0 +1,3 @@ +# NATS — verae-uptime + +None. HTTPS only. diff --git a/packages/docs-master/modules/verae-uptime/SUMMARY.md b/packages/docs-master/modules/verae-uptime/SUMMARY.md new file mode 100644 index 0000000..6d7a14e --- /dev/null +++ b/packages/docs-master/modules/verae-uptime/SUMMARY.md @@ -0,0 +1,3 @@ +# verae-uptime + +Off-box HTTPS GET of public doors. `--once` for cron (exit 2 on failure); loop serves `/status` on `:3870`. diff --git a/packages/overview/02-modules-and-repos.md b/packages/overview/02-modules-and-repos.md index 2bd1013..7625ade 100644 --- a/packages/overview/02-modules-and-repos.md +++ b/packages/overview/02-modules-and-repos.md @@ -40,6 +40,10 @@ Each runtime piece is its **own git repo** on Forgejo (`git.georgelambert.org`, | **verae-bootstrap** | `packages/verae-bootstrap` | Clone every repo; `host-deps.sh` installs OS/Node/NATS per module | | **zapier-docs-master** | `packages/docs-master` | Per-module `SUMMARY.md` + `NATS.md` | | **verae-ops** | `packages/verae-ops` | Docker, Proxmox, VMs, dedicated hardware, linking services | +| **verae-uptime** | `packages/verae-uptime` | Off-box HTTPS watch of public doors | +| **verae-backup** | `packages/verae-backup` | SQLite / JetStream / worm+tree snapshots | +| **verae-deploy** | `packages/verae-deploy` | Tagged checkout + npm ci on the target OS | +| **verae-proxmox** | `packages/verae-proxmox` | LXC 510 worker on NS1 `vmbr1` | Libraries that are **not** separate Forgejo apps today: `verae-chain-client` (inside middleware), job-poller and webhook-deliver (middleware workers, fleet-spawned). diff --git a/packages/overview/04-uptime.md b/packages/overview/04-uptime.md index 1b3ea27..433d959 100644 --- a/packages/overview/04-uptime.md +++ b/packages/overview/04-uptime.md @@ -23,7 +23,11 @@ If a tree node is paused, crashes, or fails `/health`, fleet **starts another co ## Spread across machines -`machines.json` lists hosts (`local`, `ns1` = `marchon@70.88.205.138` with `~/.ssh/id_ed25519`, optional `lan-134`). New replicas go to the **least-loaded** eligible host. Remote spawn/health/kill is SSH; workers bind loopback on the remote box. +`machines.json` lists hosts (`local`, `ns1` = `marchon@70.88.205.138` with `~/.ssh/id_ed25519`, optional `lan-134`, **`px-worker`** = LXC 510 `10.10.10.20` on `vmbr1`). New replicas go to the **least-loaded** eligible host. Remote spawn/health/kill is SSH; workers bind loopback on the remote box. + +NS1 **is** the Proxmox host. Extra worm/tree copies on `px-worker` mean a host-process crash does not take every bloom/tree replica. A second **chassis** is still the next step for disk/PSU failure. + +Off-box probe: [verae-uptime](https://git.georgelambert.org/marchon/verae-uptime) (`node src/watch.js --once`). Backup: [verae-backup](https://git.georgelambert.org/marchon/verae-backup). Tagged upgrade: [verae-deploy](https://git.georgelambert.org/marchon/verae-deploy). ## JetStream diff --git a/packages/overview/10-docs-sync.md b/packages/overview/10-docs-sync.md index 5b3a195..caabefe 100644 --- a/packages/overview/10-docs-sync.md +++ b/packages/overview/10-docs-sync.md @@ -21,6 +21,7 @@ Validated against the module list in `packages/` and Forgejo repos. - `verae-staff-iam`, `verae-staff-session`, `verae-staff-ui` - `verae-keep` - `verae-bootstrap` (clone + npm per server type; `scripts/host-deps.sh` installs OS/Node/NATS) +- `verae-uptime`, `verae-backup`, `verae-deploy`, `verae-proxmox` (LXC 510 on NS1 `vmbr1`) ## Move the catalog to another hostname diff --git a/packages/overview/11-hostnames.md b/packages/overview/11-hostnames.md index 1824254..479dd39 100644 --- a/packages/overview/11-hostnames.md +++ b/packages/overview/11-hostnames.md @@ -9,6 +9,7 @@ All of the running Verae Time × Zapier processes on **2026-09-11** live on one | `local` (NS1 all-in-one) | **NS1.GEORGELAMBERT.ORG** | `70.88.205.138` | Control plane + archive workers + keep | | `ns1` (SSH worker profile) | **NS1.GEORGELAMBERT.ORG** | `70.88.205.138` | Same box; SSH spawn unused while all-in-one is local | | `lan-134` | *(none yet)* | `70.88.205.134` | Disabled until SSH works | +| `px-worker` | **verae-px-worker** (LXC **510**) | `10.10.10.20` on `vmbr1` | Extra worm/tree; enable after `verae-proxmox` create-worker | Loopback on that box is `127.0.0.1`. Do **not** publish NATS `:4222`. diff --git a/packages/verae-backup/NATS.md b/packages/verae-backup/NATS.md new file mode 100644 index 0000000..8a3ac7c --- /dev/null +++ b/packages/verae-backup/NATS.md @@ -0,0 +1,3 @@ +# NATS — verae-backup + +Backup copies the JetStream **store directory** if present. It does not publish or subscribe. Restore JetStream only with nats-server stopped. diff --git a/packages/verae-backup/README.md b/packages/verae-backup/README.md new file mode 100644 index 0000000..e123cb9 --- /dev/null +++ b/packages/verae-backup/README.md @@ -0,0 +1,23 @@ +# verae-backup + +Snapshot and restore **zappier SQLite**, staff IAM data, keep state, fleet-runtime (worm/tree blobs), and JetStream store if present. + +**Forgejo:** https://git.georgelambert.org/marchon/verae-backup +**SSH:** `ssh://git@git.georgelambert.org:2223/marchon/verae-backup.git` + +```bash +# on NS1 +export BACKUP_ROOT=/SSD2/backups/verae +bash scripts/backup.sh +bash scripts/restore.sh --dry-run +# after stopping keep/fleet consumers: +# bash scripts/restore.sh --apply --archive /SSD2/backups/verae/20260101T031700Z.tgz +``` + +Paths: `paths.env`. Missing sources are skipped. Default retain 14 tarballs. + +User timer: `systemd/verae-backup.timer` (03:17 UTC). Enable with linger. + +Does **not** back up private keys. Restore identityFile **paths** on the operator host by hand. + +Does **not** connect to NATS; it copies the JetStream directory if `NATS_STORE` exists. diff --git a/packages/verae-backup/SUMMARY.md b/packages/verae-backup/SUMMARY.md new file mode 100644 index 0000000..9dd1352 --- /dev/null +++ b/packages/verae-backup/SUMMARY.md @@ -0,0 +1,3 @@ +# verae-backup + +Tar snapshots of SQLite, IAM, keep, worm/tree runtime, optional JetStream dir. Dry-run restore by default. diff --git a/packages/verae-backup/package.json b/packages/verae-backup/package.json new file mode 100644 index 0000000..26efc6b --- /dev/null +++ b/packages/verae-backup/package.json @@ -0,0 +1,11 @@ +{ + "name": "verae-backup", + "version": "0.1.0", + "private": true, + "description": "Backup and restore SQLite, JetStream, worm/tree, keep, and IAM data", + "scripts": { + "backup": "bash scripts/backup.sh", + "restore": "bash scripts/restore.sh", + "test": "bash scripts/test.sh" + } +} diff --git a/packages/verae-backup/paths.env b/packages/verae-backup/paths.env new file mode 100644 index 0000000..00ca39a --- /dev/null +++ b/packages/verae-backup/paths.env @@ -0,0 +1,16 @@ +# Sourced by backup.sh / restore.sh. Override in the environment. +BACKUP_ROOT="${BACKUP_ROOT:-/SSD2/backups/verae}" +KEEP=14 + +# SQLite / IAM +ZAPPIER_DB="${ZAPPIER_DB:-$HOME/verae-stack/packages/zappier/zappier.db}" +IAM_DATA="${IAM_DATA:-$HOME/verae-stack/packages/verae-staff-iam/data}" + +# Keep + fleet runtime (worm/tree blobs live under fleet-runtime data/) +KEEP_DATA="${KEEP_DATA:-$HOME/verae-keep/data}" +FLEET_RUNTIME="${FLEET_RUNTIME:-$HOME/verae-fleet-runtime}" +FLEET_PKG_DATA="${FLEET_PKG_DATA:-$HOME/verae-stack/packages/verae-fleet/data}" + +# JetStream (docker or nats-server). Missing paths are skipped. +NATS_STORE="${NATS_STORE:-/var/lib/nats/jetstream}" +NATS_DOCKER_VOL="${NATS_DOCKER_VOL:-}" diff --git a/packages/verae-backup/scripts/backup.sh b/packages/verae-backup/scripts/backup.sh new file mode 100755 index 0000000..41128f7 --- /dev/null +++ b/packages/verae-backup/scripts/backup.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Snapshot Verae data dirs into $BACKUP_ROOT/.tgz +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +# shellcheck disable=SC1091 +. "$ROOT/paths.env" +STAMP="$(date -u +%Y%m%dT%H%M%SZ)" +DEST="$BACKUP_ROOT/$STAMP" +mkdir -p "$DEST" +copy_if() { + local src="$1" name="$2" + if [[ -e "$src" ]]; then + mkdir -p "$DEST/$name" + cp -a "$src" "$DEST/$name/" 2>/dev/null || sudo cp -a "$src" "$DEST/$name/" + echo "copied $src" + else + echo "skip missing $src" + fi +} +copy_if "$ZAPPIER_DB" sqlite +copy_if "$IAM_DATA" iam +copy_if "$KEEP_DATA" keep +copy_if "$FLEET_RUNTIME" fleet-runtime +copy_if "$FLEET_PKG_DATA" fleet-pkg +copy_if "$NATS_STORE" nats +if [[ -n "${NATS_DOCKER_VOL:-}" ]]; then copy_if "$NATS_DOCKER_VOL" nats-docker; fi +tar -C "$BACKUP_ROOT" -czf "$BACKUP_ROOT/$STAMP.tgz" "$STAMP" +rm -rf "$DEST" +# retain +ls -1dt "$BACKUP_ROOT"/*.tgz 2>/dev/null | tail -n +$((KEEP + 1)) | while read -r old; do + rm -f "$old" +done +echo "backup $BACKUP_ROOT/$STAMP.tgz" +echo "$BACKUP_ROOT/$STAMP.tgz" diff --git a/packages/verae-backup/scripts/restore.sh b/packages/verae-backup/scripts/restore.sh new file mode 100755 index 0000000..5855eee --- /dev/null +++ b/packages/verae-backup/scripts/restore.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Restore a .tgz from backup.sh. Default is --dry-run. +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +# shellcheck disable=SC1091 +. "$ROOT/paths.env" +DRY=1 +ARCHIVE="" +while [[ $# -gt 0 ]]; do + case "$1" in + --apply) DRY=0 ;; + --dry-run) DRY=1 ;; + --archive) ARCHIVE="${2:-}"; shift ;; + *) echo "usage: restore.sh [--dry-run|--apply] [--archive file.tgz]" >&2; exit 1 ;; + esac + shift +done +if [[ -z "$ARCHIVE" ]]; then + ARCHIVE="$(ls -1t "$BACKUP_ROOT"/*.tgz 2>/dev/null | head -1 || true)" +fi +[[ -n "$ARCHIVE" && -f "$ARCHIVE" ]] || { echo "no archive" >&2; exit 1; } +echo "archive $ARCHIVE dry=$DRY" +TMP="$(mktemp -d)" +tar -xzf "$ARCHIVE" -C "$TMP" +INNER="$(find "$TMP" -mindepth 1 -maxdepth 1 -type d | head -1)" +restore_one() { + local from="$1" to="$2" + [[ -e "$from" ]] || { echo "skip $from"; return; } + echo "would restore $from -> $to" + if [[ $DRY -eq 0 ]]; then + mkdir -p "$(dirname "$to")" + rm -rf "$to" + cp -a "$from" "$to" + fi +} +restore_one "$INNER/sqlite/$(basename "$ZAPPIER_DB")" "$ZAPPIER_DB" +restore_one "$INNER/iam/$(basename "$IAM_DATA")" "$IAM_DATA" +restore_one "$INNER/keep/$(basename "$KEEP_DATA")" "$KEEP_DATA" +restore_one "$INNER/fleet-runtime/$(basename "$FLEET_RUNTIME")" "$FLEET_RUNTIME" +restore_one "$INNER/nats/$(basename "$NATS_STORE")" "$NATS_STORE" +rm -rf "$TMP" +if [[ $DRY -eq 1 ]]; then + echo "dry-run only. re-run with --apply after stopping keep/fleet/nats consumers." +fi diff --git a/packages/verae-backup/scripts/test.sh b/packages/verae-backup/scripts/test.sh new file mode 100755 index 0000000..9987d93 --- /dev/null +++ b/packages/verae-backup/scripts/test.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +bash -n "$ROOT/scripts/backup.sh" +bash -n "$ROOT/scripts/restore.sh" +TMP="$(mktemp -d)" +export BACKUP_ROOT="$TMP/out" +export ZAPPIER_DB="$TMP/zappier.db" +export IAM_DATA="$TMP/iam" +export KEEP_DATA="$TMP/keep" +export FLEET_RUNTIME="$TMP/runtime" +export FLEET_PKG_DATA="$TMP/fleet-pkg" +export NATS_STORE="$TMP/missing-nats" +echo hello >"$ZAPPIER_DB" +mkdir -p "$IAM_DATA" "$KEEP_DATA" "$FLEET_RUNTIME" "$FLEET_PKG_DATA" +echo x >"$KEEP_DATA/a" +# shellcheck disable=SC1091 +set +u +BACKUP_ROOT="$BACKUP_ROOT" KEEP=3 ZAPPIER_DB="$ZAPPIER_DB" IAM_DATA="$IAM_DATA" KEEP_DATA="$KEEP_DATA" FLEET_RUNTIME="$FLEET_RUNTIME" FLEET_PKG_DATA="$FLEET_PKG_DATA" NATS_STORE="$NATS_STORE" \ + bash "$ROOT/scripts/backup.sh" >/tmp/verae-backup-test.out +test -s /tmp/verae-backup-test.out +ls "$BACKUP_ROOT"/*.tgz >/dev/null +BACKUP_ROOT="$BACKUP_ROOT" ZAPPIER_DB="$ZAPPIER_DB" IAM_DATA="$IAM_DATA" KEEP_DATA="$KEEP_DATA" FLEET_RUNTIME="$FLEET_RUNTIME" FLEET_PKG_DATA="$FLEET_PKG_DATA" NATS_STORE="$NATS_STORE" \ + bash "$ROOT/scripts/restore.sh" --dry-run >/dev/null +rm -rf "$TMP" +echo OK diff --git a/packages/verae-backup/systemd/verae-backup.service b/packages/verae-backup/systemd/verae-backup.service new file mode 100644 index 0000000..5e83a1f --- /dev/null +++ b/packages/verae-backup/systemd/verae-backup.service @@ -0,0 +1,7 @@ +[Unit] +Description=Verae data backup (sqlite, keep, worm/tree, jetstream) + +[Service] +Type=oneshot +WorkingDirectory=%h/verae-src/verae-backup +ExecStart=/usr/bin/bash %h/verae-src/verae-backup/scripts/backup.sh diff --git a/packages/verae-backup/systemd/verae-backup.timer b/packages/verae-backup/systemd/verae-backup.timer new file mode 100644 index 0000000..6e225e0 --- /dev/null +++ b/packages/verae-backup/systemd/verae-backup.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Nightly Verae data backup + +[Timer] +OnCalendar=*-*-* 03:17:00 +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/packages/verae-bootstrap/README.md b/packages/verae-bootstrap/README.md index 335e791..557332a 100644 --- a/packages/verae-bootstrap/README.md +++ b/packages/verae-bootstrap/README.md @@ -44,6 +44,7 @@ bash scripts/fetch.sh ns1-all-in-one # one server type | **control-plane** | `types/control-plane/` | Fleet, edges, IAM, billing HTTP (no worm/tree floor) | | **ns1-archive** | `types/ns1-archive/` | NATS + worm ×3 + tree-node ×3 + poller + webhook + aggregator + keep | | **lan-worker** | `types/lan-worker/` | Extra SSH worker (tree-node, archive-worm). Keep disabled until SSH works (`lan-134`). | +| **proxmox-worker** | `types/proxmox-worker/` | LXC 510 on NS1 `vmbr1` (`10.10.10.20`) — extra worm/tree copies | Each type has `repos.txt` (subset of the master list) and `README.md`. diff --git a/packages/verae-bootstrap/host-deps.tsv b/packages/verae-bootstrap/host-deps.tsv index 13d2512..66f7390 100644 --- a/packages/verae-bootstrap/host-deps.tsv +++ b/packages/verae-bootstrap/host-deps.tsv @@ -36,5 +36,9 @@ verae-staff-ui|20|npm verae-staff-iam|20|npm verae-keep|20|npm verae-bootstrap|0| +verae-uptime|20|npm +verae-backup|0| +verae-deploy|20|npm +verae-proxmox|0|ssh zapier-docs-master|0|docs master-zapier-plan-draft|22|npm,native,nats,docs diff --git a/packages/verae-bootstrap/repos.txt b/packages/verae-bootstrap/repos.txt index 3a6b836..c4ba5d5 100644 --- a/packages/verae-bootstrap/repos.txt +++ b/packages/verae-bootstrap/repos.txt @@ -33,3 +33,7 @@ verae-staff-ui verae-staff-iam verae-keep verae-bootstrap +verae-uptime +verae-backup +verae-deploy +verae-proxmox diff --git a/packages/verae-bootstrap/types/ns1-all-in-one/start.sh b/packages/verae-bootstrap/types/ns1-all-in-one/start.sh index 75d7e8b..8c2fe0d 100755 --- a/packages/verae-bootstrap/types/ns1-all-in-one/start.sh +++ b/packages/verae-bootstrap/types/ns1-all-in-one/start.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash # Start HTTP control plane on NS1. Archive workers stay with verae-keep. +# Does not rewrite fleet.json / machines.json — use overlays/ns1/. set -euo pipefail STACK="${VERAE_STACK:-$HOME/verae-stack}" FLEET="$STACK/packages/verae-fleet" @@ -8,75 +9,29 @@ export STAFF_IAM_URL="${STAFF_IAM_URL:-https://iam.zapier.georgelambert.org}" export STAFF_COOKIE_DOMAIN="${STAFF_COOKIE_DOMAIN:-.zapier.georgelambert.org}" export NATS_URL="${NATS_URL:-nats://127.0.0.1:4222}" export FLEET_ENABLE_LAN134="${FLEET_ENABLE_LAN134:-}" +export FLEET_FILE="${FLEET_FILE:-$FLEET/overlays/ns1/fleet.json}" +export FLEET_MACHINES="${FLEET_MACHINES:-$FLEET/overlays/ns1/machines.json}" test -d "$FLEET" || { echo "missing $FLEET" >&2; exit 1; } - -python3 - "$FLEET" "$EDGE_PORT" <<'PY' -import json, sys -from pathlib import Path -root = Path(sys.argv[1]) -edge_port = sys.argv[2] -fleet_path = root / "fleet.json" -fleet = json.loads(fleet_path.read_text()) -for sid in ("tree-node", "archive-worm", "archive-aggregator", "job-poller", "webhook-deliver", "nats"): - if sid in fleet.get("services", {}): - fleet["services"][sid]["enabled"] = False - fleet["services"][sid]["keepFloor"] = False -fleet_path.write_text(json.dumps(fleet, indent=2) + "\n") -machines_path = root / "machines.json" -machines = json.loads(machines_path.read_text()) -for m in machines.get("machines", []): - if m.get("id") == "ns1": - m["enabled"] = False - if m.get("id") == "lan-134": - m["enabled"] = False - if m.get("id") == "local": - m["enabled"] = True - m["roles"] = ["*"] - m["capacity"] = 32 -machines_path.write_text(json.dumps(machines, indent=2) + "\n") -edge = json.loads((root / "services" / "zappier-edge.json").read_text()) -edge.setdefault("env", {})["PORT"] = edge_port -edge["env"]["BIND"] = "127.0.0.1" -edge["ports"] = {"healthBase": int(edge_port)} -(root / "services" / "zappier-edge.json").write_text(json.dumps(edge, indent=2) + "\n") -admin = f"http://127.0.0.1:{edge_port}" -for name in ( - "customer-service", - "sales-pricing", - "accounting-export", - "access-staff", - "access-web", - "staff-iam", - "staff-session", - "zappier-edge", -): - p = root / "services" / f"{name}.json" - if not p.exists(): - continue - spec = json.loads(p.read_text()) - env = spec.setdefault("env", {}) - if "ZAPPIER_ADMIN_URL" in env: - env["ZAPPIER_ADMIN_URL"] = admin - if name == "access-web": - env["ZAPPIER_EDGE_URL"] = admin - env["STAFF_IAM_URL"] = "https://iam.zapier.georgelambert.org" - env["STAFF_COOKIE_DOMAIN"] = ".zapier.georgelambert.org" - env["STAFF_AUTH"] = "1" - p.write_text(json.dumps(spec, indent=2) + "\n") -print("overlay applied edge", edge_port) -PY +test -f "$FLEET_FILE" || { echo "missing $FLEET_FILE" >&2; exit 1; } cd "$FLEET" if curl -sf http://127.0.0.1:3850/health >/dev/null; then echo "fleet already up" exit 0 fi -nohup env STAFF_IAM_URL="$STAFF_IAM_URL" STAFF_COOKIE_DOMAIN="$STAFF_COOKIE_DOMAIN" NATS_URL="$NATS_URL" node src/cli.js serve \ +nohup env \ + STAFF_IAM_URL="$STAFF_IAM_URL" \ + STAFF_COOKIE_DOMAIN="$STAFF_COOKIE_DOMAIN" \ + NATS_URL="$NATS_URL" \ + FLEET_FILE="$FLEET_FILE" \ + FLEET_MACHINES="$FLEET_MACHINES" \ + FLEET_ENABLE_LAN134="$FLEET_ENABLE_LAN134" \ + node src/cli.js serve \ >/tmp/verae-fleet-serve.out 2>&1 & echo "fleet pid $!" for i in $(seq 1 40); do if curl -sf http://127.0.0.1:3850/health >/dev/null; then - echo "fleet up" + echo "fleet up (overlay $FLEET_FILE)" exit 0 fi sleep 0.5 diff --git a/packages/verae-bootstrap/types/proxmox-worker/README.md b/packages/verae-bootstrap/types/proxmox-worker/README.md new file mode 100755 index 0000000..89fac00 --- /dev/null +++ b/packages/verae-bootstrap/types/proxmox-worker/README.md @@ -0,0 +1,17 @@ +# Server type: proxmox-worker + +Ubuntu LXC on **NS1 Proxmox** (`vmbr1` `10.10.10.20`, VMID 510). Extra `archive-worm` + `tree-node` copies so replica floors are not all on the host PID namespace. + +Provision from the Proxmox host: [verae-proxmox](https://git.georgelambert.org/marchon/verae-proxmox) `scripts/create-worker.sh`. + +```bash +export VERAE_SRC=$HOME/verae-src +export NATS_URL=nats://10.10.10.1:4222 +bash scripts/host-deps.sh --type proxmox-worker +bash scripts/fetch.sh proxmox-worker +# keep extra copies: +KEEP_UNITS=$HOME/verae-src/verae-keep/units.px-worker.json \ + bash $HOME/verae-src/verae-keep/scripts/guard.sh +``` + +Fleet machine id `px-worker`. Enable after `ssh-check`. diff --git a/packages/verae-bootstrap/types/proxmox-worker/repos.txt b/packages/verae-bootstrap/types/proxmox-worker/repos.txt new file mode 100644 index 0000000..12230e4 --- /dev/null +++ b/packages/verae-bootstrap/types/proxmox-worker/repos.txt @@ -0,0 +1,6 @@ +verae-archive-worm +verae-tree-node +verae-fleet +verae-keep +verae-bootstrap +verae-proxmox diff --git a/packages/verae-deploy/NATS.md b/packages/verae-deploy/NATS.md new file mode 100644 index 0000000..243c3eb --- /dev/null +++ b/packages/verae-deploy/NATS.md @@ -0,0 +1,3 @@ +# NATS — verae-deploy + +Deploy does not start nats-server or open client connections. Archive types still need `nats-server` from `verae-bootstrap/scripts/host-deps.sh --nats`. diff --git a/packages/verae-deploy/README.md b/packages/verae-deploy/README.md new file mode 100644 index 0000000..5667263 --- /dev/null +++ b/packages/verae-deploy/README.md @@ -0,0 +1,20 @@ +# verae-deploy + +Upgrade cloned Forgejo modules **on the machine that runs them**. Order: `host-deps.sh` → `git fetch` → checkout tag/branch → `npm ci` (or `npm install`) → `npm rebuild`. + +**Forgejo:** https://git.georgelambert.org/marchon/verae-deploy +**SSH:** `ssh://git@git.georgelambert.org:2223/marchon/verae-deploy.git` + +```bash +export VERAE_SRC=$HOME/verae-src +export VERAE_BOOTSTRAP=$HOME/verae-src/verae-bootstrap # or sibling clone +export DEPLOY_TYPE=ns1-all-in-one +bash scripts/deploy.sh main +# then: systemctl --user restart verae-keep-guard verae-fleet +``` + +Writes `$VERAE_SRC//.deployed-ref` and `$VERAE_SRC/.deployed`. Does **not** force-push. Prefer tags (`v2026-09-12`) once you cut them. + +Do **not** copy `node_modules` from macOS. Rebuild `better-sqlite3` on Linux. + +Does not speak NATS. diff --git a/packages/verae-deploy/SUMMARY.md b/packages/verae-deploy/SUMMARY.md new file mode 100644 index 0000000..ac69361 --- /dev/null +++ b/packages/verae-deploy/SUMMARY.md @@ -0,0 +1,3 @@ +# verae-deploy + +Tagged/branch checkout of every cloned module plus npm ci/rebuild on the target OS. diff --git a/packages/verae-deploy/package.json b/packages/verae-deploy/package.json new file mode 100644 index 0000000..2d7d8e8 --- /dev/null +++ b/packages/verae-deploy/package.json @@ -0,0 +1,10 @@ +{ + "name": "verae-deploy", + "version": "0.1.0", + "private": true, + "description": "Tagged checkout + npm ci + rebuild on the target OS for Verae module repos", + "scripts": { + "deploy": "bash scripts/deploy.sh", + "test": "bash scripts/test.sh" + } +} diff --git a/packages/verae-deploy/scripts/deploy.sh b/packages/verae-deploy/scripts/deploy.sh new file mode 100755 index 0000000..44103ff --- /dev/null +++ b/packages/verae-deploy/scripts/deploy.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# host-deps → git fetch → checkout ref → npm ci → npm rebuild +# Does not start processes; restart with fleet/keep after. +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +BOOT="${VERAE_BOOTSTRAP:-$ROOT/../verae-bootstrap}" +SRC="${VERAE_SRC:-$HOME/verae-src}" +REF="${1:-}" +TYPE="${DEPLOY_TYPE:-}" +if [[ -z "$REF" ]]; then + echo "usage: deploy.sh # e.g. deploy.sh v2026-09-12 or main" >&2 + exit 1 +fi +if [[ -x "$BOOT/scripts/host-deps.sh" ]]; then + if [[ -n "$TYPE" ]]; then + bash "$BOOT/scripts/host-deps.sh" --type "$TYPE" + else + bash "$BOOT/scripts/host-deps.sh" + fi +fi +LIST="$BOOT/repos.txt" +[[ -n "$TYPE" && -f "$BOOT/types/$TYPE/repos.txt" ]] && LIST="$BOOT/types/$TYPE/repos.txt" +[[ -f "$LIST" ]] || { echo "no repo list $LIST" >&2; exit 1; } +export GIT_SSH_COMMAND="${GIT_SSH_COMMAND:-ssh -o StrictHostKeyChecking=accept-new -p 2223}" +failed=0 +while read -r name; do + [[ -z "$name" || "$name" == \#* ]] && continue + dir="$SRC/$name" + if [[ ! -d "$dir/.git" ]]; then + echo "skip not cloned $name" + continue + fi + echo "deploy $name @ $REF" + git -C "$dir" fetch --prune origin + if ! git -C "$dir" checkout -q "$REF" 2>/dev/null; then + echo " no ref $REF, staying on $(git -C "$dir" rev-parse --abbrev-ref HEAD)" + fi + if [[ -f "$dir/package.json" ]]; then + ( + cd "$dir" + if [[ -f package-lock.json ]]; then + npm ci --no-audit --no-fund || npm install --no-audit --no-fund + else + npm install --no-audit --no-fund + fi + npm rebuild --no-audit --no-fund >/dev/null || true + ) || { echo " npm failed $name"; failed=1; } + fi + git -C "$dir" rev-parse --short HEAD >"$dir/.deployed-ref" + echo " $(cat "$dir/.deployed-ref")" +done < "$LIST" +echo "$REF $(date -u +%Y-%m-%dT%H:%M:%SZ)" >"$SRC/.deployed" +[[ $failed -eq 0 ]] || exit 1 +echo "deployed $REF under $SRC — restart keep/fleet yourself" diff --git a/packages/verae-deploy/scripts/test.sh b/packages/verae-deploy/scripts/test.sh new file mode 100755 index 0000000..4b26376 --- /dev/null +++ b/packages/verae-deploy/scripts/test.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +bash -n "$ROOT/scripts/deploy.sh" +grep -q host-deps.sh "$ROOT/scripts/deploy.sh" +echo OK diff --git a/packages/verae-fleet/README.md b/packages/verae-fleet/README.md index 2834e52..eeddc0e 100644 --- a/packages/verae-fleet/README.md +++ b/packages/verae-fleet/README.md @@ -41,6 +41,8 @@ Zapier cloud apps are listed but **not spawned**. NATS on NS1 is **monitored onl On a dedicated host, run **[verae-keep](https://git.georgelambert.org/marchon/verae-keep)** next to fleet: keep restarts crashed replicas unless this console paused or stopped them (`intent.json` per instance). Pause/stop/start write that file locally and over SSH. +NS1 overlay (no rewrite of git JSON): `FLEET_FILE=overlays/ns1/fleet.json` `FLEET_MACHINES=overlays/ns1/machines.json`. User systemd: `bash scripts/install-systemd.sh`. Proxmox worker: machine id `px-worker` after [verae-proxmox](https://git.georgelambert.org/marchon/verae-proxmox). + Install on Docker / Proxmox / metal: https://zapier.georgelambert.org/packages/verae-ops/README.pdf Clone: `ssh://git@git.georgelambert.org:2223/marchon/verae-fleet.git` diff --git a/packages/verae-fleet/machines.json b/packages/verae-fleet/machines.json index dd47bcc..af52c1e 100644 --- a/packages/verae-fleet/machines.json +++ b/packages/verae-fleet/machines.json @@ -42,6 +42,20 @@ "capacity": 12, "roles": ["tree-node", "archive-worm"], "notes": "Same user/key as NS1. Enable after confirming node is on PATH." + }, + { + "id": "px-worker", + "title": "Proxmox LXC 510 verae-px-worker", + "kind": "ssh", + "host": "10.10.10.20", + "user": "marchon", + "sshPort": 22, + "identityFile": "~/.ssh/id_ed25519", + "remoteDir": "~/verae-fleet-runtime", + "enabled": true, + "capacity": 12, + "roles": ["tree-node", "archive-worm"], + "notes": "Unprivileged Ubuntu CT 510 on NS1 vmbr1. ssh marchon@10.10.10.20." } ] } diff --git a/packages/verae-fleet/overlays/ns1/README.md b/packages/verae-fleet/overlays/ns1/README.md new file mode 100644 index 0000000..bae72ae --- /dev/null +++ b/packages/verae-fleet/overlays/ns1/README.md @@ -0,0 +1,13 @@ +# NS1 overlay + +Checked-in fleet + machines for **NS1.GEORGELAMBERT.ORG** (Proxmox). Do not rewrite these from `start.sh`. + +```bash +export FLEET_FILE=$PWD/overlays/ns1/fleet.json +export FLEET_MACHINES=$PWD/overlays/ns1/machines.json +node src/cli.js serve +``` + +Archive worm/tree/poller stay **disabled** here so **verae-keep** owns those processes. HTTP doors stay with fleet. + +`px-worker` (LXC 510, `10.10.10.20`) is listed disabled until `verae-proxmox/scripts/create-worker.sh` and `verae-fleet ssh-check px-worker` succeed; then set `"enabled": true`. diff --git a/packages/verae-fleet/overlays/ns1/fleet.json b/packages/verae-fleet/overlays/ns1/fleet.json new file mode 100644 index 0000000..9e3fc16 --- /dev/null +++ b/packages/verae-fleet/overlays/ns1/fleet.json @@ -0,0 +1,40 @@ +{ + "apiVersion": "verae.fleet/v1", + "control": { + "bind": "0.0.0.0", + "port": 3850, + "probeIntervalMs": 750, + "unhealthyAfterMs": 2500, + "restartBackoffMs": [200, 800, 2000], + "maxRestartsPerHour": 40 + }, + "nats": { + "url": "nats://127.0.0.1:4222", + "publicBind": false + }, + "services": { + "nats": { "min": 1, "max": 1, "keepFloor": false, "enabled": true, "managed": false }, + "zappier-edge": { "min": 1, "max": 1, "keepFloor": true, "enabled": true }, + "access-authz": { "min": 1, "max": 1, "keepFloor": true, "enabled": true }, + "access-web": { "min": 1, "max": 1, "keepFloor": true, "enabled": true }, + "access-api": { "min": 1, "max": 1, "keepFloor": true, "enabled": true }, + "access-leaf": { "min": 1, "max": 1, "keepFloor": true, "enabled": true }, + "access-zapier": { "min": 1, "max": 1, "keepFloor": true, "enabled": true }, + "access-staff": { "min": 1, "max": 1, "keepFloor": true, "enabled": true }, + "identity": { "min": 1, "max": 1, "keepFloor": true, "enabled": true }, + "jobs-events": { "min": 1, "max": 1, "keepFloor": true, "enabled": true }, + "account-balance": { "min": 1, "max": 1, "keepFloor": true, "enabled": true }, + "customer-service": { "min": 1, "max": 1, "keepFloor": true, "enabled": true }, + "sales-pricing": { "min": 1, "max": 1, "keepFloor": true, "enabled": true }, + "accounting-export": { "min": 1, "max": 1, "keepFloor": true, "enabled": true }, + "middleware-http": { "min": 1, "max": 1, "keepFloor": true, "enabled": true }, + "job-poller": { "min": 1, "max": 2, "keepFloor": false, "enabled": false }, + "webhook-deliver": { "min": 1, "max": 2, "keepFloor": false, "enabled": false }, + "archive-aggregator": { "min": 1, "max": 2, "keepFloor": false, "enabled": false }, + "archive-worm": { "min": 3, "max": 6, "keepFloor": false, "enabled": false }, + "tree-node": { "min": 3, "max": 9, "keepFloor": false, "enabled": false }, + "zapier-simulator": { "min": 0, "max": 1, "keepFloor": false, "enabled": true }, + "staff-iam": { "min": 1, "max": 1, "keepFloor": true, "enabled": true }, + "staff-session": { "min": 1, "max": 1, "keepFloor": true, "enabled": true } + } +} diff --git a/packages/verae-fleet/overlays/ns1/machines.json b/packages/verae-fleet/overlays/ns1/machines.json new file mode 100644 index 0000000..d8714b9 --- /dev/null +++ b/packages/verae-fleet/overlays/ns1/machines.json @@ -0,0 +1,54 @@ +{ + "machines": [ + { + "id": "local", + "title": "NS1 control plane (Proxmox host)", + "kind": "local", + "host": "127.0.0.1", + "enabled": true, + "capacity": 32, + "roles": ["*"] + }, + { + "id": "ns1", + "title": "NS1.GEORGELAMBERT.ORG SSH profile", + "kind": "ssh", + "host": "70.88.205.138", + "user": "marchon", + "sshPort": 22, + "identityFile": "~/.ssh/id_ed25519", + "remoteDir": "~/verae-fleet-runtime", + "enabled": false, + "capacity": 24, + "roles": ["tree-node", "archive-worm", "archive-aggregator", "job-poller", "webhook-deliver"], + "notes": "Same box as local; SSH spawn unused on all-in-one." + }, + { + "id": "lan-134", + "title": "70.88.205.134", + "kind": "ssh", + "host": "70.88.205.134", + "user": "marchon", + "sshPort": 22, + "identityFile": "~/.ssh/id_ed25519", + "remoteDir": "~/verae-fleet-runtime", + "enabled": false, + "capacity": 12, + "roles": ["tree-node", "archive-worm"] + }, + { + "id": "px-worker", + "title": "Proxmox LXC 510 verae-px-worker", + "kind": "ssh", + "host": "10.10.10.20", + "user": "marchon", + "sshPort": 22, + "identityFile": "~/.ssh/id_ed25519", + "remoteDir": "~/verae-fleet-runtime", + "enabled": true, + "capacity": 12, + "roles": ["tree-node", "archive-worm"], + "notes": "vmbr1 LXC 510. SSH as marchon@10.10.10.20 with ~/.ssh/id_ed25519." + } + ] +} diff --git a/packages/verae-fleet/scripts/install-systemd.sh b/packages/verae-fleet/scripts/install-systemd.sh new file mode 100755 index 0000000..900be9f --- /dev/null +++ b/packages/verae-fleet/scripts/install-systemd.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +UNIT_DIR="${XDG_CONFIG_HOME:-$HOME/.config}/systemd/user" +mkdir -p "$UNIT_DIR" +install -m 644 "$ROOT/systemd/verae-fleet.service" "$UNIT_DIR/verae-fleet.service" +systemctl --user daemon-reload +systemctl --user enable verae-fleet.service +loginctl enable-linger "$USER" || true +echo "enabled verae-fleet (linger). start: systemctl --user start verae-fleet" +echo "do not also nohup node src/cli.js serve" diff --git a/packages/verae-fleet/src/load.js b/packages/verae-fleet/src/load.js index 96d9662..4e62984 100644 --- a/packages/verae-fleet/src/load.js +++ b/packages/verae-fleet/src/load.js @@ -19,7 +19,9 @@ export function readJson(file) { * @param {string} [root] */ export function loadFleet(root = FLEET_ROOT, opts = {}) { - const fleetPath = path.join(root, 'fleet.json'); + const fleetPath = process.env.FLEET_FILE + ? path.resolve(process.env.FLEET_FILE) + : path.join(root, 'fleet.json'); const fleet = readJson(fleetPath); const dir = path.join(root, 'services'); const services = {}; diff --git a/packages/verae-fleet/src/machines.js b/packages/verae-fleet/src/machines.js index 0b899f7..9b2583b 100644 --- a/packages/verae-fleet/src/machines.js +++ b/packages/verae-fleet/src/machines.js @@ -21,7 +21,9 @@ export function defaultMachines() { } export function loadMachines(root, overlayPath) { - const file = path.join(root, 'machines.json'); + const file = process.env.FLEET_MACHINES + ? path.resolve(process.env.FLEET_MACHINES) + : path.join(root, 'machines.json'); let list = defaultMachines(); if (fs.existsSync(file)) { const raw = JSON.parse(fs.readFileSync(file, 'utf8')); diff --git a/packages/verae-fleet/systemd/verae-fleet.service b/packages/verae-fleet/systemd/verae-fleet.service new file mode 100644 index 0000000..c4d1c9f --- /dev/null +++ b/packages/verae-fleet/systemd/verae-fleet.service @@ -0,0 +1,19 @@ +[Unit] +Description=Verae fleet operator console +After=network.target + +[Service] +Type=simple +WorkingDirectory=%h/verae-stack/packages/verae-fleet +Environment=FLEET_FILE=%h/verae-stack/packages/verae-fleet/overlays/ns1/fleet.json +Environment=FLEET_MACHINES=%h/verae-stack/packages/verae-fleet/overlays/ns1/machines.json +Environment=STAFF_IAM_URL=https://iam.zapier.georgelambert.org +Environment=STAFF_COOKIE_DOMAIN=.zapier.georgelambert.org +Environment=NATS_URL=nats://127.0.0.1:4222 +Environment=FLEET_ENABLE_LAN134= +ExecStart=/usr/bin/node src/cli.js serve +Restart=always +RestartSec=2 + +[Install] +WantedBy=default.target diff --git a/packages/verae-keep/README.md b/packages/verae-keep/README.md index 4deb268..81a8c06 100644 --- a/packages/verae-keep/README.md +++ b/packages/verae-keep/README.md @@ -28,7 +28,14 @@ KEEP_UNITS=$HOME/verae-keep/units.ns1.json KEEP_STATE=$HOME/verae-keep/data \ nohup bash scripts/guard.sh >/tmp/verae-keep-guard.out 2>&1 & ``` -Or systemd user unit `systemd/verae-keep-guard.service` (`loginctl enable-linger`). +Or user systemd (survives reboot; do not also `nohup guard.sh`): + +```bash +bash scripts/install-systemd.sh +systemctl --user start verae-keep-guard +``` + +Proxmox guest extra copies: `KEEP_UNITS=units.px-worker.json`. Units: NATS (observe only), job-poller, webhook-deliver, archive-aggregator, archive-worm ×3, tree-node ×3. diff --git a/packages/verae-keep/scripts/install-systemd.sh b/packages/verae-keep/scripts/install-systemd.sh new file mode 100755 index 0000000..5af98c1 --- /dev/null +++ b/packages/verae-keep/scripts/install-systemd.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# User systemd for keep guard + linger so it survives reboot. +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +UNIT_DIR="${XDG_CONFIG_HOME:-$HOME/.config}/systemd/user" +mkdir -p "$UNIT_DIR" +install -m 644 "$ROOT/systemd/verae-keep-guard.service" "$UNIT_DIR/verae-keep-guard.service" +systemctl --user daemon-reload +systemctl --user enable verae-keep-guard.service +loginctl enable-linger "$USER" || true +echo "enabled verae-keep-guard (linger). start: systemctl --user start verae-keep-guard" +echo "do not also run nohup guard.sh" diff --git a/packages/verae-keep/units.px-worker.json b/packages/verae-keep/units.px-worker.json new file mode 100644 index 0000000..fbcd8a9 --- /dev/null +++ b/packages/verae-keep/units.px-worker.json @@ -0,0 +1,40 @@ +{ + "bind": "127.0.0.1", + "port": 3860, + "intervalMs": 2000, + "runtimeDir": "~/verae-fleet-runtime", + "units": [ + { + "id": "archive-worm-px-0", + "health": "http://127.0.0.1:13500/health", + "cwd": "~/verae-src/verae-fleet", + "command": ["node", "src/worker.js"], + "env": { + "FLEET_ROLE": "archive-worm", + "FLEET_SERVICE": "archive-worm", + "FLEET_INSTANCE": "archive-worm-px-0", + "FLEET_MACHINE": "px-worker", + "FLEET_HEALTH_PORT": "13500", + "FLEET_HEALTH_BIND": "127.0.0.1", + "FLEET_STATE_DIR": "data/archive-worm-px-0", + "NATS_URL": "nats://10.10.10.1:4222" + } + }, + { + "id": "tree-node-px-0", + "health": "http://127.0.0.1:13600/health", + "cwd": "~/verae-src/verae-fleet", + "command": ["node", "src/worker.js"], + "env": { + "FLEET_ROLE": "tree-node", + "FLEET_SERVICE": "tree-node", + "FLEET_INSTANCE": "tree-node-px-0", + "FLEET_MACHINE": "px-worker", + "FLEET_HEALTH_PORT": "13600", + "FLEET_HEALTH_BIND": "127.0.0.1", + "FLEET_STATE_DIR": "data/tree-node-px-0", + "NATS_URL": "nats://10.10.10.1:4222" + } + } + ] +} diff --git a/packages/verae-ops/03-proxmox.md b/packages/verae-ops/03-proxmox.md index 736a147..4817306 100644 --- a/packages/verae-ops/03-proxmox.md +++ b/packages/verae-ops/03-proxmox.md @@ -1,60 +1,50 @@ -# 3. Proxmox (LXC and QEMU) +# 3. Proxmox (NS1 is the hypervisor) -Target: one **NATS cluster VM/CT** (or three), one **HTTPS edge** VM, one or more **worker** VMs. Fleet SSH-spreads worm/tree-node onto worker hosts. +**NS1.GEORGELAMBERT.ORG** (`70.88.205.138`) **is** the Proxmox host. Control plane, Caddy, keep, and fleet run on the host. Extra worm/tree copies run in an **unprivileged Ubuntu LXC** on the private bridge. -## Suggested guests +| Bridge | Address | Use | +|--------|---------|-----| +| `vmbr0` | `70.88.205.138/28` | Public HTTPS / SSH. **Never NATS.** | +| `vmbr1` | `10.10.10.1/24` | Private. PKI `.11`, peergos-a `.12`, peergos-b `.13`, **verae-px-worker `.20`** | -| Guest | OS | vCPU / RAM / disk | Role | -|-------|----|-------------------|------| -| `nats-a` `nats-b` `nats-c` | Debian 12 LXC unprivileged | 1 / 1G / 8G | JetStream, bind `127.0.0.1` **or** a vmbr **private** bridge only | -| `edge` | Debian 12 QEMU | 2 / 2G / 20G | Caddy + zappier-edge + middleware | -| `worker-1` … | Debian 12 LXC | 2 / 2G / 20G | fleet SSH target; worm + tree-node | +## Guest we actually deploy -Use a **private bridge** (`vmbr1`, RFC1918) for NATS `4222`/`6222`. Public `vmbr0` only on `edge` (80/443). +| | | +|--|--| +| VMID | **510** | +| Name | `verae-px-worker` | +| Template | `ubuntu-24.04-standard_24.04-2_amd64.tar.zst` (`local`) | +| Disk | `SSD2` 16G | +| RAM / CPU | 2G / 2 | +| Roles | `tree-node`, `archive-worm` | +| Fleet id | `px-worker` | +| NATS | `nats://10.10.10.1:4222` (host `socat` → `127.0.0.1:4222`) | -## NATS on an LXC +Repo: [verae-proxmox](https://git.georgelambert.org/marchon/verae-proxmox) ```bash -apt-get update && apt-get install -y curl -# official nats-server binary -curl -sSL https://github.com/nats-io/nats-server/releases/latest/download/nats-server-v2.10.26-linux-amd64.tar.gz | tar -xz -install nats-server /usr/local/bin/ +# on NS1 +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-proxmox.git +cd verae-proxmox +bash scripts/create-worker.sh +ssh -i ~/.ssh/id_ed25519 marchon@10.10.10.20 true ``` -`/etc/nats/nats.conf`: +On the guest, bootstrap type `proxmox-worker` ([verae-bootstrap](https://git.georgelambert.org/marchon/verae-bootstrap)). Then `verae-fleet ssh-check px-worker` and set `"enabled": true` in `overlays/ns1/machines.json`. -``` -jetstream {} -listen: 10.10.10.11:4222 # private bridge, not 0.0.0.0 on public -cluster { - name: verae - listen: 10.10.10.11:6222 - routes: [ nats-route://10.10.10.12:6222, nats-route://10.10.10.13:6222 ] -} -http: 127.0.0.1:8222 -``` +## NATS for guests -systemd: `ExecStart=/usr/local/bin/nats-server -c /etc/nats/nats.conf`. +Loopback NATS on the host stays `127.0.0.1:4222`. `verae-nats-private-proxy.service` binds **only** `10.10.10.1:4222`. Do not listen on `vmbr0`. -## Edge VM +## Suggested future guests (not created by create-worker.sh) -Install Node 22, clone `zappier-edge` and `verae-middleware`, Caddy reverse-proxy: +| Guest | Role | +|-------|------| +| `nats-b` `nats-c` | Extra JetStream nodes on `vmbr1` when leaving all-in-one | +| more `worker-*` | Same recipe as 510 with the next free `10.10.10.2x` | -``` -zappier.example.com { - reverse_proxy 127.0.0.1:3000 -} -middleware.example.com { - reverse_proxy 127.0.0.1:3100 -} -``` +## Edge / Caddy -Middleware env: `NATS_URL=nats://10.10.10.11:4222,nats://10.10.10.12:4222,nats://10.10.10.13:4222`. +Stay on the **host** (already live). Do not put zappier-edge in the worker CT. -## Fleet on Proxmox - -On the operator workstation or a jump CT (loopback `:3850`): - -`machines.json` entries: `kind=ssh`, `user=root` or `marchon`, `host=10.10.10.2x`, `identityFile=~/.ssh/id_ed25519` (**path only**). Roles: `tree-node,archive-worm`. Capacity ≥ 3 per worker if you want floors on one host. - -`ssh-check` from the operator console before enabling `keepFloor`. +Operator console: `https://fleet.zapier.georgelambert.org/` · overlay `packages/verae-fleet/overlays/ns1/`. diff --git a/packages/verae-ops/07-maintenance.md b/packages/verae-ops/07-maintenance.md index 240f151..8590bb9 100644 --- a/packages/verae-ops/07-maintenance.md +++ b/packages/verae-ops/07-maintenance.md @@ -13,28 +13,62 @@ Green / yellow / red on the operator console: https://zapier.georgelambert.org/p ## Restart -- systemd: `systemctl restart verae-middleware zappier-edge` +- User systemd (survives reboot with linger): + - `bash packages/verae-keep/scripts/install-systemd.sh` + - `bash packages/verae-fleet/scripts/install-systemd.sh` + - `loginctl enable-linger $USER` + - `systemctl --user restart verae-keep-guard verae-fleet` - Compose: `docker compose up -d --no-deps middleware` - One replica: fleet `restart tree-node-1` (floor spawns a replacement if needed) - Pause is **not** off — paused copies do not count toward `keepFloor` +- Do not run **both** `nohup guard.sh` and the systemd unit ## Backup -| Data | Where | -|------|--------| -| zappier SQLite | `ZAPPIER_DB` path / volume | -| worm / tree blobs | instance `data/` dirs | -| NATS JetStream | nats store dir | -| fleet overlay | `packages/verae-fleet/data/` (gitignored) | +Repo: [verae-backup](https://git.georgelambert.org/marchon/verae-backup) -Do not back up private keys into git. Restore `identityFile` paths on the operator host. +```bash +export BACKUP_ROOT=/SSD2/backups/verae +bash scripts/backup.sh +bash scripts/restore.sh --dry-run +``` + +| Data | Default path | +|------|----------------| +| zappier SQLite | `~/verae-stack/packages/zappier/zappier.db` | +| worm / tree blobs | `~/verae-fleet-runtime/data/` | +| keep state | `~/verae-keep/data` | +| NATS JetStream | `/var/lib/nats/jetstream` if present | +| IAM | `~/verae-stack/packages/verae-staff-iam/data` | + +User timer: `verae-backup.timer` (03:17 UTC). Do not back up private keys into git. Restore `identityFile` paths on the operator host. After `--apply`, `curl` `/health` on edge and keep. ## Upgrade -1. `git pull` each independent repo (or refresh the monorepo snapshot). -2. `npm test` in that package. -3. Restart that unit only; NATS cluster stays up. -4. Zapier Platform app: `zapier-platform push` from `verae-zapier-app` when HTTPS contracts change. +Repo: [verae-deploy](https://git.georgelambert.org/marchon/verae-deploy) + +```bash +export VERAE_SRC=$HOME/verae-src +export DEPLOY_TYPE=ns1-all-in-one +bash scripts/deploy.sh main # or a tag vYYYY-MM-DD +systemctl --user restart verae-keep-guard verae-fleet +``` + +1. `host-deps.sh` on **this** OS. +2. `git fetch` + checkout ref (not force-push). +3. `npm ci` / `npm rebuild` (never copy macOS `node_modules`). +4. Restart that unit only; NATS stays up. +5. Zapier Platform app: `zapier-platform push` from `verae-zapier-app` when HTTPS contracts change. + +## Off-box watch + +Repo: [verae-uptime](https://git.georgelambert.org/marchon/verae-uptime) + +```bash +node src/watch.js --once # exit 2 if a public door failed +``` + +Run this on a **different** machine than NS1 (laptop cron is enough). `/status` on `:3870` in loop mode. ## Logs diff --git a/packages/verae-ops/GETTING-STARTED.md b/packages/verae-ops/GETTING-STARTED.md index c77025d..0457140 100644 --- a/packages/verae-ops/GETTING-STARTED.md +++ b/packages/verae-ops/GETTING-STARTED.md @@ -33,6 +33,8 @@ The billing and user platform is the standalone **zappier** project (`/Users/mar ## 15-minute lab (one machine) +NS1 is Proxmox. Extra worm/tree LXC: [verae-proxmox](https://git.georgelambert.org/marchon/verae-proxmox) `create-worker.sh` (VMID 510, `10.10.10.20`). Off-box watch: [verae-uptime](https://git.georgelambert.org/marchon/verae-uptime). Backup: [verae-backup](https://git.georgelambert.org/marchon/verae-backup). Tagged upgrade: [verae-deploy](https://git.georgelambert.org/marchon/verae-deploy). + Bare metal / VM (Debian or Alpine): install OS + Node first, then clone. ```bash diff --git a/packages/verae-proxmox/NATS.md b/packages/verae-proxmox/NATS.md new file mode 100644 index 0000000..8cc0d78 --- /dev/null +++ b/packages/verae-proxmox/NATS.md @@ -0,0 +1,3 @@ +# NATS — verae-proxmox + +Guests on `vmbr1` must not use the public NIC. The host runs `socat` `10.10.10.1:4222` → `127.0.0.1:4222`. Zapier cloud still never talks to NATS. diff --git a/packages/verae-proxmox/README.md b/packages/verae-proxmox/README.md new file mode 100644 index 0000000..a6e8270 --- /dev/null +++ b/packages/verae-proxmox/README.md @@ -0,0 +1,38 @@ +# verae-proxmox + +Provision an **archive worker LXC** on **NS1.GEORGELAMBERT.ORG** (this box **is** the Proxmox host). + +| | | +|--|--| +| VMID | **510** | +| Hostname | `verae-px-worker` | +| Network | `vmbr1` **10.10.10.20/24** gw `10.10.10.1` | +| Storage | SSD2 16G, unprivileged Ubuntu 24.04 | +| Roles | `tree-node`, `archive-worm` | +| NATS | `nats://10.10.10.1:4222` via host socat proxy (loopback NATS stays `127.0.0.1:4222`; **not** on `vmbr0`) | + +**Forgejo:** https://git.georgelambert.org/marchon/verae-proxmox +**SSH:** `ssh://git@git.georgelambert.org:2223/marchon/verae-proxmox.git` + +```bash +# on NS1 +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-proxmox.git +cd verae-proxmox +bash scripts/create-worker.sh +ssh marchon@10.10.10.20 +``` + +On the guest: + +```bash +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-bootstrap.git +cd verae-bootstrap +bash scripts/host-deps.sh --type proxmox-worker +export VERAE_SRC=$HOME/verae-src +export NATS_URL=nats://10.10.10.1:4222 +bash scripts/fetch.sh proxmox-worker +``` + +Then enable `px-worker` in `verae-fleet/overlays/ns1/machines.json` and `node src/cli.js ssh-check px-worker`. + +Keep on the guest uses `units.px-worker.json` (worm ×1 + tree-node ×1 extra copies). Host keep still runs the NS1 floor of 3. diff --git a/packages/verae-proxmox/SUMMARY.md b/packages/verae-proxmox/SUMMARY.md new file mode 100644 index 0000000..cf23d34 --- /dev/null +++ b/packages/verae-proxmox/SUMMARY.md @@ -0,0 +1,3 @@ +# verae-proxmox + +Create Ubuntu LXC 510 on NS1 vmbr1 for extra worm/tree copies. Private NATS proxy on 10.10.10.1:4222. diff --git a/packages/verae-proxmox/guest.env b/packages/verae-proxmox/guest.env new file mode 100644 index 0000000..9e2fbb8 --- /dev/null +++ b/packages/verae-proxmox/guest.env @@ -0,0 +1,15 @@ +# Defaults for NS1.GEORGELAMBERT.ORG (Proxmox). +VMID="${VMID:-510}" +GUEST_HOSTNAME="${GUEST_HOSTNAME:-verae-px-worker}" +BRIDGE="${BRIDGE:-vmbr1}" +IP="${IP:-10.10.10.20/24}" +GW="${GW:-10.10.10.1}" +MEMORY="${MEMORY:-2048}" +CORES="${CORES:-2}" +DISK="${DISK:-16}" +STORAGE="${STORAGE:-SSD2}" +TEMPLATE="${TEMPLATE:-/var/lib/vz/template/cache/ubuntu-24.04-standard_24.04-2_amd64.tar.zst}" +DNS="${DNS:-8.8.8.8}" +NATS_PRIVATE="${NATS_PRIVATE:-10.10.10.1}" +NATS_PORT="${NATS_PORT:-4222}" +USER_NAME="${USER_NAME:-marchon}" diff --git a/packages/verae-proxmox/package.json b/packages/verae-proxmox/package.json new file mode 100644 index 0000000..dc335c7 --- /dev/null +++ b/packages/verae-proxmox/package.json @@ -0,0 +1,10 @@ +{ + "name": "verae-proxmox", + "version": "0.1.0", + "private": true, + "description": "Provision Verae archive worker LXC guests on NS1 Proxmox (vmbr1)", + "scripts": { + "create-worker": "bash scripts/create-worker.sh", + "test": "bash scripts/test.sh" + } +} diff --git a/packages/verae-proxmox/scripts/create-worker.sh b/packages/verae-proxmox/scripts/create-worker.sh new file mode 100755 index 0000000..2c1aa48 --- /dev/null +++ b/packages/verae-proxmox/scripts/create-worker.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Create (or reuse) LXC 510 on this Proxmox host, bootstrap marchon + sshd + host-deps. +# Run on NS1 as marchon (sudo). Does not publish NATS on vmbr0. +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +# shellcheck disable=SC1091 +. "$ROOT/guest.env" +if [[ ! -f /etc/pve/local/.members ]] && [[ ! -d /etc/pve/nodes ]]; then + echo "not a Proxmox host" >&2 + exit 1 +fi +export PATH="/usr/sbin:/usr/bin:/bin:$PATH" +if ! command -v pct >/dev/null; then + echo "pct missing (need /usr/sbin/pct on the Proxmox host)" >&2 + exit 1 +fi +if [[ ! -f "$TEMPLATE" ]]; then + echo "missing template $TEMPLATE" >&2 + exit 1 +fi +if ! sudo pct status "$VMID" >/dev/null 2>&1; then + echo "pct create $VMID $GUEST_HOSTNAME $IP" + sudo pct create "$VMID" "$TEMPLATE" \ + --hostname "$GUEST_HOSTNAME" \ + --memory "$MEMORY" --cores "$CORES" --swap 512 \ + --net0 "name=eth0,bridge=${BRIDGE},ip=${IP},gw=${GW},type=veth" \ + --rootfs "${STORAGE}:${DISK}" \ + --unprivileged 1 --onboot 1 --nameserver "$DNS" \ + --features nesting=1 \ + --ostype ubuntu +else + echo "CT $VMID already exists" +fi +sudo pct start "$VMID" 2>/dev/null || true +for i in $(seq 1 30); do + sudo pct exec "$VMID" -- true 2>/dev/null && break + sleep 2 +done +PUB="" +[[ -f "$HOME/.ssh/id_ed25519.pub" ]] && PUB="$(cat "$HOME/.ssh/id_ed25519.pub")" +[[ -z "$PUB" && -f "$HOME/.ssh/authorized_keys" ]] && PUB="$(head -1 "$HOME/.ssh/authorized_keys")" +[[ -n "$PUB" ]] || { echo "no ssh public key for $USER_NAME" >&2; exit 1; } +if [[ ! -f "$HOME/.ssh/id_ed25519" ]]; then + ssh-keygen -t ed25519 -N "" -f "$HOME/.ssh/id_ed25519" -C "ns1-to-px-worker" + PUB="$(cat "$HOME/.ssh/id_ed25519.pub")" +fi +sudo pct exec "$VMID" -- bash -lc " +set -e +export DEBIAN_FRONTEND=noninteractive +apt-get update -qq +apt-get install -y --no-install-recommends openssh-server sudo curl git ca-certificates python3 make g++ xz-utils +id $USER_NAME >/dev/null 2>&1 || useradd -m -s /bin/bash $USER_NAME +echo '$USER_NAME ALL=(ALL) NOPASSWD:ALL' >/etc/sudoers.d/90-$USER_NAME +chmod 440 /etc/sudoers.d/90-$USER_NAME +install -d -m 700 -o $USER_NAME -g $USER_NAME /home/$USER_NAME/.ssh +grep -qxF '$PUB' /home/$USER_NAME/.ssh/authorized_keys 2>/dev/null || echo '$PUB' >>/home/$USER_NAME/.ssh/authorized_keys +chown $USER_NAME:$USER_NAME /home/$USER_NAME/.ssh/authorized_keys +chmod 600 /home/$USER_NAME/.ssh/authorized_keys +systemctl enable --now ssh +" +HOST_IP="${IP%%/*}" +echo "guest $GUEST_HOSTNAME $HOST_IP" +# NATS private proxy on the Proxmox host +if ! ss -lnt | grep -q "10.10.10.1:4222"; then + if command -v socat >/dev/null; then + sudo cp "$ROOT/scripts/nats-private-proxy.service" /etc/systemd/system/verae-nats-private-proxy.service + sudo systemctl daemon-reload + sudo systemctl enable --now verae-nats-private-proxy.service || true + else + sudo apt-get install -y socat || sudo apt-get install -y socat + sudo cp "$ROOT/scripts/nats-private-proxy.service" /etc/systemd/system/verae-nats-private-proxy.service + sudo systemctl daemon-reload + sudo systemctl enable --now verae-nats-private-proxy.service + fi +fi +ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=8 \ + -i "$HOME/.ssh/id_ed25519" "$USER_NAME@$HOST_IP" 'echo ssh_ok' \ + || echo "ssh not ready yet — retry: ssh $USER_NAME@$HOST_IP" +echo "NATS for guests: nats://${NATS_PRIVATE}:${NATS_PORT} (never on vmbr0)" +echo "next: on the guest, clone verae-bootstrap type proxmox-worker" diff --git a/packages/verae-proxmox/scripts/nats-private-proxy.service b/packages/verae-proxmox/scripts/nats-private-proxy.service new file mode 100644 index 0000000..1dda667 --- /dev/null +++ b/packages/verae-proxmox/scripts/nats-private-proxy.service @@ -0,0 +1,13 @@ +[Unit] +Description=Proxy loopback NATS onto vmbr1 (10.10.10.1:4222) for Proxmox guests +After=network.target + +[Service] +Type=simple +# Public NIC must stay unbound. Loopback NATS remains 127.0.0.1:4222. +ExecStart=/usr/bin/socat TCP-LISTEN:4222,bind=10.10.10.1,reuseaddr,fork TCP:127.0.0.1:4222 +Restart=always +RestartSec=2 + +[Install] +WantedBy=multi-user.target diff --git a/packages/verae-proxmox/scripts/test.sh b/packages/verae-proxmox/scripts/test.sh new file mode 100755 index 0000000..2f6d0a5 --- /dev/null +++ b/packages/verae-proxmox/scripts/test.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +bash -n "$ROOT/scripts/create-worker.sh" +test -f "$ROOT/guest.env" +test -f "$ROOT/scripts/nats-private-proxy.service" +grep -q 'bind=10.10.10.1' "$ROOT/scripts/nats-private-proxy.service" +grep -q vmbr1 "$ROOT/guest.env" +echo OK diff --git a/packages/verae-uptime/NATS.md b/packages/verae-uptime/NATS.md new file mode 100644 index 0000000..dd2a519 --- /dev/null +++ b/packages/verae-uptime/NATS.md @@ -0,0 +1,3 @@ +# NATS — verae-uptime + +Uptime watch is HTTPS GET only. It never connects to NATS. diff --git a/packages/verae-uptime/README.md b/packages/verae-uptime/README.md new file mode 100644 index 0000000..f5fc83b --- /dev/null +++ b/packages/verae-uptime/README.md @@ -0,0 +1,21 @@ +# verae-uptime + +GET-watch the public Verae Time × Zapier doors from a host that is **not** the thing being watched. If NS1 is the only watcher, NS1 dying is silent. + +**Forgejo:** https://git.georgelambert.org/marchon/verae-uptime +**SSH:** `ssh://git@git.georgelambert.org:2223/marchon/verae-uptime.git` + +Run on a laptop, a second WAN box, or (weaker) a Proxmox CT on the same chassis. + +```bash +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-uptime.git +cd verae-uptime +node src/watch.js --once # exit 2 if any required target failed +node src/watch.js # loop + http://127.0.0.1:3870/status +``` + +`probes.json` lists catalog, portal, api, zap, iam, mw, fleet, git. Optional `webhook` POSTs the snapshot when `failed > 0`. + +State: `~/.verae-uptime/status.json` and `watch.log`. + +This process does **not** speak NATS. diff --git a/packages/verae-uptime/SUMMARY.md b/packages/verae-uptime/SUMMARY.md new file mode 100644 index 0000000..493ed16 --- /dev/null +++ b/packages/verae-uptime/SUMMARY.md @@ -0,0 +1,3 @@ +# verae-uptime + +Off-box HTTPS `/health` (and homepage) probes for public doors. Loop + `/status`, or `--once` for cron. diff --git a/packages/verae-uptime/package.json b/packages/verae-uptime/package.json new file mode 100644 index 0000000..8181b42 --- /dev/null +++ b/packages/verae-uptime/package.json @@ -0,0 +1,13 @@ +{ + "name": "verae-uptime", + "version": "0.1.0", + "private": true, + "type": "module", + "description": "Off-box HTTPS health watch for Verae Time × Zapier public doors", + "scripts": { + "start": "node src/watch.js", + "once": "node src/watch.js --once", + "test": "node --test test/*.test.js" + }, + "engines": { "node": ">=20" } +} diff --git a/packages/verae-uptime/probes.json b/packages/verae-uptime/probes.json new file mode 100644 index 0000000..2c2a824 --- /dev/null +++ b/packages/verae-uptime/probes.json @@ -0,0 +1,17 @@ +{ + "intervalSec": 60, + "failAfter": 2, + "timeoutMs": 15000, + "stateDir": "~/.verae-uptime", + "webhook": "", + "targets": [ + { "id": "catalog", "url": "https://zapier.georgelambert.org/" }, + { "id": "portal", "url": "https://portal.zapier.georgelambert.org/portal/" }, + { "id": "api", "url": "https://api.zapier.georgelambert.org/health" }, + { "id": "zap", "url": "https://zap.zapier.georgelambert.org/health" }, + { "id": "iam", "url": "https://iam.zapier.georgelambert.org/" }, + { "id": "mw", "url": "https://mw.zapier.georgelambert.org/health" }, + { "id": "fleet", "url": "https://fleet.zapier.georgelambert.org/" }, + { "id": "git", "url": "https://git.georgelambert.org/" } + ] +} diff --git a/packages/verae-uptime/src/watch.js b/packages/verae-uptime/src/watch.js new file mode 100755 index 0000000..d2a014b --- /dev/null +++ b/packages/verae-uptime/src/watch.js @@ -0,0 +1,114 @@ +#!/usr/bin/env node +/** + * GET each catalog/public door. Exit 2 on --once if any required target fails. + * Loop mode writes state JSON and optional webhook on rising failure. + */ +import fs from 'node:fs'; +import http from 'node:http'; +import https from 'node:https'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const ROOT = path.join(HERE, '..'); +const ONCE = process.argv.includes('--once'); +const CFG = JSON.parse(fs.readFileSync(process.env.UPTIME_PROBES || path.join(ROOT, 'probes.json'), 'utf8')); + +function expandHome(p) { + if (!p) return p; + if (p.startsWith('~/')) return path.join(os.homedir(), p.slice(2)); + return p; +} + +const STATE_DIR = expandHome(process.env.UPTIME_STATE || CFG.stateDir || path.join(ROOT, 'data')); +const failAfter = Number(CFG.failAfter || 2); +const timeoutMs = Number(CFG.timeoutMs || 15000); +const streak = {}; + +function probe(url) { + return new Promise((resolve) => { + let settled = false; + const done = (r) => { + if (settled) return; + settled = true; + resolve(r); + }; + const lib = url.startsWith('https:') ? https : http; + const req = lib.get(url, { timeout: timeoutMs }, (res) => { + res.resume(); + const code = res.statusCode || 0; + done({ ok: code >= 200 && code < 400, code, error: '' }); + }); + req.on('timeout', () => { + req.destroy(); + done({ ok: false, code: 0, error: 'timeout' }); + }); + req.on('error', (err) => done({ ok: false, code: 0, error: err.message })); + }); +} + +async function tick() { + const results = []; + let failed = 0; + for (const t of CFG.targets || []) { + const r = await probe(t.url); + const prev = streak[t.id] || 0; + streak[t.id] = r.ok ? 0 : prev + 1; + const down = !r.ok && streak[t.id] >= failAfter && !t.optional; + if (down) failed += 1; + results.push({ + id: t.id, + url: t.url, + ok: r.ok, + code: r.code, + error: r.error, + streak: streak[t.id], + down, + }); + } + const snap = { t: new Date().toISOString(), failed, results }; + fs.mkdirSync(STATE_DIR, { recursive: true }); + fs.writeFileSync(path.join(STATE_DIR, 'status.json'), `${JSON.stringify(snap, null, 2)}\n`); + fs.appendFileSync(path.join(STATE_DIR, 'watch.log'), `${JSON.stringify(snap)}\n`); + process.stdout.write(`${JSON.stringify({ t: snap.t, failed, ids: results.map((x) => `${x.id}:${x.ok ? 'ok' : x.code || x.error}`) })}\n`); + if (failed && CFG.webhook) { + try { + const u = new URL(CFG.webhook); + const body = JSON.stringify(snap); + const lib = u.protocol === 'https:' ? https : http; + const req = lib.request(u, { method: 'POST', headers: { 'content-type': 'application/json', 'content-length': Buffer.byteLength(body) } }); + req.on('error', () => {}); + req.end(body); + } catch { + /* ignore notify */ + } + } + return failed; +} + +if (ONCE) { + tick().then((n) => process.exit(n ? 2 : 0)); +} else { + const server = http.createServer((req, res) => { + const file = path.join(STATE_DIR, 'status.json'); + if (req.url === '/health' || req.url === '/status') { + if (!fs.existsSync(file)) { + res.writeHead(200, { 'content-type': 'application/json' }); + return res.end('{"ok":true,"note":"no tick yet"}'); + } + const snap = JSON.parse(fs.readFileSync(file, 'utf8')); + res.writeHead(snap.failed ? 503 : 200, { 'content-type': 'application/json' }); + return res.end(JSON.stringify(snap)); + } + res.writeHead(404); + res.end(); + }); + const bind = process.env.UPTIME_BIND || '127.0.0.1'; + const port = Number(process.env.UPTIME_PORT || 3870); + server.listen(port, bind, () => { + process.stdout.write(`uptime listen ${bind}:${port}\n`); + }); + tick(); + setInterval(tick, Number(CFG.intervalSec || 60) * 1000); +} diff --git a/packages/verae-uptime/systemd/verae-uptime.service b/packages/verae-uptime/systemd/verae-uptime.service new file mode 100644 index 0000000..378c461 --- /dev/null +++ b/packages/verae-uptime/systemd/verae-uptime.service @@ -0,0 +1,15 @@ +[Unit] +Description=Verae off-box HTTPS door watch +After=network-online.target + +[Service] +Type=simple +WorkingDirectory=%h/verae-src/verae-uptime +Environment=UPTIME_BIND=127.0.0.1 +Environment=UPTIME_PORT=3870 +ExecStart=/usr/bin/node src/watch.js +Restart=always +RestartSec=5 + +[Install] +WantedBy=default.target diff --git a/packages/verae-uptime/test/watch.test.js b/packages/verae-uptime/test/watch.test.js new file mode 100644 index 0000000..99e592b --- /dev/null +++ b/packages/verae-uptime/test/watch.test.js @@ -0,0 +1,45 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import http from 'node:http'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { spawn } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; + +const root = path.join(path.dirname(fileURLToPath(import.meta.url)), '..'); + +test('once mode fails when a required target is down', async () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'uptime-')); + const srv = await new Promise((resolve) => { + const s = http.createServer((req, res) => { + res.writeHead(req.url === '/ok' ? 200 : 500); + res.end('x'); + }); + s.listen(0, '127.0.0.1', () => resolve(s)); + }); + const port = srv.address().port; + const cfg = path.join(tmp, 'probes.json'); + fs.writeFileSync( + cfg, + JSON.stringify({ + failAfter: 1, + timeoutMs: 2000, + stateDir: tmp, + targets: [ + { id: 'ok', url: `http://127.0.0.1:${port}/ok` }, + { id: 'bad', url: `http://127.0.0.1:${port}/bad` }, + ], + }), + ); + const code = await new Promise((resolve) => { + const p = spawn(process.execPath, [path.join(root, 'src/watch.js'), '--once'], { + env: { ...process.env, UPTIME_PROBES: cfg, UPTIME_STATE: tmp }, + }); + p.on('close', resolve); + }); + srv.close(); + assert.equal(code, 2); + const snap = JSON.parse(fs.readFileSync(path.join(tmp, 'status.json'), 'utf8')); + assert.equal(snap.failed, 1); +}); diff --git a/packages/zapier-decisions/LOG.md b/packages/zapier-decisions/LOG.md index 68beee6..5f36d68 100644 --- a/packages/zapier-decisions/LOG.md +++ b/packages/zapier-decisions/LOG.md @@ -1,5 +1,13 @@ # Action log +## 2026-09-11 — proxmox worker + deploy/monitor/backup + +- NS1 **is** Proxmox (`vmbr0` public, `vmbr1` `10.10.10.0/24`). New repo **verae-proxmox** creates LXC **510** `verae-px-worker` at `10.10.10.20`. +- NATS for guests: socat `10.10.10.1:4222` → `127.0.0.1:4222` (never vmbr0). +- **verae-uptime** off-box HTTPS GET; **verae-backup** tar snapshots; **verae-deploy** tagged npm ci. +- Fleet `overlays/ns1/` checked in; `start.sh` no longer rewrites fleet/machines JSON. +- User systemd + linger for keep and fleet. + ## 2026-09-11 — bootstrap - Created this repo to record decisions while executing cleanup items 1–5 (and a second pass on 2–5). diff --git a/scripts/build-docs-site.py b/scripts/build-docs-site.py index aaf0a37..6af2b91 100755 --- a/scripts/build-docs-site.py +++ b/scripts/build-docs-site.py @@ -74,6 +74,10 @@ SECTIONS = [ ("packages/verae-staff-ui/README.md", "verae-staff-ui"), ("packages/verae-keep/README.md", "verae-keep"), ("packages/verae-bootstrap/README.md", "verae-bootstrap (clone + deps per server type)"), + ("packages/verae-uptime/README.md", "Off-box HTTPS door watch"), + ("packages/verae-backup/README.md", "Backup SQLite / JetStream / worm+tree"), + ("packages/verae-deploy/README.md", "Tagged deploy on the target OS"), + ("packages/verae-proxmox/README.md", "Proxmox LXC worker on NS1 vmbr1"), ], ), ( @@ -590,6 +594,10 @@ def main() -> None: "verae-staff-iam", "verae-keep", "verae-bootstrap", + "verae-uptime", + "verae-backup", + "verae-deploy", + "verae-proxmox", ): pkg_root = ROOT / "packages" / pkg if pkg in {"zapier-user-docs", "overview", "docs-master", "verae-ops"}: @@ -817,6 +825,10 @@ def main() -> None: ("verae-staff-iam", "Staff users, roles, permissions"), ("verae-keep", "Host keep-alive"), ("verae-bootstrap", "Clone + deps per server type"), + ("verae-uptime", "Off-box HTTPS door watch"), + ("verae-backup", "Backup SQLite / JetStream / worm+tree"), + ("verae-deploy", "Tagged deploy on the target OS"), + ("verae-proxmox", "Proxmox LXC worker (NS1 vmbr1)"), ] git_lis = "".join( f'
  • {name} ' diff --git a/scripts/push-module-repos.sh b/scripts/push-module-repos.sh index e2868de..3c2a01c 100755 --- a/scripts/push-module-repos.sh +++ b/scripts/push-module-repos.sh @@ -70,6 +70,10 @@ create verae-staff-ui "Shared staff review HTML template" create verae-staff-iam "Internal staff users, roles, and permissions" create verae-keep "Host keep-alive: restart units unless admin paused or stopped them" create verae-bootstrap "Clone all module repos and install deps for each server type" +create verae-uptime "Off-box HTTPS health watch for public doors" +create verae-backup "Backup/restore SQLite, JetStream, worm/tree, keep, IAM" +create verae-deploy "Tagged checkout + npm ci on the target OS" +create verae-proxmox "Provision archive-worker LXC on NS1 Proxmox" push_dir "$ROOT/packages/zappier" zappier-edge push_dir "$ROOT/packages/verae-zapier-middleware" verae-middleware @@ -106,5 +110,9 @@ push_dir "$ROOT/packages/verae-staff-ui" verae-staff-ui push_dir "$ROOT/packages/verae-staff-iam" verae-staff-iam push_dir "$ROOT/packages/verae-keep" verae-keep push_dir "$ROOT/packages/verae-bootstrap" verae-bootstrap +push_dir "$ROOT/packages/verae-uptime" verae-uptime +push_dir "$ROOT/packages/verae-backup" verae-backup +push_dir "$ROOT/packages/verae-deploy" verae-deploy +push_dir "$ROOT/packages/verae-proxmox" verae-proxmox echo ALL_MODULE_REPOS_PUSHED