Proxmox worker, uptime, backup, tagged deploy

NATS.md

@@ -0,0 +1,3 @@
+# NATS — verae-backup
+
+Backup copies the JetStream **store directory** if present. It does not publish or subscribe. Restore JetStream only with nats-server stopped.

README.md

@@ -0,0 +1,23 @@
+# verae-backup
+
+Snapshot and restore **zappier SQLite**, staff IAM data, keep state, fleet-runtime (worm/tree blobs), and JetStream store if present.
+
+**Forgejo:** https://git.georgelambert.org/marchon/verae-backup  
+**SSH:** `ssh://git@git.georgelambert.org:2223/marchon/verae-backup.git`
+
+```bash
+# on NS1
+export BACKUP_ROOT=/SSD2/backups/verae
+bash scripts/backup.sh
+bash scripts/restore.sh --dry-run
+# after stopping keep/fleet consumers:
+# bash scripts/restore.sh --apply --archive /SSD2/backups/verae/20260101T031700Z.tgz
+```
+
+Paths: `paths.env`. Missing sources are skipped. Default retain 14 tarballs.
+
+User timer: `systemd/verae-backup.timer` (03:17 UTC). Enable with linger.
+
+Does **not** back up private keys. Restore identityFile **paths** on the operator host by hand.
+
+Does **not** connect to NATS; it copies the JetStream directory if `NATS_STORE` exists.

SUMMARY.md

@@ -0,0 +1,3 @@
+# verae-backup
+
+Tar snapshots of SQLite, IAM, keep, worm/tree runtime, optional JetStream dir. Dry-run restore by default.

package.json

@@ -0,0 +1,11 @@
+{
+  "name": "verae-backup",
+  "version": "0.1.0",
+  "private": true,
+  "description": "Backup and restore SQLite, JetStream, worm/tree, keep, and IAM data",
+  "scripts": {
+    "backup": "bash scripts/backup.sh",
+    "restore": "bash scripts/restore.sh",
+    "test": "bash scripts/test.sh"
+  }
+}

paths.env

@@ -0,0 +1,16 @@
+# Sourced by backup.sh / restore.sh. Override in the environment.
+BACKUP_ROOT="${BACKUP_ROOT:-/SSD2/backups/verae}"
+KEEP=14
+
+# SQLite / IAM
+ZAPPIER_DB="${ZAPPIER_DB:-$HOME/verae-stack/packages/zappier/zappier.db}"
+IAM_DATA="${IAM_DATA:-$HOME/verae-stack/packages/verae-staff-iam/data}"
+
+# Keep + fleet runtime (worm/tree blobs live under fleet-runtime data/)
+KEEP_DATA="${KEEP_DATA:-$HOME/verae-keep/data}"
+FLEET_RUNTIME="${FLEET_RUNTIME:-$HOME/verae-fleet-runtime}"
+FLEET_PKG_DATA="${FLEET_PKG_DATA:-$HOME/verae-stack/packages/verae-fleet/data}"
+
+# JetStream (docker or nats-server). Missing paths are skipped.
+NATS_STORE="${NATS_STORE:-/var/lib/nats/jetstream}"
+NATS_DOCKER_VOL="${NATS_DOCKER_VOL:-}"

scripts/backup.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+# Snapshot Verae data dirs into $BACKUP_ROOT/<utc-stamp>.tgz
+set -euo pipefail
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+# shellcheck disable=SC1091
+. "$ROOT/paths.env"
+STAMP="$(date -u +%Y%m%dT%H%M%SZ)"
+DEST="$BACKUP_ROOT/$STAMP"
+mkdir -p "$DEST"
+copy_if() {
+  local src="$1" name="$2"
+  if [[ -e "$src" ]]; then
+    mkdir -p "$DEST/$name"
+    cp -a "$src" "$DEST/$name/" 2>/dev/null || sudo cp -a "$src" "$DEST/$name/"
+    echo "copied $src"
+  else
+    echo "skip missing $src"
+  fi
+}
+copy_if "$ZAPPIER_DB" sqlite
+copy_if "$IAM_DATA" iam
+copy_if "$KEEP_DATA" keep
+copy_if "$FLEET_RUNTIME" fleet-runtime
+copy_if "$FLEET_PKG_DATA" fleet-pkg
+copy_if "$NATS_STORE" nats
+if [[ -n "${NATS_DOCKER_VOL:-}" ]]; then copy_if "$NATS_DOCKER_VOL" nats-docker; fi
+tar -C "$BACKUP_ROOT" -czf "$BACKUP_ROOT/$STAMP.tgz" "$STAMP"
+rm -rf "$DEST"
+# retain
+ls -1dt "$BACKUP_ROOT"/*.tgz 2>/dev/null | tail -n +$((KEEP + 1)) | while read -r old; do
+  rm -f "$old"
+done
+echo "backup $BACKUP_ROOT/$STAMP.tgz"
+echo "$BACKUP_ROOT/$STAMP.tgz"

scripts/restore.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# Restore a .tgz from backup.sh. Default is --dry-run.
+set -euo pipefail
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+# shellcheck disable=SC1091
+. "$ROOT/paths.env"
+DRY=1
+ARCHIVE=""
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --apply) DRY=0 ;;
+    --dry-run) DRY=1 ;;
+    --archive) ARCHIVE="${2:-}"; shift ;;
+    *) echo "usage: restore.sh [--dry-run|--apply] [--archive file.tgz]" >&2; exit 1 ;;
+  esac
+  shift
+done
+if [[ -z "$ARCHIVE" ]]; then
+  ARCHIVE="$(ls -1t "$BACKUP_ROOT"/*.tgz 2>/dev/null | head -1 || true)"
+fi
+[[ -n "$ARCHIVE" && -f "$ARCHIVE" ]] || { echo "no archive" >&2; exit 1; }
+echo "archive $ARCHIVE dry=$DRY"
+TMP="$(mktemp -d)"
+tar -xzf "$ARCHIVE" -C "$TMP"
+INNER="$(find "$TMP" -mindepth 1 -maxdepth 1 -type d | head -1)"
+restore_one() {
+  local from="$1" to="$2"
+  [[ -e "$from" ]] || { echo "skip $from"; return; }
+  echo "would restore $from -> $to"
+  if [[ $DRY -eq 0 ]]; then
+    mkdir -p "$(dirname "$to")"
+    rm -rf "$to"
+    cp -a "$from" "$to"
+  fi
+}
+restore_one "$INNER/sqlite/$(basename "$ZAPPIER_DB")" "$ZAPPIER_DB"
+restore_one "$INNER/iam/$(basename "$IAM_DATA")" "$IAM_DATA"
+restore_one "$INNER/keep/$(basename "$KEEP_DATA")" "$KEEP_DATA"
+restore_one "$INNER/fleet-runtime/$(basename "$FLEET_RUNTIME")" "$FLEET_RUNTIME"
+restore_one "$INNER/nats/$(basename "$NATS_STORE")" "$NATS_STORE"
+rm -rf "$TMP"
+if [[ $DRY -eq 1 ]]; then
+  echo "dry-run only. re-run with --apply after stopping keep/fleet/nats consumers."
+fi

scripts/test.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+set -euo pipefail
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+bash -n "$ROOT/scripts/backup.sh"
+bash -n "$ROOT/scripts/restore.sh"
+TMP="$(mktemp -d)"
+export BACKUP_ROOT="$TMP/out"
+export ZAPPIER_DB="$TMP/zappier.db"
+export IAM_DATA="$TMP/iam"
+export KEEP_DATA="$TMP/keep"
+export FLEET_RUNTIME="$TMP/runtime"
+export FLEET_PKG_DATA="$TMP/fleet-pkg"
+export NATS_STORE="$TMP/missing-nats"
+echo hello >"$ZAPPIER_DB"
+mkdir -p "$IAM_DATA" "$KEEP_DATA" "$FLEET_RUNTIME" "$FLEET_PKG_DATA"
+echo x >"$KEEP_DATA/a"
+# shellcheck disable=SC1091
+set +u
+BACKUP_ROOT="$BACKUP_ROOT" KEEP=3 ZAPPIER_DB="$ZAPPIER_DB" IAM_DATA="$IAM_DATA" KEEP_DATA="$KEEP_DATA" FLEET_RUNTIME="$FLEET_RUNTIME" FLEET_PKG_DATA="$FLEET_PKG_DATA" NATS_STORE="$NATS_STORE" \
+  bash "$ROOT/scripts/backup.sh" >/tmp/verae-backup-test.out
+test -s /tmp/verae-backup-test.out
+ls "$BACKUP_ROOT"/*.tgz >/dev/null
+BACKUP_ROOT="$BACKUP_ROOT" ZAPPIER_DB="$ZAPPIER_DB" IAM_DATA="$IAM_DATA" KEEP_DATA="$KEEP_DATA" FLEET_RUNTIME="$FLEET_RUNTIME" FLEET_PKG_DATA="$FLEET_PKG_DATA" NATS_STORE="$NATS_STORE" \
+  bash "$ROOT/scripts/restore.sh" --dry-run >/dev/null
+rm -rf "$TMP"
+echo OK

systemd/verae-backup.service

@@ -0,0 +1,7 @@
+[Unit]
+Description=Verae data backup (sqlite, keep, worm/tree, jetstream)
+
+[Service]
+Type=oneshot
+WorkingDirectory=%h/verae-src/verae-backup
+ExecStart=/usr/bin/bash %h/verae-src/verae-backup/scripts/backup.sh

systemd/verae-backup.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=Nightly Verae data backup
+
+[Timer]
+OnCalendar=*-*-* 03:17:00
+Persistent=true
+
+[Install]
+WantedBy=timers.target