From 00310350481f4a484a383f0662f0a6fbd6c60649 Mon Sep 17 00:00:00 2001 From: George Lambert Date: Fri, 11 Sep 2026 23:12:38 -0400 Subject: [PATCH] Host-deps installer and catalog hostname procedure --- HOST-DEPS.md | 63 ++++++ NATS.md | 3 + README.md | 52 +++++ SUMMARY.md | 3 + host-deps.tsv | 40 ++++ package.json | 13 ++ repos.txt | 35 ++++ scripts/clone.sh | 22 +++ scripts/fetch.sh | 18 ++ scripts/host-deps.sh | 348 +++++++++++++++++++++++++++++++++ scripts/install-deps.sh | 16 ++ scripts/test.sh | 20 ++ types/control-plane/README.md | 5 + types/control-plane/repos.txt | 23 +++ types/lan-worker/README.md | 10 + types/lan-worker/repos.txt | 5 + types/ns1-all-in-one/README.md | 29 +++ types/ns1-all-in-one/repos.txt | 27 +++ types/ns1-all-in-one/start.sh | 85 ++++++++ types/ns1-archive/README.md | 5 + types/ns1-archive/repos.txt | 8 + 21 files changed, 830 insertions(+) create mode 100644 HOST-DEPS.md create mode 100644 NATS.md create mode 100644 README.md create mode 100644 SUMMARY.md create mode 100644 host-deps.tsv create mode 100644 package.json create mode 100644 repos.txt create mode 100755 scripts/clone.sh create mode 100755 scripts/fetch.sh create mode 100755 scripts/host-deps.sh create mode 100755 scripts/install-deps.sh create mode 100755 scripts/test.sh create mode 100644 types/control-plane/README.md create mode 100644 types/control-plane/repos.txt create mode 100644 types/lan-worker/README.md create mode 100644 types/lan-worker/repos.txt create mode 100644 types/ns1-all-in-one/README.md create mode 100644 types/ns1-all-in-one/repos.txt create mode 100755 types/ns1-all-in-one/start.sh create mode 100644 types/ns1-archive/README.md create mode 100644 types/ns1-archive/repos.txt diff --git a/HOST-DEPS.md b/HOST-DEPS.md new file mode 100644 index 0000000..b78b968 --- /dev/null +++ b/HOST-DEPS.md @@ -0,0 +1,63 @@ +# Host dependencies (`scripts/host-deps.sh`) + +Every Forgejo module on `git.georgelambert.org` is listed in `host-deps.tsv`. This script **checks** that the current machine has the OS packages, Node, and (when this host runs JetStream) `nats-server`. Missing pieces are **downloaded and installed** for the OS it is running on. + +| OS family | Package tool | Node | nats-server | +|-----------|--------------|------|-------------| +| Debian / Ubuntu | `apt-get` | official `linux-x64` / `linux-arm64` tarball into `$PREFIX` | GitHub release tarball | +| Alpine | `apk` | `apk add nodejs` then musl tarball if still too old | `apk add nats-server` or glibc release (last resort) | +| macOS | Homebrew | already on PATH or brew | brew / skip | + +`$PREFIX` is `/usr/local` as root, otherwise `~/.local`. Put `$PREFIX/bin` on `PATH`. + +Do **not** copy `node_modules` from macOS onto Linux (`better-sqlite3` → `invalid ELF header`). Run `npm install` **on the target OS** (`scripts/install-deps.sh`). + +Zapier cloud never talks to NATS. `nats-server` binds **127.0.0.1:4222** only on `ns1-all-in-one` and `ns1-archive`. + +## Commands + +```bash +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-bootstrap.git +cd verae-bootstrap + +# What this box still needs (no changes) +bash scripts/host-deps.sh --check --type ns1-all-in-one + +# Install git, curl, Node 22, build tools, nats-server +bash scripts/host-deps.sh --type ns1-all-in-one + +# Single module (e.g. a thin Alpine VM that only runs worm) +bash scripts/host-deps.sh --module verae-archive-worm + +# Catalog PDF/HTML build tools (pandoc, weasyprint, pypdf) +bash scripts/host-deps.sh --docs --module zapier-docs-master + +# Then clone + npm +export VERAE_SRC=$HOME/verae-src +bash scripts/fetch.sh ns1-all-in-one +``` + +`scripts/fetch.sh` runs `host-deps.sh` first so Git/Node exist before clone. + +## Module → host needs + +| Module (Forgejo) | Node | npm | native C++ | NATS client | Notes | +|------------------|------|-----|------------|-------------|--------| +| zappier-edge | 20 | yes | **better-sqlite3** | yes | Rebuild sqlite on Linux | +| verae-middleware | **22** | yes | | yes | | +| verae-zapier-app / verae-activate | 18 | local zapier CLI only | | | Runs on Zapier cloud | +| archive-worm / tree-node / aggregator / splitter | 20 | yes | | yes | | +| fleet | 20 | yes | | | `openssh-client` | +| keep, IAM, session, access-* , identity, jobs-events, billing NATS | 20 | yes | | most | | +| overview, docs-master, user-docs, decisions, nats-accounts, ops | — | | | | Docs only (`--docs` for catalog build) | + +Full table: `host-deps.tsv`. + +## Server types and nats-server + +| Type | `nats-server` on this host? | +|------|------------------------------| +| ns1-all-in-one | yes (`--nats` implied) | +| ns1-archive | yes | +| control-plane | no (client to NS1 loopback / private URL) | +| lan-worker | no (client; lan-134 stays disabled until SSH works) | diff --git a/NATS.md b/NATS.md new file mode 100644 index 0000000..bcff40b --- /dev/null +++ b/NATS.md @@ -0,0 +1,3 @@ +# NATS — verae-bootstrap + +Bootstrap clones git repos and runs `npm install`. It does not open NATS ports. NATS is a **runtime** dependency of the `ns1-archive` and `ns1-all-in-one` server types (`127.0.0.1:4222` only). diff --git a/README.md b/README.md new file mode 100644 index 0000000..335e791 --- /dev/null +++ b/README.md @@ -0,0 +1,52 @@ +# verae-bootstrap + +Clone every Verae Time × Zapier **module repo** from Forgejo and install Node dependencies. Each **server type** has its own directory under `types/` (repos list + README + start notes). + +**Forgejo:** https://git.georgelambert.org/marchon/verae-bootstrap +**SSH:** `ssh://git@git.georgelambert.org:2223/marchon/verae-bootstrap.git` + +## Host requirements + +`scripts/host-deps.sh` **checks** this machine and **installs** anything missing (Debian/Ubuntu `apt`, Alpine `apk`, Node tarball, `nats-server` release). Table: [HOST-DEPS.md](HOST-DEPS.md). Manifest: `host-deps.tsv`. + +```bash +bash scripts/host-deps.sh --check --type ns1-all-in-one +bash scripts/host-deps.sh --type ns1-all-in-one # install +``` + +| Need | Why | +|------|-----| +| Node 20+ (`node`, `npm`); middleware wants **22** | All HTTP and worker processes | +| Git + SSH to Forgejo **port 2223** | `git clone ssh://git@git.georgelambert.org:2223/marchon/.git` | +| `python3` `make` `g++` | Native addon `better-sqlite3` on **zappier-edge** (rebuild on Linux) | +| `nats-server -js` on **127.0.0.1:4222** | Archive / all-in-one types only — never a public bind | +| `curl` | Health probes | + +Alpine first line if `bash` is missing: `apk add --no-cache bash git curl`. Zapier cloud never talks to NATS. Do not publish `:4222`. + +## Fetch + +```bash +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-bootstrap.git +cd verae-bootstrap +export VERAE_SRC=$HOME/verae-src +bash scripts/fetch.sh # every repo +bash scripts/fetch.sh ns1-all-in-one # one server type +``` + +`scripts/fetch.sh` runs `host-deps.sh` (OS + Node), then `clone.sh` (`main`), then `install-deps.sh` (`npm install` **on the target OS**, then `npm rebuild`). Do not copy `node_modules` from macOS onto Linux — native addons such as `better-sqlite3` will fail with `invalid ELF header`. + +## Server types + +| Type | Directory | What that host runs | +|------|-----------|---------------------| +| **ns1-all-in-one** | `types/ns1-all-in-one/` | Control plane + archive workers + keep (NS1 / 70.88.205.138) | +| **control-plane** | `types/control-plane/` | Fleet, edges, IAM, billing HTTP (no worm/tree floor) | +| **ns1-archive** | `types/ns1-archive/` | NATS + worm ×3 + tree-node ×3 + poller + webhook + aggregator + keep | +| **lan-worker** | `types/lan-worker/` | Extra SSH worker (tree-node, archive-worm). Keep disabled until SSH works (`lan-134`). | + +Each type has `repos.txt` (subset of the master list) and `README.md`. + +## NS1 all-in-one notes + +`:3000` on 70.88.205.138 is already a public app. zappier-edge on that box binds **127.0.0.1:13000**. Public portal remains **access-web `:3021/portal/`**. Archive workers are owned by **verae-keep**; fleet does not double-spawn them. diff --git a/SUMMARY.md b/SUMMARY.md new file mode 100644 index 0000000..2effcc0 --- /dev/null +++ b/SUMMARY.md @@ -0,0 +1,3 @@ +# verae-bootstrap + +Downloads every Forgejo module and Node dependencies. Server types (each documented under `types/`): `ns1-all-in-one`, `control-plane`, `ns1-archive`, `lan-worker`. diff --git a/host-deps.tsv b/host-deps.tsv new file mode 100644 index 0000000..13d2512 --- /dev/null +++ b/host-deps.tsv @@ -0,0 +1,40 @@ +# Forgejo module | min Node | flags (comma) +# flags: npm, native (C++ addon / better-sqlite3), nats (NATS client), +# nats_server (this host runs nats-server), ssh, docs (catalog build) +# Zapier Platform apps run in Zapier cloud; npm here is only for local zapier build. +zappier-edge|20|npm,native,nats +verae-middleware|22|npm,nats +verae-zapier-app|18|npm +verae-activate|18|npm +verae-request-splitter|20|npm,nats +verae-archive-worm|20|npm,nats +verae-archive-aggregator|20|npm,nats +verae-tree-node|20|npm,nats +verae-zapier-simulator|20|npm +zapier-user-docs|0| +verae-fleet|20|npm,ssh +overview|0| +verae-nats-process|20|npm,nats +verae-ops|0| +zappier-account-balance|20|npm,nats +zappier-customer-service|20|npm,nats +zappier-sales-pricing|20|npm,nats +zappier-accounting-export|20|npm,nats +verae-access-authz|20|npm,nats +verae-access-web|20|npm,nats +verae-access-api|20|npm,nats +verae-access-leaf|20|npm,nats +verae-access-zapier|20|npm,nats +verae-access-staff|20|npm,nats +zappier-identity|20|npm,nats +verae-jobs-events|20|npm,nats +verae-nats-accounts|0| +zapier-decisions|0| +UI-Docs|20|npm,docs +verae-staff-session|20|npm +verae-staff-ui|20|npm +verae-staff-iam|20|npm +verae-keep|20|npm +verae-bootstrap|0| +zapier-docs-master|0|docs +master-zapier-plan-draft|22|npm,native,nats,docs diff --git a/package.json b/package.json new file mode 100644 index 0000000..2426c1d --- /dev/null +++ b/package.json @@ -0,0 +1,13 @@ +{ + "name": "verae-bootstrap", + "version": "0.1.0", + "private": true, + "description": "Clone Verae Time × Zapier git repos and install dependencies for each server type", + "scripts": { + "clone": "bash scripts/clone.sh", + "deps": "bash scripts/install-deps.sh", + "host-deps": "bash scripts/host-deps.sh", + "fetch": "bash scripts/fetch.sh", + "test": "bash scripts/test.sh" + } +} diff --git a/repos.txt b/repos.txt new file mode 100644 index 0000000..3a6b836 --- /dev/null +++ b/repos.txt @@ -0,0 +1,35 @@ +zappier-edge +verae-middleware +verae-zapier-app +verae-activate +verae-request-splitter +verae-archive-worm +verae-archive-aggregator +zapier-docs-master +verae-tree-node +verae-zapier-simulator +zapier-user-docs +verae-fleet +overview +verae-nats-process +verae-ops +zappier-account-balance +zappier-customer-service +zappier-sales-pricing +zappier-accounting-export +verae-access-authz +verae-access-web +verae-access-api +verae-access-leaf +verae-access-zapier +verae-access-staff +zappier-identity +verae-jobs-events +verae-nats-accounts +zapier-decisions +UI-Docs +verae-staff-session +verae-staff-ui +verae-staff-iam +verae-keep +verae-bootstrap diff --git a/scripts/clone.sh b/scripts/clone.sh new file mode 100755 index 0000000..ffa0cb1 --- /dev/null +++ b/scripts/clone.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Clone Forgejo modules listed in a repos file. +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +GITSSH="${GITSSH:-ssh://git@git.georgelambert.org:2223/marchon}" +DEST="${VERAE_SRC:-$HOME/verae-src}" +LIST="${1:-$ROOT/repos.txt}" +export GIT_SSH_COMMAND="${GIT_SSH_COMMAND:-ssh -o StrictHostKeyChecking=accept-new -p 2223}" +mkdir -p "$DEST" +while read -r name; do + [[ -z "$name" || "$name" == \#* ]] && continue + if [[ -d "$DEST/$name/.git" ]]; then + echo "fetch $name" + git -C "$DEST/$name" fetch --prune origin + git -C "$DEST/$name" checkout -q main 2>/dev/null || git -C "$DEST/$name" checkout -q master + git -C "$DEST/$name" pull --ff-only || true + else + echo "clone $name" + git clone "$GITSSH/${name}.git" "$DEST/$name" + fi +done < "$LIST" +echo "cloned into $DEST" diff --git a/scripts/fetch.sh b/scripts/fetch.sh new file mode 100755 index 0000000..915ea3c --- /dev/null +++ b/scripts/fetch.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +TYPE="${1:-}" +LIST="$ROOT/repos.txt" +if [[ -n "$TYPE" ]]; then + LIST="$ROOT/types/$TYPE/repos.txt" + [[ -f "$LIST" ]] || { echo "unknown type $TYPE" >&2; exit 1; } +fi +# OS packages + Node (+ nats-server for archive types) before clone. +if [[ -n "$TYPE" ]]; then + bash "$ROOT/scripts/host-deps.sh" --type "$TYPE" +else + bash "$ROOT/scripts/host-deps.sh" +fi +bash "$ROOT/scripts/clone.sh" "$LIST" +bash "$ROOT/scripts/install-deps.sh" +echo "fetch done type=${TYPE:-all}" diff --git a/scripts/host-deps.sh b/scripts/host-deps.sh new file mode 100755 index 0000000..1b433ea --- /dev/null +++ b/scripts/host-deps.sh @@ -0,0 +1,348 @@ +#!/usr/bin/env bash +# Check (and optionally install) OS + Node + NATS packages needed to run +# Verae Time × Zapier modules on this host. +# +# Debian/Ubuntu and Alpine are first-class. Other Linux: print the plan. +# macOS: Homebrew when present. +# +# Usage: +# bash scripts/host-deps.sh # detect modules, install missing +# bash scripts/host-deps.sh --check # report only +# bash scripts/host-deps.sh --type ns1-all-in-one +# bash scripts/host-deps.sh --module zappier-edge +# bash scripts/host-deps.sh --nats # also install nats-server +# bash scripts/host-deps.sh --docs # pandoc / weasyprint / pypdf +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +MANIFEST="$ROOT/host-deps.tsv" +NODE_VER="${NODE_VER:-22.14.0}" +NATS_VER="${NATS_VER:-2.10.26}" +PREFIX="${PREFIX:-}" +CHECK_ONLY=0 +WANT_NATS=0 +WANT_DOCS=0 +WANT_CADDY=0 +TYPE="" +MODULES=() + +usage() { sed -n '2,16p' "$0" | tr -d '#'; exit "${1:-0}"; } + +while [[ $# -gt 0 ]]; do + case "$1" in + --check) CHECK_ONLY=1 ;; + --install) CHECK_ONLY=0 ;; + --nats) WANT_NATS=1 ;; + --docs) WANT_DOCS=1 ;; + --caddy) WANT_CADDY=1 ;; + --type) TYPE="${2:-}"; shift ;; + --module) MODULES+=("${2:-}"); shift ;; + --prefix) PREFIX="${2:-}"; shift ;; + -h|--help) usage 0 ;; + *) echo "unknown arg $1" >&2; usage 1 ;; + esac + shift +done + +# --- OS / arch --- +OS_ID="unknown" +OS_FAMILY="unknown" +if [[ "$(uname -s)" == Darwin ]]; then + OS_ID=macos + OS_FAMILY=macos +elif [[ -f /etc/os-release ]]; then + # shellcheck disable=SC1091 + . /etc/os-release + OS_ID="${ID:-unknown}" + case "$OS_ID" in + debian|ubuntu|raspbian|linuxmint) OS_FAMILY=debian ;; + alpine) OS_FAMILY=alpine ;; + *) OS_FAMILY=linux ;; + esac +fi +ARCH="$(uname -m)" +case "$ARCH" in + x86_64|amd64) NODE_ARCH=x64; NATS_ARCH=amd64; DEB_ARCH=amd64 ;; + aarch64|arm64) NODE_ARCH=arm64; NATS_ARCH=arm64; DEB_ARCH=arm64 ;; + armv7l) NODE_ARCH=armv7l; NATS_ARCH=arm7; DEB_ARCH=armhf ;; + *) NODE_ARCH=$ARCH; NATS_ARCH=$ARCH; DEB_ARCH=$ARCH ;; +esac +MUSL=0 +if [[ "$OS_FAMILY" == alpine ]]; then MUSL=1; fi + +is_root() { [[ "$(id -u)" -eq 0 ]]; } +run_root() { + if is_root; then + "$@" + elif command -v sudo >/dev/null; then + sudo "$@" + else + echo "need root (or sudo) to run: $*" >&2 + return 1 + fi +} + +if [[ -z "$PREFIX" ]]; then + if is_root; then PREFIX=/usr/local; else PREFIX="${HOME}/.local"; fi +fi +mkdir -p "$PREFIX/bin" +export PATH="$PREFIX/bin:$PATH" + +# --- module list --- +PKG_TO_REPO="$(cat <<'EOF' +zappier zappier-edge +verae-zapier-middleware verae-middleware +verae-zapier verae-zapier-app +docs-master zapier-docs-master +ui-docs UI-Docs +EOF +)" + +map_name() { + local n="$1" + local line + while read -r from to; do + [[ "$n" == "$from" ]] && { echo "$to"; return; } + done <<<"$PKG_TO_REPO" + echo "$n" +} + +collect_from_list() { + local f="$1" + while read -r name; do + [[ -z "$name" || "$name" == \#* ]] && continue + MODULES+=("$name") + done < "$f" +} + +if [[ ${#MODULES[@]} -eq 0 ]]; then + if [[ -n "$TYPE" ]]; then + collect_from_list "$ROOT/types/$TYPE/repos.txt" + elif [[ -n "${VERAE_SRC:-}" && -d "${VERAE_SRC}" ]]; then + for d in "$VERAE_SRC"/*/; do + [[ -d "$d" ]] || continue + MODULES+=("$(basename "$d")") + done + elif [[ -d "$PWD/packages" ]]; then + for d in "$PWD"/packages/*/; do + MODULES+=("$(map_name "$(basename "$d")")") + done + MODULES+=(master-zapier-plan-draft) + elif [[ -f "$PWD/package.json" ]]; then + n="$(basename "$PWD")" + MODULES+=("$(map_name "$n")") + else + collect_from_list "$ROOT/repos.txt" + fi +fi + +# types that run nats-server on this box +case "$TYPE" in + ns1-all-in-one|ns1-archive) WANT_NATS=1 ;; +esac + +# --- union flags --- +MIN_NODE=0 +NEED_NPM=0 +NEED_NATIVE=0 +NEED_NATS_CLIENT=0 +NEED_SSH=0 +NEED_DOCS=$WANT_DOCS + +while IFS='|' read -r name min flags; do + [[ -z "$name" || "$name" == \#* ]] && continue + wanted=0 + for m in "${MODULES[@]}"; do + [[ "$m" == "$name" ]] && wanted=1 + done + [[ $wanted -eq 1 ]] || continue + if [[ "$min" != 0 && "$min" -gt "$MIN_NODE" ]]; then MIN_NODE=$min; fi + [[ ",$flags," == *",npm,"* ]] && NEED_NPM=1 + [[ ",$flags," == *",native,"* ]] && NEED_NATIVE=1 + [[ ",$flags," == *",nats,"* ]] && NEED_NATS_CLIENT=1 + [[ ",$flags," == *",nats_server,"* ]] && WANT_NATS=1 + [[ ",$flags," == *",ssh,"* ]] && NEED_SSH=1 + [[ ",$flags," == *",docs,"* ]] && NEED_DOCS=1 +done < "$MANIFEST" + +echo "os=$OS_ID family=$OS_FAMILY arch=$ARCH musl=$MUSL prefix=$PREFIX" +echo "modules: ${MODULES[*]}" +echo "need: node>=${MIN_NODE:-0} npm=$NEED_NPM native=$NEED_NATIVE nats_client=$NEED_NATS_CLIENT nats_server=$WANT_NATS ssh=$NEED_SSH docs=$NEED_DOCS" + +MISSING=() +have() { command -v "$1" >/dev/null 2>&1; } + +need_cmd() { + local c="$1" + if have "$c"; then + echo "ok $c $($c --version 2>/dev/null | head -1 || true)" + else + echo "NO $c" + MISSING+=("$c") + fi +} + +node_ok() { + local want="$1" + have node || return 1 + node -e "process.exit(parseInt(process.versions.node,10)>=$want?0:1)" 2>/dev/null +} + +# --- package installers --- +apt_install() { + run_root apt-get update -qq + run_root env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends "$@" +} +apk_install() { run_root apk add --no-cache "$@"; } +brew_install() { brew install "$@"; } + +install_pkgs() { + local pkgs=("$@") + [[ ${#pkgs[@]} -eq 0 ]] && return 0 + echo "install OS packages: ${pkgs[*]}" + [[ $CHECK_ONLY -eq 1 ]] && return 0 + case "$OS_FAMILY" in + debian) apt_install "${pkgs[@]}" ;; + alpine) apk_install "${pkgs[@]}" ;; + macos) brew_install "${pkgs[@]}" ;; + *) echo "install these packages yourself: ${pkgs[*]}" >&2; return 1 ;; + esac +} + +install_node() { + local want="${1:-20}" + if node_ok "$want"; then + echo "ok node $(node -v) (>=$want)" + return 0 + fi + echo "NO node (>=$want) have=$(node -v 2>/dev/null || echo none)" + MISSING+=("node>=$want") + [[ $CHECK_ONLY -eq 1 ]] && return 0 + if [[ "$OS_FAMILY" == alpine && $MUSL -eq 1 ]]; then + apk_install nodejs npm python3 make g++ linux-headers || true + if node_ok "$want"; then echo "ok node $(node -v) (apk)"; return 0; fi + fi + if [[ "$OS_FAMILY" == debian ]]; then + apt_install ca-certificates curl xz-utils + elif [[ "$OS_FAMILY" == alpine ]]; then + apk_install curl tar xz + fi + local dist="linux-${NODE_ARCH}" + [[ $MUSL -eq 1 ]] && dist="linux-${NODE_ARCH}-musl" + local base="https://nodejs.org/dist/v${NODE_VER}" + [[ $MUSL -eq 1 ]] && base="https://unofficial-builds.nodejs.org/download/release/v${NODE_VER}" + local tar="node-v${NODE_VER}-${dist}.tar.xz" + local tmp + tmp="$(mktemp -d)" + echo "download $base/$tar" + curl -fsSL "$base/$tar" -o "$tmp/$tar" + tar -xJf "$tmp/$tar" -C "$tmp" + mkdir -p "$PREFIX" + run_root cp -a "$tmp/node-v${NODE_VER}-${dist}/." "$PREFIX/" + rm -rf "$tmp" + hash -r + node_ok "$want" || { echo "node install failed" >&2; return 1; } + echo "ok node $(node -v) → $PREFIX" +} + +install_nats() { + if have nats-server; then + echo "ok nats-server $(nats-server -v 2>/dev/null | head -1 || true)" + return 0 + fi + echo "NO nats-server" + MISSING+=("nats-server") + [[ $CHECK_ONLY -eq 1 ]] && return 0 + if [[ "$OS_FAMILY" == alpine ]]; then + apk_install nats-server 2>/dev/null && { echo "ok nats-server (apk)"; return 0; } || true + fi + local tar="nats-server-v${NATS_VER}-linux-${NATS_ARCH}.tar.gz" + local url="https://github.com/nats-io/nats-server/releases/download/v${NATS_VER}/${tar}" + local tmp + tmp="$(mktemp -d)" + echo "download $url" + curl -fsSL "$url" -o "$tmp/$tar" + tar -xzf "$tmp/$tar" -C "$tmp" + run_root install -m 0755 "$tmp/nats-server-v${NATS_VER}-linux-${NATS_ARCH}/nats-server" "$PREFIX/bin/nats-server" + rm -rf "$tmp" + have nats-server || { echo "nats-server install failed" >&2; return 1; } + echo "ok nats-server → $PREFIX/bin" +} + +# --- plan OS packages --- +PKGS=() +add_pkg() { PKGS+=("$1"); } + +case "$OS_FAMILY" in + debian) + add_pkg ca-certificates; add_pkg curl; add_pkg git; add_pkg bash; add_pkg tar; add_pkg xz-utils + [[ $NEED_NATIVE -eq 1 || $NEED_NPM -eq 1 ]] && { add_pkg python3; add_pkg make; add_pkg g++; add_pkg python3-dev; } + [[ $NEED_SSH -eq 1 ]] && add_pkg openssh-client + [[ $NEED_DOCS -eq 1 ]] && { add_pkg pandoc; add_pkg weasyprint; add_pkg python3-pip; } + [[ $WANT_CADDY -eq 1 ]] && add_pkg debian-keyring + ;; + alpine) + add_pkg ca-certificates; add_pkg curl; add_pkg git; add_pkg bash; add_pkg tar; add_pkg xz + [[ $NEED_NATIVE -eq 1 || $NEED_NPM -eq 1 ]] && { add_pkg python3; add_pkg make; add_pkg g++; add_pkg linux-headers; add_pkg libstdc++; } + [[ $NEED_SSH -eq 1 ]] && add_pkg openssh-client + [[ $NEED_DOCS -eq 1 ]] && { add_pkg pandoc; add_pkg py3-pip; add_pkg weasyprint; } + ;; + macos) + add_pkg git; add_pkg curl + [[ $NEED_DOCS -eq 1 ]] && { add_pkg pandoc; add_pkg weasyprint; } + ;; +esac + +echo "--- check ---" +need_cmd git +need_cmd curl +[[ $NEED_SSH -eq 1 ]] && need_cmd ssh +if [[ "$MIN_NODE" != 0 ]]; then + if node_ok "$MIN_NODE"; then echo "ok node $(node -v)"; else echo "NO node (>=$MIN_NODE)"; MISSING+=("node>=$MIN_NODE"); fi + need_cmd npm +fi +[[ $WANT_NATS -eq 1 ]] && { have nats-server && echo "ok nats-server" || { echo "NO nats-server"; MISSING+=("nats-server"); }; } +[[ $NEED_DOCS -eq 1 ]] && { need_cmd pandoc; have weasyprint || have weasyprint3 || echo "NO weasyprint"; } + +if [[ $CHECK_ONLY -eq 1 ]]; then + if [[ ${#MISSING[@]} -eq 0 ]]; then + echo "host deps OK" + exit 0 + fi + echo "missing: ${MISSING[*]}" + exit 2 +fi + +echo "--- install ---" +if [[ ${#PKGS[@]} -gt 0 ]]; then + UNIQ=() + while IFS= read -r p; do + [[ -n "$p" ]] && UNIQ+=("$p") + done < <(printf '%s\n' "${PKGS[@]}" | sort -u) + install_pkgs "${UNIQ[@]}" +fi +if [[ "$MIN_NODE" != 0 ]]; then + install_node "$MIN_NODE" +fi +if [[ $WANT_NATS -eq 1 ]]; then + install_nats +fi +if [[ $NEED_DOCS -eq 1 ]] && have python3; then + python3 -m pip install --user pypdf >/dev/null 2>&1 || true +fi + +MISSING=() +need_cmd git +need_cmd curl +if [[ "$MIN_NODE" != 0 ]]; then + node_ok "$MIN_NODE" || MISSING+=("node>=$MIN_NODE") + have npm || MISSING+=("npm") +fi +[[ $WANT_NATS -eq 1 ]] && { have nats-server || MISSING+=("nats-server"); } + +if [[ ${#MISSING[@]} -eq 0 ]]; then + echo "host deps OK (PATH should include $PREFIX/bin)" + exit 0 +fi +echo "still missing: ${MISSING[*]}" >&2 +exit 1 diff --git a/scripts/install-deps.sh b/scripts/install-deps.sh new file mode 100755 index 0000000..6d46eb1 --- /dev/null +++ b/scripts/install-deps.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# npm install in every cloned package that has package.json. +set -euo pipefail +DEST="${VERAE_SRC:-$HOME/verae-src}" +if [[ ! -d "$DEST" ]]; then + echo "missing $DEST — run scripts/clone.sh first" >&2 + exit 1 +fi +command -v node >/dev/null +command -v npm >/dev/null +find "$DEST" -maxdepth 2 -name package.json ! -path '*/node_modules/*' | while read -r pkg; do + dir="$(dirname "$pkg")" + echo "npm install $dir" + (cd "$dir" && npm install --no-audit --no-fund && npm rebuild --no-audit --no-fund >/dev/null) +done +echo "deps installed under $DEST" diff --git a/scripts/test.sh b/scripts/test.sh new file mode 100755 index 0000000..c21dc75 --- /dev/null +++ b/scripts/test.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +test -f "$ROOT/repos.txt" +test -f "$ROOT/types/ns1-all-in-one/repos.txt" +test -f "$ROOT/types/control-plane/repos.txt" +test -f "$ROOT/types/ns1-archive/repos.txt" +test -f "$ROOT/types/lan-worker/repos.txt" +# every type list is a subset of the master list +while read -r name; do + [[ -z "$name" || "$name" == \#* ]] && continue + grep -qx "$name" "$ROOT/repos.txt" || { echo "type repo not in master: $name" >&2; exit 1; } +done < <(cat "$ROOT"/types/*/repos.txt) +# every Forgejo module has a host-deps row +while read -r name; do + [[ -z "$name" || "$name" == \#* ]] && continue + grep -q "^${name}|" "$ROOT/host-deps.tsv" || { echo "missing host-deps.tsv row: $name" >&2; exit 1; } +done < "$ROOT/repos.txt" +bash -n "$ROOT/scripts/host-deps.sh" +echo OK diff --git a/types/control-plane/README.md b/types/control-plane/README.md new file mode 100644 index 0000000..530dc41 --- /dev/null +++ b/types/control-plane/README.md @@ -0,0 +1,5 @@ +# Server type: control-plane + +HTTP control plane only: fleet, access doors, IAM, billing, middleware. Archive floors run on an `ns1-archive` host. + +Public doors: access-web `:3021`, access-api `:3022`, access-leaf `:3023`, access-zapier `:3024`, access-staff `:3025`. Edge loopback `:3000`. Fleet `:3850`. diff --git a/types/control-plane/repos.txt b/types/control-plane/repos.txt new file mode 100644 index 0000000..64c2bca --- /dev/null +++ b/types/control-plane/repos.txt @@ -0,0 +1,23 @@ +zappier-edge +verae-middleware +verae-request-splitter +verae-zapier-simulator +verae-fleet +zappier-account-balance +zappier-customer-service +zappier-sales-pricing +zappier-accounting-export +verae-access-authz +verae-access-web +verae-access-api +verae-access-leaf +verae-access-zapier +verae-access-staff +zappier-identity +verae-jobs-events +verae-staff-session +verae-staff-ui +verae-staff-iam +verae-keep +verae-ops +verae-bootstrap diff --git a/types/lan-worker/README.md b/types/lan-worker/README.md new file mode 100644 index 0000000..6186419 --- /dev/null +++ b/types/lan-worker/README.md @@ -0,0 +1,10 @@ +# Server type: lan-worker + +Extra SSH worker (example: `lan-134` at 70.88.205.134). Roles: `tree-node`, `archive-worm`. + +Leave this type **disabled** in fleet `machines.json` until SSH works. The operator console shows a grey card for an intentionally off host. + +```bash +bash scripts/fetch.sh lan-worker +# enable in fleet only after: ssh -i ~/.ssh/id_ed25519 marchon@70.88.205.134 true +``` diff --git a/types/lan-worker/repos.txt b/types/lan-worker/repos.txt new file mode 100644 index 0000000..a2bab29 --- /dev/null +++ b/types/lan-worker/repos.txt @@ -0,0 +1,5 @@ +verae-archive-worm +verae-tree-node +verae-fleet +verae-keep +verae-bootstrap diff --git a/types/ns1-all-in-one/README.md b/types/ns1-all-in-one/README.md new file mode 100644 index 0000000..d0f9404 --- /dev/null +++ b/types/ns1-all-in-one/README.md @@ -0,0 +1,29 @@ +# Server type: ns1-all-in-one + +Dedicated NS1 box (`70.88.205.138`): NATS loopback, archive workers, HTTP doors, fleet, IAM, keep. + +## Layout + +- NATS `127.0.0.1:4222` (already running on NS1) +- Archive workers: keep (`:3860`) — job-poller, webhook-deliver, aggregator, worm ×3, tree-node ×3 +- HTTP: fleet `serve` on this host (`local` machine only) +- zappier-edge **`:13000` loopback** (host `:3000` is taken) +- Public portal: `:3021/portal/` +- Operator console: `:3850` +- IAM `:3028`, staff-session `:3027` + +## Fetch + +```bash +export VERAE_SRC=$HOME/verae-src +bash scripts/fetch.sh ns1-all-in-one +``` + +## Start (from a monorepo-shaped tree) + +```bash +export VERAE_STACK=$HOME/verae-stack # packages/ sibling layout +bash types/ns1-all-in-one/start.sh +``` + +`start.sh` disables fleet spawn of archive workers (keep already owns those ports), points edge at `:13000`, starts `verae-fleet serve`. diff --git a/types/ns1-all-in-one/repos.txt b/types/ns1-all-in-one/repos.txt new file mode 100644 index 0000000..67b5900 --- /dev/null +++ b/types/ns1-all-in-one/repos.txt @@ -0,0 +1,27 @@ +zappier-edge +verae-middleware +verae-request-splitter +verae-archive-worm +verae-archive-aggregator +verae-tree-node +verae-zapier-simulator +verae-fleet +zappier-account-balance +zappier-customer-service +zappier-sales-pricing +zappier-accounting-export +verae-access-authz +verae-access-web +verae-access-api +verae-access-leaf +verae-access-zapier +verae-access-staff +zappier-identity +verae-jobs-events +verae-nats-accounts +verae-staff-session +verae-staff-ui +verae-staff-iam +verae-keep +verae-ops +verae-bootstrap diff --git a/types/ns1-all-in-one/start.sh b/types/ns1-all-in-one/start.sh new file mode 100755 index 0000000..75d7e8b --- /dev/null +++ b/types/ns1-all-in-one/start.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# Start HTTP control plane on NS1. Archive workers stay with verae-keep. +set -euo pipefail +STACK="${VERAE_STACK:-$HOME/verae-stack}" +FLEET="$STACK/packages/verae-fleet" +EDGE_PORT="${EDGE_PORT:-13000}" +export STAFF_IAM_URL="${STAFF_IAM_URL:-https://iam.zapier.georgelambert.org}" +export STAFF_COOKIE_DOMAIN="${STAFF_COOKIE_DOMAIN:-.zapier.georgelambert.org}" +export NATS_URL="${NATS_URL:-nats://127.0.0.1:4222}" +export FLEET_ENABLE_LAN134="${FLEET_ENABLE_LAN134:-}" +test -d "$FLEET" || { echo "missing $FLEET" >&2; exit 1; } + +python3 - "$FLEET" "$EDGE_PORT" <<'PY' +import json, sys +from pathlib import Path +root = Path(sys.argv[1]) +edge_port = sys.argv[2] +fleet_path = root / "fleet.json" +fleet = json.loads(fleet_path.read_text()) +for sid in ("tree-node", "archive-worm", "archive-aggregator", "job-poller", "webhook-deliver", "nats"): + if sid in fleet.get("services", {}): + fleet["services"][sid]["enabled"] = False + fleet["services"][sid]["keepFloor"] = False +fleet_path.write_text(json.dumps(fleet, indent=2) + "\n") +machines_path = root / "machines.json" +machines = json.loads(machines_path.read_text()) +for m in machines.get("machines", []): + if m.get("id") == "ns1": + m["enabled"] = False + if m.get("id") == "lan-134": + m["enabled"] = False + if m.get("id") == "local": + m["enabled"] = True + m["roles"] = ["*"] + m["capacity"] = 32 +machines_path.write_text(json.dumps(machines, indent=2) + "\n") +edge = json.loads((root / "services" / "zappier-edge.json").read_text()) +edge.setdefault("env", {})["PORT"] = edge_port +edge["env"]["BIND"] = "127.0.0.1" +edge["ports"] = {"healthBase": int(edge_port)} +(root / "services" / "zappier-edge.json").write_text(json.dumps(edge, indent=2) + "\n") +admin = f"http://127.0.0.1:{edge_port}" +for name in ( + "customer-service", + "sales-pricing", + "accounting-export", + "access-staff", + "access-web", + "staff-iam", + "staff-session", + "zappier-edge", +): + p = root / "services" / f"{name}.json" + if not p.exists(): + continue + spec = json.loads(p.read_text()) + env = spec.setdefault("env", {}) + if "ZAPPIER_ADMIN_URL" in env: + env["ZAPPIER_ADMIN_URL"] = admin + if name == "access-web": + env["ZAPPIER_EDGE_URL"] = admin + env["STAFF_IAM_URL"] = "https://iam.zapier.georgelambert.org" + env["STAFF_COOKIE_DOMAIN"] = ".zapier.georgelambert.org" + env["STAFF_AUTH"] = "1" + p.write_text(json.dumps(spec, indent=2) + "\n") +print("overlay applied edge", edge_port) +PY + +cd "$FLEET" +if curl -sf http://127.0.0.1:3850/health >/dev/null; then + echo "fleet already up" + exit 0 +fi +nohup env STAFF_IAM_URL="$STAFF_IAM_URL" STAFF_COOKIE_DOMAIN="$STAFF_COOKIE_DOMAIN" NATS_URL="$NATS_URL" node src/cli.js serve \ + >/tmp/verae-fleet-serve.out 2>&1 & +echo "fleet pid $!" +for i in $(seq 1 40); do + if curl -sf http://127.0.0.1:3850/health >/dev/null; then + echo "fleet up" + exit 0 + fi + sleep 0.5 +done +echo "fleet did not become healthy; see /tmp/verae-fleet-serve.out" >&2 +exit 1 diff --git a/types/ns1-archive/README.md b/types/ns1-archive/README.md new file mode 100644 index 0000000..ef53718 --- /dev/null +++ b/types/ns1-archive/README.md @@ -0,0 +1,5 @@ +# Server type: ns1-archive + +NATS JetStream on loopback plus fleet/keep workers: tree-node ×3, archive-worm ×3, aggregator, job-poller, webhook-deliver. + +Do not bind NATS on a public NIC. Keep (`:3860`) restarts workers unless the operator console paused or stopped them. diff --git a/types/ns1-archive/repos.txt b/types/ns1-archive/repos.txt new file mode 100644 index 0000000..6799d9a --- /dev/null +++ b/types/ns1-archive/repos.txt @@ -0,0 +1,8 @@ +verae-archive-worm +verae-archive-aggregator +verae-tree-node +verae-fleet +verae-keep +verae-nats-accounts +verae-ops +verae-bootstrap