diff --git a/packages/docs-master/CATALOG-HOST.md b/packages/docs-master/CATALOG-HOST.md new file mode 100644 index 0000000..ca34ef4 --- /dev/null +++ b/packages/docs-master/CATALOG-HOST.md @@ -0,0 +1,81 @@ +# Move the documentation catalog to another hostname + +The catalog is a **static tree** (`site/` in the monorepo). HTML cards and PDF clicks are **path-relative**, so the same files can be served from `https://zapier.georgelambert.org/` today and `https://docs.verae-time.net/` later without rewriting links. + +Live service doors (portal, IAM, fleet, …) stay on their own hostnames. Forgejo stays on `git.georgelambert.org`. + +**Script:** `scripts/rewrite-docs-host.py` in [master-zapier-plan-draft](https://git.georgelambert.org/marchon/master-zapier-plan-draft) +**Build:** `scripts/build-docs-site.py` · **Deploy:** `scripts/deploy-docs-site.sh` + +## Default (relative) — no hostname pass + +1. Build: `python3 scripts/build-docs-site.py` +2. Rsync the `site/` directory to the new host document root (must be the site root, `/`). +3. Point DNS + TLS (Caddy `file_server` with `index index.html index.pdf README.html README.pdf`) at that tree. + +PDF annotations look like `README.pdf` and `../../index.html` (relative to the PDF). Browsers resolve them against the PDF’s own URL, so they follow the new host. + +Optional label in the HTML banner (not a link target): + +```bash +DOCS_PUBLIC_HOST=docs.verae-time.net python3 scripts/build-docs-site.py +``` + +## Absolute URLs (only if a viewer needs them) + +Bake a public origin at build time: + +```bash +DOCS_PUBLIC_URL=https://docs.verae-time.net \ +DOCS_PUBLIC_HOST=docs.verae-time.net \ + python3 scripts/build-docs-site.py +``` + +Or retarget an **already built** `site/` without rebuilding PDFs from markdown: + +```bash +# Lab origin → future docs host +python3 scripts/rewrite-docs-host.py \ + --site ./site \ + --from https://zapier.georgelambert.org \ + --to https://docs.verae-time.net + +# Strip the origin (same as the default build) +python3 scripts/rewrite-docs-host.py --site ./site --relative + +python3 scripts/rewrite-docs-host.py --site ./site --from … --to … --dry-run +``` + +The rewrite walks HTML/SVG/CSS/JS/MD text and PDF `/URI` annotations. It does **not** change `portal.zapier.georgelambert.org`, `iam.…`, or `git.georgelambert.org`. + +## Caddy (any host) + +```caddy +docs.verae-time.net { + root * /SSD2/sites/docs.verae-time.net + encode gzip zstd + file_server { + index index.html index.pdf README.html README.pdf + } + try_files {path} {path}/index.html {path}/index.pdf {path}/README.html {path}/README.pdf +} +``` + +Lab copy: `packages/verae-ops/caddy/zapier.georgelambert.org.caddy`. + +## Check + +```bash +python3 scripts/check-doc-links.py +``` + +Local files must exist for homepage hrefs and PDF URI targets. Off-site HTTPS (Forgejo, live doors) is left alone. + +## Publish lab catalog (today) + +```bash +git -C /Users/marchon/apps/zapier push ns1 main +git -C /Users/marchon/apps/zapier push ns1 main:master +bash scripts/deploy-docs-site.sh +# → marchon@70.88.205.138:/SSD2/sites/zapier.georgelambert.org +``` diff --git a/packages/docs-master/HOST-DEPS.md b/packages/docs-master/HOST-DEPS.md new file mode 100644 index 0000000..0aebf79 --- /dev/null +++ b/packages/docs-master/HOST-DEPS.md @@ -0,0 +1,22 @@ +# Host dependencies for every Forgejo module + +The installer lives in **verae-bootstrap** (not this summaries repo): + +- Script: `scripts/host-deps.sh` +- Manifest: `host-deps.tsv` (one row per Forgejo repo) +- Narrative: [HOST-DEPS.md](https://git.georgelambert.org/marchon/verae-bootstrap/src/branch/main/HOST-DEPS.md) + +It detects Debian/Ubuntu vs Alpine (and best-effort macOS), checks that git, curl, Node, native build tools, and `nats-server` are present, and **downloads/installs** whatever is missing on **this** machine. + +```bash +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-bootstrap.git +cd verae-bootstrap +bash scripts/host-deps.sh --check --type ns1-all-in-one +bash scripts/host-deps.sh --type ns1-all-in-one +export VERAE_SRC=$HOME/verae-src +bash scripts/fetch.sh ns1-all-in-one +``` + +Alpine if `bash` is not installed yet: `apk add --no-cache bash git curl`. + +Catalog hostname move (relative PDFs): [CATALOG-HOST.md](CATALOG-HOST.md). diff --git a/packages/docs-master/README.md b/packages/docs-master/README.md index 240a3b0..83b52cc 100644 --- a/packages/docs-master/README.md +++ b/packages/docs-master/README.md @@ -3,6 +3,8 @@ Summaries, NATS contracts, and message flows for every Verae Time × Zapier module. **Live catalog:** https://zapier.georgelambert.org/ +**Move the catalog to another host:** [CATALOG-HOST.md](CATALOG-HOST.md) (relative PDF links; `scripts/rewrite-docs-host.py`) +**Host OS / Node / NATS install:** [verae-bootstrap HOST-DEPS](https://git.georgelambert.org/marchon/verae-bootstrap) **Overview (start here):** https://git.georgelambert.org/marchon/overview **Operator console:** http://127.0.0.1:3850/ (Fleet · Trace · Docs) **Monorepo:** https://git.georgelambert.org/marchon/master-zapier-plan-draft (`main` and `master`) diff --git a/packages/docs-master/modules/verae-bootstrap/SUMMARY.md b/packages/docs-master/modules/verae-bootstrap/SUMMARY.md index ee89719..87f842c 100644 --- a/packages/docs-master/modules/verae-bootstrap/SUMMARY.md +++ b/packages/docs-master/modules/verae-bootstrap/SUMMARY.md @@ -1,3 +1,5 @@ # verae-bootstrap Clone Forgejo modules and install npm deps. Server types under `types/`. + +`scripts/host-deps.sh` checks this machine (Debian/Ubuntu or Alpine) and installs git, curl, Node, build tools for `better-sqlite3`, and `nats-server` when the server type runs JetStream locally. Manifest: `host-deps.tsv`. See `HOST-DEPS.md`. diff --git a/packages/overview/02-modules-and-repos.md b/packages/overview/02-modules-and-repos.md index 95da726..2bd1013 100644 --- a/packages/overview/02-modules-and-repos.md +++ b/packages/overview/02-modules-and-repos.md @@ -37,7 +37,7 @@ Each runtime piece is its **own git repo** on Forgejo (`git.georgelambert.org`, | **verae-staff-ui** | `packages/verae-staff-ui` | Shared staff review HTML (CS + access-staff) | | **verae-staff-iam** | `packages/verae-staff-iam` | Internal staff users, roles, permissions, audit | | **verae-keep** | `packages/verae-keep` | Host keep-alive + watch + guard (honors admin pause/stop) | -| **verae-bootstrap** | `packages/verae-bootstrap` | Clone every repo and install deps for each server type | +| **verae-bootstrap** | `packages/verae-bootstrap` | Clone every repo; `host-deps.sh` installs OS/Node/NATS per module | | **zapier-docs-master** | `packages/docs-master` | Per-module `SUMMARY.md` + `NATS.md` | | **verae-ops** | `packages/verae-ops` | Docker, Proxmox, VMs, dedicated hardware, linking services | diff --git a/packages/overview/10-docs-sync.md b/packages/overview/10-docs-sync.md index 381ae1c..5b3a195 100644 --- a/packages/overview/10-docs-sync.md +++ b/packages/overview/10-docs-sync.md @@ -20,7 +20,15 @@ Validated against the module list in `packages/` and Forgejo repos. - `verae-staff-iam`, `verae-staff-session`, `verae-staff-ui` - `verae-keep` -- `verae-bootstrap` (clone + npm per server type) +- `verae-bootstrap` (clone + npm per server type; `scripts/host-deps.sh` installs OS/Node/NATS) + +## Move the catalog to another hostname + +HTML and PDF catalog links are **path-relative**. Rsync `site/` to a new document root (for example `docs.verae-time.net`) without rewriting files. + +- Procedure: [CATALOG-HOST.md](https://git.georgelambert.org/marchon/zapier-docs-master/src/branch/main/CATALOG-HOST.md) in **zapier-docs-master** +- Optional absolute retarget: `python3 scripts/rewrite-docs-host.py --site ./site --from https://zapier.georgelambert.org --to https://docs.verae-time.net` +- Host packages for a new box: `bash packages/verae-bootstrap/scripts/host-deps.sh --type ns1-all-in-one` ## Still blocked (not closable here) diff --git a/packages/overview/INDEX.md b/packages/overview/INDEX.md index 0a67911..7e5ecf0 100644 --- a/packages/overview/INDEX.md +++ b/packages/overview/INDEX.md @@ -16,6 +16,8 @@ Operator console (loopback): http://127.0.0.1:3850/ · [CONSOLE.pdf](https://zap | [zapier-docs-master](https://git.georgelambert.org/marchon/zapier-docs-master) | One-line SUMMARY + NATS per module | | [zapier-user-docs](https://zapier.georgelambert.org/user-docs/README.pdf) | Signup → register → lookup | | [verae-ops](https://zapier.georgelambert.org/packages/verae-ops/README.pdf) | Docker / Proxmox / VM / metal install | +| [CATALOG-HOST](https://zapier.georgelambert.org/docs-master/CATALOG-HOST.pdf) | Serve the catalog on another hostname | +| [HOST-DEPS](https://zapier.georgelambert.org/packages/verae-bootstrap/HOST-DEPS.pdf) | OS / Node / NATS install per module | ## 2. Architecture diff --git a/packages/verae-bootstrap/HOST-DEPS.md b/packages/verae-bootstrap/HOST-DEPS.md new file mode 100644 index 0000000..b78b968 --- /dev/null +++ b/packages/verae-bootstrap/HOST-DEPS.md @@ -0,0 +1,63 @@ +# Host dependencies (`scripts/host-deps.sh`) + +Every Forgejo module on `git.georgelambert.org` is listed in `host-deps.tsv`. This script **checks** that the current machine has the OS packages, Node, and (when this host runs JetStream) `nats-server`. Missing pieces are **downloaded and installed** for the OS it is running on. + +| OS family | Package tool | Node | nats-server | +|-----------|--------------|------|-------------| +| Debian / Ubuntu | `apt-get` | official `linux-x64` / `linux-arm64` tarball into `$PREFIX` | GitHub release tarball | +| Alpine | `apk` | `apk add nodejs` then musl tarball if still too old | `apk add nats-server` or glibc release (last resort) | +| macOS | Homebrew | already on PATH or brew | brew / skip | + +`$PREFIX` is `/usr/local` as root, otherwise `~/.local`. Put `$PREFIX/bin` on `PATH`. + +Do **not** copy `node_modules` from macOS onto Linux (`better-sqlite3` → `invalid ELF header`). Run `npm install` **on the target OS** (`scripts/install-deps.sh`). + +Zapier cloud never talks to NATS. `nats-server` binds **127.0.0.1:4222** only on `ns1-all-in-one` and `ns1-archive`. + +## Commands + +```bash +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-bootstrap.git +cd verae-bootstrap + +# What this box still needs (no changes) +bash scripts/host-deps.sh --check --type ns1-all-in-one + +# Install git, curl, Node 22, build tools, nats-server +bash scripts/host-deps.sh --type ns1-all-in-one + +# Single module (e.g. a thin Alpine VM that only runs worm) +bash scripts/host-deps.sh --module verae-archive-worm + +# Catalog PDF/HTML build tools (pandoc, weasyprint, pypdf) +bash scripts/host-deps.sh --docs --module zapier-docs-master + +# Then clone + npm +export VERAE_SRC=$HOME/verae-src +bash scripts/fetch.sh ns1-all-in-one +``` + +`scripts/fetch.sh` runs `host-deps.sh` first so Git/Node exist before clone. + +## Module → host needs + +| Module (Forgejo) | Node | npm | native C++ | NATS client | Notes | +|------------------|------|-----|------------|-------------|--------| +| zappier-edge | 20 | yes | **better-sqlite3** | yes | Rebuild sqlite on Linux | +| verae-middleware | **22** | yes | | yes | | +| verae-zapier-app / verae-activate | 18 | local zapier CLI only | | | Runs on Zapier cloud | +| archive-worm / tree-node / aggregator / splitter | 20 | yes | | yes | | +| fleet | 20 | yes | | | `openssh-client` | +| keep, IAM, session, access-* , identity, jobs-events, billing NATS | 20 | yes | | most | | +| overview, docs-master, user-docs, decisions, nats-accounts, ops | — | | | | Docs only (`--docs` for catalog build) | + +Full table: `host-deps.tsv`. + +## Server types and nats-server + +| Type | `nats-server` on this host? | +|------|------------------------------| +| ns1-all-in-one | yes (`--nats` implied) | +| ns1-archive | yes | +| control-plane | no (client to NS1 loopback / private URL) | +| lan-worker | no (client; lan-134 stays disabled until SSH works) | diff --git a/packages/verae-bootstrap/README.md b/packages/verae-bootstrap/README.md index dd40d21..335e791 100644 --- a/packages/verae-bootstrap/README.md +++ b/packages/verae-bootstrap/README.md @@ -7,14 +7,22 @@ Clone every Verae Time × Zapier **module repo** from Forgejo and install Node d ## Host requirements +`scripts/host-deps.sh` **checks** this machine and **installs** anything missing (Debian/Ubuntu `apt`, Alpine `apk`, Node tarball, `nats-server` release). Table: [HOST-DEPS.md](HOST-DEPS.md). Manifest: `host-deps.tsv`. + +```bash +bash scripts/host-deps.sh --check --type ns1-all-in-one +bash scripts/host-deps.sh --type ns1-all-in-one # install +``` + | Need | Why | |------|-----| -| Node 20+ (`node`, `npm`) | All HTTP and worker processes | +| Node 20+ (`node`, `npm`); middleware wants **22** | All HTTP and worker processes | | Git + SSH to Forgejo **port 2223** | `git clone ssh://git@git.georgelambert.org:2223/marchon/.git` | -| `nats-server -js` on **127.0.0.1:4222** | Archive / billing types only — never a public bind | +| `python3` `make` `g++` | Native addon `better-sqlite3` on **zappier-edge** (rebuild on Linux) | +| `nats-server -js` on **127.0.0.1:4222** | Archive / all-in-one types only — never a public bind | | `curl` | Health probes | -Zapier cloud never talks to NATS. Do not publish `:4222`. +Alpine first line if `bash` is missing: `apk add --no-cache bash git curl`. Zapier cloud never talks to NATS. Do not publish `:4222`. ## Fetch @@ -26,7 +34,7 @@ bash scripts/fetch.sh # every repo bash scripts/fetch.sh ns1-all-in-one # one server type ``` -`scripts/clone.sh` pulls `main`. `scripts/install-deps.sh` runs `npm install` **on the target OS** (then `npm rebuild`). Do not copy `node_modules` from macOS onto Linux — native addons such as `better-sqlite3` will fail with `invalid ELF header`. +`scripts/fetch.sh` runs `host-deps.sh` (OS + Node), then `clone.sh` (`main`), then `install-deps.sh` (`npm install` **on the target OS**, then `npm rebuild`). Do not copy `node_modules` from macOS onto Linux — native addons such as `better-sqlite3` will fail with `invalid ELF header`. ## Server types diff --git a/packages/verae-bootstrap/host-deps.tsv b/packages/verae-bootstrap/host-deps.tsv new file mode 100644 index 0000000..13d2512 --- /dev/null +++ b/packages/verae-bootstrap/host-deps.tsv @@ -0,0 +1,40 @@ +# Forgejo module | min Node | flags (comma) +# flags: npm, native (C++ addon / better-sqlite3), nats (NATS client), +# nats_server (this host runs nats-server), ssh, docs (catalog build) +# Zapier Platform apps run in Zapier cloud; npm here is only for local zapier build. +zappier-edge|20|npm,native,nats +verae-middleware|22|npm,nats +verae-zapier-app|18|npm +verae-activate|18|npm +verae-request-splitter|20|npm,nats +verae-archive-worm|20|npm,nats +verae-archive-aggregator|20|npm,nats +verae-tree-node|20|npm,nats +verae-zapier-simulator|20|npm +zapier-user-docs|0| +verae-fleet|20|npm,ssh +overview|0| +verae-nats-process|20|npm,nats +verae-ops|0| +zappier-account-balance|20|npm,nats +zappier-customer-service|20|npm,nats +zappier-sales-pricing|20|npm,nats +zappier-accounting-export|20|npm,nats +verae-access-authz|20|npm,nats +verae-access-web|20|npm,nats +verae-access-api|20|npm,nats +verae-access-leaf|20|npm,nats +verae-access-zapier|20|npm,nats +verae-access-staff|20|npm,nats +zappier-identity|20|npm,nats +verae-jobs-events|20|npm,nats +verae-nats-accounts|0| +zapier-decisions|0| +UI-Docs|20|npm,docs +verae-staff-session|20|npm +verae-staff-ui|20|npm +verae-staff-iam|20|npm +verae-keep|20|npm +verae-bootstrap|0| +zapier-docs-master|0|docs +master-zapier-plan-draft|22|npm,native,nats,docs diff --git a/packages/verae-bootstrap/package.json b/packages/verae-bootstrap/package.json index c60e4ac..2426c1d 100644 --- a/packages/verae-bootstrap/package.json +++ b/packages/verae-bootstrap/package.json @@ -6,6 +6,7 @@ "scripts": { "clone": "bash scripts/clone.sh", "deps": "bash scripts/install-deps.sh", + "host-deps": "bash scripts/host-deps.sh", "fetch": "bash scripts/fetch.sh", "test": "bash scripts/test.sh" } diff --git a/packages/verae-bootstrap/scripts/fetch.sh b/packages/verae-bootstrap/scripts/fetch.sh index 1c274e0..915ea3c 100755 --- a/packages/verae-bootstrap/scripts/fetch.sh +++ b/packages/verae-bootstrap/scripts/fetch.sh @@ -7,6 +7,12 @@ if [[ -n "$TYPE" ]]; then LIST="$ROOT/types/$TYPE/repos.txt" [[ -f "$LIST" ]] || { echo "unknown type $TYPE" >&2; exit 1; } fi +# OS packages + Node (+ nats-server for archive types) before clone. +if [[ -n "$TYPE" ]]; then + bash "$ROOT/scripts/host-deps.sh" --type "$TYPE" +else + bash "$ROOT/scripts/host-deps.sh" +fi bash "$ROOT/scripts/clone.sh" "$LIST" bash "$ROOT/scripts/install-deps.sh" echo "fetch done type=${TYPE:-all}" diff --git a/packages/verae-bootstrap/scripts/host-deps.sh b/packages/verae-bootstrap/scripts/host-deps.sh new file mode 100755 index 0000000..1b433ea --- /dev/null +++ b/packages/verae-bootstrap/scripts/host-deps.sh @@ -0,0 +1,348 @@ +#!/usr/bin/env bash +# Check (and optionally install) OS + Node + NATS packages needed to run +# Verae Time × Zapier modules on this host. +# +# Debian/Ubuntu and Alpine are first-class. Other Linux: print the plan. +# macOS: Homebrew when present. +# +# Usage: +# bash scripts/host-deps.sh # detect modules, install missing +# bash scripts/host-deps.sh --check # report only +# bash scripts/host-deps.sh --type ns1-all-in-one +# bash scripts/host-deps.sh --module zappier-edge +# bash scripts/host-deps.sh --nats # also install nats-server +# bash scripts/host-deps.sh --docs # pandoc / weasyprint / pypdf +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +MANIFEST="$ROOT/host-deps.tsv" +NODE_VER="${NODE_VER:-22.14.0}" +NATS_VER="${NATS_VER:-2.10.26}" +PREFIX="${PREFIX:-}" +CHECK_ONLY=0 +WANT_NATS=0 +WANT_DOCS=0 +WANT_CADDY=0 +TYPE="" +MODULES=() + +usage() { sed -n '2,16p' "$0" | tr -d '#'; exit "${1:-0}"; } + +while [[ $# -gt 0 ]]; do + case "$1" in + --check) CHECK_ONLY=1 ;; + --install) CHECK_ONLY=0 ;; + --nats) WANT_NATS=1 ;; + --docs) WANT_DOCS=1 ;; + --caddy) WANT_CADDY=1 ;; + --type) TYPE="${2:-}"; shift ;; + --module) MODULES+=("${2:-}"); shift ;; + --prefix) PREFIX="${2:-}"; shift ;; + -h|--help) usage 0 ;; + *) echo "unknown arg $1" >&2; usage 1 ;; + esac + shift +done + +# --- OS / arch --- +OS_ID="unknown" +OS_FAMILY="unknown" +if [[ "$(uname -s)" == Darwin ]]; then + OS_ID=macos + OS_FAMILY=macos +elif [[ -f /etc/os-release ]]; then + # shellcheck disable=SC1091 + . /etc/os-release + OS_ID="${ID:-unknown}" + case "$OS_ID" in + debian|ubuntu|raspbian|linuxmint) OS_FAMILY=debian ;; + alpine) OS_FAMILY=alpine ;; + *) OS_FAMILY=linux ;; + esac +fi +ARCH="$(uname -m)" +case "$ARCH" in + x86_64|amd64) NODE_ARCH=x64; NATS_ARCH=amd64; DEB_ARCH=amd64 ;; + aarch64|arm64) NODE_ARCH=arm64; NATS_ARCH=arm64; DEB_ARCH=arm64 ;; + armv7l) NODE_ARCH=armv7l; NATS_ARCH=arm7; DEB_ARCH=armhf ;; + *) NODE_ARCH=$ARCH; NATS_ARCH=$ARCH; DEB_ARCH=$ARCH ;; +esac +MUSL=0 +if [[ "$OS_FAMILY" == alpine ]]; then MUSL=1; fi + +is_root() { [[ "$(id -u)" -eq 0 ]]; } +run_root() { + if is_root; then + "$@" + elif command -v sudo >/dev/null; then + sudo "$@" + else + echo "need root (or sudo) to run: $*" >&2 + return 1 + fi +} + +if [[ -z "$PREFIX" ]]; then + if is_root; then PREFIX=/usr/local; else PREFIX="${HOME}/.local"; fi +fi +mkdir -p "$PREFIX/bin" +export PATH="$PREFIX/bin:$PATH" + +# --- module list --- +PKG_TO_REPO="$(cat <<'EOF' +zappier zappier-edge +verae-zapier-middleware verae-middleware +verae-zapier verae-zapier-app +docs-master zapier-docs-master +ui-docs UI-Docs +EOF +)" + +map_name() { + local n="$1" + local line + while read -r from to; do + [[ "$n" == "$from" ]] && { echo "$to"; return; } + done <<<"$PKG_TO_REPO" + echo "$n" +} + +collect_from_list() { + local f="$1" + while read -r name; do + [[ -z "$name" || "$name" == \#* ]] && continue + MODULES+=("$name") + done < "$f" +} + +if [[ ${#MODULES[@]} -eq 0 ]]; then + if [[ -n "$TYPE" ]]; then + collect_from_list "$ROOT/types/$TYPE/repos.txt" + elif [[ -n "${VERAE_SRC:-}" && -d "${VERAE_SRC}" ]]; then + for d in "$VERAE_SRC"/*/; do + [[ -d "$d" ]] || continue + MODULES+=("$(basename "$d")") + done + elif [[ -d "$PWD/packages" ]]; then + for d in "$PWD"/packages/*/; do + MODULES+=("$(map_name "$(basename "$d")")") + done + MODULES+=(master-zapier-plan-draft) + elif [[ -f "$PWD/package.json" ]]; then + n="$(basename "$PWD")" + MODULES+=("$(map_name "$n")") + else + collect_from_list "$ROOT/repos.txt" + fi +fi + +# types that run nats-server on this box +case "$TYPE" in + ns1-all-in-one|ns1-archive) WANT_NATS=1 ;; +esac + +# --- union flags --- +MIN_NODE=0 +NEED_NPM=0 +NEED_NATIVE=0 +NEED_NATS_CLIENT=0 +NEED_SSH=0 +NEED_DOCS=$WANT_DOCS + +while IFS='|' read -r name min flags; do + [[ -z "$name" || "$name" == \#* ]] && continue + wanted=0 + for m in "${MODULES[@]}"; do + [[ "$m" == "$name" ]] && wanted=1 + done + [[ $wanted -eq 1 ]] || continue + if [[ "$min" != 0 && "$min" -gt "$MIN_NODE" ]]; then MIN_NODE=$min; fi + [[ ",$flags," == *",npm,"* ]] && NEED_NPM=1 + [[ ",$flags," == *",native,"* ]] && NEED_NATIVE=1 + [[ ",$flags," == *",nats,"* ]] && NEED_NATS_CLIENT=1 + [[ ",$flags," == *",nats_server,"* ]] && WANT_NATS=1 + [[ ",$flags," == *",ssh,"* ]] && NEED_SSH=1 + [[ ",$flags," == *",docs,"* ]] && NEED_DOCS=1 +done < "$MANIFEST" + +echo "os=$OS_ID family=$OS_FAMILY arch=$ARCH musl=$MUSL prefix=$PREFIX" +echo "modules: ${MODULES[*]}" +echo "need: node>=${MIN_NODE:-0} npm=$NEED_NPM native=$NEED_NATIVE nats_client=$NEED_NATS_CLIENT nats_server=$WANT_NATS ssh=$NEED_SSH docs=$NEED_DOCS" + +MISSING=() +have() { command -v "$1" >/dev/null 2>&1; } + +need_cmd() { + local c="$1" + if have "$c"; then + echo "ok $c $($c --version 2>/dev/null | head -1 || true)" + else + echo "NO $c" + MISSING+=("$c") + fi +} + +node_ok() { + local want="$1" + have node || return 1 + node -e "process.exit(parseInt(process.versions.node,10)>=$want?0:1)" 2>/dev/null +} + +# --- package installers --- +apt_install() { + run_root apt-get update -qq + run_root env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends "$@" +} +apk_install() { run_root apk add --no-cache "$@"; } +brew_install() { brew install "$@"; } + +install_pkgs() { + local pkgs=("$@") + [[ ${#pkgs[@]} -eq 0 ]] && return 0 + echo "install OS packages: ${pkgs[*]}" + [[ $CHECK_ONLY -eq 1 ]] && return 0 + case "$OS_FAMILY" in + debian) apt_install "${pkgs[@]}" ;; + alpine) apk_install "${pkgs[@]}" ;; + macos) brew_install "${pkgs[@]}" ;; + *) echo "install these packages yourself: ${pkgs[*]}" >&2; return 1 ;; + esac +} + +install_node() { + local want="${1:-20}" + if node_ok "$want"; then + echo "ok node $(node -v) (>=$want)" + return 0 + fi + echo "NO node (>=$want) have=$(node -v 2>/dev/null || echo none)" + MISSING+=("node>=$want") + [[ $CHECK_ONLY -eq 1 ]] && return 0 + if [[ "$OS_FAMILY" == alpine && $MUSL -eq 1 ]]; then + apk_install nodejs npm python3 make g++ linux-headers || true + if node_ok "$want"; then echo "ok node $(node -v) (apk)"; return 0; fi + fi + if [[ "$OS_FAMILY" == debian ]]; then + apt_install ca-certificates curl xz-utils + elif [[ "$OS_FAMILY" == alpine ]]; then + apk_install curl tar xz + fi + local dist="linux-${NODE_ARCH}" + [[ $MUSL -eq 1 ]] && dist="linux-${NODE_ARCH}-musl" + local base="https://nodejs.org/dist/v${NODE_VER}" + [[ $MUSL -eq 1 ]] && base="https://unofficial-builds.nodejs.org/download/release/v${NODE_VER}" + local tar="node-v${NODE_VER}-${dist}.tar.xz" + local tmp + tmp="$(mktemp -d)" + echo "download $base/$tar" + curl -fsSL "$base/$tar" -o "$tmp/$tar" + tar -xJf "$tmp/$tar" -C "$tmp" + mkdir -p "$PREFIX" + run_root cp -a "$tmp/node-v${NODE_VER}-${dist}/." "$PREFIX/" + rm -rf "$tmp" + hash -r + node_ok "$want" || { echo "node install failed" >&2; return 1; } + echo "ok node $(node -v) → $PREFIX" +} + +install_nats() { + if have nats-server; then + echo "ok nats-server $(nats-server -v 2>/dev/null | head -1 || true)" + return 0 + fi + echo "NO nats-server" + MISSING+=("nats-server") + [[ $CHECK_ONLY -eq 1 ]] && return 0 + if [[ "$OS_FAMILY" == alpine ]]; then + apk_install nats-server 2>/dev/null && { echo "ok nats-server (apk)"; return 0; } || true + fi + local tar="nats-server-v${NATS_VER}-linux-${NATS_ARCH}.tar.gz" + local url="https://github.com/nats-io/nats-server/releases/download/v${NATS_VER}/${tar}" + local tmp + tmp="$(mktemp -d)" + echo "download $url" + curl -fsSL "$url" -o "$tmp/$tar" + tar -xzf "$tmp/$tar" -C "$tmp" + run_root install -m 0755 "$tmp/nats-server-v${NATS_VER}-linux-${NATS_ARCH}/nats-server" "$PREFIX/bin/nats-server" + rm -rf "$tmp" + have nats-server || { echo "nats-server install failed" >&2; return 1; } + echo "ok nats-server → $PREFIX/bin" +} + +# --- plan OS packages --- +PKGS=() +add_pkg() { PKGS+=("$1"); } + +case "$OS_FAMILY" in + debian) + add_pkg ca-certificates; add_pkg curl; add_pkg git; add_pkg bash; add_pkg tar; add_pkg xz-utils + [[ $NEED_NATIVE -eq 1 || $NEED_NPM -eq 1 ]] && { add_pkg python3; add_pkg make; add_pkg g++; add_pkg python3-dev; } + [[ $NEED_SSH -eq 1 ]] && add_pkg openssh-client + [[ $NEED_DOCS -eq 1 ]] && { add_pkg pandoc; add_pkg weasyprint; add_pkg python3-pip; } + [[ $WANT_CADDY -eq 1 ]] && add_pkg debian-keyring + ;; + alpine) + add_pkg ca-certificates; add_pkg curl; add_pkg git; add_pkg bash; add_pkg tar; add_pkg xz + [[ $NEED_NATIVE -eq 1 || $NEED_NPM -eq 1 ]] && { add_pkg python3; add_pkg make; add_pkg g++; add_pkg linux-headers; add_pkg libstdc++; } + [[ $NEED_SSH -eq 1 ]] && add_pkg openssh-client + [[ $NEED_DOCS -eq 1 ]] && { add_pkg pandoc; add_pkg py3-pip; add_pkg weasyprint; } + ;; + macos) + add_pkg git; add_pkg curl + [[ $NEED_DOCS -eq 1 ]] && { add_pkg pandoc; add_pkg weasyprint; } + ;; +esac + +echo "--- check ---" +need_cmd git +need_cmd curl +[[ $NEED_SSH -eq 1 ]] && need_cmd ssh +if [[ "$MIN_NODE" != 0 ]]; then + if node_ok "$MIN_NODE"; then echo "ok node $(node -v)"; else echo "NO node (>=$MIN_NODE)"; MISSING+=("node>=$MIN_NODE"); fi + need_cmd npm +fi +[[ $WANT_NATS -eq 1 ]] && { have nats-server && echo "ok nats-server" || { echo "NO nats-server"; MISSING+=("nats-server"); }; } +[[ $NEED_DOCS -eq 1 ]] && { need_cmd pandoc; have weasyprint || have weasyprint3 || echo "NO weasyprint"; } + +if [[ $CHECK_ONLY -eq 1 ]]; then + if [[ ${#MISSING[@]} -eq 0 ]]; then + echo "host deps OK" + exit 0 + fi + echo "missing: ${MISSING[*]}" + exit 2 +fi + +echo "--- install ---" +if [[ ${#PKGS[@]} -gt 0 ]]; then + UNIQ=() + while IFS= read -r p; do + [[ -n "$p" ]] && UNIQ+=("$p") + done < <(printf '%s\n' "${PKGS[@]}" | sort -u) + install_pkgs "${UNIQ[@]}" +fi +if [[ "$MIN_NODE" != 0 ]]; then + install_node "$MIN_NODE" +fi +if [[ $WANT_NATS -eq 1 ]]; then + install_nats +fi +if [[ $NEED_DOCS -eq 1 ]] && have python3; then + python3 -m pip install --user pypdf >/dev/null 2>&1 || true +fi + +MISSING=() +need_cmd git +need_cmd curl +if [[ "$MIN_NODE" != 0 ]]; then + node_ok "$MIN_NODE" || MISSING+=("node>=$MIN_NODE") + have npm || MISSING+=("npm") +fi +[[ $WANT_NATS -eq 1 ]] && { have nats-server || MISSING+=("nats-server"); } + +if [[ ${#MISSING[@]} -eq 0 ]]; then + echo "host deps OK (PATH should include $PREFIX/bin)" + exit 0 +fi +echo "still missing: ${MISSING[*]}" >&2 +exit 1 diff --git a/packages/verae-bootstrap/scripts/test.sh b/packages/verae-bootstrap/scripts/test.sh index d36b03c..c21dc75 100755 --- a/packages/verae-bootstrap/scripts/test.sh +++ b/packages/verae-bootstrap/scripts/test.sh @@ -11,4 +11,10 @@ while read -r name; do [[ -z "$name" || "$name" == \#* ]] && continue grep -qx "$name" "$ROOT/repos.txt" || { echo "type repo not in master: $name" >&2; exit 1; } done < <(cat "$ROOT"/types/*/repos.txt) +# every Forgejo module has a host-deps row +while read -r name; do + [[ -z "$name" || "$name" == \#* ]] && continue + grep -q "^${name}|" "$ROOT/host-deps.tsv" || { echo "missing host-deps.tsv row: $name" >&2; exit 1; } +done < "$ROOT/repos.txt" +bash -n "$ROOT/scripts/host-deps.sh" echo OK diff --git a/packages/verae-ops/01-dependencies.md b/packages/verae-ops/01-dependencies.md index 6d19194..cdcfbc1 100644 --- a/packages/verae-ops/01-dependencies.md +++ b/packages/verae-ops/01-dependencies.md @@ -46,13 +46,24 @@ Optional: `verae-activate` (tiny pushable app), `verae-zapier-simulator` (trace) ## Runtime dependencies +Install (or verify) OS packages on the **target** machine — Debian/Ubuntu or Alpine: + +```bash +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-bootstrap.git +cd verae-bootstrap +bash scripts/host-deps.sh --check --type ns1-all-in-one +bash scripts/host-deps.sh --type ns1-all-in-one +``` + +Manifest: `host-deps.tsv`. Narrative: [HOST-DEPS.md](https://git.georgelambert.org/marchon/verae-bootstrap). Catalog hostname move: [CATALOG-HOST.md](https://git.georgelambert.org/marchon/zapier-docs-master/src/branch/main/CATALOG-HOST.md). + | Service | Needs | |---------|--------| -| zappier-edge | Node 20+, SQLite path or volume, `PORT`, optional Stripe keys | +| zappier-edge | Node 20+, **better-sqlite3** (python3/make/g++ on Linux), SQLite path, `PORT`, optional Stripe keys | | middleware | Node 22+, `NATS_URL` **or** `NATS_ENABLED=false`, chain URL or `MOCK_VERAE` | -| NATS | JetStream, **no public bind** | +| NATS | JetStream, **no public bind** (`host-deps.sh --nats` on archive / all-in-one) | | worm / tree-node | Node, `NATS_URL`, local state dir | -| fleet | Node, `machines.json` (SSH **paths**), HTTP `:3850` on `0.0.0.0` | +| fleet | Node, `machines.json` (SSH **paths**), HTTP `:3850` on `0.0.0.0`, `openssh-client` | | Zapier cloud app | `MIDDLEWARE_BASE_URL` = **public HTTPS of zappier-edge** (or middleware if you skip metering in lab) | ## Ports (lab defaults) diff --git a/packages/verae-ops/GETTING-STARTED.md b/packages/verae-ops/GETTING-STARTED.md index ff5fe6d..c77025d 100644 --- a/packages/verae-ops/GETTING-STARTED.md +++ b/packages/verae-ops/GETTING-STARTED.md @@ -33,6 +33,18 @@ The billing and user platform is the standalone **zappier** project (`/Users/mar ## 15-minute lab (one machine) +Bare metal / VM (Debian or Alpine): install OS + Node first, then clone. + +```bash +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-bootstrap.git +cd verae-bootstrap +bash scripts/host-deps.sh --type ns1-all-in-one +export VERAE_SRC=$HOME/verae-src +bash scripts/fetch.sh ns1-all-in-one +``` + +Docker lab from the monorepo: + ```bash git clone ssh://git@git.georgelambert.org:2223/marchon/master-zapier-plan-draft.git cd master-zapier-plan-draft diff --git a/scripts/build-docs-site.py b/scripts/build-docs-site.py index 075e0bc..aaf0a37 100755 --- a/scripts/build-docs-site.py +++ b/scripts/build-docs-site.py @@ -616,6 +616,8 @@ def main() -> None: extras = ("README.md", "SUMMARY.md", "NATS.md") if pkg == "zapier-decisions": extras = extras + ("LOG.md", "TODO.md") + if pkg == "verae-bootstrap": + extras = extras + ("HOST-DEPS.md",) for name in extras: p = pkg_root / name if p.exists(): @@ -824,8 +826,11 @@ def main() -> None: extra_pdf = [ ("docs-master/README.pdf", "Master summaries (docs-master)"), + ("docs-master/CATALOG-HOST.pdf", "Move the catalog to another hostname"), + ("docs-master/HOST-DEPS.pdf", "Host OS / Node / NATS installer"), ("docs-master/MESSAGE-FLOWS.pdf", "Numbered message flows"), ("docs-master/modules-and-nats.pdf", "NATS address table"), + ("packages/verae-bootstrap/HOST-DEPS.pdf", "Host OS / Node / NATS installer"), ("user-docs/README.pdf", "User guide index"), ("packages/verae-zapier-simulator/README.pdf", "Simulator README"), ("packages/verae-fleet/docs/CONSOLE.pdf", "Operator console"), @@ -839,8 +844,11 @@ def main() -> None: ] extra_md = [ ("docs-master/README.html", "Master summaries (docs-master)"), + ("docs-master/CATALOG-HOST.html", "Move the catalog to another hostname"), + ("docs-master/HOST-DEPS.html", "Host OS / Node / NATS installer"), ("docs-master/MESSAGE-FLOWS.html", "Numbered message flows"), ("docs-master/modules-and-nats.html", "NATS address table"), + ("packages/verae-bootstrap/HOST-DEPS.html", "Host OS / Node / NATS installer"), ("user-docs/README.html", "User guide index"), ("packages/verae-zapier-simulator/README.html", "Simulator README"), ("packages/verae-fleet/docs/CONSOLE.html", "Operator console"),