commit e6bbbaffe8deed1dd33022c17801c1d7a8972011 Author: George Lambert Date: Fri Sep 11 19:55:24 2026 -0400 Initial import of verae-bootstrap from zapier monorepo diff --git a/NATS.md b/NATS.md new file mode 100644 index 0000000..bcff40b --- /dev/null +++ b/NATS.md @@ -0,0 +1,3 @@ +# NATS — verae-bootstrap + +Bootstrap clones git repos and runs `npm install`. It does not open NATS ports. NATS is a **runtime** dependency of the `ns1-archive` and `ns1-all-in-one` server types (`127.0.0.1:4222` only). diff --git a/README.md b/README.md new file mode 100644 index 0000000..dd40d21 --- /dev/null +++ b/README.md @@ -0,0 +1,44 @@ +# verae-bootstrap + +Clone every Verae Time × Zapier **module repo** from Forgejo and install Node dependencies. Each **server type** has its own directory under `types/` (repos list + README + start notes). + +**Forgejo:** https://git.georgelambert.org/marchon/verae-bootstrap +**SSH:** `ssh://git@git.georgelambert.org:2223/marchon/verae-bootstrap.git` + +## Host requirements + +| Need | Why | +|------|-----| +| Node 20+ (`node`, `npm`) | All HTTP and worker processes | +| Git + SSH to Forgejo **port 2223** | `git clone ssh://git@git.georgelambert.org:2223/marchon/.git` | +| `nats-server -js` on **127.0.0.1:4222** | Archive / billing types only — never a public bind | +| `curl` | Health probes | + +Zapier cloud never talks to NATS. Do not publish `:4222`. + +## Fetch + +```bash +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-bootstrap.git +cd verae-bootstrap +export VERAE_SRC=$HOME/verae-src +bash scripts/fetch.sh # every repo +bash scripts/fetch.sh ns1-all-in-one # one server type +``` + +`scripts/clone.sh` pulls `main`. `scripts/install-deps.sh` runs `npm install` **on the target OS** (then `npm rebuild`). Do not copy `node_modules` from macOS onto Linux — native addons such as `better-sqlite3` will fail with `invalid ELF header`. + +## Server types + +| Type | Directory | What that host runs | +|------|-----------|---------------------| +| **ns1-all-in-one** | `types/ns1-all-in-one/` | Control plane + archive workers + keep (NS1 / 70.88.205.138) | +| **control-plane** | `types/control-plane/` | Fleet, edges, IAM, billing HTTP (no worm/tree floor) | +| **ns1-archive** | `types/ns1-archive/` | NATS + worm ×3 + tree-node ×3 + poller + webhook + aggregator + keep | +| **lan-worker** | `types/lan-worker/` | Extra SSH worker (tree-node, archive-worm). Keep disabled until SSH works (`lan-134`). | + +Each type has `repos.txt` (subset of the master list) and `README.md`. + +## NS1 all-in-one notes + +`:3000` on 70.88.205.138 is already a public app. zappier-edge on that box binds **127.0.0.1:13000**. Public portal remains **access-web `:3021/portal/`**. Archive workers are owned by **verae-keep**; fleet does not double-spawn them. diff --git a/SUMMARY.md b/SUMMARY.md new file mode 100644 index 0000000..2effcc0 --- /dev/null +++ b/SUMMARY.md @@ -0,0 +1,3 @@ +# verae-bootstrap + +Downloads every Forgejo module and Node dependencies. Server types (each documented under `types/`): `ns1-all-in-one`, `control-plane`, `ns1-archive`, `lan-worker`. diff --git a/package.json b/package.json new file mode 100644 index 0000000..c60e4ac --- /dev/null +++ b/package.json @@ -0,0 +1,12 @@ +{ + "name": "verae-bootstrap", + "version": "0.1.0", + "private": true, + "description": "Clone Verae Time × Zapier git repos and install dependencies for each server type", + "scripts": { + "clone": "bash scripts/clone.sh", + "deps": "bash scripts/install-deps.sh", + "fetch": "bash scripts/fetch.sh", + "test": "bash scripts/test.sh" + } +} diff --git a/repos.txt b/repos.txt new file mode 100644 index 0000000..3a6b836 --- /dev/null +++ b/repos.txt @@ -0,0 +1,35 @@ +zappier-edge +verae-middleware +verae-zapier-app +verae-activate +verae-request-splitter +verae-archive-worm +verae-archive-aggregator +zapier-docs-master +verae-tree-node +verae-zapier-simulator +zapier-user-docs +verae-fleet +overview +verae-nats-process +verae-ops +zappier-account-balance +zappier-customer-service +zappier-sales-pricing +zappier-accounting-export +verae-access-authz +verae-access-web +verae-access-api +verae-access-leaf +verae-access-zapier +verae-access-staff +zappier-identity +verae-jobs-events +verae-nats-accounts +zapier-decisions +UI-Docs +verae-staff-session +verae-staff-ui +verae-staff-iam +verae-keep +verae-bootstrap diff --git a/scripts/clone.sh b/scripts/clone.sh new file mode 100755 index 0000000..ffa0cb1 --- /dev/null +++ b/scripts/clone.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Clone Forgejo modules listed in a repos file. +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +GITSSH="${GITSSH:-ssh://git@git.georgelambert.org:2223/marchon}" +DEST="${VERAE_SRC:-$HOME/verae-src}" +LIST="${1:-$ROOT/repos.txt}" +export GIT_SSH_COMMAND="${GIT_SSH_COMMAND:-ssh -o StrictHostKeyChecking=accept-new -p 2223}" +mkdir -p "$DEST" +while read -r name; do + [[ -z "$name" || "$name" == \#* ]] && continue + if [[ -d "$DEST/$name/.git" ]]; then + echo "fetch $name" + git -C "$DEST/$name" fetch --prune origin + git -C "$DEST/$name" checkout -q main 2>/dev/null || git -C "$DEST/$name" checkout -q master + git -C "$DEST/$name" pull --ff-only || true + else + echo "clone $name" + git clone "$GITSSH/${name}.git" "$DEST/$name" + fi +done < "$LIST" +echo "cloned into $DEST" diff --git a/scripts/fetch.sh b/scripts/fetch.sh new file mode 100755 index 0000000..1c274e0 --- /dev/null +++ b/scripts/fetch.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +TYPE="${1:-}" +LIST="$ROOT/repos.txt" +if [[ -n "$TYPE" ]]; then + LIST="$ROOT/types/$TYPE/repos.txt" + [[ -f "$LIST" ]] || { echo "unknown type $TYPE" >&2; exit 1; } +fi +bash "$ROOT/scripts/clone.sh" "$LIST" +bash "$ROOT/scripts/install-deps.sh" +echo "fetch done type=${TYPE:-all}" diff --git a/scripts/install-deps.sh b/scripts/install-deps.sh new file mode 100755 index 0000000..6d46eb1 --- /dev/null +++ b/scripts/install-deps.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# npm install in every cloned package that has package.json. +set -euo pipefail +DEST="${VERAE_SRC:-$HOME/verae-src}" +if [[ ! -d "$DEST" ]]; then + echo "missing $DEST — run scripts/clone.sh first" >&2 + exit 1 +fi +command -v node >/dev/null +command -v npm >/dev/null +find "$DEST" -maxdepth 2 -name package.json ! -path '*/node_modules/*' | while read -r pkg; do + dir="$(dirname "$pkg")" + echo "npm install $dir" + (cd "$dir" && npm install --no-audit --no-fund && npm rebuild --no-audit --no-fund >/dev/null) +done +echo "deps installed under $DEST" diff --git a/scripts/test.sh b/scripts/test.sh new file mode 100755 index 0000000..d36b03c --- /dev/null +++ b/scripts/test.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +test -f "$ROOT/repos.txt" +test -f "$ROOT/types/ns1-all-in-one/repos.txt" +test -f "$ROOT/types/control-plane/repos.txt" +test -f "$ROOT/types/ns1-archive/repos.txt" +test -f "$ROOT/types/lan-worker/repos.txt" +# every type list is a subset of the master list +while read -r name; do + [[ -z "$name" || "$name" == \#* ]] && continue + grep -qx "$name" "$ROOT/repos.txt" || { echo "type repo not in master: $name" >&2; exit 1; } +done < <(cat "$ROOT"/types/*/repos.txt) +echo OK diff --git a/types/control-plane/README.md b/types/control-plane/README.md new file mode 100644 index 0000000..530dc41 --- /dev/null +++ b/types/control-plane/README.md @@ -0,0 +1,5 @@ +# Server type: control-plane + +HTTP control plane only: fleet, access doors, IAM, billing, middleware. Archive floors run on an `ns1-archive` host. + +Public doors: access-web `:3021`, access-api `:3022`, access-leaf `:3023`, access-zapier `:3024`, access-staff `:3025`. Edge loopback `:3000`. Fleet `:3850`. diff --git a/types/control-plane/repos.txt b/types/control-plane/repos.txt new file mode 100644 index 0000000..64c2bca --- /dev/null +++ b/types/control-plane/repos.txt @@ -0,0 +1,23 @@ +zappier-edge +verae-middleware +verae-request-splitter +verae-zapier-simulator +verae-fleet +zappier-account-balance +zappier-customer-service +zappier-sales-pricing +zappier-accounting-export +verae-access-authz +verae-access-web +verae-access-api +verae-access-leaf +verae-access-zapier +verae-access-staff +zappier-identity +verae-jobs-events +verae-staff-session +verae-staff-ui +verae-staff-iam +verae-keep +verae-ops +verae-bootstrap diff --git a/types/lan-worker/README.md b/types/lan-worker/README.md new file mode 100644 index 0000000..6186419 --- /dev/null +++ b/types/lan-worker/README.md @@ -0,0 +1,10 @@ +# Server type: lan-worker + +Extra SSH worker (example: `lan-134` at 70.88.205.134). Roles: `tree-node`, `archive-worm`. + +Leave this type **disabled** in fleet `machines.json` until SSH works. The operator console shows a grey card for an intentionally off host. + +```bash +bash scripts/fetch.sh lan-worker +# enable in fleet only after: ssh -i ~/.ssh/id_ed25519 marchon@70.88.205.134 true +``` diff --git a/types/lan-worker/repos.txt b/types/lan-worker/repos.txt new file mode 100644 index 0000000..a2bab29 --- /dev/null +++ b/types/lan-worker/repos.txt @@ -0,0 +1,5 @@ +verae-archive-worm +verae-tree-node +verae-fleet +verae-keep +verae-bootstrap diff --git a/types/ns1-all-in-one/README.md b/types/ns1-all-in-one/README.md new file mode 100644 index 0000000..d0f9404 --- /dev/null +++ b/types/ns1-all-in-one/README.md @@ -0,0 +1,29 @@ +# Server type: ns1-all-in-one + +Dedicated NS1 box (`70.88.205.138`): NATS loopback, archive workers, HTTP doors, fleet, IAM, keep. + +## Layout + +- NATS `127.0.0.1:4222` (already running on NS1) +- Archive workers: keep (`:3860`) — job-poller, webhook-deliver, aggregator, worm ×3, tree-node ×3 +- HTTP: fleet `serve` on this host (`local` machine only) +- zappier-edge **`:13000` loopback** (host `:3000` is taken) +- Public portal: `:3021/portal/` +- Operator console: `:3850` +- IAM `:3028`, staff-session `:3027` + +## Fetch + +```bash +export VERAE_SRC=$HOME/verae-src +bash scripts/fetch.sh ns1-all-in-one +``` + +## Start (from a monorepo-shaped tree) + +```bash +export VERAE_STACK=$HOME/verae-stack # packages/ sibling layout +bash types/ns1-all-in-one/start.sh +``` + +`start.sh` disables fleet spawn of archive workers (keep already owns those ports), points edge at `:13000`, starts `verae-fleet serve`. diff --git a/types/ns1-all-in-one/repos.txt b/types/ns1-all-in-one/repos.txt new file mode 100644 index 0000000..67b5900 --- /dev/null +++ b/types/ns1-all-in-one/repos.txt @@ -0,0 +1,27 @@ +zappier-edge +verae-middleware +verae-request-splitter +verae-archive-worm +verae-archive-aggregator +verae-tree-node +verae-zapier-simulator +verae-fleet +zappier-account-balance +zappier-customer-service +zappier-sales-pricing +zappier-accounting-export +verae-access-authz +verae-access-web +verae-access-api +verae-access-leaf +verae-access-zapier +verae-access-staff +zappier-identity +verae-jobs-events +verae-nats-accounts +verae-staff-session +verae-staff-ui +verae-staff-iam +verae-keep +verae-ops +verae-bootstrap diff --git a/types/ns1-all-in-one/start.sh b/types/ns1-all-in-one/start.sh new file mode 100755 index 0000000..4b6c738 --- /dev/null +++ b/types/ns1-all-in-one/start.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Start HTTP control plane on NS1. Archive workers stay with verae-keep. +set -euo pipefail +STACK="${VERAE_STACK:-$HOME/verae-stack}" +FLEET="$STACK/packages/verae-fleet" +EDGE_PORT="${EDGE_PORT:-13000}" +export STAFF_IAM_URL="${STAFF_IAM_URL:-http://127.0.0.1:3028}" +export NATS_URL="${NATS_URL:-nats://127.0.0.1:4222}" +export FLEET_ENABLE_LAN134="${FLEET_ENABLE_LAN134:-}" +test -d "$FLEET" || { echo "missing $FLEET" >&2; exit 1; } + +python3 - "$FLEET" "$EDGE_PORT" <<'PY' +import json, sys +from pathlib import Path +root = Path(sys.argv[1]) +edge_port = sys.argv[2] +fleet_path = root / "fleet.json" +fleet = json.loads(fleet_path.read_text()) +for sid in ("tree-node", "archive-worm", "archive-aggregator", "job-poller", "webhook-deliver", "nats"): + if sid in fleet.get("services", {}): + fleet["services"][sid]["enabled"] = False + fleet["services"][sid]["keepFloor"] = False +fleet_path.write_text(json.dumps(fleet, indent=2) + "\n") +machines_path = root / "machines.json" +machines = json.loads(machines_path.read_text()) +for m in machines.get("machines", []): + if m.get("id") == "ns1": + m["enabled"] = False + if m.get("id") == "lan-134": + m["enabled"] = False + if m.get("id") == "local": + m["enabled"] = True + m["roles"] = ["*"] + m["capacity"] = 32 +machines_path.write_text(json.dumps(machines, indent=2) + "\n") +edge = json.loads((root / "services" / "zappier-edge.json").read_text()) +edge.setdefault("env", {})["PORT"] = edge_port +edge["env"]["BIND"] = "127.0.0.1" +edge["ports"] = {"healthBase": int(edge_port)} +(root / "services" / "zappier-edge.json").write_text(json.dumps(edge, indent=2) + "\n") +admin = f"http://127.0.0.1:{edge_port}" +for name in ( + "customer-service", + "sales-pricing", + "accounting-export", + "access-staff", + "access-web", +): + p = root / "services" / f"{name}.json" + if not p.exists(): + continue + spec = json.loads(p.read_text()) + env = spec.setdefault("env", {}) + if "ZAPPIER_ADMIN_URL" in env or name != "access-web": + env["ZAPPIER_ADMIN_URL"] = admin + if name == "access-web": + env["ZAPPIER_EDGE_URL"] = admin + env["STAFF_IAM_URL"] = "http://127.0.0.1:3028" + env["STAFF_AUTH"] = "1" + p.write_text(json.dumps(spec, indent=2) + "\n") +print("overlay applied edge", edge_port) +PY + +cd "$FLEET" +if curl -sf http://127.0.0.1:3850/health >/dev/null; then + echo "fleet already up" + exit 0 +fi +nohup env STAFF_IAM_URL="$STAFF_IAM_URL" NATS_URL="$NATS_URL" node src/cli.js serve \ + >/tmp/verae-fleet-serve.out 2>&1 & +echo "fleet pid $!" +for i in $(seq 1 40); do + if curl -sf http://127.0.0.1:3850/health >/dev/null; then + echo "fleet up" + exit 0 + fi + sleep 0.5 +done +echo "fleet did not become healthy; see /tmp/verae-fleet-serve.out" >&2 +exit 1 diff --git a/types/ns1-archive/README.md b/types/ns1-archive/README.md new file mode 100644 index 0000000..ef53718 --- /dev/null +++ b/types/ns1-archive/README.md @@ -0,0 +1,5 @@ +# Server type: ns1-archive + +NATS JetStream on loopback plus fleet/keep workers: tree-node ×3, archive-worm ×3, aggregator, job-poller, webhook-deliver. + +Do not bind NATS on a public NIC. Keep (`:3860`) restarts workers unless the operator console paused or stopped them. diff --git a/types/ns1-archive/repos.txt b/types/ns1-archive/repos.txt new file mode 100644 index 0000000..6799d9a --- /dev/null +++ b/types/ns1-archive/repos.txt @@ -0,0 +1,8 @@ +verae-archive-worm +verae-archive-aggregator +verae-tree-node +verae-fleet +verae-keep +verae-nats-accounts +verae-ops +verae-bootstrap