From 03f557203a3fd47c8dd2b50988d1194c9a27133e Mon Sep 17 00:00:00 2001 From: George Lambert Date: Fri, 11 Sep 2026 19:52:54 -0400 Subject: [PATCH] Sync docs, add verae-bootstrap, and run the full stack on NS1. Docs match current modules: public portal is access-web /portal/, IAM and keep are listed, edge is loopback. Bootstrap clones every Forgejo repo and installs deps per server type. Fleet starts IAM and staff-session. NS1 all-in-one uses edge :13000 because :3000 is taken; archive workers stay with keep. --- packages/docs-master/README.md | 3 + packages/docs-master/modules-and-nats.md | 3 + .../modules/verae-bootstrap/NATS.md | 3 + .../modules/verae-bootstrap/SUMMARY.md | 3 + .../docs-master/modules/verae-keep/NATS.md | 3 + .../docs-master/modules/verae-keep/SUMMARY.md | 3 + .../modules/verae-staff-iam/NATS.md | 3 + .../modules/verae-staff-iam/SUMMARY.md | 3 + packages/overview/02-modules-and-repos.md | 2 + packages/overview/10-docs-sync.md | 29 +++++++ packages/ui-docs/WALKTHROUGH.md | 2 +- packages/verae-bootstrap/NATS.md | 3 + packages/verae-bootstrap/README.md | 44 ++++++++++ packages/verae-bootstrap/SUMMARY.md | 3 + packages/verae-bootstrap/package.json | 12 +++ packages/verae-bootstrap/repos.txt | 35 ++++++++ packages/verae-bootstrap/scripts/clone.sh | 22 +++++ packages/verae-bootstrap/scripts/fetch.sh | 12 +++ .../verae-bootstrap/scripts/install-deps.sh | 16 ++++ packages/verae-bootstrap/scripts/test.sh | 14 ++++ .../types/control-plane/README.md | 5 ++ .../types/control-plane/repos.txt | 23 ++++++ .../types/lan-worker/README.md | 10 +++ .../types/lan-worker/repos.txt | 5 ++ .../types/ns1-all-in-one/README.md | 29 +++++++ .../types/ns1-all-in-one/repos.txt | 27 +++++++ .../types/ns1-all-in-one/start.sh | 80 +++++++++++++++++++ .../types/ns1-archive/README.md | 5 ++ .../types/ns1-archive/repos.txt | 8 ++ packages/verae-fleet/README.md | 2 + packages/verae-fleet/fleet.json | 4 +- .../verae-fleet/services/staff-session.json | 21 +++++ packages/verae-keep/NATS.md | 3 + packages/verae-keep/SUMMARY.md | 3 + packages/verae-ops/01-dependencies.md | 13 ++- packages/verae-ops/05-dedicated-hardware.md | 6 +- packages/verae-ops/GETTING-STARTED.md | 12 ++- packages/zapier-decisions/LOG.md | 6 ++ packages/zapier-decisions/TODO.md | 2 + packages/zappier/docs/USER-MANUAL.md | 4 +- scripts/build-docs-site.py | 6 ++ scripts/push-module-repos.sh | 2 + 42 files changed, 481 insertions(+), 13 deletions(-) create mode 100644 packages/docs-master/modules/verae-bootstrap/NATS.md create mode 100644 packages/docs-master/modules/verae-bootstrap/SUMMARY.md create mode 100644 packages/docs-master/modules/verae-keep/NATS.md create mode 100644 packages/docs-master/modules/verae-keep/SUMMARY.md create mode 100644 packages/docs-master/modules/verae-staff-iam/NATS.md create mode 100644 packages/docs-master/modules/verae-staff-iam/SUMMARY.md create mode 100644 packages/overview/10-docs-sync.md create mode 100644 packages/verae-bootstrap/NATS.md create mode 100644 packages/verae-bootstrap/README.md create mode 100644 packages/verae-bootstrap/SUMMARY.md create mode 100644 packages/verae-bootstrap/package.json create mode 100644 packages/verae-bootstrap/repos.txt create mode 100755 packages/verae-bootstrap/scripts/clone.sh create mode 100755 packages/verae-bootstrap/scripts/fetch.sh create mode 100755 packages/verae-bootstrap/scripts/install-deps.sh create mode 100755 packages/verae-bootstrap/scripts/test.sh create mode 100644 packages/verae-bootstrap/types/control-plane/README.md create mode 100644 packages/verae-bootstrap/types/control-plane/repos.txt create mode 100644 packages/verae-bootstrap/types/lan-worker/README.md create mode 100644 packages/verae-bootstrap/types/lan-worker/repos.txt create mode 100644 packages/verae-bootstrap/types/ns1-all-in-one/README.md create mode 100644 packages/verae-bootstrap/types/ns1-all-in-one/repos.txt create mode 100755 packages/verae-bootstrap/types/ns1-all-in-one/start.sh create mode 100644 packages/verae-bootstrap/types/ns1-archive/README.md create mode 100644 packages/verae-bootstrap/types/ns1-archive/repos.txt create mode 100644 packages/verae-fleet/services/staff-session.json create mode 100644 packages/verae-keep/NATS.md create mode 100644 packages/verae-keep/SUMMARY.md diff --git a/packages/docs-master/README.md b/packages/docs-master/README.md index 8e76870..240a3b0 100644 --- a/packages/docs-master/README.md +++ b/packages/docs-master/README.md @@ -27,6 +27,9 @@ Summaries, NATS contracts, and message flows for every Verae Time × Zapier modu | verae-nats-process | https://git.georgelambert.org/marchon/verae-nats-process | | **zapier-docs-master** (this repo) | https://git.georgelambert.org/marchon/zapier-docs-master | | **verae-ops** | https://git.georgelambert.org/marchon/verae-ops | +| verae-staff-iam | https://git.georgelambert.org/marchon/verae-staff-iam | +| verae-keep | https://git.georgelambert.org/marchon/verae-keep | +| verae-bootstrap | https://git.georgelambert.org/marchon/verae-bootstrap | Clone (SSH port 2223): diff --git a/packages/docs-master/modules-and-nats.md b/packages/docs-master/modules-and-nats.md index b536e3d..9761785 100644 --- a/packages/docs-master/modules-and-nats.md +++ b/packages/docs-master/modules-and-nats.md @@ -51,6 +51,9 @@ Zapier cloud **never** connects to NATS. Internal services do: middleware worker | zapier-user-docs | `zapier-user-docs` | Static | — | catalog `/user-docs/` | | verae-chain-client | `verae-chain-client` | Library | — | HTTPS `api.veraetime.net` or MOCK | | fleet | `verae-fleet` | Local HTTP :3850 | operator | keepFloor + SSH hosts | +| staff-iam | `verae-staff-iam` | :3028 | staff login | none (HTTP) | +| keep | `verae-keep` | :3860 / :3861 | host supervisor | none | +| bootstrap | `verae-bootstrap` | clone/install | — | — | | overview | `overview` | Static | — | high-level map | | nats-process | `verae-nats-process` | Template worker | `verae.example.process.in` | `.out` / `.reply.*` | | docs-master | `zapier-docs-master` | Static | — | published on zapier.georgelambert.org | diff --git a/packages/docs-master/modules/verae-bootstrap/NATS.md b/packages/docs-master/modules/verae-bootstrap/NATS.md new file mode 100644 index 0000000..e14b908 --- /dev/null +++ b/packages/docs-master/modules/verae-bootstrap/NATS.md @@ -0,0 +1,3 @@ +# NATS — verae-bootstrap + +None. Install-time only. diff --git a/packages/docs-master/modules/verae-bootstrap/SUMMARY.md b/packages/docs-master/modules/verae-bootstrap/SUMMARY.md new file mode 100644 index 0000000..ee89719 --- /dev/null +++ b/packages/docs-master/modules/verae-bootstrap/SUMMARY.md @@ -0,0 +1,3 @@ +# verae-bootstrap + +Clone Forgejo modules and install npm deps. Server types under `types/`. diff --git a/packages/docs-master/modules/verae-keep/NATS.md b/packages/docs-master/modules/verae-keep/NATS.md new file mode 100644 index 0000000..d5d22db --- /dev/null +++ b/packages/docs-master/modules/verae-keep/NATS.md @@ -0,0 +1,3 @@ +# NATS — verae-keep + +None. HTTP probes only. diff --git a/packages/docs-master/modules/verae-keep/SUMMARY.md b/packages/docs-master/modules/verae-keep/SUMMARY.md new file mode 100644 index 0000000..9ebbfcb --- /dev/null +++ b/packages/docs-master/modules/verae-keep/SUMMARY.md @@ -0,0 +1,3 @@ +# verae-keep + +Host keep-alive. Restarts crashed units unless fleet paused or stopped them. diff --git a/packages/docs-master/modules/verae-staff-iam/NATS.md b/packages/docs-master/modules/verae-staff-iam/NATS.md new file mode 100644 index 0000000..91e8523 --- /dev/null +++ b/packages/docs-master/modules/verae-staff-iam/NATS.md @@ -0,0 +1,3 @@ +# NATS — verae-staff-iam + +None. HTTP `:3028`. diff --git a/packages/docs-master/modules/verae-staff-iam/SUMMARY.md b/packages/docs-master/modules/verae-staff-iam/SUMMARY.md new file mode 100644 index 0000000..b346202 --- /dev/null +++ b/packages/docs-master/modules/verae-staff-iam/SUMMARY.md @@ -0,0 +1,3 @@ +# verae-staff-iam + +Named internal staff accounts, roles, permissions, JSON sessions, login rate-limit. diff --git a/packages/overview/02-modules-and-repos.md b/packages/overview/02-modules-and-repos.md index 184a80f..95da726 100644 --- a/packages/overview/02-modules-and-repos.md +++ b/packages/overview/02-modules-and-repos.md @@ -36,6 +36,8 @@ Each runtime piece is its **own git repo** on Forgejo (`git.georgelambert.org`, | **verae-staff-session** | `packages/verae-staff-session` | Shared staff cookie login for department HTML | | **verae-staff-ui** | `packages/verae-staff-ui` | Shared staff review HTML (CS + access-staff) | | **verae-staff-iam** | `packages/verae-staff-iam` | Internal staff users, roles, permissions, audit | +| **verae-keep** | `packages/verae-keep` | Host keep-alive + watch + guard (honors admin pause/stop) | +| **verae-bootstrap** | `packages/verae-bootstrap` | Clone every repo and install deps for each server type | | **zapier-docs-master** | `packages/docs-master` | Per-module `SUMMARY.md` + `NATS.md` | | **verae-ops** | `packages/verae-ops` | Docker, Proxmox, VMs, dedicated hardware, linking services | diff --git a/packages/overview/10-docs-sync.md b/packages/overview/10-docs-sync.md new file mode 100644 index 0000000..381ae1c --- /dev/null +++ b/packages/overview/10-docs-sync.md @@ -0,0 +1,29 @@ +# Documentation sync (2026-09-11) + +Validated against the module list in `packages/` and Forgejo repos. + +## Current public doors + +| Door | Bind | Notes | +|------|------|--------| +| Customer portal | access-web `:3021/portal/` | Static + `/portal/api` → loopback edge | +| Customer API | access-api `:3022` | | +| S2S leaf | access-leaf `:3023` | No billing subjects | +| Zapier | access-zapier `:3024` | Only Zapier HTTPS | +| Staff plane | access-staff `:3025` | After IAM | +| zappier-edge | loopback `:3000` (NS1 all-in-one `:13000`) | Admin, Stripe, not a public portal | +| Operator console | fleet `:3850` on `0.0.0.0` | Pause/stop honored by keep | +| Staff IAM | `:3028` | Named users, roles, sessions | +| Keep | `:3860` + watch `:3861` + `guard.sh` | Restart unless paused/stopped | + +## New modules since the last catalog pass + +- `verae-staff-iam`, `verae-staff-session`, `verae-staff-ui` +- `verae-keep` +- `verae-bootstrap` (clone + npm per server type) + +## Still blocked (not closable here) + +- Live `api.veraetime.net` (`MOCK_VERAE=false`) +- NATS nkeys/mTLS three-node cluster +- Zapier Platform `push` diff --git a/packages/ui-docs/WALKTHROUGH.md b/packages/ui-docs/WALKTHROUGH.md index b4f3e82..21cb93c 100644 --- a/packages/ui-docs/WALKTHROUGH.md +++ b/packages/ui-docs/WALKTHROUGH.md @@ -98,7 +98,7 @@ Invoices empty state until you **Generate** a period. ## 10. Customer portal -**http://127.0.0.1:3000/portal/** — create an account or sign in. +**http://127.0.0.1:3021/portal/** — public customer door (access-web). Loopback edge still has `/portal` on `:3000`. Create an account or sign in. ![Portal login](screenshots/16-portal-login.png) diff --git a/packages/verae-bootstrap/NATS.md b/packages/verae-bootstrap/NATS.md new file mode 100644 index 0000000..bcff40b --- /dev/null +++ b/packages/verae-bootstrap/NATS.md @@ -0,0 +1,3 @@ +# NATS — verae-bootstrap + +Bootstrap clones git repos and runs `npm install`. It does not open NATS ports. NATS is a **runtime** dependency of the `ns1-archive` and `ns1-all-in-one` server types (`127.0.0.1:4222` only). diff --git a/packages/verae-bootstrap/README.md b/packages/verae-bootstrap/README.md new file mode 100644 index 0000000..dd40d21 --- /dev/null +++ b/packages/verae-bootstrap/README.md @@ -0,0 +1,44 @@ +# verae-bootstrap + +Clone every Verae Time × Zapier **module repo** from Forgejo and install Node dependencies. Each **server type** has its own directory under `types/` (repos list + README + start notes). + +**Forgejo:** https://git.georgelambert.org/marchon/verae-bootstrap +**SSH:** `ssh://git@git.georgelambert.org:2223/marchon/verae-bootstrap.git` + +## Host requirements + +| Need | Why | +|------|-----| +| Node 20+ (`node`, `npm`) | All HTTP and worker processes | +| Git + SSH to Forgejo **port 2223** | `git clone ssh://git@git.georgelambert.org:2223/marchon/.git` | +| `nats-server -js` on **127.0.0.1:4222** | Archive / billing types only — never a public bind | +| `curl` | Health probes | + +Zapier cloud never talks to NATS. Do not publish `:4222`. + +## Fetch + +```bash +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-bootstrap.git +cd verae-bootstrap +export VERAE_SRC=$HOME/verae-src +bash scripts/fetch.sh # every repo +bash scripts/fetch.sh ns1-all-in-one # one server type +``` + +`scripts/clone.sh` pulls `main`. `scripts/install-deps.sh` runs `npm install` **on the target OS** (then `npm rebuild`). Do not copy `node_modules` from macOS onto Linux — native addons such as `better-sqlite3` will fail with `invalid ELF header`. + +## Server types + +| Type | Directory | What that host runs | +|------|-----------|---------------------| +| **ns1-all-in-one** | `types/ns1-all-in-one/` | Control plane + archive workers + keep (NS1 / 70.88.205.138) | +| **control-plane** | `types/control-plane/` | Fleet, edges, IAM, billing HTTP (no worm/tree floor) | +| **ns1-archive** | `types/ns1-archive/` | NATS + worm ×3 + tree-node ×3 + poller + webhook + aggregator + keep | +| **lan-worker** | `types/lan-worker/` | Extra SSH worker (tree-node, archive-worm). Keep disabled until SSH works (`lan-134`). | + +Each type has `repos.txt` (subset of the master list) and `README.md`. + +## NS1 all-in-one notes + +`:3000` on 70.88.205.138 is already a public app. zappier-edge on that box binds **127.0.0.1:13000**. Public portal remains **access-web `:3021/portal/`**. Archive workers are owned by **verae-keep**; fleet does not double-spawn them. diff --git a/packages/verae-bootstrap/SUMMARY.md b/packages/verae-bootstrap/SUMMARY.md new file mode 100644 index 0000000..2effcc0 --- /dev/null +++ b/packages/verae-bootstrap/SUMMARY.md @@ -0,0 +1,3 @@ +# verae-bootstrap + +Downloads every Forgejo module and Node dependencies. Server types (each documented under `types/`): `ns1-all-in-one`, `control-plane`, `ns1-archive`, `lan-worker`. diff --git a/packages/verae-bootstrap/package.json b/packages/verae-bootstrap/package.json new file mode 100644 index 0000000..c60e4ac --- /dev/null +++ b/packages/verae-bootstrap/package.json @@ -0,0 +1,12 @@ +{ + "name": "verae-bootstrap", + "version": "0.1.0", + "private": true, + "description": "Clone Verae Time × Zapier git repos and install dependencies for each server type", + "scripts": { + "clone": "bash scripts/clone.sh", + "deps": "bash scripts/install-deps.sh", + "fetch": "bash scripts/fetch.sh", + "test": "bash scripts/test.sh" + } +} diff --git a/packages/verae-bootstrap/repos.txt b/packages/verae-bootstrap/repos.txt new file mode 100644 index 0000000..3a6b836 --- /dev/null +++ b/packages/verae-bootstrap/repos.txt @@ -0,0 +1,35 @@ +zappier-edge +verae-middleware +verae-zapier-app +verae-activate +verae-request-splitter +verae-archive-worm +verae-archive-aggregator +zapier-docs-master +verae-tree-node +verae-zapier-simulator +zapier-user-docs +verae-fleet +overview +verae-nats-process +verae-ops +zappier-account-balance +zappier-customer-service +zappier-sales-pricing +zappier-accounting-export +verae-access-authz +verae-access-web +verae-access-api +verae-access-leaf +verae-access-zapier +verae-access-staff +zappier-identity +verae-jobs-events +verae-nats-accounts +zapier-decisions +UI-Docs +verae-staff-session +verae-staff-ui +verae-staff-iam +verae-keep +verae-bootstrap diff --git a/packages/verae-bootstrap/scripts/clone.sh b/packages/verae-bootstrap/scripts/clone.sh new file mode 100755 index 0000000..ffa0cb1 --- /dev/null +++ b/packages/verae-bootstrap/scripts/clone.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Clone Forgejo modules listed in a repos file. +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +GITSSH="${GITSSH:-ssh://git@git.georgelambert.org:2223/marchon}" +DEST="${VERAE_SRC:-$HOME/verae-src}" +LIST="${1:-$ROOT/repos.txt}" +export GIT_SSH_COMMAND="${GIT_SSH_COMMAND:-ssh -o StrictHostKeyChecking=accept-new -p 2223}" +mkdir -p "$DEST" +while read -r name; do + [[ -z "$name" || "$name" == \#* ]] && continue + if [[ -d "$DEST/$name/.git" ]]; then + echo "fetch $name" + git -C "$DEST/$name" fetch --prune origin + git -C "$DEST/$name" checkout -q main 2>/dev/null || git -C "$DEST/$name" checkout -q master + git -C "$DEST/$name" pull --ff-only || true + else + echo "clone $name" + git clone "$GITSSH/${name}.git" "$DEST/$name" + fi +done < "$LIST" +echo "cloned into $DEST" diff --git a/packages/verae-bootstrap/scripts/fetch.sh b/packages/verae-bootstrap/scripts/fetch.sh new file mode 100755 index 0000000..1c274e0 --- /dev/null +++ b/packages/verae-bootstrap/scripts/fetch.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +TYPE="${1:-}" +LIST="$ROOT/repos.txt" +if [[ -n "$TYPE" ]]; then + LIST="$ROOT/types/$TYPE/repos.txt" + [[ -f "$LIST" ]] || { echo "unknown type $TYPE" >&2; exit 1; } +fi +bash "$ROOT/scripts/clone.sh" "$LIST" +bash "$ROOT/scripts/install-deps.sh" +echo "fetch done type=${TYPE:-all}" diff --git a/packages/verae-bootstrap/scripts/install-deps.sh b/packages/verae-bootstrap/scripts/install-deps.sh new file mode 100755 index 0000000..6d46eb1 --- /dev/null +++ b/packages/verae-bootstrap/scripts/install-deps.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# npm install in every cloned package that has package.json. +set -euo pipefail +DEST="${VERAE_SRC:-$HOME/verae-src}" +if [[ ! -d "$DEST" ]]; then + echo "missing $DEST — run scripts/clone.sh first" >&2 + exit 1 +fi +command -v node >/dev/null +command -v npm >/dev/null +find "$DEST" -maxdepth 2 -name package.json ! -path '*/node_modules/*' | while read -r pkg; do + dir="$(dirname "$pkg")" + echo "npm install $dir" + (cd "$dir" && npm install --no-audit --no-fund && npm rebuild --no-audit --no-fund >/dev/null) +done +echo "deps installed under $DEST" diff --git a/packages/verae-bootstrap/scripts/test.sh b/packages/verae-bootstrap/scripts/test.sh new file mode 100755 index 0000000..d36b03c --- /dev/null +++ b/packages/verae-bootstrap/scripts/test.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +test -f "$ROOT/repos.txt" +test -f "$ROOT/types/ns1-all-in-one/repos.txt" +test -f "$ROOT/types/control-plane/repos.txt" +test -f "$ROOT/types/ns1-archive/repos.txt" +test -f "$ROOT/types/lan-worker/repos.txt" +# every type list is a subset of the master list +while read -r name; do + [[ -z "$name" || "$name" == \#* ]] && continue + grep -qx "$name" "$ROOT/repos.txt" || { echo "type repo not in master: $name" >&2; exit 1; } +done < <(cat "$ROOT"/types/*/repos.txt) +echo OK diff --git a/packages/verae-bootstrap/types/control-plane/README.md b/packages/verae-bootstrap/types/control-plane/README.md new file mode 100644 index 0000000..530dc41 --- /dev/null +++ b/packages/verae-bootstrap/types/control-plane/README.md @@ -0,0 +1,5 @@ +# Server type: control-plane + +HTTP control plane only: fleet, access doors, IAM, billing, middleware. Archive floors run on an `ns1-archive` host. + +Public doors: access-web `:3021`, access-api `:3022`, access-leaf `:3023`, access-zapier `:3024`, access-staff `:3025`. Edge loopback `:3000`. Fleet `:3850`. diff --git a/packages/verae-bootstrap/types/control-plane/repos.txt b/packages/verae-bootstrap/types/control-plane/repos.txt new file mode 100644 index 0000000..64c2bca --- /dev/null +++ b/packages/verae-bootstrap/types/control-plane/repos.txt @@ -0,0 +1,23 @@ +zappier-edge +verae-middleware +verae-request-splitter +verae-zapier-simulator +verae-fleet +zappier-account-balance +zappier-customer-service +zappier-sales-pricing +zappier-accounting-export +verae-access-authz +verae-access-web +verae-access-api +verae-access-leaf +verae-access-zapier +verae-access-staff +zappier-identity +verae-jobs-events +verae-staff-session +verae-staff-ui +verae-staff-iam +verae-keep +verae-ops +verae-bootstrap diff --git a/packages/verae-bootstrap/types/lan-worker/README.md b/packages/verae-bootstrap/types/lan-worker/README.md new file mode 100644 index 0000000..6186419 --- /dev/null +++ b/packages/verae-bootstrap/types/lan-worker/README.md @@ -0,0 +1,10 @@ +# Server type: lan-worker + +Extra SSH worker (example: `lan-134` at 70.88.205.134). Roles: `tree-node`, `archive-worm`. + +Leave this type **disabled** in fleet `machines.json` until SSH works. The operator console shows a grey card for an intentionally off host. + +```bash +bash scripts/fetch.sh lan-worker +# enable in fleet only after: ssh -i ~/.ssh/id_ed25519 marchon@70.88.205.134 true +``` diff --git a/packages/verae-bootstrap/types/lan-worker/repos.txt b/packages/verae-bootstrap/types/lan-worker/repos.txt new file mode 100644 index 0000000..a2bab29 --- /dev/null +++ b/packages/verae-bootstrap/types/lan-worker/repos.txt @@ -0,0 +1,5 @@ +verae-archive-worm +verae-tree-node +verae-fleet +verae-keep +verae-bootstrap diff --git a/packages/verae-bootstrap/types/ns1-all-in-one/README.md b/packages/verae-bootstrap/types/ns1-all-in-one/README.md new file mode 100644 index 0000000..d0f9404 --- /dev/null +++ b/packages/verae-bootstrap/types/ns1-all-in-one/README.md @@ -0,0 +1,29 @@ +# Server type: ns1-all-in-one + +Dedicated NS1 box (`70.88.205.138`): NATS loopback, archive workers, HTTP doors, fleet, IAM, keep. + +## Layout + +- NATS `127.0.0.1:4222` (already running on NS1) +- Archive workers: keep (`:3860`) — job-poller, webhook-deliver, aggregator, worm ×3, tree-node ×3 +- HTTP: fleet `serve` on this host (`local` machine only) +- zappier-edge **`:13000` loopback** (host `:3000` is taken) +- Public portal: `:3021/portal/` +- Operator console: `:3850` +- IAM `:3028`, staff-session `:3027` + +## Fetch + +```bash +export VERAE_SRC=$HOME/verae-src +bash scripts/fetch.sh ns1-all-in-one +``` + +## Start (from a monorepo-shaped tree) + +```bash +export VERAE_STACK=$HOME/verae-stack # packages/ sibling layout +bash types/ns1-all-in-one/start.sh +``` + +`start.sh` disables fleet spawn of archive workers (keep already owns those ports), points edge at `:13000`, starts `verae-fleet serve`. diff --git a/packages/verae-bootstrap/types/ns1-all-in-one/repos.txt b/packages/verae-bootstrap/types/ns1-all-in-one/repos.txt new file mode 100644 index 0000000..67b5900 --- /dev/null +++ b/packages/verae-bootstrap/types/ns1-all-in-one/repos.txt @@ -0,0 +1,27 @@ +zappier-edge +verae-middleware +verae-request-splitter +verae-archive-worm +verae-archive-aggregator +verae-tree-node +verae-zapier-simulator +verae-fleet +zappier-account-balance +zappier-customer-service +zappier-sales-pricing +zappier-accounting-export +verae-access-authz +verae-access-web +verae-access-api +verae-access-leaf +verae-access-zapier +verae-access-staff +zappier-identity +verae-jobs-events +verae-nats-accounts +verae-staff-session +verae-staff-ui +verae-staff-iam +verae-keep +verae-ops +verae-bootstrap diff --git a/packages/verae-bootstrap/types/ns1-all-in-one/start.sh b/packages/verae-bootstrap/types/ns1-all-in-one/start.sh new file mode 100755 index 0000000..4b6c738 --- /dev/null +++ b/packages/verae-bootstrap/types/ns1-all-in-one/start.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Start HTTP control plane on NS1. Archive workers stay with verae-keep. +set -euo pipefail +STACK="${VERAE_STACK:-$HOME/verae-stack}" +FLEET="$STACK/packages/verae-fleet" +EDGE_PORT="${EDGE_PORT:-13000}" +export STAFF_IAM_URL="${STAFF_IAM_URL:-http://127.0.0.1:3028}" +export NATS_URL="${NATS_URL:-nats://127.0.0.1:4222}" +export FLEET_ENABLE_LAN134="${FLEET_ENABLE_LAN134:-}" +test -d "$FLEET" || { echo "missing $FLEET" >&2; exit 1; } + +python3 - "$FLEET" "$EDGE_PORT" <<'PY' +import json, sys +from pathlib import Path +root = Path(sys.argv[1]) +edge_port = sys.argv[2] +fleet_path = root / "fleet.json" +fleet = json.loads(fleet_path.read_text()) +for sid in ("tree-node", "archive-worm", "archive-aggregator", "job-poller", "webhook-deliver", "nats"): + if sid in fleet.get("services", {}): + fleet["services"][sid]["enabled"] = False + fleet["services"][sid]["keepFloor"] = False +fleet_path.write_text(json.dumps(fleet, indent=2) + "\n") +machines_path = root / "machines.json" +machines = json.loads(machines_path.read_text()) +for m in machines.get("machines", []): + if m.get("id") == "ns1": + m["enabled"] = False + if m.get("id") == "lan-134": + m["enabled"] = False + if m.get("id") == "local": + m["enabled"] = True + m["roles"] = ["*"] + m["capacity"] = 32 +machines_path.write_text(json.dumps(machines, indent=2) + "\n") +edge = json.loads((root / "services" / "zappier-edge.json").read_text()) +edge.setdefault("env", {})["PORT"] = edge_port +edge["env"]["BIND"] = "127.0.0.1" +edge["ports"] = {"healthBase": int(edge_port)} +(root / "services" / "zappier-edge.json").write_text(json.dumps(edge, indent=2) + "\n") +admin = f"http://127.0.0.1:{edge_port}" +for name in ( + "customer-service", + "sales-pricing", + "accounting-export", + "access-staff", + "access-web", +): + p = root / "services" / f"{name}.json" + if not p.exists(): + continue + spec = json.loads(p.read_text()) + env = spec.setdefault("env", {}) + if "ZAPPIER_ADMIN_URL" in env or name != "access-web": + env["ZAPPIER_ADMIN_URL"] = admin + if name == "access-web": + env["ZAPPIER_EDGE_URL"] = admin + env["STAFF_IAM_URL"] = "http://127.0.0.1:3028" + env["STAFF_AUTH"] = "1" + p.write_text(json.dumps(spec, indent=2) + "\n") +print("overlay applied edge", edge_port) +PY + +cd "$FLEET" +if curl -sf http://127.0.0.1:3850/health >/dev/null; then + echo "fleet already up" + exit 0 +fi +nohup env STAFF_IAM_URL="$STAFF_IAM_URL" NATS_URL="$NATS_URL" node src/cli.js serve \ + >/tmp/verae-fleet-serve.out 2>&1 & +echo "fleet pid $!" +for i in $(seq 1 40); do + if curl -sf http://127.0.0.1:3850/health >/dev/null; then + echo "fleet up" + exit 0 + fi + sleep 0.5 +done +echo "fleet did not become healthy; see /tmp/verae-fleet-serve.out" >&2 +exit 1 diff --git a/packages/verae-bootstrap/types/ns1-archive/README.md b/packages/verae-bootstrap/types/ns1-archive/README.md new file mode 100644 index 0000000..ef53718 --- /dev/null +++ b/packages/verae-bootstrap/types/ns1-archive/README.md @@ -0,0 +1,5 @@ +# Server type: ns1-archive + +NATS JetStream on loopback plus fleet/keep workers: tree-node ×3, archive-worm ×3, aggregator, job-poller, webhook-deliver. + +Do not bind NATS on a public NIC. Keep (`:3860`) restarts workers unless the operator console paused or stopped them. diff --git a/packages/verae-bootstrap/types/ns1-archive/repos.txt b/packages/verae-bootstrap/types/ns1-archive/repos.txt new file mode 100644 index 0000000..6799d9a --- /dev/null +++ b/packages/verae-bootstrap/types/ns1-archive/repos.txt @@ -0,0 +1,8 @@ +verae-archive-worm +verae-archive-aggregator +verae-tree-node +verae-fleet +verae-keep +verae-nats-accounts +verae-ops +verae-bootstrap diff --git a/packages/verae-fleet/README.md b/packages/verae-fleet/README.md index d3fe246..2834e52 100644 --- a/packages/verae-fleet/README.md +++ b/packages/verae-fleet/README.md @@ -39,6 +39,8 @@ Operator console (Fleet · Trace · Docs) at http://127.0.0.1:3850/ — [docs/CO Zapier cloud apps are listed but **not spawned**. NATS on NS1 is **monitored only** (loopback `:4222`, never a public bind). +On a dedicated host, run **[verae-keep](https://git.georgelambert.org/marchon/verae-keep)** next to fleet: keep restarts crashed replicas unless this console paused or stopped them (`intent.json` per instance). Pause/stop/start write that file locally and over SSH. + Install on Docker / Proxmox / metal: https://zapier.georgelambert.org/packages/verae-ops/README.pdf Clone: `ssh://git@git.georgelambert.org:2223/marchon/verae-fleet.git` diff --git a/packages/verae-fleet/fleet.json b/packages/verae-fleet/fleet.json index fe94b1e..eff7bda 100644 --- a/packages/verae-fleet/fleet.json +++ b/packages/verae-fleet/fleet.json @@ -33,6 +33,8 @@ "archive-aggregator": { "min": 1, "max": 2, "keepFloor": true, "enabled": true }, "archive-worm": { "min": 3, "max": 6, "keepFloor": true, "enabled": true }, "tree-node": { "min": 3, "max": 9, "keepFloor": true, "enabled": true }, - "zapier-simulator": { "min": 0, "max": 1, "keepFloor": false, "enabled": true } + "zapier-simulator": { "min": 0, "max": 1, "keepFloor": false, "enabled": true }, + "staff-iam": { "min": 1, "max": 1, "keepFloor": true, "enabled": true }, + "staff-session": { "min": 1, "max": 1, "keepFloor": true, "enabled": true } } } diff --git a/packages/verae-fleet/services/staff-session.json b/packages/verae-fleet/services/staff-session.json new file mode 100644 index 0000000..d9f6018 --- /dev/null +++ b/packages/verae-fleet/services/staff-session.json @@ -0,0 +1,21 @@ +{ + "id": "staff-session", + "title": "Staff session cookie (proxies IAM when STAFF_IAM_URL set)", + "kind": "http", + "package": "verae-staff-session", + "role": "staff-session", + "managed": true, + "runtime": "HTTP :3027", + "health": { "type": "http", "path": "/health", "timeoutMs": 2000 }, + "ports": { "healthBase": 3027 }, + "spawn": { + "cwd": "../verae-staff-session", + "command": "node", + "args": ["src/server.js"] + }, + "env": { + "PORT": "3027", + "STAFF_IAM_URL": "http://127.0.0.1:3028" + }, + "nats": { "in": [], "out": [] } +} diff --git a/packages/verae-keep/NATS.md b/packages/verae-keep/NATS.md new file mode 100644 index 0000000..7f80ff9 --- /dev/null +++ b/packages/verae-keep/NATS.md @@ -0,0 +1,3 @@ +# NATS — verae-keep + +Keep does **not** subscribe to NATS. It probes HTTP `/health` (and TCP `:4222` for NATS as observe-only). Pause/stop intent is files + fleet HTTP, not subjects. diff --git a/packages/verae-keep/SUMMARY.md b/packages/verae-keep/SUMMARY.md new file mode 100644 index 0000000..5985428 --- /dev/null +++ b/packages/verae-keep/SUMMARY.md @@ -0,0 +1,3 @@ +# verae-keep + +Host supervisor: restart crashed units unless the operator console paused or stopped them. Watch (`:3861`) restarts keep (`:3860`). `guard.sh` restarts watch. diff --git a/packages/verae-ops/01-dependencies.md b/packages/verae-ops/01-dependencies.md index 4b33501..6d19194 100644 --- a/packages/verae-ops/01-dependencies.md +++ b/packages/verae-ops/01-dependencies.md @@ -4,7 +4,10 @@ Humans / Zapier cloud │ HTTPS x-api-key ▼ - zappier-edge :3000 (meter, portal, admin, Stripe) + access-web :3021/portal access-zapier :3024 access-api :3022 + │ │ + ▼ ▼ + zappier-edge loopback :3000 (meter, admin, Stripe; not a public door) │ HTTPS internal ▼ middleware-http :3100 (/zapier/v1) @@ -34,6 +37,9 @@ Prefix `ssh://git@git.georgelambert.org:2223/marchon/.git` (SSH **2223**). | Archive merge | `verae-archive-aggregator` | https://zapier.georgelambert.org/packages/verae-archive-aggregator/README.pdf | | Splitter | `verae-request-splitter` | https://zapier.georgelambert.org/packages/verae-request-splitter/README.pdf | | Replica floors | `verae-fleet` | https://zapier.georgelambert.org/packages/verae-fleet/README.pdf | +| Keep-alive | `verae-keep` | https://zapier.georgelambert.org/packages/verae-keep/README.pdf | +| Clone + deps | `verae-bootstrap` | https://zapier.georgelambert.org/packages/verae-bootstrap/README.pdf | +| Staff IAM | `verae-staff-iam` | https://zapier.georgelambert.org/packages/verae-staff-iam/README.pdf | | This install guide | `verae-ops` | https://zapier.georgelambert.org/packages/verae-ops/README.pdf | Optional: `verae-activate` (tiny pushable app), `verae-zapier-simulator` (trace), `overview`, `zapier-docs-master`, `zapier-user-docs`, `verae-nats-process`. @@ -53,8 +59,11 @@ Optional: `verae-activate` (tiny pushable app), `verae-zapier-simulator` (trace) | Port | Bind | Process | |------|------|---------| -| 3000 | public or docker publish | zappier-edge | +| 3000 | **loopback** (lab) | zappier-edge (admin); public portal is `:3021` | +| 3021 | public | access-web `/portal/` | +| 3028 | private/LAN | staff IAM | | 3100 | public or docker publish | middleware | +| 3860 | loopback | verae-keep | | 4222 | **private only** | NATS client | | 6222 | **private only** | NATS cluster | | 8222 | private / localhost | NATS monitor | diff --git a/packages/verae-ops/05-dedicated-hardware.md b/packages/verae-ops/05-dedicated-hardware.md index d9a3cb2..322ce1a 100644 --- a/packages/verae-ops/05-dedicated-hardware.md +++ b/packages/verae-ops/05-dedicated-hardware.md @@ -8,9 +8,9 @@ NS1-style: a box that already runs NATS on **127.0.0.1:4222** (never a public bi 2. `nats-server -js` listening **127.0.0.1:4222** only (cluster later on a private NIC). 3. Clone `zappier-edge`, `verae-middleware`, `verae-fleet`. 4. systemd units (below) or `verae-fleet serve` which keepFloor-spawns workers. -5. Caddy/nginx for public HTTPS to `:3000` / `:3100`. -6. Operator console: `127.0.0.1:3850` — SSH tunnel if you are off-box: - `ssh -L 3850:127.0.0.1:3850 -L 4222:127.0.0.1:4222 user@ns1` +5. Caddy/nginx for public HTTPS to **access-web `:3021/portal/`** and **access-zapier `:3024`**. zappier-edge stays loopback (`:3000`, or `:13000` on NS1 if `:3000` is already taken). +6. Operator console: `0.0.0.0:3850`. `verae-keep` (`:3860`) + `watch` (`:3861`) + `guard.sh` keep workers up unless you paused/stopped them in the console. +7. Clone/install: [verae-bootstrap](https://git.georgelambert.org/marchon/verae-bootstrap) type `ns1-all-in-one`. ## systemd (middleware) diff --git a/packages/verae-ops/GETTING-STARTED.md b/packages/verae-ops/GETTING-STARTED.md index b142c27..ff5fe6d 100644 --- a/packages/verae-ops/GETTING-STARTED.md +++ b/packages/verae-ops/GETTING-STARTED.md @@ -13,7 +13,10 @@ This is the **top document** for install, first-up, and monitoring. Platform-spe | **access-authz** | NATS authorization step for every access plane | `:3020` | | **access-web / api / leaf / zapier / staff** | Public HTTP doors | `:3021–:3025` | | **identity** | `veraeUserId` bind/lookup | `:3026` | -| **zappier-edge** | Identity cache, Stripe, admin, portal (loopback only) | `127.0.0.1:3000` | +| **zappier-edge** | Identity cache, Stripe, admin (loopback only) | `127.0.0.1:3000` (lab; NS1 all-in-one uses `:13000` if `:3000` is taken) | +| **staff-iam** | Named staff users, roles, sessions | `:3028` | +| **staff-session** | Staff cookie (proxies IAM) | `:3027` | +| **verae-keep** | Restart crashed units unless admin paused/stopped | keep `:3860`, watch `:3861` | | **jobs-events** | Mailbox `verae.zapier.jobs.events` | `:3030` | | **account-balance** | NATS prepaid balances (SoT) | `:3010` | | **customer-service** | CS credit additions + review | `:3011` | @@ -50,8 +53,9 @@ curl -fsS http://127.0.0.1:3100/health # middleware Open: -- Customer portal: http://127.0.0.1:3000/portal — sign up, copy `x-api-key`, **Statement** tab -- Admin: http://127.0.0.1:3000/admin — **Statement** per customer +- Customer portal (public door): http://127.0.0.1:3021/portal/ — sign up, copy `x-api-key`, **Statement** tab +- Loopback edge admin: http://127.0.0.1:3000/admin — **Statement** per customer (not a public door) +- Staff IAM: http://127.0.0.1:3028/ - CS review: http://127.0.0.1:3011/ - Sales review: http://127.0.0.1:3012/ - Accounting review/export: http://127.0.0.1:3013/ @@ -68,7 +72,7 @@ cd "$MONOREPO/packages/verae-fleet" node src/cli.js serve ``` -http://127.0.0.1:3850/ — Fleet (green/yellow/red), Trace, Docs. Tree-node **min 3**; pause does not count. +http://0.0.0.0:3850/ — Fleet (green/yellow/red), Trace, Docs. Tree-node **min 3**; pause does not count. On NS1, `verae-keep` restarts workers if they crash unless you paused/stopped them here. ```bash node src/cli.js status diff --git a/packages/zapier-decisions/LOG.md b/packages/zapier-decisions/LOG.md index 6696bfc..0f5f1d4 100644 --- a/packages/zapier-decisions/LOG.md +++ b/packages/zapier-decisions/LOG.md @@ -32,6 +32,12 @@ - Disable lan-134 unless `FLEET_ENABLE_LAN134=1`. - SSH spawn timeout 8s; failed hosts skipped. +## 2026-09-11 — docs sync, bootstrap, NS1 all-in-one + +- Catalog and ops docs: public portal is access-web `:3021/portal/`; edge is loopback; IAM + keep listed. +- New `verae-bootstrap` repo: clone + npm for `ns1-all-in-one`, `control-plane`, `ns1-archive`, `lan-worker`. +- Fleet starts `staff-iam` and `staff-session`. NS1 all-in-one uses edge `:13000` because host `:3000` is taken; keep still owns archive workers. `better-sqlite3` must be rebuilt on Linux (`npm rebuild`). + ## 2026-09-11 — verae-keep on NS1 - New repo `verae-keep`: keep `:3860` restarts crashed units; watch `:3861` restarts keep; `guard.sh` restarts watch. diff --git a/packages/zapier-decisions/TODO.md b/packages/zapier-decisions/TODO.md index 54f966b..57b81c5 100644 --- a/packages/zapier-decisions/TODO.md +++ b/packages/zapier-decisions/TODO.md @@ -10,3 +10,5 @@ - [x] Turn IAM on for the running lab (fleet `STAFF_IAM_URL` + restart). - [x] IAM hardening: JSON sessions, login rate-limit, JSON 401 as well as HTML 302, credit principal = IAM username. - [x] Host keep-alive (`verae-keep`) on NS1 with watch+guard; honors admin pause/stop. +- [x] Docs synced to IAM, public portal `:3021`, keep, bootstrap server types. +- [x] `verae-bootstrap` clones all module repos and installs deps per server type. diff --git a/packages/zappier/docs/USER-MANUAL.md b/packages/zappier/docs/USER-MANUAL.md index 51c013b..babaccb 100644 --- a/packages/zappier/docs/USER-MANUAL.md +++ b/packages/zappier/docs/USER-MANUAL.md @@ -73,7 +73,7 @@ On first start the database is created and seeded with: | **Public API** | `http://localhost:3000/v1/*` | Your API customers | | **Interactive API docs** | `http://localhost:3000/docs` | Developers integrating with you | | **Admin console** | `http://localhost:3000/admin` | You (operations & accounting) | -| **Customer portal** | `http://localhost:3000/portal` | End-user customers (self-service) | +| **Customer portal** | `http://localhost:3021/portal/` (public door; loopback edge still serves `/portal` on `:3000`) | End-user customers (self-service) | | **Zapier app** | `zapier-app/` directory | No-code users via Zapier | ![Interactive API docs](screenshots/api-docs.png) @@ -191,7 +191,7 @@ snapshot. Full procedures: [ACCOUNTING.md](ACCOUNTING.md). ## 4A. Customer portal -Your customers self-serve at `http://localhost:3000/portal`: signup (or +Your customers self-serve at `http://localhost:3021/portal/` (access-web; loopback edge `/portal` remains on `:3000`): signup (or claiming an account you created, by email), sign-in with optional TOTP two-factor authentication, month-to-date usage, invoice history with print view, prepaid balance reloads (drawn down automatically at invoice issue), diff --git a/scripts/build-docs-site.py b/scripts/build-docs-site.py index 95eff37..e1113a1 100755 --- a/scripts/build-docs-site.py +++ b/scripts/build-docs-site.py @@ -42,6 +42,7 @@ SECTIONS = [ ("packages/overview/05-network-failures.md", "Local network failures"), ("packages/overview/06-address-routing.md", "Address routing"), ("packages/overview/08-diagrams.md", "Architectural diagrams"), + ("packages/overview/10-docs-sync.md", "Docs sync: portal, IAM, keep, bootstrap"), ], ), ( @@ -60,6 +61,11 @@ SECTIONS = [ ("packages/zapier-user-docs/README.md", "zapier-user-docs"), ("packages/docs-master/README.md", "zapier-docs-master"), ("packages/verae-nats-process/README.md", "verae-nats-process"), + ("packages/verae-staff-iam/README.md", "verae-staff-iam"), + ("packages/verae-staff-session/README.md", "verae-staff-session"), + ("packages/verae-staff-ui/README.md", "verae-staff-ui"), + ("packages/verae-keep/README.md", "verae-keep"), + ("packages/verae-bootstrap/README.md", "verae-bootstrap (clone + deps per server type)"), ], ), ( diff --git a/scripts/push-module-repos.sh b/scripts/push-module-repos.sh index 7411e9d..e2868de 100755 --- a/scripts/push-module-repos.sh +++ b/scripts/push-module-repos.sh @@ -69,6 +69,7 @@ create verae-staff-session "Shared staff cookie login for department HTML" create verae-staff-ui "Shared staff review HTML template" create verae-staff-iam "Internal staff users, roles, and permissions" create verae-keep "Host keep-alive: restart units unless admin paused or stopped them" +create verae-bootstrap "Clone all module repos and install deps for each server type" push_dir "$ROOT/packages/zappier" zappier-edge push_dir "$ROOT/packages/verae-zapier-middleware" verae-middleware @@ -104,5 +105,6 @@ push_dir "$ROOT/packages/verae-staff-session" verae-staff-session push_dir "$ROOT/packages/verae-staff-ui" verae-staff-ui push_dir "$ROOT/packages/verae-staff-iam" verae-staff-iam push_dir "$ROOT/packages/verae-keep" verae-keep +push_dir "$ROOT/packages/verae-bootstrap" verae-bootstrap echo ALL_MODULE_REPOS_PUSHED