From 9987a7b4c1f97d9871c0fe5fb2bba6fde7a9cc82 Mon Sep 17 00:00:00 2001 From: George Lambert Date: Fri, 11 Sep 2026 18:26:54 -0400 Subject: [PATCH] Initial import of verae-ops from zapier monorepo --- 01-dependencies.md | 64 ++++++++++ 02-docker.md | 54 ++++++++ 03-proxmox.md | 60 +++++++++ 04-virtual-servers.md | 28 +++++ 05-dedicated-hardware.md | 59 +++++++++ 06-linking-services.md | 57 +++++++++ 07-maintenance.md | 45 +++++++ GETTING-STARTED.md | 109 ++++++++++++++++ NATS.md | 10 ++ README.md | 45 +++++++ SUMMARY.md | 7 ++ docker-compose.yml | 260 +++++++++++++++++++++++++++++++++++++++ package.json | 6 + 13 files changed, 804 insertions(+) create mode 100644 01-dependencies.md create mode 100644 02-docker.md create mode 100644 03-proxmox.md create mode 100644 04-virtual-servers.md create mode 100644 05-dedicated-hardware.md create mode 100644 06-linking-services.md create mode 100644 07-maintenance.md create mode 100644 GETTING-STARTED.md create mode 100644 NATS.md create mode 100644 README.md create mode 100644 SUMMARY.md create mode 100644 docker-compose.yml create mode 100644 package.json diff --git a/01-dependencies.md b/01-dependencies.md new file mode 100644 index 0000000..4b33501 --- /dev/null +++ b/01-dependencies.md @@ -0,0 +1,64 @@ +# 1. Dependencies and data flow + +```text +Humans / Zapier cloud + │ HTTPS x-api-key + ▼ + zappier-edge :3000 (meter, portal, admin, Stripe) + │ HTTPS internal + ▼ + middleware-http :3100 (/zapier/v1) + │ + ├── HTTPS → api.veraetime.net (or MOCK_VERAE) + └── NATS → private cluster + jobs.watch / jobs.events + webhooks.deliver + archive.put / query / reply.* + │ + archive-worm × N tree-node × N + archive-aggregator job-poller + webhook-deliver +``` + +## Repositories (clone all that you run) + +Prefix `ssh://git@git.georgelambert.org:2223/marchon/.git` (SSH **2223**). + +| You need | Repo | Catalog README | +|----------|------|----------------| +| Billing HTTPS | `zappier-edge` | https://zapier.georgelambert.org/packages/zappier/README.pdf | +| Zapier adapter | `verae-middleware` | https://zapier.georgelambert.org/packages/verae-zapier-middleware/README.pdf | +| Zapier app (cloud) | `verae-zapier-app` | https://zapier.georgelambert.org/packages/verae-zapier/README.pdf | +| WORM nodes | `verae-archive-worm` | https://zapier.georgelambert.org/packages/verae-archive-worm/README.pdf | +| Merkle leaves | `verae-tree-node` | https://zapier.georgelambert.org/packages/verae-tree-node/README.pdf | +| Archive merge | `verae-archive-aggregator` | https://zapier.georgelambert.org/packages/verae-archive-aggregator/README.pdf | +| Splitter | `verae-request-splitter` | https://zapier.georgelambert.org/packages/verae-request-splitter/README.pdf | +| Replica floors | `verae-fleet` | https://zapier.georgelambert.org/packages/verae-fleet/README.pdf | +| This install guide | `verae-ops` | https://zapier.georgelambert.org/packages/verae-ops/README.pdf | + +Optional: `verae-activate` (tiny pushable app), `verae-zapier-simulator` (trace), `overview`, `zapier-docs-master`, `zapier-user-docs`, `verae-nats-process`. + +## Runtime dependencies + +| Service | Needs | +|---------|--------| +| zappier-edge | Node 20+, SQLite path or volume, `PORT`, optional Stripe keys | +| middleware | Node 22+, `NATS_URL` **or** `NATS_ENABLED=false`, chain URL or `MOCK_VERAE` | +| NATS | JetStream, **no public bind** | +| worm / tree-node | Node, `NATS_URL`, local state dir | +| fleet | Node, `machines.json` (SSH **paths**), HTTP `:3850` on `0.0.0.0` | +| Zapier cloud app | `MIDDLEWARE_BASE_URL` = **public HTTPS of zappier-edge** (or middleware if you skip metering in lab) | + +## Ports (lab defaults) + +| Port | Bind | Process | +|------|------|---------| +| 3000 | public or docker publish | zappier-edge | +| 3100 | public or docker publish | middleware | +| 4222 | **private only** | NATS client | +| 6222 | **private only** | NATS cluster | +| 8222 | private / localhost | NATS monitor | +| 3847 | 0.0.0.0 | simulator | +| 3850 | 0.0.0.0 | operator console | + +Chain: `https://api.veraetime.net` (prod) or `MOCK_VERAE=true` (lab). diff --git a/02-docker.md b/02-docker.md new file mode 100644 index 0000000..43f81b0 --- /dev/null +++ b/02-docker.md @@ -0,0 +1,54 @@ +# 2. Docker Compose + +`docker-compose.yml` in this repo starts: + +- NATS JetStream **3-node cluster** (`nats1`–`nats3`) on an internal network +- **zappier-edge** `:3000` +- **middleware** `:3100` with `NATS_URL=nats://nats1:4222` and `MOCK_VERAE=true` by default +- **archive-worm** ×3 and **tree-node** ×3 (fleet floors) as long-running Node workers +- **aggregator** + **job-poller** stand-ins via middleware workers when `NATS_ENABLED=true` + +NATS ports are **not** published to the host by default (private `verae` network). HTTPS edges are. + +## Prereqs + +Docker Engine 24+ and Compose v2. Clone **this repo** plus the application repos as siblings, **or** set `MONOREPO` to a checkout of `master-zapier-plan-draft`. + +```bash +export MONOREPO=/path/to/master-zapier-plan-draft # contains packages/ +docker compose up --build +curl -fsS http://127.0.0.1:3000/health +curl -fsS http://127.0.0.1:3100/health +``` + +Without `MONOREPO`, Compose expects `../zappier`, `../verae-zapier-middleware`, … (independent clones next to `verae-ops`). + +## Link Zapier (cloud) to the lab + +1. Publish zappier-edge on a TLS hostname (Caddy/nginx in front of `:3000`). +2. In the Zapier app env: `MIDDLEWARE_BASE_URL=https://` (edge meters, then proxies). +3. Sign up on `/portal`, copy `x-api-key`. + +## Scale + +```bash +docker compose up --scale archive-worm=3 --scale tree-node=3 +``` + +Do not publish `4222`. To debug NATS from the host: + +```bash +docker compose exec nats1 wget -qO- http://127.0.0.1:8222/healthz +``` + +## Operator console + +Fleet is **not** inside Compose by default (it SSH-spawns). On the Docker host: + +```bash +cd $MONOREPO/packages/verae-fleet +NATS_URL=nats://127.0.0.1:4222 # only if you published 4222 on localhost for lab +node src/cli.js serve # http://127.0.0.1:3850/ loopback +``` + +For Compose-only labs, skip fleet; replica counts come from `--scale`. diff --git a/03-proxmox.md b/03-proxmox.md new file mode 100644 index 0000000..736a147 --- /dev/null +++ b/03-proxmox.md @@ -0,0 +1,60 @@ +# 3. Proxmox (LXC and QEMU) + +Target: one **NATS cluster VM/CT** (or three), one **HTTPS edge** VM, one or more **worker** VMs. Fleet SSH-spreads worm/tree-node onto worker hosts. + +## Suggested guests + +| Guest | OS | vCPU / RAM / disk | Role | +|-------|----|-------------------|------| +| `nats-a` `nats-b` `nats-c` | Debian 12 LXC unprivileged | 1 / 1G / 8G | JetStream, bind `127.0.0.1` **or** a vmbr **private** bridge only | +| `edge` | Debian 12 QEMU | 2 / 2G / 20G | Caddy + zappier-edge + middleware | +| `worker-1` … | Debian 12 LXC | 2 / 2G / 20G | fleet SSH target; worm + tree-node | + +Use a **private bridge** (`vmbr1`, RFC1918) for NATS `4222`/`6222`. Public `vmbr0` only on `edge` (80/443). + +## NATS on an LXC + +```bash +apt-get update && apt-get install -y curl +# official nats-server binary +curl -sSL https://github.com/nats-io/nats-server/releases/latest/download/nats-server-v2.10.26-linux-amd64.tar.gz | tar -xz +install nats-server /usr/local/bin/ +``` + +`/etc/nats/nats.conf`: + +``` +jetstream {} +listen: 10.10.10.11:4222 # private bridge, not 0.0.0.0 on public +cluster { + name: verae + listen: 10.10.10.11:6222 + routes: [ nats-route://10.10.10.12:6222, nats-route://10.10.10.13:6222 ] +} +http: 127.0.0.1:8222 +``` + +systemd: `ExecStart=/usr/local/bin/nats-server -c /etc/nats/nats.conf`. + +## Edge VM + +Install Node 22, clone `zappier-edge` and `verae-middleware`, Caddy reverse-proxy: + +``` +zappier.example.com { + reverse_proxy 127.0.0.1:3000 +} +middleware.example.com { + reverse_proxy 127.0.0.1:3100 +} +``` + +Middleware env: `NATS_URL=nats://10.10.10.11:4222,nats://10.10.10.12:4222,nats://10.10.10.13:4222`. + +## Fleet on Proxmox + +On the operator workstation or a jump CT (loopback `:3850`): + +`machines.json` entries: `kind=ssh`, `user=root` or `marchon`, `host=10.10.10.2x`, `identityFile=~/.ssh/id_ed25519` (**path only**). Roles: `tree-node,archive-worm`. Capacity ≥ 3 per worker if you want floors on one host. + +`ssh-check` from the operator console before enabling `keepFloor`. diff --git a/04-virtual-servers.md b/04-virtual-servers.md new file mode 100644 index 0000000..915c39c --- /dev/null +++ b/04-virtual-servers.md @@ -0,0 +1,28 @@ +# 4. Virtual servers (cloud / hypervisor) + +Same topology as Proxmox, without the Proxmox UI: three small VMs or one VM for lab. + +## Single VM lab + +Install Docker, clone `verae-ops`, `docker compose up`. Put Caddy/nginx TLS in front of `:3000` and `:3100`. Do **not** open 4222 on the security group. + +## Three-VM prod-shaped + +| VM | Security group | Software | +|----|----------------|----------| +| nats | private SG only 4222/6222 between the three | nats-server -js cluster | +| edge | 80/443 public; 4222 egress to nats SG | zappier + middleware + Caddy | +| workers | private; SSH from operator IP | Node; fleet identity file | + +Cloud-init example (edge): + +```yaml +packages: [curl, git, ca-certificates] +runcmd: + - curl -fsSL https://deb.nodesource.com/setup_22.x | bash - + - apt-get install -y nodejs +``` + +Set `MIDDLEWARE_BASE_URL` in Zapier to the **public zappier hostname**. + +Snapshots: SQLite (`ZAPPIER_DB`) and worm/tree state directories. NATS JetStream store (`/var/lib/nats`). diff --git a/05-dedicated-hardware.md b/05-dedicated-hardware.md new file mode 100644 index 0000000..d9a3cb2 --- /dev/null +++ b/05-dedicated-hardware.md @@ -0,0 +1,59 @@ +# 5. Dedicated hardware + +NS1-style: a box that already runs NATS on **127.0.0.1:4222** (never a public bind), plus optional extra hosts for SSH workers. + +## This host (control + edges) + +1. Node 22+, git (SSH port **2223** to Forgejo). +2. `nats-server -js` listening **127.0.0.1:4222** only (cluster later on a private NIC). +3. Clone `zappier-edge`, `verae-middleware`, `verae-fleet`. +4. systemd units (below) or `verae-fleet serve` which keepFloor-spawns workers. +5. Caddy/nginx for public HTTPS to `:3000` / `:3100`. +6. Operator console: `127.0.0.1:3850` — SSH tunnel if you are off-box: + `ssh -L 3850:127.0.0.1:3850 -L 4222:127.0.0.1:4222 user@ns1` + +## systemd (middleware) + +``` +[Unit] +Description=verae-middleware +After=network.target nats.service + +[Service] +WorkingDirectory=/opt/verae/verae-middleware +Environment=PORT=3100 +Environment=NATS_ENABLED=true +Environment=NATS_URL=nats://127.0.0.1:4222 +Environment=VERAE_API_BASE_URL=https://api.veraetime.net +ExecStart=/usr/bin/node src/index.js +Restart=on-failure +User=verae + +[Install] +WantedBy=multi-user.target +``` + +zappier-edge similar with `PORT=3000` and `WorkingDirectory` of that clone. + +## Extra machines + +Add to fleet `machines.json`: + +```json +{ + "id": "ns1", + "kind": "ssh", + "user": "marchon", + "host": "70.88.205.138", + "port": 22, + "identityFile": "~/.ssh/id_ed25519", + "capacity": 8, + "roles": ["tree-node", "archive-worm"] +} +``` + +Never commit private key bytes. Optional `machines.secrets.json` is gitignored. + +## NATS 3-server cluster on metal + +Three boxes, private IPs, same `cluster_name`, routes to each other, `listen` on the private IP only. Clients (`NATS_URL`) list all three URLs. Target architecture: https://zapier.georgelambert.org/overview/03-nats-cluster.pdf diff --git a/06-linking-services.md b/06-linking-services.md new file mode 100644 index 0000000..4eb8241 --- /dev/null +++ b/06-linking-services.md @@ -0,0 +1,57 @@ +# 6. Linking dependent services + +Every process learns peers from **environment variables** and **fleet service JSON**. There is no service mesh. + +## Environment map + +| Process | Variable | Points at | +|---------|----------|-----------| +| Zapier cloud app | `MIDDLEWARE_BASE_URL` | Public **zappier-edge** HTTPS (prod) or middleware (lab) | +| zappier-edge | `ZAPPIER_UPSTREAM` or internal proxy target | middleware `:3100` | +| middleware | `NATS_URL` | `nats://nats1:4222,nats://nats2:4222,nats://nats3:4222` | +| middleware | `VERAE_API_BASE_URL` | `https://api.veraetime.net` | +| middleware | `MOCK_VERAE` | `true` only in lab | +| worm / tree-node / aggregator / poller | `NATS_URL` | same cluster | +| fleet | `NATS_URL` (monitor only) | same; **publicBind=false** | +| fleet SSH | `identityFile` | path on the operator disk | + +## HTTPS hops (must work) + +```text +curl -H "x-api-key: $KEY" https://zappier.example.com/v1/status +# edge meters, then +curl http://127.0.0.1:3100/health +``` + +If Zapier gets 401/402, fix keys and the rate card on zappier-edge — not NATS. + +## NATS hops (must stay private) + +| Address | From | To | +|---------|------|----| +| `verae.zapier.jobs.watch` | middleware | job-poller (queue) | +| `verae.zapier.jobs.events` | poller | middleware wait | +| `verae.zapier.webhooks.deliver` | middleware | webhook-deliver | +| `verae.archive.put` | splitter | worm / tree-node | +| `verae.archive.query` | aggregator | **all** worm/tree (broadcast) | +| `verae.archive.reply.` | worm/tree on **hit only** | aggregator | + +Bloom **miss = silence**. Do not “fix” that with a negative reply. + +## Fleet floors + +https://zapier.georgelambert.org/docs/02-architecture/fleet.pdf + +| Service | min (keepFloor) | +|---------|-----------------| +| archive-worm | 3 | +| tree-node | 3 | +| middleware-http | 1 | +| zappier-edge | 1 | +| aggregator / poller / webhook | 1 | + +Link a new host: operator console → Add machine → `ssh-check` → reconcile. + +## Adding an unplanned function + +Copy `verae-nats-process`, pick `verae...*`, add a fleet service, and if Zapier must see the result, add **one HTTPS route on middleware**. Do not teach Zapier NATS. https://zapier.georgelambert.org/overview/06-address-routing.pdf diff --git a/07-maintenance.md b/07-maintenance.md new file mode 100644 index 0000000..240f151 --- /dev/null +++ b/07-maintenance.md @@ -0,0 +1,45 @@ +# 7. Maintenance + +## Health + +| Check | Command | +|-------|---------| +| Edges | `curl -fsS https://zappier.example.com/health` and middleware `/health` | +| NATS | `curl -fsS http://127.0.0.1:8222/healthz` on each nats node (localhost) | +| Fleet | `node src/cli.js status` or http://127.0.0.1:3850/ (loopback) | +| Tree floor | status: tree-node available ≥ min, none of those paused | + +Green / yellow / red on the operator console: https://zapier.georgelambert.org/packages/verae-fleet/docs/CONSOLE.pdf + +## Restart + +- systemd: `systemctl restart verae-middleware zappier-edge` +- Compose: `docker compose up -d --no-deps middleware` +- One replica: fleet `restart tree-node-1` (floor spawns a replacement if needed) +- Pause is **not** off — paused copies do not count toward `keepFloor` + +## Backup + +| Data | Where | +|------|--------| +| zappier SQLite | `ZAPPIER_DB` path / volume | +| worm / tree blobs | instance `data/` dirs | +| NATS JetStream | nats store dir | +| fleet overlay | `packages/verae-fleet/data/` (gitignored) | + +Do not back up private keys into git. Restore `identityFile` paths on the operator host. + +## Upgrade + +1. `git pull` each independent repo (or refresh the monorepo snapshot). +2. `npm test` in that package. +3. Restart that unit only; NATS cluster stays up. +4. Zapier Platform app: `zapier-platform push` from `verae-zapier-app` when HTTPS contracts change. + +## Logs + +journald for systemd; `docker compose logs -f middleware nats1`. Redact API keys (middleware debug redaction). + +## Docs + +After doc edits: from the monorepo `npm run docs:site && npm run docs:deploy` so https://zapier.georgelambert.org/ stays current (PDF default, Markdown indexes at `/index-md.html`). diff --git a/GETTING-STARTED.md b/GETTING-STARTED.md new file mode 100644 index 0000000..b142c27 --- /dev/null +++ b/GETTING-STARTED.md @@ -0,0 +1,109 @@ +# Bring Verae Time × Zapier online + +This is the **top document** for install, first-up, and monitoring. Platform-specific detail lives in the numbered ops chapters; do not start there. + +**Catalog:** https://zapier.georgelambert.org/packages/verae-ops/GETTING-STARTED.pdf +**Billing / users (zappier-edge):** https://zapier.georgelambert.org/packages/zappier/docs/USER-MANUAL.pdf +**Operator console:** http://127.0.0.1:3850/ (loopback) · https://zapier.georgelambert.org/packages/verae-fleet/docs/CONSOLE.pdf + +## What you are turning on + +| Piece | Job | Default | +|-------|-----|---------| +| **access-authz** | NATS authorization step for every access plane | `:3020` | +| **access-web / api / leaf / zapier / staff** | Public HTTP doors | `:3021–:3025` | +| **identity** | `veraeUserId` bind/lookup | `:3026` | +| **zappier-edge** | Identity cache, Stripe, admin, portal (loopback only) | `127.0.0.1:3000` | +| **jobs-events** | Mailbox `verae.zapier.jobs.events` | `:3030` | +| **account-balance** | NATS prepaid balances (SoT) | `:3010` | +| **customer-service** | CS credit additions + review | `:3011` | +| **sales-pricing** | Per-customer sales quotes | `:3012` | +| **accounting-export** | QuickBooks IIF / CSV | `:3013` | +| **verae-middleware** | Zapier `/zapier/v1`, chain, NATS | `:3100` | +| **NATS JetStream** | Private jobs / archive | `:4222` **not public** | +| **worm ×3, tree-node ×3** | Off-chain files and Merkle leaves | fleet floor | +| **fleet** | Replica floors, SSH hosts, Trace, Docs | `:3850` on `0.0.0.0` | + +Zapier cloud talks **only** to **access-zapier** (`:3024`) over HTTPS. Browsers use access-web. Customer API keys use access-api. Staff use access-staff. zappier-edge is loopback. + +The billing and user platform is the standalone **zappier** project (`/Users/marchon/zappier`, Forgejo [zappier-edge](https://git.georgelambert.org/marchon/zappier-edge)): portal signup, TOTP, API keys, rate card, Stripe meter, PO invoices, admin users. Verae middleware does **not** replace it. + +## 15-minute lab (one machine) + +```bash +git clone ssh://git@git.georgelambert.org:2223/marchon/master-zapier-plan-draft.git +cd master-zapier-plan-draft +export MONOREPO="$PWD" +cd packages/verae-ops +docker compose up --build +``` + +Check: + +```bash +curl -fsS http://127.0.0.1:3000/health # zappier-edge (billing + users) +curl -fsS http://127.0.0.1:3010/health # account-balance (NATS SoT) +curl -fsS http://127.0.0.1:3011/health # customer-service +curl -fsS http://127.0.0.1:3012/health # sales-pricing +curl -fsS http://127.0.0.1:3100/health # middleware +``` + +Open: + +- Customer portal: http://127.0.0.1:3000/portal — sign up, copy `x-api-key`, **Statement** tab +- Admin: http://127.0.0.1:3000/admin — **Statement** per customer +- CS review: http://127.0.0.1:3011/ +- Sales review: http://127.0.0.1:3012/ +- Accounting review/export: http://127.0.0.1:3013/ +- API docs: http://127.0.0.1:3000/docs + +Do **not** publish NATS `4222`. + +## Operator console (monitor) + +On the same machine (or SSH tunnel `ssh -L 3850:127.0.0.1:3850 user@host`): + +```bash +cd "$MONOREPO/packages/verae-fleet" +node src/cli.js serve +``` + +http://127.0.0.1:3850/ — Fleet (green/yellow/red), Trace, Docs. Tree-node **min 3**; pause does not count. + +```bash +node src/cli.js status +node src/cli.js ssh-check ns1 +``` + +## Point Zapier at the lab + +1. TLS in front of `:3000` (Caddy). +2. Zapier app env: `MIDDLEWARE_BASE_URL=https://` (edge meters, then proxies to middleware). +3. Connect with the portal API key. + +## Production (pick one) + +| Where | Read | +|-------|------| +| Docker | [02-docker.pdf](02-docker.pdf) | +| Proxmox | [03-proxmox.pdf](03-proxmox.pdf) | +| Cloud VMs | [04-virtual-servers.pdf](04-virtual-servers.pdf) | +| Dedicated / NS1 | [05-dedicated-hardware.pdf](05-dedicated-hardware.pdf) | +| Env vars & NATS | [06-linking-services.pdf](06-linking-services.pdf) | +| Backup / upgrade | [07-maintenance.pdf](07-maintenance.pdf) | + +## Daily monitor + +| Check | Expect | +|-------|--------| +| `curl -fsS https://zappier.example.com/health` | 200 | +| `curl -fsS https://middleware.example.com/health` | 200 | +| NATS `http://127.0.0.1:8222/healthz` on each node | ok (localhost only) | +| Fleet tree-node available ≥ 3 | green | +| Portal login + key still works | 200 `/v1/status` with `x-api-key` | + +## If something is red + +1. Fleet row color: yellow = paused/degraded, red = below floor — `restart` that replica. +2. 401/402 from Zapier → zappier-edge keys and rate card, not NATS. +3. Archive lookup empty → bloom miss is **silence**; check worm/tree floors, not “error replies”. diff --git a/NATS.md b/NATS.md new file mode 100644 index 0000000..c485dd3 --- /dev/null +++ b/NATS.md @@ -0,0 +1,10 @@ +# NATS (ops) + +Ops does not subscribe. Runtime processes use: + +- `NATS_URL` — comma-separated client URLs on a **private** network +- JetStream required (`-js`) +- Cluster routes on port 6222 +- **No public bind** of 4222 + +See [06-linking-services.md](06-linking-services.md) and https://zapier.georgelambert.org/overview/03-nats-cluster.pdf diff --git a/README.md b/README.md new file mode 100644 index 0000000..ac6e99b --- /dev/null +++ b/README.md @@ -0,0 +1,45 @@ +# verae-ops — install, run, maintain + +How to stand up Verae Time × Zapier on **Docker**, **Proxmox**, **generic VMs**, or **dedicated hardware**, and how to **link dependent services**. + +**Forgejo:** https://git.georgelambert.org/marchon/verae-ops +**Clone:** `ssh://git@git.georgelambert.org:2223/marchon/verae-ops.git` +**Catalog:** https://zapier.georgelambert.org/packages/verae-ops/README.pdf +**System map:** https://zapier.georgelambert.org/overview/README.pdf + +This is the operations repo. Application code lives in the other independent git repositories. + +## Reading order + +0. **[GETTING-STARTED.md](GETTING-STARTED.md)** — install, first-up, monitor (read this first) +1. [01-dependencies.md](01-dependencies.md) — what must exist, who talks to whom +2. [02-docker.md](02-docker.md) — Compose (NATS 3-node cluster + HTTPS edges) +3. [03-proxmox.md](03-proxmox.md) — LXC / QEMU VMs +4. [04-virtual-servers.md](04-virtual-servers.md) — cloud or hypervisor VMs +5. [05-dedicated-hardware.md](05-dedicated-hardware.md) — bare metal (NS1-style) +6. [06-linking-services.md](06-linking-services.md) — env vars, URLs, replica floors +7. [07-maintenance.md](07-maintenance.md) — upgrade, backup, fleet, NATS + +Public PDFs of the same files: `https://zapier.georgelambert.org/packages/verae-ops/.pdf`. + +## Non-negotiables + +| Rule | Why | +|------|-----| +| Zapier cloud → **HTTPS only** (zappier-edge) | Never NATS, never `api.veraetime.net` | +| NATS binds **loopback or a private docker/VM net** | Not on the public NIC | +| Tree-node **min 3**, pause does not count | Bulk Merkle lookups | +| Private keys stay on disk; git stores **paths** | `machines.json` `identityFile` | +| Operator console HTTP binds **0.0.0.0:3850** | Not a public NATS bus; NATS stays private | + +## Quick lab (one machine) + +```bash +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-ops.git +cd verae-ops +docker compose up --build +``` + +Then: zappier http://127.0.0.1:3000/ middleware http://127.0.0.1:3100/health NATS monitoring http://127.0.0.1:8222/ + +Production layout is three NATS nodes + fleet-spread workers; see [02-docker.md](02-docker.md) and [05-dedicated-hardware.md](05-dedicated-hardware.md). diff --git a/SUMMARY.md b/SUMMARY.md new file mode 100644 index 0000000..3b26c60 --- /dev/null +++ b/SUMMARY.md @@ -0,0 +1,7 @@ +# verae-ops + +**Job:** Installation and maintenance: Docker Compose, Proxmox, virtual servers, dedicated hardware, and how to link dependent services. + +**Expects:** clones of the runtime repos (or the monorepo snapshot) and a private NATS network. + +**Sends:** nothing at runtime (documentation + compose file). diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..6c4b2db --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,260 @@ +# Lab / small-prod. NATS stays on the internal network (not published). +# Usage: +# MONOREPO=/path/to/master-zapier-plan-draft docker compose up --build +# or clone sibling repos next to this directory. + +x-node: &node + restart: unless-stopped + networks: [verae] + +services: + nats1: + image: nats:2.10-alpine + command: + - "--js" + - "--http_port=8222" + - "--cluster=nats://0.0.0.0:6222" + - "--cluster_name=verae" + - "--routes=nats://nats2:6222,nats://nats3:6222" + - "--name=nats1" + networks: [verae] + restart: unless-stopped + + nats2: + image: nats:2.10-alpine + command: + - "--js" + - "--cluster=nats://0.0.0.0:6222" + - "--cluster_name=verae" + - "--routes=nats://nats1:6222,nats://nats3:6222" + - "--name=nats2" + networks: [verae] + restart: unless-stopped + + nats3: + image: nats:2.10-alpine + command: + - "--js" + - "--cluster=nats://0.0.0.0:6222" + - "--cluster_name=verae" + - "--routes=nats://nats1:6222,nats://nats2:6222" + - "--name=nats3" + networks: [verae] + restart: unless-stopped + + zappier-edge: + <<: *node + build: + context: ${MONOREPO:-..}/packages/zappier + dockerfile: Dockerfile + environment: + PORT: "3000" + ZAPPIER_UPSTREAM: http://middleware:3100 + NATS_URL: nats://nats1:4222 + AUTHZ_URL: http://access-authz:3020 + ports: + - "3000:3000" + depends_on: [nats1, account-balance, access-authz] + + access-authz: + <<: *node + image: node:22-alpine + working_dir: /app + volumes: + - ${MONOREPO:-..}/packages/verae-access-authz:/app + command: ["node", "src/server.js"] + environment: + PORT: "3020" + NATS_URL: nats://nats1:4222 + ports: + - "3020:3020" + depends_on: [nats1] + + access-web: + <<: *node + image: node:22-alpine + working_dir: /app + volumes: + - ${MONOREPO:-..}/packages/verae-access-web:/app + command: ["node", "src/server.js"] + environment: + PORT: "3021" + AUTHZ_URL: http://access-authz:3020 + ACCOUNT_BALANCE_URL: http://account-balance:3010 + NATS_URL: nats://nats1:4222 + ports: + - "3021:3021" + depends_on: [access-authz, account-balance] + + access-api: + <<: *node + image: node:22-alpine + working_dir: /app + volumes: + - ${MONOREPO:-..}/packages/verae-access-api:/app + command: ["node", "src/server.js"] + environment: + PORT: "3022" + AUTHZ_URL: http://access-authz:3020 + ACCOUNT_BALANCE_URL: http://account-balance:3010 + ZAPPIER_UPSTREAM: http://middleware:3100 + NATS_URL: nats://nats1:4222 + ports: + - "3022:3022" + depends_on: [access-authz, middleware] + + access-leaf: + <<: *node + image: node:22-alpine + working_dir: /app + volumes: + - ${MONOREPO:-..}/packages/verae-access-leaf:/app + command: ["node", "src/server.js"] + environment: + PORT: "3023" + AUTHZ_URL: http://access-authz:3020 + LEAF_TOKEN: leaf-dev-token + NATS_URL: nats://nats1:4222 + ports: + - "3023:3023" + depends_on: [access-authz, nats1] + + access-zapier: + <<: *node + image: node:22-alpine + working_dir: /app + volumes: + - ${MONOREPO:-..}/packages/verae-access-zapier:/app + command: ["node", "src/server.js"] + environment: + PORT: "3024" + AUTHZ_URL: http://access-authz:3020 + ZAPPIER_UPSTREAM: http://middleware:3100 + NATS_URL: nats://nats1:4222 + ports: + - "3024:3024" + depends_on: [access-authz, middleware] + + account-balance: + <<: *node + image: node:22-alpine + working_dir: /app + volumes: + - ${MONOREPO:-..}/packages/zappier-account-balance:/app + command: ["node", "src/server.js"] + environment: + PORT: "3010" + NATS_URL: nats://nats1:4222 + ports: + - "3010:3010" + depends_on: [nats1] + + customer-service: + <<: *node + image: node:22-alpine + working_dir: /app + volumes: + - ${MONOREPO:-..}/packages/zappier-customer-service:/app + command: ["node", "src/server.js"] + environment: + PORT: "3011" + ZAPPIER_ADMIN_URL: http://zappier-edge:3000 + ACCOUNT_BALANCE_URL: http://account-balance:3010 + NATS_URL: nats://nats1:4222 + AUTHZ_URL: http://access-authz:3020 + ports: + - "3011:3011" + depends_on: [zappier-edge, account-balance, nats1] + + sales-pricing: + <<: *node + image: node:22-alpine + working_dir: /app + volumes: + - ${MONOREPO:-..}/packages/zappier-sales-pricing:/app + command: ["node", "src/server.js"] + environment: + PORT: "3012" + ZAPPIER_ADMIN_URL: http://zappier-edge:3000 + ACCOUNT_BALANCE_URL: http://account-balance:3010 + NATS_URL: nats://nats1:4222 + ports: + - "3012:3012" + depends_on: [zappier-edge, account-balance, nats1] + + accounting-export: + <<: *node + image: node:22-alpine + working_dir: /app + volumes: + - ${MONOREPO:-..}/packages/zappier-accounting-export:/app + command: ["node", "src/server.js"] + environment: + PORT: "3013" + ZAPPIER_ADMIN_URL: http://zappier-edge:3000 + ACCOUNT_BALANCE_URL: http://account-balance:3010 + NATS_URL: nats://nats1:4222 + ports: + - "3013:3013" + depends_on: [zappier-edge, account-balance, nats1] + + middleware: + <<: *node + build: + context: ${MONOREPO:-..}/packages/verae-zapier-middleware + dockerfile: Dockerfile + environment: + PORT: "3100" + MOCK_VERAE: "true" + NATS_ENABLED: "true" + NATS_URL: nats://nats1:4222 + ports: + - "3100:3100" + depends_on: [nats1, nats2, nats3] + + archive-worm: + <<: *node + image: node:22-alpine + working_dir: /opt/fleet + volumes: + - ${MONOREPO:-..}/packages/verae-fleet:/opt/fleet + - ${MONOREPO:-..}/packages/verae-archive-worm:/opt/worm + - worm-data:/data + command: ["node", "src/worker.js"] + environment: + NATS_URL: nats://nats1:4222 + FLEET_ROLE: archive-worm + FLEET_SERVICE: archive-worm + FLEET_HEALTH_BIND: 0.0.0.0 + FLEET_HEALTH_PORT: "13500" + FLEET_STATE_DIR: /data + deploy: + replicas: 3 + + tree-node: + <<: *node + image: node:22-alpine + working_dir: /opt/fleet + volumes: + - ${MONOREPO:-..}/packages/verae-fleet:/opt/fleet + - ${MONOREPO:-..}/packages/verae-tree-node:/opt/tree + - tree-data:/data + command: ["node", "src/worker.js"] + environment: + NATS_URL: nats://nats1:4222 + FLEET_ROLE: tree-node + FLEET_SERVICE: tree-node + FLEET_HEALTH_BIND: 0.0.0.0 + FLEET_HEALTH_PORT: "13600" + FLEET_STATE_DIR: /data + deploy: + replicas: 3 + +networks: + verae: + driver: bridge + internal: false + +volumes: + worm-data: + tree-data: diff --git a/package.json b/package.json new file mode 100644 index 0000000..510ecf9 --- /dev/null +++ b/package.json @@ -0,0 +1,6 @@ +{ + "name": "verae-ops", + "version": "0.1.0", + "private": true, + "description": "Install, Docker, Proxmox, VM, dedicated hardware, and service linking for Verae Time × Zapier" +}