diff --git a/docs/02-architecture/fleet.md b/docs/02-architecture/fleet.md new file mode 100644 index 0000000..1207b3b --- /dev/null +++ b/docs/02-architecture/fleet.md @@ -0,0 +1,21 @@ +# Fleet: replica floors, monitor, pause, restart + +Runtime copies of middleware workers, WORM archives, and **tree nodes** are declared in `packages/verae-fleet/fleet.json` (how many) and `packages/verae-fleet/services/*.json` (how each one runs). + +```text +operator --HTTP 127.0.0.1:3850--> fleet control + | spawn workers with /health + tree-node min=3 keepFloor | + archive-worm min=3 | + paused ≠ available | + unhealthy → restart v + replicas on loopback health ports +``` + +- **list** — `node src/cli.js list` +- **monitor** — `serve` probes `/health` and reconciles +- **restart** — crash or 503 → same instance id respawned +- **pause / off** — pause does not count toward `min`; keepFloor starts another tree-node. `stop` on a service disables it. +- **tree-node floor** — `fleet.json` `tree-node.min` (default 3). Do not drop this without changing the spec; bulk-summary leaf queries need several bloom-filtered nodes. + +Zapier cloud is not spawned. NATS is monitored, not bound publicly. diff --git a/docs/02-architecture/modules-and-nats.md b/docs/02-architecture/modules-and-nats.md index d27d4ff..07a26bf 100644 --- a/docs/02-architecture/modules-and-nats.md +++ b/docs/02-architecture/modules-and-nats.md @@ -34,6 +34,7 @@ Zapier cloud **never** connects to NATS. Only middleware workers and archives do | archive-worm | `verae-archive-worm` | N copies | `verae.archive.query`, `verae.archive.put` | `verae.archive.reply.` if bloom hits | | tree-node | `verae-tree-node` | N copies (WORM role) | `verae.archive.query`, `verae.archive.put` kind `tree` | `verae.archive.reply.` if bloom hits | | zapier-simulator | `verae-zapier-simulator` | Local HTTP :3847 | operator browser | in-process replay of all addresses | +| fleet | `verae-fleet` | Local HTTP :3850 | operator | spawns workers; keepFloor on tree-node | | zapier-user-docs | `zapier-user-docs` | Static | — | catalog `/user-docs/` | | verae-chain-client | `verae-chain-client` | Library | — | HTTPS `api.veraetime.net` or MOCK | | docs-master | `zapier-docs-master` | Static | — | published on zapier.georgelambert.org | diff --git a/packages/docs-master/README.md b/packages/docs-master/README.md index 1f3f5e2..c5c732f 100644 --- a/packages/docs-master/README.md +++ b/packages/docs-master/README.md @@ -20,6 +20,7 @@ Summaries, NATS contracts, and message flows for every Verae Time × Zapier modu | verae-tree-node | https://git.georgelambert.org/marchon/verae-tree-node | | verae-zapier-simulator | https://git.georgelambert.org/marchon/verae-zapier-simulator | | zapier-user-docs | https://git.georgelambert.org/marchon/zapier-user-docs | +| verae-fleet | https://git.georgelambert.org/marchon/verae-fleet | | **zapier-docs-master** (this repo) | https://git.georgelambert.org/marchon/zapier-docs-master | Clone (SSH port 2223): @@ -40,6 +41,7 @@ Clone (SSH port 2223): | tree-node | Merkle leaf proofs | archive.query, archive.put | archive.reply.* (hit only) | | zapier-simulator | Trace console (in-process) | — | — | | zapier-user-docs | Signup → lookup guide | — | — | +| verae-fleet | Replica floors + monitor | — | — | ## Documents in this repo diff --git a/packages/docs-master/modules/verae-fleet/NATS.md b/packages/docs-master/modules/verae-fleet/NATS.md new file mode 100644 index 0000000..bd95c58 --- /dev/null +++ b/packages/docs-master/modules/verae-fleet/NATS.md @@ -0,0 +1,3 @@ +# NATS — verae-fleet + +Does not subscribe. Records which workers should. NATS remains loopback. diff --git a/packages/docs-master/modules/verae-fleet/SUMMARY.md b/packages/docs-master/modules/verae-fleet/SUMMARY.md new file mode 100644 index 0000000..0fb54b9 --- /dev/null +++ b/packages/docs-master/modules/verae-fleet/SUMMARY.md @@ -0,0 +1,5 @@ +# verae-fleet + +**Job:** Service catalog, per-service configs, monitor, restart, pause/on/off, keep tree-node replica floor. + +**Config:** `fleet.json` (min/max) + `services/.json`. diff --git a/packages/verae-fleet/.gitignore b/packages/verae-fleet/.gitignore new file mode 100644 index 0000000..8fce603 --- /dev/null +++ b/packages/verae-fleet/.gitignore @@ -0,0 +1 @@ +data/ diff --git a/packages/verae-fleet/NATS.md b/packages/verae-fleet/NATS.md new file mode 100644 index 0000000..73525c7 --- /dev/null +++ b/packages/verae-fleet/NATS.md @@ -0,0 +1,15 @@ +# NATS — verae-fleet + +Fleet control **does not subscribe** to JetStream. It only records which workers should. + +| Service | NATS in | NATS out | +|---------|---------|----------| +| nats (external) | — | — (the broker) | +| middleware-http | jobs.events | jobs.watch | +| job-poller | jobs.watch | jobs.events | +| webhook-deliver | webhooks.deliver, jobs.events | HTTPS to Zapier | +| archive-aggregator | archive.reply.* | archive.query (broadcast) | +| archive-worm | archive.put, archive.query | archive.reply.* on bloom hit | +| tree-node | archive.put, archive.query | archive.reply.* on bloom hit | + +NS1 nats-server stays on `127.0.0.1:4222`. `fleet.json` `nats.publicBind` is false. diff --git a/packages/verae-fleet/README.md b/packages/verae-fleet/README.md new file mode 100644 index 0000000..b60502e --- /dev/null +++ b/packages/verae-fleet/README.md @@ -0,0 +1,32 @@ +# verae-fleet + +Operator control plane for Verae Time × Zapier **runtime** services: + +- a **list** of every service and its config file +- a **central replica spec** (`fleet.json`) — how many copies must be up +- a **monitor** of active / paused / unhealthy replicas +- **restart** when a copy is offline or fails `/health` +- **on / off / pause / resume** per service or per instance +- **keepFloor** on tree nodes so at least `min` copies stay available (paused copies do not count) + +```bash +cd packages/verae-fleet +npm test +node src/cli.js list +node src/cli.js serve # http://127.0.0.1:3850/ +``` + +Against a running daemon: + +```bash +node src/cli.js status +node src/cli.js pause tree-node-0 # floor starts another tree-node +node src/cli.js resume tree-node-0 +node src/cli.js restart tree-node-1 +node src/cli.js stop webhook-deliver # disable that service +node src/cli.js start webhook-deliver +``` + +Zapier cloud apps are listed but **not spawned**. NATS on NS1 is **monitored only** (loopback `:4222`, never a public bind). + +Clone: `ssh://git@git.georgelambert.org:2223/marchon/verae-fleet.git` diff --git a/packages/verae-fleet/SERVICES.md b/packages/verae-fleet/SERVICES.md new file mode 100644 index 0000000..3ee2c11 --- /dev/null +++ b/packages/verae-fleet/SERVICES.md @@ -0,0 +1,22 @@ +# Service list + +Central replica spec: [`fleet.json`](fleet.json). +Per-service files: [`services/`](services/). + +| Service | Config | min | max | keepFloor | Spawned by fleet | +|---------|--------|-----|-----|-----------|------------------| +| nats | `services/nats.json` | 1 | 1 | no | no (external monitor) | +| zappier-edge | `services/zappier-edge.json` | 1 | 1 | yes | yes | +| middleware-http | `services/middleware-http.json` | 1 | 1 | yes | yes | +| job-poller | `services/job-poller.json` | 1 | 2 | yes | yes | +| webhook-deliver | `services/webhook-deliver.json` | 1 | 2 | yes | yes | +| archive-aggregator | `services/archive-aggregator.json` | 1 | 2 | yes | yes | +| archive-worm | `services/archive-worm.json` | 3 | 6 | yes | yes | +| **tree-node** | `services/tree-node.json` | **3** | 9 | **yes** | yes | +| zapier-simulator | `services/zapier-simulator.json` | 0 | 1 | no | optional | +| zapier-platform-app | `services/unmanaged.json` | — | — | — | no (Zapier cloud) | +| verae-activate | `services/unmanaged.json` | — | — | — | no | +| request-splitter | `services/unmanaged.json` | — | — | — | no (library) | +| verae-chain-client | `services/unmanaged.json` | — | — | — | no (library) | + +`keepFloor`: monitor starts replacements when **available** (running, healthy, not paused) drops below `min`. That is how tree nodes stay at three live copies if one is paused, crashed, or not responding. diff --git a/packages/verae-fleet/SUMMARY.md b/packages/verae-fleet/SUMMARY.md new file mode 100644 index 0000000..162fd11 --- /dev/null +++ b/packages/verae-fleet/SUMMARY.md @@ -0,0 +1,9 @@ +# verae-fleet + +**Job:** Catalog services, hold per-service configs, monitor health, restart bad replicas, pause/resume/on/off, and keep `fleet.json` replica floors — especially tree-node `min`. + +**Expects:** operator CLI/HTTP on `127.0.0.1:3850`. Spawns `src/worker.js` children with `/health`. + +**Sends:** nothing to Zapier or public NATS. + +**Test:** `npm test` — list configs; start 3 tree-nodes; pause one still ≥3 available; unhealthy restart; stop+replace. diff --git a/packages/verae-fleet/fleet.json b/packages/verae-fleet/fleet.json new file mode 100644 index 0000000..afc9c94 --- /dev/null +++ b/packages/verae-fleet/fleet.json @@ -0,0 +1,26 @@ +{ + "apiVersion": "verae.fleet/v1", + "control": { + "bind": "127.0.0.1", + "port": 3850, + "probeIntervalMs": 750, + "unhealthyAfterMs": 2500, + "restartBackoffMs": [200, 800, 2000], + "maxRestartsPerHour": 40 + }, + "nats": { + "url": "nats://127.0.0.1:4222", + "publicBind": false + }, + "services": { + "nats": { "min": 1, "max": 1, "keepFloor": false, "enabled": true, "managed": false }, + "zappier-edge": { "min": 1, "max": 1, "keepFloor": true, "enabled": true }, + "middleware-http": { "min": 1, "max": 1, "keepFloor": true, "enabled": true }, + "job-poller": { "min": 1, "max": 2, "keepFloor": true, "enabled": true }, + "webhook-deliver": { "min": 1, "max": 2, "keepFloor": true, "enabled": true }, + "archive-aggregator": { "min": 1, "max": 2, "keepFloor": true, "enabled": true }, + "archive-worm": { "min": 3, "max": 6, "keepFloor": true, "enabled": true }, + "tree-node": { "min": 3, "max": 9, "keepFloor": true, "enabled": true }, + "zapier-simulator": { "min": 0, "max": 1, "keepFloor": false, "enabled": true } + } +} diff --git a/packages/verae-fleet/package.json b/packages/verae-fleet/package.json new file mode 100644 index 0000000..6e77001 --- /dev/null +++ b/packages/verae-fleet/package.json @@ -0,0 +1,13 @@ +{ + "name": "verae-fleet", + "version": "0.1.0", + "type": "module", + "description": "Service catalog, replica floors, monitor, restart, pause/resume for Verae Zapier workers and tree nodes", + "bin": { "verae-fleet": "./src/cli.js" }, + "scripts": { + "start": "node src/cli.js serve", + "list": "node src/cli.js list", + "test": "node --test test/*.test.js" + }, + "engines": { "node": ">=20" } +} diff --git a/packages/verae-fleet/public/index.html b/packages/verae-fleet/public/index.html new file mode 100644 index 0000000..460731f --- /dev/null +++ b/packages/verae-fleet/public/index.html @@ -0,0 +1,76 @@ + + + + + + Verae fleet monitor + + + +
+

Verae fleet monitor

+

Active replicas, replica floors, pause / restart. Tree-node keepFloor respawns until min copies are healthy and unpaused. Loopback only.

+
+
+

+ + + +
ServiceConfigmin/maxavailableinstancesactions
+
+ + + diff --git a/packages/verae-fleet/services/archive-aggregator.json b/packages/verae-fleet/services/archive-aggregator.json new file mode 100644 index 0000000..2c024d2 --- /dev/null +++ b/packages/verae-fleet/services/archive-aggregator.json @@ -0,0 +1,16 @@ +{ + "id": "archive-aggregator", + "title": "Archive reply aggregator", + "kind": "nats-worker", + "package": "verae-archive-aggregator", + "role": "archive-aggregator", + "managed": true, + "runtime": "query fan-out + merge", + "health": { "type": "http", "path": "/health", "timeoutMs": 800 }, + "ports": { "healthBase": 13400 }, + "nats": { + "in": ["verae.archive.reply.*"], + "out": ["verae.archive.query"] + }, + "notes": "Broadcast archive.query (no queue group). kinds may include tree." +} diff --git a/packages/verae-fleet/services/archive-worm.json b/packages/verae-fleet/services/archive-worm.json new file mode 100644 index 0000000..dd3360c --- /dev/null +++ b/packages/verae-fleet/services/archive-worm.json @@ -0,0 +1,16 @@ +{ + "id": "archive-worm", + "title": "Bloom-filtered WORM archive", + "kind": "nats-archive", + "package": "verae-archive-worm", + "role": "archive-worm", + "managed": true, + "runtime": "N copies, bloom miss = silence", + "health": { "type": "http", "path": "/health", "timeoutMs": 800 }, + "ports": { "healthBase": 13500 }, + "nats": { + "in": ["verae.archive.put", "verae.archive.query"], + "out": ["verae.archive.reply."] + }, + "notes": "Subscribe to query without a shared queue group." +} diff --git a/packages/verae-fleet/services/job-poller.json b/packages/verae-fleet/services/job-poller.json new file mode 100644 index 0000000..0e78df1 --- /dev/null +++ b/packages/verae-fleet/services/job-poller.json @@ -0,0 +1,16 @@ +{ + "id": "job-poller", + "title": "Job poller worker", + "kind": "nats-worker", + "package": "verae-zapier-middleware", + "role": "job-poller", + "managed": true, + "runtime": "JetStream consumer", + "health": { "type": "http", "path": "/health", "timeoutMs": 800 }, + "ports": { "healthBase": 13200 }, + "nats": { + "in": ["verae.zapier.jobs.watch"], + "out": ["verae.zapier.jobs.events"] + }, + "notes": "GET chain status; emit terminal events. Queue group job-poller." +} diff --git a/packages/verae-fleet/services/middleware-http.json b/packages/verae-fleet/services/middleware-http.json new file mode 100644 index 0000000..5b819d2 --- /dev/null +++ b/packages/verae-fleet/services/middleware-http.json @@ -0,0 +1,17 @@ +{ + "id": "middleware-http", + "title": "Verae Zapier HTTP middleware", + "kind": "http", + "package": "verae-zapier-middleware", + "role": "middleware-http", + "managed": true, + "runtime": "HTTPS :3100 /zapier/v1", + "health": { "type": "http", "path": "/health", "timeoutMs": 800 }, + "ports": { "healthBase": 13100 }, + "env": { "PORT": "3100", "NATS_URL": "nats://127.0.0.1:4222" }, + "nats": { + "in": ["verae.zapier.jobs.events"], + "out": ["verae.zapier.jobs.watch", "verae.zapier.webhooks.deliver"] + }, + "notes": "Wait path subscribes jobs.events. Never expose NATS to Zapier." +} diff --git a/packages/verae-fleet/services/nats.json b/packages/verae-fleet/services/nats.json new file mode 100644 index 0000000..60dcb37 --- /dev/null +++ b/packages/verae-fleet/services/nats.json @@ -0,0 +1,11 @@ +{ + "id": "nats", + "title": "NATS JetStream (NS1 loopback)", + "kind": "external", + "package": null, + "managed": false, + "runtime": "nats-server -js on 127.0.0.1:4222", + "health": { "type": "tcp", "host": "127.0.0.1", "port": 4222, "timeoutMs": 400 }, + "nats": { "in": [], "out": [] }, + "notes": "Do not bind 0.0.0.0. Zapier never connects here. Fleet monitors only; does not spawn nats-server." +} diff --git a/packages/verae-fleet/services/tree-node.json b/packages/verae-fleet/services/tree-node.json new file mode 100644 index 0000000..0c85964 --- /dev/null +++ b/packages/verae-fleet/services/tree-node.json @@ -0,0 +1,16 @@ +{ + "id": "tree-node", + "title": "Merkle tree-node archive", + "kind": "nats-archive", + "package": "verae-tree-node", + "role": "tree-node", + "managed": true, + "runtime": "N copies holding leaf proofs for bulk summaries", + "health": { "type": "http", "path": "/health", "timeoutMs": 800 }, + "ports": { "healthBase": 13600 }, + "nats": { + "in": ["verae.archive.put", "verae.archive.query"], + "out": ["verae.archive.reply."] + }, + "notes": "keepFloor in fleet.json must stay >= 3 so a leaf query still has a node that might hold the shard. Bloom miss = no packet." +} diff --git a/packages/verae-fleet/services/unmanaged.json b/packages/verae-fleet/services/unmanaged.json new file mode 100644 index 0000000..df7e1bf --- /dev/null +++ b/packages/verae-fleet/services/unmanaged.json @@ -0,0 +1,37 @@ +{ + "id": "_unmanaged", + "title": "Not fleet-spawned (listed for operators)", + "entries": [ + { + "id": "zapier-platform-app", + "package": "verae-zapier", + "runtime": "Zapier cloud", + "notes": "HTTPS only to zappier-edge. Never NATS." + }, + { + "id": "verae-activate", + "package": "verae-activate", + "runtime": "Zapier cloud (activate-now app)" + }, + { + "id": "request-splitter", + "package": "verae-request-splitter", + "runtime": "library in middleware" + }, + { + "id": "verae-chain-client", + "package": "verae-zapier-middleware", + "runtime": "library → api.veraetime.net or MOCK" + }, + { + "id": "zapier-user-docs", + "package": "zapier-user-docs", + "runtime": "static" + }, + { + "id": "docs-master", + "package": "docs-master", + "runtime": "static" + } + ] +} diff --git a/packages/verae-fleet/services/webhook-deliver.json b/packages/verae-fleet/services/webhook-deliver.json new file mode 100644 index 0000000..6c058e3 --- /dev/null +++ b/packages/verae-fleet/services/webhook-deliver.json @@ -0,0 +1,16 @@ +{ + "id": "webhook-deliver", + "title": "Zapier REST Hook deliverer", + "kind": "nats-worker", + "package": "verae-zapier-middleware", + "role": "webhook-deliver", + "managed": true, + "runtime": "JetStream consumer + HTTPS POST", + "health": { "type": "http", "path": "/health", "timeoutMs": 800 }, + "ports": { "healthBase": 13300 }, + "nats": { + "in": ["verae.zapier.webhooks.deliver", "verae.zapier.jobs.events"], + "out": [] + }, + "notes": "HTTPS POST to Zapier hook URL. Not a NATS client in Zapier cloud." +} diff --git a/packages/verae-fleet/services/zapier-simulator.json b/packages/verae-fleet/services/zapier-simulator.json new file mode 100644 index 0000000..e454710 --- /dev/null +++ b/packages/verae-fleet/services/zapier-simulator.json @@ -0,0 +1,14 @@ +{ + "id": "zapier-simulator", + "title": "Zapier interface simulator + trace console", + "kind": "http", + "package": "verae-zapier-simulator", + "role": "zapier-simulator", + "managed": true, + "runtime": "HTTP 127.0.0.1:3847", + "health": { "type": "http", "path": "/health", "timeoutMs": 800 }, + "ports": { "healthBase": 13847 }, + "env": { "SIM_PORT": "3847" }, + "nats": { "in": [], "out": [] }, + "notes": "Optional. min 0 in central fleet.json. Does not talk to live NATS." +} diff --git a/packages/verae-fleet/services/zappier-edge.json b/packages/verae-fleet/services/zappier-edge.json new file mode 100644 index 0000000..a43ca91 --- /dev/null +++ b/packages/verae-fleet/services/zappier-edge.json @@ -0,0 +1,14 @@ +{ + "id": "zappier-edge", + "title": "Zappier commercial HTTPS edge", + "kind": "http", + "package": "zappier", + "role": "zappier-edge", + "managed": true, + "runtime": "HTTPS :3000 (portal, admin, meter)", + "health": { "type": "http", "path": "/health", "timeoutMs": 800 }, + "ports": { "healthBase": 13000 }, + "env": { "PORT": "3000" }, + "nats": { "in": [], "out": [] }, + "notes": "Zapier x-api-key lands here. HTTPS to middleware only." +} diff --git a/packages/verae-fleet/src/cli.js b/packages/verae-fleet/src/cli.js new file mode 100644 index 0000000..a8c4127 --- /dev/null +++ b/packages/verae-fleet/src/cli.js @@ -0,0 +1,129 @@ +#!/usr/bin/env node +/** + * verae-fleet — catalog, monitor, restart, pause. + * + * Local (no daemon): list + * Daemon: serve | monitor + * Against daemon: status | start | stop | pause | resume | restart + */ +import http from 'node:http'; +import { loadFleet, listServices } from './load.js'; +import { Supervisor } from './supervisor.js'; +import { Monitor } from './monitor.js'; +import { startControlServer } from './server.js'; + +const loaded = loadFleet(); +const [cmd = 'help', target] = process.argv.slice(2); +const BASE = process.env.FLEET_URL || `http://127.0.0.1:${loaded.control.port}`; + +function print(obj) { + process.stdout.write(`${typeof obj === 'string' ? obj : JSON.stringify(obj, null, 2)}\n`); +} + +function help() { + print(`verae-fleet + + list services, config files, replica min/max + serve start floors + monitor + UI ${BASE}/ + monitor same without printing extra + status GET daemon status + start turn on, spawn up to min + stop turn off (service disable) or stop one replica + pause pause (does not count toward tree-node floor) + resume + restart kill + respawn + reconcile force floor check + +Central spec: fleet.json +Per service: services/.json +`); +} + +function api(method, pathname) { + return new Promise((resolve, reject) => { + const u = new URL(pathname, BASE); + const req = http.request( + { hostname: u.hostname, port: u.port, path: u.pathname, method, timeout: 8000 }, + (res) => { + const chunks = []; + res.on('data', (c) => chunks.push(c)); + res.on('end', () => { + const raw = Buffer.concat(chunks).toString('utf8'); + try { + resolve({ status: res.statusCode, body: JSON.parse(raw) }); + } catch { + resolve({ status: res.statusCode, body: raw }); + } + }); + }, + ); + req.on('error', reject); + req.end(); + }); +} + +async function main() { + if (cmd === 'help' || cmd === '-h' || cmd === '--help') return help(); + if (cmd === 'list') { + for (const r of listServices(loaded)) { + print( + `${r.id.padEnd(22)} min=${String(r.min).padStart(2)} max=${String(r.max).padStart(2)} floor=${r.keepFloor ? 'yes' : 'no '} managed=${r.managed ? 'yes' : 'no '} ${r.configPath}`, + ); + } + print(`\ncentral replica spec: ${loaded.fleetPath}`); + print(`tree-node floor: min=${loaded.services['tree-node'].min} keepFloor=${loaded.services['tree-node'].keepFloor}`); + return; + } + if (cmd === 'serve' || cmd === 'monitor') { + const sup = new Supervisor({ loaded }); + for (const id of Object.keys(loaded.services)) { + const s = loaded.services[id]; + if (s.managed && s.enabled && s.min > 0) await sup.startService(id); + } + const mon = new Monitor(sup); + await mon.tick(); + mon.start(); + startControlServer(sup, mon); + print(`fleet ${cmd} pid=${process.pid}\nUI ${BASE}/\nSIGINT stops all managed replicas`); + process.on('SIGINT', async () => { + mon.stop(); + await sup.stopAll(); + process.exit(0); + }); + process.on('SIGTERM', async () => { + mon.stop(); + await sup.stopAll(); + process.exit(0); + }); + return; + } + + const instance = target && /-\d+$/.test(target); + const map = { + status: ['GET', '/api/status'], + reconcile: ['POST', '/api/reconcile'], + }; + if (map[cmd] && !target) { + const [m, p] = map[cmd]; + const r = await api(m, p); + print(r.body); + if (r.status >= 400) process.exitCode = 1; + return; + } + if (!target) { + help(); + process.exitCode = 1; + return; + } + const pathName = instance + ? `/api/instances/${target}/${cmd}` + : `/api/services/${target}/${cmd}`; + const r = await api('POST', pathName); + print(r.body); + if (r.status >= 400) process.exitCode = 1; +} + +main().catch((err) => { + print({ error: err.message, hint: 'Is `verae-fleet serve` running?' }); + process.exitCode = 1; +}); diff --git a/packages/verae-fleet/src/health.js b/packages/verae-fleet/src/health.js new file mode 100644 index 0000000..38755cd --- /dev/null +++ b/packages/verae-fleet/src/health.js @@ -0,0 +1,90 @@ +/** + * Probe a replica: HTTP /health or TCP connect. + * @module health + */ + +import http from 'node:http'; +import net from 'node:net'; +import fs from 'node:fs'; +import path from 'node:path'; + +export function httpProbe(port, pathname = '/health', timeoutMs = 800) { + return new Promise((resolve) => { + const req = http.get( + { host: '127.0.0.1', port, path: pathname, timeout: timeoutMs }, + (res) => { + const chunks = []; + res.on('data', (c) => chunks.push(c)); + res.on('end', () => { + let body = {}; + try { + body = JSON.parse(Buffer.concat(chunks).toString('utf8') || '{}'); + } catch { + body = {}; + } + resolve({ + ok: res.statusCode >= 200 && res.statusCode < 300 && body.ok !== false, + statusCode: res.statusCode, + paused: Boolean(body.paused), + body, + }); + }); + }, + ); + req.on('error', (err) => resolve({ ok: false, error: err.message, paused: false })); + req.on('timeout', () => { + req.destroy(); + resolve({ ok: false, error: 'timeout', paused: false }); + }); + }); +} + +export function tcpProbe(host, port, timeoutMs = 400) { + return new Promise((resolve) => { + const sock = net.connect({ host, port }); + const done = (ok, error) => { + try { + sock.destroy(); + } catch { + /* ignore */ + } + resolve({ ok, error, paused: false }); + }; + sock.setTimeout(timeoutMs); + sock.on('connect', () => done(true)); + sock.on('error', (err) => done(false, err.message)); + sock.on('timeout', () => done(false, 'timeout')); + }); +} + +export function postControl(port, pathname) { + return new Promise((resolve, reject) => { + const req = http.request( + { host: '127.0.0.1', port, path: pathname, method: 'POST', timeout: 800 }, + (res) => { + const chunks = []; + res.on('data', (c) => chunks.push(c)); + res.on('end', () => { + try { + resolve(JSON.parse(Buffer.concat(chunks).toString('utf8') || '{}')); + } catch { + resolve({}); + } + }); + }, + ); + req.on('error', reject); + req.end(); + }); +} + +export function heartbeatFresh(stateDir, maxAgeMs) { + const file = path.join(stateDir, 'heartbeat.json'); + try { + const st = fs.statSync(file); + const body = JSON.parse(fs.readFileSync(file, 'utf8')); + return { ok: Date.now() - st.mtimeMs < maxAgeMs && body.ok !== false, body, ageMs: Date.now() - st.mtimeMs }; + } catch (err) { + return { ok: false, error: err.message }; + } +} diff --git a/packages/verae-fleet/src/load.js b/packages/verae-fleet/src/load.js new file mode 100644 index 0000000..d8ed9da --- /dev/null +++ b/packages/verae-fleet/src/load.js @@ -0,0 +1,87 @@ +/** + * Load central fleet.json + per-service JSON configs. + * @module load + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +export const FLEET_ROOT = path.resolve(HERE, '..'); + +export function readJson(file) { + return JSON.parse(fs.readFileSync(file, 'utf8')); +} + +/** + * @param {string} [root] + */ +export function loadFleet(root = FLEET_ROOT) { + const fleetPath = path.join(root, 'fleet.json'); + const fleet = readJson(fleetPath); + const dir = path.join(root, 'services'); + const services = {}; + const unmanaged = []; + for (const name of fs.readdirSync(dir).sort()) { + if (!name.endsWith('.json')) continue; + const spec = readJson(path.join(dir, name)); + const configPath = path.join('services', name); + if (spec.id === '_unmanaged') { + for (const e of spec.entries || []) { + unmanaged.push({ ...e, managed: false, configPath }); + } + continue; + } + const central = fleet.services?.[spec.id] || {}; + services[spec.id] = { + ...spec, + configPath, + min: central.min ?? 0, + max: central.max ?? 1, + keepFloor: central.keepFloor !== false, + enabled: central.enabled !== false, + managed: central.managed === false ? false : spec.managed !== false, + }; + } + return { + root, + fleetPath: path.relative(root, fleetPath) || 'fleet.json', + control: fleet.control, + nats: fleet.nats, + services, + unmanaged, + }; +} + +export function listServices(loaded) { + const rows = Object.values(loaded.services).map((s) => ({ + id: s.id, + title: s.title, + kind: s.kind, + managed: s.managed, + enabled: s.enabled, + min: s.min, + max: s.max, + keepFloor: s.keepFloor, + configPath: s.configPath, + package: s.package, + runtime: s.runtime, + })); + for (const u of loaded.unmanaged) { + rows.push({ + id: u.id, + title: u.id, + kind: 'unmanaged', + managed: false, + enabled: false, + min: 0, + max: 0, + keepFloor: false, + configPath: u.configPath, + package: u.package, + runtime: u.runtime, + }); + } + return rows; +} diff --git a/packages/verae-fleet/src/monitor.js b/packages/verae-fleet/src/monitor.js new file mode 100644 index 0000000..6cdb2e1 --- /dev/null +++ b/packages/verae-fleet/src/monitor.js @@ -0,0 +1,38 @@ +/** + * Periodic probe + reconcile so replica floors (especially tree-node) stay met. + * @module monitor + */ + +export class Monitor { + /** + * @param {import('./supervisor.js').Supervisor} supervisor + * @param {{ intervalMs?: number }} [opts] + */ + constructor(supervisor, opts = {}) { + this.supervisor = supervisor; + this.intervalMs = opts.intervalMs ?? supervisor.loaded.control?.probeIntervalMs ?? 750; + this.timer = null; + this.last = null; + } + + async tick() { + const actions = await this.supervisor.reconcile(); + this.last = { t: new Date().toISOString(), actions, status: this.supervisor.status() }; + return this.last; + } + + start() { + if (this.timer) return; + this.timer = setInterval(() => { + this.tick().catch((err) => { + this.supervisor.log('monitor-error', { error: err.message }); + }); + }, this.intervalMs); + this.timer.unref?.(); + } + + stop() { + if (this.timer) clearInterval(this.timer); + this.timer = null; + } +} diff --git a/packages/verae-fleet/src/server.js b/packages/verae-fleet/src/server.js new file mode 100644 index 0000000..f909f01 --- /dev/null +++ b/packages/verae-fleet/src/server.js @@ -0,0 +1,85 @@ +/** + * Operator HTTP: list, status, pause/resume/stop/start/restart. + * Binds 127.0.0.1 only. + * @module server + */ + +import http from 'node:http'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { listServices } from './load.js'; + +const PUBLIC = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', 'public'); + +function json(res, code, obj) { + res.writeHead(code, { 'content-type': 'application/json', 'cache-control': 'no-store' }); + res.end(JSON.stringify(obj, null, 2)); +} + +/** + * @param {import('./supervisor.js').Supervisor} sup + * @param {import('./monitor.js').Monitor} [mon] + */ +export function startControlServer(sup, mon) { + const bind = sup.loaded.control?.bind || '127.0.0.1'; + const port = Number(sup.loaded.control?.port || 3850); + + const server = http.createServer(async (req, res) => { + const url = new URL(req.url, `http://${bind}:${port}`); + try { + if (req.method === 'GET' && url.pathname === '/health') { + return json(res, 200, { ok: true, role: 'fleet-control' }); + } + if (req.method === 'GET' && url.pathname === '/api/services') { + return json(res, 200, { services: listServices(sup.loaded), fleetPath: sup.loaded.fleetPath }); + } + if (req.method === 'GET' && url.pathname === '/api/status') { + return json(res, 200, { ...sup.status(), monitor: mon?.last || null }); + } + if (req.method === 'POST' && url.pathname === '/api/reconcile') { + const actions = await sup.reconcile(); + return json(res, 200, { actions, status: sup.status() }); + } + const svc = url.pathname.match(/^\/api\/services\/([^/]+)\/(start|stop|pause|resume)$/); + if (svc && req.method === 'POST') { + const [, id, op] = svc; + const fn = { + start: () => sup.startService(id), + stop: () => sup.stopService(id), + pause: () => sup.pauseService(id), + resume: () => sup.resumeService(id), + }[op]; + const result = await fn(); + return json(res, 200, { op, id, result, status: sup.status().services[id] }); + } + const inst = url.pathname.match(/^\/api\/instances\/([^/]+)\/(pause|resume|restart|stop|unhealthy)$/); + if (inst && req.method === 'POST') { + const [, id, op] = inst; + const fn = { + pause: () => sup.pauseInstance(id), + resume: () => sup.resumeInstance(id), + restart: () => sup.restartInstance(id), + stop: () => sup.stopInstance(id, { replace: true }), + unhealthy: () => sup.markUnhealthy(id), + }[op]; + const result = await fn(); + return json(res, 200, { op, id, result: result?.id || result, status: sup.status() }); + } + if (req.method === 'GET' && (url.pathname === '/' || url.pathname === '/index.html')) { + const dest = path.join(PUBLIC, 'index.html'); + res.writeHead(200, { 'content-type': 'text/html; charset=utf-8' }); + fs.createReadStream(dest).pipe(res); + return; + } + json(res, 404, { error: 'not found' }); + } catch (err) { + json(res, 400, { error: err.message }); + } + }); + + server.listen(port, bind, () => { + process.stdout.write(`fleet control http://${bind}:${port}/\n`); + }); + return server; +} diff --git a/packages/verae-fleet/src/supervisor.js b/packages/verae-fleet/src/supervisor.js new file mode 100644 index 0000000..94ec6c1 --- /dev/null +++ b/packages/verae-fleet/src/supervisor.js @@ -0,0 +1,323 @@ +/** + * Spawn, pause, resume, stop, restart replicas. Keep replica floor (tree-node min). + * @module supervisor + */ + +import { spawn } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { loadFleet } from './load.js'; +import { httpProbe, postControl } from './health.js'; + +const WORKER = path.join(path.dirname(fileURLToPath(import.meta.url)), 'worker.js'); + +export class Supervisor { + /** + * @param {object} [opts] + * @param {ReturnType} [opts.loaded] + * @param {string} [opts.stateRoot] + */ + constructor(opts = {}) { + this.loaded = opts.loaded || loadFleet(); + this.stateRoot = opts.stateRoot || path.join(this.loaded.root, 'data', 'runtime'); + fs.mkdirSync(this.stateRoot, { recursive: true }); + /** @type {Map} */ + this.instances = new Map(); + this.events = []; + this._seq = 0; + } + + log(type, detail) { + const ev = { t: new Date().toISOString(), type, ...detail }; + this.events.push(ev); + if (this.events.length > 500) this.events.splice(0, this.events.length - 400); + return ev; + } + + spec(serviceId) { + const s = this.loaded.services[serviceId]; + if (!s) throw new Error(`unknown service ${serviceId}`); + return s; + } + + nextIndex(serviceId) { + const used = new Set( + [...this.instances.values()].filter((i) => i.service === serviceId).map((i) => i.index), + ); + const max = this.spec(serviceId).max ?? 8; + for (let i = 0; i < max; i += 1) if (!used.has(i)) return i; + return null; + } + + /** + * @param {string} serviceId + * @param {{ index?: number }} [opts] + */ + async startOne(serviceId, opts = {}) { + const spec = this.spec(serviceId); + if (!spec.managed) { + this.log('skip', { service: serviceId, reason: 'unmanaged' }); + return null; + } + if (!spec.enabled) { + this.log('skip', { service: serviceId, reason: 'disabled' }); + return null; + } + const index = opts.index ?? this.nextIndex(serviceId); + if (index == null) { + this.log('skip', { service: serviceId, reason: 'at-max' }); + return null; + } + const instance = `${serviceId}-${index}`; + if (this.instances.has(instance)) return this.instances.get(instance); + const healthPort = (spec.ports?.healthBase || 19000) + index; + const stateDir = path.join(this.stateRoot, instance); + fs.mkdirSync(stateDir, { recursive: true }); + const child = spawn(process.execPath, [WORKER], { + env: { + ...process.env, + FLEET_ROLE: spec.role || serviceId, + FLEET_SERVICE: serviceId, + FLEET_INSTANCE: instance, + FLEET_HEALTH_PORT: String(healthPort), + FLEET_STATE_DIR: stateDir, + }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + const rec = { + id: instance, + service: serviceId, + index, + pid: child.pid, + healthPort, + stateDir, + child, + paused: false, + startedAt: Date.now(), + restarts: 0, + lastError: null, + }; + child.stderr?.on('data', () => {}); + child.on('exit', (code, signal) => { + rec.exitCode = code; + rec.signal = signal; + rec.pid = null; + this.log('exit', { instance, service: serviceId, code, signal }); + }); + this.instances.set(instance, rec); + this.log('start', { instance, service: serviceId, pid: rec.pid, healthPort }); + const h = await waitForHealth(healthPort, 4000); + rec.healthy = h.ok; + rec.lastProbe = h; + return rec; + } + + async stopInstance(instanceId, { replace = true } = {}) { + const rec = this.instances.get(instanceId); + if (!rec) return null; + rec.stopping = true; + if (rec.child && rec.pid) { + rec.child.kill('SIGTERM'); + await waitExit(rec.child, 2000); + } + this.instances.delete(instanceId); + this.log('stop', { instance: instanceId, service: rec.service }); + if (replace) await this.reconcile(rec.service); + return rec; + } + + async pauseInstance(instanceId, { replace = true } = {}) { + const rec = this.instances.get(instanceId); + if (!rec) throw new Error(`no instance ${instanceId}`); + await postControl(rec.healthPort, '/pause'); + rec.paused = true; + this.log('pause', { instance: instanceId, service: rec.service }); + if (replace) await this.reconcile(rec.service); + return rec; + } + + async resumeInstance(instanceId) { + const rec = this.instances.get(instanceId); + if (!rec) throw new Error(`no instance ${instanceId}`); + await postControl(rec.healthPort, '/resume'); + rec.paused = false; + this.log('resume', { instance: instanceId, service: rec.service }); + return rec; + } + + async restartInstance(instanceId) { + const rec = this.instances.get(instanceId); + if (!rec) throw new Error(`no instance ${instanceId}`); + const { service, index } = rec; + rec.child?.kill('SIGTERM'); + await waitExit(rec.child, 2000); + this.instances.delete(instanceId); + const next = await this.startOne(service, { index }); + if (next) next.restarts = (rec.restarts || 0) + 1; + this.log('restart', { instance: instanceId, service }); + return next; + } + + async markUnhealthy(instanceId) { + const rec = this.instances.get(instanceId); + if (!rec) throw new Error(`no instance ${instanceId}`); + await postControl(rec.healthPort, '/unhealthy'); + return rec; + } + + async probe(rec) { + const spec = this.spec(rec.service); + const r = await httpProbe(rec.healthPort, spec.health?.path || '/health', spec.health?.timeoutMs || 800); + rec.paused = Boolean(r.paused); + rec.lastProbe = r; + rec.healthy = r.ok; + return r; + } + + available(serviceId) { + return [...this.instances.values()].filter( + (i) => i.service === serviceId && i.pid && !i.paused && i.healthy !== false, + ); + } + + /** + * Ensure running healthy unpaused copies >= min when keepFloor. + * @param {string} [only] + */ + async reconcile(only) { + const actions = []; + const ids = only ? [only] : Object.keys(this.loaded.services); + for (const serviceId of ids) { + const spec = this.loaded.services[serviceId]; + if (!spec?.managed || !spec.enabled) continue; + for (const rec of [...this.instances.values()].filter((i) => i.service === serviceId)) { + if (!rec.pid && !rec.stopping) { + this.instances.delete(rec.id); + continue; + } + if (rec.pid) await this.probe(rec); + if (rec.pid && rec.healthy === false && !rec.paused) { + this.log('unhealthy', { instance: rec.id, service: serviceId }); + await this.restartInstance(rec.id); + actions.push({ op: 'restart', instance: rec.id }); + } + } + const avail = this.available(serviceId).length; + const want = spec.keepFloor ? spec.min : Math.min(spec.min, spec.max); + if (avail < want) { + const need = want - avail; + for (let n = 0; n < need; n += 1) { + const started = await this.startOne(serviceId); + if (started) actions.push({ op: 'start', instance: started.id }); + } + } + } + return actions; + } + + async startService(serviceId) { + const spec = this.spec(serviceId); + spec.enabled = true; + const have = [...this.instances.values()].filter((i) => i.service === serviceId && i.pid).length; + const want = spec.min || 1; + const started = []; + for (let i = have; i < want; i += 1) { + const rec = await this.startOne(serviceId); + if (rec) started.push(rec.id); + } + return started; + } + + async stopService(serviceId, { holdFloor = false } = {}) { + const spec = this.spec(serviceId); + if (!holdFloor) spec.enabled = false; + const ids = [...this.instances.values()].filter((i) => i.service === serviceId).map((i) => i.id); + for (const id of ids) await this.stopInstance(id, { replace: false }); + if (holdFloor) await this.reconcile(serviceId); + return ids; + } + + async pauseService(serviceId) { + const ids = [...this.instances.values()].filter((i) => i.service === serviceId && i.pid).map((i) => i.id); + for (const id of ids) await this.pauseInstance(id, { replace: false }); + await this.reconcile(serviceId); + return ids; + } + + async resumeService(serviceId) { + const ids = [...this.instances.values()].filter((i) => i.service === serviceId).map((i) => i.id); + for (const id of ids) await this.resumeInstance(id); + return ids; + } + + async stopAll() { + for (const id of [...this.instances.keys()]) { + await this.stopInstance(id, { replace: false }); + } + } + + status() { + const services = {}; + for (const spec of Object.values(this.loaded.services)) { + const inst = [...this.instances.values()].filter((i) => i.service === spec.id); + const available = inst.filter((i) => i.pid && !i.paused && i.healthy !== false).length; + services[spec.id] = { + id: spec.id, + title: spec.title, + configPath: spec.configPath, + managed: spec.managed, + enabled: spec.enabled, + min: spec.min, + max: spec.max, + keepFloor: spec.keepFloor, + running: inst.filter((i) => i.pid).length, + paused: inst.filter((i) => i.paused).length, + available, + belowFloor: spec.keepFloor && spec.enabled && spec.managed && available < spec.min, + instances: inst.map((i) => ({ + id: i.id, + pid: i.pid, + healthPort: i.healthPort, + paused: i.paused, + healthy: i.healthy, + restarts: i.restarts, + })), + }; + } + return { + control: this.loaded.control, + nats: this.loaded.nats, + services, + events: this.events.slice(-50), + }; + } +} + +function waitExit(child, ms) { + return new Promise((resolve) => { + if (!child || child.exitCode != null) return resolve(); + const t = setTimeout(() => { + try { + child.kill('SIGKILL'); + } catch { + /* ignore */ + } + resolve(); + }, ms); + child.once('exit', () => { + clearTimeout(t); + resolve(); + }); + }); +} + +async function waitForHealth(port, ms) { + const deadline = Date.now() + ms; + while (Date.now() < deadline) { + const r = await httpProbe(port, '/health', 300); + if (r.ok || r.statusCode === 503) return r; + await new Promise((r) => setTimeout(r, 50)); + } + return { ok: false, error: 'start-timeout' }; +} diff --git a/packages/verae-fleet/src/worker.js b/packages/verae-fleet/src/worker.js new file mode 100644 index 0000000..5b3fe56 --- /dev/null +++ b/packages/verae-fleet/src/worker.js @@ -0,0 +1,113 @@ +#!/usr/bin/env node +/** + * Fleet-managed replica: health HTTP + pause + heartbeat. + * Roles tree-node / archive-worm hold an in-process WormArchive. + */ +import http from 'node:http'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const role = process.env.FLEET_ROLE || 'unknown'; +const service = process.env.FLEET_SERVICE || role; +const instance = process.env.FLEET_INSTANCE || `${service}-0`; +const port = Number(process.env.FLEET_HEALTH_PORT || 0); +const stateDir = process.env.FLEET_STATE_DIR || path.join(process.cwd(), 'data', instance); +const startedAt = new Date().toISOString(); + +fs.mkdirSync(stateDir, { recursive: true }); + +let paused = false; +let forceUnhealthy = false; +let archive = null; +let puts = 0; + +if (role === 'tree-node' || role === 'archive-worm') { + try { + const wormPath = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + '../../verae-archive-worm/src/archive.js', + ); + const { WormArchive } = await import(wormPath); + archive = new WormArchive(instance); + } catch { + archive = { + archiveId: instance, + put() { + puts += 1; + }, + query() { + return null; + }, + }; + } +} + +function snapshot() { + return { + ok: !paused && !forceUnhealthy, + paused, + forceUnhealthy, + role, + service, + instance, + port, + pid: process.pid, + startedAt, + puts, + archiveId: archive?.archiveId || null, + }; +} + +function beat() { + const body = { ...snapshot(), ts: Date.now() }; + fs.writeFileSync(path.join(stateDir, 'heartbeat.json'), JSON.stringify(body)); +} + +const server = http.createServer((req, res) => { + const url = new URL(req.url, 'http://127.0.0.1'); + const json = (code, obj) => { + res.writeHead(code, { 'content-type': 'application/json' }); + res.end(JSON.stringify(obj)); + }; + if (url.pathname === '/health' && req.method === 'GET') { + const snap = snapshot(); + return json(snap.ok ? 200 : 503, snap); + } + if (url.pathname === '/info' && req.method === 'GET') return json(200, snapshot()); + if (url.pathname === '/pause' && req.method === 'POST') { + paused = true; + beat(); + return json(200, snapshot()); + } + if (url.pathname === '/resume' && req.method === 'POST') { + paused = false; + forceUnhealthy = false; + beat(); + return json(200, snapshot()); + } + if (url.pathname === '/unhealthy' && req.method === 'POST') { + forceUnhealthy = true; + beat(); + return json(200, snapshot()); + } + json(404, { error: 'not found' }); +}); + +server.listen(port, '127.0.0.1', () => { + beat(); + process.stdout.write(`fleet-worker ${instance} health http://127.0.0.1:${port}/health\n`); +}); + +const iv = setInterval(beat, 400); +function shutdown() { + clearInterval(iv); + try { + server.close(); + } catch { + /* ignore */ + } + process.exit(0); +} +process.on('SIGTERM', shutdown); +process.on('SIGINT', shutdown); diff --git a/packages/verae-fleet/test/fleet.test.js b/packages/verae-fleet/test/fleet.test.js new file mode 100644 index 0000000..90e7a29 --- /dev/null +++ b/packages/verae-fleet/test/fleet.test.js @@ -0,0 +1,99 @@ +import { describe, it, after } from 'node:test'; +import assert from 'node:assert/strict'; +import { loadFleet, listServices } from '../src/load.js'; +import { Supervisor } from '../src/supervisor.js'; +import { Monitor } from '../src/monitor.js'; + +function treeOnly(healthBase) { + const loaded = loadFleet(); + for (const s of Object.values(loaded.services)) { + s.enabled = s.id === 'tree-node'; + if (s.id === 'tree-node') { + s.min = 3; + s.max = 6; + s.keepFloor = true; + s.ports = { healthBase }; + } + } + return loaded; +} + +describe('verae-fleet', () => { + /** @type {Supervisor[]} */ + const supervisors = []; + after(async () => { + for (const s of supervisors) await s.stopAll(); + }); + + it('lists every service with a config file and central replica spec', () => { + const loaded = loadFleet(); + const rows = listServices(loaded); + const ids = rows.map((r) => r.id); + for (const need of [ + 'nats', + 'zappier-edge', + 'middleware-http', + 'job-poller', + 'webhook-deliver', + 'archive-aggregator', + 'archive-worm', + 'tree-node', + 'zapier-simulator', + 'zapier-platform-app', + ]) { + assert.ok(ids.includes(need), `missing ${need}`); + } + const tree = loaded.services['tree-node']; + assert.equal(tree.min, 3); + assert.equal(tree.keepFloor, true); + assert.equal(tree.configPath, 'services/tree-node.json'); + assert.equal(loaded.fleetPath, 'fleet.json'); + }); + + it('starts the tree-node floor (3 available copies)', async () => { + const sup = new Supervisor({ loaded: treeOnly(14600) }); + supervisors.push(sup); + await sup.startService('tree-node'); + const st = sup.status().services['tree-node']; + assert.equal(st.available, 3); + assert.equal(st.belowFloor, false); + assert.equal(st.instances.length, 3); + }); + + it('pauses one tree-node and respawns so min available stays 3', async () => { + const sup = new Supervisor({ loaded: treeOnly(14700) }); + supervisors.push(sup); + await sup.startService('tree-node'); + await sup.pauseInstance('tree-node-0'); + const st = sup.status().services['tree-node']; + assert.ok(st.available >= 3, `available ${st.available}`); + assert.ok(st.paused >= 1); + assert.ok(st.running >= 4); + assert.equal(st.belowFloor, false); + }); + + it('restarts an unhealthy replica and keeps the floor', async () => { + const sup = new Supervisor({ loaded: treeOnly(14800) }); + supervisors.push(sup); + await sup.startService('tree-node'); + const before = sup.instances.get('tree-node-1').pid; + await sup.markUnhealthy('tree-node-1'); + const mon = new Monitor(sup, { intervalMs: 50 }); + await mon.tick(); + const rec = sup.instances.get('tree-node-1'); + assert.ok(rec.pid); + assert.notEqual(rec.pid, before); + assert.equal(rec.healthy, true); + assert.equal(sup.status().services['tree-node'].available, 3); + }); + + it('stop+replace of a replica keeps min tree-nodes available', async () => { + const sup = new Supervisor({ loaded: treeOnly(14900) }); + supervisors.push(sup); + await sup.startService('tree-node'); + await sup.stopInstance('tree-node-2', { replace: true }); + const st = sup.status().services['tree-node']; + assert.equal(st.available, 3); + assert.ok(st.instances.some((i) => i.id === 'tree-node-2' || i.id === 'tree-node-0')); + }); +}); diff --git a/scripts/build-docs-site.py b/scripts/build-docs-site.py index 51b8b81..47f8a2d 100755 --- a/scripts/build-docs-site.py +++ b/scripts/build-docs-site.py @@ -32,6 +32,7 @@ SECTIONS = [ ("docs/02-architecture/archive-nats.md", "Archive NATS, bloom, multi-receipt"), ("docs/02-architecture/modules-and-nats.md", "Module catalog and NATS addresses"), ("docs/02-architecture/tree-nodes.md", "Tree nodes and bulk Merkle summaries"), + ("docs/02-architecture/fleet.md", "Fleet replica floors and monitor"), ], ), ( @@ -139,6 +140,7 @@ def main() -> None: "verae-tree-node", "verae-zapier-simulator", "zapier-user-docs", + "verae-fleet", "docs-master", ): readme = ROOT / "packages" / pkg / "README.md" @@ -232,6 +234,7 @@ def main() -> None: ("verae-tree-node", "Merkle leaf proofs (bulk summaries)"), ("verae-zapier-simulator", "Zapier interface + trace console"), ("zapier-user-docs", "Signup-to-usage user guide"), + ("verae-fleet", "Service catalog, replica floors, monitor"), ] git_lis = "".join( f'
  • {name} ' diff --git a/scripts/push-module-repos.sh b/scripts/push-module-repos.sh index 204d9cf..1b93ec1 100755 --- a/scripts/push-module-repos.sh +++ b/scripts/push-module-repos.sh @@ -46,6 +46,7 @@ create zapier-docs-master "Master summaries, NATS contracts, message flows" create verae-tree-node "Merkle leaf proofs for hashes only sealed as a bulk summary" create verae-zapier-simulator "Zapier interface simulator with hop-by-hop trace console" create zapier-user-docs "User docs from signup through hash register, chain lookup, tree-node query" +create verae-fleet "Service catalog, replica floors, monitor, restart, pause" push_dir "$ROOT/packages/zappier" zappier-edge push_dir "$ROOT/packages/verae-zapier-middleware" verae-middleware @@ -58,5 +59,6 @@ push_dir "$ROOT/packages/docs-master" zapier-docs-master push_dir "$ROOT/packages/verae-tree-node" verae-tree-node push_dir "$ROOT/packages/verae-zapier-simulator" verae-zapier-simulator push_dir "$ROOT/packages/zapier-user-docs" zapier-user-docs +push_dir "$ROOT/packages/verae-fleet" verae-fleet echo ALL_MODULE_REPOS_PUSHED