diff --git a/docs/02-architecture/fleet.md b/docs/02-architecture/fleet.md index 72ccac5..8392257 100644 --- a/docs/02-architecture/fleet.md +++ b/docs/02-architecture/fleet.md @@ -12,6 +12,8 @@ operator --HTTP 127.0.0.1:3850--> fleet control replicas on loopback health ports ``` +Operator console (Fleet · Trace · Docs): http://127.0.0.1:3850/ — see `packages/verae-fleet/docs/CONSOLE.md`. + - **list** — `node src/cli.js list` - **monitor** — `serve` probes `/health` and reconciles - **restart** — crash or 503 → same instance id respawned diff --git a/packages/docs-master/README.md b/packages/docs-master/README.md index 11052f6..01e81e7 100644 --- a/packages/docs-master/README.md +++ b/packages/docs-master/README.md @@ -3,6 +3,8 @@ Summaries, NATS contracts, and message flows for every Verae Time × Zapier module. **Live catalog:** https://zapier.georgelambert.org/ +**Overview (start here):** https://git.georgelambert.org/marchon/overview +**Operator console:** http://127.0.0.1:3850/ (Fleet · Trace · Docs) **Monorepo:** https://git.georgelambert.org/marchon/master-zapier-plan-draft (`main` and `master`) ## Git repos (Forgejo on NS1) diff --git a/packages/docs-master/modules/verae-fleet/SUMMARY.md b/packages/docs-master/modules/verae-fleet/SUMMARY.md index 7c9900c..b4ac4b0 100644 --- a/packages/docs-master/modules/verae-fleet/SUMMARY.md +++ b/packages/docs-master/modules/verae-fleet/SUMMARY.md @@ -2,4 +2,4 @@ **Job:** Service catalog, per-service configs, monitor, restart, pause/on/off, keep tree-node replica floor. -**Config:** `fleet.json` (min/max) + `machines.json` (hosts) + `services/.json`. Monitor shows message RTT min/avg/p50/p90. +**Config:** `fleet.json` (min/max) + `machines.json` (hosts) + `services/.json`. Operator console at `:3850` (Fleet · Trace · Docs). diff --git a/packages/overview/02-modules-and-repos.md b/packages/overview/02-modules-and-repos.md index 720b30f..663ede2 100644 --- a/packages/overview/02-modules-and-repos.md +++ b/packages/overview/02-modules-and-repos.md @@ -15,7 +15,7 @@ Each runtime piece is its **own git repo** on Forgejo (`git.georgelambert.org`, | **verae-archive-worm** | `packages/verae-archive-worm` | Bloom WORM node | | **verae-archive-aggregator** | `packages/verae-archive-aggregator` | Merge archive replies | | **verae-tree-node** | `packages/verae-tree-node` | Merkle leaf proofs (bulk summaries) | -| **verae-fleet** | `packages/verae-fleet` | Catalog, min copies, pause/restart, SSH hosts | +| **verae-fleet** | `packages/verae-fleet` | Catalog, min copies, pause/restart, SSH hosts, operator console | | **verae-zapier-simulator** | `packages/verae-zapier-simulator` | Trace console before `zapier-platform push` | | **zapier-user-docs** | `packages/zapier-user-docs` | Customer signup → register → lookup | | **zapier-docs-master** | `packages/docs-master` | Per-module `SUMMARY.md` + `NATS.md` | diff --git a/packages/overview/04-uptime.md b/packages/overview/04-uptime.md index df78f9b..1b3ea27 100644 --- a/packages/overview/04-uptime.md +++ b/packages/overview/04-uptime.md @@ -13,7 +13,7 @@ Uptime is **NATS durability + fleet replica floors + more than one machine**, no | archive-aggregator, job-poller, webhook-deliver | 1 | yes | | zappier-edge, middleware-http | 1 | yes | -If a tree node is paused, crashes, or fails `/health`, fleet **starts another copy** until three are available. Monitor: http://127.0.0.1:3850/ (green / yellow / red rows). +If a tree node is paused, crashes, or fails `/health`, fleet **starts another copy** until three are available. Operator console: http://127.0.0.1:3850/ (Fleet tab — green / yellow / red; Trace tab for hop tests; Docs tab for reading order). ## Restart and pause diff --git a/packages/overview/INDEX.md b/packages/overview/INDEX.md index 90191ee..d35059f 100644 --- a/packages/overview/INDEX.md +++ b/packages/overview/INDEX.md @@ -1,26 +1,29 @@ # Documentation index -Pointers into the rest of the documentation. Live HTML catalog: https://zapier.georgelambert.org/ +Read **high-level first**, then **architecture**, then the rest. Live catalog: https://zapier.georgelambert.org/ -## This overview +Operator console (loopback): http://127.0.0.1:3850/ -| Topic | File | -|-------|------| -| TOC / intro | [README.md](README.md) | -| System | [01-system.md](01-system.md) | -| Modules and repos | [02-modules-and-repos.md](02-modules-and-repos.md) | -| NATS 3-server cluster | [03-nats-cluster.md](03-nats-cluster.md) | -| Uptime | [04-uptime.md](04-uptime.md) | -| Local network failures | [05-network-failures.md](05-network-failures.md) | -| Address routing | [06-address-routing.md](06-address-routing.md) | -| External search / storage | [07-external-resources.md](07-external-resources.md) | -| Diagrams | [08-diagrams.md](08-diagrams.md) | -| Expansion template | [09-expansion-template.md](09-expansion-template.md) | +## 1. High-level (non-technical) -## Catalog (HTML) +| Document | What it is | +|----------|------------| +| [README.md](README.md) | Overview (start here) | +| [01-system.md](01-system.md) | What runs where, in plain language | +| [02-modules-and-repos.md](02-modules-and-repos.md) | Short names of every independent repo | +| [zapier-docs-master](https://git.georgelambert.org/marchon/zapier-docs-master) | One-line SUMMARY + NATS per module | +| [zapier-user-docs](https://git.georgelambert.org/marchon/zapier-user-docs) | Signup → register → lookup | + +## 2. Architecture | Document | URL | |----------|-----| +| NATS cluster | [03-nats-cluster.md](03-nats-cluster.md) | +| Uptime | [04-uptime.md](04-uptime.md) | +| Network failures | [05-network-failures.md](05-network-failures.md) | +| Address routing | [06-address-routing.md](06-address-routing.md) | +| External resources | [07-external-resources.md](07-external-resources.md) | +| Diagrams | [08-diagrams.md](08-diagrams.md) | | Catalog home | https://zapier.georgelambert.org/ | | Composition | https://zapier.georgelambert.org/docs/02-architecture/composition.html | | NATS gateway | https://zapier.georgelambert.org/docs/02-architecture/nats-gateway.html | @@ -29,6 +32,13 @@ Pointers into the rest of the documentation. Live HTML catalog: https://zapier.g | Module NATS map | https://zapier.georgelambert.org/docs/02-architecture/modules-and-nats.html | | Tree nodes | https://zapier.georgelambert.org/docs/02-architecture/tree-nodes.html | | Fleet | https://zapier.georgelambert.org/docs/02-architecture/fleet.html | + +## 3. Remaining documentation + +| Document | URL | +|----------|-----| +| Expansion template | [09-expansion-template.md](09-expansion-template.md) | +| Operator console | https://git.georgelambert.org/marchon/verae-fleet/src/branch/main/docs/CONSOLE.md | | User guide | https://zapier.georgelambert.org/user-docs/README.md | | Tree-node lookup (users) | https://zapier.georgelambert.org/packages/zapier-user-docs/09-lookup-tree-nodes.html | | Zapier developer setup | https://zapier.georgelambert.org/docs/04-activate/SETUP-ZAPIER-DEVELOPER.html | @@ -48,8 +58,8 @@ In **zapier-docs-master**: `modules//SUMMARY.md` and `NATS.md` for ever | Surface | URL | |---------|-----| -| Fleet monitor | http://127.0.0.1:3850/ | -| Zapier simulator | http://127.0.0.1:3847/ | +| Operator console (Fleet · Trace · Docs) | http://127.0.0.1:3850/ | +| Standalone simulator | http://127.0.0.1:3847/ | ## A–Z subject index @@ -68,3 +78,8 @@ In **zapier-docs-master**: `modules//SUMMARY.md` and `NATS.md` for ever | SSH hosts | verae-fleet machines.json | | Zapier never NATS | 01-system | | zappier-edge | metering, portal | +| operator console | http://127.0.0.1:3850/ ; verae-fleet docs/CONSOLE.md | + +--- + +Design: **Scott Lindsey**, **George Lambert**, **NATS.IO**, and **Grok-Code** by [Grok.com](https://grok.com). diff --git a/packages/overview/README.md b/packages/overview/README.md index 7e96762..5ccd0c4 100644 --- a/packages/overview/README.md +++ b/packages/overview/README.md @@ -46,3 +46,59 @@ Zapier → HTTPS `zappier-edge` → HTTPS `verae-middleware` → **NATS cluster* | [zapier-docs-master](https://git.georgelambert.org/marchon/zapier-docs-master) | Per-module SUMMARY + NATS | Clone any of them: `git clone ssh://git@git.georgelambert.org:2223/marchon/.git` (SSH port **2223**). + +## Operator console (loopback) + +Fleet, message **Trace**, and **Docs** share one shell at http://127.0.0.1:3850/ (`verae-fleet`). Not public. See [verae-fleet CONSOLE.md](https://git.georgelambert.org/marchon/verae-fleet/src/branch/main/docs/CONSOLE.md). + +![Fleet tab](screenshots/console-fleet.png) + +![Trace tab](screenshots/console-trace.png) + +![Docs tab](screenshots/console-docs.png) + +## Where to read next + +Start at the top. Architecture is second. Everything else is reference. + +### 1. High-level (non-technical) + +| Document | What it is | +|----------|------------| +| **This overview** | Whole system in one place | +| [Modules and repositories](02-modules-and-repos.md) | Short names of every independent repo | +| [zapier-docs-master](https://git.georgelambert.org/marchon/zapier-docs-master) | One-line SUMMARY + NATS per module | +| [zapier-user-docs](https://git.georgelambert.org/marchon/zapier-user-docs) | Signup → register a hash → lookup | + +### 2. Architecture + +| Document | What it is | +|----------|------------| +| [NATS 3-server cluster](03-nats-cluster.md) | Who may connect; private URLs | +| [Uptime](04-uptime.md) | Replica floors, restart, SSH hosts | +| [Local network failures](05-network-failures.md) | Reconnect, bloom silence, failover | +| [Address routing](06-address-routing.md) | Unplanned functions as new subjects | +| [External resources](07-external-resources.md) | Chain, WORM, tree-node search | +| [Diagrams](08-diagrams.md) | SVG + mermaid | +| [Fleet](https://zapier.georgelambert.org/docs/02-architecture/fleet.html) | Operator replica control | +| [Module NATS map](https://zapier.georgelambert.org/docs/02-architecture/modules-and-nats.html) | Address table | +| [Archive / bloom](https://zapier.georgelambert.org/docs/02-architecture/archive-nats.html) | Multipart and multi-receipt | +| [Tree nodes](https://zapier.georgelambert.org/docs/02-architecture/tree-nodes.html) | Bulk Merkle leaves | +| [Composition](https://zapier.georgelambert.org/docs/02-architecture/composition.html) | zappier + middleware HTTPS | + +### 3. Remaining documentation + +| Document | What it is | +|----------|------------| +| [Documentation index](INDEX.md) | A–Z and catalog URLs | +| [Expansion template](09-expansion-template.md) / [verae-nats-process](https://git.georgelambert.org/marchon/verae-nats-process) | Copy this to add an address | +| [Operator console](https://git.georgelambert.org/marchon/verae-fleet/src/branch/main/docs/CONSOLE.md) | Screenshots of Fleet / Trace / Docs | +| [Simulator](https://git.georgelambert.org/marchon/verae-zapier-simulator) | Standalone trace (also inside the console) | +| [Zapier developer setup](https://zapier.georgelambert.org/docs/04-activate/SETUP-ZAPIER-DEVELOPER.html) | CLI / login / push | +| Live catalog | https://zapier.georgelambert.org/ | + +--- + +## Credits + +Design: **Scott Lindsey**, **George Lambert**, **NATS.IO**, and **Grok-Code** by [Grok.com](https://grok.com). diff --git a/packages/overview/screenshots/console-docs.png b/packages/overview/screenshots/console-docs.png new file mode 100644 index 0000000..975c6a1 Binary files /dev/null and b/packages/overview/screenshots/console-docs.png differ diff --git a/packages/overview/screenshots/console-fleet.png b/packages/overview/screenshots/console-fleet.png new file mode 100644 index 0000000..a599cb8 Binary files /dev/null and b/packages/overview/screenshots/console-fleet.png differ diff --git a/packages/overview/screenshots/console-trace.png b/packages/overview/screenshots/console-trace.png new file mode 100644 index 0000000..9915079 Binary files /dev/null and b/packages/overview/screenshots/console-trace.png differ diff --git a/packages/verae-fleet/README.md b/packages/verae-fleet/README.md index 8a0d2f9..9e8e88e 100644 --- a/packages/verae-fleet/README.md +++ b/packages/verae-fleet/README.md @@ -35,7 +35,7 @@ node src/cli.js ssh-check ns1 # marchon@70.88.205.138 with ~/.ssh/id_ed25519 Private keys stay on disk (`~/.ssh/id_ed25519`); git stores only the path. Optional overrides: `machines.secrets.json` (gitignored). -The UI shows message-processing **min / avg / p50 / p90** RTT per service, instance, and machine. +Operator console (Fleet · Trace · Docs) at http://127.0.0.1:3850/ — see [docs/CONSOLE.md](docs/CONSOLE.md). Message-processing **min / avg / p50 / p90** RTT is on the Fleet tab. Zapier cloud apps are listed but **not spawned**. NATS on NS1 is **monitored only** (loopback `:4222`, never a public bind). diff --git a/packages/verae-fleet/docs/CONSOLE.md b/packages/verae-fleet/docs/CONSOLE.md new file mode 100644 index 0000000..42816e9 --- /dev/null +++ b/packages/verae-fleet/docs/CONSOLE.md @@ -0,0 +1,28 @@ +# Operator console + +Loopback UI at **http://127.0.0.1:3850/** — Fleet, Trace (simulator), and Docs in one shell. Not a public site. NATS stays private; SSH identity files stay on disk. + +```bash +cd packages/verae-fleet +node src/cli.js serve +``` + +## Tabs + +| Tab | Purpose | +|-----|---------| +| **Fleet** | Machines (SSH/local), replica floors, green/yellow/red, RTT min/avg/p50/p90 spark, pause/restart/off with confirm, ssh-check toast | +| **Trace** | In-process Zapier simulator: run a step, hop log, monitors/suggestions | +| **Docs** | Reading order: overview → module summaries → architecture → remaining docs | + +## Screenshots + +![Fleet](screenshots/console-fleet.png) + +![Trace](screenshots/console-trace.png) + +![Docs](screenshots/console-docs.png) + +Design tokens match the zappier portal/admin (indigo shell). Customer portal is unchanged. + +See [overview](https://git.georgelambert.org/marchon/overview). diff --git a/packages/verae-fleet/docs/screenshots/console-docs.png b/packages/verae-fleet/docs/screenshots/console-docs.png new file mode 100644 index 0000000..975c6a1 Binary files /dev/null and b/packages/verae-fleet/docs/screenshots/console-docs.png differ diff --git a/packages/verae-fleet/docs/screenshots/console-fleet.png b/packages/verae-fleet/docs/screenshots/console-fleet.png new file mode 100644 index 0000000..a599cb8 Binary files /dev/null and b/packages/verae-fleet/docs/screenshots/console-fleet.png differ diff --git a/packages/verae-fleet/docs/screenshots/console-trace.png b/packages/verae-fleet/docs/screenshots/console-trace.png new file mode 100644 index 0000000..9915079 Binary files /dev/null and b/packages/verae-fleet/docs/screenshots/console-trace.png differ diff --git a/packages/verae-fleet/public/index.html b/packages/verae-fleet/public/index.html index 29f0fc4..7f578e3 100644 --- a/packages/verae-fleet/public/index.html +++ b/packages/verae-fleet/public/index.html @@ -3,146 +3,333 @@ - Verae fleet monitor + Verae operator console -
-

Verae fleet monitor

-

Spread replicas across machines. Track message-processing RTT (min / avg / p50 / p90) for capacity planning. Paused copies do not count toward the tree-node floor.

+
+
+
loopback · not public NATS
+

Verae operator console

+

Fleet, message trace, and docs in one shell. Pause does not count toward replica floors. SSH keys stay on disk.

+
+
-

-

Machines

-
- - - - - - - - - -
- - - -
MachineSSH / hostKindCapacityRunningRTT ms min / avg / p50 / p90
-

Services

- - - -
ServiceStatemin/maxavailableRTT ms min / avg / p50 / p90instancesactions
+
+

+

Machines

+
+ + + + + + + + + +
+
+

Services

+
+
+
+
+ +
+

Response

+ 
Run a step.
+

Hop trace

+
+

Monitors

+
+
+
+
+
+

Reading order: high-level first, then architecture, then the rest. Operator console is loopback only.

+

High-level (start here)

+ +

Architecture

+ +

Remaining documentation

+ +
+
diff --git a/packages/verae-fleet/src/server.js b/packages/verae-fleet/src/server.js index 7baaf07..8d89e19 100644 --- a/packages/verae-fleet/src/server.js +++ b/packages/verae-fleet/src/server.js @@ -9,6 +9,7 @@ import fs from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { listServices } from './load.js'; +import { ACTIONS, Simulator } from '../../verae-zapier-simulator/src/pipeline.js'; const PUBLIC = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', 'public'); @@ -24,12 +25,27 @@ function json(res, code, obj) { export function startControlServer(sup, mon) { const bind = sup.loaded.control?.bind || '127.0.0.1'; const port = Number(sup.loaded.control?.port || 3850); + const sim = new Simulator(); const server = http.createServer(async (req, res) => { const url = new URL(req.url, `http://${bind}:${port}`); try { if (req.method === 'GET' && url.pathname === '/health') { - return json(res, 200, { ok: true, role: 'fleet-control' }); + return json(res, 200, { ok: true, role: 'operator-console' }); + } + if (req.method === 'GET' && url.pathname === '/api/sim/actions') { + return json(res, 200, { actions: ACTIONS, state: sim.snapshot() }); + } + if (req.method === 'POST' && url.pathname === '/api/sim/reset') { + sim.reset(); + return json(res, 200, { ok: true, state: sim.snapshot() }); + } + if (req.method === 'POST' && url.pathname === '/api/sim/run') { + const chunks = []; + for await (const c of req) chunks.push(c); + const body = JSON.parse(Buffer.concat(chunks).toString('utf8') || '{}'); + const result = await sim.run(body); + return json(res, 200, result); } if (req.method === 'GET' && url.pathname === '/api/services') { return json(res, 200, { services: listServices(sup.loaded), fleetPath: sup.loaded.fleetPath }); @@ -92,6 +108,14 @@ export function startControlServer(sup, mon) { fs.createReadStream(dest).pipe(res); return; } + if (req.method === 'GET' && url.pathname.startsWith('/docs/screenshots/')) { + const dest = path.normalize(path.join(PUBLIC, '..', url.pathname)); + const root = path.normalize(path.join(PUBLIC, '..')); + if (!dest.startsWith(root) || !fs.existsSync(dest)) return json(res, 404, { error: 'not found' }); + res.writeHead(200, { 'content-type': 'image/png' }); + fs.createReadStream(dest).pipe(res); + return; + } json(res, 404, { error: 'not found' }); } catch (err) { json(res, 400, { error: err.message });