From a73875f4efea91f980f6608043e5817b31beae83 Mon Sep 17 00:00:00 2001 From: George Lambert Date: Fri, 11 Sep 2026 23:59:38 -0400 Subject: [PATCH] Cut the test environment over to the 3-node Proxmox NATS cluster. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fleet/keep/middleware NATS_URL lists 10.10.10.21–23. Product streams are replicas=3. px-worker worm/tree run on that cluster. Backup timer and px-worker uptime watch are on. MOCK_VERAE and Zapier push unchanged. --- packages/overview/03-nats-cluster.md | 2 +- .../types/ns1-all-in-one/start.sh | 3 +- packages/verae-fleet/overlays/ns1/fleet.json | 2 +- packages/verae-fleet/services/access-api.json | 2 +- .../verae-fleet/services/access-authz.json | 2 +- .../verae-fleet/services/access-leaf.json | 2 +- packages/verae-fleet/services/access-web.json | 2 +- .../verae-fleet/services/access-zapier.json | 2 +- .../verae-fleet/services/account-balance.json | 2 +- .../services/accounting-export.json | 2 +- .../services/customer-service.json | 2 +- packages/verae-fleet/services/identity.json | 2 +- .../verae-fleet/services/jobs-events.json | 2 +- .../verae-fleet/services/middleware-http.json | 5 +- .../verae-fleet/services/sales-pricing.json | 2 +- .../verae-fleet/services/zappier-edge.json | 2 +- .../verae-fleet/systemd/verae-fleet.service | 3 +- .../systemd/verae-keep-guard.service | 1 + .../systemd/verae-keep-guard.service.bak | 15 ++++ packages/verae-keep/units.ns1.json | 89 ++++++++++++++----- packages/verae-keep/units.px-worker.json | 4 +- packages/verae-nats-accounts/nats.conf | 4 + packages/verae-nats-cluster/README.md | 13 ++- packages/verae-nats-cluster/client.env | 3 + .../verae-nats-cluster/scripts/cutover-ns1.sh | 10 +++ .../scripts/ensure-streams.sh | 34 +++++++ packages/verae-ops/GETTING-STARTED.md | 6 +- .../src/nats/connection.js | 2 +- packages/zapier-decisions/LOG.md | 7 ++ 29 files changed, 184 insertions(+), 43 deletions(-) create mode 100644 packages/verae-keep/systemd/verae-keep-guard.service.bak create mode 100644 packages/verae-nats-cluster/client.env create mode 100755 packages/verae-nats-cluster/scripts/cutover-ns1.sh create mode 100755 packages/verae-nats-cluster/scripts/ensure-streams.sh diff --git a/packages/overview/03-nats-cluster.md b/packages/overview/03-nats-cluster.md index 9e0d5bb..16e2066 100644 --- a/packages/overview/03-nats-cluster.md +++ b/packages/overview/03-nats-cluster.md @@ -18,7 +18,7 @@ Today’s NS1 box (`NS1.GEORGELAMBERT.ORG`, `70.88.205.138`) still runs the **la nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222 ``` -(`nats-a` VMID 511, `nats-b` 512, `nats-c` 513). Cluster routes on `:6222`. Nothing on `vmbr0`. Cut over workers with `NATS_URL` listing all three; then the same configs move to separate hardware. Fleet SSH hosts run **workers**, not extra public NATS listeners. +(`nats-a` VMID 511, `nats-b` 512, `nats-c` 513). Cluster routes on `:6222`. Nothing on `vmbr0`. **Test env is cut over:** fleet/keep/middleware `NATS_URL` lists all three; product streams `ZAPIER_*` and `VERAE_ARCHIVE` are `replicas=3`. Same configs later move to separate hardware. Fleet SSH hosts run **workers**, not extra public NATS listeners. ![NATS cluster](diagrams/nats-cluster.svg) diff --git a/packages/verae-bootstrap/types/ns1-all-in-one/start.sh b/packages/verae-bootstrap/types/ns1-all-in-one/start.sh index 8c2fe0d..14dce4a 100755 --- a/packages/verae-bootstrap/types/ns1-all-in-one/start.sh +++ b/packages/verae-bootstrap/types/ns1-all-in-one/start.sh @@ -7,7 +7,8 @@ FLEET="$STACK/packages/verae-fleet" EDGE_PORT="${EDGE_PORT:-13000}" export STAFF_IAM_URL="${STAFF_IAM_URL:-https://iam.zapier.georgelambert.org}" export STAFF_COOKIE_DOMAIN="${STAFF_COOKIE_DOMAIN:-.zapier.georgelambert.org}" -export NATS_URL="${NATS_URL:-nats://127.0.0.1:4222}" +export NATS_URL="${NATS_URL:-nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222}" +export JETSTREAM_REPLICAS="${JETSTREAM_REPLICAS:-3}" export FLEET_ENABLE_LAN134="${FLEET_ENABLE_LAN134:-}" export FLEET_FILE="${FLEET_FILE:-$FLEET/overlays/ns1/fleet.json}" export FLEET_MACHINES="${FLEET_MACHINES:-$FLEET/overlays/ns1/machines.json}" diff --git a/packages/verae-fleet/overlays/ns1/fleet.json b/packages/verae-fleet/overlays/ns1/fleet.json index 9e3fc16..3018a2d 100644 --- a/packages/verae-fleet/overlays/ns1/fleet.json +++ b/packages/verae-fleet/overlays/ns1/fleet.json @@ -9,7 +9,7 @@ "maxRestartsPerHour": 40 }, "nats": { - "url": "nats://127.0.0.1:4222", + "url": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222", "publicBind": false }, "services": { diff --git a/packages/verae-fleet/services/access-api.json b/packages/verae-fleet/services/access-api.json index 1d5a7ae..c7ba3e1 100644 --- a/packages/verae-fleet/services/access-api.json +++ b/packages/verae-fleet/services/access-api.json @@ -18,7 +18,7 @@ "AUTHZ_URL": "http://127.0.0.1:3020", "ACCOUNT_BALANCE_URL": "http://127.0.0.1:3010", "ZAPPIER_UPSTREAM": "http://127.0.0.1:3100", - "NATS_URL": "nats://127.0.0.1:4222" + "NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222" }, "nats": { "in": [], diff --git a/packages/verae-fleet/services/access-authz.json b/packages/verae-fleet/services/access-authz.json index f104616..4f45235 100644 --- a/packages/verae-fleet/services/access-authz.json +++ b/packages/verae-fleet/services/access-authz.json @@ -15,7 +15,7 @@ }, "env": { "PORT": "3020", - "NATS_URL": "nats://127.0.0.1:4222" + "NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222" }, "nats": { "in": ["verae.access.authz.check"], diff --git a/packages/verae-fleet/services/access-leaf.json b/packages/verae-fleet/services/access-leaf.json index 8676ac4..514efba 100644 --- a/packages/verae-fleet/services/access-leaf.json +++ b/packages/verae-fleet/services/access-leaf.json @@ -18,7 +18,7 @@ "AUTHZ_URL": "http://127.0.0.1:3020", "LEAF_TOKEN": "leaf-dev-token", "NATS_POLICY_PATH": "../verae-nats-accounts/policy.json", - "NATS_URL": "nats://127.0.0.1:4222" + "NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222" }, "nats": { "in": ["verae.access.leaf.in"], diff --git a/packages/verae-fleet/services/access-web.json b/packages/verae-fleet/services/access-web.json index 213e80d..5d50163 100644 --- a/packages/verae-fleet/services/access-web.json +++ b/packages/verae-fleet/services/access-web.json @@ -17,7 +17,7 @@ "PORT": "3021", "AUTHZ_URL": "http://127.0.0.1:3020", "ACCOUNT_BALANCE_URL": "http://127.0.0.1:3010", - "NATS_URL": "nats://127.0.0.1:4222", + "NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222", "ZAPPIER_EDGE_URL": "http://127.0.0.1:3000" }, "nats": { diff --git a/packages/verae-fleet/services/access-zapier.json b/packages/verae-fleet/services/access-zapier.json index a3244b5..be8ecd0 100644 --- a/packages/verae-fleet/services/access-zapier.json +++ b/packages/verae-fleet/services/access-zapier.json @@ -17,7 +17,7 @@ "PORT": "3024", "AUTHZ_URL": "http://127.0.0.1:3020", "ZAPPIER_UPSTREAM": "http://127.0.0.1:3100", - "NATS_URL": "nats://127.0.0.1:4222" + "NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222" }, "nats": { "in": [], diff --git a/packages/verae-fleet/services/account-balance.json b/packages/verae-fleet/services/account-balance.json index d7f6532..94113ac 100644 --- a/packages/verae-fleet/services/account-balance.json +++ b/packages/verae-fleet/services/account-balance.json @@ -15,7 +15,7 @@ }, "env": { "PORT": "3010", - "NATS_URL": "nats://127.0.0.1:4222" + "NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222" }, "nats": { "in": [ diff --git a/packages/verae-fleet/services/accounting-export.json b/packages/verae-fleet/services/accounting-export.json index e1bca79..af1a15e 100644 --- a/packages/verae-fleet/services/accounting-export.json +++ b/packages/verae-fleet/services/accounting-export.json @@ -17,7 +17,7 @@ "PORT": "3013", "ZAPPIER_ADMIN_URL": "http://127.0.0.1:3000", "ACCOUNT_BALANCE_URL": "http://127.0.0.1:3010", - "NATS_URL": "nats://127.0.0.1:4222", + "NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222", "STAFF_IAM_URL": "http://127.0.0.1:3028", "STAFF_AUTH": "1" }, diff --git a/packages/verae-fleet/services/customer-service.json b/packages/verae-fleet/services/customer-service.json index cff1017..461bdd5 100644 --- a/packages/verae-fleet/services/customer-service.json +++ b/packages/verae-fleet/services/customer-service.json @@ -17,7 +17,7 @@ "PORT": "3011", "ZAPPIER_ADMIN_URL": "http://127.0.0.1:3000", "ACCOUNT_BALANCE_URL": "http://127.0.0.1:3010", - "NATS_URL": "nats://127.0.0.1:4222", + "NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222", "AUTHZ_URL": "http://127.0.0.1:3020", "STAFF_IAM_URL": "http://127.0.0.1:3028", "STAFF_AUTH": "1" diff --git a/packages/verae-fleet/services/identity.json b/packages/verae-fleet/services/identity.json index f49b84a..a1a8700 100644 --- a/packages/verae-fleet/services/identity.json +++ b/packages/verae-fleet/services/identity.json @@ -15,7 +15,7 @@ }, "env": { "PORT": "3026", - "NATS_URL": "nats://127.0.0.1:4222" + "NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222" }, "nats": { "in": ["verae.identity.bind", "verae.identity.lookup"], diff --git a/packages/verae-fleet/services/jobs-events.json b/packages/verae-fleet/services/jobs-events.json index 21033e1..0b97c02 100644 --- a/packages/verae-fleet/services/jobs-events.json +++ b/packages/verae-fleet/services/jobs-events.json @@ -15,7 +15,7 @@ }, "env": { "PORT": "3030", - "NATS_URL": "nats://127.0.0.1:4222" + "NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222" }, "nats": { "in": ["verae.zapier.jobs.events"], diff --git a/packages/verae-fleet/services/middleware-http.json b/packages/verae-fleet/services/middleware-http.json index d7a89d5..84197a6 100644 --- a/packages/verae-fleet/services/middleware-http.json +++ b/packages/verae-fleet/services/middleware-http.json @@ -15,9 +15,10 @@ }, "env": { "PORT": "3100", - "NATS_URL": "nats://127.0.0.1:4222", + "NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222", "NATS_ENABLED": "true", - "MOCK_VERAE": "true" + "MOCK_VERAE": "true", + "JETSTREAM_REPLICAS": "3" }, "nats": { "in": ["verae.zapier.jobs.events"], diff --git a/packages/verae-fleet/services/sales-pricing.json b/packages/verae-fleet/services/sales-pricing.json index 082498e..3d4f7c6 100644 --- a/packages/verae-fleet/services/sales-pricing.json +++ b/packages/verae-fleet/services/sales-pricing.json @@ -17,7 +17,7 @@ "PORT": "3012", "ZAPPIER_ADMIN_URL": "http://127.0.0.1:3000", "ACCOUNT_BALANCE_URL": "http://127.0.0.1:3010", - "NATS_URL": "nats://127.0.0.1:4222", + "NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222", "AUTHZ_URL": "http://127.0.0.1:3020", "STAFF_IAM_URL": "http://127.0.0.1:3028", "STAFF_AUTH": "1" diff --git a/packages/verae-fleet/services/zappier-edge.json b/packages/verae-fleet/services/zappier-edge.json index 917cdb5..2cd1a8b 100644 --- a/packages/verae-fleet/services/zappier-edge.json +++ b/packages/verae-fleet/services/zappier-edge.json @@ -22,7 +22,7 @@ "CS_SERVICE_URL": "http://127.0.0.1:3011", "SALES_PRICING_URL": "http://127.0.0.1:3012", "ACCOUNTING_EXPORT_URL": "http://127.0.0.1:3013", - "NATS_URL": "nats://127.0.0.1:4222", + "NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222", "AUTHZ_URL": "http://127.0.0.1:3020", "ACCOUNT_BALANCE_URL": "http://127.0.0.1:3010", "IDENTITY_URL": "http://127.0.0.1:3026" diff --git a/packages/verae-fleet/systemd/verae-fleet.service b/packages/verae-fleet/systemd/verae-fleet.service index c4d1c9f..04b2f7f 100644 --- a/packages/verae-fleet/systemd/verae-fleet.service +++ b/packages/verae-fleet/systemd/verae-fleet.service @@ -9,7 +9,8 @@ Environment=FLEET_FILE=%h/verae-stack/packages/verae-fleet/overlays/ns1/fleet.js Environment=FLEET_MACHINES=%h/verae-stack/packages/verae-fleet/overlays/ns1/machines.json Environment=STAFF_IAM_URL=https://iam.zapier.georgelambert.org Environment=STAFF_COOKIE_DOMAIN=.zapier.georgelambert.org -Environment=NATS_URL=nats://127.0.0.1:4222 +Environment=NATS_URL=nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222 +Environment=JETSTREAM_REPLICAS=3 Environment=FLEET_ENABLE_LAN134= ExecStart=/usr/bin/node src/cli.js serve Restart=always diff --git a/packages/verae-keep/systemd/verae-keep-guard.service b/packages/verae-keep/systemd/verae-keep-guard.service index 75d69a8..c101b1c 100644 --- a/packages/verae-keep/systemd/verae-keep-guard.service +++ b/packages/verae-keep/systemd/verae-keep-guard.service @@ -7,6 +7,7 @@ Type=simple WorkingDirectory=%h/verae-keep Environment=KEEP_UNITS=%h/verae-keep/units.ns1.json Environment=KEEP_STATE=%h/verae-keep/data +Environment=NATS_URL=nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222 ExecStart=/usr/bin/bash %h/verae-keep/scripts/guard.sh Restart=always RestartSec=2 diff --git a/packages/verae-keep/systemd/verae-keep-guard.service.bak b/packages/verae-keep/systemd/verae-keep-guard.service.bak new file mode 100644 index 0000000..75d69a8 --- /dev/null +++ b/packages/verae-keep/systemd/verae-keep-guard.service.bak @@ -0,0 +1,15 @@ +[Unit] +Description=Verae keep-alive guard (restarts watch + keep) +After=network.target + +[Service] +Type=simple +WorkingDirectory=%h/verae-keep +Environment=KEEP_UNITS=%h/verae-keep/units.ns1.json +Environment=KEEP_STATE=%h/verae-keep/data +ExecStart=/usr/bin/bash %h/verae-keep/scripts/guard.sh +Restart=always +RestartSec=2 + +[Install] +WantedBy=default.target diff --git a/packages/verae-keep/units.ns1.json b/packages/verae-keep/units.ns1.json index 07d3b5b..7cc5366 100644 --- a/packages/verae-keep/units.ns1.json +++ b/packages/verae-keep/units.ns1.json @@ -7,15 +7,22 @@ { "id": "nats", "title": "NATS JetStream (loopback)", - "tcp": { "host": "127.0.0.1", "port": 4222 }, + "tcp": { + "host": "10.10.10.21", + "port": 4222 + }, "managed": false, - "defaultIntent": "run" + "defaultIntent": "run", + "env": {} }, { "id": "job-poller-0", "health": "http://127.0.0.1:13200/health", "cwd": "~/verae-fleet-runtime", - "command": ["node", "src/worker.js"], + "command": [ + "node", + "src/worker.js" + ], "env": { "FLEET_ROLE": "job-poller", "FLEET_SERVICE": "job-poller", @@ -23,14 +30,19 @@ "FLEET_MACHINE": "ns1", "FLEET_HEALTH_PORT": "13200", "FLEET_HEALTH_BIND": "127.0.0.1", - "FLEET_STATE_DIR": "data/job-poller-0" + "FLEET_STATE_DIR": "data/job-poller-0", + "NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222", + "JETSTREAM_REPLICAS": "3" } }, { "id": "webhook-deliver-0", "health": "http://127.0.0.1:13300/health", "cwd": "~/verae-fleet-runtime", - "command": ["node", "src/worker.js"], + "command": [ + "node", + "src/worker.js" + ], "env": { "FLEET_ROLE": "webhook-deliver", "FLEET_SERVICE": "webhook-deliver", @@ -38,14 +50,19 @@ "FLEET_MACHINE": "ns1", "FLEET_HEALTH_PORT": "13300", "FLEET_HEALTH_BIND": "127.0.0.1", - "FLEET_STATE_DIR": "data/webhook-deliver-0" + "FLEET_STATE_DIR": "data/webhook-deliver-0", + "NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222", + "JETSTREAM_REPLICAS": "3" } }, { "id": "archive-aggregator-0", "health": "http://127.0.0.1:13400/health", "cwd": "~/verae-fleet-runtime", - "command": ["node", "src/worker.js"], + "command": [ + "node", + "src/worker.js" + ], "env": { "FLEET_ROLE": "archive-aggregator", "FLEET_SERVICE": "archive-aggregator", @@ -53,14 +70,19 @@ "FLEET_MACHINE": "ns1", "FLEET_HEALTH_PORT": "13400", "FLEET_HEALTH_BIND": "127.0.0.1", - "FLEET_STATE_DIR": "data/archive-aggregator-0" + "FLEET_STATE_DIR": "data/archive-aggregator-0", + "NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222", + "JETSTREAM_REPLICAS": "3" } }, { "id": "archive-worm-0", "health": "http://127.0.0.1:13500/health", "cwd": "~/verae-fleet-runtime", - "command": ["node", "src/worker.js"], + "command": [ + "node", + "src/worker.js" + ], "env": { "FLEET_ROLE": "archive-worm", "FLEET_SERVICE": "archive-worm", @@ -68,14 +90,19 @@ "FLEET_MACHINE": "ns1", "FLEET_HEALTH_PORT": "13500", "FLEET_HEALTH_BIND": "127.0.0.1", - "FLEET_STATE_DIR": "data/archive-worm-0" + "FLEET_STATE_DIR": "data/archive-worm-0", + "NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222", + "JETSTREAM_REPLICAS": "3" } }, { "id": "archive-worm-1", "health": "http://127.0.0.1:13501/health", "cwd": "~/verae-fleet-runtime", - "command": ["node", "src/worker.js"], + "command": [ + "node", + "src/worker.js" + ], "env": { "FLEET_ROLE": "archive-worm", "FLEET_SERVICE": "archive-worm", @@ -83,14 +110,19 @@ "FLEET_MACHINE": "ns1", "FLEET_HEALTH_PORT": "13501", "FLEET_HEALTH_BIND": "127.0.0.1", - "FLEET_STATE_DIR": "data/archive-worm-1" + "FLEET_STATE_DIR": "data/archive-worm-1", + "NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222", + "JETSTREAM_REPLICAS": "3" } }, { "id": "archive-worm-2", "health": "http://127.0.0.1:13502/health", "cwd": "~/verae-fleet-runtime", - "command": ["node", "src/worker.js"], + "command": [ + "node", + "src/worker.js" + ], "env": { "FLEET_ROLE": "archive-worm", "FLEET_SERVICE": "archive-worm", @@ -98,14 +130,19 @@ "FLEET_MACHINE": "ns1", "FLEET_HEALTH_PORT": "13502", "FLEET_HEALTH_BIND": "127.0.0.1", - "FLEET_STATE_DIR": "data/archive-worm-2" + "FLEET_STATE_DIR": "data/archive-worm-2", + "NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222", + "JETSTREAM_REPLICAS": "3" } }, { "id": "tree-node-0", "health": "http://127.0.0.1:13600/health", "cwd": "~/verae-fleet-runtime", - "command": ["node", "src/worker.js"], + "command": [ + "node", + "src/worker.js" + ], "env": { "FLEET_ROLE": "tree-node", "FLEET_SERVICE": "tree-node", @@ -113,14 +150,19 @@ "FLEET_MACHINE": "ns1", "FLEET_HEALTH_PORT": "13600", "FLEET_HEALTH_BIND": "127.0.0.1", - "FLEET_STATE_DIR": "data/tree-node-0" + "FLEET_STATE_DIR": "data/tree-node-0", + "NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222", + "JETSTREAM_REPLICAS": "3" } }, { "id": "tree-node-1", "health": "http://127.0.0.1:13601/health", "cwd": "~/verae-fleet-runtime", - "command": ["node", "src/worker.js"], + "command": [ + "node", + "src/worker.js" + ], "env": { "FLEET_ROLE": "tree-node", "FLEET_SERVICE": "tree-node", @@ -128,14 +170,19 @@ "FLEET_MACHINE": "ns1", "FLEET_HEALTH_PORT": "13601", "FLEET_HEALTH_BIND": "127.0.0.1", - "FLEET_STATE_DIR": "data/tree-node-1" + "FLEET_STATE_DIR": "data/tree-node-1", + "NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222", + "JETSTREAM_REPLICAS": "3" } }, { "id": "tree-node-2", "health": "http://127.0.0.1:13602/health", "cwd": "~/verae-fleet-runtime", - "command": ["node", "src/worker.js"], + "command": [ + "node", + "src/worker.js" + ], "env": { "FLEET_ROLE": "tree-node", "FLEET_SERVICE": "tree-node", @@ -143,7 +190,9 @@ "FLEET_MACHINE": "ns1", "FLEET_HEALTH_PORT": "13602", "FLEET_HEALTH_BIND": "127.0.0.1", - "FLEET_STATE_DIR": "data/tree-node-2" + "FLEET_STATE_DIR": "data/tree-node-2", + "NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222", + "JETSTREAM_REPLICAS": "3" } } ] diff --git a/packages/verae-keep/units.px-worker.json b/packages/verae-keep/units.px-worker.json index fbcd8a9..66c751a 100644 --- a/packages/verae-keep/units.px-worker.json +++ b/packages/verae-keep/units.px-worker.json @@ -17,7 +17,7 @@ "FLEET_HEALTH_PORT": "13500", "FLEET_HEALTH_BIND": "127.0.0.1", "FLEET_STATE_DIR": "data/archive-worm-px-0", - "NATS_URL": "nats://10.10.10.1:4222" + "NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222" } }, { @@ -33,7 +33,7 @@ "FLEET_HEALTH_PORT": "13600", "FLEET_HEALTH_BIND": "127.0.0.1", "FLEET_STATE_DIR": "data/tree-node-px-0", - "NATS_URL": "nats://10.10.10.1:4222" + "NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222" } } ] diff --git a/packages/verae-nats-accounts/nats.conf b/packages/verae-nats-accounts/nats.conf index 6e2bede..857af4b 100644 --- a/packages/verae-nats-accounts/nats.conf +++ b/packages/verae-nats-accounts/nats.conf @@ -2,6 +2,10 @@ # INTERNAL: middleware, account-balance, authz, identity. # LEAF: access-leaf and remote leaf nodes — no billing. +# Loopback sketch. The live 3-node test cluster is verae-nats-cluster +# (10.10.10.21–23 on vmbr1) and currently runs without accounts so every +# existing NATS_URL keeps working. Flip this file onto those guests only +# after clients use nats://internal:internal-dev@host:4222. listen: 127.0.0.1:4222 http: 127.0.0.1:8222 jetstream {} diff --git a/packages/verae-nats-cluster/README.md b/packages/verae-nats-cluster/README.md index 75543c3..fd0f381 100644 --- a/packages/verae-nats-cluster/README.md +++ b/packages/verae-nats-cluster/README.md @@ -28,6 +28,17 @@ bash scripts/status.sh bash scripts/test.sh ``` -Cut over a worker later with `NATS_URL=nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222` (for example `px-worker`). Zapier cloud never talks to NATS. +Test-env cut-over (keep/fleet/middleware on NS1): + +```bash +# 1) create product streams (replicas=3) +bash scripts/ensure-streams.sh +# 2) fleet overlay nats.url + services/*.json already list the three URLs +# 3) restart fleet and keep with NATS_URL from client.env +``` + +`JETSTREAM_REPLICAS=3` on middleware. Host `127.0.0.1:4222` can stay up unused. Zapier cloud never talks to NATS. + +nkeys/mTLS: `verae-nats-accounts` still has the INTERNAL/LEAF sketch. Do **not** enable accounts on this cluster until every client passes credentials in `NATS_URL`. Private `vmbr1` is the current isolation. Hardware move: same three configs, three boxes, private NIC only — change IPs in `cluster.env` and `conf/nats.conf.tmpl`. diff --git a/packages/verae-nats-cluster/client.env b/packages/verae-nats-cluster/client.env new file mode 100644 index 0000000..4afa412 --- /dev/null +++ b/packages/verae-nats-cluster/client.env @@ -0,0 +1,3 @@ +# Test-environment client URL. Private vmbr1 only. Lab loopback stays until cut-over. +export NATS_URL="${NATS_URL:-nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222}" +export JETSTREAM_REPLICAS="${JETSTREAM_REPLICAS:-3}" diff --git a/packages/verae-nats-cluster/scripts/cutover-ns1.sh b/packages/verae-nats-cluster/scripts/cutover-ns1.sh new file mode 100755 index 0000000..6fa073c --- /dev/null +++ b/packages/verae-nats-cluster/scripts/cutover-ns1.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# Point NS1 test modules at the 3-node vmbr1 cluster. Does not change MOCK_VERAE or Zapier. +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +# shellcheck disable=SC1091 +. "$ROOT/client.env" +export PATH="/usr/sbin:/usr/bin:/bin:$PATH" +bash "$ROOT/scripts/ensure-streams.sh" +echo "NATS_URL=$NATS_URL" +echo "streams ensured. restart keep + fleet on the host after copying overlay/service JSON." diff --git a/packages/verae-nats-cluster/scripts/ensure-streams.sh b/packages/verae-nats-cluster/scripts/ensure-streams.sh new file mode 100755 index 0000000..4b661be --- /dev/null +++ b/packages/verae-nats-cluster/scripts/ensure-streams.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Create product JetStream streams with replicas=3 on the Proxmox cluster. +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +# shellcheck disable=SC1091 +. "$ROOT/client.env" +export PATH="/usr/sbin:/usr/bin:/bin:/usr/local/bin:$PATH" +VMID="${1:-511}" +sudo pct exec "$VMID" -- bash -lc " +set -e +export DEBIAN_FRONTEND=noninteractive +export PATH=/usr/local/bin:/usr/bin:/bin +export NATS_URL=nats://10.10.10.21:4222 +if [[ ! -x /usr/local/bin/nats ]]; then + apt-get install -y --no-install-recommends unzip >/dev/null + curl -fsSL https://github.com/nats-io/natscli/releases/download/v0.1.6/nats-0.1.6-linux-amd64.zip -o /tmp/natscli.zip + rm -rf /tmp/natscli && mkdir -p /tmp/natscli + unzip -o /tmp/natscli.zip -d /tmp/natscli >/dev/null + BIN=\$(find /tmp/natscli -type f -name nats | head -1) + install -m 0755 \"\$BIN\" /usr/local/bin/nats +fi +add() { + local name=\$1 subj=\$2 + nats stream info \"\$name\" >/dev/null 2>&1 && return 0 + nats stream add \"\$name\" --subjects=\"\$subj\" --replicas=3 --storage=file --retention=limits --discard=old --max-msgs=-1 --max-bytes=-1 --max-age=24h --dupe-window=2m --defaults +} +add ZAPIER_JOBS 'verae.zapier.jobs.watch' +add ZAPIER_EVENTS 'verae.zapier.jobs.events' +add ZAPIER_WEBHOOKS 'verae.zapier.webhooks.deliver' +add ZAPIER_USAGE 'verae.zapier.usage' +add VERAE_ARCHIVE 'verae.archive.>' +nats stream ls +" +echo "streams ready on cluster (replicas=3)" diff --git a/packages/verae-ops/GETTING-STARTED.md b/packages/verae-ops/GETTING-STARTED.md index 0457140..9dbb722 100644 --- a/packages/verae-ops/GETTING-STARTED.md +++ b/packages/verae-ops/GETTING-STARTED.md @@ -75,7 +75,11 @@ Open: - Accounting review/export: http://127.0.0.1:3013/ - API docs: http://127.0.0.1:3000/docs -Do **not** publish NATS `4222`. +Test env NATS is the **three LXC cluster** on `vmbr1`: + +`nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222` + +Host loopback `:4222` may still listen but modules no longer use it. Do **not** publish NATS on `vmbr0`. ## Operator console (monitor) diff --git a/packages/verae-zapier-middleware/src/nats/connection.js b/packages/verae-zapier-middleware/src/nats/connection.js index 5630b38..25e48c6 100644 --- a/packages/verae-zapier-middleware/src/nats/connection.js +++ b/packages/verae-zapier-middleware/src/nats/connection.js @@ -78,7 +78,7 @@ export async function ensureStreams(manager) { retention: 'limits', storage: 'file', max_age: 24 * 60 * 60 * 1e9, // 24h in ns - num_replicas: 1, + num_replicas: Number.parseInt(process.env.JETSTREAM_REPLICAS || '1', 10) || 1, }); log.info('stream created', { stream: def.name, subjects: def.subjects }); } diff --git a/packages/zapier-decisions/LOG.md b/packages/zapier-decisions/LOG.md index 1860670..7405342 100644 --- a/packages/zapier-decisions/LOG.md +++ b/packages/zapier-decisions/LOG.md @@ -1,5 +1,12 @@ # Action log +## 2026-09-12 — test env on the 3-node cluster + +- Fleet, keep, middleware, billing, and access planes `NATS_URL` → `10.10.10.21–23:4222`. +- Streams ZAPIER_JOBS/EVENTS/WEBHOOKS/USAGE + VERAE_ARCHIVE replicas=3. +- px-worker keep worm/tree 200 on the cluster. Backup timer enabled. Uptime loop on px-worker. +- MOCK_VERAE still true. Zapier push still waiting on login. nkeys/mTLS not flipped (would require client creds). + ## 2026-09-12 — NATS cluster on three Proxmox LXC - New repo **verae-nats-cluster**: LXC 511/512/513 (`nats-a/b/c`, `10.10.10.21–23`) JetStream cluster on `vmbr1`.