Cut the test environment over to the 3-node Proxmox NATS cluster.
Some checks are pending
offline / test (push) Waiting to run
Some checks are pending
offline / test (push) Waiting to run
Fleet/keep/middleware NATS_URL lists 10.10.10.21–23. Product streams are replicas=3. px-worker worm/tree run on that cluster. Backup timer and px-worker uptime watch are on. MOCK_VERAE and Zapier push unchanged.
This commit is contained in:
parent
69e28af4d8
commit
a73875f4ef
29 changed files with 184 additions and 43 deletions
|
|
@ -18,7 +18,7 @@ Today’s NS1 box (`NS1.GEORGELAMBERT.ORG`, `70.88.205.138`) still runs the **la
|
|||
nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222
|
||||
```
|
||||
|
||||
(`nats-a` VMID 511, `nats-b` 512, `nats-c` 513). Cluster routes on `:6222`. Nothing on `vmbr0`. Cut over workers with `NATS_URL` listing all three; then the same configs move to separate hardware. Fleet SSH hosts run **workers**, not extra public NATS listeners.
|
||||
(`nats-a` VMID 511, `nats-b` 512, `nats-c` 513). Cluster routes on `:6222`. Nothing on `vmbr0`. **Test env is cut over:** fleet/keep/middleware `NATS_URL` lists all three; product streams `ZAPIER_*` and `VERAE_ARCHIVE` are `replicas=3`. Same configs later move to separate hardware. Fleet SSH hosts run **workers**, not extra public NATS listeners.
|
||||
|
||||

|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,8 @@ FLEET="$STACK/packages/verae-fleet"
|
|||
EDGE_PORT="${EDGE_PORT:-13000}"
|
||||
export STAFF_IAM_URL="${STAFF_IAM_URL:-https://iam.zapier.georgelambert.org}"
|
||||
export STAFF_COOKIE_DOMAIN="${STAFF_COOKIE_DOMAIN:-.zapier.georgelambert.org}"
|
||||
export NATS_URL="${NATS_URL:-nats://127.0.0.1:4222}"
|
||||
export NATS_URL="${NATS_URL:-nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222}"
|
||||
export JETSTREAM_REPLICAS="${JETSTREAM_REPLICAS:-3}"
|
||||
export FLEET_ENABLE_LAN134="${FLEET_ENABLE_LAN134:-}"
|
||||
export FLEET_FILE="${FLEET_FILE:-$FLEET/overlays/ns1/fleet.json}"
|
||||
export FLEET_MACHINES="${FLEET_MACHINES:-$FLEET/overlays/ns1/machines.json}"
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@
|
|||
"maxRestartsPerHour": 40
|
||||
},
|
||||
"nats": {
|
||||
"url": "nats://127.0.0.1:4222",
|
||||
"url": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222",
|
||||
"publicBind": false
|
||||
},
|
||||
"services": {
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@
|
|||
"AUTHZ_URL": "http://127.0.0.1:3020",
|
||||
"ACCOUNT_BALANCE_URL": "http://127.0.0.1:3010",
|
||||
"ZAPPIER_UPSTREAM": "http://127.0.0.1:3100",
|
||||
"NATS_URL": "nats://127.0.0.1:4222"
|
||||
"NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222"
|
||||
},
|
||||
"nats": {
|
||||
"in": [],
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@
|
|||
},
|
||||
"env": {
|
||||
"PORT": "3020",
|
||||
"NATS_URL": "nats://127.0.0.1:4222"
|
||||
"NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222"
|
||||
},
|
||||
"nats": {
|
||||
"in": ["verae.access.authz.check"],
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@
|
|||
"AUTHZ_URL": "http://127.0.0.1:3020",
|
||||
"LEAF_TOKEN": "leaf-dev-token",
|
||||
"NATS_POLICY_PATH": "../verae-nats-accounts/policy.json",
|
||||
"NATS_URL": "nats://127.0.0.1:4222"
|
||||
"NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222"
|
||||
},
|
||||
"nats": {
|
||||
"in": ["verae.access.leaf.in"],
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@
|
|||
"PORT": "3021",
|
||||
"AUTHZ_URL": "http://127.0.0.1:3020",
|
||||
"ACCOUNT_BALANCE_URL": "http://127.0.0.1:3010",
|
||||
"NATS_URL": "nats://127.0.0.1:4222",
|
||||
"NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222",
|
||||
"ZAPPIER_EDGE_URL": "http://127.0.0.1:3000"
|
||||
},
|
||||
"nats": {
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@
|
|||
"PORT": "3024",
|
||||
"AUTHZ_URL": "http://127.0.0.1:3020",
|
||||
"ZAPPIER_UPSTREAM": "http://127.0.0.1:3100",
|
||||
"NATS_URL": "nats://127.0.0.1:4222"
|
||||
"NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222"
|
||||
},
|
||||
"nats": {
|
||||
"in": [],
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@
|
|||
},
|
||||
"env": {
|
||||
"PORT": "3010",
|
||||
"NATS_URL": "nats://127.0.0.1:4222"
|
||||
"NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222"
|
||||
},
|
||||
"nats": {
|
||||
"in": [
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@
|
|||
"PORT": "3013",
|
||||
"ZAPPIER_ADMIN_URL": "http://127.0.0.1:3000",
|
||||
"ACCOUNT_BALANCE_URL": "http://127.0.0.1:3010",
|
||||
"NATS_URL": "nats://127.0.0.1:4222",
|
||||
"NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222",
|
||||
"STAFF_IAM_URL": "http://127.0.0.1:3028",
|
||||
"STAFF_AUTH": "1"
|
||||
},
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@
|
|||
"PORT": "3011",
|
||||
"ZAPPIER_ADMIN_URL": "http://127.0.0.1:3000",
|
||||
"ACCOUNT_BALANCE_URL": "http://127.0.0.1:3010",
|
||||
"NATS_URL": "nats://127.0.0.1:4222",
|
||||
"NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222",
|
||||
"AUTHZ_URL": "http://127.0.0.1:3020",
|
||||
"STAFF_IAM_URL": "http://127.0.0.1:3028",
|
||||
"STAFF_AUTH": "1"
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@
|
|||
},
|
||||
"env": {
|
||||
"PORT": "3026",
|
||||
"NATS_URL": "nats://127.0.0.1:4222"
|
||||
"NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222"
|
||||
},
|
||||
"nats": {
|
||||
"in": ["verae.identity.bind", "verae.identity.lookup"],
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@
|
|||
},
|
||||
"env": {
|
||||
"PORT": "3030",
|
||||
"NATS_URL": "nats://127.0.0.1:4222"
|
||||
"NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222"
|
||||
},
|
||||
"nats": {
|
||||
"in": ["verae.zapier.jobs.events"],
|
||||
|
|
|
|||
|
|
@ -15,9 +15,10 @@
|
|||
},
|
||||
"env": {
|
||||
"PORT": "3100",
|
||||
"NATS_URL": "nats://127.0.0.1:4222",
|
||||
"NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222",
|
||||
"NATS_ENABLED": "true",
|
||||
"MOCK_VERAE": "true"
|
||||
"MOCK_VERAE": "true",
|
||||
"JETSTREAM_REPLICAS": "3"
|
||||
},
|
||||
"nats": {
|
||||
"in": ["verae.zapier.jobs.events"],
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@
|
|||
"PORT": "3012",
|
||||
"ZAPPIER_ADMIN_URL": "http://127.0.0.1:3000",
|
||||
"ACCOUNT_BALANCE_URL": "http://127.0.0.1:3010",
|
||||
"NATS_URL": "nats://127.0.0.1:4222",
|
||||
"NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222",
|
||||
"AUTHZ_URL": "http://127.0.0.1:3020",
|
||||
"STAFF_IAM_URL": "http://127.0.0.1:3028",
|
||||
"STAFF_AUTH": "1"
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@
|
|||
"CS_SERVICE_URL": "http://127.0.0.1:3011",
|
||||
"SALES_PRICING_URL": "http://127.0.0.1:3012",
|
||||
"ACCOUNTING_EXPORT_URL": "http://127.0.0.1:3013",
|
||||
"NATS_URL": "nats://127.0.0.1:4222",
|
||||
"NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222",
|
||||
"AUTHZ_URL": "http://127.0.0.1:3020",
|
||||
"ACCOUNT_BALANCE_URL": "http://127.0.0.1:3010",
|
||||
"IDENTITY_URL": "http://127.0.0.1:3026"
|
||||
|
|
|
|||
|
|
@ -9,7 +9,8 @@ Environment=FLEET_FILE=%h/verae-stack/packages/verae-fleet/overlays/ns1/fleet.js
|
|||
Environment=FLEET_MACHINES=%h/verae-stack/packages/verae-fleet/overlays/ns1/machines.json
|
||||
Environment=STAFF_IAM_URL=https://iam.zapier.georgelambert.org
|
||||
Environment=STAFF_COOKIE_DOMAIN=.zapier.georgelambert.org
|
||||
Environment=NATS_URL=nats://127.0.0.1:4222
|
||||
Environment=NATS_URL=nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222
|
||||
Environment=JETSTREAM_REPLICAS=3
|
||||
Environment=FLEET_ENABLE_LAN134=
|
||||
ExecStart=/usr/bin/node src/cli.js serve
|
||||
Restart=always
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ Type=simple
|
|||
WorkingDirectory=%h/verae-keep
|
||||
Environment=KEEP_UNITS=%h/verae-keep/units.ns1.json
|
||||
Environment=KEEP_STATE=%h/verae-keep/data
|
||||
Environment=NATS_URL=nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222
|
||||
ExecStart=/usr/bin/bash %h/verae-keep/scripts/guard.sh
|
||||
Restart=always
|
||||
RestartSec=2
|
||||
|
|
|
|||
15
packages/verae-keep/systemd/verae-keep-guard.service.bak
Normal file
15
packages/verae-keep/systemd/verae-keep-guard.service.bak
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
[Unit]
|
||||
Description=Verae keep-alive guard (restarts watch + keep)
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
WorkingDirectory=%h/verae-keep
|
||||
Environment=KEEP_UNITS=%h/verae-keep/units.ns1.json
|
||||
Environment=KEEP_STATE=%h/verae-keep/data
|
||||
ExecStart=/usr/bin/bash %h/verae-keep/scripts/guard.sh
|
||||
Restart=always
|
||||
RestartSec=2
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
|
|
@ -7,15 +7,22 @@
|
|||
{
|
||||
"id": "nats",
|
||||
"title": "NATS JetStream (loopback)",
|
||||
"tcp": { "host": "127.0.0.1", "port": 4222 },
|
||||
"tcp": {
|
||||
"host": "10.10.10.21",
|
||||
"port": 4222
|
||||
},
|
||||
"managed": false,
|
||||
"defaultIntent": "run"
|
||||
"defaultIntent": "run",
|
||||
"env": {}
|
||||
},
|
||||
{
|
||||
"id": "job-poller-0",
|
||||
"health": "http://127.0.0.1:13200/health",
|
||||
"cwd": "~/verae-fleet-runtime",
|
||||
"command": ["node", "src/worker.js"],
|
||||
"command": [
|
||||
"node",
|
||||
"src/worker.js"
|
||||
],
|
||||
"env": {
|
||||
"FLEET_ROLE": "job-poller",
|
||||
"FLEET_SERVICE": "job-poller",
|
||||
|
|
@ -23,14 +30,19 @@
|
|||
"FLEET_MACHINE": "ns1",
|
||||
"FLEET_HEALTH_PORT": "13200",
|
||||
"FLEET_HEALTH_BIND": "127.0.0.1",
|
||||
"FLEET_STATE_DIR": "data/job-poller-0"
|
||||
"FLEET_STATE_DIR": "data/job-poller-0",
|
||||
"NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222",
|
||||
"JETSTREAM_REPLICAS": "3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "webhook-deliver-0",
|
||||
"health": "http://127.0.0.1:13300/health",
|
||||
"cwd": "~/verae-fleet-runtime",
|
||||
"command": ["node", "src/worker.js"],
|
||||
"command": [
|
||||
"node",
|
||||
"src/worker.js"
|
||||
],
|
||||
"env": {
|
||||
"FLEET_ROLE": "webhook-deliver",
|
||||
"FLEET_SERVICE": "webhook-deliver",
|
||||
|
|
@ -38,14 +50,19 @@
|
|||
"FLEET_MACHINE": "ns1",
|
||||
"FLEET_HEALTH_PORT": "13300",
|
||||
"FLEET_HEALTH_BIND": "127.0.0.1",
|
||||
"FLEET_STATE_DIR": "data/webhook-deliver-0"
|
||||
"FLEET_STATE_DIR": "data/webhook-deliver-0",
|
||||
"NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222",
|
||||
"JETSTREAM_REPLICAS": "3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "archive-aggregator-0",
|
||||
"health": "http://127.0.0.1:13400/health",
|
||||
"cwd": "~/verae-fleet-runtime",
|
||||
"command": ["node", "src/worker.js"],
|
||||
"command": [
|
||||
"node",
|
||||
"src/worker.js"
|
||||
],
|
||||
"env": {
|
||||
"FLEET_ROLE": "archive-aggregator",
|
||||
"FLEET_SERVICE": "archive-aggregator",
|
||||
|
|
@ -53,14 +70,19 @@
|
|||
"FLEET_MACHINE": "ns1",
|
||||
"FLEET_HEALTH_PORT": "13400",
|
||||
"FLEET_HEALTH_BIND": "127.0.0.1",
|
||||
"FLEET_STATE_DIR": "data/archive-aggregator-0"
|
||||
"FLEET_STATE_DIR": "data/archive-aggregator-0",
|
||||
"NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222",
|
||||
"JETSTREAM_REPLICAS": "3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "archive-worm-0",
|
||||
"health": "http://127.0.0.1:13500/health",
|
||||
"cwd": "~/verae-fleet-runtime",
|
||||
"command": ["node", "src/worker.js"],
|
||||
"command": [
|
||||
"node",
|
||||
"src/worker.js"
|
||||
],
|
||||
"env": {
|
||||
"FLEET_ROLE": "archive-worm",
|
||||
"FLEET_SERVICE": "archive-worm",
|
||||
|
|
@ -68,14 +90,19 @@
|
|||
"FLEET_MACHINE": "ns1",
|
||||
"FLEET_HEALTH_PORT": "13500",
|
||||
"FLEET_HEALTH_BIND": "127.0.0.1",
|
||||
"FLEET_STATE_DIR": "data/archive-worm-0"
|
||||
"FLEET_STATE_DIR": "data/archive-worm-0",
|
||||
"NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222",
|
||||
"JETSTREAM_REPLICAS": "3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "archive-worm-1",
|
||||
"health": "http://127.0.0.1:13501/health",
|
||||
"cwd": "~/verae-fleet-runtime",
|
||||
"command": ["node", "src/worker.js"],
|
||||
"command": [
|
||||
"node",
|
||||
"src/worker.js"
|
||||
],
|
||||
"env": {
|
||||
"FLEET_ROLE": "archive-worm",
|
||||
"FLEET_SERVICE": "archive-worm",
|
||||
|
|
@ -83,14 +110,19 @@
|
|||
"FLEET_MACHINE": "ns1",
|
||||
"FLEET_HEALTH_PORT": "13501",
|
||||
"FLEET_HEALTH_BIND": "127.0.0.1",
|
||||
"FLEET_STATE_DIR": "data/archive-worm-1"
|
||||
"FLEET_STATE_DIR": "data/archive-worm-1",
|
||||
"NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222",
|
||||
"JETSTREAM_REPLICAS": "3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "archive-worm-2",
|
||||
"health": "http://127.0.0.1:13502/health",
|
||||
"cwd": "~/verae-fleet-runtime",
|
||||
"command": ["node", "src/worker.js"],
|
||||
"command": [
|
||||
"node",
|
||||
"src/worker.js"
|
||||
],
|
||||
"env": {
|
||||
"FLEET_ROLE": "archive-worm",
|
||||
"FLEET_SERVICE": "archive-worm",
|
||||
|
|
@ -98,14 +130,19 @@
|
|||
"FLEET_MACHINE": "ns1",
|
||||
"FLEET_HEALTH_PORT": "13502",
|
||||
"FLEET_HEALTH_BIND": "127.0.0.1",
|
||||
"FLEET_STATE_DIR": "data/archive-worm-2"
|
||||
"FLEET_STATE_DIR": "data/archive-worm-2",
|
||||
"NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222",
|
||||
"JETSTREAM_REPLICAS": "3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "tree-node-0",
|
||||
"health": "http://127.0.0.1:13600/health",
|
||||
"cwd": "~/verae-fleet-runtime",
|
||||
"command": ["node", "src/worker.js"],
|
||||
"command": [
|
||||
"node",
|
||||
"src/worker.js"
|
||||
],
|
||||
"env": {
|
||||
"FLEET_ROLE": "tree-node",
|
||||
"FLEET_SERVICE": "tree-node",
|
||||
|
|
@ -113,14 +150,19 @@
|
|||
"FLEET_MACHINE": "ns1",
|
||||
"FLEET_HEALTH_PORT": "13600",
|
||||
"FLEET_HEALTH_BIND": "127.0.0.1",
|
||||
"FLEET_STATE_DIR": "data/tree-node-0"
|
||||
"FLEET_STATE_DIR": "data/tree-node-0",
|
||||
"NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222",
|
||||
"JETSTREAM_REPLICAS": "3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "tree-node-1",
|
||||
"health": "http://127.0.0.1:13601/health",
|
||||
"cwd": "~/verae-fleet-runtime",
|
||||
"command": ["node", "src/worker.js"],
|
||||
"command": [
|
||||
"node",
|
||||
"src/worker.js"
|
||||
],
|
||||
"env": {
|
||||
"FLEET_ROLE": "tree-node",
|
||||
"FLEET_SERVICE": "tree-node",
|
||||
|
|
@ -128,14 +170,19 @@
|
|||
"FLEET_MACHINE": "ns1",
|
||||
"FLEET_HEALTH_PORT": "13601",
|
||||
"FLEET_HEALTH_BIND": "127.0.0.1",
|
||||
"FLEET_STATE_DIR": "data/tree-node-1"
|
||||
"FLEET_STATE_DIR": "data/tree-node-1",
|
||||
"NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222",
|
||||
"JETSTREAM_REPLICAS": "3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "tree-node-2",
|
||||
"health": "http://127.0.0.1:13602/health",
|
||||
"cwd": "~/verae-fleet-runtime",
|
||||
"command": ["node", "src/worker.js"],
|
||||
"command": [
|
||||
"node",
|
||||
"src/worker.js"
|
||||
],
|
||||
"env": {
|
||||
"FLEET_ROLE": "tree-node",
|
||||
"FLEET_SERVICE": "tree-node",
|
||||
|
|
@ -143,7 +190,9 @@
|
|||
"FLEET_MACHINE": "ns1",
|
||||
"FLEET_HEALTH_PORT": "13602",
|
||||
"FLEET_HEALTH_BIND": "127.0.0.1",
|
||||
"FLEET_STATE_DIR": "data/tree-node-2"
|
||||
"FLEET_STATE_DIR": "data/tree-node-2",
|
||||
"NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222",
|
||||
"JETSTREAM_REPLICAS": "3"
|
||||
}
|
||||
}
|
||||
]
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@
|
|||
"FLEET_HEALTH_PORT": "13500",
|
||||
"FLEET_HEALTH_BIND": "127.0.0.1",
|
||||
"FLEET_STATE_DIR": "data/archive-worm-px-0",
|
||||
"NATS_URL": "nats://10.10.10.1:4222"
|
||||
"NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222"
|
||||
}
|
||||
},
|
||||
{
|
||||
|
|
@ -33,7 +33,7 @@
|
|||
"FLEET_HEALTH_PORT": "13600",
|
||||
"FLEET_HEALTH_BIND": "127.0.0.1",
|
||||
"FLEET_STATE_DIR": "data/tree-node-px-0",
|
||||
"NATS_URL": "nats://10.10.10.1:4222"
|
||||
"NATS_URL": "nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222"
|
||||
}
|
||||
}
|
||||
]
|
||||
|
|
|
|||
|
|
@ -2,6 +2,10 @@
|
|||
# INTERNAL: middleware, account-balance, authz, identity.
|
||||
# LEAF: access-leaf and remote leaf nodes — no billing.
|
||||
|
||||
# Loopback sketch. The live 3-node test cluster is verae-nats-cluster
|
||||
# (10.10.10.21–23 on vmbr1) and currently runs without accounts so every
|
||||
# existing NATS_URL keeps working. Flip this file onto those guests only
|
||||
# after clients use nats://internal:internal-dev@host:4222.
|
||||
listen: 127.0.0.1:4222
|
||||
http: 127.0.0.1:8222
|
||||
jetstream {}
|
||||
|
|
|
|||
|
|
@ -28,6 +28,17 @@ bash scripts/status.sh
|
|||
bash scripts/test.sh
|
||||
```
|
||||
|
||||
Cut over a worker later with `NATS_URL=nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222` (for example `px-worker`). Zapier cloud never talks to NATS.
|
||||
Test-env cut-over (keep/fleet/middleware on NS1):
|
||||
|
||||
```bash
|
||||
# 1) create product streams (replicas=3)
|
||||
bash scripts/ensure-streams.sh
|
||||
# 2) fleet overlay nats.url + services/*.json already list the three URLs
|
||||
# 3) restart fleet and keep with NATS_URL from client.env
|
||||
```
|
||||
|
||||
`JETSTREAM_REPLICAS=3` on middleware. Host `127.0.0.1:4222` can stay up unused. Zapier cloud never talks to NATS.
|
||||
|
||||
nkeys/mTLS: `verae-nats-accounts` still has the INTERNAL/LEAF sketch. Do **not** enable accounts on this cluster until every client passes credentials in `NATS_URL`. Private `vmbr1` is the current isolation.
|
||||
|
||||
Hardware move: same three configs, three boxes, private NIC only — change IPs in `cluster.env` and `conf/nats.conf.tmpl`.
|
||||
|
|
|
|||
3
packages/verae-nats-cluster/client.env
Normal file
3
packages/verae-nats-cluster/client.env
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
# Test-environment client URL. Private vmbr1 only. Lab loopback stays until cut-over.
|
||||
export NATS_URL="${NATS_URL:-nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222}"
|
||||
export JETSTREAM_REPLICAS="${JETSTREAM_REPLICAS:-3}"
|
||||
10
packages/verae-nats-cluster/scripts/cutover-ns1.sh
Executable file
10
packages/verae-nats-cluster/scripts/cutover-ns1.sh
Executable file
|
|
@ -0,0 +1,10 @@
|
|||
#!/usr/bin/env bash
|
||||
# Point NS1 test modules at the 3-node vmbr1 cluster. Does not change MOCK_VERAE or Zapier.
|
||||
set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
. "$ROOT/client.env"
|
||||
export PATH="/usr/sbin:/usr/bin:/bin:$PATH"
|
||||
bash "$ROOT/scripts/ensure-streams.sh"
|
||||
echo "NATS_URL=$NATS_URL"
|
||||
echo "streams ensured. restart keep + fleet on the host after copying overlay/service JSON."
|
||||
34
packages/verae-nats-cluster/scripts/ensure-streams.sh
Executable file
34
packages/verae-nats-cluster/scripts/ensure-streams.sh
Executable file
|
|
@ -0,0 +1,34 @@
|
|||
#!/usr/bin/env bash
|
||||
# Create product JetStream streams with replicas=3 on the Proxmox cluster.
|
||||
set -euo pipefail
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
# shellcheck disable=SC1091
|
||||
. "$ROOT/client.env"
|
||||
export PATH="/usr/sbin:/usr/bin:/bin:/usr/local/bin:$PATH"
|
||||
VMID="${1:-511}"
|
||||
sudo pct exec "$VMID" -- bash -lc "
|
||||
set -e
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
export PATH=/usr/local/bin:/usr/bin:/bin
|
||||
export NATS_URL=nats://10.10.10.21:4222
|
||||
if [[ ! -x /usr/local/bin/nats ]]; then
|
||||
apt-get install -y --no-install-recommends unzip >/dev/null
|
||||
curl -fsSL https://github.com/nats-io/natscli/releases/download/v0.1.6/nats-0.1.6-linux-amd64.zip -o /tmp/natscli.zip
|
||||
rm -rf /tmp/natscli && mkdir -p /tmp/natscli
|
||||
unzip -o /tmp/natscli.zip -d /tmp/natscli >/dev/null
|
||||
BIN=\$(find /tmp/natscli -type f -name nats | head -1)
|
||||
install -m 0755 \"\$BIN\" /usr/local/bin/nats
|
||||
fi
|
||||
add() {
|
||||
local name=\$1 subj=\$2
|
||||
nats stream info \"\$name\" >/dev/null 2>&1 && return 0
|
||||
nats stream add \"\$name\" --subjects=\"\$subj\" --replicas=3 --storage=file --retention=limits --discard=old --max-msgs=-1 --max-bytes=-1 --max-age=24h --dupe-window=2m --defaults
|
||||
}
|
||||
add ZAPIER_JOBS 'verae.zapier.jobs.watch'
|
||||
add ZAPIER_EVENTS 'verae.zapier.jobs.events'
|
||||
add ZAPIER_WEBHOOKS 'verae.zapier.webhooks.deliver'
|
||||
add ZAPIER_USAGE 'verae.zapier.usage'
|
||||
add VERAE_ARCHIVE 'verae.archive.>'
|
||||
nats stream ls
|
||||
"
|
||||
echo "streams ready on cluster (replicas=3)"
|
||||
|
|
@ -75,7 +75,11 @@ Open:
|
|||
- Accounting review/export: http://127.0.0.1:3013/
|
||||
- API docs: http://127.0.0.1:3000/docs
|
||||
|
||||
Do **not** publish NATS `4222`.
|
||||
Test env NATS is the **three LXC cluster** on `vmbr1`:
|
||||
|
||||
`nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222`
|
||||
|
||||
Host loopback `:4222` may still listen but modules no longer use it. Do **not** publish NATS on `vmbr0`.
|
||||
|
||||
## Operator console (monitor)
|
||||
|
||||
|
|
|
|||
|
|
@ -78,7 +78,7 @@ export async function ensureStreams(manager) {
|
|||
retention: 'limits',
|
||||
storage: 'file',
|
||||
max_age: 24 * 60 * 60 * 1e9, // 24h in ns
|
||||
num_replicas: 1,
|
||||
num_replicas: Number.parseInt(process.env.JETSTREAM_REPLICAS || '1', 10) || 1,
|
||||
});
|
||||
log.info('stream created', { stream: def.name, subjects: def.subjects });
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,12 @@
|
|||
# Action log
|
||||
|
||||
## 2026-09-12 — test env on the 3-node cluster
|
||||
|
||||
- Fleet, keep, middleware, billing, and access planes `NATS_URL` → `10.10.10.21–23:4222`.
|
||||
- Streams ZAPIER_JOBS/EVENTS/WEBHOOKS/USAGE + VERAE_ARCHIVE replicas=3.
|
||||
- px-worker keep worm/tree 200 on the cluster. Backup timer enabled. Uptime loop on px-worker.
|
||||
- MOCK_VERAE still true. Zapier push still waiting on login. nkeys/mTLS not flipped (would require client creds).
|
||||
|
||||
## 2026-09-12 — NATS cluster on three Proxmox LXC
|
||||
|
||||
- New repo **verae-nats-cluster**: LXC 511/512/513 (`nats-a/b/c`, `10.10.10.21–23`) JetStream cluster on `vmbr1`.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue