Initial import of verae-fleet from zapier monorepo

test/classify.test.js

@@ -0,0 +1,36 @@
+import { describe, it } from 'node:test';
+import assert from 'node:assert/strict';
+import { classifyService } from '../src/classify.js';
+
+describe('classifyService', () => {
+  it('operational when available meets min and none paused', () => {
+    assert.equal(
+      classifyService({ min: 3, available: 3, running: 3, paused: 0, instances: [{ pid: 1, healthy: true }] }),
+      'operational',
+    );
+  });
+
+  it('degraded when a replica is paused even if floor is still met', () => {
+    assert.equal(
+      classifyService({
+        min: 3,
+        available: 3,
+        running: 4,
+        paused: 1,
+        instances: [
+          { pid: 1, paused: true, healthy: false },
+          { pid: 2, paused: false, healthy: true },
+        ],
+      }),
+      'degraded',
+    );
+  });
+
+  it('not operational when min>0 and nothing available', () => {
+    assert.equal(classifyService({ min: 1, available: 0, running: 0, paused: 0, instances: [] }), 'down');
+  });
+
+  it('optional idle (min 0, not running) is operational', () => {
+    assert.equal(classifyService({ min: 0, available: 0, running: 0, paused: 0 }), 'operational');
+  });
+});

test/fleet.test.js

@@ -0,0 +1,141 @@
+import { describe, it, after } from 'node:test';
+import assert from 'node:assert/strict';
+import { loadFleet, listServices, FLEET_ROOT } from '../src/load.js';
+import { Supervisor } from '../src/supervisor.js';
+import { Monitor } from '../src/monitor.js';
+
+function treeOnly(healthBase) {
+  const loaded = loadFleet(FLEET_ROOT, { overlay: false });
+  for (const s of Object.values(loaded.services)) {
+    s.enabled = s.id === 'tree-node';
+    if (s.id === 'tree-node') {
+      s.min = 3;
+      s.max = 6;
+      s.keepFloor = true;
+      s.ports = { healthBase };
+    }
+  }
+  loaded.machinesOverlay = null;
+  loaded.machines = loaded.machines.filter((m) => m.kind === 'local' && m.enabled);
+  if (!loaded.machines.length) {
+    loaded.machines = [
+      { id: 'local', title: 'local', kind: 'local', host: '127.0.0.1', enabled: true, capacity: 32, roles: ['*'] },
+    ];
+  }
+  return loaded;
+}
+
+describe('verae-fleet', () => {
+  /** @type {Supervisor[]} */
+  const supervisors = [];
+  after(async () => {
+    for (const s of supervisors) await s.stopAll();
+  });
+
+  it('lists every service with a config file and central replica spec', () => {
+    const loaded = loadFleet();
+    const rows = listServices(loaded);
+    const ids = rows.map((r) => r.id);
+    for (const need of [
+      'nats',
+      'zappier-edge',
+      'middleware-http',
+      'job-poller',
+      'webhook-deliver',
+      'archive-aggregator',
+      'archive-worm',
+      'tree-node',
+      'zapier-simulator',
+      'zapier-platform-app',
+    ]) {
+      assert.ok(ids.includes(need), `missing ${need}`);
+    }
+    const tree = loaded.services['tree-node'];
+    assert.equal(tree.min, 3);
+    assert.equal(tree.keepFloor, true);
+    assert.equal(tree.configPath, 'services/tree-node.json');
+    assert.equal(loaded.fleetPath, 'fleet.json');
+  });
+
+  it('starts the tree-node floor (3 available copies)', async () => {
+    const sup = new Supervisor({ loaded: treeOnly(14600) });
+    supervisors.push(sup);
+    await sup.startService('tree-node');
+    const st = sup.status().services['tree-node'];
+    assert.equal(st.available, 3);
+    assert.equal(st.belowFloor, false);
+    assert.equal(st.instances.length, 3);
+  });
+
+  it('pauses one tree-node and respawns so min available stays 3', async () => {
+    const sup = new Supervisor({ loaded: treeOnly(14700) });
+    supervisors.push(sup);
+    await sup.startService('tree-node');
+    await sup.pauseInstance('tree-node-0');
+    const st = sup.status().services['tree-node'];
+    assert.ok(st.available >= 3, `available ${st.available}`);
+    assert.ok(st.paused >= 1);
+    assert.ok(st.running >= 4);
+    assert.equal(st.belowFloor, false);
+  });
+
+  it('restarts an unhealthy replica and keeps the floor', async () => {
+    const sup = new Supervisor({ loaded: treeOnly(14800) });
+    supervisors.push(sup);
+    await sup.startService('tree-node');
+    const before = sup.instances.get('tree-node-1').pid;
+    await sup.markUnhealthy('tree-node-1');
+    const mon = new Monitor(sup, { intervalMs: 50 });
+    await mon.tick();
+    const rec = sup.instances.get('tree-node-1');
+    assert.ok(rec.pid);
+    assert.notEqual(rec.pid, before);
+    assert.equal(rec.healthy, true);
+    assert.equal(sup.status().services['tree-node'].available, 3);
+  });
+
+  it('stop+replace of a replica keeps min tree-nodes available', async () => {
+    const sup = new Supervisor({ loaded: treeOnly(14900) });
+    supervisors.push(sup);
+    await sup.startService('tree-node');
+    await sup.stopInstance('tree-node-2', { replace: true });
+    const st = sup.status().services['tree-node'];
+    assert.equal(st.available, 3);
+    assert.ok(st.instances.some((i) => i.id === 'tree-node-2' || i.id === 'tree-node-0'));
+  });
+
+  it('spreads replicas across two defined machines', async () => {
+    const loaded = treeOnly(15100);
+    loaded.services['tree-node'].min = 4;
+    loaded.services['tree-node'].max = 6;
+    loaded.machines = [
+      { id: 'rack-a', title: 'A', kind: 'local', host: '127.0.0.1', enabled: true, capacity: 8, roles: ['*'] },
+      { id: 'rack-b', title: 'B', kind: 'local', host: '127.0.0.1', enabled: true, capacity: 8, roles: ['*'] },
+    ];
+    const sup = new Supervisor({ loaded });
+    supervisors.push(sup);
+    await sup.startService('tree-node');
+    const hosts = new Set(sup.status().services['tree-node'].instances.map((i) => i.machine));
+    assert.ok(hosts.has('rack-a') && hosts.has('rack-b'), `hosts ${[...hosts]}`);
+    const machines = sup.status().machines;
+    assert.equal(machines.length, 2);
+    assert.ok(machines.every((m) => m.running >= 1));
+  });
+
+  it('records message-processing RTT percentiles', async () => {
+    const loaded = treeOnly(15200);
+    loaded.services['tree-node'].min = 1;
+    loaded.services['tree-node'].max = 2;
+    const sup = new Supervisor({ loaded });
+    supervisors.push(sup);
+    await sup.startService('tree-node');
+    const rec = [...sup.instances.values()][0];
+    for (let i = 0; i < 8; i += 1) await sup.probe(rec);
+    const rtt = rec.rtt;
+    assert.ok(rtt.count >= 8, `count ${rtt.count}`);
+    assert.ok(rtt.minMs <= rtt.p50Ms && rtt.p50Ms <= rtt.p90Ms);
+    const svc = sup.status().services['tree-node'].rtt;
+    assert.ok(svc.count >= 8);
+    assert.ok(svc.minMs != null && svc.avgMs != null);
+  });
+});

test/rtt.test.js

@@ -0,0 +1,42 @@
+import { describe, it } from 'node:test';
+import assert from 'node:assert/strict';
+import { summarizeRtt, RttWindow } from '../src/rtt.js';
+import { pickMachine } from '../src/machines.js';
+
+describe('RTT planning stats', () => {
+  it('computes min, avg, p50, p90 in order', () => {
+    const s = summarizeRtt([10, 20, 30, 40, 50, 60, 70, 80, 90, 100]);
+    assert.equal(s.count, 10);
+    assert.equal(s.minMs, 10);
+    assert.equal(s.avgMs, 55);
+    assert.ok(s.p50Ms >= s.minMs && s.p50Ms <= s.p90Ms);
+    assert.ok(s.p90Ms <= 100);
+  });
+
+  it('empty window is nulls', () => {
+    const s = new RttWindow().stats();
+    assert.equal(s.count, 0);
+    assert.equal(s.minMs, null);
+  });
+});
+
+describe('machine placement', () => {
+  it('spreads onto the least-loaded eligible host', () => {
+    const machines = [
+      { id: 'a', enabled: true, capacity: 4, roles: ['*'] },
+      { id: 'b', enabled: true, capacity: 4, roles: ['*'] },
+    ];
+    const instances = [{ machine: 'a', pid: 1 }];
+    const pick = pickMachine(machines, instances, 'tree-node', 'tree-node');
+    assert.equal(pick.id, 'b');
+  });
+
+  it('skips machines that do not allow the role', () => {
+    const machines = [
+      { id: 'edge', enabled: true, capacity: 8, roles: ['zappier-edge'] },
+      { id: 'trees', enabled: true, capacity: 8, roles: ['tree-node'] },
+    ];
+    const pick = pickMachine(machines, [], 'tree-node', 'tree-node');
+    assert.equal(pick.id, 'trees');
+  });
+});

test/ssh.test.js

@@ -0,0 +1,56 @@
+import { describe, it } from 'node:test';
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { expandHome, shellQuote, sshBaseArgs, sshTarget } from '../src/ssh.js';
+import { normalizeMachine, pickMachine } from '../src/machines.js';
+
+describe('SSH remote machine config', () => {
+  it('expands ~ in identityFile and quotes remote commands', () => {
+    assert.equal(expandHome('~/foo'), `${os.homedir()}/foo`);
+    assert.equal(shellQuote("a'b"), `'a'"'"'b'`);
+  });
+
+  it('builds ssh argv with user, host, port, and key path (not key bytes)', () => {
+    const key = path.join(os.tmpdir(), 'verae-fleet-test-id');
+    fs.writeFileSync(key, 'not-a-real-key\n', { mode: 0o600 });
+    const m = normalizeMachine({
+      id: 'ns1',
+      kind: 'ssh',
+      host: '70.88.205.138',
+      user: 'marchon',
+      sshPort: 22,
+      identityFile: key,
+    });
+    assert.equal(m.kind, 'ssh');
+    assert.equal(sshTarget(m), 'marchon@70.88.205.138');
+    const args = sshBaseArgs(m);
+    assert.ok(args.includes('-i'));
+    assert.ok(args.includes(key));
+    assert.ok(!args.some((a) => a.includes('BEGIN')));
+    assert.equal(m.user, 'marchon');
+    fs.unlinkSync(key);
+  });
+
+  it('places load on ssh hosts when they have free capacity', () => {
+    const machines = [
+      normalizeMachine({ id: 'local', kind: 'local', host: '127.0.0.1', capacity: 2, roles: ['*'] }),
+      normalizeMachine({
+        id: 'ns1',
+        kind: 'ssh',
+        host: '70.88.205.138',
+        user: 'marchon',
+        identityFile: '~/.ssh/id_ed25519',
+        capacity: 8,
+        roles: ['tree-node'],
+      }),
+    ];
+    const instances = [
+      { machine: 'local', pid: 1 },
+      { machine: 'local', pid: 2 },
+    ];
+    const pick = pickMachine(machines, instances, 'tree-node', 'tree-node');
+    assert.equal(pick.id, 'ns1');
+  });
+});