master-zapier-plan-draft/packages/verae-fleet/test/fleet.test.js

import { describe, it, after } from 'node:test';
import assert from 'node:assert/strict';
import { loadFleet, listServices, FLEET_ROOT } from '../src/load.js';
import { Supervisor, spawnLocal } from '../src/supervisor.js';
import { Monitor } from '../src/monitor.js';

function treeOnly(healthBase) {
  const loaded = loadFleet(FLEET_ROOT, { overlay: false });
  for (const s of Object.values(loaded.services)) {
    s.enabled = s.id === 'tree-node';
    if (s.id === 'tree-node') {
      s.min = 3;
      s.max = 6;
      s.keepFloor = true;
      s.ports = { healthBase };
    }
  }
  loaded.machinesOverlay = null;
  loaded.machines = loaded.machines.filter((m) => m.kind === 'local' && m.enabled);
  if (!loaded.machines.length) {
    loaded.machines = [
      { id: 'local', title: 'local', kind: 'local', host: '127.0.0.1', enabled: true, capacity: 32, roles: ['*'] },
    ];
  }
  return loaded;
}

describe('verae-fleet', () => {
  /** @type {Supervisor[]} */
  const supervisors = [];
  after(async () => {
    for (const s of supervisors) await s.stopAll();
  });

  it('spawnLocal runs spawn.command instead of the stub worker', async () => {
    const child = spawnLocal(
      FLEET_ROOT,
      { spawn: { command: process.execPath, args: ['-e', 'process.stdout.write("ok"); process.exit(0)'] } },
      { ...process.env },
      9,
    );
    const out = [];
    child.stdout.on('data', (c) => out.push(c));
    const code = await new Promise((resolve) => child.on('exit', resolve));
    assert.equal(code, 0);
    assert.equal(Buffer.concat(out).toString(), 'ok');
  });

  it('lists every service with a config file and central replica spec', () => {
    const loaded = loadFleet();
    const rows = listServices(loaded);
    const ids = rows.map((r) => r.id);
    for (const need of [
      'nats',
      'zappier-edge',
      'access-authz',
      'access-web',
      'access-api',
      'access-leaf',
      'access-zapier',
      'access-staff',
      'identity',
      'jobs-events',
      'account-balance',
      'customer-service',
      'sales-pricing',
      'accounting-export',
      'middleware-http',
      'job-poller',
      'webhook-deliver',
      'archive-aggregator',
      'archive-worm',
      'tree-node',
      'zapier-simulator',
      'zapier-platform-app',
    ]) {
      assert.ok(ids.includes(need), `missing ${need}`);
    }
    const tree = loaded.services['tree-node'];
    assert.equal(tree.min, 3);
    assert.equal(tree.keepFloor, true);
    assert.equal(tree.configPath, 'services/tree-node.json');
    assert.equal(loaded.fleetPath, 'fleet.json');
    const lan = loaded.machines.find((m) => m.id === 'lan-134');
    if (lan) assert.equal(lan.enabled, false);
  });

  it('starts the tree-node floor (3 available copies)', async () => {
    const sup = new Supervisor({ loaded: treeOnly(14600) });
    supervisors.push(sup);
    await sup.startService('tree-node');
    const st = sup.status().services['tree-node'];
    assert.equal(st.available, 3);
    assert.equal(st.belowFloor, false);
    assert.equal(st.instances.length, 3);
  });

  it('pauses one tree-node and respawns so min available stays 3', async () => {
    const sup = new Supervisor({ loaded: treeOnly(14700) });
    supervisors.push(sup);
    await sup.startService('tree-node');
    await sup.pauseInstance('tree-node-0');
    const st = sup.status().services['tree-node'];
    assert.ok(st.available >= 3, `available ${st.available}`);
    assert.ok(st.paused >= 1);
    assert.ok(st.running >= 4);
    assert.equal(st.belowFloor, false);
  });

  it('restarts an unhealthy replica and keeps the floor', async () => {
    const sup = new Supervisor({ loaded: treeOnly(14800) });
    supervisors.push(sup);
    await sup.startService('tree-node');
    const before = sup.instances.get('tree-node-1').pid;
    await sup.markUnhealthy('tree-node-1');
    const mon = new Monitor(sup, { intervalMs: 50 });
    await mon.tick();
    const rec = sup.instances.get('tree-node-1');
    assert.ok(rec.pid);
    assert.notEqual(rec.pid, before);
    assert.equal(rec.healthy, true);
    assert.equal(sup.status().services['tree-node'].available, 3);
  });

  it('stop+replace of a replica keeps min tree-nodes available', async () => {
    const sup = new Supervisor({ loaded: treeOnly(14900) });
    supervisors.push(sup);
    await sup.startService('tree-node');
    await sup.stopInstance('tree-node-2', { replace: true });
    const st = sup.status().services['tree-node'];
    assert.equal(st.available, 3);
    assert.ok(st.instances.some((i) => i.id === 'tree-node-2' || i.id === 'tree-node-0'));
  });

  it('spreads replicas across two defined machines', async () => {
    const loaded = treeOnly(15100);
    loaded.services['tree-node'].min = 4;
    loaded.services['tree-node'].max = 6;
    loaded.machines = [
      { id: 'rack-a', title: 'A', kind: 'local', host: '127.0.0.1', enabled: true, capacity: 8, roles: ['*'] },
      { id: 'rack-b', title: 'B', kind: 'local', host: '127.0.0.1', enabled: true, capacity: 8, roles: ['*'] },
    ];
    const sup = new Supervisor({ loaded });
    supervisors.push(sup);
    await sup.startService('tree-node');
    const hosts = new Set(sup.status().services['tree-node'].instances.map((i) => i.machine));
    assert.ok(hosts.has('rack-a') && hosts.has('rack-b'), `hosts ${[...hosts]}`);
    const machines = sup.status().machines;
    assert.equal(machines.length, 2);
    assert.ok(machines.every((m) => m.running >= 1));
  });

  it('records message-processing RTT percentiles', async () => {
    const loaded = treeOnly(15200);
    loaded.services['tree-node'].min = 1;
    loaded.services['tree-node'].max = 2;
    const sup = new Supervisor({ loaded });
    supervisors.push(sup);
    await sup.startService('tree-node');
    const rec = [...sup.instances.values()][0];
    for (let i = 0; i < 8; i += 1) await sup.probe(rec);
    const rtt = rec.rtt;
    assert.ok(rtt.count >= 8, `count ${rtt.count}`);
    assert.ok(rtt.minMs <= rtt.p50Ms && rtt.p50Ms <= rtt.p90Ms);
    const svc = sup.status().services['tree-node'].rtt;
    assert.ok(svc.count >= 8);
    assert.ok(svc.minMs != null && svc.avgMs != null);
  });
});