/** * Spawn, pause, resume, stop, restart replicas. Keep replica floor (tree-node min). * @module supervisor */ import { spawn } from 'node:child_process'; import fs from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { loadFleet } from './load.js'; import { httpProbe, postControl } from './health.js'; import { classifyService } from './classify.js'; import { pickMachine, upsertMachine, saveOverlay } from './machines.js'; import { summarizeRtt } from './rtt.js'; import { sshSpawnWorker, sshKillWorker, sshHttp, sshCheck } from './ssh.js'; const WORKER = path.join(path.dirname(fileURLToPath(import.meta.url)), 'worker.js'); /** * Real HTTP apps (zappier-edge, middleware) set `spawn` in services/*.json. * Everything else is the fleet health worker. */ export function spawnLocal(root, spec, env, healthPort) { const merged = { ...env, ...(spec.env || {}), PORT: String(healthPort) }; if (spec.spawn?.command) { const cwd = path.resolve(root, spec.spawn.cwd || '.'); return spawn(spec.spawn.command, spec.spawn.args || [], { env: merged, cwd, stdio: ['ignore', 'pipe', 'pipe'], }); } return spawn(process.execPath, [WORKER], { env: merged, stdio: ['ignore', 'pipe', 'pipe'] }); } export class Supervisor { /** * @param {object} [opts] * @param {ReturnType} [opts.loaded] * @param {string} [opts.stateRoot] */ constructor(opts = {}) { this.loaded = opts.loaded || loadFleet(); this.stateRoot = opts.stateRoot || path.join(this.loaded.root, 'data', 'runtime'); fs.mkdirSync(this.stateRoot, { recursive: true }); /** @type {Map} */ this.instances = new Map(); this.events = []; this._seq = 0; if (!Array.isArray(this.loaded.machines) || !this.loaded.machines.length) { this.loaded.machines = [ { id: 'local', title: 'local', kind: 'local', host: '127.0.0.1', enabled: true, capacity: 32, roles: ['*'] }, ]; } } machines() { return this.loaded.machines; } addMachine(spec) { const m = upsertMachine(this.loaded.machines, spec); if (this.loaded.machinesOverlay) saveOverlay(this.loaded.machinesOverlay, this.loaded.machines); this.log('machine-add', { id: m.id, host: m.host, kind: m.kind }); return m; } setMachineEnabled(id, enabled) { const m = this.loaded.machines.find((x) => x.id === id); if (!m) throw new Error(`unknown machine ${id}`); m.enabled = Boolean(enabled); if (this.loaded.machinesOverlay) saveOverlay(this.loaded.machinesOverlay, this.loaded.machines); return m; } async checkMachine(id) { const m = this.machines().find((x) => x.id === id); if (!m) throw new Error(`unknown machine ${id}`); if (m.kind === 'ssh') return sshCheck(m); if (m.kind === 'agent') { const r = await httpProbe({ host: m.host, port: m.agentPort, path: '/health', timeoutMs: 2000 }); return { ok: r.ok, kind: 'agent', body: r.body }; } return { ok: true, kind: 'local', host: m.host }; } machineOf(rec) { return this.machines().find((x) => x.id === rec.machine); } async controlInstance(rec, pathname, body) { if (rec.kind === 'ssh') { const m = this.machineOf(rec); if (!m) throw new Error(`machine ${rec.machine} missing`); return sshHttp(m, 'POST', pathname, { port: rec.healthPort, body }); } return postControl({ host: rec.host || '127.0.0.1', port: rec.healthPort, path: pathname, body, }); } log(type, detail) { const ev = { t: new Date().toISOString(), type, ...detail }; this.events.push(ev); if (this.events.length > 500) this.events.splice(0, this.events.length - 400); return ev; } spec(serviceId) { const s = this.loaded.services[serviceId]; if (!s) throw new Error(`unknown service ${serviceId}`); return s; } nextIndex(serviceId) { const used = new Set( [...this.instances.values()].filter((i) => i.service === serviceId).map((i) => i.index), ); const max = this.spec(serviceId).max ?? 8; for (let i = 0; i < max; i += 1) if (!used.has(i)) return i; return null; } /** * @param {string} serviceId * @param {{ index?: number }} [opts] */ async startOne(serviceId, opts = {}) { const spec = this.spec(serviceId); if (!spec.managed) { this.log('skip', { service: serviceId, reason: 'unmanaged' }); return null; } if (!spec.enabled) { this.log('skip', { service: serviceId, reason: 'disabled' }); return null; } const index = opts.index ?? this.nextIndex(serviceId); if (index == null) { this.log('skip', { service: serviceId, reason: 'at-max' }); return null; } const instance = `${serviceId}-${index}`; if (this.instances.has(instance)) return this.instances.get(instance); const machine = opts.machine || pickMachine( this.machines(), [...this.instances.values()], serviceId, spec.role || serviceId, opts.excludeMachines || [], ); if (!machine) { this.log('skip', { service: serviceId, reason: 'no-machine-capacity' }); return null; } const healthPort = (spec.ports?.healthBase || 19000) + index; const stateDir = path.join(this.stateRoot, instance); fs.mkdirSync(stateDir, { recursive: true }); const env = { ...process.env, FLEET_ROLE: spec.role || serviceId, FLEET_SERVICE: serviceId, FLEET_INSTANCE: instance, FLEET_MACHINE: machine.id, FLEET_HEALTH_PORT: String(healthPort), FLEET_HEALTH_BIND: machine.kind === 'local' ? '127.0.0.1' : '127.0.0.1', FLEET_STATE_DIR: machine.kind === 'ssh' ? `${machine.remoteDir || '~/verae-fleet-runtime'}/data/${instance}` : stateDir, FLEET_MESSAGE_HZ: process.env.FLEET_MESSAGE_HZ || '8', }; let child = null; let pid = null; if (machine.kind === 'ssh') { try { const spawned = await sshSpawnWorker(machine, { instance, env }); pid = spawned.pid; } catch (err) { this.log('skip', { service: serviceId, reason: 'ssh-spawn-failed', machine: machine.id, error: err.message }); const exclude = [...(opts.excludeMachines || []), machine.id]; if (exclude.length < this.machines().length) { return this.startOne(serviceId, { ...opts, excludeMachines: exclude }); } return null; } } else if (machine.kind === 'agent') { try { const spawned = await postControl({ host: machine.host, port: machine.agentPort, path: '/spawn', body: { env, instance }, timeoutMs: 4000, }); pid = spawned.pid || null; } catch (err) { this.log('skip', { service: serviceId, reason: 'agent-spawn-failed', machine: machine.id, error: err.message }); return null; } } else { child = spawnLocal(this.loaded.root, spec, env, healthPort); pid = child.pid; } const rec = { id: instance, service: serviceId, index, machine: machine.id, host: machine.host, pid, healthPort, stateDir, child, kind: machine.kind, _machine: machine, paused: false, startedAt: Date.now(), restarts: 0, lastError: null, rttSamples: [], }; if (child) { child.stderr?.on('data', () => {}); child.on('exit', (code, signal) => { rec.exitCode = code; rec.signal = signal; rec.pid = null; this.log('exit', { instance, service: serviceId, code, signal }); }); } this.instances.set(instance, rec); this.log('start', { instance, service: serviceId, machine: machine.id, pid: rec.pid, healthPort }); const h = await waitForHealth(rec, 8000); rec.healthy = h.ok; rec.lastProbe = h; return rec; } async stopInstance(instanceId, { replace = true } = {}) { const rec = this.instances.get(instanceId); if (!rec) return null; rec.stopping = true; if (rec.kind === 'ssh') { const m = this.machines().find((x) => x.id === rec.machine); if (m) { try { await sshKillWorker(m, rec.pid, instanceId); } catch { /* ignore */ } } } else if (rec.kind === 'agent') { const m = this.machines().find((x) => x.id === rec.machine); if (m) { try { await postControl({ host: m.host, port: m.agentPort, path: '/kill', body: { instance: instanceId } }); } catch { /* ignore */ } } } else if (rec.child && rec.pid) { rec.child.kill('SIGTERM'); await waitExit(rec.child, 2000); } this.instances.delete(instanceId); this.log('stop', { instance: instanceId, service: rec.service }); if (replace) await this.reconcile(rec.service); return rec; } async pauseInstance(instanceId, { replace = true } = {}) { const rec = this.instances.get(instanceId); if (!rec) throw new Error(`no instance ${instanceId}`); await this.controlInstance(rec, '/pause'); rec.paused = true; this.log('pause', { instance: instanceId, service: rec.service }); if (replace) await this.reconcile(rec.service); return rec; } async resumeInstance(instanceId) { const rec = this.instances.get(instanceId); if (!rec) throw new Error(`no instance ${instanceId}`); await this.controlInstance(rec, '/resume'); rec.paused = false; this.log('resume', { instance: instanceId, service: rec.service }); return rec; } async restartInstance(instanceId) { const rec = this.instances.get(instanceId); if (!rec) throw new Error(`no instance ${instanceId}`); const { service, index, machine } = rec; await this.stopInstance(instanceId, { replace: false }); const next = await this.startOne(service, { index, machine: this.machines().find((m) => m.id === machine) }); if (next) next.restarts = (rec.restarts || 0) + 1; this.log('restart', { instance: instanceId, service }); return next; } async markUnhealthy(instanceId) { const rec = this.instances.get(instanceId); if (!rec) throw new Error(`no instance ${instanceId}`); await this.controlInstance(rec, '/unhealthy'); return rec; } async probe(rec) { const spec = this.spec(rec.service); const pathName = spec.health?.path || '/health'; let r; if (rec.kind === 'ssh') { const m = this.machineOf(rec); try { const body = await sshHttp(m, 'GET', pathName, { port: rec.healthPort }); r = { ok: body.ok !== false, paused: Boolean(body.paused), body, statusCode: body.ok === false ? 503 : 200 }; } catch (err) { r = { ok: false, paused: false, error: err.message }; } } else { r = await httpProbe({ host: rec.host || '127.0.0.1', port: rec.healthPort, path: pathName, timeoutMs: spec.health?.timeoutMs || 800, }); } rec.paused = Boolean(r.paused); rec.lastProbe = r; rec.healthy = r.ok; if (r.ok && !rec.paused) { try { const msg = await this.controlInstance(rec, '/message'); if (Number.isFinite(msg.rttMs)) { rec.rttSamples = rec.rttSamples || []; rec.rttSamples.push(msg.rttMs); if (rec.rttSamples.length > 200) rec.rttSamples.splice(0, rec.rttSamples.length - 200); } rec.rtt = msg.rtt || r.body?.rtt || summarizeRtt(rec.rttSamples); } catch { rec.rtt = r.body?.rtt || summarizeRtt(rec.rttSamples || []); } } return r; } available(serviceId) { return [...this.instances.values()].filter( (i) => i.service === serviceId && i.pid && !i.paused && i.healthy !== false, ); } /** * Ensure running healthy unpaused copies >= min when keepFloor. * @param {string} [only] */ async reconcile(only) { const actions = []; const ids = only ? [only] : Object.keys(this.loaded.services); for (const serviceId of ids) { const spec = this.loaded.services[serviceId]; if (!spec?.managed || !spec.enabled) continue; for (const rec of [...this.instances.values()].filter((i) => i.service === serviceId)) { if (!rec.pid && !rec.stopping) { this.instances.delete(rec.id); continue; } if (rec.pid) await this.probe(rec); if (rec.pid && rec.healthy === false && !rec.paused) { this.log('unhealthy', { instance: rec.id, service: serviceId }); await this.restartInstance(rec.id); actions.push({ op: 'restart', instance: rec.id }); } } const avail = this.available(serviceId).length; const want = spec.keepFloor ? spec.min : Math.min(spec.min, spec.max); if (avail < want) { const need = want - avail; for (let n = 0; n < need; n += 1) { const started = await this.startOne(serviceId); if (started) actions.push({ op: 'start', instance: started.id }); } } } return actions; } async startService(serviceId) { const spec = this.spec(serviceId); spec.enabled = true; const have = [...this.instances.values()].filter((i) => i.service === serviceId && i.pid).length; const want = spec.min || 1; const started = []; for (let i = have; i < want; i += 1) { const rec = await this.startOne(serviceId); if (rec) started.push(rec.id); } return started; } async stopService(serviceId, { holdFloor = false } = {}) { const spec = this.spec(serviceId); if (!holdFloor) spec.enabled = false; const ids = [...this.instances.values()].filter((i) => i.service === serviceId).map((i) => i.id); for (const id of ids) await this.stopInstance(id, { replace: false }); if (holdFloor) await this.reconcile(serviceId); return ids; } async pauseService(serviceId) { const ids = [...this.instances.values()].filter((i) => i.service === serviceId && i.pid).map((i) => i.id); for (const id of ids) await this.pauseInstance(id, { replace: false }); await this.reconcile(serviceId); return ids; } async resumeService(serviceId) { const ids = [...this.instances.values()].filter((i) => i.service === serviceId).map((i) => i.id); for (const id of ids) await this.resumeInstance(id); return ids; } async stopAll() { for (const id of [...this.instances.keys()]) { await this.stopInstance(id, { replace: false }); } } status() { const services = {}; for (const spec of Object.values(this.loaded.services)) { const inst = [...this.instances.values()].filter((i) => i.service === spec.id); const available = inst.filter((i) => i.pid && !i.paused && i.healthy !== false).length; services[spec.id] = { id: spec.id, title: spec.title, configPath: spec.configPath, managed: spec.managed, enabled: spec.enabled, min: spec.min, max: spec.max, keepFloor: spec.keepFloor, running: inst.filter((i) => i.pid).length, paused: inst.filter((i) => i.paused).length, available, belowFloor: spec.keepFloor && spec.enabled && spec.managed && available < spec.min, instances: inst.map((i) => ({ id: i.id, pid: i.pid, machine: i.machine || 'local', host: i.host || '127.0.0.1', healthPort: i.healthPort, paused: i.paused, healthy: i.healthy, restarts: i.restarts, rtt: i.rtt || summarizeRtt(i.rttSamples || []), })), }; const allSamples = inst.flatMap((i) => i.rttSamples || []); services[spec.id].rtt = summarizeRtt(allSamples); services[spec.id].health = classifyService(services[spec.id]); } const machines = this.machines().map((m) => { const inst = [...this.instances.values()].filter((i) => i.machine === m.id); const samples = inst.flatMap((i) => i.rttSamples || []); return { ...m, running: inst.filter((i) => i.pid).length, available: inst.filter((i) => i.pid && !i.paused && i.healthy !== false).length, rtt: summarizeRtt(samples), }; }); return { control: this.loaded.control, nats: this.loaded.nats, machines, services, events: this.events.slice(-50), }; } } function waitExit(child, ms) { return new Promise((resolve) => { if (!child || child.exitCode != null) return resolve(); const t = setTimeout(() => { try { child.kill('SIGKILL'); } catch { /* ignore */ } resolve(); }, ms); child.once('exit', () => { clearTimeout(t); resolve(); }); }); } async function waitForHealth(rec, ms) { const deadline = Date.now() + ms; while (Date.now() < deadline) { let r; if (rec.kind === 'ssh' && rec._machine) { try { const body = await sshHttp(rec._machine, 'GET', '/health', { port: rec.healthPort }); r = { ok: body.ok !== false, statusCode: 200, paused: Boolean(body.paused), body }; } catch { r = { ok: false }; } } else { r = await httpProbe({ host: rec.host || '127.0.0.1', port: rec.healthPort, path: '/health', timeoutMs: 400, }); } if (r.ok || r.statusCode === 503) return r; await new Promise((res) => setTimeout(res, 80)); } return { ok: false, error: 'start-timeout' }; }