Add verae-keep: host supervisor with watch+guard on NS1.

Restarts crashed units unless the admin console paused or stopped
them. Fleet writes per-instance intent.json. Tested on 138:
crash-restart, pause-hold, watch respawn of keep.

packages/verae-keep/src/expand.js

@@ -0,0 +1,9 @@
+import os from 'node:os';
+import path from 'node:path';
+
+export function expandHome(p) {
+  if (!p) return p;
+  if (p === '~') return os.homedir();
+  if (p.startsWith('~/')) return path.join(os.homedir(), p.slice(2));
+  return p;
+}

packages/verae-keep/src/intent.js

@@ -0,0 +1,67 @@
+import fs from 'node:fs';
+import path from 'node:path';
+
+export const STATES = ['run', 'pause', 'stop'];
+
+export function emptyStore() {
+  return { units: {} };
+}
+
+export function loadStore(file) {
+  try {
+    return JSON.parse(fs.readFileSync(file, 'utf8'));
+  } catch {
+    return emptyStore();
+  }
+}
+
+export function saveStore(file, store) {
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  const tmp = `${file}.tmp`;
+  fs.writeFileSync(tmp, JSON.stringify(store, null, 2));
+  fs.renameSync(tmp, file);
+}
+
+export function record(store, id, state, source = 'keep') {
+  if (!STATES.includes(state)) throw new Error(`invalid intent ${state}`);
+  store.units[id] = { state, source, updatedAt: new Date().toISOString() };
+  return store.units[id];
+}
+
+export function readUnitFile(file) {
+  try {
+    const j = JSON.parse(fs.readFileSync(file, 'utf8'));
+    if (!STATES.includes(j.state)) return null;
+    return {
+      state: j.state,
+      source: j.source || 'file',
+      updatedAt: j.updatedAt || '',
+    };
+  } catch {
+    return null;
+  }
+}
+
+export function writeUnitFile(file, id, state, source = 'keep') {
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  const body = { id, state, source, updatedAt: new Date().toISOString() };
+  fs.writeFileSync(file, JSON.stringify(body, null, 2));
+  return body;
+}
+
+/** Newest updatedAt wins. Missing timestamps lose. */
+export function pickIntent(...candidates) {
+  const ok = candidates.filter((c) => c && STATES.includes(c.state));
+  if (!ok.length) return null;
+  ok.sort((a, b) => String(b.updatedAt || '').localeCompare(String(a.updatedAt || '')));
+  return ok[0];
+}
+
+export function fromFleetService(sv, instanceId) {
+  if (!sv) return null;
+  if (sv.enabled === false) return { state: 'stop', source: 'fleet', updatedAt: new Date().toISOString() };
+  const inst = (sv.instances || []).find((i) => i.id === instanceId);
+  if (inst?.paused) return { state: 'pause', source: 'fleet', updatedAt: new Date().toISOString() };
+  if (sv.enabled) return { state: 'run', source: 'fleet', updatedAt: new Date().toISOString() };
+  return null;
+}

packages/verae-keep/src/keep.js

@@ -0,0 +1,237 @@
+#!/usr/bin/env node
+/**
+ * Host keep-alive. Restarts units that should be running.
+ * Honors admin pause/stop (intent files + optional fleet /api/status).
+ */
+import fs from 'node:fs';
+import http from 'node:http';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { expandHome } from './expand.js';
+import {
+  loadStore,
+  saveStore,
+  record,
+  pickIntent,
+  writeUnitFile,
+  fromFleetService,
+} from './intent.js';
+import { httpProbe, httpPost, tcpProbe } from './probe.js';
+import { spawnUnit, killPid } from './spawn.js';
+import { loadUnitsFile, scanRuntimeIntents } from './units.js';
+
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+const ROOT = path.join(HERE, '..');
+const UNITS_FILE = expandHome(process.env.KEEP_UNITS || path.join(ROOT, 'units.json'));
+const STATE_DIR = expandHome(process.env.KEEP_STATE || path.join(ROOT, 'data'));
+const STORE_FILE = path.join(STATE_DIR, 'intent.json');
+const LOG_DIR = path.join(STATE_DIR, 'logs');
+const FLEET_URL = (process.env.FLEET_URL || '').replace(/\/$/, '');
+
+const cfg = loadUnitsFile(UNITS_FILE);
+const store = loadStore(STORE_FILE);
+const last = {};
+const restarts = {};
+
+function log(event, extra = {}) {
+  const line = JSON.stringify({ t: new Date().toISOString(), event, ...extra });
+  fs.mkdirSync(STATE_DIR, { recursive: true });
+  fs.appendFileSync(path.join(STATE_DIR, 'keep.log'), `${line}\n`);
+  process.stdout.write(`${line}\n`);
+}
+
+function unitById(id) {
+  return cfg.units.find((u) => u.id === id);
+}
+
+function intentOf(unit, runtimeIntents, fleetMap) {
+  const fromStore = store.units[unit.id];
+  const fromRuntime = runtimeIntents[unit.id];
+  const fromFleet = fleetMap[unit.id];
+  const picked = pickIntent(fromStore, fromRuntime, fromFleet);
+  if (picked) return picked;
+  return { state: unit.defaultIntent || 'run', source: 'default', updatedAt: '' };
+}
+
+async function probe(unit) {
+  if (unit.healthUrl) return httpProbe(unit.healthUrl);
+  if (unit.tcp) return tcpProbe(unit.tcp.host || '127.0.0.1', Number(unit.tcp.port));
+  return { ok: false, paused: false, error: 'no probe' };
+}
+
+function healthBase(unit) {
+  if (!unit.healthUrl) return null;
+  try {
+    const u = new URL(unit.healthUrl);
+    return `${u.protocol}//${u.host}`;
+  } catch {
+    return null;
+  }
+}
+
+function bumpRestart(id) {
+  const hour = Math.floor(Date.now() / 3600000);
+  const slot = restarts[id] || { hour, n: 0 };
+  if (slot.hour !== hour) {
+    slot.hour = hour;
+    slot.n = 0;
+  }
+  slot.n += 1;
+  restarts[id] = slot;
+  return slot.n <= 40;
+}
+
+async function apply(unit, intent) {
+  const p = await probe(unit);
+  last[unit.id] = { intent, probe: { ok: p.ok, paused: p.paused, error: p.error || null, pid: p.pid || null } };
+  const base = healthBase(unit);
+  if (!unit.managed) {
+    last[unit.id].action = p.ok ? 'observe' : 'down-unmanaged';
+    return last[unit.id];
+  }
+  if (intent.state === 'stop') {
+    if (p.ok || p.paused || p.pid) {
+      killPid(p.pid || last[unit.id]?.spawnedPid);
+      last[unit.id].action = 'killed';
+      log('stop', { id: unit.id });
+    } else last[unit.id].action = 'already-stopped';
+    return last[unit.id];
+  }
+  if (intent.state === 'pause') {
+    if (p.ok && !p.paused && base) {
+      await httpPost(`${base}${unit.pausePath}`);
+      last[unit.id].action = 'paused';
+      log('pause', { id: unit.id });
+    } else if (!p.ok && !p.paused) {
+      last[unit.id].action = 'leave-down';
+    } else last[unit.id].action = 'already-paused';
+    return last[unit.id];
+  }
+  // run
+  if (p.ok) {
+    last[unit.id].action = 'healthy';
+    return last[unit.id];
+  }
+  if (p.paused && base) {
+    await httpPost(`${base}${unit.resumePath}`);
+    last[unit.id].action = 'resumed';
+    log('resume', { id: unit.id });
+    return last[unit.id];
+  }
+  if (!unit.command) {
+    last[unit.id].action = 'down-no-command';
+    return last[unit.id];
+  }
+  if (!bumpRestart(unit.id)) {
+    last[unit.id].action = 'restart-capped';
+    log('restart-capped', { id: unit.id });
+    return last[unit.id];
+  }
+  const spawned = spawnUnit(unit, { logDir: LOG_DIR });
+  last[unit.id].spawnedPid = spawned.pid;
+  last[unit.id].action = 'spawned';
+  log('spawn', { id: unit.id, pid: spawned.pid });
+  return last[unit.id];
+}
+
+async function fleetMap() {
+  const map = {};
+  if (!FLEET_URL) return map;
+  try {
+    const r = await httpProbe(`${FLEET_URL}/api/status`, { timeoutMs: 2500 });
+    const services = r.body?.services || {};
+    for (const unit of cfg.units) {
+      const sid = unit.env?.FLEET_SERVICE || unit.id.replace(/-\d+$/, '');
+      const sv = services[sid];
+      const via = fromFleetService(sv, unit.id);
+      if (via) map[unit.id] = via;
+    }
+  } catch {
+    /* fleet optional */
+  }
+  return map;
+}
+
+async function tick() {
+  const runtimeIntents = cfg.runtimeDir ? scanRuntimeIntents(cfg.runtimeDir) : {};
+  const fromFleet = await fleetMap();
+  const actions = [];
+  for (const unit of cfg.units) {
+    const intent = intentOf(unit, runtimeIntents, fromFleet);
+    const row = await apply(unit, intent);
+    actions.push({ id: unit.id, intent: intent.state, source: intent.source, action: row.action });
+  }
+  return { t: new Date().toISOString(), actions };
+}
+
+function setIntent(id, state, source = 'admin') {
+  const unit = unitById(id);
+  if (!unit) throw Object.assign(new Error(`unknown unit ${id}`), { status: 404 });
+  const row = record(store, id, state, source);
+  saveStore(STORE_FILE, store);
+  if (cfg.runtimeDir) {
+    writeUnitFile(path.join(cfg.runtimeDir, 'data', id, 'intent.json'), id, state, source);
+  }
+  log('intent', { id, state, source });
+  return row;
+}
+
+const server = http.createServer(async (req, res) => {
+  const url = new URL(req.url || '/', `http://127.0.0.1:${cfg.port}`);
+  const json = (code, obj) => {
+    res.writeHead(code, { 'content-type': 'application/json', 'cache-control': 'no-store' });
+    res.end(JSON.stringify(obj, null, 2));
+  };
+  try {
+    if (req.method === 'GET' && url.pathname === '/health') {
+      return json(200, { ok: true, role: 'verae-keep', units: cfg.units.length, pid: process.pid });
+    }
+    if (req.method === 'GET' && url.pathname === '/status') {
+      return json(200, {
+        ok: true,
+        units: cfg.units.map((u) => ({
+          id: u.id,
+          title: u.title,
+          intent: store.units[u.id] || { state: u.defaultIntent, source: 'default' },
+          last: last[u.id] || null,
+        })),
+      });
+    }
+    const m = url.pathname.match(/^\/intent\/([^/]+)$/);
+    if (req.method === 'POST' && m) {
+      const chunks = [];
+      for await (const c of req) chunks.push(c);
+      const body = JSON.parse(Buffer.concat(chunks).toString('utf8') || '{}');
+      const row = setIntent(decodeURIComponent(m[1]), body.state, body.source || 'admin');
+      return json(200, row);
+    }
+    if (req.method === 'POST' && url.pathname === '/reconcile') {
+      return json(200, await tick());
+    }
+    json(404, { error: 'not found' });
+  } catch (err) {
+    json(err.status || 400, { error: err.message });
+  }
+});
+
+const bind = process.env.KEEP_BIND || cfg.bind || '127.0.0.1';
+const port = Number(process.env.KEEP_PORT || cfg.port || 3860);
+
+if (process.argv[1] === fileURLToPath(import.meta.url)) {
+  server.listen(port, bind, () => {
+    log('listen', { url: `http://${bind}:${port}/`, units: cfg.units.map((u) => u.id) });
+  });
+  const iv = setInterval(() => {
+    tick().catch((err) => log('tick-error', { error: err.message }));
+  }, cfg.intervalMs);
+  tick().catch((err) => log('tick-error', { error: err.message }));
+  const stop = () => {
+    clearInterval(iv);
+    server.close();
+    process.exit(0);
+  };
+  process.on('SIGTERM', stop);
+  process.on('SIGINT', stop);
+}
+
+export { tick, setIntent, cfg, store };

packages/verae-keep/src/probe.js

@@ -0,0 +1,83 @@
+import http from 'node:http';
+import net from 'node:net';
+
+export function httpProbe(url, { timeoutMs = 1500 } = {}) {
+  return new Promise((resolve) => {
+    const u = new URL(url);
+    const req = http.request(
+      {
+        hostname: u.hostname,
+        port: u.port,
+        path: `${u.pathname}${u.search}`,
+        method: 'GET',
+        timeout: timeoutMs,
+      },
+      (res) => {
+        const chunks = [];
+        res.on('data', (c) => chunks.push(c));
+        res.on('end', () => {
+          const raw = Buffer.concat(chunks).toString('utf8');
+          let body = {};
+          try {
+            body = JSON.parse(raw);
+          } catch {
+            body = { raw };
+          }
+          resolve({
+            ok: res.statusCode >= 200 && res.statusCode < 300 && body.ok !== false,
+            paused: Boolean(body.paused),
+            statusCode: res.statusCode,
+            pid: body.pid || null,
+            body,
+          });
+        });
+      },
+    );
+    req.on('error', (err) => resolve({ ok: false, paused: false, error: err.message }));
+    req.on('timeout', () => {
+      req.destroy();
+      resolve({ ok: false, paused: false, error: 'timeout' });
+    });
+    req.end();
+  });
+}
+
+export function httpPost(url, { timeoutMs = 1500 } = {}) {
+  return new Promise((resolve) => {
+    const u = new URL(url);
+    const req = http.request(
+      {
+        hostname: u.hostname,
+        port: u.port,
+        path: `${u.pathname}${u.search}`,
+        method: 'POST',
+        timeout: timeoutMs,
+      },
+      (res) => {
+        const chunks = [];
+        res.on('data', (c) => chunks.push(c));
+        res.on('end', () => resolve({ ok: res.statusCode < 400, statusCode: res.statusCode }));
+      },
+    );
+    req.on('error', (err) => resolve({ ok: false, error: err.message }));
+    req.on('timeout', () => {
+      req.destroy();
+      resolve({ ok: false, error: 'timeout' });
+    });
+    req.end();
+  });
+}
+
+export function tcpProbe(host, port, { timeoutMs = 800 } = {}) {
+  return new Promise((resolve) => {
+    const sock = net.connect({ host, port, timeout: timeoutMs }, () => {
+      sock.end();
+      resolve({ ok: true, paused: false });
+    });
+    sock.on('error', (err) => resolve({ ok: false, paused: false, error: err.message }));
+    sock.on('timeout', () => {
+      sock.destroy();
+      resolve({ ok: false, paused: false, error: 'timeout' });
+    });
+  });
+}

packages/verae-keep/src/spawn.js

@@ -0,0 +1,48 @@
+import { spawn } from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
+import { expandHome } from './expand.js';
+
+export function spawnUnit(unit, { logDir } = {}) {
+  const cwd = expandHome(unit.cwd || process.cwd());
+  const cmd = unit.command?.[0];
+  if (!cmd) throw new Error(`unit ${unit.id} has no command`);
+  const args = unit.command.slice(1);
+  fs.mkdirSync(cwd, { recursive: true });
+  let stdout = 'ignore';
+  let stderr = 'ignore';
+  if (logDir) {
+    fs.mkdirSync(logDir, { recursive: true });
+    const log = fs.openSync(path.join(logDir, `${unit.id}.log`), 'a');
+    stdout = log;
+    stderr = log;
+  }
+  const child = spawn(cmd, args, {
+    cwd,
+    env: { ...process.env, ...(unit.env || {}) },
+    detached: true,
+    stdio: ['ignore', stdout, stderr],
+  });
+  child.unref();
+  return { pid: child.pid, child };
+}
+
+export function killPid(pid, signal = 'SIGTERM') {
+  if (!pid) return false;
+  try {
+    process.kill(Number(pid), signal);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+export function pidAlive(pid) {
+  if (!pid) return false;
+  try {
+    process.kill(Number(pid), 0);
+    return true;
+  } catch {
+    return false;
+  }
+}

packages/verae-keep/src/units.js

@@ -0,0 +1,52 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { expandHome } from './expand.js';
+import { readUnitFile } from './intent.js';
+
+export function loadUnitsFile(file) {
+  const raw = JSON.parse(fs.readFileSync(file, 'utf8'));
+  const runtimeDir = expandHome(raw.runtimeDir || '');
+  return {
+    bind: raw.bind || '127.0.0.1',
+    port: Number(raw.port || 3860),
+    intervalMs: Number(raw.intervalMs || 2000),
+    runtimeDir,
+    units: (raw.units || []).map((u) => normalizeUnit(u, runtimeDir)),
+  };
+}
+
+function normalizeUnit(u, runtimeDir) {
+  const healthUrl = typeof u.health === 'string' ? u.health : u.health?.url;
+  const tcp = u.health?.tcp || (u.tcp ? { host: u.tcp.host, port: u.tcp.port } : null);
+  return {
+    id: u.id,
+    title: u.title || u.id,
+    healthUrl: healthUrl || null,
+    tcp: tcp || null,
+    cwd: expandHome(u.cwd || runtimeDir || process.cwd()),
+    command: u.command || null,
+    env: u.env || {},
+    defaultIntent: u.defaultIntent || 'run',
+    managed: u.managed !== false,
+    pausePath: u.pausePath || '/pause',
+    resumePath: u.resumePath || '/resume',
+  };
+}
+
+export function scanRuntimeIntents(runtimeDir) {
+  const root = expandHome(runtimeDir);
+  const data = path.join(root, 'data');
+  const out = {};
+  let names = [];
+  try {
+    names = fs.readdirSync(data);
+  } catch {
+    return out;
+  }
+  for (const name of names) {
+    const file = path.join(data, name, 'intent.json');
+    const intent = readUnitFile(file);
+    if (intent) out[name] = intent;
+  }
+  return out;
+}

packages/verae-keep/src/watch.js

@@ -0,0 +1,115 @@
+#!/usr/bin/env node
+/**
+ * Backup for verae-keep. If keep's /health fails, spawn keep again.
+ */
+import { spawn } from 'node:child_process';
+import fs from 'node:fs';
+import http from 'node:http';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { httpProbe } from './probe.js';
+import { killPid, pidAlive } from './spawn.js';
+import { expandHome } from './expand.js';
+
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+const ROOT = path.join(HERE, '..');
+const KEEP_URL = (process.env.KEEP_URL || 'http://127.0.0.1:3860').replace(/\/$/, '');
+const STATE_DIR = expandHome(process.env.KEEP_STATE || path.join(ROOT, 'data'));
+const PID_FILE = path.join(STATE_DIR, 'keep.pid');
+const INTERVAL = Number(process.env.WATCH_INTERVAL_MS || 2000);
+const BIND = process.env.WATCH_BIND || '127.0.0.1';
+const PORT = Number(process.env.WATCH_PORT || 3861);
+
+let child = null;
+let restarts = 0;
+
+function log(event, extra = {}) {
+  const line = JSON.stringify({ t: new Date().toISOString(), event, ...extra });
+  fs.mkdirSync(STATE_DIR, { recursive: true });
+  fs.appendFileSync(path.join(STATE_DIR, 'watch.log'), `${line}\n`);
+  process.stdout.write(`${line}\n`);
+}
+
+function readPid() {
+  try {
+    return Number(fs.readFileSync(PID_FILE, 'utf8').trim());
+  } catch {
+    return null;
+  }
+}
+
+function writePid(pid) {
+  fs.mkdirSync(STATE_DIR, { recursive: true });
+  fs.writeFileSync(PID_FILE, String(pid));
+}
+
+function startKeep() {
+  const existing = readPid();
+  if (existing && pidAlive(existing)) {
+    log('adopt', { pid: existing });
+    return existing;
+  }
+  const env = { ...process.env };
+  const proc = spawn(process.execPath, [path.join(HERE, 'keep.js')], {
+    cwd: ROOT,
+    env,
+    detached: true,
+    stdio: ['ignore', 'pipe', 'pipe'],
+  });
+  proc.unref();
+  child = proc;
+  writePid(proc.pid);
+  restarts += 1;
+  log('start-keep', { pid: proc.pid, restarts });
+  proc.stdout?.on('data', (c) => process.stdout.write(c));
+  proc.stderr?.on('data', (c) => process.stderr.write(c));
+  proc.on('exit', (code, signal) => {
+    log('keep-exit', { code, signal, pid: proc.pid });
+    if (readPid() === proc.pid) {
+      try {
+        fs.unlinkSync(PID_FILE);
+      } catch {
+        /* ignore */
+      }
+    }
+  });
+  return proc.pid;
+}
+
+async function tick() {
+  const h = await httpProbe(`${KEEP_URL}/health`, { timeoutMs: 1500 });
+  if (h.ok) return { ok: true, keep: h.body };
+  log('keep-down', { error: h.error || h.statusCode });
+  const old = readPid();
+  if (old && pidAlive(old)) killPid(old, 'SIGTERM');
+  startKeep();
+  return { ok: false, restarted: true };
+}
+
+const server = http.createServer(async (req, res) => {
+  const url = new URL(req.url || '/', `http://127.0.0.1:${PORT}`);
+  const json = (code, obj) => {
+    res.writeHead(code, { 'content-type': 'application/json' });
+    res.end(JSON.stringify(obj));
+  };
+  if (url.pathname === '/health') {
+    return json(200, { ok: true, role: 'verae-keep-watch', pid: process.pid, keepPid: readPid(), restarts });
+  }
+  if (url.pathname === '/tick' && req.method === 'POST') {
+    return json(200, await tick());
+  }
+  json(404, { error: 'not found' });
+});
+
+server.listen(PORT, BIND, () => {
+  log('listen', { url: `http://${BIND}:${PORT}/`, keep: KEEP_URL });
+  startKeep();
+  setInterval(() => {
+    tick().catch((err) => log('tick-error', { error: err.message }));
+  }, INTERVAL);
+});
+
+process.on('SIGTERM', () => {
+  if (child?.pid) killPid(child.pid, 'SIGTERM');
+  process.exit(0);
+});