Add verae-keep: host supervisor with watch+guard on NS1.

Restarts crashed units unless the admin console paused or stopped
them. Fleet writes per-instance intent.json. Tested on 138:
crash-restart, pause-hold, watch respawn of keep.

packages/verae-keep/test/intent.test.js

@@ -0,0 +1,33 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { emptyStore, record, pickIntent, writeUnitFile, readUnitFile, fromFleetService } from '../src/intent.js';
+
+test('record and pick newest intent', () => {
+  const s = emptyStore();
+  record(s, 'a', 'run', 'keep');
+  const older = { state: 'stop', source: 'file', updatedAt: '2020-01-01T00:00:00Z' };
+  const picked = pickIntent(s.units.a, older);
+  assert.equal(picked.state, 'run');
+  assert.equal(picked.source, 'keep');
+});
+
+test('unit file round-trip', () => {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'keep-intent-'));
+  const file = path.join(dir, 'intent.json');
+  writeUnitFile(file, 'x', 'pause', 'admin');
+  const got = readUnitFile(file);
+  assert.equal(got.state, 'pause');
+  assert.equal(got.source, 'admin');
+});
+
+test('fleet enabled false is stop; paused instance is pause', () => {
+  assert.equal(fromFleetService({ enabled: false, instances: [] }, 't-0').state, 'stop');
+  assert.equal(
+    fromFleetService({ enabled: true, instances: [{ id: 't-0', paused: true }] }, 't-0').state,
+    'pause',
+  );
+  assert.equal(fromFleetService({ enabled: true, instances: [{ id: 't-0', paused: false }] }, 't-0').state, 'run');
+});

packages/verae-keep/test/keep.test.js

@@ -0,0 +1,111 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import http from 'node:http';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { spawn } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+
+const root = path.join(path.dirname(fileURLToPath(import.meta.url)), '..');
+
+function listen(handler) {
+  return new Promise((resolve) => {
+    const s = http.createServer(handler);
+    s.listen(0, '127.0.0.1', () => resolve(s));
+  });
+}
+
+test('keep restarts a crashed unit and leaves a paused unit down', async () => {
+  const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'keep-lab-'));
+  const dummy = path.join(tmp, 'dummy.js');
+  fs.writeFileSync(
+    dummy,
+    `import http from 'node:http';
+const port = Number(process.env.PORT);
+let paused = false;
+http.createServer((req, res) => {
+  const u = new URL(req.url, 'http://127.0.0.1');
+  const body = JSON.stringify({ ok: !paused, paused, pid: process.pid });
+  if (u.pathname === '/pause' && req.method === 'POST') paused = true;
+  if (u.pathname === '/resume' && req.method === 'POST') paused = false;
+  res.writeHead(paused ? 503 : 200, { 'content-type': 'application/json' });
+  res.end(body);
+}).listen(port, '127.0.0.1');
+`,
+  );
+  const port = await new Promise((resolve) => {
+    const s = http.createServer();
+    s.listen(0, '127.0.0.1', () => {
+      const p = s.address().port;
+      s.close(() => resolve(p));
+    });
+  });
+  const keepPort = port + 1;
+  const units = {
+    bind: '127.0.0.1',
+    port: keepPort,
+    intervalMs: 400,
+    runtimeDir: tmp,
+    units: [
+      {
+        id: 'dummy-0',
+        health: `http://127.0.0.1:${port}/health`,
+        cwd: tmp,
+        command: [process.execPath, dummy],
+        env: { PORT: String(port) },
+      },
+    ],
+  };
+  const unitsFile = path.join(tmp, 'units.json');
+  fs.writeFileSync(unitsFile, JSON.stringify(units));
+  const child = spawn(process.execPath, ['src/keep.js'], {
+    cwd: root,
+    env: { ...process.env, KEEP_UNITS: unitsFile, KEEP_STATE: path.join(tmp, 'state'), KEEP_PORT: String(keepPort) },
+    stdio: ['ignore', 'pipe', 'pipe'],
+  });
+  const waitHealth = async (url, wantOk) => {
+    for (let i = 0; i < 25; i += 1) {
+      try {
+        const r = await fetch(url);
+        const b = await r.json();
+        if (Boolean(b.ok) === wantOk) return b;
+      } catch {
+        if (!wantOk) return { ok: false };
+      }
+      await new Promise((r) => setTimeout(r, 200));
+    }
+    throw new Error(`timeout ${url} wantOk=${wantOk}`);
+  };
+  try {
+    await waitHealth(`http://127.0.0.1:${keepPort}/health`, true);
+    const up = await waitHealth(`http://127.0.0.1:${port}/health`, true);
+    assert.equal(up.ok, true);
+    const pid1 = up.pid;
+    process.kill(pid1, 'SIGKILL');
+    const up2 = await waitHealth(`http://127.0.0.1:${port}/health`, true);
+    assert.notEqual(up2.pid, pid1);
+
+    await fetch(`http://127.0.0.1:${keepPort}/intent/dummy-0`, {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body: JSON.stringify({ state: 'stop' }),
+    });
+    await new Promise((r) => setTimeout(r, 800));
+    try {
+      process.kill(up2.pid, 'SIGKILL');
+    } catch {
+      /* already stopped */
+    }
+    await new Promise((r) => setTimeout(r, 900));
+    let down = false;
+    try {
+      await fetch(`http://127.0.0.1:${port}/health`, { signal: AbortSignal.timeout(400) });
+    } catch {
+      down = true;
+    }
+    assert.equal(down, true);
+  } finally {
+    child.kill('SIGTERM');
+  }
+});