Add verae-keep: host supervisor with watch+guard on NS1.
Some checks are pending
offline / test (push) Waiting to run
Some checks are pending
offline / test (push) Waiting to run
Restarts crashed units unless the admin console paused or stopped them. Fleet writes per-instance intent.json. Tested on 138: crash-restart, pause-hold, watch respawn of keep.
This commit is contained in:
parent
2bb3884a1b
commit
baaf1c2275
20 changed files with 1049 additions and 1 deletions
237
packages/verae-keep/src/keep.js
Normal file
237
packages/verae-keep/src/keep.js
Normal file
|
|
@ -0,0 +1,237 @@
|
|||
#!/usr/bin/env node
|
||||
/**
|
||||
* Host keep-alive. Restarts units that should be running.
|
||||
* Honors admin pause/stop (intent files + optional fleet /api/status).
|
||||
*/
|
||||
import fs from 'node:fs';
|
||||
import http from 'node:http';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { expandHome } from './expand.js';
|
||||
import {
|
||||
loadStore,
|
||||
saveStore,
|
||||
record,
|
||||
pickIntent,
|
||||
writeUnitFile,
|
||||
fromFleetService,
|
||||
} from './intent.js';
|
||||
import { httpProbe, httpPost, tcpProbe } from './probe.js';
|
||||
import { spawnUnit, killPid } from './spawn.js';
|
||||
import { loadUnitsFile, scanRuntimeIntents } from './units.js';
|
||||
|
||||
const HERE = path.dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = path.join(HERE, '..');
|
||||
const UNITS_FILE = expandHome(process.env.KEEP_UNITS || path.join(ROOT, 'units.json'));
|
||||
const STATE_DIR = expandHome(process.env.KEEP_STATE || path.join(ROOT, 'data'));
|
||||
const STORE_FILE = path.join(STATE_DIR, 'intent.json');
|
||||
const LOG_DIR = path.join(STATE_DIR, 'logs');
|
||||
const FLEET_URL = (process.env.FLEET_URL || '').replace(/\/$/, '');
|
||||
|
||||
const cfg = loadUnitsFile(UNITS_FILE);
|
||||
const store = loadStore(STORE_FILE);
|
||||
const last = {};
|
||||
const restarts = {};
|
||||
|
||||
function log(event, extra = {}) {
|
||||
const line = JSON.stringify({ t: new Date().toISOString(), event, ...extra });
|
||||
fs.mkdirSync(STATE_DIR, { recursive: true });
|
||||
fs.appendFileSync(path.join(STATE_DIR, 'keep.log'), `${line}\n`);
|
||||
process.stdout.write(`${line}\n`);
|
||||
}
|
||||
|
||||
function unitById(id) {
|
||||
return cfg.units.find((u) => u.id === id);
|
||||
}
|
||||
|
||||
function intentOf(unit, runtimeIntents, fleetMap) {
|
||||
const fromStore = store.units[unit.id];
|
||||
const fromRuntime = runtimeIntents[unit.id];
|
||||
const fromFleet = fleetMap[unit.id];
|
||||
const picked = pickIntent(fromStore, fromRuntime, fromFleet);
|
||||
if (picked) return picked;
|
||||
return { state: unit.defaultIntent || 'run', source: 'default', updatedAt: '' };
|
||||
}
|
||||
|
||||
async function probe(unit) {
|
||||
if (unit.healthUrl) return httpProbe(unit.healthUrl);
|
||||
if (unit.tcp) return tcpProbe(unit.tcp.host || '127.0.0.1', Number(unit.tcp.port));
|
||||
return { ok: false, paused: false, error: 'no probe' };
|
||||
}
|
||||
|
||||
function healthBase(unit) {
|
||||
if (!unit.healthUrl) return null;
|
||||
try {
|
||||
const u = new URL(unit.healthUrl);
|
||||
return `${u.protocol}//${u.host}`;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function bumpRestart(id) {
|
||||
const hour = Math.floor(Date.now() / 3600000);
|
||||
const slot = restarts[id] || { hour, n: 0 };
|
||||
if (slot.hour !== hour) {
|
||||
slot.hour = hour;
|
||||
slot.n = 0;
|
||||
}
|
||||
slot.n += 1;
|
||||
restarts[id] = slot;
|
||||
return slot.n <= 40;
|
||||
}
|
||||
|
||||
async function apply(unit, intent) {
|
||||
const p = await probe(unit);
|
||||
last[unit.id] = { intent, probe: { ok: p.ok, paused: p.paused, error: p.error || null, pid: p.pid || null } };
|
||||
const base = healthBase(unit);
|
||||
if (!unit.managed) {
|
||||
last[unit.id].action = p.ok ? 'observe' : 'down-unmanaged';
|
||||
return last[unit.id];
|
||||
}
|
||||
if (intent.state === 'stop') {
|
||||
if (p.ok || p.paused || p.pid) {
|
||||
killPid(p.pid || last[unit.id]?.spawnedPid);
|
||||
last[unit.id].action = 'killed';
|
||||
log('stop', { id: unit.id });
|
||||
} else last[unit.id].action = 'already-stopped';
|
||||
return last[unit.id];
|
||||
}
|
||||
if (intent.state === 'pause') {
|
||||
if (p.ok && !p.paused && base) {
|
||||
await httpPost(`${base}${unit.pausePath}`);
|
||||
last[unit.id].action = 'paused';
|
||||
log('pause', { id: unit.id });
|
||||
} else if (!p.ok && !p.paused) {
|
||||
last[unit.id].action = 'leave-down';
|
||||
} else last[unit.id].action = 'already-paused';
|
||||
return last[unit.id];
|
||||
}
|
||||
// run
|
||||
if (p.ok) {
|
||||
last[unit.id].action = 'healthy';
|
||||
return last[unit.id];
|
||||
}
|
||||
if (p.paused && base) {
|
||||
await httpPost(`${base}${unit.resumePath}`);
|
||||
last[unit.id].action = 'resumed';
|
||||
log('resume', { id: unit.id });
|
||||
return last[unit.id];
|
||||
}
|
||||
if (!unit.command) {
|
||||
last[unit.id].action = 'down-no-command';
|
||||
return last[unit.id];
|
||||
}
|
||||
if (!bumpRestart(unit.id)) {
|
||||
last[unit.id].action = 'restart-capped';
|
||||
log('restart-capped', { id: unit.id });
|
||||
return last[unit.id];
|
||||
}
|
||||
const spawned = spawnUnit(unit, { logDir: LOG_DIR });
|
||||
last[unit.id].spawnedPid = spawned.pid;
|
||||
last[unit.id].action = 'spawned';
|
||||
log('spawn', { id: unit.id, pid: spawned.pid });
|
||||
return last[unit.id];
|
||||
}
|
||||
|
||||
async function fleetMap() {
|
||||
const map = {};
|
||||
if (!FLEET_URL) return map;
|
||||
try {
|
||||
const r = await httpProbe(`${FLEET_URL}/api/status`, { timeoutMs: 2500 });
|
||||
const services = r.body?.services || {};
|
||||
for (const unit of cfg.units) {
|
||||
const sid = unit.env?.FLEET_SERVICE || unit.id.replace(/-\d+$/, '');
|
||||
const sv = services[sid];
|
||||
const via = fromFleetService(sv, unit.id);
|
||||
if (via) map[unit.id] = via;
|
||||
}
|
||||
} catch {
|
||||
/* fleet optional */
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
async function tick() {
|
||||
const runtimeIntents = cfg.runtimeDir ? scanRuntimeIntents(cfg.runtimeDir) : {};
|
||||
const fromFleet = await fleetMap();
|
||||
const actions = [];
|
||||
for (const unit of cfg.units) {
|
||||
const intent = intentOf(unit, runtimeIntents, fromFleet);
|
||||
const row = await apply(unit, intent);
|
||||
actions.push({ id: unit.id, intent: intent.state, source: intent.source, action: row.action });
|
||||
}
|
||||
return { t: new Date().toISOString(), actions };
|
||||
}
|
||||
|
||||
function setIntent(id, state, source = 'admin') {
|
||||
const unit = unitById(id);
|
||||
if (!unit) throw Object.assign(new Error(`unknown unit ${id}`), { status: 404 });
|
||||
const row = record(store, id, state, source);
|
||||
saveStore(STORE_FILE, store);
|
||||
if (cfg.runtimeDir) {
|
||||
writeUnitFile(path.join(cfg.runtimeDir, 'data', id, 'intent.json'), id, state, source);
|
||||
}
|
||||
log('intent', { id, state, source });
|
||||
return row;
|
||||
}
|
||||
|
||||
const server = http.createServer(async (req, res) => {
|
||||
const url = new URL(req.url || '/', `http://127.0.0.1:${cfg.port}`);
|
||||
const json = (code, obj) => {
|
||||
res.writeHead(code, { 'content-type': 'application/json', 'cache-control': 'no-store' });
|
||||
res.end(JSON.stringify(obj, null, 2));
|
||||
};
|
||||
try {
|
||||
if (req.method === 'GET' && url.pathname === '/health') {
|
||||
return json(200, { ok: true, role: 'verae-keep', units: cfg.units.length, pid: process.pid });
|
||||
}
|
||||
if (req.method === 'GET' && url.pathname === '/status') {
|
||||
return json(200, {
|
||||
ok: true,
|
||||
units: cfg.units.map((u) => ({
|
||||
id: u.id,
|
||||
title: u.title,
|
||||
intent: store.units[u.id] || { state: u.defaultIntent, source: 'default' },
|
||||
last: last[u.id] || null,
|
||||
})),
|
||||
});
|
||||
}
|
||||
const m = url.pathname.match(/^\/intent\/([^/]+)$/);
|
||||
if (req.method === 'POST' && m) {
|
||||
const chunks = [];
|
||||
for await (const c of req) chunks.push(c);
|
||||
const body = JSON.parse(Buffer.concat(chunks).toString('utf8') || '{}');
|
||||
const row = setIntent(decodeURIComponent(m[1]), body.state, body.source || 'admin');
|
||||
return json(200, row);
|
||||
}
|
||||
if (req.method === 'POST' && url.pathname === '/reconcile') {
|
||||
return json(200, await tick());
|
||||
}
|
||||
json(404, { error: 'not found' });
|
||||
} catch (err) {
|
||||
json(err.status || 400, { error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
const bind = process.env.KEEP_BIND || cfg.bind || '127.0.0.1';
|
||||
const port = Number(process.env.KEEP_PORT || cfg.port || 3860);
|
||||
|
||||
if (process.argv[1] === fileURLToPath(import.meta.url)) {
|
||||
server.listen(port, bind, () => {
|
||||
log('listen', { url: `http://${bind}:${port}/`, units: cfg.units.map((u) => u.id) });
|
||||
});
|
||||
const iv = setInterval(() => {
|
||||
tick().catch((err) => log('tick-error', { error: err.message }));
|
||||
}, cfg.intervalMs);
|
||||
tick().catch((err) => log('tick-error', { error: err.message }));
|
||||
const stop = () => {
|
||||
clearInterval(iv);
|
||||
server.close();
|
||||
process.exit(0);
|
||||
};
|
||||
process.on('SIGTERM', stop);
|
||||
process.on('SIGINT', stop);
|
||||
}
|
||||
|
||||
export { tick, setIntent, cfg, store };
|
||||
Loading…
Add table
Add a link
Reference in a new issue