Add verae-keep: host supervisor with watch+guard on NS1.
Some checks are pending
offline / test (push) Waiting to run

Restarts crashed units unless the admin console paused or stopped
them. Fleet writes per-instance intent.json. Tested on 138:
crash-restart, pause-hold, watch respawn of keep.
This commit is contained in:
George Lambert 2026-09-11 19:25:11 -04:00
parent 2bb3884a1b
commit baaf1c2275
20 changed files with 1049 additions and 1 deletions

View file

@ -135,6 +135,21 @@ export async function sshSpawnWorker(machine, { instance, env }) {
return { pid, remoteDir };
}
export async function sshWriteIntent(machine, instance, state) {
const remoteDir = machine.remoteDir || '~/verae-fleet-runtime';
const body = JSON.stringify({
id: instance,
state,
source: 'fleet',
updatedAt: new Date().toISOString(),
});
const dest = `${remoteDir}/data/${instance}/intent.json`;
await sshExec(
machine,
`mkdir -p ${remoteDir}/data/${instance} && printf %s ${shellQuote(body)} > ${dest}`,
);
}
export async function sshKillWorker(machine, pid, instance) {
const cmd = pid
? `kill -TERM ${Number(pid)} 2>/dev/null || true`

View file

@ -12,7 +12,7 @@ import { httpProbe, postControl } from './health.js';
import { classifyService } from './classify.js';
import { pickMachine, upsertMachine, saveOverlay } from './machines.js';
import { summarizeRtt } from './rtt.js';
import { sshSpawnWorker, sshKillWorker, sshHttp, sshCheck } from './ssh.js';
import { sshSpawnWorker, sshKillWorker, sshHttp, sshCheck, sshWriteIntent } from './ssh.js';
const WORKER = path.join(path.dirname(fileURLToPath(import.meta.url)), 'worker.js');
@ -237,13 +237,41 @@ export class Supervisor {
const h = await waitForHealth(rec, 8000);
rec.healthy = h.ok;
rec.lastProbe = h;
await this.persistIntent(rec, 'run');
return rec;
}
async persistIntent(rec, state) {
if (!rec) return;
const body = {
id: rec.id,
state,
source: 'fleet',
updatedAt: new Date().toISOString(),
};
try {
fs.mkdirSync(rec.stateDir, { recursive: true });
fs.writeFileSync(path.join(rec.stateDir, 'intent.json'), JSON.stringify(body, null, 2));
} catch {
/* ignore */
}
if (rec.kind === 'ssh') {
const m = this.machines().find((x) => x.id === rec.machine);
if (m) {
try {
await sshWriteIntent(m, rec.id, state);
} catch {
/* ignore */
}
}
}
}
async stopInstance(instanceId, { replace = true } = {}) {
const rec = this.instances.get(instanceId);
if (!rec) return null;
rec.stopping = true;
await this.persistIntent(rec, 'stop');
if (rec.kind === 'ssh') {
const m = this.machines().find((x) => x.id === rec.machine);
if (m) {
@ -277,6 +305,7 @@ export class Supervisor {
if (!rec) throw new Error(`no instance ${instanceId}`);
await this.controlInstance(rec, '/pause');
rec.paused = true;
await this.persistIntent(rec, 'pause');
this.log('pause', { instance: instanceId, service: rec.service });
if (replace) await this.reconcile(rec.service);
return rec;
@ -287,6 +316,7 @@ export class Supervisor {
if (!rec) throw new Error(`no instance ${instanceId}`);
await this.controlInstance(rec, '/resume');
rec.paused = false;
await this.persistIntent(rec, 'run');
this.log('resume', { instance: instanceId, service: rec.service });
return rec;
}