Add verae-keep: host supervisor with watch+guard on NS1.
Some checks are pending
offline / test (push) Waiting to run
Some checks are pending
offline / test (push) Waiting to run
Restarts crashed units unless the admin console paused or stopped them. Fleet writes per-instance intent.json. Tested on 138: crash-restart, pause-hold, watch respawn of keep.
This commit is contained in:
parent
2bb3884a1b
commit
baaf1c2275
20 changed files with 1049 additions and 1 deletions
|
|
@ -135,6 +135,21 @@ export async function sshSpawnWorker(machine, { instance, env }) {
|
|||
return { pid, remoteDir };
|
||||
}
|
||||
|
||||
export async function sshWriteIntent(machine, instance, state) {
|
||||
const remoteDir = machine.remoteDir || '~/verae-fleet-runtime';
|
||||
const body = JSON.stringify({
|
||||
id: instance,
|
||||
state,
|
||||
source: 'fleet',
|
||||
updatedAt: new Date().toISOString(),
|
||||
});
|
||||
const dest = `${remoteDir}/data/${instance}/intent.json`;
|
||||
await sshExec(
|
||||
machine,
|
||||
`mkdir -p ${remoteDir}/data/${instance} && printf %s ${shellQuote(body)} > ${dest}`,
|
||||
);
|
||||
}
|
||||
|
||||
export async function sshKillWorker(machine, pid, instance) {
|
||||
const cmd = pid
|
||||
? `kill -TERM ${Number(pid)} 2>/dev/null || true`
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ import { httpProbe, postControl } from './health.js';
|
|||
import { classifyService } from './classify.js';
|
||||
import { pickMachine, upsertMachine, saveOverlay } from './machines.js';
|
||||
import { summarizeRtt } from './rtt.js';
|
||||
import { sshSpawnWorker, sshKillWorker, sshHttp, sshCheck } from './ssh.js';
|
||||
import { sshSpawnWorker, sshKillWorker, sshHttp, sshCheck, sshWriteIntent } from './ssh.js';
|
||||
|
||||
const WORKER = path.join(path.dirname(fileURLToPath(import.meta.url)), 'worker.js');
|
||||
|
||||
|
|
@ -237,13 +237,41 @@ export class Supervisor {
|
|||
const h = await waitForHealth(rec, 8000);
|
||||
rec.healthy = h.ok;
|
||||
rec.lastProbe = h;
|
||||
await this.persistIntent(rec, 'run');
|
||||
return rec;
|
||||
}
|
||||
|
||||
async persistIntent(rec, state) {
|
||||
if (!rec) return;
|
||||
const body = {
|
||||
id: rec.id,
|
||||
state,
|
||||
source: 'fleet',
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
try {
|
||||
fs.mkdirSync(rec.stateDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(rec.stateDir, 'intent.json'), JSON.stringify(body, null, 2));
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
if (rec.kind === 'ssh') {
|
||||
const m = this.machines().find((x) => x.id === rec.machine);
|
||||
if (m) {
|
||||
try {
|
||||
await sshWriteIntent(m, rec.id, state);
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async stopInstance(instanceId, { replace = true } = {}) {
|
||||
const rec = this.instances.get(instanceId);
|
||||
if (!rec) return null;
|
||||
rec.stopping = true;
|
||||
await this.persistIntent(rec, 'stop');
|
||||
if (rec.kind === 'ssh') {
|
||||
const m = this.machines().find((x) => x.id === rec.machine);
|
||||
if (m) {
|
||||
|
|
@ -277,6 +305,7 @@ export class Supervisor {
|
|||
if (!rec) throw new Error(`no instance ${instanceId}`);
|
||||
await this.controlInstance(rec, '/pause');
|
||||
rec.paused = true;
|
||||
await this.persistIntent(rec, 'pause');
|
||||
this.log('pause', { instance: instanceId, service: rec.service });
|
||||
if (replace) await this.reconcile(rec.service);
|
||||
return rec;
|
||||
|
|
@ -287,6 +316,7 @@ export class Supervisor {
|
|||
if (!rec) throw new Error(`no instance ${instanceId}`);
|
||||
await this.controlInstance(rec, '/resume');
|
||||
rec.paused = false;
|
||||
await this.persistIntent(rec, 'run');
|
||||
this.log('resume', { instance: instanceId, service: rec.service });
|
||||
return rec;
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue