Add Proxmox worker CT, off-box watch, backup, and tagged deploy.
Some checks are pending
offline / test (push) Waiting to run

NS1 is the Proxmox host. verae-proxmox creates LXC 510 (verae-px-worker
10.10.10.20 on vmbr1) with a private NATS proxy on 10.10.10.1:4222.
verae-uptime GET-watches public doors; verae-backup snapshots SQLite and
worm/tree data; verae-deploy does host-deps + checkout + npm ci.
Fleet overlays/ns1 are checked in (start.sh no longer rewrites JSON).
User systemd + linger for keep and fleet survive reboot.
This commit is contained in:
George Lambert 2026-09-11 23:35:44 -04:00
parent 2d51d7a0dd
commit 7a5e25639e
69 changed files with 1087 additions and 118 deletions

View file

@ -0,0 +1,114 @@
#!/usr/bin/env node
/**
* GET each catalog/public door. Exit 2 on --once if any required target fails.
* Loop mode writes state JSON and optional webhook on rising failure.
*/
import fs from 'node:fs';
import http from 'node:http';
import https from 'node:https';
import os from 'node:os';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
const HERE = path.dirname(fileURLToPath(import.meta.url));
const ROOT = path.join(HERE, '..');
const ONCE = process.argv.includes('--once');
const CFG = JSON.parse(fs.readFileSync(process.env.UPTIME_PROBES || path.join(ROOT, 'probes.json'), 'utf8'));
function expandHome(p) {
if (!p) return p;
if (p.startsWith('~/')) return path.join(os.homedir(), p.slice(2));
return p;
}
const STATE_DIR = expandHome(process.env.UPTIME_STATE || CFG.stateDir || path.join(ROOT, 'data'));
const failAfter = Number(CFG.failAfter || 2);
const timeoutMs = Number(CFG.timeoutMs || 15000);
const streak = {};
function probe(url) {
return new Promise((resolve) => {
let settled = false;
const done = (r) => {
if (settled) return;
settled = true;
resolve(r);
};
const lib = url.startsWith('https:') ? https : http;
const req = lib.get(url, { timeout: timeoutMs }, (res) => {
res.resume();
const code = res.statusCode || 0;
done({ ok: code >= 200 && code < 400, code, error: '' });
});
req.on('timeout', () => {
req.destroy();
done({ ok: false, code: 0, error: 'timeout' });
});
req.on('error', (err) => done({ ok: false, code: 0, error: err.message }));
});
}
async function tick() {
const results = [];
let failed = 0;
for (const t of CFG.targets || []) {
const r = await probe(t.url);
const prev = streak[t.id] || 0;
streak[t.id] = r.ok ? 0 : prev + 1;
const down = !r.ok && streak[t.id] >= failAfter && !t.optional;
if (down) failed += 1;
results.push({
id: t.id,
url: t.url,
ok: r.ok,
code: r.code,
error: r.error,
streak: streak[t.id],
down,
});
}
const snap = { t: new Date().toISOString(), failed, results };
fs.mkdirSync(STATE_DIR, { recursive: true });
fs.writeFileSync(path.join(STATE_DIR, 'status.json'), `${JSON.stringify(snap, null, 2)}\n`);
fs.appendFileSync(path.join(STATE_DIR, 'watch.log'), `${JSON.stringify(snap)}\n`);
process.stdout.write(`${JSON.stringify({ t: snap.t, failed, ids: results.map((x) => `${x.id}:${x.ok ? 'ok' : x.code || x.error}`) })}\n`);
if (failed && CFG.webhook) {
try {
const u = new URL(CFG.webhook);
const body = JSON.stringify(snap);
const lib = u.protocol === 'https:' ? https : http;
const req = lib.request(u, { method: 'POST', headers: { 'content-type': 'application/json', 'content-length': Buffer.byteLength(body) } });
req.on('error', () => {});
req.end(body);
} catch {
/* ignore notify */
}
}
return failed;
}
if (ONCE) {
tick().then((n) => process.exit(n ? 2 : 0));
} else {
const server = http.createServer((req, res) => {
const file = path.join(STATE_DIR, 'status.json');
if (req.url === '/health' || req.url === '/status') {
if (!fs.existsSync(file)) {
res.writeHead(200, { 'content-type': 'application/json' });
return res.end('{"ok":true,"note":"no tick yet"}');
}
const snap = JSON.parse(fs.readFileSync(file, 'utf8'));
res.writeHead(snap.failed ? 503 : 200, { 'content-type': 'application/json' });
return res.end(JSON.stringify(snap));
}
res.writeHead(404);
res.end();
});
const bind = process.env.UPTIME_BIND || '127.0.0.1';
const port = Number(process.env.UPTIME_PORT || 3870);
server.listen(port, bind, () => {
process.stdout.write(`uptime listen ${bind}:${port}\n`);
});
tick();
setInterval(tick, Number(CFG.intervalSec || 60) * 1000);
}