Add Proxmox worker CT, off-box watch, backup, and tagged deploy.
Some checks are pending
offline / test (push) Waiting to run
Some checks are pending
offline / test (push) Waiting to run
NS1 is the Proxmox host. verae-proxmox creates LXC 510 (verae-px-worker 10.10.10.20 on vmbr1) with a private NATS proxy on 10.10.10.1:4222. verae-uptime GET-watches public doors; verae-backup snapshots SQLite and worm/tree data; verae-deploy does host-deps + checkout + npm ci. Fleet overlays/ns1 are checked in (start.sh no longer rewrites JSON). User systemd + linger for keep and fleet survive reboot.
This commit is contained in:
parent
2d51d7a0dd
commit
7a5e25639e
69 changed files with 1087 additions and 118 deletions
114
packages/verae-uptime/src/watch.js
Executable file
114
packages/verae-uptime/src/watch.js
Executable file
|
|
@ -0,0 +1,114 @@
|
|||
#!/usr/bin/env node
|
||||
/**
|
||||
* GET each catalog/public door. Exit 2 on --once if any required target fails.
|
||||
* Loop mode writes state JSON and optional webhook on rising failure.
|
||||
*/
|
||||
import fs from 'node:fs';
|
||||
import http from 'node:http';
|
||||
import https from 'node:https';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const HERE = path.dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = path.join(HERE, '..');
|
||||
const ONCE = process.argv.includes('--once');
|
||||
const CFG = JSON.parse(fs.readFileSync(process.env.UPTIME_PROBES || path.join(ROOT, 'probes.json'), 'utf8'));
|
||||
|
||||
function expandHome(p) {
|
||||
if (!p) return p;
|
||||
if (p.startsWith('~/')) return path.join(os.homedir(), p.slice(2));
|
||||
return p;
|
||||
}
|
||||
|
||||
const STATE_DIR = expandHome(process.env.UPTIME_STATE || CFG.stateDir || path.join(ROOT, 'data'));
|
||||
const failAfter = Number(CFG.failAfter || 2);
|
||||
const timeoutMs = Number(CFG.timeoutMs || 15000);
|
||||
const streak = {};
|
||||
|
||||
function probe(url) {
|
||||
return new Promise((resolve) => {
|
||||
let settled = false;
|
||||
const done = (r) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
resolve(r);
|
||||
};
|
||||
const lib = url.startsWith('https:') ? https : http;
|
||||
const req = lib.get(url, { timeout: timeoutMs }, (res) => {
|
||||
res.resume();
|
||||
const code = res.statusCode || 0;
|
||||
done({ ok: code >= 200 && code < 400, code, error: '' });
|
||||
});
|
||||
req.on('timeout', () => {
|
||||
req.destroy();
|
||||
done({ ok: false, code: 0, error: 'timeout' });
|
||||
});
|
||||
req.on('error', (err) => done({ ok: false, code: 0, error: err.message }));
|
||||
});
|
||||
}
|
||||
|
||||
async function tick() {
|
||||
const results = [];
|
||||
let failed = 0;
|
||||
for (const t of CFG.targets || []) {
|
||||
const r = await probe(t.url);
|
||||
const prev = streak[t.id] || 0;
|
||||
streak[t.id] = r.ok ? 0 : prev + 1;
|
||||
const down = !r.ok && streak[t.id] >= failAfter && !t.optional;
|
||||
if (down) failed += 1;
|
||||
results.push({
|
||||
id: t.id,
|
||||
url: t.url,
|
||||
ok: r.ok,
|
||||
code: r.code,
|
||||
error: r.error,
|
||||
streak: streak[t.id],
|
||||
down,
|
||||
});
|
||||
}
|
||||
const snap = { t: new Date().toISOString(), failed, results };
|
||||
fs.mkdirSync(STATE_DIR, { recursive: true });
|
||||
fs.writeFileSync(path.join(STATE_DIR, 'status.json'), `${JSON.stringify(snap, null, 2)}\n`);
|
||||
fs.appendFileSync(path.join(STATE_DIR, 'watch.log'), `${JSON.stringify(snap)}\n`);
|
||||
process.stdout.write(`${JSON.stringify({ t: snap.t, failed, ids: results.map((x) => `${x.id}:${x.ok ? 'ok' : x.code || x.error}`) })}\n`);
|
||||
if (failed && CFG.webhook) {
|
||||
try {
|
||||
const u = new URL(CFG.webhook);
|
||||
const body = JSON.stringify(snap);
|
||||
const lib = u.protocol === 'https:' ? https : http;
|
||||
const req = lib.request(u, { method: 'POST', headers: { 'content-type': 'application/json', 'content-length': Buffer.byteLength(body) } });
|
||||
req.on('error', () => {});
|
||||
req.end(body);
|
||||
} catch {
|
||||
/* ignore notify */
|
||||
}
|
||||
}
|
||||
return failed;
|
||||
}
|
||||
|
||||
if (ONCE) {
|
||||
tick().then((n) => process.exit(n ? 2 : 0));
|
||||
} else {
|
||||
const server = http.createServer((req, res) => {
|
||||
const file = path.join(STATE_DIR, 'status.json');
|
||||
if (req.url === '/health' || req.url === '/status') {
|
||||
if (!fs.existsSync(file)) {
|
||||
res.writeHead(200, { 'content-type': 'application/json' });
|
||||
return res.end('{"ok":true,"note":"no tick yet"}');
|
||||
}
|
||||
const snap = JSON.parse(fs.readFileSync(file, 'utf8'));
|
||||
res.writeHead(snap.failed ? 503 : 200, { 'content-type': 'application/json' });
|
||||
return res.end(JSON.stringify(snap));
|
||||
}
|
||||
res.writeHead(404);
|
||||
res.end();
|
||||
});
|
||||
const bind = process.env.UPTIME_BIND || '127.0.0.1';
|
||||
const port = Number(process.env.UPTIME_PORT || 3870);
|
||||
server.listen(port, bind, () => {
|
||||
process.stdout.write(`uptime listen ${bind}:${port}\n`);
|
||||
});
|
||||
tick();
|
||||
setInterval(tick, Number(CFG.intervalSec || 60) * 1000);
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue