Proxmox worker, uptime, backup, tagged deploy

NATS.md

@@ -0,0 +1,3 @@
+# NATS — verae-uptime
+
+Uptime watch is HTTPS GET only. It never connects to NATS.

README.md

@@ -0,0 +1,21 @@
+# verae-uptime
+
+GET-watch the public Verae Time × Zapier doors from a host that is **not** the thing being watched. If NS1 is the only watcher, NS1 dying is silent.
+
+**Forgejo:** https://git.georgelambert.org/marchon/verae-uptime  
+**SSH:** `ssh://git@git.georgelambert.org:2223/marchon/verae-uptime.git`
+
+Run on a laptop, a second WAN box, or (weaker) a Proxmox CT on the same chassis.
+
+```bash
+git clone ssh://git@git.georgelambert.org:2223/marchon/verae-uptime.git
+cd verae-uptime
+node src/watch.js --once          # exit 2 if any required target failed
+node src/watch.js                 # loop + http://127.0.0.1:3870/status
+```
+
+`probes.json` lists catalog, portal, api, zap, iam, mw, fleet, git. Optional `webhook` POSTs the snapshot when `failed > 0`.
+
+State: `~/.verae-uptime/status.json` and `watch.log`.
+
+This process does **not** speak NATS.

SUMMARY.md

@@ -0,0 +1,3 @@
+# verae-uptime
+
+Off-box HTTPS `/health` (and homepage) probes for public doors. Loop + `/status`, or `--once` for cron.

package.json

@@ -0,0 +1,13 @@
+{
+  "name": "verae-uptime",
+  "version": "0.1.0",
+  "private": true,
+  "type": "module",
+  "description": "Off-box HTTPS health watch for Verae Time × Zapier public doors",
+  "scripts": {
+    "start": "node src/watch.js",
+    "once": "node src/watch.js --once",
+    "test": "node --test test/*.test.js"
+  },
+  "engines": { "node": ">=20" }
+}

probes.json

@@ -0,0 +1,17 @@
+{
+  "intervalSec": 60,
+  "failAfter": 2,
+  "timeoutMs": 15000,
+  "stateDir": "~/.verae-uptime",
+  "webhook": "",
+  "targets": [
+    { "id": "catalog", "url": "https://zapier.georgelambert.org/" },
+    { "id": "portal", "url": "https://portal.zapier.georgelambert.org/portal/" },
+    { "id": "api", "url": "https://api.zapier.georgelambert.org/health" },
+    { "id": "zap", "url": "https://zap.zapier.georgelambert.org/health" },
+    { "id": "iam", "url": "https://iam.zapier.georgelambert.org/" },
+    { "id": "mw", "url": "https://mw.zapier.georgelambert.org/health" },
+    { "id": "fleet", "url": "https://fleet.zapier.georgelambert.org/" },
+    { "id": "git", "url": "https://git.georgelambert.org/" }
+  ]
+}

src/watch.js

@@ -0,0 +1,114 @@
+#!/usr/bin/env node
+/**
+ * GET each catalog/public door. Exit 2 on --once if any required target fails.
+ * Loop mode writes state JSON and optional webhook on rising failure.
+ */
+import fs from 'node:fs';
+import http from 'node:http';
+import https from 'node:https';
+import os from 'node:os';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+const ROOT = path.join(HERE, '..');
+const ONCE = process.argv.includes('--once');
+const CFG = JSON.parse(fs.readFileSync(process.env.UPTIME_PROBES || path.join(ROOT, 'probes.json'), 'utf8'));
+
+function expandHome(p) {
+  if (!p) return p;
+  if (p.startsWith('~/')) return path.join(os.homedir(), p.slice(2));
+  return p;
+}
+
+const STATE_DIR = expandHome(process.env.UPTIME_STATE || CFG.stateDir || path.join(ROOT, 'data'));
+const failAfter = Number(CFG.failAfter || 2);
+const timeoutMs = Number(CFG.timeoutMs || 15000);
+const streak = {};
+
+function probe(url) {
+  return new Promise((resolve) => {
+    let settled = false;
+    const done = (r) => {
+      if (settled) return;
+      settled = true;
+      resolve(r);
+    };
+    const lib = url.startsWith('https:') ? https : http;
+    const req = lib.get(url, { timeout: timeoutMs }, (res) => {
+      res.resume();
+      const code = res.statusCode || 0;
+      done({ ok: code >= 200 && code < 400, code, error: '' });
+    });
+    req.on('timeout', () => {
+      req.destroy();
+      done({ ok: false, code: 0, error: 'timeout' });
+    });
+    req.on('error', (err) => done({ ok: false, code: 0, error: err.message }));
+  });
+}
+
+async function tick() {
+  const results = [];
+  let failed = 0;
+  for (const t of CFG.targets || []) {
+    const r = await probe(t.url);
+    const prev = streak[t.id] || 0;
+    streak[t.id] = r.ok ? 0 : prev + 1;
+    const down = !r.ok && streak[t.id] >= failAfter && !t.optional;
+    if (down) failed += 1;
+    results.push({
+      id: t.id,
+      url: t.url,
+      ok: r.ok,
+      code: r.code,
+      error: r.error,
+      streak: streak[t.id],
+      down,
+    });
+  }
+  const snap = { t: new Date().toISOString(), failed, results };
+  fs.mkdirSync(STATE_DIR, { recursive: true });
+  fs.writeFileSync(path.join(STATE_DIR, 'status.json'), `${JSON.stringify(snap, null, 2)}\n`);
+  fs.appendFileSync(path.join(STATE_DIR, 'watch.log'), `${JSON.stringify(snap)}\n`);
+  process.stdout.write(`${JSON.stringify({ t: snap.t, failed, ids: results.map((x) => `${x.id}:${x.ok ? 'ok' : x.code || x.error}`) })}\n`);
+  if (failed && CFG.webhook) {
+    try {
+      const u = new URL(CFG.webhook);
+      const body = JSON.stringify(snap);
+      const lib = u.protocol === 'https:' ? https : http;
+      const req = lib.request(u, { method: 'POST', headers: { 'content-type': 'application/json', 'content-length': Buffer.byteLength(body) } });
+      req.on('error', () => {});
+      req.end(body);
+    } catch {
+      /* ignore notify */
+    }
+  }
+  return failed;
+}
+
+if (ONCE) {
+  tick().then((n) => process.exit(n ? 2 : 0));
+} else {
+  const server = http.createServer((req, res) => {
+    const file = path.join(STATE_DIR, 'status.json');
+    if (req.url === '/health' || req.url === '/status') {
+      if (!fs.existsSync(file)) {
+        res.writeHead(200, { 'content-type': 'application/json' });
+        return res.end('{"ok":true,"note":"no tick yet"}');
+      }
+      const snap = JSON.parse(fs.readFileSync(file, 'utf8'));
+      res.writeHead(snap.failed ? 503 : 200, { 'content-type': 'application/json' });
+      return res.end(JSON.stringify(snap));
+    }
+    res.writeHead(404);
+    res.end();
+  });
+  const bind = process.env.UPTIME_BIND || '127.0.0.1';
+  const port = Number(process.env.UPTIME_PORT || 3870);
+  server.listen(port, bind, () => {
+    process.stdout.write(`uptime listen ${bind}:${port}\n`);
+  });
+  tick();
+  setInterval(tick, Number(CFG.intervalSec || 60) * 1000);
+}

systemd/verae-uptime.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=Verae off-box HTTPS door watch
+After=network-online.target
+
+[Service]
+Type=simple
+WorkingDirectory=%h/verae-src/verae-uptime
+Environment=UPTIME_BIND=127.0.0.1
+Environment=UPTIME_PORT=3870
+ExecStart=/usr/bin/node src/watch.js
+Restart=always
+RestartSec=5
+
+[Install]
+WantedBy=default.target

test/watch.test.js

@@ -0,0 +1,45 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import http from 'node:http';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { spawn } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+
+const root = path.join(path.dirname(fileURLToPath(import.meta.url)), '..');
+
+test('once mode fails when a required target is down', async () => {
+  const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'uptime-'));
+  const srv = await new Promise((resolve) => {
+    const s = http.createServer((req, res) => {
+      res.writeHead(req.url === '/ok' ? 200 : 500);
+      res.end('x');
+    });
+    s.listen(0, '127.0.0.1', () => resolve(s));
+  });
+  const port = srv.address().port;
+  const cfg = path.join(tmp, 'probes.json');
+  fs.writeFileSync(
+    cfg,
+    JSON.stringify({
+      failAfter: 1,
+      timeoutMs: 2000,
+      stateDir: tmp,
+      targets: [
+        { id: 'ok', url: `http://127.0.0.1:${port}/ok` },
+        { id: 'bad', url: `http://127.0.0.1:${port}/bad` },
+      ],
+    }),
+  );
+  const code = await new Promise((resolve) => {
+    const p = spawn(process.execPath, [path.join(root, 'src/watch.js'), '--once'], {
+      env: { ...process.env, UPTIME_PROBES: cfg, UPTIME_STATE: tmp },
+    });
+    p.on('close', resolve);
+  });
+  srv.close();
+  assert.equal(code, 2);
+  const snap = JSON.parse(fs.readFileSync(path.join(tmp, 'status.json'), 'utf8'));
+  assert.equal(snap.failed, 1);
+});