Spread fleet replicas across extra machines and show message RTT percentiles

machines.json plus Add-machine UI place copies on the least-loaded host. Monitor samples processing RTT and displays min, avg, p50, and p90 for capacity planning. Remote hosts run src/agent.js.
2026-09-11 13:25:05 -04:00 · 2026-09-11 13:25:05 -04:00 · b325f6f697
commit b325f6f697
parent 6fbc808bb1
17 changed files with 673 additions and 53 deletions
--- a/packages/verae-fleet/public/index.html
+++ b/packages/verae-fleet/public/index.html
@ -11,9 +11,10 @@
    header { background:var(--ink); color:#f6f3ee; padding:.9rem 1.1rem; }
    header h1 { margin:0; font-size:1.1rem; }
    header p { margin:.35rem 0 0; color:#c5d0d8; font-size:13px; }
-    main { padding:1rem; max-width:1100px; margin:0 auto; }
+    main { padding:1rem; max-width:1280px; margin:0 auto; }
+    h2 { font-size:13px; letter-spacing:.08em; text-transform:uppercase; color:var(--muted); margin:1.2rem 0 .4rem; }
    table { width:100%; border-collapse:collapse; background:#fff; }
-    th, td { text-align:left; padding:.4rem .5rem; border-bottom:1px solid var(--line); font-size:13px; }
+    th, td { text-align:left; padding:.4rem .5rem; border-bottom:1px solid var(--line); font-size:13px; vertical-align:top; }
    th { font-size:11px; letter-spacing:.06em; text-transform:uppercase; color:var(--muted); }
    tr.state-operational { background:#e3f6e8; }
    tr.state-degraded { background:#fff3bf; }
@ -25,28 +26,65 @@
    button { margin-right:.25rem; font:650 12px system-ui; border:1px solid var(--line); background:#fff; border-radius:5px; padding:.25rem .45rem; cursor:pointer; }
    code { font:12px ui-monospace,Menlo,monospace; }
    .inst { font:12px ui-monospace,Menlo,monospace; margin:.15rem 0; }
+    .rtt { font:12px ui-monospace,Menlo,monospace; white-space:nowrap; }
+    form.add { display:flex; flex-wrap:wrap; gap:.4rem; margin:.5rem 0 1rem; align-items:end; }
+    form.add label { font-size:11px; color:var(--muted); display:flex; flex-direction:column; gap:.15rem; }
+    form.add input, form.add select { padding:.35rem .45rem; border:1px solid var(--line); border-radius:5px; font:13px ui-monospace,Menlo,monospace; }
  </style>
 </head>
 <body>
 <header>
  <h1>Verae fleet monitor</h1>
-  <p>Active replicas, replica floors, pause / restart. Tree-node <code>keepFloor</code> respawns until <code>min</code> copies are healthy and unpaused. Loopback only.</p>
+  <p>Spread replicas across machines. Track message-processing RTT (min / avg / p50 / p90) for capacity planning. Paused copies do not count toward the tree-node floor.</p>
 </header>
 <main>
  <p id="meta"></p>
+  <h2>Machines</h2>
+  <form class="add" id="addMachine">
+    <label>id <input name="id" required placeholder="ns1-b"/></label>
+    <label>host <input name="host" required placeholder="10.0.0.12"/></label>
+    <label>kind
+      <select name="kind"><option value="local">local</option><option value="agent">agent</option></select>
+    </label>
+    <label>capacity <input name="capacity" type="number" min="1" value="8"/></label>
+    <label>roles <input name="roles" value="*" placeholder="tree-node,archive-worm"/></label>
+    <button type="submit">Add machine</button>
+  </form>
  <table>
-    <thead><tr><th>Service</th><th>State</th><th>Config</th><th>min/max</th><th>available</th><th>instances</th><th>actions</th></tr></thead>
+    <thead><tr><th>Machine</th><th>Host</th><th>Kind</th><th>Capacity</th><th>Running</th><th>RTT ms min / avg / p50 / p90</th><th></th></tr></thead>
+    <tbody id="machines"></tbody>
+  </table>
+  <h2>Services</h2>
+  <table>
+    <thead><tr><th>Service</th><th>State</th><th>min/max</th><th>available</th><th>RTT ms min / avg / p50 / p90</th><th>instances</th><th>actions</th></tr></thead>
    <tbody id="rows"></tbody>
  </table>
 </main>
 <script>
 async function j(url, opts) { const r = await fetch(url, opts); return r.json(); }
 async function act(path) { await j(path, { method:'POST' }); await draw(); }
+function rttCell(r) {
+  if (!r || !r.count) return '<span class="rtt">—</span>';
+  return `<span class="rtt">${r.minMs} / ${r.avgMs} / ${r.p50Ms} / ${r.p90Ms} <span style="color:#5b6d78">n=${r.count}</span></span>`;
+}
 async function draw() {
  const s = await j('/api/status');
-  document.getElementById('meta').textContent = 'probe ' + (s.monitor?.t || '—') + ' · NATS ' + (s.nats?.url || '');
-  const tb = document.getElementById('rows');
-  tb.innerHTML = Object.values(s.services).map((sv) => {
+  document.getElementById('meta').textContent = 'probe ' + (s.monitor?.t || '—') + ' · NATS ' + (s.nats?.url || '') + ' · machines ' + (s.machines || []).filter(m => m.enabled).length;
+  document.getElementById('machines').innerHTML = (s.machines || []).map((m) => {
+    const on = m.enabled;
+    return `<tr class="${on ? (m.running > 0 ? 'state-operational' : '') : 'state-down'}">
+      <td><strong>${m.id}</strong><br><span style="color:#5b6d78">${m.title || ''}</span></td>
+      <td><code>${m.host}${m.kind === 'agent' ? ':' + m.agentPort : ''}</code></td>
+      <td>${m.kind} ${on ? '<span class="pill good">on</span>' : '<span class="pill bad">off</span>'}</td>
+      <td>${m.running} / ${m.capacity}</td>
+      <td>${m.available} avail</td>
+      <td>${rttCell(m.rtt)}</td>
+      <td>
+        <button onclick="act('/api/machines/${m.id}/${on ? 'disable' : 'enable'}')">${on ? 'disable' : 'enable'}</button>
+      </td>
+    </tr>`;
+  }).join('');
+  document.getElementById('rows').innerHTML = Object.values(s.services).map((sv) => {
    const health = sv.health || 'down';
    const healthLabel = health === 'operational' ? 'operational' : health === 'degraded' ? 'degraded' : 'not operational';
    const healthCls = health === 'operational' ? 'good' : health === 'degraded' ? 'warn' : 'bad';
@ -54,7 +92,8 @@ async function draw() {
    const inst = (sv.instances || []).map((i) => {
      const st = !i.pid ? 'dead' : i.paused ? 'paused' : i.healthy === false ? 'unhealthy' : 'up';
      const cls = st === 'up' ? 'good' : st === 'paused' ? 'warn' : 'bad';
-      return `<div class="inst"><span class="pill ${cls}">${st}</span> ${i.id} pid=${i.pid || '—'} :${i.healthPort}
+      return `<div class="inst"><span class="pill ${cls}">${st}</span> ${i.id} @${i.machine || 'local'} pid=${i.pid || '—'} :${i.healthPort}
+        ${i.rtt && i.rtt.count ? `rtt ${i.rtt.minMs}/${i.rtt.avgMs}/${i.rtt.p50Ms}/${i.rtt.p90Ms}` : ''}
        <button onclick="act('/api/instances/${i.id}/pause')">pause</button>
        <button onclick="act('/api/instances/${i.id}/resume')">resume</button>
        <button onclick="act('/api/instances/${i.id}/restart')">restart</button>
@ -63,9 +102,9 @@ async function draw() {
    return `<tr class="state-${health}">
      <td><strong>${sv.id}</strong><br><span style="color:#5b6d78">${sv.title || ''}</span></td>
      <td><span class="pill ${healthCls}">${healthLabel}</span></td>
-      <td><code>${sv.configPath || ''}</code></td>
      <td>${sv.min} / ${sv.max} ${floor}</td>
-      <td>${sv.available} running=${sv.running} paused=${sv.paused}</td>
+      <td>${sv.available} run=${sv.running} paused=${sv.paused}</td>
+      <td>${rttCell(sv.rtt)}</td>
      <td>${inst}</td>
      <td>
        <button onclick="act('/api/services/${sv.id}/start')">on</button>
@ -76,6 +115,25 @@ async function draw() {
    </tr>`;
  }).join('');
 }
+document.getElementById('addMachine').addEventListener('submit', async (e) => {
+  e.preventDefault();
+  const f = e.target;
+  const roles = f.roles.value.split(',').map((s) => s.trim()).filter(Boolean);
+  await j('/api/machines', {
+    method: 'POST',
+    headers: { 'content-type': 'application/json' },
+    body: JSON.stringify({
+      id: f.id.value,
+      host: f.host.value,
+      kind: f.kind.value,
+      capacity: Number(f.capacity.value || 8),
+      roles,
+      enabled: true,
+    }),
+  });
+  f.reset();
+  await draw();
+});
 draw();
 setInterval(draw, 1500);
 </script>