From e666e7bf3cb02871bab2e28ecbcb3b07d25625d8 Mon Sep 17 00:00:00 2001 From: George Lambert Date: Sat, 3 Oct 2026 23:02:02 -0400 Subject: [PATCH] WIP snapshot before drive cleanup 2026-10-03 --- .agents/JOB_BUS.md | 1 + .jobbus.json | 6 + AGENTS.md | 6 + LOG.md | 4 + STATUS.md | 8 +- TODO.md | 1 + decisions/D001-prepaid-sot.md | 7 + .../D006-libsql-business-jetstream-work.md | 39 ++ decisions/index.md | 10 + jobs/pieces.yaml | 23 + plans/2026-10-03-forgejo-sources.md | 178 ++++++ ...026-10-03-libsql-multihost-architecture.md | 295 ++++++++++ plans/2026-10-03-system-docs-map.md | 383 +++++++++++++ plans/qa/2026-10-03-baseline.md | 344 ++++++++++++ plans/qa/2026-10-03-libsql-dryrun.md | 89 +++ .../2026-10-03-retest-completedAt-credits.md | 86 +++ plans/qa/2026-10-03-retest-mg3.md | 255 +++++++++ plans/qa/2026-10-03-retest-wpl.md | 243 +++++++++ plans/reviews/2026-10-03-arch-review.md | 220 ++++++++ plans/reviews/2026-10-03-pm-board.md | 511 ++++++++++++++++++ scripts/jobbus-farm.sh | 120 ++++ 21 files changed, 2828 insertions(+), 1 deletion(-) create mode 120000 .agents/JOB_BUS.md create mode 100644 .jobbus.json create mode 100644 AGENTS.md create mode 100644 decisions/D006-libsql-business-jetstream-work.md create mode 100644 decisions/index.md create mode 100644 jobs/pieces.yaml create mode 100644 plans/2026-10-03-forgejo-sources.md create mode 100644 plans/2026-10-03-libsql-multihost-architecture.md create mode 100644 plans/2026-10-03-system-docs-map.md create mode 100644 plans/qa/2026-10-03-baseline.md create mode 100644 plans/qa/2026-10-03-libsql-dryrun.md create mode 100644 plans/qa/2026-10-03-retest-completedAt-credits.md create mode 100644 plans/qa/2026-10-03-retest-mg3.md create mode 100644 plans/qa/2026-10-03-retest-wpl.md create mode 100644 plans/reviews/2026-10-03-arch-review.md create mode 100644 plans/reviews/2026-10-03-pm-board.md create mode 100755 scripts/jobbus-farm.sh diff --git a/.agents/JOB_BUS.md b/.agents/JOB_BUS.md new file mode 120000 index 0000000..e546bba --- /dev/null +++ b/.agents/JOB_BUS.md @@ -0,0 +1 @@ +/Users/marchon/job-bus/docs/JOB_BUS_LLM_INSTRUCTIONS.md \ No newline at end of file diff --git a/.jobbus.json b/.jobbus.json new file mode 100644 index 0000000..ab460ca --- /dev/null +++ b/.jobbus.json @@ -0,0 +1,6 @@ +{ + "enabled": true, + "manifest": "jobs/pieces.yaml", + "farm": "scripts/jobbus-farm.sh", + "instructions": ".agents/JOB_BUS.md" +} diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..2c41576 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,6 @@ +# Agent notes + +## Job-bus multi-build +This repo farms multi-build pieces via NATS job-bus. See [.agents/JOB_BUS.md](.agents/JOB_BUS.md). +Edit `jobs/pieces.yaml`, then run `./scripts/jobbus-farm.sh` (or `make farm`). +CPU pieces → Xeons (137/138); GPU pieces → Mac. Always set `requires` per piece. diff --git a/LOG.md b/LOG.md index 4a37102..10ee026 100644 --- a/LOG.md +++ b/LOG.md @@ -1,5 +1,9 @@ # Action log +## 2026-10-03 — D006 libSQL business SoT + +Accepted **D006**: all billing, customer, pricing, accounting, and related business data live in **libSQL**. JetStream only as needed for multi-host work queues. Core NATS only for Verae timestamp job-complete subjects. Go timestamping DBs unchanged. Single-host `billing.db` on 138 remains the lab stand-in until libSQL cutover. + ## 2026-09-12 — exhaustive optimal-config study - 8c/16G ZFS factorial: file/memory × r=1/r=3, 4 KiB, reconnect, UDP echo, MQTT QoS0 on nats-a :1883 (then removed). diff --git a/STATUS.md b/STATUS.md index 779ef4a..cb33d59 100644 --- a/STATUS.md +++ b/STATUS.md @@ -1,4 +1,10 @@ -# Current status — 2026-09-12 +# Current status — 2026-10-03 + +**Multi-host cutover plan in flight:** [plans/2026-10-03-libsql-multihost-architecture.md](plans/2026-10-03-libsql-multihost-architecture.md) (D006: business SoT = **libSQL**; JetStream = work only). Arch review: **approve-with-changes** (A1–A8 merged). PM board: [plans/reviews/2026-10-03-pm-board.md](plans/reviews/2026-10-03-pm-board.md). Docs map: [plans/2026-10-03-system-docs-map.md](plans/2026-10-03-system-docs-map.md). + +--- + +# Prior status — 2026-09-12 Verae Time × Zapier is a **working lab** on one Proxmox host (**NS1.GEORGELAMBERT.ORG**, `70.88.205.138`). Public HTTPS doors, catalog, keep/fleet, and a **test 3-node NATS cluster** on separate LXC guests are up. The test environment **points at that cluster**. Zapier Platform `push` and live `api.veraetime.net` are still waiting on operator login / bind credentials. diff --git a/TODO.md b/TODO.md index d929275..6f50c5b 100644 --- a/TODO.md +++ b/TODO.md @@ -13,3 +13,4 @@ - [x] Host keep-alive (`verae-keep`) on NS1 with watch+guard; honors admin pause/stop. - [x] Docs synced to IAM, public portal `:3021`, keep, bootstrap server types. - [x] `verae-bootstrap` clones all module repos and installs deps per server type. +- [ ] Cut over billing/customers/pricing/usage ledger from single-host billing.db to shared libSQL (D006); keep JetStream for jobs only. diff --git a/decisions/D001-prepaid-sot.md b/decisions/D001-prepaid-sot.md index 9b1bace..e5b1d83 100644 --- a/decisions/D001-prepaid-sot.md +++ b/decisions/D001-prepaid-sot.md @@ -4,3 +4,10 @@ **Step:** 1 Prepaid cents, credits, usage, and payments persist in `zappier-account-balance` (`books.json`). zappier-edge `balanceCents` is a cache filled from the books reply. Tests without NATS/HTTP books still mutate the cache so unit tests stay hermetic. + +## Addendum (2026-10-03, D006) + +Durable prepaid/credits/usage/payments store is shared **libSQL**, not per-process +`books.json` or per-host `billing.db`. Single writer path still required. Edge +`balanceCents` remains a cache. Lab `books.json` / `billing.db` are migration sources. + diff --git a/decisions/D006-libsql-business-jetstream-work.md b/decisions/D006-libsql-business-jetstream-work.md new file mode 100644 index 0000000..11cb5a2 --- /dev/null +++ b/decisions/D006-libsql-business-jetstream-work.md @@ -0,0 +1,39 @@ +# D006 — Business data in libSQL; JetStream only for work + +**Status:** accepted +**Step:** 6 +**Date:** 2026-10-03 + +Launch topology is **3–5 machines** (gateway, bridge, delivery, and related planes). Shared business state must not live as per-host SQLite files. + +## Decision + +1. **System of record for business / accounting / customers / pricing** is **libSQL** (Turso’s SQLite fork; server mode or hosted). That includes: + - customers, segments, tiers, balances, Stripe / PO metadata + - pricing and channel×segment rate cards + - invoices and invoice lines + - metered usage ledger (`usage_entries` and its append-only hash chain) + - other accounting and CRM-style records for the Zapier/Verae edge stack + +2. **NATS JetStream** is used **as necessary** for multi-host **work distribution** (job queues, competing consumers, retries, durable event fans that are not the money ledger). Examples: `ZAPIER_JOBS`, worker pull/ack, bridge task streams. + +3. **Core NATS (ephemeral)** remains mandatory for Verae timestamp **job-complete / jobId** notifications (`verae.ts.completed`, `verae.ts.receipt`). Those must **not** move to JetStream. + +4. **Timestamping-svc Go** `blockchain.db` and other Go timestamping SQLite files stay on that service; this decision does not relocate them. + +## Relationship to earlier decisions + +- **D001** (single prepaid writer) still holds: one logical writer path for prepaid/credits. The durable store for that path becomes **libSQL**, not per-process `books.json` or a private `billing.db` on each machine. Edge `balanceCents` may remain a cache filled from the billing/libSQL service. +- Lab `billing.db` / `zappier.db` on 138 are the current single-host stand-in; migrate that schema into shared libSQL before splitting planes across hosts. + +## Non-goals + +- JetStream as the customer/balance/invoice/pricing database +- Litestream as a multi-writer fabric (Litestream remains backup/DR for a primary if used) +- Per-machine copies of customer or pricing tables + +## Implementation sketch + +- Keep repositories behind a small DB interface; swap better-sqlite3 / modernc file opens for libSQL client when the shared server is up. +- One billing writer path owns usage hash-chain appends (`BEGIN`-style sequencing). +- Gateway/bridge/delivery call billing APIs or the shared libSQL endpoint; they do not each open a local billing file for production. diff --git a/decisions/index.md b/decisions/index.md new file mode 100644 index 0000000..71f3d9b --- /dev/null +++ b/decisions/index.md @@ -0,0 +1,10 @@ +# Decisions + +6 files in this book. + +- [D001-prepaid-sot.md](D001-prepaid-sot.md) +- [D002-identity.md](D002-identity.md) +- [D003-public-doors.md](D003-public-doors.md) +- [D004-leaf-accounts.md](D004-leaf-accounts.md) +- [D005-fleet-local-first.md](D005-fleet-local-first.md) +- [D006-libsql-business-jetstream-work.md](D006-libsql-business-jetstream-work.md) diff --git a/jobs/pieces.yaml b/jobs/pieces.yaml new file mode 100644 index 0000000..2ace19a --- /dev/null +++ b/jobs/pieces.yaml @@ -0,0 +1,23 @@ +# Multi-build piece manifest for job-bus. +# Copy to jobs/pieces.yaml in your repo and edit. +# Each piece becomes one tracked job; kind routes cpu→Xeons, gpu→Mac. +batch: "zapier-decisions-EDITME" +defaults: + kind: cpu + requires: [] + timeout_sec: 3600 + cleanup: none # or on_success; full wipe via: job-cleanup --batch +pieces: + - id: piece-01 + kind: cpu + requires: [bin:bash] + cmd: ["make", "piece-01"] + # cwd: defaults to repo root on submitter; set worker-visible path if needed + - id: piece-02 + kind: cpu + requires: [ffmpeg, pip:numpy] + cmd: ["python3", "scripts/build_piece.py", "--piece", "2"] + - id: piece-gpu-hero + kind: gpu + requires: [pip:Pillow] + cmd: ["python3", "scripts/gpu_hero.py"] diff --git a/plans/2026-10-03-forgejo-sources.md b/plans/2026-10-03-forgejo-sources.md new file mode 100644 index 0000000..66e73a2 --- /dev/null +++ b/plans/2026-10-03-forgejo-sources.md @@ -0,0 +1,178 @@ +# Verae × Zapier Forgejo sources + +Research date: 2026-10-03 + +Sources: +- Forgejo API `https://git.georgelambert.org/api/v1/repos/search` (q=zapier,zappier,verae,nats,account-balance,access) plus direct `/repos/marchon/` +- Host `marchon@70.88.205.138`: catalog `/SSD2/sites/zapier.georgelambert.org/packages`, live `/opt/verae/{zappier-edge,verae-zapier-middleware}`, worktree `/home/marchon/zappier-edge-work` + +## Executive summary — what is SoT + +| Concern | Source of truth (design) | What is actually live on 138 today | +|---|---|---| +| **Billing / customers / pricing / invoices / usage ledger** | **`marchon/zappier-edge`** (+ D006: migrate durable store to **shared libSQL**). Department UIs are thin clients. | **`/opt/verae/zappier-edge`** with **`billing.db`** (ledger) + **`zappier.db`** (auth/sessions), WAL SQLite. Worktree `/home/marchon/zappier-edge-work` is ahead of Forgejo. Service `zappier-edge.service`. | +| **Prepaid balance book** | Design D001: **`marchon/zappier-account-balance`** persists **`books.json`**; edge `balanceCents` is a cache. D006: prepaid writer path stays single, store becomes libSQL. | No `books.json` found under `/opt/verae` or `/home/marchon`. Live prepaid appears folded into edge SQLite/`balanceCents` until account-balance/libSQL cutover. | +| **Public doors (access planes)** | **`verae-access-{zapier,web,api,leaf,staff}`** + **`verae-access-authz`** (D003). Edge stays loopback. | Catalog packages exist; **not** confirmed as separate live systemd units on this pass. Live public surface for Zapier metering is **`zap.verae.ai` → zappier-edge :3040** plus **`verae-zapier-middleware.service`**. | +| **NATS / JetStream fabric** | **`verae-nats-cluster`** (LXC 511–513), **`verae-nats-accounts`**, **`verae-nats-process`** template. JetStream streams designed: `ZAPIER_JOBS`, `ZAPIER_EVENTS`, `ZAPIER_WEBHOOKS`, `ZAPIER_USAGE`, `VERAE_ARCHIVE`. | Live: `nats-public-edge.service`, `verae-nats-private-proxy.service`, various leaf workers. **D006 / standing rule:** JetStream for **work queues only**; **core NATS** for timestamp **job-complete/jobId** (`verae.ts.completed`, `verae.ts.receipt`) — never JetStream for those. | +| **Jobs (Zapier timestamp path)** | **`verae-middleware`** owns `/zapier/v1`, job wait, NATS publish/consume; **`verae-jobs-events`** mailbox for `verae.zapier.jobs.events`; splitter/worm/tree for archive path. | **`/opt/verae/verae-zapier-middleware`** + `verae-zapier-middleware.service` active. Edge also does direct Verae/batch metering on `:3040`. | +| **Decisions / libSQL** | **`zapier-decisions`** D006 (2026-10-03): libSQL = business SoT; JetStream = multi-host work; core NATS = job-complete. | Decision file present in catalog `packages/zapier-decisions/decisions/D006-libsql-business-jetstream-work.md`. | + +**Important:** `/SSD2/sites/zapier.georgelambert.org/packages/*` are **catalog doc trees** (README/PDF), **not** git working copies (no `.git`). Live code under `/opt/verae/zappier-edge` is a **deployed tree without `.git`**. Clone from Forgejo SSH port **2223**. + +## Catalog package ↔ Forgejo repo + +| Catalog package | Forgejo `full_name` | Default branch | Forgejo updated | Size | Active? | One-line purpose | Clone (HTTPS) | Clone (SSH) | +|---|---|---|---|---|---|---|---|---| +| `zappier` | `marchon/zappier-edge` | main | 2026-09-23T14:00:59 | 187072 | LIVE on 138 (/opt/verae/zappier-edge); worktree Oct 2 ahead of Forgejo Sep 23 | Metered HTTPS edge: portal, admin, Stripe, rate card, usage ledger; proxies Verae to middleware | `https://git.georgelambert.org/marchon/zappier-edge.git` | `ssh://git@git.georgelambert.org:2223/marchon/zappier-edge.git` | +| `zappier-account-balance` | `marchon/zappier-account-balance` | main | 2026-09-23T14:00:59 | 48 | Catalog/Forgejo batch ~Sep 22–23; not confirmed as live systemd unit | Prepaid balance SoT via NATS (books.json design); edge balanceCents is cache | `https://git.georgelambert.org/marchon/zappier-account-balance.git` | `ssh://git@git.georgelambert.org:2223/marchon/zappier-account-balance.git` | +| `zappier-customer-service` | `marchon/zappier-customer-service` | main | 2026-09-23T14:00:59 | 68 | Catalog/Forgejo batch ~Sep 22–23; not confirmed as live systemd unit | CS goodwill credits / statement review over verae.billing.* | `https://git.georgelambert.org/marchon/zappier-customer-service.git` | `ssh://git@git.georgelambert.org:2223/marchon/zappier-customer-service.git` | +| `zappier-sales-pricing` | `marchon/zappier-sales-pricing` | main | 2026-09-23T14:00:59 | 59 | Catalog/Forgejo batch ~Sep 22–23; not confirmed as live systemd unit | Sales per-customer tier/multiplier; reviews via NATS statement | `https://git.georgelambert.org/marchon/zappier-sales-pricing.git` | `ssh://git@git.georgelambert.org:2223/marchon/zappier-sales-pricing.git` | +| `zappier-accounting-export` | `marchon/zappier-accounting-export` | main | 2026-09-23T14:00:59 | 55 | Catalog/Forgejo batch ~Sep 22–23; not confirmed as live systemd unit | QuickBooks IIF + accounting CSV from edge invoices + NATS statements | `https://git.georgelambert.org/marchon/zappier-accounting-export.git` | `ssh://git@git.georgelambert.org:2223/marchon/zappier-accounting-export.git` | +| `zappier-identity` | `marchon/zappier-identity` | main | 2026-09-23T14:00:59 | 33 | Catalog/Forgejo batch ~Sep 22–23; not confirmed as live systemd unit | Bind/lookup veraeUserId (not JWTs) | `https://git.georgelambert.org/marchon/zappier-identity.git` | `ssh://git@git.georgelambert.org:2223/marchon/zappier-identity.git` | +| `verae-zapier-middleware` | `marchon/verae-middleware` | main | 2026-09-22T06:36:26 | 145 | LIVE on 138 (verae-zapier-middleware.service) | Zapier HTTPS /zapier/v1/* + NATS workers; Zapier never speaks NATS | `https://git.georgelambert.org/marchon/verae-middleware.git` | `ssh://git@git.georgelambert.org:2223/marchon/verae-middleware.git` | +| `verae-zapier` | `marchon/verae-zapier-app` | main | 2026-09-22T06:17:43 | 102 | Catalog/Forgejo batch ~Sep 22–23; not confirmed as live systemd unit | Zapier Platform CLI app (timestamp/wait/batch/verify/hooks) | `https://git.georgelambert.org/marchon/verae-zapier-app.git` | `ssh://git@git.georgelambert.org:2223/marchon/verae-zapier-app.git` | +| `verae-zapier-simulator` | `marchon/verae-zapier-simulator` | main | 2026-09-23T14:00:58 | 73 | Catalog/Forgejo batch ~Sep 22–23; not confirmed as live systemd unit | In-process Zapier editor stand-in with hop trace console | `https://git.georgelambert.org/marchon/verae-zapier-simulator.git` | `ssh://git@git.georgelambert.org:2223/marchon/verae-zapier-simulator.git` | +| `verae-activate` | `marchon/verae-activate` | main | 2026-09-23T14:00:47 | 57 | Catalog/Forgejo batch ~Sep 22–23; not confirmed as live systemd unit | Activate-now Zapier app (Add Numbers, Echo, SHA256, mock stamp) | `https://git.georgelambert.org/marchon/verae-activate.git` | `ssh://git@git.georgelambert.org:2223/marchon/verae-activate.git` | +| `verae-access-zapier` | `marchon/verae-access-zapier` | main | 2026-09-23T14:00:57 | 33 | Catalog/Forgejo batch ~Sep 22–23; not confirmed as live systemd unit | Public Zapier HTTPS door (:3024) into middleware | `https://git.georgelambert.org/marchon/verae-access-zapier.git` | `ssh://git@git.georgelambert.org:2223/marchon/verae-access-zapier.git` | +| `verae-access-web` | `marchon/verae-access-web` | main | 2026-09-23T14:00:57 | 39 | Catalog/Forgejo batch ~Sep 22–23; not confirmed as live systemd unit | Public customer web/portal door (:3021) | `https://git.georgelambert.org/marchon/verae-access-web.git` | `ssh://git@git.georgelambert.org:2223/marchon/verae-access-web.git` | +| `verae-access-api` | `marchon/verae-access-api` | main | 2026-09-23T14:00:57 | 33 | Catalog/Forgejo batch ~Sep 22–23; not confirmed as live systemd unit | Public direct customer API door (:3022) | `https://git.georgelambert.org/marchon/verae-access-api.git` | `ssh://git@git.georgelambert.org:2223/marchon/verae-access-api.git` | +| `verae-access-leaf` | `marchon/verae-access-leaf` | main | 2026-09-23T14:00:57 | 37 | Catalog/Forgejo batch ~Sep 22–23; not confirmed as live systemd unit | S2S leaf plane (:3023); no billing subjects | `https://git.georgelambert.org/marchon/verae-access-leaf.git` | `ssh://git@git.georgelambert.org:2223/marchon/verae-access-leaf.git` | +| `verae-access-staff` | `marchon/verae-access-staff` | main | 2026-09-23T14:00:57 | 55 | Catalog/Forgejo batch ~Sep 22–23; not confirmed as live systemd unit | Staff CS/sales/accounting plane (:3025) | `https://git.georgelambert.org/marchon/verae-access-staff.git` | `ssh://git@git.georgelambert.org:2223/marchon/verae-access-staff.git` | +| `verae-access-authz` | `marchon/verae-access-authz` | main | 2026-09-23T14:00:57 | 41 | Catalog/Forgejo batch ~Sep 22–23; not confirmed as live systemd unit | NATS authz.check gate for all access planes | `https://git.georgelambert.org/marchon/verae-access-authz.git` | `ssh://git@git.georgelambert.org:2223/marchon/verae-access-authz.git` | +| `verae-jobs-events` | `marchon/verae-jobs-events` | main | 2026-09-23T14:00:58 | 33 | Catalog/Forgejo batch ~Sep 22–23; not confirmed as live systemd unit | Mailbox for verae.zapier.jobs.events (passthrough today) | `https://git.georgelambert.org/marchon/verae-jobs-events.git` | `ssh://git@git.georgelambert.org:2223/marchon/verae-jobs-events.git` | +| `verae-nats-accounts` | `marchon/verae-nats-accounts` | main | 2026-09-22T06:36:30 | 33 | Catalog/Forgejo batch ~Sep 22–23; not confirmed as live systemd unit | INTERNAL vs LEAF NATS account/subject policy | `https://git.georgelambert.org/marchon/verae-nats-accounts.git` | `ssh://git@git.georgelambert.org:2223/marchon/verae-nats-accounts.git` | +| `verae-nats-cluster` | `marchon/verae-nats-cluster` | main | 2026-09-22T06:16:14 | 3666 | Catalog/Forgejo batch ~Sep 22–23; not confirmed as live systemd unit | 3× Proxmox LXC JetStream cluster (10.10.10.21–23) | `https://git.georgelambert.org/marchon/verae-nats-cluster.git` | `ssh://git@git.georgelambert.org:2223/marchon/verae-nats-cluster.git` | +| `verae-nats-process` | `marchon/verae-nats-process` | main | 2026-09-23T14:00:58 | 40 | Catalog/Forgejo batch ~Sep 22–23; not confirmed as live systemd unit | Template for a new addressed NATS process | `https://git.georgelambert.org/marchon/verae-nats-process.git` | `ssh://git@git.georgelambert.org:2223/marchon/verae-nats-process.git` | +| `verae-request-splitter` | `marchon/verae-request-splitter` | main | 2026-09-23T14:00:58 | 42 | Catalog/Forgejo batch ~Sep 22–23; not confirmed as live systemd unit | Split chain hash vs off-chain archive.put | `https://git.georgelambert.org/marchon/verae-request-splitter.git` | `ssh://git@git.georgelambert.org:2223/marchon/verae-request-splitter.git` | +| `verae-archive-worm` | `marchon/verae-archive-worm` | main | 2026-09-23T14:00:58 | 43 | Catalog/Forgejo batch ~Sep 22–23; not confirmed as live systemd unit | Bloom-filtered WORM archive node | `https://git.georgelambert.org/marchon/verae-archive-worm.git` | `ssh://git@git.georgelambert.org:2223/marchon/verae-archive-worm.git` | +| `verae-archive-aggregator` | `marchon/verae-archive-aggregator` | main | 2026-09-23T14:00:58 | 44 | Catalog/Forgejo batch ~Sep 22–23; not confirmed as live systemd unit | Merge archive.query replies into job JSON | `https://git.georgelambert.org/marchon/verae-archive-aggregator.git` | `ssh://git@git.georgelambert.org:2223/marchon/verae-archive-aggregator.git` | +| `verae-tree-node` | `marchon/verae-tree-node` | main | 2026-09-23T14:00:58 | 42 | Catalog/Forgejo batch ~Sep 22–23; not confirmed as live systemd unit | Merkle leaf proofs for bulk summaries | `https://git.georgelambert.org/marchon/verae-tree-node.git` | `ssh://git@git.georgelambert.org:2223/marchon/verae-tree-node.git` | +| `verae-fleet` | `marchon/verae-fleet` | main | 2026-09-22T06:37:31 | 3896 | Catalog/Forgejo batch ~Sep 22–23; not confirmed as live systemd unit | Operator catalog, replica floors, pause/restart console | `https://git.georgelambert.org/marchon/verae-fleet.git` | `ssh://git@git.georgelambert.org:2223/marchon/verae-fleet.git` | +| `verae-keep` | `marchon/verae-keep` | main | 2026-09-22T06:35:54 | 43 | Catalog/Forgejo batch ~Sep 22–23; not confirmed as live systemd unit | Host keep-alive; restarts unless operator paused/stopped | `https://git.georgelambert.org/marchon/verae-keep.git` | `ssh://git@git.georgelambert.org:2223/marchon/verae-keep.git` | +| `verae-bootstrap` | `marchon/verae-bootstrap` | main | 2026-09-22T06:17:17 | 54 | Catalog/Forgejo batch ~Sep 22–23; not confirmed as live systemd unit | Clone all module repos + host-deps per server type | `https://git.georgelambert.org/marchon/verae-bootstrap.git` | `ssh://git@git.georgelambert.org:2223/marchon/verae-bootstrap.git` | +| `verae-ops` | `marchon/verae-ops` | main | 2026-09-11T19:00:06 | 86 | Older Forgejo (Sep 11); still the install book | Docker/Proxmox/VM/hardware install and service linking | `https://git.georgelambert.org/marchon/verae-ops.git` | `ssh://git@git.georgelambert.org:2223/marchon/verae-ops.git` | +| `verae-deploy` | `marchon/verae-deploy` | main | 2026-09-11T18:36:11 | 25 | Older Forgejo (Sep 11) | Tagged checkout + npm ci on target OS | `https://git.georgelambert.org/marchon/verae-deploy.git` | `ssh://git@git.georgelambert.org:2223/marchon/verae-deploy.git` | +| `verae-proxmox` | `marchon/verae-proxmox` | main | 2026-09-22T06:36:28 | 28 | Catalog/Forgejo batch ~Sep 22–23; not confirmed as live systemd unit | Provision archive-worker LXC on NS1 | `https://git.georgelambert.org/marchon/verae-proxmox.git` | `ssh://git@git.georgelambert.org:2223/marchon/verae-proxmox.git` | +| `verae-backup` | `marchon/verae-backup` | main | 2026-09-22T06:31:38 | 28 | Catalog/Forgejo batch ~Sep 22–23; not confirmed as live systemd unit | Backup/restore SQLite, JetStream, worm/tree, IAM | `https://git.georgelambert.org/marchon/verae-backup.git` | `ssh://git@git.georgelambert.org:2223/marchon/verae-backup.git` | +| `verae-uptime` | `marchon/verae-uptime` | main | 2026-09-22T06:36:26 | 28 | Catalog/Forgejo batch ~Sep 22–23; not confirmed as live systemd unit | Off-box HTTPS watch of public doors | `https://git.georgelambert.org/marchon/verae-uptime.git` | `ssh://git@git.georgelambert.org:2223/marchon/verae-uptime.git` | +| `verae-staff-iam` | `marchon/verae-staff-iam` | main | 2026-09-23T14:00:58 | 49 | Catalog/Forgejo batch ~Sep 22–23; not confirmed as live systemd unit | Internal staff users/roles/permissions SoT | `https://git.georgelambert.org/marchon/verae-staff-iam.git` | `ssh://git@git.georgelambert.org:2223/marchon/verae-staff-iam.git` | +| `verae-staff-session` | `marchon/verae-staff-session` | main | 2026-09-23T14:00:58 | 37 | Catalog/Forgejo batch ~Sep 22–23; not confirmed as live systemd unit | Shared staff cookie login | `https://git.georgelambert.org/marchon/verae-staff-session.git` | `ssh://git@git.georgelambert.org:2223/marchon/verae-staff-session.git` | +| `verae-staff-ui` | `marchon/verae-staff-ui` | main | 2026-09-23T14:00:58 | 32 | Catalog/Forgejo batch ~Sep 22–23; not confirmed as live systemd unit | Shared staff review HTML template | `https://git.georgelambert.org/marchon/verae-staff-ui.git` | `ssh://git@git.georgelambert.org:2223/marchon/verae-staff-ui.git` | +| `overview` | `marchon/overview` | main | 2026-09-22T06:41:52 | 1567 | Catalog/Forgejo batch ~Sep 22–23; not confirmed as live systemd unit | High-level system map and NATS/uptime docs | `https://git.georgelambert.org/marchon/overview.git` | `ssh://git@git.georgelambert.org:2223/marchon/overview.git` | +| `docs-master` | `marchon/zapier-docs-master` | main | 2026-09-22T06:32:02 | 197 | Catalog/Forgejo batch ~Sep 22–23; not confirmed as live systemd unit | Per-module SUMMARY/NATS contracts and message flows | `https://git.georgelambert.org/marchon/zapier-docs-master.git` | `ssh://git@git.georgelambert.org:2223/marchon/zapier-docs-master.git` | +| `dev-overview` | `marchon/dev-overview` | main | 2026-09-22T06:17:14 | 327 | Catalog/Forgejo batch ~Sep 22–23; not confirmed as live systemd unit | Designed vs running NATS routes; Zapier→Verae path | `https://git.georgelambert.org/marchon/dev-overview.git` | `ssh://git@git.georgelambert.org:2223/marchon/dev-overview.git` | +| `zapier-decisions` | `marchon/zapier-decisions` | main | 2026-09-22T06:44:58 | 1920 | Active decision log; D006 dated 2026-10-03 in catalog copy | Architecture decisions (D001–D006), logs, benches | `https://git.georgelambert.org/marchon/zapier-decisions.git` | `ssh://git@git.georgelambert.org:2223/marchon/zapier-decisions.git` | +| `zapier-user-docs` | `marchon/zapier-user-docs` | main | 2026-09-23T14:00:59 | 52 | Catalog/Forgejo batch ~Sep 22–23; not confirmed as live systemd unit | Customer signup → register → chain/tree lookup | `https://git.georgelambert.org/marchon/zapier-user-docs.git` | `ssh://git@git.georgelambert.org:2223/marchon/zapier-user-docs.git` | +| `ui-docs` | `marchon/UI-Docs` | main | 2026-09-23T14:00:57 | 21512 | Catalog/Forgejo batch ~Sep 22–23; not confirmed as live systemd unit | Operator/staff/portal walkthrough screenshots + review PDF | `https://git.georgelambert.org/marchon/UI-Docs.git` | `ssh://git@git.georgelambert.org:2223/marchon/UI-Docs.git` | + +## Related Forgejo repos (search hits, not catalog packages) + +| full_name | updated | purpose | clone | +|---|---|---|---| +| `marchon/master-zapier-plan-draft` | 2026-09-20T19:06:04 | Monorepo plan/snapshot workspace; not the only clone path | `https://git.georgelambert.org/marchon/master-zapier-plan-draft.git` | +| `marchon/nats-service-endpoints` | 2026-09-23 | Canonical NATS endpoint catalog | `https://git.georgelambert.org/marchon/nats-service-endpoints.git` | +| `marchon/verae-mcp-timestamping-server` | 2026-09-29 | MCP timestamping (live `/opt/verae/...`) | `https://git.georgelambert.org/marchon/verae-mcp-timestamping-server.git` | +| `marchon/verae-mcp-timestamping-server-go` | 2026-09-29 | Go MCP + tsnats sibling | `https://git.georgelambert.org/marchon/verae-mcp-timestamping-server-go.git` | + +## Notes: billing, libSQL, sqlite, JetStream, books.json, account-balance + +- **billing:** Live ledger/rate card/customers/invoices in **zappier-edge** `billing.db` (`BILLING_DB`); auth in `zappier.db`. Docs in edge `docs/BILLING-DB.md`, `CHANNEL-BILLING.md`, `PEERGOS-BILLING.md`. Department repos (CS/sales/accounting-export) call NATS + edge admin; they are not the money DB. +- **sqlite:** Lab/single-host stand-in. Edge uses better-sqlite3 + WAL. Go timestamping-svc DBs stay separate (`blockchain.db`) — do not relocate (D006). +- **libSQL:** Accepted in **D006** as shared SoT for business data across 3–5 machines. Not yet the live store on 138. +- **JetStream:** Designed streams for jobs/events/webhooks/usage/archive. Use for **work distribution** only. **Forbidden** for timestamp job-complete/jobId notify (core NATS subjects). +- **books.json:** Documented as persistence for **zappier-account-balance**. Not found on 138 in this survey; design SoT for prepaid until libSQL migration absorbs it (D001∩D006). +- **account-balance:** Forgejo `marchon/zappier-account-balance`; subjects `verae.billing.statement.get|balance.get|balance.adjust` + usage/payment/credit pubs; queue group `account-balance`; HTTP `:3010`. + +## Host inventory (138) + +### Catalog packages (`/SSD2/sites/zapier.georgelambert.org/packages`) +Doc snapshots dated ~2026-09-14. Names match table above. No `.git` remotes. + +### Live `/opt/verae` Zapier-related +- `/opt/verae/zappier-edge` — `zappier-edge.service` **active**; `billing.db`/`zappier.db` present; **no `.git`** +- `/opt/verae/verae-zapier-middleware` — `verae-zapier-middleware.service` **active**; README points at Forgejo `verae-middleware` +- Work copy: `/home/marchon/zappier-edge-work` (mtime Oct 2) — treat as practical edge SoT until Forgejo catch-up + +## Recommended clone list for implementation + +SSH base: `ssh://git@git.georgelambert.org:2223/marchon/.git` (port **2223**). + +### Must clone first (critical path) +```bash +mkdir -p ~/verae-src && cd ~/verae-src +git clone ssh://git@git.georgelambert.org:2223/marchon/zappier-edge.git +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-middleware.git +git clone ssh://git@git.georgelambert.org:2223/marchon/zappier-account-balance.git +git clone ssh://git@git.georgelambert.org:2223/marchon/zapier-decisions.git +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-access-zapier.git +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-access-web.git +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-access-api.git +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-access-authz.git +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-nats-cluster.git +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-nats-accounts.git +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-jobs-events.git +git clone ssh://git@git.georgelambert.org:2223/marchon/overview.git +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-bootstrap.git +``` + +### Next (jobs / archive path + Zapier app) +```bash +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-zapier-app.git +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-request-splitter.git +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-archive-worm.git +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-archive-aggregator.git +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-tree-node.git +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-access-leaf.git +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-access-staff.git +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-zapier-simulator.git +``` + +### Billing department satellites +```bash +git clone ssh://git@git.georgelambert.org:2223/marchon/zappier-customer-service.git +git clone ssh://git@git.georgelambert.org:2223/marchon/zappier-sales-pricing.git +git clone ssh://git@git.georgelambert.org:2223/marchon/zappier-accounting-export.git +git clone ssh://git@git.georgelambert.org:2223/marchon/zappier-identity.git +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-staff-iam.git +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-staff-session.git +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-staff-ui.git +``` + +### Ops / docs (as needed) +```bash +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-ops.git +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-fleet.git +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-keep.git +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-deploy.git +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-backup.git +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-uptime.git +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-proxmox.git +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-nats-process.git +git clone ssh://git@git.georgelambert.org:2223/marchon/verae-activate.git +git clone ssh://git@git.georgelambert.org:2223/marchon/zapier-docs-master.git +git clone ssh://git@git.georgelambert.org:2223/marchon/zapier-user-docs.git +git clone ssh://git@git.georgelambert.org:2223/marchon/UI-Docs.git +git clone ssh://git@git.georgelambert.org:2223/marchon/dev-overview.git +git clone ssh://git@git.georgelambert.org:2223/marchon/master-zapier-plan-draft.git +``` + +Or use bootstrap: clone `verae-bootstrap` then `bash scripts/fetch.sh ns1-all-in-one` (full `types/` may be incomplete in the **catalog** copy; prefer Forgejo clone of bootstrap). + +## Search hit counts (API) + +| q | results (limit 50) | +|---|---| +| zapier | 7 | +| zappier | 6 | +| verae | 32 | +| nats | 4 | +| account-balance | 1 | +| access | 7 (includes unrelated MCP mirror) | + +## Caveats + +1. Forgejo `updated_at` for most module repos clusters on **2026-09-22/23** (likely bulk push). **Live edge code on 138 is newer (Oct 2)** than Forgejo edge — sync/push before treating Forgejo as current for billing. +2. Catalog package `verae-fleet` README incorrectly linked `verae-keep` in one scrape; overview table maps fleet→`verae-fleet` correctly. +3. Catalog `docs-master` README first Forgejo link hit `verae-bootstrap` via grep noise; real docs repo is **`zapier-docs-master`**. +4. `master-zapier-plan-draft` is a large monorepo snapshot — useful archaeology, not the independent-module clone path. + diff --git a/plans/2026-10-03-libsql-multihost-architecture.md b/plans/2026-10-03-libsql-multihost-architecture.md new file mode 100644 index 0000000..861a07b --- /dev/null +++ b/plans/2026-10-03-libsql-multihost-architecture.md @@ -0,0 +1,295 @@ +# Verae × Zapier Multi-Host Architecture & Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (or executing-plans) task-by-task. Steps use checkbox (`- [ ]`) syntax. + +**Goal:** Run the Verae × Zapier commercial stack across **3–5 machines** with **libSQL** as the system of record for all business/billing/customer/pricing/accounting data, **JetStream only for work queues**, and proven end-to-end userflows under continuous QA. + +**Architecture:** Public HTTPS doors → zappier-edge (meter) → middleware (stamp) → `api.veraetime.net`; private NATS for workers; shared libSQL for money & CRM; Go timestamping DBs stay local to timestamping-svc. + +**Tech Stack:** Node (edge, middleware, access planes, CS/sales/acct), NATS JetStream (3-node lab → production), libSQL / go-libsql or `@libsql/client`, better-sqlite3 only as local/dev adapter, Litestream optional for libSQL file primary backup, Jest + gate scripts + E2E userflow harness. + +**Spec / SoT docs:** +- Catalog: https://zapier.georgelambert.org/ +- Map: `/tmp/zapier-system-docs-map.md` +- Decisions: D001–D006 (`packages/zapier-decisions/decisions/`) +- Composition: `docs/02-architecture/composition.md` +- NATS: `docs/02-architecture/nats-subjects.md`, `overview/03-nats-cluster.md` +- Ops: `packages/verae-ops/GETTING-STARTED.md` +- Digests-only stamps: mcp.verae.ai + D006 note + live swagger + +**Forgejo org:** https://git.georgelambert.org/marchon/ (modules listed in §2) + +## Global Constraints + +- **D006:** Business/billing/customers/pricing/accounting/usage ledger → **libSQL only** (shared). +- **JetStream:** work queues as necessary (`ZAPIER_JOBS`, `ZAPIER_WEBHOOKS`, archive put/query). Not money SoT. +- **Core NATS only** for `verae.ts.completed` / `verae.ts.receipt` (no JetStream on that path). +- **D001 refined:** single prepaid **writer path**; durable store is libSQL (not per-host `books.json` / `billing.db`). +- **D002:** public credential = zappier `x-api-key`; `veraeUserId` join; JWT = `tokenRef` server-side. +- **D003:** only access planes are public HTTP; edge loopback (except intentional public `zap.verae.ai` edge). +- **D004:** leaf must not touch billing. +- **Digests only** for stamp submit (`POST /api/batch/timestamp` multihash; MCP shape). +- **Never** modify Go timestamping-svc SQLite (`blockchain.db`, etc.) as part of this plan. +- **WAL** (or libSQL equivalent durability) on all SQL stores. +- Credits language in product copy; ledger may keep `*_cents` column names as credit units. +- Deploy never rsyncs live `*.db*` over production data. +- **ZAPIER_USAGE** / `verae.zapier.usage` is telemetry or export only — never usage/money SoT. +- Litestream (if used) backs up a **single** libSQL primary; never multi-writer fabric. +- **Single meter/prepaid writer** process owns `usage_entries` append + balance adjust; other hosts call that API. +- Dual-run: libSQL primary for writes; file/SQLite fallback is **read-only** (or kill-switched). +- Auth (`admin_users`, `portal_sessions`): v1 remains edge-local `zappier.db`; not in `verae_business`. +- Product edge bind on 138: `127.0.0.1:3040` (`zap.verae.ai`); `ZAPPIER_UPSTREAM` defaults to `http://127.0.0.1:3130` for live lab smoke (not `:3100` mock unless intentional). +- Catalog docs that call account-balance “prepaid SoT” are stale after D006; Phase 0 patches them. + +--- + +## 1. System map (from catalog + live lab) + +```text +Internet + ├─ Zapier cloud ──HTTPS──► access-zapier :3024 ──► zappier-edge (meter) ──► middleware :3100 + ├─ Browsers ────HTTPS──► access-web :3021 /portal + ├─ Customer API ─HTTPS──► access-api :3022 + ├─ Leaf S2S ────HTTPS──► access-leaf :3023 (no billing) + └─ Staff ───────HTTPS──► access-staff :3025 + IAM :3028 + +Private + ├─ NATS JetStream 10.10.10.21–23 (jobs.watch, webhooks.deliver, archive.*) + ├─ Core NATS (verae.ts.completed, verae.ts.receipt, billing request-reply) + ├─ libSQL (TARGET) ← customers, pricing, usage chain, invoices, balances + ├─ Workers job-poller, webhook-deliver, archive-worm×N, tree-node×N + └─ api.veraetime.net / MOCK_VERAE +``` + +### Critical userflows (must pass QA) + +1. **Signup → API key → Statement** (portal via access-web). +2. **Stamp digests (single + batch)** via zap door → edge meter → middleware → Verae → 202 jobIds → credits headers. +3. **Async complete → Zapier REST Hook** (`jobs.watch` → poll → `jobs.events` → `webhooks.deliver`). +4. **CS credit adjust → balance visible** on portal/admin (single prepaid writer). +5. **Sales channel×segment quote → meter charge** matches rate card. +6. **Accounting export** sees payments/usage consistent with libSQL. +7. **Leaf plane** can stamp/authz but **cannot** adjust billing (D004). + +--- + +## 2. Forgejo source elements (implementation clones) + +| Concern | Repo | Notes | +|---------|------|-------| +| Billing edge | `marchon/zappier-edge` | Live `/opt/verae/zappier-edge` on 138; also catalog `packages/zappier` | +| Prepaid mailbox (legacy) | `marchon/zappier-account-balance` | `books.json` → migrate into libSQL | +| CS / Sales / Acct | `zappier-customer-service`, `zappier-sales-pricing`, `zappier-accounting-export` | | +| Identity | `zappier-identity` | | +| Middleware | `marchon/verae-middleware` (catalog: verae-zapier-middleware) | | +| Zapier apps | `verae-zapier-app`, `verae-activate` | | +| Access planes | `verae-access-{web,api,leaf,zapier,staff,authz}` | | +| Jobs/archive | `verae-jobs-events`, `verae-archive-*`, `verae-tree-node`, `verae-request-splitter` | | +| Ops | `verae-ops`, `verae-bootstrap`, `verae-fleet`, `verae-keep`, `verae-nats-cluster` | | +| Decisions | `zapier-decisions` | D001–D006 | +| Monorepo mirror | `master-zapier-plan-draft` | docs + packages composition | + +--- + +## 3. Conflict register (resolve before/with build) + +| ID | Conflict | Resolution (locked) | +|----|----------|---------------------| +| C1 | D001 `books.json` SoT vs live `billing.db` vs D006 libSQL | **libSQL is SoT**; migrate both books + billing.db schema; account-balance becomes libSQL-backed service or folds into billing API | +| C2 | README “imported, not yet wired” vs STATUS lab up | Treat STATUS + live doors as operational truth; update README composition status in Phase 0 docs | +| C3 | overview `zmw_` Bearer vs composition `x-api-key` | **Public = x-api-key** (D002/composition); fix overview/models | +| C4 | Edge ports `:3000` / `:13000` / live `:3040` (`zap.verae.ai`) | Document env matrix; lab doors proxy to loopback; zap.verae.ai may remain public edge | +| C5 | JetStream `jobs.events` vs ban on JetStream for job-complete | **Different subjects:** Zapier job lifecycle may use JetStream `verae.zapier.jobs.events`; Verae chain notify `verae.ts.*` stays **core NATS** | +| C6 | TimestampRequest “Hello world” data vs digests-only | Digests-only + multihash batch (already fixed on live swagger; sync catalog models) | +| C7 | Dual prepaid writers (edge SQLite + account-balance) | One writer API on libSQL; edge caches only | +| C8 | Per-host SQLite on 3–5 machines | Forbidden for business data; libSQL shared | +| C9 | Go `blockchain.db` in scope creep | Out of scope — do not touch | +| C10 | `ZAPIER_USAGE` vs libSQL usage ledger | Demote to telemetry/export only; never money SoT | +| C11 | Dual middleware `:3100` mock vs `:3130` product upstream | Env matrix; product smoke uses **3130** | +| C12 | D001 file still says books.json SoT | D001 addendum → libSQL (done with this amend) | +| C13 | Auth store edge-local vs libSQL | **v1 edge-local** `zappier.db` | +| C14 | Concurrent hash-chain append across hosts | Single meter writer / serialized append | +| C15 | GETTING-STARTED calls account-balance SoT | Phase 0 doc patch → libSQL | +| C16 | Live `:3010` + `billing.db` dual writers | Cutover runbook: one authoritative adjust path | +| C17 | Litestream multi-writer temptation | Litestream = backup of one primary only | +| C18 | Digests-only vs upload-file stamp docs | Client hasher + digests-only only | + +--- + +## 4. Target data ownership (libSQL) + +**Database:** `verae_business` (name TBD) on shared libSQL. + +| Domain | Tables / collections | Writer | +|--------|----------------------|--------| +| Customers / identity bind | `customers`, identity map | billing API + identity service | +| Pricing | `tiers`, `price_endpoints`, `channel_rates`, `channel_rate_meta` | admin / sales-pricing | +| Metered ledger | `usage_entries` (+ `prev_hash`/`row_hash` chain) | **single** meter writer | +| Prepaid | balances, credits, payments (from books.json) | billing adjust API | +| Invoices | `invoices`, `invoice_lines` | admin / Stripe jobs | +| Auth (optional split) | `admin_users`, `portal_sessions` | may stay edge-local or move to libSQL `auth` schema | + + +### 4.1 Launch topology (3–5 machines) + +- One libSQL primary (dedicated host or named primary among the 3–5). +- Replicas if any are read-only; writers always target primary URL (`LIBSQL_URL`). +- Backup: filesystem snapshot and/or Litestream of the **primary only**; tested restore before Phase 4. +- Fail closed: metered stamp paths return **503** if BusinessStore write unavailable (no silent local SQLite fallback in production). +- Network: libSQL on private fabric only (same class as NATS `10.10.10.0/24`); not public. +- Schema: `verae_business`; auth stays out for v1 (C13). + +**Interface:** `BusinessStore` (TS) with drivers: `LibsqlBusinessStore` (prod), `SqliteFileBusinessStore` (dev/test). + +--- + +## 5. NATS usage matrix + +| Subject / stream | Transport | Purpose | +|------------------|-----------|---------| +| `verae.zapier.jobs.watch` | JetStream | Poll Verae status | +| `verae.zapier.jobs.events` | JetStream | Terminal Zapier-pipeline job events | +| `verae.zapier.usage` / `ZAPIER_USAGE` | JetStream optional | **Telemetry/export only; NOT ledger SoT** | +| `verae.zapier.webhooks.deliver` | JetStream | Hook delivery retries | +| `verae.archive.*` | JetStream / fanout | WORM/tree | +| `verae.billing.*` | Core request-reply (+ optional pub) | Statement/balance; eventually thin facade over libSQL | +| `verae.ts.completed`, `verae.ts.receipt` | **Core NATS only** | Chain job-complete notify | +| `verae.access.*`, `verae.identity.*` | Core / as designed | Authz & bind | + +--- + +## 6. Constraint (TOC) + +> The system constraint is **shared business SoT cutover (libSQL + single prepaid writer)** because every multi-host userflow (meter, CS credit, sales quote, portal statement, accounting export) blocks on one consistent customer/balance/usage store; JetStream and doors already exist in the lab. + +Critical chain: **Conflict freeze → BusinessStore+libSQL → Migrate data → Rewire edge/CS/sales/acct → E2E userflow tests → QA whattochange loop**. + +Feeding chains (parallel): digests-only docs sync; NATS subject clarification; access-plane smoke; Forgejo clone inventory. + +--- + +## 7. Phased implementation (PM DAG) + +### Phase 0 — Docs & conflict freeze +- [ ] Publish this plan to catalog `docs/plans/` + `zapier-decisions/plans/` +- [ ] Patch conflict C2/C3/C5/C6 in overview + models + README status blurb +- [ ] Architecture review sign-off artifact: `plans/reviews/2026-10-03-arch-review.md` + +### Phase 1 — BusinessStore + libSQL skeleton (TDD) +- [ ] Define `BusinessStore` interface (customers, usage.record+verifyChain, rates, invoices, balances) +- [ ] Failing tests against interface +- [ ] `LibsqlBusinessStore` + docker/libsql-server or `:memory:`/file for CI +- [ ] Keep `SqliteFileBusinessStore` wrapping current repos for lab + +### Phase 2 — Migrate lab data +- [ ] One-shot migrator: `billing.db` + `books.json` → libSQL +- [ ] Preserve usage hash-chain tip integrity (verify before/after); refuse cutover if verifyChain fails +- [ ] Dual-run: writes libSQL only when `BUSINESS_STORE=libsql`; file fallback **read-only** +- [ ] Account-balance becomes libSQL-backed facade for `verae.billing.*` **or** is stopped after HTTP BusinessStore cutover (pick one in Phase 3) + +### Phase 3 — Rewire writers +- [ ] zappier-edge meter → BusinessStore only +- [ ] account-balance service reads/writes libSQL (or deprecate mailbox in favor of HTTP billing API) +- [ ] CS / sales / accounting-export point at same store +- [ ] Remove second writer paths (C7) + +### Phase 4 — Multi-host topology +- [ ] Deploy libSQL on dedicated host (or primary of 3–5) +- [ ] Edge/middleware/CS on separate hosts with `LIBSQL_URL` +- [ ] JetStream cluster already present — verify workers only +- [ ] Confirm leaf cannot call billing adjust + +### Phase 5 — Userflow E2E + gates +- [ ] Automate flows 1–7 (§1) in `harness/` or `packages/verae-zapier-simulator` extensions +- [ ] Jest unit + integration + gate scripts green +- [ ] Live smoke on 138 with MOCK_VERAE then optional live Verae + +### Phase 6 — QA loop (repeat until pass) +- [ ] QA finds bugs → `/whattochange` pack (a/b/c/d + todos) +- [ ] Engineers implement; PM reorders PreReqs +- [ ] Re-run full suite + userflows +- [ ] Exit when all critical userflows pass twice consecutively + +--- + +## 8. Test requirements (“tests for everything”) + +| Layer | Must cover | +|-------|------------| +| Unit | BusinessStore ops, hash-chain append/tamper, channel rate resolve, digests-only batch mapping, D004 leaf deny | +| Integration | edge→middleware mock stamp; billing adjust→statement; JetStream watch→event (testcontainers/nats) | +| Contract | OpenAPI digests-only examples; NATS payload schemas | +| E2E userflow | Flows 1–7 against lab compose or NS1 | +| Regression | Existing zappier-edge Jest suite must stay green during driver swap | +| Non-goals | Go timestamping-svc blockchain tests | + +--- + +## 9. Roles (agent orchestration) + +| Role | Responsibility | Artifact | +|------|----------------|----------| +| **Architect** | Review plan, expand conflict register, approve SoT | `plans/reviews/*-arch-review.md` | +| **PM** | DAG, task assignment, merge gates, buffer | `plans/reviews/*-pm-board.md` | +| **Engineers** | Implement Phases 1–5 in worktrees | PRs / commits citing this plan | +| **QA** | Break userflows; whattochange packs; retest | `plans/qa/*` + NotImplemented leftovers | +| **Orchestrator (this session)** | Launch agents, merge gates, enforce D006/JetStream rules | STATUS updates | + +--- + +## 10. Acceptance criteria (project buffer) + +1. One libSQL URL serves customers/pricing/usage/invoices/balances for all commercial services. +2. No production business writes to per-host `billing.db` / `books.json`. +3. JetStream used only for work/archive streams listed in §5. +4. `verae.ts.completed` / `verae.ts.receipt` remain core NATS. +5. Digests-only multihash batch path green with credit headers. +6. Userflows 1–7 pass twice in a row under QA. +7. Go timestamping DBs untouched. +8. Catalog docs match runtime (conflicts C2–C6, C10–C18 closed or owned). +9. `ZAPIER_USAGE` is non-SoT; no service writes authoritative balances there. +10. Production metered paths fail closed if BusinessStore writer is down. +11. Concurrent append contract test proves single-writer (or serialized) hash-chain safety. +12. Live env matrix documents `:3040` edge, `:3130` product upstream, `:3100` mock, `:3010` facade/retire. +13. D001 file contains D006 addendum; GETTING-STARTED / 01-system no longer call books.json SoT. + +--- + +## 11. Out of scope + +- Replacing api.veraetime.net / Go chain storage +- Zapier Platform public directory listing (private push may remain operator-gated) +- Peergos Seaweed fleet redesign (consume existing billing APIs only) +- lan-134 enablement (D005) +- Any JetStream binding for `verae.ts.completed` / `verae.ts.receipt` +- Editing `/var/lib/timestamping-svc/data/blockchain.db` or timestamping-svc test `results.db` +- Multi-primary libSQL or Litestream multi-writer +- Moving portal auth to libSQL in the same cutover as billing (defer) + +### Architecture review +See `plans/reviews/2026-10-03-arch-review.md` (**approve-with-changes**; A1–A8 merged herein). + +--- + +## Task checklist (engineer entry) + +### Task 1: BusinessStore interface + failing tests +**Files:** Create under zappier-edge `src/db/business-store.ts`, `tests/business-store.contract.test.ts` +**Produces:** Interface used by later tasks +- [ ] Write contract tests for recordUsage/verifyChain/findCustomer/getBalance/adjustBalance +- [ ] Run tests — expect fail +- [ ] Minimal SqliteFile adapter pass +- [ ] Commit citing this plan + +### Task 2: LibsqlBusinessStore +- [ ] Add `@libsql/client` +- [ ] Implement same contract +- [ ] CI uses file or embedded libsql +- [ ] Commit + +### Task 3: Migrator billing.db + books.json → libSQL +- [ ] Script + tests with fixtures +- [ ] Chain verify pre/post +- [ ] Commit + +*(PM expands Tasks 4+ after arch review.)* diff --git a/plans/2026-10-03-system-docs-map.md b/plans/2026-10-03-system-docs-map.md new file mode 100644 index 0000000..1b26567 --- /dev/null +++ b/plans/2026-10-03-system-docs-map.md @@ -0,0 +1,383 @@ +# Verae × Zapier System Documentation Map + +**Source root:** `/SSD2/sites/zapier.georgelambert.org` on `70.88.205.138` (NS1.GEORGELAMBERT.ORG) +**Public catalog:** https://zapier.georgelambert.org/ +**Mapped:** 2026-10-03 +**Scope:** Documentation tree only (not live `/opt/verae/zappier-edge` runtime audit) + +--- + +## 1. Docs structure + +### Site layout + +| Path | Role | +|------|------| +| `/SSD2/sites/zapier.georgelambert.org/` | Published catalog (Caddy) | +| `README.md` | Monorepo entry; composition of zappier + middleware + Zapier app | +| `TODO.md` | Middleware phase gates 0–15 | +| `docs/` | Numbered product/architecture books + modules/models | +| `docs-master/` | Per-module SUMMARY + NATS contracts + MESSAGE-FLOWS | +| `overview/` | System overview book (01–11) + diagrams | +| `packages/` | ~41 independent module packages (README + PDF each) | +| `live/` | Screenshots of public doors | +| `user-docs/` | Customer-facing guides | +| `simulator/` | Static simulator UI copy | +| `sphinx/` | Sphinx HTML build of modules/models | +| `research/` | Vendor Zapier research | + +### `docs/` tree + +``` +docs/ + 00-sources/ workspace-brief, provenance, getting-started-research, + FUNCTIONS/PLATFORM/MCP refs, veraetime-openapi.yaml, LOGIN/MONGO/RESTART + 01-product/ features-a-m, api-gap-analysis, billing-and-keys, zapier-billing-research + 02-architecture/ + overview, composition, modules-and-nats, nats-subjects, nats-gateway, + archive-nats, fleet, tree-nodes + 03-zapier/ auth, operations, README + 04-activate/ SETUP-ZAPIER-DEVELOPER + api/ middleware-openapi.yaml + developer/ debugging + modules/ + models/ data model sheets (+ models-pdf/) + modules/ per-source-module API sheets (+ modules-pdf/) + plans/ phase-gates.md + sphinx/ RST/MD sources for Sphinx + OPEN.md, WORK-LOG.md +``` + +### `docs-master/` + +- `README.md`, `composition.md`, `modules-and-nats.md`, `archive-nats.md` +- `MESSAGE-FLOWS.md`, `HOST-DEPS.md`, `CATALOG-HOST.md` +- `modules//{SUMMARY,NATS}.md` for each package + +### `overview/` + +| Doc | Topic | +|-----|--------| +| `01-system.md` | Planes, hard rule (Zapier never NATS) | +| `02-modules-and-repos.md` | Forgejo repos | +| `03-nats-cluster.md` | 3-server JetStream | +| `04-uptime.md` | Replica floors | +| `05-network-failures.md` | Reconnect / silence | +| `06-address-routing.md` | New functions = new subjects | +| `07-external-resources.md` | Chain / storage | +| `08-diagrams.md` | SVG + mermaid | +| `09-expansion-template.md` | verae-nats-process | +| `10-docs-sync.md` | Catalog sync | +| `11-hostnames.md` | Ports, DNS, public doors | +| `NATS.md`, `SUMMARY.md`, `INDEX.md` | Shortcuts | + +### `plans/` + +- Single plan book: `docs/plans/phase-gates.md` (gates 0–15; mirrors root `TODO.md`) + +--- + +## 2. Key document summaries + +### Architecture / overview + +**`docs/02-architecture/overview.md`** +- Connect Zapier to Verae without exposing JWTs, forcing Zapier to poll, or coupling billing to the chain API. +- Components: `verae-zapier` (Zapier cloud), middleware HTTPS, NATS JetStream, workers, `api.veraetime.net`. +- Flows: async create → `jobs.watch` → poll → `jobs.events` → webhook; wait path; auth `/auth/me`. +- Flags: `NATS_ENABLED`, `MOCK_VERAE`, `DEBUG_VERAE`. +- Still describes public auth as middleware `zmw_` Bearer (pre-composition framing). + +**`docs/02-architecture/composition.md`** +- Target path: Zapier → zappier `/v1/*` (`x-api-key`) → middleware `/zapier/v1/*`. +- Ownership split: billing/portal = zappier; stamp/NATS/hooks = middleware; nouns = verae-zapier. +- Zapier never talks to NATS or live Verae; payment invoices ≠ timestamp receipts. + +**`docs/02-architecture/modules-and-nats.md`** +- Master module table (edge, middleware, poller, webhook, splitter, aggregator, WORM, tree-node, fleet, simulator). +- Subjects: `verae.zapier.jobs.{watch,events}`, `webhooks.deliver`, `verae.archive.{put,query,reply.*}`. +- Zapier cloud never on NATS. + +**`docs/02-architecture/nats-subjects.md`** +- Streams: `ZAPIER_JOBS`, `ZAPIER_EVENTS`, `ZAPIER_WEBHOOKS`, optional `ZAPIER_USAGE`. +- Payload schemas + ack/Nak semantics for poller and webhook deliver. +- Prefer `tokenRef` over JWT in messages; NATS private + auth. + +**`docs/02-architecture/archive-nats.md` / `fleet.md` / `tree-nodes.md`** +- Off-chain WORM + Merkle leaf proofs; bloom miss = silence. +- Fleet replica floors (worm/tree min 3); operator console `:3850`. + +**`overview/01-system.md`** +- Planes: Zapier, web, API, leaf, staff, authz, commercial edge, account-balance, CS/sales/acct, middleware, NATS, workers, archives, chain, fleet. +- One-sentence request path through meter → stamp → watch/events → billing NATS. + +**`overview/11-hostnames.md`** +- NS1 `70.88.205.138`; LXC 510 worker; NATS LXC 511–513 at `10.10.10.21–23`. +- Full service→port→DNS table (portal/api/zap/staff/iam/mw/fleet/git/catalog). +- zappier-edge on `127.0.0.1:13000` (tunnel); note that host `:3000` is “another app”. +- NATS never public; public token edge documented separately in nats-cluster package. + +**`overview/NATS.md` / `03-nats-cluster.md`** +- Central 3-server JetStream; new functions = new addresses; Zapier never connects. + +### STATUS / decisions / NATS lab + +**`packages/zapier-decisions/STATUS.md` (2026-09-12)** +- Working lab on NS1; public doors 8/8 OK; keep+fleet up; test 3-node JetStream on `vmbr1`. +- Product streams replicas=3: `ZAPIER_JOBS`, `ZAPIER_EVENTS`, `ZAPIER_WEBHOOKS`, `ZAPIER_USAGE`, `VERAE_ARCHIVE`. +- Clients use `nats://10.10.10.21:4222,...`; host loopback `:4222` unused by clients. +- Open: Zapier Platform push; `MOCK_VERAE=false`; nkeys/mTLS; lan-134; reboot ownership. +- Mock path submit→jobId→wait→receipt proven; live chain seal not proven. + +**Decisions (`packages/zapier-decisions/decisions/`)** +- **D001** — Account-balance (`books.json`) is sole prepaid writer; edge `balanceCents` is cache. +- **D002** — `veraeUserId` join key; public credential = zappier API key; JWT = `tokenRef` server-side. +- **D003** — Access planes are only public HTTP doors (`:3021–3025`); edge on loopback `:3000`. +- **D004** — Leaf cannot touch billing (policy allow-list). +- **D005** — HTTP spawn local-first; `lan-134` disabled. +- **D006** (2026-10-03) — Business SoT → **libSQL**; JetStream = work queues only; **core NATS** for `verae.ts.completed` / `verae.ts.receipt`; Go timestamping DBs stay local; lab `billing.db`/`zappier.db`/`books.json` are single-host stand-ins. + +**`docs-master/MESSAGE-FLOWS.md`** +- Async stamp, wait, idempotent hash, `includeAttached`, multipart, batch Merkle, simulator replay. + +**`docs/plans/phase-gates.md` / `TODO.md`** +- Gates 0–12 largely passed in source/mock; 13–15 (live Verae, private Zapier push, hardening) open. +- Root README still says “9 and 12+ open” in places — stale vs gate logs. + +**`docs/OPEN.md`** +- Needs human: Zapier login/push; live Verae gaps (hash lookup etc.); PDF artwork; LTS/share not live; NATS public bind left loopback (superseded by cluster/token edge notes elsewhere). + +--- + +## 3. Packages (`packages/`) — what each does + +| Package | Role (from README) | +|---------|-------------------| +| **overview** | High-level system book; Zapier never NATS; address routing | +| **docs-master** | Per-module SUMMARY/NATS; MESSAGE-FLOWS; catalog host move | +| **dev-overview** | Designed vs running NATS fabric map; credentials cheat sheet | +| **ui-docs** | Operator/staff/portal screenshot walkthrough + UI-REVIEW.pdf | +| **zapier-decisions** | STATUS, LOG, TODO, D001–D006, NATS bench reports | +| **zapier-user-docs** | Customer guide: signup → stamp → lookup → tree proofs | +| **zappier** (Forgejo: zappier-edge) | Metered API, portal, admin, Stripe, invoices, rate card | +| **zappier-account-balance** | Prepaid SoT (`books.json`); NATS `verae.billing.*` | +| **zappier-customer-service** | CS credits/review UI `:3011` | +| **zappier-sales-pricing** | Per-customer pricing UI `:3012` | +| **zappier-accounting-export** | QuickBooks IIF / CSV export `:3013` | +| **zappier-identity** | Bind/lookup `veraeUserId` ↔ customer (`:3026`) | +| **verae-zapier-middleware** | `/zapier/v1/*`, NATS workers, mock/live chain client `:3100` | +| **verae-zapier** | Full Zapier Platform app (creates/searches/triggers) | +| **verae-activate** | Tiny “Add Numbers” push-first Zapier app | +| **verae-zapier-simulator** | In-process Zapier+trace console `:3847` | +| **verae-access-authz** | Default-deny `verae.access.authz.check` `:3020` | +| **verae-access-web** | Browser portal plane `:3021` | +| **verae-access-api** | Direct customer API plane `:3022` | +| **verae-access-leaf** | S2S leaf plane `:3023` (no billing) | +| **verae-access-zapier** | Zapier HTTPS front door `:3024` | +| **verae-access-staff** | Staff plane UI `:3025` | +| **verae-staff-iam** | Staff users/roles/audit `:3028` | +| **verae-staff-session** | Shared staff cookie login `:3027` | +| **verae-staff-ui** | Shared review HTML template | +| **verae-request-splitter** | Hash vs archive puts (in-process today) | +| **verae-archive-worm** | Bloom WORM off-chain store | +| **verae-archive-aggregator** | Fan-out query + merge replies | +| **verae-tree-node** | Merkle leaf proof WORM (`kind=tree`) | +| **verae-jobs-events** | Mailbox for `jobs.events` `:3030` (passthrough today) | +| **verae-fleet** | Replica floors, spawn, operator console `:3850` | +| **verae-keep** | Crash restart unless paused (`:3860`/`:3861`) | +| **verae-ops** | Docker/Proxmox/metal install + linking runbooks | +| **verae-bootstrap** | Clone all Forgejo modules + host-deps | +| **verae-deploy** | Tag checkout + npm ci on target hosts | +| **verae-proxmox** | LXC worker (510) provision | +| **verae-nats-cluster** | LXC 511–513 JetStream cluster scripts | +| **verae-nats-accounts** | Leaf allow-list + future nkeys sketch | +| **verae-nats-process** | Template for new addressed workers | +| **verae-uptime** | Off-box HTTPS door probes `:3870` | +| **verae-backup** | Tar snapshots under `/SSD2/backups/verae/` | + +--- + +## 4. Extracted system facts (as documented) + +### Services & ports (NS1 lab catalog) + +| Service | Port | Bind (docs) | Public door / DNS | +|---------|------|-------------|-------------------| +| Docs catalog | 443 | Caddy | **zapier.georgelambert.org** | +| Forgejo | 443 / SSH **2223** | Caddy | **git.georgelambert.org** | +| Fleet console | 3850 | 0.0.0.0 | fleet.zapier.georgelambert.org | +| Portal (access-web) | 3021 | 0.0.0.0 | portal.zapier… | +| Customer API | 3022 | 0.0.0.0 | api.zapier… | +| Leaf | 3023 | 0.0.0.0 | leaf.zapier… | +| Zapier plane | 3024 | 0.0.0.0 | **zap.zapier.georgelambert.org** | +| Staff plane | 3025 | 0.0.0.0 | staff.zapier… | +| Authz | 3020 | 0.0.0.0 | internal | +| Identity | 3026 | 0.0.0.0 | id.zapier… | +| Staff session | 3027 | 0.0.0.0 | login.zapier… | +| Staff IAM | 3028 | 0.0.0.0 | iam.zapier… | +| Account-balance | 3010 | 0.0.0.0 | books.zapier… | +| CS / Sales / Acct | 3011 / 3012 / 3013 | 0.0.0.0 | cs / sales / acct | +| Middleware | 3100 | 0.0.0.0 | mw.zapier… | +| jobs.events mailbox | 3030 | 0.0.0.0 | events.zapier… | +| zappier-edge (admin) | **13000** (loopback) or **:3000** in older docs | 127.0.0.1 | tunnel only | +| Keep / watch | 3860 / 3861 | 127.0.0.1 | — | +| Simulator | 3847 | optional | sim.zapier… | +| Uptime status | 3870 | 127.0.0.1 | — | +| Workers | 13200–13602 | 127.0.0.1 | — | +| NATS clients | 4222 | private `10.10.10.21–23` | never public product path | +| NATS cluster routes | 6222 | vmbr1 | never public | +| NATS monitor | 8222 | loopback on nodes | — | +| Optional public NATS edge | 4222 / WSS | token-gated on 138 | nats.zapier… (not for Zapier) | + +**Also documented elsewhere (live product, not this catalog’s primary story):** `zap.verae.ai` / `zappier-edge.service` on `:3040` under `/opt/verae/zappier-edge` — see Conflicts. + +### NATS subjects & streams + +**JetStream product streams (replicas=3):** +`ZAPIER_JOBS`, `ZAPIER_EVENTS`, `ZAPIER_WEBHOOKS`, `ZAPIER_USAGE`, `VERAE_ARCHIVE` + +| Subject | Kind | Purpose | +|---------|------|---------| +| `verae.zapier.jobs.watch` | JS work | Poll chain status | +| `verae.zapier.jobs.events` | JS events | Terminal job outcomes → waiters/webhooks | +| `verae.zapier.webhooks.deliver` | JS work | POST Zapier REST Hook | +| `verae.zapier.usage` | optional JS | Usage export | +| `verae.archive.put` | JS | Off-chain put | +| `verae.archive.query` | broadcast | Bloom query | +| `verae.archive.reply.` | reply | Hit-only | +| `verae.billing.statement.get` | req-reply | Statement | +| `verae.billing.balance.get` | req-reply | Balance | +| `verae.billing.balance.adjust` | req-reply | Credits/reloads | +| `verae.billing.usage.recorded` | pub | Meter | +| `verae.billing.payment.recorded` | pub | Payments | +| `verae.billing.credit.applied` | pub | CS credits | +| `verae.access.authz.check` | req-reply | Allow/deny | +| `verae.access.{web,api,zapier,leaf}.*` | ingress | Plane fronts | +| `verae.identity.bind` / `.lookup` | req-reply | Identity map | +| `verae.ts.completed` / `verae.ts.receipt` | **core NATS** (D006 / standing rule) | Job-complete notify — **not** JetStream | + +Client URL (lab): `nats://10.10.10.21:4222,nats://10.10.10.22:4222,nats://10.10.10.23:4222` + +### Databases / data stores (documented) + +| Store | Owner | Contents | +|-------|-------|----------| +| **`books.json`** | zappier-account-balance (D001) | Prepaid, credits, usage, payments (lab SoT) | +| **`zappier.db`** | zappier-edge | Auth/sessions/admin (also older monolith docs) | +| **`billing.db`** | live edge / D006 note | Customers, usage_entries (+ hash chain), rates, invoices (138 stand-in) | +| **libSQL (target)** | D006 | Shared business SoT across 3–5 hosts | +| Middleware file JSON / `data/` | middleware | Tenants, webhooks, blobs (MVP; Postgres/Redis Phase 15) | +| WORM / tree blobs | archive-worm, tree-node | Off-chain metadata/files/leaf proofs | +| JetStream store | nats-a/b/c | Durable work/events/archive streams | +| Keep/fleet runtime | `~/verae-*-runtime` | Intent + spawned unit state | +| Backups | `/SSD2/backups/verae/` | Tar snapshots (not private keys) | +| Go `blockchain.db` | timestamping-svc | **Do not relocate** (D006) | + +### Public doors (D003) + +- Browser → `:3021` / portal.zapier… +- Customer API → `:3022` / api.zapier… +- Zapier → `:3024` / zap.zapier… +- Leaf → `:3023` +- Staff → `:3025` (+ iam/login) +- Catalog → zapier.georgelambert.org +- Git → git.georgelambert.org +- Edge admin/Stripe/identity stay loopback (not a public door) + +### Identity model (D002 + access planes) + +- **Public credential:** zappier API key (`x-api-key`); portal signup/regen. +- **Middleware legacy:** `zmw_` / `zmt_` Bearer (internal / pre-composition). +- **Chain auth:** Verae login → JWT held as `tokenRef` server-side; never in Zapier. +- **Join key:** `veraeUserId` (`vu_…`) via `zappier-identity` bind/lookup. +- **Staff:** `verae-staff-iam` roles (owner, cs, sales, accounting, operator, …) + session cookie. +- **Planes:** zapier / web / api / leaf / staff — each must pass `authz.check` before internal subjects; leaf cannot bill. + +### Userflows (documented) + +1. **Signup** → portal → API key (Free). +2. **Connect Zap** → HTTPS access-zapier / edge with API key. +3. **Create Timestamp (async)** → meter → middleware → chain → `jobs.watch` → 202 `{jobId}` → poll → `jobs.events` → REST Hook. +4. **Create and Wait** → same, HTTP holds until event or timeout → `pending`+`jobId`. +5. **Idempotent SHA-256** → original seal / `existing: true`. +6. **Attachments / includeAttached** → archive.put + query/merge. +7. **Batch Merkle** → root on chain; leaves on tree-nodes; lookup via `kinds=["tree"]`. +8. **CS credit / portal reload / sales pricing / accounting export** via `verae.billing.*`. +9. **Simulator** replays 1–7 in-process with fault injection. + +--- + +## 5. Conflicts / outdated claims + +| Topic | Claim A | Claim B | Notes | +|-------|---------|---------|-------| +| Wiring status | Root README: packages “imported, **not yet wired**” | STATUS: working lab, doors 8/8, NATS cluster live | README composition blurb is stale | +| Public auth | Overview / verae-zapier: Bearer **`zmw_`** | Composition / D003 / billing docs: **`x-api-key`** via zappier | Transition incomplete in docs | +| Prepaid SoT | D001: **`books.json`** via account-balance | Live edge memory / D006: **`billing.db`** + **`zappier.db`**; target **libSQL** | Three eras coexist in writing | +| Edge port | D003 / ops: **`:3000`** loopback | Hostnames: **`:13000`**; “`:3000` is another app” | Also live **`zap.verae.ai:3040`** outside this catalog | +| Public Zap host | Catalog: **zap.zapier.georgelambert.org** | Product memory: **zap.verae.ai** | Dual branding / cutover | +| NATS location | Many docs: loopback `:4222` | STATUS / nats-cluster: **10.10.10.21–23**; loopback unused by clients | Hostnames table still lists loopback NATS | +| Job-complete bus | Architecture: **`verae.zapier.jobs.events` on JetStream** | D006 + standing rule: **`verae.ts.*` on core NATS only** | Naming + persistence conflict; do not JetStream job-complete notify | +| Gate progress | README: phases **9 and 12+ open** | phase-gates gate logs: **9 and 12 pass** | Stale README sentence | +| MOCK / live | STATUS: still **MOCK_VERAE** | Some defaults printed as `MOCK_VERAE=false` | Live seal not proven | +| NATS accounts | D004 sketches accounts | STATUS: cluster **without** nkeys; isolation = vmbr1 | Expected | +| Metering owner | Middleware `PLAN_LIMITS` in older overview | Composition: metering on **zappier** only | Correct direction = composition | +| Hash lookup | Features a–m: mock done | Live OpenAPI: **no** hash route; client 501 | Gap documented | +| docs-master diagram | Still shows “NS1 loopback :4222” for JetStream | Cluster cut-over done | Update modules-and-nats diagram | + +--- + +## 6. Source-of-truth files (prefer these) + +| Concern | Prefer | +|---------|--------| +| Current lab status | `packages/zapier-decisions/STATUS.md` | +| Accepted architecture decisions | `packages/zapier-decisions/decisions/D00*.md` (esp. **D003**, **D006**) | +| System planes / hard rules | `overview/01-system.md` | +| Hostnames & ports | `overview/11-hostnames.md` | +| NATS streams (Zapier jobs/webhooks/archive) | `docs/02-architecture/nats-subjects.md` + STATUS streams list | +| Billing subjects | `docs-master/modules-and-nats.md` + `packages/zappier-account-balance/README.md` | +| Message userflows | `docs-master/MESSAGE-FLOWS.md` | +| Composition ownership | `docs/02-architecture/composition.md` | +| Product features a–m | `docs/01-product/features-a-m.md` | +| Ops bring-up | `packages/verae-ops/GETTING-STARTED.md` (+ PDF) | +| Module catalog | `docs-master/README.md` + `packages/*/README.md` | +| Phase implementation order | `docs/plans/phase-gates.md` / `TODO.md` | +| Customer narrative | `packages/zapier-user-docs/` | +| Designed vs running NATS | `packages/dev-overview/` | + +**Standing overrides outside this tree (operator rules / memory):** +- Never JetStream for timestamp job-complete (`verae.ts.completed` / `verae.ts.receipt`). +- Live commercial edge on 138: `/opt/verae/zappier-edge`, `billing.db` + `zappier.db`, service `:3040`, public `zap.verae.ai`. +- D006: migrate business data to shared **libSQL** before multi-host split. + +--- + +## 7. Quick system map (ASCII) + +```text + [Zapier cloud: verae-zapier / activate] + HTTPS only + | + +------------------+------------------+ + | | + access-zapier :3024 (alt) zappier-edge /v1 + | | + +---------> middleware :3100 <--------+ + | \ + chain HTTPS \-- NATS JetStream (jobs/webhooks/archive) + api.veraetime.net | + or MOCK_VERAE workers: poller, webhook, + aggregator, worm×N, tree×N + Browser :3021 API :3022 Leaf :3023 Staff :3025 + \ | / + authz.check :3020 + | + billing.* ---- account-balance (books → libSQL) + identity.* --- zappier-identity + fleet :3850 / keep :3860 (ops) + catalog: zapier.georgelambert.org +``` + +--- + +*End of documentation map. Absolute path of this report: `/tmp/zapier-system-docs-map.md`* diff --git a/plans/qa/2026-10-03-baseline.md b/plans/qa/2026-10-03-baseline.md new file mode 100644 index 0000000..731b6eb --- /dev/null +++ b/plans/qa/2026-10-03-baseline.md @@ -0,0 +1,344 @@ +# QA Baseline — Verae × Zapier Userflows 1–7 vs D006 Multi-Host + +**Role:** QA (evaluate only; no product feature implementation) +**Date:** 2026-10-03 (UTC probe ~06:04–06:07) +**Host:** `NS1.GEORGELAMBERT.ORG` / `70.88.205.138` +**Inputs:** +- Plan: [`../2026-10-03-libsql-multihost-architecture.md`](../2026-10-03-libsql-multihost-architecture.md) +- Arch review: [`../reviews/2026-10-03-arch-review.md`](../reviews/2026-10-03-arch-review.md) (approve-with-changes; A1–A8 merged; full sign-off still open) +- PM board: [`../reviews/2026-10-03-pm-board.md`](../reviews/2026-10-03-pm-board.md) +- Decision: D006 (libSQL business SoT; JetStream = work only; core NATS for `verae.ts.*`) +**Scope:** Readiness of critical userflows **1–7** for multi-host cutover. This is a **baseline** (WP-Q precursor). Per PM §5, formal WP-Q entry requires MG3; this pack documents why that gate is not met and what must change first. + +--- + +## Summary judgment + +Lab doors and digests-only stamp submit on `:3040` → `:3130` work today, but **D006 multi-host readiness fails**: there is **no libSQL**, **no BusinessStore**, and **two live prepaid/ledger writers** (`billing.db` on edge + empty `books.json` behind `:3010`). Userflows 1, 3–6 are not E2E-proven; flow 2 is partially green; flow 7 has only weak 404 evidence for billing deny. Do **not** start Phase 4 host split or treat WP-Q as “in” until single-writer libSQL lands. + +--- + +## Live lab evidence (2026-10-03) + +### Health matrix + +| Port | Role | `/health` | Notes | +|------|------|-----------|-------| +| `:3040` | zappier-edge | **200** `{"ok":true,"role":"zappier-edge"}` | Bind `127.0.0.1`; systemd `zappier-edge.service` | +| `:3100` | middleware (lab/mock) | **200** `mockVerae:true`, `natsConnected:false` | Catalog/default; **not** product upstream | +| `:3130` | middleware (product) | **200** `mockVerae:false`, `natsConnected:true` | Edge `ZAPPIER_UPSTREAM` target | +| `:3010` | zappier-account-balance | **200** `nats:false` | Loopback; **not** a systemd unit; fleet-spawned | +| `:3021`–`:3025` | access planes | **200** each | web / api / leaf / zapier / staff | + +### Env / upstream + +- Edge secrets (`/SSD2/secrets/zap-verae/zappier-edge.env`): `PORT=3040`, **`ZAPPIER_UPSTREAM=http://127.0.0.1:3130`** (confirms arch C11 / plan A1). +- Product MW env: `PORT=3130`, `HOST=127.0.0.1`, `MOCK_VERAE=false`, `NATS_URL=nats://127.0.0.1:4222`. +- Account-balance (pid 4179): cwd under moved stack package; `FLEET_STATE_DIR=.../verae-fleet/data/runtime/account-balance-0`; `NATS_URL` lists `10.10.10.21–23` but health reports **`nats:false`**. + +### Digest-only stamp (flow 2 probe) + +- Digest: `sha256("qa-baseline")` = `edf8f682e42104442bb4637c665f50ada9873fa2fe6958ae00a6c05b2e5181cb` +- `POST /v1/timestamps` with `x-api-key: key-ada`, `x-billing-channel: api`, body `{sha256s:[…]}` → **HTTP 202**, `created:1` then idempotent `existing:1`, credit headers present (`X-Verae-Credits-*`). +- Items-shaped digest-only body also **202**. +- Single `POST /v1/timestamp` → **202** with `jobId`, but receipt `completedAt: "0001-01-01T00:00:00Z"` (placeholder / incomplete receipt shape for async complete). + +### Dual-writer / C16 (flows 1, 4–6) + +| Store | Path | Observation | +|-------|------|-------------| +| Edge ledger | `/opt/verae/zappier-edge/billing.db` | **Exists** (WAL). `usage_entries` count **3** for `cust_1` @ 10 each after probes; `customers.balance_cents` for Ada = **0** | +| Account-balance books | `FLEET_STATE_DIR/.../data/books.json` | **Exists**, empty: `prepaid:{}`, `usage:[]`, `credits:[]`, `payments:[]` | +| Statement HTTP | `GET :3010/statement/cust_1` | **200** `{prepaidCents:0, usage:[], …}` — **does not see** edge `usage_entries` | +| Edge response | batch stamp | `balanceCredits:0`, period spend headers show **30** after three 10-credit meters | + +**C16 confirmed live:** edge writes metered usage into `billing.db`; account-balance serves a separate empty `books.json`. No shared SoT. Splitting hosts now would fork money. + +### Go timestamping DB (C9) + +- `/var/lib/timestamping-svc/data/blockchain.db` **exists** (~31 MB, mtime 2026-09-25). +- **Not modified** by this QA probe (read `ls`/`stat` only). + +### libSQL / BusinessStore / harness + +- No `LIBSQL_URL` / `sqld` / libSQL container observed on 138 for business data. +- Local clone `/Users/marchon/apps/verae/zappier-edge/src/db/`: **no** `business-store.ts` (`NO_BUSINESS_STORE`). +- `verae-zapier-simulator` exists (messaging/trace) but is **not** the WP-P automated harness for flows 1–7. +- Arch checklist still open: `nats-subjects.md` `verae.ts.*` / `ZAPIER_USAGE` non-SoT; published env matrix; GETTING-STARTED SoT language. +- Host `nats stream ls` to `10.10.10.21:4222` **timed out** from this probe (MW still reports `natsConnected:true` via loopback `:4222` proxy). STATUS still lists product stream **`ZAPIER_USAGE`** (C10 risk). + +### Leaf D004 (flow 7 probe) + +- `:3023/health` **200** leaf plane up. +- `POST :3023/v1/billing/adjust` → **404** `not found` (route absent). This is **not** yet an automated policy deny of `verae.billing.balance.adjust` (PM WP-D DoD). + +--- + +## Userflow readiness (1–7) + +| # | Userflow | Today | Blocker vs D006 | +|---|----------|-------|-----------------| +| 1 | Signup → API key → Statement | Partial: seed keys exist (`key-ada`…); statement via `:3010` empty/out of band with edge ledger | Dual SoT; no portal E2E harness; account-balance `nats:false` | +| 2 | Stamp digests single + batch + credit headers | **Mostly green** on lab (`:3040`→`:3130`, 202, headers, idempotent existing) | Still local SQLite meter; zero balance still meters; receipt placeholder on single path; not libSQL | +| 3 | Async complete → Zapier REST Hook | **Not proven** | `completedAt` zero-time on single stamp; no WP-P watch→events→webhooks.deliver E2E; Platform push still open in STATUS | +| 4 | CS credit adjust → portal balance | **Broken / dual path** | Adjust would hit books/`verae.billing.*` while meter hits `billing.db`; nats false on balance service | +| 5 | Sales quote → meter match | **Unproven** | Rate tables live in edge SQLite; sales package not wired to shared SoT | +| 6 | Accounting export consistency | **Fails if run today** | Export cannot reconcile empty books vs non-empty `usage_entries` | +| 7 | Leaf stamp OK / billing deny | Health only | Need authz allow-list tests; 404 ≠ D004 proof | + +**PM QA entry (board §5):** MG3 single writer, regression green, NATS rules verified, harness skeleton, D004 automated, migrator tip report, Go DB untouched note, docs C2/C3/C5/C6 — **not met** (only Go-untouched + partial digests-only + doors up). + +--- + +## a) What to change + +| # | Component | Problem | Evidence | Severity | Type | +|---|-----------|---------|----------|----------|------| +| 1 | Prepaid / ledger SoT (C7/C16) | Two writers: edge `billing.db` meters usage; `:3010` serves empty `books.json` statement | usage_n=3 vs statement `usage:[]`; both processes up | **critical** | design flaw / dual writer | +| 2 | Missing libSQL + BusinessStore (D006 Phases 1–2) | No shared business DB or interface; multi-host impossible without per-host SQLite fork | No `business-store.ts`; no LIBSQL on 138 | **critical** | missing foundation | +| 3 | Account-balance NATS / facade | Health `nats:false` while CS/portal expect `verae.billing.*` | `:3010/health` JSON; env has NATS_URL but not connected | **high** | ops / wiring | +| 4 | Userflow E2E harness (WP-P) | Flows 1–7 not automated; simulator ≠ harness | No `plans/qa` flow reports; simulator README is messaging-only | **high** | missing gate | +| 5 | Async complete / receipts (flow 3) | Single-stamp receipt shows epoch-zero `completedAt`; REST Hook path unproven | curl body `0001-01-01T00:00:00Z` | **high** | incomplete product path | +| 6 | Zero-balance metering (flows 2/4) | Ada `balance_cents=0` still stamps and accumulates period spend | customers row + 202 responses + `X-Verae-Credits-Balance: 0` | **medium** | missing gate / prepaid policy | +| 7 | Dual middleware confusion (C11) | `:3100` mock vs `:3130` product; docs/scripts can point wrong | health JSON both ports; edge env → 3130 | **medium** | doc/ops drift | +| 8 | D004 leaf deny proof (flow 7) | Only 404 on invent-ed HTTP path; no automated NATS/HTTP deny | curl 404; WP-D tests not evidenced on lab | **medium** | missing test | +| 9 | `ZAPIER_USAGE` / nats-subjects docs (C5/C10) | STATUS still lists usage stream as product; catalog subjects omit core-only `verae.ts.*` | STATUS.md streams table; arch checklist open | **medium** | doc drift / SoT confusion | +| 10 | Env matrix / Phase 0 leftovers | Arch full sign-off blocked; GETTING-STARTED may still call books SoT | arch review checklist unchecked items | **medium** | doc freeze incomplete | +| 11 | Account-balance not systemd-managed | Process from fleet overlay / moved path; restart/ownership unclear vs edge unit | no `*account-balance*.service`; pid under `marchon` | **low** | ops hygiene | + +### What is working (preserve) + +- Digests-only batch + single stamp on product edge `:3040` with `key-ada` and credit headers. +- Product upstream correctly set to **`:3130`** (not mock `:3100`). +- Access-plane health on `:3021–3025`. +- Edge split DBs present and WAL: `billing.db` (business tables) + `zappier.db` (auth). +- Idempotent re-stamp (`created:0`, `existing:1`, spend 0). +- Standing rules respected in this probe: **no** JetStream for `verae.ts.*`; **no** writes to `blockchain.db`. +- Plan + A1–A8 + D001 addendum + D006 direction are the right target. + +--- + +## b) What to change it to + +| # | Component | Target behavior | Acceptance criteria | Constraints | +|---|-----------|-----------------|---------------------|-------------| +| 1 | Single prepaid/ledger writer | One authoritative adjust + `recordUsage` path on shared store; other path read-only or stopped | Grep: no prod writes to `books.json` / per-host `billing.db` as SoT; adjust on CS visible on portal statement and edge balance headers | D001 single writer; do not dual-append hash chain | +| 2 | libSQL BusinessStore | `LibsqlBusinessStore` (+ SqliteFile for CI) behind `BusinessStore`; lab migrator from `billing.db` + `books.json` | Contract tests green; migrator tip hash verify; `BUSINESS_STORE=libsql` writes only to libSQL | Auth stays edge-local `zappier.db` (C13); fail closed (503) if writer down | +| 3 | Billing facade | `:3010` (or successor) is libSQL-backed listener for `verae.billing.*` **or** folded into edge HTTP API — pick one (plan Phase 3) | Health `nats:true` (if NATS facade kept); statement usage matches `usage_entries` | Keep subjects stable for CS/sales | +| 4 | Harness 1–7 | Automated runner under simulator/harness producing `plans/qa/` report | Flows 1,2,4 runnable under MOCK; then 3,5,6,7; exit 0 artifact | Zapier cloud push optional for first MOCK pass | +| 5 | Async complete | Real completion times / receipts; `jobs.watch` → `jobs.events` → `webhooks.deliver` | E2E: jobId reaches terminal event + hook delivery (or recorded attempt) | `verae.ts.completed` / `verae.ts.receipt` remain **core NATS only** | +| 6 | Prepaid gate | Metering respects prepaid/credits policy (402 or configured allow) | Seed funded customer; zero-balance path returns expected product code | Do not rewrite historical ledger mid-test | +| 7 | Env matrix | Documented `:3040` / `:3130` product / `:3100` mock / `:3010` facade | STATUS + ops GETTING-STARTED + plan cite same matrix | Do not “fix” smoke by pointing edge at 3100 | +| 8 | D004 | Leaf cannot call balance adjust (HTTP door + NATS authz) | `verae-access-leaf` / authz tests green; live deny ≠ bare 404 | Leaf may still stamp/authz | +| 9 | NATS docs | `nats-subjects.md` Non-streams: `verae.ts.*` core-only; `ZAPIER_USAGE` telemetry non-SoT | Arch checklist boxes checked; STATUS wording demotes usage stream | No JetStream bind for job-complete | +| 10 | Phase 0 close | C2/C3/C5/C6 closed or owned; GETTING-STARTED SoT → libSQL | Arch full sign-off or explicit waivers | — | +| 11 | Balance process ops | systemd (or fleet-documented) unit with clear state dir | `systemctl status` shows balance facade; books path documented until migration | Deploy never rsyncs `*.db*` | + +--- + +## c) How to implement + +Aligned with PM DAG; **do not skip merge gates**. + +```mermaid +flowchart TD + P0[Phase 0 docs/env/NATS freeze] --> P1[Phase 1 BusinessStore TDD] + P1 --> P2[Phase 2 migrator + dual-read] + P2 --> MG2[MG2 writers unlock] + MG2 --> P3[Phase 3 single writer rewire] + P3 --> MG3[MG3 single writer proof] + MG3 --> P4[Phase 4 multi-host LIBSQL_URL] + MG3 --> P5[Phase 5 harness 1-7] + P4 --> P5 + P5 --> MG4[MG4 green] + MG4 --> P6[Phase 6 QA whattochange loop] +``` + +### Phase 0 — Conflict / docs freeze (feeding; ~1–2 days) + +- **Goal:** Close open arch checklist items that confuse operators and agents. +- **Changes:** Publish env matrix; patch `nats-subjects.md`; demote `ZAPIER_USAGE` in STATUS; fix overview `zmw_` / digests examples (C3/C6); GETTING-STARTED SoT → libSQL. +- **Validation:** Arch checklist; `rg` for stale SoT claims. +- **Effort:** 1–2 agent-days (WP-A/B/C). + +### Phase 1 — BusinessStore + libSQL skeleton (critical; WP-F/G/H) + +- **Goal:** Interface + SqliteFile + Libsql drivers with contract tests (including hash-chain). +- **Changes:** `zappier-edge` `src/db/business-store.ts`, contract tests, CI libSQL. +- **Validation:** Jest contract green both drivers (MG1). +- **Effort:** ~3 days serial. + +### Phase 2 — Migrate + dual-read (WP-J/K) + +- **Goal:** Import live `billing.db` + empty/legacy `books.json`; verify chain tip; reads dual-run; **writes still locked**. +- **Validation:** Migrator report archived under `plans/qa/`; MG2. +- **Effort:** ~2 days. + +### Phase 3 — Rewire writers (WP-L/M/N) — only after MG2 + +- **Goal:** Edge meter + account-balance + CS/sales/acct → one BusinessStore; remove second writer. +- **Validation:** Statement usage == ledger; CS adjust visible on portal/edge headers; MG3 grep proof. +- **Effort:** ~3–4 days. + +### Phase 4 — Multi-host (WP-O) — only after MG3 + +- **Goal:** `LIBSQL_URL` primary; edge/MW/CS on separate hosts possible. +- **Validation:** Cross-host adjust/meter/statement same balance. + +### Phase 5 — Harness (WP-P) + +- **Goal:** Automate flows 1–7; first pass MOCK_VERAE OK. +- **Validation:** Report under `plans/qa/`; MG4. + +### Phase 6 — QA loop (WP-Q) + +- **Goal:** Break → whattochange → fix → retest until **two consecutive** full greens (project buffer). + +**Explicit non-goals until gates pass:** JetStream for `verae.ts.*`; editing `blockchain.db`; Litestream multi-writer; multi-host split before MG3. + +--- + +## d) Expected results / still incomplete + +### After implementation (realistic) + +| Metric / behavior | Today (baseline) | Expected after Phases 0–5 | +|-------------------|------------------|---------------------------| +| Business SoT | Split `billing.db` + empty `books.json` | One libSQL `verae_business` | +| Flow 2 digests stamp | 202 + headers on single host SQLite | Same UX on libSQL meter writer | +| Flow 1/4 statement vs meter | Divergent (usage 3 vs `[]`) | Identical customer/usage/balance | +| Flow 3 async → hook | Unproven; zero `completedAt` | Proven under harness (MOCK then optional live) | +| Flow 5/6 quote & export | Unproven / inconsistent | Same rate tables + export = ledger | +| Flow 7 D004 | 404 only | Automated deny | +| Multi-host money | Unsafe | Safe only post-MG3 | +| `blockchain.db` | Untouched | Still untouched | + +### Still missing, incorrect, or incomplete (even after this todo list) + +| Gap | Why it persists | +|-----|-----------------| +| Zapier Platform public `push` + live workspace Zaps | Needs operator Zapier login (STATUS); out of first cutover buffer | +| Live `api.veraetime.net` seal proof beyond lab MW | Creds / OpenAPI gaps (hash lookup 501); separate from libSQL SoT | +| NATS nkeys/mTLS | Cluster isolation today is `vmbr1`, not accounts | +| Portal auth on multi-edge sticky sessions | C13: auth stays local `zappier.db` for v1 | +| libSQL HA / failover RPO | Launch = one primary + backup; not multi-primary | +| Peergos / Go timestamping redesign | Out of scope | +| Full architecture sign-off | Depends on remaining checklist (docs), not only code | + +### Success tiers + +| Tier | Criteria | +|------|----------| +| **Minimum viable (this baseline exit)** | Dual-writer named daily owner; env matrix published; harness skeleton for 1/2/4; no host split | +| **MG3 / single-writer** | libSQL SoT; one adjust path; statement == meter | +| **Production-ready multi-host** | MG3 + WP-O + flows 1–7 twice green; fail-closed meter; D004 automated | +| **Complete** | + Platform push + live Verae seal + nkeys — separate projects | + +--- + +## e) Prioritized todo list with PreReqs + +``` +Item 1 — [P0] — Freeze dual-writer authority for lab (C16 runbook) +Document which process may adjust/meter each day until MG3; forbid CS adjust against books while edge meters billing.db without a sync plan. +PreReqs: (none) + +Item 2 — [P0] — Publish live env matrix (:3040 / :3130 / :3100 / :3010) +Add to STATUS + ops docs; match edge ZAPPIER_UPSTREAM=http://127.0.0.1:3130 evidence. +PreReqs: (none) + +Item 3 — [P0] — BusinessStore interface + failing contract tests (WP-F) +Create business-store.ts + contract suite (customers, usage chain, rates, invoices, balances). +PreReqs: (Item 2) + +Item 4 — [P0] — SqliteFileBusinessStore adapter (WP-G) +Wrap existing edge repos; keep Jest green. +PreReqs: (Item 3) + +Item 5 — [P0] — LibsqlBusinessStore + CI libSQL (WP-H) +Driver + reachable LIBSQL_URL in CI/lab; MG1. +PreReqs: (Item 4) + +Item 6 — [P0] — Migrator billing.db + books.json → libSQL with chain verify (WP-K) +Refuse cutover on verifyChain failure; archive tip report under plans/qa/; MG2. +PreReqs: (Item 5) + +Item 7 — [P0] — Rewire edge meter to BusinessStore only (WP-L) +Remove production writes that treat local billing.db as SoT once flag on. +PreReqs: (Item 6) + +Item 8 — [P0] — Fold or rewire account-balance to libSQL facade (WP-M) +One verae.billing.* listener; empty books.json import-only; nats connected if facade kept. +PreReqs: (Item 7) + +Item 9 — [P1] — Rewire CS / sales / accounting-export to same store (WP-N) +Credit adjust, rates, export read libSQL; MG3 proof. +PreReqs: (Item 8) + +Item 10 — [P1] — Patch nats-subjects + demote ZAPIER_USAGE (C5/C10) +Document verae.ts.* core-only; usage stream telemetry-only in STATUS/catalog. +PreReqs: (none) + +Item 11 — [P1] — Automate D004 leaf billing deny (WP-D) +Tests prove leaf cannot balance.adjust; replace 404-only smoke. +PreReqs: (none) + +Item 12 — [P1] — Userflow harness skeleton flows 1, 2, 4 (WP-P start) +MOCK_VERAE runnable report artifact under plans/qa/. +PreReqs: (Item 9) + +Item 13 — [P1] — Extend harness to flows 3, 5, 6, 7 +Async hook path, quote match, export consistency, leaf deny E2E; MG4. +PreReqs: (Item 12 + Item 11) + +Item 14 — [P2] — Fix async receipt completedAt / completion wiring +Eliminate 0001-01-01 placeholders on product stamp path used by Zapier wait/hook. +PreReqs: (Item 12) + +Item 15 — [P2] — Prepaid zero-balance policy gate +Define and test 402 vs allow-with-debt; seed funded fixture for harness. +PreReqs: (Item 7) + +Item 16 — [P2] — Multi-host LIBSQL_URL topology (WP-O) +Only after MG3; cross-host meter/adjust/statement. +PreReqs: (Item 9) + +Item 17 — [P2] — systemd/fleet hygiene for account-balance facade +Unit or documented fleet runtime; stable state dir until books retired. +PreReqs: (Item 8) + +Item 18 — [P3] — Phase 0 leftover docs (C2/C3/C6, GETTING-STARTED) +Close arch full sign-off checklist items not covered by Item 2/10. +PreReqs: (Item 2 + Item 10) +``` + +**Dependency order (for “Do all”):** +Item 1 ∥ Item 2 ∥ Item 10 ∥ Item 11 → Item 3 → Item 4 → Item 5 → Item 6 → Item 7 → Item 8 → Item 9 → Item 12 → (Item 13 ∥ Item 14 ∥ Item 15) → Item 16 → Item 17; Item 18 can trail after Item 2+10. + +**QA note:** Formal WP-Q churn should wait for **Item 9 (MG3)** + **Item 12**. This baseline pack is the readiness report that unlocks engineering focus on Items 1–9. + +--- + +## Probe command index (reproducible) + +```bash +ssh -i ~/.ssh/id_ed25519 -o IdentitiesOnly=yes marchon@70.88.205.138 \ + 'for p in 3040 3100 3130 3010 3021 3022 3023 3024 3025; do + curl -sS -m 5 http://127.0.0.1:$p/health; echo; done' + +# Digest-only batch (do not use production customer keys outside lab seeds) +DIGEST=$(printf "qa-baseline" | sha256sum | awk "{print \$1}") +curl -sS -D - -X POST http://127.0.0.1:3040/v1/timestamps \ + -H "content-type: application/json" -H "x-api-key: key-ada" \ + -H "x-billing-channel: api" \ + -d "{\"sha256s\":[\"$DIGEST\"]}" + +# Dual-writer check (read-only) +sqlite3 /opt/verae/zappier-edge/billing.db "SELECT COUNT(*) FROM usage_entries;" +curl -sS http://127.0.0.1:3010/statement/cust_1 +# blockchain.db: ls/stat only — never migrate/edit +``` + +--- + +*End of QA baseline. No product code was changed by this review.* diff --git a/plans/qa/2026-10-03-libsql-dryrun.md b/plans/qa/2026-10-03-libsql-dryrun.md new file mode 100644 index 0000000..ecb941e --- /dev/null +++ b/plans/qa/2026-10-03-libsql-dryrun.md @@ -0,0 +1,89 @@ +# libSQL migrator dry-run (WP-K) — PASS + +**Date:** 2026-10-03T06:24–06:26Z +**Host:** NS1 `70.88.205.138` (`NS1.GEORGELAMBERT.ORG`) +**Role:** Engineer (safe dry-run only) +**Result:** **PASS** + +## Constraints honored + +- Did **not** modify `/var/lib/timestamping-svc/data/blockchain.db` or any Go timestamping DB (stat-only; mtime 2026-09-25). +- Did **not** set `BUSINESS_STORE=libsql` (or any `LIBSQL_*`) on `zappier-edge.service`. +- Did **not** delete/truncate live `/opt/verae/zappier-edge/billing.db`. +- All migrator I/O under `/tmp/libsql-dryrun-20261003T062452Z/` copies only. + +## Workspace + +| Item | Path | +|------|------| +| Dry-run dir | `/tmp/libsql-dryrun-20261003T062452Z/` | +| Source copy | `…/source-billing.db` (90112 bytes) | +| Dest libSQL file | `…/dest.db` (`file:/tmp/libsql-dryrun-20261003T062452Z/dest.db`, 94208 bytes) | +| Migrator | `/opt/verae/zappier-edge/dist/db/migrate-to-libsql.js` (same content/mtime as `/tmp/zappier-edge-work`) | +| Runner | `…/run-migrate.mjs` (uses opt `node_modules`; chdir to `/opt/verae/zappier-edge`) | +| JSON report | `…/migrate-report.json` | + +`/tmp/zappier-edge-work` had no `node_modules`; used live opt install for `@libsql/client` + `better-sqlite3`. Migrator source identical in both trees. + +## Commands + +```bash +# 1) Checkpoint + consistent copy (as verae-zapier → /tmp dry-run dir) +sudo -u verae-zapier sqlite3 /opt/verae/zappier-edge/billing.db "PRAGMA wal_checkpoint(TRUNCATE);" +# → 0|0|0 +sudo chmod 777 /tmp/libsql-dryrun-20261003T062452Z +sudo -u verae-zapier python3 -c "import sqlite3; s=sqlite3.connect(\"/opt/verae/zappier-edge/billing.db\"); d=sqlite3.connect(\"/tmp/libsql-dryrun-20261003T062452Z/source-billing.db\"); s.backup(d); d.close(); s.close()" +sudo chown marchon:marchon /tmp/libsql-dryrun-20261003T062452Z/source-billing.db + +# 2) Run migrator (node ESM runner → migrateBillingDbToLibsql) +cd /opt/verae/zappier-edge +node /tmp/libsql-dryrun-20261003T062452Z/run-migrate.mjs +``` + +## Row counts + +| Table | Live billing.db | Source copy | Dest libSQL | Match | +|-------|-----------------|-------------|-------------|-------| +| customers | 4 | 4 | 4 | yes | +| usage_entries | 6 | 6 | 6 | yes | +| channel_rates | 108 | 108 | 108 | yes | +| channel_rate_meta | 1 | 1 | 1 | yes | +| tiers | 3 | 3 | 3 | yes | +| price_endpoints | 11 | 11 | 11 | yes | +| invoices | 0 | 0 | 0 | yes | +| invoice_lines | 0 | 0 | 0 | yes | + +Customers on dest: `cust-platform`, `cust_1`, `cust_2`, `cust_3`. + +## Chain / tip hashes + +| Phase | ok | length | tipHash | +|-------|----|--------|---------| +| before (source copy via `verifyUsageLedgerChain`) | true | 6 | `8d58a2986f46a9775712354234d44075eea1257226f93f9fb33a19b444ddebd5` | +| after (dest via `LibsqlBusinessStore.verifyUsageChain`) | true | 6 | `8d58a2986f46a9775712354234d44075eea1257226f93f9fb33a19b444ddebd5` | +| live billing.db (post-run read) | — | 6 | `8d58a2986f46a9775712354234d44075eea1257226f93f9fb33a19b444ddebd5` | + +Genesis `prev_hash` = 64 zeros; tip id = 6. Before tip === after tip; lengths match. + +## Dual-run / safety post-checks + +| Check | Result | +|-------|--------| +| `sourcePreserved: true` (migrator report) | PASS | +| Source copy still 90112 bytes after migrate | PASS | +| Live `billing.db` still 90112 bytes; counts 4/6; same tip | PASS | +| `BUSINESS_STORE` / `LIBSQL_*` absent from unit + not applied | PASS | +| zappier-edge `/health` still 200 | PASS | +| blockchain.db untouched (mtime 2026-09-25, ~31MB) | PASS | +| booksJson | `absent` (path not provided; stub skip) | + +## Pass/fail + +**PASS** — WP-K migrator dry-run on a WAL-checkpointed copy of live `billing.db` → `file:…/dest.db` succeeded with matching row counts and identical usage ledger tip hash. Live service remains on sqlite-file BusinessStore; no cutover. + +## Non-goals (explicitly not done) + +- No `BUSINESS_STORE=libsql` flip +- No rsync/deploy of edge +- No books.json prepaid import (schema stub) +- No JetStream / multi-host work diff --git a/plans/qa/2026-10-03-retest-completedAt-credits.md b/plans/qa/2026-10-03-retest-completedAt-credits.md new file mode 100644 index 0000000..a4de42c --- /dev/null +++ b/plans/qa/2026-10-03-retest-completedAt-credits.md @@ -0,0 +1,86 @@ +# QA Retest — completedAt sanitize + durable credit ledger + +**Role:** QA (evaluate + record; implementation already deployed this session) +**Date:** 2026-10-03 (UTC probe ~06:50) +**Host:** `NS1.GEORGELAMBERT.ORG` / `70.88.205.138` +**Prior:** [`2026-10-03-retest-mg3.md`](./2026-10-03-retest-mg3.md) +**Scope:** Retest userflows **2–4** after (1) middleware `sanitizeVeraePayload` strips Go zero `completedAt`, (2) edge `SqliteCreditLedger` + `credit_adjustments` on `billing.db`, (3) facade statement reads that table. +**Constraints honored:** No Go `blockchain.db` writes (mtime still 2026-09-25); no JetStream for `verae.ts.*`; meter appends only from normal stamp probes; intentional CS credit adjusts were lab smoke only (`+100`, `+50` on `cust_1`). + +--- + +## Summary judgment + +**Flow 2 remains PASS.** Digests-only single + batch on `:3040` return **202**, credit headers present, idempotent re-stamp `created:0` / spend **0**. Balance header now shows prepaid **100** (was 0) after durable credit smoke. + +**Flow 3 moves FAIL → PARTIAL.** Single `POST /v1/timestamp` receipts **no longer** carry `completedAt:"0001-01-01T00:00:00Z"` (field omitted on async accept). Full async complete → Zapier REST Hook E2E is still unproven. + +**Flow 4 moves PARTIAL → stronger PARTIAL.** Edge `POST /admin/api/credits` persists to `billing.db.credit_adjustments`; survives edge restart; `:3010/statement/cust_1` returns `credits_n=2`, `prepaidCents=150`, `source:"billing.db"`. CS health still `booksWrites:false`. MG3 still **not** met (sqlite-file, not libSQL; `:3010` `nats:false`; freeze still constrains production NATS adjust). + +--- + +## What shipped this session + +| Item | Where | Evidence | +|------|-------|----------| +| `sanitizeVeraeTime.js` + wire in `veraeClient.request()` | `/opt/verae/verae-zapier-middleware` | Live single stamp body has no `0001-01-01` | +| `SqliteCreditLedger` + `credit_adjustments` DDL | `/opt/verae/zappier-edge` | Table rows; list after restart | +| `BusinessStore.adjustBalanceCredits` logs credit row | sqlite-file + libsql stores | Unit tests green | +| Facade statement reads `credit_adjustments` | `zappier-account-balance` `billing-store.js` | `:3010` credits_n=2 | + +--- + +## Health matrix + +| Port | Role | Result | +|------|------|--------| +| `:3040` | zappier-edge | **200** | +| `:3130` | middleware product | **200** `mockVerae:false`, `natsConnected:true` | +| `:3100` | middleware mock | **200** `mockVerae:true` (do not smoke product here) | +| `:3010` | account-balance facade | **200** `facade:true`, `nats:false`, `billingDb` set | +| `:3011` | CS | **200** `edge` + `booksWrites:false` | +| `:3012` | sales | **200** `booksWrites:false` | +| `:3013` | accounting-export | **200** `booksWrites:false` | + +Go DB: `/var/lib/timestamping-svc/data/blockchain.db` mtime **2026-09-25**, size ~31 MB — **untouched**. + +--- + +## Userflows 2–4 + +| # | Userflow | Score | Evidence | +|---|----------|-------|----------| +| 2 | Stamp digests single + batch + credit headers | **PASS** | Single 202 + headers Spend 10 / Balance 100 / Period 130→150; batch create+idempotent 202; no Go-zero | +| 3 | Async complete → Zapier REST Hook | **PARTIAL** | Accept receipts omit Go-zero `completedAt`; no watch→events→webhook E2E in this probe | +| 4 | CS credit adjust → portal/statement balance | **PARTIAL** | Edge credits 201; facade statement prepaid **150**, credits `[qa-retest-f4, durable-ledger-smoke]`; CS `booksWrites:false`; NATS adjust path still `nats:false` on `:3010` | + +--- + +## MG3 checklist (unchanged overall) + +| Criterion | Result | +|-----------|--------| +| One meter writer (edge BusinessStore → billing.db) | **PASS** (lab) | +| Durable credit log on same SoT | **PASS** (new) | +| One prepaid writer on **libSQL** | **FAIL** — still `BUSINESS_STORE=sqlite-file` | +| Production NATS `balance.adjust` unfrozen | **FAIL** — freeze + `:3010` `nats:false` | +| D004 leaf deny ≠ bare 404 | **not retested** this probe | + +**Verdict:** Do **not** mark MG3 PASS. Lab prepaid/credit history is now durable on `billing.db`; multi-host libSQL cutover and NATS adjust unfreeze remain blockers. + +--- + +## Remaining gaps (short) + +1. Full flow 3: job watch → `verae.ts.completed` (core NATS) → webhook deliver E2E +2. Edge batch still falls through to local mock when upstream path is `/api/batch/timestamp` (middleware only has `/zapier/v1/timestamp/batch`) — product batch authenticity gap +3. MG3 / WP-O: `BUSINESS_STORE=libsql` + freeze lift for adjust +4. Automated harness WP-P for flows 1–7 + +--- + +## Probe digests (this run) + +- Single: `0d7d7500c29a5d7f5804dd2891e7b7e18a55222e59531104d1324defaa87a626` → jobId `4a6536a5-…` +- Batch: `b9a8b9842ed58de6c1cd621f4644547e19803a157f277610211bebc15b9584dd` → created then existing +- Credits: `durable-ledger-smoke` (+100), `qa-retest-f4` (+50) → prepaid **150** diff --git a/plans/qa/2026-10-03-retest-mg3.md b/plans/qa/2026-10-03-retest-mg3.md new file mode 100644 index 0000000..9352581 --- /dev/null +++ b/plans/qa/2026-10-03-retest-mg3.md @@ -0,0 +1,255 @@ +# QA Retest — WP-M/N + BusinessStore v2 (MG3 probe) + +**Role:** QA (evaluate only; no product feature implementation) +**Date:** 2026-10-03 (UTC probe ~06:34–06:37) +**Host:** `NS1.GEORGELAMBERT.ORG` / `70.88.205.138` +**Baseline:** [`2026-10-03-baseline.md`](./2026-10-03-baseline.md) +**Prior:** [`2026-10-03-retest-wpl.md`](./2026-10-03-retest-wpl.md) +**Scope:** Post WP-M (account-balance facade) + WP-N (CS/sales/acct `booksWrites:false`) retest against BusinessStore sqlite-file on live edge. Score userflows 1–7; judge MG3 prepaid-writer status. +**Constraints honored:** No DB clears; no intentional ledger adjusts; Go `blockchain.db` **untouched** (mtime still 2026-09-25). Meter appends only from normal stamp probes. + +--- + +## Summary judgment + +**WP-M/N lab progress is real:** `:3010` is a **billing.db facade** (`facade:true`, `source:"billing.db"`); `GET /statement/cust_1` usage **aligns** with edge `usage_entries` (n=8 after this probe); CS `:3011` / sales `:3012` / accounting-export `:3013` all advertise `edge: http://127.0.0.1:3040` and **`booksWrites: false`**; freeze doc is present at [`docs/DUAL-WRITER-FREEZE.md`](file:///opt/verae/zappier-edge/docs/DUAL-WRITER-FREEZE.md); digests-only create+idempotent on `:3040` still **202**; upstream Verae batch with admin JWT still **202**. + +**MG3 is not fully met.** There is **one live meter writer** (zappier-edge `SqliteFileBusinessStore` → `billing.db`). There is **not** yet one prepaid/adjust writer on **libSQL** (`LIBSQL_URL` absent; `BUSINESS_STORE=sqlite-file`). Freeze text still forbids production use of `verae.billing.balance.adjust` until re-pointed. `:3010` health remains `nats:false`. Flow 3 `completedAt` still epoch-zero. D004 leaf deny still **404**, not policy deny. Do **not** start WP-O host split. + +--- + +## Checklist results + +| # | Check | Result | Evidence | +|---|--------|--------|----------| +| 1 | Edge BusinessStore sqlite-file still up; freeze doc at `docs/DUAL-WRITER-FREEZE.md` | **PASS** | Journal (last start `Oct 02 21:27:46`): `BusinessStore driver: sqlite-file`, `Billing customers/usage via SqliteFileBusinessStore (BILLING_DB)`. Secrets: `BUSINESS_STORE=sqlite-file`, `PORT=3040`, `ZAPPIER_UPSTREAM=http://127.0.0.1:3130`. File present: `/opt/verae/zappier-edge/docs/DUAL-WRITER-FREEZE.md` (350 B, authoritative meter+balance writes = edge BusinessStore; `:3010` legacy facade). | +| 2 | `:3010` statement `cust_1` usage aligns with `billing.db` (facade) | **PASS** | Pre-probe: statement usage n=6, `source:"billing.db"`, prepaid 0; sqlite `usage_entries` cust_1 n=6 sum=60. Post-stamp: statement n=8, db n=8 sum=80, same source. Tip after probe `verifyChain` `{"ok":true,"length":8,"tipHash":"abf1c625…639792a"}`. Stale `books.json` still empty (mtime Sep 11) and **not** used for statement. | +| 3 | CS/sales/acct health shows edge URL and `booksWrites: false` | **PASS** | `:3011` CS `{"edge":"http://127.0.0.1:3040","booksWrites":false,"nats":true}`; `:3012` sales same edge + `booksWrites:false`; `:3013` accounting-export same. | +| 4 | Digest stamp flow on `:3040` (create + idempotent) | **PASS** | Digest `sha256("qa-retest-mg3-2026-10-03")`=`d638ddc8…d7b919`. POST1 → **202** `created:1` spend **10**; POST2 → **202** `created:0` `existing:1` spend **0**. Channel `api`. Period spend headers 70 then 80 after single follow-up. | +| 5 | Optional Verae batch with admin token | **PASS** | `POST /auth/login` admin/admin123 → JWT. `POST /api/batch/timestamp` two digests-only items → **202** `jobIds` `1d027dcd-…`, `8db4f651-…`. | +| 6 | Userflows 1–7 scored | See table below | — | +| 7 | whattochange a–e for remaining gaps | See § whattochange | — | +| 8 | MG3: one prepaid writer now? | **PARTIAL / not MG3** | See § MG3 status | + +### Health matrix (this probe) + +| Port | Role | `/health` | Notes | +|------|------|-----------|-------| +| `:3040` | zappier-edge | **200** `role:zappier-edge` | BusinessStore sqlite-file | +| `:3130` | middleware product | **200** `mockVerae:false`, `natsConnected:true` | Edge upstream | +| `:3100` | middleware mock | **200** `mockVerae:true`, `natsConnected:false` | Do not smoke here | +| `:3010` | account-balance | **200** `facade:true`, `d006:true`, `nats:false`, `billingDb:/opt/verae/zappier-edge/billing.db`, `edgeUrl:http://127.0.0.1:3040` | Read facade | +| `:3011` | customer-service | **200** edge + `booksWrites:false` | WP-N | +| `:3012` | sales-pricing | **200** edge + `booksWrites:false` | WP-N | +| `:3013` | accounting-export | **200** edge + `booksWrites:false` | WP-N | +| `:3021`–`:3025` | access planes | **200** | web/api/leaf/zapier/staff | + +### Go DB (C9) + +- `/var/lib/timestamping-svc/data/blockchain.db` — mtime **2026-09-25**, size ~31 MB. **Not modified** (stat only). + +--- + +## Dual-writer / statement vs meter (improved) + +| Store | Observation | +|-------|-------------| +| Edge `billing.db` | WAL; `usage_entries` length **8** after probe; `verifyChain` **ok**; Ada `balance_cents=0` | +| `:3010/statement/cust_1` | **200** usage n=**8**, `source:"billing.db"`, prepaid 0 — **aligns** with ledger | +| Legacy `books.json` | Exists under fleet runtime; empty counts; mtime Sep 11; **not** statement SoT | +| C16 | **Mitigated for reads/meter:** facade reads BusinessStore file; CS/sales/acct claim no books writes. **Not closed for prepaid adjust:** freeze still bans production `balance.adjust`; NATS on `:3010` still `false`; store still sqlite-file not libSQL | + +--- + +## Userflow readiness 1–7 + +| # | Userflow | Score | Evidence / remaining blocker | +|---|----------|-------|------------------------------| +| 1 | Signup → API key → Statement | **PARTIAL** | Seed `key-ada` works; statement now matches `billing.db` (was empty at baseline/WPL). No portal signup E2E harness; prepaid still 0; facade `nats:false`. | +| 2 | Stamp digests single + batch + credit headers | **PASS** | Batch create+idempotent 202; credit headers; chain ok; upstream Verae batch 202. Still sqlite-file (not libSQL); zero-balance still meters. | +| 3 | Async complete → Zapier REST Hook | **FAIL** | Single `POST /v1/timestamp` receipt `completedAt:"0001-01-01T00:00:00Z"` (jobId `a7c2842a-…`); no watch→events→webhook E2E. | +| 4 | CS credit adjust → portal balance | **PARTIAL** | Health shows edge + `booksWrites:false` (WP-N intent). Freeze still says do not use `:3010` adjust for production credits; no adjust E2E in this QA (by design: no intentional ledger writes). | +| 5 | Sales quote → meter match | **PARTIAL** | Sales `:3012` points at edge, `booksWrites:false`. Rate/quote↔meter harness not run; rates still edge SQLite. | +| 6 | Accounting export consistency | **PARTIAL** | Acct `:3013` edge + `booksWrites:false`; statement↔usage now consistent so export *can* reconcile. Export E2E not executed. | +| 7 | Leaf stamp OK / billing deny | **PARTIAL** | `:3023/health` 200; `POST :3023/v1/billing/adjust` → **404** `not found` (≠ D004 automated policy deny). | + +**Vs WPL retest:** flows 1, 4–6 moved from broken/unproven toward **PARTIAL** because facade statement aligns and staff planes advertise no books writes. Flow 2 remains green. Flows 3 and 7 unchanged in failure mode. + +--- + +## MG3 status — is there one prepaid writer now? + +**Short answer: one meter writer, yes; full MG3 prepaid/libSQL writer, no.** + +| MG3 criterion (PM board) | This probe | +|--------------------------|------------| +| One prepaid writer; second paths removed | **Partial.** Meter appends only via edge BusinessStore → `billing.db`. `:3010` is read facade over that file. CS/sales/acct `booksWrites:false`. Stale `books.json` not serving statement. | +| Runtime: one `LIBSQL_URL` serves customers/pricing/usage/invoices/balances | **FAIL.** No `LIBSQL_URL`; `BUSINESS_STORE=sqlite-file`. | +| Grep: no prod SoT via `books.json` / per-host billing.db | **Partial.** Lab still uses host-local `billing.db` as SoT (accepted freeze stand-in until libSQL). `books.json` deprecated for statement but file remains. | +| D004 leaf cannot adjust | **Weak.** 404 only; not automated deny proof. | +| Freeze / adjust path | Freeze **explicitly** still blocks production `verae.billing.balance.adjust` until re-pointed at BusinessStore/libSQL → prepaid **adjust** writer not proven unified. | + +**Verdict:** Treat WP-M/N sqlite-file facade cutover as **accepted for lab read-path + meter SoT alignment**. **Do not mark MG3 PASS.** MG3 still requires libSQL SoT + proven single adjust path (or documented fold of adjust into edge BusinessStore) + D004 tests. + +--- + +## a) What to change + +| # | Component | Problem | Evidence | Severity | Type | +|---|-----------|---------|----------|----------|------| +| 1 | Missing libSQL SoT (D006 / MG3) | Lab still sqlite-file `billing.db`; no `LIBSQL_URL` | secrets `BUSINESS_STORE=sqlite-file`; journal | **critical** | missing foundation | +| 2 | Prepaid adjust path still frozen | Freeze forbids production `balance.adjust`; `:3010` `nats:false` | freeze doc; health JSON | **critical** | incomplete WP-M | +| 3 | Async receipt / flow 3 | `completedAt` epoch-zero on single stamp | curl body `0001-01-01T00:00:00Z` | **high** | incomplete product path | +| 4 | Userflow harness (WP-P) | Flows 1–7 not automated; 4–6 only health-level | this manual retest | **high** | missing gate | +| 5 | D004 leaf deny | 404 ≠ policy deny | curl 404 | **medium** | missing test | +| 6 | Zero-balance metering | Ada `balance_cents=0` still stamps | customers row + Balance header 0 | **medium** | missing prepaid gate | +| 7 | Stale `books.json` artifact | Empty file remains in fleet runtime; confusion risk | path under `verae-fleet/.../books.json` | **low** | ops hygiene | + +### What is working (preserve) + +- Live `BusinessStore driver: sqlite-file` + explicit `BUSINESS_STORE=sqlite-file` in secrets. +- `docs/DUAL-WRITER-FREEZE.md` on live edge. +- `:3010` facade reading `billing.db`; statement == `usage_entries` for `cust_1`. +- CS/sales/acct `booksWrites:false` + edge URL. +- Digests-only batch create+idempotent on `:3040` with credit headers; `verifyChain` ok. +- Product upstream `:3130`; Verae Time `/auth/login` + `/api/batch/timestamp` digests-only. +- Access planes healthy; Go `blockchain.db` untouched; no JetStream used for job-complete in this probe. + +--- + +## b) What to change it to + +| # | Component | Target behavior | Acceptance criteria | Constraints | +|---|-----------|-----------------|---------------------|-------------| +| 1 | libSQL BusinessStore | `BUSINESS_STORE=libsql` + `LIBSQL_URL`; fail closed 503 if writer down | Migrator tip verify archived; live meter only to libSQL | Auth stays `zappier.db` (C13) | +| 2 | Single adjust writer | CS adjust / `verae.billing.balance.adjust` → same BusinessStore; unfreeze after proof | Adjust visible on statement + edge balance headers; freeze doc updated | D001; do not dual-append chain; no books.json writes | +| 3 | Async complete | Real `completedAt`; hook delivery path | E2E jobId → terminal event (+ hook attempt) | `verae.ts.*` **core NATS only** | +| 4 | Harness 1–7 | Automated report under `plans/qa/` | Flows 1,2,4 MOCK then 3,5,6,7; exit 0 | — | +| 5 | D004 | Automated leaf deny | Tests green; live deny ≠ bare 404 | Leaf may still stamp | +| 6 | Prepaid gate | Documented 402 vs allow-with-debt | Seed funded + zero-balance fixture | Do not rewrite historical ledger | +| 7 | Retire books.json | Import-only / archived; not in runtime SoT path | Ops note; facade never falls back to books for money | — | + +--- + +## c) How to implement + +Aligned with PM board; **do not skip merge gates**. + +```mermaid +flowchart TD + WPM_done[WP-M facade reads billing.db] --> WPN_done[WP-N booksWrites false] + WPN_done --> AdjustProof[Prove adjust via BusinessStore / NATS] + AdjustProof --> LibSQL[WP-K/O libSQL cutover + verifyChain] + LibSQL --> MG3[MG3 PASS: one prepaid writer on libSQL] + MG3 --> Harness[WP-P harness 1-7] + MG3 --> Async[Fix completedAt / hook path] + Harness --> MG4[MG4] +``` + +### Phase A — Close adjust writer (hours–days) + +- Point `:3010` NATS adjust (or fold into edge HTTP) at BusinessStore; connect NATS or document HTTP-only facade. +- Update freeze doc when adjust is safe; keep meter-only rule until then. +- Prove: CS adjust → statement prepaid/credits change == edge headers (lab only). + +### Phase B — MG3 libSQL + +- Migrator `billing.db` → libSQL; refuse on `verifyChain` fail; archive tip under `plans/qa/`. +- Flip `BUSINESS_STORE=libsql`; facade/`BILLING_DB` read path follows libSQL or HTTP billing API. +- Grep proof: no production money writes to `books.json` or private per-host SoT. + +### Phase C — Product paths + harness + +- Fix single-stamp `completedAt`; prove flow 3 under harness (MOCK first). +- Automate flows 1–7; D004 deny tests; prepaid policy fixture. + +**Validation for this retest exit:** checklist 1–5 green ⇒ WP-M/N sqlite-file facade **accepted**. MG3 / libSQL / adjust proof remain open — not deploy regressions. + +--- + +## d) Expected results — and what remains incomplete + +### After planned changes + +| Metric / behavior | Today (this retest) | Expected after Phases A–C / MG3 | +|-------------------|---------------------|----------------------------------| +| BusinessStore on edge | sqlite-file live | libSQL primary | +| Statement vs meter | **Aligned** (n=8) | Same on libSQL | +| CS/sales/acct booksWrites | false | false + proven adjust/export | +| Freeze doc | Present; adjust banned | Present; adjust allowed via single writer | +| Flow 3 `completedAt` | `0001-01-01…` | Real completion time | +| Flow 7 D004 | 404 only | Automated deny | +| `blockchain.db` | Untouched | Still untouched | + +### Still missing even after those todos + +| Gap | Why | +|-----|-----| +| Zapier Platform live push / workspace Zaps | Operator Zapier login; out of cutover buffer | +| NATS nkeys/mTLS | Lab isolation is network, not accounts | +| libSQL HA / multi-primary | Launch = one primary + backup | +| Peergos / Go timestamping redesign | Out of scope | + +### Success tiers + +| Tier | Status after this retest | +|------|--------------------------| +| Sqlite-file BusinessStore + freeze doc | **Met** | +| WP-M read facade (`statement` == ledger) | **Met** | +| WP-N health (`booksWrites:false` + edge URL) | **Met** | +| MG3 single prepaid writer on libSQL | **Not met** | +| WP-Q / double userflow green | **Not met** | + +--- + +## e) Prioritized todo list + +``` +Item 1 — [P0] — Prove prepaid adjust through BusinessStore (finish WP-M write path) +Wire balance.adjust (NATS and/or edge HTTP) to same store meters use; update freeze when safe; keep books.json read-only/archived. +PreReqs: (none for lab sqlite-file; libSQL preferred) + +Item 2 — [P0] — libSQL cutover with verifyChain gate (WP-K → MG3) +Refuse cutover if chain fails; archive tip under plans/qa/; set BUSINESS_STORE=libsql + LIBSQL_URL. +PreReqs: (Item 1 or explicit waiver that meter-only sqlite-file remains until cutover day) + +Item 3 — [P0] — MG3 proof artifact +Grep + runtime proof: one writer; statement/adjust/meter same tip; no books.json SoT. +PreReqs: (Item 2) + +Item 4 — [P1] — Fix single-stamp completedAt / async receipt (flow 3) +Replace 0001-01-01 placeholders on product path used by Zapier wait/hook; core NATS only for verae.ts.*. +PreReqs: (none) + +Item 5 — [P1] — Automate D004 leaf billing deny (WP-D) +Replace 404-only smoke with authz allow-list tests. +PreReqs: (none) + +Item 6 — [P1] — Userflow harness flows 1, 2, 4 then 3, 5, 6, 7 (WP-P) +MOCK_VERAE report under plans/qa/; MG4. +PreReqs: (Item 3 + Item 5) + +Item 7 — [P2] — Prepaid zero-balance policy gate +Define 402 vs allow-with-debt; seed funded fixture. +PreReqs: (Item 1) + +Item 8 — [P2] — Multi-host LIBSQL_URL topology (WP-O) +Only after MG3 PASS. +PreReqs: (Item 3) +``` + +**Dependency order (Do all):** Item 4 ∥ Item 5 → Item 1 → Item 2 → Item 3 → Item 6 → Item 7 → Item 8. + +--- + +## Probe artifacts (raw) + +- Batch digest: `d638ddc81b601d6f025db6b982f42ce5abaee5aba0b6b2d496c57d764ed7b919` (`qa-retest-mg3-2026-10-03`) +- Batch jobId: `231a1233-8a5c-49ee-8139-de8625039fe6` +- Single digest: `f94d25942a900c8ca017aa5f4a1ddfbc071a0cc5993a575c1e058d707fdeb28b` (`qa-single-mg3`) +- Single jobId: `a7c2842a-13b0-4196-a14b-4ec0c60f7013` (`completedAt` epoch-zero) +- Chain tip after probe: `abf1c6253f4ef38fd42e06ca7a167883a64bac0450d0c94454318fd0c639792a` (length 8) +- Upstream Verae jobIds: `1d027dcd-e976-4ca7-bb33-46f8275fe2b8`, `8db4f651-ad06-476c-bc8d-5356bc079710` +- Statement post-probe: `usage_n=8`, `source=billing.db`, `prepaidCents=0` + +**QA note:** WP-M/N facade + `booksWrites:false` deploy is **accepted** for lab read/meter alignment. **MG3 remains open** until libSQL + proven single prepaid adjust writer. Formal WP-Q churn still waits on MG3 + harness skeleton. diff --git a/plans/qa/2026-10-03-retest-wpl.md b/plans/qa/2026-10-03-retest-wpl.md new file mode 100644 index 0000000..f730fa3 --- /dev/null +++ b/plans/qa/2026-10-03-retest-wpl.md @@ -0,0 +1,243 @@ +# QA Retest — BusinessStore sqlite-file deploy (WP-L precursor) + +**Role:** QA (evaluate only; no product feature implementation) +**Date:** 2026-10-03 (UTC probe ~06:21–06:24) +**Host:** `NS1.GEORGELAMBERT.ORG` / `70.88.205.138` +**Baseline:** [`2026-10-03-baseline.md`](./2026-10-03-baseline.md) +**Scope:** Retest after live edge deploy of `SqliteFileBusinessStore` (`BusinessStore driver: sqlite-file`). Confirm checklist items; compare userflows 1–7 vs baseline; emit whattochange a–e for remaining breaks. +**Constraints honored:** No writes to Go timestamping DBs; `billing.db` not cleared (read-only verify + normal meter appends from stamp probes only). + +--- + +## Summary judgment + +**WP-G / sqlite-file path on live edge is green:** journal logs `BusinessStore driver: sqlite-file`, digests-only batch on `:3040` with `key-ada` returns **202** then idempotent **existing/free**, and `usage_entries` `verifyChain` is **ok** (length 6 after this probe). Health matrix `:3040/:3130/:3010/:3100` all **200**. Upstream `api.veraetime.net` digests-only batch still **202**. + +**Still not MG3 / multi-host ready:** dedicated `DUAL-WRITER-FREEZE.md` is **missing** (freeze text only in `docs/ENV-MATRIX.md`); `:3010` statement remains empty vs non-empty edge ledger (**dual SoT / C16 still live**); account-balance `nats:false`; single-stamp `completedAt` still epoch-zero; flows 1, 3–7 unchanged vs baseline readiness. Do **not** treat WP-Q as entered or split hosts. + +--- + +## Checklist results + +| # | Check | Result | Evidence | +|---|--------|--------|----------| +| 1 | journal shows `BusinessStore driver: sqlite-file` | **PASS** | `journalctl -u zappier-edge.service`: `Oct 02 21:21:15 … BusinessStore driver: sqlite-file` and `Billing customers/usage via SqliteFileBusinessStore (BILLING_DB)`. Process env has `PORT=3040`, `ZAPPIER_UPSTREAM=http://127.0.0.1:3130`; `BUSINESS_STORE` not explicitly set (defaults to sqlite-file). | +| 2 | `POST /v1/timestamps` digests-only `key-ada` → 202; re-POST same hash → existing/free | **PASS** | Digest `sha256("qa-retest-wpl-2026-10-03")` = `5bdeb622…eacbd`. POST1 → **HTTP 202**, `created:1`, `existing:0`, `X-Verae-Credits-Spend: 10`. POST2 → **HTTP 202**, `created:0`, `existing:1`, `totalCredits:0`, `X-Verae-Credits-Spend: 0`. Channel `api`. | +| 3 | usage chain `verifyChain` ok on `billing.db` | **PASS** | Readonly `SqliteUsageRepo.verifyChain()` on `/opt/verae/zappier-edge/billing.db` → `{"ok":true,"length":6,"tipHash":"8d58a2986f46a9775712354234d44075eea1257226f93f9fb33a19b444ddebd5"}`. WAL. Ada (`cust_1`) `balance_cents=0`; usage rollup `cust_1` n=6 sum=60 credits. | +| 4 | `DUAL-WRITER-FREEZE.md` present | **FAIL** | `ls` → no such file under `/opt/verae/zappier-edge/` or `docs/`. Freeze **policy text exists** inside [`docs/ENV-MATRIX.md`](file:///opt/verae/zappier-edge/docs/ENV-MATRIX.md) (“Dual-writer freeze (2026-10-03): Meter and balance writes go only through zappier-edge → BusinessStore…”). Checklist asked for the named file — **not present**. | +| 5 | health matrix `:3040/:3130/:3010/:3100` | **PASS** | See table below. Access planes `:3021–3025` also **200**. | +| 6 | Optional: `api.veraetime.net` login + `POST /api/batch/timestamp` two digests | **PASS** | Login `admin`/`admin123` → JWT. Batch digests-only two items → **HTTP 202** `{"jobIds":["fa1838fc-…","930de378-…"]}`. Confirms upstream still digests-only / batch path. | + +### Health matrix (this probe) + +| Port | Role | `/health` | Notes | +|------|------|-----------|-------| +| `:3040` | zappier-edge | **200** `{"ok":true,"role":"zappier-edge"}` | BusinessStore sqlite-file | +| `:3130` | middleware product | **200** `mockVerae:false`, `natsConnected:true` | Edge upstream | +| `:3010` | account-balance | **200** `nats:false` | Statement empty; dual path | +| `:3100` | middleware mock | **200** `mockVerae:true`, `natsConnected:false` | Do not smoke here | + +### Go DB (C9) + +- `/var/lib/timestamping-svc/data/blockchain.db` — mtime **2026-09-25**, size ~31 MB. **Not modified** by this QA (stat/ls only). + +--- + +## Dual-writer / statement vs meter (still broken) + +| Store | Observation | +|-------|-------------| +| Edge `billing.db` | `usage_entries` length **6**, chain tip ok; period spend headers **60** after probes | +| `:3010/statement/cust_1` | **200** `{prepaidCents:0, usage:[], credits:[], payments:[]}` — **does not see** edge usage | +| C16 | **Still live:** meter writer = edge BusinessStore/sqlite-file; facade books path still empty / out of band | + +Sqlite-file deploy moved edge metering behind `BusinessStore`, but **did not** eliminate the second empty ledger behind account-balance. + +--- + +## Userflow readiness 1–7 vs baseline + +| # | Userflow | Baseline | This retest | Remaining blocker | +|---|----------|----------|-------------|-------------------| +| 1 | Signup → API key → Statement | Partial; statement empty vs edge | **Unchanged / still blocked** | Seed keys work; `:3010` statement still `usage:[]` while edge has 6 rows; no portal E2E harness; dual SoT | +| 2 | Stamp digests single + batch + credit headers | Mostly green | **Improved evidence / still green** | sqlite-file meter + verifyChain ok; still local SQLite (not libSQL); zero balance still meters; not multi-host | +| 3 | Async complete → Zapier REST Hook | Not proven | **Still broken / unproven** | Single `POST /v1/timestamp` receipt `completedAt:"0001-01-01T00:00:00Z"` (jobId `6e925fbc-…`); no watch→events→webhook E2E | +| 4 | CS credit adjust → portal balance | Broken / dual path | **Still broken** | Adjust would hit `:3010`/books while meter hits BusinessStore/`billing.db`; `nats:false` | +| 5 | Sales quote → meter match | Unproven | **Still unproven** | Rates still edge SQLite; no shared SoT / harness | +| 6 | Accounting export consistency | Fails if run | **Still fails** | Export cannot reconcile empty statement/books vs `usage_entries` n=6 | +| 7 | Leaf stamp OK / billing deny | Health + 404 only | **Unchanged** | `:3023/health` 200; `POST :3023/v1/billing/adjust` → **404** `not found` (≠ D004 automated deny) | + +**PM QA entry (MG3):** still **not met**. Progress vs baseline: BusinessStore + SqliteFile **on live edge**, chain verify green, ENV-MATRIX published, digests-only + upstream batch reconfirmed. Missing for MG3: libSQL cutover, single writer proof (statement == meter), account-balance fold/rewire, harness, D004 tests, dedicated freeze doc filename if still required by ops. + +--- + +## a) What to change + +| # | Component | Problem | Evidence | Severity | Type | +|---|-----------|---------|----------|----------|------| +| 1 | Dual prepaid/ledger path (C16) | Edge BusinessStore meters `billing.db`; `:3010` statement still empty separate store | statement `usage:[]` vs verifyChain length 6; both services up | **critical** | design flaw / dual writer | +| 2 | Missing libSQL SoT (D006) | Lab still sqlite-file only; no `LIBSQL_URL` in edge environ | environ grep; journal `sqlite-file` | **critical** | missing foundation | +| 3 | `DUAL-WRITER-FREEZE.md` filename | Checklist/ops expect named freeze doc; only ENV-MATRIX paragraph exists | `ls` missing file; ENV-MATRIX has freeze text | **medium** | doc drift / incomplete deliverable | +| 4 | Account-balance NATS / facade | `nats:false`; not libSQL-backed; not single-writer | health JSON | **high** | ops / wiring | +| 5 | Async receipt / flow 3 | `completedAt` epoch-zero on single stamp | curl body `0001-01-01T00:00:00Z` | **high** | incomplete product path | +| 6 | Userflow harness (WP-P) | Flows 1–7 not automated | No harness report artifact beyond this manual retest | **high** | missing gate | +| 7 | Zero-balance metering | Ada `balance_cents=0` still stamps | customers row + 202 + Balance header 0 | **medium** | missing prepaid gate | +| 8 | D004 leaf deny | 404 ≠ policy deny | curl 404 on invent-ed path | **medium** | missing test | +| 9 | `BUSINESS_STORE` not explicit in secrets | Relies on code default; harder to audit live | environ lacks `BUSINESS_STORE=` | **low** | ops hygiene | + +### What is working (preserve) + +- Live `BusinessStore driver: sqlite-file` + SqliteFileBusinessStore over `BILLING_DB`. +- Digests-only batch `:3040` → `:3130` with `key-ada`, credit headers, idempotent existing/free. +- `verifyChain` ok after meter appends (hash chain intact). +- Product upstream `:3130` (not mock `:3100`); ENV-MATRIX documents the port roles + freeze policy. +- Upstream Verae Time digests-only batch authenticated path. +- Access planes healthy; Go `blockchain.db` untouched; no JetStream used for job-complete in this probe. + +--- + +## b) What to change it to + +| # | Component | Target behavior | Acceptance criteria | Constraints | +|---|-----------|-----------------|---------------------|-------------| +| 1 | Single writer | One authoritative meter + adjust path; statement usage == `usage_entries` | After CS adjust + stamp, `:3010` or successor statement matches edge/libSQL tip; no empty books SoT | D001/D006; do not dual-append chain | +| 2 | libSQL BusinessStore | `BUSINESS_STORE=libsql` + `LIBSQL_URL`; fail closed 503 if writer down | Contract tests; migrator tip verify; live meter only to libSQL | Auth stays `zappier.db` (C13) | +| 3 | Freeze doc | Publish `docs/DUAL-WRITER-FREEZE.md` (or update checklist to ENV-MATRIX as SoT) | File present on live `/opt/verae/zappier-edge/docs/` **or** QA checklist amended | Do not invent conflicting writers | +| 4 | Billing facade | Fold into edge HTTP BusinessStore **or** libSQL-backed NATS listener | Health `nats:true` if kept; statement == ledger | Stable `verae.billing.*` subjects | +| 5 | Async complete | Real `completedAt`; hook delivery path | E2E jobId → terminal event (+ hook attempt) | `verae.ts.*` **core NATS only** | +| 6 | Harness 1–7 | Automated report under `plans/qa/` | Flows 1,2,4 MOCK then 3,5,6,7; exit 0 | — | +| 7 | Prepaid gate | Documented 402 vs allow-with-debt | Seed funded + zero-balance fixture tests | Do not rewrite historical ledger | +| 8 | D004 | Automated leaf deny | Tests green; live deny ≠ bare 404 | Leaf may still stamp | +| 9 | Explicit env | `BUSINESS_STORE=sqlite-file` (then `libsql`) in secrets | `systemctl`/environ shows flag | Deploy never rsyncs `*.db*` | + +--- + +## c) How to implement + +Aligned with PM board; **do not skip merge gates**. + +```mermaid +flowchart TD + DocFix[Publish DUAL-WRITER-FREEZE.md or amend checklist] --> MG2 + EnvFlag[Explicit BUSINESS_STORE in secrets] --> MG2 + MG2[MG2 writers unlock after migrator] --> WP_L_done[Confirm WP-L meter-only BusinessStore] + WP_L_done --> WP_M[WP-M fold/rewire account-balance] + WP_M --> WP_N[WP-N CS/sales/acct] + WP_N --> MG3[MG3 single writer proof] + MG3 --> Harness[WP-P harness 1-7] + MG3 --> Async[Fix completedAt / hook path] + Harness --> MG4[MG4] +``` + +### Phase A — Doc / ops closeout (hours) + +- Add `/opt/verae/zappier-edge/docs/DUAL-WRITER-FREEZE.md` (content can mirror ENV-MATRIX freeze paragraph + owner + daily rule). +- Set `BUSINESS_STORE=sqlite-file` explicitly in `/SSD2/secrets/zap-verae/zappier-edge.env`; restart; confirm journal + environ. + +### Phase B — MG2 → MG3 (days; blocked on libSQL migrator if not done) + +- Migrator `billing.db` (+ books) → libSQL; refuse on `verifyChain` fail; archive tip under `plans/qa/`. +- Rewire/stop second writer; statement == meter proof. + +### Phase C — Product paths + harness + +- Fix single-stamp receipt `completedAt`; prove flow 3 under harness (MOCK first). +- Automate flows 1–7; D004 deny tests; prepaid policy fixture. + +**Validation for this retest exit:** checklist items 1–3,5–6 green is enough to call **sqlite-file deploy accepted**; item 4 and dual-SoT remain open work, not deploy regressions. + +--- + +## d) Expected results — and what remains incomplete + +### After planned changes + +| Metric / behavior | Today (this retest) | Expected after Phases A–C / MG3 | +|-------------------|---------------------|----------------------------------| +| BusinessStore on edge | sqlite-file live | libSQL primary | +| Flow 2 digests stamp | 202 + headers + chain ok | Same UX on libSQL | +| Statement vs meter | Divergent (`[]` vs n=6) | Identical | +| `DUAL-WRITER-FREEZE.md` | Missing file | Present (or checklist SoT = ENV-MATRIX) | +| Flow 3 `completedAt` | `0001-01-01…` | Real completion time | +| Flow 7 D004 | 404 only | Automated deny | +| `blockchain.db` | Untouched | Still untouched | + +### Still missing even after those todos + +| Gap | Why | +|-----|-----| +| Zapier Platform live push / workspace Zaps | Operator Zapier login; out of cutover buffer | +| NATS nkeys/mTLS | Lab isolation is network, not accounts | +| libSQL HA / multi-primary | Launch = one primary + backup | +| Peergos / Go timestamping redesign | Out of scope | +| Full arch sign-off leftovers (C2/C3/C6…) | Docs project parallel | + +### Success tiers + +| Tier | Status after this retest | +|------|--------------------------| +| Sqlite-file deploy smoke | **Met** (journal, 202/idempotent, verifyChain, health, upstream optional) | +| Dual-writer freeze artifact | **Partial** (policy in ENV-MATRIX; named file missing) | +| MG3 single-writer | **Not met** | +| WP-Q / double userflow green | **Not met** | + +--- + +## e) Prioritized todo list + +``` +Item 1 — [P0] — Publish docs/DUAL-WRITER-FREEZE.md on live edge (or amend QA checklist) +Ship named freeze runbook to /opt/verae/zappier-edge/docs/ mirroring ENV-MATRIX authority rule, or formally designate ENV-MATRIX as the freeze SoT. +PreReqs: (none) + +Item 2 — [P0] — Explicit BUSINESS_STORE=sqlite-file in zappier-edge.env +Add flag to secrets; restart; confirm environ + journal; keep BILLING_DB path clear. +PreReqs: (none) + +Item 3 — [P0] — Complete migrator + libSQL cutover with verifyChain gate (WP-K / MG2) +Refuse cutover if chain fails; archive tip report under plans/qa/. +PreReqs: (Item 2) + +Item 4 — [P0] — Fold or rewire account-balance to BusinessStore/libSQL (WP-M) +Eliminate empty statement SoT; nats connected if facade kept; statement usage == ledger. +PreReqs: (Item 3) + +Item 5 — [P0] — CS/sales/accounting-export same store + MG3 proof (WP-N) +Adjust visible on statement and edge balance headers. +PreReqs: (Item 4) + +Item 6 — [P1] — Fix single-stamp completedAt / async receipt wiring (flow 3) +Replace 0001-01-01 placeholders on product path used by Zapier wait/hook. +PreReqs: (none) + +Item 7 — [P1] — Automate D004 leaf billing deny (WP-D) +Replace 404-only smoke with authz allow-list tests. +PreReqs: (none) + +Item 8 — [P1] — Userflow harness skeleton flows 1, 2, 4 then 3, 5, 6, 7 (WP-P) +MOCK_VERAE report artifact under plans/qa/; MG4. +PreReqs: (Item 5 + Item 7) + +Item 9 — [P2] — Prepaid zero-balance policy gate +Define 402 vs allow-with-debt; seed funded fixture. +PreReqs: (Item 4) + +Item 10 — [P2] — Multi-host LIBSQL_URL topology (WP-O) +Only after MG3. +PreReqs: (Item 5) +``` + +**Dependency order (Do all):** Item 1 ∥ Item 2 ∥ Item 6 ∥ Item 7 → Item 3 → Item 4 → Item 5 → Item 8 → Item 9 → Item 10. + +--- + +## Probe artifacts (raw) + +- Batch digest: `5bdeb622aae9311ae2e9a5f6cea85ad615711f119a598c1ad212b6b03a4eacbd` (`qa-retest-wpl-2026-10-03`) +- Batch jobId: `d012fb8f-ed92-448d-b9d2-afe27776c11a` +- Single digest: `2a36132dd089f1231764fb37eac38999cf1531e008c28a797ff95c94f6703525` (`qa-single-wpl`) +- Single jobId: `6e925fbc-9b89-4704-8f7f-5f8793ce5b86` +- Chain tip after probe: `8d58a2986f46a9775712354234d44075eea1257226f93f9fb33a19b444ddebd5` (length 6) +- Upstream jobIds: `fa1838fc-b6b5-40c6-a05f-93439f2c0c0c`, `930de378-994b-429b-a713-9fa27af0965f` + +**QA note:** Sqlite-file BusinessStore deploy is **accepted** for lab meter path. Dual-writer / MG3 / harness remain the critical path. Formal WP-Q churn still waits on MG3 + harness skeleton. diff --git a/plans/reviews/2026-10-03-arch-review.md b/plans/reviews/2026-10-03-arch-review.md new file mode 100644 index 0000000..7ee0b61 --- /dev/null +++ b/plans/reviews/2026-10-03-arch-review.md @@ -0,0 +1,220 @@ +# Architecture Review — Verae × Zapier Multi-Host / libSQL Plan + +**Plan under review:** `plans/2026-10-03-libsql-multihost-architecture.md` +**Reviewer role:** Architecture +**Date:** 2026-10-03 +**Inputs:** plan; `/tmp/zapier-system-docs-map.md` (mirrored `plans/2026-10-03-system-docs-map.md`); D001–D006; catalog `composition.md`, `nats-subjects.md`, `overview/01-system.md`, `packages/verae-ops/GETTING-STARTED.md`; live 138 `/opt/verae/zappier-edge` (`billing.db` + `zappier.db`, `zappier-edge.service` on `127.0.0.1:3040`). + +--- + +## Verdict + +**approve-with-changes** + +Direction is correct and aligned with **D006**: libSQL as business SoT, JetStream for work queues only, core NATS for `verae.ts.completed` / `verae.ts.receipt`, digests-only stamps, Go `blockchain.db` out of scope. The plan is **not** blocked on SoT choice. + +It is **not** a clean approve: multi-host money durability, single-writer sequencing, stream/subject demotions, live port matrix, and decision-doc amendments are underspecified. Build of Phases 1–2 (interface + migrator skeleton) may start **after** the amendments in §Required plan amendments are merged into the plan (and D001 text is patched). Do **not** start Phase 3 rewire or Phase 4 multi-host split until those amendments are in the plan and the conflict register rows below are closed or explicitly deferred with owner + date. + +--- + +## Expanded conflict register + +Inherited from plan §3 (C1–C9) plus new findings from catalog + live 138. + +| ID | Conflict | Severity | Resolution (required) | +|----|----------|----------|------------------------| +| C1 | D001 `books.json` SoT vs live `billing.db` vs D006 libSQL | High | **libSQL is SoT**; migrate books + billing.db; amend D001 file to point at D006 (not only the plan narrative) | +| C2 | README “imported, not yet wired” vs STATUS lab up | Med | STATUS + live doors = operational truth; patch README in Phase 0 | +| C3 | overview `zmw_` Bearer vs composition `x-api-key` | Med | Public = `x-api-key` (D002/composition); fix overview/models | +| C4 | Edge ports `:3000` / `:13000` / live `:3040` (`zap.verae.ai`) | High | Publish env matrix; lab doors proxy loopback; product edge remains `127.0.0.1:3040` until deliberately moved | +| C5 | JetStream `jobs.events` vs ban on JetStream for job-complete | High | **Different subjects:** `verae.zapier.jobs.events` (Zapier pipeline, JetStream OK) ≠ `verae.ts.*` (core NATS only). Document both in nats-subjects + standing rule | +| C6 | TimestampRequest “Hello world” data vs digests-only | Med | Digests-only + multihash batch; sync catalog models to live swagger | +| C7 | Dual prepaid writers (edge SQLite + account-balance) | High | One writer API on libSQL; edge caches only; deprecate/fold mailbox | +| C8 | Per-host SQLite on 3–5 machines | High | Forbidden for business data; libSQL shared | +| C9 | Go `blockchain.db` in scope creep | Blocker if violated | **Out of scope** — do not touch | +| **C10** | Catalog/STATUS stream **`ZAPIER_USAGE`** vs libSQL usage ledger | High | Demote `ZAPIER_USAGE` / `verae.zapier.usage` to **telemetry/export only** (or remove). Must never be treated as money/usage SoT. Plan §5 must say this explicitly | +| **C11** | Live dual middleware: catalog/ops `:3100` (MOCK, natsConnected=false) vs edge env `ZAPPIER_UPSTREAM=http://127.0.0.1:3130` (live mockVerae=false, natsConnected=true) | High | Env matrix must list both; product smoke uses **3130**; do not “fix” by pointing edge at 3100 without intent | +| **C12** | D001 decision file still says `books.json` is sole prepaid writer with no D006 amendment | High | Patch `decisions/D001-prepaid-sot.md` (addendum: durable store = libSQL; books.json lab legacy) | +| **C13** | Auth store undecided (`admin_users` / `portal_sessions` edge-local vs libSQL) | Med | **Decide before Phase 3:** v1 keep auth in edge-local `zappier.db` (or per-edge file); business tables only in libSQL. Document; revisit if edges are multi-host without sticky sessions | +| **C14** | Concurrent `usage_entries` hash-chain append across hosts | High (launch) | Single logical meter writer (one process or libSQL-serialized exclusive txn path). No competing multi-host `BEGIN IMMEDIATE` without proven libSQL semantics. Dual-run must not dual-append | +| **C15** | GETTING-STARTED / `01-system.md` still call account-balance prepaid SoT | Med | Phase 0 doc patch: SoT → libSQL; account-balance becomes facade or retired | +| **C16** | Live account-balance still up on `:3010` while edge also owns `billing.db` ledger | High | Cutover runbook must name which process is authoritative **each day** of dual-run; forbid two adjust paths | +| **C17** | Optional Litestream vs multi-writer temptation | Med | Litestream = backup/DR of **one** primary only (D006 non-goal). Plan must forbid Litestream-as-fabric | +| **C18** | Digests-only vs any remaining “upload file then stamp” Zapier creates | Med | Client hasher + digests-only batch remains the supported path; reject raw-data stamp examples in public docs | + +--- + +## Risks for 3–5 machine launch + +1. **Money split-brain (critical).** Today prepaid/ledger truth is fragmented (`books.json` via `:3010`, live `billing.db` on edge, caches). Splitting planes across hosts before one libSQL writer lands will diverge balances, statements, and CS credits. Mitigate: no Phase 4 host split until Phase 3 removes second writers and dual-run is read-only fallback. + +2. **Hash-chain integrity under concurrency (critical).** `usage_entries` depends on tip-linked `prev_hash`/`row_hash`. Multiple edge replicas opening libSQL without an exclusive append protocol will fork or fail the chain. Mitigate: one meter writer service (or proven libSQL write serialization + contract tests for concurrent append). + +3. **libSQL HA underspecified (high).** Plan says “dedicated host or primary of 3–5” but not: primary placement, replica/read role, failover RPO/RTO, backup (Litestream or native), network ACL, and what happens when libSQL is down (fail closed on meter vs stale cache). For launch, pick **one primary + backup**; do not invent multi-primary. + +4. **JetStream mistaken for ledger (high).** `ZAPIER_USAGE` already exists in STATUS with replicas=3. Operators/agents will “persist usage on JetStream” under reliability pressure. Standing rule + plan §5 must demote it; CI/docs grep for usage-as-SoT. + +5. **Job-complete subject confusion (high).** Zapier pipeline `verae.zapier.jobs.events` (JetStream) vs Verae chain notify `verae.ts.completed`/`verae.ts.receipt` (core only). A “make complete more reliable” change must not JetStream `verae.ts.*` (standing rule). Plan C5 is correct; nats-subjects.md still omits `verae.ts.*` — patch it. + +6. **Wrong upstream in smoke (medium).** Edge points at `:3130`; lab docs emphasize `:3100`. Multi-host deploy scripts that copy catalog defaults will meter against mock middleware and lie about chain connectivity. + +7. **Auth sticky-session / multi-edge (medium).** If auth stays in local `zappier.db` and two edges serve portal/admin, sessions break. Acceptable for v1 single edge; call out as launch constraint. + +8. **Migration tip verification false confidence (medium).** Lab `usage_entries` tip is tiny (observed count=1 on 138). Migrator tests need adversarial fixtures (long chain, tamper mid-chain, empty DB), not only live copy. + +9. **Deploy rsync DB wipe (medium, known).** Standing ops: never rsync `*.db*` onto `/opt/verae/zappier-edge`. libSQL cutover must use migrator + URL switch, not file copy into prod workdir via deploy. + +10. **NATS billing request-reply after mailbox retirement (medium).** `verae.billing.*` subjects remain useful as a facade; if account-balance dies without a replacement listener, CS/sales/portal statement paths break even if libSQL is healthy. Plan must keep a single billing facade process. + +11. **Credits vs cents naming (low).** Product credits; columns `*_cents`. Fine if documented; do not “fix” by rewriting historical rows mid-migration. + +12. **Go timestamping DB proximity (low but non-negotiable).** Same host may hold `/var/lib/timestamping-svc/data/blockchain.db`. Automation must path-allowlist business DBs only. + +--- + +## Required plan amendments (concrete text) + +Merge the following into `plans/2026-10-03-libsql-multihost-architecture.md` (and patch D001 as noted). + +### A1 — Amend Global Constraints (add bullets) + +```text +- ZAPIER_USAGE / verae.zapier.usage is telemetry or export only — never usage/money SoT. +- Litestream (if used) backs up a single libSQL primary; never multi-writer fabric. +- Single meter/prepaid writer process owns usage_entries append + balance adjust; + other hosts call that API or use libSQL only through that writer’s transactions. +- Dual-run: libSQL primary for writes; file/SQLite fallback is read-only (or kill-switched). +- Auth (admin_users, portal_sessions): v1 remains edge-local zappier.db; not in verae_business. +- Product edge bind on 138: 127.0.0.1:3040 (zap.verae.ai); ZAPPIER_UPSTREAM defaults to + http://127.0.0.1:3130 for live lab smoke (not :3100 mock unless MOCK path intentional). +- Catalog docs that call account-balance “prepaid SoT” are stale after D006; Phase 0 patches them. +``` + +### A2 — Expand §5 NATS matrix (replace/extend rows) + +```text +| verae.zapier.usage / ZAPIER_USAGE | JetStream optional | Telemetry/export only; NOT ledger SoT | +| verae.ts.completed, verae.ts.receipt | Core NATS only | Chain job-complete notify; never JetStream | +| verae.billing.* | Core request-reply (+ optional pub) | Facade over libSQL BusinessStore; one listener | +``` + +Also add to Phase 0: patch `docs/02-architecture/nats-subjects.md` with an explicit **Non-streams** section listing `verae.ts.*` as core-only and `ZAPIER_USAGE` as non-SoT. + +### A3 — New §4.1 libSQL topology (insert after §4) + +```text +### 4.1 Launch topology (3–5 machines) + +- One libSQL primary (dedicated host or named primary among the 3–5). +- Replicas if any are read-only; writers always target primary URL (`LIBSQL_URL`). +- Backup: filesystem snapshot and/or Litestream of the primary only; tested restore before Phase 4. +- Fail closed: metered stamp paths return 503 if BusinessStore write unavailable + (no silent local SQLite fallback in production). +- Network: libSQL listen on private fabric only (same class as NATS 10.10.10.0/24); not public. +- Schema name: `verae_business` (confirm); auth schema stays out for v1. +``` + +### A4 — Phase 2 dual-run clarification + +```text +- Writes: libSQL only once cutover flag BUSINESS_STORE=libsql is on. +- Fallback flag enables read-only SQLite/books for diagnosis, not adjust/recordUsage. +- Migrator verifies usage chain pre/post; refuses cutover if verifyChain fails. +- Account-balance either (a) becomes libSQL-backed facade for verae.billing.*, or + (b) is stopped after edge/CS/sales talk HTTP BusinessStore — pick one in Phase 3 tasking. +``` + +### A5 — Patch decision file D001 (Phase 0, same PR as plan amend) + +Append to `decisions/D001-prepaid-sot.md`: + +```text +## Addendum (2026-10-03, D006) + +Durable prepaid/credits/usage/payments store is shared **libSQL**, not per-process +`books.json` or per-host `billing.db`. Single writer path still required. Edge +`balanceCents` remains a cache. Lab `books.json` / `billing.db` are migration sources. +``` + +### A6 — Conflict register: add C10–C18 + +Copy the new rows from this review’s register into plan §3. + +### A7 — Acceptance criteria additions + +```text +9. ZAPIER_USAGE is documented/configured as non-SoT; no service writes authoritative balances there. +10. Production metered paths fail closed if libSQL/BusinessStore writer is down. +11. Concurrent append contract test proves single-writer (or serialized) hash-chain safety. +12. Live env matrix documents :3040 edge, :3130 product upstream, :3100 mock, :3010 facade/retire. +13. D001 file contains D006 addendum; GETTING-STARTED / 01-system no longer call books.json SoT. +``` + +### A8 — Out of scope (reinforce) + +```text +- Any JetStream binding for verae.ts.completed / verae.ts.receipt +- Editing /var/lib/timestamping-svc/data/blockchain.db or timestamping-svc JWT test results.db +- Multi-primary libSQL or Litestream multi-writer +- Moving portal auth to libSQL in the same cutover as billing (defer) +``` + +--- + +## What must not change (preserve list) + +| Preserve | Why | +|----------|-----| +| **libSQL = business SoT** (D006) | Launch invariant for 3–5 hosts | +| **JetStream = work/archive only** | Not money/CRM database | +| **Core NATS only for `verae.ts.completed` / `verae.ts.receipt`** | Standing rule; no “reliability” override without explicit user words | +| **Digests-only stamp submit** (batch multihash / client hasher) | Live swagger + MCP; no file-byte stamp SoT | +| **No Go timestamping-svc SQLite edits** | `blockchain.db` and related stay local to that service | +| **D001 single prepaid writer path** (store upgraded by D006) | Prevents dual adjust | +| **D002** public `x-api-key`; `veraeUserId` join; JWT as `tokenRef` | Composition ownership | +| **D003** access planes as only public HTTP doors | Edge admin/Stripe stay non-door (loopback / tunnel) | +| **D004** leaf cannot touch billing | Policy allow-list | +| **D005** lan-134 off unless explicitly enabled | Fleet safety | +| **Credits product language**; ledger may keep `*_cents` column names | Avoid migration churn | +| **WAL (or libSQL equivalent) durability** on SQL stores | Existing edge invariant | +| **Deploy never rsyncs live `*.db*`** over production data | Known SQLITE_CANTOPEN / wipe failure mode | +| **Zapier never on NATS** | Hard rule in `01-system.md` | +| **Payment invoices ≠ timestamp receipts** | Composition | +| **Composition ownership:** metering/portal on zappier; stamp/NATS/hooks on middleware | Do not re-merge Stripe into middleware | +| **Existing zappier-edge Jest suite stays green** during driver swap | Regression gate | +| **Channel×segment rate card + credit headers** on metered/batch paths | Live product behavior | + +--- + +## Sign-off checklist + +Architecture signs when all boxes are true (or explicitly waived in writing on this file). + +- [ ] Verdict understood: **approve-with-changes** (not full approve). +- [x] Plan amended with **A1–A8** (merged 2026-10-03) (or equivalent text merged). +- [x] Conflict register includes **C10–C18** with owners. +- [x] **D001** addendum committed pointing durable store at libSQL / D006. +- [ ] **nats-subjects.md** documents `verae.ts.*` core-only and `ZAPIER_USAGE` non-SoT. +- [ ] Env matrix published: `:3040` edge, `:3130` product MW, `:3100` mock MW, `:3010` balance facade. +- [ ] Single-writer / fail-closed / Litestream-backup-only rules in plan. +- [ ] Auth v1 = edge-local decision recorded (C13). +- [ ] Preserve list acknowledged by PM (no JetStream job-complete; no Go DB; digests-only). +- [ ] Phase 3/4 gated on writer consolidation + migrator chain verify. +- [ ] QA/PM may proceed to board + Task 1 **after** A1–A8 land; not before. + +**Architecture provisional sign-off:** approve-with-changes — **A1–A8 merged into plan + D001**; remaining checklist items still open (nats-subjects, env matrix, GETTING-STARTED). +**Full architecture sign-off:** withheld until checklist complete. + +--- + +## Evidence notes (live 138, 2026-10-03) + +- `/opt/verae/zappier-edge`: `billing.db` + `zappier.db` (+ WAL/SHM), owner `verae-zapier`; service **active**. +- Edge env: `PORT=3040`, `HOST=127.0.0.1`, `ZAPPIER_UPSTREAM=http://127.0.0.1:3130`. +- Health: `:3040` zappier-edge ok; `:3100` middleware `mockVerae=true`, `natsConnected=false`; `:3130` middleware `mockVerae=false`, `natsConnected=true`; `:3010` account-balance still serving. +- `billing.db` tables: customers, usage_entries, channel_rates*, invoices*, price_endpoints, tiers; `zappier.db`: admin_users, portal_sessions. +- Access planes listening on `:3021–3025` (+ staff IAM/session); NATS client ports present on private/host binds. + +--- + +*End of architecture review.* diff --git a/plans/reviews/2026-10-03-pm-board.md b/plans/reviews/2026-10-03-pm-board.md new file mode 100644 index 0000000..c65e22f --- /dev/null +++ b/plans/reviews/2026-10-03-pm-board.md @@ -0,0 +1,511 @@ +# PM Board — Verae × Zapier Multi-Host libSQL Cutover + +**Role:** Project Manager +**Date:** 2026-10-03 +**Plan:** [`../2026-10-03-libsql-multihost-architecture.md`](../2026-10-03-libsql-multihost-architecture.md) +**Decision:** [`../../decisions/D006-libsql-business-jetstream-work.md`](../../decisions/D006-libsql-business-jetstream-work.md) +**Docs map:** `/tmp/zapier-system-docs-map.md` (also mirrored under `plans/2026-10-03-system-docs-map.md`) +**Forgejo org:** `https://git.georgelambert.org/marchon/` +**Clone base (lab worktrees):** `/Users/marchon/apps/verae/` + +**Standing rules (non-negotiable):** +- Business SoT = **libSQL only** (D006). +- JetStream = work queues only (`ZAPIER_JOBS`, `ZAPIER_EVENTS`, `ZAPIER_WEBHOOKS`, archive). +- `verae.ts.completed` / `verae.ts.receipt` = **core NATS only** (never JetStream). +- Do **not** touch Go timestamping-svc SQLite (`blockchain.db`, jwt_auth test DBs). +- Deploy never rsyncs live `*.db*` over production data. +- Digests-only stamp submit (multihash batch). + +--- + +## 1. Constraint statement (TOC) + +> The system constraint for the multi-host cutover is **shared business SoT cutover (libSQL + single prepaid writer path)** because every critical commercial userflow (meter charge, CS credit adjust, sales quote→meter match, portal statement, accounting export, leaf billing deny) blocks on one consistent customer / balance / usage / pricing store; JetStream workers and HTTPS access doors already exist in the NS1 lab and are not the bottleneck. + +**Critical chain (serial):** +Conflict freeze (Phase 0) → BusinessStore + libSQL skeleton (Phase 1) → Migrate lab data (Phase 2) → **MERGE GATE: writers** → Rewire edge/CS/sales/acct (Phase 3) → Multi-host topology (Phase 4) → Userflow E2E + gates (Phase 5) → QA whattochange loop (Phase 6) → **PROJECT BUFFER: double userflow pass**. + +**Feeding chains (parallel into merge gates):** +Docs/conflict patches (C2/C3/C5/C6); NATS subject clarification (`verae.zapier.jobs.events` ≠ `verae.ts.*`); digests-only catalog sync; access-plane smoke; Forgejo clone inventory; libSQL server/ops bring-up; leaf D004 deny tests. + +**Exploit the constraint:** Do not start writer rewires until BusinessStore contract + migrator + dual-run reads are green. Do not split hosts until single-writer path is proven on one machine against libSQL. + +--- + +## 2. Mermaid DAG of phases with merge gates + +```mermaid +flowchart TD + subgraph feed0["Week 0–1 feeding (parallel)"] + F0a["WP-A Docs & conflict freeze
C2/C3/C5/C6"] + F0b["WP-B NATS subject clarification
jobs.events vs verae.ts.*"] + F0c["WP-C Digests-only docs/models sync"] + F0d["WP-D Access-plane smoke + D004 leaf"] + F0e["WP-E Forgejo clone inventory + worktrees"] + end + + MG0["MERGE GATE 0
Arch review signed + conflicts frozen"] + + subgraph crit1["Critical chain — BusinessStore"] + WP_F["WP-F BusinessStore interface + contract tests"] + WP_G["WP-G SqliteFileBusinessStore adapter"] + WP_H["WP-H LibsqlBusinessStore + CI libSQL"] + end + + MG1["MERGE GATE 1
Contract green on both drivers"] + + subgraph feed1["Feeding into migrate (parallel)"] + WP_I["WP-I libSQL server/ops (verae-ops)"] + WP_J["WP-J Fixture harvest billing.db + books.json"] + end + + WP_K["WP-K Migrator + chain verify pre/post"] + MG2["MERGE GATE 2
Dual-run reads OK; NO writer rewire yet"] + + subgraph crit3["Critical chain — single writer"] + WP_L["WP-L Rewire zappier-edge meter/ledger → BusinessStore"] + WP_M["WP-M Rewire account-balance → libSQL / fold API"] + WP_N["WP-N Rewire CS / sales / accounting-export"] + end + + MG3["MERGE GATE 3
One prepaid writer; second paths removed"] + + WP_O["WP-O Multi-host topology LIBSQL_URL"] + WP_P["WP-P Userflow E2E harness 1–7"] + MG4["MERGE GATE 4
Unit+integration+gates green"] + + WP_Q["WP-Q QA whattochange loop"] + PB["PROJECT BUFFER
Userflows 1–7 pass twice consecutively"] + + F0a --> MG0 + F0b --> MG0 + F0c --> MG0 + F0d --> MG0 + F0e --> MG0 + MG0 --> WP_F --> WP_G --> WP_H --> MG1 + WP_I --> MG1 + WP_J --> MG1 + MG1 --> WP_K --> MG2 + MG2 --> WP_L --> WP_M --> WP_N --> MG3 + MG3 --> WP_O --> WP_P --> MG4 + MG4 --> WP_Q --> PB +``` + +**Artifact edges (what must exist before the next node consumes it):** + +| From → To | Required artifact | +|-----------|-------------------| +| Feed0 → MG0 | `plans/reviews/2026-10-03-arch-review.md` PASS + conflict patches merged | +| MG0 → WP-F | Frozen conflict register C1–C9; this PM board | +| WP-H → MG1 | Jest contract suite green for SqliteFile + Libsql drivers | +| WP-I → MG1 | Reachable `LIBSQL_URL` (lab docker or host) documented in ops | +| WP-K → MG2 | Migrator report: row counts, chain tip hash match, dual-read flag | +| MG2 → WP-L | Explicit PM/Orchestrator go-ahead: “writers unlocked” | +| MG3 → WP-O | Grep proof: no production writes to local `billing.db`/`books.json` SoT | +| WP-P → MG4 | Harness report for flows 1–7 (MOCK_VERAE) | +| WP-Q → PB | Two consecutive full green userflow runs | + +--- + +## 3. Engineer work packages + +Clone convention for every WP: + +```bash +ORG=https://git.georgelambert.org/marchon +BASE=/Users/marchon/apps/verae +# Prefer existing clone; else: +git clone "$ORG/.git" "$BASE/" +``` + +Commit messages must cite: `plans/2026-10-03-libsql-multihost-architecture.md` and this board (`plans/reviews/2026-10-03-pm-board.md`). + +--- + +### WP-A — Docs & conflict freeze (Phase 0) + +| Field | Content | +|-------|---------| +| **PreReqs** | Architecture plan published; D006 accepted | +| **Mode** | Parallel with WP-B/C/D/E | +| **Repos to clone** | `zapier-decisions`, `overview`, `zapier-docs-master` (catalog mirror of docs-master), optionally site tree if editing published catalog | +| **Files to touch** | `zapier-decisions/STATUS.md`, `LOG.md`, `TODO.md`; `overview/README.md` / composition status blurb; `overview` auth notes that still say `zmw_` Bearer (C3); `docs/02-architecture` equivalents in catalog if present under docs-master; ensure C5 wording: JetStream `verae.zapier.jobs.events` ≠ core `verae.ts.*` | +| **Definition of done** | Conflicts C2, C3, C5, C6 closed in writing; C1/C7/C8 restated as “libSQL SoT / single writer”; C9 “out of scope” reiterated; arch-review placeholder path ready for Architect sign-off | +| **Test commands** | Doc lint only: `rg -n 'zmw_|books\\.json is sole|JetStream.*verae\\.ts' overview zapier-docs-master zapier-decisions` — residual false claims listed and fixed or explicitly deferred with why | + +--- + +### WP-B — NATS subject clarification + +| Field | Content | +|-------|---------| +| **PreReqs** | None (feeding) | +| **Mode** | Parallel | +| **Repos** | `zapier-decisions`, `verae-middleware`, `verae-nats-cluster`, `zapier-docs-master` | +| **Files** | `verae-middleware/src/nats/subjects.js`; `verae-middleware/NATS.md`; `zapier-docs-master` / architecture `nats-subjects` copies; `verae-nats-cluster` stream list docs; `zapier-decisions/NATS.md` | +| **DoD** | Matrix matches architecture §5: JetStream for `verae.zapier.jobs.{watch,events}`, `webhooks.deliver`, archive; **core only** for `verae.ts.completed` / `verae.ts.receipt`; billing `verae.billing.*` remains core request-reply | +| **Tests** | `cd verae-middleware && npm run test:unit`; `rg -n 'verae\\.ts\\.(completed|receipt)' verae-middleware` must not show JetStream stream binds; `cd verae-nats-cluster && npm run status` (lab) — confirm product streams list does **not** include timestamp job-complete | + +--- + +### WP-C — Digests-only docs / models sync + +| Field | Content | +|-------|---------| +| **PreReqs** | None | +| **Mode** | Parallel | +| **Repos** | `zapier-docs-master`, `verae-middleware`, `verae-zapier-app`, `zapier-decisions` | +| **Files** | Catalog models / OpenAPI examples that still show “Hello world” data; `verae-middleware` route docs for batch timestamp; Zapier create field descriptions | +| **DoD** | All public stamp examples are multihash digests; no plaintext body-as-data examples in SoT docs | +| **Tests** | `cd verae-middleware && npm run test:hash` (and batch-related unit tests if present); `rg -n 'Hello world' docs models openapi` → zero in stamp request examples | + +--- + +### WP-D — Access-plane smoke + D004 leaf deny + +| Field | Content | +|-------|---------| +| **PreReqs** | Lab doors up (STATUS) preferred; else local unit tests | +| **Mode** | Parallel | +| **Repos** | `verae-access-leaf`, `verae-access-web`, `verae-access-api`, `verae-access-zapier`, `verae-access-authz`, `zappier-identity` | +| **Files** | Leaf allow-list / policy tests under each `test/`; authz check paths | +| **DoD** | Automated proof leaf cannot call `verae.billing.balance.adjust` / prepaid adjust HTTP; web/api/zapier smoke health | +| **Tests** | `cd verae-access-leaf && npm test`; `cd verae-access-authz && npm test`; `cd verae-access-web && npm test`; same for api/zapier if scripts exist | + +--- + +### WP-E — Forgejo clone inventory + worktrees + +| Field | Content | +|-------|---------| +| **PreReqs** | None | +| **Mode** | Parallel | +| **Repos** | `verae-bootstrap` (source of `repos.txt`) | +| **Files** | Produce checklist under `zapier-decisions/plans/reviews/clone-inventory-2026-10-03.md` (or append section to this board’s §7); do not invent new repos | +| **DoD** | Every WP’s clone list verified present under `/Users/marchon/apps/verae` or cloned; `repos.txt` vs WP matrix diff empty for in-scope modules | +| **Tests** | `while read r; do test -d /Users/marchon/apps/verae/$r/.git || echo MISSING $r; done < verae-bootstrap/repos.txt` | + +--- + +### WP-F — BusinessStore interface + failing contract tests (Phase 1 start) + +| Field | Content | +|-------|---------| +| **PreReqs** | **MG0 PASS** | +| **Mode** | Serial critical chain | +| **Repos** | `zappier-edge` | +| **Files** | **Create** `src/db/business-store.ts` (interface); **Create** `tests/business-store.contract.test.ts`; may re-export types from `src/usage.ts`, `src/db/customer-repo.ts`, `src/db/pricing-store.ts`, `src/db/invoice-repo.ts` | +| **Interface minimum** | `findCustomer`, `saveCustomer`, `recordUsage`, `verifyUsageChain`, `listUsage`, `getBalance`, `adjustBalance`, `getRates` / `resolveChannelRate`, `listInvoices` / `saveInvoice` | +| **DoD** | Contract tests exist and **fail** against missing impl (red); methods documented; hash-chain fields (`prev_hash`, `row_hash`) required even if current `SqliteUsageRepo` lacks them (extend schema in WP-G) | +| **Tests** | `cd zappier-edge && npm test -- tests/business-store.contract.test.ts` → expect fail | + +--- + +### WP-G — SqliteFileBusinessStore (dev/lab adapter) + +| Field | Content | +|-------|---------| +| **PreReqs** | WP-F red tests exist | +| **Mode** | Serial | +| **Repos** | `zappier-edge` | +| **Files** | **Create** `src/db/sqlite-file-business-store.ts`; adapt `src/db/usage-repo.ts` (add hash chain columns + verify); wire factory in `src/index.ts` / `src/app.ts` behind flag `BUSINESS_STORE=sqlite-file` (default keep current behavior until WP-L) | +| **DoD** | Contract tests **pass** on file/memory SQLite; existing Jest suite still green | +| **Tests** | `cd zappier-edge && npm test -- tests/business-store.contract.test.ts`; `cd zappier-edge && npm test` | + +--- + +### WP-H — LibsqlBusinessStore + CI + +| Field | Content | +|-------|---------| +| **PreReqs** | WP-G green | +| **Mode** | Serial (can overlap WP-I ops once interface stable) | +| **Repos** | `zappier-edge` | +| **Files** | **Create** `src/db/libsql-business-store.ts`; `package.json` add `@libsql/client`; `.env.example` `LIBSQL_URL`, `LIBSQL_AUTH_TOKEN`; Jest config for libsql file URL or embedded | +| **DoD** | Same contract suite passes with `BUSINESS_STORE=libsql` and `LIBSQL_URL=file:…` (or testcontainers); WAL/durable semantics documented | +| **Tests** | `cd zappier-edge && BUSINESS_STORE=libsql LIBSQL_URL=file:./tmp-business.db npm test -- tests/business-store.contract.test.ts`; full `npm test` | + +--- + +### WP-I — libSQL server / ops bring-up + +| Field | Content | +|-------|---------| +| **PreReqs** | None for draft runbook; MG1 needs live URL | +| **Mode** | Parallel feeding into MG1 | +| **Repos** | `verae-ops`, `verae-deploy`, `verae-nats-cluster` (topology reference only), `zapier-decisions` | +| **Files** | New runbook under `verae-ops` (e.g. `libsql.md` / section in `GETTING-STARTED.md`); env matrix for 3–5 hosts; **do not** relocate Go DBs | +| **DoD** | Documented `LIBSQL_URL` for lab; backup note (Litestream optional for primary file — not multi-writer); firewall private-only | +| **Tests** | Manual: `curl`/client ping create table + select; record connection string in STATUS | + +--- + +### WP-J — Fixture harvest (billing.db + books.json) + +| Field | Content | +|-------|---------| +| **PreReqs** | Read-only access to lab 138 paths (or sanitized copies) | +| **Mode** | Parallel into MG1 | +| **Repos** | `zappier-edge`, `zappier-account-balance`, `zapier-decisions` | +| **Files** | **Create** `zappier-edge/tests/fixtures/` (sanitized); `scripts/export-lab-business-fixtures.md` instructions; never commit live secrets/API keys | +| **Sources** | Live stand-ins: `/opt/verae/zappier-edge/billing.db` (BILLING_DB), `zappier.db` (auth may stay local), account-balance `books.json`; seed keys in lab only | +| **DoD** | Anonymized fixtures with known chain tip + prepaid balances for migrator tests | +| **Tests** | Fixture checksums documented; `sqlite3 fixture.db 'select count(*) from usage_entries;'` | + +--- + +### WP-K — Migrator billing.db + books.json → libSQL (Phase 2) + +| Field | Content | +|-------|---------| +| **PreReqs** | **MG1 PASS**; fixtures from WP-J; `LIBSQL_URL` from WP-I | +| **Mode** | Serial | +| **Repos** | `zappier-edge`, `zappier-account-balance` | +| **Files** | **Create** `zappier-edge/scripts/migrate-to-libsql.ts` (or `.mjs`); **Create** `tests/migrate-to-libsql.test.ts`; map tables: customers, tiers, price_endpoints, channel_rates*, usage_entries(+hashes), invoices*, balances/credits/payments from books | +| **DoD** | One-shot migrator; `verifyUsageChain` true before and after; dual-run read path flag `BUSINESS_READ_PRIMARY=libsql` with file fallback; **no production writer cutover yet** | +| **Tests** | `cd zappier-edge && npm test -- tests/migrate-to-libsql.test.ts`; dry-run against fixtures; print before/after tip hashes | + +--- + +### MERGE GATE 2 — Writers still locked + +Engineers **must not** start WP-L/M/N until Orchestrator/PM records: + +``` +MERGE GATE 2: PASS +- migrator tip hashes match +- dual-run reads validated +- writers unlocked: yes +``` + +--- + +### WP-L — Rewire zappier-edge meter / ledger → BusinessStore (Phase 3) + +| Field | Content | +|-------|---------| +| **PreReqs** | **MG2 PASS** | +| **Mode** | Serial (constraint path) | +| **Repos** | `zappier-edge` | +| **Files** | `src/meter.ts`, `src/usage.ts`, `src/ledger.ts`, `src/billing-nats.ts`, `src/credits.ts`, `src/portal.ts`, `src/pricing.ts`, `src/invoicing.ts`, `src/db/*-repo.ts`, `src/index.ts`, `src/app.ts` — inject BusinessStore; edge `balanceCents` remains cache filled from store/billing API | +| **DoD** | Meter appends usage **only** via BusinessStore; prepaid adjust goes through single writer API; no direct better-sqlite3 business writes outside adapter | +| **Tests** | `cd zappier-edge && npm test`; focus `tests/meter.test.ts`, `tests/usage.test.ts`, `tests/credit.test.ts`, `tests/portal.test.ts`, `tests/db-*.test.ts`, `tests/billing-delta.test.ts` | + +--- + +### WP-M — Rewire account-balance (books.json → libSQL) + +| Field | Content | +|-------|---------| +| **PreReqs** | WP-L in progress or done (coordinate subjects) | +| **Mode** | Serial after WP-L starts (shared `verae.billing.*` contracts) | +| **Repos** | `zappier-account-balance`, `zappier-edge` | +| **Files** | `zappier-account-balance/src/persist.js`, `books.js`, `server.js`, `subjects.js`; tests under `test/`; README SoT language → libSQL | +| **DoD** | NATS `verae.billing.balance.*` / statement backed by libSQL BusinessStore or HTTP billing API; `books.json` deprecated (import-only); D001 single writer preserved | +| **Tests** | `cd zappier-account-balance && npm test`; integration: adjust → edge statement sees same balance | + +--- + +### WP-N — Rewire CS / sales / accounting-export + +| Field | Content | +|-------|---------| +| **PreReqs** | WP-M billing subjects stable | +| **Mode** | Parallel with each other **after** WP-M contract freeze; else serial behind WP-M | +| **Repos** | `zappier-customer-service`, `zappier-sales-pricing`, `zappier-accounting-export` | +| **Files** | Each package `src/nats-billing.js`, `src/server.js`; staff UI pages as needed | +| **DoD** | CS credit adjust → libSQL balance; sales channel×segment rates → same rate tables meter uses; accounting export reads payments/usage from libSQL | +| **Tests** | `cd zappier-customer-service && npm test`; `cd zappier-sales-pricing && npm test`; `cd zappier-accounting-export && npm test` | + +--- + +### MERGE GATE 3 — Single writer proof + +| Check | Pass criteria | +|-------|---------------| +| Grep | No code path treats `books.json` or per-host `billing.db` as production SoT | +| Runtime | One `LIBSQL_URL` serves customers/pricing/usage/invoices/balances | +| D004 | Leaf still cannot adjust billing | +| Regression | `zappier-edge` full Jest green | + +--- + +### WP-O — Multi-host topology (Phase 4) + +| Field | Content | +|-------|---------| +| **PreReqs** | **MG3 PASS** | +| **Mode** | Serial | +| **Repos** | `verae-ops`, `verae-deploy`, `verae-fleet`, `verae-keep`, `verae-bootstrap`, `zappier-edge`, `verae-middleware` | +| **Files** | Host env templates (`LIBSQL_URL`, `NATS_URL=nats://10.10.10.21:4222,...`); deploy scripts; STATUS topology section | +| **DoD** | libSQL on dedicated/primary host; edge/middleware/CS on separate hosts possible; JetStream workers unchanged; leaf cannot bill | +| **Tests** | Cross-host: meter on host A, CS adjust on host B, portal statement on host C — same balance; `verae-nats-cluster` `npm run status` | + +--- + +### WP-P — Userflow E2E harness (Phase 5) + +| Field | Content | +|-------|---------| +| **PreReqs** | MG3; WP-O preferred for multi-host proof, MOCK_VERAE OK for first pass | +| **Mode** | Serial into MG4 | +| **Repos** | `verae-zapier-simulator`, `zappier-edge`, `verae-middleware`, `verae-access-web`, `verae-access-zapier`, `verae-access-leaf` | +| **Files** | Extend simulator/`harness` under simulator or `packages`-style tests; scripts for flows 1–7 from architecture §1 | +| **Flows** | (1) Signup→API key→Statement (2) Stamp digests single+batch + credit headers (3) Async complete→REST Hook via jobs.watch→events→webhooks.deliver (4) CS credit→portal balance (5) Sales quote→meter match (6) Accounting export consistency (7) Leaf stamp OK / billing deny | +| **DoD** | Automated report artifact under `zapier-decisions/plans/qa/` | +| **Tests** | `cd verae-zapier-simulator && npm test`; middleware `npm test` + selected `test:gate*`; edge `npm test`; flow runner script (to be added) exit 0 | + +--- + +### WP-Q — QA whattochange loop (Phase 6) + +| Field | Content | +|-------|---------| +| **PreReqs** | **MG4 PASS** (suite green once) | +| **Mode** | Loop until project buffer | +| **Repos** | `zapier-decisions` + whichever repos bugs land in | +| **Files** | `plans/qa/-whattochange/` packs; `NotImplemented/` leftovers per recommended-changes skill | +| **DoD** | Each bug → a/b/c/d + PreReq todos; engineers implement; re-run WP-P | +| **Tests** | Full suite + userflows after each pack | + +--- + +## 4. Parallel vs serial schedule — first 2 weeks + +Assume agent-days; calendar = 10 working days. Critical chain owns the constraint; feeding work never blocks writers early. + +| Day | Parallel (feeding) | Serial (critical chain) | Gate | +|-----|--------------------|-------------------------|------| +| **1** | WP-A, WP-B, WP-C, WP-D, WP-E kickoff | — | Aim MG0 EOD day 2 | +| **2** | Finish A–E; Architect signs `2026-10-03-arch-review.md` | — | **MG0** | +| **3** | WP-I ops draft; WP-J fixture planning | **WP-F** interface + red tests | | +| **4** | WP-I lab libSQL up; WP-J sanitize fixtures | **WP-G** SqliteFile adapter | | +| **5** | WP-I harden docs; fixture finalize | **WP-H** LibsqlBusinessStore | **MG1** target EOD day 5–6 | +| **6** | Buffer if MG1 slip | **WP-K** migrator TDD | | +| **7** | — | Finish WP-K; dual-run reads | **MG2** (writers unlock) | +| **8** | — | **WP-L** edge rewire | | +| **9** | WP-N prep (read-only) | **WP-M** account-balance | | +| **10** | **WP-N** CS/sales/acct (parallel agents) | Finish L/M integration | **MG3** stretch; else day 11 | + +**Week 2 overflow (if MG3 slips):** do **not** start WP-O multi-host. Burn down writer bugs; keep project buffer intact. + +**Week 2 if on track (days 11–14):** WP-O topology → WP-P harness → MG4 → start WP-Q. + +**Explicit non-goals in first 2 weeks:** live `MOCK_VERAE=false` seal proof; Zapier Platform public push; lan-134; Go DB moves; JetStream for `verae.ts.*`. + +--- + +## 5. QA entry criteria + +QA (WP-Q / userflow break attempts) may begin only when **all** of the following are true: + +1. **MG3 PASS** — single prepaid writer on libSQL; no dual SoT writes. +2. **Regression green:** `zappier-edge` `npm test`; `zappier-account-balance` `npm test`; CS/sales/acct `npm test`; `verae-middleware` `npm test` (unit + non-NATS integration at minimum). +3. **NATS rules verified:** product JetStream streams only for Zapier jobs/webhooks/archive; `verae.ts.completed` / `verae.ts.receipt` core-only (grep + cluster status). +4. **Harness exists (WP-P skeleton):** at least flows 1, 2, 4 runnable under `MOCK_VERAE=true`. +5. **D004 automated:** leaf billing adjust denied. +6. **Migrator evidence:** tip hash report archived under `plans/qa/`. +7. **Go timestamping DBs:** confirmation note that no PR touched them. +8. **Docs:** C2/C3/C5/C6 closed or explicitly ticketed with owner. + +**QA exit (feeds project buffer):** whattochange packs filed for every failure; no silent skips; leftovers under `NotImplemented/` with why. + +--- + +## 6. Buffer policy + +### Merge gate before rewiring writers (MG2) + +- **Hard stop:** WP-L / WP-M / WP-N are blocked until MG2 PASS. +- Rationale (Critical Chain feeding buffer): incomplete BusinessStore or bad migration hitting the meter writer destroys the only constraint we are exploiting (consistent SoT). +- Gate owner: PM / Orchestrator. Engineers may prepare branches but must not merge writer cutover. + +### Other merge gates + +| Gate | Absorbs variation from | Failure action | +|------|------------------------|----------------| +| MG0 | Docs/conflict feeding | Re-run WP-A/B/C; no Phase 1 code | +| MG1 | Driver + ops + fixtures | Fix contract/CI URL; no migrator on prod-like data | +| MG3 | Writer rewires | Roll back feature flag to SqliteFile/dual-read; do not split hosts | +| MG4 | E2E harness | Fix failures; do not declare QA loop “done” | + +### Project buffer = double userflow pass + +- After MG4 and WP-Q churn, **acceptance requires userflows 1–7 green twice consecutively** (architecture §10 / Phase 6 exit). +- First pass proves the system; second pass absorbs latent race/order bugs without consuming schedule fantasy. +- Do **not** spend project buffer on early scope (Platform push, live seal, Peergos redesign). +- If second pass fails, return to WP-Q; do not “waive” the second pass. + +### Constraint buffer + +- Before each critical-chain WP (F, G, H, K, L), feeding artifacts listed in §2 must be staged so the constraint agent never waits on clone/docs/URL discovery mid-implementation. + +--- + +## 7. Explicit Forgejo repos each WP clones + +Base URL: `https://git.georgelambert.org/marchon/.git` + +| WP | Repos to clone | +|----|----------------| +| **WP-A** | `zapier-decisions`, `overview`, `zapier-docs-master` | +| **WP-B** | `zapier-decisions`, `verae-middleware`, `verae-nats-cluster`, `zapier-docs-master` | +| **WP-C** | `zapier-docs-master`, `verae-middleware`, `verae-zapier-app`, `zapier-decisions` | +| **WP-D** | `verae-access-leaf`, `verae-access-web`, `verae-access-api`, `verae-access-zapier`, `verae-access-authz`, `zappier-identity` | +| **WP-E** | `verae-bootstrap` (plus any MISSING from `repos.txt` needed by later WPs) | +| **WP-F** | `zappier-edge` | +| **WP-G** | `zappier-edge` | +| **WP-H** | `zappier-edge` | +| **WP-I** | `verae-ops`, `verae-deploy`, `zapier-decisions` | +| **WP-J** | `zappier-edge`, `zappier-account-balance`, `zapier-decisions` | +| **WP-K** | `zappier-edge`, `zappier-account-balance` | +| **WP-L** | `zappier-edge` | +| **WP-M** | `zappier-account-balance`, `zappier-edge` | +| **WP-N** | `zappier-customer-service`, `zappier-sales-pricing`, `zappier-accounting-export` | +| **WP-O** | `verae-ops`, `verae-deploy`, `verae-fleet`, `verae-keep`, `verae-bootstrap`, `zappier-edge`, `verae-middleware` | +| **WP-P** | `verae-zapier-simulator`, `zappier-edge`, `verae-middleware`, `verae-access-web`, `verae-access-zapier`, `verae-access-leaf` | +| **WP-Q** | `zapier-decisions` + bug-target repos as discovered | + +### Union set for cutover agents (minimum clone list) + +``` +zapier-decisions +overview +zapier-docs-master +verae-bootstrap +verae-ops +verae-deploy +verae-fleet +verae-keep +verae-nats-cluster +zappier-edge +zappier-account-balance +zappier-customer-service +zappier-sales-pricing +zappier-accounting-export +zappier-identity +verae-middleware +verae-zapier-app +verae-zapier-simulator +verae-access-authz +verae-access-web +verae-access-api +verae-access-leaf +verae-access-zapier +``` + +**Out of clone scope for this cutover (do not pull for WPs above):** Go timestamping-svc repos, Peergos/Seaweed redesign trees, `verae-activate` (unless WP-C needs tiny-app copy sync). + +--- + +## Orchestrator checklist (next actions) + +- [ ] Architect produces `plans/reviews/2026-10-03-arch-review.md` → **MG0** +- [ ] Launch parallel agents: WP-A, WP-B, WP-C, WP-D, WP-E +- [ ] After MG0: engineer on WP-F → G → H; parallel WP-I, WP-J +- [ ] **Stop before writers** until MG2 +- [ ] Enforce D006 + core-NATS job-complete on every PR review + +--- + +*End of PM board. Planning artifact only — no implementation in this change.* diff --git a/scripts/jobbus-farm.sh b/scripts/jobbus-farm.sh new file mode 100755 index 0000000..4f85abe --- /dev/null +++ b/scripts/jobbus-farm.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +# Farm multi-build pieces from jobs/pieces.yaml (or PIECES_FILE) onto job-bus. +set -euo pipefail +export PATH="${HOME}/bin:/opt/homebrew/bin:/usr/local/bin:${PATH}" +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" + +if [[ -f "${HOME}/.config/nats/job-bus.env" ]]; then + set -a; # shellcheck disable=SC1090 + source "${HOME}/.config/nats/job-bus.env"; set +a +elif [[ -f /etc/nats/job-bus.env ]]; then + set -a; source /etc/nats/job-bus.env; set +a +else + echo "missing job-bus.env" >&2; exit 1 +fi + +PIECES_FILE="${PIECES_FILE:-$ROOT/jobs/pieces.yaml}" +[[ -f "$PIECES_FILE" ]] || { echo "missing $PIECES_FILE (copy from job-bus template)" >&2; exit 1; } + +PY="${JOB_BUS_PYTHON:-}" +if [[ -z "$PY" ]]; then + if [[ -x "$HOME/job-bus/venv/bin/python" ]]; then PY="$HOME/job-bus/venv/bin/python" + elif [[ -x /opt/job-bus/venv/bin/python ]]; then PY=/opt/job-bus/venv/bin/python + else PY=python3 + fi +fi + +WATCH=0 +CLEAN_AFTER=0 +PROGRESS=0 +while [[ $# -gt 0 ]]; do + case "$1" in + --watch) WATCH=1; shift ;; + --progress) PROGRESS=1; shift ;; + --cleanup-after) CLEAN_AFTER=1; shift ;; + -h|--help) + echo "usage: scripts/jobbus-farm.sh [--watch] [--progress] [--cleanup-after]" + exit 0 ;; + *) echo "unknown: $1" >&2; exit 1 ;; + esac +done + +# Ensure PyYAML available in venv lightly +"$PY" -c 'import yaml' 2>/dev/null || "$PY" -m pip install -q pyyaml + +MAP=$("$PY" - "$PIECES_FILE" "$ROOT" <<'PY' +import json, os, sys, datetime +from pathlib import Path +try: + import yaml +except ImportError: + raise SystemExit("PyYAML required") + +path, root = sys.argv[1], sys.argv[2] +doc = yaml.safe_load(open(path)) +defaults = doc.get("defaults") or {} +repo = Path(root).name +date = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") +batch = (doc.get("batch") or f"{repo}-{date}").replace("{{REPO_NAME}}", repo).replace("{{DATE}}", date) +pieces = doc.get("pieces") or [] +out_dir = Path(root) / "jobs" / "submitted" / batch +out_dir.mkdir(parents=True, exist_ok=True) +ids = [] +for i, p in enumerate(pieces, 1): + pid = str(p.get("id") or f"piece-{i:02d}") + kind = p.get("kind") or defaults.get("kind") or "cpu" + requires = p.get("requires") if "requires" in p else defaults.get("requires") or [] + if requires is None: + requires = [] + cmd = p.get("cmd") + if not cmd: + raise SystemExit(f"piece {pid} missing cmd") + job_id = f"job-{batch}-{pid}" + body = { + "id": job_id, + "kind": kind, + "requires": list(requires), + "cmd": list(cmd), + "timeout_sec": int(p.get("timeout_sec") or defaults.get("timeout_sec") or 3600), + "cleanup": p.get("cleanup") or defaults.get("cleanup") or "none", + "meta": {"batch": batch, "piece": pid, "repo": repo, **(p.get("meta") or {})}, + "ts": datetime.datetime.now(datetime.timezone.utc).isoformat().replace("+00:00", "Z"), + } + if p.get("cwd"): + body["cwd"] = p["cwd"] + else: + # default cwd: repo root — only works if worker can see this path (shared mount). + # Until SSHFS is set, prefer pieces that use absolute worker paths or staged inputs. + body["cwd"] = root + jf = out_dir / f"{pid}.json" + jf.write_text(json.dumps(body, indent=2) + "\n") + ids.append({"id": job_id, "kind": kind, "file": str(jf), "piece": pid}) +print(json.dumps({"batch": batch, "jobs": ids, "count": len(ids)})) +PY +) + +BATCH=$(echo "$MAP" | "$PY" -c 'import json,sys; print(json.load(sys.stdin)["batch"])') +COUNT=$(echo "$MAP" | "$PY" -c 'import json,sys; print(json.load(sys.stdin)["count"])') +echo "batch=$BATCH pieces=$COUNT" + +echo "$MAP" | "$PY" -c 'import json,sys; d=json.load(sys.stdin); print("\n".join(j["file"] for j in d["jobs"]))' | while read -r jf; do + submit-job --json "$jf" +done + +echo "submitted $COUNT jobs for batch $BATCH" +echo "track: job-progress --batch $BATCH --expect $COUNT" +echo "clean: job-cleanup --batch $BATCH" + +if [[ "$PROGRESS" -eq 1 ]]; then + job-progress --batch "$BATCH" --expect "$COUNT" || true +fi + +if [[ "$WATCH" -eq 1 ]]; then + # simple wait via status subscription count*2 (running+done) approx using progress expect + job-progress --batch "$BATCH" --expect "$COUNT" || true +fi + +if [[ "$CLEAN_AFTER" -eq 1 ]]; then + job-cleanup --batch "$BATCH" || true +fi