From 697278209917d7f781a14c7a948df306676b8de9 Mon Sep 17 00:00:00 2001 From: George Lambert Date: Mon, 26 Oct 2026 21:27:30 -0400 Subject: [PATCH] wip: Mac uncommitted snapshot 2026-10-26 (auto, secrets/large files excluded) --- .agents/JOB_BUS.md | 1 - .jobbus.json | 6 -- 01-what-you-get.md | 19 ------ 02-signup-zappier-portal.md | 24 ------- 03-connect-zapier.md | 19 ------ 04-register-a-hash.md | 29 -------- 05-wait-async-hooks.md | 22 ------ 06-lookup-central-chain.md | 24 ------- 07-attachments-and-metadata.md | 16 ----- 08-bulk-merkle-summaries.md | 17 ----- 09-lookup-tree-nodes.md | 31 --------- 10-reading-receipts.md | 28 -------- 11-errors-billing-retries.md | 14 ---- 12-security.md | 10 --- AGENTS.md | 6 -- NATS.md | 5 -- README.md | 51 -------------- SUMMARY.md | 9 --- jobs/pieces.yaml | 23 ------- package.json | 6 -- scripts/jobbus-farm.sh | 120 --------------------------------- 21 files changed, 480 deletions(-) delete mode 120000 .agents/JOB_BUS.md delete mode 100644 .jobbus.json delete mode 100644 01-what-you-get.md delete mode 100644 02-signup-zappier-portal.md delete mode 100644 03-connect-zapier.md delete mode 100644 04-register-a-hash.md delete mode 100644 05-wait-async-hooks.md delete mode 100644 06-lookup-central-chain.md delete mode 100644 07-attachments-and-metadata.md delete mode 100644 08-bulk-merkle-summaries.md delete mode 100644 09-lookup-tree-nodes.md delete mode 100644 10-reading-receipts.md delete mode 100644 11-errors-billing-retries.md delete mode 100644 12-security.md delete mode 100644 AGENTS.md delete mode 100644 NATS.md delete mode 100644 README.md delete mode 100644 SUMMARY.md delete mode 100644 jobs/pieces.yaml delete mode 100644 package.json delete mode 100755 scripts/jobbus-farm.sh diff --git a/.agents/JOB_BUS.md b/.agents/JOB_BUS.md deleted file mode 120000 index e546bba..0000000 --- a/.agents/JOB_BUS.md +++ /dev/null @@ -1 +0,0 @@ -/Users/marchon/job-bus/docs/JOB_BUS_LLM_INSTRUCTIONS.md \ No newline at end of file diff --git a/.jobbus.json b/.jobbus.json deleted file mode 100644 index ab460ca..0000000 --- a/.jobbus.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "enabled": true, - "manifest": "jobs/pieces.yaml", - "farm": "scripts/jobbus-farm.sh", - "instructions": ".agents/JOB_BUS.md" -} diff --git a/01-what-you-get.md b/01-what-you-get.md deleted file mode 100644 index f7ddf8c..0000000 --- a/01-what-you-get.md +++ /dev/null @@ -1,19 +0,0 @@ -# 1. What you get - -Verae Time proves that a SHA-256 existed at a given time. Zapier lets you do that from the tools you already use (Drive, email, CRM, forms) without writing HTTP clients. - -Three kinds of proof: - -| Proof | Where it lives | Zapier search | -|-------|----------------|---------------| -| **Itemized seal** | Main Verae chain (`api.veraetime.net`) | Find Timestamp by SHA256 | -| **Bulk Merkle root** | Main chain, one seal for many items | Create Batch Timestamps | -| **Leaf inclusion** | External tree-node archives (NATS, never Zapier) | Find Hash (tree nodes + central chain) | - -You also get: - -- A **zappier** customer portal: account, API key, usage, invoices, prepaid balance. -- A **Zapier** integration: creates, searches, and a “timestamp completed” trigger. -- Optional **public metadata**, **encrypted private metadata**, and **file attachments** stored on WORM archives — not on the blockchain. - -Zapier talks **HTTPS only** to zappier-edge. zappier meters the call and forwards to Verae middleware. Middleware talks to the chain and, when asked, to archive/tree-node NATS servers. You never configure NATS in Zapier. diff --git a/02-signup-zappier-portal.md b/02-signup-zappier-portal.md deleted file mode 100644 index ec8d8b4..0000000 --- a/02-signup-zappier-portal.md +++ /dev/null @@ -1,24 +0,0 @@ -# 2. Sign up for a Verae / zappier account - -You need an API key **before** you can connect Zapier. - -1. Open the customer portal (`/portal` on the zappier host your operator gave you). -2. Choose **Create an account**. -3. Enter name, email, and a password of at least 8 characters. -4. You start on the **Free** plan and receive an **API key immediately**. -5. Copy the key. It is sent as `x-api-key` on every Zapier request. -6. Signup also **registers you on Verae central** and stores a stable `veraeUserId` on your account. That id rides every internal hop (billing, authz, job watch) so CS/sales and the chain can see the same person. You never paste the Verae JWT into Zapier — that token stays on the server (`tokenRef` on NATS). - -If your company already created an account (you received a key by email), sign up with the **same email**. The existing plan and key are kept; your password is attached to that account. - -## After signup - -- **Dashboard** — usage this month, included credit, billable overage, prepaid balance. -- **API & pricing** — live rate card (free / fixed / variable per operation, including batch). -- **Invoices** — draft / issued / paid. -- **Billing** — reload $1–$10,000; prepaid is drawn down automatically. Customer-service can add goodwill credits; sales can set a per-customer multiplier; accounting can export invoices to QuickBooks (IIF/CSV). -- **Security** — optional 2FA. - -Regenerating the API key **invalidates the old one immediately**. Update the Zapier authentication field or every Zap step will 401. - -Simulator: action **Sign up (zappier portal)** then **Connect Zapier app**. diff --git a/03-connect-zapier.md b/03-connect-zapier.md deleted file mode 100644 index e94e74b..0000000 --- a/03-connect-zapier.md +++ /dev/null @@ -1,19 +0,0 @@ -# 3. Connect the Zapier app - -1. Sign in at [zapier.com](https://zapier.com) (or your Zapier workspace). -2. Create a Zap → add **Verae Time** (or **Verae Activate** while the full app is in private invite). -3. When Zapier asks for authentication, paste: - - **API key** from the zappier portal. - - **API base URL** if the operator asked you to (hosted edge). Leave blank only for local-mock Activate steps that do not hit the chain. -4. Click **Test**. Zapier calls zappier-edge over HTTPS (`whoami` / a cheap GET). You should see your email and plan. A 401 means the key was regenerated or mistyped. - -You do **not** enter NATS URLs, Verae JWT, or `api.veraetime.net` credentials. Those stay on middleware. - -## Plans - -| Plan | Typical access | -|------|----------------| -| Free | Single timestamp, lookup, Add Numbers / Echo / SHA256 helpers | -| Paid | Batch timestamps, wait-for-certificate, attached metadata, tree-node lookup | - -A 403 `PLAN_UPGRADE_REQUIRED` means that Zap step is paid-only. A 402 means you exhausted included credit — reload in the portal (the error should include an upgrade URL). diff --git a/04-register-a-hash.md b/04-register-a-hash.md deleted file mode 100644 index 7910de7..0000000 --- a/04-register-a-hash.md +++ /dev/null @@ -1,29 +0,0 @@ -# 4. Register a SHA-256 (create timestamp) - -Two Zapier **creates**: - -| Action | What happens | What you get back | -|--------|----------------|-------------------| -| **Create Timestamp (async)** | Submit hash, return immediately | `jobId`, `sha256`, `existing` | -| **Create Timestamp and Wait** | Submit hash, hold the Zap step until the certificate exists (or timeout) | `status`, `receipts[]`, `completedAt` | - -## Inputs - -- **Data** — raw text/bytes. Middleware hashes it with SHA-256. Prefer this when the previous Zap step has the document contents. -- **SHA256** — 64 hex characters if you already hashed the file outside Zapier (recommended for large files: hash locally, send only the digest). -- **Hash algorithm** — `SHA256` (default). - -Do not send the whole PDF to the blockchain. Only the digest is sealed. - -## Already registered - -If that SHA-256 was sealed before, you get the **original** `jobId` and certificate. A second chain write is not created. `existing: true`. - -## Example Zap - -1. Trigger: “New file in Google Drive”. -2. Action (optional): compute SHA-256 in a Code step, or pass file contents as **Data**. -3. Action: **Create Timestamp and Wait**. -4. Action: store `jobId`, `sha256`, and certificate in your sheet/CRM. - -Simulator: scenario **Register** (wait + `includeAttached`). diff --git a/05-wait-async-hooks.md b/05-wait-async-hooks.md deleted file mode 100644 index 10a0081..0000000 --- a/05-wait-async-hooks.md +++ /dev/null @@ -1,22 +0,0 @@ -# 5. Wait, async, and completed hooks - -Timestamping is not always instant. Middleware publishes `verae.zapier.jobs.watch`; a poller waits on the chain; then `verae.zapier.jobs.events` fires. Zapier never sees those subjects. - -## Choose a pattern - -| Pattern | Use when | -|---------|----------| -| **Wait** | The next Zap step needs the certificate now (Slack message, PDF receipt). | -| **Async + Find Job Status** | You stored `jobId` and will poll later. | -| **Timestamp Completed trigger** | You want a second Zap to run when the seal exists (REST Hook). | - -If **Wait** returns `status: pending`, the chain or NATS event did not finish inside `WAIT_TIMEOUT_MS`. Pair async creates with the **Timestamp Completed** trigger instead of raising the timeout blindly. - -## Trigger setup - -1. New Zap → Verae Time → **Timestamp Completed**. -2. Zapier registers a REST Hook URL with middleware (`/zapier/v1/webhooks/subscribe`). -3. When `jobs.events` says `timestamp.completed`, middleware HTTPS POSTs that URL. -4. Turning the Zap off unsubscribes the hook. - -The hook payload is the same completed-job JSON as wait (seal receipt, optional attached metadata). diff --git a/06-lookup-central-chain.md b/06-lookup-central-chain.md deleted file mode 100644 index 0cff10b..0000000 --- a/06-lookup-central-chain.md +++ /dev/null @@ -1,24 +0,0 @@ -# 6. Look up a hash on the central Verae chain - -Zapier search: **Find Timestamp by SHA256**. - -Middleware: `GET /zapier/v1/hashes/{sha256}`. - -This asks the **main Verae chain** (or its mock) whether that digest was **itemized** as its own seal. - -| Result | Meaning | -|--------|---------| -| One record, `exists: true`, `itemizedOnMainChain: true` | This hash has its own certificate. Use `jobId` / `receipts[0]`. | -| Empty / `exists: false` | Not itemized on the main chain. It may still exist as a **leaf** of a bulk summary — use [tree-node lookup](09-lookup-tree-nodes.md). | - -Searches in Zapier must return a list. Zero hits is a miss (the Zap can follow the “not found” path). Errors other than 404 fail the step. - -## When central lookup is the right tool - -- You registered with **Create Timestamp** (single item). -- You stored the `sha256` from that Zap and want to confirm the seal later. -- You are verifying someone else’s **itemized** hash (they gave you 64 hex chars that they sealed individually). - -## When it is the wrong tool - -- The hash was submitted in **Create Batch Timestamps**. The chain stored the Merkle **root**, not this leaf. Central lookup will miss. Continue to tree nodes. diff --git a/07-attachments-and-metadata.md b/07-attachments-and-metadata.md deleted file mode 100644 index c4608e5..0000000 --- a/07-attachments-and-metadata.md +++ /dev/null @@ -1,16 +0,0 @@ -# 7. Attachments and metadata (not on chain) - -The blockchain stores hash + time + certificate. Everything else is **off-chain WORM**: - -| Field | Chain | Archive (`verae.archive.put`) | -|-------|-------|-------------------------------| -| SHA-256 | yes | key | -| Public metadata JSON | no | `kind: publicMeta` | -| Encrypted private metadata | no | `kind: privateMeta` | -| File bytes | no | `kind: file` + `contentSha256` | - -Set **includeAttached** on wait/lookup when you need those records back. Middleware then broadcasts `verae.archive.query`. Each archive uses a bloom filter: **if it does not have the hash, it sends nothing**. Archives that have data reply; the aggregator merges them into `receipts[]` and `files[]`. - -Bloom false positives are possible (empty reply after a maybe). False negatives must be rare. - -Private metadata is only returned on authenticated archive replies. Zapier still never talks to NATS; it only sees the JSON middleware built. diff --git a/08-bulk-merkle-summaries.md b/08-bulk-merkle-summaries.md deleted file mode 100644 index 83710e9..0000000 --- a/08-bulk-merkle-summaries.md +++ /dev/null @@ -1,17 +0,0 @@ -# 8. Bulk Merkle summaries - -**Create Batch Timestamps** takes one payload per line (or an `items[]` array). - -Middleware: - -1. SHA-256 each item (leaf). -2. Build a Merkle tree. -3. Seal **only the root** on the main Verae chain (one certificate, one `jobId`). -4. Put each leaf’s **inclusion proof** on a sharded **tree-node** archive (`kind: tree`). -5. Return `jobId`, `merkleRoot`, `leafCount`, and the leaf hashes. - -Those member hashes are **not individually itemized** on the main chain. Looking them up with **Find Timestamp by SHA256** returns empty. That is expected. - -Why batch: one chain write for thousands of documents, cheaper and faster, while each document can still prove inclusion later via its proof + the root certificate. - -Simulator: scenario **Batch Merkle**, then copy the first leaf into the SHA256 field. diff --git a/09-lookup-tree-nodes.md b/09-lookup-tree-nodes.md deleted file mode 100644 index 61695ce..0000000 --- a/09-lookup-tree-nodes.md +++ /dev/null @@ -1,31 +0,0 @@ -# 9. Look up a hash on tree-node archives - -Zapier search: **Find Hash (tree nodes + central chain)**. - -Middleware: `GET /zapier/v1/hashes/{sha256}?includeAttached=true&includeTree=true`. - -## Sequence (you never configure this) - -1. Central chain lookup (same as chapter 6). -2. If itemized, return that seal (and attached metadata if requested). -3. If miss, middleware broadcasts `verae.archive.query` with `kinds: ["tree", …]` to **every** tree-node NATS server. -4. A node whose bloom filter does not contain the hash **stays silent**. -5. A node that holds the leaf replies on `verae.archive.reply.` with `{ merkleRoot, proof, leafIndex, chainSealJobId }`. -6. Middleware checks the proof against the root, loads the **root’s** chain seal, and returns both receipts. - -## Reading the result in a Zap - -- `exists: true` and `itemizedOnMainChain: false` — this hash was in a bulk summary. -- `proofOk: true` — the leaf really is under `merkleRoot`. -- `receipts` includes `kind: seal` (of the root) and `kind: tree-leaf` (the proof). -- `archiveId` tells you which tree node answered (for support). - -If every node is silent, the hash is unknown **or** the archive fleet is down. The simulator flags “all archives silent” when puts were known to exist; operators should treat that as an outage, not a miss. - -## Zap pattern - -1. Search: Find Hash (tree nodes + central chain). -2. Filter / Paths: - - Found + itemized → treat as a first-class seal. - - Found + not itemized → store Merkle proof + root certificate. - - Not found → optionally **Create Timestamp** to itemize it now. diff --git a/10-reading-receipts.md b/10-reading-receipts.md deleted file mode 100644 index 6ea16ec..0000000 --- a/10-reading-receipts.md +++ /dev/null @@ -1,28 +0,0 @@ -# 10. Reading receipts - -Completed JSON (wait, hook, or lookup): - -```json -{ - "jobId": "…", - "status": "completed", - "sha256": "…", - "receipts": [ - { "kind": "seal", "timestamp": "…", "certificate": "…", "blockIndex": 42 }, - { "kind": "metadata-attach", "attachedAt": "…", "publicMetadata": {} }, - { "kind": "tree-leaf", "merkleRoot": "…", "proof": [], "itemizedOnMainChain": false } - ], - "files": [{ "id": "invoice.pdf", "sha256": "…", "archiveId": "archive-b" }], - "archivesQueried": true, - "archiveReplies": 2 -} -``` - -| `kind` | Meaning | -|--------|---------| -| `seal` | Chain certificate. If `of: merkleRoot`, this seal covers a batch, not the leaf itself. | -| `metadata-attach` | Public or private metadata added later. | -| `file-attach` | Off-chain file (content hash in the record). | -| `tree-leaf` | Inclusion proof for a hash that is not itemized on the main chain. | - -**Verify Timestamp** takes a `certificate` string and returns `{ valid, timestamp, blockIndex }`. For a bulk leaf, verify the **root** certificate, then trust the leaf only if `proofOk` is true. diff --git a/11-errors-billing-retries.md b/11-errors-billing-retries.md deleted file mode 100644 index bf9e1f2..0000000 --- a/11-errors-billing-retries.md +++ /dev/null @@ -1,14 +0,0 @@ -# 11. Errors, billing, retries - -| HTTP | Zapier mapping | What you should do | -|------|----------------|--------------------| -| 400 | validation | SHA-256 must be 64 hex chars; supply Data or SHA256. | -| 401 | auth | Re-paste API key; it may have been regenerated. | -| 402 | QuotaExceeded | Reload prepaid credit in the portal (`upgradeUrl` in the error). | -| 403 | PlanUpgradeRequired | Batch / wait / tree lookup may be paid-only. | -| 404 | empty search | Not itemized — try tree-node search before creating a new seal. | -| 504 / pending | timeout | Use the Completed trigger; do not loop wait in the same step. | - -Middleware retries `jobs.watch` / `jobs.events` once when NATS drops (simulator: enable **recover**). Zapier itself retries failed HTTP according to your Zap settings — keep creates **idempotent** (re-registering the same hash returns the original seal). - -Delays: if the simulator flags hops over 200ms, do not raise Zapier’s step timeout until chain poll interval and archive query window (`WAIT_ARCHIVE_MS`) are measured on NS1. diff --git a/12-security.md b/12-security.md deleted file mode 100644 index f403402..0000000 --- a/12-security.md +++ /dev/null @@ -1,10 +0,0 @@ -# 12. Security: what Zapier never sees - -- Zapier **never** connects to NATS, tree nodes, WORM archives, or `api.veraetime.net`. -- NS1 `nats-server` stays on **127.0.0.1:4222**. Operators use `scripts/nats-tunnel.sh`; it is not a public bind. -- File bytes and private metadata never go on chain. Private metadata is only on authenticated archive replies. -- API keys are `x-api-key` / Bearer tokens on HTTPS. Treat them like passwords; regenerating kills old Zaps. -- The Verae central JWT from `/auth/login` is **not** your Zapier/portal token. Middleware holds it (or re-logins via `tokenRef`). `veraeUserId` is the public correlation id (`vu_…`). -- Bloom filters are **not** an access-control list. On a hit, middleware still checks tenant/share before returning private records. - -If a trace (simulator or `DEBUG_VERAE`) ever shows a `zapier-platform-app` hop with a `verae.*` subject, that is a bug — do not push the app. diff --git a/AGENTS.md b/AGENTS.md deleted file mode 100644 index 2c41576..0000000 --- a/AGENTS.md +++ /dev/null @@ -1,6 +0,0 @@ -# Agent notes - -## Job-bus multi-build -This repo farms multi-build pieces via NATS job-bus. See [.agents/JOB_BUS.md](.agents/JOB_BUS.md). -Edit `jobs/pieces.yaml`, then run `./scripts/jobbus-farm.sh` (or `make farm`). -CPU pieces → Xeons (137/138); GPU pieces → Mac. Always set `requires` per piece. diff --git a/NATS.md b/NATS.md deleted file mode 100644 index e9f2d63..0000000 --- a/NATS.md +++ /dev/null @@ -1,5 +0,0 @@ -# NATS — zapier-user-docs - -This documentation repo does **not** subscribe to NATS. It describes what middleware and tree nodes do so users can choose the right Zapier search. - -Users never set NATS URLs in Zapier. diff --git a/README.md b/README.md deleted file mode 100644 index 27014d3..0000000 --- a/README.md +++ /dev/null @@ -1,51 +0,0 @@ -# Verae Time for Zapier — user guide - -[![standard-readme compliant](https://img.shields.io/badge/readme%20style-standard-brightgreen.svg?style=flat-square)](https://github.com/RichardLitt/standard-readme) - -How to go from **zero account** to **registering SHA-256 hashes**, **looking them up on the central Verae chain**,... - -This is the customer-facing guide (signup → Zaps). Developers: see [zapier-docs-master](https://git.georgelambert.org/marchon/zapier-docs-master) and the simulator at [verae-zapier-simulator](https://git.georgelambert.org/marchon/verae-zapier-simulator). - -**Live catalog:** https://zapier.georgelambert.org/ -**Clone:** `ssh://git@git.georgelambert.org:2223/marchon/zapier-user-docs.git` - -## Contents - -1. [What you get](01-what-you-get.md) -2. [Sign up for a Verae / zappier account](02-signup-zappier-portal.md) -3. [Connect the Zapier app](03-connect-zapier.md) -4. [Register a SHA-256 (create timestamp)](04-register-a-hash.md) -5. [Wait, async, and completed hooks](05-wait-async-hooks.md) -6. [Look up a hash on the central Verae chain](06-lookup-central-chain.md) -7. [Attachments and metadata (not on chain)](07-attachments-and-metadata.md) -8. [Bulk Merkle summaries](08-bulk-merkle-summaries.md) -9. [Look up a hash on tree-node archives](09-lookup-tree-nodes.md) -10. [Reading receipts](10-reading-receipts.md) -11. [Errors, billing, retries](11-errors-billing-retries.md) -12. [Security: what Zapier never sees](12-security.md) - -## Try it without going live - -The [interface simulator](https://git.georgelambert.org/marchon/verae-zapier-simulator) is a Zapier-like form plus a trace console. Run **Batch Merkle**, copy a leaf SHA-256, then **Find Timestamp by SHA256** (miss) vs **Find Hash (tree nodes)** (proof). - -## Install - -```sh -git clone ssh://git@git.georgelambert.org:2223/marchon/zapier-user-docs.git -cd zapier-user-docs -``` - -## Usage - -```sh -# No single entrypoint is declared in this tree. -ls -``` - -## Contributing - -Questions: open an issue at https://git.georgelambert.org/marchon/zapier-user-docs/issues. Pull requests are accepted. - -## License - -UNLICENSED © George Lambert diff --git a/SUMMARY.md b/SUMMARY.md deleted file mode 100644 index 5868f33..0000000 --- a/SUMMARY.md +++ /dev/null @@ -1,9 +0,0 @@ -# zapier-user-docs - -**Job:** Teach a customer to sign up, connect Zapier, register SHA-256s, look them up on the central Verae chain, and query tree-node archives for hashes only present in a bulk Merkle summary. - -**Expects messages from:** none (static docs). - -**Sends messages to:** none. - -**Published on:** https://zapier.georgelambert.org/user-docs/ diff --git a/jobs/pieces.yaml b/jobs/pieces.yaml deleted file mode 100644 index 497a630..0000000 --- a/jobs/pieces.yaml +++ /dev/null @@ -1,23 +0,0 @@ -# Multi-build piece manifest for job-bus. -# Copy to jobs/pieces.yaml in your repo and edit. -# Each piece becomes one tracked job; kind routes cpu→Xeons, gpu→Mac. -batch: "zapier-user-docs-EDITME" -defaults: - kind: cpu - requires: [] - timeout_sec: 3600 - cleanup: none # or on_success; full wipe via: job-cleanup --batch -pieces: - - id: piece-01 - kind: cpu - requires: [bin:bash] - cmd: ["make", "piece-01"] - # cwd: defaults to repo root on submitter; set worker-visible path if needed - - id: piece-02 - kind: cpu - requires: [ffmpeg, pip:numpy] - cmd: ["python3", "scripts/build_piece.py", "--piece", "2"] - - id: piece-gpu-hero - kind: gpu - requires: [pip:Pillow] - cmd: ["python3", "scripts/gpu_hero.py"] diff --git a/package.json b/package.json deleted file mode 100644 index 4764bec..0000000 --- a/package.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "name": "zapier-user-docs", - "version": "0.1.0", - "private": true, - "description": "Verae Time × Zapier user documentation from signup through hash register, central lookup, and tree-node queries" -} diff --git a/scripts/jobbus-farm.sh b/scripts/jobbus-farm.sh deleted file mode 100755 index 4f85abe..0000000 --- a/scripts/jobbus-farm.sh +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env bash -# Farm multi-build pieces from jobs/pieces.yaml (or PIECES_FILE) onto job-bus. -set -euo pipefail -export PATH="${HOME}/bin:/opt/homebrew/bin:/usr/local/bin:${PATH}" -ROOT="$(cd "$(dirname "$0")/.." && pwd)" -cd "$ROOT" - -if [[ -f "${HOME}/.config/nats/job-bus.env" ]]; then - set -a; # shellcheck disable=SC1090 - source "${HOME}/.config/nats/job-bus.env"; set +a -elif [[ -f /etc/nats/job-bus.env ]]; then - set -a; source /etc/nats/job-bus.env; set +a -else - echo "missing job-bus.env" >&2; exit 1 -fi - -PIECES_FILE="${PIECES_FILE:-$ROOT/jobs/pieces.yaml}" -[[ -f "$PIECES_FILE" ]] || { echo "missing $PIECES_FILE (copy from job-bus template)" >&2; exit 1; } - -PY="${JOB_BUS_PYTHON:-}" -if [[ -z "$PY" ]]; then - if [[ -x "$HOME/job-bus/venv/bin/python" ]]; then PY="$HOME/job-bus/venv/bin/python" - elif [[ -x /opt/job-bus/venv/bin/python ]]; then PY=/opt/job-bus/venv/bin/python - else PY=python3 - fi -fi - -WATCH=0 -CLEAN_AFTER=0 -PROGRESS=0 -while [[ $# -gt 0 ]]; do - case "$1" in - --watch) WATCH=1; shift ;; - --progress) PROGRESS=1; shift ;; - --cleanup-after) CLEAN_AFTER=1; shift ;; - -h|--help) - echo "usage: scripts/jobbus-farm.sh [--watch] [--progress] [--cleanup-after]" - exit 0 ;; - *) echo "unknown: $1" >&2; exit 1 ;; - esac -done - -# Ensure PyYAML available in venv lightly -"$PY" -c 'import yaml' 2>/dev/null || "$PY" -m pip install -q pyyaml - -MAP=$("$PY" - "$PIECES_FILE" "$ROOT" <<'PY' -import json, os, sys, datetime -from pathlib import Path -try: - import yaml -except ImportError: - raise SystemExit("PyYAML required") - -path, root = sys.argv[1], sys.argv[2] -doc = yaml.safe_load(open(path)) -defaults = doc.get("defaults") or {} -repo = Path(root).name -date = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") -batch = (doc.get("batch") or f"{repo}-{date}").replace("{{REPO_NAME}}", repo).replace("{{DATE}}", date) -pieces = doc.get("pieces") or [] -out_dir = Path(root) / "jobs" / "submitted" / batch -out_dir.mkdir(parents=True, exist_ok=True) -ids = [] -for i, p in enumerate(pieces, 1): - pid = str(p.get("id") or f"piece-{i:02d}") - kind = p.get("kind") or defaults.get("kind") or "cpu" - requires = p.get("requires") if "requires" in p else defaults.get("requires") or [] - if requires is None: - requires = [] - cmd = p.get("cmd") - if not cmd: - raise SystemExit(f"piece {pid} missing cmd") - job_id = f"job-{batch}-{pid}" - body = { - "id": job_id, - "kind": kind, - "requires": list(requires), - "cmd": list(cmd), - "timeout_sec": int(p.get("timeout_sec") or defaults.get("timeout_sec") or 3600), - "cleanup": p.get("cleanup") or defaults.get("cleanup") or "none", - "meta": {"batch": batch, "piece": pid, "repo": repo, **(p.get("meta") or {})}, - "ts": datetime.datetime.now(datetime.timezone.utc).isoformat().replace("+00:00", "Z"), - } - if p.get("cwd"): - body["cwd"] = p["cwd"] - else: - # default cwd: repo root — only works if worker can see this path (shared mount). - # Until SSHFS is set, prefer pieces that use absolute worker paths or staged inputs. - body["cwd"] = root - jf = out_dir / f"{pid}.json" - jf.write_text(json.dumps(body, indent=2) + "\n") - ids.append({"id": job_id, "kind": kind, "file": str(jf), "piece": pid}) -print(json.dumps({"batch": batch, "jobs": ids, "count": len(ids)})) -PY -) - -BATCH=$(echo "$MAP" | "$PY" -c 'import json,sys; print(json.load(sys.stdin)["batch"])') -COUNT=$(echo "$MAP" | "$PY" -c 'import json,sys; print(json.load(sys.stdin)["count"])') -echo "batch=$BATCH pieces=$COUNT" - -echo "$MAP" | "$PY" -c 'import json,sys; d=json.load(sys.stdin); print("\n".join(j["file"] for j in d["jobs"]))' | while read -r jf; do - submit-job --json "$jf" -done - -echo "submitted $COUNT jobs for batch $BATCH" -echo "track: job-progress --batch $BATCH --expect $COUNT" -echo "clean: job-cleanup --batch $BATCH" - -if [[ "$PROGRESS" -eq 1 ]]; then - job-progress --batch "$BATCH" --expect "$COUNT" || true -fi - -if [[ "$WATCH" -eq 1 ]]; then - # simple wait via status subscription count*2 (running+done) approx using progress expect - job-progress --batch "$BATCH" --expect "$COUNT" || true -fi - -if [[ "$CLEAN_AFTER" -eq 1 ]]; then - job-cleanup --batch "$BATCH" || true -fi