commit 0f369b1551361266fa1921595ae516dc95f3763b Author: George Lambert Date: Fri Sep 11 14:36:47 2026 -0400 Initial import of verae-archive-worm from zapier monorepo diff --git a/NATS.md b/NATS.md new file mode 100644 index 0000000..a64f14f --- /dev/null +++ b/NATS.md @@ -0,0 +1,9 @@ +# NATS — verae-archive-worm + +| Direction | Address | From / to | Body | +|-----------|---------|-----------|------| +| IN | `verae.archive.put` | splitter | `{ sha256, tenantId, kind, record, traceId }` | +| IN | `verae.archive.query` | aggregator (broadcast) | `{ correlationId, sha256, tenantId, kinds[], traceId }` | +| OUT | `verae.archive.reply.` | aggregator | `{ archiveId, sha256, records[], traceId }` **only if bloom.mightHave(sha256)** | + +Subscribe to query **without** a shared queue group so every archive sees every query. diff --git a/README.md b/README.md new file mode 100644 index 0000000..49cad10 --- /dev/null +++ b/README.md @@ -0,0 +1,34 @@ +# verae-archive-worm + +Bloom-filtered append-only WORM node. Stores metadata and files **off chain**. A bloom **miss is silence** (no NATS reply). + +**Forgejo:** https://git.georgelambert.org/marchon/verae-archive-worm +**Catalog:** https://zapier.georgelambert.org/packages/verae-archive-worm/README.pdf +**NATS:** https://zapier.georgelambert.org/docs-master/modules/verae-archive-worm/NATS.pdf + +## Run + +```bash +cd packages/verae-archive-worm +npm test +``` + +Fleet keeps **min 3** copies (`keepFloor`). Pause does not count as available. + +## Addresses + +| Subject | Role | +|---------|------| +| `verae.archive.put` | Append `{ sha256, kind, record }` and add the hash to the bloom | +| `verae.archive.query` | Broadcast (no queue group). Hit → `verae.archive.reply.` | +| miss | **no message** | + +Kinds: `publicMeta`, `privateMeta`, `file`. Tree leaves live on **verae-tree-node** (`kind=tree`). + +## Depends on + +- NATS JetStream reachable on the **private** URL (`NATS_URL`) +- archive-aggregator (query fan-out / merge) +- fleet replica floor + +Does not talk to Zapier or the chain. diff --git a/SUMMARY.md b/SUMMARY.md new file mode 100644 index 0000000..fe2ac10 --- /dev/null +++ b/SUMMARY.md @@ -0,0 +1,14 @@ +# verae-archive-worm + +**Job:** Append-only WORM store of metadata and files. Bloom filter: miss → stay silent on NATS. + +**Expects:** +- `verae.archive.put` — append record, add sha256 to bloom +- `verae.archive.query` — `{ correlationId, sha256, tenantId, kinds[] }` (every node sees this; **not** a shared queue group) + +**Sends:** +- `verae.archive.reply.` **only on bloom hit** — `{ archiveId, sha256, records[] }` + +**Does not** write to the blockchain. + +**Test:** `npm test` — miss is `null`; put then query returns records. diff --git a/package.json b/package.json new file mode 100644 index 0000000..b361d1e --- /dev/null +++ b/package.json @@ -0,0 +1,8 @@ +{ + "name": "verae-archive-worm", + "version": "0.1.0", + "type": "module", + "description": "Bloom-filtered WORM archive node for off-chain metadata and files", + "scripts": { "test": "node --test test/*.test.js" }, + "engines": { "node": ">=20" } +} diff --git a/src/archive.js b/src/archive.js new file mode 100644 index 0000000..1963e97 --- /dev/null +++ b/src/archive.js @@ -0,0 +1,40 @@ +/** + * In-process WORM archive: append-only records + bloom. + * @module archive + */ + +import { Bloom } from './bloom.js'; + +export class WormArchive { + /** + * @param {string} archiveId + */ + constructor(archiveId) { + this.archiveId = archiveId; + this.bloom = new Bloom(); + /** @type {Map} */ + this.byHash = new Map(); + } + + /** + * @param {{ sha256: string, kind: string, record: object }} put + */ + put(put) { + const key = String(put.sha256).toLowerCase(); + const list = this.byHash.get(key) ?? []; + list.push({ kind: put.kind, record: put.record, archiveId: this.archiveId, storedAt: new Date().toISOString() }); + this.byHash.set(key, list); + this.bloom.add(key); + } + + /** + * Bloom miss → null (caller must stay silent). Hit → records (may be empty if false positive). + * @param {string} sha256 + * @returns {object[]|null} + */ + query(sha256) { + const key = String(sha256).toLowerCase(); + if (!this.bloom.mightHave(key)) return null; + return this.byHash.get(key) ?? []; + } +} diff --git a/src/bloom.js b/src/bloom.js new file mode 100644 index 0000000..d5e23d0 --- /dev/null +++ b/src/bloom.js @@ -0,0 +1,46 @@ +/** + * Tiny Bloom filter for SHA256 membership (false positives ok). + * @module bloom + */ + +import { createHash } from 'node:crypto'; + +export class Bloom { + /** + * @param {number} [m=16384] bit count + * @param {number} [k=4] hash rounds + */ + constructor(m = 16384, k = 4) { + this.m = m; + this.k = k; + this.bits = Buffer.alloc(Math.ceil(m / 8)); + } + + /** + * @param {string} s + * @returns {number[]} + */ + hashes(s) { + const digest = createHash('sha256').update(String(s)).digest(); + const out = []; + for (let i = 0; i < this.k; i += 1) { + const n = digest.readUInt32BE((i * 4) % 28); + out.push(n % this.m); + } + return out; + } + + add(s) { + for (const i of this.hashes(s)) { + this.bits[i >> 3] |= 1 << (i & 7); + } + } + + /** + * @param {string} s + * @returns {boolean} maybe (never a guaranteed yes) + */ + mightHave(s) { + return this.hashes(s).every((i) => (this.bits[i >> 3] & (1 << (i & 7))) !== 0); + } +} diff --git a/test/archive.test.js b/test/archive.test.js new file mode 100644 index 0000000..4d9a5e4 --- /dev/null +++ b/test/archive.test.js @@ -0,0 +1,28 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { WormArchive } from '../src/archive.js'; +import { Bloom } from '../src/bloom.js'; + +describe('WormArchive + Bloom', () => { + it('miss stays silent (null)', () => { + const a = new WormArchive('archive-a'); + assert.equal(a.query('a'.repeat(64)), null); + }); + + it('put then query returns records', () => { + const a = new WormArchive('archive-b'); + const sha = 'b'.repeat(64); + a.put({ sha256: sha, kind: 'publicMeta', record: { publicMetadata: { t: 1 } } }); + const hit = a.query(sha); + assert.ok(Array.isArray(hit)); + assert.equal(hit[0].kind, 'publicMeta'); + assert.equal(hit[0].archiveId, 'archive-b'); + }); + + it('bloom reports maybe after add', () => { + const b = new Bloom(); + assert.equal(b.mightHave('hello'), false); + b.add('hello'); + assert.equal(b.mightHave('hello'), true); + }); +});