George Lambert
b4150c8250
Compose-ready workspace: packages/zappier (rate card, portal, Stripe), packages/verae-zapier-middleware (timestamp + NATS), packages/verae-zapier (CLI app), vendor/zapier-platform, and research/zapier vendor corpus. Gate 0 structure checks pass. Product code and research are not yet wired.
66 lines
2 KiB
Python
66 lines
2 KiB
Python
#!/usr/bin/env python3
|
|
"""Load the full Zapier template sitemap into zapier.templates."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
from datetime import datetime, timezone
|
|
|
|
from pymongo import ASCENDING, UpdateOne
|
|
|
|
|
|
def main() -> int:
|
|
from pymongo import MongoClient
|
|
|
|
uri = os.environ.get("ZAPIER_MONGO_URI") or os.environ.get("MDB_MCP_CONNECTION_STRING")
|
|
if not uri:
|
|
print("ZAPIER_MONGO_URI required", file=sys.stderr)
|
|
return 1
|
|
path = sys.argv[1] if len(sys.argv) > 1 else "raw/templates-by-slug.json"
|
|
data = json.load(open(path))
|
|
client = MongoClient(uri, serverSelectionTimeoutMS=15000)
|
|
db = client.get_default_database()
|
|
if db is None:
|
|
db = client["zapier"]
|
|
col = db["templates"]
|
|
now = datetime.now(timezone.utc)
|
|
# flatten unique by template id + app pair
|
|
seen = set()
|
|
batch = []
|
|
n = 0
|
|
for slug, items in data.items():
|
|
for it in items:
|
|
tid = str(it.get("id") or "")
|
|
apps = tuple(it.get("apps") or [])
|
|
key = (tid, apps)
|
|
if not tid or key in seen:
|
|
continue
|
|
seen.add(key)
|
|
_id = f"{tid}:{'+'.join(apps)}"
|
|
doc = {
|
|
"_id": _id,
|
|
"template_id": tid,
|
|
"title": it.get("title"),
|
|
"url": it.get("url"),
|
|
"apps": list(apps),
|
|
"updated_at": now,
|
|
}
|
|
batch.append(UpdateOne({"_id": _id}, {"$set": doc}, upsert=True))
|
|
if len(batch) >= 1000:
|
|
col.bulk_write(batch, ordered=False)
|
|
n += len(batch)
|
|
batch = []
|
|
if n % 50000 == 0:
|
|
print("wrote", n, flush=True)
|
|
if batch:
|
|
col.bulk_write(batch, ordered=False)
|
|
n += len(batch)
|
|
col.create_index([("apps", ASCENDING)])
|
|
col.create_index([("template_id", ASCENDING)])
|
|
print("upserted", n, "estimated", col.estimated_document_count())
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|