Milestone 0: import zappier billing, Verae middleware, and Zapier research
Compose-ready workspace: packages/zappier (rate card, portal, Stripe), packages/verae-zapier-middleware (timestamp + NATS), packages/verae-zapier (CLI app), vendor/zapier-platform, and research/zapier vendor corpus. Gate 0 structure checks pass. Product code and research are not yet wired.
This commit is contained in:
commit
b4150c8250
1364 changed files with 6814366 additions and 0 deletions
256
research/zapier/scripts/assemble-all-contacts.py
Normal file
256
research/zapier/scripts/assemble-all-contacts.py
Normal file
|
|
@ -0,0 +1,256 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Merge catalog + curated contacts + scrape into contacts-all.csv/json/md."""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import json
|
||||
import re
|
||||
from collections import Counter
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def load_jsonl(path: Path) -> list[dict]:
|
||||
rows = []
|
||||
if not path.exists():
|
||||
return rows
|
||||
for line in path.open():
|
||||
line = line.strip()
|
||||
if line:
|
||||
rows.append(json.loads(line))
|
||||
return rows
|
||||
|
||||
|
||||
ZAPIER_FIRST_PARTY = {
|
||||
"filter",
|
||||
"webhook",
|
||||
"formatter",
|
||||
"schedule",
|
||||
"paths",
|
||||
"code",
|
||||
"email",
|
||||
"delay",
|
||||
"ai",
|
||||
"looping",
|
||||
"rss",
|
||||
"sms",
|
||||
"email-parser",
|
||||
"zapier-tables",
|
||||
"zapier-chrome-extension",
|
||||
"zapier-manager",
|
||||
"sub-zap",
|
||||
"digest",
|
||||
"storage",
|
||||
"translate",
|
||||
"weather",
|
||||
"zapier-central",
|
||||
"interfaces",
|
||||
"chatbots",
|
||||
"agents",
|
||||
}
|
||||
|
||||
BAD_EMAIL = re.compile(
|
||||
r"(@2x\.|votredomaine|example\.|yourdomain|company\.com$|"
|
||||
r"sentry\.|wixpress|schema\.org|\.png$|\.jpe?g$|\.mp$|\.css$)",
|
||||
re.I,
|
||||
)
|
||||
UNIXISH = re.compile(r"^1[5-9]\d{8}$") # 10-digit unix timestamps 2017–2033
|
||||
|
||||
|
||||
def first(*vals):
|
||||
for v in vals:
|
||||
if v:
|
||||
return v
|
||||
return None
|
||||
|
||||
|
||||
def clean_email(e: str | None) -> str | None:
|
||||
if not e:
|
||||
return None
|
||||
e = e.strip().lower()
|
||||
if "@" not in e or "." not in e.split("@")[-1]:
|
||||
return None
|
||||
if BAD_EMAIL.search(e):
|
||||
return None
|
||||
tld = e.rsplit(".", 1)[-1]
|
||||
if len(tld) < 2 or len(tld) > 24:
|
||||
return None
|
||||
return e
|
||||
|
||||
|
||||
def clean_phone(p: str | None, slug: str) -> str | None:
|
||||
if not p or slug in ZAPIER_FIRST_PARTY:
|
||||
return None
|
||||
raw = str(p).strip()
|
||||
if raw.count(".") >= 2:
|
||||
return None # IP / version / CSS
|
||||
digits = re.sub(r"\D", "", raw)
|
||||
if len(digits) < 10 or len(digits) > 15:
|
||||
return None
|
||||
if UNIXISH.match(digits):
|
||||
return None
|
||||
if len(set(digits)) < 4:
|
||||
return None
|
||||
# require some phone-like formatting or a leading +
|
||||
if not re.search(r"[+()\-]", raw) and " " not in raw:
|
||||
# bare digit strings from page IDs — drop
|
||||
return None
|
||||
return raw
|
||||
|
||||
|
||||
def main() -> None:
|
||||
catalog = load_jsonl(ROOT / "raw" / "all-apps.jsonl")
|
||||
scrape_rows = load_jsonl(ROOT / "raw" / "contacts-all-scrape.jsonl")
|
||||
scrape = {r["slug"]: r for r in scrape_rows}
|
||||
curated = {}
|
||||
cpath = ROOT / "contacts.json"
|
||||
if cpath.exists():
|
||||
curated = json.loads(cpath.read_text())
|
||||
|
||||
identified = {}
|
||||
ipath = ROOT / "identified-vendors.json"
|
||||
if ipath.exists():
|
||||
ident = json.loads(ipath.read_text())
|
||||
for vert, rows in ident.get("verticals", {}).items():
|
||||
for r in rows:
|
||||
identified.setdefault(r["slug"], []).append(vert)
|
||||
|
||||
out = []
|
||||
for app in catalog:
|
||||
slug = app["slug"]
|
||||
s = scrape.get(slug, {})
|
||||
c = curated.get(slug, {})
|
||||
scrape_emails = [e for e in (s.get("emails") or []) if clean_email(e)]
|
||||
scrape_phones = [p for p in (s.get("phones") or []) if clean_phone(p, slug)]
|
||||
rec = {
|
||||
"name": app.get("name"),
|
||||
"legal_name": c.get("legal_name") or app.get("name"),
|
||||
"slug": slug,
|
||||
"website": first(c.get("website"), s.get("website"), app.get("external_url")),
|
||||
"hq_address": c.get("hq_address"),
|
||||
"phone": first(c.get("phone"), scrape_phones[0] if scrape_phones else None),
|
||||
"email": first(clean_email(c.get("email")), scrape_emails[0] if scrape_emails else None),
|
||||
"all_emails": "; ".join(
|
||||
dict.fromkeys(
|
||||
[e for e in ([c.get("email")] if c.get("email") else []) + scrape_emails if clean_email(e)]
|
||||
)
|
||||
),
|
||||
"sales_form": first(c.get("sales_form"), (s.get("contact_urls") or [None])[0] if s.get("contact_urls") else None),
|
||||
"support": c.get("support"),
|
||||
"linkedin": c.get("linkedin"),
|
||||
"zapier_url": "https://zapier.com"
|
||||
+ (app.get("app_profile_url") or f"/apps/{slug}/integrations"),
|
||||
"zap_usage": app.get("zap_usage_count") or 0,
|
||||
"popularity_rank": app.get("popularity"),
|
||||
"categories": "; ".join(app.get("category_titles") or app.get("categories") or []),
|
||||
"description": (app.get("description") or "").replace("\n", " ").strip(),
|
||||
"is_premium": bool(app.get("is_premium")),
|
||||
"is_beta": bool(app.get("is_beta")),
|
||||
"in_vendors_txt_verticals": "; ".join(identified.get(slug, [])),
|
||||
"contact_source": "curated" if c else ("scrape" if s else "catalog_only"),
|
||||
"scrape_error": s.get("error"),
|
||||
}
|
||||
if rec["email"] and not rec["all_emails"]:
|
||||
rec["all_emails"] = rec["email"]
|
||||
out.append(rec)
|
||||
|
||||
out.sort(key=lambda r: (-(r["zap_usage"] or 0), (r["name"] or "").lower()))
|
||||
|
||||
csv_path = ROOT / "contacts-all.csv"
|
||||
fields = [
|
||||
"name",
|
||||
"legal_name",
|
||||
"slug",
|
||||
"website",
|
||||
"hq_address",
|
||||
"phone",
|
||||
"email",
|
||||
"all_emails",
|
||||
"sales_form",
|
||||
"support",
|
||||
"linkedin",
|
||||
"zapier_url",
|
||||
"zap_usage",
|
||||
"popularity_rank",
|
||||
"categories",
|
||||
"description",
|
||||
"is_premium",
|
||||
"is_beta",
|
||||
"in_vendors_txt_verticals",
|
||||
"contact_source",
|
||||
"scrape_error",
|
||||
]
|
||||
with csv_path.open("w", newline="") as f:
|
||||
w = csv.DictWriter(f, fieldnames=fields, extrasaction="ignore")
|
||||
w.writeheader()
|
||||
w.writerows(out)
|
||||
|
||||
json_path = ROOT / "contacts-all.json"
|
||||
json_path.write_text(json.dumps(out, indent=2, ensure_ascii=False))
|
||||
|
||||
n = len(out)
|
||||
phone = sum(1 for r in out if r["phone"])
|
||||
email = sum(1 for r in out if r["email"])
|
||||
form = sum(1 for r in out if r["sales_form"])
|
||||
web = sum(1 for r in out if r["website"])
|
||||
hq = sum(1 for r in out if r["hq_address"])
|
||||
any_direct = sum(1 for r in out if r["phone"] or r["email"] or r["sales_form"])
|
||||
src = Counter(r["contact_source"] for r in out)
|
||||
|
||||
md = []
|
||||
A = md.append
|
||||
A("# All Zapier providers — contact directory")
|
||||
A("")
|
||||
A(f"_Generated {date.today().isoformat()} from the live Zapier public catalog snapshot ({n} apps)._")
|
||||
A("")
|
||||
A("This is the **full directory**, not only the 15 `Vendors.txt` verticals.")
|
||||
A("")
|
||||
A("| Deliverable | Path |")
|
||||
A("|-------------|------|")
|
||||
A("| Spreadsheet | [contacts-all.csv](contacts-all.csv) |")
|
||||
A("| JSON | [contacts-all.json](contacts-all.json) |")
|
||||
A("| 15-vertical subset (richer HQ/legal) | [CONTACTS.md](CONTACTS.md) / [contacts.csv](contacts.csv) |")
|
||||
A("")
|
||||
A("## Coverage")
|
||||
A("")
|
||||
A("| Field | Count | Share |")
|
||||
A("|-------|------:|------:|")
|
||||
A(f"| Apps in Zapier catalog | {n:,} | 100% |")
|
||||
A(f"| Website | {web:,} | {web/n:.0%} |")
|
||||
A(f"| Phone or email or sales form | {any_direct:,} | {any_direct/n:.0%} |")
|
||||
A(f"| Email | {email:,} | {email/n:.0%} |")
|
||||
A(f"| Phone | {phone:,} | {phone/n:.0%} |")
|
||||
A(f"| Sales / contact URL | {form:,} | {form/n:.0%} |")
|
||||
A(f"| HQ address (mostly curated 15-vertical set) | {hq:,} | {hq/n:.0%} |")
|
||||
A("")
|
||||
A(f"Sources: curated deep-research {src.get('curated',0):,} · site scrape {src.get('scrape',0):,} · catalog-only {src.get('catalog_only',0):,}.")
|
||||
A("")
|
||||
A("Every row has at least **name + Zapier listing URL**. Email/phone are first-party public values when the vendor HTML published them. Many SaaS sites are JavaScript-only or form-only; those rows still include website + Zapier page.")
|
||||
A("")
|
||||
A("## Top 50 by Zap usage")
|
||||
A("")
|
||||
A("| Name | Zaps | Phone | Email | Website |")
|
||||
A("|------|-----:|-------|-------|---------|")
|
||||
for r in out[:50]:
|
||||
A(
|
||||
f"| [{r['name']}]({r['zapier_url']}) | {r['zap_usage']:,} | "
|
||||
f"{r['phone'] or '—'} | {r['email'] or '—'} | {r['website'] or '—'} |"
|
||||
)
|
||||
A("")
|
||||
A("## Notes")
|
||||
A("")
|
||||
A("- Do not treat scraped phones as verified switchboards without checking the sales/contact URL.")
|
||||
A("- The 256 vendors from `Vendors.txt` keep the richer legal-name / HQ / LinkedIn fields from the curated pass.")
|
||||
A("- Re-run `python3 scripts/scrape-all-contacts.py` then this assembler to refresh.")
|
||||
A("")
|
||||
(ROOT / "CONTACTS-ALL.md").write_text("\n".join(md))
|
||||
print(
|
||||
f"wrote {n} rows csv={csv_path.stat().st_size} "
|
||||
f"email={email} phone={phone} form={form}"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
169
research/zapier/scripts/assemble-capabilities.py
Normal file
169
research/zapier/scripts/assemble-capabilities.py
Normal file
|
|
@ -0,0 +1,169 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Assemble capabilities-all.csv/json + sibling products by website domain."""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import json
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def load_jsonl(path: Path) -> list[dict]:
|
||||
rows = []
|
||||
if not path.exists():
|
||||
return rows
|
||||
for line in path.open():
|
||||
line = line.strip()
|
||||
if line:
|
||||
rows.append(json.loads(line))
|
||||
return rows
|
||||
|
||||
|
||||
def registrable(url: str | None) -> str | None:
|
||||
if not url:
|
||||
return None
|
||||
if not url.startswith("http"):
|
||||
url = "https://" + url
|
||||
try:
|
||||
host = (urlparse(url).hostname or "").lower()
|
||||
except Exception:
|
||||
return None
|
||||
if not host:
|
||||
return None
|
||||
host = host[4:] if host.startswith("www.") else host
|
||||
# collapse obvious product hosts into parent
|
||||
return host
|
||||
|
||||
|
||||
def names(items) -> str:
|
||||
return " | ".join(
|
||||
f"{i.get('label')} ({i.get('key')})"
|
||||
for i in items or []
|
||||
if i.get("label")
|
||||
)
|
||||
|
||||
|
||||
def required_inputs(items) -> str:
|
||||
bits = []
|
||||
for i in items or []:
|
||||
req = [n["key"] for n in i.get("inputs") or [] if n.get("required")]
|
||||
if req:
|
||||
bits.append(f"{i.get('key')}: {', '.join(req)}")
|
||||
return " ; ".join(bits)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
catalog = load_jsonl(ROOT / "raw" / "all-apps.jsonl")
|
||||
by_slug = {a["slug"]: a for a in catalog}
|
||||
caps_rows = load_jsonl(ROOT / "raw" / "capabilities-scrape.jsonl")
|
||||
caps = {r["slug"]: r for r in caps_rows}
|
||||
|
||||
# sibling products: other Zapier apps on the same website domain
|
||||
domain_apps = defaultdict(list)
|
||||
for a in catalog:
|
||||
d = registrable(a.get("external_url"))
|
||||
if d:
|
||||
domain_apps[d].append(a["name"])
|
||||
|
||||
out = []
|
||||
for a in catalog:
|
||||
slug = a["slug"]
|
||||
c = caps.get(slug, {})
|
||||
domain = registrable(a.get("external_url"))
|
||||
siblings = [
|
||||
n for n in domain_apps.get(domain or "", []) if n != a.get("name")
|
||||
]
|
||||
trig = c.get("triggers") or []
|
||||
acts = c.get("actions") or []
|
||||
seas = c.get("searches") or []
|
||||
impl = c.get("implementation") or a.get("current_implementation_id") or ""
|
||||
rec = {
|
||||
"name": a.get("name"),
|
||||
"slug": slug,
|
||||
"used_for": c.get("description") or a.get("description") or "",
|
||||
"categories": "; ".join(
|
||||
c.get("categories") or a.get("category_titles") or a.get("categories") or []
|
||||
),
|
||||
"partner_tier": c.get("partner_tier") or "",
|
||||
"premium": c.get("premium") if c.get("premium") is not None else a.get("is_premium"),
|
||||
"zap_usage": a.get("zap_usage_count") or 0,
|
||||
"website": a.get("external_url") or "",
|
||||
"api_docs_url": "", # filled from overviews if present
|
||||
"zapier_url": "https://zapier.com"
|
||||
+ (a.get("app_profile_url") or f"/apps/{slug}/integrations"),
|
||||
"mcp_url": f"https://zapier.com/mcp/{slug}",
|
||||
"implementation": impl,
|
||||
"calling_convention": (
|
||||
"instant webhook"
|
||||
if c.get("instant_trigger_count")
|
||||
else ("polling trigger" if c.get("trigger_count") else "actions only / no trigger")
|
||||
),
|
||||
"trigger_count": c.get("trigger_count") or 0,
|
||||
"instant_trigger_count": c.get("instant_trigger_count") or 0,
|
||||
"action_count": c.get("action_count") or 0,
|
||||
"search_count": c.get("search_count") or 0,
|
||||
"triggers": names(trig),
|
||||
"actions": names(acts),
|
||||
"searches": names(seas),
|
||||
"required_inputs": required_inputs(acts + seas),
|
||||
"alternatives": " | ".join(c.get("alternatives") or []),
|
||||
"commonly_paired_with": " | ".join(c.get("paired_apps") or []),
|
||||
"sibling_zapier_apps": " | ".join(siblings[:25]),
|
||||
"help_articles": " | ".join(c.get("help_articles") or []),
|
||||
"error": c.get("error") or "",
|
||||
}
|
||||
out.append(rec)
|
||||
|
||||
# attach api_docs from overviews if present
|
||||
ov_path = ROOT / "raw" / "overviews.json"
|
||||
if ov_path.exists():
|
||||
ov = json.loads(ov_path.read_text())
|
||||
for rec in out:
|
||||
rec["api_docs_url"] = (ov.get(rec["slug"]) or {}).get("api_docs_url") or ""
|
||||
|
||||
out.sort(key=lambda r: (-(r["zap_usage"] or 0), (r["name"] or "").lower()))
|
||||
|
||||
fields = list(out[0].keys()) if out else []
|
||||
with (ROOT / "capabilities-all.csv").open("w", newline="") as f:
|
||||
w = csv.DictWriter(f, fieldnames=fields, extrasaction="ignore")
|
||||
w.writeheader()
|
||||
w.writerows(out)
|
||||
(ROOT / "capabilities-all.json").write_text(json.dumps(out, indent=2, ensure_ascii=False))
|
||||
|
||||
scraped = sum(1 for r in out if r["trigger_count"] or r["action_count"] or r["search_count"])
|
||||
md = []
|
||||
A = md.append
|
||||
A("# Zapier provider capabilities")
|
||||
A("")
|
||||
A(f"_Generated {date.today().isoformat()}. Catalog {len(out):,} apps. Controls scraped for {scraped:,}._")
|
||||
A("")
|
||||
A("Each app’s public Zapier page lists **triggers** (reads), **actions** (writes), **searches**, required **inputs**, popular pairings, and alternatives. Auth method (OAuth vs API key) is **not** published on the directory page.")
|
||||
A("")
|
||||
A("| File | Contents |")
|
||||
A("|------|----------|")
|
||||
A("| [capabilities-all.csv](capabilities-all.csv) | One row per app: used-for, trigger/action lists, calling convention, siblings |")
|
||||
A("| [capabilities-all.json](capabilities-all.json) | Same, plus structured trigger/action objects in the scrape JSONL |")
|
||||
A("| [raw/capabilities-scrape.jsonl](raw/capabilities-scrape.jsonl) | Full per-control records (keys, hook vs poll, input fields) |")
|
||||
A("")
|
||||
A("## Top 25 by Zap usage (controls)")
|
||||
A("")
|
||||
A("| App | Used for | Triggers | Instant | Actions | Searches | Convention |")
|
||||
A("|-----|----------|--------:|--------:|--------:|---------:|------------|")
|
||||
for r in out[:25]:
|
||||
used = (r["used_for"] or "").replace("|", "/")[:90]
|
||||
A(
|
||||
f"| [{r['name']}]({r['zapier_url']}) | {used} | {r['trigger_count']} | "
|
||||
f"{r['instant_trigger_count']} | {r['action_count']} | {r['search_count']} | {r['calling_convention']} |"
|
||||
)
|
||||
A("")
|
||||
(ROOT / "CAPABILITIES.md").write_text("\n".join(md))
|
||||
print(f"wrote {len(out)} scraped_with_controls={scraped}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
35
research/zapier/scripts/clone-zapier-repos.sh
Executable file
35
research/zapier/scripts/clone-zapier-repos.sh
Executable file
|
|
@ -0,0 +1,35 @@
|
|||
#!/bin/sh
|
||||
# Shallow-clone official Zapier repos used to build connectors / applications.
|
||||
# Default dest: $ROOT/repos (ROOT defaults to this project).
|
||||
set -eu
|
||||
ROOT="${ROOT:-$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)}"
|
||||
DEST="${1:-$ROOT/repos}"
|
||||
mkdir -p "$DEST"
|
||||
REPOS="
|
||||
zapier-platform
|
||||
sdk
|
||||
zapier-mcp
|
||||
connectors
|
||||
install-zapier
|
||||
agent-skills
|
||||
marketplace
|
||||
visual-builder
|
||||
resthooks
|
||||
zapier-platform-cli
|
||||
zapier-platform-core
|
||||
zapier-platform-schema
|
||||
zapier-platform-example-app-github
|
||||
zapier-platform-example-app-oauth2
|
||||
zapier-platform-example-app-minimal
|
||||
gtm-cheat-codes
|
||||
"
|
||||
for r in $REPOS; do
|
||||
if [ -d "$DEST/$r/.git" ]; then
|
||||
echo "UPDATE $r"
|
||||
git -C "$DEST/$r" fetch --depth=1 origin && git -C "$DEST/$r" reset --hard FETCH_HEAD
|
||||
else
|
||||
echo "CLONE $r"
|
||||
git clone --depth 1 "https://github.com/zapier/$r.git" "$DEST/$r" || echo "FAIL $r"
|
||||
fi
|
||||
done
|
||||
echo "DONE $DEST"
|
||||
11
research/zapier/scripts/dev-env.sh
Executable file
11
research/zapier/scripts/dev-env.sh
Executable file
|
|
@ -0,0 +1,11 @@
|
|||
#!/bin/sh
|
||||
# Zapier coding-set environment. Usage: source scripts/dev-env.sh
|
||||
export PATH="$HOME/.npm-global/bin:$PATH"
|
||||
# Mongo via SSH tunnel: ./scripts/mongo-tunnel.sh
|
||||
if [ -f "$HOME/.mcp-env" ]; then
|
||||
# shellcheck disable=SC1091
|
||||
. "$HOME/.mcp-env"
|
||||
fi
|
||||
echo "zapier-platform: $(command -v zapier-platform 2>/dev/null || echo missing)"
|
||||
echo "zapier-sdk: $(command -v zapier-sdk 2>/dev/null || echo missing)"
|
||||
echo "ZAPIER_MONGO_URI: ${ZAPIER_MONGO_URI:+set}"
|
||||
33
research/zapier/scripts/ensure-mongo-tunnel.sh
Executable file
33
research/zapier/scripts/ensure-mongo-tunnel.sh
Executable file
|
|
@ -0,0 +1,33 @@
|
|||
#!/bin/sh
|
||||
# Start the NS1 Mongo SSH tunnel if nothing is already listening on 127.0.0.1:27017.
|
||||
set -eu
|
||||
ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)"
|
||||
TUNNEL="$ROOT/scripts/mongo-tunnel.sh"
|
||||
LOG="${ZAPIER_TUNNEL_LOG:-$HOME/.grok/zapier-mongo-tunnel.log}"
|
||||
|
||||
if nc -z 127.0.0.1 27017 2>/dev/null; then
|
||||
echo "mongo tunnel: already up (127.0.0.1:27017)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ ! -x "$TUNNEL" ]; then
|
||||
echo "missing $TUNNEL" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$(dirname "$LOG")"
|
||||
nohup "$TUNNEL" >>"$LOG" 2>&1 &
|
||||
echo $! >"${ZAPIER_TUNNEL_PIDFILE:-$HOME/.grok/zapier-mongo-tunnel.pid}"
|
||||
|
||||
i=0
|
||||
while [ "$i" -lt 20 ]; do
|
||||
if nc -z 127.0.0.1 27017 2>/dev/null; then
|
||||
echo "mongo tunnel: started (pid $(cat "${ZAPIER_TUNNEL_PIDFILE:-$HOME/.grok/zapier-mongo-tunnel.pid}"))"
|
||||
exit 0
|
||||
fi
|
||||
i=$((i + 1))
|
||||
sleep 0.25
|
||||
done
|
||||
|
||||
echo "mongo tunnel: failed to bind 127.0.0.1:27017 — see $LOG" >&2
|
||||
exit 1
|
||||
122
research/zapier/scripts/generate-diagrams-pdf.py
Normal file
122
research/zapier/scripts/generate-diagrams-pdf.py
Normal file
|
|
@ -0,0 +1,122 @@
|
|||
#!/usr/bin/env python3
|
||||
"""One-page-per-figure atlas of docs/diagrams/*.svg → docs/architecture-diagrams.pdf."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from reportlab.lib import colors
|
||||
from reportlab.lib.enums import TA_CENTER
|
||||
from reportlab.lib.pagesizes import letter
|
||||
from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet
|
||||
from reportlab.lib.units import inch
|
||||
from reportlab.platypus import Image, PageBreak, Paragraph, SimpleDocTemplate, Spacer
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DIAG = ROOT / "docs" / "diagrams"
|
||||
OUT = ROOT / "docs" / "architecture-diagrams.pdf"
|
||||
NAVY = colors.HexColor("#0F2744")
|
||||
TEAL = colors.HexColor("#1A6B6B")
|
||||
|
||||
TITLES = {
|
||||
"01-high-level-architecture.svg": "High-level architecture",
|
||||
"02-security-boundaries.svg": "Security boundaries",
|
||||
"03-auth-two-hop.svg": "Two-hop authentication",
|
||||
"04-flow-async-timestamp.svg": "Create Timestamp (async + hook)",
|
||||
"05-flow-wait.svg": "Create Timestamp and Wait",
|
||||
"06-nats-topology.svg": "NATS subject topology",
|
||||
"07-operations-map.svg": "Operations map",
|
||||
"08-middleware-internals.svg": "Middleware internals",
|
||||
"09-phase-roadmap.svg": "Phase roadmap",
|
||||
"10-workspace-integration.svg": "Workspace integration",
|
||||
"11-zapier-billing.svg": "Zapier and Verae billing layers",
|
||||
"12-peergos-ipfs-tiered-storage.svg": "Peergos / IPFS / cold retrieve",
|
||||
}
|
||||
|
||||
|
||||
def header_footer(canvas, doc):
|
||||
canvas.saveState()
|
||||
w, h = letter
|
||||
canvas.setFillColor(NAVY)
|
||||
canvas.rect(0, h - 28, w, 28, fill=1, stroke=0)
|
||||
canvas.setFillColor(colors.white)
|
||||
canvas.setFont("Helvetica", 8)
|
||||
canvas.drawString(0.7 * inch, h - 18, "Verae Time x Zapier")
|
||||
canvas.drawRightString(w - 0.7 * inch, h - 18, "Architecture diagrams")
|
||||
canvas.setFillColor(TEAL)
|
||||
canvas.rect(0, 0, w, 22, fill=1, stroke=0)
|
||||
canvas.setFillColor(colors.white)
|
||||
canvas.setFont("Helvetica", 8)
|
||||
canvas.drawString(0.7 * inch, 8, "Internal · 18 August 2026 · SVG sources in docs/diagrams/")
|
||||
canvas.drawRightString(w - 0.7 * inch, 8, str(doc.page))
|
||||
canvas.restoreState()
|
||||
|
||||
|
||||
def main():
|
||||
base = getSampleStyleSheet()
|
||||
title_s = ParagraphStyle(
|
||||
"t", parent=base["Title"], fontName="Helvetica-Bold", fontSize=22,
|
||||
textColor=NAVY, alignment=TA_CENTER, spaceAfter=10,
|
||||
)
|
||||
cap = ParagraphStyle(
|
||||
"c", parent=base["Normal"], fontName="Helvetica", fontSize=10,
|
||||
textColor=NAVY, alignment=TA_CENTER, spaceBefore=6, spaceAfter=4,
|
||||
)
|
||||
body = ParagraphStyle(
|
||||
"b", parent=base["Normal"], fontName="Helvetica", fontSize=10,
|
||||
textColor=colors.HexColor("#334155"), alignment=TA_CENTER, leading=14,
|
||||
)
|
||||
tmp = Path(tempfile.mkdtemp(prefix="diag-pdf-"))
|
||||
svgs = sorted(DIAG.glob("*.svg"))
|
||||
usable_w = letter[0] - 1.2 * inch
|
||||
usable_h = letter[1] - 1.6 * inch
|
||||
|
||||
story = [
|
||||
Spacer(1, 2.2 * inch),
|
||||
Paragraph("Architecture diagram atlas", title_s),
|
||||
Paragraph(
|
||||
"Vector sources live in docs/diagrams/. This PDF is a print/review pack "
|
||||
"of the same figures used in getting-started.pdf.",
|
||||
body,
|
||||
),
|
||||
PageBreak(),
|
||||
]
|
||||
for svg in svgs:
|
||||
png = tmp / (svg.stem + ".png")
|
||||
subprocess.run(
|
||||
["rsvg-convert", "-w", "1800", "-f", "png", "-o", str(png), str(svg)],
|
||||
check=True,
|
||||
)
|
||||
# keep aspect, fit page
|
||||
# unknown px size: use viewBox from svg
|
||||
text = svg.read_text(encoding="utf-8", errors="ignore")
|
||||
import re
|
||||
m = re.search(r'viewBox="0 0 (\d+) (\d+)"', text)
|
||||
vw, vh = (int(m.group(1)), int(m.group(2))) if m else (1100, 620)
|
||||
scale = min(usable_w / vw, usable_h / vh)
|
||||
iw, ih = vw * scale, vh * scale
|
||||
story.append(Paragraph(TITLES.get(svg.name, svg.stem), cap))
|
||||
story.append(Image(str(png), width=iw, height=ih))
|
||||
story.append(Paragraph(svg.name, body))
|
||||
story.append(PageBreak())
|
||||
if story[-1].__class__.__name__ == "PageBreak":
|
||||
story.pop()
|
||||
|
||||
doc = SimpleDocTemplate(
|
||||
str(OUT),
|
||||
pagesize=letter,
|
||||
leftMargin=0.6 * inch,
|
||||
rightMargin=0.6 * inch,
|
||||
topMargin=0.5 * inch,
|
||||
bottomMargin=0.4 * inch,
|
||||
title="Verae Time x Zapier — architecture diagram atlas",
|
||||
author="Verae / Zapier research workspace",
|
||||
)
|
||||
doc.build(story, onFirstPage=header_footer, onLaterPages=header_footer)
|
||||
print(f"wrote {OUT} ({OUT.stat().st_size} bytes, {len(svgs)} figures)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
1011
research/zapier/scripts/generate-diagrams.py
Normal file
1011
research/zapier/scripts/generate-diagrams.py
Normal file
File diff suppressed because it is too large
Load diff
1523
research/zapier/scripts/generate-getting-started-pdf.py
Normal file
1523
research/zapier/scripts/generate-getting-started-pdf.py
Normal file
File diff suppressed because it is too large
Load diff
758
research/zapier/scripts/generate-zapier-billing-pdf.py
Normal file
758
research/zapier/scripts/generate-zapier-billing-pdf.py
Normal file
|
|
@ -0,0 +1,758 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Render docs/zapier-billing.pdf — Zapier cost structure for Verae planning.
|
||||
|
||||
Platypus Paragraphs for bullets (never ListFlowable). Every table cell is a Paragraph.
|
||||
Avoid '|' in body text (Helvetica renders it like a capital I).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from reportlab.lib import colors
|
||||
from reportlab.lib.enums import TA_CENTER, TA_LEFT
|
||||
from reportlab.lib.pagesizes import letter
|
||||
from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet
|
||||
from reportlab.lib.units import inch
|
||||
from reportlab.platypus import (
|
||||
PageBreak,
|
||||
Paragraph,
|
||||
Preformatted,
|
||||
SimpleDocTemplate,
|
||||
Spacer,
|
||||
Table,
|
||||
TableStyle,
|
||||
)
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
OUT = ROOT / "docs" / "zapier-billing.pdf"
|
||||
|
||||
NAVY = colors.HexColor("#0F2744")
|
||||
TEAL = colors.HexColor("#1A6B6B")
|
||||
SLATE = colors.HexColor("#334155")
|
||||
RULE = colors.HexColor("#CBD5E1")
|
||||
ROW = colors.HexColor("#F1F5F9")
|
||||
HEAD_BG = colors.HexColor("#0F2744")
|
||||
HEAD_FG = colors.white
|
||||
CODE_BG = colors.HexColor("#F8FAFC")
|
||||
|
||||
|
||||
def styles():
|
||||
base = getSampleStyleSheet()
|
||||
return {
|
||||
"cover_kicker": ParagraphStyle(
|
||||
"cover_kicker", parent=base["Normal"], fontName="Helvetica",
|
||||
fontSize=10, textColor=TEAL, alignment=TA_CENTER, spaceAfter=10,
|
||||
),
|
||||
"cover_title": ParagraphStyle(
|
||||
"cover_title", parent=base["Title"], fontName="Helvetica-Bold",
|
||||
fontSize=24, leading=30, textColor=NAVY, alignment=TA_CENTER, spaceAfter=10,
|
||||
),
|
||||
"cover_sub": ParagraphStyle(
|
||||
"cover_sub", parent=base["Normal"], fontName="Helvetica",
|
||||
fontSize=11.5, leading=16, textColor=SLATE, alignment=TA_CENTER, spaceAfter=8,
|
||||
),
|
||||
"h1": ParagraphStyle(
|
||||
"h1", parent=base["Heading1"], fontName="Helvetica-Bold",
|
||||
fontSize=14.5, leading=18, textColor=NAVY, spaceBefore=14, spaceAfter=7,
|
||||
),
|
||||
"h2": ParagraphStyle(
|
||||
"h2", parent=base["Heading2"], fontName="Helvetica-Bold",
|
||||
fontSize=11.5, leading=15, textColor=TEAL, spaceBefore=10, spaceAfter=5,
|
||||
),
|
||||
"body": ParagraphStyle(
|
||||
"body", parent=base["Normal"], fontName="Helvetica",
|
||||
fontSize=9.4, leading=13, textColor=SLATE, alignment=TA_LEFT, spaceAfter=6,
|
||||
),
|
||||
"bullet": ParagraphStyle(
|
||||
"bullet", parent=base["Normal"], fontName="Helvetica",
|
||||
fontSize=9.4, leading=12.8, textColor=SLATE, leftIndent=14,
|
||||
firstLineIndent=-10, spaceAfter=3,
|
||||
),
|
||||
"toc": ParagraphStyle(
|
||||
"toc", parent=base["Normal"], fontName="Helvetica",
|
||||
fontSize=10, leading=15, textColor=NAVY, leftIndent=6, spaceAfter=2,
|
||||
),
|
||||
"cell": ParagraphStyle(
|
||||
"cell", parent=base["Normal"], fontName="Helvetica",
|
||||
fontSize=7.8, leading=10.4, textColor=SLATE,
|
||||
),
|
||||
"cell_h": ParagraphStyle(
|
||||
"cell_h", parent=base["Normal"], fontName="Helvetica-Bold",
|
||||
fontSize=7.8, leading=10.4, textColor=HEAD_FG,
|
||||
),
|
||||
"code": ParagraphStyle(
|
||||
"code", parent=base["Code"], fontName="Courier",
|
||||
fontSize=7.6, leading=10.2, textColor=NAVY, backColor=CODE_BG,
|
||||
leftIndent=4, rightIndent=4, spaceBefore=3, spaceAfter=7,
|
||||
),
|
||||
"caption": ParagraphStyle(
|
||||
"caption", parent=base["Normal"], fontName="Helvetica-Oblique",
|
||||
fontSize=8, leading=11, textColor=colors.HexColor("#64748B"), spaceAfter=8,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
S = styles()
|
||||
USABLE = letter[0] - 1.4 * inch
|
||||
|
||||
|
||||
def P(text, style="body"):
|
||||
return Paragraph(text, S[style])
|
||||
|
||||
|
||||
def B(text):
|
||||
return Paragraph(f"• {text}", S["bullet"])
|
||||
|
||||
|
||||
def H1(text):
|
||||
return Paragraph(text, S["h1"])
|
||||
|
||||
|
||||
def H2(text):
|
||||
return Paragraph(text, S["h2"])
|
||||
|
||||
|
||||
def CODE(text):
|
||||
return Preformatted(text.rstrip() + "\n", S["code"])
|
||||
|
||||
|
||||
def tbl(headers, rows, widths=None):
|
||||
if widths is None:
|
||||
widths = [USABLE / len(headers)] * len(headers)
|
||||
data = [[Paragraph(h, S["cell_h"]) for h in headers]]
|
||||
for row in rows:
|
||||
data.append([Paragraph(c, S["cell"]) for c in row])
|
||||
t = Table(data, colWidths=widths, repeatRows=1)
|
||||
cmds = [
|
||||
("BACKGROUND", (0, 0), (-1, 0), HEAD_BG),
|
||||
("VALIGN", (0, 0), (-1, -1), "TOP"),
|
||||
("LEFTPADDING", (0, 0), (-1, -1), 4),
|
||||
("RIGHTPADDING", (0, 0), (-1, -1), 4),
|
||||
("TOPPADDING", (0, 0), (-1, -1), 3.5),
|
||||
("BOTTOMPADDING", (0, 0), (-1, -1), 3.5),
|
||||
("GRID", (0, 0), (-1, -1), 0.3, RULE),
|
||||
("LINEBELOW", (0, 0), (-1, 0), 1, TEAL),
|
||||
]
|
||||
for i in range(1, len(data)):
|
||||
if i % 2 == 0:
|
||||
cmds.append(("BACKGROUND", (0, i), (-1, i), ROW))
|
||||
t.setStyle(TableStyle(cmds))
|
||||
return t
|
||||
|
||||
|
||||
def header_footer(canvas, doc):
|
||||
canvas.saveState()
|
||||
w, h = letter
|
||||
canvas.setFillColor(NAVY)
|
||||
canvas.rect(0, h - 28, w, 28, fill=1, stroke=0)
|
||||
canvas.setFillColor(colors.white)
|
||||
canvas.setFont("Helvetica", 8)
|
||||
canvas.drawString(0.7 * inch, h - 18, "Verae Time · Zapier billing brief")
|
||||
canvas.drawRightString(w - 0.7 * inch, h - 18, "Cost structure and variable pricing")
|
||||
canvas.setFillColor(TEAL)
|
||||
canvas.rect(0, 0, w, 22, fill=1, stroke=0)
|
||||
canvas.setFillColor(colors.white)
|
||||
canvas.setFont("Helvetica", 8)
|
||||
canvas.drawString(0.7 * inch, 8, "Internal planning · 18 August 2026 · USD list from zapier.com/pricing")
|
||||
canvas.drawRightString(w - 0.7 * inch, 8, f"{doc.page}")
|
||||
canvas.restoreState()
|
||||
|
||||
|
||||
def cover_footer(canvas, doc):
|
||||
canvas.saveState()
|
||||
w, _ = letter
|
||||
canvas.setFillColor(NAVY)
|
||||
canvas.rect(0, 0, w, 56, fill=1, stroke=0)
|
||||
canvas.setFillColor(TEAL)
|
||||
canvas.rect(0, 56, w, 4, fill=1, stroke=0)
|
||||
canvas.setFillColor(colors.white)
|
||||
canvas.setFont("Helvetica", 9)
|
||||
canvas.drawCentredString(w / 2, 28, "Confirm live rates before any customer quote")
|
||||
canvas.setFont("Helvetica", 8)
|
||||
canvas.drawCentredString(w / 2, 14, "18 August 2026")
|
||||
canvas.restoreState()
|
||||
|
||||
|
||||
def story():
|
||||
out = []
|
||||
|
||||
out.append(Spacer(1, 1.5 * inch))
|
||||
out.append(P("VERAE TIME · COMMERCIAL PLANNING", "cover_kicker"))
|
||||
out.append(P("Zapier cost structure<br/>and billing models", "cover_title"))
|
||||
out.append(
|
||||
P(
|
||||
"How Zapier charges for Zaps, MCP, AI, Code, Agents, and embed — "
|
||||
"with worked examples — so we can design Verae’s meter without "
|
||||
"subsidizing or colliding with Zapier.",
|
||||
"cover_sub",
|
||||
)
|
||||
)
|
||||
out.append(Spacer(1, 14))
|
||||
out.append(
|
||||
tbl(
|
||||
["Source", "As of", "Use"],
|
||||
[
|
||||
[
|
||||
"zapier.com/pricing (official machine-readable page)",
|
||||
"17–18 Aug 2026",
|
||||
"Plan levels, task tiers, overflow, add-ons",
|
||||
],
|
||||
[
|
||||
"Zapier “What is a task?” (updated June 2026)",
|
||||
"June 2026",
|
||||
"What counts; AI 1/3/5; MCP = 2",
|
||||
],
|
||||
[
|
||||
"verae-zapier-middleware PLAN_LIMITS",
|
||||
"This repo",
|
||||
"Our second meter (timestamps / verify / batch)",
|
||||
],
|
||||
],
|
||||
[2.5 * inch, 1.5 * inch, 2.5 * inch],
|
||||
)
|
||||
)
|
||||
out.append(PageBreak())
|
||||
|
||||
out.append(H1("Contents"))
|
||||
for line in [
|
||||
"1. The one idea that unlocks the rest",
|
||||
"2. What a task is (and is not)",
|
||||
"3. Plan levels — what the customer can build",
|
||||
"4. Variable pricing inside the task pool",
|
||||
"5. Task tiers and list price",
|
||||
"6. Overflow (pay-per-task)",
|
||||
"7. Other Zapier products — different models",
|
||||
"8. Worked examples",
|
||||
"9. Verae’s meter today",
|
||||
"10. How this should shape our billing",
|
||||
"11. Planning checklist",
|
||||
]:
|
||||
out.append(P(line, "toc"))
|
||||
out.append(
|
||||
P(
|
||||
"Companion narrative: docs/zapier-billing.md. Architecture guide: getting-started.md §9–10.",
|
||||
"caption",
|
||||
)
|
||||
)
|
||||
|
||||
# 1
|
||||
out.append(H1("1. The one idea that unlocks the rest"))
|
||||
out.append(
|
||||
P(
|
||||
"Zapier does not charge per Zap, per connected app, or (on Professional) per seat. "
|
||||
"It charges for <b>successful work</b>. You buy two things that travel together: "
|
||||
"a <b>plan level</b> (feature set) and a <b>task tier</b> (monthly allowance). "
|
||||
"Enterprise swaps the monthly reset for an annual task pool."
|
||||
)
|
||||
)
|
||||
out.append(
|
||||
P(
|
||||
"Three other economies sit <b>beside</b> that pool, not inside it:"
|
||||
)
|
||||
)
|
||||
out.append(
|
||||
tbl(
|
||||
["Economy", "Unit", "Used for"],
|
||||
[
|
||||
["Core Zapier", "Task", "Zaps, AI by Zapier, Code by Zapier, MCP, SDK"],
|
||||
["Agents add-on", "Activity", "Zapier Agents — does not draw tasks"],
|
||||
["Chatbots add-on", "Bot-count tier", "Zapier Chatbots — not task-metered"],
|
||||
["Verae (us)", "Timestamp / verify / batch", "Middleware PLAN_LIMITS — a second invoice"],
|
||||
],
|
||||
[1.7 * inch, 1.8 * inch, 3.0 * inch],
|
||||
)
|
||||
)
|
||||
out.append(
|
||||
P(
|
||||
"Verae never appears on the Zapier invoice unless we sign a White Label / "
|
||||
"Powered by Zapier contract and resell Zapier usage ourselves.",
|
||||
"caption",
|
||||
)
|
||||
)
|
||||
|
||||
# 2
|
||||
out.append(H1("2. What a task is (and is not)"))
|
||||
out.append(
|
||||
P(
|
||||
"A task is counted when Zapier <b>successfully completes</b> a unit of work. "
|
||||
"Failed steps are free on Zapier (they can still cost us if we already accepted the job)."
|
||||
)
|
||||
)
|
||||
out.append(
|
||||
tbl(
|
||||
["Counts as tasks", "Does not count"],
|
||||
[
|
||||
[
|
||||
"Successful action in another product (Create Timestamp, Slack, Drive, Sheets, webhook)",
|
||||
"The trigger — including polling every 1–15 minutes",
|
||||
],
|
||||
[
|
||||
"Successful Zapier MCP tool execute (read or write)",
|
||||
"Filter, Paths, Formatter, Delay, Looping, Sub-Zap, Digest, Manager, Storage",
|
||||
],
|
||||
[
|
||||
"AI by Zapier and Code by Zapier (see multipliers / extra runtime)",
|
||||
"Zapier Tables and Forms triggers and actions",
|
||||
],
|
||||
[
|
||||
"SDK execute once beta pricing starts (today: beta is free)",
|
||||
"Building or testing a Zap until a step succeeds in production",
|
||||
],
|
||||
],
|
||||
[3.25 * inch, 3.25 * inch],
|
||||
)
|
||||
)
|
||||
out.append(
|
||||
P(
|
||||
"One shared allowance for the whole account. There is no separate MCP budget. "
|
||||
"A five-step Zap that is trigger + Filter + Formatter + Timestamp + Slack uses "
|
||||
"<b>two</b> tasks, not five."
|
||||
)
|
||||
)
|
||||
|
||||
# 3
|
||||
out.append(H1("3. Plan levels — what the customer can build"))
|
||||
out.append(P("Plans are cumulative. Task <b>volume</b> is chosen separately (section 5)."))
|
||||
out.append(
|
||||
tbl(
|
||||
["", "Free", "Professional", "Team", "Enterprise"],
|
||||
[
|
||||
["Seats", "1", "1", "25", "Unlimited"],
|
||||
["Zap shape", "Two-step only", "Multi-step", "Multi-step", "Multi-step"],
|
||||
["Polling", "15 min", "2 min", "1 min", "1 min"],
|
||||
["Premium apps, webhooks", "No", "Yes", "Yes", "Yes"],
|
||||
["Filters, Paths, Formatter, AI", "No", "Yes", "Yes", "Yes"],
|
||||
["Shared Zaps, SAML SSO", "No", "No", "Yes", "Yes"],
|
||||
["SCIM, app controls, TAM, BYOM", "No", "No", "No", "Yes"],
|
||||
["Task cycle", "Monthly", "Monthly", "Monthly", "Annual pool"],
|
||||
["Pay-per-task overflow", "No", "Optional", "Optional", "Custom"],
|
||||
[
|
||||
"Entry price (annual, USD/mo)",
|
||||
"$0 (100 tasks)",
|
||||
"$19.99 (750 tasks)",
|
||||
"$69 (2,000 tasks)",
|
||||
"Sales",
|
||||
],
|
||||
],
|
||||
[1.55 * inch, 1.15 * inch, 1.35 * inch, 1.25 * inch, 1.2 * inch],
|
||||
)
|
||||
)
|
||||
out.append(
|
||||
P(
|
||||
"14-day Professional trial, no card. Non-profit: extra 15% off the subscription, "
|
||||
"not on pay-per-task. Live chat on Professional only at the 2,000+ task tier.",
|
||||
"caption",
|
||||
)
|
||||
)
|
||||
out.append(
|
||||
P(
|
||||
"<b>Verae implication:</b> a Free customer can only run “new file → Create Timestamp.” "
|
||||
"A catalog write, Slack notify, or Wait plus another action requires Professional."
|
||||
)
|
||||
)
|
||||
|
||||
# 4
|
||||
out.append(H1("4. Variable pricing inside the task pool"))
|
||||
out.append(
|
||||
P(
|
||||
"Not every successful step costs one task. Zapier uses multipliers so expensive "
|
||||
"compute consumes more of the <b>same</b> allowance. Confirm /pricing/rates before "
|
||||
"a contract; the page was not fetchable at write time. These multipliers are what "
|
||||
"Zapier publishes on the main pricing page and the June 2026 task article."
|
||||
)
|
||||
)
|
||||
out.append(
|
||||
tbl(
|
||||
["Work", "Tasks per success", "Notes"],
|
||||
[
|
||||
["Typical third-party action (Verae Create Timestamp, Slack, Drive, Sheets)", "1", "The default. Design around this."],
|
||||
["Standard AI by Zapier (default model)", "1", "Same as a normal action"],
|
||||
["Advanced AI by Zapier", "3", "Confirm on the rate card"],
|
||||
["Premium AI by Zapier", "5", "Confirm on the rate card"],
|
||||
["Zapier MCP execute (read or write)", "2", "Discover / inspect / enable are free meta-tools"],
|
||||
[
|
||||
"Code by Zapier",
|
||||
"0, then 1 per extra 30 s",
|
||||
"Included: Free 1s, Pro/Team 30s, Enterprise 2 min. Extended runtime opt-in 1–8 min on paid.",
|
||||
],
|
||||
["Zapier SDK", "Free in beta", "Expect it to join the task pool when beta ends"],
|
||||
],
|
||||
[2.5 * inch, 1.5 * inch, 2.5 * inch],
|
||||
)
|
||||
)
|
||||
out.append(
|
||||
P(
|
||||
"Failed steps = 0 Zapier tasks. Autoreplay and customer retries can still hit Verae. "
|
||||
"That is our problem, not Zapier’s."
|
||||
)
|
||||
)
|
||||
|
||||
# 5
|
||||
out.append(H1("5. Task tiers and list price (USD, August 2026)"))
|
||||
out.append(
|
||||
P(
|
||||
"Self-serve is sold as plan × tier. Annual is about 33% off monthly. "
|
||||
"Implied dollars per task = list price ÷ included tasks (order-of-magnitude only)."
|
||||
)
|
||||
)
|
||||
out.append(H2("Professional"))
|
||||
out.append(
|
||||
tbl(
|
||||
["Tasks / mo", "Annual / mo", "Monthly / mo", "Implied $/task (annual)"],
|
||||
[
|
||||
["750", "$19.99", "$29.99", "$0.027"],
|
||||
["1,500", "$39.00", "$58.50", "$0.026"],
|
||||
["2,000", "$49.00", "$73.50", "$0.025"],
|
||||
["5,000", "$89.00", "$133.50", "$0.018"],
|
||||
["10,000", "$129.00", "$193.50", "$0.013"],
|
||||
["20,000", "$189.00", "$283.50", "$0.0095"],
|
||||
["50,000", "$289.00", "$433.50", "$0.0058"],
|
||||
["100,000", "$489.00", "$733.50", "$0.0049"],
|
||||
["200,000", "$769.00", "$1,149.00", "$0.0038"],
|
||||
["500,000", "$1,499.00", "$2,199.00", "$0.0030"],
|
||||
["1,000,000", "$2,199.00", "$3,299.00", "$0.0022"],
|
||||
["2,000,000", "$3,389.00", "$5,099.00", "$0.0017"],
|
||||
],
|
||||
[1.6 * inch, 1.6 * inch, 1.6 * inch, 1.7 * inch],
|
||||
)
|
||||
)
|
||||
out.append(H2("Team (starts at 2,000)"))
|
||||
out.append(
|
||||
tbl(
|
||||
["Tasks / mo", "Annual / mo", "Monthly / mo", "Implied $/task (annual)"],
|
||||
[
|
||||
["2,000", "$69.00", "$103.50", "$0.035"],
|
||||
["5,000", "$119.00", "$178.50", "$0.024"],
|
||||
["10,000", "$169.00", "$253.50", "$0.017"],
|
||||
["20,000", "$249.00", "$373.50", "$0.012"],
|
||||
["50,000", "$399.00", "$598.50", "$0.008"],
|
||||
["100,000", "$599.00", "$898.50", "$0.006"],
|
||||
["1,000,000", "$2,499.00", "$3,749.00", "$0.0025"],
|
||||
["2,000,000", "$3,999.00", "$5,999.00", "$0.0020"],
|
||||
],
|
||||
[1.6 * inch, 1.6 * inch, 1.6 * inch, 1.7 * inch],
|
||||
)
|
||||
)
|
||||
out.append(
|
||||
P(
|
||||
"Team’s entry dollars-per-task is <b>higher</b> than Pro because the customer is "
|
||||
"buying seats, shared connections, and SSO — not cheaper tasks. Intermediate "
|
||||
"tiers (300k, 400k, 750k, 1.25M, 1.5M, 1.75M) exist on the live page. Above 2M: Sales.",
|
||||
"caption",
|
||||
)
|
||||
)
|
||||
|
||||
# 6
|
||||
out.append(H1("6. Overflow (pay-per-task)"))
|
||||
out.append(
|
||||
tbl(
|
||||
["Setting", "What happens"],
|
||||
[
|
||||
[
|
||||
"On (paid plans)",
|
||||
"Zaps and MCP keep running. Extra tasks bill at 1.25× the plan’s base dollars-per-task (annual) or 2.5× (monthly). Hard ceiling: 3× subscribed tasks, then pause.",
|
||||
],
|
||||
["Off", "Everything stops at the allowance."],
|
||||
["Free", "No overflow. Hits 100 and stops."],
|
||||
["Enterprise", "Annual pool instead of a monthly reset; overflow is contractual."],
|
||||
],
|
||||
[1.6 * inch, 4.9 * inch],
|
||||
)
|
||||
)
|
||||
out.append(
|
||||
P(
|
||||
"Worked overage: Professional 750 annual → base ≈ $19.99 / 750 = <b>$0.0267</b>. "
|
||||
"Overflow ≈ <b>$0.0333</b> per task. Same plan billed monthly: base ≈ $0.0400, "
|
||||
"overflow ≈ <b>$0.100</b>. Overflow on a monthly subscription is about 3× the "
|
||||
"annual overflow rate."
|
||||
)
|
||||
)
|
||||
|
||||
# 7
|
||||
out.append(H1("7. Other Zapier products — different models"))
|
||||
out.append(
|
||||
tbl(
|
||||
["Product", "Model", "Relation to tasks"],
|
||||
[
|
||||
["Zap workflows", "Task", "Core"],
|
||||
["Zapier MCP", "Same pool; 2 tasks per successful execute", "Meta-tools free"],
|
||||
["Zapier SDK", "Beta free; expect tasks later", "Consume, not publish"],
|
||||
["AI by Zapier", "Task × model tier (1 / 3 / 5)", "Inside the Zap"],
|
||||
["Code by Zapier", "Included seconds + 1 task / 30 s extra", "Inside the Zap"],
|
||||
["Tables / Forms", "Plan caps (records, pages, upload size)", "0 tasks when used in Zaps"],
|
||||
[
|
||||
"Agents",
|
||||
"Activities (Free 400/mo; paid ~$33.33/mo annual for 1,500). Per-run cap 10 / 40.",
|
||||
"Does not use tasks",
|
||||
],
|
||||
["Chatbots", "Bot-count tiers (Free 2; paid ≈5 / ≈20)", "Does not use tasks"],
|
||||
["Canvas / Copilot", "Included; Free Copilot has a daily message limit", "Not a usage meter"],
|
||||
[
|
||||
"Directory integration (us)",
|
||||
"$0 to publish. Customer’s Zapier plan pays tasks.",
|
||||
"Partner is not billed",
|
||||
],
|
||||
[
|
||||
"White Label / Powered by Zapier / embed",
|
||||
"Usage-based to the product company. End users may not need their own Zapier bill.",
|
||||
"Sales contract",
|
||||
],
|
||||
["NLA / AI Actions", "Retired", "Do not design around this"],
|
||||
],
|
||||
[1.8 * inch, 2.8 * inch, 1.9 * inch],
|
||||
)
|
||||
)
|
||||
|
||||
# 8
|
||||
out.append(H1("8. Worked examples"))
|
||||
out.append(
|
||||
P(
|
||||
"Assume Professional billed <b>annually</b> unless noted, and that every named "
|
||||
"action succeeds. Verae units are <b>our</b> meter."
|
||||
)
|
||||
)
|
||||
|
||||
out.append(H2("A. Two-step timestamp (Free-capable)"))
|
||||
out.append(P("New file in Drive → Verae: Create Timestamp · 1 Zapier task and 1 Verae stamp per file."))
|
||||
out.append(
|
||||
tbl(
|
||||
["Volume", "Zapier tasks", "Fits", "Zapier $ (annual)", "Verae"],
|
||||
[
|
||||
["80 files / mo", "80", "Free", "$0", "80 stamps — over our free (50)"],
|
||||
["200", "200", "Pro 750", "$19.99", "200 — still under our starter (500)"],
|
||||
["600", "600", "Pro 750", "$19.99", "600 — over our starter; we 402 first"],
|
||||
],
|
||||
[1.3 * inch, 1.2 * inch, 1.1 * inch, 1.4 * inch, 1.5 * inch],
|
||||
)
|
||||
)
|
||||
out.append(
|
||||
P(
|
||||
"Lesson: the customer can be fine on Zapier and still hit <b>our</b> 402. "
|
||||
"The two meters trip at different points."
|
||||
)
|
||||
)
|
||||
|
||||
out.append(H2("B. Production pattern we recommend"))
|
||||
out.append(
|
||||
P(
|
||||
"New file → Filter → Create Timestamp → Formatter → write catalog "
|
||||
"(Zapier Tables = <b>0</b> tasks) → Slack. Billable: Timestamp + Slack = "
|
||||
"<b>2 tasks per file that passes the filter</b>."
|
||||
)
|
||||
)
|
||||
out.append(
|
||||
tbl(
|
||||
["Files / mo", "Pass filter", "Zapier tasks", "Cheapest Pro tier", "Zapier $", "Verae stamps"],
|
||||
[
|
||||
["1,000", "20% (200)", "400", "750", "$19.99", "200"],
|
||||
["1,000", "100%", "2,000", "2,000", "$49", "1,000 (our pro)"],
|
||||
["10,000", "100%", "20,000", "20,000", "$189", "10,000 (contract)"],
|
||||
],
|
||||
[1.05 * inch, 1.1 * inch, 1.05 * inch, 1.25 * inch, 0.95 * inch, 1.1 * inch],
|
||||
)
|
||||
)
|
||||
out.append(
|
||||
P(
|
||||
"Filter early: 800 of 1,000 files dropped costs 0 Zapier tasks and 0 Verae stamps. "
|
||||
"That is the single best cost control we can teach customers."
|
||||
)
|
||||
)
|
||||
|
||||
out.append(H2("C. Create and Wait versus hook"))
|
||||
out.append(B("Wait in the Zap: 1 Zapier task. Fine when the user needs the certificate in the same run."))
|
||||
out.append(
|
||||
B(
|
||||
"Async create + Timestamp Completed hook + update row: 1 task to create + 1 task "
|
||||
"when the hook fires an action = 2 tasks, but the Zap does not sit open. Prefer at volume."
|
||||
)
|
||||
)
|
||||
out.append(
|
||||
B(
|
||||
"Anti-pattern: polling Find Job Status every minute. Each successful search is a task."
|
||||
)
|
||||
)
|
||||
|
||||
out.append(H2("D. Same work via Zapier MCP (an agent)"))
|
||||
out.append(
|
||||
P(
|
||||
"Agent executes Create Timestamp, write Sheet, Slack. Three executes × 2 tasks = "
|
||||
"<b>6 Zapier tasks per file</b>, plus Agent activities if they used Zapier Agents."
|
||||
)
|
||||
)
|
||||
out.append(
|
||||
tbl(
|
||||
["Files / mo", "Zapier tasks", "Cheapest Pro", "Zapier $", "Same stamps via Example B"],
|
||||
[
|
||||
["200", "1,200", "1,500 ($39)", "$39", "400 tasks / $19.99"],
|
||||
["1,000", "6,000", "10,000 ($129)", "$129", "2,000 tasks / $49"],
|
||||
],
|
||||
[1.2 * inch, 1.2 * inch, 1.5 * inch, 1.1 * inch, 1.5 * inch],
|
||||
)
|
||||
)
|
||||
out.append(
|
||||
P(
|
||||
"<b>Planning rule:</b> MCP-shaped clients burn Zapier 2× per hop. Do not price "
|
||||
"Verae as if the customer’s only cost is our stamp."
|
||||
)
|
||||
)
|
||||
|
||||
out.append(H2("E. AI enrichment in the Zap"))
|
||||
out.append(
|
||||
P(
|
||||
"New file → Standard AI extract (1) → Create Timestamp (1) → Sheet (1) = "
|
||||
"<b>3 tasks</b>. Swap Premium AI: 5+1+1 = <b>7 tasks</b>. "
|
||||
"1,000 files: 3,000 vs 7,000 tasks → $89 vs $129 on Pro annual. "
|
||||
"Model choice is the customer’s Zapier bill unless we bury AI inside our action "
|
||||
"(we should not)."
|
||||
)
|
||||
)
|
||||
|
||||
out.append(H2("F. Overflow month"))
|
||||
out.append(
|
||||
P(
|
||||
"Customer on Pro 750 annual ($19.99). A campaign pushes 1,400 successful "
|
||||
"timestamp-only runs."
|
||||
)
|
||||
)
|
||||
out.append(CODE(
|
||||
"Included: 750\n"
|
||||
"Overflow: 650 × ~$0.0333 ≈ $22\n"
|
||||
"Total Zapier that month ≈ $42\n"
|
||||
"Still under the 3× ceiling (2,250)\n"
|
||||
"\n"
|
||||
"Same usage on monthly Pro 750 ($29.99):\n"
|
||||
"Overflow: 650 × ~$0.100 ≈ $65\n"
|
||||
"Plus subscription ≈ $95"
|
||||
))
|
||||
|
||||
out.append(H2("G. White Label — we are the billed party"))
|
||||
out.append(
|
||||
P(
|
||||
"If Verae embeds Zapier and Zapier bills <b>us</b> per task, then 10,000 "
|
||||
"customer files × 2 tasks = 20,000 tasks ≈ <b>$189/mo</b> at Pro annual list "
|
||||
"<b>plus</b> our timestamp COGS. That number belongs in our COGS, not in the "
|
||||
"end customer’s Zapier account."
|
||||
)
|
||||
)
|
||||
|
||||
# 9
|
||||
out.append(H1("9. Verae’s meter today"))
|
||||
out.append(P("From verae-zapier-middleware PLAN_LIMITS — independent of Zapier."))
|
||||
out.append(
|
||||
tbl(
|
||||
["Verae plan", "Timestamps / mo", "Verifications", "Batch", "RPM"],
|
||||
[
|
||||
["free", "50", "50", "no", "30"],
|
||||
["starter", "500", "500", "yes, max 10", "120"],
|
||||
["pro", "5,000", "5,000", "yes, max 100", "600"],
|
||||
["enterprise", "unlimited / contract", "contract", "yes", "3,000"],
|
||||
],
|
||||
[1.4 * inch, 1.5 * inch, 1.3 * inch, 1.2 * inch, 1.1 * inch],
|
||||
)
|
||||
)
|
||||
out.append(
|
||||
P(
|
||||
"Over-quota → HTTP 402 QUOTA_EXCEEDED. Batch on free → 403 PLAN_UPGRADE_REQUIRED. "
|
||||
"These fire whether or not Zapier is still inside its task allowance."
|
||||
)
|
||||
)
|
||||
|
||||
# 10
|
||||
out.append(H1("10. How this should shape our billing"))
|
||||
out.append(B(
|
||||
"<b>1. Never bundle “unlimited Zapier.”</b> We do not control their task tier, "
|
||||
"MCP multiplier, or overflow toggle."
|
||||
))
|
||||
out.append(B(
|
||||
"<b>2. Meter what we uniquely do:</b> accepted timestamp jobs, verifies, maybe "
|
||||
"stored GB / pin-days / Glacier restores — not Zap steps."
|
||||
))
|
||||
out.append(B(
|
||||
"<b>3. Mirror Zapier’s shape if we want familiarity:</b> plan + included units + "
|
||||
"optional overage with a ceiling. Customers already understand 402 versus pause."
|
||||
))
|
||||
out.append(B(
|
||||
"<b>4. Do not copy Zapier’s 2× MCP tax onto our API.</b> POST /zapier/v1/timestamp "
|
||||
"should stay one Verae unit whether the Zap used 1 task or an agent used 2."
|
||||
))
|
||||
out.append(B(
|
||||
"<b>5. Price batch as a plan gate.</b> Zapier still charges 1 task for the batch "
|
||||
"action. We decide whether 100 items cost 1 or 100 of our units. Today we count "
|
||||
"batch size against batchMaxItems and timestamp quota."
|
||||
))
|
||||
out.append(B(
|
||||
"<b>6. Package for the hook pattern.</b> Include enough monthly stamps that "
|
||||
"create + hook + catalog is cheaper on our side than polling status."
|
||||
))
|
||||
out.append(B(
|
||||
"<b>7. Three motions, three packages.</b> Directory user = they pay Zapier + they "
|
||||
"pay Verae. MCP/agent user = they already pay 2 Zapier tasks per hop; keep us "
|
||||
"simple. Embed/White Label = Zapier may bill us; either cover that COGS or sell "
|
||||
"only the stamp and let them bring their own Zapier."
|
||||
))
|
||||
out.append(B(
|
||||
"<b>8. Storage / Peergos / Glacier is a third meter</b> (getting-started §10). "
|
||||
"Do not hide pin-days or restore fees inside “one timestamp.”"
|
||||
))
|
||||
out.append(B(
|
||||
"<b>9. Bill on accepted jobId (HTTP 202),</b> not on Zapier’s success callback. "
|
||||
"A failed later Zap step is free for them and already spent for us."
|
||||
))
|
||||
out.append(B(
|
||||
"<b>10. Quote two lines</b> in every proposal: “Your Zapier plan (estimate N tasks)” "
|
||||
"and “Verae (M timestamps).” Never a single blended number."
|
||||
))
|
||||
out.append(Spacer(1, 6))
|
||||
out.append(
|
||||
P(
|
||||
"At Pro 750 annual, Zapier’s included work is about <b>2.7 cents per task</b>. "
|
||||
"If Create Timestamp is one Zapier task + one Verae stamp, the customer’s "
|
||||
"Zapier share of a two-step Zap is ~$0.027. Price our stamp from chain/ops "
|
||||
"cost, not 1:1 to that 2.7¢ — but if we charge dollars per stamp while Zapier "
|
||||
"is cents per step, SMB Zaps will feel Verae-expensive even when Zapier is "
|
||||
"the bigger invoice at volume."
|
||||
)
|
||||
)
|
||||
|
||||
# 11
|
||||
out.append(H1("11. Planning checklist"))
|
||||
out.append(B("For each target Zap, count billable Zapier <b>actions</b>, not steps on the canvas."))
|
||||
out.append(B("Multiply MCP executes by 2."))
|
||||
out.append(B("Apply AI 1 / 3 / 5 if they enrich in the Zap."))
|
||||
out.append(B("Pick the cheapest Zapier tier that covers that task count (or run overflow math)."))
|
||||
out.append(B("Count Verae timestamps / verifies / batch separately; check the 50 / 500 / 5,000 tripwires."))
|
||||
out.append(B("If we embed Zapier, put Zapier list price into our COGS."))
|
||||
out.append(B("Keep Peergos pin / Glacier restore off the timestamp SKU."))
|
||||
out.append(B("Re-check zapier.com/pricing before any contract. This brief is dated 18 August 2026."))
|
||||
|
||||
out.append(Spacer(1, 14))
|
||||
out.append(
|
||||
P(
|
||||
"Related: docs/zapier-billing.md · getting-started.md §9–10 · "
|
||||
"docs/diagrams/11-zapier-billing.svg · LOGIN.md (customer must have a Zapier plan to run Zaps).",
|
||||
"caption",
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
OUT.parent.mkdir(parents=True, exist_ok=True)
|
||||
doc = SimpleDocTemplate(
|
||||
str(OUT),
|
||||
pagesize=letter,
|
||||
leftMargin=0.7 * inch,
|
||||
rightMargin=0.7 * inch,
|
||||
topMargin=0.55 * inch,
|
||||
bottomMargin=0.45 * inch,
|
||||
title="Zapier cost structure and billing models — Verae planning brief",
|
||||
author="Verae / Zapier research workspace",
|
||||
subject="Official Zapier pricing (Aug 2026) with examples for Verae billing design",
|
||||
)
|
||||
doc.build(story(), onFirstPage=cover_footer, onLaterPages=header_footer)
|
||||
print(f"wrote {OUT} ({OUT.stat().st_size} bytes)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
474
research/zapier/scripts/ingest-coding-set.py
Executable file
474
research/zapier/scripts/ingest-coding-set.py
Executable file
|
|
@ -0,0 +1,474 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Load remaining coding-set artifacts into zapier.platform_reference.
|
||||
|
||||
- exported-schema.json + definition example
|
||||
- OpenAPI specs exploded to api_function
|
||||
- Platform News 2025–2026
|
||||
- example-app source as kind=template
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(os.environ.get("ZAPIER_RESEARCH_ROOT") or Path(__file__).resolve().parents[1])
|
||||
REPOS = ROOT / "repos"
|
||||
NOW = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
print("pyyaml required", file=sys.stderr)
|
||||
raise
|
||||
|
||||
|
||||
def clip(s: str, n: int = 80000) -> str:
|
||||
s = s or ""
|
||||
return s if len(s) <= n else s[: n - 20] + "\n\n…[truncated]"
|
||||
|
||||
|
||||
def rec(**kw) -> dict:
|
||||
kind, key = kw["kind"], kw["key"]
|
||||
_id = kw.get("_id") or f"{kind}:{key}".lower()
|
||||
_id = re.sub(r"[^a-z0-9:._/\-]+", "-", _id)[:200]
|
||||
return {
|
||||
"_id": _id,
|
||||
"kind": kind,
|
||||
"key": key,
|
||||
"title": kw.get("title") or key,
|
||||
"summary": kw.get("summary") or "",
|
||||
"body": clip(kw.get("body") or ""),
|
||||
"usage": kw.get("usage") or "",
|
||||
"signature": kw.get("signature") or "",
|
||||
"aliases": kw.get("aliases") or [],
|
||||
"flags": kw.get("flags") or [],
|
||||
"args": kw.get("args") or [],
|
||||
"examples": kw.get("examples") or [],
|
||||
"source_url": kw.get("source_url") or "",
|
||||
"source_repo": kw.get("source_repo") or "",
|
||||
"source_path": kw.get("source_path") or "",
|
||||
"section": kw.get("section") or "",
|
||||
"tags": sorted(set(kw.get("tags") or [])),
|
||||
"related": kw.get("related") or [],
|
||||
"meta": kw.get("meta") or {},
|
||||
"ingested_at": NOW,
|
||||
}
|
||||
|
||||
|
||||
def ingest_schema() -> list[dict]:
|
||||
rows = []
|
||||
path = REPOS / "zapier-platform" / "packages" / "schema" / "exported-schema.json"
|
||||
if not path.exists():
|
||||
return rows
|
||||
data = json.loads(path.read_text())
|
||||
version = data.get("version") or "unknown"
|
||||
schemas = data.get("schemas") or {}
|
||||
rows.append(
|
||||
rec(
|
||||
kind="schema_json",
|
||||
key="exported-schema",
|
||||
title=f"zapier-platform-schema {version} (exported JSON)",
|
||||
summary="Machine-checkable JSON Schema bundle for App definitions.",
|
||||
body=json.dumps(data, indent=2),
|
||||
source_url="https://github.com/zapier/zapier-platform/blob/main/packages/schema/exported-schema.json",
|
||||
source_repo="zapier/zapier-platform",
|
||||
source_path="zapier-platform/packages/schema/exported-schema.json",
|
||||
section="schema",
|
||||
tags=["schema", "json", "platform"],
|
||||
meta={"platform_schema_version": version, "schema_count": len(schemas)},
|
||||
)
|
||||
)
|
||||
for name, spec in schemas.items():
|
||||
required = spec.get("required") or []
|
||||
props = list((spec.get("properties") or {}).keys())
|
||||
body = [
|
||||
f"# `{name}`",
|
||||
"",
|
||||
f"> {spec.get('description') or name}",
|
||||
"",
|
||||
"## High-level description",
|
||||
"",
|
||||
spec.get("description") or name,
|
||||
"",
|
||||
"## Internals",
|
||||
"",
|
||||
f"JSON Schema id `{spec.get('id')}` from zapier-platform-schema {version}. "
|
||||
"Used by `zapier-platform validate` / build.",
|
||||
"",
|
||||
"## Typed inputs (properties)",
|
||||
"",
|
||||
"```ts",
|
||||
f"// required: {', '.join(required) or '—'}",
|
||||
]
|
||||
for p, ps in (spec.get("properties") or {}).items():
|
||||
ref = ps.get("$ref") or ps.get("type") or "any"
|
||||
opt = "?" if p not in required else ""
|
||||
body.append(f" {p}{opt}: {ref}; // {(ps.get('description') or '')[:120]}")
|
||||
body += ["```", "", "## Schema", "", "```json", json.dumps(spec, indent=2), "```"]
|
||||
rows.append(
|
||||
rec(
|
||||
kind="schema_type",
|
||||
key=name,
|
||||
title=name,
|
||||
summary=spec.get("description") or name,
|
||||
body="\n".join(body),
|
||||
signature=f"{name} required=[{', '.join(required)}]",
|
||||
source_url="https://github.com/zapier/zapier-platform/blob/main/packages/schema/exported-schema.json",
|
||||
source_repo="zapier/zapier-platform",
|
||||
source_path="zapier-platform/packages/schema/exported-schema.json",
|
||||
section="schema",
|
||||
tags=["schema", "json"],
|
||||
meta={"id": spec.get("id"), "required": required, "properties": props, "version": version},
|
||||
)
|
||||
)
|
||||
ex = REPOS / "zapier-platform" / "packages" / "schema" / "examples" / "definition.json"
|
||||
if ex.exists():
|
||||
rows.append(
|
||||
rec(
|
||||
kind="schema_json",
|
||||
key="example-definition",
|
||||
title="Minimal valid App definition.json",
|
||||
summary="Official example App JSON used by schema tests.",
|
||||
body=ex.read_text(),
|
||||
source_path="zapier-platform/packages/schema/examples/definition.json",
|
||||
source_repo="zapier/zapier-platform",
|
||||
section="schema",
|
||||
tags=["schema", "example"],
|
||||
)
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def _schema_to_ts(sch: dict | None, depth: int = 0) -> str:
|
||||
if not sch or depth > 4:
|
||||
return "unknown"
|
||||
if "$ref" in sch:
|
||||
return sch["$ref"].split("/")[-1]
|
||||
if "anyOf" in sch:
|
||||
return " | ".join(_schema_to_ts(s, depth + 1) for s in sch["anyOf"][:6])
|
||||
if "oneOf" in sch:
|
||||
return " | ".join(_schema_to_ts(s, depth + 1) for s in sch["oneOf"][:6])
|
||||
t = sch.get("type")
|
||||
if t == "array":
|
||||
return f"Array<{_schema_to_ts(sch.get('items') or {}, depth + 1)}>"
|
||||
if t == "object":
|
||||
return "object"
|
||||
if isinstance(t, list):
|
||||
return " | ".join(str(x) for x in t)
|
||||
return t or "unknown"
|
||||
|
||||
|
||||
def ingest_openapi() -> list[dict]:
|
||||
rows = []
|
||||
files = [
|
||||
ROOT / "raw" / "openapi" / "actions.yaml",
|
||||
ROOT / "raw" / "openapi" / "connections.yaml",
|
||||
ROOT / "raw" / "openapi" / "trigger-inbox.yaml",
|
||||
ROOT / "raw" / "openapi" / "promotions-openapi.yaml",
|
||||
ROOT / "raw" / "openapi" / "workflow-api-schema.json",
|
||||
]
|
||||
for path in files:
|
||||
if not path.exists():
|
||||
continue
|
||||
text = path.read_text(encoding="utf-8", errors="replace")
|
||||
try:
|
||||
spec = yaml.safe_load(text)
|
||||
except Exception as e:
|
||||
print("skip", path, e)
|
||||
continue
|
||||
if not isinstance(spec, dict):
|
||||
continue
|
||||
title = (spec.get("info") or {}).get("title") or path.stem
|
||||
api_ver = (spec.get("info") or {}).get("version")
|
||||
servers = [s.get("url") for s in (spec.get("servers") or []) if isinstance(s, dict)]
|
||||
paths = spec.get("paths") or {}
|
||||
rows.append(
|
||||
rec(
|
||||
kind="openapi",
|
||||
key=path.name,
|
||||
title=f"{title} ({path.name})",
|
||||
summary=(spec.get("info") or {}).get("description", "")[:400],
|
||||
body=text,
|
||||
source_url=f"https://docs.zapier.com/api-reference/specs/{path.name}" if path.suffix == ".yaml" else "https://api.zapier.com/schema",
|
||||
section="api-reference",
|
||||
tags=["openapi", "api"],
|
||||
meta={"title": title, "version": api_ver, "servers": servers, "path_count": len(paths)},
|
||||
)
|
||||
)
|
||||
for pth, item in paths.items():
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
for method, op in item.items():
|
||||
if method.startswith("x-") or method in {"parameters", "servers"}:
|
||||
continue
|
||||
if not isinstance(op, dict):
|
||||
continue
|
||||
oid = op.get("operationId") or f"{method}_{pth}"
|
||||
params = []
|
||||
for prm in (item.get("parameters") or []) + (op.get("parameters") or []):
|
||||
if isinstance(prm, dict):
|
||||
params.append(
|
||||
{
|
||||
"name": prm.get("name"),
|
||||
"in": prm.get("in"),
|
||||
"required": bool(prm.get("required")),
|
||||
"type": _schema_to_ts(prm.get("schema") or {}),
|
||||
"description": prm.get("description") or "",
|
||||
}
|
||||
)
|
||||
rb = ((op.get("requestBody") or {}).get("content") or {})
|
||||
req_schema = None
|
||||
if "application/json" in rb:
|
||||
req_schema = (rb["application/json"] or {}).get("schema")
|
||||
resps = {}
|
||||
for code, r in (op.get("responses") or {}).items():
|
||||
if isinstance(r, dict):
|
||||
resps[str(code)] = r.get("description") or ""
|
||||
typed_in = ["```ts", "type Input = {"]
|
||||
for prm in params:
|
||||
opt = "" if prm["required"] else "?"
|
||||
typed_in.append(f" {prm['name']}{opt}: {prm['type']}; // in {prm['in']} — {prm['description'][:100]}")
|
||||
if req_schema:
|
||||
typed_in.append(f" body?: {_schema_to_ts(req_schema)}; // request JSON")
|
||||
if len(typed_in) == 2:
|
||||
typed_in.append(" // no parameters")
|
||||
typed_in += ["};", "```"]
|
||||
server = servers[0] if servers else "https://api.zapier.com"
|
||||
body = "\n".join(
|
||||
[
|
||||
f"# `{oid}`",
|
||||
"",
|
||||
f"> {op.get('summary') or oid}",
|
||||
"",
|
||||
"## High-level description",
|
||||
"",
|
||||
op.get("description") or op.get("summary") or oid,
|
||||
"",
|
||||
"## Internals",
|
||||
"",
|
||||
f"`{method.upper()} {server}{pth}` from **{title}**. "
|
||||
"Authenticate per spec (OAuth / partner JWT / embed secret). "
|
||||
"This is a Zapier *public* API — not `z.request` inside a connector.",
|
||||
"",
|
||||
"## Typed inputs",
|
||||
"",
|
||||
*typed_in,
|
||||
"",
|
||||
"## Outputs",
|
||||
"",
|
||||
"```json",
|
||||
json.dumps(resps, indent=2),
|
||||
"```",
|
||||
"",
|
||||
f"- Tags: {', '.join(op.get('tags') or []) or '—'}",
|
||||
f"- Security: {json.dumps(op.get('security') or spec.get('security') or [])}",
|
||||
"",
|
||||
"## Example",
|
||||
"",
|
||||
"```http",
|
||||
f"{method.upper()} {pth} HTTP/1.1",
|
||||
f"Host: {server.replace('https://','')}",
|
||||
"Authorization: Bearer <token>",
|
||||
"```",
|
||||
]
|
||||
)
|
||||
rows.append(
|
||||
rec(
|
||||
kind="api_function",
|
||||
key=str(oid),
|
||||
title=f"{method.upper()} {pth} — {op.get('summary') or oid}",
|
||||
summary=op.get("summary") or op.get("description") or oid,
|
||||
body=body,
|
||||
usage=f"{method.upper()} {pth}",
|
||||
signature=f"{method.upper()} {pth}",
|
||||
args=params,
|
||||
source_url=f"https://docs.zapier.com/api-reference",
|
||||
section="api-reference",
|
||||
tags=["api", "openapi", (op.get("tags") or ["api"])[0].lower().replace(" ", "-")],
|
||||
meta={
|
||||
"method": method.upper(),
|
||||
"path": pth,
|
||||
"spec": path.name,
|
||||
"api_title": title,
|
||||
"servers": servers,
|
||||
"request_schema": req_schema,
|
||||
"responses": resps,
|
||||
},
|
||||
)
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def ingest_news() -> list[dict]:
|
||||
rows = []
|
||||
news_root = ROOT / "raw" / "docs" / "integrations" / "news"
|
||||
if not news_root.exists():
|
||||
return rows
|
||||
for md in sorted(news_root.rglob("*.md")):
|
||||
text = md.read_text(encoding="utf-8", errors="replace")
|
||||
rel = str(md.relative_to(news_root))
|
||||
title = ""
|
||||
for line in text.splitlines():
|
||||
if line.startswith("#"):
|
||||
title = re.sub(r"^#+\s*", "", line).strip()
|
||||
break
|
||||
rows.append(
|
||||
rec(
|
||||
kind="platform_news",
|
||||
key=rel.replace(".md", ""),
|
||||
title=title or rel,
|
||||
summary=title or rel,
|
||||
body=text,
|
||||
source_url=f"https://docs.zapier.com/integrations/news/{rel.replace('.md','')}",
|
||||
section="news",
|
||||
tags=["news", "changelog", rel.split("/")[0] if "/" in rel else "news"],
|
||||
meta={"year": rel.split("/")[0] if rel[:4].isdigit() else None},
|
||||
)
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
SOURCE_GLOBS = ("*.js", "*.ts", "*.json", "*.md")
|
||||
SKIP_NAMES = {"package-lock.json"}
|
||||
|
||||
|
||||
def ingest_templates() -> list[dict]:
|
||||
rows = []
|
||||
base = REPOS / "zapier-platform" / "example-apps"
|
||||
if not base.exists():
|
||||
return rows
|
||||
for d in sorted(p for p in base.iterdir() if p.is_dir()):
|
||||
files = {}
|
||||
for p in d.rglob("*"):
|
||||
if not p.is_file():
|
||||
continue
|
||||
if p.name in SKIP_NAMES or "node_modules" in p.parts:
|
||||
continue
|
||||
if p.suffix not in {".js", ".ts", ".json", ".md"}:
|
||||
continue
|
||||
rel = str(p.relative_to(d))
|
||||
try:
|
||||
files[rel] = p.read_text(encoding="utf-8", errors="replace")
|
||||
except Exception:
|
||||
continue
|
||||
auth = next((t for t in ("oauth2", "oauth1", "session-auth", "basic-auth", "digest-auth", "custom-auth") if t in d.name), None)
|
||||
pattern = next((t for t in ("trigger", "create", "search", "rest-hooks", "files", "middleware", "resource", "callback", "line-items", "dynamic-dropdown") if t in d.name), "example")
|
||||
parts = [f"# example-app `{d.name}`", "", f"Auth: `{auth or 'n/a'}` · Pattern: `{pattern}`", ""]
|
||||
for rel, src in sorted(files.items()):
|
||||
parts += [f"## `{rel}`", "", f"```{p.suffix.lstrip('.') if False else rel.split('.')[-1]}", src[:12000], "```", ""]
|
||||
rows.append(
|
||||
rec(
|
||||
kind="template",
|
||||
key=d.name,
|
||||
title=f"Source template: {d.name}",
|
||||
summary=f"Full example-app source ({auth or 'n/a'} / {pattern}).",
|
||||
body="\n".join(parts),
|
||||
source_url=f"https://github.com/zapier/zapier-platform/tree/main/example-apps/{d.name}",
|
||||
source_repo="zapier/zapier-platform",
|
||||
source_path=f"zapier-platform/example-apps/{d.name}",
|
||||
section="templates",
|
||||
tags=["template", "example", pattern] + ([auth] if auth else []),
|
||||
related=[f"zapier-platform init --template {d.name}"] if auth or d.name in {"minimal", "files", "callback"} else [],
|
||||
meta={"files": list(files), "auth": auth, "pattern": pattern, "local": f"repos/zapier-platform/example-apps/{d.name}"},
|
||||
)
|
||||
)
|
||||
# golden local project
|
||||
gold = ROOT / "scratch" / "oauth2-typescript"
|
||||
if gold.exists():
|
||||
srcs = {}
|
||||
for p in (gold / "src").rglob("*"):
|
||||
if p.is_file():
|
||||
srcs[str(p.relative_to(gold))] = p.read_text(encoding="utf-8", errors="replace")
|
||||
rows.append(
|
||||
rec(
|
||||
kind="template",
|
||||
key="scratch-oauth2-typescript",
|
||||
title="Golden local project: scratch/oauth2-typescript",
|
||||
summary="CLI 19.1.0 `zapier-platform init --template oauth2 --language typescript`. Structurally valid after build.",
|
||||
body="\n".join(
|
||||
line
|
||||
for k, v in sorted(srcs.items())
|
||||
for line in (f"## `{k}`", "", "```ts", v, "```", "")
|
||||
),
|
||||
source_path="scratch/oauth2-typescript",
|
||||
section="templates",
|
||||
tags=["template", "golden", "oauth2", "typescript"],
|
||||
related=["init", "validate", "z.request"],
|
||||
meta={"cli_version": "19.1.0", "validated": True},
|
||||
)
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def main() -> int:
|
||||
rows: list[dict] = []
|
||||
print("schema…")
|
||||
rows += ingest_schema()
|
||||
print("openapi…")
|
||||
rows += ingest_openapi()
|
||||
print("news…")
|
||||
rows += ingest_news()
|
||||
print("templates…")
|
||||
rows += ingest_templates()
|
||||
counts: dict[str, int] = {}
|
||||
for r in rows:
|
||||
counts[r["kind"]] = counts.get(r["kind"], 0) + 1
|
||||
out = ROOT / "raw" / "coding-set.jsonl"
|
||||
with out.open("w") as f:
|
||||
for r in rows:
|
||||
f.write(json.dumps(r, ensure_ascii=False) + "\n")
|
||||
print("wrote", len(rows), counts, "→", out)
|
||||
|
||||
rows.append(
|
||||
rec(
|
||||
kind="guide",
|
||||
key="coding-set",
|
||||
title="Coding set — schema, APIs, news, templates",
|
||||
summary="After build-new-connector: schema_json, api_function, platform_news, template.",
|
||||
body="""# Coding set
|
||||
|
||||
Start: `db.platform_reference.findOne({kind:'guide', key:'build-new-connector'})`
|
||||
|
||||
Then:
|
||||
|
||||
```js
|
||||
db.platform_reference.findOne({kind:'schema_json', key:'exported-schema'})
|
||||
db.platform_reference.find({kind:'api_function'}).sort({key:1})
|
||||
db.platform_reference.find({kind:'platform_news'})
|
||||
db.platform_reference.find({kind:'template', key:'oauth2-typescript'})
|
||||
db.platform_reference.findOne({kind:'template', key:'scratch-oauth2-typescript'})
|
||||
```
|
||||
|
||||
Local golden app: `scratch/oauth2-typescript` (CLI 19.1.0, `zapier-platform validate` clean after build).
|
||||
CLIs: `~/.npm-global/bin/zapier-platform` and `zapier-sdk` (login still required for push/SDK).
|
||||
""",
|
||||
section="guide",
|
||||
tags=["guide", "coding-set"],
|
||||
)
|
||||
)
|
||||
|
||||
uri = os.environ.get("ZAPIER_MONGO_URI") or os.environ.get("MDB_MCP_CONNECTION_STRING")
|
||||
if not uri:
|
||||
print("no mongo uri")
|
||||
return 0
|
||||
from pymongo import MongoClient, ReplaceOne
|
||||
|
||||
col = MongoClient(uri).get_database("zapier")["platform_reference"]
|
||||
ops = [ReplaceOne({"_id": r["_id"]}, r, upsert=True) for r in rows]
|
||||
for i in range(0, len(ops), 250):
|
||||
col.bulk_write(ops[i : i + 250], ordered=False)
|
||||
print("mongo kinds", {k: col.count_documents({"kind": k}) for k in ("schema_json", "api_function", "platform_news", "template", "schema_type")})
|
||||
col.database["meta"].update_one(
|
||||
{"_id": "ingest"},
|
||||
{"$set": {"coding_set_ingested_at": NOW}},
|
||||
upsert=True,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
1152
research/zapier/scripts/ingest-function-reference.py
Normal file
1152
research/zapier/scripts/ingest-function-reference.py
Normal file
File diff suppressed because it is too large
Load diff
1256
research/zapier/scripts/ingest-mcp-reference.py
Normal file
1256
research/zapier/scripts/ingest-mcp-reference.py
Normal file
File diff suppressed because it is too large
Load diff
1032
research/zapier/scripts/ingest-platform-reference.py
Executable file
1032
research/zapier/scripts/ingest-platform-reference.py
Executable file
File diff suppressed because it is too large
Load diff
86
research/zapier/scripts/ingest-templates-sitemap.py
Normal file
86
research/zapier/scripts/ingest-templates-sitemap.py
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Download Zapier template sitemaps and group URLs by app slug."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import ssl
|
||||
import time
|
||||
import urllib.request
|
||||
from collections import defaultdict
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
OUT = ROOT / "raw" / "templates-by-slug.json"
|
||||
INDEXES = [
|
||||
"https://zapier.com/sitemaps/zap-templates/LZB2XiWzbJFggRYdnRGwcoAgEkTnHHEh",
|
||||
"https://zapier.com/sitemaps/templates/Xf3Lk9YbRzQnmgP2eCJtA5Wxv0NdMu6T",
|
||||
]
|
||||
LOC_RE = re.compile(r"<loc>\s*([^<\s]+)\s*</loc>", re.I)
|
||||
# /apps/{a}/integrations/{b}/{id}/{slug}
|
||||
PATH_RE = re.compile(
|
||||
r"/apps/([^/]+)/integrations/([^/]+)/(\d+)/([^/?#]+)"
|
||||
)
|
||||
|
||||
|
||||
def fetch(url: str) -> str:
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0 research"})
|
||||
ctx = ssl.create_default_context()
|
||||
with urllib.request.urlopen(req, timeout=60, context=ctx) as r:
|
||||
return r.read().decode("utf-8", errors="ignore")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
pages = []
|
||||
for idx in INDEXES:
|
||||
xml = fetch(idx)
|
||||
found = LOC_RE.findall(xml)
|
||||
print(f"index {idx} pages={len(found)}", flush=True)
|
||||
pages.extend(found)
|
||||
pages = list(dict.fromkeys(pages))
|
||||
print("unique sitemap pages", len(pages), flush=True)
|
||||
|
||||
locs = []
|
||||
with ThreadPoolExecutor(max_workers=12) as ex:
|
||||
futs = {ex.submit(fetch, u): u for u in pages}
|
||||
n = 0
|
||||
for fut in as_completed(futs):
|
||||
try:
|
||||
xml = fut.result()
|
||||
locs.extend(LOC_RE.findall(xml))
|
||||
except Exception as e:
|
||||
print("fail", futs[fut], e, flush=True)
|
||||
n += 1
|
||||
if n % 50 == 0:
|
||||
print(f"pages {n}/{len(pages)} locs={len(locs)}", flush=True)
|
||||
time.sleep(0.01)
|
||||
|
||||
by_slug = defaultdict(list)
|
||||
seen = set()
|
||||
for loc in locs:
|
||||
m = PATH_RE.search(loc)
|
||||
if not m:
|
||||
continue
|
||||
a, b, tid, slug = m.group(1), m.group(2), m.group(3), m.group(4)
|
||||
title = slug.replace("-", " ")
|
||||
rec = {
|
||||
"id": tid,
|
||||
"title": title,
|
||||
"url": loc,
|
||||
"apps": [a, b],
|
||||
}
|
||||
key = (tid, a, b)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
by_slug[a].append(rec)
|
||||
if a != b:
|
||||
by_slug[b].append(rec)
|
||||
|
||||
OUT.write_text(json.dumps({k: v for k, v in by_slug.items()}, indent=2))
|
||||
print("slugs", len(by_slug), "unique templates", len(seen), "rows", sum(len(v) for v in by_slug.values()))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
293
research/zapier/scripts/load-mongodb.py
Normal file
293
research/zapier/scripts/load-mongodb.py
Normal file
|
|
@ -0,0 +1,293 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Merge all Zapier research artifacts and upsert into MongoDB."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def load_jsonl(path: Path) -> list[dict]:
|
||||
if not path.exists():
|
||||
return []
|
||||
rows = []
|
||||
for line in path.open():
|
||||
line = line.strip()
|
||||
if line:
|
||||
rows.append(json.loads(line))
|
||||
return rows
|
||||
|
||||
|
||||
def load_json(path: Path):
|
||||
if not path.exists():
|
||||
return None
|
||||
return json.loads(path.read_text())
|
||||
|
||||
|
||||
def domain(url: str | None) -> str | None:
|
||||
if not url:
|
||||
return None
|
||||
if not str(url).startswith("http"):
|
||||
url = "https://" + url
|
||||
try:
|
||||
host = (urlparse(url).hostname or "").lower()
|
||||
except Exception:
|
||||
return None
|
||||
if host.startswith("www."):
|
||||
host = host[4:]
|
||||
return host or None
|
||||
|
||||
|
||||
def main() -> int:
|
||||
try:
|
||||
from pymongo import ASCENDING, DESCENDING, MongoClient
|
||||
except ImportError:
|
||||
print("pymongo missing", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
uri = os.environ.get("ZAPIER_MONGO_URI") or os.environ.get("MDB_MCP_CONNECTION_STRING")
|
||||
if not uri:
|
||||
print("Set ZAPIER_MONGO_URI (after opening the SSH tunnel)", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
catalog = load_jsonl(ROOT / "raw" / "all-apps-full.jsonl") or load_jsonl(
|
||||
ROOT / "raw" / "all-apps.jsonl"
|
||||
)
|
||||
contacts_all = {r["slug"]: r for r in (load_json(ROOT / "contacts-all.json") or [])}
|
||||
curated = load_json(ROOT / "contacts.json") or {}
|
||||
caps_rows = load_jsonl(ROOT / "raw" / "capabilities-scrape.jsonl")
|
||||
caps = {r["slug"]: r for r in caps_rows}
|
||||
identified = load_json(ROOT / "identified-vendors.json") or {}
|
||||
templates_by_slug = load_json(ROOT / "raw" / "templates-by-slug.json") or {}
|
||||
commercial = {r["slug"]: r for r in load_jsonl(ROOT / "raw" / "commercial-scrape.jsonl")}
|
||||
extras = {r["slug"]: r for r in load_jsonl(ROOT / "raw" / "extras-scrape.jsonl")}
|
||||
help_auth = load_json(ROOT / "raw" / "help-auth-by-slug.json") or {}
|
||||
api_docs = {r["slug"]: r for r in load_jsonl(ROOT / "raw" / "api-docs-scrape.jsonl")}
|
||||
verticals = {}
|
||||
for vert, rows in (identified.get("verticals") or {}).items():
|
||||
for r in rows:
|
||||
verticals.setdefault(r["slug"], []).append(vert)
|
||||
|
||||
# sibling products by domain
|
||||
by_dom = {}
|
||||
for a in catalog:
|
||||
d = domain(a.get("external_url"))
|
||||
if d:
|
||||
by_dom.setdefault(d, []).append({"name": a.get("name"), "slug": a.get("slug")})
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
docs = []
|
||||
for a in catalog:
|
||||
slug = a["slug"]
|
||||
c = contacts_all.get(slug) or {}
|
||||
cur = curated.get(slug) or {}
|
||||
cap = caps.get(slug) or {}
|
||||
d = domain(a.get("external_url"))
|
||||
siblings = [s for s in by_dom.get(d or "", []) if s.get("slug") != slug]
|
||||
docs.append(
|
||||
{
|
||||
"_id": slug,
|
||||
"slug": slug,
|
||||
"name": a.get("name"),
|
||||
"legal_name": cur.get("legal_name") or c.get("legal_name") or a.get("name"),
|
||||
"description": a.get("description") or cap.get("description"),
|
||||
"website": cur.get("website") or c.get("website") or a.get("external_url"),
|
||||
"domain": d,
|
||||
"contact": {
|
||||
"hq_address": cur.get("hq_address") or c.get("hq_address"),
|
||||
"phone": cur.get("phone") or c.get("phone"),
|
||||
"email": cur.get("email") or c.get("email"),
|
||||
"sales_form": cur.get("sales_form") or c.get("sales_form"),
|
||||
"support": cur.get("support") or c.get("support"),
|
||||
"linkedin": cur.get("linkedin") or c.get("linkedin"),
|
||||
"notes": cur.get("notes"),
|
||||
},
|
||||
"zapier": {
|
||||
"url": "https://zapier.com"
|
||||
+ (a.get("app_profile_url") or f"/apps/{slug}/integrations"),
|
||||
"mcp_url": f"https://zapier.com/mcp/{slug}",
|
||||
"zap_usage": a.get("zap_usage_count") or 0,
|
||||
"popularity_rank": a.get("popularity"),
|
||||
"request_count": a.get("request_count"),
|
||||
"age_in_days": a.get("age_in_days"),
|
||||
"days_since_last_update": a.get("days_since_last_update"),
|
||||
"api_docs_url": a.get("api_docs_url"),
|
||||
"learn_more_url": a.get("learn_more_url"),
|
||||
"implementation": cap.get("implementation")
|
||||
or a.get("current_implementation_id"),
|
||||
"partner_tier": cap.get("partner_tier"),
|
||||
"is_premium": a.get("is_premium"),
|
||||
"is_beta": a.get("is_beta"),
|
||||
"is_built_in": a.get("is_built_in"),
|
||||
"is_featured": a.get("is_featured"),
|
||||
"is_public": a.get("is_public"),
|
||||
"is_upcoming": a.get("is_upcoming"),
|
||||
"invite_url": a.get("invite_url"),
|
||||
"hashtag": a.get("hashtag"),
|
||||
"canonical_id": a.get("canonical_id"),
|
||||
"categories": a.get("category_titles") or a.get("categories") or [],
|
||||
},
|
||||
"capabilities": {
|
||||
"trigger_count": cap.get("trigger_count") or 0,
|
||||
"instant_trigger_count": cap.get("instant_trigger_count") or 0,
|
||||
"action_count": cap.get("action_count") or 0,
|
||||
"search_count": cap.get("search_count") or 0,
|
||||
"calling_convention": (
|
||||
"instant webhook"
|
||||
if cap.get("instant_trigger_count")
|
||||
else (
|
||||
"polling trigger"
|
||||
if cap.get("trigger_count")
|
||||
else "actions only / no trigger"
|
||||
)
|
||||
),
|
||||
"triggers": cap.get("triggers") or [],
|
||||
"actions": cap.get("actions") or [],
|
||||
"searches": cap.get("searches") or [],
|
||||
"error": cap.get("error"),
|
||||
},
|
||||
"ecosystem": {
|
||||
"alternatives": cap.get("alternatives") or [],
|
||||
"paired_apps": cap.get("paired_apps") or [],
|
||||
"sibling_zapier_apps": siblings,
|
||||
"help_articles": (extras.get(slug) or {}).get("help_articles")
|
||||
or [
|
||||
{"title": t} for t in (cap.get("help_articles") or [])
|
||||
],
|
||||
},
|
||||
"templates": {
|
||||
"count": len(templates_by_slug.get(slug) or [])
|
||||
or (extras.get(slug) or {}).get("template_count_zapier")
|
||||
or 0,
|
||||
"zapier_reported_count": (extras.get(slug) or {}).get(
|
||||
"template_count_zapier"
|
||||
),
|
||||
"featured": (extras.get(slug) or {}).get("featured_templates") or [],
|
||||
"items": (templates_by_slug.get(slug) or [])[:50],
|
||||
},
|
||||
"overview": (extras.get(slug) or {}).get("overview") or "",
|
||||
"auth": {
|
||||
"primary": (help_auth.get(slug) or {}).get("primary_auth")
|
||||
or (
|
||||
"oauth"
|
||||
if "oauth2" in ((api_docs.get(slug) or {}).get("signals") or [])
|
||||
else (
|
||||
"api_key"
|
||||
if "api_key" in ((api_docs.get(slug) or {}).get("signals") or [])
|
||||
else None
|
||||
)
|
||||
),
|
||||
"signals": sorted(
|
||||
set(
|
||||
((help_auth.get(slug) or {}).get("auth_signals") or [])
|
||||
+ ((api_docs.get(slug) or {}).get("signals") or [])
|
||||
)
|
||||
),
|
||||
"prerequisites": (help_auth.get(slug) or {}).get("prerequisites") or [],
|
||||
"connect_steps": (help_auth.get(slug) or {}).get("connect_steps") or [],
|
||||
"source": (
|
||||
"zapier_help"
|
||||
if slug in help_auth
|
||||
else ("api_docs" if slug in api_docs else None)
|
||||
),
|
||||
"api_docs": {
|
||||
"url": (api_docs.get(slug) or {}).get("api_docs_url"),
|
||||
"title": (api_docs.get(slug) or {}).get("title"),
|
||||
"signals": (api_docs.get(slug) or {}).get("signals") or [],
|
||||
"error": (api_docs.get(slug) or {}).get("error"),
|
||||
},
|
||||
},
|
||||
"commercial": {
|
||||
"flags": (commercial.get(slug) or {}).get("flags") or [],
|
||||
"price_mentions": (commercial.get(slug) or {}).get("price_mentions")
|
||||
or [],
|
||||
"pages": (commercial.get(slug) or {}).get("pages") or [],
|
||||
},
|
||||
"vendors_txt_verticals": verticals.get(slug) or [],
|
||||
"updated_at": now,
|
||||
}
|
||||
)
|
||||
|
||||
client = MongoClient(uri, serverSelectionTimeoutMS=8000)
|
||||
db = client.get_default_database()
|
||||
if db is None:
|
||||
db = client["zapier"]
|
||||
col = db["apps"]
|
||||
# replace set
|
||||
slugs = [d["_id"] for d in docs]
|
||||
if slugs:
|
||||
col.delete_many({"_id": {"$nin": slugs}})
|
||||
ops = 0
|
||||
from pymongo import ReplaceOne
|
||||
|
||||
batch = []
|
||||
for doc in docs:
|
||||
batch.append(ReplaceOne({"_id": doc["_id"]}, doc, upsert=True))
|
||||
if len(batch) >= 500:
|
||||
col.bulk_write(batch, ordered=False)
|
||||
ops += len(batch)
|
||||
batch = []
|
||||
if batch:
|
||||
col.bulk_write(batch, ordered=False)
|
||||
ops += len(batch)
|
||||
|
||||
col.create_index([("name", ASCENDING)])
|
||||
col.create_index([("zapier.zap_usage", DESCENDING)])
|
||||
col.create_index([("zapier.categories", ASCENDING)])
|
||||
col.create_index([("domain", ASCENDING)])
|
||||
col.create_index([("vendors_txt_verticals", ASCENDING)])
|
||||
col.create_index([("contact.email", ASCENDING)])
|
||||
|
||||
db["meta"].replace_one(
|
||||
{"_id": "ingest"},
|
||||
{
|
||||
"_id": "ingest",
|
||||
"app_count": len(docs),
|
||||
"loaded_at": now,
|
||||
"source": "zapier research workspace",
|
||||
},
|
||||
upsert=True,
|
||||
)
|
||||
help_col = db["help_articles"]
|
||||
help_batch = []
|
||||
help_path = ROOT / "raw" / "help-articles.jsonl"
|
||||
if help_path.exists():
|
||||
for line in help_path.open():
|
||||
rec = json.loads(line)
|
||||
hid = rec.get("url") or rec.get("final_url")
|
||||
if not hid:
|
||||
continue
|
||||
help_batch.append(
|
||||
ReplaceOne(
|
||||
{"_id": hid},
|
||||
{
|
||||
"_id": hid,
|
||||
"slug": rec.get("slug"),
|
||||
"title": rec.get("help_title") or rec.get("title"),
|
||||
"url": rec.get("url"),
|
||||
"auth_signals": rec.get("auth_signals") or [],
|
||||
"prerequisites": rec.get("prerequisites"),
|
||||
"connect_steps": rec.get("connect_steps"),
|
||||
"text": rec.get("text"),
|
||||
"updated_at": now,
|
||||
},
|
||||
upsert=True,
|
||||
)
|
||||
)
|
||||
if help_batch:
|
||||
help_col.bulk_write(help_batch, ordered=False)
|
||||
help_col.create_index([("slug", ASCENDING)])
|
||||
print("help_articles", help_col.estimated_document_count())
|
||||
|
||||
print(f"upserted {ops} apps into {db.name}.apps")
|
||||
print("count", col.estimated_document_count())
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
66
research/zapier/scripts/load-templates-collection.py
Normal file
66
research/zapier/scripts/load-templates-collection.py
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Load the full Zapier template sitemap into zapier.templates."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from pymongo import ASCENDING, UpdateOne
|
||||
|
||||
|
||||
def main() -> int:
|
||||
from pymongo import MongoClient
|
||||
|
||||
uri = os.environ.get("ZAPIER_MONGO_URI") or os.environ.get("MDB_MCP_CONNECTION_STRING")
|
||||
if not uri:
|
||||
print("ZAPIER_MONGO_URI required", file=sys.stderr)
|
||||
return 1
|
||||
path = sys.argv[1] if len(sys.argv) > 1 else "raw/templates-by-slug.json"
|
||||
data = json.load(open(path))
|
||||
client = MongoClient(uri, serverSelectionTimeoutMS=15000)
|
||||
db = client.get_default_database()
|
||||
if db is None:
|
||||
db = client["zapier"]
|
||||
col = db["templates"]
|
||||
now = datetime.now(timezone.utc)
|
||||
# flatten unique by template id + app pair
|
||||
seen = set()
|
||||
batch = []
|
||||
n = 0
|
||||
for slug, items in data.items():
|
||||
for it in items:
|
||||
tid = str(it.get("id") or "")
|
||||
apps = tuple(it.get("apps") or [])
|
||||
key = (tid, apps)
|
||||
if not tid or key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
_id = f"{tid}:{'+'.join(apps)}"
|
||||
doc = {
|
||||
"_id": _id,
|
||||
"template_id": tid,
|
||||
"title": it.get("title"),
|
||||
"url": it.get("url"),
|
||||
"apps": list(apps),
|
||||
"updated_at": now,
|
||||
}
|
||||
batch.append(UpdateOne({"_id": _id}, {"$set": doc}, upsert=True))
|
||||
if len(batch) >= 1000:
|
||||
col.bulk_write(batch, ordered=False)
|
||||
n += len(batch)
|
||||
batch = []
|
||||
if n % 50000 == 0:
|
||||
print("wrote", n, flush=True)
|
||||
if batch:
|
||||
col.bulk_write(batch, ordered=False)
|
||||
n += len(batch)
|
||||
col.create_index([("apps", ASCENDING)])
|
||||
col.create_index([("template_id", ASCENDING)])
|
||||
print("upserted", n, "estimated", col.estimated_document_count())
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
311
research/zapier/scripts/md_to_guide_pdf.py
Normal file
311
research/zapier/scripts/md_to_guide_pdf.py
Normal file
|
|
@ -0,0 +1,311 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Render a Markdown guide to a letter PDF (headings, tables, code, bullets).
|
||||
|
||||
Usage:
|
||||
python3 scripts/md_to_guide_pdf.py INPUT.md OUTPUT.pdf [title]
|
||||
python3 scripts/md_to_guide_pdf.py --concat OUT.pdf TITLE file1.md file2.md ...
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from reportlab.lib import colors
|
||||
from reportlab.lib.enums import TA_CENTER, TA_LEFT
|
||||
from reportlab.lib.pagesizes import letter
|
||||
from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet
|
||||
from reportlab.lib.units import inch
|
||||
from reportlab.platypus import (
|
||||
PageBreak,
|
||||
Paragraph,
|
||||
Preformatted,
|
||||
SimpleDocTemplate,
|
||||
Spacer,
|
||||
Table,
|
||||
TableStyle,
|
||||
)
|
||||
|
||||
NAVY = colors.HexColor("#0F2744")
|
||||
TEAL = colors.HexColor("#1A6B6B")
|
||||
SLATE = colors.HexColor("#334155")
|
||||
RULE = colors.HexColor("#CBD5E1")
|
||||
ROW = colors.HexColor("#F1F5F9")
|
||||
CODE_BG = colors.HexColor("#F8FAFC")
|
||||
USABLE = letter[0] - 1.4 * inch
|
||||
|
||||
|
||||
def _styles():
|
||||
base = getSampleStyleSheet()
|
||||
return {
|
||||
"cover_kicker": ParagraphStyle(
|
||||
"k", parent=base["Normal"], fontName="Helvetica", fontSize=10,
|
||||
textColor=TEAL, alignment=TA_CENTER, spaceAfter=8,
|
||||
),
|
||||
"cover_title": ParagraphStyle(
|
||||
"t", parent=base["Title"], fontName="Helvetica-Bold", fontSize=20,
|
||||
leading=26, textColor=NAVY, alignment=TA_CENTER, spaceAfter=10,
|
||||
),
|
||||
"h1": ParagraphStyle(
|
||||
"h1", parent=base["Heading1"], fontName="Helvetica-Bold",
|
||||
fontSize=13.5, leading=17, textColor=NAVY, spaceBefore=12, spaceAfter=6,
|
||||
),
|
||||
"h2": ParagraphStyle(
|
||||
"h2", parent=base["Heading2"], fontName="Helvetica-Bold",
|
||||
fontSize=11.2, leading=14.5, textColor=TEAL, spaceBefore=9, spaceAfter=4,
|
||||
),
|
||||
"h3": ParagraphStyle(
|
||||
"h3", parent=base["Heading3"], fontName="Helvetica-Bold",
|
||||
fontSize=10, leading=13, textColor=NAVY, spaceBefore=7, spaceAfter=3,
|
||||
),
|
||||
"body": ParagraphStyle(
|
||||
"b", parent=base["Normal"], fontName="Helvetica", fontSize=9.1,
|
||||
leading=12.4, textColor=SLATE, alignment=TA_LEFT, spaceAfter=5,
|
||||
),
|
||||
"bullet": ParagraphStyle(
|
||||
"bu", parent=base["Normal"], fontName="Helvetica", fontSize=9.1,
|
||||
leading=12.2, textColor=SLATE, leftIndent=14, firstLineIndent=-10, spaceAfter=2,
|
||||
),
|
||||
"cell": ParagraphStyle(
|
||||
"c", parent=base["Normal"], fontName="Helvetica", fontSize=7.4,
|
||||
leading=10, textColor=SLATE,
|
||||
),
|
||||
"cell_h": ParagraphStyle(
|
||||
"ch", parent=base["Normal"], fontName="Helvetica-Bold", fontSize=7.4,
|
||||
leading=10, textColor=colors.white,
|
||||
),
|
||||
"code": ParagraphStyle(
|
||||
"co", parent=base["Code"], fontName="Courier", fontSize=7.2,
|
||||
leading=9.6, textColor=NAVY, backColor=CODE_BG, leftIndent=4,
|
||||
rightIndent=4, spaceBefore=2, spaceAfter=6,
|
||||
),
|
||||
"caption": ParagraphStyle(
|
||||
"ca", parent=base["Normal"], fontName="Helvetica-Oblique", fontSize=8,
|
||||
leading=11, textColor=colors.HexColor("#64748B"), spaceAfter=8, alignment=TA_CENTER,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
S = _styles()
|
||||
|
||||
|
||||
def _esc(text: str) -> str:
|
||||
return (
|
||||
text.replace("&", "&")
|
||||
.replace("<", "<")
|
||||
.replace(">", ">")
|
||||
)
|
||||
|
||||
|
||||
def _inline(text: str) -> str:
|
||||
text = _esc(text)
|
||||
text = re.sub(r"\[([^\]]+)\]\(([^)]+)\)", r"\1", text)
|
||||
text = re.sub(r"`([^`]+)`", r"<font face='Courier' size='8'>\1</font>", text)
|
||||
text = re.sub(r"\*\*([^*]+)\*\*", r"<b>\1</b>", text)
|
||||
text = re.sub(r"(?<!\*)\*([^*]+)\*(?!\*)", r"<i>\1</i>", text)
|
||||
return text.replace("|", "/")
|
||||
|
||||
|
||||
def _ascii_code(text: str) -> str:
|
||||
repl = {
|
||||
"│": "|", "─": "-", "┌": "+", "┐": "+", "└": "+", "┘": "+",
|
||||
"├": "+", "┤": "+", "┬": "+", "┴": "+", "┼": "+",
|
||||
"►": ">", "▼": "v", "▲": "^", "◄": "<",
|
||||
"→": "->", "←": "<-", "↔": "<->", "⇒": "=>",
|
||||
"—": "--", "–": "-", "…": "...", "×": "x",
|
||||
}
|
||||
for a, b in repl.items():
|
||||
text = text.replace(a, b)
|
||||
return text
|
||||
|
||||
|
||||
def _table(header: list[str], rows: list[list[str]]):
|
||||
n = max(1, len(header))
|
||||
# weight first column a bit if many cols
|
||||
if n == 1:
|
||||
widths = [USABLE]
|
||||
elif n == 2:
|
||||
widths = [2.2 * inch, USABLE - 2.2 * inch]
|
||||
else:
|
||||
first = 1.5 * inch
|
||||
rest = (USABLE - first) / (n - 1)
|
||||
widths = [first] + [rest] * (n - 1)
|
||||
if n >= 4:
|
||||
widths = [USABLE / n] * n
|
||||
data = [[Paragraph(_inline(h), S["cell_h"]) for h in header]]
|
||||
for row in rows:
|
||||
padded = (row + [""] * n)[:n]
|
||||
data.append([Paragraph(_inline(c), S["cell"]) for c in padded])
|
||||
t = Table(data, colWidths=widths, repeatRows=1)
|
||||
cmds = [
|
||||
("BACKGROUND", (0, 0), (-1, 0), NAVY),
|
||||
("VALIGN", (0, 0), (-1, -1), "TOP"),
|
||||
("LEFTPADDING", (0, 0), (-1, -1), 3),
|
||||
("RIGHTPADDING", (0, 0), (-1, -1), 3),
|
||||
("TOPPADDING", (0, 0), (-1, -1), 3),
|
||||
("BOTTOMPADDING", (0, 0), (-1, -1), 3),
|
||||
("GRID", (0, 0), (-1, -1), 0.3, RULE),
|
||||
("LINEBELOW", (0, 0), (-1, 0), 1, TEAL),
|
||||
]
|
||||
for i in range(1, len(data)):
|
||||
if i % 2 == 0:
|
||||
cmds.append(("BACKGROUND", (0, i), (-1, i), ROW))
|
||||
t.setStyle(TableStyle(cmds))
|
||||
return t
|
||||
|
||||
|
||||
def _split_row(line: str) -> list[str]:
|
||||
line = line.strip().strip("|")
|
||||
return [c.strip() for c in line.split("|")]
|
||||
|
||||
|
||||
def md_to_flowables(md: str, skip_first_h1: bool = False) -> list:
|
||||
lines = md.replace("\r\n", "\n").split("\n")
|
||||
out = []
|
||||
i = 0
|
||||
skipped = False
|
||||
while i < len(lines):
|
||||
line = lines[i]
|
||||
|
||||
if line.startswith("```"):
|
||||
buf = []
|
||||
i += 1
|
||||
while i < len(lines) and not lines[i].startswith("```"):
|
||||
buf.append(lines[i])
|
||||
i += 1
|
||||
i += 1
|
||||
block = _ascii_code("\n".join(buf))
|
||||
out.append(Preformatted(block + "\n", S["code"]))
|
||||
continue
|
||||
|
||||
if re.match(r"^\s*\|", line) and i + 1 < len(lines) and re.search(r"\|\s*-+", lines[i + 1]):
|
||||
header = _split_row(line)
|
||||
i += 2
|
||||
rows = []
|
||||
while i < len(lines) and re.match(r"^\s*\|", lines[i]):
|
||||
rows.append(_split_row(lines[i]))
|
||||
i += 1
|
||||
out.append(_table(header, rows))
|
||||
out.append(Spacer(1, 6))
|
||||
continue
|
||||
|
||||
if line.strip() == "---":
|
||||
out.append(Spacer(1, 8))
|
||||
i += 1
|
||||
continue
|
||||
|
||||
if line.startswith("# "):
|
||||
if skip_first_h1 and not skipped:
|
||||
skipped = True
|
||||
i += 1
|
||||
continue
|
||||
out.append(Paragraph(_inline(line[2:].strip()), S["h1"]))
|
||||
i += 1
|
||||
continue
|
||||
if line.startswith("## "):
|
||||
out.append(Paragraph(_inline(line[3:].strip()), S["h2"]))
|
||||
i += 1
|
||||
continue
|
||||
if line.startswith("### "):
|
||||
out.append(Paragraph(_inline(line[4:].strip()), S["h3"]))
|
||||
i += 1
|
||||
continue
|
||||
|
||||
m = re.match(r"^(\s*)[-*]\s+(.*)$", line)
|
||||
if m:
|
||||
out.append(Paragraph("• " + _inline(m.group(2)), S["bullet"]))
|
||||
i += 1
|
||||
continue
|
||||
m = re.match(r"^(\s*)\d+\.\s+(.*)$", line)
|
||||
if m:
|
||||
out.append(Paragraph("• " + _inline(m.group(2)), S["bullet"]))
|
||||
i += 1
|
||||
continue
|
||||
|
||||
if not line.strip():
|
||||
i += 1
|
||||
continue
|
||||
|
||||
out.append(Paragraph(_inline(line.strip()), S["body"]))
|
||||
i += 1
|
||||
return out
|
||||
|
||||
|
||||
def header_footer_factory(kicker: str):
|
||||
def header_footer(canvas, doc):
|
||||
canvas.saveState()
|
||||
w, h = letter
|
||||
canvas.setFillColor(NAVY)
|
||||
canvas.rect(0, h - 28, w, 28, fill=1, stroke=0)
|
||||
canvas.setFillColor(colors.white)
|
||||
canvas.setFont("Helvetica", 8)
|
||||
canvas.drawString(0.7 * inch, h - 18, "Verae Time x Zapier")
|
||||
canvas.drawRightString(w - 0.7 * inch, h - 18, kicker[:70])
|
||||
canvas.setFillColor(TEAL)
|
||||
canvas.rect(0, 0, w, 22, fill=1, stroke=0)
|
||||
canvas.setFillColor(colors.white)
|
||||
canvas.setFont("Helvetica", 8)
|
||||
canvas.drawString(0.7 * inch, 8, "Internal · 18 August 2026")
|
||||
canvas.drawRightString(w - 0.7 * inch, 8, str(doc.page))
|
||||
canvas.restoreState()
|
||||
|
||||
return header_footer
|
||||
|
||||
|
||||
def cover_footer(canvas, doc):
|
||||
header_footer_factory("")(canvas, doc)
|
||||
|
||||
|
||||
def build_pdf(parts: list[tuple[str, str]], dest: Path, title: str, subtitle: str):
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
story = [
|
||||
Spacer(1, 1.7 * inch),
|
||||
Paragraph("VERAE TIME · ZAPIER WORKSPACE", S["cover_kicker"]),
|
||||
Paragraph(_esc(title).replace("\n", "<br/>"), S["cover_title"]),
|
||||
Paragraph(_esc(subtitle), S["caption"]),
|
||||
PageBreak(),
|
||||
]
|
||||
for idx, (label, md) in enumerate(parts):
|
||||
if idx:
|
||||
story.append(PageBreak())
|
||||
if label:
|
||||
story.append(Paragraph(_inline(label), S["h1"]))
|
||||
story.extend(md_to_flowables(md, skip_first_h1=bool(label)))
|
||||
|
||||
doc = SimpleDocTemplate(
|
||||
str(dest),
|
||||
pagesize=letter,
|
||||
leftMargin=0.7 * inch,
|
||||
rightMargin=0.7 * inch,
|
||||
topMargin=0.55 * inch,
|
||||
bottomMargin=0.45 * inch,
|
||||
title=title,
|
||||
author="Verae / Zapier research workspace",
|
||||
subject=subtitle,
|
||||
)
|
||||
hf = header_footer_factory(title)
|
||||
doc.build(story, onFirstPage=cover_footer, onLaterPages=hf)
|
||||
print(f"wrote {dest} ({dest.stat().st_size} bytes)")
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
if len(argv) >= 2 and argv[0] == "--concat":
|
||||
dest = Path(argv[1])
|
||||
title = argv[2]
|
||||
files = [Path(p) for p in argv[3:]]
|
||||
parts = [(p.name, p.read_text(encoding="utf-8")) for p in files]
|
||||
build_pdf(parts, dest, title, "Combined from " + ", ".join(p.name for p in files))
|
||||
return 0
|
||||
if len(argv) < 2:
|
||||
print(__doc__)
|
||||
return 2
|
||||
src = Path(argv[0])
|
||||
dest = Path(argv[1])
|
||||
title = argv[2] if len(argv) > 2 else src.stem.replace("-", " ").title()
|
||||
build_pdf([(None, src.read_text(encoding="utf-8"))], dest, title, str(src))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv[1:]))
|
||||
3
research/zapier/scripts/mongo-tunnel.sh
Executable file
3
research/zapier/scripts/mongo-tunnel.sh
Executable file
|
|
@ -0,0 +1,3 @@
|
|||
#!/bin/sh
|
||||
# Forward local 27017 to MongoDB on NS1 (127.0.0.1:27017). Keep this running.
|
||||
exec ssh -N -L 27017:127.0.0.1:27017 ns1
|
||||
131
research/zapier/scripts/refine-help.py
Normal file
131
research/zapier/scripts/refine-help.py
Normal file
|
|
@ -0,0 +1,131 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Re-parse stored help article text into cleaner auth/connect fields."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SRC = ROOT / "raw" / "help-articles.jsonl"
|
||||
OUT = ROOT / "raw" / "help-auth-by-slug.json"
|
||||
|
||||
AUTH = [
|
||||
("oauth", re.compile(
|
||||
r"\bOAuth 2(?:\.0)?\b|\bOAuth\b|Sign in with Google|Sign in with Microsoft|"
|
||||
r"Grant Zapier permission to access|new browser tab or window will open|"
|
||||
r"Log into .+ to authenticate",
|
||||
re.I,
|
||||
)),
|
||||
("api_key", re.compile(
|
||||
r"\bAPI key\b|\bAPI token\b|\bAPI Key\b|\bpersonal access token\b|"
|
||||
r"\bsecret key\b|paste your (?:API|token)",
|
||||
re.I,
|
||||
)),
|
||||
("username_password", re.compile(r"enter your (?:username|email).{0,40}password", re.I)),
|
||||
("basic_auth", re.compile(r"\bBasic Auth(?:entication)?\b", re.I)),
|
||||
("webhook", re.compile(r"webhook URL|catch hook|Copy the webhook", re.I)),
|
||||
("invite_only", re.compile(r"invite-only|invitation (?:URL|link)", re.I)),
|
||||
("premium_zapier", re.compile(r"Premium app|available on paid Zapier", re.I)),
|
||||
]
|
||||
|
||||
|
||||
def strip_chrome(text: str) -> str:
|
||||
# drop everything before the article H1-ish "How to get started" / "Common Problems"
|
||||
for pat in (
|
||||
r"How to get started with .+? on Zapier",
|
||||
r"Common Problems with ",
|
||||
r"Updated \w+ \d+, 20\d\d",
|
||||
):
|
||||
m = re.search(pat, text)
|
||||
if m:
|
||||
text = text[m.start() :]
|
||||
break
|
||||
for cut in (
|
||||
"Related articles",
|
||||
"Was this article helpful",
|
||||
"Have more questions",
|
||||
"My Requests",
|
||||
):
|
||||
i = text.find(cut)
|
||||
if i > 300:
|
||||
text = text[:i]
|
||||
return text.strip()
|
||||
|
||||
|
||||
def section(text: str, start: str, ends: list[str], maxlen=2000) -> str:
|
||||
m = re.search(start, text, re.I)
|
||||
if not m:
|
||||
return ""
|
||||
rest = text[m.start() :]
|
||||
end = len(rest)
|
||||
for ep in ends:
|
||||
n = re.search(ep, rest[m.end() - m.start() + 1 :], re.I)
|
||||
if n:
|
||||
end = min(end, (m.end() - m.start() + 1) + n.start())
|
||||
return re.sub(r"\s+", " ", rest[:end]).strip()[:maxlen]
|
||||
|
||||
|
||||
def main() -> None:
|
||||
by = defaultdict(list)
|
||||
for line in SRC.open():
|
||||
r = json.loads(line)
|
||||
by[r["slug"]].append(r)
|
||||
|
||||
out = {}
|
||||
for slug, arts in by.items():
|
||||
signals = set()
|
||||
pieces = []
|
||||
prereqs = []
|
||||
connects = []
|
||||
help_docs = []
|
||||
for a in arts:
|
||||
raw = strip_chrome(a.get("text") or "")
|
||||
title = a.get("help_title") or a.get("title") or ""
|
||||
help_docs.append({"title": title, "url": a.get("url")})
|
||||
for name, rx in AUTH:
|
||||
if rx.search(raw):
|
||||
signals.add(name)
|
||||
pr = section(
|
||||
raw,
|
||||
r"Prerequisites",
|
||||
[r"Connect .+ to Zapier", r"To create an app connection", r"How to connect", r"Using .+ in Zaps", r"Triggers and actions"],
|
||||
)
|
||||
if pr:
|
||||
prereqs.append(pr)
|
||||
cn = section(
|
||||
raw,
|
||||
r"(?:To create an app connection to|To connect your .+ account to Zapier|Connect [A-Z][\w .+-]+ to Zapier)",
|
||||
[
|
||||
r"Your .+ account is now successfully connected",
|
||||
r"About .+ app",
|
||||
r"Using .+ in Zaps",
|
||||
r"Available triggers",
|
||||
r"Triggers, searches, and actions",
|
||||
r"Common problems",
|
||||
],
|
||||
)
|
||||
if cn and not cn.startswith("Connect 20"):
|
||||
connects.append(cn)
|
||||
pieces.append(raw[:4000])
|
||||
# pick a primary auth
|
||||
order = ["oauth", "api_key", "basic_auth", "username_password", "webhook", "invite_only"]
|
||||
primary = next((x for x in order if x in signals), None)
|
||||
out[slug] = {
|
||||
"slug": slug,
|
||||
"auth_signals": sorted(signals),
|
||||
"primary_auth": primary,
|
||||
"prerequisites": prereqs[:2],
|
||||
"connect_steps": connects[:2],
|
||||
"help_articles": help_docs,
|
||||
"excerpt": pieces[0][:2500] if pieces else "",
|
||||
}
|
||||
OUT.write_text(json.dumps(out, indent=2))
|
||||
from collections import Counter
|
||||
c = Counter(v["primary_auth"] or "unknown" for v in out.values())
|
||||
print("slugs", len(out), "auth", dict(c))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
23
research/zapier/scripts/restart-grok.sh
Executable file
23
research/zapier/scripts/restart-grok.sh
Executable file
|
|
@ -0,0 +1,23 @@
|
|||
#!/bin/sh
|
||||
# Quit the current Grok TUI first (/quit), then run this from a normal shell.
|
||||
# Brings up the Mongo tunnel and resumes the latest session in this workspace.
|
||||
set -eu
|
||||
ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)"
|
||||
cd "$ROOT"
|
||||
|
||||
export PATH="$HOME/.npm-global/bin:$PATH"
|
||||
if [ -f "$HOME/.mcp-env" ]; then
|
||||
# shellcheck disable=SC1091
|
||||
. "$HOME/.mcp-env"
|
||||
fi
|
||||
|
||||
"$ROOT/scripts/ensure-mongo-tunnel.sh"
|
||||
|
||||
if ! command -v grok >/dev/null 2>&1; then
|
||||
echo "grok is not on PATH" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "starting grok --resume in $ROOT"
|
||||
echo "after it opens: /mcps (press r to refresh) then /zapier-build"
|
||||
exec grok --resume
|
||||
226
research/zapier/scripts/scrape-all-contacts.py
Normal file
226
research/zapier/scripts/scrape-all-contacts.py
Normal file
|
|
@ -0,0 +1,226 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Scrape public contact emails/phones from Zapier app vendor websites.
|
||||
|
||||
Resumable: skips slugs already present in the output JSONL.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
import ssl
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
CATALOG = ROOT / "raw" / "all-apps.jsonl"
|
||||
OUT = ROOT / "raw" / "contacts-all-scrape.jsonl"
|
||||
WORKERS = 24
|
||||
TIMEOUT = 7
|
||||
PATHS = ("", "/contact", "/contact-us", "/contact-sales", "/company/contact")
|
||||
|
||||
EMAIL_RE = re.compile(r"[A-Z0-9._%+\-]+@[A-Z0-9.\-]+\.[A-Z]{2,}", re.I)
|
||||
PHONE_RE = re.compile(
|
||||
r"(?:\+\d{1,3}[\s.\-()]*)?(?:\(?\d{2,4}\)?[\s.\-]*){2,4}\d{3,4}"
|
||||
)
|
||||
BAD_EMAIL = re.compile(
|
||||
r"(example\.com|sentry\.io|wixpress|schema\.org|godaddy|cloudflare|"
|
||||
r"jquery|webpack|png|jpe?g|svg|css|woff2?|localhost|yourdomain|"
|
||||
r"company\.com|email\.com|domain\.com|yourcompany|name@|"
|
||||
r"user@|test@|noreply|no-reply|donotreply|mailer-daemon|"
|
||||
r"github\.com|google\.com$|googleapis|gstatic|w3\.org|"
|
||||
r"sentry\.|bugsnag|facebook\.com|twitter\.com)$",
|
||||
re.I,
|
||||
)
|
||||
|
||||
|
||||
def load_done() -> set[str]:
|
||||
done = set()
|
||||
if OUT.exists():
|
||||
for line in OUT.open():
|
||||
try:
|
||||
done.add(json.loads(line)["slug"])
|
||||
except Exception:
|
||||
continue
|
||||
return done
|
||||
|
||||
|
||||
def norm_url(u: str | None) -> str | None:
|
||||
if not u:
|
||||
return None
|
||||
u = u.strip()
|
||||
if not u or u.startswith("@"):
|
||||
return None
|
||||
if not u.startswith("http"):
|
||||
u = "https://" + u
|
||||
return u.rstrip("/")
|
||||
|
||||
|
||||
def fetch(url: str) -> tuple[str, str]:
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
headers={
|
||||
"User-Agent": "Mozilla/5.0 (compatible; vendor-contact-research/1.0)",
|
||||
"Accept": "text/html,application/xhtml+xml",
|
||||
},
|
||||
)
|
||||
ctx = ssl.create_default_context()
|
||||
with urllib.request.urlopen(req, timeout=TIMEOUT, context=ctx) as r:
|
||||
raw = r.read(220_000)
|
||||
ctype = (r.headers.get("Content-Type") or "").lower()
|
||||
final = r.geturl()
|
||||
if "html" not in ctype and not raw[:80].lstrip().lower().startswith(
|
||||
(b"<!doctype", b"<html")
|
||||
):
|
||||
return final, ""
|
||||
return final, raw.decode("utf-8", errors="ignore")
|
||||
|
||||
|
||||
def emails_from(text: str) -> list[str]:
|
||||
found = []
|
||||
for e in EMAIL_RE.findall(text):
|
||||
e = e.strip(".,;:()<>\"'").lower()
|
||||
if BAD_EMAIL.search(e):
|
||||
continue
|
||||
if any(x in e for x in ("noreply", "no-reply", "donotreply")):
|
||||
continue
|
||||
if len(e) > 80:
|
||||
continue
|
||||
found.append(e)
|
||||
pri, rest = [], []
|
||||
for e in dict.fromkeys(found):
|
||||
local = e.split("@")[0]
|
||||
if any(
|
||||
k in local
|
||||
for k in (
|
||||
"sales",
|
||||
"partner",
|
||||
"hello",
|
||||
"info",
|
||||
"contact",
|
||||
"support",
|
||||
"press",
|
||||
"biz",
|
||||
"enterprise",
|
||||
"help",
|
||||
)
|
||||
):
|
||||
pri.append(e)
|
||||
else:
|
||||
rest.append(e)
|
||||
return (pri + rest)[:6]
|
||||
|
||||
|
||||
def phones_from(text: str) -> list[str]:
|
||||
text = html.unescape(re.sub(r"<[^>]+>", " ", text))
|
||||
text = re.sub(r"\s+", " ", text)
|
||||
out, seen = [], set()
|
||||
for m in PHONE_RE.findall(text):
|
||||
digits = re.sub(r"\D", "", m)
|
||||
if len(digits) < 10 or len(digits) > 15:
|
||||
continue
|
||||
if digits.startswith("000") or len(set(digits)) < 3:
|
||||
continue
|
||||
# skip years / zip-like
|
||||
if digits.startswith("20") and len(digits) == 10:
|
||||
continue
|
||||
if digits not in seen:
|
||||
seen.add(digits)
|
||||
out.append(re.sub(r"\s+", " ", m).strip())
|
||||
if len(out) >= 4:
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
def scrape_one(app: dict) -> dict:
|
||||
website = norm_url(app.get("external_url"))
|
||||
rec = {
|
||||
"slug": app["slug"],
|
||||
"name": app.get("name"),
|
||||
"website": website,
|
||||
"zapier_url": "https://zapier.com"
|
||||
+ (app.get("app_profile_url") or f"/apps/{app['slug']}/integrations"),
|
||||
"emails": [],
|
||||
"phones": [],
|
||||
"contact_urls": [],
|
||||
"pages": 0,
|
||||
"error": None,
|
||||
}
|
||||
if not website:
|
||||
rec["error"] = "no website"
|
||||
return rec
|
||||
parsed = urllib.parse.urlparse(website)
|
||||
if not parsed.netloc:
|
||||
rec["error"] = "bad website"
|
||||
return rec
|
||||
origin = f"{parsed.scheme}://{parsed.netloc}"
|
||||
texts = []
|
||||
last_err = None
|
||||
for path in PATHS:
|
||||
url = website if path == "" else origin + path
|
||||
try:
|
||||
final, text = fetch(url)
|
||||
if text and len(text) > 200:
|
||||
texts.append((final, text))
|
||||
except Exception as e:
|
||||
last_err = f"{type(e).__name__}"
|
||||
continue
|
||||
if len(texts) >= 2:
|
||||
break
|
||||
rec["pages"] = len(texts)
|
||||
blob = " ".join(t for _, t in texts)
|
||||
rec["emails"] = emails_from(blob)
|
||||
contactish = " ".join(
|
||||
t
|
||||
for u, t in texts
|
||||
if any(x in u.lower() for x in ("contact", "about", "sales", "company"))
|
||||
)
|
||||
rec["phones"] = phones_from(contactish or blob)
|
||||
rec["contact_urls"] = [
|
||||
u
|
||||
for u, _ in texts
|
||||
if any(x in u.lower() for x in ("contact", "sales", "demo"))
|
||||
][:4]
|
||||
if not texts and last_err:
|
||||
rec["error"] = last_err
|
||||
return rec
|
||||
|
||||
|
||||
def main() -> int:
|
||||
apps = [json.loads(l) for l in CATALOG.open()]
|
||||
done = load_done()
|
||||
todo = [a for a in apps if a["slug"] not in done]
|
||||
print(f"catalog={len(apps)} done={len(done)} todo={len(todo)}", flush=True)
|
||||
if not todo:
|
||||
return 0
|
||||
OUT.parent.mkdir(parents=True, exist_ok=True)
|
||||
ok_email = 0
|
||||
n = 0
|
||||
t0 = time.time()
|
||||
with OUT.open("a") as out, ThreadPoolExecutor(max_workers=WORKERS) as ex:
|
||||
futs = {ex.submit(scrape_one, a): a["slug"] for a in todo}
|
||||
for fut in as_completed(futs):
|
||||
rec = fut.result()
|
||||
out.write(json.dumps(rec, ensure_ascii=False) + "\n")
|
||||
out.flush()
|
||||
n += 1
|
||||
if rec.get("emails"):
|
||||
ok_email += 1
|
||||
if n % 100 == 0:
|
||||
rate = n / max(time.time() - t0, 1)
|
||||
print(
|
||||
f"{n}/{len(todo)} emails={ok_email} "
|
||||
f"{rate:.1f}/s last={rec['slug']}",
|
||||
flush=True,
|
||||
)
|
||||
print(f"finished batch n={n} emails={ok_email}", flush=True)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
94
research/zapier/scripts/scrape-api-docs.py
Normal file
94
research/zapier/scripts/scrape-api-docs.py
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Fetch vendor API docs homepages and classify auth style."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import ssl
|
||||
import urllib.request
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
CATALOG = ROOT / "raw" / "all-apps-full.jsonl"
|
||||
OUT = ROOT / "raw" / "api-docs-scrape.jsonl"
|
||||
|
||||
SIGNALS = [
|
||||
("oauth2", re.compile(r"\bOAuth\s*2(?:\.0)?\b|\bauthorization code\b|\bclient_id\b", re.I)),
|
||||
("api_key", re.compile(r"\bAPI[- ]key\b|\bx-api-key\b|\bBearer token\b|\bpersonal access token\b", re.I)),
|
||||
("basic", re.compile(r"\bBasic Auth(?:entication)?\b", re.I)),
|
||||
("jwt", re.compile(r"\bJWT\b|\bJSON Web Token\b", re.I)),
|
||||
("webhook", re.compile(r"\bwebhook\b", re.I)),
|
||||
("openapi", re.compile(r"\bOpenAPI\b|\bSwagger\b", re.I)),
|
||||
("graphql", re.compile(r"\bGraphQL\b", re.I)),
|
||||
("rest", re.compile(r"\bREST(?:ful)? API\b", re.I)),
|
||||
]
|
||||
|
||||
|
||||
def load_done() -> set[str]:
|
||||
done = set()
|
||||
if OUT.exists():
|
||||
for line in OUT.open():
|
||||
try:
|
||||
done.add(json.loads(line)["slug"])
|
||||
except Exception:
|
||||
continue
|
||||
return done
|
||||
|
||||
|
||||
def fetch(url: str) -> str:
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
headers={"User-Agent": "Mozilla/5.0 research", "Accept": "text/html,application/xhtml+xml"},
|
||||
)
|
||||
ctx = ssl.create_default_context()
|
||||
with urllib.request.urlopen(req, timeout=10, context=ctx) as r:
|
||||
raw = r.read(160_000)
|
||||
return raw.decode("utf-8", errors="ignore")
|
||||
|
||||
|
||||
def scrape(app: dict) -> dict:
|
||||
url = (app.get("api_docs_url") or "").strip()
|
||||
rec = {"slug": app["slug"], "api_docs_url": url or None, "signals": [], "title": None, "error": None}
|
||||
if not url:
|
||||
rec["error"] = "no api_docs_url"
|
||||
return rec
|
||||
if not url.startswith("http"):
|
||||
url = "https://" + url
|
||||
rec["api_docs_url"] = url
|
||||
try:
|
||||
html = fetch(url)
|
||||
except Exception as e:
|
||||
rec["error"] = type(e).__name__
|
||||
return rec
|
||||
text = re.sub(r"<script[\s\S]*?</script>", " ", html, flags=re.I)
|
||||
text = re.sub(r"<style[\s\S]*?</style>", " ", text, flags=re.I)
|
||||
text = re.sub(r"<[^>]+>", " ", text)
|
||||
text = re.sub(r"\s+", " ", text)
|
||||
rec["signals"] = [n for n, rx in SIGNALS if rx.search(text)]
|
||||
tm = re.search(r"<title[^>]*>([^<]{3,160})</title>", html, re.I)
|
||||
rec["title"] = tm.group(1).strip() if tm else None
|
||||
rec["excerpt"] = text[:800]
|
||||
return rec
|
||||
|
||||
|
||||
def main() -> None:
|
||||
apps = [json.loads(l) for l in CATALOG.open()]
|
||||
done = load_done()
|
||||
todo = [a for a in apps if a["slug"] not in done]
|
||||
print("todo", len(todo), flush=True)
|
||||
n = 0
|
||||
with OUT.open("a") as out, ThreadPoolExecutor(max_workers=28) as ex:
|
||||
futs = {ex.submit(scrape, a): a["slug"] for a in todo}
|
||||
for fut in as_completed(futs):
|
||||
rec = fut.result()
|
||||
out.write(json.dumps(rec, ensure_ascii=False) + "\n")
|
||||
out.flush()
|
||||
n += 1
|
||||
if n % 300 == 0:
|
||||
print(f"{n}/{len(todo)} last={rec['slug']} sig={rec.get('signals')}", flush=True)
|
||||
print("finished", n)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
162
research/zapier/scripts/scrape-capabilities.py
Normal file
162
research/zapier/scripts/scrape-capabilities.py
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Scrape Zapier integration controls (triggers/actions/searches) per app.
|
||||
|
||||
Resumable JSONL keyed by slug.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import ssl
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
CATALOG = ROOT / "raw" / "all-apps.jsonl"
|
||||
OUT = ROOT / "raw" / "capabilities-scrape.jsonl"
|
||||
WORKERS = 16
|
||||
TIMEOUT = 20
|
||||
|
||||
|
||||
def texts_from_ast(node) -> str:
|
||||
parts = []
|
||||
|
||||
def walk(n):
|
||||
if isinstance(n, dict):
|
||||
if n.get("rawText"):
|
||||
parts.append(n["rawText"])
|
||||
for c in n.get("childNodes") or []:
|
||||
walk(c)
|
||||
|
||||
walk(node)
|
||||
return re.sub(r"\s+", " ", " ".join(parts)).strip()
|
||||
|
||||
|
||||
def slim_action(it: dict) -> dict:
|
||||
needs = []
|
||||
for n in it.get("needs") or []:
|
||||
needs.append(
|
||||
{
|
||||
"key": n.get("key"),
|
||||
"label": n.get("label"),
|
||||
"required": bool(n.get("required")),
|
||||
"type": n.get("type"),
|
||||
}
|
||||
)
|
||||
return {
|
||||
"key": it.get("key"),
|
||||
"label": it.get("label"),
|
||||
"hook": bool(it.get("isHook")),
|
||||
"hidden": bool(it.get("isHidden")),
|
||||
"help": texts_from_ast(it.get("helpTextHtmlAst"))[:400],
|
||||
"inputs": needs,
|
||||
}
|
||||
|
||||
|
||||
def load_done() -> set[str]:
|
||||
done = set()
|
||||
if OUT.exists():
|
||||
for line in OUT.open():
|
||||
try:
|
||||
done.add(json.loads(line)["slug"])
|
||||
except Exception:
|
||||
continue
|
||||
return done
|
||||
|
||||
|
||||
def fetch_page(slug: str) -> dict:
|
||||
url = f"https://zapier.com/apps/{slug}/integrations"
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
headers={
|
||||
"User-Agent": "Mozilla/5.0 (compatible; vendor-research/1.0)",
|
||||
"Accept": "text/html",
|
||||
},
|
||||
)
|
||||
ctx = ssl.create_default_context()
|
||||
with urllib.request.urlopen(req, timeout=TIMEOUT, context=ctx) as r:
|
||||
html = r.read().decode("utf-8", errors="ignore")
|
||||
m = re.search(
|
||||
r'<script id="__NEXT_DATA__" type="application/json">(.*?)</script>',
|
||||
html,
|
||||
)
|
||||
if not m:
|
||||
return {"slug": slug, "error": "no_next_data"}
|
||||
d = json.loads(m.group(1))
|
||||
details = d["props"]["pageProps"]["oneAppPage"]["appDetails"]
|
||||
app = details.get("app") or {}
|
||||
impl = details.get("currentImplementation") or {}
|
||||
reads = [slim_action(x) for x in impl.get("reads") or [] if not x.get("isHidden")]
|
||||
writes = [slim_action(x) for x in impl.get("writes") or [] if not x.get("isHidden")]
|
||||
searches = [slim_action(x) for x in impl.get("searches") or [] if not x.get("isHidden")]
|
||||
return {
|
||||
"slug": slug,
|
||||
"name": app.get("name"),
|
||||
"description": app.get("description"),
|
||||
"partner_tier": app.get("partnerTier"),
|
||||
"premium": app.get("isPremium"),
|
||||
"beta": app.get("isBeta"),
|
||||
"implementation": impl.get("selectedApi"),
|
||||
"categories": [c.get("title") for c in details.get("appCategories") or []],
|
||||
"trigger_count": len(reads),
|
||||
"action_count": len(writes),
|
||||
"search_count": len(searches),
|
||||
"instant_trigger_count": sum(1 for t in reads if t["hook"]),
|
||||
"triggers": reads,
|
||||
"actions": writes,
|
||||
"searches": searches,
|
||||
"alternatives": [
|
||||
(a.get("app") or {}).get("name")
|
||||
for a in details.get("topAlternatives") or []
|
||||
if (a.get("app") or {}).get("name")
|
||||
],
|
||||
"paired_apps": [
|
||||
(a.get("app") or {}).get("name")
|
||||
for a in details.get("pairedAppsExcludingZapierBuiltinApps") or []
|
||||
if (a.get("app") or {}).get("name")
|
||||
][:20],
|
||||
"help_articles": [
|
||||
h.get("title") for h in details.get("helpContent") or [] if h.get("title")
|
||||
],
|
||||
"mcp_url": f"https://zapier.com/mcp/{slug}",
|
||||
"error": None,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
apps = [json.loads(l) for l in CATALOG.open()]
|
||||
done = load_done()
|
||||
todo = [a["slug"] for a in apps if a["slug"] not in done]
|
||||
print(f"catalog={len(apps)} done={len(done)} todo={len(todo)}", flush=True)
|
||||
if not todo:
|
||||
return 0
|
||||
n = 0
|
||||
errs = 0
|
||||
t0 = time.time()
|
||||
with OUT.open("a") as out, ThreadPoolExecutor(max_workers=WORKERS) as ex:
|
||||
futs = {ex.submit(fetch_page, slug): slug for slug in todo}
|
||||
for fut in as_completed(futs):
|
||||
slug = futs[fut]
|
||||
try:
|
||||
rec = fut.result()
|
||||
except Exception as e:
|
||||
rec = {"slug": slug, "error": f"{type(e).__name__}: {e}"}
|
||||
errs += 1
|
||||
out.write(json.dumps(rec, ensure_ascii=False) + "\n")
|
||||
out.flush()
|
||||
n += 1
|
||||
if n % 50 == 0:
|
||||
rate = n / max(time.time() - t0, 1)
|
||||
print(
|
||||
f"{n}/{len(todo)} err={errs} {rate:.1f}/s last={slug}",
|
||||
flush=True,
|
||||
)
|
||||
print(f"finished n={n} err={errs}", flush=True)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
145
research/zapier/scripts/scrape-commercial.py
Normal file
145
research/zapier/scripts/scrape-commercial.py
Normal file
|
|
@ -0,0 +1,145 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Scrape public pricing/security pages for Zapier vendor websites."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import ssl
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
CATALOG = ROOT / "raw" / "all-apps-full.jsonl"
|
||||
OUT = ROOT / "raw" / "commercial-scrape.jsonl"
|
||||
PATHS = (
|
||||
"/pricing",
|
||||
"/plans",
|
||||
"/pricing/",
|
||||
"/security",
|
||||
"/trust",
|
||||
"/compliance",
|
||||
"/legal/security",
|
||||
"/privacy",
|
||||
)
|
||||
FLAGS = {
|
||||
"soc2": re.compile(r"\bSOC\s*2\b", re.I),
|
||||
"soc2_type2": re.compile(r"SOC\s*2\s*Type\s*I{1,2}", re.I),
|
||||
"hipaa": re.compile(r"\bHIPAA\b", re.I),
|
||||
"gdpr": re.compile(r"\bGDPR\b", re.I),
|
||||
"iso27001": re.compile(r"ISO\s*/?\s*27001", re.I),
|
||||
"ccpa": re.compile(r"\bCCPA\b|\bCPRA\b", re.I),
|
||||
"fedramp": re.compile(r"\bFedRAMP\b", re.I),
|
||||
"pci": re.compile(r"\bPCI[\s-]?DSS\b", re.I),
|
||||
}
|
||||
PRICE_RE = re.compile(
|
||||
r"(?:from\s+)?(?:US\$|\$|€|£)\s?\d{1,4}(?:[.,]\d{2})?(?:\s*/\s*(?:mo|month|user|seat|yr|year))?",
|
||||
re.I,
|
||||
)
|
||||
|
||||
|
||||
def load_done() -> set[str]:
|
||||
done = set()
|
||||
if OUT.exists():
|
||||
for line in OUT.open():
|
||||
try:
|
||||
done.add(json.loads(line)["slug"])
|
||||
except Exception:
|
||||
continue
|
||||
return done
|
||||
|
||||
|
||||
def norm(u: str | None) -> str | None:
|
||||
if not u:
|
||||
return None
|
||||
u = u.strip()
|
||||
if not u.startswith("http"):
|
||||
u = "https://" + u
|
||||
return u.rstrip("/")
|
||||
|
||||
|
||||
def fetch(url: str) -> str:
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
headers={"User-Agent": "Mozilla/5.0 research", "Accept": "text/html"},
|
||||
)
|
||||
ctx = ssl.create_default_context()
|
||||
with urllib.request.urlopen(req, timeout=8, context=ctx) as r:
|
||||
raw = r.read(180_000)
|
||||
ctype = (r.headers.get("Content-Type") or "").lower()
|
||||
if "html" not in ctype and not raw[:80].lstrip().lower().startswith(
|
||||
(b"<!doctype", b"<html")
|
||||
):
|
||||
return ""
|
||||
return raw.decode("utf-8", errors="ignore")
|
||||
|
||||
|
||||
def textish(html: str) -> str:
|
||||
html = re.sub(r"<script[\s\S]*?</script>", " ", html, flags=re.I)
|
||||
html = re.sub(r"<style[\s\S]*?</style>", " ", html, flags=re.I)
|
||||
html = re.sub(r"<[^>]+>", " ", html)
|
||||
return re.sub(r"\s+", " ", html)
|
||||
|
||||
|
||||
def scrape(app: dict) -> dict:
|
||||
website = norm(app.get("external_url"))
|
||||
rec = {
|
||||
"slug": app["slug"],
|
||||
"website": website,
|
||||
"pages": [],
|
||||
"flags": [],
|
||||
"price_mentions": [],
|
||||
"error": None,
|
||||
}
|
||||
if not website:
|
||||
rec["error"] = "no website"
|
||||
return rec
|
||||
parsed = urllib.parse.urlparse(website)
|
||||
origin = f"{parsed.scheme}://{parsed.netloc}"
|
||||
texts = []
|
||||
for path in ("",) + PATHS[:6]:
|
||||
url = website if path == "" else origin + path
|
||||
try:
|
||||
html = fetch(url)
|
||||
except Exception:
|
||||
continue
|
||||
if html and len(html) > 200:
|
||||
texts.append((url, textish(html)[:20000]))
|
||||
rec["pages"].append(url)
|
||||
if len(texts) >= 3:
|
||||
break
|
||||
blob = " ".join(t for _, t in texts)
|
||||
rec["flags"] = [k for k, rx in FLAGS.items() if rx.search(blob)]
|
||||
prices = []
|
||||
for m in PRICE_RE.findall(blob):
|
||||
s = re.sub(r"\s+", " ", m).strip()
|
||||
if s not in prices:
|
||||
prices.append(s)
|
||||
if len(prices) >= 8:
|
||||
break
|
||||
rec["price_mentions"] = prices
|
||||
return rec
|
||||
|
||||
|
||||
def main() -> None:
|
||||
apps = [json.loads(l) for l in CATALOG.open()]
|
||||
done = load_done()
|
||||
todo = [a for a in apps if a["slug"] not in done]
|
||||
print(f"todo={len(todo)} done={len(done)}", flush=True)
|
||||
n = 0
|
||||
with OUT.open("a") as out, ThreadPoolExecutor(max_workers=20) as ex:
|
||||
futs = {ex.submit(scrape, a): a["slug"] for a in todo}
|
||||
for fut in as_completed(futs):
|
||||
rec = fut.result()
|
||||
out.write(json.dumps(rec, ensure_ascii=False) + "\n")
|
||||
out.flush()
|
||||
n += 1
|
||||
if n % 200 == 0:
|
||||
print(f"{n}/{len(todo)} last={rec['slug']} flags={rec.get('flags')}", flush=True)
|
||||
print("finished", n)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
104
research/zapier/scripts/scrape-extras.py
Normal file
104
research/zapier/scripts/scrape-extras.py
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Pull featured Zap templates, help article URLs, and overview text from Zapier pages."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import ssl
|
||||
import urllib.request
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
CATALOG = ROOT / "raw" / "all-apps-full.jsonl"
|
||||
OUT = ROOT / "raw" / "extras-scrape.jsonl"
|
||||
|
||||
|
||||
def load_done() -> set[str]:
|
||||
done = set()
|
||||
if OUT.exists():
|
||||
for line in OUT.open():
|
||||
try:
|
||||
done.add(json.loads(line)["slug"])
|
||||
except Exception:
|
||||
continue
|
||||
return done
|
||||
|
||||
|
||||
def texts(node) -> str:
|
||||
parts = []
|
||||
|
||||
def walk(n):
|
||||
if isinstance(n, dict):
|
||||
if n.get("rawText"):
|
||||
parts.append(n["rawText"])
|
||||
for c in n.get("childNodes") or []:
|
||||
walk(c)
|
||||
|
||||
walk(node)
|
||||
return re.sub(r"\s+", " ", " ".join(parts)).strip()
|
||||
|
||||
|
||||
def fetch(slug: str) -> dict:
|
||||
url = f"https://zapier.com/apps/{slug}/integrations"
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0 research"})
|
||||
ctx = ssl.create_default_context()
|
||||
with urllib.request.urlopen(req, timeout=20, context=ctx) as r:
|
||||
html = r.read().decode("utf-8", errors="ignore")
|
||||
m = re.search(r'<script id="__NEXT_DATA__" type="application/json">(.*?)</script>', html)
|
||||
if not m:
|
||||
return {"slug": slug, "error": "no_next_data"}
|
||||
page = json.loads(m.group(1))["props"]["pageProps"]["oneAppPage"]
|
||||
details = page.get("appDetails") or {}
|
||||
pt = page.get("paginatedZapTemplates") or {}
|
||||
featured = []
|
||||
for t in pt.get("results") or []:
|
||||
apps = [a.get("slug") for a in (t.get("apps") or []) if a.get("slug")]
|
||||
featured.append(
|
||||
{
|
||||
"id": t.get("id"),
|
||||
"title": t.get("title"),
|
||||
"url": "https://zapier.com" + (t.get("canonicalPageRelativePath") or ""),
|
||||
"description": t.get("metaDescription"),
|
||||
"apps": apps,
|
||||
}
|
||||
)
|
||||
help_ = []
|
||||
for h in details.get("helpContent") or []:
|
||||
help_.append({"title": h.get("title"), "url": h.get("canonicalUrl")})
|
||||
overview = texts(page.get("integrationOverviewHtmlAst"))[:2000]
|
||||
return {
|
||||
"slug": slug,
|
||||
"template_count_zapier": pt.get("count"),
|
||||
"featured_templates": featured,
|
||||
"help_articles": help_,
|
||||
"overview": overview,
|
||||
"error": None,
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
apps = [json.loads(l) for l in CATALOG.open()]
|
||||
done = load_done()
|
||||
todo = [a["slug"] for a in apps if a["slug"] not in done]
|
||||
print(f"todo={len(todo)} done={len(done)}", flush=True)
|
||||
n = err = 0
|
||||
with OUT.open("a") as out, ThreadPoolExecutor(max_workers=28) as ex:
|
||||
futs = {ex.submit(fetch, s): s for s in todo}
|
||||
for fut in as_completed(futs):
|
||||
slug = futs[fut]
|
||||
try:
|
||||
rec = fut.result()
|
||||
except Exception as e:
|
||||
rec = {"slug": slug, "error": f"{type(e).__name__}: {e}"}
|
||||
err += 1
|
||||
out.write(json.dumps(rec, ensure_ascii=False) + "\n")
|
||||
out.flush()
|
||||
n += 1
|
||||
if n % 200 == 0:
|
||||
print(f"{n}/{len(todo)} err={err} last={slug}", flush=True)
|
||||
print("finished", n, "err", err)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
96
research/zapier/scripts/scrape-help-extra.py
Normal file
96
research/zapier/scripts/scrape-help-extra.py
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Fetch remaining help.zapier.com sitemap articles."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import ssl
|
||||
import urllib.request
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
URLS = json.loads((ROOT / "raw" / "help-sitemap-new-urls.json").read_text())
|
||||
OUT = ROOT / "raw" / "help-articles-extra.jsonl"
|
||||
|
||||
# copy of helpers (keep this file standalone)
|
||||
AUTH_PATTERNS = [
|
||||
("oauth", re.compile(r"\bOAuth\b|Grant Zapier permission|new browser tab or window will open", re.I)),
|
||||
("api_key", re.compile(r"\bAPI key\b|\bAPI token\b|\bpersonal access token\b", re.I)),
|
||||
("username_password", re.compile(r"username and password|email and password", re.I)),
|
||||
("webhook", re.compile(r"webhook URL|catch hook", re.I)),
|
||||
("premium", re.compile(r"Premium app", re.I)),
|
||||
]
|
||||
|
||||
|
||||
def html_to_text(html: str) -> str:
|
||||
html = re.sub(r"<script[\s\S]*?</script>", " ", html, flags=re.I)
|
||||
html = re.sub(r"<style[\s\S]*?</style>", " ", html, flags=re.I)
|
||||
html = re.sub(r"<[^>]+>", " ", html)
|
||||
return re.sub(r"\s+", " ", html).strip()
|
||||
|
||||
|
||||
def fetch_article(url: str) -> dict:
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0 research"})
|
||||
ctx = ssl.create_default_context()
|
||||
with urllib.request.urlopen(req, timeout=25, context=ctx) as r:
|
||||
html = r.read().decode("utf-8", errors="ignore")
|
||||
final = r.geturl()
|
||||
text = html_to_text(html)
|
||||
for cut in ("Related articles", "Was this article helpful"):
|
||||
i = text.find(cut)
|
||||
if i > 400:
|
||||
text = text[:i]
|
||||
title = ""
|
||||
m = re.search(r"/articles/\d+-([^/?#]+)", url)
|
||||
if m:
|
||||
title = m.group(1).replace("-", " ")
|
||||
slug = None
|
||||
m2 = re.search(r"get started with (.+?) on zapier", title, re.I)
|
||||
m3 = re.search(r"common problems with (.+?)(?: on zapier)?$", title, re.I)
|
||||
if m2:
|
||||
slug = m2.group(1).strip().lower().replace(" ", "-")
|
||||
elif m3:
|
||||
slug = m3.group(1).strip().lower().replace(" ", "-")
|
||||
return {
|
||||
"url": url,
|
||||
"final_url": final,
|
||||
"title": title,
|
||||
"help_title": title,
|
||||
"slug": slug,
|
||||
"auth_signals": [n for n, rx in AUTH_PATTERNS if rx.search(text)],
|
||||
"text": text[:12000],
|
||||
"error": None,
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
done = set()
|
||||
if OUT.exists():
|
||||
for line in OUT.open():
|
||||
try:
|
||||
done.add(json.loads(line)["url"])
|
||||
except Exception:
|
||||
pass
|
||||
todo = [u for u in URLS if u not in done]
|
||||
print("todo", len(todo), flush=True)
|
||||
n = err = 0
|
||||
with OUT.open("a") as out, ThreadPoolExecutor(max_workers=16) as ex:
|
||||
futs = {ex.submit(fetch_article, u): u for u in todo}
|
||||
for fut in as_completed(futs):
|
||||
url = futs[fut]
|
||||
try:
|
||||
rec = fut.result()
|
||||
except Exception as e:
|
||||
rec = {"url": url, "error": f"{type(e).__name__}: {e}"}
|
||||
err += 1
|
||||
out.write(json.dumps(rec, ensure_ascii=False) + "\n")
|
||||
out.flush()
|
||||
n += 1
|
||||
if n % 50 == 0:
|
||||
print(f"{n}/{len(todo)} err={err}", flush=True)
|
||||
print("finished", n, "err", err)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
132
research/zapier/scripts/scrape-help.py
Normal file
132
research/zapier/scripts/scrape-help.py
Normal file
|
|
@ -0,0 +1,132 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Fetch Zapier help articles and extract auth / connect guidance."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import ssl
|
||||
import urllib.request
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
URLS = ROOT / "raw" / "help-urls.json"
|
||||
OUT = ROOT / "raw" / "help-articles.jsonl"
|
||||
|
||||
AUTH_PATTERNS = [
|
||||
("oauth", re.compile(r"\bOAuth\b|\bSign in with\b|\bAuthorize\b", re.I)),
|
||||
("api_key", re.compile(r"\bAPI key\b|\bAPI token\b|\baccess token\b|\bsecret key\b", re.I)),
|
||||
("username_password", re.compile(r"username and password|email and password", re.I)),
|
||||
("session", re.compile(r"\bsession\b.*\blogin\b|\blog in to generate", re.I)),
|
||||
("basic_auth", re.compile(r"\bBasic Auth\b|\bbasic authentication\b", re.I)),
|
||||
("webhook", re.compile(r"\bwebhook URL\b|\bcatch hook\b", re.I)),
|
||||
("invite_only", re.compile(r"invite-only|invitation link", re.I)),
|
||||
("premium", re.compile(r"\bpremium (?:app|account)\b", re.I)),
|
||||
]
|
||||
|
||||
|
||||
def html_to_text(html: str) -> str:
|
||||
html = re.sub(r"<script[\s\S]*?</script>", " ", html, flags=re.I)
|
||||
html = re.sub(r"<style[\s\S]*?</style>", " ", html, flags=re.I)
|
||||
html = re.sub(r"<[^>]+>", " ", html)
|
||||
html = re.sub(r" ", " ", html)
|
||||
html = re.sub(r"&", "&", html)
|
||||
html = re.sub(r"<", "<", html)
|
||||
html = re.sub(r">", ">", html)
|
||||
return re.sub(r"\s+", " ", html).strip()
|
||||
|
||||
|
||||
def extract_section(text: str, start_pat: str, end_pats: list[str]) -> str:
|
||||
m = re.search(start_pat, text, re.I)
|
||||
if not m:
|
||||
return ""
|
||||
start = m.start()
|
||||
end = len(text)
|
||||
for ep in end_pats:
|
||||
n = re.search(ep, text[m.end() :], re.I)
|
||||
if n:
|
||||
end = min(end, m.end() + n.start())
|
||||
return text[start:end].strip()[:2500]
|
||||
|
||||
|
||||
def fetch_article(url: str) -> dict:
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
headers={"User-Agent": "Mozilla/5.0 research", "Accept": "text/html"},
|
||||
)
|
||||
ctx = ssl.create_default_context()
|
||||
with urllib.request.urlopen(req, timeout=25, context=ctx) as r:
|
||||
html = r.read().decode("utf-8", errors="ignore")
|
||||
final = r.geturl()
|
||||
text = html_to_text(html)
|
||||
# drop chrome
|
||||
for cut in ("Related articles", "Was this article helpful", "Have more questions"):
|
||||
i = text.find(cut)
|
||||
if i > 400:
|
||||
text = text[:i]
|
||||
auth = [name for name, rx in AUTH_PATTERNS if rx.search(text)]
|
||||
connect = extract_section(
|
||||
text,
|
||||
r"Connect .+ to Zapier|To create an app connection|How to connect",
|
||||
[r"Prerequisites", r"Using .+ with Zapier", r"Triggers", r"Actions", r"Common problems"],
|
||||
)
|
||||
prereq = extract_section(
|
||||
text,
|
||||
r"Prerequisites",
|
||||
[r"Connect .+ to Zapier", r"How to connect", r"Using .+ with Zapier", r"Triggers"],
|
||||
)
|
||||
return {
|
||||
"url": url,
|
||||
"final_url": final,
|
||||
"title": (re.search(r"How to get started[^.]{0,80}|Common Problems[^.]{0,80}", text) or type("x", (), {"group": lambda s: ""})()).group()
|
||||
if False
|
||||
else "",
|
||||
"auth_signals": auth,
|
||||
"prerequisites": prereq[:1500],
|
||||
"connect_steps": connect[:2500],
|
||||
"text": text[:12000],
|
||||
"error": None,
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
by_slug = json.loads(URLS.read_text())
|
||||
# unique urls
|
||||
jobs = []
|
||||
for slug, arts in by_slug.items():
|
||||
for a in arts:
|
||||
jobs.append((slug, a.get("title"), a["url"]))
|
||||
done = {}
|
||||
if OUT.exists():
|
||||
for line in OUT.open():
|
||||
try:
|
||||
rec = json.loads(line)
|
||||
done[rec["url"]] = True
|
||||
except Exception:
|
||||
pass
|
||||
todo = [j for j in jobs if j[2] not in done]
|
||||
print(f"articles={len(jobs)} todo={len(todo)}", flush=True)
|
||||
n = err = 0
|
||||
with OUT.open("a") as out, ThreadPoolExecutor(max_workers=16) as ex:
|
||||
futs = {ex.submit(fetch_article, url): (slug, title, url) for slug, title, url in todo}
|
||||
for fut in as_completed(futs):
|
||||
slug, title, url = futs[fut]
|
||||
try:
|
||||
rec = fut.result()
|
||||
except Exception as e:
|
||||
rec = {"url": url, "error": f"{type(e).__name__}: {e}", "auth_signals": []}
|
||||
err += 1
|
||||
rec["slug"] = slug
|
||||
rec["help_title"] = title
|
||||
if not rec.get("title"):
|
||||
rec["title"] = title
|
||||
out.write(json.dumps(rec, ensure_ascii=False) + "\n")
|
||||
out.flush()
|
||||
n += 1
|
||||
if n % 50 == 0:
|
||||
print(f"{n}/{len(todo)} err={err} last={slug}", flush=True)
|
||||
print("finished", n, "err", err)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
56
research/zapier/scripts/zapier-status.sh
Executable file
56
research/zapier/scripts/zapier-status.sh
Executable file
|
|
@ -0,0 +1,56 @@
|
|||
#!/bin/sh
|
||||
# Print coding-set readiness: tunnel, CLIs, Mongo ping, login files.
|
||||
set -u
|
||||
ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)"
|
||||
export PATH="$HOME/.npm-global/bin:$PATH"
|
||||
if [ -f "$HOME/.mcp-env" ]; then
|
||||
# shellcheck disable=SC1091
|
||||
. "$HOME/.mcp-env"
|
||||
fi
|
||||
|
||||
ok() { printf ' ok %s\n' "$1"; }
|
||||
bad() { printf ' FAIL %s\n' "$1"; }
|
||||
|
||||
echo "workspace: $ROOT"
|
||||
|
||||
if nc -z 127.0.0.1 27017 2>/dev/null; then
|
||||
ok "127.0.0.1:27017 listening"
|
||||
else
|
||||
bad "127.0.0.1:27017 not listening (run scripts/ensure-mongo-tunnel.sh)"
|
||||
fi
|
||||
|
||||
if command -v zapier-platform >/dev/null 2>&1; then
|
||||
ok "zapier-platform $(zapier-platform --version 2>/dev/null | head -1 | tr -d '* ' )"
|
||||
else
|
||||
bad "zapier-platform not on PATH (source scripts/dev-env.sh)"
|
||||
fi
|
||||
|
||||
if command -v zapier-sdk >/dev/null 2>&1; then
|
||||
ok "zapier-sdk $(zapier-sdk --version 2>/dev/null)"
|
||||
else
|
||||
bad "zapier-sdk not on PATH"
|
||||
fi
|
||||
|
||||
if [ -f "$HOME/.zapierrc" ]; then
|
||||
ok "~/.zapierrc (platform CLI logged in)"
|
||||
else
|
||||
bad "~/.zapierrc missing — run: zapier-platform login"
|
||||
fi
|
||||
|
||||
if [ -n "${ZAPIER_MONGO_URI:-}" ]; then
|
||||
if command -v mongosh >/dev/null 2>&1; then
|
||||
n="$(mongosh --quiet "$ZAPIER_MONGO_URI" --eval 'db.getSiblingDB("zapier").platform_reference.estimatedDocumentCount()' 2>/dev/null || true)"
|
||||
if [ -n "$n" ]; then
|
||||
ok "mongo platform_reference=$n"
|
||||
else
|
||||
bad "mongosh could not count platform_reference"
|
||||
fi
|
||||
else
|
||||
ok "ZAPIER_MONGO_URI set (mongosh not installed locally)"
|
||||
fi
|
||||
else
|
||||
bad "ZAPIER_MONGO_URI unset (source ~/.mcp-env)"
|
||||
fi
|
||||
|
||||
echo "resume: $ROOT/scripts/restart-grok.sh"
|
||||
echo "playbook: db.platform_reference.findOne({ kind: 'guide', key: 'build-new-connector' })"
|
||||
Loading…
Add table
Add a link
Reference in a new issue