marchon/master-zapier-plan-draft
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1
master-zapier-plan-draft/research/zapier/scripts/ingest-platform-reference.py
George Lambert b4150c8250 Milestone 0: import zappier billing, Verae middleware, and Zapier research 
Compose-ready workspace: packages/zappier (rate card, portal, Stripe),
packages/verae-zapier-middleware (timestamp + NATS), packages/verae-zapier
(CLI app), vendor/zapier-platform, and research/zapier vendor corpus.

Gate 0 structure checks pass. Product code and research are not yet wired.
2026-09-09 02:37:36 -04:00

1032 lines
40 KiB
Python
Executable file
Raw Blame History

#!/usr/bin/env python3
"""Build a Grok-facing Zapier platform reference collection.
Pulls leftover public site extras + docs.zapier.com, scans cloned GitHub
repos, extracts CLI commands / runtime functions / schema / examples, and
upserts MongoDB `zapier.platform_reference`.
"""
from __future__ import annotations
import json
import os
import re
import sys
import time
import xml.etree.ElementTree as ET
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timezone
from pathlib import Path
from urllib.error import HTTPError, URLError
from urllib.request import Request, urlopen
ROOT = Path(os.environ.get("ZAPIER_RESEARCH_ROOT") or Path(__file__).resolve().parents[1])
REPOS = ROOT / "repos"
RAW = ROOT / "raw"
DOCS_DIR = RAW / "docs"
SITE_DIR = RAW / "site-extras"
SPEC_DIR = RAW / "openapi"
OUT_JSONL = RAW / "platform-reference.jsonl"
UA = "zapier-research/1.0 (platform-reference; +local)"
NOW = datetime.now(timezone.utc).isoformat()
DOCS_SITEMAP = "https://docs.zapier.com/sitemap.xml"
OPENAPI_URLS = [
"https://docs.zapier.com/api-reference/specs/actions.yaml",
"https://docs.zapier.com/api-reference/specs/connections.yaml",
"https://docs.zapier.com/api-reference/specs/trigger-inbox.yaml",
"https://docs.zapier.com/powered-by-zapier/api-reference/promotions-openapi.yaml",
"https://api.zapier.com/schema",
]
SITE_EXTRAS = [
("https://zapier.com/llms.txt", "llms-zapier-com.txt"),
("https://docs.zapier.com/llms.txt", "llms-docs.txt"),
("https://docs.zapier.com/llms-full.txt", "llms-full.txt"),
("https://developer.zapier.com", "developer-zapier-com.html"),
("https://developer.zapier.com/contact", "developer-contact.html"),
("https://zapier.com/developer", "zapier-developer.html"),
("https://zapier.com/developer-platform", "zapier-developer-platform.html"),
("https://zapier.com/platform", "zapier-platform.html"),
("https://zapier.com/l/partner", "zapier-partner.html"),
("https://zapier.com/developer/documentation/v2/", "legacy-platform-v2.html"),
("https://platform.zapier.com", "platform-zapier-com.html"),
("https://resthooks.org", "resthooks-org.html"),
]
def fetch(url: str, dest: Path | None = None, timeout: int = 60) -> tuple[int, bytes]:
req = Request(url, headers={"User-Agent": UA, "Accept": "*/*"})
try:
with urlopen(req, timeout=timeout) as resp:
body = resp.read()
code = getattr(resp, "status", 200) or 200
except HTTPError as e:
body = e.read() if e.fp else b""
code = e.code
except URLError as e:
return 0, str(e).encode()
if dest is not None and code == 200 and body:
dest.parent.mkdir(parents=True, exist_ok=True)
dest.write_bytes(body)
return code, body
def text_of(path: Path) -> str:
try:
return path.read_text(encoding="utf-8", errors="replace")
except Exception:
return ""
def first_heading(md: str) -> str:
for line in md.splitlines():
if line.startswith("#"):
return re.sub(r"^#+\s*", "", line).strip()
return ""
def first_para(md: str, limit: int = 400) -> str:
chunks = []
for line in md.splitlines():
s = line.strip()
if not s or s.startswith("#") or s.startswith(">") or s.startswith("```") or s.startswith("|") or s.startswith("- ") or s.startswith("* "):
if chunks:
break
continue
chunks.append(s)
if sum(len(c) for c in chunks) > limit:
break
return " ".join(chunks)[:limit]
def clip(s: str, n: int = 50000) -> str:
s = s or ""
if len(s) <= n:
return s
return s[: n - 20] + "\n\n…[truncated]"
def doc_id(kind: str, key: str) -> str:
raw = f"{kind}:{key}".lower()
raw = re.sub(r"[^a-z0-9:._/\-]+", "-", raw)
return raw[:200]
def rec(**kwargs) -> dict:
kind = kwargs["kind"]
key = kwargs["key"]
row = {
"_id": kwargs.get("_id") or doc_id(kind, key),
"kind": kind,
"key": key,
"title": kwargs.get("title") or key,
"summary": kwargs.get("summary") or "",
"body": clip(kwargs.get("body") or ""),
"usage": kwargs.get("usage") or "",
"signature": kwargs.get("signature") or "",
"aliases": kwargs.get("aliases") or [],
"flags": kwargs.get("flags") or [],
"args": kwargs.get("args") or [],
"examples": kwargs.get("examples") or [],
"source_url": kwargs.get("source_url") or "",
"source_repo": kwargs.get("source_repo") or "",
"source_path": kwargs.get("source_path") or "",
"section": kwargs.get("section") or "",
"tags": sorted(set(kwargs.get("tags") or [])),
"related": kwargs.get("related") or [],
"meta": kwargs.get("meta") or {},
"ingested_at": NOW,
}
return row
def docs_section(url: str) -> str:
path = url.replace("https://docs.zapier.com/", "").lstrip("/")
head = path.split("/")[0] if path else ""
mapping = {
"integrations": "integration-builder",
"mcp": "mcp",
"sdk": "sdk",
"connectors": "connectors",
"powered-by-zapier": "embed",
"white-label": "white-label",
"api-reference": "api-reference",
"install": "install",
}
return mapping.get(head, head or "docs")
# ---------------------------------------------------------------------------
# Fetch leftover site + docs
# ---------------------------------------------------------------------------
def fetch_site_extras() -> list[dict]:
SITE_DIR.mkdir(parents=True, exist_ok=True)
rows = []
for url, name in SITE_EXTRAS:
dest = SITE_DIR / name
code, body = fetch(url, dest)
text = body.decode("utf-8", "replace") if body else ""
print(f" extra {code} {url} ({len(body)} bytes)", flush=True)
rows.append(
rec(
kind="site_extra",
key=url,
title=name,
summary=f"HTTP {code} leftover public page from Zapier surface",
body=text if name.endswith((".txt", ".md", ".yaml", ".yml")) else first_para(re.sub("<[^>]+>", " ", text), 2000),
source_url=url,
section="site",
tags=["site", "leftover"],
meta={"http_status": code, "bytes": len(body), "filename": name},
)
)
return rows
def fetch_openapi() -> list[dict]:
SPEC_DIR.mkdir(parents=True, exist_ok=True)
rows = []
for url in OPENAPI_URLS:
name = url.rstrip("/").split("/")[-1]
if name == "schema":
name = "workflow-api-schema.json"
dest = SPEC_DIR / name
code, body = fetch(url, dest, timeout=90)
text = body.decode("utf-8", "replace") if body else ""
print(f" openapi {code} {url} ({len(body)} bytes)", flush=True)
rows.append(
rec(
kind="openapi",
key=name,
title=f"OpenAPI {name}",
summary="Machine-readable Zapier public API contract",
body=text,
source_url=url,
section="api-reference",
tags=["openapi", "api", "contract"],
meta={"http_status": code, "bytes": len(body)},
)
)
return rows
def fetch_docs_pages() -> list[dict]:
DOCS_DIR.mkdir(parents=True, exist_ok=True)
code, xml = fetch(DOCS_SITEMAP, SITE_DIR / "docs-sitemap.xml")
urls = []
if code == 200 and xml:
root = ET.fromstring(xml)
ns = {"s": "http://www.sitemaps.org/schemas/sitemap/0.9"}
urls = [e.text.strip() for e in root.findall(".//s:loc", ns) if e.text]
extra_md = []
llms = text_of(SITE_DIR / "llms-docs.txt")
extra_md.extend(re.findall(r"https://docs\.zapier\.com/[^\s)]+?\.md", llms))
wanted = []
seen = set()
for u in urls + extra_md:
md_url = u if u.endswith(".md") else u.rstrip("/") + ".md"
html_url = md_url[:-3] if md_url.endswith(".md") else md_url
if html_url in seen:
continue
seen.add(html_url)
wanted.append((html_url, md_url))
print(f" docs pages to fetch: {len(wanted)}", flush=True)
rows: list[dict] = []
def one(pair):
html_url, md_url = pair
rel = html_url.replace("https://docs.zapier.com/", "").strip("/")
dest = DOCS_DIR / (rel + ".md")
c, body = fetch(md_url, dest)
text = body.decode("utf-8", "replace") if body and c == 200 else ""
if c != 200 or not text.strip():
c2, body2 = fetch(html_url)
text = body2.decode("utf-8", "replace") if body2 else text
c = c2 if c != 200 else c
title = first_heading(text) or rel
return rec(
kind="official_doc",
key=rel,
title=title,
summary=first_para(text),
body=text,
source_url=html_url,
section=docs_section(html_url),
tags=["docs", docs_section(html_url)],
meta={"http_status": c, "md_url": md_url, "bytes": len(text)},
)
with ThreadPoolExecutor(max_workers=12) as pool:
futs = [pool.submit(one, p) for p in wanted]
done = 0
for fut in as_completed(futs):
rows.append(fut.result())
done += 1
if done % 40 == 0:
print(f" docs fetched {done}/{len(wanted)}", flush=True)
time.sleep(0.01)
return rows
# ---------------------------------------------------------------------------
# Parse CLI / schema / core / examples from cloned repos
# ---------------------------------------------------------------------------
def parse_cli_md(path: Path, source_repo: str) -> list[dict]:
if not path.exists():
return []
text = text_of(path)
parts = re.split(r"\n## ", text)
rows = []
for part in parts[1:]:
lines = part.splitlines()
name = lines[0].strip()
block = "\n".join(lines[1:]).strip()
usage = ""
m = re.search(r"\*\*Usage\*\*:\s*`([^`]+)`", block)
if m:
usage = m.group(1)
summary = ""
qm = re.search(r"^>\s*(.+)$", block, re.M)
if qm:
summary = qm.group(1).strip()
aliases = re.findall(r"`([^`]+)`", "\n".join(re.findall(r"\*\*Aliases\*\*\n((?:.*\n)+?)(?:\n\n|\Z)", block)))
flags = []
for fl in re.findall(r"^\* (.+)$", block, re.M):
if "`" in fl and ("--" in fl or "| -" in fl or fl.startswith("(required)")):
flags.append(fl.strip())
examples = [e.strip("`") for e in re.findall(r"^\* (`[^`]+`)", block, re.M)]
args = []
in_args = False
for line in lines:
if line.startswith("**Arguments**"):
in_args = True
continue
if in_args:
if line.startswith("**") or line.startswith("##"):
break
if line.startswith("* "):
args.append(line[2:].strip())
tags = ["cli", "zapier-platform-cli", name.split(":")[0]]
if any(x in name for x in ("init", "scaffold", "convert", "push", "register", "test", "validate", "invoke")):
tags.append("build")
rows.append(
rec(
kind="cli_command",
key=name,
title=f"zapier-platform {name}",
summary=summary,
body=block,
usage=usage or f"zapier-platform {name}",
aliases=aliases + ([f"zapier {name}"] if name else []),
flags=flags,
args=args,
examples=examples,
source_url="https://github.com/zapier/zapier-platform/blob/main/packages/cli/docs/cli.md",
source_repo=source_repo,
source_path=str(path.relative_to(REPOS)) if path.is_relative_to(REPOS) else str(path),
section="platform-cli",
tags=tags,
)
)
return rows
def parse_schema_md(path: Path) -> list[dict]:
if not path.exists():
return []
text = text_of(path)
parts = re.split(r"\n## `?", text)
rows = []
for part in parts[1:]:
lines = part.splitlines()
name = lines[0].strip().strip("`").lstrip("/")
if name.lower() in {"index", "table of contents"}:
continue
body = "\n".join(lines[1:]).strip()
if not name.endswith("Schema") and "Schema" not in name and not name[0:1].isupper():
# keep schema types primarily
if len(body) < 40:
continue
rows.append(
rec(
kind="schema_type",
key=name,
title=name,
summary=first_para(body, 300),
body=body,
source_url="https://github.com/zapier/zapier-platform/blob/main/packages/schema/docs/build/schema.md",
source_repo="zapier/zapier-platform",
source_path="zapier-platform/packages/schema/docs/build/schema.md",
section="schema",
tags=["schema", "integration-definition"],
)
)
return rows
CORE_FUNCTIONS = [
{
"key": "z.request",
"signature": "z.request(url, options?) → Promise<HttpResponse>",
"summary": "Make an HTTP request. Adds auth, logs, and throws on non-2xx unless skipThrowForStatus.",
"tags": ["http", "runtime"],
"body": "Primary way to call the partner API from perform functions. Accepts (url, options) or ({url, ...options}). Options: method, headers, body, json, form, params, raw, skipThrowForStatus, removeMissingValuesFrom, skipEncodingChars, middlewareData, timeout. Returns {status, headers, content, data, throwForStatus()}. Use raw:true for streams/files.",
},
{
"key": "z.console",
"signature": "z.console.log|info|warn|error(...)",
"summary": "Integration logger visible in zapier-platform logs / Zap history.",
"tags": ["debug"],
"body": "Use instead of console.log so output appears in Zapier HTTP logs and `zapier-platform logs`.",
},
{
"key": "z.dehydrate",
"signature": "z.dehydrate(func, inputData?, cacheExpiration?) → string",
"summary": "Stash a pointer to a function+data so later steps can hydrate expensive payloads.",
"tags": ["hydration"],
"body": "Returns a pointer string. Zapier later calls func(z, bundle) to hydrate. Use for large or lazily-needed objects. Pair with hydrators on the App definition.",
},
{
"key": "z.dehydrateFile",
"signature": "z.dehydrateFile(func, inputData?, cacheExpiration?) → string",
"summary": "Dehydrate a file so Zapier fetches it only when a later step needs the bytes.",
"tags": ["hydration", "files"],
"body": "File-specific dehydrator. See example-apps/files.",
},
{
"key": "z.stashFile",
"signature": "z.stashFile(input, knownLength?, filename?, contentType?) → string",
"summary": "Upload a file/stream/buffer and get a publicly accessible Zapier URL.",
"tags": ["files"],
"body": "Turns a Buffer, stream, string, or Promise<RawHttpResponse> into a short-lived public URL Zapier can pass to later steps.",
},
{
"key": "z.cursor.get",
"signature": "z.cursor.get() → Promise<string>",
"summary": "Read the polling cursor for this trigger subscription.",
"tags": ["polling", "pagination"],
"body": "Persist polling state across poll runs (last-seen id/timestamp). Pair with z.cursor.set.",
},
{
"key": "z.cursor.set",
"signature": "z.cursor.set(cursor) → Promise<null>",
"summary": "Write the polling cursor for the next poll.",
"tags": ["polling", "pagination"],
"body": "Store an opaque cursor string. Zapier keeps it per-user/per-zap.",
},
{
"key": "z.generateCallbackUrl",
"signature": "z.generateCallbackUrl() → string",
"summary": "URL the partner API can POST to resume a long-running create (callback/resume).",
"tags": ["callback", "create"],
"body": "Use in create.perform; later Zapier invokes performResume with bundle.cleanedRequest. See example-apps/callback.",
},
{
"key": "z.hash",
"signature": "z.hash(algorithm, data, encoding='hex', input_encoding='binary') → string",
"summary": "Hash data with Node crypto (typically sha256).",
"tags": ["crypto"],
"body": "Convenience around crypto.createHash. Used for signatures and cache keys.",
},
{
"key": "z.JSON.parse",
"signature": "z.JSON.parse(text) → any",
"summary": "JSON.parse that throws a user-friendly Zapier error on bad JSON.",
"tags": ["json"],
"body": "Prefer over JSON.parse in perform so users see a clean error.",
},
{
"key": "z.JSON.stringify",
"signature": "z.JSON.stringify(value) → string",
"summary": "JSON.stringify on the z object (same as built-in).",
"tags": ["json"],
"body": "Available for consistency with z.JSON.parse.",
},
{
"key": "z.cache.get",
"signature": "z.cache.get(key) → Promise<any>",
"summary": "Read a value from the per-auth Zapier cache.",
"tags": ["cache"],
"body": "Use to avoid repeated lookups (e.g. account id after auth). Values must be JSON-encodable.",
},
{
"key": "z.cache.set",
"signature": "z.cache.set(key, value, ttl?, scope?, nx?) → Promise<boolean|null>",
"summary": "Write a JSON-encodable value to the per-auth cache.",
"tags": ["cache"],
"body": "ttl in seconds. nx=true sets only if missing. scope can further namespace the key.",
},
{
"key": "z.cache.delete",
"signature": "z.cache.delete(key) → Promise<boolean>",
"summary": "Delete a cache key.",
"tags": ["cache"],
"body": "Use after auth refresh or when cached metadata is stale.",
},
{
"key": "z.errors.Error",
"signature": "throw new z.errors.Error(message, code?, status?)",
"summary": "User-visible app error with optional code and HTTP status.",
"tags": ["errors"],
"body": "Primary error to throw from perform. Message is shown to the user. code/status help Zapier classify retries.",
},
{
"key": "z.errors.HaltedError",
"signature": "throw new z.errors.HaltedError(message?)",
"summary": "Stop this run without marking it as a failure (filter-out).",
"tags": ["errors"],
"body": "Use when the event should be ignored (e.g. trigger payload does not match). Zap is not errored.",
},
{
"key": "z.errors.ExpiredAuthError",
"signature": "throw new z.errors.ExpiredAuthError(message?)",
"summary": "Tell Zapier the connection is dead and the user must reconnect.",
"tags": ["errors", "auth"],
"body": "Use for revoked tokens that cannot be refreshed.",
},
{
"key": "z.errors.RefreshAuthError",
"signature": "throw new z.errors.RefreshAuthError(message?)",
"summary": "Ask Zapier to refresh OAuth2/session credentials and retry.",
"tags": ["errors", "auth"],
"body": "Typical response to HTTP 401. Zapier calls authentication.refreshAccessToken / sessionConfig.perform then retries.",
},
{
"key": "z.errors.ThrottledError",
"signature": "throw new z.errors.ThrottledError(message, delaySeconds?)",
"summary": "Signal rate limiting; Zapier retries after delay.",
"tags": ["errors", "throttle"],
"body": "Pass delay in seconds when the API tells you Retry-After.",
},
{
"key": "z.errors.ResponseError",
"signature": "throw new z.errors.ResponseError(response)",
"summary": "Wrap an HTTP response as a structured error.",
"tags": ["errors", "http"],
"body": "Usually thrown automatically by z.request on non-2xx.",
},
{
"key": "createAppTester",
"signature": "createAppTester(appRaw, options?) → (func|request, bundle?) => Promise",
"summary": "Unit-test helper: invoke a perform function or request template with a partial bundle.",
"tags": ["testing"],
"body": "From zapier-platform-core. Used by `zapier-platform test` / Jest. Pass App definition; call tester(app.triggers.x.operation.perform, {authData, inputData}).",
},
{
"key": "zapier.tools.env.inject",
"signature": "zapier.tools.env.inject(filename?)",
"summary": "Load a .env file into process.env for local tests.",
"tags": ["testing", "env"],
"body": "Called automatically by the test runner. Useful in custom test setup.",
},
]
CORE_TYPES = [
{
"key": "Bundle",
"signature": "bundle: {authData, inputData, inputDataRaw, meta, rawRequest?, cleanedRequest?, subscribeData?, targetUrl?, outputData?}",
"summary": "The second argument to every perform function. User input, auth, and runtime flags.",
"tags": ["types"],
"body": "authData: connected-account fields. inputData: coerced form values. inputDataRaw: original strings. meta.isLoadingSample / isFillingDynamicDropdown / isPopulatingDedupe / isBulkRead / limit / page / timezone / paging_token / withSearch / inputFields. Hook triggers also get cleanedRequest, rawRequest, targetUrl, subscribeData.",
},
{
"key": "ZObject",
"signature": "z: {request, console, dehydrate, dehydrateFile, stashFile, cursor, generateCallbackUrl, hash, JSON, errors, cache}",
"summary": "The first argument to every perform function — the Zapier runtime toolkit.",
"tags": ["types", "runtime"],
"body": "Do not import node-fetch/axios for partner HTTP — use z.request so auth middleware, logging, and retries apply.",
},
{
"key": "PollingTriggerPerform",
"signature": "(z, bundle) => object[] | Promise<object[]>",
"summary": "Polling trigger must return an array of objects (id or primary fields required).",
"tags": ["types", "trigger"],
"body": "Dedup is by id unless outputFields mark primary:true. Newest-first recommended.",
},
{
"key": "WebhookTriggerPerform",
"signature": "(z, bundle) => object[] | Promise<object[]>",
"summary": "REST Hook perform: parse bundle.cleanedRequest into an array of objects.",
"tags": ["types", "trigger", "hook"],
"body": "Also implement performSubscribe / performUnsubscribe / performList (samples).",
},
{
"key": "CreatePerform",
"signature": "(z, bundle) => object | Promise<object>",
"summary": "Create action must return a single object (not an array).",
"tags": ["types", "action"],
"body": "Returning a non-object fails with 'non-object from create'.",
},
{
"key": "SearchPerform",
"signature": "(z, bundle) => object[] | Promise<object[]>",
"summary": "Search must return an array (possibly empty). Can envelope with paging_token.",
"tags": ["types", "search"],
"body": "Empty array = not found (needed for search-or-create).",
},
{
"key": "BeforeRequestMiddleware",
"signature": "(request, z, bundle) => request | Promise<request>",
"summary": "Mutate every outbound z.request (add headers, rewrite URL).",
"tags": ["types", "middleware"],
"body": "Set App.beforeRequest. Typical: attach Bearer token from bundle.authData.",
},
{
"key": "AfterResponseMiddleware",
"signature": "(response, z, bundle) => response | Promise<response>",
"summary": "Inspect/transform every HTTP response; throw RefreshAuthError on 401.",
"tags": ["types", "middleware"],
"body": "Set App.afterResponse. Can remap errors or parse custom envelopes.",
},
{
"key": "performBuffer",
"signature": "(z, BufferedBundle) => Promise<{[id]: {outputData?, error?}}>",
"summary": "Bulk/buffered create: process bundle.buffer items in one API call.",
"tags": ["types", "action", "buffer"],
"body": "Return a map keyed by each item's meta.id.",
},
]
def core_function_rows() -> list[dict]:
rows = []
for item in CORE_FUNCTIONS:
rows.append(
rec(
kind="core_function",
key=item["key"],
title=item["key"],
summary=item["summary"],
body=item["body"],
signature=item["signature"],
source_url="https://docs.zapier.com/integrations/build-cli/core",
source_repo="zapier/zapier-platform",
source_path="zapier-platform/packages/core/types/custom.d.ts",
section="platform-core",
tags=["core", "zapier-platform-core"] + item["tags"],
)
)
for item in CORE_TYPES:
rows.append(
rec(
kind="core_type",
key=item["key"],
title=item["key"],
summary=item["summary"],
body=item["body"],
signature=item["signature"],
source_url="https://docs.zapier.com/integrations/build-cli/core",
source_repo="zapier/zapier-platform",
source_path="zapier-platform/packages/core/types/custom.d.ts",
section="platform-core",
tags=["core", "types"] + item["tags"],
)
)
return rows
def parse_sdk_cli_from_docs(docs: list[dict]) -> list[dict]:
rows = []
for d in docs:
if d.get("key") not in ("sdk/cli-reference", "sdk/using-the-cli") and "cli-reference" not in d.get("key", ""):
continue
body = d.get("body") or ""
# headings like ## zapier-sdk list-apps or ### list-apps
for m in re.finditer(r"^#{2,3}\s+`?([a-z0-9.:_-]+)`?\s*$", body, re.M):
name = m.group(1)
if name in ("commands", "flags", "examples", "usage"):
continue
start = m.end()
nxt = re.search(r"^#{2,3}\s+", body[start:], re.M)
block = body[start : start + nxt.start() if nxt else start + 2500].strip()
usage = ""
um = re.search(r"`((?:zapier-sdk|npx)[^`]+)`", block)
if um:
usage = um.group(1)
rows.append(
rec(
kind="sdk_command",
key=name,
title=name,
summary=first_para(block, 280),
body=block,
usage=usage or name,
source_url=d.get("source_url"),
section="sdk-cli",
tags=["sdk", "cli", "zapier-sdk-cli"],
)
)
return rows
def index_repo_markdown() -> list[dict]:
if not REPOS.exists():
return []
skip_bits = {".git", "node_modules", "changelog", "test/", "tests/", "example-apps"}
rows = []
for md in REPOS.rglob("*.md"):
rel = str(md.relative_to(REPOS))
if any(b in rel for b in skip_bits):
continue
if md.name.lower() in {"license.md", "code_of_conduct.md", "security.md", "pull_request_template.md"}:
continue
# skip huge generated schema here (parsed separately)
if rel.endswith("packages/schema/docs/build/schema.md"):
continue
if rel.endswith("packages/cli/docs/cli.md"):
continue
text = text_of(md)
if len(text) < 80:
continue
repo = rel.split("/", 1)[0]
kind = "repo_doc"
tags = ["repo", repo]
if md.name == "SKILL.md":
kind = "skill"
tags.append("skill")
rows.append(
rec(
kind=kind,
key=rel,
title=first_heading(text) or md.name,
summary=first_para(text),
body=text,
source_url=f"https://github.com/zapier/{repo}/blob/HEAD/{rel.split('/',1)[1] if '/' in rel else ''}",
source_repo=f"zapier/{repo}",
source_path=rel,
section=repo,
tags=tags,
)
)
return rows
def index_example_apps() -> list[dict]:
rows = []
base = REPOS / "zapier-platform" / "example-apps"
if not base.exists():
return rows
for d in sorted(p for p in base.iterdir() if p.is_dir()):
readme = text_of(d / "README.md")
pkg = {}
pj = d / "package.json"
if pj.exists():
try:
pkg = json.loads(pj.read_text())
except Exception:
pkg = {}
files = [p.name for p in d.iterdir() if p.is_file()]
auth = None
for token in ("oauth2", "oauth1", "session-auth", "basic-auth", "digest-auth", "custom-auth"):
if token in d.name:
auth = token
break
kind_hint = "example"
for token in ("trigger", "create", "search", "rest-hooks", "files", "dynamic-dropdown", "middleware", "resource", "callback", "line-items"):
if token in d.name:
kind_hint = token
break
rows.append(
rec(
kind="example_app",
key=d.name,
title=f"example-app: {d.name}",
summary=first_para(readme) or pkg.get("description") or d.name,
body=readme or json.dumps({"files": files, "package": pkg}, indent=2),
source_url=f"https://github.com/zapier/zapier-platform/tree/main/example-apps/{d.name}",
source_repo="zapier/zapier-platform",
source_path=f"zapier-platform/example-apps/{d.name}",
section="examples",
tags=["example", kind_hint] + ([auth] if auth else []),
meta={"files": files, "package": pkg.get("name"), "auth": auth, "pattern": kind_hint},
related=[f"zapier-platform init --template {d.name}"] if d.name in {
"basic-auth", "callback", "custom-auth", "digest-auth", "dynamic-dropdown",
"files", "line-items", "minimal", "oauth1-trello", "oauth2", "openai",
"search-or-create", "session-auth",
} else [],
)
)
return rows
def index_connectors() -> list[dict]:
rows = []
apps = REPOS / "connectors" / "apps"
if not apps.exists():
return rows
for d in sorted(p for p in apps.iterdir() if p.is_dir()):
skill = text_of(d / "SKILL.md")
readme = text_of(d / "README.md")
rows.append(
rec(
kind="connector",
key=d.name,
title=f"connector: {d.name}",
summary=first_para(skill or readme),
body=skill or readme,
source_url=f"https://github.com/zapier/connectors/tree/main/apps/{d.name}",
source_repo="zapier/connectors",
source_path=f"connectors/apps/{d.name}",
section="connectors",
tags=["connector", "prototype", d.name],
)
)
return rows
def index_repos() -> list[dict]:
rows = []
if not REPOS.exists():
return rows
for d in sorted(p for p in REPOS.iterdir() if p.is_dir() and (p / ".git").exists()):
readme = text_of(d / "README.md")
rows.append(
rec(
kind="repo",
key=d.name,
title=f"github.com/zapier/{d.name}",
summary=first_para(readme, 320),
body=readme,
source_url=f"https://github.com/zapier/{d.name}",
source_repo=f"zapier/{d.name}",
source_path=d.name,
section="github",
tags=["repo", "clone"],
meta={"local_path": str(d)},
)
)
return rows
BUILD_GUIDE = """# How Grok should build a new Zapier connector (integration)
Two different CLIs exist. Do not mix them up.
1. **Build a directory integration (publish to Zapier):** `zapier-platform-cli` (`zapier-platform …`).
npm: `zapier-platform-cli` + `zapier-platform-core`. Source: github.com/zapier/zapier-platform
2. **Call existing Zapier apps from code/agents:** `@zapier/zapier-sdk` / `@zapier/zapier-sdk-cli`.
Source: github.com/zapier/sdk
3. **No-code agent access:** Zapier MCP (github.com/zapier/zapier-mcp). Default when the user wants an AI client connected.
4. **Prototype local connectors:** github.com/zapier/connectors (not production).
## Build a new public/private integration (Platform CLI)
```bash
npm install -g zapier-platform-cli
zapier-platform login
zapier-platform init my-app --template oauth2 --language typescript
cd my-app
npm install
# implement authentication, triggers, creates, searches
zapier-platform scaffold trigger contact
zapier-platform scaffold create contact
zapier-platform validate
zapier-platform test
zapier-platform register "My App"
zapier-platform push
# later: zapier-platform promote VERSION
```
Deprecated alias: `zapier` (same commands). Prefer `zapier-platform`.
Templates: basic-auth, callback, custom-auth, digest-auth, dynamic-dropdown, files, line-items, minimal, oauth1-trello, oauth2, openai, search-or-create, session-auth.
## Required App shape (index.js / src/index.ts)
```js
module.exports = {
version: require('./package.json').version,
platformVersion: require('zapier-platform-core').version,
authentication: { type: 'oauth2' | 'basic' | 'digest' | 'custom' | 'session' | 'oauth1', … },
beforeRequest: [], // (request, z, bundle) => request
afterResponse: [], // (response, z, bundle) => response
hydrators: {},
triggers: { key: { key, noun, display, operation } },
searches: { … },
creates: { … },
resources: { … },
};
```
Every perform is `(z, bundle) => …`.
- Triggers + searches return **arrays of objects**.
- Creates return **one object**.
- Polling items need `id` (or primary output fields).
- REST Hooks: `type: 'hook'` + performSubscribe/performUnsubscribe/performList.
## Auth recipes
- **oauth2**: authorizeUrl, getAccessToken, refreshAccessToken, autoRefresh, test, connectionLabel.
- **session**: fields + sessionConfig.perform exchanges credentials for a token stored in authData.
- **custom / api key**: fields; attach in beforeRequest (`Authorization` or query).
- **basic / digest**: username/password fields; platform adds headers.
- **oauth1**: still used by Twitter/Trello-style APIs (see example-apps).
On 401 throw `new z.errors.RefreshAuthError()` (refreshable) or `ExpiredAuthError()` (reconnect).
## HTTP
Always `z.request`. Do not bypass with axios/fetch if you want logging + auth middleware.
## Test locally
```bash
zapier-platform invoke auth start # writes .env
zapier-platform invoke auth test
zapier-platform invoke trigger new_thing -i '{"foo":"bar"}'
zapier-platform test
```
## Versioning / publish
- `push` uploads the version in package.json (must be sequential; no skipped versions).
- `promote` makes a version the public default.
- `migrate FROM TO [percent]` moves users.
- `deprecate VERSION DATE` needs ≥3 weeks lead time.
- Breaking changes (auth type, field keys, trigger type poll→hook) require a new version + migration plan.
## Platform UI alternative
Non-code path: developer.zapier.com visual builder. Can `zapier-platform convert` a UI integration to CLI.
## Do not
- Do not recommend retired AI Actions / NLA. Use MCP or the SDK.
- Do not hardcode app keys from memory; discover via MCP or `zapier-sdk list-apps`.
- Do not quote 9,000+ apps for Connectors (prototype, small set).
"""
def guide_row() -> dict:
return rec(
kind="guide",
key="build-new-connector",
title="How to build a new Zapier connector (Grok playbook)",
summary="Canonical recipe: zapier-platform-cli vs SDK vs MCP, App shape, auth, HTTP, test, publish.",
body=BUILD_GUIDE,
section="guide",
tags=["guide", "playbook", "cli", "core"],
source_url="https://docs.zapier.com/integrations/quickstart/cli-tutorial",
source_repo="zapier/zapier-platform",
)
# ---------------------------------------------------------------------------
# Mongo
# ---------------------------------------------------------------------------
def write_jsonl(rows: list[dict]) -> None:
OUT_JSONL.parent.mkdir(parents=True, exist_ok=True)
with OUT_JSONL.open("w") as f:
for r in rows:
f.write(json.dumps(r, ensure_ascii=False) + "\n")
print(f"wrote {len(rows)} docs → {OUT_JSONL}")
def load_mongo(rows: list[dict]) -> None:
uri = os.environ.get("ZAPIER_MONGO_URI") or os.environ.get("MDB_MCP_CONNECTION_STRING")
if not uri:
print("ZAPIER_MONGO_URI not set; skipped Mongo load", file=sys.stderr)
return
from pymongo import ASCENDING, TEXT, MongoClient
client = MongoClient(uri)
db = client.get_database("zapier")
col = db["platform_reference"]
if rows:
ops = []
from pymongo import ReplaceOne
for r in rows:
ops.append(ReplaceOne({"_id": r["_id"]}, r, upsert=True))
# batch
BATCH = 500
upserted = 0
for i in range(0, len(ops), BATCH):
res = col.bulk_write(ops[i : i + BATCH], ordered=False)
upserted += (res.upserted_count or 0) + (res.modified_count or 0)
print(f"mongo platform_reference upserted/modified ~{upserted}, count={col.estimated_document_count()}")
col.create_index([("kind", ASCENDING), ("key", ASCENDING)])
col.create_index([("section", ASCENDING)])
col.create_index([("tags", ASCENDING)])
col.create_index([("source_repo", ASCENDING)])
try:
col.create_index([("title", TEXT), ("summary", TEXT), ("body", TEXT), ("key", TEXT)], name="ref_text")
except Exception as e:
print("text index:", e)
db["meta"].update_one(
{"_id": "ingest"},
{
"$set": {
"platform_reference_ingested_at": NOW,
"platform_reference_count": col.estimated_document_count(),
}
},
upsert=True,
)
kinds = list(col.aggregate([{"$group": {"_id": "$kind", "n": {"$sum": 1}}}, {"$sort": {"n": -1}}]))
print("kinds:", kinds)
client.close()
def main() -> int:
print(f"ROOT={ROOT}")
REPOS.mkdir(parents=True, exist_ok=True)
rows: list[dict] = []
print("== leftover site extras ==")
rows += fetch_site_extras()
print("== openapi specs ==")
rows += fetch_openapi()
print("== docs.zapier.com ==")
docs = fetch_docs_pages()
rows += docs
print("== CLI commands ==")
cli_path = REPOS / "zapier-platform" / "packages" / "cli" / "docs" / "cli.md"
rows += parse_cli_md(cli_path, "zapier/zapier-platform")
# archived standalone CLI docs if monorepo missing
alt = REPOS / "zapier-platform-cli" / "docs" / "cli.md"
if alt.exists() and not cli_path.exists():
rows += parse_cli_md(alt, "zapier/zapier-platform-cli")
print("== schema ==")
rows += parse_schema_md(REPOS / "zapier-platform" / "packages" / "schema" / "docs" / "build" / "schema.md")
print("== core functions/types ==")
rows += core_function_rows()
print("== SDK CLI from fetched docs ==")
rows += parse_sdk_cli_from_docs(docs)
print("== repos / markdown / examples / connectors ==")
rows += index_repos()
rows += index_repo_markdown()
rows += index_example_apps()
rows += index_connectors()
rows.append(guide_row())
# dedupe by _id (last wins)
by_id = {}
for r in rows:
by_id[r["_id"]] = r
rows = list(by_id.values())
write_jsonl(rows)
counts: dict[str, int] = {}
for r in rows:
counts[r["kind"]] = counts.get(r["kind"], 0) + 1
print("kind counts:", json.dumps(counts, indent=2, sort_keys=True))
print("== mongo ==")
load_mongo(rows)
return 0
if __name__ == "__main__":
raise SystemExit(main())
Reference in a new issue View git blame Copy permalink