George Lambert
b4150c8250
Compose-ready workspace: packages/zappier (rate card, portal, Stripe), packages/verae-zapier-middleware (timestamp + NATS), packages/verae-zapier (CLI app), vendor/zapier-platform, and research/zapier vendor corpus. Gate 0 structure checks pass. Product code and research are not yet wired.
474 lines
19 KiB
Python
Executable file
474 lines
19 KiB
Python
Executable file
#!/usr/bin/env python3
|
||
"""Load remaining coding-set artifacts into zapier.platform_reference.
|
||
|
||
- exported-schema.json + definition example
|
||
- OpenAPI specs exploded to api_function
|
||
- Platform News 2025–2026
|
||
- example-app source as kind=template
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import os
|
||
import re
|
||
import sys
|
||
from datetime import datetime, timezone
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(os.environ.get("ZAPIER_RESEARCH_ROOT") or Path(__file__).resolve().parents[1])
|
||
REPOS = ROOT / "repos"
|
||
NOW = datetime.now(timezone.utc).isoformat()
|
||
|
||
try:
|
||
import yaml
|
||
except ImportError:
|
||
print("pyyaml required", file=sys.stderr)
|
||
raise
|
||
|
||
|
||
def clip(s: str, n: int = 80000) -> str:
|
||
s = s or ""
|
||
return s if len(s) <= n else s[: n - 20] + "\n\n…[truncated]"
|
||
|
||
|
||
def rec(**kw) -> dict:
|
||
kind, key = kw["kind"], kw["key"]
|
||
_id = kw.get("_id") or f"{kind}:{key}".lower()
|
||
_id = re.sub(r"[^a-z0-9:._/\-]+", "-", _id)[:200]
|
||
return {
|
||
"_id": _id,
|
||
"kind": kind,
|
||
"key": key,
|
||
"title": kw.get("title") or key,
|
||
"summary": kw.get("summary") or "",
|
||
"body": clip(kw.get("body") or ""),
|
||
"usage": kw.get("usage") or "",
|
||
"signature": kw.get("signature") or "",
|
||
"aliases": kw.get("aliases") or [],
|
||
"flags": kw.get("flags") or [],
|
||
"args": kw.get("args") or [],
|
||
"examples": kw.get("examples") or [],
|
||
"source_url": kw.get("source_url") or "",
|
||
"source_repo": kw.get("source_repo") or "",
|
||
"source_path": kw.get("source_path") or "",
|
||
"section": kw.get("section") or "",
|
||
"tags": sorted(set(kw.get("tags") or [])),
|
||
"related": kw.get("related") or [],
|
||
"meta": kw.get("meta") or {},
|
||
"ingested_at": NOW,
|
||
}
|
||
|
||
|
||
def ingest_schema() -> list[dict]:
|
||
rows = []
|
||
path = REPOS / "zapier-platform" / "packages" / "schema" / "exported-schema.json"
|
||
if not path.exists():
|
||
return rows
|
||
data = json.loads(path.read_text())
|
||
version = data.get("version") or "unknown"
|
||
schemas = data.get("schemas") or {}
|
||
rows.append(
|
||
rec(
|
||
kind="schema_json",
|
||
key="exported-schema",
|
||
title=f"zapier-platform-schema {version} (exported JSON)",
|
||
summary="Machine-checkable JSON Schema bundle for App definitions.",
|
||
body=json.dumps(data, indent=2),
|
||
source_url="https://github.com/zapier/zapier-platform/blob/main/packages/schema/exported-schema.json",
|
||
source_repo="zapier/zapier-platform",
|
||
source_path="zapier-platform/packages/schema/exported-schema.json",
|
||
section="schema",
|
||
tags=["schema", "json", "platform"],
|
||
meta={"platform_schema_version": version, "schema_count": len(schemas)},
|
||
)
|
||
)
|
||
for name, spec in schemas.items():
|
||
required = spec.get("required") or []
|
||
props = list((spec.get("properties") or {}).keys())
|
||
body = [
|
||
f"# `{name}`",
|
||
"",
|
||
f"> {spec.get('description') or name}",
|
||
"",
|
||
"## High-level description",
|
||
"",
|
||
spec.get("description") or name,
|
||
"",
|
||
"## Internals",
|
||
"",
|
||
f"JSON Schema id `{spec.get('id')}` from zapier-platform-schema {version}. "
|
||
"Used by `zapier-platform validate` / build.",
|
||
"",
|
||
"## Typed inputs (properties)",
|
||
"",
|
||
"```ts",
|
||
f"// required: {', '.join(required) or '—'}",
|
||
]
|
||
for p, ps in (spec.get("properties") or {}).items():
|
||
ref = ps.get("$ref") or ps.get("type") or "any"
|
||
opt = "?" if p not in required else ""
|
||
body.append(f" {p}{opt}: {ref}; // {(ps.get('description') or '')[:120]}")
|
||
body += ["```", "", "## Schema", "", "```json", json.dumps(spec, indent=2), "```"]
|
||
rows.append(
|
||
rec(
|
||
kind="schema_type",
|
||
key=name,
|
||
title=name,
|
||
summary=spec.get("description") or name,
|
||
body="\n".join(body),
|
||
signature=f"{name} required=[{', '.join(required)}]",
|
||
source_url="https://github.com/zapier/zapier-platform/blob/main/packages/schema/exported-schema.json",
|
||
source_repo="zapier/zapier-platform",
|
||
source_path="zapier-platform/packages/schema/exported-schema.json",
|
||
section="schema",
|
||
tags=["schema", "json"],
|
||
meta={"id": spec.get("id"), "required": required, "properties": props, "version": version},
|
||
)
|
||
)
|
||
ex = REPOS / "zapier-platform" / "packages" / "schema" / "examples" / "definition.json"
|
||
if ex.exists():
|
||
rows.append(
|
||
rec(
|
||
kind="schema_json",
|
||
key="example-definition",
|
||
title="Minimal valid App definition.json",
|
||
summary="Official example App JSON used by schema tests.",
|
||
body=ex.read_text(),
|
||
source_path="zapier-platform/packages/schema/examples/definition.json",
|
||
source_repo="zapier/zapier-platform",
|
||
section="schema",
|
||
tags=["schema", "example"],
|
||
)
|
||
)
|
||
return rows
|
||
|
||
|
||
def _schema_to_ts(sch: dict | None, depth: int = 0) -> str:
|
||
if not sch or depth > 4:
|
||
return "unknown"
|
||
if "$ref" in sch:
|
||
return sch["$ref"].split("/")[-1]
|
||
if "anyOf" in sch:
|
||
return " | ".join(_schema_to_ts(s, depth + 1) for s in sch["anyOf"][:6])
|
||
if "oneOf" in sch:
|
||
return " | ".join(_schema_to_ts(s, depth + 1) for s in sch["oneOf"][:6])
|
||
t = sch.get("type")
|
||
if t == "array":
|
||
return f"Array<{_schema_to_ts(sch.get('items') or {}, depth + 1)}>"
|
||
if t == "object":
|
||
return "object"
|
||
if isinstance(t, list):
|
||
return " | ".join(str(x) for x in t)
|
||
return t or "unknown"
|
||
|
||
|
||
def ingest_openapi() -> list[dict]:
|
||
rows = []
|
||
files = [
|
||
ROOT / "raw" / "openapi" / "actions.yaml",
|
||
ROOT / "raw" / "openapi" / "connections.yaml",
|
||
ROOT / "raw" / "openapi" / "trigger-inbox.yaml",
|
||
ROOT / "raw" / "openapi" / "promotions-openapi.yaml",
|
||
ROOT / "raw" / "openapi" / "workflow-api-schema.json",
|
||
]
|
||
for path in files:
|
||
if not path.exists():
|
||
continue
|
||
text = path.read_text(encoding="utf-8", errors="replace")
|
||
try:
|
||
spec = yaml.safe_load(text)
|
||
except Exception as e:
|
||
print("skip", path, e)
|
||
continue
|
||
if not isinstance(spec, dict):
|
||
continue
|
||
title = (spec.get("info") or {}).get("title") or path.stem
|
||
api_ver = (spec.get("info") or {}).get("version")
|
||
servers = [s.get("url") for s in (spec.get("servers") or []) if isinstance(s, dict)]
|
||
paths = spec.get("paths") or {}
|
||
rows.append(
|
||
rec(
|
||
kind="openapi",
|
||
key=path.name,
|
||
title=f"{title} ({path.name})",
|
||
summary=(spec.get("info") or {}).get("description", "")[:400],
|
||
body=text,
|
||
source_url=f"https://docs.zapier.com/api-reference/specs/{path.name}" if path.suffix == ".yaml" else "https://api.zapier.com/schema",
|
||
section="api-reference",
|
||
tags=["openapi", "api"],
|
||
meta={"title": title, "version": api_ver, "servers": servers, "path_count": len(paths)},
|
||
)
|
||
)
|
||
for pth, item in paths.items():
|
||
if not isinstance(item, dict):
|
||
continue
|
||
for method, op in item.items():
|
||
if method.startswith("x-") or method in {"parameters", "servers"}:
|
||
continue
|
||
if not isinstance(op, dict):
|
||
continue
|
||
oid = op.get("operationId") or f"{method}_{pth}"
|
||
params = []
|
||
for prm in (item.get("parameters") or []) + (op.get("parameters") or []):
|
||
if isinstance(prm, dict):
|
||
params.append(
|
||
{
|
||
"name": prm.get("name"),
|
||
"in": prm.get("in"),
|
||
"required": bool(prm.get("required")),
|
||
"type": _schema_to_ts(prm.get("schema") or {}),
|
||
"description": prm.get("description") or "",
|
||
}
|
||
)
|
||
rb = ((op.get("requestBody") or {}).get("content") or {})
|
||
req_schema = None
|
||
if "application/json" in rb:
|
||
req_schema = (rb["application/json"] or {}).get("schema")
|
||
resps = {}
|
||
for code, r in (op.get("responses") or {}).items():
|
||
if isinstance(r, dict):
|
||
resps[str(code)] = r.get("description") or ""
|
||
typed_in = ["```ts", "type Input = {"]
|
||
for prm in params:
|
||
opt = "" if prm["required"] else "?"
|
||
typed_in.append(f" {prm['name']}{opt}: {prm['type']}; // in {prm['in']} — {prm['description'][:100]}")
|
||
if req_schema:
|
||
typed_in.append(f" body?: {_schema_to_ts(req_schema)}; // request JSON")
|
||
if len(typed_in) == 2:
|
||
typed_in.append(" // no parameters")
|
||
typed_in += ["};", "```"]
|
||
server = servers[0] if servers else "https://api.zapier.com"
|
||
body = "\n".join(
|
||
[
|
||
f"# `{oid}`",
|
||
"",
|
||
f"> {op.get('summary') or oid}",
|
||
"",
|
||
"## High-level description",
|
||
"",
|
||
op.get("description") or op.get("summary") or oid,
|
||
"",
|
||
"## Internals",
|
||
"",
|
||
f"`{method.upper()} {server}{pth}` from **{title}**. "
|
||
"Authenticate per spec (OAuth / partner JWT / embed secret). "
|
||
"This is a Zapier *public* API — not `z.request` inside a connector.",
|
||
"",
|
||
"## Typed inputs",
|
||
"",
|
||
*typed_in,
|
||
"",
|
||
"## Outputs",
|
||
"",
|
||
"```json",
|
||
json.dumps(resps, indent=2),
|
||
"```",
|
||
"",
|
||
f"- Tags: {', '.join(op.get('tags') or []) or '—'}",
|
||
f"- Security: {json.dumps(op.get('security') or spec.get('security') or [])}",
|
||
"",
|
||
"## Example",
|
||
"",
|
||
"```http",
|
||
f"{method.upper()} {pth} HTTP/1.1",
|
||
f"Host: {server.replace('https://','')}",
|
||
"Authorization: Bearer <token>",
|
||
"```",
|
||
]
|
||
)
|
||
rows.append(
|
||
rec(
|
||
kind="api_function",
|
||
key=str(oid),
|
||
title=f"{method.upper()} {pth} — {op.get('summary') or oid}",
|
||
summary=op.get("summary") or op.get("description") or oid,
|
||
body=body,
|
||
usage=f"{method.upper()} {pth}",
|
||
signature=f"{method.upper()} {pth}",
|
||
args=params,
|
||
source_url=f"https://docs.zapier.com/api-reference",
|
||
section="api-reference",
|
||
tags=["api", "openapi", (op.get("tags") or ["api"])[0].lower().replace(" ", "-")],
|
||
meta={
|
||
"method": method.upper(),
|
||
"path": pth,
|
||
"spec": path.name,
|
||
"api_title": title,
|
||
"servers": servers,
|
||
"request_schema": req_schema,
|
||
"responses": resps,
|
||
},
|
||
)
|
||
)
|
||
return rows
|
||
|
||
|
||
def ingest_news() -> list[dict]:
|
||
rows = []
|
||
news_root = ROOT / "raw" / "docs" / "integrations" / "news"
|
||
if not news_root.exists():
|
||
return rows
|
||
for md in sorted(news_root.rglob("*.md")):
|
||
text = md.read_text(encoding="utf-8", errors="replace")
|
||
rel = str(md.relative_to(news_root))
|
||
title = ""
|
||
for line in text.splitlines():
|
||
if line.startswith("#"):
|
||
title = re.sub(r"^#+\s*", "", line).strip()
|
||
break
|
||
rows.append(
|
||
rec(
|
||
kind="platform_news",
|
||
key=rel.replace(".md", ""),
|
||
title=title or rel,
|
||
summary=title or rel,
|
||
body=text,
|
||
source_url=f"https://docs.zapier.com/integrations/news/{rel.replace('.md','')}",
|
||
section="news",
|
||
tags=["news", "changelog", rel.split("/")[0] if "/" in rel else "news"],
|
||
meta={"year": rel.split("/")[0] if rel[:4].isdigit() else None},
|
||
)
|
||
)
|
||
return rows
|
||
|
||
|
||
SOURCE_GLOBS = ("*.js", "*.ts", "*.json", "*.md")
|
||
SKIP_NAMES = {"package-lock.json"}
|
||
|
||
|
||
def ingest_templates() -> list[dict]:
|
||
rows = []
|
||
base = REPOS / "zapier-platform" / "example-apps"
|
||
if not base.exists():
|
||
return rows
|
||
for d in sorted(p for p in base.iterdir() if p.is_dir()):
|
||
files = {}
|
||
for p in d.rglob("*"):
|
||
if not p.is_file():
|
||
continue
|
||
if p.name in SKIP_NAMES or "node_modules" in p.parts:
|
||
continue
|
||
if p.suffix not in {".js", ".ts", ".json", ".md"}:
|
||
continue
|
||
rel = str(p.relative_to(d))
|
||
try:
|
||
files[rel] = p.read_text(encoding="utf-8", errors="replace")
|
||
except Exception:
|
||
continue
|
||
auth = next((t for t in ("oauth2", "oauth1", "session-auth", "basic-auth", "digest-auth", "custom-auth") if t in d.name), None)
|
||
pattern = next((t for t in ("trigger", "create", "search", "rest-hooks", "files", "middleware", "resource", "callback", "line-items", "dynamic-dropdown") if t in d.name), "example")
|
||
parts = [f"# example-app `{d.name}`", "", f"Auth: `{auth or 'n/a'}` · Pattern: `{pattern}`", ""]
|
||
for rel, src in sorted(files.items()):
|
||
parts += [f"## `{rel}`", "", f"```{p.suffix.lstrip('.') if False else rel.split('.')[-1]}", src[:12000], "```", ""]
|
||
rows.append(
|
||
rec(
|
||
kind="template",
|
||
key=d.name,
|
||
title=f"Source template: {d.name}",
|
||
summary=f"Full example-app source ({auth or 'n/a'} / {pattern}).",
|
||
body="\n".join(parts),
|
||
source_url=f"https://github.com/zapier/zapier-platform/tree/main/example-apps/{d.name}",
|
||
source_repo="zapier/zapier-platform",
|
||
source_path=f"zapier-platform/example-apps/{d.name}",
|
||
section="templates",
|
||
tags=["template", "example", pattern] + ([auth] if auth else []),
|
||
related=[f"zapier-platform init --template {d.name}"] if auth or d.name in {"minimal", "files", "callback"} else [],
|
||
meta={"files": list(files), "auth": auth, "pattern": pattern, "local": f"repos/zapier-platform/example-apps/{d.name}"},
|
||
)
|
||
)
|
||
# golden local project
|
||
gold = ROOT / "scratch" / "oauth2-typescript"
|
||
if gold.exists():
|
||
srcs = {}
|
||
for p in (gold / "src").rglob("*"):
|
||
if p.is_file():
|
||
srcs[str(p.relative_to(gold))] = p.read_text(encoding="utf-8", errors="replace")
|
||
rows.append(
|
||
rec(
|
||
kind="template",
|
||
key="scratch-oauth2-typescript",
|
||
title="Golden local project: scratch/oauth2-typescript",
|
||
summary="CLI 19.1.0 `zapier-platform init --template oauth2 --language typescript`. Structurally valid after build.",
|
||
body="\n".join(
|
||
line
|
||
for k, v in sorted(srcs.items())
|
||
for line in (f"## `{k}`", "", "```ts", v, "```", "")
|
||
),
|
||
source_path="scratch/oauth2-typescript",
|
||
section="templates",
|
||
tags=["template", "golden", "oauth2", "typescript"],
|
||
related=["init", "validate", "z.request"],
|
||
meta={"cli_version": "19.1.0", "validated": True},
|
||
)
|
||
)
|
||
return rows
|
||
|
||
|
||
def main() -> int:
|
||
rows: list[dict] = []
|
||
print("schema…")
|
||
rows += ingest_schema()
|
||
print("openapi…")
|
||
rows += ingest_openapi()
|
||
print("news…")
|
||
rows += ingest_news()
|
||
print("templates…")
|
||
rows += ingest_templates()
|
||
counts: dict[str, int] = {}
|
||
for r in rows:
|
||
counts[r["kind"]] = counts.get(r["kind"], 0) + 1
|
||
out = ROOT / "raw" / "coding-set.jsonl"
|
||
with out.open("w") as f:
|
||
for r in rows:
|
||
f.write(json.dumps(r, ensure_ascii=False) + "\n")
|
||
print("wrote", len(rows), counts, "→", out)
|
||
|
||
rows.append(
|
||
rec(
|
||
kind="guide",
|
||
key="coding-set",
|
||
title="Coding set — schema, APIs, news, templates",
|
||
summary="After build-new-connector: schema_json, api_function, platform_news, template.",
|
||
body="""# Coding set
|
||
|
||
Start: `db.platform_reference.findOne({kind:'guide', key:'build-new-connector'})`
|
||
|
||
Then:
|
||
|
||
```js
|
||
db.platform_reference.findOne({kind:'schema_json', key:'exported-schema'})
|
||
db.platform_reference.find({kind:'api_function'}).sort({key:1})
|
||
db.platform_reference.find({kind:'platform_news'})
|
||
db.platform_reference.find({kind:'template', key:'oauth2-typescript'})
|
||
db.platform_reference.findOne({kind:'template', key:'scratch-oauth2-typescript'})
|
||
```
|
||
|
||
Local golden app: `scratch/oauth2-typescript` (CLI 19.1.0, `zapier-platform validate` clean after build).
|
||
CLIs: `~/.npm-global/bin/zapier-platform` and `zapier-sdk` (login still required for push/SDK).
|
||
""",
|
||
section="guide",
|
||
tags=["guide", "coding-set"],
|
||
)
|
||
)
|
||
|
||
uri = os.environ.get("ZAPIER_MONGO_URI") or os.environ.get("MDB_MCP_CONNECTION_STRING")
|
||
if not uri:
|
||
print("no mongo uri")
|
||
return 0
|
||
from pymongo import MongoClient, ReplaceOne
|
||
|
||
col = MongoClient(uri).get_database("zapier")["platform_reference"]
|
||
ops = [ReplaceOne({"_id": r["_id"]}, r, upsert=True) for r in rows]
|
||
for i in range(0, len(ops), 250):
|
||
col.bulk_write(ops[i : i + 250], ordered=False)
|
||
print("mongo kinds", {k: col.count_documents({"kind": k}) for k in ("schema_json", "api_function", "platform_news", "template", "schema_type")})
|
||
col.database["meta"].update_one(
|
||
{"_id": "ingest"},
|
||
{"$set": {"coding_set_ingested_at": NOW}},
|
||
upsert=True,
|
||
)
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|