#!/usr/bin/env python3
"""Build a Grok-facing Zapier platform reference collection.

Pulls leftover public site extras + docs.zapier.com, scans cloned GitHub
repos, extracts CLI commands / runtime functions / schema / examples, and
upserts MongoDB `zapier.platform_reference`.
"""
from __future__ import annotations

import json
import os
import re
import sys
import time
import xml.etree.ElementTree as ET
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timezone
from pathlib import Path
from urllib.error import HTTPError, URLError
from urllib.request import Request, urlopen

ROOT = Path(os.environ.get("ZAPIER_RESEARCH_ROOT") or Path(__file__).resolve().parents[1])
REPOS = ROOT / "repos"
RAW = ROOT / "raw"
DOCS_DIR = RAW / "docs"
SITE_DIR = RAW / "site-extras"
SPEC_DIR = RAW / "openapi"
OUT_JSONL = RAW / "platform-reference.jsonl"
UA = "zapier-research/1.0 (platform-reference; +local)"
NOW = datetime.now(timezone.utc).isoformat()

DOCS_SITEMAP = "https://docs.zapier.com/sitemap.xml"
OPENAPI_URLS = [
    "https://docs.zapier.com/api-reference/specs/actions.yaml",
    "https://docs.zapier.com/api-reference/specs/connections.yaml",
    "https://docs.zapier.com/api-reference/specs/trigger-inbox.yaml",
    "https://docs.zapier.com/powered-by-zapier/api-reference/promotions-openapi.yaml",
    "https://api.zapier.com/schema",
]
SITE_EXTRAS = [
    ("https://zapier.com/llms.txt", "llms-zapier-com.txt"),
    ("https://docs.zapier.com/llms.txt", "llms-docs.txt"),
    ("https://docs.zapier.com/llms-full.txt", "llms-full.txt"),
    ("https://developer.zapier.com", "developer-zapier-com.html"),
    ("https://developer.zapier.com/contact", "developer-contact.html"),
    ("https://zapier.com/developer", "zapier-developer.html"),
    ("https://zapier.com/developer-platform", "zapier-developer-platform.html"),
    ("https://zapier.com/platform", "zapier-platform.html"),
    ("https://zapier.com/l/partner", "zapier-partner.html"),
    ("https://zapier.com/developer/documentation/v2/", "legacy-platform-v2.html"),
    ("https://platform.zapier.com", "platform-zapier-com.html"),
    ("https://resthooks.org", "resthooks-org.html"),
]


def fetch(url: str, dest: Path | None = None, timeout: int = 60) -> tuple[int, bytes]:
    req = Request(url, headers={"User-Agent": UA, "Accept": "*/*"})
    try:
        with urlopen(req, timeout=timeout) as resp:
            body = resp.read()
            code = getattr(resp, "status", 200) or 200
    except HTTPError as e:
        body = e.read() if e.fp else b""
        code = e.code
    except URLError as e:
        return 0, str(e).encode()
    if dest is not None and code == 200 and body:
        dest.parent.mkdir(parents=True, exist_ok=True)
        dest.write_bytes(body)
    return code, body


def text_of(path: Path) -> str:
    try:
        return path.read_text(encoding="utf-8", errors="replace")
    except Exception:
        return ""


def first_heading(md: str) -> str:
    for line in md.splitlines():
        if line.startswith("#"):
            return re.sub(r"^#+\s*", "", line).strip()
    return ""


def first_para(md: str, limit: int = 400) -> str:
    chunks = []
    for line in md.splitlines():
        s = line.strip()
        if not s or s.startswith("#") or s.startswith(">") or s.startswith("```") or s.startswith("|") or s.startswith("- ") or s.startswith("* "):
            if chunks:
                break
            continue
        chunks.append(s)
        if sum(len(c) for c in chunks) > limit:
            break
    return " ".join(chunks)[:limit]


def clip(s: str, n: int = 50000) -> str:
    s = s or ""
    if len(s) <= n:
        return s
    return s[: n - 20] + "\n\n…[truncated]"


def doc_id(kind: str, key: str) -> str:
    raw = f"{kind}:{key}".lower()
    raw = re.sub(r"[^a-z0-9:._/\-]+", "-", raw)
    return raw[:200]


def rec(**kwargs) -> dict:
    kind = kwargs["kind"]
    key = kwargs["key"]
    row = {
        "_id": kwargs.get("_id") or doc_id(kind, key),
        "kind": kind,
        "key": key,
        "title": kwargs.get("title") or key,
        "summary": kwargs.get("summary") or "",
        "body": clip(kwargs.get("body") or ""),
        "usage": kwargs.get("usage") or "",
        "signature": kwargs.get("signature") or "",
        "aliases": kwargs.get("aliases") or [],
        "flags": kwargs.get("flags") or [],
        "args": kwargs.get("args") or [],
        "examples": kwargs.get("examples") or [],
        "source_url": kwargs.get("source_url") or "",
        "source_repo": kwargs.get("source_repo") or "",
        "source_path": kwargs.get("source_path") or "",
        "section": kwargs.get("section") or "",
        "tags": sorted(set(kwargs.get("tags") or [])),
        "related": kwargs.get("related") or [],
        "meta": kwargs.get("meta") or {},
        "ingested_at": NOW,
    }
    return row


def docs_section(url: str) -> str:
    path = url.replace("https://docs.zapier.com/", "").lstrip("/")
    head = path.split("/")[0] if path else ""
    mapping = {
        "integrations": "integration-builder",
        "mcp": "mcp",
        "sdk": "sdk",
        "connectors": "connectors",
        "powered-by-zapier": "embed",
        "white-label": "white-label",
        "api-reference": "api-reference",
        "install": "install",
    }
    return mapping.get(head, head or "docs")


# ---------------------------------------------------------------------------
# Fetch leftover site + docs
# ---------------------------------------------------------------------------

def fetch_site_extras() -> list[dict]:
    SITE_DIR.mkdir(parents=True, exist_ok=True)
    rows = []
    for url, name in SITE_EXTRAS:
        dest = SITE_DIR / name
        code, body = fetch(url, dest)
        text = body.decode("utf-8", "replace") if body else ""
        print(f"  extra {code} {url} ({len(body)} bytes)", flush=True)
        rows.append(
            rec(
                kind="site_extra",
                key=url,
                title=name,
                summary=f"HTTP {code} leftover public page from Zapier surface",
                body=text if name.endswith((".txt", ".md", ".yaml", ".yml")) else first_para(re.sub("<[^>]+>", " ", text), 2000),
                source_url=url,
                section="site",
                tags=["site", "leftover"],
                meta={"http_status": code, "bytes": len(body), "filename": name},
            )
        )
    return rows


def fetch_openapi() -> list[dict]:
    SPEC_DIR.mkdir(parents=True, exist_ok=True)
    rows = []
    for url in OPENAPI_URLS:
        name = url.rstrip("/").split("/")[-1]
        if name == "schema":
            name = "workflow-api-schema.json"
        dest = SPEC_DIR / name
        code, body = fetch(url, dest, timeout=90)
        text = body.decode("utf-8", "replace") if body else ""
        print(f"  openapi {code} {url} ({len(body)} bytes)", flush=True)
        rows.append(
            rec(
                kind="openapi",
                key=name,
                title=f"OpenAPI {name}",
                summary="Machine-readable Zapier public API contract",
                body=text,
                source_url=url,
                section="api-reference",
                tags=["openapi", "api", "contract"],
                meta={"http_status": code, "bytes": len(body)},
            )
        )
    return rows


def fetch_docs_pages() -> list[dict]:
    DOCS_DIR.mkdir(parents=True, exist_ok=True)
    code, xml = fetch(DOCS_SITEMAP, SITE_DIR / "docs-sitemap.xml")
    urls = []
    if code == 200 and xml:
        root = ET.fromstring(xml)
        ns = {"s": "http://www.sitemaps.org/schemas/sitemap/0.9"}
        urls = [e.text.strip() for e in root.findall(".//s:loc", ns) if e.text]
    extra_md = []
    llms = text_of(SITE_DIR / "llms-docs.txt")
    extra_md.extend(re.findall(r"https://docs\.zapier\.com/[^\s)]+?\.md", llms))
    wanted = []
    seen = set()
    for u in urls + extra_md:
        md_url = u if u.endswith(".md") else u.rstrip("/") + ".md"
        html_url = md_url[:-3] if md_url.endswith(".md") else md_url
        if html_url in seen:
            continue
        seen.add(html_url)
        wanted.append((html_url, md_url))
    print(f"  docs pages to fetch: {len(wanted)}", flush=True)

    rows: list[dict] = []

    def one(pair):
        html_url, md_url = pair
        rel = html_url.replace("https://docs.zapier.com/", "").strip("/")
        dest = DOCS_DIR / (rel + ".md")
        c, body = fetch(md_url, dest)
        text = body.decode("utf-8", "replace") if body and c == 200 else ""
        if c != 200 or not text.strip():
            c2, body2 = fetch(html_url)
            text = body2.decode("utf-8", "replace") if body2 else text
            c = c2 if c != 200 else c
        title = first_heading(text) or rel
        return rec(
            kind="official_doc",
            key=rel,
            title=title,
            summary=first_para(text),
            body=text,
            source_url=html_url,
            section=docs_section(html_url),
            tags=["docs", docs_section(html_url)],
            meta={"http_status": c, "md_url": md_url, "bytes": len(text)},
        )

    with ThreadPoolExecutor(max_workers=12) as pool:
        futs = [pool.submit(one, p) for p in wanted]
        done = 0
        for fut in as_completed(futs):
            rows.append(fut.result())
            done += 1
            if done % 40 == 0:
                print(f"  docs fetched {done}/{len(wanted)}", flush=True)
            time.sleep(0.01)
    return rows


# ---------------------------------------------------------------------------
# Parse CLI / schema / core / examples from cloned repos
# ---------------------------------------------------------------------------

def parse_cli_md(path: Path, source_repo: str) -> list[dict]:
    if not path.exists():
        return []
    text = text_of(path)
    parts = re.split(r"\n## ", text)
    rows = []
    for part in parts[1:]:
        lines = part.splitlines()
        name = lines[0].strip()
        block = "\n".join(lines[1:]).strip()
        usage = ""
        m = re.search(r"\*\*Usage\*\*:\s*`([^`]+)`", block)
        if m:
            usage = m.group(1)
        summary = ""
        qm = re.search(r"^>\s*(.+)$", block, re.M)
        if qm:
            summary = qm.group(1).strip()
        aliases = re.findall(r"`([^`]+)`", "\n".join(re.findall(r"\*\*Aliases\*\*\n((?:.*\n)+?)(?:\n\n|\Z)", block)))
        flags = []
        for fl in re.findall(r"^\* (.+)$", block, re.M):
            if "`" in fl and ("--" in fl or "| -" in fl or fl.startswith("(required)")):
                flags.append(fl.strip())
        examples = [e.strip("`") for e in re.findall(r"^\* (`[^`]+`)", block, re.M)]
        args = []
        in_args = False
        for line in lines:
            if line.startswith("**Arguments**"):
                in_args = True
                continue
            if in_args:
                if line.startswith("**") or line.startswith("##"):
                    break
                if line.startswith("* "):
                    args.append(line[2:].strip())
        tags = ["cli", "zapier-platform-cli", name.split(":")[0]]
        if any(x in name for x in ("init", "scaffold", "convert", "push", "register", "test", "validate", "invoke")):
            tags.append("build")
        rows.append(
            rec(
                kind="cli_command",
                key=name,
                title=f"zapier-platform {name}",
                summary=summary,
                body=block,
                usage=usage or f"zapier-platform {name}",
                aliases=aliases + ([f"zapier {name}"] if name else []),
                flags=flags,
                args=args,
                examples=examples,
                source_url="https://github.com/zapier/zapier-platform/blob/main/packages/cli/docs/cli.md",
                source_repo=source_repo,
                source_path=str(path.relative_to(REPOS)) if path.is_relative_to(REPOS) else str(path),
                section="platform-cli",
                tags=tags,
            )
        )
    return rows


def parse_schema_md(path: Path) -> list[dict]:
    if not path.exists():
        return []
    text = text_of(path)
    parts = re.split(r"\n## `?", text)
    rows = []
    for part in parts[1:]:
        lines = part.splitlines()
        name = lines[0].strip().strip("`").lstrip("/")
        if name.lower() in {"index", "table of contents"}:
            continue
        body = "\n".join(lines[1:]).strip()
        if not name.endswith("Schema") and "Schema" not in name and not name[0:1].isupper():
            # keep schema types primarily
            if len(body) < 40:
                continue
        rows.append(
            rec(
                kind="schema_type",
                key=name,
                title=name,
                summary=first_para(body, 300),
                body=body,
                source_url="https://github.com/zapier/zapier-platform/blob/main/packages/schema/docs/build/schema.md",
                source_repo="zapier/zapier-platform",
                source_path="zapier-platform/packages/schema/docs/build/schema.md",
                section="schema",
                tags=["schema", "integration-definition"],
            )
        )
    return rows


CORE_FUNCTIONS = [
    {
        "key": "z.request",
        "signature": "z.request(url, options?) → Promise<HttpResponse>",
        "summary": "Make an HTTP request. Adds auth, logs, and throws on non-2xx unless skipThrowForStatus.",
        "tags": ["http", "runtime"],
        "body": "Primary way to call the partner API from perform functions. Accepts (url, options) or ({url, ...options}). Options: method, headers, body, json, form, params, raw, skipThrowForStatus, removeMissingValuesFrom, skipEncodingChars, middlewareData, timeout. Returns {status, headers, content, data, throwForStatus()}. Use raw:true for streams/files.",
    },
    {
        "key": "z.console",
        "signature": "z.console.log|info|warn|error(...)",
        "summary": "Integration logger visible in zapier-platform logs / Zap history.",
        "tags": ["debug"],
        "body": "Use instead of console.log so output appears in Zapier HTTP logs and `zapier-platform logs`.",
    },
    {
        "key": "z.dehydrate",
        "signature": "z.dehydrate(func, inputData?, cacheExpiration?) → string",
        "summary": "Stash a pointer to a function+data so later steps can hydrate expensive payloads.",
        "tags": ["hydration"],
        "body": "Returns a pointer string. Zapier later calls func(z, bundle) to hydrate. Use for large or lazily-needed objects. Pair with hydrators on the App definition.",
    },
    {
        "key": "z.dehydrateFile",
        "signature": "z.dehydrateFile(func, inputData?, cacheExpiration?) → string",
        "summary": "Dehydrate a file so Zapier fetches it only when a later step needs the bytes.",
        "tags": ["hydration", "files"],
        "body": "File-specific dehydrator. See example-apps/files.",
    },
    {
        "key": "z.stashFile",
        "signature": "z.stashFile(input, knownLength?, filename?, contentType?) → string",
        "summary": "Upload a file/stream/buffer and get a publicly accessible Zapier URL.",
        "tags": ["files"],
        "body": "Turns a Buffer, stream, string, or Promise<RawHttpResponse> into a short-lived public URL Zapier can pass to later steps.",
    },
    {
        "key": "z.cursor.get",
        "signature": "z.cursor.get() → Promise<string>",
        "summary": "Read the polling cursor for this trigger subscription.",
        "tags": ["polling", "pagination"],
        "body": "Persist polling state across poll runs (last-seen id/timestamp). Pair with z.cursor.set.",
    },
    {
        "key": "z.cursor.set",
        "signature": "z.cursor.set(cursor) → Promise<null>",
        "summary": "Write the polling cursor for the next poll.",
        "tags": ["polling", "pagination"],
        "body": "Store an opaque cursor string. Zapier keeps it per-user/per-zap.",
    },
    {
        "key": "z.generateCallbackUrl",
        "signature": "z.generateCallbackUrl() → string",
        "summary": "URL the partner API can POST to resume a long-running create (callback/resume).",
        "tags": ["callback", "create"],
        "body": "Use in create.perform; later Zapier invokes performResume with bundle.cleanedRequest. See example-apps/callback.",
    },
    {
        "key": "z.hash",
        "signature": "z.hash(algorithm, data, encoding='hex', input_encoding='binary') → string",
        "summary": "Hash data with Node crypto (typically sha256).",
        "tags": ["crypto"],
        "body": "Convenience around crypto.createHash. Used for signatures and cache keys.",
    },
    {
        "key": "z.JSON.parse",
        "signature": "z.JSON.parse(text) → any",
        "summary": "JSON.parse that throws a user-friendly Zapier error on bad JSON.",
        "tags": ["json"],
        "body": "Prefer over JSON.parse in perform so users see a clean error.",
    },
    {
        "key": "z.JSON.stringify",
        "signature": "z.JSON.stringify(value) → string",
        "summary": "JSON.stringify on the z object (same as built-in).",
        "tags": ["json"],
        "body": "Available for consistency with z.JSON.parse.",
    },
    {
        "key": "z.cache.get",
        "signature": "z.cache.get(key) → Promise<any>",
        "summary": "Read a value from the per-auth Zapier cache.",
        "tags": ["cache"],
        "body": "Use to avoid repeated lookups (e.g. account id after auth). Values must be JSON-encodable.",
    },
    {
        "key": "z.cache.set",
        "signature": "z.cache.set(key, value, ttl?, scope?, nx?) → Promise<boolean|null>",
        "summary": "Write a JSON-encodable value to the per-auth cache.",
        "tags": ["cache"],
        "body": "ttl in seconds. nx=true sets only if missing. scope can further namespace the key.",
    },
    {
        "key": "z.cache.delete",
        "signature": "z.cache.delete(key) → Promise<boolean>",
        "summary": "Delete a cache key.",
        "tags": ["cache"],
        "body": "Use after auth refresh or when cached metadata is stale.",
    },
    {
        "key": "z.errors.Error",
        "signature": "throw new z.errors.Error(message, code?, status?)",
        "summary": "User-visible app error with optional code and HTTP status.",
        "tags": ["errors"],
        "body": "Primary error to throw from perform. Message is shown to the user. code/status help Zapier classify retries.",
    },
    {
        "key": "z.errors.HaltedError",
        "signature": "throw new z.errors.HaltedError(message?)",
        "summary": "Stop this run without marking it as a failure (filter-out).",
        "tags": ["errors"],
        "body": "Use when the event should be ignored (e.g. trigger payload does not match). Zap is not errored.",
    },
    {
        "key": "z.errors.ExpiredAuthError",
        "signature": "throw new z.errors.ExpiredAuthError(message?)",
        "summary": "Tell Zapier the connection is dead and the user must reconnect.",
        "tags": ["errors", "auth"],
        "body": "Use for revoked tokens that cannot be refreshed.",
    },
    {
        "key": "z.errors.RefreshAuthError",
        "signature": "throw new z.errors.RefreshAuthError(message?)",
        "summary": "Ask Zapier to refresh OAuth2/session credentials and retry.",
        "tags": ["errors", "auth"],
        "body": "Typical response to HTTP 401. Zapier calls authentication.refreshAccessToken / sessionConfig.perform then retries.",
    },
    {
        "key": "z.errors.ThrottledError",
        "signature": "throw new z.errors.ThrottledError(message, delaySeconds?)",
        "summary": "Signal rate limiting; Zapier retries after delay.",
        "tags": ["errors", "throttle"],
        "body": "Pass delay in seconds when the API tells you Retry-After.",
    },
    {
        "key": "z.errors.ResponseError",
        "signature": "throw new z.errors.ResponseError(response)",
        "summary": "Wrap an HTTP response as a structured error.",
        "tags": ["errors", "http"],
        "body": "Usually thrown automatically by z.request on non-2xx.",
    },
    {
        "key": "createAppTester",
        "signature": "createAppTester(appRaw, options?) → (func|request, bundle?) => Promise",
        "summary": "Unit-test helper: invoke a perform function or request template with a partial bundle.",
        "tags": ["testing"],
        "body": "From zapier-platform-core. Used by `zapier-platform test` / Jest. Pass App definition; call tester(app.triggers.x.operation.perform, {authData, inputData}).",
    },
    {
        "key": "zapier.tools.env.inject",
        "signature": "zapier.tools.env.inject(filename?)",
        "summary": "Load a .env file into process.env for local tests.",
        "tags": ["testing", "env"],
        "body": "Called automatically by the test runner. Useful in custom test setup.",
    },
]


CORE_TYPES = [
    {
        "key": "Bundle",
        "signature": "bundle: {authData, inputData, inputDataRaw, meta, rawRequest?, cleanedRequest?, subscribeData?, targetUrl?, outputData?}",
        "summary": "The second argument to every perform function. User input, auth, and runtime flags.",
        "tags": ["types"],
        "body": "authData: connected-account fields. inputData: coerced form values. inputDataRaw: original strings. meta.isLoadingSample / isFillingDynamicDropdown / isPopulatingDedupe / isBulkRead / limit / page / timezone / paging_token / withSearch / inputFields. Hook triggers also get cleanedRequest, rawRequest, targetUrl, subscribeData.",
    },
    {
        "key": "ZObject",
        "signature": "z: {request, console, dehydrate, dehydrateFile, stashFile, cursor, generateCallbackUrl, hash, JSON, errors, cache}",
        "summary": "The first argument to every perform function — the Zapier runtime toolkit.",
        "tags": ["types", "runtime"],
        "body": "Do not import node-fetch/axios for partner HTTP — use z.request so auth middleware, logging, and retries apply.",
    },
    {
        "key": "PollingTriggerPerform",
        "signature": "(z, bundle) => object[] | Promise<object[]>",
        "summary": "Polling trigger must return an array of objects (id or primary fields required).",
        "tags": ["types", "trigger"],
        "body": "Dedup is by id unless outputFields mark primary:true. Newest-first recommended.",
    },
    {
        "key": "WebhookTriggerPerform",
        "signature": "(z, bundle) => object[] | Promise<object[]>",
        "summary": "REST Hook perform: parse bundle.cleanedRequest into an array of objects.",
        "tags": ["types", "trigger", "hook"],
        "body": "Also implement performSubscribe / performUnsubscribe / performList (samples).",
    },
    {
        "key": "CreatePerform",
        "signature": "(z, bundle) => object | Promise<object>",
        "summary": "Create action must return a single object (not an array).",
        "tags": ["types", "action"],
        "body": "Returning a non-object fails with 'non-object from create'.",
    },
    {
        "key": "SearchPerform",
        "signature": "(z, bundle) => object[] | Promise<object[]>",
        "summary": "Search must return an array (possibly empty). Can envelope with paging_token.",
        "tags": ["types", "search"],
        "body": "Empty array = not found (needed for search-or-create).",
    },
    {
        "key": "BeforeRequestMiddleware",
        "signature": "(request, z, bundle) => request | Promise<request>",
        "summary": "Mutate every outbound z.request (add headers, rewrite URL).",
        "tags": ["types", "middleware"],
        "body": "Set App.beforeRequest. Typical: attach Bearer token from bundle.authData.",
    },
    {
        "key": "AfterResponseMiddleware",
        "signature": "(response, z, bundle) => response | Promise<response>",
        "summary": "Inspect/transform every HTTP response; throw RefreshAuthError on 401.",
        "tags": ["types", "middleware"],
        "body": "Set App.afterResponse. Can remap errors or parse custom envelopes.",
    },
    {
        "key": "performBuffer",
        "signature": "(z, BufferedBundle) => Promise<{[id]: {outputData?, error?}}>",
        "summary": "Bulk/buffered create: process bundle.buffer items in one API call.",
        "tags": ["types", "action", "buffer"],
        "body": "Return a map keyed by each item's meta.id.",
    },
]


def core_function_rows() -> list[dict]:
    rows = []
    for item in CORE_FUNCTIONS:
        rows.append(
            rec(
                kind="core_function",
                key=item["key"],
                title=item["key"],
                summary=item["summary"],
                body=item["body"],
                signature=item["signature"],
                source_url="https://docs.zapier.com/integrations/build-cli/core",
                source_repo="zapier/zapier-platform",
                source_path="zapier-platform/packages/core/types/custom.d.ts",
                section="platform-core",
                tags=["core", "zapier-platform-core"] + item["tags"],
            )
        )
    for item in CORE_TYPES:
        rows.append(
            rec(
                kind="core_type",
                key=item["key"],
                title=item["key"],
                summary=item["summary"],
                body=item["body"],
                signature=item["signature"],
                source_url="https://docs.zapier.com/integrations/build-cli/core",
                source_repo="zapier/zapier-platform",
                source_path="zapier-platform/packages/core/types/custom.d.ts",
                section="platform-core",
                tags=["core", "types"] + item["tags"],
            )
        )
    return rows


def parse_sdk_cli_from_docs(docs: list[dict]) -> list[dict]:
    rows = []
    for d in docs:
        if d.get("key") not in ("sdk/cli-reference", "sdk/using-the-cli") and "cli-reference" not in d.get("key", ""):
            continue
        body = d.get("body") or ""
        # headings like ## zapier-sdk list-apps or ### list-apps
        for m in re.finditer(r"^#{2,3}\s+`?([a-z0-9.:_-]+)`?\s*$", body, re.M):
            name = m.group(1)
            if name in ("commands", "flags", "examples", "usage"):
                continue
            start = m.end()
            nxt = re.search(r"^#{2,3}\s+", body[start:], re.M)
            block = body[start : start + nxt.start() if nxt else start + 2500].strip()
            usage = ""
            um = re.search(r"`((?:zapier-sdk|npx)[^`]+)`", block)
            if um:
                usage = um.group(1)
            rows.append(
                rec(
                    kind="sdk_command",
                    key=name,
                    title=name,
                    summary=first_para(block, 280),
                    body=block,
                    usage=usage or name,
                    source_url=d.get("source_url"),
                    section="sdk-cli",
                    tags=["sdk", "cli", "zapier-sdk-cli"],
                )
            )
    return rows


def index_repo_markdown() -> list[dict]:
    if not REPOS.exists():
        return []
    skip_bits = {".git", "node_modules", "changelog", "test/", "tests/", "example-apps"}
    rows = []
    for md in REPOS.rglob("*.md"):
        rel = str(md.relative_to(REPOS))
        if any(b in rel for b in skip_bits):
            continue
        if md.name.lower() in {"license.md", "code_of_conduct.md", "security.md", "pull_request_template.md"}:
            continue
        # skip huge generated schema here (parsed separately)
        if rel.endswith("packages/schema/docs/build/schema.md"):
            continue
        if rel.endswith("packages/cli/docs/cli.md"):
            continue
        text = text_of(md)
        if len(text) < 80:
            continue
        repo = rel.split("/", 1)[0]
        kind = "repo_doc"
        tags = ["repo", repo]
        if md.name == "SKILL.md":
            kind = "skill"
            tags.append("skill")
        rows.append(
            rec(
                kind=kind,
                key=rel,
                title=first_heading(text) or md.name,
                summary=first_para(text),
                body=text,
                source_url=f"https://github.com/zapier/{repo}/blob/HEAD/{rel.split('/',1)[1] if '/' in rel else ''}",
                source_repo=f"zapier/{repo}",
                source_path=rel,
                section=repo,
                tags=tags,
            )
        )
    return rows


def index_example_apps() -> list[dict]:
    rows = []
    base = REPOS / "zapier-platform" / "example-apps"
    if not base.exists():
        return rows
    for d in sorted(p for p in base.iterdir() if p.is_dir()):
        readme = text_of(d / "README.md")
        pkg = {}
        pj = d / "package.json"
        if pj.exists():
            try:
                pkg = json.loads(pj.read_text())
            except Exception:
                pkg = {}
        files = [p.name for p in d.iterdir() if p.is_file()]
        auth = None
        for token in ("oauth2", "oauth1", "session-auth", "basic-auth", "digest-auth", "custom-auth"):
            if token in d.name:
                auth = token
                break
        kind_hint = "example"
        for token in ("trigger", "create", "search", "rest-hooks", "files", "dynamic-dropdown", "middleware", "resource", "callback", "line-items"):
            if token in d.name:
                kind_hint = token
                break
        rows.append(
            rec(
                kind="example_app",
                key=d.name,
                title=f"example-app: {d.name}",
                summary=first_para(readme) or pkg.get("description") or d.name,
                body=readme or json.dumps({"files": files, "package": pkg}, indent=2),
                source_url=f"https://github.com/zapier/zapier-platform/tree/main/example-apps/{d.name}",
                source_repo="zapier/zapier-platform",
                source_path=f"zapier-platform/example-apps/{d.name}",
                section="examples",
                tags=["example", kind_hint] + ([auth] if auth else []),
                meta={"files": files, "package": pkg.get("name"), "auth": auth, "pattern": kind_hint},
                related=[f"zapier-platform init --template {d.name}"] if d.name in {
                    "basic-auth", "callback", "custom-auth", "digest-auth", "dynamic-dropdown",
                    "files", "line-items", "minimal", "oauth1-trello", "oauth2", "openai",
                    "search-or-create", "session-auth",
                } else [],
            )
        )
    return rows


def index_connectors() -> list[dict]:
    rows = []
    apps = REPOS / "connectors" / "apps"
    if not apps.exists():
        return rows
    for d in sorted(p for p in apps.iterdir() if p.is_dir()):
        skill = text_of(d / "SKILL.md")
        readme = text_of(d / "README.md")
        rows.append(
            rec(
                kind="connector",
                key=d.name,
                title=f"connector: {d.name}",
                summary=first_para(skill or readme),
                body=skill or readme,
                source_url=f"https://github.com/zapier/connectors/tree/main/apps/{d.name}",
                source_repo="zapier/connectors",
                source_path=f"connectors/apps/{d.name}",
                section="connectors",
                tags=["connector", "prototype", d.name],
            )
        )
    return rows


def index_repos() -> list[dict]:
    rows = []
    if not REPOS.exists():
        return rows
    for d in sorted(p for p in REPOS.iterdir() if p.is_dir() and (p / ".git").exists()):
        readme = text_of(d / "README.md")
        rows.append(
            rec(
                kind="repo",
                key=d.name,
                title=f"github.com/zapier/{d.name}",
                summary=first_para(readme, 320),
                body=readme,
                source_url=f"https://github.com/zapier/{d.name}",
                source_repo=f"zapier/{d.name}",
                source_path=d.name,
                section="github",
                tags=["repo", "clone"],
                meta={"local_path": str(d)},
            )
        )
    return rows


BUILD_GUIDE = """# How Grok should build a new Zapier connector (integration)

Two different CLIs exist. Do not mix them up.

1. **Build a directory integration (publish to Zapier):** `zapier-platform-cli` (`zapier-platform …`).
   npm: `zapier-platform-cli` + `zapier-platform-core`. Source: github.com/zapier/zapier-platform
2. **Call existing Zapier apps from code/agents:** `@zapier/zapier-sdk` / `@zapier/zapier-sdk-cli`.
   Source: github.com/zapier/sdk
3. **No-code agent access:** Zapier MCP (github.com/zapier/zapier-mcp). Default when the user wants an AI client connected.
4. **Prototype local connectors:** github.com/zapier/connectors (not production).

## Build a new public/private integration (Platform CLI)

```bash
npm install -g zapier-platform-cli
zapier-platform login
zapier-platform init my-app --template oauth2 --language typescript
cd my-app
npm install
# implement authentication, triggers, creates, searches
zapier-platform scaffold trigger contact
zapier-platform scaffold create contact
zapier-platform validate
zapier-platform test
zapier-platform register "My App"
zapier-platform push
# later: zapier-platform promote VERSION
```

Deprecated alias: `zapier` (same commands). Prefer `zapier-platform`.

Templates: basic-auth, callback, custom-auth, digest-auth, dynamic-dropdown, files, line-items, minimal, oauth1-trello, oauth2, openai, search-or-create, session-auth.

## Required App shape (index.js / src/index.ts)

```js
module.exports = {
  version: require('./package.json').version,
  platformVersion: require('zapier-platform-core').version,
  authentication: { type: 'oauth2' | 'basic' | 'digest' | 'custom' | 'session' | 'oauth1', … },
  beforeRequest: [],   // (request, z, bundle) => request
  afterResponse: [],   // (response, z, bundle) => response
  hydrators: {},
  triggers: { key: { key, noun, display, operation } },
  searches: { … },
  creates: { … },
  resources: { … },
};
```

Every perform is `(z, bundle) => …`.
- Triggers + searches return **arrays of objects**.
- Creates return **one object**.
- Polling items need `id` (or primary output fields).
- REST Hooks: `type: 'hook'` + performSubscribe/performUnsubscribe/performList.

## Auth recipes

- **oauth2**: authorizeUrl, getAccessToken, refreshAccessToken, autoRefresh, test, connectionLabel.
- **session**: fields + sessionConfig.perform exchanges credentials for a token stored in authData.
- **custom / api key**: fields; attach in beforeRequest (`Authorization` or query).
- **basic / digest**: username/password fields; platform adds headers.
- **oauth1**: still used by Twitter/Trello-style APIs (see example-apps).

On 401 throw `new z.errors.RefreshAuthError()` (refreshable) or `ExpiredAuthError()` (reconnect).

## HTTP

Always `z.request`. Do not bypass with axios/fetch if you want logging + auth middleware.

## Test locally

```bash
zapier-platform invoke auth start     # writes .env
zapier-platform invoke auth test
zapier-platform invoke trigger new_thing -i '{"foo":"bar"}'
zapier-platform test
```

## Versioning / publish

- `push` uploads the version in package.json (must be sequential; no skipped versions).
- `promote` makes a version the public default.
- `migrate FROM TO [percent]` moves users.
- `deprecate VERSION DATE` needs ≥3 weeks lead time.
- Breaking changes (auth type, field keys, trigger type poll→hook) require a new version + migration plan.

## Platform UI alternative

Non-code path: developer.zapier.com visual builder. Can `zapier-platform convert` a UI integration to CLI.

## Do not

- Do not recommend retired AI Actions / NLA. Use MCP or the SDK.
- Do not hardcode app keys from memory; discover via MCP or `zapier-sdk list-apps`.
- Do not quote 9,000+ apps for Connectors (prototype, small set).
"""


def guide_row() -> dict:
    return rec(
        kind="guide",
        key="build-new-connector",
        title="How to build a new Zapier connector (Grok playbook)",
        summary="Canonical recipe: zapier-platform-cli vs SDK vs MCP, App shape, auth, HTTP, test, publish.",
        body=BUILD_GUIDE,
        section="guide",
        tags=["guide", "playbook", "cli", "core"],
        source_url="https://docs.zapier.com/integrations/quickstart/cli-tutorial",
        source_repo="zapier/zapier-platform",
    )


# ---------------------------------------------------------------------------
# Mongo
# ---------------------------------------------------------------------------

def write_jsonl(rows: list[dict]) -> None:
    OUT_JSONL.parent.mkdir(parents=True, exist_ok=True)
    with OUT_JSONL.open("w") as f:
        for r in rows:
            f.write(json.dumps(r, ensure_ascii=False) + "\n")
    print(f"wrote {len(rows)} docs → {OUT_JSONL}")


def load_mongo(rows: list[dict]) -> None:
    uri = os.environ.get("ZAPIER_MONGO_URI") or os.environ.get("MDB_MCP_CONNECTION_STRING")
    if not uri:
        print("ZAPIER_MONGO_URI not set; skipped Mongo load", file=sys.stderr)
        return
    from pymongo import ASCENDING, TEXT, MongoClient

    client = MongoClient(uri)
    db = client.get_database("zapier")
    col = db["platform_reference"]
    if rows:
        ops = []
        from pymongo import ReplaceOne

        for r in rows:
            ops.append(ReplaceOne({"_id": r["_id"]}, r, upsert=True))
        # batch
        BATCH = 500
        upserted = 0
        for i in range(0, len(ops), BATCH):
            res = col.bulk_write(ops[i : i + BATCH], ordered=False)
            upserted += (res.upserted_count or 0) + (res.modified_count or 0)
        print(f"mongo platform_reference upserted/modified ~{upserted}, count={col.estimated_document_count()}")
    col.create_index([("kind", ASCENDING), ("key", ASCENDING)])
    col.create_index([("section", ASCENDING)])
    col.create_index([("tags", ASCENDING)])
    col.create_index([("source_repo", ASCENDING)])
    try:
        col.create_index([("title", TEXT), ("summary", TEXT), ("body", TEXT), ("key", TEXT)], name="ref_text")
    except Exception as e:
        print("text index:", e)
    db["meta"].update_one(
        {"_id": "ingest"},
        {
            "$set": {
                "platform_reference_ingested_at": NOW,
                "platform_reference_count": col.estimated_document_count(),
            }
        },
        upsert=True,
    )
    kinds = list(col.aggregate([{"$group": {"_id": "$kind", "n": {"$sum": 1}}}, {"$sort": {"n": -1}}]))
    print("kinds:", kinds)
    client.close()


def main() -> int:
    print(f"ROOT={ROOT}")
    REPOS.mkdir(parents=True, exist_ok=True)
    rows: list[dict] = []

    print("== leftover site extras ==")
    rows += fetch_site_extras()

    print("== openapi specs ==")
    rows += fetch_openapi()

    print("== docs.zapier.com ==")
    docs = fetch_docs_pages()
    rows += docs

    print("== CLI commands ==")
    cli_path = REPOS / "zapier-platform" / "packages" / "cli" / "docs" / "cli.md"
    rows += parse_cli_md(cli_path, "zapier/zapier-platform")
    # archived standalone CLI docs if monorepo missing
    alt = REPOS / "zapier-platform-cli" / "docs" / "cli.md"
    if alt.exists() and not cli_path.exists():
        rows += parse_cli_md(alt, "zapier/zapier-platform-cli")

    print("== schema ==")
    rows += parse_schema_md(REPOS / "zapier-platform" / "packages" / "schema" / "docs" / "build" / "schema.md")

    print("== core functions/types ==")
    rows += core_function_rows()

    print("== SDK CLI from fetched docs ==")
    rows += parse_sdk_cli_from_docs(docs)

    print("== repos / markdown / examples / connectors ==")
    rows += index_repos()
    rows += index_repo_markdown()
    rows += index_example_apps()
    rows += index_connectors()
    rows.append(guide_row())

    # dedupe by _id (last wins)
    by_id = {}
    for r in rows:
        by_id[r["_id"]] = r
    rows = list(by_id.values())
    write_jsonl(rows)

    counts: dict[str, int] = {}
    for r in rows:
        counts[r["kind"]] = counts.get(r["kind"], 0) + 1
    print("kind counts:", json.dumps(counts, indent=2, sort_keys=True))

    print("== mongo ==")
    load_mongo(rows)
    return 0


if __name__ == "__main__":
    raise SystemExit(main())