#!/usr/bin/env python3
"""Load remaining coding-set artifacts into zapier.platform_reference.

- exported-schema.json + definition example
- OpenAPI specs exploded to api_function
- Platform News 2025–2026
- example-app source as kind=template
"""
from __future__ import annotations

import json
import os
import re
import sys
from datetime import datetime, timezone
from pathlib import Path

ROOT = Path(os.environ.get("ZAPIER_RESEARCH_ROOT") or Path(__file__).resolve().parents[1])
REPOS = ROOT / "repos"
NOW = datetime.now(timezone.utc).isoformat()

try:
    import yaml
except ImportError:
    print("pyyaml required", file=sys.stderr)
    raise


def clip(s: str, n: int = 80000) -> str:
    s = s or ""
    return s if len(s) <= n else s[: n - 20] + "\n\n…[truncated]"


def rec(**kw) -> dict:
    kind, key = kw["kind"], kw["key"]
    _id = kw.get("_id") or f"{kind}:{key}".lower()
    _id = re.sub(r"[^a-z0-9:._/\-]+", "-", _id)[:200]
    return {
        "_id": _id,
        "kind": kind,
        "key": key,
        "title": kw.get("title") or key,
        "summary": kw.get("summary") or "",
        "body": clip(kw.get("body") or ""),
        "usage": kw.get("usage") or "",
        "signature": kw.get("signature") or "",
        "aliases": kw.get("aliases") or [],
        "flags": kw.get("flags") or [],
        "args": kw.get("args") or [],
        "examples": kw.get("examples") or [],
        "source_url": kw.get("source_url") or "",
        "source_repo": kw.get("source_repo") or "",
        "source_path": kw.get("source_path") or "",
        "section": kw.get("section") or "",
        "tags": sorted(set(kw.get("tags") or [])),
        "related": kw.get("related") or [],
        "meta": kw.get("meta") or {},
        "ingested_at": NOW,
    }


def ingest_schema() -> list[dict]:
    rows = []
    path = REPOS / "zapier-platform" / "packages" / "schema" / "exported-schema.json"
    if not path.exists():
        return rows
    data = json.loads(path.read_text())
    version = data.get("version") or "unknown"
    schemas = data.get("schemas") or {}
    rows.append(
        rec(
            kind="schema_json",
            key="exported-schema",
            title=f"zapier-platform-schema {version} (exported JSON)",
            summary="Machine-checkable JSON Schema bundle for App definitions.",
            body=json.dumps(data, indent=2),
            source_url="https://github.com/zapier/zapier-platform/blob/main/packages/schema/exported-schema.json",
            source_repo="zapier/zapier-platform",
            source_path="zapier-platform/packages/schema/exported-schema.json",
            section="schema",
            tags=["schema", "json", "platform"],
            meta={"platform_schema_version": version, "schema_count": len(schemas)},
        )
    )
    for name, spec in schemas.items():
        required = spec.get("required") or []
        props = list((spec.get("properties") or {}).keys())
        body = [
            f"# `{name}`",
            "",
            f"> {spec.get('description') or name}",
            "",
            "## High-level description",
            "",
            spec.get("description") or name,
            "",
            "## Internals",
            "",
            f"JSON Schema id `{spec.get('id')}` from zapier-platform-schema {version}. "
            "Used by `zapier-platform validate` / build.",
            "",
            "## Typed inputs (properties)",
            "",
            "```ts",
            f"// required: {', '.join(required) or '—'}",
        ]
        for p, ps in (spec.get("properties") or {}).items():
            ref = ps.get("$ref") or ps.get("type") or "any"
            opt = "?" if p not in required else ""
            body.append(f"  {p}{opt}: {ref}; // {(ps.get('description') or '')[:120]}")
        body += ["```", "", "## Schema", "", "```json", json.dumps(spec, indent=2), "```"]
        rows.append(
            rec(
                kind="schema_type",
                key=name,
                title=name,
                summary=spec.get("description") or name,
                body="\n".join(body),
                signature=f"{name} required=[{', '.join(required)}]",
                source_url="https://github.com/zapier/zapier-platform/blob/main/packages/schema/exported-schema.json",
                source_repo="zapier/zapier-platform",
                source_path="zapier-platform/packages/schema/exported-schema.json",
                section="schema",
                tags=["schema", "json"],
                meta={"id": spec.get("id"), "required": required, "properties": props, "version": version},
            )
        )
    ex = REPOS / "zapier-platform" / "packages" / "schema" / "examples" / "definition.json"
    if ex.exists():
        rows.append(
            rec(
                kind="schema_json",
                key="example-definition",
                title="Minimal valid App definition.json",
                summary="Official example App JSON used by schema tests.",
                body=ex.read_text(),
                source_path="zapier-platform/packages/schema/examples/definition.json",
                source_repo="zapier/zapier-platform",
                section="schema",
                tags=["schema", "example"],
            )
        )
    return rows


def _schema_to_ts(sch: dict | None, depth: int = 0) -> str:
    if not sch or depth > 4:
        return "unknown"
    if "$ref" in sch:
        return sch["$ref"].split("/")[-1]
    if "anyOf" in sch:
        return " | ".join(_schema_to_ts(s, depth + 1) for s in sch["anyOf"][:6])
    if "oneOf" in sch:
        return " | ".join(_schema_to_ts(s, depth + 1) for s in sch["oneOf"][:6])
    t = sch.get("type")
    if t == "array":
        return f"Array<{_schema_to_ts(sch.get('items') or {}, depth + 1)}>"
    if t == "object":
        return "object"
    if isinstance(t, list):
        return " | ".join(str(x) for x in t)
    return t or "unknown"


def ingest_openapi() -> list[dict]:
    rows = []
    files = [
        ROOT / "raw" / "openapi" / "actions.yaml",
        ROOT / "raw" / "openapi" / "connections.yaml",
        ROOT / "raw" / "openapi" / "trigger-inbox.yaml",
        ROOT / "raw" / "openapi" / "promotions-openapi.yaml",
        ROOT / "raw" / "openapi" / "workflow-api-schema.json",
    ]
    for path in files:
        if not path.exists():
            continue
        text = path.read_text(encoding="utf-8", errors="replace")
        try:
            spec = yaml.safe_load(text)
        except Exception as e:
            print("skip", path, e)
            continue
        if not isinstance(spec, dict):
            continue
        title = (spec.get("info") or {}).get("title") or path.stem
        api_ver = (spec.get("info") or {}).get("version")
        servers = [s.get("url") for s in (spec.get("servers") or []) if isinstance(s, dict)]
        paths = spec.get("paths") or {}
        rows.append(
            rec(
                kind="openapi",
                key=path.name,
                title=f"{title} ({path.name})",
                summary=(spec.get("info") or {}).get("description", "")[:400],
                body=text,
                source_url=f"https://docs.zapier.com/api-reference/specs/{path.name}" if path.suffix == ".yaml" else "https://api.zapier.com/schema",
                section="api-reference",
                tags=["openapi", "api"],
                meta={"title": title, "version": api_ver, "servers": servers, "path_count": len(paths)},
            )
        )
        for pth, item in paths.items():
            if not isinstance(item, dict):
                continue
            for method, op in item.items():
                if method.startswith("x-") or method in {"parameters", "servers"}:
                    continue
                if not isinstance(op, dict):
                    continue
                oid = op.get("operationId") or f"{method}_{pth}"
                params = []
                for prm in (item.get("parameters") or []) + (op.get("parameters") or []):
                    if isinstance(prm, dict):
                        params.append(
                            {
                                "name": prm.get("name"),
                                "in": prm.get("in"),
                                "required": bool(prm.get("required")),
                                "type": _schema_to_ts(prm.get("schema") or {}),
                                "description": prm.get("description") or "",
                            }
                        )
                rb = ((op.get("requestBody") or {}).get("content") or {})
                req_schema = None
                if "application/json" in rb:
                    req_schema = (rb["application/json"] or {}).get("schema")
                resps = {}
                for code, r in (op.get("responses") or {}).items():
                    if isinstance(r, dict):
                        resps[str(code)] = r.get("description") or ""
                typed_in = ["```ts", "type Input = {"]
                for prm in params:
                    opt = "" if prm["required"] else "?"
                    typed_in.append(f"  {prm['name']}{opt}: {prm['type']}; // in {prm['in']} — {prm['description'][:100]}")
                if req_schema:
                    typed_in.append(f"  body?: {_schema_to_ts(req_schema)}; // request JSON")
                if len(typed_in) == 2:
                    typed_in.append("  // no parameters")
                typed_in += ["};", "```"]
                server = servers[0] if servers else "https://api.zapier.com"
                body = "\n".join(
                    [
                        f"# `{oid}`",
                        "",
                        f"> {op.get('summary') or oid}",
                        "",
                        "## High-level description",
                        "",
                        op.get("description") or op.get("summary") or oid,
                        "",
                        "## Internals",
                        "",
                        f"`{method.upper()} {server}{pth}` from **{title}**. "
                        "Authenticate per spec (OAuth / partner JWT / embed secret). "
                        "This is a Zapier *public* API — not `z.request` inside a connector.",
                        "",
                        "## Typed inputs",
                        "",
                        *typed_in,
                        "",
                        "## Outputs",
                        "",
                        "```json",
                        json.dumps(resps, indent=2),
                        "```",
                        "",
                        f"- Tags: {', '.join(op.get('tags') or []) or '—'}",
                        f"- Security: {json.dumps(op.get('security') or spec.get('security') or [])}",
                        "",
                        "## Example",
                        "",
                        "```http",
                        f"{method.upper()} {pth} HTTP/1.1",
                        f"Host: {server.replace('https://','')}",
                        "Authorization: Bearer <token>",
                        "```",
                    ]
                )
                rows.append(
                    rec(
                        kind="api_function",
                        key=str(oid),
                        title=f"{method.upper()} {pth} — {op.get('summary') or oid}",
                        summary=op.get("summary") or op.get("description") or oid,
                        body=body,
                        usage=f"{method.upper()} {pth}",
                        signature=f"{method.upper()} {pth}",
                        args=params,
                        source_url=f"https://docs.zapier.com/api-reference",
                        section="api-reference",
                        tags=["api", "openapi", (op.get("tags") or ["api"])[0].lower().replace(" ", "-")],
                        meta={
                            "method": method.upper(),
                            "path": pth,
                            "spec": path.name,
                            "api_title": title,
                            "servers": servers,
                            "request_schema": req_schema,
                            "responses": resps,
                        },
                    )
                )
    return rows


def ingest_news() -> list[dict]:
    rows = []
    news_root = ROOT / "raw" / "docs" / "integrations" / "news"
    if not news_root.exists():
        return rows
    for md in sorted(news_root.rglob("*.md")):
        text = md.read_text(encoding="utf-8", errors="replace")
        rel = str(md.relative_to(news_root))
        title = ""
        for line in text.splitlines():
            if line.startswith("#"):
                title = re.sub(r"^#+\s*", "", line).strip()
                break
        rows.append(
            rec(
                kind="platform_news",
                key=rel.replace(".md", ""),
                title=title or rel,
                summary=title or rel,
                body=text,
                source_url=f"https://docs.zapier.com/integrations/news/{rel.replace('.md','')}",
                section="news",
                tags=["news", "changelog", rel.split("/")[0] if "/" in rel else "news"],
                meta={"year": rel.split("/")[0] if rel[:4].isdigit() else None},
            )
        )
    return rows


SOURCE_GLOBS = ("*.js", "*.ts", "*.json", "*.md")
SKIP_NAMES = {"package-lock.json"}


def ingest_templates() -> list[dict]:
    rows = []
    base = REPOS / "zapier-platform" / "example-apps"
    if not base.exists():
        return rows
    for d in sorted(p for p in base.iterdir() if p.is_dir()):
        files = {}
        for p in d.rglob("*"):
            if not p.is_file():
                continue
            if p.name in SKIP_NAMES or "node_modules" in p.parts:
                continue
            if p.suffix not in {".js", ".ts", ".json", ".md"}:
                continue
            rel = str(p.relative_to(d))
            try:
                files[rel] = p.read_text(encoding="utf-8", errors="replace")
            except Exception:
                continue
        auth = next((t for t in ("oauth2", "oauth1", "session-auth", "basic-auth", "digest-auth", "custom-auth") if t in d.name), None)
        pattern = next((t for t in ("trigger", "create", "search", "rest-hooks", "files", "middleware", "resource", "callback", "line-items", "dynamic-dropdown") if t in d.name), "example")
        parts = [f"# example-app `{d.name}`", "", f"Auth: `{auth or 'n/a'}` · Pattern: `{pattern}`", ""]
        for rel, src in sorted(files.items()):
            parts += [f"## `{rel}`", "", f"```{p.suffix.lstrip('.') if False else rel.split('.')[-1]}", src[:12000], "```", ""]
        rows.append(
            rec(
                kind="template",
                key=d.name,
                title=f"Source template: {d.name}",
                summary=f"Full example-app source ({auth or 'n/a'} / {pattern}).",
                body="\n".join(parts),
                source_url=f"https://github.com/zapier/zapier-platform/tree/main/example-apps/{d.name}",
                source_repo="zapier/zapier-platform",
                source_path=f"zapier-platform/example-apps/{d.name}",
                section="templates",
                tags=["template", "example", pattern] + ([auth] if auth else []),
                related=[f"zapier-platform init --template {d.name}"] if auth or d.name in {"minimal", "files", "callback"} else [],
                meta={"files": list(files), "auth": auth, "pattern": pattern, "local": f"repos/zapier-platform/example-apps/{d.name}"},
            )
        )
    # golden local project
    gold = ROOT / "scratch" / "oauth2-typescript"
    if gold.exists():
        srcs = {}
        for p in (gold / "src").rglob("*"):
            if p.is_file():
                srcs[str(p.relative_to(gold))] = p.read_text(encoding="utf-8", errors="replace")
        rows.append(
            rec(
                kind="template",
                key="scratch-oauth2-typescript",
                title="Golden local project: scratch/oauth2-typescript",
                summary="CLI 19.1.0 `zapier-platform init --template oauth2 --language typescript`. Structurally valid after build.",
                body="\n".join(
                    line
                    for k, v in sorted(srcs.items())
                    for line in (f"## `{k}`", "", "```ts", v, "```", "")
                ),
                source_path="scratch/oauth2-typescript",
                section="templates",
                tags=["template", "golden", "oauth2", "typescript"],
                related=["init", "validate", "z.request"],
                meta={"cli_version": "19.1.0", "validated": True},
            )
        )
    return rows


def main() -> int:
    rows: list[dict] = []
    print("schema…")
    rows += ingest_schema()
    print("openapi…")
    rows += ingest_openapi()
    print("news…")
    rows += ingest_news()
    print("templates…")
    rows += ingest_templates()
    counts: dict[str, int] = {}
    for r in rows:
        counts[r["kind"]] = counts.get(r["kind"], 0) + 1
    out = ROOT / "raw" / "coding-set.jsonl"
    with out.open("w") as f:
        for r in rows:
            f.write(json.dumps(r, ensure_ascii=False) + "\n")
    print("wrote", len(rows), counts, "→", out)

    rows.append(
        rec(
            kind="guide",
            key="coding-set",
            title="Coding set — schema, APIs, news, templates",
            summary="After build-new-connector: schema_json, api_function, platform_news, template.",
            body="""# Coding set

Start: `db.platform_reference.findOne({kind:'guide', key:'build-new-connector'})`

Then:

```js
db.platform_reference.findOne({kind:'schema_json', key:'exported-schema'})
db.platform_reference.find({kind:'api_function'}).sort({key:1})
db.platform_reference.find({kind:'platform_news'})
db.platform_reference.find({kind:'template', key:'oauth2-typescript'})
db.platform_reference.findOne({kind:'template', key:'scratch-oauth2-typescript'})
```

Local golden app: `scratch/oauth2-typescript` (CLI 19.1.0, `zapier-platform validate` clean after build).
CLIs: `~/.npm-global/bin/zapier-platform` and `zapier-sdk` (login still required for push/SDK).
""",
            section="guide",
            tags=["guide", "coding-set"],
        )
    )

    uri = os.environ.get("ZAPIER_MONGO_URI") or os.environ.get("MDB_MCP_CONNECTION_STRING")
    if not uri:
        print("no mongo uri")
        return 0
    from pymongo import MongoClient, ReplaceOne

    col = MongoClient(uri).get_database("zapier")["platform_reference"]
    ops = [ReplaceOne({"_id": r["_id"]}, r, upsert=True) for r in rows]
    for i in range(0, len(ops), 250):
        col.bulk_write(ops[i : i + 250], ordered=False)
    print("mongo kinds", {k: col.count_documents({"kind": k}) for k in ("schema_json", "api_function", "platform_news", "template", "schema_type")})
    col.database["meta"].update_one(
        {"_id": "ingest"},
        {"$set": {"coding_set_ingested_at": NOW}},
        upsert=True,
    )
    return 0


if __name__ == "__main__":
    raise SystemExit(main())