#!/usr/bin/env python3
"""Fetch vendor API docs homepages and classify auth style."""
from __future__ import annotations

import json
import re
import ssl
import urllib.request
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
CATALOG = ROOT / "raw" / "all-apps-full.jsonl"
OUT = ROOT / "raw" / "api-docs-scrape.jsonl"

SIGNALS = [
    ("oauth2", re.compile(r"\bOAuth\s*2(?:\.0)?\b|\bauthorization code\b|\bclient_id\b", re.I)),
    ("api_key", re.compile(r"\bAPI[- ]key\b|\bx-api-key\b|\bBearer token\b|\bpersonal access token\b", re.I)),
    ("basic", re.compile(r"\bBasic Auth(?:entication)?\b", re.I)),
    ("jwt", re.compile(r"\bJWT\b|\bJSON Web Token\b", re.I)),
    ("webhook", re.compile(r"\bwebhook\b", re.I)),
    ("openapi", re.compile(r"\bOpenAPI\b|\bSwagger\b", re.I)),
    ("graphql", re.compile(r"\bGraphQL\b", re.I)),
    ("rest", re.compile(r"\bREST(?:ful)? API\b", re.I)),
]


def load_done() -> set[str]:
    done = set()
    if OUT.exists():
        for line in OUT.open():
            try:
                done.add(json.loads(line)["slug"])
            except Exception:
                continue
    return done


def fetch(url: str) -> str:
    req = urllib.request.Request(
        url,
        headers={"User-Agent": "Mozilla/5.0 research", "Accept": "text/html,application/xhtml+xml"},
    )
    ctx = ssl.create_default_context()
    with urllib.request.urlopen(req, timeout=10, context=ctx) as r:
        raw = r.read(160_000)
        return raw.decode("utf-8", errors="ignore")


def scrape(app: dict) -> dict:
    url = (app.get("api_docs_url") or "").strip()
    rec = {"slug": app["slug"], "api_docs_url": url or None, "signals": [], "title": None, "error": None}
    if not url:
        rec["error"] = "no api_docs_url"
        return rec
    if not url.startswith("http"):
        url = "https://" + url
        rec["api_docs_url"] = url
    try:
        html = fetch(url)
    except Exception as e:
        rec["error"] = type(e).__name__
        return rec
    text = re.sub(r"<script[\s\S]*?</script>", " ", html, flags=re.I)
    text = re.sub(r"<style[\s\S]*?</style>", " ", text, flags=re.I)
    text = re.sub(r"<[^>]+>", " ", text)
    text = re.sub(r"\s+", " ", text)
    rec["signals"] = [n for n, rx in SIGNALS if rx.search(text)]
    tm = re.search(r"<title[^>]*>([^<]{3,160})</title>", html, re.I)
    rec["title"] = tm.group(1).strip() if tm else None
    rec["excerpt"] = text[:800]
    return rec


def main() -> None:
    apps = [json.loads(l) for l in CATALOG.open()]
    done = load_done()
    todo = [a for a in apps if a["slug"] not in done]
    print("todo", len(todo), flush=True)
    n = 0
    with OUT.open("a") as out, ThreadPoolExecutor(max_workers=28) as ex:
        futs = {ex.submit(scrape, a): a["slug"] for a in todo}
        for fut in as_completed(futs):
            rec = fut.result()
            out.write(json.dumps(rec, ensure_ascii=False) + "\n")
            out.flush()
            n += 1
            if n % 300 == 0:
                print(f"{n}/{len(todo)} last={rec['slug']} sig={rec.get('signals')}", flush=True)
    print("finished", n)


if __name__ == "__main__":
    main()