#!/usr/bin/env python3
"""Fetch remaining help.zapier.com sitemap articles."""
from __future__ import annotations

import json
import re
import ssl
import urllib.request
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
URLS = json.loads((ROOT / "raw" / "help-sitemap-new-urls.json").read_text())
OUT = ROOT / "raw" / "help-articles-extra.jsonl"

# copy of helpers (keep this file standalone)
AUTH_PATTERNS = [
    ("oauth", re.compile(r"\bOAuth\b|Grant Zapier permission|new browser tab or window will open", re.I)),
    ("api_key", re.compile(r"\bAPI key\b|\bAPI token\b|\bpersonal access token\b", re.I)),
    ("username_password", re.compile(r"username and password|email and password", re.I)),
    ("webhook", re.compile(r"webhook URL|catch hook", re.I)),
    ("premium", re.compile(r"Premium app", re.I)),
]


def html_to_text(html: str) -> str:
    html = re.sub(r"<script[\s\S]*?</script>", " ", html, flags=re.I)
    html = re.sub(r"<style[\s\S]*?</style>", " ", html, flags=re.I)
    html = re.sub(r"<[^>]+>", " ", html)
    return re.sub(r"\s+", " ", html).strip()


def fetch_article(url: str) -> dict:
    req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0 research"})
    ctx = ssl.create_default_context()
    with urllib.request.urlopen(req, timeout=25, context=ctx) as r:
        html = r.read().decode("utf-8", errors="ignore")
        final = r.geturl()
    text = html_to_text(html)
    for cut in ("Related articles", "Was this article helpful"):
        i = text.find(cut)
        if i > 400:
            text = text[:i]
    title = ""
    m = re.search(r"/articles/\d+-([^/?#]+)", url)
    if m:
        title = m.group(1).replace("-", " ")
    slug = None
    m2 = re.search(r"get started with (.+?) on zapier", title, re.I)
    m3 = re.search(r"common problems with (.+?)(?: on zapier)?$", title, re.I)
    if m2:
        slug = m2.group(1).strip().lower().replace(" ", "-")
    elif m3:
        slug = m3.group(1).strip().lower().replace(" ", "-")
    return {
        "url": url,
        "final_url": final,
        "title": title,
        "help_title": title,
        "slug": slug,
        "auth_signals": [n for n, rx in AUTH_PATTERNS if rx.search(text)],
        "text": text[:12000],
        "error": None,
    }


def main() -> None:
    done = set()
    if OUT.exists():
        for line in OUT.open():
            try:
                done.add(json.loads(line)["url"])
            except Exception:
                pass
    todo = [u for u in URLS if u not in done]
    print("todo", len(todo), flush=True)
    n = err = 0
    with OUT.open("a") as out, ThreadPoolExecutor(max_workers=16) as ex:
        futs = {ex.submit(fetch_article, u): u for u in todo}
        for fut in as_completed(futs):
            url = futs[fut]
            try:
                rec = fut.result()
            except Exception as e:
                rec = {"url": url, "error": f"{type(e).__name__}: {e}"}
                err += 1
            out.write(json.dumps(rec, ensure_ascii=False) + "\n")
            out.flush()
            n += 1
            if n % 50 == 0:
                print(f"{n}/{len(todo)} err={err}", flush=True)
    print("finished", n, "err", err)


if __name__ == "__main__":
    main()