George Lambert
b4150c8250
Compose-ready workspace: packages/zappier (rate card, portal, Stripe), packages/verae-zapier-middleware (timestamp + NATS), packages/verae-zapier (CLI app), vendor/zapier-platform, and research/zapier vendor corpus. Gate 0 structure checks pass. Product code and research are not yet wired.
96 lines
3.2 KiB
Python
96 lines
3.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Fetch remaining help.zapier.com sitemap articles."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
import ssl
|
|
import urllib.request
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
URLS = json.loads((ROOT / "raw" / "help-sitemap-new-urls.json").read_text())
|
|
OUT = ROOT / "raw" / "help-articles-extra.jsonl"
|
|
|
|
# copy of helpers (keep this file standalone)
|
|
AUTH_PATTERNS = [
|
|
("oauth", re.compile(r"\bOAuth\b|Grant Zapier permission|new browser tab or window will open", re.I)),
|
|
("api_key", re.compile(r"\bAPI key\b|\bAPI token\b|\bpersonal access token\b", re.I)),
|
|
("username_password", re.compile(r"username and password|email and password", re.I)),
|
|
("webhook", re.compile(r"webhook URL|catch hook", re.I)),
|
|
("premium", re.compile(r"Premium app", re.I)),
|
|
]
|
|
|
|
|
|
def html_to_text(html: str) -> str:
|
|
html = re.sub(r"<script[\s\S]*?</script>", " ", html, flags=re.I)
|
|
html = re.sub(r"<style[\s\S]*?</style>", " ", html, flags=re.I)
|
|
html = re.sub(r"<[^>]+>", " ", html)
|
|
return re.sub(r"\s+", " ", html).strip()
|
|
|
|
|
|
def fetch_article(url: str) -> dict:
|
|
req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0 research"})
|
|
ctx = ssl.create_default_context()
|
|
with urllib.request.urlopen(req, timeout=25, context=ctx) as r:
|
|
html = r.read().decode("utf-8", errors="ignore")
|
|
final = r.geturl()
|
|
text = html_to_text(html)
|
|
for cut in ("Related articles", "Was this article helpful"):
|
|
i = text.find(cut)
|
|
if i > 400:
|
|
text = text[:i]
|
|
title = ""
|
|
m = re.search(r"/articles/\d+-([^/?#]+)", url)
|
|
if m:
|
|
title = m.group(1).replace("-", " ")
|
|
slug = None
|
|
m2 = re.search(r"get started with (.+?) on zapier", title, re.I)
|
|
m3 = re.search(r"common problems with (.+?)(?: on zapier)?$", title, re.I)
|
|
if m2:
|
|
slug = m2.group(1).strip().lower().replace(" ", "-")
|
|
elif m3:
|
|
slug = m3.group(1).strip().lower().replace(" ", "-")
|
|
return {
|
|
"url": url,
|
|
"final_url": final,
|
|
"title": title,
|
|
"help_title": title,
|
|
"slug": slug,
|
|
"auth_signals": [n for n, rx in AUTH_PATTERNS if rx.search(text)],
|
|
"text": text[:12000],
|
|
"error": None,
|
|
}
|
|
|
|
|
|
def main() -> None:
|
|
done = set()
|
|
if OUT.exists():
|
|
for line in OUT.open():
|
|
try:
|
|
done.add(json.loads(line)["url"])
|
|
except Exception:
|
|
pass
|
|
todo = [u for u in URLS if u not in done]
|
|
print("todo", len(todo), flush=True)
|
|
n = err = 0
|
|
with OUT.open("a") as out, ThreadPoolExecutor(max_workers=16) as ex:
|
|
futs = {ex.submit(fetch_article, u): u for u in todo}
|
|
for fut in as_completed(futs):
|
|
url = futs[fut]
|
|
try:
|
|
rec = fut.result()
|
|
except Exception as e:
|
|
rec = {"url": url, "error": f"{type(e).__name__}: {e}"}
|
|
err += 1
|
|
out.write(json.dumps(rec, ensure_ascii=False) + "\n")
|
|
out.flush()
|
|
n += 1
|
|
if n % 50 == 0:
|
|
print(f"{n}/{len(todo)} err={err}", flush=True)
|
|
print("finished", n, "err", err)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|