George Lambert
b4150c8250
Compose-ready workspace: packages/zappier (rate card, portal, Stripe), packages/verae-zapier-middleware (timestamp + NATS), packages/verae-zapier (CLI app), vendor/zapier-platform, and research/zapier vendor corpus. Gate 0 structure checks pass. Product code and research are not yet wired.
94 lines
3.2 KiB
Python
94 lines
3.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Fetch vendor API docs homepages and classify auth style."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
import ssl
|
|
import urllib.request
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
CATALOG = ROOT / "raw" / "all-apps-full.jsonl"
|
|
OUT = ROOT / "raw" / "api-docs-scrape.jsonl"
|
|
|
|
SIGNALS = [
|
|
("oauth2", re.compile(r"\bOAuth\s*2(?:\.0)?\b|\bauthorization code\b|\bclient_id\b", re.I)),
|
|
("api_key", re.compile(r"\bAPI[- ]key\b|\bx-api-key\b|\bBearer token\b|\bpersonal access token\b", re.I)),
|
|
("basic", re.compile(r"\bBasic Auth(?:entication)?\b", re.I)),
|
|
("jwt", re.compile(r"\bJWT\b|\bJSON Web Token\b", re.I)),
|
|
("webhook", re.compile(r"\bwebhook\b", re.I)),
|
|
("openapi", re.compile(r"\bOpenAPI\b|\bSwagger\b", re.I)),
|
|
("graphql", re.compile(r"\bGraphQL\b", re.I)),
|
|
("rest", re.compile(r"\bREST(?:ful)? API\b", re.I)),
|
|
]
|
|
|
|
|
|
def load_done() -> set[str]:
|
|
done = set()
|
|
if OUT.exists():
|
|
for line in OUT.open():
|
|
try:
|
|
done.add(json.loads(line)["slug"])
|
|
except Exception:
|
|
continue
|
|
return done
|
|
|
|
|
|
def fetch(url: str) -> str:
|
|
req = urllib.request.Request(
|
|
url,
|
|
headers={"User-Agent": "Mozilla/5.0 research", "Accept": "text/html,application/xhtml+xml"},
|
|
)
|
|
ctx = ssl.create_default_context()
|
|
with urllib.request.urlopen(req, timeout=10, context=ctx) as r:
|
|
raw = r.read(160_000)
|
|
return raw.decode("utf-8", errors="ignore")
|
|
|
|
|
|
def scrape(app: dict) -> dict:
|
|
url = (app.get("api_docs_url") or "").strip()
|
|
rec = {"slug": app["slug"], "api_docs_url": url or None, "signals": [], "title": None, "error": None}
|
|
if not url:
|
|
rec["error"] = "no api_docs_url"
|
|
return rec
|
|
if not url.startswith("http"):
|
|
url = "https://" + url
|
|
rec["api_docs_url"] = url
|
|
try:
|
|
html = fetch(url)
|
|
except Exception as e:
|
|
rec["error"] = type(e).__name__
|
|
return rec
|
|
text = re.sub(r"<script[\s\S]*?</script>", " ", html, flags=re.I)
|
|
text = re.sub(r"<style[\s\S]*?</style>", " ", text, flags=re.I)
|
|
text = re.sub(r"<[^>]+>", " ", text)
|
|
text = re.sub(r"\s+", " ", text)
|
|
rec["signals"] = [n for n, rx in SIGNALS if rx.search(text)]
|
|
tm = re.search(r"<title[^>]*>([^<]{3,160})</title>", html, re.I)
|
|
rec["title"] = tm.group(1).strip() if tm else None
|
|
rec["excerpt"] = text[:800]
|
|
return rec
|
|
|
|
|
|
def main() -> None:
|
|
apps = [json.loads(l) for l in CATALOG.open()]
|
|
done = load_done()
|
|
todo = [a for a in apps if a["slug"] not in done]
|
|
print("todo", len(todo), flush=True)
|
|
n = 0
|
|
with OUT.open("a") as out, ThreadPoolExecutor(max_workers=28) as ex:
|
|
futs = {ex.submit(scrape, a): a["slug"] for a in todo}
|
|
for fut in as_completed(futs):
|
|
rec = fut.result()
|
|
out.write(json.dumps(rec, ensure_ascii=False) + "\n")
|
|
out.flush()
|
|
n += 1
|
|
if n % 300 == 0:
|
|
print(f"{n}/{len(todo)} last={rec['slug']} sig={rec.get('signals')}", flush=True)
|
|
print("finished", n)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|