George Lambert
1b199ca4d4
Some checks are pending
offline / test (push) Waiting to run
Zapier is one ingress. Direct web, customer API, and S2S leaf nodes are their own services. Every hop to an internal subject must pass verae.access.authz.check (default deny by plane).
391 lines
14 KiB
Python
391 lines
14 KiB
Python
#!/usr/bin/env python3
|
||
"""Generate per-module Markdown + RST API docs from JS/TS sources."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import re
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
OUT_MD = ROOT / "docs" / "modules"
|
||
OUT_RST = ROOT / "docs" / "sphinx" / "modules"
|
||
|
||
PACKAGES = [
|
||
("zappier", ROOT / "packages" / "zappier" / "src", (".ts",)),
|
||
("verae-zapier-middleware", ROOT / "packages" / "verae-zapier-middleware" / "src", (".js",)),
|
||
("verae-activate", ROOT / "packages" / "verae-activate", (".js",)),
|
||
("verae-zapier", ROOT / "packages" / "verae-zapier", (".js",)),
|
||
("verae-fleet", ROOT / "packages" / "verae-fleet" / "src", (".js",)),
|
||
("verae-tree-node", ROOT / "packages" / "verae-tree-node" / "src", (".js",)),
|
||
("verae-archive-worm", ROOT / "packages" / "verae-archive-worm" / "src", (".js",)),
|
||
("verae-archive-aggregator", ROOT / "packages" / "verae-archive-aggregator" / "src", (".js",)),
|
||
("verae-request-splitter", ROOT / "packages" / "verae-request-splitter" / "src", (".js",)),
|
||
("verae-nats-process", ROOT / "packages" / "verae-nats-process" / "src", (".js",)),
|
||
("verae-zapier-simulator", ROOT / "packages" / "verae-zapier-simulator" / "src", (".js",)),
|
||
("zappier-account-balance", ROOT / "packages" / "zappier-account-balance" / "src", (".js",)),
|
||
("zappier-customer-service", ROOT / "packages" / "zappier-customer-service" / "src", (".js",)),
|
||
("zappier-sales-pricing", ROOT / "packages" / "zappier-sales-pricing" / "src", (".js",)),
|
||
("zappier-accounting-export", ROOT / "packages" / "zappier-accounting-export" / "src", (".js",)),
|
||
("verae-access-authz", ROOT / "packages" / "verae-access-authz" / "src", (".js",)),
|
||
("verae-access-web", ROOT / "packages" / "verae-access-web" / "src", (".js",)),
|
||
("verae-access-api", ROOT / "packages" / "verae-access-api" / "src", (".js",)),
|
||
("verae-access-leaf", ROOT / "packages" / "verae-access-leaf" / "src", (".js",)),
|
||
("verae-access-zapier", ROOT / "packages" / "verae-access-zapier" / "src", (".js",)),
|
||
]
|
||
|
||
REPO_READMES = [
|
||
"overview",
|
||
"verae-ops",
|
||
"zappier",
|
||
"verae-zapier-middleware",
|
||
"verae-zapier",
|
||
"verae-activate",
|
||
"verae-request-splitter",
|
||
"verae-archive-worm",
|
||
"verae-archive-aggregator",
|
||
"verae-tree-node",
|
||
"verae-fleet",
|
||
"verae-zapier-simulator",
|
||
"zapier-user-docs",
|
||
"docs-master",
|
||
"verae-nats-process",
|
||
"zappier-account-balance",
|
||
"zappier-customer-service",
|
||
"zappier-sales-pricing",
|
||
"zappier-accounting-export",
|
||
"verae-access-authz",
|
||
"verae-access-web",
|
||
"verae-access-api",
|
||
"verae-access-leaf",
|
||
"verae-access-zapier",
|
||
]
|
||
|
||
SKIP_PARTS = {"test", "tests", "node_modules", "dist"}
|
||
|
||
FN_RE = re.compile(
|
||
r"""^(?P<indent> *)(?:export\s+)?(?:async\s+)?function\s+(?P<name>[A-Za-z0-9_]+)\s*\((?P<params>[^)]*)\)""",
|
||
re.M,
|
||
)
|
||
ARROW_RE = re.compile(
|
||
r"""^(?P<indent> *)(?:export\s+)?(?:const|let|var)\s+(?P<name>[A-Za-z0-9_]+)\s*=\s*(?:async\s*)?\((?P<params>[^)]*)\)\s*=>""",
|
||
re.M,
|
||
)
|
||
CLASS_RE = re.compile(r"""^(?:export\s+)?class\s+(?P<name>[A-Za-z0-9_]+)""", re.M)
|
||
IFACE_RE = re.compile(r"""^(?:export\s+)?interface\s+(?P<name>[A-Za-z0-9_]+)""", re.M)
|
||
TYPE_RE = re.compile(r"""^(?:export\s+)?type\s+(?P<name>[A-Za-z0-9_]+)\s*=""", re.M)
|
||
METHOD_RE = re.compile(
|
||
r"""^ (?:async\s+)?(?P<name>[A-Za-z0-9_]+)\s*\((?P<params>[^)]*)\)""",
|
||
re.M,
|
||
)
|
||
EXPORTS_RE = re.compile(r"""^export\s+(?:async\s+)?(?:function|class|const|let|type|interface|enum)\s+([A-Za-z0-9_]+)""", re.M)
|
||
MODULE_EXPORTS_RE = re.compile(r"""module\.exports\s*=\s*\{([^}]+)\}""", re.S)
|
||
REQUIRE_RE = re.compile(r"""require\(['\"]([^'\"]+)['\"]\)""")
|
||
IMPORT_RE = re.compile(r"""from ['\"]([^'\"]+)['\"]""")
|
||
JSDOC_RE = re.compile(r"""/\*\*(.*?)\*/""", re.S)
|
||
PARAM_RE = re.compile(r"""@param\s+(?:\{([^}]+)\}\s+)?(?:\[)?([A-Za-z0-9_.]+)""")
|
||
RETURN_RE = re.compile(r"""@returns?\s+(?:\{([^}]+)\})?\s*(.*)""")
|
||
CALL_RE = re.compile(r"""\b([A-Za-z0-9_]+)\s*\(""")
|
||
|
||
|
||
def should_skip(path: Path) -> bool:
|
||
return any(p in SKIP_PARTS for p in path.parts)
|
||
|
||
|
||
def rel_mod(pkg: str, src_root: Path, file: Path) -> str:
|
||
rel = file.relative_to(src_root).with_suffix("")
|
||
return f"{pkg}/{rel.as_posix()}"
|
||
|
||
|
||
def parse_jsdoc_blocks(text: str) -> list[dict]:
|
||
blocks = []
|
||
for m in JSDOC_RE.finditer(text):
|
||
body = m.group(1)
|
||
params = PARAM_RE.findall(body)
|
||
ret = RETURN_RE.search(body)
|
||
summary = " ".join(
|
||
line.strip(" *")
|
||
for line in body.splitlines()
|
||
if line.strip() and not line.strip().startswith("*") and "@" not in line
|
||
).strip()
|
||
blocks.append(
|
||
{
|
||
"summary": summary[:400],
|
||
"params": [{"type": t or "unknown", "name": n} for t, n in params],
|
||
"returns_type": (ret.group(1) if ret else "") or "unknown",
|
||
"returns_note": (ret.group(2).strip() if ret else ""),
|
||
"end": m.end(),
|
||
}
|
||
)
|
||
return blocks
|
||
|
||
|
||
def nearest_jsdoc(blocks: list[dict], pos: int) -> dict | None:
|
||
best = None
|
||
for b in blocks:
|
||
if b["end"] <= pos and pos - b["end"] < 180:
|
||
best = b
|
||
return best
|
||
|
||
|
||
def parse_file(path: Path) -> dict:
|
||
text = path.read_text(encoding="utf-8", errors="replace")
|
||
blocks = parse_jsdoc_blocks(text)
|
||
functions = []
|
||
for rx in (FN_RE, ARROW_RE):
|
||
for m in rx.finditer(text):
|
||
name = m.group("name")
|
||
params = m.group("params").strip()
|
||
js = nearest_jsdoc(blocks, m.start())
|
||
functions.append(
|
||
{
|
||
"name": name,
|
||
"params": params,
|
||
"jsdoc": js,
|
||
"kind": "function",
|
||
}
|
||
)
|
||
classes = [{"name": m.group("name")} for m in CLASS_RE.finditer(text)]
|
||
types = [{"name": m.group("name"), "kind": "interface"} for m in IFACE_RE.finditer(text)]
|
||
types += [{"name": m.group("name"), "kind": "type"} for m in TYPE_RE.finditer(text)]
|
||
methods = []
|
||
for m in METHOD_RE.finditer(text):
|
||
name = m.group("name")
|
||
if name in {"if", "for", "while", "switch", "catch", "function"}:
|
||
continue
|
||
methods.append({"name": name, "params": m.group("params").strip()})
|
||
exports = EXPORTS_RE.findall(text)
|
||
me = MODULE_EXPORTS_RE.search(text)
|
||
if me:
|
||
exports += re.findall(r"([A-Za-z0-9_]+)\s*[:,]", me.group(1))
|
||
requires = REQUIRE_RE.findall(text)
|
||
imports = IMPORT_RE.findall(text)
|
||
calls = []
|
||
for c in CALL_RE.findall(text):
|
||
if c[0].isupper() or c in {
|
||
"if",
|
||
"for",
|
||
"while",
|
||
"switch",
|
||
"catch",
|
||
"function",
|
||
"return",
|
||
"await",
|
||
"typeof",
|
||
"new",
|
||
"super",
|
||
"Number",
|
||
"String",
|
||
"Boolean",
|
||
"Object",
|
||
"Array",
|
||
"JSON",
|
||
"Buffer",
|
||
"Error",
|
||
"Date",
|
||
"Promise",
|
||
"Math",
|
||
"console",
|
||
"require",
|
||
"describe",
|
||
"it",
|
||
"expect",
|
||
}:
|
||
continue
|
||
calls.append(c)
|
||
# unique preserve order
|
||
seen = set()
|
||
call_uniq = []
|
||
for c in calls:
|
||
if c not in seen:
|
||
seen.add(c)
|
||
call_uniq.append(c)
|
||
return {
|
||
"path": str(path),
|
||
"functions": functions,
|
||
"classes": classes,
|
||
"types": types,
|
||
"methods": methods[:80],
|
||
"exports": list(dict.fromkeys(exports)),
|
||
"requires": list(dict.fromkeys(requires)),
|
||
"imports": list(dict.fromkeys(imports)),
|
||
"calls": call_uniq[:60],
|
||
"lines": text.count("\n") + 1,
|
||
}
|
||
|
||
|
||
def render_md(pkg: str, rel: str, info: dict) -> str:
|
||
lines = [
|
||
f"# `{rel}`",
|
||
"",
|
||
f"**Package:** `{pkg}` ",
|
||
f"**Source:** `{info['path'].replace(str(ROOT) + '/', '')}` ",
|
||
f"**Lines:** {info['lines']}",
|
||
"",
|
||
"## What this module is",
|
||
"",
|
||
f"Implementation module in `{pkg}`. The tables below are extracted from the source (signatures + JSDoc).",
|
||
"",
|
||
"## Exports",
|
||
"",
|
||
]
|
||
if info["exports"]:
|
||
lines.append(", ".join(f"`{e}`" for e in info["exports"]))
|
||
else:
|
||
lines.append("_No named `export` / `module.exports` keys detected._")
|
||
lines += ["", "## Types / interfaces / classes", ""]
|
||
if info["types"] or info["classes"]:
|
||
lines += ["| Kind | Name |", "|------|------|"]
|
||
for t in info["types"]:
|
||
lines.append(f"| {t['kind']} | `{t['name']}` |")
|
||
for c in info["classes"]:
|
||
lines.append(f"| class | `{c['name']}` |")
|
||
else:
|
||
lines.append("_None extracted._")
|
||
lines += ["", "## Functions", ""]
|
||
if info["functions"]:
|
||
lines += [
|
||
"| Name | Parameters | Param types (JSDoc) | Returns | Calls (same file / helpers) |",
|
||
"|------|------------|---------------------|---------|-----------------------------|",
|
||
]
|
||
for fn in info["functions"]:
|
||
js = fn["jsdoc"] or {}
|
||
ptypes = ", ".join(f"{p['name']}: `{p['type']}`" for p in js.get("params", [])) or "—"
|
||
ret = f"`{js.get('returns_type', 'unknown')}`"
|
||
if js.get("returns_note"):
|
||
ret += f" — {js['returns_note'][:120]}"
|
||
params = fn["params"].replace("|", "\\|") or "(none)"
|
||
lines.append(
|
||
f"| `{fn['name']}` | `{params}` | {ptypes} | {ret} | see Call graph |"
|
||
)
|
||
if js.get("summary"):
|
||
lines.append(f"| | _{js['summary']}_ | | | |")
|
||
else:
|
||
lines.append("_No top-level functions extracted._")
|
||
if info["methods"]:
|
||
lines += ["", "## Methods (class / object)", ""]
|
||
lines += ["| Name | Parameters |", "|------|------------|"]
|
||
for m in info["methods"]:
|
||
lines.append(f"| `{m['name']}` | `{m['params'] or '(none)'}` |")
|
||
lines += ["", "## What it imports / requires", ""]
|
||
deps = info["imports"] + info["requires"]
|
||
if deps:
|
||
for d in deps:
|
||
lines.append(f"- `{d}`")
|
||
else:
|
||
lines.append("_No imports detected._")
|
||
lines += ["", "## Call graph (identifiers invoked)", ""]
|
||
if info["calls"]:
|
||
lines.append(", ".join(f"`{c}`" for c in info["calls"]))
|
||
lines.append("")
|
||
lines.append(
|
||
"Each identifier is a call site in this file. Follow the import list to see the defining module; "
|
||
"open that module’s MD for parameter and return types."
|
||
)
|
||
else:
|
||
lines.append("_No local call identifiers extracted._")
|
||
lines += [
|
||
"",
|
||
"## Return values (how to read this)",
|
||
"",
|
||
"- HTTP route handlers return Express `res.json(...)` bodies (see route docs).",
|
||
"- Zapier `perform` functions return a **single object** (creates) or an **array** (triggers/searches).",
|
||
"- Pricing functions return integer **cents** on `Quote.totalCents`.",
|
||
"",
|
||
]
|
||
return "\n".join(lines) + "\n"
|
||
|
||
|
||
def render_rst(title: str, md_rel: str) -> str:
|
||
return f"""{title}
|
||
{'=' * len(title)}
|
||
|
||
.. include:: /{md_rel}
|
||
:parser: myst_parser.sphinx_
|
||
"""
|
||
|
||
|
||
def main() -> None:
|
||
OUT_MD.mkdir(parents=True, exist_ok=True)
|
||
OUT_RST.mkdir(parents=True, exist_ok=True)
|
||
index_entries = []
|
||
rst_entries = []
|
||
count = 0
|
||
for pkg, src_root, suffixes in PACKAGES:
|
||
if not src_root.exists():
|
||
continue
|
||
files = sorted(
|
||
p
|
||
for p in src_root.rglob("*")
|
||
if p.suffix in suffixes and p.is_file() and not should_skip(p)
|
||
)
|
||
for f in files:
|
||
if "node_modules" in f.parts:
|
||
continue
|
||
rel = rel_mod(pkg, src_root, f)
|
||
info = parse_file(f)
|
||
md = render_md(pkg, rel, info)
|
||
md_path = OUT_MD / f"{rel}.md"
|
||
md_path.parent.mkdir(parents=True, exist_ok=True)
|
||
md_path.write_text(md, encoding="utf-8")
|
||
rst_path = OUT_RST / f"{rel}.rst"
|
||
rst_path.parent.mkdir(parents=True, exist_ok=True)
|
||
rst_title = rel.replace("/", ".")
|
||
# myst include path relative to sphinx conf dir
|
||
include = f"../modules/{rel}.md"
|
||
# Sphinx include is relative to the RST file: copy MD beside it.
|
||
sibling_md = rst_path.with_suffix(".md")
|
||
sibling_md.write_text(md, encoding="utf-8")
|
||
rst_path.write_text(
|
||
f"{rst_title}\n{'=' * len(rst_title)}\n\n"
|
||
f"Generated API sheet for ``{rel}``.\n\n"
|
||
f".. include:: {sibling_md.name}\n"
|
||
f" :parser: myst_parser.sphinx_\n",
|
||
encoding="utf-8",
|
||
)
|
||
index_entries.append(rel)
|
||
rst_entries.append(f"modules/{rel}")
|
||
count += 1
|
||
|
||
idx = ["# Module documentation index", "", "One MD file per source module.", ""]
|
||
for rel in index_entries:
|
||
idx.append(f"- [{rel}]({rel}.md)")
|
||
(OUT_MD / "README.md").write_text("\n".join(idx) + "\n", encoding="utf-8")
|
||
|
||
repos_dir = ROOT / "docs" / "sphinx" / "repos"
|
||
repos_dir.mkdir(parents=True, exist_ok=True)
|
||
repo_entries = []
|
||
for name in REPO_READMES:
|
||
src = ROOT / "packages" / name / "README.md"
|
||
if not src.exists():
|
||
continue
|
||
dest = repos_dir / f"{name}.md"
|
||
dest.write_text(src.read_text(encoding="utf-8"), encoding="utf-8")
|
||
repo_entries.append(f"repos/{name}")
|
||
|
||
repo_toc = "\n ".join(repo_entries) or "repos/overview"
|
||
toctree = "\n ".join(rst_entries)
|
||
sphinx_index = f"""Verae × Zapier module reference
|
||
=================================
|
||
|
||
Public HTML: https://zapier.georgelambert.org/sphinx/
|
||
LaTeX/PDF: https://zapier.georgelambert.org/sphinx/verae-zapier-modules.pdf
|
||
Markdown indexes: https://zapier.georgelambert.org/index-md.html
|
||
|
||
.. toctree::
|
||
:maxdepth: 1
|
||
:caption: Repository READMEs
|
||
|
||
{repo_toc}
|
||
|
||
.. toctree::
|
||
:maxdepth: 2
|
||
:caption: API modules
|
||
|
||
{toctree}
|
||
"""
|
||
(ROOT / "docs" / "sphinx" / "index.rst").write_text(sphinx_index, encoding="utf-8")
|
||
print(f"wrote {count} module docs + {len(repo_entries)} repo READMEs")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|