Milestone 2: module/model docs (MD, Sphinx RST, PDF) and work log
74 source modules documented with extracted signatures, JSDoc params, imports, and call graphs. 10 first-class models (PriceRule through StatusResponse) have their own MD/RST/PDF. Sphinx HTML builds from docs/sphinx. Per-module PDFs in docs/modules-pdf and docs/models-pdf. Middleware gates 2–6 and 10 pass on MOCK_VERAE without NATS. Learned: RST includes are relative to the RST file; keep one PDF per module; Add Numbers remains the only push required tomorrow.
This commit is contained in:
parent
b814501441
commit
10c663cc0c
347 changed files with 17200 additions and 14 deletions
326
scripts/gen-module-docs.py
Normal file
326
scripts/gen-module-docs.py
Normal file
|
|
@ -0,0 +1,326 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Generate per-module Markdown + RST API docs from JS/TS sources."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
OUT_MD = ROOT / "docs" / "modules"
|
||||
OUT_RST = ROOT / "docs" / "sphinx" / "modules"
|
||||
|
||||
PACKAGES = [
|
||||
("zappier", ROOT / "packages" / "zappier" / "src", (".ts",)),
|
||||
("verae-zapier-middleware", ROOT / "packages" / "verae-zapier-middleware" / "src", (".js",)),
|
||||
("verae-activate", ROOT / "packages" / "verae-activate", (".js",)),
|
||||
("verae-zapier", ROOT / "packages" / "verae-zapier", (".js",)),
|
||||
]
|
||||
|
||||
SKIP_PARTS = {"test", "tests", "node_modules", "dist"}
|
||||
|
||||
FN_RE = re.compile(
|
||||
r"""^(?P<indent> *)(?:export\s+)?(?:async\s+)?function\s+(?P<name>[A-Za-z0-9_]+)\s*\((?P<params>[^)]*)\)""",
|
||||
re.M,
|
||||
)
|
||||
ARROW_RE = re.compile(
|
||||
r"""^(?P<indent> *)(?:export\s+)?(?:const|let|var)\s+(?P<name>[A-Za-z0-9_]+)\s*=\s*(?:async\s*)?\((?P<params>[^)]*)\)\s*=>""",
|
||||
re.M,
|
||||
)
|
||||
CLASS_RE = re.compile(r"""^(?:export\s+)?class\s+(?P<name>[A-Za-z0-9_]+)""", re.M)
|
||||
IFACE_RE = re.compile(r"""^(?:export\s+)?interface\s+(?P<name>[A-Za-z0-9_]+)""", re.M)
|
||||
TYPE_RE = re.compile(r"""^(?:export\s+)?type\s+(?P<name>[A-Za-z0-9_]+)\s*=""", re.M)
|
||||
METHOD_RE = re.compile(
|
||||
r"""^ (?:async\s+)?(?P<name>[A-Za-z0-9_]+)\s*\((?P<params>[^)]*)\)""",
|
||||
re.M,
|
||||
)
|
||||
EXPORTS_RE = re.compile(r"""^export\s+(?:async\s+)?(?:function|class|const|let|type|interface|enum)\s+([A-Za-z0-9_]+)""", re.M)
|
||||
MODULE_EXPORTS_RE = re.compile(r"""module\.exports\s*=\s*\{([^}]+)\}""", re.S)
|
||||
REQUIRE_RE = re.compile(r"""require\(['\"]([^'\"]+)['\"]\)""")
|
||||
IMPORT_RE = re.compile(r"""from ['\"]([^'\"]+)['\"]""")
|
||||
JSDOC_RE = re.compile(r"""/\*\*(.*?)\*/""", re.S)
|
||||
PARAM_RE = re.compile(r"""@param\s+(?:\{([^}]+)\}\s+)?(?:\[)?([A-Za-z0-9_.]+)""")
|
||||
RETURN_RE = re.compile(r"""@returns?\s+(?:\{([^}]+)\})?\s*(.*)""")
|
||||
CALL_RE = re.compile(r"""\b([A-Za-z0-9_]+)\s*\(""")
|
||||
|
||||
|
||||
def should_skip(path: Path) -> bool:
|
||||
return any(p in SKIP_PARTS for p in path.parts)
|
||||
|
||||
|
||||
def rel_mod(pkg: str, src_root: Path, file: Path) -> str:
|
||||
rel = file.relative_to(src_root).with_suffix("")
|
||||
return f"{pkg}/{rel.as_posix()}"
|
||||
|
||||
|
||||
def parse_jsdoc_blocks(text: str) -> list[dict]:
|
||||
blocks = []
|
||||
for m in JSDOC_RE.finditer(text):
|
||||
body = m.group(1)
|
||||
params = PARAM_RE.findall(body)
|
||||
ret = RETURN_RE.search(body)
|
||||
summary = " ".join(
|
||||
line.strip(" *")
|
||||
for line in body.splitlines()
|
||||
if line.strip() and not line.strip().startswith("*") and "@" not in line
|
||||
).strip()
|
||||
blocks.append(
|
||||
{
|
||||
"summary": summary[:400],
|
||||
"params": [{"type": t or "unknown", "name": n} for t, n in params],
|
||||
"returns_type": (ret.group(1) if ret else "") or "unknown",
|
||||
"returns_note": (ret.group(2).strip() if ret else ""),
|
||||
"end": m.end(),
|
||||
}
|
||||
)
|
||||
return blocks
|
||||
|
||||
|
||||
def nearest_jsdoc(blocks: list[dict], pos: int) -> dict | None:
|
||||
best = None
|
||||
for b in blocks:
|
||||
if b["end"] <= pos and pos - b["end"] < 180:
|
||||
best = b
|
||||
return best
|
||||
|
||||
|
||||
def parse_file(path: Path) -> dict:
|
||||
text = path.read_text(encoding="utf-8", errors="replace")
|
||||
blocks = parse_jsdoc_blocks(text)
|
||||
functions = []
|
||||
for rx in (FN_RE, ARROW_RE):
|
||||
for m in rx.finditer(text):
|
||||
name = m.group("name")
|
||||
params = m.group("params").strip()
|
||||
js = nearest_jsdoc(blocks, m.start())
|
||||
functions.append(
|
||||
{
|
||||
"name": name,
|
||||
"params": params,
|
||||
"jsdoc": js,
|
||||
"kind": "function",
|
||||
}
|
||||
)
|
||||
classes = [{"name": m.group("name")} for m in CLASS_RE.finditer(text)]
|
||||
types = [{"name": m.group("name"), "kind": "interface"} for m in IFACE_RE.finditer(text)]
|
||||
types += [{"name": m.group("name"), "kind": "type"} for m in TYPE_RE.finditer(text)]
|
||||
methods = []
|
||||
for m in METHOD_RE.finditer(text):
|
||||
name = m.group("name")
|
||||
if name in {"if", "for", "while", "switch", "catch", "function"}:
|
||||
continue
|
||||
methods.append({"name": name, "params": m.group("params").strip()})
|
||||
exports = EXPORTS_RE.findall(text)
|
||||
me = MODULE_EXPORTS_RE.search(text)
|
||||
if me:
|
||||
exports += re.findall(r"([A-Za-z0-9_]+)\s*[:,]", me.group(1))
|
||||
requires = REQUIRE_RE.findall(text)
|
||||
imports = IMPORT_RE.findall(text)
|
||||
calls = []
|
||||
for c in CALL_RE.findall(text):
|
||||
if c[0].isupper() or c in {
|
||||
"if",
|
||||
"for",
|
||||
"while",
|
||||
"switch",
|
||||
"catch",
|
||||
"function",
|
||||
"return",
|
||||
"await",
|
||||
"typeof",
|
||||
"new",
|
||||
"super",
|
||||
"Number",
|
||||
"String",
|
||||
"Boolean",
|
||||
"Object",
|
||||
"Array",
|
||||
"JSON",
|
||||
"Buffer",
|
||||
"Error",
|
||||
"Date",
|
||||
"Promise",
|
||||
"Math",
|
||||
"console",
|
||||
"require",
|
||||
"describe",
|
||||
"it",
|
||||
"expect",
|
||||
}:
|
||||
continue
|
||||
calls.append(c)
|
||||
# unique preserve order
|
||||
seen = set()
|
||||
call_uniq = []
|
||||
for c in calls:
|
||||
if c not in seen:
|
||||
seen.add(c)
|
||||
call_uniq.append(c)
|
||||
return {
|
||||
"path": str(path),
|
||||
"functions": functions,
|
||||
"classes": classes,
|
||||
"types": types,
|
||||
"methods": methods[:80],
|
||||
"exports": list(dict.fromkeys(exports)),
|
||||
"requires": list(dict.fromkeys(requires)),
|
||||
"imports": list(dict.fromkeys(imports)),
|
||||
"calls": call_uniq[:60],
|
||||
"lines": text.count("\n") + 1,
|
||||
}
|
||||
|
||||
|
||||
def render_md(pkg: str, rel: str, info: dict) -> str:
|
||||
lines = [
|
||||
f"# `{rel}`",
|
||||
"",
|
||||
f"**Package:** `{pkg}` ",
|
||||
f"**Source:** `{info['path'].replace(str(ROOT) + '/', '')}` ",
|
||||
f"**Lines:** {info['lines']}",
|
||||
"",
|
||||
"## What this module is",
|
||||
"",
|
||||
f"Implementation module in `{pkg}`. The tables below are extracted from the source (signatures + JSDoc).",
|
||||
"",
|
||||
"## Exports",
|
||||
"",
|
||||
]
|
||||
if info["exports"]:
|
||||
lines.append(", ".join(f"`{e}`" for e in info["exports"]))
|
||||
else:
|
||||
lines.append("_No named `export` / `module.exports` keys detected._")
|
||||
lines += ["", "## Types / interfaces / classes", ""]
|
||||
if info["types"] or info["classes"]:
|
||||
lines += ["| Kind | Name |", "|------|------|"]
|
||||
for t in info["types"]:
|
||||
lines.append(f"| {t['kind']} | `{t['name']}` |")
|
||||
for c in info["classes"]:
|
||||
lines.append(f"| class | `{c['name']}` |")
|
||||
else:
|
||||
lines.append("_None extracted._")
|
||||
lines += ["", "## Functions", ""]
|
||||
if info["functions"]:
|
||||
lines += [
|
||||
"| Name | Parameters | Param types (JSDoc) | Returns | Calls (same file / helpers) |",
|
||||
"|------|------------|---------------------|---------|-----------------------------|",
|
||||
]
|
||||
for fn in info["functions"]:
|
||||
js = fn["jsdoc"] or {}
|
||||
ptypes = ", ".join(f"{p['name']}: `{p['type']}`" for p in js.get("params", [])) or "—"
|
||||
ret = f"`{js.get('returns_type', 'unknown')}`"
|
||||
if js.get("returns_note"):
|
||||
ret += f" — {js['returns_note'][:120]}"
|
||||
params = fn["params"].replace("|", "\\|") or "(none)"
|
||||
lines.append(
|
||||
f"| `{fn['name']}` | `{params}` | {ptypes} | {ret} | see Call graph |"
|
||||
)
|
||||
if js.get("summary"):
|
||||
lines.append(f"| | _{js['summary']}_ | | | |")
|
||||
else:
|
||||
lines.append("_No top-level functions extracted._")
|
||||
if info["methods"]:
|
||||
lines += ["", "## Methods (class / object)", ""]
|
||||
lines += ["| Name | Parameters |", "|------|------------|"]
|
||||
for m in info["methods"]:
|
||||
lines.append(f"| `{m['name']}` | `{m['params'] or '(none)'}` |")
|
||||
lines += ["", "## What it imports / requires", ""]
|
||||
deps = info["imports"] + info["requires"]
|
||||
if deps:
|
||||
for d in deps:
|
||||
lines.append(f"- `{d}`")
|
||||
else:
|
||||
lines.append("_No imports detected._")
|
||||
lines += ["", "## Call graph (identifiers invoked)", ""]
|
||||
if info["calls"]:
|
||||
lines.append(", ".join(f"`{c}`" for c in info["calls"]))
|
||||
lines.append("")
|
||||
lines.append(
|
||||
"Each identifier is a call site in this file. Follow the import list to see the defining module; "
|
||||
"open that module’s MD for parameter and return types."
|
||||
)
|
||||
else:
|
||||
lines.append("_No local call identifiers extracted._")
|
||||
lines += [
|
||||
"",
|
||||
"## Return values (how to read this)",
|
||||
"",
|
||||
"- HTTP route handlers return Express `res.json(...)` bodies (see route docs).",
|
||||
"- Zapier `perform` functions return a **single object** (creates) or an **array** (triggers/searches).",
|
||||
"- Pricing functions return integer **cents** on `Quote.totalCents`.",
|
||||
"",
|
||||
]
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def render_rst(title: str, md_rel: str) -> str:
|
||||
return f"""{title}
|
||||
{'=' * len(title)}
|
||||
|
||||
.. include:: /{md_rel}
|
||||
:parser: myst_parser.sphinx_
|
||||
"""
|
||||
|
||||
|
||||
def main() -> None:
|
||||
OUT_MD.mkdir(parents=True, exist_ok=True)
|
||||
OUT_RST.mkdir(parents=True, exist_ok=True)
|
||||
index_entries = []
|
||||
rst_entries = []
|
||||
count = 0
|
||||
for pkg, src_root, suffixes in PACKAGES:
|
||||
if not src_root.exists():
|
||||
continue
|
||||
files = sorted(
|
||||
p
|
||||
for p in src_root.rglob("*")
|
||||
if p.suffix in suffixes and p.is_file() and not should_skip(p)
|
||||
)
|
||||
for f in files:
|
||||
if "node_modules" in f.parts:
|
||||
continue
|
||||
rel = rel_mod(pkg, src_root, f)
|
||||
info = parse_file(f)
|
||||
md = render_md(pkg, rel, info)
|
||||
md_path = OUT_MD / f"{rel}.md"
|
||||
md_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
md_path.write_text(md, encoding="utf-8")
|
||||
rst_path = OUT_RST / f"{rel}.rst"
|
||||
rst_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
rst_title = rel.replace("/", ".")
|
||||
# myst include path relative to sphinx conf dir
|
||||
include = f"../modules/{rel}.md"
|
||||
# Sphinx include is relative to the RST file: copy MD beside it.
|
||||
sibling_md = rst_path.with_suffix(".md")
|
||||
sibling_md.write_text(md, encoding="utf-8")
|
||||
rst_path.write_text(
|
||||
f"{rst_title}\n{'=' * len(rst_title)}\n\n"
|
||||
f"Generated API sheet for ``{rel}``.\n\n"
|
||||
f".. include:: {sibling_md.name}\n"
|
||||
f" :literal:\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
index_entries.append(rel)
|
||||
rst_entries.append(f"modules/{rel}")
|
||||
count += 1
|
||||
|
||||
idx = ["# Module documentation index", "", "One MD file per source module.", ""]
|
||||
for rel in index_entries:
|
||||
idx.append(f"- [{rel}]({rel}.md)")
|
||||
(OUT_MD / "README.md").write_text("\n".join(idx) + "\n", encoding="utf-8")
|
||||
|
||||
toctree = "\n ".join(rst_entries)
|
||||
sphinx_index = f"""Verae × Zapier module reference
|
||||
=================================
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
:caption: Modules
|
||||
|
||||
{toctree}
|
||||
"""
|
||||
(ROOT / "docs" / "sphinx" / "index.rst").write_text(sphinx_index, encoding="utf-8")
|
||||
print(f"wrote {count} module docs")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue