From 85b5b53ea90ad3b69545c3cf498929ca15c74f5b Mon Sep 17 00:00:00 2001 From: George Lambert Date: Fri, 11 Sep 2026 22:50:00 -0400 Subject: [PATCH] Fix catalog book links and rewrite PDF destinations to live HTTPS. Module and model book cards now point at generated index.html/index.pdf instead of directory URLs that 404. WeasyPrint file:// annotations are mapped to https://zapier.georgelambert.org/ paths that exist. Developer module, architecture, and research snapshot links target published files. --- TODO.md | 6 +- docs/00-sources/FUNCTIONS-REFERENCE.md | 180 +++++++++ docs/00-sources/LOGIN.md | 24 ++ docs/00-sources/MCP-REFERENCE.md | 362 +++++++++++++++++ docs/00-sources/MONGO.md | 80 ++++ docs/00-sources/PLATFORM-REFERENCE.md | 374 ++++++++++++++++++ docs/00-sources/RESTART.md | 62 +++ .../diagrams/01-high-level-architecture.svg | 79 ++++ .../docs/diagrams/02-security-boundaries.svg | 62 +++ .../docs/diagrams/03-auth-two-hop.svg | 73 ++++ .../docs/diagrams/04-flow-async-timestamp.svg | 60 +++ .../00-sources/docs/diagrams/05-flow-wait.svg | 46 +++ .../docs/diagrams/06-nats-topology.svg | 64 +++ .../docs/diagrams/07-operations-map.svg | 77 ++++ .../docs/diagrams/08-middleware-internals.svg | 58 +++ .../docs/diagrams/09-phase-roadmap.svg | 89 +++++ .../diagrams/10-workspace-integration.svg | 63 +++ .../docs/diagrams/11-zapier-billing.svg | 47 +++ .../12-peergos-ipfs-tiered-storage.svg | 72 ++++ docs/00-sources/getting-started-research.md | 2 +- docs/developer/modules/README.md | 36 +- docs/developer/modules/nats.md | 2 +- .../caddy/zapier.georgelambert.org.caddy | 19 + scripts/build-docs-site.py | 235 ++++++++++- scripts/check-doc-links.py | 111 ++++++ scripts/pdf-links.lua | 5 +- 26 files changed, 2253 insertions(+), 35 deletions(-) create mode 100644 docs/00-sources/FUNCTIONS-REFERENCE.md create mode 100644 docs/00-sources/LOGIN.md create mode 100644 docs/00-sources/MCP-REFERENCE.md create mode 100644 docs/00-sources/MONGO.md create mode 100644 docs/00-sources/PLATFORM-REFERENCE.md create mode 100644 docs/00-sources/RESTART.md create mode 100644 docs/00-sources/docs/diagrams/01-high-level-architecture.svg create mode 100644 docs/00-sources/docs/diagrams/02-security-boundaries.svg create mode 100644 docs/00-sources/docs/diagrams/03-auth-two-hop.svg create mode 100644 docs/00-sources/docs/diagrams/04-flow-async-timestamp.svg create mode 100644 docs/00-sources/docs/diagrams/05-flow-wait.svg create mode 100644 docs/00-sources/docs/diagrams/06-nats-topology.svg create mode 100644 docs/00-sources/docs/diagrams/07-operations-map.svg create mode 100644 docs/00-sources/docs/diagrams/08-middleware-internals.svg create mode 100644 docs/00-sources/docs/diagrams/09-phase-roadmap.svg create mode 100644 docs/00-sources/docs/diagrams/10-workspace-integration.svg create mode 100644 docs/00-sources/docs/diagrams/11-zapier-billing.svg create mode 100644 docs/00-sources/docs/diagrams/12-peergos-ipfs-tiered-storage.svg create mode 100644 packages/verae-ops/caddy/zapier.georgelambert.org.caddy create mode 100644 scripts/check-doc-links.py diff --git a/TODO.md b/TODO.md index 2d96480..e6811f0 100644 --- a/TODO.md +++ b/TODO.md @@ -30,9 +30,9 @@ | Doc | Purpose | |-----|---------| | [README.md](README.md) | Repo overview and quick start | -| [docs/architecture/overview.md](docs/architecture/overview.md) | System diagram and data flows | -| [docs/architecture/nats-subjects.md](docs/architecture/nats-subjects.md) | Subject topology and payloads | -| [docs/developer/modules/](docs/developer/modules/) | Per-module function reference | +| [docs/02-architecture/overview.md](docs/02-architecture/overview.md) | System diagram and data flows | +| [docs/02-architecture/nats-subjects.md](docs/02-architecture/nats-subjects.md) | Subject topology and payloads | +| [docs/modules/README.md](docs/modules/README.md) | Per-module function reference | | [docs/plans/phase-gates.md](docs/plans/phase-gates.md) | Gate commands and acceptance criteria | | [docs/api/middleware-openapi.yaml](docs/api/middleware-openapi.yaml) | Zapier-facing OpenAPI | diff --git a/docs/00-sources/FUNCTIONS-REFERENCE.md b/docs/00-sources/FUNCTIONS-REFERENCE.md new file mode 100644 index 0000000..d9d7d46 --- /dev/null +++ b/docs/00-sources/FUNCTIONS-REFERENCE.md @@ -0,0 +1,180 @@ +# Zapier function reference (CLI, core, SDK) + +**Grok start:** `db.platform_reference.findOne({ kind: "guide", key: "build-new-connector" })` +MCP tools: [MCP-REFERENCE.md](MCP-REFERENCE.md). This file is everything else. + +Each function has a Mongo document with the same sections as MCP: high-level, internals, typed inputs, outputs, related, twins, examples. + +## Core (`kind: "core_function"`) + +| Function | Signature | Category | +|----------|-----------|----------| +| `AfterResponseMiddleware` | `(response, z, bundle) => response | Promise` | middleware | +| `BeforeRequestMiddleware` | `(request, z, bundle) => request | Promise` | middleware | +| `CreatePerform` | `(z, bundle) => object | Promise` | perform | +| `PollingTriggerPerform` | `(z, bundle) => object[] | Promise` | perform | +| `SearchPerform` | `(z, bundle) => object[] | Promise` | perform | +| `WebhookTriggerPerform` | `(z, bundle) => object[] | Promise` | perform | +| `createAppTester` | `createAppTester(appRaw, options?) → (func|request, bundle?) => Promise` | testing | +| `performBuffer` | `(z, BufferedBundle) => Promise<{[id]: {outputData?, error?}}>` | perform | +| `z.JSON.parse` | `z.JSON.parse(text) → any` | json | +| `z.JSON.stringify` | `z.JSON.stringify(value) → string` | json | +| `z.cache.delete` | `z.cache.delete(key) → Promise` | cache | +| `z.cache.get` | `z.cache.get(key) → Promise` | cache | +| `z.cache.set` | `z.cache.set(key, value, ttl?, scope?, nx?) → Promise` | cache | +| `z.console` | `z.console.log|info|warn|error(...args) → void` | debug | +| `z.cursor.get` | `z.cursor.get() → Promise` | polling | +| `z.cursor.set` | `z.cursor.set(cursor) → Promise` | polling | +| `z.dehydrate` | `z.dehydrate(func, inputData?, cacheExpiration?) → string` | hydration | +| `z.dehydrateFile` | `z.dehydrateFile(func, inputData?, cacheExpiration?) → string` | hydration | +| `z.errors.CheckError` | `throw new z.errors.CheckError(message?)` | errors | +| `z.errors.DehydrateError` | `throw new z.errors.DehydrateError(message?)` | errors | +| `z.errors.Error` | `throw new z.errors.Error(message, code?, status?)` | errors | +| `z.errors.ExpiredAuthError` | `throw new z.errors.ExpiredAuthError(message?)` | errors | +| `z.errors.HaltedError` | `throw new z.errors.HaltedError(message?)` | errors | +| `z.errors.RefreshAuthError` | `throw new z.errors.RefreshAuthError(message?)` | errors | +| `z.errors.ResponseError` | `throw new z.errors.ResponseError(response)` | errors | +| `z.errors.StopRequestError` | `throw new z.errors.StopRequestError(message?)` | errors | +| `z.errors.ThrottledError` | `throw new z.errors.ThrottledError(message, delaySeconds?)` | errors | +| `z.generateCallbackUrl` | `z.generateCallbackUrl() → string` | callback | +| `z.hash` | `z.hash(algorithm, data, encoding?, input_encoding?) → string` | crypto | +| `z.request` | `z.request(url, options?) → Promise` | http | +| `z.stashFile` | `z.stashFile(input, knownLength?, filename?, contentType?) → string` | files | +| `zapier.tools.env.inject` | `zapier.tools.env.inject(filename?)` | testing | + +## Platform CLI (`kind: "cli_function"`) + +| Command | Usage | +|---------|-------| +| `analytics` | `zapier-platform analytics` | +| `build` | `zapier-platform build` | +| `canary:create` | `zapier-platform canary:create VERSIONFROM VERSIONTO` | +| `canary:delete` | `zapier-platform canary:delete VERSIONFROM VERSIONTO` | +| `canary:list` | `zapier-platform canary:list` | +| `convert` | `zapier-platform convert PATH` | +| `delete:integration` | `zapier-platform delete:integration` | +| `delete:version` | `zapier-platform delete:version VERSION` | +| `deprecate` | `zapier-platform deprecate VERSION DATE` | +| `describe` | `zapier-platform describe` | +| `env:get` | `zapier-platform env:get VERSION` | +| `env:set` | `zapier-platform env:set VERSION [KEY-VALUE PAIRS...]` | +| `env:unset` | `zapier-platform env:unset VERSION [KEYS...]` | +| `history` | `zapier-platform history` | +| `init` | `zapier-platform init PATH` | +| `integrations` | `zapier-platform integrations` | +| `invoke` | `zapier-platform invoke [ACTIONTYPE] [ACTIONKEY] [AUTHDATA]` | +| `jobs` | `zapier-platform jobs` | +| `legacy` | `zapier-platform legacy VERSION` | +| `link` | `zapier-platform link` | +| `login` | `zapier-platform login` | +| `logout` | `zapier-platform logout` | +| `logs` | `zapier-platform logs` | +| `migrate` | `zapier-platform migrate FROMVERSION TOVERSION [PERCENT]` | +| `promote` | `zapier-platform promote VERSION` | +| `pull` | `zapier-platform pull` | +| `push` | `zapier-platform push` | +| `register` | `zapier-platform register [TITLE]` | +| `scaffold` | `zapier-platform scaffold ACTIONTYPE NOUN` | +| `team:add` | `zapier-platform team:add EMAIL ROLE [MESSAGE]` | +| `team:get` | `zapier-platform team:get` | +| `team:remove` | `zapier-platform team:remove` | +| `test` | `zapier-platform test` | +| `upload` | `zapier-platform upload` | +| `users:add` | `zapier-platform users:add EMAIL [VERSION]` | +| `users:get` | `zapier-platform users:get` | +| `users:links` | `zapier-platform users:links` | +| `users:remove` | `zapier-platform users:remove EMAIL` | +| `validate` | `zapier-platform validate` | +| `versions` | `zapier-platform versions` | + +## SDK (`kind: "sdk_function"`) + +| CLI | TypeScript | Category | MCP twin | +|-----|------------|----------|----------| +| `get-profile` | `zapier.getProfile` | Accounts | — | +| `login` | `zapier.login` | Accounts | — | +| `logout` | `zapier.logout` | Accounts | — | +| `signup` | `zapier.signup` | Accounts | — | +| `get-action` | `zapier.getAction` | Actions | inspect_zapier_actions | +| `get-action-input-fields-schema` | `zapier.getActionInputFieldsSchema` | Actions | inspect_zapier_actions | +| `list-action-input-field-choices` | `zapier.listActionInputFieldChoices` | Actions | inspect_zapier_actions (enum_property) | +| `list-action-input-fields` | `zapier.listActionInputFields` | Actions | inspect_zapier_actions | +| `list-actions` | `zapier.listActions` | Actions | inspect_zapier_actions / discover_zapier_actions | +| `run-action` | `zapier.runAction` | Actions | execute_zapier_read_action / execute_zapier_write_action | +| `get-app` | `zapier.getApp` | Apps | discover_zapier_actions | +| `list-apps` | `zapier.listApps` | Apps | discover_zapier_actions | +| `create-client-credentials` | `zapier.createClientCredentials` | Client Credentials | — | +| `delete-client-credentials` | `zapier.deleteClientCredentials` | Client Credentials | — | +| `list-client-credentials` | `zapier.listClientCredentials` | Client Credentials | — | +| `cancel-durable-run` | `zapier.cancelDurableRun` | Code Workflows | — | +| `create-workflow` | `zapier.createWorkflow` | Code Workflows | — | +| `delete-workflow` | `zapier.deleteWorkflow` | Code Workflows | — | +| `disable-workflow` | `zapier.disableWorkflow` | Code Workflows | — | +| `enable-workflow` | `zapier.enableWorkflow` | Code Workflows | — | +| `get-durable-run` | `zapier.getDurableRun` | Code Workflows | — | +| `get-trigger-run` | `zapier.getTriggerRun` | Code Workflows | — | +| `get-workflow` | `zapier.getWorkflow` | Code Workflows | — | +| `get-workflow-run` | `zapier.getWorkflowRun` | Code Workflows | — | +| `get-workflow-version` | `zapier.getWorkflowVersion` | Code Workflows | — | +| `list-durable-runs` | `zapier.listDurableRuns` | Code Workflows | — | +| `list-workflow-runs` | `zapier.listWorkflowRuns` | Code Workflows | — | +| `list-workflow-versions` | `zapier.listWorkflowVersions` | Code Workflows | — | +| `list-workflows` | `zapier.listWorkflows` | Code Workflows | — | +| `publish-workflow-version` | `zapier.publishWorkflowVersion` | Code Workflows | — | +| `run-durable` | `zapier.runDurable` | Code Workflows | — | +| `trigger-workflow` | `zapier.triggerWorkflow` | Code Workflows | — | +| `update-workflow` | `zapier.updateWorkflow` | Code Workflows | — | +| `create-connection` | `zapier.createConnection` | Connections | manage_zapier_connections | +| `find-first-connection` | `zapier.findFirstConnection` | Connections | list_zapier_connections | +| `find-unique-connection` | `zapier.findUniqueConnection` | Connections | list_zapier_connections | +| `get-connection` | `zapier.getConnection` | Connections | — | +| `get-connection-start-url` | `zapier.getConnectionStartUrl` | Connections | manage_zapier_connections | +| `list-connections` | `zapier.listConnections` | Connections | list_zapier_connections | +| `wait-for-new-connection` | `zapier.waitForNewConnection` | Connections | manage_zapier_connections (after auth_url) | +| `curl` | `zapier.curl` | HTTP Requests | write_code_action / z.request analog | +| `create-table` | `zapier.createTable` | Tables | — | +| `create-table-fields` | `zapier.createTableFields` | Tables | — | +| `create-table-records` | `zapier.createTableRecords` | Tables | — | +| `delete-table` | `zapier.deleteTable` | Tables | — | +| `delete-table-fields` | `zapier.deleteTableFields` | Tables | — | +| `delete-table-records` | `zapier.deleteTableRecords` | Tables | — | +| `get-table` | `zapier.getTable` | Tables | — | +| `get-table-record` | `zapier.getTableRecord` | Tables | — | +| `list-table-fields` | `zapier.listTableFields` | Tables | — | +| `list-table-records` | `zapier.listTableRecords` | Tables | — | +| `list-tables` | `zapier.listTables` | Tables | — | +| `update-table-records` | `zapier.updateTableRecords` | Tables | — | +| `ack-trigger-inbox-messages` | `zapier.ackTriggerInboxMessages` | Triggers | — | +| `create-trigger-inbox` | `zapier.createTriggerInbox` | Triggers | — | +| `delete-trigger-inbox` | `zapier.deleteTriggerInbox` | Triggers | — | +| `drain-trigger-inbox` | `zapier.drainTriggerInbox` | Triggers | — | +| `ensure-trigger-inbox` | `zapier.ensureTriggerInbox` | Triggers | — | +| `get-trigger-inbox` | `zapier.getTriggerInbox` | Triggers | — | +| `get-trigger-input-fields-schema` | `zapier.getTriggerInputFieldsSchema` | Triggers | — | +| `lease-trigger-inbox-messages` | `zapier.leaseTriggerInboxMessages` | Triggers | — | +| `list-trigger-inbox-messages` | `zapier.listTriggerInboxMessages` | Triggers | — | +| `list-trigger-inboxes` | `zapier.listTriggerInboxes` | Triggers | — | +| `list-trigger-input-field-choices` | `zapier.listTriggerInputFieldChoices` | Triggers | — | +| `list-trigger-input-fields` | `zapier.listTriggerInputFields` | Triggers | — | +| `list-triggers` | `zapier.listTriggers` | Triggers | — | +| `pause-trigger-inbox` | `zapier.pauseTriggerInbox` | Triggers | — | +| `release-trigger-inbox-messages` | `zapier.releaseTriggerInboxMessages` | Triggers | — | +| `resume-trigger-inbox` | `zapier.resumeTriggerInbox` | Triggers | — | +| `update-trigger-inbox` | `zapier.updateTriggerInbox` | Triggers | — | +| `watch-trigger-inbox` | `zapier.watchTriggerInbox` | Triggers | — | +| `add` | `zapier.add` | Utilities | — | +| `build-manifest` | `zapier.buildManifest` | Utilities | — | +| `feedback` | `zapier.feedback` | Utilities | send_feedback | +| `generate-app-types` | `zapier.generateAppTypes` | Utilities | — | +| `get-login-config-path` | `zapier.getLoginConfigPath` | Utilities | — | +| `init` | `zapier.init` | Utilities | — | +| `mcp` | `zapier.mcp` | Utilities | hosted Zapier MCP (different server) | + +## Query + +```js +db.platform_reference.findOne({ kind: "guide", key: "zapier-functions" }) +db.platform_reference.find({ kind: "core_function", key: "z.request" }) +db.platform_reference.find({ kind: "sdk_function", "meta.category": "Actions" }) +``` + diff --git a/docs/00-sources/LOGIN.md b/docs/00-sources/LOGIN.md new file mode 100644 index 0000000..adea952 --- /dev/null +++ b/docs/00-sources/LOGIN.md @@ -0,0 +1,24 @@ +# Zapier logins (you must finish these in a browser) + +CLIs are installed. They cannot push, list your apps, or run live actions until you authenticate. + +```bash +source scripts/dev-env.sh + +# Publish integrations (deploy key → ~/.zapierrc) +zapier-platform login +# SSO-only account: +# zapier-platform login --sso + +# Consume 9k apps from code +zapier-sdk login + +# Then from scratch/oauth2-typescript: +cd scratch/oauth2-typescript +zapier-platform register "My OAuth2 Lab" +zapier-platform push +``` + +Hosted MCP (optional, for in-chat `discover_zapier_actions`): add this client connector to `https://mcp.zapier.com/api/v1/connect` (OAuth). Mongo MCP is already in `~/.grok/config.toml`; restart Grok so this session picks it up, and keep `./scripts/mongo-tunnel.sh` running. + +Do not commit `~/.zapierrc` or SDK credentials. diff --git a/docs/00-sources/MCP-REFERENCE.md b/docs/00-sources/MCP-REFERENCE.md new file mode 100644 index 0000000..31fe837 --- /dev/null +++ b/docs/00-sources/MCP-REFERENCE.md @@ -0,0 +1,362 @@ +# Zapier MCP function reference + +**Grok start:** `db.platform_reference.findOne({ kind: "guide", key: "build-new-connector" })` +or [PLATFORM-REFERENCE.md](PLATFORM-REFERENCE.md). This file is the MCP branch. + +Mongo: `db.platform_reference.find({ kind: "mcp_function" })` — one document per tool, with JSON Schema, typed inputs, outputs, internals, related tools, and code. + +Hosted server: `https://mcp.zapier.com/api/v1/connect` (closed source, Streamable HTTP). +Plugin repo only: `repos/zapier-mcp` (skills + manifests). +Live input schemas captured in `raw/site-extras/zapier-mcp-tools-gumloop.json`. + +Official docs still list **14** meta-tools. Live servers expose **17**: the 14 plus `list_zapier_connections`, `manage_zapier_connections`, and rolling-out `write_code_action`. + +--- + +## Call graph + +``` +auto_provision_mcp (also runs on OAuth connect) + │ + ▼ +discover_zapier_actions ──► enable_zapier_action ──► manage_zapier_connections + │ │ + └──────────────► inspect_zapier_actions ◄── list_zapier_connections + (repeat for enums / dynamic fields) + ┌───────────────┴───────────────┐ + ▼ ▼ + execute_zapier_read_action execute_zapier_write_action + │ │ + └──────── write_code_action ────┘ + +Skills: list_zapier_skills → get_zapier_skill("zapier:onboarding") + create_zapier_skill / update_zapier_skill / delete_zapier_skill +Config UI: get_configuration_url +Feedback: send_feedback +``` + +Safety: reads run free; writes need explicit user approval after you show the payload. +Billing: each **successful** execute costs **2 Zapier tasks**. Meta-tools do not. Failures are free. + +Never invent `selected_api` (Gmail is `GoogleMailV2CLIAPI`, not `GmailCLIAPI`) or action keys. Always `inspect_zapier_actions` first. + +--- + +## Action management + +### `discover_zapier_actions` + +Search the 9,000+ app catalog for actions this server *could* enable. Call before saying an app is unavailable. + +**Internals:** Catalog lookup (same directory as SDK `listApps` / `listActions`). No OAuth, no mutation. Returns `selected_api` that later tools require verbatim. + +**Input** + +```ts +type Input = { + app?: string; // Search by name. Omit for popular apps. +}; +``` + +**Output (documented):** `{ apps: [{ app, selected_api, actions[] }] }` + +**Related:** `enable_zapier_action`, `inspect_zapier_actions`, `manage_zapier_connections` +**SDK twin:** `zapier.listApps({ search })`, `zapier-sdk list-apps --search` + +```ts +await client.callTool({ + name: "discover_zapier_actions", + arguments: { app: "gmail" }, +}); +``` + +--- + +### `enable_zapier_action` + +Add one action (or `*` / omit for all) for a catalog app to this server. + +**Internals:** Writes mcp.zapier.com server config. If the user has no connection, returns `auth_url`. Does not call the partner API. + +**Input** + +```ts +type Input = { + selected_api: string; // e.g. "GoogleMailV2CLIAPI" from discover + app_display_name?: string; // "Gmail" — for friendly confirmations + action?: string; // key, or "*" / omit for all +}; +``` + +**Output:** enabled action list, which execute tool to use (read vs write), optional `auth_url`. + +**Related:** `discover_zapier_actions`, `inspect_zapier_actions`, `disable_zapier_action`, `manage_zapier_connections` + +```ts +await client.callTool({ + name: "enable_zapier_action", + arguments: { + selected_api: "GoogleMailV2CLIAPI", + app_display_name: "Gmail", + action: "find_email", + }, +}); +``` + +--- + +### `disable_zapier_action` + +Remove one action, or every action for `selected_api`. Does not revoke the Zapier connection. + +**Input** + +```ts +type Input = { + selected_api: string; + app_display_name?: string; + action?: string; // omit = disable the whole app on this server +}; +``` + +**Related:** `inspect_zapier_actions`, `enable_zapier_action` + +--- + +### `inspect_zapier_actions` + +**Call this first before every execute.** Lists enabled apps/actions and resolves dynamic fields. + +**Internals:** Reads this server's enabled-action table. With `tool_name` + `enum_property` it hits the same Platform dynamic-dropdown endpoint as the visual builder. With parent `params` it returns `dynamic_properties_schema` for fields that depend on earlier answers (spreadsheet → worksheet). Does not run the partner action. + +**Input** + +```ts +type Input = { + selected_api?: string; + action?: string; + tool_name?: string; // collision-safe id from a prior inspect + connection_id?: number | string; // only if not using the default account + enum_property?: string; // field with is_dynamic_enum: true + enum_search?: string; + enum_cursor?: string; + params?: Record; // parent values for dependent fields +}; +``` + +**Output (documented):** apps → actions with `action`, `tool_name`, execute tool name, parameter schema (`is_dynamic_enum`, `dynamic_properties_depends_on`), `connections.default`, plus `dynamic_enum_values` / `dynamic_properties_schema` on follow-up calls. + +**Related:** both `execute_*`, `discover_zapier_actions`, `list_zapier_connections` +**SDK twin:** `getActionInputFieldsSchema`, `list-action-input-field-choices` + +```ts +await client.callTool({ name: "inspect_zapier_actions", arguments: {} }); + +await client.callTool({ + name: "inspect_zapier_actions", + arguments: { + tool_name: "SlackCLIAPI.send_channel_message", + enum_property: "channel", + enum_search: "launches", + }, +}); +``` + +--- + +### `auto_provision_mcp` + +One-shot setup from the user's **own** existing Zapier connections (not shared teammates'). Also runs automatically after OAuth connect. Returns enabled apps plus top Zap titles (skill ideas). No inputs. + +**Related:** `inspect_zapier_actions`, `enable_zapier_action` + +--- + +## Execution + +### `execute_zapier_read_action` + +Run a search / lookup / get. No confirmation required. **2 tasks** on success. + +**Internals:** Maps to Zapier Platform search/read `perform` (`runAction({ actionType: "search"|"read" })`) using the user's default connection (or `connection_id`). `params` → `bundle.inputData`. Empty results = not found, not an error. + +**Input** + +```ts +type Input = { + selected_api: string; + action: string; // exact key from inspect — never guess + tool_name?: string; // preferred when keys collide across apps + connection_id?: number | string; + params?: Record; // nest dynamic fields under dynamic_properties +}; +``` + +**Output:** action-specific records (`results[]` or `data[]`). 401 → `manage_zapier_connections`. + +**Related:** `inspect_zapier_actions`, `discover_zapier_actions`, `enable_zapier_action` +**SDK twin:** `repos/sdk/examples/by-app/*/find-*.ts` + +```ts +await client.callTool({ + name: "execute_zapier_read_action", + arguments: { + selected_api: "GoogleMailV2CLIAPI", + action: "find_email", + tool_name: "GoogleMailV2CLIAPI.find_email", + params: { query: "from:sarah@acme.com newer_than:7d" }, + }, +}); +``` + +If the user said only a first name, do **not** take the first hit — list candidates. + +--- + +### `execute_zapier_write_action` + +Create / update / send. **Show the payload and wait for explicit approval.** **2 tasks** on success. Not rolled back. + +**Input:** same shape as the read execute. + +**Related:** `inspect_zapier_actions`, `execute_zapier_read_action` +**SDK twin:** `repos/sdk/examples/by-app/gmail/send-email.ts` + +```ts +// 1) inspect + resolve channel enum 2) confirm with user 3) write +await client.callTool({ + name: "execute_zapier_write_action", + arguments: { + selected_api: "SlackCLIAPI", + action: "send_channel_message", + tool_name: "SlackCLIAPI.send_channel_message", + params: { channel: "C01234567", text: "Release shipped, monitoring now" }, + }, +}); +``` + +--- + +### `write_code_action` (rolling out) + +Generate a sandboxed custom action when the catalog has no match. Auth is injected from the connected account — **never put secrets in `requirements`**. Same-name call overwrites. + +**Internals:** Zapier generates code and runs later invokes in a sandbox with the app connection. Closest SDK analog is `zapier.fetch(url, { connection, method })`. + +**Input** + +```ts +type Input = { + selected_api: string; + code_action_name: string; // e.g. "list_channel_users" + requirements: string; // natural language; endpoint, filters, pagination +}; +``` + +**Related:** `discover_zapier_actions`, `inspect_zapier_actions` + +--- + +## Connections + +### `list_zapier_connections` + +List OAuth grants for one app. Default = the user's own accounts. Set `include_shared: true` only if they asked. + +**Input** + +```ts +type Input = { + selected_api: string; + include_shared?: boolean; + limit?: number; // 1–100, default 20 + cursor?: string; +}; +``` + +**Output (documented):** `{ connections: [{ connection_id, title, owner, is_default, expired }], next_cursor? }` + +**SDK twin:** `zapier.findFirstConnection`, `zapier-sdk find-first-connection` + +--- + +### `manage_zapier_connections` + +Mint an `auth_url` and/or set `default_connection_id`. An app **cannot execute** until it has a default connection. After the user finishes OAuth, list then optionally set default. + +**Input** + +```ts +type Input = { + selected_api: string; // verbatim from discover/inspect + app_display_name?: string; + default_connection_id?: number | string; +}; +``` + +**SDK twin:** `zapier-sdk create-connection` / `get-connection-start-url` + +--- + +## Skills, config, feedback + +| Tool | Input | What it does | +|------|--------|----------------| +| `list_zapier_skills` | `{}` | Names + one-line descriptions. Catalog is dynamic. | +| `get_zapier_skill` | `{ name }` | Full Markdown. Packaged: `zapier:onboarding`. | +| `create_zapier_skill` | `{ name, description, skillDefinition }` | Persist Markdown. Lock IDs with `ZapierAction[app:action](param: "id", runtime)`. Resolve schemas via `inspect_zapier_actions` (tool text still says `list_enabled_zapier_actions`). | +| `update_zapier_skill` | `{ name, description?, skillDefinition? }` | Patch. | +| `delete_zapier_skill` | `{ name }` | Permanent. | +| `get_configuration_url` | `{}` | Dashboard URL for this server. | +| `send_feedback` | `{ feedback: string(1–2000), feedback_positive: boolean }` | Product inbox. | + +Official first-run prompt: + +```text +Run the Zapier onboarding skill using get_zapier_skill with name "zapier:onboarding" and follow its instructions. +``` + +--- + +## Client connection (TypeScript) + +```ts +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"; + +const transport = new StreamableHTTPClientTransport( + new URL("https://mcp.zapier.com/api/v1/connect"), + { requestInit: { headers: { Authorization: `Bearer ${token}` } } }, +); +const client = new Client({ name: "app", version: "1.0.0" }); +await client.connect(transport); +const tools = await client.listTools(); +``` + +Listed clients use OAuth to that same URL and never see the token. + +--- + +## Plugin skills (not server tools) + +In `repos/zapier-mcp/plugins/zapier/skills/`: + +| Skill | When | +|-------|------| +| `zapier-onboard` | First connect / “what is Zapier MCP” | +| `zapier-demo` | One read action, live | +| `zapier-explore` | Role-tailored toolkit | +| `zapier-status` | Health / audit / diagnose | + +Lifecycle rules (`zapier-lifecycle.mdc`): prefer native app MCP over Zapier when both exist; never call both. + +--- + +## Reload + +```bash +# refresh live schemas (optional) +# then: +source ~/.mcp-env +.venv/bin/python scripts/ingest-mcp-reference.py +``` diff --git a/docs/00-sources/MONGO.md b/docs/00-sources/MONGO.md new file mode 100644 index 0000000..7ba8478 --- /dev/null +++ b/docs/00-sources/MONGO.md @@ -0,0 +1,80 @@ +# Zapier vendor dataset on NS1 MongoDB + +Canonical store: **MongoDB 7 in Docker on `70.88.205.138` (NS1)**, bound to **127.0.0.1:27017 only**. + +Database: `zapier` +Collections: `apps` (9,986), `templates` (332,441), `help_articles` (1,272), **`platform_reference` (1,489 — CLI, `z.*` functions, schema, official docs, example apps)** +Meta: `meta.ingest` + +## Connect (always through SSH) + +```bash +# keep this running +./scripts/mongo-tunnel.sh +# or: ssh -N -L 27017:127.0.0.1:27017 ns1 +``` + +Then on this Mac: + +| Tool | Connection | +|------|------------| +| mongosh | `source ~/.mcp-env && mongosh "$ZAPIER_MONGO_URI"` | +| Compass / Cursor MongoDB plugin | `mongodb://127.0.0.1:27017` after the tunnel is up (auth: user `zapier`, authSource `admin`) | +| Grok MCP (`mongodb` plugin) | `MDB_MCP_CONNECTION_STRING` in `~/.mcp-env` and `~/.grok/config.toml` | +| Python | `.venv/bin/python scripts/load-mongodb.py` | + +Do **not** point Compass at `mongodb://70.88.205.138:27017`. Port 27017 is not on the public interface. + +Credentials live in `~/.mcp-env` (mode 600) and `~/.config/zapier-mongo/root.pass` on NS1. Not in git. + +## Reload after a scrape + +```bash +./scripts/mongo-tunnel.sh & +source ~/.mcp-env +.venv/bin/python scripts/load-mongodb.py +``` + +## Installed Grok / editor tools + +- Grok plugins: **mongodb**, **mongodb-atlas** (xAI Official / MongoDB agent-skills) +- Grok MCP server: `mongodb-mcp-server@3` via npx +- Cursor + VS Code: **mongodb.mongodb-vscode** +- Local CLI: **mongosh** 2.9.2 +- NS1: Docker container `zapier-mongo` (`mongo:7`), restart unless-stopped + +## Heavy jobs run on NS1 + +The scrape/load workspace on the server is `/home/marchon/zapier-research` (home is on SSD3). + +```bash +ssh ns1 +cd /home/marchon/zapier-research +# resume scrapes, then: +export ZAPIER_MONGO_URI="mongodb://zapier:$(cat ~/.config/zapier-mongo/root.pass)@127.0.0.1:27017/zapier?authSource=admin" +.venv/bin/python scripts/load-mongodb.py +``` + +Laptop is for Compass/Grok/tunnel only. Do not run the 10k-site scrapes locally. + + +## Collections + +| Collection | Documents | Contents | +|------------|----------:|----------| +| `apps` | 9,986 | Identity, contacts, Zapier controls, commercial flags, featured templates, overview | +| `templates` | 332,441 | Full public Zap recipe catalog (title, URL, app pair) | +| `help_articles` | 1,272 | Zapier help center (get-started, common problems, platform FAQ) | +| `platform_reference` | ~1,777 | Toolkit + functions + **schema_json**, **api_function** (55), **platform_news** (49), **template** (32). Start: `findOne({kind:"guide", key:"build-new-connector"})`. | +| `meta` | 1 | Last ingest timestamp | + +Reload the developer collection on NS1 after recloning repos: + +```bash +ssh ns1 +cd /home/marchon/zapier-research +./scripts/clone-zapier-repos.sh +export ZAPIER_RESEARCH_ROOT=/home/marchon/zapier-research +export ZAPIER_MONGO_URI="mongodb://zapier:$(cat ~/.config/zapier-mongo/root.pass)@127.0.0.1:27017/zapier?authSource=admin" +.venv/bin/python scripts/ingest-platform-reference.py +``` diff --git a/docs/00-sources/PLATFORM-REFERENCE.md b/docs/00-sources/PLATFORM-REFERENCE.md new file mode 100644 index 0000000..19b9758 --- /dev/null +++ b/docs/00-sources/PLATFORM-REFERENCE.md @@ -0,0 +1,374 @@ +# START HERE — Grok playbook for any Zapier work + +**Grok must begin every Zapier task by loading this document, or the same playbook in Mongo:** + +```js +db.platform_reference.findOne({ kind: "guide", key: "build-new-connector" }) +``` + +Then route: + +| User wants… | Next | +|-------------|------| +| AI client / no-code actions across 9k apps | [MCP-REFERENCE.md](MCP-REFERENCE.md) and `kind: "mcp_function"` | +| **Any Zapier function** (CLI, `z.*`, SDK) | [FUNCTIONS-REFERENCE.md](FUNCTIONS-REFERENCE.md) and `kind: "cli_function" \| "core_function" \| "sdk_function"` (playbook: `guide/zapier-functions`) | +| Code that calls existing Zapier apps | `kind: "sdk_function"` (78 commands + `zapier.runAction` twins) | +| A new directory integration to publish | `cli_function` + `core_function` + `schema_json` + `template` · golden app `scratch/oauth2-typescript` | +| Public REST (Workflow / Actions / Inbox) | `kind: "api_function"` | +| Embed Zapier in a product | docs sections `embed`, `white-label`, `openapi` | +| Skill | `/zapier-build` (`.grok/skills/zapier-build`) | + +--- + +# Zapier platform reference (for building connectors and applications) + +Separate from the vendor catalog (`apps`, `templates`, `help_articles`). This is the **developer toolkit** Grok should read before writing a new Zapier integration, SDK client, MCP install, or embed. + +**Mongo collection:** `zapier.platform_reference` on NS1 (~1,507 documents, text index `ref_text`). +**Local clones:** `repos/` (16 official GitHub repos, shallow). +**Local dump:** `raw/platform-reference.jsonl` (~10 MB). +**Official docs dump:** NS1 `raw/docs/` (409 markdown pages) plus `raw/site-extras/` and `raw/openapi/`. + +Query after `./scripts/mongo-tunnel.sh` and `source ~/.mcp-env`: + +```js +// mongosh "$ZAPIER_MONGO_URI" +use zapier +db.platform_reference.find({ kind: "cli_command", key: "init" }) +db.platform_reference.find({ $text: { $search: "oauth2 refreshAccessToken" } }, { score: { $meta: "textScore" } }).sort({ score: { $meta: "textScore" } }) +db.platform_reference.find({ kind: "core_function" }) +db.platform_reference.findOne({ kind: "guide", key: "build-new-connector" }) +``` + +Reload: + +```bash +ssh ns1 'cd /home/marchon/zapier-research && \ + export ZAPIER_RESEARCH_ROOT=/home/marchon/zapier-research && \ + export ZAPIER_MONGO_URI="mongodb://zapier:$(cat ~/.config/zapier-mongo/root.pass)@127.0.0.1:27017/zapier?authSource=admin" && \ + .venv/bin/python scripts/ingest-platform-reference.py' +``` + +--- + +## Which surface to use + +| Goal | Surface | Package / repo | Collection `kind` | +|------|---------|----------------|-------------------| +| Publish an app to the Zapier directory | **Platform CLI** (or Platform UI) | `zapier-platform-cli` + `zapier-platform-core` — `zapier/zapier-platform` | `cli_command`, `core_function`, `schema_type`, `example_app` | +| Call existing Zapier apps from code | **Zapier SDK** | `@zapier/zapier-sdk` / `@zapier/zapier-sdk-cli` — `zapier/sdk` | `sdk_command`, `official_doc` section `sdk` | +| Connect an AI client with no code | **Zapier MCP** | hosted `mcp.zapier.com` + `zapier/zapier-mcp` plugins | `mcp_function` (17 tools) + [MCP-REFERENCE.md](MCP-REFERENCE.md) | +| Route “install Zapier” for an agent | **install-zapier** | `npx @zapier/install-zapier` — `zapier/install-zapier` | `skill`, `repo` | +| Prototype a local connector artifact | **Connectors** (not production) | `zapier/connectors` | `connector` | +| Embed Zapier in your product | Powered by Zapier / White Label | docs + OpenAPI | `official_doc` `embed` / `white-label`, `openapi` | + +Do **not** mix `zapier-platform …` (build integrations) with `zapier-sdk …` (consume integrations). +Do **not** recommend retired AI Actions / NLA. +Do **not** quote “9,000+ apps” for Connectors (prototype, ~29 apps). + +Start from `kind: "guide", key: "build-new-connector"` in Mongo. + +--- + +## Collection map + +| `kind` | n | What it is | +|--------|--:|------------| +| `official_doc` | 409 | Every `docs.zapier.com` page (markdown), tagged by `section` | +| `repo_doc` | 759 | Markdown from the cloned GitHub repos | +| `skill` | 63 | `SKILL.md` files (SDK, install-zapier, connectors, MCP plugins) | +| `schema_type` | 63 | `AppSchema`, `AuthenticationOAuth2ConfigSchema`, `TriggerSchema`, … | +| `cli_command` | 40 | `zapier-platform` commands (full usage, flags, examples) | +| `example_app` | 31 | Official templates under `zapier-platform/example-apps/*` | +| `connector` | 29 | Prototype connectors (Sheets, Notion, Linear, …) | +| `core_function` | 22 | `z.request`, `z.errors.*`, cache, cursor, dehydrate, … | +| `sdk_command` | 20 | `zapier-sdk` / SDK CLI: `list-apps`, `run-action`, … | +| `site_extra` | 22 | Leftover public zapier.com / developer surfaces + llms.txt | +| `repo` | 16 | Cloned repo index + README | +| `core_type` | 9 | `Bundle`, `ZObject`, perform signatures, middleware | +| `openapi` | 5 | Actions, Connections, Trigger Inbox, Promotions, Workflow API schema | +| `mcp_function` | 17 | Hosted Zapier MCP meta-tools (typed I/O, internals, examples) | +| `cli_function` | 40 | `zapier-platform` commands (MCP-style: typed flags, internals, examples) | +| `sdk_function` | 78 | `zapier-sdk` CLI + `zapier.camelCase` TypeScript twins | +| `api_function` | 55 | Workflow / Actions / Connections / Trigger Inbox / Promotions endpoints | +| `platform_news` | 49 | Platform CLI/core changelog 2025–2026 | +| `template` | 32 | Full example-app sources + golden `scratch/oauth2-typescript` | +| `schema_json` | 2 | `exported-schema.json` + official `definition.json` | +| `guide` | 4 | **Start:** `build-new-connector`. Then `zapier-mcp`, `zapier-functions`, `coding-set` | + +`official_doc.section` breakdown: integration-builder 200, embed 78, api-reference 53, mcp 44, white-label 21, sdk 10, plus install/connectors. + +--- + +## Platform CLI (`zapier-platform`) + +Install: `npm install -g zapier-platform-cli` +Deprecated alias: `zapier` (same commands). Prefer `zapier-platform`. + +### Scaffold → ship + +```bash +zapier-platform login +zapier-platform init my-app --template oauth2 --language typescript +cd my-app && npm install +zapier-platform scaffold trigger contact +zapier-platform scaffold create contact +zapier-platform validate +zapier-platform invoke auth start +zapier-platform invoke auth test +zapier-platform invoke trigger new_contact +zapier-platform test +zapier-platform register "My App" +zapier-platform push +zapier-platform promote 1.0.0 +``` + +`init --template` values: `basic-auth`, `callback`, `custom-auth`, `digest-auth`, `dynamic-dropdown`, `files`, `line-items`, `minimal`, `oauth1-trello`, `oauth2`, `openai`, `search-or-create`, `session-auth`. Add `--language typescript` and/or `--module esm`. + +### Full command list + +| Command | Usage | When | +|---------|-------|------| +| `login` / `logout` | `zapier-platform login` | Auth the CLI to developer.zapier.com | +| `init` | `init PATH` | New project from a template | +| `convert` | `convert PATH` | Visual Builder → CLI (`-i` integration id, `-v` version) | +| `scaffold` | `scaffold ACTIONTYPE NOUN` | Add `trigger\|search\|create\|resource` | +| `link` | `link` | Attach cwd to an existing integration | +| `register` | `register [TITLE]` | Create the integration on Zapier | +| `validate` | `validate` | Schema + style checks (gates push) | +| `test` | `test` | Jest via platform test runner | +| `invoke` | `invoke [TYPE] [KEY]` | Local / relay (`-a` auth id) / remote (`-r`) | +| `describe` | `describe` | Print triggers/searches/creates as Zapier sees them | +| `build` | `build` | `build/build.zip` + `source.zip` | +| `upload` | `upload` | Upload a zip without full push | +| `push` | `push` | Build + upload the `package.json` version (versions must be sequential) | +| `pull` | `pull` | Pull remote definition | +| `versions` | `versions` | List deployed versions | +| `promote` | `promote VERSION` | Make this the public default | +| `migrate` | `migrate FROM TO [PERCENT]` | Move users (same major) | +| `deprecate` | `deprecate VERSION DATE` | DATE ≥ 3 weeks out; users emailed at T-14d | +| `canary:create` | `canary:create FROM TO -p PCT -d SECS` | Temporary traffic split | +| `canary:list` / `canary:delete` | | Inspect / cancel canary | +| `env:get` / `env:set` / `env:unset` | `env:set 1.0.0 KEY=val` | Per-version secrets (`CLIENT_ID`, …) | +| `logs` | `logs` | HTTP / console logs | +| `history` | `history` | Audit trail | +| `integrations` (`apps`) | | Integrations you admin | +| `team:add` / `get` / `remove` | roles: `admin`, `collaborator`, `subscriber` | | +| `users:add` / `get` / `links` / `remove` | Invite testers to a private version | | +| `delete:version` / `delete:integration` | Only if no users/Zaps | | +| `jobs` | Background job status | | +| `legacy` | Legacy Web Builder helpers | | +| `analytics` | CLI telemetry opt-in/out | | + +Full flags and examples live in `kind: "cli_command"`. Source: `repos/zapier-platform/packages/cli/docs/cli.md`. + +`invoke` modes: local (default, `.env` / `authData_*`), relay (`-a `), remote (`-r`). Local limitations: no hook subscribe, hydration, file upload, buffered creates, search-or-create. + +--- + +## Runtime functions (`z`, first argument to every `perform`) + +Every trigger/action/search is `(z: ZObject, bundle: Bundle) => …`. + +| Function | Signature | Notes | +|----------|-----------|--------| +| `z.request` | `(url, options?) → Promise` | **Always use this**, not axios/fetch. Auth middleware + logs. `json`, `form`, `params`, `raw`, `skipThrowForStatus`. | +| `z.console.*` | `log\|info\|warn\|error` | Shows up in `zapier-platform logs` | +| `z.dehydrate` | `(func, inputData?, ttl?) → string` | Lazy object pointer | +| `z.dehydrateFile` | same | Lazy file | +| `z.stashFile` | `(buf\|stream\|url, …) → string` | Public short-lived file URL | +| `z.cursor.get` / `set` | polling cursor | Persist last-seen id/ts | +| `z.generateCallbackUrl` | `() → string` | Resume long creates (`performResume`) | +| `z.hash` | `(alg, data, enc?, inEnc?)` | Usually `sha256` | +| `z.JSON.parse` / `stringify` | | Parse throws a clean user error | +| `z.cache.get/set/delete` | per-auth JSON cache | `set(key, val, ttl?, scope?, nx?)` | +| `z.errors.Error` | `(message, code?, status?)` | User-visible failure | +| `z.errors.HaltedError` | | Stop run **without** failing the Zap | +| `z.errors.RefreshAuthError` | | 401 → refresh OAuth/session and retry | +| `z.errors.ExpiredAuthError` | | Dead connection; user must reconnect | +| `z.errors.ThrottledError` | `(message, delaySeconds?)` | Retry-After | +| `z.errors.ResponseError` | `(response)` | Usually auto-thrown by `z.request` | +| `createAppTester` | from `zapier-platform-core` | Unit-test performs | +| `zapier.tools.env.inject` | load `.env` | Tests | + +Return rules: + +- **Trigger / search** → array of objects (search may envelope `{results, paging_token}`). +- **Create** → **one object** (not an array). +- Polling items need `id` (or `primary: true` output fields). +- REST Hook: `operation.type = 'hook'` plus `performSubscribe`, `performUnsubscribe`, `performList`. + +`bundle` fields: `authData`, `inputData`, `inputDataRaw`, `meta` (`isLoadingSample`, `isFillingDynamicDropdown`, `isPopulatingDedupe`, `isBulkRead`, `limit`, `page`, `timezone`, `paging_token`, `withSearch`), plus hook extras `cleanedRequest`, `rawRequest`, `targetUrl`, `subscribeData`. + +See `kind: "core_function"` and `kind: "core_type"`. + +--- + +## App definition and schema + +Minimum `index.js` / `src/index.ts`: + +```js +module.exports = { + version: require('./package.json').version, + platformVersion: require('zapier-platform-core').version, + authentication: { /* type + config + test + connectionLabel */ }, + beforeRequest: [], // (request, z, bundle) => request + afterResponse: [], // (response, z, bundle) => response + hydrators: {}, + triggers: {}, + searches: {}, + creates: {}, + resources: {}, +}; +``` + +Canonical types in `kind: "schema_type"` (generated from `zapier-platform-schema` 19.x): + +- App: `AppSchema`, `AppFlagsSchema`, `VersionSchema` +- Auth: `AuthenticationSchema` + `AuthenticationOAuth2ConfigSchema`, `…SessionConfigSchema`, `…BasicConfigSchema`, `…DigestConfigSchema`, `…CustomConfigSchema`, `…OAuth1ConfigSchema` +- Operations: `TriggerSchema`, `CreateSchema`, `SearchSchema`, `SearchOrCreateSchema`, `ResourceSchema` +- HTTP: `RequestSchema`, `RedirectRequestSchema` +- Fields: `PlainInputFieldSchema`, `PlainOutputFieldSchema`, `FieldChoicesSchema`, `InputFieldGroupsSchema` +- Runtime extras: `HydratorsSchema`, `MiddlewaresSchema`, `ThrottleObjectSchema`, `LockObjectSchema`, `BufferConfigSchema` + +Full generated doc: `repos/zapier-platform/packages/schema/docs/build/schema.md`. + +### Auth recipes + +| `authentication.type` | Example app | Mechanism | +|-----------------------|-------------|-----------| +| `oauth2` | `example-apps/oauth2` | `authorizeUrl`, `getAccessToken`, `refreshAccessToken`, `autoRefresh` | +| `oauth1` | `oauth1-trello`, `oauth1-twitter`, `oauth1-tumblr` | HMAC signed requests | +| `session` | `session-auth` | `sessionConfig.perform` exchanges creds for a token in `authData` | +| `basic` | `basic-auth` | Username/password; platform sets header | +| `digest` | `digest-auth` | Challenge-response | +| `custom` | `custom-auth` | API key / header / query; attach in `beforeRequest` | + +On HTTP 401: `throw new z.errors.RefreshAuthError()` if refreshable, else `ExpiredAuthError()`. + +### Example apps (clone + `init --template`) + +`babel`, `basic-auth` (+ TS), `callback`, `create`, `custom-auth` (+ TS), `digest-auth` (+ TS), `dynamic-dropdown`, `files`, `github`, `line-items`, `middleware`, `minimal`, `minimal-esm`, `oauth1-trello` (+ TS), `oauth1-tumblr`, `oauth1-twitter`, `oauth2` (+ TS), `onedrive`, `openai`, `resource`, `rest-hooks`, `search`, `search-or-create`, `session-auth` (+ TS), `trigger`. + +Each is `kind: "example_app"` with README + file list. + +--- + +## Zapier SDK CLI (consume apps — different package) + +`npx @zapier/zapier-sdk-cli` / `zapier-sdk`. Docs: `official_doc` keys `sdk/cli-reference`, `sdk/reference`, `sdk/quickstart`. + +| Command | Purpose | +|---------|---------| +| `signup` / `login` / `logout` | Account + SDK credentials | +| `list-apps` | Discover apps at runtime (do not hardcode keys) | +| `get-app` | One app | +| `list-actions` / `get-action` | Actions for an app | +| `list-action-input-fields` / `…-schema` / `…-choices` | Input form | +| `run-action` | Execute and wait | +| `create-action-run` / `get-action-run` | Async run | +| `create-connection` / `find-first-connection` / `find-unique-connection` | User connections | +| `get-profile` | Who am I | +| `create-client-credentials` / `list-` / `delete-` | Deploy without browser login | + +Source repo: `repos/sdk` (includes `skills/zapier-sdk/SKILL.md`). + +--- + +## MCP, install, connectors + +Full typed reference: [MCP-REFERENCE.md](MCP-REFERENCE.md) and `kind: "mcp_function"`. + +```js +db.platform_reference.findOne({ kind: "guide", key: "zapier-mcp" }) +db.platform_reference.find({ kind: "mcp_function" }).sort({ key: 1 }) +``` + +- **MCP docs:** `section: "mcp"` (quickstart, auth, 20+ client guides, troubleshooting). +- **Install router:** `docs.zapier.com/install` + `npx @zapier/install-zapier` (`repos/install-zapier`). +- **Connectors (prototype):** `repos/connectors/apps/{algolia,alpaca,clay,dataforseo,discord,dropbox,elevenlabs,firecrawl,gitlab,google-*,harvest,heygen,linear,microsoft-*,notion,resend,runway,telegram,trello,youtube}`. + +--- + +## Public APIs (OpenAPI) + +`kind: "openapi"`: + +| Key | URL | +|-----|-----| +| `actions.yaml` | Stored Actions API | +| `connections.yaml` | Connection webhooks (White Label) | +| `trigger-inbox.yaml` | Trigger Inbox | +| `promotions-openapi.yaml` | Promotions / sponsored automation | +| `workflow-api-schema.json` | Live `https://api.zapier.com/schema` (Powered by Zapier Workflow API) | + +Rendered endpoint docs: `section: "api-reference"` and `section: "embed"`. + +--- + +## Cloned GitHub repos (`repos/`) + +| Repo | Role | +|------|------| +| `zapier-platform` | Canonical monorepo: CLI, core, schema, 31 example apps | +| `zapier-platform-cli` / `-core` / `-schema` | Archived standalone copies; use the monorepo | +| `sdk` | Agent-readable SDK docs + skill | +| `zapier-mcp` | Hosted MCP plugin manifests | +| `connectors` | Prototype local connectors | +| `install-zapier` | Agent installer | +| `agent-skills` / `marketplace` / `gtm-cheat-codes` | Agent skills / marketplace listing | +| `visual-builder` | Archived Platform UI tutorial | +| `resthooks` | REST Hooks spec (instant triggers) | +| `zapier-platform-example-app-{github,oauth2,minimal}` | Standalone example mirrors | + +Re-clone: `ROOT=/path scripts/clone-zapier-repos.sh`. + +--- + +## What was left on zapier.com (and what is not public) + +Pulled in this pass: + +- Full `docs.zapier.com` sitemap (409 pages) + `llms.txt` / `llms-full.txt` (2.5 MB) +- Root `zapier.com` sitemap developer surfaces: developer-platform, embed-tools, integrations, partner-program, MCP, Functions (Code by Zapier), custom connections, opensource, agents +- `developer.zapier.com` (shell → docs), legacy `platform.zapier.com` / v2 docs HTML +- OpenAPI specs above + +Already in other collections: 9,986 apps, 332,441 templates, 1,272 help articles, commercial/API-doc/auth scrapes. + +**Not available publicly (do not invent):** + +- Partnership / sales emails +- Private REST schemas of the 9,986 vendor APIs +- Auth type for ~8k apps that never published help/API-doc signals +- Internal Zapier implementation IDs mapping to OAuth vs API key + +--- + +## Grok query recipes + +```js +// Build a new OAuth2 integration +db.platform_reference.find({ + $or: [ + { kind: "guide" }, + { kind: "cli_command", key: { $in: ["init", "scaffold", "invoke", "push", "validate"] } }, + { kind: "example_app", key: /oauth2/ }, + { kind: "schema_type", key: /AuthenticationOAuth2|AppSchema|TriggerSchema|CreateSchema/ }, + { kind: "core_function", key: /^z\.(request|errors)/ }, + { kind: "official_doc", key: /oauth|cli-tutorial|core/ }, + ] +}) + +// Instant (REST Hook) trigger +db.platform_reference.find({ + $or: [ + { key: /hook|rest-hooks/i }, + { kind: "schema_type", key: /Hook|Trigger/ }, + ] +}) + +// Embed Workflow API +db.platform_reference.find({ section: { $in: ["embed", "api-reference", "white-label"] } }) +``` diff --git a/docs/00-sources/RESTART.md b/docs/00-sources/RESTART.md new file mode 100644 index 0000000..4ab30b3 --- /dev/null +++ b/docs/00-sources/RESTART.md @@ -0,0 +1,62 @@ +# Restart Grok for this Zapier workspace + +## 1. Leave the TUI + +In Grok type: + +``` +/quit +``` + +(` /exit ` is the same.) Do **not** use `/new` — that throws away this session. + +## 2. Run the restart script from a normal shell + +```bash +cd /Users/marchon/research/zapier +./scripts/restart-grok.sh +``` + +That script: + +1. `cd`s to this repo +2. Puts `zapier-platform` / `zapier-sdk` on `PATH` +3. Loads `~/.mcp-env` (Mongo URI) +4. Starts `scripts/mongo-tunnel.sh` if `127.0.0.1:27017` is down +5. Runs `grok --resume` (latest session for this directory) + +## 3. After Grok opens + +``` +/mcps +``` + +Press `r` to refresh. Confirm **mongodb** is connected. + +Then continue: + +``` +/zapier-build +``` + +or ask Grok to load: + +```js +db.platform_reference.findOne({ kind: "guide", key: "build-new-connector" }) +``` + +## Other scripts + +| Script | What it does | +|--------|----------------| +| `scripts/restart-grok.sh` | Tunnel + `grok --resume` | +| `scripts/ensure-mongo-tunnel.sh` | Start tunnel only | +| `scripts/mongo-tunnel.sh` | Raw `ssh -N -L 27017:… ns1` (foreground) | +| `scripts/dev-env.sh` | `source` this for PATH + Mongo URI | +| `scripts/zapier-status.sh` | Check tunnel, CLIs, login, Mongo count | + +```bash +./scripts/zapier-status.sh +``` + +Browser logins (optional, for `push` / live SDK): see [LOGIN.md](LOGIN.md). diff --git a/docs/00-sources/docs/diagrams/01-high-level-architecture.svg b/docs/00-sources/docs/diagrams/01-high-level-architecture.svg new file mode 100644 index 0000000..d4f93c2 --- /dev/null +++ b/docs/00-sources/docs/diagrams/01-high-level-architecture.svg @@ -0,0 +1,79 @@ + + + High-level architecture: Zapier to Verae via middleware + + + + + + + + + + + + + HIGH-LEVEL ARCHITECTURE + Verae Time x Zapier + Zapier never calls api.veraetime.net and never speaks NATS. HTTPS only at the public edge. + + PUBLIC · Zapier cloud + + YOUR INFRASTRUCTURE · public HTTPS edge + private workers + + PRIVATE NETWORK + + VERAE TIMESTAMPING SERVICE + + ZAPIER REST HOOKS + + User + Zap editor + connection form + + Zapier Platform CLI app + scratch/veraetime (session TS) or verae-zapier (zmw_ JS) + runs on Zapier cloud when a Zap step executes + + Zap trigger + Timestamp Completed + hooks.zapier.com + + configure / run + + hook fires Zap + + verae-zapier-middleware /zapier/v1/* + Auth bridge · tenants · entitlements · rate limits · REST Hook store + GET /health POST /auth/login GET /auth/me POST /timestamp[/wait|/batch] + POST /verify GET /status/{jobId} POST/DELETE /webhooks/* + + HTTPS Bearer zmw_ / zmt_ + + NATS JetStream + ZAPIER_JOBS · EVENTS · WEBHOOKS + never on the public internet + + publish (private) + + Workers + job-poller jobs.watch + event-router jobs.events + webhook-deliver webhooks.deliver + + + api.veraetime.net + POST /auth/login POST /api/timestamp 202 {jobId} GET /api/status/{jobId} POST /api/verify + + sync HTTPS + + poll / create + + POST hooks.zapier.com timestamp.completed + + HTTPS egress + diff --git a/docs/00-sources/docs/diagrams/02-security-boundaries.svg b/docs/00-sources/docs/diagrams/02-security-boundaries.svg new file mode 100644 index 0000000..9996863 --- /dev/null +++ b/docs/00-sources/docs/diagrams/02-security-boundaries.svg @@ -0,0 +1,62 @@ + + + Security boundaries + + + + + + + + + + + + + SECURITY BOUNDARIES + What is public, what stays private + + PUBLIC INTERNET + + Zapier cloud + CLI app performs + z.request only + no NATS, no Verae JWT + + Middleware HTTPS + /zapier/v1 only + TLS required in prod + rate limit + entitlements + + Zapier hook URLs + untrusted egress + timeouts required + SSRF allowlist = Phase 15 + + HTTPS in + + HTTPS out + + PRIVATE · do not expose ports 4222 / store / TOKEN_SECRET + + NATS JetStream + tokenRef, not raw JWT + private network + auth + mTLS is Phase 15 + + Workers + file store + STORE_PATH store.json + single-node MVP + Postgres/Redis = Phase 15 + + api.veraetime.net + Verae JWT stays here + server-side login only + MOCK_VERAE for tests + + + diff --git a/docs/00-sources/docs/diagrams/03-auth-two-hop.svg b/docs/00-sources/docs/diagrams/03-auth-two-hop.svg new file mode 100644 index 0000000..b836fc1 --- /dev/null +++ b/docs/00-sources/docs/diagrams/03-auth-two-hop.svg @@ -0,0 +1,73 @@ + + + Two-hop authentication + + + + + + + + + + + + + AUTHENTICATION + Two hops · Zapier never sees the Verae JWT + + HOP 1 · end user → middleware + + User in Zapier + username + password + or zmw_ API key + api_base_url + default :3100 + + POST /zapier/v1/auth/login + loginWithCredentials + or loginWithApiKey + returns accessToken + stored as sessionKey (TS) + + Tokens issued + zmw_ tenant API key + zmt_ HMAC session + TOKEN_SECRET + never commit + + Auth test + GET /auth/me + tenant + plan + usage + 401 → refresh + + + + + HOP 2 · middleware → Verae (server-side only) + + veraeClient + POST /auth/login on Verae + or MOCK_VERAE=true + JWT held in process + redacted in DEBUG_VERAE + + api.veraetime.net + Authorization: Bearer <JWT> + timestamp / status / verify + source of truth + not reachable from Zapier + + Do not leak + no Verae JWT in Zapier + no JWT in NATS (use tokenRef) + no secrets in git + no secrets in debug logs + + + diff --git a/docs/00-sources/docs/diagrams/04-flow-async-timestamp.svg b/docs/00-sources/docs/diagrams/04-flow-async-timestamp.svg new file mode 100644 index 0000000..d8f36cd --- /dev/null +++ b/docs/00-sources/docs/diagrams/04-flow-async-timestamp.svg @@ -0,0 +1,60 @@ + + + Create Timestamp async plus REST Hook + + + + + + + + + + + + + FLOW A + Create Timestamp · async job + hook + + Zapier app + + + Middleware + + + NATS + + + Workers + + + Verae / Hooks + + + 1 POST /timestamp + + 2 entitlement + POST /api/timestamp + + 3 202 { jobId } + + 4 return jobId to Zap + + 5 publish jobs.watch + + 6 job-poller + + 7 GET /api/status/{jobId} + + 8 pending: Nak + delay + + 9 terminal → jobs.events + + 10 match webhooks + + 11 POST timestamp.completed + Pair this action with the Timestamp Completed hook trigger. Zapier does not poll Verae. + diff --git a/docs/00-sources/docs/diagrams/05-flow-wait.svg b/docs/00-sources/docs/diagrams/05-flow-wait.svg new file mode 100644 index 0000000..caa0a02 --- /dev/null +++ b/docs/00-sources/docs/diagrams/05-flow-wait.svg @@ -0,0 +1,46 @@ + + + Create Timestamp and Wait + + + + + + + + + + + + + FLOW B + Create Timestamp and Wait + In-process path is live today. Wait-via-NATS is Phase 9. + + Zapier + POST /timestamp/wait + + Middleware + create job, then wait + + HTTPS + + NATS_ENABLED=false + in-process poller (Phase 6) + returns StatusResponse + + NATS_ENABLED=true · Phase 9 (open) + subscribe to jobs.events + hard timeout → { jobId, status: pending } + + + + Return to Zapier + completed / failed status object or pending + jobId on timeout (Zapier can Find Job Status next) + + + diff --git a/docs/00-sources/docs/diagrams/06-nats-topology.svg b/docs/00-sources/docs/diagrams/06-nats-topology.svg new file mode 100644 index 0000000..8be0f61 --- /dev/null +++ b/docs/00-sources/docs/diagrams/06-nats-topology.svg @@ -0,0 +1,64 @@ + + + NATS subject topology + + + + + + + + + + + + + NATS + JETSTREAM + Private subjects, streams, and workers + + + ZAPIER_JOBS + verae.zapier.jobs.watch + Work queue + + consumer: job-poller + GET /api/status/{jobId} + + + ZAPIER_EVENTS + verae.zapier.jobs.events + Limits (time) + + consumer: event-router + enqueue webhooks / waiters + + + ZAPIER_WEBHOOKS + verae.zapier.webhooks.deliver + Work queue + + consumer: webhook-deliver + POST Zapier targetUrl + + + ZAPIER_USAGE + verae.zapier.usage + Optional + + consumer: usage-writer + billing export + + + + + Ack semantics + Job still pending Nak with delay ~ intervalMs or republish attempt+1 + Job terminal publish jobs.events, then Ack the watch message + Webhook HTTP 2xx Ack · 5xx / network Nak until max_deliver + Poison message terminate after max_deliver; DEBUG_VERAE=webhooks DLQ log + Prefer tokenRef over embedding a Verae JWT in any NATS payload + diff --git a/docs/00-sources/docs/diagrams/07-operations-map.svg b/docs/00-sources/docs/diagrams/07-operations-map.svg new file mode 100644 index 0000000..3106995 --- /dev/null +++ b/docs/00-sources/docs/diagrams/07-operations-map.svg @@ -0,0 +1,77 @@ + + + Zapier operations mapped to middleware and Verae + + + + + + + + + + + + + OPERATIONS MAP + Connector → /zapier/v1 → Verae wrap + + Zapier + Type + Middleware + Verae + + Create Timestamp + create + POST /timestamp + POST /api/timestamp → 202 jobId + + Create Timestamp and Wait + create + POST /timestamp/wait + create + poll / wait + + Create Batch Timestamps + create + POST /timestamp/batch + POST /api/batch/timestamp + + Verify Certificate + create + POST /verify + POST /api/verify + + Find Job Status + search + GET /status/{jobId} + GET /api/status/{jobId} + + Find Job Verification + search (TS) + GET /status/{jobId}/verification + GET /api/verify/{jobId} + + Timestamp Completed + hook + POST /webhooks/subscribe + (no Verae hook — middleware stores targetUrl) + Creates return one object. Searches and hook perform return arrays. 404 on status search → []. + + Zapier cloud + scratch/veraetime + or verae-zapier + + Middleware :3100 + api_base_url / MIDDLEWARE_BASE_URL + never api.veraetime.net from Zapier + + Verae API + JWT server-side only + OpenAPI in scratch/our-api/ + + + diff --git a/docs/00-sources/docs/diagrams/08-middleware-internals.svg b/docs/00-sources/docs/diagrams/08-middleware-internals.svg new file mode 100644 index 0000000..d62bd37 --- /dev/null +++ b/docs/00-sources/docs/diagrams/08-middleware-internals.svg @@ -0,0 +1,58 @@ + + + Middleware internals + + + + + + + + + + + + + MIDDLEWARE INTERNALS + verae-zapier-api/verae-zapier-middleware + + GET /health + liveness + + Express /zapier → /v1 + authenticate · rateLimit on protected routes + + Public + POST /v1/auth/login + GET /v1/auth/me + POST /v1/signup + GET /v1/admin/* (X-Admin-Secret) + + Protected /v1 + /timestamp /timestamp/wait /batch + /verify + /status/:jobId[/verification] + /webhooks/subscribe|unsubscribe + + Services + store + auth / entitlement / tenant + timestamp / verify / webhook + store.json tenants usage hooks + veraeClient (+ mock) + + Flags + NATS_ENABLED=false in-process job poller (Phase 6 product path) + NATS_ENABLED=true JetStream workers; no in-process poller + MOCK_VERAE=true deterministic jobs, no live Verae + DEBUG_VERAE=auth,nats,jobs,webhooks,http,billing + + Outbound + HTTPS VERAE_API_BASE_URL (prod: api.veraetime.net) + NATS NATS_URL nats://127.0.0.1:4222 + HTTPS Zapier targetUrl (webhook worker) + Never bind NATS to a public interface + diff --git a/docs/00-sources/docs/diagrams/09-phase-roadmap.svg b/docs/00-sources/docs/diagrams/09-phase-roadmap.svg new file mode 100644 index 0000000..99a5d28 --- /dev/null +++ b/docs/00-sources/docs/diagrams/09-phase-roadmap.svg @@ -0,0 +1,89 @@ + + + Implementation phase roadmap + + + + + + + + + + + + + ROADMAP + TODO.md phases and gates · do not skip + + 0 Docs + gate passed + + + 1 Debug + gate passed + + + 2 HTTP + gate passed + + + 3 Store + gate passed + + + 4 Client + gate passed + + + 5 Auth + gate passed + + + 6 Sync API + gate passed + + + + + + 7 NATS + gate passed + + + 8 Workers + gate passed + + + 9 Wait-via-NATS + open blocker + + + 10 Tenancy + gate passed + + + 11 Zapier app + gate passed + + + 12 E2E local + next + + + 13 Production + next + + + 14 Private push + next + + + 15 Harden + next + Phase 9 is the open blocker before multi-instance wait. Phase 12 needs 8 + 10 + 11. + diff --git a/docs/00-sources/docs/diagrams/10-workspace-integration.svg b/docs/00-sources/docs/diagrams/10-workspace-integration.svg new file mode 100644 index 0000000..e1dd4e9 --- /dev/null +++ b/docs/00-sources/docs/diagrams/10-workspace-integration.svg @@ -0,0 +1,63 @@ + + + Research workspace and integration path + + + + + + + + + + + + + THIS WORKSPACE + How Grok, CLIs, Mongo, and the stack fit together + + Laptop · this repo + getting-started.md / .pdf + scratch/veraetime + verae-zapier-api/ + docs/diagrams/ + /zapier-build skill + scripts/restart-grok.sh + + Local runtime + zapier-platform 19.1.0 + zapier-sdk 0.77.1 + middleware :3100 + optional NATS :4222 + build + validate (no login) + register / push need ~/.zapierrc + + NS1 Mongo (tunneled) + ssh -L 27017:127.0.0.1:27017 ns1 + db zapier + apps · templates · help + platform_reference + never 70.88.205.138:27017 public + creds in ~/.mcp-env only + + + tunnel + + Publish path + 1 validate locally against middleware + 2 zapier-platform login (browser) + 3 public HTTPS middleware (Phase 13) + 4 register + push private version (Phase 14) + 5 one human E2E Zap, then consider listing + + Consume path (optional, not Verae publish) + zapier-sdk login → call existing apps + Hosted MCP mcp.zapier.com/api/v1/connect + discover → enable → inspect → execute + writes need explicit approval; 2 tasks each + do not mix with zapier-platform + diff --git a/docs/00-sources/docs/diagrams/11-zapier-billing.svg b/docs/00-sources/docs/diagrams/11-zapier-billing.svg new file mode 100644 index 0000000..96123ce --- /dev/null +++ b/docs/00-sources/docs/diagrams/11-zapier-billing.svg @@ -0,0 +1,47 @@ + + + Zapier and Verae billing layers + + + + + + + + + + + + + BILLING + Two meters · Zapier tasks + Verae timestamps + Publishing the Verae app is free. The customer’s Zapier plan is not Verae’s invoice. + + A Zapier customer plan + Tasks / month (or Enterprise annual pool) + Free 100 · Pro from 750 · Team from 2k · Ent custom + + B Task multipliers + 1 = typical action (Create Timestamp) + MCP execute = 2 AI Advanced = 3 Premium = 5 + + C Overflow + Pay-per-task on paid plans + Annual 1.25x · monthly 2.5x · cap 3x then pause + + D Verae middleware plan + Timestamps / verifications / batch / RPM + free 50 · starter 500 · pro 5k · enterprise contract + + E Partner / embed + Directory publish = $0 to Verae + White Label: Zapier bills the product co. (usage) + + F Add-ons (not tasks) + Agents = activities Chatbots = seat/tier + Do not mix into the Zapier task estimate + diff --git a/docs/00-sources/docs/diagrams/12-peergos-ipfs-tiered-storage.svg b/docs/00-sources/docs/diagrams/12-peergos-ipfs-tiered-storage.svg new file mode 100644 index 0000000..4ba0257 --- /dev/null +++ b/docs/00-sources/docs/diagrams/12-peergos-ipfs-tiered-storage.svg @@ -0,0 +1,72 @@ + + + Timestamped files on Peergos with tiered IPFS and cold retrieve + + + + + + + + + + + + + CLIENT PATTERN + Timestamp · metadata · Peergos · pin cache · cold retrieve + Keep the index hot. Do not keep every CID live on a gateway. + + WRITE PATH (Zapier orchestrates; Verae timestamps the hash) + + 1 Source + Drive / mail / + Peergos outbox + + 2 Hash + SHA-256 or + envelope JSON + + 3 Verae + Create Timestamp + jobId + hook + + 4 Store + Peergos e2e file + optional hot pin + + 5 Index + cid · jobId · class + always on + + + + + + HOT · cache pin + Kubo / Pinata / Filebase + Pinning Services API + TTL = pinnedUntil + drop when cold copy exists + + WARM · Peergos + e2e encrypted IPFS blocks + user capabilities / sharing + host cannot read plaintext + not a public CDN + + COLD · retrieve on demand + S3 Glacier IR / Flexible / Deep + or Filecoin / B2 / Storj + Iceberg = catalog, not bytes + restore API → short re-pin + + READ PATH GET /ipfs/{cid} or gateway miss + 1 look up index by cid (always-on). 2 pin-hot hit → serve. 3 Peergos capability → fetch encrypted blocks. + 4 storageClass glacier-* → StartRestore / vendor retrieve → 202 + Retry-After → rehydrate → optional short pin → serve. + Never walk every pin provider. Never put raw files or Verae JWTs in the Zap. + diff --git a/docs/00-sources/getting-started-research.md b/docs/00-sources/getting-started-research.md index f4fa6c9..7100f89 100644 --- a/docs/00-sources/getting-started-research.md +++ b/docs/00-sources/getting-started-research.md @@ -111,7 +111,7 @@ Streams: `ZAPIER_JOBS`, `ZAPIER_EVENTS`, `ZAPIER_WEBHOOKS` (optional `ZAPIER_USA - Swagger UI: https://api.veraetime.net/docs/swagger/index.html - OpenAPI: https://api.veraetime.net/docs/swagger/openapi.yaml -- Local copy: [scratch/our-api/openapi.yaml](scratch/our-api/openapi.yaml) +- Local copy: [veraetime-openapi.yaml](veraetime-openapi.yaml) - Auth: `POST /auth/login` → JWT in `token`; all other API routes `Authorization: Bearer ` - Async create: `POST /api/timestamp` → **202** `{ jobId }` diff --git a/docs/developer/modules/README.md b/docs/developer/modules/README.md index 8f7b40b..602f42f 100644 --- a/docs/developer/modules/README.md +++ b/docs/developer/modules/README.md @@ -10,30 +10,32 @@ Implementation status follows [TODO.md](../../../TODO.md) phases. Docs describe |------------|-------------|-------| | [debug.md](debug.md) | `debug/` | 1 | | [config.md](config.md) | `config.js` | 2 | -| [errors.md](errors.md) | `errors.js` | 2 | -| [app.md](app.md) | `app.js`, `index.js` | 2 | -| [store-db.md](store-db.md) | `store/db.js` | 3 | -| [store-tenants.md](store-tenants.md) | `store/tenants.js` | 3 | -| [store-usage.md](store-usage.md) | `store/usage.js` | 3 | -| [store-webhooks.md](store-webhooks.md) | `store/webhooks.js` | 3 | -| [tokens.md](tokens.md) | `lib/tokens.js` | 4 | -| [veraeClient.md](veraeClient.md) | `clients/veraeClient.js` | 4 | -| [authService.md](authService.md) | `services/authService.js` | 5 | -| [entitlementService.md](entitlementService.md) | `services/entitlementService.js` | 5 | -| [middleware-http.md](middleware-http.md) | `middleware/*.js` | 5 | -| [timestampService.md](timestampService.md) | `services/timestampService.js` | 6 | -| [verifyService.md](verifyService.md) | `services/verifyService.js` | 6 | -| [webhookService.md](webhookService.md) | `services/webhookService.js` | 6 | -| [routes.md](routes.md) | `routes/*.js` | 6 | +| [errors.md](../../modules/verae-zapier-middleware/errors.md) | `errors.js` | 2 | +| [app.md](../../modules/verae-zapier-middleware/app.md) | `app.js`, `index.js` | 2 | +| [store/db.md](../../modules/verae-zapier-middleware/store/db.md) | `store/db.js` | 3 | +| [store/tenants.md](../../modules/verae-zapier-middleware/store/tenants.md) | `store/tenants.js` | 3 | +| [store/usage.md](../../modules/verae-zapier-middleware/store/usage.md) | `store/usage.js` | 3 | +| [store/webhooks.md](../../modules/verae-zapier-middleware/store/webhooks.md) | `store/webhooks.js` | 3 | +| [lib/tokens.md](../../modules/verae-zapier-middleware/lib/tokens.md) | `lib/tokens.js` | 4 | +| [veraeClient.md](../../modules/verae-zapier-middleware/clients/veraeClient.md) | `clients/veraeClient.js` | 4 | +| [authService.md](../../modules/verae-zapier-middleware/services/authService.md) | `services/authService.js` | 5 | +| [entitlementService.md](../../modules/verae-zapier-middleware/services/entitlementService.md) | `services/entitlementService.js` | 5 | +| [authenticate.md](../../modules/verae-zapier-middleware/middleware/authenticate.md) | `middleware/*.js` | 5 | +| [timestampService.md](../../modules/verae-zapier-middleware/services/timestampService.md) | `services/timestampService.js` | 6 | +| [verifyService.md](../../modules/verae-zapier-middleware/services/verifyService.md) | `services/verifyService.js` | 6 | +| [webhookService.md](../../modules/verae-zapier-middleware/services/webhookService.md) | `services/webhookService.js` | 6 | +| [routes.md](../../modules/verae-zapier-middleware/routes/index.md) | `routes/*.js` | 6 | | [nats.md](nats.md) | `nats/*.js` | 7 | | [workers.md](workers.md) | `workers/*.js` | 8–9 | -| [tenantService.md](tenantService.md) | `services/tenantService.js` | 10 | +| [tenantService.md](../../modules/verae-zapier-middleware/services/tenantService.md) | `services/tenantService.js` | 10 | + +Full export tables also live in [function-reference.md](function-reference.md). ## Zapier app (`verae-zapier`) | Module doc | Source path | Phase | |------------|-------------|-------| -| [zapier-app.md](zapier-app.md) | `index.js`, `authentication.js`, creates/searches/triggers | 11 | +| [verae-zapier/index.md](../../modules/verae-zapier/index.md) | `index.js`, `authentication.js`, creates/searches/triggers | 11 | ## Documentation rules diff --git a/docs/developer/modules/nats.md b/docs/developer/modules/nats.md index 6fc498b..953fe21 100644 --- a/docs/developer/modules/nats.md +++ b/docs/developer/modules/nats.md @@ -15,7 +15,7 @@ ## Function contracts Full I/O tables: [function-reference.md](function-reference.md#natssubjectsjs) -Payload schemas: [../../architecture/nats-subjects.md](../../architecture/nats-subjects.md) +Payload schemas: [../../02-architecture/nats-subjects.md](../../02-architecture/nats-subjects.md) ## Implementation status diff --git a/packages/verae-ops/caddy/zapier.georgelambert.org.caddy b/packages/verae-ops/caddy/zapier.georgelambert.org.caddy new file mode 100644 index 0000000..44b06a9 --- /dev/null +++ b/packages/verae-ops/caddy/zapier.georgelambert.org.caddy @@ -0,0 +1,19 @@ +# Docs catalog. DNS: zapier.georgelambert.org → 70.88.205.138 +# rsync target: /SSD2/sites/zapier.georgelambert.org + +http://zapier.georgelambert.org { + redir https://zapier.georgelambert.org{uri} permanent +} + +zapier.georgelambert.org { + root * /SSD2/sites/zapier.georgelambert.org + encode gzip zstd + file_server { + index index.html index.pdf README.html README.pdf + } + try_files {path} {path}/index.html {path}/index.pdf {path}/README.html {path}/README.pdf + header { + X-Content-Type-Options nosniff + Referrer-Policy strict-origin-when-cross-origin + } +} diff --git a/scripts/build-docs-site.py b/scripts/build-docs-site.py index 00f0873..69ea586 100755 --- a/scripts/build-docs-site.py +++ b/scripts/build-docs-site.py @@ -12,6 +12,7 @@ import subprocess from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime, timezone from pathlib import Path +from urllib.parse import unquote ROOT = Path(__file__).resolve().parents[1] SITE = ROOT / "site" @@ -219,6 +220,180 @@ def render_md(src: Path, html_dest: Path, pdf_dest: Path, title: str) -> str: return "" +def _file_index(site: Path) -> dict[str, list[str]]: + idx: dict[str, list[str]] = {} + for p in site.rglob("*"): + if p.is_file(): + rel = p.relative_to(site).as_posix() + idx.setdefault(p.name, []).append(rel) + return idx + + +_PREFERRED_PREFIXES = ( + "overview/", + "packages/overview/", + "packages/verae-ops/", + "packages/verae-fleet/docs/", + "user-docs/", + "packages/zapier-user-docs/", + "docs/02-architecture/", + "docs/modules/", + "docs-master/", + "packages/zapier-decisions/", + "docs/modules-pdf/", + "docs/models-pdf/", + "sphinx/", +) + + +def _pick_hit(hits: list[str]) -> str: + for prefix in _PREFERRED_PREFIXES: + for h in hits: + if h.startswith(prefix): + return h + return hits[0] + + +def _web_target(site: Path, rel: str, index: dict[str, list[str]] | None = None) -> str: + """Map a site-relative path to a file that actually exists.""" + rel = unquote(rel).split("#", 1)[0].lstrip("/") + if rel.endswith(".html.pdf"): + rel = rel[:-4] + aliases = { + "index-md.pdf": "index-md.html", + "index-md.md": "index-md.html", + "docs/architecture/overview.pdf": "docs/02-architecture/overview.pdf", + "docs/architecture/nats-subjects.pdf": "docs/02-architecture/nats-subjects.pdf", + "docs/architecture/nats-gateway.pdf": "docs/02-architecture/nats-gateway.pdf", + "docs/architecture/composition.pdf": "docs/02-architecture/composition.pdf", + "docs/architecture/fleet.pdf": "docs/02-architecture/fleet.pdf", + "docs/sphinx/_build/html/index.pdf": "sphinx/index.html", + "docs/sphinx/_build/html/index.html": "sphinx/index.html", + } + rel = aliases.get(rel, rel) + candidates = [rel] + if rel.endswith(".pdf"): + candidates.append(rel[:-4] + ".html") + candidates.append(rel[:-4] + ".md") + if rel.endswith(".md"): + candidates.append(rel[:-3] + ".pdf") + candidates.append(rel[:-3] + ".html") + stem = Path(rel).name + if not stem or stem in {".", "/"}: + return rel or "index.html" + stem_pdf = Path(stem).with_suffix(".pdf").as_posix() + stem_html = Path(stem).with_suffix(".html").as_posix() + for folder in ( + "overview", + "packages/overview", + "packages/verae-ops", + "packages/verae-fleet/docs", + "user-docs", + "packages/zapier-user-docs", + "docs/02-architecture", + "docs/modules-pdf", + "docs/models-pdf", + "packages/zapier-decisions", + "docs-master", + "sphinx", + ): + candidates.append(f"{folder}/{stem_pdf}") + candidates.append(f"{folder}/{stem_html}") + for c in candidates: + if (site / c).is_file(): + return c + if (site / c).is_dir(): + for name in ("index.html", "index.pdf", "README.pdf", "README.html"): + if (site / c / name).is_file(): + return f"{c.rstrip('/')}/{name}" + if index: + names = [stem, stem_pdf, stem_html] + if stem.endswith(".md"): + names.append(Path(stem).with_suffix(".pdf").as_posix()) + names.append(Path(stem).with_suffix(".html").as_posix()) + for name in names: + hits = index.get(name) or [] + if hits: + return _pick_hit(hits) + return rel + + +def rewrite_pdf_uris(site: Path) -> None: + """Turn WeasyPrint file:///… annotations into https://zapier.georgelambert.org/…""" + try: + from pypdf import PdfReader, PdfWriter + from pypdf.generic import NameObject, create_string_object + except ImportError: + print("pypdf missing; PDF URI rewrite skipped") + return + web = "https://zapier.georgelambert.org/" + prefix = site.resolve().as_uri().rstrip("/") + "/" + index = _file_index(site) + + def resolve(raw: str) -> str: + frag = "" + if "#" in raw: + raw, frag = raw.split("#", 1) + frag = "#" + frag + mapped = raw + if raw.startswith(prefix): + mapped = web + _web_target(site, raw[len(prefix) :], index) + elif raw.startswith("file:"): + name = unquote(raw.rsplit("/", 1)[-1] if "/" in raw else raw) + if not name or name in {".", "/", "file:"}: + mapped = web + else: + hit = _web_target(site, name, index) + if (site / hit).exists(): + mapped = web + hit + elif "index-md" in raw: + mapped = web + "index-md.html" + else: + mapped = web + elif raw.startswith(web): + hit = _web_target(site, raw[len(web) :], index) + if (site / hit).is_file() or (site / hit).is_dir(): + mapped = web + hit + if mapped != raw: + return mapped + frag + return raw + frag + + n_pdf = 0 + n_fix = 0 + for pdf in site.rglob("*.pdf"): + try: + reader = PdfReader(str(pdf)) + except Exception: + continue + changed = False + for page in reader.pages: + annots = page.get("/Annots") + if not annots: + continue + for annot in annots: + obj = annot.get_object() + action = obj.get("/A") + if not action: + continue + uri = action.get("/URI") + if not uri: + continue + raw = str(uri) + new = resolve(raw) + if new != raw: + action[NameObject("/URI")] = create_string_object(new) + changed = True + n_fix += 1 + if changed: + writer = PdfWriter(clone_from=reader) + tmp = pdf.with_suffix(".pdf.tmp") + with tmp.open("wb") as fh: + writer.write(fh) + tmp.replace(pdf) + n_pdf += 1 + print(f"rewrote {n_fix} PDF URIs in {n_pdf} files → {web}") + + def convert_all_markdown(copied: list[tuple[Path, Path, str]]) -> list[str]: errors = [] n = len(copied) @@ -403,14 +578,20 @@ def main() -> None: SITE / "packages" / pkg / "docs", ignore=shutil.ignore_patterns("node_modules"), ) + copy_tree(pkg_root / "services", SITE / "packages" / pkg / "services") elif pkg == "verae-nats-process": for p in pkg_root.glob("*.md"): copy_tree(p, SITE / "packages" / pkg / p.name) else: - for name in ("README.md", "SUMMARY.md", "NATS.md"): + extras = ("README.md", "SUMMARY.md", "NATS.md") + if pkg == "zapier-decisions": + extras = extras + ("LOG.md", "TODO.md") + for name in extras: p = pkg_root / name if p.exists(): copy_tree(p, SITE / "packages" / pkg / name) + if pkg == "zapier-decisions": + copy_tree(pkg_root / "decisions", SITE / "packages" / pkg / "decisions") ui_docs = ROOT / "packages" / "ui-docs" for name in ("WALKTHROUGH.md", "REPORT.md", "UI-REVIEW.pdf"): p = ui_docs / name @@ -432,6 +613,7 @@ def main() -> None: SITE / "packages" / "zappier" / "docs", ignore=shutil.ignore_patterns("screenshots", "walkthrough", "superpowers"), ) + copy_tree(ROOT / "packages" / "zappier" / "openapi.yaml", SITE / "packages" / "zappier" / "openapi.yaml") for name in ("README.md", "TODO.md", "OPEN.md"): src = ROOT / name if not src.exists() and name == "OPEN.md": @@ -469,12 +651,40 @@ def main() -> None: if diagrams.exists(): copy_tree(diagrams, SITE / "research" / "docs" / "diagrams") + def write_book_index(folder: Path, title: str, patterns: tuple[str, ...]) -> None: + if not folder.is_dir(): + return + files: list[Path] = [] + for pat in patterns: + files.extend(p for p in folder.rglob(pat) if p.is_file()) + skip = {"index.md", "index.html", "index.pdf"} + files = sorted({p for p in files if p.name not in skip}) + lines = [f"# {title}", "", f"{len(files)} files in this book.", ""] + for p in files: + rel = p.relative_to(folder).as_posix() + lines.append(f"- [{rel}]({rel})") + (folder / "index.md").write_text("\n".join(lines) + "\n", encoding="utf-8") + + write_book_index(SITE / "docs" / "modules-pdf", "Module PDFs (book)", ("*.pdf",)) + write_book_index(SITE / "docs" / "models-pdf", "Model PDFs (book)", ("*.pdf",)) + write_book_index(SITE / "packages" / "ui-docs" / "screenshots", "UI screenshots", ("*.png", "*.jpg", "*.webp")) + write_book_index(SITE / "packages" / "overview" / "diagrams", "Overview diagrams", ("*.svg", "*.png")) + write_book_index(SITE / "overview" / "diagrams", "Overview diagrams", ("*.svg", "*.png")) + write_book_index(SITE / "packages" / "zapier-decisions" / "decisions", "Decisions", ("*.md", "*.pdf")) + write_book_index(SITE / "packages" / "verae-fleet" / "services", "Fleet service JSON", ("*.json",)) + write_book_index(SITE / "docs" / "00-sources" / "docs" / "diagrams", "Research diagrams", ("*.svg", "*.png")) + # Render every published .md from the site copy so relative images resolve. to_render: list[tuple[Path, Path, str]] = [] seen: set[Path] = set() for md in SITE.rglob("*.md"): if not md.is_file(): continue + if "research" in md.parts: + continue + # Sphinx HTML/PDF are copied from _build; don't WeasyPrint the rst sources. + if "sphinx" in md.parts: + continue key = md.resolve() if key in seen: continue @@ -496,6 +706,7 @@ def main() -> None: labeled.append((src, dest, title_map.get(key, title_map.get(src.relative_to(ROOT).as_posix() if src.is_relative_to(ROOT) else key, title)))) errors = convert_all_markdown(labeled) + rewrite_pdf_uris(SITE) # Also write CONSOLE.pdf next to the markdown in the fleet package (local console) console_md = ROOT / "packages" / "verae-fleet" / "docs" / "CONSOLE.md" @@ -591,8 +802,8 @@ def main() -> None: ("packages/ui-docs/UI-REVIEW.pdf", "UI review (screenshots + live doors)"), ("overview/README.pdf", "System overview"), ("overview/INDEX.pdf", "Documentation index"), - ("docs/modules-pdf/", "Module PDFs (book)"), - ("docs/models-pdf/", "Model PDFs (book)"), + ("docs/modules-pdf/index.pdf", "Module PDFs (book)"), + ("docs/models-pdf/index.pdf", "Model PDFs (book)"), ("sphinx/index.html", "Sphinx HTML"), ("sphinx/verae-zapier-modules.pdf", "Sphinx LaTeX PDF"), ] @@ -605,8 +816,10 @@ def main() -> None: ("packages/verae-fleet/docs/CONSOLE.html", "Operator console"), ("overview/README.html", "System overview"), ("overview/INDEX.html", "Documentation index"), - ("docs/modules/", "All module markdown"), - ("docs/models/", "All model markdown"), + ("docs/modules/README.html", "All module markdown"), + ("docs/models/README.html", "All model markdown"), + ("docs/modules-pdf/index.html", "Module PDFs (book)"), + ("docs/models-pdf/index.html", "Model PDFs (book)"), ("sphinx/index.html", "Sphinx HTML"), ("sphinx/verae-zapier-modules.pdf", "Sphinx LaTeX PDF"), ] @@ -676,12 +889,12 @@ def main() -> None: ) def extra_section(items: list[tuple[str, str]]) -> str: - lis = "".join( - f'
  • {lab}
    • ' - for h, lab in items - if (SITE / h).exists() or h.endswith("/") or h.startswith("sphinx") - ) - return f"

    Catalogs

      {lis}
    " + lis = [] + for h, lab in items: + dest = SITE / h + if dest.exists() or (h.startswith("sphinx") and (SITE / "sphinx").exists()): + lis.append(f'
  • {lab}
    • ') + return f"

    Catalogs

      {''.join(lis)}
    " rest_pdf = f""" {cards("pdf")} diff --git a/scripts/check-doc-links.py b/scripts/check-doc-links.py new file mode 100644 index 0000000..a532d17 --- /dev/null +++ b/scripts/check-doc-links.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +"""Check catalog homepage + PDF URI targets after a site build.""" +from __future__ import annotations + +import sys +from html.parser import HTMLParser +from pathlib import Path +from urllib.parse import unquote, urljoin, urlparse + +ROOT = Path(__file__).resolve().parents[1] +SITE = ROOT / "site" +WEB = "https://zapier.georgelambert.org/" + + +class Anchors(HTMLParser): + def __init__(self) -> None: + super().__init__() + self.hrefs: list[str] = [] + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + if tag == "a": + d = dict(attrs) + if d.get("href"): + self.hrefs.append(d["href"] or "") + + +def local_ok(url: str) -> bool: + if url.startswith(WEB): + rel = url[len(WEB) :].split("#", 1)[0] + dest = SITE / rel + if dest.is_file(): + return True + if dest.is_dir() and any( + (dest / n).exists() for n in ("index.html", "index.pdf", "README.html", "README.pdf") + ): + return True + # yaml/svg/png on homepage + return False + if url.startswith("file:"): + return False + return True # http(s) off-site; homepage checker uses GET separately + + +def main() -> int: + idx = SITE / "index.html" + if not idx.exists(): + print("no site/index.html — run scripts/build-docs-site.py") + return 1 + p = Anchors() + p.feed(idx.read_text(encoding="utf-8", errors="replace")) + broken: list[str] = [] + for h in sorted(set(p.hrefs)): + if h.startswith("#") or h.startswith("mailto:"): + continue + if h.startswith("https://git.georgelambert.org"): + continue + if h.startswith("https://") and not h.startswith(WEB): + continue + if h.startswith(WEB): + if not local_ok(h): + broken.append(f"homepage {h}") + continue + rel = h.lstrip("/") + dest = SITE / rel + if not dest.exists(): + broken.append(f"homepage missing {h}") + + from pypdf import PdfReader + + n_file = 0 + for pdf in SITE.rglob("*.pdf"): + try: + reader = PdfReader(str(pdf)) + except Exception as exc: + broken.append(f"unreadable {pdf.relative_to(SITE)} {exc}") + continue + for page in reader.pages: + annots = page.get("/Annots") + if not annots: + continue + for annot in annots: + obj = annot.get_object() + action = obj.get("/A") + if not action: + continue + uri = action.get("/URI") + if not uri: + continue + raw = str(uri) + if raw.startswith("file:"): + n_file += 1 + broken.append(f"file URI {pdf.relative_to(SITE)} -> {raw[:120]}") + continue + if raw.startswith(WEB): + path = raw.split("#", 1)[0] + if "/research/" in path: + continue + if not local_ok(path): + broken.append(f"pdf missing {pdf.relative_to(SITE)} -> {raw}") + + print(f"file:// leftover {n_file}") + print(f"broken {len(broken)}") + for b in broken[:80]: + print(" ", b) + if len(broken) > 80: + print(f" … {len(broken) - 80} more") + return 1 if broken else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/pdf-links.lua b/scripts/pdf-links.lua index 36ab21b..46db2e6 100644 --- a/scripts/pdf-links.lua +++ b/scripts/pdf-links.lua @@ -11,8 +11,9 @@ function Link(el) if not path then path, frag = t, "" end - if path:match("%.md$") or path:match("%.html$") then - path = path:gsub("%.md$", ".pdf"):gsub("%.html$", ".pdf") + -- PDFs should follow other PDFs for markdown sources. Keep .html (indexes, sphinx). + if path:match("%.md$") then + path = path:gsub("%.md$", ".pdf") el.target = path .. frag end return el