Milestone 0: import zappier billing, Verae middleware, and Zapier research

Compose-ready workspace: packages/zappier (rate card, portal, Stripe),
packages/verae-zapier-middleware (timestamp + NATS), packages/verae-zapier
(CLI app), vendor/zapier-platform, and research/zapier vendor corpus.

Gate 0 structure checks pass. Product code and research are not yet wired.
This commit is contained in:
George Lambert 2026-09-09 02:37:36 -04:00
commit b4150c8250
1364 changed files with 6814366 additions and 0 deletions

View file

@ -0,0 +1 @@
dist/

View file

@ -0,0 +1,64 @@
# Zapier Schema-to-TS
This is a tool to convert the JSON Schema schemas from
zapier-platform-schema into TypeScript types and interfaces that can be
used in downstream integration application code.
This tool is not intended to be pushed to NPM, but rather to be used
during development of the Zapier Platform CLI. This is why it has been
placed as a top-level package in this repo, and not in `packages/`.
## Development
- `pnpm install` for getting started.
- `pnpm test` for running unit tests.
- `pnpm build` for building this compiler into runnable js inside of `./dist/`.
- `pnpm generate-types` to actually generate the TypeScript interfaces.
- By default, this will read `packages/schema/exported-schema.json` as input.
- By default, this will write `packages/core/types/zapier.generated.d.ts` as output.
## "Publishing"
This tool is configured via run `pnpm generate-types` on every commit,
via husky. This will keep the generated TypeScript interfaces up to date
with the latest schema definitions.
## How it Works
This tool reads the contents of `packages/schema/exported-schema.json`,
and, starting with the AppSchema, recursively generates relevant types
and interfaces that it references. Ultimately, it writes a
`schemas.generated.d.ts` file in `packages/core/types/`. These types are
referenced and combined with the other types of zapier-platform-core to
provide a complete set of typings for integration developers to use.
Notably, there is an "override" system in `overrides.ts` that allows for
the raw types and interfaces to be modified or skipped as they are
encountered during conversion from JSONSchema into TypeScript. The
ts-morph library, a wrapper of TypeScript internals, is used to assemble
and write the final output.
## Rationale
Converting our JSON Schema schemas into TypeScript interfaces has been a
longstanding goal at Zapier (See issue
[#8](https://github.com/zapier/zapier-platform/issues/8) and
[#233](https://github.com/zapier/zapier-platform/issues/233)).
While the JSON schemas exist and are useful at the point of uploading
integrations to the Zapier platform, AND the fact there are plenty of
open-source schema TypeScript projects, none would address the fact that
there are features of JavaScript that are necessary when writing a
Zapier CLI integration. Specifically, our JSON Schemas are unable to
express the concepts of functions or promises, or connect the
definitions of input field definitions to the data provided in
bundle.inputData objects in perform functions.
In a similar vein, previous attempts, even to use the
`json-schema-to-typescript` library have floundered, as the level of
documentation and references it provides were undesirable. This project
was born as a HackWeek project in April 2024 to address some of these
issues, and has since been refined and improved.
This schema-to-ts project was rewritten in April 2025 to introduce the
much neater override system and any remove all dependency on the
`json-schema-to-typescript` library.

View file

@ -0,0 +1,37 @@
{
"name": "zapier-schema-to-ts",
"version": "0.1.0",
"description": "Converts zapier-platform-schema JsonSchema definitions to TypeScript types",
"author": "Thomas Cranny <thomas.cranny@zapier.com>",
"private": true,
"main": "src/main.ts",
"type": "module",
"scripts": {
"test": "tsc && vitest --run",
"test:v2": "vitest src/v2",
"clean": "rm -rf dist",
"generate-types": "tsx ./src/main.ts --schema-json ../packages/schema/exported-schema.json --output ../packages/core/types/schemas.generated.d.ts",
"git-add": "git add ../packages/core/types/schemas.generated.d.ts",
"precommit": "pnpm generate-types && pnpm git-add"
},
"dependencies": {
"@commander-js/extra-typings": "^14.0.0",
"commander": "^14.0.0",
"deepmerge": "^4.3.1",
"json-schema-to-typescript": "15.0.4",
"marked": "^16.1.1",
"pino": "^9.7.0",
"pino-pretty": "^13.0.0",
"prettier": "^3.6.2",
"ts-morph": "^26.0.0",
"tsx": "^4.20.3",
"word-wrap": "^1.2.5"
},
"devDependencies": {
"@types/json-schema": "^7.0.15",
"@types/node": "^20.17.30",
"tsx": "^4.19.3",
"typescript": "^5.8.2",
"vitest": "^3.2.4"
}
}

View file

@ -0,0 +1,53 @@
import { commentToTsDocString, reflowLines } from './comments.js';
import { describe, expect, it } from 'vitest';
import { lexer } from 'marked';
describe('comments to TsDoc strings', () => {
it('should handle short single-line docs', () => {
const expected = '/** hello */\n';
const actual = commentToTsDocString('hello');
expect(actual).toBe(expected);
});
it.each([
['hello', '/** hello */\n'],
[
'123456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789 123456789',
'/**\n * 123456789 123456789 123456789 123456789 123456789 123456789\n * 123456789 123456789 123456789\n */\n',
],
])('should produce expected outputs', (input, expected) => {
const actual = commentToTsDocString(input);
expect(actual).toEqual(expected);
});
});
describe('reflowing rules', () => {
it.each([
['foobar', ['foobar']],
['123456789 123456789 0123456789', ['123456789 123456789', '0123456789']],
])('should reflow lines', (input, expected) => {
const tokens = lexer(input);
const actual = reflowLines(tokens, { width: 20 });
expect(actual).toEqual(expected);
});
// These are now just markdown to markdown; so should remain
// unchanged from the schema. Originally used {@link}, but markdown
// was nicer.
it('should convert solitary links to TSDoc', () => {
const input = '[A link](http://some.domain.com)';
const expected = [`[A link](http://some.domain.com)`];
const tokens = lexer(input);
const actual = reflowLines(tokens, { width: 20 });
expect(actual).toEqual(expected);
});
it('should not wrap solitary links', () => {
const input = '[A link](http://reeeeeaaaallll.loooooong.domain.com)';
const expected = [`[A link](http://reeeeeaaaallll.loooooong.domain.com)`];
const tokens = lexer(input);
const actual = reflowLines(tokens, { width: 20 });
expect(actual).toEqual(expected);
});
});

View file

@ -0,0 +1,61 @@
import type { Token, Tokens, TokensList } from 'marked';
import { lexer } from 'marked';
import wrap from 'word-wrap';
const SINGLE_LINE_CONTENT_WIDTH = 60;
const MULTILINE_DEFAULT_CONTENT_WIDTH = 65;
/**
* Zapier-platform-schemas is thoroughly documented with Markdown
* content. This function converts this content into TS/JS block
* comments (i.e. /** style), preserving even tables and bulleted lists,
* but adding sensible line wrapping for readability.
*/
export const commentToTsDocString = (comment: string): string => {
const stripped = comment?.trim();
if (!stripped) return '';
const tokens = lexer(stripped);
const lines = reflowLines(tokens);
if (lines.length === 1 && lines[0]!.length < SINGLE_LINE_CONTENT_WIDTH) {
return `/** ${stripped} */\n`;
}
const docsLines = lines.map((line) => ` * ${line}`).join('\n');
return `/**\n${docsLines}\n */\n`;
};
// Exported only for testing purposes.
export const reflowLines = (
tokens: TokensList,
{ width = MULTILINE_DEFAULT_CONTENT_WIDTH } = {},
): string[] => {
return tokens.flatMap((t) => reflowToken(t, width));
};
const reflowToken = (token: Token, width: number): string[] => {
if (isSolitaryLink(token)) {
const link = token.tokens[0];
const docLink = `[${link.text}](${link.href})`;
return [docLink];
}
return wrap(token.raw, { width, indent: '', trim: true }).split(/\n/);
};
const isSolitaryLink = (
token: Token,
): token is Tokens.Paragraph & { tokens: [Tokens.Link] } =>
token.type === 'paragraph' &&
token.tokens?.length === 1 &&
token.tokens[0]?.type === 'link';
export function docStringLines(
comment: string | undefined,
trailingContent?: string,
): string[] | undefined {
if (comment === undefined) {
return undefined;
}
const tokens = lexer(comment);
return [reflowLines(tokens).join('\n') + (trailingContent ?? '')];
}

View file

@ -0,0 +1,295 @@
import {
IGNORE,
IGNORE_BUT_FOLLOW_REFS,
type AddPropertyContext,
type CompilerContext,
type CompilerOptions,
type SchemaPath,
type VersionInfo,
} from './types.ts';
import { IMPORTS, INTERFACE_OVERRIDES, TYPE_OVERRIDES } from './overrides.ts';
import { Project, SourceFile } from 'ts-morph';
import { idToTypeName, refToSchemaName } from './utils.ts';
import type { JSONSchema4 } from 'json-schema';
import Statistics from './statistics.ts';
import { docStringLines } from './comments.ts';
import { format } from './formatter.ts';
import fs from 'node:fs';
import { logger } from './utils.ts';
import renderType from './renderType.ts';
export async function compileV3(options: CompilerOptions) {
const { version, schemas } = JSON.parse(
fs.readFileSync(options.schemaJson!, 'utf8'),
);
logger.info({ version }, 'Loaded %d schemas', Object.keys(schemas).length);
const project = new Project();
const file = project.createSourceFile('dummy-value.ts');
addPreamble(file, {
compilerVersion: options.compilerVersion,
platformVersion: version,
});
addImports(file);
const ctx: CompilerContext = {
file,
schemas,
schemasToRender: ['/AppSchema'], // "Entrypoint" schema. More will get added.
renderedSchemas: new Set(),
ignoreUnusedOverrides: options.ignoreUnusedOverrides,
stats: new Statistics(),
};
while (ctx.schemasToRender.length > 0) {
const schemaName = ctx.schemasToRender.shift()!;
addTopLevelType(ctx, schemaName);
}
// Done! Format it with prettier and write it to the output file.
const rawTypeScript = file.getFullText();
const formatted = await format(rawTypeScript);
fs.writeFileSync(options.output!, formatted);
logger.info({ output: options.output }, 'Wrote generated TypeScript to file');
reportStatistics(ctx);
}
function reportStatistics(ctx: CompilerContext) {
const unusedTypeOverrides = ctx.stats.findUnusedTypeOverrides(TYPE_OVERRIDES);
if (unusedTypeOverrides.length > 0) {
logger.error({ unused: unusedTypeOverrides }, 'Unused type overrides');
} else {
logger.info('All type overrides were used');
}
const unusedInterfaceSelfOverrides =
ctx.stats.findUnusedInterfaceSelfOverrides(INTERFACE_OVERRIDES);
if (unusedInterfaceSelfOverrides.length > 0) {
logger.error(
{ unused: unusedInterfaceSelfOverrides },
'Unused interface signature overrides',
);
} else {
logger.info('All interface signature overrides were used');
}
const unusedInterfacePropertyOverrides =
ctx.stats.findUnusedInterfacePropertyOverrides(INTERFACE_OVERRIDES);
if (unusedInterfacePropertyOverrides.length > 0) {
logger.error(
{ unused: unusedInterfacePropertyOverrides },
'Unused interface property overrides',
);
} else {
logger.info('All interface property overrides were used');
}
if (
unusedTypeOverrides.length > 0 ||
unusedInterfaceSelfOverrides.length > 0 ||
unusedInterfacePropertyOverrides.length > 0
) {
if (ctx.ignoreUnusedOverrides !== true) {
throw new Error(
'Please make sure all type overrides are invoked, or ignore with --ignore-unused-overrides',
);
}
logger.warn('Ignoring unused overrides');
}
}
function addPreamble(file: SourceFile, options: VersionInfo) {
logger.debug({ options }, 'Adding preamble to file');
const preamble = `/**
* This file was automatically generated by Zapier's schema-to-ts tool.
* DO NOT MODIFY IT BY HAND. Instead, modify the source JSON Schema
* files, and/or the schema-to-ts tool and run its CLI to regenerate
* these typings.
*
* zapier-platform-schema version: ${options.platformVersion}
* schema-to-ts compiler version: ${options.compilerVersion}
*/`;
file.insertText(0, (writer) => writer.writeLine(preamble));
}
function addImports(file: SourceFile) {
logger.debug(
{ modules: IMPORTS.map((i) => i.moduleSpecifier) },
'Adding %d import sections to file',
IMPORTS.length,
);
file.addImportDeclarations(IMPORTS);
}
function addTopLevelType(ctx: CompilerContext, schemaPath: SchemaPath) {
const schema = ctx.schemas[refToSchemaName(schemaPath)];
if (!schema) {
logger.fatal({ schemaPath }, 'Top-level schema not found');
throw new Error(`Top-level schema not found: ${schemaPath}`);
}
// Skip if we've already added this type.
if (ctx.renderedSchemas.has(schemaPath)) {
logger.trace('Skipping already rendered top-level type %s', schemaPath);
return;
}
ctx.renderedSchemas.add(schemaPath);
if (isInterface(schema)) {
addInterface(ctx, schemaPath);
} else {
addType(ctx, schemaPath);
}
}
/**
* Detect if a schema should be rendered as an interface. Otherwise a
* plain type will be created.
*/
function isInterface(schema: JSONSchema4) {
return (
typeof schema === 'object' &&
schema !== null &&
'type' in schema &&
schema.type === 'object' &&
'properties' in schema
);
}
function addType(ctx: CompilerContext, schemaPath: SchemaPath) {
const typeName = idToTypeName(schemaPath);
const schema = ctx.schemas[refToSchemaName(schemaPath)];
if (!schema) {
logger.fatal({ schemaPath }, 'Top-level schema not found');
throw new Error(`Top-level schema not found: ${schemaPath}`);
}
const override = TYPE_OVERRIDES[schemaPath];
if (override) {
ctx.stats.incTypeOverride(schemaPath);
}
if (typeof override === 'function') {
logger.debug({ typeName }, "Type '%s': using override function", typeName);
override({
compilerCtx: ctx,
file: ctx.file,
typeName,
schemaPath,
schema,
});
return;
}
logger.debug(
{ typeName, override },
"Adding default type '%s' %s",
typeName,
override ? 'WITH OVERRIDES' : '',
);
if (override === IGNORE) {
return;
}
const { rawType, referencedTypes } = renderType(schema);
if (referencedTypes) {
ctx.schemasToRender.push(...referencedTypes);
}
if (override === IGNORE_BUT_FOLLOW_REFS) {
return;
}
ctx.file.addTypeAlias({
name: typeName,
isExported: true,
docs: docStringLines(schema.description),
type: rawType,
leadingTrivia: '\n',
...override,
});
}
function addInterface(ctx: CompilerContext, schemaPath: SchemaPath) {
const schema = ctx.schemas[refToSchemaName(schemaPath)];
if (!schema) {
logger.fatal({ schemaPath }, 'Top-level schema not found');
throw new Error(`Top-level schema not found: ${schemaPath}`);
}
const overrides = INTERFACE_OVERRIDES[schemaPath]?.self;
if (overrides) {
ctx.stats.incInterfaceSelfOverride(schemaPath);
}
logger.debug(
{ schemaPath, overrides },
"Adding interface '%s' %s",
schemaPath,
overrides ? 'WITH OVERRIDES' : '',
);
const iface = ctx.file.addInterface({
name: idToTypeName(schemaPath),
isExported: true,
docs: docStringLines(schema.description),
...overrides,
});
const requiredProperties = Array.isArray(schema.required)
? schema.required
: [];
Object.entries(schema.properties ?? {}).forEach(([key, value]) => {
addInterfaceProperty({
compilerCtx: ctx,
iface,
schemaPath,
key,
value,
isRequired: requiredProperties.includes(key),
});
});
}
function addInterfaceProperty(ctx: AddPropertyContext) {
const { compilerCtx, iface, schemaPath, key, value, isRequired } = ctx;
let override = INTERFACE_OVERRIDES[schemaPath]?.properties?.[key];
if (override) {
compilerCtx.stats.incInterfacePropertyOverride(schemaPath, key);
}
if (typeof override === 'function') {
override(ctx);
return;
}
if (typeof override === 'string') {
override = { type: override };
}
if (override === IGNORE) {
return;
}
const { rawType, referencedTypes } = renderType(value);
if (referencedTypes) {
compilerCtx.schemasToRender.push(...referencedTypes);
}
if (override === IGNORE_BUT_FOLLOW_REFS) {
return;
}
iface.addProperty({
name: key,
type: rawType,
docs: docStringLines(value.description),
hasQuestionToken: !isRequired,
leadingTrivia: '\n',
...override,
});
}

View file

@ -0,0 +1,20 @@
import { format as prettify, type Options as PrettierOptions } from 'prettier';
import { logger } from './utils.js';
const DEFAULT_OPTIONS: PrettierOptions = {
bracketSpacing: true,
trailingComma: 'all',
singleQuote: true,
printWidth: 80,
semi: true,
tabWidth: 2,
useTabs: false,
};
/** Format a Typescript module with Prettier. */
export const format = async (ts: string): Promise<string> => {
const options = { parser: 'typescript', ...DEFAULT_OPTIONS };
logger.debug({ options }, 'Finalised prettier options');
const result = prettify(ts, options);
return result;
};

View file

@ -0,0 +1,54 @@
import { Command, Option } from '@commander-js/extra-typings';
import type { CompilerOptions } from './types.js';
import { compileV3 } from './compiler.ts';
import { logger } from './utils.js';
const program = new Command()
.name(process.env.npm_package_name ?? 'unknown')
.version(process.env.npm_package_version ?? 'unknown')
.description(process.env.npm_package_description ?? 'unknown')
.addOption(
new Option('-l, --log-level <level>')
.choices(['fatal', 'error', 'warn', 'info', 'debug', 'trace', 'silent'])
.env('LOG_LEVEL')
.default('info'),
)
.option(
'-s, --schema-json <file>',
'The `exported-schema.json` file from zapier-platform-schema to compile from. Typically the latest built output from zapier-platform-schema.',
'../schema/exported-schema.json',
)
.option(
'-o, --output <file>',
'The file to write the generated TypeScript to. Typically intended to be put in ../core/types as a generated module.',
'../core/types/schemas.generated.d.ts',
)
.option(
'-i, --ignore-unused-overrides',
'Ignore unused type overrides. This is useful when you are migrating from zapier-platform-schema to schema-to-ts.',
);
const main = async () => {
const startTime = performance.now();
program.parse();
const options = program.opts();
if (options.logLevel) {
logger.level = options.logLevel;
}
logger.debug({ options }, 'Parsed CLI options');
const compilerOptions: CompilerOptions = {
...options,
compilerVersion: process.env.npm_package_version!,
};
logger.debug({ compilerOptions }, 'Finalised compiler options');
logger.info('Using V3 compiler');
await compileV3(compilerOptions);
const endTime = performance.now();
logger.info('Compilation took %d ms', Math.round(endTime - startTime));
return;
};
await main();

View file

@ -0,0 +1,173 @@
import type { ImportDeclarationStructure, OptionalKind } from 'ts-morph';
import {
IGNORE_BUT_FOLLOW_REFS,
type InterfaceOverridesMap,
type TypeOverrideMap,
} from './types.ts';
import { docStringLines } from './comments.ts';
export const IMPORTS: OptionalKind<ImportDeclarationStructure>[] = [
{
isTypeOnly: true,
moduleSpecifier: './custom',
namedImports: [
'AfterResponseMiddleware',
'BeforeRequestMiddleware',
'PerformFunction',
],
},
{
isTypeOnly: true,
moduleSpecifier: './inputs',
namedImports: ['InputFields', 'InferInputData'],
},
{
isTypeOnly: true,
moduleSpecifier: './functions',
namedImports: [
'PollingTriggerPerform',
'WebhookTriggerPerform',
'WebhookTriggerPerformList',
'WebhookTriggerPerformSubscribe',
'WebhookTriggerPerformUnsubscribe',
'HookToPollTriggerPerformList',
'HookToPollTriggerPerformSubscribe',
'HookToPollTriggerPerformUnsubscribe',
'CreatePerform',
'CreatePerformResume',
'CreatePerformGet',
'SearchPerform',
'SearchPerformGet',
'SearchPerformResume',
'OAuth2AuthorizeUrl',
'OAuth2GetAccessToken',
'OAuth2RefreshAccessToken',
],
},
];
const KeyTypeParam = {
name: '$Key',
constraint: 'string',
default: 'string',
};
const InputFieldsTypeParam = {
name: '$InputFields',
constraint: 'InputFields',
default: 'InputFields',
};
export const INTERFACE_OVERRIDES: InterfaceOverridesMap = {
// AppSchema is renamed to BaseApp. The Triggers, Creates, and
// Searches are deliberately omitted, and separately handled by a the
// `App` type from `./apps.d.ts` in zapier-platform-core, which
// extends this BaseApp type.
'/AppSchema': {
self: { name: 'BaseApp' },
properties: {
beforeRequest: 'BeforeRequestMiddleware | BeforeRequestMiddleware[]',
afterResponse: 'AfterResponseMiddleware | AfterResponseMiddleware[]',
creates: IGNORE_BUT_FOLLOW_REFS,
triggers: IGNORE_BUT_FOLLOW_REFS,
searches: IGNORE_BUT_FOLLOW_REFS,
},
},
'/TriggerSchema': {
self: { typeParameters: [KeyTypeParam, InputFieldsTypeParam] },
properties: {
key: '$Key',
operation:
'BasicPollingOperation<$InputFields> | BasicHookOperation<$InputFields> | BasicHookToPollOperation<$InputFields>',
},
},
'/CreateSchema': {
self: { typeParameters: [KeyTypeParam, InputFieldsTypeParam] },
properties: {
key: '$Key',
operation: 'BasicCreateOperation<$InputFields>',
},
},
'/SearchSchema': {
self: { typeParameters: [KeyTypeParam, InputFieldsTypeParam] },
properties: {
key: '$Key',
operation: 'BasicSearchOperation<$InputFields>',
},
},
'/BasicPollingOperationSchema': {
self: { typeParameters: [InputFieldsTypeParam] },
properties: {
inputFields: '$InputFields',
perform: 'Request | PollingTriggerPerform<InferInputData<$InputFields>>',
},
},
'/BasicHookOperationSchema': {
self: { typeParameters: [InputFieldsTypeParam] },
properties: {
inputFields: '$InputFields',
perform: 'WebhookTriggerPerform<InferInputData<$InputFields>>',
performList:
'Request | WebhookTriggerPerformList<InferInputData<$InputFields>>',
performSubscribe:
'Request | WebhookTriggerPerformSubscribe<InferInputData<$InputFields>>',
performUnsubscribe:
'Request | WebhookTriggerPerformUnsubscribe<InferInputData<$InputFields>>',
},
},
'/BasicHookToPollOperationSchema': {
self: { typeParameters: [InputFieldsTypeParam] },
properties: {
inputFields: '$InputFields',
performList:
'Request | HookToPollTriggerPerformList<InferInputData<$InputFields>>',
performSubscribe:
'Request | HookToPollTriggerPerformSubscribe<InferInputData<$InputFields>>',
performUnsubscribe:
'Request | HookToPollTriggerPerformUnsubscribe<InferInputData<$InputFields>>',
},
},
'/BasicCreateOperationSchema': {
self: { typeParameters: [InputFieldsTypeParam] },
properties: {
inputFields: '$InputFields',
perform: 'Request | CreatePerform<InferInputData<$InputFields>>',
performResume: 'CreatePerformResume<InferInputData<$InputFields>>',
performGet: 'Request | CreatePerformGet<InferInputData<$InputFields>>',
},
},
'/BasicSearchOperationSchema': {
self: { typeParameters: [InputFieldsTypeParam] },
properties: {
inputFields: '$InputFields',
perform: 'Request | SearchPerform<InferInputData<$InputFields>>',
performGet: 'Request | SearchPerformGet<InferInputData<$InputFields>>',
performResume: 'SearchPerformResume<InferInputData<$InputFields>>',
},
},
};
export const TYPE_OVERRIDES: TypeOverrideMap = {
'/FunctionSchema': ({ file, typeName, schema }) => {
file.addTypeAlias({
name: typeName,
isExported: true,
docs: docStringLines(
schema.description,
'\n\n@deprecated Prefer using the perform types from the `functions` module.',
),
type: 'PerformFunction',
leadingTrivia: '\n',
});
},
// Don't render this type as it's replaced by an import. We do want
// the plain input field type it references to be rendered, though.
'/InputFieldsSchema': IGNORE_BUT_FOLLOW_REFS,
// Don't render these types because they're reimplemented by
// apps.d.ts in zapier-platform-core
'/TriggersSchema': IGNORE_BUT_FOLLOW_REFS,
'/CreatesSchema': IGNORE_BUT_FOLLOW_REFS,
'/SearchesSchema': IGNORE_BUT_FOLLOW_REFS,
};

View file

@ -0,0 +1,143 @@
import { describe, expect, it } from 'vitest';
import type { JSONSchema4 } from 'json-schema';
import type { SchemaPath } from './types.ts';
import renderType from './renderType.ts';
describe('type rendering', () => {
it.each<[JSONSchema4, string]>([
[{ type: 'string' }, 'string'],
[{ type: 'string', enum: ['foo', 'bar'] }, "'foo' | 'bar'"],
])('renders string type from %s', (schema, expected) => {
const result = renderType(schema);
expect(result.rawType).toBe(expected);
});
it.each<[JSONSchema4, string]>([
[{ $ref: '/FoobarSchema' }, 'Foobar'],
[{ type: 'string' }, 'string'],
[{ type: 'number' }, 'number'],
[{ type: 'integer' }, 'number'],
[{ type: 'boolean' }, 'boolean'],
[{ type: 'null' }, 'null'],
])('renders basic type of %s', (schema, expected) => {
const result = renderType(schema);
expect(result.rawType).toBe(expected);
});
it.each<[JSONSchema4, string]>([
[{ type: 'array' }, 'unknown[]'],
[{ type: 'array', items: { type: 'string' } }, 'string[]'],
[
{
type: 'array',
items: { oneOf: [{ type: 'string' }, { type: 'number' }] },
},
'(string | number)[]',
],
])('renders array type of %s', (schema, expected) => {
const result = renderType(schema);
expect(result.rawType).toBe(expected);
});
it.each<[JSONSchema4, string]>([
[{ type: 'object', additionalProperties: false }, '{}'],
[{ type: 'object' }, 'Record<string, unknown>'],
[{ type: 'object', additionalProperties: true }, 'Record<string, unknown>'],
[
{ type: 'object', properties: { name: { type: 'string' } } },
'{ name: string }',
],
[{ type: 'object', additionalProperties: false }, '{}'],
[{ type: 'object', additionalProperties: true }, 'Record<string, unknown>'],
[
{
type: 'object',
additionalProperties: false,
properties: { name: { type: 'string' } },
},
'{ name: string }',
],
[
{
type: 'object',
additionalProperties: false,
properties: { name: { type: 'string' }, age: { type: 'number' } },
},
'{ name: string; age: number }',
],
[
{
type: 'object',
patternProperties: {
'^[a-zA-Z]+[a-zA-Z0-9]*$': { type: 'string' },
},
additionalProperties: false,
},
'Record<string, string>',
],
])('renders object type of %s', (schema, expected) => {
const result = renderType(schema);
expect(result.rawType).toBe(expected);
});
it.each<[JSONSchema4, string]>([
[{ oneOf: [{ type: 'string' }] }, 'string'],
[{ oneOf: [{ type: 'string' }, { type: 'number' }] }, 'string | number'],
[
{ oneOf: [{ type: 'string' }, { $ref: '/FoobarSchema' }] },
'string | Foobar',
],
[{ anyOf: [{ type: 'string' }] }, 'string'],
[{ anyOf: [{ type: 'string' }, { type: 'number' }] }, 'string | number'],
[
{ anyOf: [{ type: 'string' }, { $ref: '/FoobarSchema' }] },
'string | Foobar',
],
])('renders union type of %s', (schema, expected) => {
const result = renderType(schema);
expect(result.rawType).toBe(expected);
});
});
describe('reference extraction', () => {
it.each<[JSONSchema4]>([
[{ type: 'string' }],
[{ type: 'number' }],
[{ type: 'integer' }],
[{ type: 'boolean' }],
[{ type: 'object', properties: { name: { type: 'string' } } }],
])('Does not extract anything from %s', (schema) => {
const result = renderType(schema);
expect(result.referencedTypes).toBeUndefined();
});
it.each<[JSONSchema4, Set<SchemaPath> | undefined]>([
[{ $ref: '/FoobarSchema' }, new Set(['/FoobarSchema'])],
[
{ type: 'array', items: { $ref: '/FoobarSchema' } },
new Set(['/FoobarSchema']),
],
[
{
type: 'object',
properties: { name: { $ref: '/FoobarSchema' } },
},
new Set(['/FoobarSchema']),
],
[{ oneOf: [{ $ref: '/FoobarSchema' }] }, new Set(['/FoobarSchema'])],
[
{
oneOf: [
{ type: 'string' },
{ type: 'array', items: { $ref: '/FoobarSchema' } },
],
},
new Set(['/FoobarSchema']),
],
[{ anyOf: [{ $ref: '/FoobarSchema' }] }, new Set(['/FoobarSchema'])],
])('Extracts references from %s', (schema, expected) => {
const result = renderType(schema);
expect(result.referencedTypes).toEqual(expected);
});
});

View file

@ -0,0 +1,165 @@
import { idToTypeName, logger } from './utils.ts';
import type { JSONSchema4 } from 'json-schema';
import type { SchemaPath } from './types.ts';
type RenderResult = {
/**
* The raw type that can be inserted into a TypeScript type.
*/
rawType: string;
/**
* An optional set of /XyxSchema references that were referenced and will need to be
* rendered.
*/
referencedTypes?: Set<SchemaPath>;
};
/**
* Render a JSONSchema object into a TypeScript type. Returns a string
* of the rawType that can be inserted as raw TypeScript code, and an
* optional set of /XyxSchema references that were referenced and will
* need to be rendered.
*/
export default function renderType(schema: JSONSchema4): RenderResult {
if (schema.$ref) {
return {
rawType: idToTypeName(schema.$ref),
referencedTypes: new Set([schema.$ref as SchemaPath]),
};
}
if (schema.type === 'string') {
return renderStringType(schema);
}
if (schema.type === 'number' || schema.type === 'integer') {
return { rawType: 'number' };
}
if (schema.type === 'boolean') {
return { rawType: 'boolean' };
}
if (schema.type === 'null') {
return { rawType: 'null' };
}
if (schema.type === 'object') {
return renderObjectType(schema);
}
if (schema.type === 'array') {
return renderArrayType(schema);
}
if (schema.oneOf) {
return renderOneOfType(schema);
}
if (schema.anyOf) {
return renderAnyOfType(schema);
}
logger.error(
{ schema },
'Schema not supported. Add support to renderType().',
);
throw new Error(
`Schema not supported, add support to renderType(): ${JSON.stringify(schema)}`,
);
}
const renderStringType = (schema: JSONSchema4): RenderResult => {
if (schema.enum) {
return { rawType: `'${schema.enum.join("' | '")}'` };
}
return { rawType: 'string' };
};
/**
* Renders the type for an object schema.
*/
export const renderObjectType = (schema: JSONSchema4): RenderResult => {
// PatternProperties become records
if (schema.patternProperties) {
if (Object.keys(schema.patternProperties).length !== 1) {
logger.error(
{ schema },
'Only PatternProperties with a single entry are supported.',
);
throw new Error(
'Only PatternProperties with a single entry are supported.',
);
}
const [value] = Object.values(schema.patternProperties);
const { rawType, referencedTypes } = renderType(value!);
return { rawType: `Record<string, ${rawType}>`, referencedTypes };
}
// Unspecified key types.
if (
schema.type === 'object' &&
schema.additionalProperties !== false &&
!schema.properties
) {
return { rawType: 'Record<string, unknown>' };
}
// No properties.
if (!schema.properties) {
return { rawType: '{}' };
}
const properties = Object.entries(schema.properties).map(
([key, value]): RenderResult => {
const { rawType, referencedTypes } = renderType(value);
return { rawType: `${key}: ${rawType}`, referencedTypes };
},
);
const rawType = properties.map((p) => p.rawType).join('; ');
const referencedTypes = new Set(
properties.flatMap((p) => [...(p.referencedTypes ?? [])]),
);
return {
rawType: `{ ${rawType} }`,
referencedTypes: referencedTypes.size > 0 ? referencedTypes : undefined,
};
};
/**
* Renders the type for an array schema.
*/
const renderArrayType = (schema: JSONSchema4): RenderResult => {
if (schema.items) {
const { rawType, referencedTypes } = renderType(schema.items);
if (rawType.includes('|')) {
// Parentheses preserve the union inside the array.
return { rawType: `(${rawType})[]`, referencedTypes };
}
return { rawType: `${rawType}[]`, referencedTypes };
}
return { rawType: 'unknown[]' };
};
const renderOneOfType = (schema: JSONSchema4): RenderResult => {
if (schema.oneOf) {
const types = schema.oneOf.map((type) => renderType(type));
const referencedTypes = new Set(
types.flatMap((t) => [...(t.referencedTypes ?? [])]),
);
return {
rawType: types.map((t) => t.rawType).join(' | '),
referencedTypes,
};
}
return { rawType: 'unknown' };
};
const renderAnyOfType = (schema: JSONSchema4): RenderResult => {
if (schema.anyOf) {
const types = schema.anyOf.map((type) => renderType(type));
const referencedTypes = new Set(
types.flatMap((t) => [...(t.referencedTypes ?? [])]),
);
return {
rawType: types.map((t) => t.rawType).join(' | '),
referencedTypes,
};
}
return { rawType: 'unknown' };
};

View file

@ -0,0 +1,76 @@
import type {
InterfaceOverridesMap,
SchemaPath,
TypeOverrideMap,
} from './types.ts';
export default class Statistics {
private readonly typeStats: Record<string, number> = {};
private readonly interfaceSelfStats: Record<string, number> = {};
private readonly interfacePropertyStats: Record<
string,
Record<string, number>
> = {};
public incInterfaceSelfOverride(schemaPath: SchemaPath) {
this.interfaceSelfStats[schemaPath] ??= 0;
this.interfaceSelfStats[schemaPath]++;
}
public incInterfacePropertyOverride(
schemaPath: SchemaPath,
propertyName: string,
) {
this.interfacePropertyStats[schemaPath] ??= {};
this.interfacePropertyStats[schemaPath][propertyName] ??= 0;
this.interfacePropertyStats[schemaPath][propertyName]++;
}
public incTypeOverride(schemaPath: SchemaPath) {
this.typeStats[schemaPath] ??= 0;
this.typeStats[schemaPath]++;
}
public findUnusedTypeOverrides(typeOverrides: TypeOverrideMap): string[] {
return Object.keys(typeOverrides).filter((key) => !this.typeStats[key]);
}
public findUnusedInterfaceSelfOverrides(
interfaceOverrides: InterfaceOverridesMap,
): string[] {
return Object.entries(interfaceOverrides)
.filter(
([key, overrides]) => overrides.self && !this.interfaceSelfStats[key],
)
.map(([key]) => key);
}
public findUnusedInterfacePropertyOverrides(
interfaceOverrides: InterfaceOverridesMap,
): string[] {
return Object.keys(interfaceOverrides).flatMap((interfacePath) =>
this.findUnusedInterfacePropertyOverridesInInterface(
interfaceOverrides,
interfacePath as SchemaPath,
).map((p) => `${interfacePath}.${p}`),
);
}
private findUnusedInterfacePropertyOverridesInInterface(
interfaceOverrides: InterfaceOverridesMap,
interfacePath: SchemaPath,
): string[] {
const overrides = interfaceOverrides[interfacePath];
if (!overrides || !overrides.properties) {
return [];
}
return Object.entries(overrides.properties)
.filter(
([property]) =>
!this.interfacePropertyStats[interfacePath] ||
!this.interfacePropertyStats[interfacePath][property],
)
.map(([property]) => property);
}
}

View file

@ -0,0 +1,165 @@
import type {
InterfaceDeclaration,
InterfaceDeclarationStructure,
PropertySignatureStructure,
SourceFile,
TypeAliasDeclarationStructure,
} from 'ts-morph';
import type { JSONSchema4 } from 'json-schema';
import type { LevelWithSilent } from 'pino';
import type Statistics from './statistics.ts';
export type ZapierSchemaDocument = {
version: string;
schemas: Record<SchemaPath, TopLevelSchema>;
};
export interface CliOptions {
/**
* @default "info"
*/
logLevel?: LevelWithSilent;
/**
* Path to the `exported-schema.json` file from zapier-platform-schema
* to compile from. Typically the latest built output from
* zapier-platform-schema.
*
* @default "../schema/exported-schema.json"
*/
schemaJson?: string;
/**
* The file to write the generated TypeScript to.
*
* @default "../core/types/schemas.generated.d.ts"
*/
output?: string;
/**
* Whether to ignore unused type overrides.
*
* @default false
*/
ignoreUnusedOverrides?: boolean;
}
export interface CompilerOptions extends CliOptions {
/** The version of this schema-to-ts compiler */
compilerVersion: string;
}
export interface VersionInfo {
/** The version of this schema-to-ts compiler */
compilerVersion: string;
/** The zapier-platform version of schemas that are being compiled. */
platformVersion: string;
}
/** The ID of a schema in the exported-schema.json. */
export type SchemaPath = `/${string}Schema`; // e.g. /AppSchema
/** All schemas in the exported-schema.json have an ID. */
export type TopLevelSchema = JSONSchema4 & { id: SchemaPath };
export type CompilerContext = {
file: SourceFile;
schemas: Record<string, TopLevelSchema>;
/**
* Queue of schema names that need to be rendered. Added as
* encountered traversing the schema tree.
*/
schemasToRender: SchemaPath[];
/**
* Schemas that have already been rendered.
*/
renderedSchemas: Set<SchemaPath>;
/**
* Whether to ignore unused type overrides.
*/
ignoreUnusedOverrides?: boolean;
stats: Statistics;
};
export type AddTypeContext = {
compilerCtx: CompilerContext;
file: SourceFile;
typeName: string;
schemaPath: SchemaPath;
schema: JSONSchema4;
};
export type TypeOverrideFunction = (ctx: AddTypeContext) => void;
/**
* Overrides can be partial options to the `addTypeAlias` function, or a
* function that can add, modify, or ignore the type from scratch.
*/
export type TypeOverrides =
| TypeOverrideFunction
| typeof IGNORE
| typeof IGNORE_BUT_FOLLOW_REFS
| Partial<TypeAliasDeclarationStructure>;
export type TypeOverrideMap = Record<SchemaPath, TypeOverrides>;
export type InterfaceOverridesMap = Record<SchemaPath, InterfaceOverrides>;
export type AddPropertyContext = {
compilerCtx: CompilerContext;
iface: InterfaceDeclaration;
schemaPath: SchemaPath;
key: string;
value: JSONSchema4;
isRequired: boolean;
};
export type PropertyOverrides =
| string
| Partial<PropertySignatureStructure>
| typeof IGNORE
| typeof IGNORE_BUT_FOLLOW_REFS
| ((ctx: AddPropertyContext) => void);
/**
* Specify overrides for an interface and itself properties.
*/
export type InterfaceOverrides = {
/**
* Overrides for the interface declaration itself. Will be merged with
* the default interface declaration.
*/
self?: Partial<InterfaceDeclarationStructure>;
/**
* Collection of optional overrides for properties of the interface.
* Each value can be one of three things:
* - A string, which will be used as the type of the property.
* - An object, which will be merged with the default property declaration.
* - A function, that can fully add, modify, or ignore the property entirely.
*/
properties?: Record<string, PropertyOverrides>;
};
/**
* Special symbol that can be used instead of a no-op function for types
* and property overrides that will cause the type or property to be
* ignored. Any references to other types that would also be included
* will also be ignored, though they may be included if they are
* referenced by something else that is not ignored.
*/
export const IGNORE = Symbol();
/**
* Special symbol that can be used instead of a no-op function in the
* collection of overrides for properties, that will cause the property
* to be ignored, but the references of the original schema to continue
* to be followed.
*/
export const IGNORE_BUT_FOLLOW_REFS = Symbol();

View file

@ -0,0 +1,58 @@
import type { SchemaPath, ZapierSchemaDocument } from './types.ts';
import { existsSync } from 'fs';
import { pino } from 'pino';
import { readFileSync } from 'fs';
const logLevel = process.env.LOG_LEVEL?.toLowerCase() ?? 'info';
export function idToTypeName(name: string) {
return name.replace(/^\/?/, '').replace(/Schema$/, '');
}
export function refToSchemaName(name: SchemaPath) {
return name.replace(/^\/?/, '');
}
export const logger = pino({
level: logLevel,
transport: {
target: 'pino-pretty',
options: {
ignore: 'time,pid,hostname',
singleLine: true,
},
},
});
export const loadExportedSchemas = (
schemaJsonPath: string,
): ZapierSchemaDocument => {
if (!existsSync(schemaJsonPath)) {
logger.fatal(
{ schemaJsonPath },
'Schema-json file does not exist, aborting',
);
throw new Error(`Schema-json file does not exist: ${schemaJsonPath}`);
} else {
logger.info(
{ schemaJsonPath },
'Successfully found schema-json file to compile.',
);
}
const { version, schemas } = JSON.parse(
readFileSync(schemaJsonPath, 'utf-8'),
);
logger.info(
{
schemaJsonPath,
version,
numRawSchemas: Object.keys(schemas).length,
},
'Loaded %d raw JsonSchemas from zapier-platform-schemas v%s to compile',
Object.keys(schemas).length,
version,
);
return { version, schemas };
};

View file

@ -0,0 +1,29 @@
{
"compilerOptions": {
"rootDir": "./src",
/* Base Options */
"esModuleInterop": true,
"skipLibCheck": true,
"moduleDetection": "force",
/* Strictness */
"strict": true,
"noUncheckedIndexedAccess": true,
"forceConsistentCasingInFileNames": true,
/* Language and Environment */
"target": "ESNext",
"lib": ["ES2022"],
/* Modules */
"noEmit": true /* Run by tsx */,
"module": "NodeNext",
"moduleResolution": "NodeNext",
"resolveJsonModule": true,
"verbatimModuleSyntax": true,
"allowImportingTsExtensions": true
},
"exclude": ["node_modules", "./src/**/*.spec.ts"],
"include": ["./src/**/*.ts"]
}