Milestone 0: import zappier billing, Verae middleware, and Zapier research

Compose-ready workspace: packages/zappier (rate card, portal, Stripe),
packages/verae-zapier-middleware (timestamp + NATS), packages/verae-zapier
(CLI app), vendor/zapier-platform, and research/zapier vendor corpus.

Gate 0 structure checks pass. Product code and research are not yet wired.

vendor/zapier-platform/packages/schema/lib/utils/buildDocs.js

@@ -0,0 +1,231 @@
+'use strict';
+
+const util = require('util');
+
+const _ = require('lodash');
+const toc = require('markdown-toc');
+
+const packageJson = require('../../package.json');
+const links = require('./links');
+
+const NO_DESCRIPTION = '_No description given._';
+const COMBOS = ['anyOf', 'allOf', 'oneOf'];
+const { SKIP_KEY } = require('../constants');
+const hiddenRefs = [];
+
+const walkSchemas = (InitSchema, callback) => {
+  const recurse = (Schema, parents) => {
+    parents = parents || [];
+    callback(Schema, parents);
+    Schema.dependencies.forEach((childSchema) => {
+      const newParents = parents.concat([InitSchema]);
+      recurse(childSchema, newParents);
+    });
+  };
+  recurse(InitSchema);
+};
+
+const collectSchemas = (InitSchema) => {
+  const schemas = {};
+  walkSchemas(InitSchema, (Schema) => {
+    if (!_.get(Schema, 'schema.docAnnotation.hide')) {
+      schemas[Schema.id] = Schema;
+    } else {
+      hiddenRefs.push(Schema.id);
+    }
+  });
+  return schemas;
+};
+
+const BREAK_LENGTH = 96;
+const prepQuote = (val) => val.replace('`', '');
+const quote = (val, triple, indent = '') =>
+  // either ``` with optional indentation or `
+  triple && val.length > BREAK_LENGTH
+    ? '```\n' +
+      val
+        .match(/[^\r\n]+/g)
+        .map((line) => indent + line)
+        .join('\n') +
+      '\n' +
+      indent +
+      '```'
+    : `\`${prepQuote(val)}\``;
+const quoteOrNa = (val, triple = false, indent = '') =>
+  val ? quote(val, triple, indent) : '_n/a_';
+
+const formatExample = (example) => {
+  const ex = _.isPlainObject(example) ? _.omit(example, SKIP_KEY) : example;
+  return `* ${quoteOrNa(
+    util.inspect(ex, { depth: null, breakLength: BREAK_LENGTH }),
+    true,
+    '  ',
+  )}`.replace(/\s+\n/gm, '\n');
+};
+
+// Generate a display of the type (or link to a $ref).
+const typeOrLink = (schema) => {
+  if (schema.type === 'array' && schema.items) {
+    return `${quoteOrNa(schema.type)}[${typeOrLink(schema.items)}]`;
+  }
+  if (schema.$ref) {
+    if (!hiddenRefs.includes(schema.$ref)) {
+      return `[${schema.$ref}](${links.anchor(schema.$ref)})`;
+    }
+    return;
+  }
+  for (let i = 0; i < COMBOS.length; i++) {
+    const key = COMBOS[i];
+    if (schema[key] && schema[key].length) {
+      return `${key}(${schema[key]
+        .map(typeOrLink)
+        .filter(Boolean)
+        .join(', ')})`;
+    }
+  }
+  if (schema.enum && schema.enum.length) {
+    return `${quoteOrNa(schema.type)} in (${schema.enum
+      .map(util.inspect)
+      .map(quoteOrNa)
+      .join(', ')})`;
+  }
+  return quoteOrNa(schema.type);
+};
+
+// Properly quote and display examples.
+const makeExampleSection = (Schema) => {
+  const examples = Schema.schema.examples || [];
+  if (!examples.length) {
+    return '';
+  }
+  return `\
+#### Examples
+
+${examples.map(formatExample).join('\n')}
+`;
+};
+
+// Properly quote and display anti-examples.
+const makeAntiExampleSection = (Schema) => {
+  const antiExamples = Schema.schema.antiExamples || [];
+  if (!antiExamples.length) {
+    return '';
+  }
+  return `\
+#### Anti-Examples
+
+${antiExamples
+  .map(({ example, reason }) => {
+    const formattedAntiExample = formatExample(example);
+    // If block quote, newline and indent the reason.
+    // Otherwise, show the reason inline w/ the anti-example and separated by a dash.
+    return formattedAntiExample.endsWith('```')
+      ? `${formattedAntiExample}\n  _${reason}_`
+      : `${formattedAntiExample} - _${reason}_`;
+  })
+  .join('\n')}
+`;
+};
+
+const processProperty = (key, property, propIsRequired) => {
+  let isRequired = propIsRequired ? '**yes**' : 'no';
+  if (_.get(property, 'docAnnotation.hide')) {
+    return '';
+  } else if (_.get(property, 'docAnnotation.required')) {
+    // can also support keys besides "required"
+    const annotation = property.docAnnotation.required;
+    if (annotation.type === 'replace') {
+      isRequired = annotation.value;
+    } else if (annotation.type === 'append') {
+      isRequired += annotation.value;
+    } else {
+      throw new Error(`unrecognized docAnnotation type: ${annotation.type}`);
+    }
+  }
+  return `${quoteOrNa(key)} | ${isRequired} | ${typeOrLink(property)} | ${
+    property.description || NO_DESCRIPTION
+  }`;
+};
+
+// Enumerate the properties as a table.
+const makePropertiesSection = (Schema) => {
+  const properties =
+    Schema.schema.properties || Schema.schema.patternProperties || {};
+  if (!Object.keys(properties).length) {
+    return '';
+  }
+  const required = Schema.schema.required || [];
+  return `\
+#### Properties
+
+Key | Required | Type | Description
+--- | -------- | ---- | -----------
+${Object.keys(properties)
+  .map((key) => {
+    const property = properties[key];
+    return processProperty(key, property, required.includes(key));
+  })
+  .join('\n')}
+`;
+};
+
+// Given a "root" schema, create some markdown.
+const makeMarkdownSection = (Schema) => {
+  return `\
+## ${Schema.id}
+
+${Schema.schema.description || NO_DESCRIPTION}
+
+#### Details
+
+* **Type** - ${typeOrLink(Schema.schema)}${
+    Schema.schema.pattern
+      ? `
+* **Pattern** - ${quoteOrNa(Schema.schema.pattern)}`
+      : ''
+  }
+* [**Source Code**](${links.makeCodeLink(Schema.id)})
+
+${makePropertiesSection(Schema)}
+${makeExampleSection(Schema)}
+${makeAntiExampleSection(Schema)}
+`.trim();
+};
+
+// Generate the final markdown.
+const buildDocs = (InitSchema) => {
+  const schemas = collectSchemas(InitSchema);
+  const markdownSections = _.chain(schemas)
+    .values()
+    .sortBy('id')
+    .map(makeMarkdownSection)
+    .join('\n\n-----\n\n');
+  const docs = `\
+<!-- {% raw %} -->
+# \`zapier-platform-schema\` Generated Documentation
+
+This is automatically generated by the \`npm run docs\` command in \`zapier-platform-schema\` version ${quoteOrNa(
+    packageJson.version,
+  )}.
+
+To see the docs for a different version, switch to the corresponding version tag in the GitHub repository.
+
+For example, to view docs for version 17.7.0, switch to the tag 'zapier-platform-schema@17.7.0'.
+
+Alternatively, modify the URL directly: https://github.com/zapier/zapier-platform/blob/zapier-platform-schema@17.7.0/packages/schema/docs/build/schema.md, replacing 17.7.0 with your desired version.
+
+
+-----
+
+## Index
+<!-- toc -->
+
+-----
+
+${markdownSections}
+<!-- {% endraw %} -->
+`.trim();
+  return toc.insert(docs, { maxdepth: 2, bullets: '*' });
+};
+
+module.exports = buildDocs;

vendor/zapier-platform/packages/schema/lib/utils/exportSchema.js

@@ -0,0 +1,23 @@
+'use strict';
+
+const _ = require('lodash');
+const packageJson = require('../../package.json');
+
+const exportSchema = (InitSchema) => {
+  const exportedSchema = {
+    version: packageJson.version,
+    schemas: {},
+  };
+  const addAndRecurse = (Schema) => {
+    exportedSchema.schemas[Schema.id.replace('/', '')] = _.omit(
+      Schema.schema,
+      'examples',
+      'antiExamples',
+    );
+    Schema.dependencies.map(addAndRecurse);
+  };
+  addAndRecurse(InitSchema);
+  return exportedSchema;
+};
+
+module.exports = exportSchema;

vendor/zapier-platform/packages/schema/lib/utils/links.js

@@ -0,0 +1,27 @@
+const _ = require('lodash');
+
+const packageJson = require('../../package.json');
+const constants = require('../constants.js');
+
+// From '</SomeSchema>' to 'SomeSchema'.
+const filename = (val) => _.trim(String(val), '/<>');
+
+// From '/SomeSchema' to '#someschema'.
+const anchor = (val) => '#' + filename(val.toLowerCase());
+
+const makeCodeLink = (id) =>
+  `${constants.ROOT_GITHUB}/blob/zapier-platform-schema@${
+    packageJson.version
+  }/packages/schema/lib/schemas/${filename(id)}.js`;
+const makeDocLink = (id) =>
+  _.template(constants.DOC_URL_TEMPLATE)({
+    version: packageJson.version,
+    anchor: anchor(id),
+  });
+
+module.exports = {
+  filename,
+  anchor,
+  makeCodeLink,
+  makeDocLink,
+};

vendor/zapier-platform/packages/schema/lib/utils/makeSchema.js

@@ -0,0 +1,27 @@
+'use strict';
+
+const _ = require('lodash');
+
+const makeValidator = require('./makeValidator');
+
+const getRawSchema = (schema) => schema.schema;
+
+const getDependencies = (schema) => schema.dependencies;
+
+const flattenDependencies = (schemas) => {
+  schemas = schemas || [];
+  return _.flatten(schemas.map(getDependencies).concat(schemas));
+};
+
+const makeSchema = (schemaDefinition, schemaDependencies) => {
+  const dependencies = flattenDependencies(schemaDependencies);
+  const validatorDependencies = dependencies.map(getRawSchema);
+  return {
+    dependencies,
+    id: schemaDefinition.id,
+    schema: schemaDefinition,
+    validate: makeValidator(schemaDefinition, validatorDependencies).validate,
+  };
+};
+
+module.exports = makeSchema;

vendor/zapier-platform/packages/schema/lib/utils/makeValidator.js

@@ -0,0 +1,151 @@
+'use strict';
+
+const jsonschema = require('jsonschema');
+const links = require('./links');
+const functionalConstraints = require('../functional-constraints');
+const { flattenDeep, get } = require('lodash');
+
+const ambiguousTypes = ['anyOf', 'oneOf', 'allOf'];
+
+const makeLinks = (error, makerFunc) => {
+  if (typeof error.schema === 'string') {
+    return [makerFunc(error.schema)];
+  }
+  if (
+    ambiguousTypes.includes(error.name) &&
+    error.argument &&
+    error.argument.length
+  ) {
+    // no way to know what the subschema was, so don't create links for it
+    return error.argument
+      .map((s) => (s.includes('subschema') ? '' : makerFunc(s)))
+      .filter(Boolean);
+  }
+  return [];
+};
+
+const removeFirstAndLastChar = (s) => s.slice(1, -1);
+// always return a string
+const makePath = (path, newSegment) =>
+  (path ? [path, newSegment].join('.') : newSegment) || '';
+
+const processBaseError = (err, path) => {
+  const completePath = makePath(path, err.property)
+    .replace(/\.instance\.?/g, '.')
+    .replace(/\.instance$/, '')
+    .replace(/\.$/, ''); // Remove any trailing dots
+
+  const subSchemas = err.message.match(/\[subschema \d+\]/g);
+  if (subSchemas) {
+    subSchemas.forEach((subschema, idx) => {
+      // err.schema is either an anonymous schema object or the name of a named schema
+      if (typeof err.schema === 'string') {
+        // this is basically only for FieldChoicesSchema and I'm not sure why
+        err.message += ' Consult the docs below for valid subschemas.';
+      } else {
+        // the subschemas have a type property
+        err.message = err.message.replace(
+          subschema,
+          err.schema[err.name][idx].type || 'unknown',
+        );
+      }
+    });
+  }
+
+  err.property = completePath;
+  return err;
+};
+
+/**
+ * We have a lot of `anyOf` schemas that return ambiguous errors. This recurses down the schema until it finds the errors that cause the failures replaces the ambiguity.
+ * @param {ValidationError} validationError an individual error
+ * @param {string} path current path in the error chain
+ * @param {Validator} validator validator object to pass around that has all the schemas
+ * @param {object} definition the original schema we're defining
+ */
+const cleanError = (validationError, path, validator, definition) => {
+  if (ambiguousTypes.includes(validationError.name)) {
+    // flatObjectSchema requires each property to be a type. instead of recursing down, it's more valuable to say "hey, it's not of these types"
+    if (validationError.argument.every((s) => s.includes('subschema'))) {
+      return processBaseError(validationError, path);
+    }
+
+    // Try against each of A, B, and C to take a guess as to which it's closed to
+    // errorGroups will be an array of arrays of errors
+    const errorGroups = validationError.argument.map((schemaName, idx) => {
+      // this is what we'll validate against next
+      let nextSchema;
+      // schemaName is either "[subschema n]" or "/NamedSchema"
+      if (schemaName.startsWith('[subschema')) {
+        const maybeNamedSchema = validator.schemas[validationError.schema];
+
+        if (maybeNamedSchema) {
+          nextSchema = maybeNamedSchema[validationError.name][idx];
+        } else {
+          // hoist the anonymous subschema up
+          nextSchema = validationError.schema[validationError.name][idx];
+        }
+      } else {
+        nextSchema = validator.schemas[removeFirstAndLastChar(schemaName)];
+      }
+
+      if (validationError.instance === undefined) {
+        // Work around a jsonschema bug: When the value being validated is
+        // falsy, validationError.instance isn't available
+        // See https://github.com/tdegrunt/jsonschema/issues/263
+        const fullPath =
+          path.replace(/^instance\./, '') +
+          validationError.property.replace(/^instance\./, '');
+        validationError.instance = get(definition, fullPath);
+      }
+
+      const res = validator.validate(validationError.instance, nextSchema);
+
+      return res.errors.map((e) =>
+        cleanError(
+          e,
+          makePath(path, validationError.property),
+          validator,
+          definition,
+        ),
+      );
+    });
+
+    // find the group with the fewest errors, that's probably the most accurate
+    // if we're goign to tweak what gets returned, this is where we'll do it
+    // a possible improvement could be treating a longer path favorably, like the python implementation does
+    errorGroups.sort((a, b) => a.length - b.length);
+    return errorGroups[0];
+  } else {
+    // base case
+    return processBaseError(validationError, path);
+  }
+};
+
+const makeValidator = (mainSchema, subSchemas) => {
+  const schemas = [mainSchema].concat(subSchemas || []);
+  const v = new jsonschema.Validator();
+  schemas.forEach((Schema) => {
+    v.addSchema(Schema, Schema.id);
+  });
+  return {
+    validate: (definition) => {
+      const results = v.validate(definition, mainSchema);
+      const allErrors = results.errors.concat(
+        functionalConstraints.run(definition, mainSchema),
+      );
+      const cleanedErrors = flattenDeep(
+        allErrors.map((e) => cleanError(e, '', v, definition)),
+      );
+
+      results.errors = cleanedErrors.map((error) => {
+        error.codeLinks = makeLinks(error, links.makeCodeLink);
+        error.docLinks = makeLinks(error, links.makeDocLink);
+        return error;
+      });
+      return results;
+    },
+  };
+};
+
+module.exports = makeValidator;