feat: add conformance test for authorization server metadata (#170)

2026-04-01 21:42:23 +09:00 · 2026-04-01 21:42:23 +09:00 · 050d9cf93a
commit 050d9cf93a
parent cb0f4c3601
7 changed files with 479 additions and 10 deletions
--- a/src/index.ts
+++ b/src/index.ts
@ -10,6 +10,10 @@ import {
  printServerSummary,
  runInteractiveMode
 } from './runner';
+import {
+  printAuthorizationServerSummary,
+  runAuthorizationServerConformanceTest
+} from './runner/authorization-server';
 import {
  listScenarios,
  listClientScenarios,
@ -24,11 +28,17 @@ import {
  listScenariosForSpec,
  listClientScenariosForSpec,
  getScenarioSpecVersions,
+  listClientScenariosForAuthorizationServer,
+  listClientScenariosForAuthorizationServerForSpec,
  resolveSpecVersion
 } from './scenarios';
 import type { SpecVersion } from './scenarios';
 import { ConformanceCheck } from './types';
-import { ClientOptionsSchema, ServerOptionsSchema } from './schemas';
+import {
+  AuthorizationServerOptionsSchema,
+  ClientOptionsSchema,
+  ServerOptionsSchema
+} from './schemas';
 import {
  loadExpectedFailures,
  evaluateBaseline,
@ -44,12 +54,19 @@ import packageJson from '../package.json';
 function filterScenariosBySpecVersion(
  allScenarios: string[],
  version: SpecVersion,
-  command: 'client' | 'server'
+  command: 'client' | 'server' | 'authorization'
 ): string[] {
-  const versionScenarios =
-    command === 'client'
-      ? listScenariosForSpec(version)
-      : listClientScenariosForSpec(version);
+  let versionScenarios: string[];
+  if (command === 'client') {
+    versionScenarios = listScenariosForSpec(version);
+  } else if (command === 'server') {
+    versionScenarios = listClientScenariosForSpec(version);
+  } else if (command === 'authorization') {
+    versionScenarios =
+      listClientScenariosForAuthorizationServerForSpec(version);
+  } else {
+    versionScenarios = [];
+  }
  const allowed = new Set(versionScenarios);
  return allScenarios.filter((s) => allowed.has(s));
 }
@ -437,6 +454,87 @@ program
    }
  });

+// Authorization command - tests an authorization server implementation
+program
+  .command('authorization')
+  .description(
+    'Run conformance tests against an authorization server implementation'
+  )
+  .requiredOption('--url <url>', 'URL of the authorization server issuer')
+  .option('-o, --output-dir <path>', 'Save results to this directory')
+  .option(
+    '--spec-version <version>',
+    'Filter scenarios by spec version (cumulative for date versions)'
+  )
+  .action(async (options) => {
+    try {
+      // Validate options with Zod
+      const validated = AuthorizationServerOptionsSchema.parse(options);
+      const outputDir = options.outputDir;
+      const specVersionFilter = options.specVersion
+        ? resolveSpecVersion(options.specVersion)
+        : undefined;
+
+      let scenarios: string[];
+      scenarios = listClientScenariosForAuthorizationServer();
+      if (specVersionFilter) {
+        scenarios = filterScenariosBySpecVersion(
+          scenarios,
+          specVersionFilter,
+          'authorization'
+        );
+      }
+      console.log(
+        `Running test (${scenarios.length} scenarios) against ${validated.url}\n`
+      );
+
+      const allResults: { scenario: string; checks: ConformanceCheck[] }[] = [];
+      for (const scenarioName of scenarios) {
+        console.log(`\n=== Running scenario: ${scenarioName} ===`);
+        try {
+          const result = await runAuthorizationServerConformanceTest(
+            validated.url,
+            scenarioName,
+            outputDir
+          );
+          allResults.push({ scenario: scenarioName, checks: result.checks });
+        } catch (error) {
+          console.error(`Failed to run scenario ${scenarioName}:`, error);
+          allResults.push({
+            scenario: scenarioName,
+            checks: [
+              {
+                id: scenarioName,
+                name: scenarioName,
+                description: 'Failed to run scenario',
+                status: 'FAILURE',
+                timestamp: new Date().toISOString(),
+                errorMessage:
+                  error instanceof Error ? error.message : String(error)
+              }
+            ]
+          });
+        }
+      }
+      const { totalFailed } = printAuthorizationServerSummary(allResults);
+      process.exit(totalFailed > 0 ? 1 : 0);
+    } catch (error) {
+      if (error instanceof ZodError) {
+        console.error('Validation error:');
+        error.issues.forEach((err) => {
+          console.error(`  ${err.path.join('.')}: ${err.message}`);
+        });
+        console.error('\nAvailable authorization server scenarios:');
+        listClientScenariosForAuthorizationServer().forEach((s) =>
+          console.error(`  - ${s}`)
+        );
+        process.exit(1);
+      }
+      console.error('Authorization server test error:', error);
+      process.exit(1);
+    }
+  });
+
 // Tier check command
 program.addCommand(createTierCheckCommand());

@ -446,6 +544,7 @@ program
  .description('List available test scenarios')
  .option('--client', 'List client scenarios')
  .option('--server', 'List server scenarios')
+  .option('--authorization', 'List authorization server scenarios')
  .option(
    '--spec-version <version>',
    'Filter scenarios by spec version (cumulative for date versions)'
@ -455,7 +554,10 @@ program
      ? resolveSpecVersion(options.specVersion)
      : undefined;

-    if (options.server || (!options.client && !options.server)) {
+    if (
+      options.server ||
+      (!options.client && !options.server && !options.authorization)
+    ) {
      console.log('Server scenarios (test against a server):');
      let serverScenarios = listClientScenarios();
      if (specVersionFilter) {
@ -471,7 +573,10 @@ program
      });
    }

-    if (options.client || (!options.client && !options.server)) {
+    if (
+      options.client ||
+      (!options.client && !options.server && !options.authorization)
+    ) {
      if (options.server || (!options.client && !options.server)) {
        console.log('');
      }
@ -489,6 +594,31 @@ program
        console.log(`  - ${s}${v ? ` [${v}]` : ''}`);
      });
    }
+
+    if (
+      options.authorization ||
+      (!options.authorization && !options.server && !options.client)
+    ) {
+      if (!(options.authorization && !options.server && !options.client)) {
+        console.log('');
+      }
+      console.log(
+        'Authorization server scenarios (test against an authorization server):'
+      );
+      let authorizationServerScenarios =
+        listClientScenariosForAuthorizationServer();
+      if (specVersionFilter) {
+        authorizationServerScenarios = filterScenariosBySpecVersion(
+          authorizationServerScenarios,
+          specVersionFilter,
+          'authorization'
+        );
+      }
+      authorizationServerScenarios.forEach((s) => {
+        const v = getScenarioSpecVersions(s);
+        console.log(`  - ${s}${v ? ` [${v}]` : ''}`);
+      });
+    }
  });

 program.parse();
--- a/src/runner/authorization-server.ts
+++ b/src/runner/authorization-server.ts
@ -0,0 +1,74 @@
+import { promises as fs } from 'fs';
+import path from 'path';
+import { ConformanceCheck } from '../types';
+import { getClientScenarioForAuthorizationServer } from '../scenarios';
+import { createResultDir } from './utils';
+
+export async function runAuthorizationServerConformanceTest(
+  serverUrl: string,
+  scenarioName: string,
+  outputDir?: string
+): Promise<{
+  checks: ConformanceCheck[];
+  resultDir?: string;
+  scenarioDescription: string;
+}> {
+  let resultDir: string | undefined;
+
+  if (outputDir) {
+    resultDir = createResultDir(
+      outputDir,
+      scenarioName,
+      'authorization-server'
+    );
+    await fs.mkdir(resultDir, { recursive: true });
+  }
+
+  // Scenario is guaranteed to exist by CLI validation
+  const scenario = getClientScenarioForAuthorizationServer(scenarioName)!;
+
+  console.log(
+    `Running client scenario for authorization server '${scenarioName}' against server: ${serverUrl}`
+  );
+
+  const checks = await scenario.run(serverUrl);
+
+  if (resultDir) {
+    await fs.writeFile(
+      path.join(resultDir, 'checks.json'),
+      JSON.stringify(checks, null, 2)
+    );
+
+    console.log(`Results saved to ${resultDir}`);
+  }
+
+  return {
+    checks,
+    resultDir,
+    scenarioDescription: scenario.description
+  };
+}
+
+export function printAuthorizationServerSummary(
+  allResults: { scenario: string; checks: ConformanceCheck[] }[]
+): { totalPassed: number; totalFailed: number } {
+  console.log('\n\n=== SUMMARY ===');
+  let totalPassed = 0;
+  let totalFailed = 0;
+
+  for (const result of allResults) {
+    const passed = result.checks.filter((c) => c.status === 'SUCCESS').length;
+    const failed = result.checks.filter((c) => c.status === 'FAILURE').length;
+    totalPassed += passed;
+    totalFailed += failed;
+
+    const status = failed === 0 ? '✓' : '✗';
+    console.log(
+      `${status} ${result.scenario}: ${passed} passed, ${failed} failed`
+    );
+  }
+
+  console.log(`\nTotal: ${totalPassed} passed, ${totalFailed} failed`);
+
+  return { totalPassed, totalFailed };
+}
--- a/src/scenarios/authorization-server/authorization-server-metadata.test.ts
+++ b/src/scenarios/authorization-server/authorization-server-metadata.test.ts
@ -0,0 +1,51 @@
+import { describe, it, expect, vi } from 'vitest';
+import { AuthorizationServerMetadataEndpointScenario } from './authorization-server-metadata.js';
+import { request } from 'undici';
+
+vi.mock('undici', () => ({
+  request: vi.fn()
+}));
+
+const mockedRequest = vi.mocked(request);
+
+describe('AuthorizationServerMetadataEndpointScenario (SUCCESS only)', () => {
+  it('returns SUCCESS for valid authorization server metadata', async () => {
+    const scenario = new AuthorizationServerMetadataEndpointScenario();
+    const serverUrl = 'https://example.com';
+
+    mockedRequest.mockResolvedValue({
+      statusCode: 200,
+      headers: {
+        'content-type': 'application/json'
+      },
+      body: {
+        json: async () => ({
+          issuer: 'https://example.com',
+          authorization_endpoint: 'https://example.com/auth',
+          token_endpoint: 'https://example.com/token',
+          response_types_supported: ['code']
+        })
+      }
+    } as any);
+
+    const checks = await scenario.run(serverUrl);
+
+    expect(checks).toHaveLength(1);
+
+    const check = checks[0];
+    expect(check.status).toBe('SUCCESS');
+    expect(check.errorMessage).toBeUndefined();
+    expect(check.details).toBeDefined();
+    expect(check.details!.contentType).toContain('application/json');
+    expect((check.details!.body as any).issuer).toBe('https://example.com');
+    expect((check.details!.body as any).authorization_endpoint).toBe(
+      'https://example.com/auth'
+    );
+    expect((check.details!.body as any).token_endpoint).toBe(
+      'https://example.com/token'
+    );
+    expect((check.details!.body as any).response_types_supported).toEqual([
+      'code'
+    ]);
+  });
+});
--- a/src/scenarios/authorization-server/authorization-server-metadata.ts
+++ b/src/scenarios/authorization-server/authorization-server-metadata.ts
@ -0,0 +1,155 @@
+/**
+ * Authorization server metadata endpoint test scenarios for MCP authorization servers
+ */
+import {
+  ClientScenarioForAuthorizationServer,
+  ConformanceCheck,
+  SpecVersion
+} from '../../types';
+import { request } from 'undici';
+
+type Status = 'SUCCESS' | 'FAILURE';
+
+export class AuthorizationServerMetadataEndpointScenario implements ClientScenarioForAuthorizationServer {
+  name = 'authorization-server-metadata-endpoint';
+  specVersions: SpecVersion[] = ['2025-03-26', '2025-06-18', '2025-11-25'];
+  description = `Test authorization server metadata endpoint.
+
+**Authorization Server Implementation Requirements:**
+
+**Endpoint**: \`authorization server metadata\`
+
+**Requirements**:
+- HTTP response status code MUST be 200 OK
+- Content-Type header MUST be application/json
+- Return a JSON response including issuer, authorization_endpoint, token_endpoint and response_types_supported
+- The issuer value MUST match the URI obtained by removing the well-known URI string from the authorization server metadata URI.`;
+
+  async run(serverUrl: string): Promise<ConformanceCheck[]> {
+    let status: Status = 'SUCCESS';
+    let errorMessage: string | undefined;
+    let details: any;
+    let response: any | null = null;
+    try {
+      const wellKnownUrls = this.createWellKnownUrl(serverUrl);
+
+      for (const url of wellKnownUrls) {
+        try {
+          const checkResponse = await request(url, { method: 'GET' });
+          if (checkResponse.statusCode === 200) {
+            response = checkResponse;
+            break;
+          }
+        } catch {
+          // Ignore the error and proceed to the next loop.
+        }
+      }
+
+      if (!response) {
+        throw new Error(
+          'All authorization server metadata endpoints returned invalid status code.'
+        );
+      }
+
+      this.validateContentType(response.headers['content-type']);
+
+      const body = await this.parseJson(response);
+      const errors: string[] = [];
+      this.validateMetadataBody(body, serverUrl, errors);
+
+      if (errors.length > 0) {
+        status = 'FAILURE';
+        errorMessage = errors.join(', ');
+      }
+
+      details = {
+        contentType: response.headers['content-type'],
+        body
+      };
+    } catch (error) {
+      status = 'FAILURE';
+      errorMessage = error instanceof Error ? error.message : String(error);
+    }
+
+    return [
+      {
+        id: 'authorization-server-metadata',
+        name: 'AuthorizationServerMetadata',
+        description: 'Valid authorization server metadata response',
+        status,
+        timestamp: new Date().toISOString(),
+        errorMessage,
+        specReferences: [
+          {
+            id: 'Authorization-Server-Metadata',
+            url: 'https://datatracker.ietf.org/doc/html/rfc8414'
+          }
+        ],
+        ...(details ? { details } : {})
+      }
+    ];
+  }
+
+  private createWellKnownUrl(serverUrl: string): string[] {
+    const base = new URL(serverUrl);
+    const origin = base.origin;
+    const path = base.pathname.replace(/\/$/, '');
+
+    const urls = new Set<string>();
+    urls.add(`${origin}/.well-known/oauth-authorization-server${path}`);
+    urls.add(`${origin}/.well-known/openid-configuration${path}`);
+    urls.add(`${origin}${path}/.well-known/openid-configuration`);
+
+    return Array.from(urls);
+  }
+
+  private validateContentType(contentType?: string | string[]): void {
+    const valid =
+      typeof contentType === 'string' &&
+      contentType.toLowerCase().includes('application/json');
+
+    if (!valid) {
+      throw new Error(`Invalid Content-Type: ${contentType ?? '(missing)'}`);
+    }
+  }
+
+  private async parseJson(response: any): Promise<Record<string, any>> {
+    const body = await response.body.json();
+    if (typeof body !== 'object' || body === null) {
+      throw new Error('Response body is not an object');
+    }
+    return body;
+  }
+
+  private validateMetadataBody(
+    body: Record<string, any>,
+    serverUrl: string,
+    errors: string[]
+  ): void {
+    this.assertString(
+      body.authorization_endpoint,
+      'authorization_endpoint',
+      errors
+    );
+    this.assertString(body.token_endpoint, 'token_endpoint', errors);
+
+    if (
+      !Array.isArray(body.response_types_supported) ||
+      !body.response_types_supported.includes('code')
+    ) {
+      errors.push(
+        'Response body does not include valid "response_types_supported" claim'
+      );
+    }
+
+    if (body.issuer !== serverUrl) {
+      errors.push(`Invalid issuer: ${body.issuer ?? '(missing)'}`);
+    }
+  }
+
+  private assertString(value: unknown, name: string, errors: string[]): void {
+    if (typeof value !== 'string' || value.length === 0) {
+      errors.push(`Response body does not include valid "${name}" claim`);
+    }
+  }
+}
--- a/src/scenarios/index.ts
+++ b/src/scenarios/index.ts
@ -1,4 +1,9 @@
-import { Scenario, ClientScenario, SpecVersion } from '../types';
+import {
+  Scenario,
+  ClientScenario,
+  ClientScenarioForAuthorizationServer,
+  SpecVersion
+} from '../types';
 import { InitializeScenario } from './client/initialize';
 import { ToolsCallScenario } from './client/tools_call';
 import { ElicitationClientDefaultsScenario } from './client/elicitation-defaults';
@ -60,6 +65,7 @@ import {
  extensionScenariosList
 } from './client/auth/index';
 import { listMetadataScenarios } from './client/auth/discovery-metadata';
+import { AuthorizationServerMetadataEndpointScenario } from './authorization-server/authorization-server-metadata';

 // Pending client scenarios (not yet fully tested/implemented)
 const pendingClientScenariosList: ClientScenario[] = [
@ -142,6 +148,23 @@ export const clientScenarios = new Map<string, ClientScenario>(
  allClientScenariosList.map((scenario) => [scenario.name, scenario])
 );

+// All client scenarios for authorization server
+const allClientScenariosListForAuthorizationServer: ClientScenario[] = [
+  // Authorization server scenarios
+  new AuthorizationServerMetadataEndpointScenario()
+];
+
+// Client scenarios map for authorization server - built from list
+export const clientScenariosForAuthorizationServer = new Map<
+  string,
+  ClientScenario
+>(
+  allClientScenariosListForAuthorizationServer.map((scenario) => [
+    scenario.name,
+    scenario
+  ])
+);
+
 // All client test scenarios (core + backcompat + extensions)
 const scenariosList: Scenario[] = [
  new InitializeScenario(),
@ -180,6 +203,12 @@ export function getClientScenario(name: string): ClientScenario | undefined {
  return clientScenarios.get(name);
 }

+export function getClientScenarioForAuthorizationServer(
+  name: string
+): ClientScenarioForAuthorizationServer | undefined {
+  return clientScenariosForAuthorizationServer.get(name);
+}
+
 export function listScenarios(): string[] {
  return Array.from(scenarios.keys());
 }
@ -212,6 +241,10 @@ export function listBackcompatScenarios(): string[] {
  return backcompatScenariosList.map((scenario) => scenario.name);
 }

+export function listClientScenariosForAuthorizationServer(): string[] {
+  return Array.from(clientScenariosForAuthorizationServer.keys());
+}
+
 export function listDraftScenarios(): string[] {
  return draftScenariosList.map((scenario) => scenario.name);
 }
@ -248,11 +281,21 @@ export function listClientScenariosForSpec(version: SpecVersion): string[] {
    .map((s) => s.name);
 }

+export function listClientScenariosForAuthorizationServerForSpec(
+  version: SpecVersion
+): string[] {
+  return allClientScenariosListForAuthorizationServer
+    .filter((s) => s.specVersions.includes(version))
+    .map((s) => s.name);
+}
+
 export function getScenarioSpecVersions(
  name: string
 ): SpecVersion[] | undefined {
  return (
-    scenarios.get(name)?.specVersions ?? clientScenarios.get(name)?.specVersions
+    scenarios.get(name)?.specVersions ??
+    clientScenarios.get(name)?.specVersions ??
+    clientScenariosForAuthorizationServer.get(name)?.specVersions
  );
 }

--- a/src/schemas.ts
+++ b/src/schemas.ts
@ -38,6 +38,15 @@ export const ServerOptionsSchema = z.object({

 export type ServerOptions = z.infer<typeof ServerOptionsSchema>;

+// Authorization server command options schema
+export const AuthorizationServerOptionsSchema = z.object({
+  url: z.string().url('Invalid authorization server URL')
+});
+
+export type AuthorizationServerOptions = z.infer<
+  typeof AuthorizationServerOptionsSchema
+>;
+
 // Interactive command options schema
 export const InteractiveOptionsSchema = z.object({
  scenario: z
--- a/src/types.ts
+++ b/src/types.ts
@ -60,3 +60,10 @@ export interface ClientScenario {
  specVersions: SpecVersion[];
  run(serverUrl: string): Promise<ConformanceCheck[]>;
 }
+
+export interface ClientScenarioForAuthorizationServer {
+  name: string;
+  description: string;
+  specVersions: SpecVersion[];
+  run(serverUrl: string): Promise<ConformanceCheck[]>;
+}