[scenario] Add test for broken oauth client (#12)

* starting on broken auth client

* switch to middleware approach

* add broken example

* fix failure testing

* subset matching

examples/clients/typescript/auth-test-broken1.ts

@@ -0,0 +1,99 @@
+#!/usr/bin/env node
+
+import { Client } from '@modelcontextprotocol/sdk/client/index.js';
+import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js';
+import { handle401, withOAuthRetry } from './helpers/withOAuthRetry.js';
+import { ConformanceOAuthProvider } from './helpers/ConformanceOAuthProvider.js';
+import { FetchLike } from '@modelcontextprotocol/sdk/shared/transport.js';
+import {
+  auth,
+  UnauthorizedError
+} from '@modelcontextprotocol/sdk/client/auth.js';
+
+export const handle401Broken = async (
+  response: Response,
+  provider: ConformanceOAuthProvider,
+  next: FetchLike,
+  serverUrl: string | URL
+): Promise<void> => {
+  // BROKEN: Use root-based PRM discovery exclusively, regardless of input.
+  const resourceMetadataUrl = new URL(
+    '/.well-known/oauth-protected-resource',
+    typeof serverUrl === 'string' ? serverUrl : serverUrl.origin
+  );
+
+  let result = await auth(provider, {
+    serverUrl,
+    resourceMetadataUrl,
+    fetchFn: next
+  });
+
+  if (result === 'REDIRECT') {
+    // Ordinarily, we'd wait for the callback to be handled here,
+    // but in our conformance provider, we get the authorization code
+    // during the redirect handling, so we can go straight to
+    // retrying the auth step.
+    // await provider.waitForCallback();
+
+    const authorizationCode = await provider.getAuthCode();
+
+    // TODO: this retry logic should be incorporated into the typescript SDK
+    result = await auth(provider, {
+      serverUrl,
+      resourceMetadataUrl,
+      authorizationCode,
+      fetchFn: next
+    });
+    if (result !== 'AUTHORIZED') {
+      throw new UnauthorizedError(
+        `Authentication failed with result: ${result}`
+      );
+    }
+  }
+};
+
+async function main(): Promise<void> {
+  const serverUrl = process.argv[2];
+
+  if (!serverUrl) {
+    console.error('Usage: auth-test <server-url>');
+    process.exit(1);
+  }
+
+  console.log(`Connecting to MCP server at: ${serverUrl}`);
+
+  const client = new Client(
+    {
+      name: 'test-auth-client',
+      version: '1.0.0'
+    },
+    {
+      capabilities: {}
+    }
+  );
+
+  // Create a custom fetch that uses the OAuth middleware with retry logic
+  const oauthFetch = withOAuthRetry(
+    'test-auth-client',
+    new URL(serverUrl),
+    handle401Broken
+  )(fetch);
+
+  const transport = new StreamableHTTPClientTransport(new URL(serverUrl), {
+    fetch: oauthFetch
+  });
+
+  // Connect to the server - OAuth is handled automatically by the middleware
+  await client.connect(transport);
+  console.log('✅ Successfully connected to MCP server');
+
+  await client.listTools();
+  console.log('✅ Successfully listed tools');
+
+  await transport.close();
+  console.log('✅ Connection closed successfully');
+
+  process.exit(0);
+}
+
+main();

examples/clients/typescript/auth-test.ts

@@ -2,8 +2,7 @@
 
 import { Client } from '@modelcontextprotocol/sdk/client/index.js';
 import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js';
-import { ConformanceOAuthProvider } from './helpers/ConformanceOAuthProvider.js';
-import { UnauthorizedError } from '@modelcontextprotocol/sdk/client/auth.js';
+import { withOAuthRetry } from './helpers/withOAuthRetry.js';
 
 async function main(): Promise<void> {
   const serverUrl = process.argv[2];
@@ -25,47 +24,19 @@ async function main(): Promise<void> {
     }
   );
 
-  const authProvider = new ConformanceOAuthProvider(
-    'http://localhost:3000/callback',
-    {
-      client_name: 'test-auth-client',
-      redirect_uris: ['http://localhost:3000/callback']
-    }
-  );
+  // Create a custom fetch that uses the OAuth middleware with retry logic
+  const oauthFetch = withOAuthRetry(
+    'test-auth-client',
+    new URL(serverUrl)
+  )(fetch);
 
-  let transport = new StreamableHTTPClientTransport(new URL(serverUrl), {
-    authProvider
+  const transport = new StreamableHTTPClientTransport(new URL(serverUrl), {
+    fetch: oauthFetch
   });
 
-  // Try to connect - handle OAuth if needed
-  try {
-    await client.connect(transport);
-    console.log('✅ Successfully connected to MCP server');
-  } catch (error) {
-    if (error instanceof UnauthorizedError) {
-      console.log('🔐 OAuth required - handling authorization...');
-
-      // The provider will automatically fetch the auth code
-      const authCode = await authProvider.getAuthCode();
-
-      // Complete the auth flow
-      await transport.finishAuth(authCode);
-
-      // Close the old transport
-      await transport.close();
-
-      // Create a new transport with the authenticated provider
-      transport = new StreamableHTTPClientTransport(new URL(serverUrl), {
-        authProvider: authProvider
-      });
-
-      // Connect with the new transport
-      await client.connect(transport);
-      console.log('✅ Successfully connected with authentication');
-    } else {
-      throw error;
-    }
-  }
+  // Connect to the server - OAuth is handled automatically by the middleware
+  await client.connect(transport);
+  console.log('✅ Successfully connected to MCP server');
 
   await client.listTools();
   console.log('✅ Successfully listed tools');

examples/clients/typescript/helpers/withOAuthRetry.ts

@@ -0,0 +1,109 @@
+import {
+  auth,
+  extractResourceMetadataUrl,
+  UnauthorizedError
+} from '@modelcontextprotocol/sdk/client/auth.js';
+import type { FetchLike } from '@modelcontextprotocol/sdk/shared/transport.js';
+import type { Middleware } from '@modelcontextprotocol/sdk/client/middleware.js';
+import { ConformanceOAuthProvider } from './ConformanceOAuthProvider';
+
+export const handle401 = async (
+  response: Response,
+  provider: ConformanceOAuthProvider,
+  next: FetchLike,
+  serverUrl: string | URL
+): Promise<void> => {
+  const resourceMetadataUrl = extractResourceMetadataUrl(response);
+
+  let result = await auth(provider, {
+    serverUrl,
+    resourceMetadataUrl,
+    fetchFn: next
+  });
+
+  if (result === 'REDIRECT') {
+    // Ordinarily, we'd wait for the callback to be handled here,
+    // but in our conformance provider, we get the authorization code
+    // during the redirect handling, so we can go straight to
+    // retrying the auth step.
+    // await provider.waitForCallback();
+
+    const authorizationCode = await provider.getAuthCode();
+
+    // TODO: this retry logic should be incorporated into the typescript SDK
+    result = await auth(provider, {
+      serverUrl,
+      resourceMetadataUrl,
+      authorizationCode,
+      fetchFn: next
+    });
+    if (result !== 'AUTHORIZED') {
+      throw new UnauthorizedError(
+        `Authentication failed with result: ${result}`
+      );
+    }
+  }
+};
+/**
+ * Creates a fetch wrapper that handles OAuth authentication with retry logic.
+ *
+ * Unlike the SDK's withOAuth, this version:
+ * - Automatically handles authorization redirects by retrying with fresh tokens
+ * - Does not throw UnauthorizedError on redirect, but instead retries
+ * - Calls next() instead of throwing for redirect-based auth
+ *
+ * @param provider - OAuth client provider for authentication
+ * @param baseUrl - Base URL for OAuth server discovery (defaults to request URL domain)
+ * @returns A fetch middleware function
+ */
+export const withOAuthRetry = (
+  clientName: string,
+  baseUrl?: string | URL,
+  handle401Fn: typeof handle401 = handle401
+): Middleware => {
+  const provider = new ConformanceOAuthProvider(
+    'http://localhost:3000/callback',
+    {
+      client_name: clientName,
+      redirect_uris: ['http://localhost:3000/callback']
+    }
+  );
+  return (next: FetchLike) => {
+    return async (
+      input: string | URL,
+      init?: RequestInit
+    ): Promise<Response> => {
+      const makeRequest = async (): Promise<Response> => {
+        const headers = new Headers(init?.headers);
+
+        // Add authorization header if tokens are available
+        const tokens = await provider.tokens();
+        if (tokens) {
+          headers.set('Authorization', `Bearer ${tokens.access_token}`);
+        }
+
+        return await next(input, { ...init, headers });
+      };
+
+      let response = await makeRequest();
+
+      // Handle 401 responses by attempting re-authentication
+      if (response.status === 401) {
+        const serverUrl =
+          baseUrl ||
+          (typeof input === 'string' ? new URL(input).origin : input.origin);
+        await handle401Fn(response, provider, next, serverUrl);
+
+        response = await makeRequest();
+      }
+
+      // If we still have a 401 after re-auth attempt, throw an error
+      if (response.status === 401) {
+        const url = typeof input === 'string' ? input : input.toString();
+        throw new UnauthorizedError(`Authentication failed for ${url}`);
+      }
+
+      return response;
+    };
+  };
+};

src/scenarios/client/auth/basic-dcr.test.ts

@@ -1,5 +1,8 @@
 import { describe, test } from '@jest/globals';
-import { runClientAgainstScenario } from './helpers/testClient.js';
+import {
+  runClientAgainstScenario,
+  SpawnedClientRunner
+} from './test_helpers/testClient.js';
 import path from 'path';
 
 describe('PRM Path-Based Discovery', () => {
@@ -8,6 +11,19 @@ describe('PRM Path-Based Discovery', () => {
       process.cwd(),
       'examples/clients/typescript/auth-test.ts'
     );
-    await runClientAgainstScenario(clientPath, 'auth/basic-dcr');
+    const runner = new SpawnedClientRunner(clientPath);
+    await runClientAgainstScenario(runner, 'auth/basic-dcr');
+  });
+
+  test('bad client requests root PRM location', async () => {
+    const clientPath = path.join(
+      process.cwd(),
+      'examples/clients/typescript/auth-test-broken1.ts'
+    );
+    const runner = new SpawnedClientRunner(clientPath);
+    await runClientAgainstScenario(runner, 'auth/basic-dcr', [
+      // There will be other failures, but this is the one that matters
+      'prm-priority-order'
+    ]);
   });
 });

src/scenarios/client/auth/basic-dcr.ts

@@ -3,6 +3,7 @@ import { ScenarioUrls } from '../../../types.js';
 import { createAuthServer } from './helpers/createAuthServer.js';
 import { createServer } from './helpers/createServer.js';
 import { ServerLifecycle } from './helpers/serverLifecycle.js';
+import { Request, Response } from 'express';
 
 export class AuthBasicDCRScenario implements Scenario {
   name = 'auth-basic-dcr';
@@ -25,6 +26,38 @@ export class AuthBasicDCRScenario implements Scenario {
       () => this.baseUrl,
       () => this.authBaseUrl
     );
+
+    // For this scenario, reject PRM requests at root location since we have the path-based PRM.
+    app.get(
+      '/.well-known/oauth-protected-resource',
+      (req: Request, res: Response) => {
+        this.checks.push({
+          id: 'prm-priority-order',
+          name: 'PRM Priority Order',
+          description:
+            'Client requested PRM metadata at root location on a server with path-based PRM',
+          status: 'FAILURE',
+          timestamp: new Date().toISOString(),
+          specReferences: [
+            {
+              id: 'mcp-authorization-prm',
+              url: 'https://modelcontextprotocol.io/specification/draft/basic/authorization#protected-resource-metadata-discovery-requirements'
+            }
+          ],
+          details: {
+            url: req.url,
+            path: req.path
+          }
+        });
+
+        // Return 404 to indicate PRM is not available at root location
+        res.status(404).json({
+          error: 'not_found',
+          error_description: 'PRM metadata not available at root location'
+        });
+      }
+    );
+
     this.baseUrl = await this.server.start(app);
 
     return { serverUrl: `${this.baseUrl}/mcp` };

src/scenarios/client/auth/basic-metadata-var1.test.ts

@@ -1,5 +1,8 @@
 import { describe, test } from '@jest/globals';
-import { runClientAgainstScenario } from './helpers/testClient.js';
+import {
+  runClientAgainstScenario,
+  SpawnedClientRunner
+} from './test_helpers/testClient.js';
 import path from 'path';
 
 describe('OAuth Metadata at OpenID Configuration Path', () => {
@@ -8,6 +11,7 @@ describe('OAuth Metadata at OpenID Configuration Path', () => {
       process.cwd(),
       'examples/clients/typescript/auth-test.ts'
     );
-    await runClientAgainstScenario(clientPath, 'auth/basic-metadata-var1');
+    const runner = new SpawnedClientRunner(clientPath);
+    await runClientAgainstScenario(runner, 'auth/basic-metadata-var1');
   });
 });

src/scenarios/client/auth/test_helpers/testClient.ts

@@ -0,0 +1,167 @@
+import { getScenario } from '../../../index.js';
+import { spawn } from 'child_process';
+
+const CLIENT_TIMEOUT = 10000; // 10 seconds for client to complete
+
+/**
+ * Represents a client that can be executed against a scenario.
+ * Implementations can run client code inline or by spawning a process.
+ */
+export interface ClientRunner {
+  /**
+   * Run the client against the given server URL.
+   * Should reject if the client fails.
+   */
+  run(serverUrl: string): Promise<void>;
+}
+
+/**
+ * Client runner that spawns a shell process to execute a client file.
+ */
+export class SpawnedClientRunner implements ClientRunner {
+  constructor(private clientPath: string) {}
+
+  async run(serverUrl: string): Promise<void> {
+    await new Promise<void>((resolve, reject) => {
+      const clientProcess = spawn('npx', ['tsx', this.clientPath, serverUrl], {
+        stdio: ['ignore', 'pipe', 'pipe']
+      });
+
+      let stdout = '';
+      let stderr = '';
+
+      clientProcess.stdout?.on('data', (data) => {
+        stdout += data.toString();
+      });
+
+      clientProcess.stderr?.on('data', (data) => {
+        stderr += data.toString();
+      });
+
+      const timeout = setTimeout(() => {
+        clientProcess.kill('SIGTERM');
+        reject(
+          new Error(
+            `Client failed to complete within ${CLIENT_TIMEOUT}ms\nStdout: ${stdout}\nStderr: ${stderr}`
+          )
+        );
+      }, CLIENT_TIMEOUT);
+
+      clientProcess.on('exit', (code) => {
+        clearTimeout(timeout);
+        if (code === 0) {
+          resolve();
+        } else {
+          reject(
+            new Error(
+              `Client exited with code ${code}\nStdout: ${stdout}\nStderr: ${stderr}`
+            )
+          );
+        }
+      });
+
+      clientProcess.on('error', (error) => {
+        clearTimeout(timeout);
+        reject(
+          new Error(
+            `Failed to start client: ${error.message}\nStdout: ${stdout}\nStderr: ${stderr}`
+          )
+        );
+      });
+    });
+  }
+}
+
+/**
+ * Client runner that executes a client function inline without spawning a shell.
+ */
+export class InlineClientRunner implements ClientRunner {
+  constructor(private clientFn: (serverUrl: string) => Promise<void>) {}
+
+  async run(serverUrl: string): Promise<void> {
+    await this.clientFn(serverUrl);
+  }
+}
+
+export async function runClientAgainstScenario(
+  clientRunner: ClientRunner | string,
+  scenarioName: string,
+  expectedFailureSlugs: string[] = []
+): Promise<void> {
+  // Handle backward compatibility: if string is passed, treat as file path
+  const runner =
+    typeof clientRunner === 'string'
+      ? new SpawnedClientRunner(clientRunner)
+      : clientRunner;
+
+  const scenario = getScenario(scenarioName);
+  if (!scenario) {
+    throw new Error(`Scenario ${scenarioName} not found`);
+  }
+
+  // Start the scenario server
+  const urls = await scenario.start();
+  const serverUrl = urls.serverUrl;
+
+  try {
+    // Run the client
+    try {
+      await runner.run(serverUrl);
+    } catch (err) {
+      if (expectedFailureSlugs.length === 0) {
+        throw err; // Unexpected failure
+      }
+      // Otherwise, expected failure - continue to checks verification
+    }
+
+    // Get checks from the scenario
+    const checks = scenario.getChecks();
+
+    // Verify checks were returned
+    if (checks.length === 0) {
+      throw new Error('No checks returned from scenario');
+    }
+
+    // Filter out INFO checks
+    const nonInfoChecks = checks.filter((c) => c.status !== 'INFO');
+
+    // Check for expected failures
+    if (expectedFailureSlugs.length > 0) {
+      // Verify that the expected failures are present
+      for (const slug of expectedFailureSlugs) {
+        const check = checks.find((c) => c.id === slug);
+        if (!check) {
+          throw new Error(`Expected failure check ${slug} not found`);
+        }
+      }
+
+      // Verify that only the expected checks failed
+      const failures = nonInfoChecks.filter((c) => c.status === 'FAILURE');
+      const failureSlugs = failures.map((c) => c.id);
+      // Check that failureSlugs contains all expectedFailureSlugs
+      expect(failureSlugs).toEqual(
+        expect.arrayContaining(expectedFailureSlugs)
+      );
+    } else {
+      // Default: expect all checks to pass
+      const failures = nonInfoChecks.filter((c) => c.status === 'FAILURE');
+      if (failures.length > 0) {
+        const failureMessages = failures
+          .map((c) => `${c.name}: ${c.errorMessage || c.description}`)
+          .join('\n  ');
+        throw new Error(`Scenario failed with checks:\n  ${failureMessages}`);
+      }
+
+      // All non-INFO checks should be SUCCESS
+      const successes = nonInfoChecks.filter((c) => c.status === 'SUCCESS');
+      if (successes.length !== nonInfoChecks.length) {
+        throw new Error(
+          `Expected all checks to pass but got ${successes.length}/${nonInfoChecks.length}`
+        );
+      }
+    }
+  } finally {
+    // Stop the scenario server
+    await scenario.stop();
+  }
+}