[scenario] Add test for broken oauth client (#12)

* starting on broken auth client

* switch to middleware approach

* add broken example

* fix failure testing

* subset matching

src/scenarios/client/auth/basic-dcr.test.ts

@@ -1,5 +1,8 @@
 import { describe, test } from '@jest/globals';
-import { runClientAgainstScenario } from './helpers/testClient.js';
+import {
+  runClientAgainstScenario,
+  SpawnedClientRunner
+} from './test_helpers/testClient.js';
 import path from 'path';
 
 describe('PRM Path-Based Discovery', () => {
@@ -8,6 +11,19 @@ describe('PRM Path-Based Discovery', () => {
       process.cwd(),
       'examples/clients/typescript/auth-test.ts'
     );
-    await runClientAgainstScenario(clientPath, 'auth/basic-dcr');
+    const runner = new SpawnedClientRunner(clientPath);
+    await runClientAgainstScenario(runner, 'auth/basic-dcr');
+  });
+
+  test('bad client requests root PRM location', async () => {
+    const clientPath = path.join(
+      process.cwd(),
+      'examples/clients/typescript/auth-test-broken1.ts'
+    );
+    const runner = new SpawnedClientRunner(clientPath);
+    await runClientAgainstScenario(runner, 'auth/basic-dcr', [
+      // There will be other failures, but this is the one that matters
+      'prm-priority-order'
+    ]);
   });
 });

src/scenarios/client/auth/basic-dcr.ts

@@ -3,6 +3,7 @@ import { ScenarioUrls } from '../../../types.js';
 import { createAuthServer } from './helpers/createAuthServer.js';
 import { createServer } from './helpers/createServer.js';
 import { ServerLifecycle } from './helpers/serverLifecycle.js';
+import { Request, Response } from 'express';
 
 export class AuthBasicDCRScenario implements Scenario {
   name = 'auth-basic-dcr';
@@ -25,6 +26,38 @@ export class AuthBasicDCRScenario implements Scenario {
       () => this.baseUrl,
       () => this.authBaseUrl
     );
+
+    // For this scenario, reject PRM requests at root location since we have the path-based PRM.
+    app.get(
+      '/.well-known/oauth-protected-resource',
+      (req: Request, res: Response) => {
+        this.checks.push({
+          id: 'prm-priority-order',
+          name: 'PRM Priority Order',
+          description:
+            'Client requested PRM metadata at root location on a server with path-based PRM',
+          status: 'FAILURE',
+          timestamp: new Date().toISOString(),
+          specReferences: [
+            {
+              id: 'mcp-authorization-prm',
+              url: 'https://modelcontextprotocol.io/specification/draft/basic/authorization#protected-resource-metadata-discovery-requirements'
+            }
+          ],
+          details: {
+            url: req.url,
+            path: req.path
+          }
+        });
+
+        // Return 404 to indicate PRM is not available at root location
+        res.status(404).json({
+          error: 'not_found',
+          error_description: 'PRM metadata not available at root location'
+        });
+      }
+    );
+
     this.baseUrl = await this.server.start(app);
 
     return { serverUrl: `${this.baseUrl}/mcp` };

src/scenarios/client/auth/basic-metadata-var1.test.ts

@@ -1,5 +1,8 @@
 import { describe, test } from '@jest/globals';
-import { runClientAgainstScenario } from './helpers/testClient.js';
+import {
+  runClientAgainstScenario,
+  SpawnedClientRunner
+} from './test_helpers/testClient.js';
 import path from 'path';
 
 describe('OAuth Metadata at OpenID Configuration Path', () => {
@@ -8,6 +11,7 @@ describe('OAuth Metadata at OpenID Configuration Path', () => {
       process.cwd(),
       'examples/clients/typescript/auth-test.ts'
     );
-    await runClientAgainstScenario(clientPath, 'auth/basic-metadata-var1');
+    const runner = new SpawnedClientRunner(clientPath);
+    await runClientAgainstScenario(runner, 'auth/basic-metadata-var1');
   });
 });

src/scenarios/client/auth/test_helpers/testClient.ts

@@ -0,0 +1,167 @@
+import { getScenario } from '../../../index.js';
+import { spawn } from 'child_process';
+
+const CLIENT_TIMEOUT = 10000; // 10 seconds for client to complete
+
+/**
+ * Represents a client that can be executed against a scenario.
+ * Implementations can run client code inline or by spawning a process.
+ */
+export interface ClientRunner {
+  /**
+   * Run the client against the given server URL.
+   * Should reject if the client fails.
+   */
+  run(serverUrl: string): Promise<void>;
+}
+
+/**
+ * Client runner that spawns a shell process to execute a client file.
+ */
+export class SpawnedClientRunner implements ClientRunner {
+  constructor(private clientPath: string) {}
+
+  async run(serverUrl: string): Promise<void> {
+    await new Promise<void>((resolve, reject) => {
+      const clientProcess = spawn('npx', ['tsx', this.clientPath, serverUrl], {
+        stdio: ['ignore', 'pipe', 'pipe']
+      });
+
+      let stdout = '';
+      let stderr = '';
+
+      clientProcess.stdout?.on('data', (data) => {
+        stdout += data.toString();
+      });
+
+      clientProcess.stderr?.on('data', (data) => {
+        stderr += data.toString();
+      });
+
+      const timeout = setTimeout(() => {
+        clientProcess.kill('SIGTERM');
+        reject(
+          new Error(
+            `Client failed to complete within ${CLIENT_TIMEOUT}ms\nStdout: ${stdout}\nStderr: ${stderr}`
+          )
+        );
+      }, CLIENT_TIMEOUT);
+
+      clientProcess.on('exit', (code) => {
+        clearTimeout(timeout);
+        if (code === 0) {
+          resolve();
+        } else {
+          reject(
+            new Error(
+              `Client exited with code ${code}\nStdout: ${stdout}\nStderr: ${stderr}`
+            )
+          );
+        }
+      });
+
+      clientProcess.on('error', (error) => {
+        clearTimeout(timeout);
+        reject(
+          new Error(
+            `Failed to start client: ${error.message}\nStdout: ${stdout}\nStderr: ${stderr}`
+          )
+        );
+      });
+    });
+  }
+}
+
+/**
+ * Client runner that executes a client function inline without spawning a shell.
+ */
+export class InlineClientRunner implements ClientRunner {
+  constructor(private clientFn: (serverUrl: string) => Promise<void>) {}
+
+  async run(serverUrl: string): Promise<void> {
+    await this.clientFn(serverUrl);
+  }
+}
+
+export async function runClientAgainstScenario(
+  clientRunner: ClientRunner | string,
+  scenarioName: string,
+  expectedFailureSlugs: string[] = []
+): Promise<void> {
+  // Handle backward compatibility: if string is passed, treat as file path
+  const runner =
+    typeof clientRunner === 'string'
+      ? new SpawnedClientRunner(clientRunner)
+      : clientRunner;
+
+  const scenario = getScenario(scenarioName);
+  if (!scenario) {
+    throw new Error(`Scenario ${scenarioName} not found`);
+  }
+
+  // Start the scenario server
+  const urls = await scenario.start();
+  const serverUrl = urls.serverUrl;
+
+  try {
+    // Run the client
+    try {
+      await runner.run(serverUrl);
+    } catch (err) {
+      if (expectedFailureSlugs.length === 0) {
+        throw err; // Unexpected failure
+      }
+      // Otherwise, expected failure - continue to checks verification
+    }
+
+    // Get checks from the scenario
+    const checks = scenario.getChecks();
+
+    // Verify checks were returned
+    if (checks.length === 0) {
+      throw new Error('No checks returned from scenario');
+    }
+
+    // Filter out INFO checks
+    const nonInfoChecks = checks.filter((c) => c.status !== 'INFO');
+
+    // Check for expected failures
+    if (expectedFailureSlugs.length > 0) {
+      // Verify that the expected failures are present
+      for (const slug of expectedFailureSlugs) {
+        const check = checks.find((c) => c.id === slug);
+        if (!check) {
+          throw new Error(`Expected failure check ${slug} not found`);
+        }
+      }
+
+      // Verify that only the expected checks failed
+      const failures = nonInfoChecks.filter((c) => c.status === 'FAILURE');
+      const failureSlugs = failures.map((c) => c.id);
+      // Check that failureSlugs contains all expectedFailureSlugs
+      expect(failureSlugs).toEqual(
+        expect.arrayContaining(expectedFailureSlugs)
+      );
+    } else {
+      // Default: expect all checks to pass
+      const failures = nonInfoChecks.filter((c) => c.status === 'FAILURE');
+      if (failures.length > 0) {
+        const failureMessages = failures
+          .map((c) => `${c.name}: ${c.errorMessage || c.description}`)
+          .join('\n  ');
+        throw new Error(`Scenario failed with checks:\n  ${failureMessages}`);
+      }
+
+      // All non-INFO checks should be SUCCESS
+      const successes = nonInfoChecks.filter((c) => c.status === 'SUCCESS');
+      if (successes.length !== nonInfoChecks.length) {
+        throw new Error(
+          `Expected all checks to pass but got ${successes.length}/${nonInfoChecks.length}`
+        );
+      }
+    }
+  } finally {
+    // Stop the scenario server
+    await scenario.stop();
+  }
+}