Add sep-835 limit retry scenario, add suite options to server command and update pending scenarios (#49)

* lefthook: move deps installs to optional local change * add test scenario for infinite retry and negative test * Add broken client for retry limit negative test and fix lefthook formatting 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com> * fix www-authenticate in scope scenario * add suites and shift pending to reflect current pending * lint --------- Co-authored-by: Claude <noreply@anthropic.com>
2025-11-21 16:49:38 +00:00 · 2025-11-21 16:49:38 +00:00 · b64e054918
commit b64e054918
parent f2aaedaaa4
6 changed files with 402 additions and 73 deletions
--- a/src/index.ts
+++ b/src/index.ts
@ -14,6 +14,7 @@ import {
  listScenarios,
  listClientScenarios,
  listActiveClientScenarios,
+  listPendingClientScenarios,
  listAuthScenarios,
  listMetadataScenarios
 } from './scenarios';
@ -53,7 +54,9 @@ program

        const suites: Record<string, () => string[]> = {
          auth: listAuthScenarios,
-          metadata: listMetadataScenarios
+          metadata: listMetadataScenarios,
+          'sep-835': () =>
+            listAuthScenarios().filter((name) => name.startsWith('auth/scope-'))
        };

        const suiteName = options.suite.toLowerCase();
@ -123,8 +126,9 @@ program
          totalWarnings += warnings;

          const status = failed === 0 ? '✓' : '✗';
+          const warningStr = warnings > 0 ? `, ${warnings} warnings` : '';
          console.log(
-            `${status} ${result.scenario}: ${passed} passed, ${failed} failed`
+            `${status} ${result.scenario}: ${passed} passed, ${failed} failed${warningStr}`
          );

          if (verbose && failed > 0) {
@ -149,7 +153,7 @@ program
        console.error('Either --scenario or --suite is required');
        console.error('\nAvailable client scenarios:');
        listScenarios().forEach((s) => console.error(`  - ${s}`));
-        console.error('\nAvailable suites: auth, metadata');
+        console.error('\nAvailable suites: auth, metadata, sep-835');
        process.exit(1);
      }

@ -193,7 +197,12 @@ program
  .requiredOption('--url <url>', 'URL of the server to test')
  .option(
    '--scenario <scenario>',
-    'Scenario to test (defaults to all scenarios if not specified)'
+    'Scenario to test (defaults to active suite if not specified)'
+  )
+  .option(
+    '--suite <suite>',
+    'Suite to run: "active" (default, excludes pending), "all", or "pending"',
+    'active'
  )
  .action(async (options) => {
    try {
@ -213,10 +222,24 @@ program
        );
        process.exit(failed > 0 ? 1 : 0);
      } else {
-        // Run all active scenarios
-        const scenarios = listActiveClientScenarios();
+        // Run scenarios based on suite
+        const suite = options.suite?.toLowerCase() || 'active';
+        let scenarios: string[];
+
+        if (suite === 'all') {
+          scenarios = listClientScenarios();
+        } else if (suite === 'active') {
+          scenarios = listActiveClientScenarios();
+        } else if (suite === 'pending') {
+          scenarios = listPendingClientScenarios();
+        } else {
+          console.error(`Unknown suite: ${suite}`);
+          console.error('Available suites: active, all, pending');
+          process.exit(1);
+        }
+
        console.log(
-          `Running ${scenarios.length} scenarios against ${validated.url}\n`
+          `Running ${suite} suite (${scenarios.length} scenarios) against ${validated.url}\n`
        );

        const allResults: { scenario: string; checks: ConformanceCheck[] }[] =
--- a/src/scenarios/client/auth/index.test.ts
+++ b/src/scenarios/client/auth/index.test.ts
@ -9,24 +9,18 @@ import { runClient as noCimdClient } from '../../../../examples/clients/typescri
 import { runClient as ignoreScopeClient } from '../../../../examples/clients/typescript/auth-test-ignore-scope';
 import { runClient as partialScopesClient } from '../../../../examples/clients/typescript/auth-test-partial-scopes';
 import { runClient as ignore403Client } from '../../../../examples/clients/typescript/auth-test-ignore-403';
+import { runClient as noRetryLimitClient } from '../../../../examples/clients/typescript/auth-test-no-retry-limit';
 import { setLogLevel } from '../../../../examples/clients/typescript/helpers/logger';

 beforeAll(() => {
  setLogLevel('error');
 });

-const skipScenarios = new Set<string>([
-  // Waiting on typescript-sdk support in bearerAuth middleware to include
-  // scope in WWW-Authenticate header
-  // https://github.com/modelcontextprotocol/typescript-sdk/pull/1133
-  'auth/scope-from-www-authenticate',
-  // Waiting on typescript-sdk support for using scopes_supported from PRM
-  // to request scopes.
-  // https://github.com/modelcontextprotocol/typescript-sdk/pull/1133
-  'auth/scope-from-scopes-supported',
-  // Waiting on typescript-sdk support for CIMD
-  // https://github.com/modelcontextprotocol/typescript-sdk/pull/1127
-  'auth/basic-cimd'
+const skipScenarios = new Set<string>([]);
+
+const allowClientErrorScenarios = new Set<string>([
+  // Client is expected to give up (error) after limited retries, but check should pass
+  'auth/scope-retry-limit'
 ]);

 describe('Client Auth Scenarios', () => {
@ -38,7 +32,9 @@ describe('Client Auth Scenarios', () => {
        return;
      }
      const runner = new InlineClientRunner(goodClient);
-      await runClientAgainstScenario(runner, scenario.name);
+      await runClientAgainstScenario(runner, scenario.name, {
+        allowClientError: allowClientErrorScenarios.has(scenario.name)
+      });
    });
  }
 });
@ -46,23 +42,23 @@ describe('Client Auth Scenarios', () => {
 describe('Negative tests', () => {
  test('bad client requests root PRM location', async () => {
    const runner = new InlineClientRunner(badPrmClient);
-    await runClientAgainstScenario(runner, 'auth/metadata-default', [
-      'prm-priority-order'
-    ]);
+    await runClientAgainstScenario(runner, 'auth/metadata-default', {
+      expectedFailureSlugs: ['prm-priority-order']
+    });
  });

  test('client ignores scope from WWW-Authenticate header', async () => {
    const runner = new InlineClientRunner(ignoreScopeClient);
-    await runClientAgainstScenario(runner, 'auth/scope-from-www-authenticate', [
-      'scope-from-www-authenticate'
-    ]);
+    await runClientAgainstScenario(runner, 'auth/scope-from-www-authenticate', {
+      expectedFailureSlugs: ['scope-from-www-authenticate']
+    });
  });

  test('client only requests subset of scopes_supported', async () => {
    const runner = new InlineClientRunner(partialScopesClient);
-    await runClientAgainstScenario(runner, 'auth/scope-from-scopes-supported', [
-      'scope-from-scopes-supported'
-    ]);
+    await runClientAgainstScenario(runner, 'auth/scope-from-scopes-supported', {
+      expectedFailureSlugs: ['scope-from-scopes-supported']
+    });
  });

  test('client requests scope even if scopes_supported is empty', async () => {
@ -70,21 +66,31 @@ describe('Negative tests', () => {
    await runClientAgainstScenario(
      runner,
      'auth/scope-omitted-when-undefined',
-      ['scope-omitted-when-undefined']
+      {
+        expectedFailureSlugs: ['scope-omitted-when-undefined']
+      }
    );
  });

  test('client only responds to 401, not 403', async () => {
    const runner = new InlineClientRunner(ignore403Client);
-    await runClientAgainstScenario(runner, 'auth/scope-step-up', [
-      'scope-step-up-escalation'
-    ]);
+    await runClientAgainstScenario(runner, 'auth/scope-step-up', {
+      expectedFailureSlugs: ['scope-step-up-escalation']
+    });
  });

  test('client uses DCR instead of CIMD when server supports it', async () => {
    const runner = new InlineClientRunner(noCimdClient);
-    await runClientAgainstScenario(runner, 'auth/basic-cimd', [
-      'cimd-client-id-used'
-    ]);
+    await runClientAgainstScenario(runner, 'auth/basic-cimd', {
+      expectedFailureSlugs: ['cimd-client-id-used']
+    });
+  });
+
+  test('client retries auth infinitely without limit', async () => {
+    const runner = new InlineClientRunner(noRetryLimitClient);
+    await runClientAgainstScenario(runner, 'auth/scope-retry-limit', {
+      expectedFailureSlugs: ['scope-retry-limit'],
+      allowClientError: true
+    });
  });
 });
--- a/src/scenarios/client/auth/index.ts
+++ b/src/scenarios/client/auth/index.ts
@ -9,7 +9,8 @@ import {
  ScopeFromWwwAuthenticateScenario,
  ScopeFromScopesSupportedScenario,
  ScopeOmittedWhenUndefinedScenario,
-  ScopeStepUpAuthScenario
+  ScopeStepUpAuthScenario,
+  ScopeRetryLimitScenario
 } from './scope-handling';

 export const authScenariosList: Scenario[] = [
@ -20,5 +21,6 @@ export const authScenariosList: Scenario[] = [
  new ScopeFromWwwAuthenticateScenario(),
  new ScopeFromScopesSupportedScenario(),
  new ScopeOmittedWhenUndefinedScenario(),
-  new ScopeStepUpAuthScenario()
+  new ScopeStepUpAuthScenario(),
+  new ScopeRetryLimitScenario()
 ];
--- a/src/scenarios/client/auth/scope-handling.ts
+++ b/src/scenarios/client/auth/scope-handling.ts
@ -351,7 +351,7 @@ export class ScopeStepUpAuthScenario implements Scenario {
          .status(403)
          .set(
            'WWW-Authenticate',
-            `Bearer scope="${requiredScopes.join(' ')}", resource_metadata="${resourceMetadataUrl()}"`
+            `Bearer scope="${requiredScopes.join(' ')}", resource_metadata="${resourceMetadataUrl()}", error="insufficient_scope"`
          )
          .json({
            error: 'insufficient_scope',
@ -421,3 +421,173 @@ export class ScopeStepUpAuthScenario implements Scenario {
    return this.checks;
  }
 }
+
+/**
+ * Scenario 5: Client implements retry limits for scope escalation
+ *
+ * Tests that clients SHOULD implement retry limits to avoid infinite
+ * authorization loops when receiving repeated 403 insufficient_scope errors.
+ * The server always returns 403 with the same scope requirement, and clients
+ * should stop retrying after a reasonable number of attempts (3 or fewer).
+ */
+export class ScopeRetryLimitScenario implements Scenario {
+  name = 'auth/scope-retry-limit';
+  description =
+    'Tests that client implements retry limits to prevent infinite authorization loops on repeated 403 responses';
+  private authServer = new ServerLifecycle();
+  private server = new ServerLifecycle();
+  private checks: ConformanceCheck[] = [];
+
+  async start(): Promise<ScenarioUrls> {
+    this.checks = [];
+
+    const requiredScope = 'mcp:admin';
+    const tokenVerifier = new MockTokenVerifier(this.checks, []);
+    let authRequestCount = 0;
+
+    const authApp = createAuthServer(this.checks, this.authServer.getUrl, {
+      tokenVerifier,
+      onAuthorizationRequest: (data) => {
+        authRequestCount++;
+        this.checks.push({
+          id: 'scope-retry-auth-attempt',
+          name: `Authorization attempt ${authRequestCount}`,
+          description: `Client made authorization request attempt ${authRequestCount}`,
+          status: 'INFO',
+          timestamp: data.timestamp,
+          specReferences: [SpecReferences.MCP_SCOPE_CHALLENGE_HANDLING],
+          details: {
+            attemptNumber: authRequestCount,
+            requestedScope: data.scope || 'none'
+          }
+        });
+      }
+    });
+    await this.authServer.start(authApp);
+
+    const resourceMetadataUrl = () =>
+      `${this.server.getUrl()}/.well-known/oauth-protected-resource/mcp`;
+
+    const maxAttempts = 3;
+    let mcpRequestWithTokenCount = 0;
+
+    const alwaysDeny403Middleware = async (
+      req: Request,
+      res: Response,
+      next: NextFunction
+    ) => {
+      let body = req.body;
+      if (typeof body === 'string') {
+        body = JSON.parse(body);
+      }
+      const method = body?.method;
+
+      if (method === 'initialize' || method?.startsWith('notifications/')) {
+        return next();
+      }
+
+      const authHeader = req.headers.authorization;
+      if (!authHeader || !authHeader.startsWith('Bearer ')) {
+        return res
+          .status(401)
+          .set(
+            'WWW-Authenticate',
+            `Bearer scope="${requiredScope}", resource_metadata="${resourceMetadataUrl()}"`
+          )
+          .json({
+            error: 'invalid_token',
+            error_description: 'Missing Authorization header'
+          });
+      }
+
+      mcpRequestWithTokenCount++;
+
+      if (mcpRequestWithTokenCount > maxAttempts) {
+        return res.status(410).json({
+          error: 'test_complete',
+          error_description:
+            'Test is over - client exceeded maximum retry attempts'
+        });
+      }
+
+      return res
+        .status(403)
+        .set(
+          'WWW-Authenticate',
+          `Bearer error="insufficient_scope", scope="${requiredScope}", resource_metadata="${resourceMetadataUrl()}", error_description="Scope upgrade will never succeed"`
+        )
+        .json({
+          error: 'insufficient_scope',
+          error_description: 'Scope upgrade will never succeed'
+        });
+    };
+
+    const baseApp = createServer(
+      this.checks,
+      this.server.getUrl,
+      this.authServer.getUrl,
+      {
+        prmPath: '/.well-known/oauth-protected-resource/mcp',
+        requiredScopes: [requiredScope],
+        scopesSupported: [requiredScope],
+        includeScopeInWwwAuth: true,
+        authMiddleware: alwaysDeny403Middleware,
+        tokenVerifier
+      }
+    );
+
+    await this.server.start(baseApp);
+
+    return { serverUrl: `${this.server.getUrl()}/mcp` };
+  }
+
+  async stop() {
+    await this.authServer.stop();
+    await this.server.stop();
+  }
+
+  getChecks(): ConformanceCheck[] {
+    const authAttempts = this.checks.filter(
+      (c) => c.id === 'scope-retry-auth-attempt'
+    ).length;
+
+    if (authAttempts === 0) {
+      this.checks.push({
+        id: 'scope-retry-limit',
+        name: 'Client retry limit for scope escalation',
+        description: 'Client did not make any authorization requests',
+        status: 'FAILURE',
+        timestamp: new Date().toISOString(),
+        specReferences: [SpecReferences.MCP_SCOPE_CHALLENGE_HANDLING]
+      });
+    } else if (authAttempts <= 3) {
+      this.checks.push({
+        id: 'scope-retry-limit',
+        name: 'Client retry limit for scope escalation',
+        description: `Client correctly limited retry attempts to ${authAttempts} (3 or fewer)`,
+        status: 'SUCCESS',
+        timestamp: new Date().toISOString(),
+        specReferences: [SpecReferences.MCP_SCOPE_CHALLENGE_HANDLING],
+        details: {
+          authorizationAttempts: authAttempts,
+          maxAllowed: 3
+        }
+      });
+    } else {
+      this.checks.push({
+        id: 'scope-retry-limit',
+        name: 'Client retry limit for scope escalation',
+        description: `Client made ${authAttempts} authorization attempts (more than 3). Clients SHOULD implement retry limits to avoid infinite loops.`,
+        status: 'FAILURE',
+        timestamp: new Date().toISOString(),
+        specReferences: [SpecReferences.MCP_SCOPE_CHALLENGE_HANDLING],
+        details: {
+          authorizationAttempts: authAttempts,
+          maxAllowed: 3
+        }
+      });
+    }
+
+    return this.checks;
+  }
+}
--- a/src/scenarios/client/auth/test_helpers/testClient.ts
+++ b/src/scenarios/client/auth/test_helpers/testClient.ts
@ -83,16 +83,19 @@ export class InlineClientRunner implements ClientRunner {
  }
 }

+export interface RunClientOptions {
+  expectedFailureSlugs?: string[];
+  allowClientError?: boolean;
+}
+
 export async function runClientAgainstScenario(
-  clientRunner: ClientRunner | string,
+  clientRunner: ClientRunner,
  scenarioName: string,
-  expectedFailureSlugs: string[] = []
+  options: RunClientOptions = {}
 ): Promise<void> {
-  // Handle backward compatibility: if string is passed, treat as file path
-  const runner =
-    typeof clientRunner === 'string'
-      ? new SpawnedClientRunner(clientRunner)
-      : clientRunner;
+  const { expectedFailureSlugs = [], allowClientError = false } = options;
+
+  const runner = clientRunner;

  const scenario = getScenario(scenarioName);
  if (!scenario) {
@ -108,10 +111,10 @@ export async function runClientAgainstScenario(
    try {
      await runner.run(serverUrl);
    } catch (err) {
-      if (expectedFailureSlugs.length === 0) {
+      if (expectedFailureSlugs.length === 0 && !allowClientError) {
        throw err; // Unexpected failure
      }
-      // Otherwise, expected failure - continue to checks verification
+      // Otherwise, expected failure or allowed error - continue to checks verification
    }

    // Get checks from the scenario
--- a/src/scenarios/index.ts
+++ b/src/scenarios/index.ts
@ -58,7 +58,12 @@ const pendingClientScenariosList: ClientScenario[] = [
  // JSON Schema 2020-12 (SEP-1613)
  // This test is pending until the SDK includes PR #1135 which preserves
  // $schema, $defs, and additionalProperties fields in tool schemas.
-  new JsonSchema2020_12Scenario()
+  new JsonSchema2020_12Scenario(),
+
+  // On hold until elicitation schema types are fixed
+  // https://github.com/modelcontextprotocol/modelcontextprotocol/pull/1863
+  new ToolsCallElicitationScenario(),
+  new ElicitationDefaultsScenario()
 ];

 // All client scenarios
@ -90,7 +95,7 @@ const allClientScenariosList: ClientScenario[] = [
  new ElicitationDefaultsScenario(),

  // Elicitation scenarios (SEP-1330) - pending
-  ...pendingClientScenariosList,
+  new ElicitationEnumsScenario(),

  // Resources scenarios
  new ResourcesListScenario(),
@ -159,6 +164,10 @@ export function listActiveClientScenarios(): string[] {
  return activeClientScenariosList.map((scenario) => scenario.name);
 }

+export function listPendingClientScenarios(): string[] {
+  return pendingClientScenariosList.map((scenario) => scenario.name);
+}
+
 export function listAuthScenarios(): string[] {
  return authScenariosList.map((scenario) => scenario.name);
 }