feat: add conformance scenario for SEP-2164 resource-not-found error code (#200)

* feat: add conformance scenario for SEP-2164 resource-not-found error code

Adds a draft-scoped scenario verifying servers return -32602 (Invalid
Params) for non-existent resources instead of the legacy -32002 or an
empty contents array. Includes a sep-2164.yaml traceability file mapping
each MUST/MUST NOT/SHOULD to its check per the proposed SEP-2484 process.

* fix: align SEP-2164 check severities with spec text (#246)

Splits sep-2164-error-code into:
- sep-2164-no-empty-contents (MUST NOT -> FAILURE)
- sep-2164-error-code (SHOULD -> WARNING)

Updates sep-2164.yaml to quote the spec text verbatim and adds an
excluded row for the client-side -32002 backcompat SHOULD.

See #245 for the SHOULD/WARNING tier-scoring policy discussion.

---------

Co-authored-by: Paul Carleton <paulcarletonjr@gmail.com>

src/scenarios/index.ts

@@ -45,7 +45,8 @@ import {
   ResourcesReadBinaryScenario,
   ResourcesTemplateReadScenario,
   ResourcesSubscribeScenario,
-  ResourcesUnsubscribeScenario
+  ResourcesUnsubscribeScenario,
+  ResourcesNotFoundErrorScenario
 } from './server/resources';
 
 import {
@@ -123,6 +124,9 @@ const allClientScenariosList: ClientScenario[] = [
   new ResourcesSubscribeScenario(),
   new ResourcesUnsubscribeScenario(),
 
+  // Resources error handling (SEP-2164)
+  new ResourcesNotFoundErrorScenario(),
+
   // Prompts scenarios
   new PromptsListScenario(),
   new PromptsGetSimpleScenario(),

src/scenarios/server/resources.ts

@@ -6,7 +6,8 @@ import { ClientScenario, ConformanceCheck, SpecVersion } from '../../types';
 import { connectToServer } from './client-helper';
 import {
   TextResourceContents,
-  BlobResourceContents
+  BlobResourceContents,
+  McpError
 } from '@modelcontextprotocol/sdk/types.js';
 
 export class ResourcesListScenario implements ClientScenario {
@@ -435,6 +436,165 @@ Example request:
   }
 }
 
+export class ResourcesNotFoundErrorScenario implements ClientScenario {
+  name = 'sep-2164-resource-not-found';
+  specVersions: SpecVersion[] = ['draft'];
+  description = `Test error handling for non-existent resources (SEP-2164).
+
+**Server Implementation Requirements:**
+
+**Endpoint**: \`resources/read\`
+
+When a client requests a URI that does not correspond to any resource, the server:
+
+- **MUST NOT** return a result with an empty \`contents\` array
+- **SHOULD** return a JSON-RPC error with code \`-32602\` (Invalid Params)
+- **SHOULD** include the requested \`uri\` in the error \`data\` field
+
+Example error response:
+
+\`\`\`json
+{
+  "jsonrpc": "2.0",
+  "id": 2,
+  "error": {
+    "code": -32602,
+    "message": "Resource not found",
+    "data": {
+      "uri": "test://nonexistent-resource-for-conformance-testing"
+    }
+  }
+}
+\`\`\`
+
+This scenario does not require the server to register any specific resource — it tests behavior when reading a URI the server does not recognize.`;
+
+  async run(serverUrl: string): Promise<ConformanceCheck[]> {
+    const checks: ConformanceCheck[] = [];
+    const nonexistentUri =
+      'test://nonexistent-resource-for-conformance-testing';
+    const specReferences = [
+      {
+        id: 'SEP-2164',
+        url: 'https://modelcontextprotocol.io/specification/draft/server/resources#error-handling'
+      }
+    ];
+
+    let connection;
+    try {
+      connection = await connectToServer(serverUrl);
+    } catch (error) {
+      checks.push({
+        id: 'sep-2164-error-code',
+        name: 'ResourcesNotFoundErrorCode',
+        description:
+          'Server returns -32602 (Invalid Params) for non-existent resource',
+        status: 'FAILURE',
+        timestamp: new Date().toISOString(),
+        errorMessage: `Failed to connect: ${error instanceof Error ? error.message : String(error)}`,
+        specReferences
+      });
+      return checks;
+    }
+
+    let caughtError: unknown;
+    let result: { contents: unknown[] } | undefined;
+    try {
+      result = await connection.client.readResource({ uri: nonexistentUri });
+    } catch (error) {
+      caughtError = error;
+    }
+
+    // Check 1: MUST NOT return an empty contents array
+    const returnedEmptyContents =
+      result !== undefined && (result.contents?.length ?? 0) === 0;
+    checks.push({
+      id: 'sep-2164-no-empty-contents',
+      name: 'ResourcesNotFoundNoEmptyContents',
+      description:
+        'Server does not return an empty contents array for a non-existent resource',
+      status: returnedEmptyContents ? 'FAILURE' : 'SUCCESS',
+      timestamp: new Date().toISOString(),
+      errorMessage: returnedEmptyContents
+        ? 'Server returned a result with an empty contents array. Servers MUST NOT return an empty contents array for non-existent resources.'
+        : undefined,
+      specReferences,
+      details: {
+        requestedUri: nonexistentUri,
+        receivedResult: result !== undefined,
+        contentsLength: result?.contents?.length
+      }
+    });
+
+    // Check 2: SHOULD return JSON-RPC error with code -32602
+    const errorCode =
+      caughtError instanceof McpError ? caughtError.code : undefined;
+    let errorCodeMessage: string | undefined;
+    if (result !== undefined) {
+      errorCodeMessage = `Server returned a result instead of an error (contents length: ${result.contents?.length ?? 'undefined'}). Servers SHOULD return a JSON-RPC error for non-existent resources.`;
+    } else if (!(caughtError instanceof McpError)) {
+      errorCodeMessage = `Expected a JSON-RPC error, got: ${caughtError instanceof Error ? caughtError.message : String(caughtError)}`;
+    } else if (errorCode !== -32602) {
+      errorCodeMessage =
+        `Expected error code -32602 (Invalid Params), got ${errorCode}. ` +
+        (errorCode === -32002
+          ? 'Code -32002 was used in earlier spec versions but SEP-2164 standardizes on -32602.'
+          : '');
+    }
+
+    checks.push({
+      id: 'sep-2164-error-code',
+      name: 'ResourcesNotFoundErrorCode',
+      description:
+        'Server returns -32602 (Invalid Params) for non-existent resource (SHOULD)',
+      status: errorCodeMessage === undefined ? 'SUCCESS' : 'WARNING',
+      timestamp: new Date().toISOString(),
+      errorMessage: errorCodeMessage,
+      specReferences,
+      details: {
+        requestedUri: nonexistentUri,
+        receivedErrorCode: errorCode,
+        receivedResult: result !== undefined
+      }
+    });
+
+    // Check 3: SHOULD include uri in error data field
+    const errorData =
+      caughtError instanceof McpError
+        ? (caughtError.data as { uri?: string } | undefined)
+        : undefined;
+    const dataUriMatches = errorData?.uri === nonexistentUri;
+
+    checks.push({
+      id: 'sep-2164-data-uri',
+      name: 'ResourcesNotFoundDataUri',
+      description:
+        'Server includes the requested URI in the error data field (SHOULD)',
+      status:
+        caughtError instanceof McpError
+          ? dataUriMatches
+            ? 'SUCCESS'
+            : 'WARNING'
+          : 'FAILURE',
+      timestamp: new Date().toISOString(),
+      errorMessage:
+        caughtError instanceof McpError
+          ? dataUriMatches
+            ? undefined
+            : `Error data.uri is ${JSON.stringify(errorData?.uri)}, expected "${nonexistentUri}". This is a SHOULD requirement.`
+          : 'No JSON-RPC error received; cannot evaluate data field.',
+      specReferences,
+      details: {
+        requestedUri: nonexistentUri,
+        receivedDataUri: errorData?.uri
+      }
+    });
+
+    await connection.close();
+    return checks;
+  }
+}
+
 export class ResourcesUnsubscribeScenario implements ClientScenario {
   name = 'resources-unsubscribe';
   specVersions: SpecVersion[] = ['2025-06-18', '2025-11-25'];

src/scenarios/server/sep-2164.yaml

@@ -0,0 +1,9 @@
+sep: 2164
+spec_url: https://modelcontextprotocol.io/specification/draft/server/resources#error-handling
+requirements:
+  - text: 'Servers MUST NOT return an empty contents array for a non-existent resource'
+    check: sep-2164-no-empty-contents
+  - text: 'Servers SHOULD return standard JSON-RPC errors for common failure cases: Resource not found: -32602 (Invalid Params)'
+    check: sep-2164-error-code
+  - text: 'clients SHOULD also accept -32002 as a resource not found error'
+    excluded: 'Client-side error handling is implementation-defined; not protocol-observable'