Merge pull request #293 from koic/conformance_client

Support running MCP client conformance tests against the Ruby SDK

Rakefile

@@ -20,7 +20,8 @@ desc "Run MCP conformance tests (PORT, SCENARIO, SPEC_VERSION, VERBOSE)"
 task :conformance do |t|
   next unless npx_available?(t.name)
 
-  require_relative "conformance/runner"
+  require_relative "conformance/server_runner"
+  require_relative "conformance/client_runner"
 
   options = {}
   options[:port] = Integer(ENV["PORT"]) if ENV["PORT"]
@@ -28,7 +29,8 @@ task :conformance do |t|
   options[:spec_version] = ENV["SPEC_VERSION"] if ENV["SPEC_VERSION"]
   options[:verbose] = true if ENV["VERBOSE"]
 
-  Conformance::Runner.new(**options).run
+  Conformance::ServerRunner.new(**options).run
+  Conformance::ClientRunner.new(**options.except(:port)).run
 end
 
 desc "List available conformance scenarios"
@@ -36,6 +38,7 @@ task :conformance_list do |t|
   next unless npx_available?(t.name)
 
   system("npx", "--yes", "@modelcontextprotocol/conformance", "list", "--server")
+  system("npx", "--yes", "@modelcontextprotocol/conformance", "list", "--client")
 end
 
 desc "Start the conformance server (PORT)"

conformance/README.md

@@ -15,22 +15,22 @@ Validates the Ruby SDK's conformance to the MCP specification using [`@modelcont
 bundle exec rake conformance
 ```
 
-Starts the conformance server, runs all active scenarios against it, prints a pass/fail
+Runs both server and client conformance tests in sequence. Server conformance starts the
-summary for each scenario, and exits with a non-zero status code if any unexpected failures
+conformance server and tests it. Client conformance spawns test servers for each scenario
-are detected. Scenarios listed in `expected_failures.yml` are allowed to fail without
+and invokes `conformance/client.rb` against them. Scenarios listed in `expected_failures.yml`
-affecting the exit code.
+are allowed to fail without affecting the exit code.
 
 ### Environment variables
 
-| Variable       | Description                          | Default |
+| Variable       | Description                           | Default |
-|----------------|--------------------------------------|---------|
+|----------------|---------------------------------------|---------|
-| `PORT`         | Server port                          | `9292`  |
+| `PORT`         | Server port (server conformance only) | `9292`  |
-| `SCENARIO`     | Run a single scenario by name        | (all)   |
+| `SCENARIO`     | Run a single scenario by name         | (all)   |
-| `SPEC_VERSION` | Filter scenarios by spec version     | (all)   |
+| `SPEC_VERSION` | Filter scenarios by spec version      | (all)   |
-| `VERBOSE`      | Show raw JSON output when set        | (off)   |
+| `VERBOSE`      | Show raw JSON output when set         | (off)   |
 
 ```bash
-# Run a single scenario
+# Run a single server scenario
 bundle exec rake conformance SCENARIO=ping
 
 # Use a different port with verbose output
@@ -91,8 +91,10 @@ submissions.
 ```
 conformance/
   server.rb              # Conformance server (Rack + Puma, default port 9292)
-  runner.rb              # Starts the server, runs npx conformance, exits with result code
+  server_runner.rb       # Starts the server, runs npx conformance server, exits with result code
-  expected_failures.yml  # Baseline of known-failing scenarios
+  client.rb              # Conformance client (invoked by npx conformance client)
+  client_runner.rb       # Runs npx conformance client, exits with result code
+  expected_failures.yml  # Baseline of known-failing scenarios (server and client)
   README.md              # This file
 ```
 

conformance/client.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+# Conformance test client for the MCP Ruby SDK.
+# Invoked by the conformance runner:
+#   MCP_CONFORMANCE_SCENARIO=<scenario> bundle exec ruby conformance/client.rb <server-url>
+#
+# The server URL is passed as the last positional argument.
+# The scenario name is read from the MCP_CONFORMANCE_SCENARIO environment variable,
+# which is set automatically by the conformance test runner.
+
+require_relative "../lib/mcp"
+
+scenario = ENV["MCP_CONFORMANCE_SCENARIO"]
+server_url = ARGV.last
+
+unless scenario && server_url
+  abort("Usage: MCP_CONFORMANCE_SCENARIO=<scenario> ruby conformance/client.rb <server-url>")
+end
+
+transport = MCP::Client::HTTP.new(url: server_url)
+client = MCP::Client.new(transport: transport)
+client.connect(client_info: { name: "ruby-sdk-conformance-client", version: MCP::VERSION })
+
+case scenario
+when "initialize"
+  client.tools
+when "tools_call"
+  tools = client.tools
+  add_numbers = tools.find { |t| t.name == "add_numbers" }
+  abort("Tool add_numbers not found") unless add_numbers
+  client.call_tool(tool: add_numbers, arguments: { a: 1, b: 2 })
+else
+  abort("Unknown or unsupported scenario: #{scenario}")
+end

conformance/client_runner.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+# Runs `npx @modelcontextprotocol/conformance client` against the conformance client script.
+require "English"
+
+module Conformance
+  class ClientRunner
+    def initialize(scenario: nil, spec_version: nil, verbose: false)
+      @scenario = scenario
+      @spec_version = spec_version
+      @verbose = verbose
+    end
+
+    def run
+      command = build_command
+      puts "Command: #{command.join(" ")}\n\n"
+
+      system(*command)
+      conformance_exit_code = $CHILD_STATUS.exitstatus
+      exit(conformance_exit_code || 1) unless conformance_exit_code == 0
+    end
+
+    private
+
+    def build_command
+      expected_failures_yml = File.expand_path("expected_failures.yml", __dir__)
+      client_script = File.expand_path("client.rb", __dir__)
+
+      npx_command = [
+        "npx",
+        "--yes",
+        "@modelcontextprotocol/conformance",
+        "client",
+        "--command",
+        "bundle exec ruby #{client_script}",
+      ]
+      npx_command += if @scenario
+        ["--scenario", @scenario]
+      else
+        ["--suite", "all"]
+      end
+      npx_command += ["--spec-version", @spec_version] if @spec_version
+      npx_command += ["--verbose"] if @verbose
+      npx_command += ["--expected-failures", expected_failures_yml]
+      npx_command
+    end
+  end
+end

conformance/expected_failures.yml

@@ -1 +1,28 @@
 server: []
+client:
+  # TODO: SSE reconnection not implemented in Ruby client.
+  - sse-retry
+  # TODO: Elicitation not implemented in Ruby client.
+  - elicitation-sep1034-client-defaults
+  # TODO: OAuth/auth not implemented in Ruby client.
+  - auth/metadata-default
+  - auth/metadata-var1
+  - auth/metadata-var2
+  - auth/metadata-var3
+  - auth/basic-cimd
+  - auth/scope-from-www-authenticate
+  - auth/scope-from-scopes-supported
+  - auth/scope-omitted-when-undefined
+  - auth/scope-step-up
+  - auth/scope-retry-limit
+  - auth/token-endpoint-auth-basic
+  - auth/token-endpoint-auth-post
+  - auth/token-endpoint-auth-none
+  - auth/pre-registration
+  - auth/2025-03-26-oauth-metadata-backcompat
+  - auth/2025-03-26-oauth-endpoint-fallback
+  - auth/client-credentials-jwt
+  - auth/client-credentials-basic
+  - auth/cross-app-access-complete-flow
+  - auth/offline-access-scope
+  - auth/offline-access-not-supported

conformance/server_runner.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+# Starts the conformance server and runs `npx @modelcontextprotocol/conformance` against it.
+require "English"
+require "net/http"
+require_relative "server"
+
+module Conformance
+  class ServerRunner
+    # Timeout for waiting for the Puma server to start.
+    SERVER_START_TIMEOUT = 20
+    SERVER_POLL_INTERVAL = 0.5
+    SERVER_HEALTH_CHECK_RETRIES = (SERVER_START_TIMEOUT / SERVER_POLL_INTERVAL).to_i
+
+    def initialize(port: Server::DEFAULT_PORT, scenario: nil, spec_version: nil, verbose: false)
+      @port = port
+      @scenario = scenario
+      @spec_version = spec_version
+      @verbose = verbose
+    end
+
+    def run
+      command = build_command
+      server_pid = start_server
+
+      run_conformance(command, server_pid: server_pid)
+    end
+
+    private
+
+    def build_command
+      expected_failures_yml = File.expand_path("expected_failures.yml", __dir__)
+
+      npx_command = ["npx", "--yes", "@modelcontextprotocol/conformance", "server", "--url", "http://localhost:#{@port}/mcp"]
+      npx_command += ["--scenario", @scenario] if @scenario
+      npx_command += ["--spec-version", @spec_version] if @spec_version
+      npx_command += ["--verbose"] if @verbose
+      npx_command += ["--expected-failures", expected_failures_yml]
+      npx_command
+    end
+
+    def start_server
+      puts "Starting conformance server on port #{@port}..."
+
+      server_pid = fork do
+        Conformance::Server.new(port: @port).start
+      end
+
+      health_url = URI("http://localhost:#{@port}/health")
+      ready = false
+      SERVER_HEALTH_CHECK_RETRIES.times do
+        begin
+          response = Net::HTTP.get_response(health_url)
+          if response.code == "200"
+            ready = true
+            break
+          end
+        rescue Errno::ECONNREFUSED, Errno::ECONNRESET, Net::ReadTimeout
+          # not ready yet
+        end
+        sleep(SERVER_POLL_INTERVAL)
+      end
+
+      unless ready
+        warn("ERROR: Conformance server did not start within #{SERVER_START_TIMEOUT} seconds")
+        terminate_server(server_pid)
+        exit(1)
+      end
+
+      puts "Server ready. Running conformance tests..."
+
+      server_pid
+    end
+
+    def run_conformance(command, server_pid:)
+      puts "Command: #{command.join(" ")}\n\n"
+
+      conformance_exit_code = nil
+      begin
+        system(*command)
+        conformance_exit_code = $CHILD_STATUS.exitstatus
+      ensure
+        terminate_server(server_pid)
+      end
+
+      exit(conformance_exit_code || 1) unless conformance_exit_code == 0
+    end
+
+    def terminate_server(pid)
+      Process.kill("TERM", pid)
+    rescue Errno::ESRCH
+      # process already exited
+    ensure
+      begin
+        Process.wait(pid)
+      rescue Errno::ECHILD
+        # already reaped
+      end
+    end
+  end
+end