modelcontextprotocol/ruby-sdk
Watch
1
0
Fork
You've already forked ruby-sdk
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #293 from koic/conformance_client

Browse source 
Support running MCP client conformance tests against the Ruby SDK
This commit is contained in:
Koichi ITO 2026-05-10 23:34:17 +09:00 • committed by GitHub
commit e47607defa
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 131 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

7
Rakefile
View file

@ -20,7 +20,8 @@ desc "Run MCP conformance tests (PORT, SCENARIO, SPEC_VERSION, VERBOSE)"
task :conformance do |t|
next unless npx_available?(t.name)
require_relative "conformance/runner"
require_relative "conformance/server_runner"
require_relative "conformance/client_runner"
options = {}
options[:port] = Integer(ENV["PORT"]) if ENV["PORT"]
@ -28,7 +29,8 @@ task :conformance do |t|
options[:spec_version] = ENV["SPEC_VERSION"] if ENV["SPEC_VERSION"]
options[:verbose] = true if ENV["VERBOSE"]
Conformance::Runner.new(**options).run
Conformance::ServerRunner.new(**options).run
Conformance::ClientRunner.new(**options.except(:port)).run
end
desc "List available conformance scenarios"
@ -36,6 +38,7 @@ task :conformance_list do |t|
next unless npx_available?(t.name)
system("npx", "--yes", "@modelcontextprotocol/conformance", "list", "--server")
system("npx", "--yes", "@modelcontextprotocol/conformance", "list", "--client")
end
desc "Start the conformance server (PORT)"

28
conformance/README.md
View file

@ -15,22 +15,22 @@ Validates the Ruby SDK's conformance to the MCP specification using [`@modelcont
bundle exec rake conformance
```
Starts the conformance server, runs all active scenarios against it, prints a pass/fail
summary for each scenario, and exits with a non-zero status code if any unexpected failures
are detected. Scenarios listed in `expected_failures.yml` are allowed to fail without
affecting the exit code.
Runs both server and client conformance tests in sequence. Server conformance starts the
conformance server and tests it. Client conformance spawns test servers for each scenario
and invokes `conformance/client.rb` against them. Scenarios listed in `expected_failures.yml`
are allowed to fail without affecting the exit code.
### Environment variables
| Variable | Description | Default |
|----------------|--------------------------------------|---------|
| `PORT` | Server port | `9292` |
| `SCENARIO` | Run a single scenario by name | (all) |
| `SPEC_VERSION` | Filter scenarios by spec version | (all) |
| `VERBOSE` | Show raw JSON output when set | (off) |
| Variable | Description | Default |
|----------------|---------------------------------------|---------|
| `PORT` | Server port (server conformance only) | `9292` |
| `SCENARIO` | Run a single scenario by name | (all) |
| `SPEC_VERSION` | Filter scenarios by spec version | (all) |
| `VERBOSE` | Show raw JSON output when set | (off) |
```bash
# Run a single scenario
# Run a single server scenario
bundle exec rake conformance SCENARIO=ping
# Use a different port with verbose output
@ -91,8 +91,10 @@ submissions.
```
conformance/
server.rb # Conformance server (Rack + Puma, default port 9292)
runner.rb # Starts the server, runs npx conformance, exits with result code
expected_failures.yml # Baseline of known-failing scenarios
server_runner.rb # Starts the server, runs npx conformance server, exits with result code
client.rb # Conformance client (invoked by npx conformance client)
client_runner.rb # Runs npx conformance client, exits with result code
expected_failures.yml # Baseline of known-failing scenarios (server and client)
README.md # This file
```

34
conformance/client.rb Normal file
View file

@ -0,0 +1,34 @@
# frozen_string_literal: true
# Conformance test client for the MCP Ruby SDK.
# Invoked by the conformance runner:
# MCP_CONFORMANCE_SCENARIO=<scenario> bundle exec ruby conformance/client.rb <server-url>
#
# The server URL is passed as the last positional argument.
# The scenario name is read from the MCP_CONFORMANCE_SCENARIO environment variable,
# which is set automatically by the conformance test runner.
require_relative "../lib/mcp"
scenario = ENV["MCP_CONFORMANCE_SCENARIO"]
server_url = ARGV.last
unless scenario && server_url
abort("Usage: MCP_CONFORMANCE_SCENARIO=<scenario> ruby conformance/client.rb <server-url>")
end
transport = MCP::Client::HTTP.new(url: server_url)
client = MCP::Client.new(transport: transport)
client.connect(client_info: { name: "ruby-sdk-conformance-client", version: MCP::VERSION })
case scenario
when "initialize"
client.tools
when "tools_call"
tools = client.tools
add_numbers = tools.find { |t| t.name == "add_numbers" }
abort("Tool add_numbers not found") unless add_numbers
client.call_tool(tool: add_numbers, arguments: { a: 1, b: 2 })
else
abort("Unknown or unsupported scenario: #{scenario}")
end

48
conformance/client_runner.rb Normal file
View file

@ -0,0 +1,48 @@
# frozen_string_literal: true
# Runs `npx @modelcontextprotocol/conformance client` against the conformance client script.
require "English"
module Conformance
class ClientRunner
def initialize(scenario: nil, spec_version: nil, verbose: false)
@scenario = scenario
@spec_version = spec_version
@verbose = verbose
end
def run
command = build_command
puts "Command: #{command.join(" ")}\n\n"
system(*command)
conformance_exit_code = $CHILD_STATUS.exitstatus
exit(conformance_exit_code || 1) unless conformance_exit_code == 0
end
private
def build_command
expected_failures_yml = File.expand_path("expected_failures.yml", __dir__)
client_script = File.expand_path("client.rb", __dir__)
npx_command = [
"npx",
"--yes",
"@modelcontextprotocol/conformance",
"client",
"--command",
"bundle exec ruby #{client_script}",
]
npx_command += if @scenario
["--scenario", @scenario]
else
["--suite", "all"]
end
npx_command += ["--spec-version", @spec_version] if @spec_version
npx_command += ["--verbose"] if @verbose
npx_command += ["--expected-failures", expected_failures_yml]
npx_command
end
end
end

27
conformance/expected_failures.yml
View file

@ -1 +1,28 @@
server: []
client:
# TODO: SSE reconnection not implemented in Ruby client.
- sse-retry
# TODO: Elicitation not implemented in Ruby client.
- elicitation-sep1034-client-defaults
# TODO: OAuth/auth not implemented in Ruby client.
- auth/metadata-default
- auth/metadata-var1
- auth/metadata-var2
- auth/metadata-var3
- auth/basic-cimd
- auth/scope-from-www-authenticate
- auth/scope-from-scopes-supported
- auth/scope-omitted-when-undefined
- auth/scope-step-up
- auth/scope-retry-limit
- auth/token-endpoint-auth-basic
- auth/token-endpoint-auth-post
- auth/token-endpoint-auth-none
- auth/pre-registration
- auth/2025-03-26-oauth-metadata-backcompat
- auth/2025-03-26-oauth-endpoint-fallback
- auth/client-credentials-jwt
- auth/client-credentials-basic
- auth/cross-app-access-complete-flow
- auth/offline-access-scope
- auth/offline-access-not-supported

101
conformance/server_runner.rb Normal file
View file

@ -0,0 +1,101 @@
# frozen_string_literal: true
# Starts the conformance server and runs `npx @modelcontextprotocol/conformance` against it.
require "English"
require "net/http"
require_relative "server"
module Conformance
class ServerRunner
# Timeout for waiting for the Puma server to start.
SERVER_START_TIMEOUT = 20
SERVER_POLL_INTERVAL = 0.5
SERVER_HEALTH_CHECK_RETRIES = (SERVER_START_TIMEOUT / SERVER_POLL_INTERVAL).to_i
def initialize(port: Server::DEFAULT_PORT, scenario: nil, spec_version: nil, verbose: false)
@port = port
@scenario = scenario
@spec_version = spec_version
@verbose = verbose
end
def run
command = build_command
server_pid = start_server
run_conformance(command, server_pid: server_pid)
end
private
def build_command
expected_failures_yml = File.expand_path("expected_failures.yml", __dir__)
npx_command = ["npx", "--yes", "@modelcontextprotocol/conformance", "server", "--url", "http://localhost:#{@port}/mcp"]
npx_command += ["--scenario", @scenario] if @scenario
npx_command += ["--spec-version", @spec_version] if @spec_version
npx_command += ["--verbose"] if @verbose
npx_command += ["--expected-failures", expected_failures_yml]
npx_command
end
def start_server
puts "Starting conformance server on port #{@port}..."
server_pid = fork do
Conformance::Server.new(port: @port).start
end
health_url = URI("http://localhost:#{@port}/health")
ready = false
SERVER_HEALTH_CHECK_RETRIES.times do
begin
response = Net::HTTP.get_response(health_url)
if response.code == "200"
ready = true
break
end
rescue Errno::ECONNREFUSED, Errno::ECONNRESET, Net::ReadTimeout
# not ready yet
end
sleep(SERVER_POLL_INTERVAL)
end
unless ready
warn("ERROR: Conformance server did not start within #{SERVER_START_TIMEOUT} seconds")
terminate_server(server_pid)
exit(1)
end
puts "Server ready. Running conformance tests..."
server_pid
end
def run_conformance(command, server_pid:)
puts "Command: #{command.join(" ")}\n\n"
conformance_exit_code = nil
begin
system(*command)
conformance_exit_code = $CHILD_STATUS.exitstatus
ensure
terminate_server(server_pid)
end
exit(conformance_exit_code || 1) unless conformance_exit_code == 0
end
def terminate_server(pid)
Process.kill("TERM", pid)
rescue Errno::ESRCH
# process already exited
ensure
begin
Process.wait(pid)
rescue Errno::ECHILD
# already reaped
end
end
end
end