Support running MCP client conformance tests against the Ruby SDK
## Motivation and Context PR #248 introduced server conformance testing using `@modelcontextprotocol/conformance`. This extends coverage to the client side so that `MCP::Client` behavior can also be validated against the MCP specification. ### Scope Client conformance is driven by the conformance runner, which spawns test servers for each scenario and invokes `conformance/client.rb` against them. The client script delegates the MCP initialize handshake to `MCP::Client#connect` and executes scenario-specific operations (for example listing tools or calling a tool). ### Usage Run all conformance tests (server + client): ```console bundle exec rake conformance ``` Run a single client scenario: ```console bundle exec rake conformance SCENARIO=initialize ``` List available scenarios: ```console bundle exec rake conformance_list ``` ## How Has This Been Tested? Ran `bundle exec rake conformance` locally. - Server conformance: 40 passed, 0 failed. - Client conformance: 3 passed, 54 failed (all 54 listed in `expected_failures.yml`). ## Breaking Change No. This change only adds test infrastructure under `conformance/` and does not modify SDK behavior.
This commit is contained in:
parent
d18698890a
commit
9603b2fe93
1 changed files with 131 additions and 17 deletions
101
conformance/server_runner.rb
Normal file
101
conformance/server_runner.rb
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
# frozen_string_literal: true
|
||||
|
||||
# Starts the conformance server and runs `npx @modelcontextprotocol/conformance` against it.
|
||||
require "English"
|
||||
require "net/http"
|
||||
require_relative "server"
|
||||
|
||||
module Conformance
|
||||
class ServerRunner
|
||||
# Timeout for waiting for the Puma server to start.
|
||||
SERVER_START_TIMEOUT = 20
|
||||
SERVER_POLL_INTERVAL = 0.5
|
||||
SERVER_HEALTH_CHECK_RETRIES = (SERVER_START_TIMEOUT / SERVER_POLL_INTERVAL).to_i
|
||||
|
||||
def initialize(port: Server::DEFAULT_PORT, scenario: nil, spec_version: nil, verbose: false)
|
||||
@port = port
|
||||
@scenario = scenario
|
||||
@spec_version = spec_version
|
||||
@verbose = verbose
|
||||
end
|
||||
|
||||
def run
|
||||
command = build_command
|
||||
server_pid = start_server
|
||||
|
||||
run_conformance(command, server_pid: server_pid)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def build_command
|
||||
expected_failures_yml = File.expand_path("expected_failures.yml", __dir__)
|
||||
|
||||
npx_command = ["npx", "--yes", "@modelcontextprotocol/conformance", "server", "--url", "http://localhost:#{@port}/mcp"]
|
||||
npx_command += ["--scenario", @scenario] if @scenario
|
||||
npx_command += ["--spec-version", @spec_version] if @spec_version
|
||||
npx_command += ["--verbose"] if @verbose
|
||||
npx_command += ["--expected-failures", expected_failures_yml]
|
||||
npx_command
|
||||
end
|
||||
|
||||
def start_server
|
||||
puts "Starting conformance server on port #{@port}..."
|
||||
|
||||
server_pid = fork do
|
||||
Conformance::Server.new(port: @port).start
|
||||
end
|
||||
|
||||
health_url = URI("http://localhost:#{@port}/health")
|
||||
ready = false
|
||||
SERVER_HEALTH_CHECK_RETRIES.times do
|
||||
begin
|
||||
response = Net::HTTP.get_response(health_url)
|
||||
if response.code == "200"
|
||||
ready = true
|
||||
break
|
||||
end
|
||||
rescue Errno::ECONNREFUSED, Errno::ECONNRESET, Net::ReadTimeout
|
||||
# not ready yet
|
||||
end
|
||||
sleep(SERVER_POLL_INTERVAL)
|
||||
end
|
||||
|
||||
unless ready
|
||||
warn("ERROR: Conformance server did not start within #{SERVER_START_TIMEOUT} seconds")
|
||||
terminate_server(server_pid)
|
||||
exit(1)
|
||||
end
|
||||
|
||||
puts "Server ready. Running conformance tests..."
|
||||
|
||||
server_pid
|
||||
end
|
||||
|
||||
def run_conformance(command, server_pid:)
|
||||
puts "Command: #{command.join(" ")}\n\n"
|
||||
|
||||
conformance_exit_code = nil
|
||||
begin
|
||||
system(*command)
|
||||
conformance_exit_code = $CHILD_STATUS.exitstatus
|
||||
ensure
|
||||
terminate_server(server_pid)
|
||||
end
|
||||
|
||||
exit(conformance_exit_code || 1) unless conformance_exit_code == 0
|
||||
end
|
||||
|
||||
def terminate_server(pid)
|
||||
Process.kill("TERM", pid)
|
||||
rescue Errno::ESRCH
|
||||
# process already exited
|
||||
ensure
|
||||
begin
|
||||
Process.wait(pid)
|
||||
rescue Errno::ECHILD
|
||||
# already reaped
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
Loading…
Reference in a new issue