From 634735bfa71e36a09cb88ea3abf7e034c32dfc72 Mon Sep 17 00:00:00 2001 From: Maciek Kisiel Date: Tue, 27 Jan 2026 15:45:50 +0000 Subject: [PATCH] conformance: create everything-client with basic scenarios and configure GitHub action to run it - implement all non-auth scenarios (client OAuth is not yet supported) - sse-retry is also implemented, but failing due to a bug in the SDK - configure client-conformance job in the conformance GitHub action - create a client-conformance.sh script and rename conformance.sh to server-conformance.sh --- .github/workflows/conformance.yml | 17 ++ .github/workflows/nightly.yml | 4 +- conformance/baseline.yml | 19 +- conformance/everything-client/main.go | 248 ++++++++++++++++++ scripts/client-conformance.sh | 85 ++++++ .../{conformance.sh => server-conformance.sh} | 0 6 files changed, 370 insertions(+), 3 deletions(-) create mode 100644 conformance/everything-client/main.go create mode 100755 scripts/client-conformance.sh rename scripts/{conformance.sh => server-conformance.sh} (100%) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index f9a84f9..66acdf8 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -36,3 +36,20 @@ jobs: url: http://localhost:3001/mcp suite: active expected-failures: ./conformance/baseline.yml + + client-conformance: + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4 + - name: Set up Go + uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5 + with: + go-version: "^1.25" + - name: "Run conformance tests" + uses: modelcontextprotocol/conformance@c2f3fdaf781dcd5a862cb0d2f6454c1c210bf0f0 # v0.1.11 + with: + mode: client + command: go run ./conformance/everything-client/main.go + suite: core + expected-failures: ./conformance/baseline.yml diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 0aa95df..e2b5a52 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -32,5 +32,5 @@ jobs: uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4.1.0 with: node-version: "20" - - name: Run conformance tests - run: ./scripts/conformance.sh + - name: Run server conformance tests + run: ./scripts/server-conformance.sh diff --git a/conformance/baseline.yml b/conformance/baseline.yml index fd4e80a..c583549 100644 --- a/conformance/baseline.yml +++ b/conformance/baseline.yml @@ -1,4 +1,21 @@ server: # All tests should pass. client: - # Tests are expected to fail. List will be populated after running the tests. +- auth/basic-cimd +- auth/metadata-default +- auth/metadata-var1 +- auth/metadata-var2 +- auth/metadata-var3 +- auth/2025-03-26-oauth-metadata-backcompat +- auth/2025-03-26-oauth-endpoint-fallback +- auth/scope-from-www-authenticate +- auth/scope-from-scopes-supported +- auth/scope-omitted-when-undefined +- auth/scope-step-up +- auth/scope-retry-limit +- auth/token-endpoint-auth-basic +- auth/token-endpoint-auth-post +- auth/token-endpoint-auth-none +- auth/client-credentials-jwt +- auth/client-credentials-basic +- sse-retry diff --git a/conformance/everything-client/main.go b/conformance/everything-client/main.go new file mode 100644 index 0000000..9674dbb --- /dev/null +++ b/conformance/everything-client/main.go @@ -0,0 +1,248 @@ +// Copyright 2025 The Go MCP SDK Authors. All rights reserved. +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +// The conformance client implements features required for MCP conformance testing. +// It mirrors the functionality of the TypeScript conformance client at +// https://github.com/modelcontextprotocol/typescript-sdk/blob/main/src/conformance/everything-client.ts +package main + +import ( + "context" + "fmt" + "log" + "os" + "slices" + "sort" + "strings" + + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +// scenarioHandler is the function signature for all conformance test scenarios. +// It takes a context and the server URL to connect to. +type scenarioHandler func(ctx context.Context, serverURL string) error + +var ( + // registry stores all registered scenario handlers. + registry = make(map[string]scenarioHandler) +) + +// registerScenario registers a new scenario handler with the given name. +// This function should be called during init() by scenario implementations. +func registerScenario(name string, handler scenarioHandler) { + if _, exists := registry[name]; exists { + log.Fatalf("Scenario %q is already registered", name) + } + registry[name] = handler +} + +func init() { + registerScenario("initialize", runBasicClient) + registerScenario("tools_call", runToolsCallClient) + registerScenario("elicitation-sep1034-client-defaults", runElicitationDefaultsClient) + registerScenario("sse-retry", runSSERetryClient) +} + +// ============================================================================ +// Basic scenarios +// ============================================================================ + +func runBasicClient(ctx context.Context, serverURL string) error { + session, err := connectToServer(ctx, serverURL) + if err != nil { + return err + } + defer session.Close() + + _, err = session.ListTools(ctx, nil) + if err != nil { + return fmt.Errorf("session.ListTools(): %v", err) + } + + return nil +} + +func runToolsCallClient(ctx context.Context, serverURL string) error { + session, err := connectToServer(ctx, serverURL) + if err != nil { + return err + } + defer session.Close() + + tools, err := session.ListTools(ctx, nil) + if err != nil { + return fmt.Errorf("session.ListTools(): %v", err) + } + + idx := slices.IndexFunc(tools.Tools, func(t *mcp.Tool) bool { + return t.Name == "add_numbers" + }) + if idx == -1 { + return fmt.Errorf("tool %q not found", "add_numbers") + } + + _, err = session.CallTool(ctx, &mcp.CallToolParams{ + Name: "add_numbers", + Arguments: map[string]any{"a": 5, "b": 3}, + }) + if err != nil { + return fmt.Errorf("session.CallTool('add_numbers'): %v", err) + } + + return nil +} + +// ============================================================================ +// Elicitation scenarios +// ============================================================================ + +func runElicitationDefaultsClient(ctx context.Context, serverURL string) error { + elicitationHandler := func(ctx context.Context, req *mcp.ElicitRequest) (*mcp.ElicitResult, error) { + return &mcp.ElicitResult{ + Action: "accept", + Content: map[string]any{}, + }, nil + } + + session, err := connectToServer(ctx, serverURL, withClientOptions(&mcp.ClientOptions{ + ElicitationHandler: elicitationHandler, + })) + if err != nil { + return err + } + defer session.Close() + + tools, err := session.ListTools(ctx, nil) + if err != nil { + return fmt.Errorf("session.ListTools(): %v", err) + } + + var testToolName = "test_client_elicitation_defaults" + idx := slices.IndexFunc(tools.Tools, func(t *mcp.Tool) bool { + return t.Name == testToolName + }) + if idx == -1 { + return fmt.Errorf("tool %q not found", testToolName) + } + + _, err = session.CallTool(ctx, &mcp.CallToolParams{ + Name: testToolName, + Arguments: map[string]any{}, + }) + if err != nil { + return fmt.Errorf("session.CallTool(%q): %v", testToolName, err) + } + + return nil +} + +// ============================================================================ +// SSE retry scenario +// ============================================================================ + +func runSSERetryClient(ctx context.Context, serverURL string) error { + // TODO: this scenario is not passing yet. It requires a fix in the client SSE handling. + session, err := connectToServer(ctx, serverURL) + if err != nil { + return err + } + defer session.Close() + log.Printf("Connected to server %q", serverURL) + + tools, err := session.ListTools(ctx, nil) + if err != nil { + return fmt.Errorf("session.ListTools(): %v", err) + } + + var testToolName = "test_reconnection" + idx := slices.IndexFunc(tools.Tools, func(t *mcp.Tool) bool { + return t.Name == testToolName + }) + if idx == -1 { + return fmt.Errorf("tool %q not found", testToolName) + } + + _, err = session.CallTool(ctx, &mcp.CallToolParams{ + Name: testToolName, + Arguments: map[string]any{}, + }) + if err != nil { + return fmt.Errorf("session.CallTool(%q): %v", testToolName, err) + } + + return nil +} + +// ============================================================================ +// Main entry point +// ============================================================================ + +func main() { + if len(os.Args) != 2 { + printUsageAndExit("Usage: %s ", os.Args[0]) + } + + serverURL := os.Args[1] + scenarioName := os.Getenv("MCP_CONFORMANCE_SCENARIO") + + if scenarioName == "" { + printUsageAndExit("MCP_CONFORMANCE_SCENARIO not set") + } + + handler, ok := registry[scenarioName] + if !ok { + printUsageAndExit("Unknown scenario: %q", scenarioName) + } + + ctx := context.Background() + if err := handler(ctx, serverURL); err != nil { + log.Fatalf("Scenario %q failed: %v", scenarioName, err) + } +} + +func printUsageAndExit(format string, args ...any) { + var scenarios []string + for name := range registry { + scenarios = append(scenarios, name) + } + sort.Strings(scenarios) + + msg := fmt.Sprintf(format, args...) + log.Fatalf("%s\nAvailable scenarios:\n - %s", msg, strings.Join(scenarios, "\n - ")) +} + +type connectConfig struct { + clientOptions *mcp.ClientOptions +} + +type connectOption func(*connectConfig) + +func withClientOptions(opts *mcp.ClientOptions) connectOption { + return func(c *connectConfig) { + c.clientOptions = opts + } +} + +// connectToServer connects to the MCP server and returns a client session. +// The caller is responsible for closing the session. +func connectToServer(ctx context.Context, serverURL string, opts ...connectOption) (*mcp.ClientSession, error) { + config := &connectConfig{} + for _, opt := range opts { + opt(config) + } + + client := mcp.NewClient(&mcp.Implementation{ + Name: "test-client", + Version: "1.0.0", + }, config.clientOptions) + + transport := &mcp.StreamableClientTransport{Endpoint: serverURL} + + session, err := client.Connect(ctx, transport, nil) + if err != nil { + return nil, fmt.Errorf("client.Connect(): %v", err) + } + + return session, nil +} diff --git a/scripts/client-conformance.sh b/scripts/client-conformance.sh new file mode 100755 index 0000000..c093c75 --- /dev/null +++ b/scripts/client-conformance.sh @@ -0,0 +1,85 @@ +#!/bin/bash +# Copyright 2025 The Go MCP SDK Authors. All rights reserved. +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file. + +# Run MCP conformance tests against the Go SDK conformance client. + +set -e + +RESULT_DIR="" +WORKDIR="" +CONFORMANCE_REPO="" +FINAL_EXIT_CODE=0 + +usage() { + echo "Usage: $0 [options]" + echo "" + echo "Run MCP conformance tests against the Go SDK conformance client." + echo "" + echo "Options:" + echo " --result_dir Save results to the specified directory" + echo " --conformance_repo Run conformance tests from a local checkout" + echo " instead of using the latest npm release" + echo " --help Show this help message" +} + +# Parse arguments. +while [[ $# -gt 0 ]]; do + case $1 in + --result_dir) + RESULT_DIR="$2" + shift 2 + ;; + --conformance_repo) + CONFORMANCE_REPO="$2" + shift 2 + ;; + --help) + usage + exit 0 + ;; + *) + echo "Unknown option: $1" + usage + exit 1 + ;; + esac +done + +# Set up the work directory. +if [ -n "$RESULT_DIR" ]; then + mkdir -p "$RESULT_DIR" + WORKDIR="$RESULT_DIR" +else + WORKDIR=$(mktemp -d) +fi + +# Build the conformance server. +go build -o "$WORKDIR/conformance-client" ./conformance/everything-client + +# Run conformance tests from the work directory to avoid writing results to the repo. +echo "Running conformance tests..." +if [ -n "$CONFORMANCE_REPO" ]; then + # Run from local checkout using npm run start. + (cd "$WORKDIR" && \ + npm --prefix "$CONFORMANCE_REPO" run start -- \ + client --command "$WORKDIR/conformance-client" \ + --suite core \ + ${RESULT_DIR:+--output-dir "$RESULT_DIR"}) || FINAL_EXIT_CODE=$? +else + (cd "$WORKDIR" && \ + npx @modelcontextprotocol/conformance@latest \ + client --command "$WORKDIR/conformance-client" \ + --suite core \ + ${RESULT_DIR:+--output-dir "$RESULT_DIR"}) || FINAL_EXIT_CODE=$? +fi + +echo "" +if [ -n "$RESULT_DIR" ]; then + echo "See $RESULT_DIR for details." +else + echo "Run with --result_dir to save results." +fi + +exit $FINAL_EXIT_CODE diff --git a/scripts/conformance.sh b/scripts/server-conformance.sh similarity index 100% rename from scripts/conformance.sh rename to scripts/server-conformance.sh