using System.Diagnostics; using System.Runtime.InteropServices; using System.Text; using System.Text.RegularExpressions; using ModelContextProtocol.Tests.Utils; namespace ModelContextProtocol.ConformanceTests; /// /// Shared fixture that starts a single ConformanceServer instance for all tests in /// . This avoids TCP port TIME_WAIT conflicts /// that occur when each test starts and stops its own server on the same port. /// public class ConformanceServerFixture : IAsyncLifetime { // Use different ports for each target framework to allow parallel execution // net10.0 -> 3001, net9.0 -> 3002, net8.0 -> 3003 private static int GetPortForTargetFramework() { var testBinaryDir = AppContext.BaseDirectory; var targetFramework = Path.GetFileName(testBinaryDir.TrimEnd(Path.DirectorySeparatorChar)); return targetFramework switch { "net10.0" => 3001, "net9.0" => 3002, "net8.0" => 3003, _ => 3001 // Default fallback }; } private Task? _serverTask; private CancellationTokenSource? _serverCts; public string ServerUrl { get; } = $"http://localhost:{GetPortForTargetFramework()}"; public async ValueTask InitializeAsync() { _serverCts = new CancellationTokenSource(); _serverTask = Task.Run(() => ConformanceServer.Program.MainAsync( ["--urls", ServerUrl], cancellationToken: _serverCts.Token)); // Wait for server to be ready (retry for up to 30 seconds) var timeout = TimeSpan.FromSeconds(30); var stopwatch = Stopwatch.StartNew(); using var httpClient = new HttpClient { Timeout = TestConstants.HttpClientPollingTimeout }; while (stopwatch.Elapsed < timeout) { try { await httpClient.GetAsync($"{ServerUrl}/health"); return; } catch (HttpRequestException) { // Connection refused means server not ready yet } catch (TaskCanceledException) { // Timeout means server might be processing, give it more time } await Task.Delay(500); } throw new InvalidOperationException("ConformanceServer failed to start within the timeout period"); } public async ValueTask DisposeAsync() { if (_serverCts != null) { _serverCts.Cancel(); if (_serverTask != null) { try { await _serverTask.WaitAsync(TestConstants.DefaultTimeout); } catch { // Ignore exceptions during shutdown } } _serverCts.Dispose(); } } } /// /// Runs the official MCP conformance tests against the ConformanceServer. /// Uses a shared so the server is started once /// and reused across all tests, avoiding TCP port conflicts on Windows. /// public class ServerConformanceTests(ConformanceServerFixture fixture, ITestOutputHelper output) : IClassFixture { [Fact] public async Task RunConformanceTests() { Assert.SkipWhen(!NodeHelpers.IsNodeInstalled(), "Node.js is not installed. Skipping conformance tests."); var result = await RunConformanceTestsAsync($"server --url {fixture.ServerUrl}"); Assert.True(result.Success, $"Conformance tests failed.\n\nStdout:\n{result.Output}\n\nStderr:\n{result.Error}"); } [Fact] public async Task RunPendingConformanceTest_JsonSchema202012() { Assert.SkipWhen(!NodeHelpers.IsNodeInstalled(), "Node.js is not installed. Skipping conformance tests."); Assert.SkipWhen( RuntimeInformation.IsOSPlatform(OSPlatform.Windows), "Pending Node-based conformance scenario is unstable on Windows due to a libuv shutdown assertion."); var result = await RunConformanceTestsAsync($"server --url {fixture.ServerUrl} --scenario json-schema-2020-12"); Assert.True(result.Success, $"Conformance test failed.\n\nStdout:\n{result.Output}\n\nStderr:\n{result.Error}"); } [Fact] public async Task RunPendingConformanceTest_ServerSsePolling() { Assert.SkipWhen(!NodeHelpers.IsNodeInstalled(), "Node.js is not installed. Skipping conformance tests."); Assert.SkipWhen( RuntimeInformation.IsOSPlatform(OSPlatform.Windows), "Pending Node-based conformance scenario is unstable on Windows due to a libuv shutdown assertion."); var result = await RunConformanceTestsAsync($"server --url {fixture.ServerUrl} --scenario server-sse-polling"); Assert.True(result.Success, $"Conformance test failed.\n\nStdout:\n{result.Output}\n\nStderr:\n{result.Error}"); } [Fact] public async Task RunConformanceTest_HttpHeaderValidation() { Assert.SkipWhen(!NodeHelpers.IsNodeInstalled(), "Node.js is not installed. Skipping conformance tests."); Assert.SkipWhen(!NodeHelpers.HasSep2243Scenarios(), "SEP-2243 conformance scenarios not yet available."); var result = await RunConformanceTestsAsync($"server --url {fixture.ServerUrl} --scenario http-header-validation"); Assert.True(result.Success, $"Conformance test failed.\n\nStdout:\n{result.Output}\n\nStderr:\n{result.Error}"); } [Fact] public async Task RunConformanceTest_HttpCustomHeaderServerValidation() { Assert.SkipWhen(!NodeHelpers.IsNodeInstalled(), "Node.js is not installed. Skipping conformance tests."); Assert.SkipWhen(!NodeHelpers.HasSep2243Scenarios(), "SEP-2243 conformance scenarios not yet available."); var result = await RunConformanceTestsAsync($"server --url {fixture.ServerUrl} --scenario http-custom-header-server-validation"); Assert.True(result.Success, $"Conformance test failed.\n\nStdout:\n{result.Output}\n\nStderr:\n{result.Error}"); } // SEP-2322 (Multi Round-Trip Requests / IncompleteResult) conformance scenarios. // The csharp-sdk ConformanceServer surfaces the matching tools/prompts via // ConformanceServer.Tools.IncompleteResultTools and ConformanceServer.Prompts.IncompleteResultPrompts. // Each scenario uses the conformance harness's RawMcpSession, which negotiates DRAFT-2026-v1 // so the csharp-sdk emits InputRequiredResult on the wire. These tests skip until the // upstream conformance package ships with SEP-2322 scenarios // (https://github.com/modelcontextprotocol/conformance/pull/188). [Theory] [InlineData("incomplete-result-basic-elicitation")] [InlineData("incomplete-result-basic-sampling")] [InlineData("incomplete-result-basic-list-roots")] [InlineData("incomplete-result-request-state")] [InlineData("incomplete-result-multiple-input-requests")] [InlineData("incomplete-result-multi-round")] [InlineData("incomplete-result-missing-input-response")] [InlineData("incomplete-result-non-tool-request")] public async Task RunMrtrConformanceTest(string scenario) { Assert.SkipWhen(!NodeHelpers.IsNodeInstalled(), "Node.js is not installed. Skipping conformance tests."); Assert.SkipWhen(!NodeHelpers.HasMrtrScenarios(), "SEP-2322 MRTR conformance scenarios not yet available in the published @modelcontextprotocol/conformance package."); var result = await RunConformanceTestsAsync( $"server --url {fixture.ServerUrl} --scenario {scenario}"); Assert.True(result.Success, $"MRTR conformance test '{scenario}' failed.\n\nStdout:\n{result.Output}\n\nStderr:\n{result.Error}"); } private async Task<(bool Success, string Output, string Error)> RunConformanceTestsAsync(string arguments) { var startInfo = NodeHelpers.ConformanceTestStartInfo(arguments); var outputBuilder = new StringBuilder(); var errorBuilder = new StringBuilder(); var process = new Process { StartInfo = startInfo }; // Protect callbacks with try/catch to prevent ITestOutputHelper from // throwing on a background thread if events arrive after the test completes. DataReceivedEventHandler outputHandler = (sender, e) => { if (e.Data != null) { try { output.WriteLine(e.Data); } catch { } outputBuilder.AppendLine(e.Data); } }; DataReceivedEventHandler errorHandler = (sender, e) => { if (e.Data != null) { try { output.WriteLine(e.Data); } catch { } errorBuilder.AppendLine(e.Data); } }; process.OutputDataReceived += outputHandler; process.ErrorDataReceived += errorHandler; process.Start(); process.BeginOutputReadLine(); process.BeginErrorReadLine(); using var cts = new CancellationTokenSource(TimeSpan.FromMinutes(5)); try { await process.WaitForExitAsync(cts.Token); } catch (OperationCanceledException) { process.Kill(entireProcessTree: true); process.OutputDataReceived -= outputHandler; process.ErrorDataReceived -= errorHandler; return ( Success: false, Output: outputBuilder.ToString(), Error: errorBuilder.ToString() + "\nProcess timed out after 5 minutes and was killed." ); } process.OutputDataReceived -= outputHandler; process.ErrorDataReceived -= errorHandler; var stdoutText = outputBuilder.ToString(); var stderrText = errorBuilder.ToString(); // The Node.js conformance runner can crash during cleanup on Windows with a libuv // assertion ("!(handle->flags & UV_HANDLE_CLOSING)") that produces a non-zero exit // code even though every conformance check passed. When that happens, fall back to // parsing the "Test Results:" summary in stdout to decide success. bool success = process.ExitCode == 0 || ConformanceOutputIndicatesSuccess(stdoutText); return ( Success: success, Output: stdoutText, Error: stderrText ); } /// /// Parses the conformance runner output for a "Test Results:" line such as /// "Passed: 3/3, 0 failed, 0 warnings" and returns true when all checks passed /// and none failed. /// private static bool ConformanceOutputIndicatesSuccess(string output) { // Match lines like "Passed: 3/3, 0 failed, 0 warnings" var match = Regex.Match(output, @"Passed:\s*(\d+)/(\d+),\s*(\d+)\s*failed"); if (!match.Success) { return false; } int passed = int.Parse(match.Groups[1].Value); int total = int.Parse(match.Groups[2].Value); int failed = int.Parse(match.Groups[3].Value); return passed == total && failed == 0 && total > 0; } }