From 7b12dc401039a5256f150156370afed8a21d935d Mon Sep 17 00:00:00 2001 From: Christopher Hertel Date: Sat, 15 Aug 2026 02:24:48 +0200 Subject: [PATCH] [Tests] Introduce client-server integration tests (#419) * [Tests] Introduce client-server integration tests * Add an in-process loopback that connects a real client transport to a real server transport, draining both Fiber loops in one synchronous pass instead of polling. A stalled exchange fails where it happened rather than hanging. * Cover protocol version negotiation across both implementations, where the unit tests only ever drive one side against a canned counter-offer. * Cover the server-to-client round-trips end to end: elicitation, sampling, roots, plus progress and logging notifications. * Add the `integration` test suite and a `make integration-tests` target. * [Tests] Run the integration suite in CI The unit job pins `--testsuite=unit`, so the new suite would never have run on GitHub. It goes in that job rather than its own: the loopback leans on Fibers, and the matrix is what covers them down to the PHP 8.1 floor. * [Tests] Run integration tests against a real server process Replaces the in-process loopback with the setup from examples/client: the client spawns a fixture server over the real StdioTransport, so the tests exercise the shipped transports and no longer nest both sides' Fibers. Moves the suite into its own CI job, matrixed over PHP 8.1-8.5. --- .github/workflows/pipeline.yaml | 22 +++++ Makefile | 5 +- phpunit.xml.dist | 3 + tests/Integration/ElicitationTest.php | 107 +++++++++++++++++++++ tests/Integration/Fixture/elicitation.php | 49 ++++++++++ tests/Integration/Fixture/handshake.php | 32 ++++++ tests/Integration/Fixture/notification.php | 39 ++++++++ tests/Integration/Fixture/roots.php | 43 +++++++++ tests/Integration/Fixture/sampling.php | 42 ++++++++ tests/Integration/HandshakeTest.php | 85 ++++++++++++++++ tests/Integration/IntegrationTestCase.php | 83 ++++++++++++++++ tests/Integration/NotificationTest.php | 75 +++++++++++++++ tests/Integration/RootsTest.php | 98 +++++++++++++++++++ tests/Integration/SamplingTest.php | 98 +++++++++++++++++++ 14 files changed, 780 insertions(+), 1 deletion(-) create mode 100644 tests/Integration/ElicitationTest.php create mode 100644 tests/Integration/Fixture/elicitation.php create mode 100644 tests/Integration/Fixture/handshake.php create mode 100644 tests/Integration/Fixture/notification.php create mode 100644 tests/Integration/Fixture/roots.php create mode 100644 tests/Integration/Fixture/sampling.php create mode 100644 tests/Integration/HandshakeTest.php create mode 100644 tests/Integration/IntegrationTestCase.php create mode 100644 tests/Integration/NotificationTest.php create mode 100644 tests/Integration/RootsTest.php create mode 100644 tests/Integration/SamplingTest.php diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index 9b76066..f7396e6 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -49,6 +49,28 @@ jobs: - name: Tests run: vendor/bin/phpunit --testsuite=unit + integration: + runs-on: ubuntu-latest + strategy: + matrix: + php: ['8.1', '8.2', '8.3', '8.4', '8.5'] + + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Setup PHP + uses: shivammathur/setup-php@v2 + with: + php-version: ${{ matrix.php }} + coverage: "none" + + - name: Install Composer + uses: "ramsey/composer-install@v4" + + - name: Tests + run: vendor/bin/phpunit --testsuite=integration + inspector: runs-on: ubuntu-latest steps: diff --git a/Makefile b/Makefile index 667aa04..c886bc5 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: deps-stable deps-low cs phpstan tests unit-tests inspector-tests coverage ci ci-stable ci-lowest conformance-tests conformance-server conformance-client docs +.PHONY: deps-stable deps-low cs phpstan tests unit-tests integration-tests inspector-tests coverage ci ci-stable ci-lowest conformance-tests conformance-server conformance-client docs deps-stable: composer update --prefer-stable @@ -18,6 +18,9 @@ tests: unit-tests: vendor/bin/phpunit --testsuite=unit +integration-tests: + vendor/bin/phpunit --testsuite=integration + inspector-tests: vendor/bin/phpunit --testsuite=inspector diff --git a/phpunit.xml.dist b/phpunit.xml.dist index 54c2a8e..eb07b79 100644 --- a/phpunit.xml.dist +++ b/phpunit.xml.dist @@ -12,6 +12,9 @@ tests/Unit + + tests/Integration + examples/server/oauth-microsoft/tests diff --git a/tests/Integration/ElicitationTest.php b/tests/Integration/ElicitationTest.php new file mode 100644 index 0000000..dc16847 --- /dev/null +++ b/tests/Integration/ElicitationTest.php @@ -0,0 +1,107 @@ +connect('elicitation', $this->clientAnswering( + new ElicitResult(ElicitAction::Accept, ['name' => 'Ada']), + )); + + $result = $client->callTool('ask_name'); + + $this->assertFalse($result->isError); + $this->assertInstanceOf(TextContent::class, $result->content[0]); + $this->assertSame('accept:Ada', $result->content[0]->text); + } + + #[TestDox('a declined elicitation reaches the tool as a decline, not an error')] + public function testDeclinedElicitation(): void + { + $client = $this->connect('elicitation', $this->clientAnswering( + new ElicitResult(ElicitAction::Decline), + )); + + $result = $client->callTool('ask_name'); + + $this->assertInstanceOf(TextContent::class, $result->content[0]); + $this->assertSame('decline:', $result->content[0]->text); + } + + #[TestDox('a client that does not advertise elicitation is not asked')] + public function testCapabilityIsVisibleToTheServer(): void + { + // The tool consults supportsElicitation(), which answers from the + // capabilities this client sent during the handshake. + $client = $this->connect('elicitation'); + + $result = $client->callTool('ask_name'); + + $this->assertInstanceOf(TextContent::class, $result->content[0]); + $this->assertSame('unsupported', $result->content[0]->text); + } + + #[TestDox('a client advertising elicitation without a handler fails the tool call')] + public function testAdvertisedCapabilityWithoutHandler(): void + { + // The client answers "method not found", which the gateway raises inside + // the tool as a ClientException rather than leaving it waiting. + $client = $this->connect( + 'elicitation', + $this->clientBuilder()->setCapabilities(new ClientCapabilities(elicitation: true)), + ); + + $result = $client->callTool('ask_name'); + + $this->assertInstanceOf(TextContent::class, $result->content[0]); + $this->assertSame('Client does not handle "elicitation/create" requests.', $result->content[0]->text); + } + + private function clientAnswering(ElicitResult $answer): ClientBuilder + { + $callback = new class($answer) implements ElicitationCallbackInterface { + public function __construct(private readonly ElicitResult $answer) + { + } + + public function __invoke(ElicitRequest $request): ElicitResult + { + return $this->answer; + } + }; + + return $this->clientBuilder() + ->setCapabilities(new ClientCapabilities(elicitation: true)) + ->addRequestHandler(new ElicitationRequestHandler($callback)); + } +} diff --git a/tests/Integration/Fixture/elicitation.php b/tests/Integration/Fixture/elicitation.php new file mode 100644 index 0000000..ad588a9 --- /dev/null +++ b/tests/Integration/Fixture/elicitation.php @@ -0,0 +1,49 @@ +setServerInfo('integration-server', '1.0.0') + ->addTool( + static function (RequestContext $context): string { + $gateway = $context->getClientGateway(); + + if (!$gateway->supportsElicitation()) { + return 'unsupported'; + } + + try { + $result = $gateway->elicit('What is your name?', new ElicitationSchema([ + 'name' => new StringSchemaDefinition(title: 'Name'), + ])); + } catch (ClientException $e) { + return $e->getMessage(); + } + + return sprintf('%s:%s', $result->action->value, $result->content['name'] ?? ''); + }, + name: 'ask_name', + description: 'Asks the client for a name.', + ) + ->build() + ->run(new StdioTransport()); diff --git a/tests/Integration/Fixture/handshake.php b/tests/Integration/Fixture/handshake.php new file mode 100644 index 0000000..776f451 --- /dev/null +++ b/tests/Integration/Fixture/handshake.php @@ -0,0 +1,32 @@ +setServerInfo('integration-server', '1.0.0') + ->setInstructions('Be brief.'); + +if (is_string($pinned = getenv('MCP_INTEGRATION_PROTOCOL_VERSION')) && '' !== $pinned) { + $builder->setProtocolVersion(ProtocolVersion::from($pinned)); +} + +$builder->build()->run(new StdioTransport()); diff --git a/tests/Integration/Fixture/notification.php b/tests/Integration/Fixture/notification.php new file mode 100644 index 0000000..166e84c --- /dev/null +++ b/tests/Integration/Fixture/notification.php @@ -0,0 +1,39 @@ +setServerInfo('integration-server', '1.0.0') + ->addTool( + static function (RequestContext $context): string { + $gateway = $context->getClientGateway(); + + $gateway->log(LoggingLevel::Info, 'starting work'); + $gateway->progress(0.5, 1.0, 'halfway'); + $gateway->progress(1.0, 1.0, 'done'); + + return 'finished'; + }, + name: 'work', + description: 'Reports progress and logs while working.', + ) + ->build() + ->run(new StdioTransport()); diff --git a/tests/Integration/Fixture/roots.php b/tests/Integration/Fixture/roots.php new file mode 100644 index 0000000..a4b6382 --- /dev/null +++ b/tests/Integration/Fixture/roots.php @@ -0,0 +1,43 @@ +setServerInfo('integration-server', '1.0.0') + ->addTool( + static function (RequestContext $context): string { + $gateway = $context->getClientGateway(); + + if (!$gateway->supportsRoots()) { + return 'unsupported'; + } + + $described = []; + foreach ($gateway->listRoots()->roots as $root) { + $described[] = sprintf('%s (%s)', $root->uri, $root->name ?? '-'); + } + + return implode(', ', $described); + }, + name: 'inspect_roots', + description: 'Reports the workspace roots the client exposes.', + ) + ->build() + ->run(new StdioTransport()); diff --git a/tests/Integration/Fixture/sampling.php b/tests/Integration/Fixture/sampling.php new file mode 100644 index 0000000..bb37bbf --- /dev/null +++ b/tests/Integration/Fixture/sampling.php @@ -0,0 +1,42 @@ +setServerInfo('integration-server', '1.0.0') + ->addTool( + static function (RequestContext $context, string $text): string { + try { + $result = $context->getClientGateway()->sample($text, maxTokens: 64); + } catch (ClientException $e) { + return $e->getMessage(); + } + + assert($result->content instanceof TextContent); + + return sprintf('%s said: %s', $result->model, $result->content->text); + }, + name: 'summarize', + description: 'Summarizes text by asking the client to sample.', + ) + ->build() + ->run(new StdioTransport()); diff --git a/tests/Integration/HandshakeTest.php b/tests/Integration/HandshakeTest.php new file mode 100644 index 0000000..5e0455f --- /dev/null +++ b/tests/Integration/HandshakeTest.php @@ -0,0 +1,85 @@ +clientBuilder(); + if (null !== $clientVersion) { + $client->setProtocolVersion($clientVersion); + } + + $connected = $this->connect( + 'handshake', + $client, + null !== $serverVersion ? ['MCP_INTEGRATION_PROTOCOL_VERSION' => $serverVersion->value] : [], + ); + + $this->assertSame($expected, $connected->getProtocolVersion()); + } + + /** + * @return iterable + */ + public static function provideNegotiations(): iterable + { + $latest = ProtocolVersion::latestHandshake(); + + yield 'both unconfigured' => [null, null, $latest]; + + // Whichever end of the supported range it sits at. + foreach (ProtocolVersion::handshakeVersions() as $version) { + yield \sprintf('client asks for %s', $version->value) => [$version, null, $version]; + } + + // A pinned server answers with its pin, and the client continues on it. + yield 'server pins an older revision' => [ProtocolVersion::V2025_11_25, ProtocolVersion::V2025_03_26, ProtocolVersion::V2025_03_26]; + yield 'server pins a newer revision' => [ProtocolVersion::V2024_11_05, ProtocolVersion::V2025_11_25, ProtocolVersion::V2025_11_25]; + yield 'both pin the same revision' => [ProtocolVersion::V2025_06_18, ProtocolVersion::V2025_06_18, ProtocolVersion::V2025_06_18]; + + // Neither side reaches the modern era through `initialize`, so + // configuring it falls back to the handshake set on both ends. + yield 'client configured modern' => [ProtocolVersion::V2026_07_28, null, $latest]; + yield 'server configured modern' => [ProtocolVersion::V2025_06_18, ProtocolVersion::V2026_07_28, ProtocolVersion::V2025_06_18]; + yield 'both configured modern' => [ProtocolVersion::V2026_07_28, ProtocolVersion::V2026_07_28, $latest]; + } + + #[TestDox('the handshake carries the server identity to the client')] + public function testServerInfoIsExchanged(): void + { + $client = $this->connect('handshake'); + + $this->assertSame('integration-server', $client->getServerInfo()->name); + $this->assertSame('1.0.0', $client->getServerInfo()->version); + $this->assertSame('Be brief.', $client->getInstructions()); + $this->assertTrue($client->isConnected()); + } + + #[TestDox('the negotiated revision is unset before the handshake')] + public function testProtocolVersionIsNullBeforeConnecting(): void + { + $this->assertNull($this->clientBuilder()->build()->getProtocolVersion()); + } +} diff --git a/tests/Integration/IntegrationTestCase.php b/tests/Integration/IntegrationTestCase.php new file mode 100644 index 0000000..aa0dd9c --- /dev/null +++ b/tests/Integration/IntegrationTestCase.php @@ -0,0 +1,83 @@ + + */ +abstract class IntegrationTestCase extends TestCase +{ + /** + * Both sides answer immediately, so anything reaching this is a deadlock. + * Far below the SDK's two-minute default, to fail rather than hang. + */ + private const TIMEOUT = 5; + + private ?Client $client = null; + + protected function clientBuilder(): ClientBuilder + { + return Client::builder() + ->setClientInfo('integration-client', '1.0.0') + ->setInitTimeout(self::TIMEOUT) + ->setRequestTimeout(self::TIMEOUT); + } + + /** + * Spawn a fixture server and connect a client to it. + * + * The returned client has completed the handshake. + * + * @param string $fixture basename of a script in {@see Fixture} + * @param array $env added to the server process environment + */ + protected function connect(string $fixture, ?ClientBuilder $client = null, array $env = []): Client + { + $script = __DIR__.'/Fixture/'.$fixture.'.php'; + + $this->client = ($client ?? $this->clientBuilder())->build(); + + try { + $this->client->connect(new StdioTransport( + command: \PHP_BINARY, + args: [$script], + // proc_open() replaces the environment rather than adding to it. + env: [] === $env ? null : array_merge(getenv(), $env), + )); + } catch (ConnectionException $e) { + // The transport discards the child's stderr, so a fixture dying on + // startup arrives here as a bare timeout. + $this->fail(\sprintf('Could not connect to fixture server "%s": %s. Run `%s %s` to see why.', $fixture, $e->getMessage(), \PHP_BINARY, $script)); + } + + return $this->client; + } + + protected function tearDown(): void + { + $this->client?->disconnect(); + $this->client = null; + } +} diff --git a/tests/Integration/NotificationTest.php b/tests/Integration/NotificationTest.php new file mode 100644 index 0000000..0948718 --- /dev/null +++ b/tests/Integration/NotificationTest.php @@ -0,0 +1,75 @@ +connect('notification'); + + $updates = []; + $result = $client->callTool('work', [], static function (float $progress, ?float $total, ?string $message) use (&$updates): void { + $updates[] = [$progress, $total, $message]; + }); + + $this->assertSame([[0.5, 1.0, 'halfway'], [1.0, 1.0, 'done']], $updates); + $this->assertInstanceOf(TextContent::class, $result->content[0]); + $this->assertSame('finished', $result->content[0]->text); + } + + #[TestDox('progress is skipped when the caller asked for none')] + public function testProgressIsSkippedWithoutAToken(): void + { + // Without an onProgress callback the request carries no progress token, + // so the gateway drops the notification instead of sending it. + $client = $this->connect('notification'); + + $result = $client->callTool('work'); + + $this->assertInstanceOf(TextContent::class, $result->content[0]); + $this->assertSame('finished', $result->content[0]->text); + } + + #[TestDox('log notifications reach a registered logging handler')] + public function testLoggingReachesTheClient(): void + { + $logged = []; + $client = $this->connect( + 'notification', + $this->clientBuilder()->addNotificationHandler(new LoggingNotificationHandler( + static function (LoggingMessageNotification $notification) use (&$logged): void { + $logged[] = [$notification->level, $notification->data]; + }, + )), + ); + + $client->callTool('work'); + + $this->assertSame([[LoggingLevel::Info, 'starting work']], $logged); + } +} diff --git a/tests/Integration/RootsTest.php b/tests/Integration/RootsTest.php new file mode 100644 index 0000000..6dfd065 --- /dev/null +++ b/tests/Integration/RootsTest.php @@ -0,0 +1,98 @@ +connect('roots', $this->clientExposing( + new Root('file:///workspace/app', 'App'), + new Root('file:///workspace/docs', 'Docs'), + )); + + $result = $client->callTool('inspect_roots'); + + $this->assertInstanceOf(TextContent::class, $result->content[0]); + $this->assertSame('file:///workspace/app (App), file:///workspace/docs (Docs)', $result->content[0]->text); + } + + #[TestDox('an empty root list is a valid answer, not a failure')] + public function testEmptyRootList(): void + { + $client = $this->connect('roots', $this->clientExposing()); + + $result = $client->callTool('inspect_roots'); + + $this->assertInstanceOf(TextContent::class, $result->content[0]); + $this->assertSame('', $result->content[0]->text); + } + + #[TestDox('a client that does not advertise roots is not asked')] + public function testCapabilityIsVisibleToTheServer(): void + { + $client = $this->connect('roots'); + + $result = $client->callTool('inspect_roots'); + + $this->assertInstanceOf(TextContent::class, $result->content[0]); + $this->assertSame('unsupported', $result->content[0]->text); + } + + #[TestDox('the client can announce that its roots changed')] + public function testRootsListChangedNotification(): void + { + $client = $this->connect('roots', $this->clientExposing(new Root('file:///workspace'))); + + // A notification has no reply, so what this pins down is that sending + // one mid-session leaves the connection usable. + $client->sendRootsListChanged(); + + $this->assertTrue($client->isConnected()); + $this->assertInstanceOf(TextContent::class, $client->callTool('inspect_roots')->content[0]); + } + + private function clientExposing(Root ...$roots): ClientBuilder + { + $callback = new class(array_values($roots)) implements RootsCallbackInterface { + /** @param list $roots */ + public function __construct(private readonly array $roots) + { + } + + public function __invoke(ListRootsRequest $request): ListRootsResult + { + return new ListRootsResult($this->roots); + } + }; + + return $this->clientBuilder() + ->setCapabilities(new ClientCapabilities(roots: true, rootsListChanged: true)) + ->addRequestHandler(new ListRootsRequestHandler($callback)); + } +} diff --git a/tests/Integration/SamplingTest.php b/tests/Integration/SamplingTest.php new file mode 100644 index 0000000..c581cf8 --- /dev/null +++ b/tests/Integration/SamplingTest.php @@ -0,0 +1,98 @@ +connect('sampling', $this->clientSampling()); + + $result = $client->callTool('summarize', ['text' => 'a long report']); + + $this->assertInstanceOf(TextContent::class, $result->content[0]); + $this->assertSame('test-model said: a long report', $result->content[0]->text); + } + + #[TestDox('the prompt the tool passed arrives at the client')] + public function testPromptReachesTheClient(): void + { + // The client stays in this process, so what it was asked can be + // collected by reference even though the tool asking runs in another. + /** @var \ArrayObject $seen */ + $seen = new \ArrayObject(); + $client = $this->connect('sampling', $this->clientSampling($seen)); + + $client->callTool('summarize', ['text' => 'inspect me']); + + $this->assertCount(1, $seen); + $this->assertInstanceOf(TextContent::class, $seen[0]->messages[0]->content); + $this->assertSame('inspect me', $seen[0]->messages[0]->content->text); + $this->assertSame(64, $seen[0]->maxTokens); + } + + #[TestDox('a client that cannot sample refuses instead of stalling the tool')] + public function testClientWithoutSamplingRefuses(): void + { + // The gateway has no supportsSampling() to consult, so the tool finds + // out by asking and the refusal surfaces as a ClientException. + $client = $this->connect('sampling'); + + $result = $client->callTool('summarize', ['text' => 'anything']); + + $this->assertInstanceOf(TextContent::class, $result->content[0]); + $this->assertSame('Client does not handle "sampling/createMessage" requests.', $result->content[0]->text); + } + + /** + * @param \ArrayObject|null $seen collects what the server asked for + */ + private function clientSampling(?\ArrayObject $seen = null): ClientBuilder + { + $callback = new class($seen ?? new \ArrayObject()) implements SamplingCallbackInterface { + /** @param \ArrayObject $seen */ + public function __construct(private readonly \ArrayObject $seen) + { + } + + public function __invoke(CreateSamplingMessageRequest $request): CreateSamplingMessageResult + { + $this->seen[] = $request; + + $prompt = $request->messages[0]->content; + \assert($prompt instanceof TextContent); + + return new CreateSamplingMessageResult(Role::Assistant, new TextContent($prompt->text), 'test-model'); + } + }; + + return $this->clientBuilder() + ->setCapabilities(new ClientCapabilities(sampling: true)) + ->addRequestHandler(new SamplingRequestHandler($callback)); + } +}