diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index 9b76066b..f7396e60 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -49,6 +49,28 @@ jobs: - name: Tests run: vendor/bin/phpunit --testsuite=unit + integration: + runs-on: ubuntu-latest + strategy: + matrix: + php: ['8.1', '8.2', '8.3', '8.4', '8.5'] + + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Setup PHP + uses: shivammathur/setup-php@v2 + with: + php-version: ${{ matrix.php }} + coverage: "none" + + - name: Install Composer + uses: "ramsey/composer-install@v4" + + - name: Tests + run: vendor/bin/phpunit --testsuite=integration + inspector: runs-on: ubuntu-latest steps: diff --git a/Makefile b/Makefile index 667aa046..c886bc5e 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: deps-stable deps-low cs phpstan tests unit-tests inspector-tests coverage ci ci-stable ci-lowest conformance-tests conformance-server conformance-client docs +.PHONY: deps-stable deps-low cs phpstan tests unit-tests integration-tests inspector-tests coverage ci ci-stable ci-lowest conformance-tests conformance-server conformance-client docs deps-stable: composer update --prefer-stable @@ -18,6 +18,9 @@ tests: unit-tests: vendor/bin/phpunit --testsuite=unit +integration-tests: + vendor/bin/phpunit --testsuite=integration + inspector-tests: vendor/bin/phpunit --testsuite=inspector diff --git a/phpunit.xml.dist b/phpunit.xml.dist index 54c2a8e1..eb07b79a 100644 --- a/phpunit.xml.dist +++ b/phpunit.xml.dist @@ -12,6 +12,9 @@ tests/Unit + + tests/Integration + examples/server/oauth-microsoft/tests diff --git a/tests/Integration/ElicitationTest.php b/tests/Integration/ElicitationTest.php new file mode 100644 index 00000000..dc168479 --- /dev/null +++ b/tests/Integration/ElicitationTest.php @@ -0,0 +1,107 @@ +connect('elicitation', $this->clientAnswering( + new ElicitResult(ElicitAction::Accept, ['name' => 'Ada']), + )); + + $result = $client->callTool('ask_name'); + + $this->assertFalse($result->isError); + $this->assertInstanceOf(TextContent::class, $result->content[0]); + $this->assertSame('accept:Ada', $result->content[0]->text); + } + + #[TestDox('a declined elicitation reaches the tool as a decline, not an error')] + public function testDeclinedElicitation(): void + { + $client = $this->connect('elicitation', $this->clientAnswering( + new ElicitResult(ElicitAction::Decline), + )); + + $result = $client->callTool('ask_name'); + + $this->assertInstanceOf(TextContent::class, $result->content[0]); + $this->assertSame('decline:', $result->content[0]->text); + } + + #[TestDox('a client that does not advertise elicitation is not asked')] + public function testCapabilityIsVisibleToTheServer(): void + { + // The tool consults supportsElicitation(), which answers from the + // capabilities this client sent during the handshake. + $client = $this->connect('elicitation'); + + $result = $client->callTool('ask_name'); + + $this->assertInstanceOf(TextContent::class, $result->content[0]); + $this->assertSame('unsupported', $result->content[0]->text); + } + + #[TestDox('a client advertising elicitation without a handler fails the tool call')] + public function testAdvertisedCapabilityWithoutHandler(): void + { + // The client answers "method not found", which the gateway raises inside + // the tool as a ClientException rather than leaving it waiting. + $client = $this->connect( + 'elicitation', + $this->clientBuilder()->setCapabilities(new ClientCapabilities(elicitation: true)), + ); + + $result = $client->callTool('ask_name'); + + $this->assertInstanceOf(TextContent::class, $result->content[0]); + $this->assertSame('Client does not handle "elicitation/create" requests.', $result->content[0]->text); + } + + private function clientAnswering(ElicitResult $answer): ClientBuilder + { + $callback = new class($answer) implements ElicitationCallbackInterface { + public function __construct(private readonly ElicitResult $answer) + { + } + + public function __invoke(ElicitRequest $request): ElicitResult + { + return $this->answer; + } + }; + + return $this->clientBuilder() + ->setCapabilities(new ClientCapabilities(elicitation: true)) + ->addRequestHandler(new ElicitationRequestHandler($callback)); + } +} diff --git a/tests/Integration/Fixture/elicitation.php b/tests/Integration/Fixture/elicitation.php new file mode 100644 index 00000000..ad588a94 --- /dev/null +++ b/tests/Integration/Fixture/elicitation.php @@ -0,0 +1,49 @@ +setServerInfo('integration-server', '1.0.0') + ->addTool( + static function (RequestContext $context): string { + $gateway = $context->getClientGateway(); + + if (!$gateway->supportsElicitation()) { + return 'unsupported'; + } + + try { + $result = $gateway->elicit('What is your name?', new ElicitationSchema([ + 'name' => new StringSchemaDefinition(title: 'Name'), + ])); + } catch (ClientException $e) { + return $e->getMessage(); + } + + return sprintf('%s:%s', $result->action->value, $result->content['name'] ?? ''); + }, + name: 'ask_name', + description: 'Asks the client for a name.', + ) + ->build() + ->run(new StdioTransport()); diff --git a/tests/Integration/Fixture/handshake.php b/tests/Integration/Fixture/handshake.php new file mode 100644 index 00000000..776f4513 --- /dev/null +++ b/tests/Integration/Fixture/handshake.php @@ -0,0 +1,32 @@ +setServerInfo('integration-server', '1.0.0') + ->setInstructions('Be brief.'); + +if (is_string($pinned = getenv('MCP_INTEGRATION_PROTOCOL_VERSION')) && '' !== $pinned) { + $builder->setProtocolVersion(ProtocolVersion::from($pinned)); +} + +$builder->build()->run(new StdioTransport()); diff --git a/tests/Integration/Fixture/notification.php b/tests/Integration/Fixture/notification.php new file mode 100644 index 00000000..166e84c6 --- /dev/null +++ b/tests/Integration/Fixture/notification.php @@ -0,0 +1,39 @@ +setServerInfo('integration-server', '1.0.0') + ->addTool( + static function (RequestContext $context): string { + $gateway = $context->getClientGateway(); + + $gateway->log(LoggingLevel::Info, 'starting work'); + $gateway->progress(0.5, 1.0, 'halfway'); + $gateway->progress(1.0, 1.0, 'done'); + + return 'finished'; + }, + name: 'work', + description: 'Reports progress and logs while working.', + ) + ->build() + ->run(new StdioTransport()); diff --git a/tests/Integration/Fixture/roots.php b/tests/Integration/Fixture/roots.php new file mode 100644 index 00000000..a4b63824 --- /dev/null +++ b/tests/Integration/Fixture/roots.php @@ -0,0 +1,43 @@ +setServerInfo('integration-server', '1.0.0') + ->addTool( + static function (RequestContext $context): string { + $gateway = $context->getClientGateway(); + + if (!$gateway->supportsRoots()) { + return 'unsupported'; + } + + $described = []; + foreach ($gateway->listRoots()->roots as $root) { + $described[] = sprintf('%s (%s)', $root->uri, $root->name ?? '-'); + } + + return implode(', ', $described); + }, + name: 'inspect_roots', + description: 'Reports the workspace roots the client exposes.', + ) + ->build() + ->run(new StdioTransport()); diff --git a/tests/Integration/Fixture/sampling.php b/tests/Integration/Fixture/sampling.php new file mode 100644 index 00000000..bb37bbfa --- /dev/null +++ b/tests/Integration/Fixture/sampling.php @@ -0,0 +1,42 @@ +setServerInfo('integration-server', '1.0.0') + ->addTool( + static function (RequestContext $context, string $text): string { + try { + $result = $context->getClientGateway()->sample($text, maxTokens: 64); + } catch (ClientException $e) { + return $e->getMessage(); + } + + assert($result->content instanceof TextContent); + + return sprintf('%s said: %s', $result->model, $result->content->text); + }, + name: 'summarize', + description: 'Summarizes text by asking the client to sample.', + ) + ->build() + ->run(new StdioTransport()); diff --git a/tests/Integration/HandshakeTest.php b/tests/Integration/HandshakeTest.php new file mode 100644 index 00000000..5e0455ff --- /dev/null +++ b/tests/Integration/HandshakeTest.php @@ -0,0 +1,85 @@ +clientBuilder(); + if (null !== $clientVersion) { + $client->setProtocolVersion($clientVersion); + } + + $connected = $this->connect( + 'handshake', + $client, + null !== $serverVersion ? ['MCP_INTEGRATION_PROTOCOL_VERSION' => $serverVersion->value] : [], + ); + + $this->assertSame($expected, $connected->getProtocolVersion()); + } + + /** + * @return iterable + */ + public static function provideNegotiations(): iterable + { + $latest = ProtocolVersion::latestHandshake(); + + yield 'both unconfigured' => [null, null, $latest]; + + // Whichever end of the supported range it sits at. + foreach (ProtocolVersion::handshakeVersions() as $version) { + yield \sprintf('client asks for %s', $version->value) => [$version, null, $version]; + } + + // A pinned server answers with its pin, and the client continues on it. + yield 'server pins an older revision' => [ProtocolVersion::V2025_11_25, ProtocolVersion::V2025_03_26, ProtocolVersion::V2025_03_26]; + yield 'server pins a newer revision' => [ProtocolVersion::V2024_11_05, ProtocolVersion::V2025_11_25, ProtocolVersion::V2025_11_25]; + yield 'both pin the same revision' => [ProtocolVersion::V2025_06_18, ProtocolVersion::V2025_06_18, ProtocolVersion::V2025_06_18]; + + // Neither side reaches the modern era through `initialize`, so + // configuring it falls back to the handshake set on both ends. + yield 'client configured modern' => [ProtocolVersion::V2026_07_28, null, $latest]; + yield 'server configured modern' => [ProtocolVersion::V2025_06_18, ProtocolVersion::V2026_07_28, ProtocolVersion::V2025_06_18]; + yield 'both configured modern' => [ProtocolVersion::V2026_07_28, ProtocolVersion::V2026_07_28, $latest]; + } + + #[TestDox('the handshake carries the server identity to the client')] + public function testServerInfoIsExchanged(): void + { + $client = $this->connect('handshake'); + + $this->assertSame('integration-server', $client->getServerInfo()->name); + $this->assertSame('1.0.0', $client->getServerInfo()->version); + $this->assertSame('Be brief.', $client->getInstructions()); + $this->assertTrue($client->isConnected()); + } + + #[TestDox('the negotiated revision is unset before the handshake')] + public function testProtocolVersionIsNullBeforeConnecting(): void + { + $this->assertNull($this->clientBuilder()->build()->getProtocolVersion()); + } +} diff --git a/tests/Integration/IntegrationTestCase.php b/tests/Integration/IntegrationTestCase.php new file mode 100644 index 00000000..aa0dd9c0 --- /dev/null +++ b/tests/Integration/IntegrationTestCase.php @@ -0,0 +1,83 @@ + + */ +abstract class IntegrationTestCase extends TestCase +{ + /** + * Both sides answer immediately, so anything reaching this is a deadlock. + * Far below the SDK's two-minute default, to fail rather than hang. + */ + private const TIMEOUT = 5; + + private ?Client $client = null; + + protected function clientBuilder(): ClientBuilder + { + return Client::builder() + ->setClientInfo('integration-client', '1.0.0') + ->setInitTimeout(self::TIMEOUT) + ->setRequestTimeout(self::TIMEOUT); + } + + /** + * Spawn a fixture server and connect a client to it. + * + * The returned client has completed the handshake. + * + * @param string $fixture basename of a script in {@see Fixture} + * @param array $env added to the server process environment + */ + protected function connect(string $fixture, ?ClientBuilder $client = null, array $env = []): Client + { + $script = __DIR__.'/Fixture/'.$fixture.'.php'; + + $this->client = ($client ?? $this->clientBuilder())->build(); + + try { + $this->client->connect(new StdioTransport( + command: \PHP_BINARY, + args: [$script], + // proc_open() replaces the environment rather than adding to it. + env: [] === $env ? null : array_merge(getenv(), $env), + )); + } catch (ConnectionException $e) { + // The transport discards the child's stderr, so a fixture dying on + // startup arrives here as a bare timeout. + $this->fail(\sprintf('Could not connect to fixture server "%s": %s. Run `%s %s` to see why.', $fixture, $e->getMessage(), \PHP_BINARY, $script)); + } + + return $this->client; + } + + protected function tearDown(): void + { + $this->client?->disconnect(); + $this->client = null; + } +} diff --git a/tests/Integration/NotificationTest.php b/tests/Integration/NotificationTest.php new file mode 100644 index 00000000..09487185 --- /dev/null +++ b/tests/Integration/NotificationTest.php @@ -0,0 +1,75 @@ +connect('notification'); + + $updates = []; + $result = $client->callTool('work', [], static function (float $progress, ?float $total, ?string $message) use (&$updates): void { + $updates[] = [$progress, $total, $message]; + }); + + $this->assertSame([[0.5, 1.0, 'halfway'], [1.0, 1.0, 'done']], $updates); + $this->assertInstanceOf(TextContent::class, $result->content[0]); + $this->assertSame('finished', $result->content[0]->text); + } + + #[TestDox('progress is skipped when the caller asked for none')] + public function testProgressIsSkippedWithoutAToken(): void + { + // Without an onProgress callback the request carries no progress token, + // so the gateway drops the notification instead of sending it. + $client = $this->connect('notification'); + + $result = $client->callTool('work'); + + $this->assertInstanceOf(TextContent::class, $result->content[0]); + $this->assertSame('finished', $result->content[0]->text); + } + + #[TestDox('log notifications reach a registered logging handler')] + public function testLoggingReachesTheClient(): void + { + $logged = []; + $client = $this->connect( + 'notification', + $this->clientBuilder()->addNotificationHandler(new LoggingNotificationHandler( + static function (LoggingMessageNotification $notification) use (&$logged): void { + $logged[] = [$notification->level, $notification->data]; + }, + )), + ); + + $client->callTool('work'); + + $this->assertSame([[LoggingLevel::Info, 'starting work']], $logged); + } +} diff --git a/tests/Integration/RootsTest.php b/tests/Integration/RootsTest.php new file mode 100644 index 00000000..6dfd0650 --- /dev/null +++ b/tests/Integration/RootsTest.php @@ -0,0 +1,98 @@ +connect('roots', $this->clientExposing( + new Root('file:///workspace/app', 'App'), + new Root('file:///workspace/docs', 'Docs'), + )); + + $result = $client->callTool('inspect_roots'); + + $this->assertInstanceOf(TextContent::class, $result->content[0]); + $this->assertSame('file:///workspace/app (App), file:///workspace/docs (Docs)', $result->content[0]->text); + } + + #[TestDox('an empty root list is a valid answer, not a failure')] + public function testEmptyRootList(): void + { + $client = $this->connect('roots', $this->clientExposing()); + + $result = $client->callTool('inspect_roots'); + + $this->assertInstanceOf(TextContent::class, $result->content[0]); + $this->assertSame('', $result->content[0]->text); + } + + #[TestDox('a client that does not advertise roots is not asked')] + public function testCapabilityIsVisibleToTheServer(): void + { + $client = $this->connect('roots'); + + $result = $client->callTool('inspect_roots'); + + $this->assertInstanceOf(TextContent::class, $result->content[0]); + $this->assertSame('unsupported', $result->content[0]->text); + } + + #[TestDox('the client can announce that its roots changed')] + public function testRootsListChangedNotification(): void + { + $client = $this->connect('roots', $this->clientExposing(new Root('file:///workspace'))); + + // A notification has no reply, so what this pins down is that sending + // one mid-session leaves the connection usable. + $client->sendRootsListChanged(); + + $this->assertTrue($client->isConnected()); + $this->assertInstanceOf(TextContent::class, $client->callTool('inspect_roots')->content[0]); + } + + private function clientExposing(Root ...$roots): ClientBuilder + { + $callback = new class(array_values($roots)) implements RootsCallbackInterface { + /** @param list $roots */ + public function __construct(private readonly array $roots) + { + } + + public function __invoke(ListRootsRequest $request): ListRootsResult + { + return new ListRootsResult($this->roots); + } + }; + + return $this->clientBuilder() + ->setCapabilities(new ClientCapabilities(roots: true, rootsListChanged: true)) + ->addRequestHandler(new ListRootsRequestHandler($callback)); + } +} diff --git a/tests/Integration/SamplingTest.php b/tests/Integration/SamplingTest.php new file mode 100644 index 00000000..c581cf86 --- /dev/null +++ b/tests/Integration/SamplingTest.php @@ -0,0 +1,98 @@ +connect('sampling', $this->clientSampling()); + + $result = $client->callTool('summarize', ['text' => 'a long report']); + + $this->assertInstanceOf(TextContent::class, $result->content[0]); + $this->assertSame('test-model said: a long report', $result->content[0]->text); + } + + #[TestDox('the prompt the tool passed arrives at the client')] + public function testPromptReachesTheClient(): void + { + // The client stays in this process, so what it was asked can be + // collected by reference even though the tool asking runs in another. + /** @var \ArrayObject $seen */ + $seen = new \ArrayObject(); + $client = $this->connect('sampling', $this->clientSampling($seen)); + + $client->callTool('summarize', ['text' => 'inspect me']); + + $this->assertCount(1, $seen); + $this->assertInstanceOf(TextContent::class, $seen[0]->messages[0]->content); + $this->assertSame('inspect me', $seen[0]->messages[0]->content->text); + $this->assertSame(64, $seen[0]->maxTokens); + } + + #[TestDox('a client that cannot sample refuses instead of stalling the tool')] + public function testClientWithoutSamplingRefuses(): void + { + // The gateway has no supportsSampling() to consult, so the tool finds + // out by asking and the refusal surfaces as a ClientException. + $client = $this->connect('sampling'); + + $result = $client->callTool('summarize', ['text' => 'anything']); + + $this->assertInstanceOf(TextContent::class, $result->content[0]); + $this->assertSame('Client does not handle "sampling/createMessage" requests.', $result->content[0]->text); + } + + /** + * @param \ArrayObject|null $seen collects what the server asked for + */ + private function clientSampling(?\ArrayObject $seen = null): ClientBuilder + { + $callback = new class($seen ?? new \ArrayObject()) implements SamplingCallbackInterface { + /** @param \ArrayObject $seen */ + public function __construct(private readonly \ArrayObject $seen) + { + } + + public function __invoke(CreateSamplingMessageRequest $request): CreateSamplingMessageResult + { + $this->seen[] = $request; + + $prompt = $request->messages[0]->content; + \assert($prompt instanceof TextContent); + + return new CreateSamplingMessageResult(Role::Assistant, new TextContent($prompt->text), 'test-model'); + } + }; + + return $this->clientBuilder() + ->setCapabilities(new ClientCapabilities(sampling: true)) + ->addRequestHandler(new SamplingRequestHandler($callback)); + } +}