Adds a comprehensive conformance test suite for the Kotlin MCP SDK, covering core protocol operations, tool calls, elicitation, resources, prompts, and 20 OAuth/auth scenarios - Conformance server and client implementations - OAuth/auth test scenarios: JWT, authorization code flow, client credentials, PKCE, scope handling, cross-app access, client registration - CI workflow - Baseline file for tracking expected failures - Shell script fixes: - #592 - #593 - #596 ## Remaining known failures (tracked issues, will be fixed directly in `main`) - [x] `tools-call-with-logging`, `tools-call-with-progress`, `tools-call-sampling`, `tools-call-elicitation`, `elicitation-sep1034-defaults`- see #599, - [x] `elicitation-sep1330-enums` - #587 #600 - [x] `initialize` - #588 - [x] `tools_call`, `auth/scope-step-up`, `auth/scope-retry-limit` - #589 - [ ] `elicitation-sep1034-client-defaults` - #414 - [x] `sse-retry` - #590 - [ ] `resources-templates-read` - #591 ## Breaking Changes from #596 - `StreamableHttpClientTransport` and `mcpStreamableHttp`/`mcpStreamableHttpTransport`: old constructors accepting `Duration` timeout are now `@Deprecated` — use the new overloads with `ReconnectionOptions` instead - `StreamableHttpClientTransport.close()` no longer calls `terminateSession()` automatically ## Types of changes - [x] Bug fix (non-breaking change which fixes an issue) - [x] New feature (non-breaking change which adds functionality) - [x] Breaking change (fix or feature that would cause existing functionality to change) - [ ] Documentation update ## Checklist - [x] I have read the [MCP Documentation](https://modelcontextprotocol.io) - [x] My code follows the repository's style guidelines - [x] New and existing tests pass locally - [x] I have added appropriate error handling - [x] I have added or updated documentation as needed --------- Co-authored-by: Konstantin Pavlov <1517853+kpavlov@users.noreply.github.com>
This commit is contained in:
Pavel Gorgulov
GitHub
parent
338f94804f
commit
e88b34170c
1 changed files with 3186 additions and 1122 deletions
185
.github/workflows/conformance.yml
vendored
185
.github/workflows/conformance.yml
vendored
|
|
@ -7,67 +7,166 @@ on:
|
|||
push:
|
||||
branches: [ main ]
|
||||
|
||||
env:
|
||||
JAVA_VERSION: '21'
|
||||
JAVA_DISTRIBUTION: temurin
|
||||
NODE_VERSION: '22'
|
||||
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
# Cancel only when the run is NOT on `main` branch
|
||||
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
|
||||
|
||||
jobs:
|
||||
run-conformance:
|
||||
runs-on: ${{ matrix.os }}
|
||||
name: Run Conformance Tests on ${{ matrix.os }}
|
||||
server:
|
||||
runs-on: ubuntu-latest
|
||||
name: Conformance Server Tests
|
||||
timeout-minutes: 20
|
||||
env:
|
||||
JAVA_OPTS: "-Xmx8g -Dfile.encoding=UTF-8 -Djava.awt.headless=true -Dkotlin.daemon.jvm.options=-Xmx6g"
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- os: ubuntu-latest
|
||||
max-workers: 3
|
||||
- os: windows-latest
|
||||
max-workers: 3
|
||||
- os: macos-latest
|
||||
max-workers: 2
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Set up JDK 21
|
||||
- name: Set up JDK
|
||||
uses: actions/setup-java@v5
|
||||
with:
|
||||
java-version: '21'
|
||||
distribution: 'temurin'
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: '22' # increase only after https://github.com/nodejs/node/issues/56645 will be fixed
|
||||
|
||||
- name: Setup Conformance Tests
|
||||
working-directory: conformance-test
|
||||
run: |-
|
||||
npm install -g @modelcontextprotocol/conformance@0.1.8
|
||||
java-version: ${{ env.JAVA_VERSION }}
|
||||
distribution: ${{ env.JAVA_DISTRIBUTION }}
|
||||
|
||||
- name: Setup Gradle
|
||||
uses: gradle/actions/setup-gradle@v5
|
||||
with:
|
||||
add-job-summary: 'always'
|
||||
cache-read-only: ${{ github.ref != 'refs/heads/main' }}
|
||||
gradle-home-cache-includes: |
|
||||
caches
|
||||
notifications
|
||||
sdks
|
||||
../.konan/**
|
||||
|
||||
- name: Run Conformance Tests
|
||||
run: |-
|
||||
./gradlew :conformance-test:test --no-daemon --max-workers ${{ matrix.max-workers }}
|
||||
- name: Build
|
||||
run: ./gradlew :conformance-test:installDist
|
||||
|
||||
- name: Upload Conformance Results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
- name: Start server
|
||||
run: |
|
||||
MCP_PORT=3001 conformance-test/build/install/conformance-test/bin/conformance-test &
|
||||
for i in $(seq 1 30); do
|
||||
if curl -s -o /dev/null http://localhost:3001/mcp; then
|
||||
echo "Server is ready"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
- name: Run conformance tests
|
||||
uses: modelcontextprotocol/conformance@v0.1.15
|
||||
with:
|
||||
name: conformance-results-${{ matrix.os }}
|
||||
path: conformance-test/results/
|
||||
mode: server
|
||||
url: http://localhost:3001/mcp
|
||||
suite: active
|
||||
node-version: ${{ env.NODE_VERSION }}
|
||||
expected-failures: ./conformance-test/conformance-baseline.yml
|
||||
|
||||
client:
|
||||
runs-on: ubuntu-latest
|
||||
name: "Conformance Client Tests: ${{ matrix.scenario }}"
|
||||
timeout-minutes: 20
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
scenario:
|
||||
- initialize
|
||||
- tools_call
|
||||
- elicitation-sep1034-client-defaults
|
||||
- sse-retry
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Set up JDK
|
||||
uses: actions/setup-java@v5
|
||||
with:
|
||||
java-version: ${{ env.JAVA_VERSION }}
|
||||
distribution: ${{ env.JAVA_DISTRIBUTION }}
|
||||
|
||||
- name: Setup Gradle
|
||||
uses: gradle/actions/setup-gradle@v5
|
||||
with:
|
||||
cache-read-only: ${{ github.ref != 'refs/heads/main' }}
|
||||
|
||||
- name: Build
|
||||
run: ./gradlew :conformance-test:installDist
|
||||
|
||||
- name: Run conformance tests
|
||||
uses: modelcontextprotocol/conformance@v0.1.15
|
||||
with:
|
||||
mode: client
|
||||
command: conformance-test/build/install/conformance-test/bin/conformance-client
|
||||
scenario: ${{ matrix.scenario }}
|
||||
node-version: ${{ env.NODE_VERSION }}
|
||||
expected-failures: ./conformance-test/conformance-baseline.yml
|
||||
|
||||
auth:
|
||||
runs-on: ubuntu-latest
|
||||
name: Conformance Auth Tests
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Set up JDK
|
||||
uses: actions/setup-java@v5
|
||||
with:
|
||||
java-version: ${{ env.JAVA_VERSION }}
|
||||
distribution: ${{ env.JAVA_DISTRIBUTION }}
|
||||
|
||||
- name: Setup Gradle
|
||||
uses: gradle/actions/setup-gradle@v5
|
||||
with:
|
||||
cache-read-only: ${{ github.ref != 'refs/heads/main' }}
|
||||
|
||||
- name: Build
|
||||
run: ./gradlew :conformance-test:installDist
|
||||
|
||||
- name: Run conformance tests
|
||||
uses: modelcontextprotocol/conformance@v0.1.15
|
||||
with:
|
||||
mode: client
|
||||
command: conformance-test/build/install/conformance-test/bin/conformance-client
|
||||
suite: auth
|
||||
node-version: ${{ env.NODE_VERSION }}
|
||||
expected-failures: ./conformance-test/conformance-baseline.yml
|
||||
|
||||
auth-scenarios:
|
||||
runs-on: ubuntu-latest
|
||||
name: "Conformance Auth Scenario: ${{ matrix.scenario }}"
|
||||
timeout-minutes: 20
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
scenario:
|
||||
- auth/client-credentials-jwt
|
||||
- auth/client-credentials-basic
|
||||
- auth/cross-app-access-complete-flow
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Set up JDK
|
||||
uses: actions/setup-java@v5
|
||||
with:
|
||||
java-version: ${{ env.JAVA_VERSION }}
|
||||
distribution: ${{ env.JAVA_DISTRIBUTION }}
|
||||
|
||||
- name: Setup Gradle
|
||||
uses: gradle/actions/setup-gradle@v5
|
||||
with:
|
||||
cache-read-only: ${{ github.ref != 'refs/heads/main' }}
|
||||
|
||||
- name: Build
|
||||
run: ./gradlew :conformance-test:installDist
|
||||
|
||||
- name: Run conformance tests
|
||||
uses: modelcontextprotocol/conformance@v0.1.15
|
||||
with:
|
||||
mode: client
|
||||
command: conformance-test/build/install/conformance-test/bin/conformance-client
|
||||
scenario: ${{ matrix.scenario }}
|
||||
node-version: ${{ env.NODE_VERSION }}
|
||||
expected-failures: ./conformance-test/conformance-baseline.yml
|
||||
|
|
|
|||
Loading…
Reference in a new issue