diff --git a/.dockerignore b/.dockerignore index 8953253..43f469d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,3 +2,6 @@ target/ assets/ resources/ releases/ +.conformance-suite/ +conformance-logs/ +conformance-results/ diff --git a/.github/workflows/mcp_conformance.yml b/.github/workflows/mcp_conformance.yml new file mode 100644 index 0000000..1a47c6b --- /dev/null +++ b/.github/workflows/mcp_conformance.yml @@ -0,0 +1,116 @@ +name: MCP Conformance + +on: + workflow_dispatch: + pull_request: + branches: [main] + push: + branches: [main] + +permissions: + contents: read + +concurrency: + group: mcp-conformance-${{ github.ref }} + cancel-in-progress: true + +env: + MCP_CONFORMANCE_VERSION: 0.2.0-alpha.11 + MCP_CONFORMANCE_SOURCE_SHA: c321dd32035556e6769d3724a8ee97d87c3faaac # pragma: allowlist secret + MCP_CONFORMANCE_SPEC_VERSION: 2026-07-28 + MCP_CONFORMANCE_SERVER_ID: 3f33286667d34b65a31c3bafd30e4c21 + CF_CONTROLPLANE_IMAGE: ghcr.io/ibm/mcp-context-forge:latest + CF_DATAPLANE_IMAGE: contextforge-data-plane:conformance + +jobs: + conformance: + name: MCP 2026-07-28 through nginx and dataplane + runs-on: ubuntu-latest + timeout-minutes: 30 + + steps: + - name: Check out data plane + uses: actions/checkout@v6.0.2 + + - name: Build data plane from the checked-out source + run: make conformance-image + + - name: Check out the alpha.11 conformance fixture + uses: actions/checkout@v6.0.2 + with: + repository: modelcontextprotocol/conformance + ref: ${{ env.MCP_CONFORMANCE_SOURCE_SHA }} + path: .conformance-suite + persist-credentials: false + + - name: Install Node.js 22 + uses: actions/setup-node@v6.5.0 + with: + node-version: 22 + cache: npm + cache-dependency-path: .conformance-suite/package-lock.json + + - name: Install official conformance dependencies + working-directory: .conformance-suite + run: | + test "$(node -p "require('./package.json').version")" = "${MCP_CONFORMANCE_VERSION}" + npm ci --ignore-scripts + + - name: Pull external stack images + env: + MCP_CONFORMANCE_TOKEN: pull-only + run: >- + docker compose -f tests/conformance/docker-compose.yml + pull redis control-plane nginx + + - name: Start the fixture and control plane + env: + MCP_CONFORMANCE_TOKEN: bootstrap-only + run: tests/conformance/start-fixture-and-control-plane.sh + + - name: Register the official fixture through the control plane + env: + MCP_CONFORMANCE_TOKEN: bootstrap-only + run: tests/conformance/register-fixture.sh + + - name: Start dataplane and nginx + run: tests/conformance/start-dataplane-and-nginx.sh + + - name: Run MCP 2026-07-28 requirements + id: runner + run: tests/conformance/run-conformance.sh + + - name: Print conformance failures and warnings + if: always() + run: | + if [ ! -d conformance-results ]; then + echo "No conformance results were produced." + exit 0 + fi + find conformance-results -type f -name checks.json -print0 | + while IFS= read -r -d '' checks_file; do + jq --arg file "${checks_file}" \ + '[.[] | select(.status == "FAILURE" or .status == "WARNING")] | + if length > 0 then {file: $file, findings: .} else empty end' \ + "${checks_file}" + done + + - name: Print live stack logs + if: always() + env: + MCP_CONFORMANCE_TOKEN: diagnostics-only + run: >- + docker compose -f tests/conformance/docker-compose.yml + logs --no-color || true + + - name: Stop the live stack + if: always() + env: + MCP_CONFORMANCE_TOKEN: cleanup-only + run: tests/conformance/stop-live-stack.sh + + - name: Enforce conformance baseline + if: always() + env: + RUNNER_STATUS: ${{ steps.runner.outputs.status }} + run: test "${RUNNER_STATUS}" = "0" diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 5205758..8227358 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -24,13 +24,13 @@ jobs: steps: - uses: actions/checkout@v6.0.2 - name: Install mdBook - uses: taiki-e/install-action@v2.75.27 + uses: taiki-e/install-action@v2.85.11 with: tool: mdbook@0.5.3 - name: Build docs run: mdbook build _context/wiki - name: Upload Pages artifact - uses: actions/upload-pages-artifact@v4 + uses: actions/upload-pages-artifact@v5.0.0 with: path: _context/wiki/book @@ -51,4 +51,4 @@ jobs: steps: - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@v4 + uses: actions/deploy-pages@v5.0.0 diff --git a/.gitignore b/.gitignore index 5efd0ef..5c8c16a 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,10 @@ target # Generated by gateway local runs contextforge-data-plane.log.* +# Generated by local MCP conformance runs +.conformance-suite/ +conformance-logs/ +conformance-results/ # Generated by mdBook (wiki) _context/wiki/book/ diff --git a/Cargo.lock b/Cargo.lock index 6abd30c..28ebb8e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1018,7 +1018,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2266,7 +2266,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2518,9 +2518,9 @@ dependencies = [ [[package]] name = "rmcp" -version = "3.1.1" +version = "3.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "094c075f6698deef5a657cf4df6b684dff65157d255978b92b552ec22503f17a" +checksum = "c8dddc5b1924b9a59fba420166160ca2c4663a4e01803e52eda33070f56d63c8" dependencies = [ "async-trait", "base64 0.23.1", @@ -2552,9 +2552,9 @@ dependencies = [ [[package]] name = "rmcp-macros" -version = "3.1.1" +version = "3.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "737d947bcfd946fae6a179a4ef6487be6dcf25c930c2393856b820f1386e52a6" +checksum = "6898e24cd16342b59bfa8a53c2c04b9cf62fc8a2cfea57b9c038b09984bfc521" dependencies = [ "darling", "proc-macro2", @@ -2653,7 +2653,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3886,7 +3886,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index eacf3e9..a7841d8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,7 @@ repository = "https://github.com/contextforge-org/contextforge-data-plane" # Keep dependencies here only when at least two workspace members inherit them. contextforge-data-plane-cpex = { path = "./crates/contextforge-data-plane-cpex" } contextforge-data-plane-apis = { path = "./crates/contextforge-data-plane-apis"} -rmcp = { version = "3.1.1", features = [ +rmcp = { version = "3.1.2", features = [ "server", "client", "auth", diff --git a/Makefile b/Makefile index cdc0725..cf44396 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,5 @@ IMAGE_NAME := contextforge-data-plane:latest +CF_DATAPLANE_IMAGE ?= contextforge-data-plane:conformance SERVICES ?= nginx control-plane redis postgres pgbouncer data-plane fast_time_server register_fast_time ARGS ?= @@ -6,7 +7,7 @@ ARGS ?= DETECT_SECRETS_SPEC ?= git+https://github.com/ibm/detect-secrets.git@076672a9a01abdfc7ecee2e7d14f08cdccb73976 DETECT_SECRETS_EXCLUDE := '(?x)(Cargo\.lock$$|\.lock$$)|^\.secrets\.baseline$$' -.PHONY: help docker-prod compose-up compose-down docs-serve pre-commit secrets-scan-all configure-git +.PHONY: help docker-prod compose-up compose-down conformance-image conformance docs-serve pre-commit secrets-scan-all configure-git help: ## Show available commands @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-22s\033[0m %s\n", $$1, $$2}' @@ -24,6 +25,12 @@ compose-up: ## Launch stack: nginx, control plane, redis, postgres, pgbouncer, d compose-down: ## Tear down the stack docker compose -f docker/docker-compose.yml stop $(SERVICES) $(ARGS) +conformance-image: ## Build the current checkout for MCP conformance + docker build -t "$(CF_DATAPLANE_IMAGE)" -f docker/Dockerfile . + +conformance: conformance-image ## Build and run official MCP 2026-07-28 conformance locally + CF_DATAPLANE_IMAGE="$(CF_DATAPLANE_IMAGE)" tests/conformance/run-local.sh + docs-serve: ## Serve the wiki book locally at http://127.0.0.1:3000 mdbook serve _context/wiki --hostname 127.0.0.1 --port 3000 --open diff --git a/_context/wiki/testing.md b/_context/wiki/testing.md index 8d16a43..c713505 100644 --- a/_context/wiki/testing.md +++ b/_context/wiki/testing.md @@ -34,6 +34,21 @@ Protocol tests and fixtures should target MCP `2026-07-28`, use `server/discover These run in `cargo nextest run` with no Docker dependencies. +## MCP Conformance CI + +`.github/workflows/mcp_conformance.yml` runs the pinned official conformance +suite `0.2.0-alpha.11` with `--requirements 2026-07-28`. Its small live path is +official runner → nginx → published `latest` dataplane → official fixture, +with the published `latest` control plane registering and publishing the +fixture through Redis. The control plane uses ephemeral SQLite, so PostgreSQL +is unnecessary. The harness lives in `tests/conformance/`. + +Because this conformance CLI cannot set a bearer header, nginx adds an +ephemeral control-plane token when one is absent; there is no auth proxy or +repository-owned JavaScript. A route probe prevents control-plane fallback. +Counts appear directly in the Actions log, and `expected-failures.yml` guards +the current baseline. The job does not retain a separate conformance artifact. + ## Full-Stack Integration Harness [`cf-integration`](https://github.com/contextforge-org/contextforge-dev-tools) wires the external ContextForge control plane to this dataplane the way production intends: the stock upstream Compose stack, plus exactly two intentional differences — nginx routes only `/servers/{virtual_host_id}/mcp` to the dataplane (as `/contextforge-rs/servers/{virtual_host_id}/mcp`), and the control plane runs with `DATAPLANE_PUBLISHER=true` so virtual server configs reach the dataplane through Redis. diff --git a/tests/conformance/docker-compose.yml b/tests/conformance/docker-compose.yml new file mode 100644 index 0000000..f941f3c --- /dev/null +++ b/tests/conformance/docker-compose.yml @@ -0,0 +1,113 @@ +name: contextforge-conformance + +services: + redis: + image: redis:8.8.1-alpine3.23 + command: [redis-server, --save, "", --appendonly, "no"] + networks: [contextforge] + healthcheck: + test: [CMD, redis-cli, ping] + interval: 2s + timeout: 2s + retries: 30 + + control-plane: + image: ${CF_CONTROLPLANE_IMAGE:-ghcr.io/ibm/mcp-context-forge:latest} + ports: + - "127.0.0.1:4444:4444" + networks: [contextforge] + extra_hosts: + - host.docker.internal:host-gateway + environment: + HOST: 0.0.0.0 + PORT: "4444" + DATABASE_URL: sqlite:////tmp/contextforge-conformance.db + CACHE_TYPE: redis + REDIS_URL: redis://redis:6379/0 + JWT_SECRET_KEY: contextforge-conformance-jwt-secret-at-least-32-bytes + JWT_ALGORITHM: RS256 + JWT_PUBLIC_KEY_PATH: /keys/jwt.key.pub + JWT_PRIVATE_KEY_PATH: /keys/jwt.key + AUTH_ENCRYPTION_SECRET: contextforge-conformance-encryption-secret-at-least-32-bytes # pragma: allowlist secret + AUTH_REQUIRED: "true" + BASIC_AUTH_USER: admin + BASIC_AUTH_PASSWORD: conformance-admin-password # pragma: allowlist secret + PLATFORM_ADMIN_EMAIL: admin@example.com + PLATFORM_ADMIN_PASSWORD: conformance-admin-password # pragma: allowlist secret + MCPGATEWAY_SKIP_MIGRATIONS: "false" + MCPGATEWAY_UI_ENABLED: "false" + MCPGATEWAY_ADMIN_API_ENABLED: "true" + SECURE_COOKIES: "false" + SSRF_ALLOW_PRIVATE_NETWORKS: "true" + DATAPLANE_PUBLISHER: "true" + DATAPLANE_PUBLISHER_INTERVAL_SECONDS: "2" + PASSWORD_CHANGE_ENFORCEMENT_ENABLED: "false" + ADMIN_REQUIRE_PASSWORD_CHANGE_ON_BOOTSTRAP: "false" + REQUIRE_PASSWORD_CHANGE_FOR_DEFAULT_PASSWORD: "false" + GATEWAY_TOOL_NAME_SEPARATOR: _ + GUNICORN_WORKERS: "1" + LOG_LEVEL: INFO + volumes: + - ../../assets:/keys:ro + depends_on: + redis: + condition: service_healthy + healthcheck: + test: [CMD, curl, --fail, http://127.0.0.1:4444/health] + interval: 5s + timeout: 5s + retries: 60 + start_period: 10s + + data-plane: + image: ${CF_DATAPLANE_IMAGE:-ghcr.io/contextforge-org/contextforge-data-plane:latest} + networks: [contextforge] + extra_hosts: + - host.docker.internal:host-gateway + environment: + CONTEXTFORGE_DATA_PLANE_ADDRESS: 0.0.0.0:4445 + CONTEXTFORGE_DATA_PLANE_REDIS_HOSTNAME: redis + CONTEXTFORGE_DATA_PLANE_REDIS_PORT: "6379" + CONTEXTFORGE_DATA_PLANE_REDIS_CONNECTION_MODE: plain-text + CONTEXTFORGE_DATA_PLANE_UPSTREAM_CONNECTION_MODE: plain-text-or-tls + CONTEXTFORGE_DATA_PLANE_TOKEN_VERIFICATION_PUBLIC_KEY: /keys/jwt.key.pub + CONTEXTFORGE_DATA_PLANE_TOKEN_VERIFICATION_PRIVATE_KEY: /keys/jwt.key # pragma: allowlist secret + CONTEXTFORGE_DATA_PLANE_USER_CONFIG_CACHE_EXPIRY_SECONDS: "0" + CONTEXTFORGE_GATEWAY_RS_MCP_ALLOWED_HOSTS: >- + 127.0.0.1:${MCP_CONFORMANCE_PORT:-8080},localhost:${MCP_CONFORMANCE_PORT:-8080} + CONTEXTFORGE_GATEWAY_RS_MCP_ALLOWED_ORIGINS: >- + http://127.0.0.1:${MCP_CONFORMANCE_PORT:-8080},http://localhost:${MCP_CONFORMANCE_PORT:-8080} + RUST_LOG: info + volumes: + - ../../assets:/keys:ro + depends_on: + redis: + condition: service_healthy + + nginx: + image: nginx:1.30.4-alpine3.24 + ports: + - "127.0.0.1:${MCP_CONFORMANCE_PORT:-8080}:80" + networks: [contextforge] + environment: + MCP_CONFORMANCE_TOKEN: ${MCP_CONFORMANCE_TOKEN:?} + NGINX_ENVSUBST_TEMPLATE_DIR: /templates + NGINX_ENVSUBST_TEMPLATE_SUFFIX: .template + NGINX_ENVSUBST_OUTPUT_DIR: /etc/nginx + volumes: + - ./nginx.conf.template:/templates/nginx.conf.template:ro + depends_on: + control-plane: + condition: service_healthy + # Full readiness, routing, and auth are verified by start-dataplane-and-nginx.sh. + data-plane: + condition: service_started + healthcheck: + test: [CMD, curl, --fail, http://127.0.0.1/health] + interval: 2s + timeout: 2s + retries: 30 + +networks: + contextforge: + driver: bridge diff --git a/tests/conformance/expected-failures.yml b/tests/conformance/expected-failures.yml new file mode 100644 index 0000000..4f5007b --- /dev/null +++ b/tests/conformance/expected-failures.yml @@ -0,0 +1,45 @@ +server: + - server-stateless:sep-2575-server-declares-prompts-in-discover + - server-stateless:sep-2575-discover-capabilities-match-handlers + - server-stateless:sep-2575-server-rejects-undeclared-capability + - server-stateless:sep-2575-missing-capability-http-400 + - server-stateless:sep-2575-http-server-no-independent-requests-on-stream + - server-stateless:sep-2575-server-no-log-without-loglevel + - completion-complete:completion-complete + - tools-list:tools-list + - tools-call-simple-text:tools-call-simple-text + - tools-call-image:tools-call-image + - tools-call-audio:tools-call-audio + - tools-call-embedded-resource:tools-call-embedded-resource + - tools-call-mixed-content:tools-call-mixed-content + - tools-call-error:tools-call-error + - tools-call-with-progress:tools-call-with-progress + - resources-list:resources-list + - resources-read-text:resources-read-text + - resources-read-binary:resources-read-binary + - resources-templates-read:resources-templates-read + - sep-2164-resource-not-found:sep-2164-error-code + - sep-2164-resource-not-found:sep-2164-data-uri + - prompts-list:prompts-list + - prompts-get-simple:prompts-get-simple + - prompts-get-with-args:prompts-get-with-args + - prompts-get-embedded-resource:prompts-get-embedded-resource + - prompts-get-with-image:prompts-get-with-image + - caching:sep-2549-tools-list-caching-hints + - caching:sep-2549-prompts-list-caching-hints + - caching:sep-2549-resources-list-caching-hints + - caching:sep-2549-resources-templates-list-caching-hints + - caching:sep-2549-ttl-non-negative + - caching:sep-2549-cache-scope-valid + - input-required-result-basic-elicitation:sep-2322-elicitation-incomplete + - input-required-result-basic-sampling:sep-2322-sampling-incomplete + - input-required-result-basic-list-roots:sep-2322-list-roots-incomplete + - input-required-result-request-state:sep-2322-request-state-incomplete + - input-required-result-multiple-input-requests:sep-2322-multiple-inputs-incomplete + - input-required-result-multi-round:sep-2322-multi-round-r1 + - input-required-result-missing-input-response:sep-2322-missing-response-rerequests + - input-required-result-non-tool-request:sep-2322-non-tool-incomplete + - input-required-result-result-type:sep-2322-result-type-included + - input-required-result-tampered-state:sep-2322-reject-tampered-state + - input-required-result-capability-check:sep-2322-respect-client-capabilities + - input-required-result-ignore-extra-params:sep-2322-ignore-unexpected-params diff --git a/tests/conformance/nginx.conf.template b/tests/conformance/nginx.conf.template new file mode 100644 index 0000000..404e8d1 --- /dev/null +++ b/tests/conformance/nginx.conf.template @@ -0,0 +1,65 @@ +user nginx; +worker_processes auto; +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + access_log /var/log/nginx/access.log; + resolver 127.0.0.11 valid=5s ipv6=off; + + map $http_authorization $conformance_authorization { + default $http_authorization; + "" "Bearer ${MCP_CONFORMANCE_TOKEN}"; + } + + server { + listen 80 default_server; + server_name _; + + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header Authorization $conformance_authorization; + proxy_set_header Mcp-Session-Id $http_mcp_session_id; + proxy_set_header Mcp-Protocol-Version $http_mcp_protocol_version; + proxy_set_header Origin $http_origin; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $http_host; + proxy_set_header Connection ""; + + location ~ ^/servers/([^/]+)/mcp/?$ { + set $dataplane http://data-plane:4445; + proxy_pass $dataplane/contextforge-rs/servers/$1/mcp$is_args$args; + proxy_request_buffering on; + proxy_buffering off; + proxy_cache off; + proxy_connect_timeout 30s; + proxy_send_timeout 1h; + proxy_read_timeout 1h; + add_header X-Accel-Buffering "no" always; + add_header X-CF-Conformance-Backend dataplane always; + } + + location = /health { + set $control_plane http://control-plane:4444; + proxy_pass $control_plane/health; + proxy_connect_timeout 5s; + proxy_read_timeout 5s; + } + + location / { + set $control_plane http://control-plane:4444; + proxy_pass $control_plane; + proxy_buffering off; + proxy_connect_timeout 30s; + proxy_send_timeout 120s; + proxy_read_timeout 120s; + } + } +} diff --git a/tests/conformance/register-fixture.sh b/tests/conformance/register-fixture.sh new file mode 100755 index 0000000..ffae024 --- /dev/null +++ b/tests/conformance/register-fixture.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${GITHUB_ENV:?GITHUB_ENV must be set}" +: "${MCP_CONFORMANCE_SERVER_ID:?MCP_CONFORMANCE_SERVER_ID must be set}" + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +compose_file="${script_dir}/docker-compose.yml" + +bootstrap_token="$({ + docker compose -f "${compose_file}" exec -T control-plane \ + python3 -m mcpgateway.utils.create_jwt_token \ + --username admin@example.com --admin --exp 120 +} 2>/dev/null | tail -n 1)" +test -n "${bootstrap_token}" +if [ "${GITHUB_ACTIONS:-}" = "true" ]; then + echo "::add-mask::${bootstrap_token}" +fi + +api_request() { + local method="$1" + local path="$2" + local body="${3-}" + local args=( + --silent --show-error --fail-with-body + --request "${method}" + --header "Authorization: Bearer ${bootstrap_token}" + --header "Content-Type: application/json" + "http://127.0.0.1:4444${path}" + ) + if [ -n "${body}" ]; then + args+=(--data "${body}") + fi + curl "${args[@]}" +} + +gateway="$(api_request POST /gateways '{ + "name": "_", + "url": "http://host.docker.internal:3000/mcp", + "transport": "STREAMABLEHTTP", + "authType": "authheaders", + "authHeaders": [{"key": "Host", "value": "localhost:3000"}], + "description": "Official MCP alpha.11 conformance fixture" +}')" +gateway_id="$(jq --exit-status --raw-output '.id' <<< "${gateway}")" + +api_request POST \ + "/gateways/${gateway_id}/tools/refresh?include_resources=true&include_prompts=true" \ + '{}' > /dev/null + +tool_ids='[]' +resource_ids='[]' +prompt_ids='[]' +has_tool=0 +has_resource=0 +has_prompt=0 +for _ in $(seq 1 120); do + tools="$(api_request GET /tools)" + resources="$(api_request GET /resources)" + prompts="$(api_request GET /prompts)" + + tool_ids="$(jq --compact-output --arg id "${gateway_id}" \ + '[.[] | select((.gateway_id // .gatewayId) == $id) | .id]' <<< "${tools}")" + resource_ids="$(jq --compact-output --arg id "${gateway_id}" \ + '[.[] | select((.gateway_id // .gatewayId) == $id) | .id]' <<< "${resources}")" + prompt_ids="$(jq --compact-output --arg id "${gateway_id}" \ + '[.[] | select((.gateway_id // .gatewayId) == $id) | .id]' <<< "${prompts}")" + + has_tool="$(jq --arg id "${gateway_id}" \ + '[.[] | select((.gateway_id // .gatewayId) == $id and .name == "test_simple_text")] | length' \ + <<< "${tools}")" + has_resource="$(jq --arg id "${gateway_id}" \ + '[.[] | select((.gateway_id // .gatewayId) == $id and .uri == "test://static-text")] | length' \ + <<< "${resources}")" + has_prompt="$(jq --arg id "${gateway_id}" \ + '[.[] | select((.gateway_id // .gatewayId) == $id and .name == "test_simple_prompt")] | length' \ + <<< "${prompts}")" + if [ "${has_tool}" -gt 0 ] && [ "${has_resource}" -gt 0 ] && [ "${has_prompt}" -gt 0 ]; then + break + fi + sleep 0.5 +done +test "${has_tool}" -gt 0 +test "${has_resource}" -gt 0 +test "${has_prompt}" -gt 0 + +server_payload="$(jq --null-input --compact-output \ + --arg id "${MCP_CONFORMANCE_SERVER_ID}" \ + --argjson tools "${tool_ids}" \ + --argjson resources "${resource_ids}" \ + --argjson prompts "${prompt_ids}" \ + '{server: { + id: $id, + name: "Official MCP Conformance Server", + description: "Virtual server for alpha.11 conformance", + associated_tools: $tools, + associated_resources: $resources, + associated_prompts: $prompts + }}')" +api_request POST /servers "${server_payload}" > /dev/null + +token_response="$(api_request POST /v1/tokens '{ + "name": "MCP conformance CI", + "description": "Ephemeral dataplane token", + "expires_in_days": 1, + "user_email": "admin@example.com" +}')" +conformance_token="$(jq --exit-status --raw-output '.access_token' <<< "${token_response}")" +if [ "${GITHUB_ACTIONS:-}" = "true" ]; then + echo "::add-mask::${conformance_token}" +fi +echo "MCP_CONFORMANCE_TOKEN=${conformance_token}" >> "${GITHUB_ENV}" diff --git a/tests/conformance/run-conformance.sh b/tests/conformance/run-conformance.sh new file mode 100755 index 0000000..8302110 --- /dev/null +++ b/tests/conformance/run-conformance.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${GITHUB_OUTPUT:?GITHUB_OUTPUT must be set}" +: "${MCP_CONFORMANCE_SERVER_ID:?MCP_CONFORMANCE_SERVER_ID must be set}" +: "${MCP_CONFORMANCE_SPEC_VERSION:?MCP_CONFORMANCE_SPEC_VERSION must be set}" + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +repo_root="$(cd -- "${script_dir}/../.." && pwd)" +suite_dir="${MCP_CONFORMANCE_SUITE_DIR:-${repo_root}/.conformance-suite}" +conformance_port="${MCP_CONFORMANCE_PORT:-8080}" + +set +e +( + cd "${suite_dir}" + npm start -- \ + server \ + --url "http://127.0.0.1:${conformance_port}/servers/${MCP_CONFORMANCE_SERVER_ID}/mcp" \ + --requirements "${MCP_CONFORMANCE_SPEC_VERSION}" \ + --expected-failures "${script_dir}/expected-failures.yml" \ + --output-dir "${repo_root}/conformance-results" +) +runner_status="$?" +set -e + +echo "status=${runner_status}" >> "${GITHUB_OUTPUT}" diff --git a/tests/conformance/run-local.sh b/tests/conformance/run-local.sh new file mode 100755 index 0000000..86bf25d --- /dev/null +++ b/tests/conformance/run-local.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +repo_root="$(cd -- "${script_dir}/../.." && pwd)" +compose_file="${script_dir}/docker-compose.yml" + +export MCP_CONFORMANCE_VERSION="${MCP_CONFORMANCE_VERSION:-0.2.0-alpha.11}" +export MCP_CONFORMANCE_SOURCE_SHA="${MCP_CONFORMANCE_SOURCE_SHA:-c321dd32035556e6769d3724a8ee97d87c3faaac}" +export MCP_CONFORMANCE_SPEC_VERSION="${MCP_CONFORMANCE_SPEC_VERSION:-2026-07-28}" +export MCP_CONFORMANCE_SERVER_ID="${MCP_CONFORMANCE_SERVER_ID:-3f33286667d34b65a31c3bafd30e4c21}" +export MCP_CONFORMANCE_SUITE_DIR="${MCP_CONFORMANCE_SUITE_DIR:-${repo_root}/.conformance-suite}" +export CF_CONTROLPLANE_IMAGE="${CF_CONTROLPLANE_IMAGE:-ghcr.io/ibm/mcp-context-forge:latest}" +export CF_DATAPLANE_IMAGE="${CF_DATAPLANE_IMAGE:-contextforge-data-plane:conformance}" + +for command in curl docker git jq node npm; do + if ! command -v "${command}" > /dev/null 2>&1; then + echo "Required command not found: ${command}" >&2 + exit 1 + fi +done +docker compose version > /dev/null + +if [ -e "${MCP_CONFORMANCE_SUITE_DIR}" ] && [ ! -d "${MCP_CONFORMANCE_SUITE_DIR}/.git" ]; then + echo "MCP_CONFORMANCE_SUITE_DIR is not a git checkout: ${MCP_CONFORMANCE_SUITE_DIR}" >&2 + exit 1 +fi + +if [ ! -d "${MCP_CONFORMANCE_SUITE_DIR}/.git" ]; then + echo "Checking out the official conformance suite." + git clone --filter=blob:none \ + https://github.com/modelcontextprotocol/conformance.git \ + "${MCP_CONFORMANCE_SUITE_DIR}" + git -C "${MCP_CONFORMANCE_SUITE_DIR}" checkout --detach "${MCP_CONFORMANCE_SOURCE_SHA}" +fi + +suite_sha="$(git -C "${MCP_CONFORMANCE_SUITE_DIR}" rev-parse HEAD)" +if [ "${suite_sha}" != "${MCP_CONFORMANCE_SOURCE_SHA}" ]; then + echo "Conformance checkout is at ${suite_sha}; expected ${MCP_CONFORMANCE_SOURCE_SHA}." >&2 + echo "Use a checkout at the pinned commit or set MCP_CONFORMANCE_SUITE_DIR." >&2 + exit 1 +fi + +( + echo "Installing official conformance dependencies." + cd "${MCP_CONFORMANCE_SUITE_DIR}" + test "$(node -p "require('./package.json').version")" = "${MCP_CONFORMANCE_VERSION}" + npm ci --ignore-scripts +) + +state_dir="$(mktemp -d "${TMPDIR:-/tmp}/contextforge-conformance.XXXXXX")" +export GITHUB_ENV="${state_dir}/github-env" +export GITHUB_OUTPUT="${state_dir}/github-output" +touch "${GITHUB_ENV}" "${GITHUB_OUTPUT}" + +# shellcheck disable=SC2329 # Invoked by the trap below. +cleanup() { + local status="$?" + trap - EXIT INT TERM + if [ "${status}" -ne 0 ]; then + echo "Conformance run failed; printing live stack logs." >&2 + MCP_CONFORMANCE_TOKEN=diagnostics-only \ + docker compose -f "${compose_file}" logs --no-color || true + fi + MCP_CONFORMANCE_TOKEN="${MCP_CONFORMANCE_TOKEN:-cleanup-only}" \ + "${script_dir}/stop-live-stack.sh" || true + rm -f -- "${GITHUB_ENV}" "${GITHUB_OUTPUT}" + rmdir -- "${state_dir}" + exit "${status}" +} +trap cleanup EXIT INT TERM + +MCP_CONFORMANCE_TOKEN=pull-only \ + docker compose -f "${compose_file}" pull redis control-plane nginx +echo "Starting the fixture and control plane." +MCP_CONFORMANCE_TOKEN=bootstrap-only \ + "${script_dir}/start-fixture-and-control-plane.sh" +echo "Registering the fixture through the control plane." +MCP_CONFORMANCE_TOKEN=bootstrap-only \ + "${script_dir}/register-fixture.sh" + +set -a +# shellcheck disable=SC1090 +source "${GITHUB_ENV}" +set +a + +echo "Starting the dataplane and nginx." +"${script_dir}/start-dataplane-and-nginx.sh" +echo "Running MCP ${MCP_CONFORMANCE_SPEC_VERSION} conformance." +"${script_dir}/run-conformance.sh" + +runner_status="$(sed -n 's/^status=//p' "${GITHUB_OUTPUT}" | tail -n 1)" +if [ -z "${runner_status}" ]; then + echo "Conformance runner did not report a status." >&2 + exit 1 +fi +exit "${runner_status}" diff --git a/tests/conformance/start-dataplane-and-nginx.sh b/tests/conformance/start-dataplane-and-nginx.sh new file mode 100755 index 0000000..c2a793e --- /dev/null +++ b/tests/conformance/start-dataplane-and-nginx.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${MCP_CONFORMANCE_SERVER_ID:?MCP_CONFORMANCE_SERVER_ID must be set}" + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +repo_root="$(cd -- "${script_dir}/../.." && pwd)" +log_dir="${repo_root}/conformance-logs" +conformance_port="${MCP_CONFORMANCE_PORT:-8080}" + +docker compose -f "${script_dir}/docker-compose.yml" \ + up -d --wait data-plane nginx + +endpoint="http://127.0.0.1:${conformance_port}/servers/${MCP_CONFORMANCE_SERVER_ID}/mcp" +request='{ + "jsonrpc": "2.0", + "id": 1, + "method": "server/discover", + "params": { + "_meta": { + "io.modelcontextprotocol/protocolVersion": "2026-07-28", + "io.modelcontextprotocol/clientInfo": { + "name": "ci-route-probe", + "version": "1.0.0" + }, + "io.modelcontextprotocol/clientCapabilities": {} + } + } +}' + +for _ in $(seq 1 120); do + curl --silent --show-error \ + --dump-header "${log_dir}/route-probe-headers.txt" \ + --output "${log_dir}/route-probe-body.txt" \ + --request POST \ + --header 'Content-Type: application/json' \ + --header 'Accept: application/json, text/event-stream' \ + --header 'MCP-Protocol-Version: 2026-07-28' \ + --header 'MCP-Method: server/discover' \ + --data "${request}" \ + "${endpoint}" || true + if grep --ignore-case --quiet '^X-CF-Conformance-Backend: dataplane' \ + "${log_dir}/route-probe-headers.txt" \ + && sed -n 's/^data: //p' "${log_dir}/route-probe-body.txt" \ + | jq --exit-status \ + '.result.supportedVersions | index("2026-07-28") != null' \ + > /dev/null 2>&1; then + exit 0 + fi + sleep 0.5 +done + +echo "Modern MCP route did not reach the dataplane through nginx" >&2 +cat "${log_dir}/route-probe-headers.txt" >&2 +cat "${log_dir}/route-probe-body.txt" >&2 +exit 1 diff --git a/tests/conformance/start-fixture-and-control-plane.sh b/tests/conformance/start-fixture-and-control-plane.sh new file mode 100755 index 0000000..a255270 --- /dev/null +++ b/tests/conformance/start-fixture-and-control-plane.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +repo_root="$(cd -- "${script_dir}/../.." && pwd)" +suite_dir="${MCP_CONFORMANCE_SUITE_DIR:-${repo_root}/.conformance-suite}" +log_dir="${repo_root}/conformance-logs" + +mkdir -p "${log_dir}" "${repo_root}/conformance-results" + +( + cd "${suite_dir}" + PORT=3000 npm exec -- tsx examples/servers/typescript/everything-server.ts +) > "${log_dir}/reference-server.log" 2>&1 & +echo "$!" > "${log_dir}/reference-server.pid" + +docker compose -f "${script_dir}/docker-compose.yml" \ + up -d --wait redis control-plane + +for _ in $(seq 1 120); do + if curl --silent --output /dev/null http://127.0.0.1:3000/mcp; then + exit 0 + fi + sleep 0.25 +done + +echo "Timed out waiting for the official conformance fixture" >&2 +exit 1 diff --git a/tests/conformance/stop-live-stack.sh b/tests/conformance/stop-live-stack.sh new file mode 100755 index 0000000..e88cf94 --- /dev/null +++ b/tests/conformance/stop-live-stack.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +repo_root="$(cd -- "${script_dir}/../.." && pwd)" +pid_file="${repo_root}/conformance-logs/reference-server.pid" + +if [ -f "${pid_file}" ]; then + kill "$(cat "${pid_file}")" 2>/dev/null || true +fi + +docker compose -f "${script_dir}/docker-compose.yml" \ + down --volumes --remove-orphans