diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml new file mode 100644 index 0000000..30ab0e0 --- /dev/null +++ b/.github/workflows/conformance.yml @@ -0,0 +1,29 @@ +name: Conformance Tests + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +concurrency: + group: conformance-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + server-conformance: + runs-on: ubuntu-latest + continue-on-error: true + steps: + - uses: actions/checkout@v6 + - uses: ruby/setup-ruby@v1 + with: + ruby-version: '4.0' # Specify the latest supported Ruby version. + bundler-cache: true + - uses: actions/setup-node@v4 + with: + node-version: '24' # Specify the latest Node.js version. + - run: bundle exec rake conformance diff --git a/README.md b/README.md index e814e96..88d6b79 100644 --- a/README.md +++ b/README.md @@ -1028,6 +1028,12 @@ The client provides a wrapper class for tools returned by the server: This class provides easy access to tool properties like name, description, input schema, and output schema. +## Conformance Testing + +The `conformance/` directory contains a test server and runner that validate the SDK against the MCP specification using [`@modelcontextprotocol/conformance`](https://github.com/modelcontextprotocol/conformance). + +See [conformance/README.md](conformance/README.md) for usage instructions. + ## Documentation - [SDK API documentation](https://rubydoc.info/gems/mcp) diff --git a/Rakefile b/Rakefile index 0d0ad08..b9eab74 100644 --- a/Rakefile +++ b/Rakefile @@ -14,4 +14,46 @@ require "rubocop/rake_task" RuboCop::RakeTask.new -task default: [:test, :rubocop] +task default: [:rubocop, :test, :conformance] + +desc "Run MCP conformance tests (PORT, SCENARIO, SPEC_VERSION, VERBOSE)" +task :conformance do |t| + next unless npx_available?(t.name) + + require_relative "conformance/runner" + + options = {} + options[:port] = Integer(ENV["PORT"]) if ENV["PORT"] + options[:scenario] = ENV["SCENARIO"] if ENV["SCENARIO"] + options[:spec_version] = ENV["SPEC_VERSION"] if ENV["SPEC_VERSION"] + options[:verbose] = true if ENV["VERBOSE"] + + Conformance::Runner.new(**options).run +end + +desc "List available conformance scenarios" +task :conformance_list do |t| + next unless npx_available?(t.name) + + system("npx", "--yes", "@modelcontextprotocol/conformance", "list", "--server") +end + +desc "Start the conformance server (PORT)" +task :conformance_server do + require_relative "conformance/server" + + options = {} + options[:port] = Integer(ENV["PORT"]) if ENV["PORT"] + + Conformance::Server.new(**options).start +rescue Interrupt + puts "Exiting..." + exit +end + +def npx_available?(task_name) + return true if system("which", "npx", out: File::NULL, err: File::NULL) + + warn("Skipping #{task_name}: npx is not installed. Install Node.js to run this task: https://nodejs.org/") + false +end diff --git a/conformance/README.md b/conformance/README.md new file mode 100644 index 0000000..159def0 --- /dev/null +++ b/conformance/README.md @@ -0,0 +1,103 @@ +# MCP Conformance Tests + +Validates the Ruby SDK's conformance to the MCP specification using [`@modelcontextprotocol/conformance`](https://github.com/modelcontextprotocol/conformance). + +## Prerequisites + +- Node.js (for `npx`) +- `bundle install` completed + +## Running the Tests + +### Run all scenarios + +```bash +bundle exec rake conformance +``` + +Starts the conformance server, runs all active scenarios against it, prints a pass/fail +summary for each scenario, and exits with a non-zero status code if any unexpected failures +are detected. Scenarios listed in `expected_failures.yml` are allowed to fail without +affecting the exit code. + +### Environment variables + +| Variable | Description | Default | +|----------------|--------------------------------------|---------| +| `PORT` | Server port | `9292` | +| `SCENARIO` | Run a single scenario by name | (all) | +| `SPEC_VERSION` | Filter scenarios by spec version | (all) | +| `VERBOSE` | Show raw JSON output when set | (off) | + +```bash +# Run a single scenario +bundle exec rake conformance SCENARIO=ping + +# Use a different port with verbose output +bundle exec rake conformance PORT=3000 VERBOSE=1 + +# Start the server on a specific port +bundle exec rake conformance_server PORT=3000 +``` + +### Start the server and test separately + +```bash +# Terminal 1: start the server +bundle exec rake conformance_server + +# Terminal 2: run all scenarios +npx @modelcontextprotocol/conformance server --url http://localhost:9292/mcp + +# Terminal 2: run a single scenario +npx @modelcontextprotocol/conformance server --url http://localhost:9292/mcp --scenario ping +``` + +Keeps the server alive between test runs, which avoids the startup overhead when iterating +on a single scenario. Stop the server with Ctrl+C when done. + +### List available scenarios + +```bash +bundle exec rake conformance_list +``` + +Prints all scenario names that can be passed to `SCENARIO`. + +## SDK Tier Report + +The [MCP SDK Tier system](https://modelcontextprotocol.io/community/sdk-tiers) requires SDK +maintainers to self-assess and report results to the SDK Working Group via +[modelcontextprotocol/modelcontextprotocol issues](https://github.com/modelcontextprotocol/modelcontextprotocol/issues). + +To generate a full tier assessment report, use the `/mcp-sdk-tier-audit` slash command from +the [modelcontextprotocol/conformance](https://github.com/modelcontextprotocol/conformance) +repository with the conformance server running: + +```bash +# Terminal 1 (this repository): start the conformance server +bundle exec rake conformance_server + +# Terminal 2 (conformance repository): run the tier audit skill as a slash command in Claude Code +/mcp-sdk-tier-audit /path/to/modelcontextprotocol/ruby-sdk http://localhost:9292/mcp +``` + +The skill evaluates conformance pass rate, issue label taxonomy, triage metrics, documentation +coverage, and policy compliance, then produces a markdown report suitable for tier advancement +submissions. + +## File Structure + +``` +conformance/ + server.rb # Conformance server (Rack + Puma, default port 9292) + runner.rb # Starts the server, runs npx conformance, exits with result code + expected_failures.yml # Baseline of known-failing scenarios + README.md # This file +``` + +## Known Limitations + +Known-failing scenarios are registered in `conformance/expected_failures.yml`. They are allowed to +fail without affecting the exit code and are tracked to catch regressions. +These are shown in the output of `bundle exec rake conformance`. diff --git a/conformance/expected_failures.yml b/conformance/expected_failures.yml new file mode 100644 index 0000000..98aa64e --- /dev/null +++ b/conformance/expected_failures.yml @@ -0,0 +1,9 @@ +server: + # Server-to-client requests (sampling/createMessage, elicitation/create) are not + # implemented: `Transport#send_request` does not exist in the current SDK. + - tools-call-sampling + - tools-call-elicitation + - elicitation-sep1034-defaults + - elicitation-sep1330-enums + # TODO: The SDK does not extract `_meta.progressToken` from tool call requests or deliver `notifications/progress` to tools. + - tools-call-with-progress diff --git a/conformance/runner.rb b/conformance/runner.rb new file mode 100644 index 0000000..de38621 --- /dev/null +++ b/conformance/runner.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +# Starts the conformance server and runs `npx @modelcontextprotocol/conformance` against it. +require "English" +require "net/http" +require_relative "server" + +module Conformance + class Runner + # Empirical timeout: Puma typically starts well within this window. + SERVER_START_TIMEOUT = 20 + SERVER_POLL_INTERVAL = 0.5 + SERVER_HEALTH_CHECK_RETRIES = (SERVER_START_TIMEOUT / SERVER_POLL_INTERVAL).to_i + + def initialize(port: Server::DEFAULT_PORT, scenario: nil, spec_version: nil, verbose: false) + @port = port + @scenario = scenario + @spec_version = spec_version + @verbose = verbose + end + + def run + command = build_command + server_pid = start_server + + run_conformance(command, server_pid: server_pid) + end + + private + + def build_command + expected_failures_yml = File.expand_path("expected_failures.yml", __dir__) + + npx_command = ["npx", "--yes", "@modelcontextprotocol/conformance", "server", "--url", "http://localhost:#{@port}/mcp"] + npx_command += ["--scenario", @scenario] if @scenario + npx_command += ["--spec-version", @spec_version] if @spec_version + npx_command += ["--verbose"] if @verbose + npx_command += ["--expected-failures", expected_failures_yml] + npx_command + end + + def start_server + puts "Starting conformance server on port #{@port}..." + + server_pid = fork do + Conformance::Server.new(port: @port).start + end + + health_url = URI("http://localhost:#{@port}/health") + ready = false + SERVER_HEALTH_CHECK_RETRIES.times do + begin + response = Net::HTTP.get_response(health_url) + if response.code == "200" + ready = true + break + end + rescue Errno::ECONNREFUSED, Errno::ECONNRESET, Net::ReadTimeout + # not ready yet + end + sleep(SERVER_POLL_INTERVAL) + end + + unless ready + warn("ERROR: Conformance server did not start within #{SERVER_START_TIMEOUT} seconds") + terminate_server(server_pid) + exit(1) + end + + puts "Server ready. Running conformance tests..." + + server_pid + end + + def run_conformance(command, server_pid:) + puts "Command: #{command.join(" ")}\n\n" + + conformance_exit_code = nil + begin + system(*command) + conformance_exit_code = $CHILD_STATUS.exitstatus + ensure + terminate_server(server_pid) + end + + exit(conformance_exit_code || 1) + end + + def terminate_server(pid) + Process.kill("TERM", pid) + rescue Errno::ESRCH + # process already exited + ensure + begin + Process.wait(pid) + rescue Errno::ECHILD + # already reaped + end + end + end +end diff --git a/conformance/server.rb b/conformance/server.rb new file mode 100644 index 0000000..5bdd047 --- /dev/null +++ b/conformance/server.rb @@ -0,0 +1,544 @@ +# frozen_string_literal: true + +require "rackup" +require "json" +require "uri" +require_relative "../lib/mcp" + +module Conformance + # 1x1 red PNG pixel (matches TypeScript SDK and Python SDK) + BASE64_1X1_PNG = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==" + + # Minimal WAV file (matches TypeScript SDK and Python SDK) + BASE64_MINIMAL_WAV = "UklGRiYAAABXQVZFZm10IBAAAAABAAEAQB8AAAB9AAACABAAZGF0YQIAAAA=" + + module Tools + class TestSimpleText < MCP::Tool + tool_name "test_simple_text" + description "A tool that returns simple text content" + + class << self + def call(**_args) + MCP::Tool::Response.new([MCP::Content::Text.new("This is a simple text response for testing.").to_h]) + end + end + end + + class TestImageContent < MCP::Tool + tool_name "test_image_content" + description "A tool that returns image content" + + class << self + def call(**_args) + MCP::Tool::Response.new([MCP::Content::Image.new(BASE64_1X1_PNG, "image/png").to_h]) + end + end + end + + class TestAudioContent < MCP::Tool + tool_name "test_audio_content" + description "A tool that returns audio content" + + class << self + def call(**_args) + MCP::Tool::Response.new([MCP::Content::Audio.new(BASE64_MINIMAL_WAV, "audio/wav").to_h]) + end + end + end + + class TestEmbeddedResource < MCP::Tool + tool_name "test_embedded_resource" + description "A tool that returns embedded resource content" + + class << self + def call(**_args) + text_contents = MCP::Resource::TextContents.new( + uri: "test://embedded-resource", + mime_type: "text/plain", + text: "This is an embedded resource content.", + ) + MCP::Tool::Response.new([MCP::Content::EmbeddedResource.new(text_contents).to_h]) + end + end + end + + class TestMultipleContentTypes < MCP::Tool + tool_name "test_multiple_content_types" + description "A tool that returns multiple content types" + + class << self + def call(**_args) + MCP::Tool::Response.new([ + MCP::Content::Text.new("Multiple content types test:").to_h, + MCP::Content::Image.new(BASE64_1X1_PNG, "image/png").to_h, + MCP::Content::EmbeddedResource.new( + MCP::Resource::TextContents.new( + uri: "test://mixed-content-resource", + mime_type: "application/json", + text: '{"test":"data","value":123}', + ), + ).to_h, + ]) + end + end + end + + class TestErrorHandling < MCP::Tool + tool_name "test_error_handling" + description "A tool that intentionally returns an error for testing" + + class << self + def call(**_args) + MCP::Tool::Response.new( + [MCP::Content::Text.new("This tool intentionally returns an error for testing").to_h], + error: true, + ) + end + end + end + + class JsonSchema202012Tool < MCP::Tool + tool_name "json_schema_2020_12_tool" + description "Tool with JSON Schema 2020-12 features" + input_schema( + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + address: { + type: "object", + properties: { + street: { type: "string" }, + city: { type: "string" }, + }, + }, + }, + properties: { + name: { type: "string" }, + address: { "$ref": "#/$defs/address" }, + }, + additionalProperties: false, + ) + + class << self + def call(**_args) + MCP::Tool::Response.new([MCP::Content::Text.new("Processed with JSON Schema 2020-12").to_h]) + end + end + end + + class TestToolWithLogging < MCP::Tool + tool_name "test_tool_with_logging" + description "A tool that sends log messages during execution" + + class << self + def call(server_context:, **_args) + server_context.notify_log_message(data: "Tool execution started", level: "info", logger: "test_logger") + sleep(0.05) # Required by the conformance test to verify clients handle interleaved notifications (same as TypeScript SDK). + server_context.notify_log_message(data: "Tool processing data", level: "info", logger: "test_logger") + sleep(0.05) # Same as above. + server_context.notify_log_message(data: "Tool execution completed", level: "info", logger: "test_logger") + MCP::Tool::Response.new([MCP::Content::Text.new("Logging complete (3 messages sent)").to_h]) + end + end + end + + # test_tool_with_progress: the actual progress dispatch is in `tools_call_handler` + class TestToolWithProgress < MCP::Tool + tool_name "test_tool_with_progress" + description "A tool that reports progress notifications" + + class << self + def call(**_args) + MCP::Tool::Response.new([MCP::Content::Text.new("Progress complete").to_h]) + end + end + end + + # TODO: Implement when `Transport` supports server-to-client requests. + class TestSampling < MCP::Tool + tool_name "test_sampling" + description "A tool that requests LLM sampling from the client" + input_schema( + properties: { prompt: { type: "string" } }, + required: ["prompt"], + ) + + class << self + def call(prompt:) + MCP::Tool::Response.new( + [MCP::Content::Text.new("Sampling not supported in this SDK version").to_h], + error: true, + ) + end + end + end + + # TODO: Implement when `Transport` supports server-to-client requests. + class TestElicitation < MCP::Tool + tool_name "test_elicitation" + description "A tool that requests user input from the client" + input_schema( + properties: { message: { type: "string" } }, + required: ["message"], + ) + + class << self + def call(message:) + MCP::Tool::Response.new( + [MCP::Content::Text.new("Elicitation not supported in this SDK version").to_h], + error: true, + ) + end + end + end + + # TODO: Implement when `Transport` supports server-to-client requests. + class TestElicitationSep1034Defaults < MCP::Tool + tool_name "test_elicitation_sep1034_defaults" + description "A tool that tests elicitation with default values" + + class << self + def call(**_args) + MCP::Tool::Response.new( + [MCP::Content::Text.new("Elicitation not supported in this SDK version").to_h], + error: true, + ) + end + end + end + + # TODO: Implement when `Transport` supports server-to-client requests. + class TestElicitationSep1330Enums < MCP::Tool + tool_name "test_elicitation_sep1330_enums" + description "A tool that tests elicitation with enum schemas" + + class << self + def call(**_args) + MCP::Tool::Response.new( + [MCP::Content::Text.new("Elicitation not supported in this SDK version").to_h], + error: true, + ) + end + end + end + + class TestReconnection < MCP::Tool + tool_name "test_reconnection" + description "A tool that triggers SSE stream closure to test client reconnection behavior" + + class << self + def call(**_args) + MCP::Tool::Response.new([MCP::Content::Text.new("Reconnection test completed").to_h]) + end + end + end + end + + module Prompts + class TestSimplePrompt < MCP::Prompt + prompt_name "test_simple_prompt" + description "A simple prompt for testing with no arguments" + + class << self + def template(_args, server_context: nil) + MCP::Prompt::Result.new( + messages: [ + MCP::Prompt::Message.new( + role: "user", + content: MCP::Content::Text.new("This is a simple prompt for testing."), + ), + ], + ) + end + end + end + + class TestPromptWithArguments < MCP::Prompt + prompt_name "test_prompt_with_arguments" + description "A prompt with required arguments for testing" + arguments [ + MCP::Prompt::Argument.new(name: "arg1", description: "First test argument", required: true), + MCP::Prompt::Argument.new(name: "arg2", description: "Second test argument", required: true), + ] + + class << self + def template(args, server_context: nil) + arg1 = args.dig(:arg1) || args.dig("arg1") || "" + arg2 = args.dig(:arg2) || args.dig("arg2") || "" + MCP::Prompt::Result.new( + messages: [ + MCP::Prompt::Message.new( + role: "user", + content: MCP::Content::Text.new("Prompt with arguments: arg1='#{arg1}', arg2='#{arg2}'"), + ), + ], + ) + end + end + end + + class TestPromptWithEmbeddedResource < MCP::Prompt + prompt_name "test_prompt_with_embedded_resource" + description "A prompt with an embedded resource for testing" + arguments [ + MCP::Prompt::Argument.new(name: "resourceUri", description: "URI of the resource to embed", required: true), + ] + + class << self + def template(args, server_context: nil) + resource_uri = args.dig(:resourceUri) || args.dig("resourceUri") || "test://example-resource" + MCP::Prompt::Result.new( + messages: [ + MCP::Prompt::Message.new( + role: "user", + content: MCP::Content::EmbeddedResource.new( + MCP::Resource::TextContents.new( + uri: resource_uri, + mime_type: "text/plain", + text: "Embedded resource content for testing.", + ), + ), + ), + MCP::Prompt::Message.new( + role: "user", + content: MCP::Content::Text.new("Please process the embedded resource above."), + ), + ], + ) + end + end + end + + class TestPromptWithImage < MCP::Prompt + prompt_name "test_prompt_with_image" + description "A prompt with image content for testing" + + class << self + def template(_args, server_context: nil) + MCP::Prompt::Result.new( + messages: [ + MCP::Prompt::Message.new( + role: "user", + content: MCP::Content::Image.new(BASE64_1X1_PNG, "image/png"), + ), + MCP::Prompt::Message.new( + role: "user", + content: MCP::Content::Text.new("Please analyze the image above."), + ), + ], + ) + end + end + end + end + + class Server + DEFAULT_PORT = 9292 + + class DnsRebindingProtection + LOCALHOST_PATTERNS = /\A(localhost|127\.0\.0\.1|\[::1\]|::1)(:\d+)?\z/i.freeze + + def initialize(app) + @app = app + end + + def call(env) + host = env["HTTP_HOST"] || env["SERVER_NAME"] || "" + + unless localhost?(host) + return [ + 403, + { "Content-Type" => "application/json" }, + [{ error: "Forbidden: DNS rebinding protection - invalid Host header '#{host}'" }.to_json], + ] + end + + origin = env["HTTP_ORIGIN"] + if origin && !origin.empty? + begin + origin_host = URI.parse(origin).host.to_s + unless localhost?(origin_host) + return [ + 403, + { "Content-Type" => "application/json" }, + [{ error: "Forbidden: DNS rebinding protection - invalid Origin '#{origin}'" }.to_json], + ] + end + rescue URI::InvalidURIError + return [ + 403, + { "Content-Type" => "application/json" }, + [{ error: "Forbidden: invalid Origin header" }.to_json], + ] + end + end + + @app.call(env) + end + + private + + def localhost?(host) + host.empty? || host.match?(LOCALHOST_PATTERNS) + end + end + + def initialize(port: DEFAULT_PORT) + @port = port + end + + def start + server = build_server + transport = build_transport(server) + configure_handlers(server) + rack_app = build_rack_app(transport) + + puts "MCP Conformance Server starting on http://localhost:#{@port}/mcp" + + Rackup::Handler.get("puma").run(rack_app, Port: @port, Host: "localhost", Silent: true) + end + + private + + def build_server + MCP::Server.new( + name: "ruby-sdk-conformance-server", + version: MCP::VERSION, + tools: [ + Tools::TestSimpleText, + Tools::TestImageContent, + Tools::TestAudioContent, + Tools::TestEmbeddedResource, + Tools::TestMultipleContentTypes, + Tools::TestErrorHandling, + Tools::JsonSchema202012Tool, + Tools::TestToolWithLogging, + Tools::TestToolWithProgress, + Tools::TestSampling, + Tools::TestElicitation, + Tools::TestElicitationSep1034Defaults, + Tools::TestElicitationSep1330Enums, + Tools::TestReconnection, + ], + prompts: [ + Prompts::TestSimplePrompt, + Prompts::TestPromptWithArguments, + Prompts::TestPromptWithEmbeddedResource, + Prompts::TestPromptWithImage, + ], + resources: resources, + resource_templates: resource_templates, + capabilities: { + tools: { listChanged: true }, + prompts: { listChanged: true }, + resources: { listChanged: true, subscribe: true }, + logging: {}, + completions: {}, + }, + ) + end + + def resources + [ + MCP::Resource.new( + uri: "test://static-text", + name: "static-text", + description: "A static text resource for testing", + mime_type: "text/plain", + ), + MCP::Resource.new( + uri: "test://static-binary", + name: "static-binary", + description: "A static binary (PNG) resource for testing", + mime_type: "image/png", + ), + MCP::Resource.new( + uri: "test://watched-resource", + name: "watched-resource", + description: "A resource for subscription testing", + mime_type: "text/plain", + ), + ] + end + + def resource_templates + [ + MCP::ResourceTemplate.new( + uri_template: "test://template/{id}/data", + name: "template-resource", + description: "A parameterized resource template for testing", + mime_type: "application/json", + ), + ] + end + + def build_transport(server) + transport = MCP::Server::Transports::StreamableHTTPTransport.new(server) + server.transport = transport + transport + end + + def configure_handlers(server) + server.logging_message_notification = MCP::LoggingMessageNotification.new(level: "debug") + server.server_context = server + + configure_resources_read_handler(server) + end + + def configure_resources_read_handler(server) + server.resources_read_handler do |params| + uri = params[:uri].to_s + + case uri + when "test://static-text" + [ + MCP::Resource::TextContents.new( + text: "This is the content of the static text resource.", + uri: uri, + mime_type: "text/plain", + ).to_h, + ] + when "test://static-binary" + [ + MCP::Resource::BlobContents.new( + data: BASE64_1X1_PNG, + uri: uri, + mime_type: "image/png", + ).to_h, + ] + when %r{\Atest://template/(.+)/data\z} + id = Regexp.last_match(1) + content = { id: id, templateTest: true, data: "Data for ID: #{id}" }.to_json + + [ + MCP::Resource::TextContents.new( + text: content, + uri: uri, + mime_type: "application/json", + ).to_h, + ] + else + [] + end + end + end + + def build_rack_app(transport) + mcp_app = proc do |env| + request = Rack::Request.new(env) + + if request.path_info == "/health" + [200, { "Content-Type" => "application/json" }, ['{"status":"ok"}']] + elsif request.path_info == "/mcp" || request.path_info == "/" + transport.handle_request(request) + else + [404, { "Content-Type" => "application/json" }, ['{"error":"Not found"}']] + end + end + + Rack::Builder.new do + use(DnsRebindingProtection) + run(mcp_app) + end + end + end +end