Add validation tools for server.json schemas and examples (modelcontextprotocol#176)

tadasant · claude · chlowell · web-flow · commit 774a0df59232 · 2025-07-15T11:32:54.000-07:00
This is a first PR for modelcontextprotocol#159 - adding some validation scripts. I want to get to a place where we are confident that making schema changes is in sync with the rest of our code. Step one is making sure the schemas are in sync with our examples. Step two will be making sure the schemas are in sync with our actual Registry implementation code. This is just for the JSON right now; if we're feeling good about this approach I'll do an analogous change for the OpenAPI schemas as well. I'll also follow with adding usage of these scripts to our CI pipeline so we have automated checks on them with every change. ## Summary - Added CLI validation tools for JSON schemas and examples in `docs/server-json/` - Fixed some minor validation errors found in `examples.md` - Ensures schema consistency and example validity in CI/CD pipelines ## Changes ### New Validation Tools 1. **`tools/validate-schemas`** - Validates that `schema.json` and `registry-schema.json` are valid JSON Schema documents 2. **`tools/validate-examples`** - Validates all JSON examples in `examples.md` against both schemas ### Key Features - ✅ Validates JSON schema syntax and structure - ✅ Validates all examples against both base and registry schemas - ✅ Tracks validation counts to prevent silent failures - ✅ Proper exit codes for CI integration - ✅ Clear error messages for debugging ### Fixes to examples.md - Added missing `repository.id` field in NuGet example - Changed `repository.source` from "gitlab" to "github" in Docker example ### Usage ```bash # From repository root ./tools/validate-schemas.sh ./tools/validate-examples.sh ``` ## Testing I ran these scripts locally with minor (breaking) changes, which they each correctly reported. As submitted, they are both passing. 🤖 Co-authored with [Claude Code](https://claude.ai/code) --------- Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Charles Lowell <10964656+chlowell@users.noreply.github.com>
diff --git a/.gitignore b/.gitignore
@@ -5,3 +5,5 @@ build/*
 **/bin
 cmd/registry/registry
 publisher
+validate-examples
+validate-schemas
diff --git a/.golangci.yml b/.golangci.yml
@@ -71,6 +71,11 @@ linters:
     - whitespace
 
 linters-settings:
+  revive:
+    rules:
+      - name: use-any
+        disabled: false
+        severity: error
   cyclop:
     max-complexity: 50
   funlen:
diff --git a/docs/server-json/README.md b/docs/server-json/README.md
@@ -15,3 +15,25 @@ References:
 - [schema.json](./schema.json) - The official JSON schema specification for this representation
 - [examples.md](./examples.md) - Example manifestations of the JSON schema
 - [registry-schema.json](./registry-schema.json) - A more constrained version of `schema.json` that the official registry supports
+
+## Validation Tools
+
+Two validation tools are provided in the repository's `tools/` directory:
+- **validate-schemas** - Validates that `schema.json` and `registry-schema.json` are valid JSON Schema documents
+- **validate-examples** - Validates that all JSON examples in `examples.md` conform to both schemas
+
+### Usage
+
+From the repository root:
+
+#### Validate JSON Schemas
+
+```bash
+./tools/validate-schemas.sh
+```
+
+#### Validate Examples
+
+```bash
+./tools/validate-examples.sh
+```
diff --git a/docs/server-json/examples.md b/docs/server-json/examples.md
@@ -219,7 +219,7 @@ The `dnx` tool ships with the .NET 10 SDK, starting with Preview 6.
   "repository": {
     "url": "https://github.com/joelverhagen/Knapcode.SampleMcpServer",
     "source": "github",
-    "id": "def456gh-ijkl-7890-mnop-qrstuvwxyz13"
+    "id": "example-nuget-id-0000-1111-222222222222"
   },
   "version_detail": {
     "version": "0.3.0-beta"
@@ -251,7 +251,7 @@ The `dnx` tool ships with the .NET 10 SDK, starting with Preview 6.
   "description": "MCP server for database operations with support for multiple database types",
   "repository": {
     "url": "https://github.com/example/mcp-database",
-    "source": "gitlab",
+    "source": "github",
     "id": "ghi789jk-lmno-1234-pqrs-tuvwxyz56789"
   },
   "version_detail": {
diff --git a/go.mod b/go.mod
@@ -5,6 +5,7 @@ go 1.23.0
 require (
 	github.com/caarlos0/env/v11 v11.3.1
 	github.com/google/uuid v1.6.0
+	github.com/santhosh-tekuri/jsonschema/v5 v5.3.1
 	github.com/stretchr/testify v1.10.0
 	github.com/swaggo/files v1.0.1
 	github.com/swaggo/http-swagger v1.3.4
diff --git a/go.sum b/go.sum
@@ -34,6 +34,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
 github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
+github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4=
+github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY=
 github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
 github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
diff --git a/internal/api/handlers/v0/auth.go b/internal/api/handlers/v0/auth.go
@@ -64,7 +64,7 @@ func StartAuthHandler(authService auth.Service) http.HandlerFunc {
 		// Return successful response
 		w.Header().Set("Content-Type", "application/json")
 		w.WriteHeader(http.StatusOK)
-		if err := json.NewEncoder(w).Encode(map[string]interface{}{
+		if err := json.NewEncoder(w).Encode(map[string]any{
 			"flow_info":    flowInfo,
 			"status_token": statusToken,
 			"expires_in":   300, // 5 minutes
@@ -98,7 +98,7 @@ func CheckAuthStatusHandler(authService auth.Service) http.HandlerFunc {
 				// Auth is still pending
 				w.Header().Set("Content-Type", "application/json")
 				w.WriteHeader(http.StatusOK)
-				if err := json.NewEncoder(w).Encode(map[string]interface{}{
+				if err := json.NewEncoder(w).Encode(map[string]any{
 					"status": "pending",
 				}); err != nil {
 					http.Error(w, "Failed to encode response", http.StatusInternalServerError)
@@ -115,7 +115,7 @@ func CheckAuthStatusHandler(authService auth.Service) http.HandlerFunc {
 		// Authentication completed successfully
 		w.Header().Set("Content-Type", "application/json")
 		w.WriteHeader(http.StatusOK)
-		if err := json.NewEncoder(w).Encode(map[string]interface{}{
+		if err := json.NewEncoder(w).Encode(map[string]any{
 			"status": "complete",
 			"token":  token,
 		}); err != nil {
diff --git a/internal/api/handlers/v0/publish_test.go b/internal/api/handlers/v0/publish_test.go
@@ -61,7 +61,7 @@ func TestPublishHandler(t *testing.T) {
 	testCases := []struct {
 		name             string
 		method           string
-		requestBody      interface{}
+		requestBody      any
 		authHeader       string
 		setupMocks       func(*MockRegistryService, *MockAuthService)
 		expectedStatus   int
diff --git a/internal/database/database.go b/internal/database/database.go
@@ -19,7 +19,7 @@ var (
 // Database defines the interface for database operations on MCPRegistry entries
 type Database interface {
 	// List retrieves all MCPRegistry entries with optional filtering
-	List(ctx context.Context, filter map[string]interface{}, cursor string, limit int) ([]*model.Server, string, error)
+	List(ctx context.Context, filter map[string]any, cursor string, limit int) ([]*model.Server, string, error)
 	// GetByID retrieves a single ServerDetail by it's ID
 	GetByID(ctx context.Context, id string) (*model.ServerDetail, error)
 	// Publish adds a new ServerDetail to the database
diff --git a/internal/database/import.go b/internal/database/import.go
@@ -33,7 +33,7 @@ func ReadSeedFile(path string) ([]model.ServerDetail, error) {
 	var servers []model.ServerDetail
 	if err := json.Unmarshal(fileContent, &servers); err != nil {
 		// Try parsing as a raw JSON array and then convert to our model
-		var rawData []map[string]interface{}
+		var rawData []map[string]any
 		if jsonErr := json.Unmarshal(fileContent, &rawData); jsonErr != nil {
 			return nil, fmt.Errorf("failed to parse JSON: %w (original error: %w)", jsonErr, err)
 		}
diff --git a/internal/database/memory.go b/internal/database/memory.go
@@ -90,7 +90,7 @@ func compareSemanticVersions(version1, version2 string) int {
 //gocognit:ignore
 func (db *MemoryDB) List(
 	ctx context.Context,
-	filter map[string]interface{},
+	filter map[string]any,
 	cursor string,
 	limit int,
 ) ([]*model.Server, string, error) {
diff --git a/internal/database/mongo.go b/internal/database/mongo.go
@@ -75,7 +75,7 @@ func NewMongoDB(ctx context.Context, connectionURI, databaseName, collectionName
 // List retrieves MCPRegistry entries with optional filtering and pagination
 func (db *MongoDB) List(
 	ctx context.Context,
-	filter map[string]interface{},
+	filter map[string]any,
 	cursor string,
 	limit int,
 ) ([]*model.Server, string, error) {
diff --git a/tools/publisher/main.go b/tools/publisher/main.go
@@ -290,7 +290,7 @@ func createCommand() {
 // publishToRegistry sends the MCP server details to the registry with authentication
 func publishToRegistry(registryURL string, mcpData []byte, token string) error {
 	// Parse the MCP JSON data
-	var mcpDetails map[string]interface{}
+	var mcpDetails map[string]any
 	err := json.Unmarshal(mcpData, &mcpDetails)
 	if err != nil {
 		return fmt.Errorf("error parsing server.json file: %w", err)
diff --git a/tools/validate-examples.sh b/tools/validate-examples.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+# Validate examples in docs/server-json/examples.md
+# For more information, see docs/server-json/README.md
+
+set -e
+
+cd "$(dirname "$0")/.."
+exec go run tools/validate-examples/main.go "$@"
diff --git a/tools/validate-examples/main.go b/tools/validate-examples/main.go
@@ -0,0 +1,168 @@
+// validate-examples validates JSON examples in docs/server-json/examples.md
+// against both schema.json and registry-schema.json.
+//
+// For more information, see docs/server-json/README.md
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/santhosh-tekuri/jsonschema/v5"
+)
+
+const (
+	// expectedExampleCount is the number of JSON examples we expect to find in examples.md
+	// IMPORTANT: Only change this count if you have intentionally added or removed examples
+	// from the examples.md file. This check prevents accidental formatting changes from
+	// causing examples to be skipped during validation.
+	expectedExampleCount = 8
+)
+
+func main() {
+	log.SetFlags(0) // Remove timestamp from logs
+
+	if err := runValidation(); err != nil {
+		log.Fatalf("Error: %v", err)
+	}
+}
+
+func runValidation() error {
+	basePath := filepath.Join("docs", "server-json")
+
+	examplesPath := filepath.Join(basePath, "examples.md")
+	schemaPath := filepath.Join(basePath, "schema.json")
+	registrySchemaPath := filepath.Join(basePath, "registry-schema.json")
+
+	examples, err := extractExamples(examplesPath)
+	if err != nil {
+		return fmt.Errorf("failed to extract examples: %w", err)
+	}
+
+	log.Printf("Found %d examples in examples.md\n", len(examples))
+
+	if len(examples) != expectedExampleCount {
+		return fmt.Errorf("expected %d examples but found %d - if this is intentional, update expectedExampleCount in %s",
+			expectedExampleCount, len(examples), "tools/validate-examples/main.go")
+	}
+
+	log.Println()
+
+	baseSchema, err := compileSchema(schemaPath)
+	if err != nil {
+		return fmt.Errorf("failed to compile schema.json: %w", err)
+	}
+
+	registrySchema, err := compileSchema(registrySchemaPath)
+	if err != nil {
+		return fmt.Errorf("failed to compile registry-schema.json: %w", err)
+	}
+
+	validatedCount := 0
+	for i, example := range examples {
+		log.Printf("Example %d:", i+1)
+
+		var data any
+		if err := json.Unmarshal([]byte(example.content), &data); err != nil {
+			log.Printf("  ❌ Invalid JSON: %v", err)
+			continue
+		}
+
+		baseValid := false
+		registryValid := false
+
+		if err := baseSchema.Validate(data); err != nil {
+			log.Printf("  Validating against schema.json: ❌")
+			log.Printf("    Error: %v", err)
+		} else {
+			log.Printf("  Validating against schema.json: ✅")
+			baseValid = true
+		}
+
+		if err := registrySchema.Validate(data); err != nil {
+			log.Printf("  Validating against registry-schema.json: ❌")
+			log.Printf("    Error: %v", err)
+		} else {
+			log.Printf("  Validating against registry-schema.json: ✅")
+			registryValid = true
+		}
+
+		// Only count as validated if both schemas passed
+		if baseValid && registryValid {
+			validatedCount++
+		}
+
+		log.Println()
+	}
+
+	if validatedCount != expectedExampleCount {
+		return fmt.Errorf("validation failed: expected %d examples to pass both validations but only %d did",
+			expectedExampleCount, validatedCount)
+	}
+
+	log.Printf("Successfully validated all %d examples!", validatedCount)
+	return nil
+}
+
+type example struct {
+	content string
+	line    int
+}
+
+func extractExamples(path string) ([]example, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	content := string(data)
+
+	// Regex to match JSON code blocks in markdown
+	// Captures everything between ```json and ```
+	re := regexp.MustCompile("(?s)```json\n(.*?)\n```")
+	matches := re.FindAllStringSubmatchIndex(content, -1)
+
+	var examples []example
+	for _, match := range matches {
+		if len(match) < 4 {
+			// should never happen
+			return nil, fmt.Errorf("invalid match - expected at least 4 indices but got %d", len(match))
+		}
+		start, end := match[2], match[3]
+		// line numbers start at 1
+		line := 1 + strings.Count(content[:start], "\n")
+		examples = append(examples, example{
+			content: content[start:end],
+			line:    line,
+		})
+	}
+
+	return examples, nil
+}
+
+func compileSchema(path string) (*jsonschema.Schema, error) {
+	compiler := jsonschema.NewCompiler()
+	compiler.Draft = jsonschema.Draft7
+
+	// For registry-schema.json, we need to register the base schema it references
+	if strings.Contains(path, "registry-schema.json") {
+		basePath := filepath.Join(filepath.Dir(path), "schema.json")
+		baseData, err := os.ReadFile(basePath)
+		if err != nil {
+			return nil, fmt.Errorf("failed to read base schema: %w", err)
+		}
+
+		// Add the base schema to the compiler with the expected URL
+		if err := compiler.AddResource("https://modelcontextprotocol.io/schemas/draft/2025-07-09/server.json", bytes.NewReader(baseData)); err != nil {
+			return nil, fmt.Errorf("failed to add base schema resource: %w", err)
+		}
+	}
+
+	return compiler.Compile(path)
+}
diff --git a/tools/validate-schemas.sh b/tools/validate-schemas.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+# Validate JSON schema files
+# For more information, see docs/server-json/README.md
+
+set -e
+
+cd "$(dirname "$0")/.."
+exec go run tools/validate-schemas/main.go "$@"
diff --git a/tools/validate-schemas/main.go b/tools/validate-schemas/main.go

Original file line number	Diff line number	Diff line change
`@@ -33,7 +33,7 @@ func ReadSeedFile(path string) ([]model.ServerDetail, error) {`
`33`	`33`	`var servers []model.ServerDetail`
`34`	`34`	`if err := json.Unmarshal(fileContent, &servers); err != nil {`
`35`	`35`	`// Try parsing as a raw JSON array and then convert to our model`
`36`		`- var rawData []map[string]interface{}`
	`36`	`+ var rawData []map[string]any`
`37`	`37`	`if jsonErr := json.Unmarshal(fileContent, &rawData); jsonErr != nil {`
`38`	`38`	`return nil, fmt.Errorf("failed to parse JSON: %w (original error: %w)", jsonErr, err)`
`39`	`39`	`}`