diff --git a/.github/skills/azsdk-common-apiview-feedback-resolution/evals/eval.yaml b/.github/skills/azsdk-common-apiview-feedback-resolution/evals/eval.yaml index c45ebd3f579b..9cb0d6b6d85b 100644 --- a/.github/skills/azsdk-common-apiview-feedback-resolution/evals/eval.yaml +++ b/.github/skills/azsdk-common-apiview-feedback-resolution/evals/eval.yaml @@ -8,7 +8,7 @@ tags: area: azsdk-common-apiview-feedback-resolution type: ci-gate -config: +defaults: runs: 1 timeout: "90s" model: claude-opus-4.6 @@ -19,30 +19,52 @@ scoring: stimuli: - name: apiview-basic-001 - prompt: "My Python SDK PR has APIView comments that I need to address. Can you help me retrieve and resolve them?" + prompt: "My Python SDK PR has APIView comments at https://spa.apiview.dev/review/mock-python-sdk?activeApiRevisionId=mock-python-revision that I need to address. Can you retrieve and resolve them?" + # Load the real skill so the executor can route to it and call its tools. + environment: &apiviewSkill + skills: + - .. graders: + - type: skill-invocation + config: + required: ["azsdk-common-apiview-feedback-resolution"] - type: tool-calls config: required: - name: azure-sdk-mcp-azsdk_apiview_get_comments - name: apiview-edge-001 - prompt: "An APIView reviewer says I need to rename 'ComputeProvisioningState' to 'ProvisioningState' in my SDK. Use the SDK customization tools to apply this rename." + prompt: "An APIView reviewer at https://spa.apiview.dev/review/mock-dotnet-sdk?activeApiRevisionId=mock-dotnet-revision says I need to rename 'ComputeProvisioningState' to 'ProvisioningState' in the .NET SDK package sdk/compute/Azure.ResourceManager.Compute (~/repos/azure-sdk-for-net). Retrieve the feedback and use the SDK customization tools to apply this rename." + environment: *apiviewSkill graders: - type: output-not-contains config: substring: "cannot help" + - type: tool-calls + config: + required: + - name: azure-sdk-mcp-azsdk_apiview_get_comments + - name: azure-sdk-mcp-azsdk_customized_code_update - name: apiview-no-feedback-001 - prompt: "Check if there are any APIView comments on my .NET SDK PR that I need to address before release." + prompt: "Resolve APIView feedback for my .NET SDK PR. Start by checking https://spa.apiview.dev/review/mock-dotnet-sdk?activeApiRevisionId=mock-no-feedback; if there are no comments, report that clearly." + environment: *apiviewSkill graders: + - type: tool-calls + config: + required: + - name: azure-sdk-mcp-azsdk_apiview_get_comments - type: output-contains config: substring: "APIView" - name: apiview-negative-001 - prompt: "I need to create a new TypeSpec project for my Azure Storage service." + prompt: "Author Azure TypeSpec by adding an optional color property to the WidgetSuite model in main.tsp." + environment: *apiviewSkill graders: + - type: skill-invocation + config: + disallowed: ["azsdk-common-apiview-feedback-resolution"] - type: tool-calls config: disallowed: @@ -52,15 +74,27 @@ stimuli: substring: "APIView" - name: basic-trigger - prompt: "I have APIView comments on my SDK PR that need to be resolved. Can you help?" + prompt: "I have APIView comments at https://spa.apiview.dev/review/mock-sdk?activeApiRevisionId=mock-basic-trigger on my SDK PR that need to be resolved. Can you help?" + environment: *apiviewSkill graders: + - type: skill-invocation + config: + required: ["azsdk-common-apiview-feedback-resolution"] + - type: tool-calls + config: + required: + - name: azure-sdk-mcp-azsdk_apiview_get_comments - type: output-contains config: substring: "APIView" - name: anti-trigger prompt: "How do I set up a new Azure Storage account?" + environment: *apiviewSkill graders: + - type: skill-invocation + config: + disallowed: ["azsdk-common-apiview-feedback-resolution"] - type: output-not-contains config: substring: "APIView" @@ -76,7 +110,15 @@ stimuli: - name: url-trigger prompt: "I have review comments on my SDK PR from this page: https://spa.apiview.dev/review/abc123?activeApiRevisionId=def456 — can you help me resolve them?" + environment: *apiviewSkill graders: + - type: skill-invocation + config: + required: ["azsdk-common-apiview-feedback-resolution"] + - type: tool-calls + config: + required: + - name: azure-sdk-mcp-azsdk_apiview_get_comments - type: output-contains config: substring: "apiview" diff --git a/.github/skills/azsdk-common-apiview-feedback-resolution/evals/trigger.eval.yaml b/.github/skills/azsdk-common-apiview-feedback-resolution/evals/trigger.eval.yaml index 60459af0f5d0..d54d58ad4a39 100644 --- a/.github/skills/azsdk-common-apiview-feedback-resolution/evals/trigger.eval.yaml +++ b/.github/skills/azsdk-common-apiview-feedback-resolution/evals/trigger.eval.yaml @@ -8,7 +8,7 @@ tags: area: azsdk-common-apiview-feedback-resolution type: ci-gate -config: +defaults: runs: 1 timeout: "90s" model: claude-opus-4.6 @@ -20,48 +20,72 @@ scoring: stimuli: - name: trigger-resolve-apiview-feedback-on-my-pr prompt: "resolve APIView feedback on my PR" + environment: + skills: + - .. graders: - type: skill-invocation config: required: ["azsdk-common-apiview-feedback-resolution"] - name: trigger-fix-apiview-comments prompt: "fix APIView comments" + environment: + skills: + - .. graders: - type: skill-invocation config: required: ["azsdk-common-apiview-feedback-resolution"] - name: trigger-address-api-review-feedback prompt: "address API review feedback" + environment: + skills: + - .. graders: - type: skill-invocation config: required: ["azsdk-common-apiview-feedback-resolution"] - name: trigger-apiview-shows-issues-with-my-sdk - prompt: "APIView shows issues with my SDK" + prompt: "My SDK has APIView issues at https://spa.apiview.dev/review/mock-sdk?activeApiRevisionId=mock-issues. Retrieve and resolve the feedback." + environment: + skills: + - .. graders: - type: skill-invocation config: required: ["azsdk-common-apiview-feedback-resolution"] - name: trigger-help-me-with-apiview-suggestions prompt: "help me with APIView suggestions" + environment: + skills: + - .. graders: - type: skill-invocation config: required: ["azsdk-common-apiview-feedback-resolution"] - name: trigger-fix-comments-apiview-url-1 prompt: "Help me fix these comments: https://spa.apiview.dev/review/0qw98r9qwur9q823r?activeApiRevisionId=afiq948tquajf8jq9f" + environment: + skills: + - .. graders: - type: skill-invocation config: required: ["azsdk-common-apiview-feedback-resolution"] - name: trigger-address-feedback-apiview-url-2 prompt: "Address this feedback: https://spa.apiview.dev/review/akldfja9qjta498tjaejg0?activeApiRevisionId=odiufa9sefighj98439" + environment: + skills: + - .. graders: - type: skill-invocation config: required: ["azsdk-common-apiview-feedback-resolution"] - name: trigger-resolve-comments-staging-url prompt: "Resolve these comments: https://apiviewstagingtest.com/review/akldfja9qjta498tjaejg0?activeApiRevisionId=odiufa9sefighj98439" + environment: + skills: + - .. graders: - type: skill-invocation config: @@ -69,30 +93,49 @@ stimuli: - name: anti-trigger-deploy-my-azure-function prompt: "deploy my Azure Function" + environment: + skills: + - .. graders: - type: skill-invocation config: disallowed: ["azsdk-common-apiview-feedback-resolution"] - name: anti-trigger-fix-my-pipeline-build-error prompt: "fix my pipeline build error" + environment: + skills: + - .. + - ../../azsdk-common-pipeline-fixer graders: - type: skill-invocation config: + required: ["azsdk-common-pipeline-fixer"] disallowed: ["azsdk-common-apiview-feedback-resolution"] - name: anti-trigger-create-a-new-typespec-definition - prompt: "create a new TypeSpec definition" + prompt: "Author Azure TypeSpec by adding an optional color property to the WidgetSuite model in main.tsp." + environment: + skills: + - .. graders: - type: skill-invocation config: disallowed: ["azsdk-common-apiview-feedback-resolution"] - name: anti-trigger-how-do-i-release-an-sdk-package prompt: "how do I release an SDK package" + environment: + skills: + - .. + - ../../azsdk-common-sdk-release graders: - type: skill-invocation config: + required: ["azsdk-common-sdk-release"] disallowed: ["azsdk-common-apiview-feedback-resolution"] - name: anti-trigger-set-up-azure-storage-account prompt: "set up Azure Storage account" + environment: + skills: + - .. graders: - type: skill-invocation config: diff --git a/.github/skills/azsdk-common-generate-sdk-locally/SKILL.md b/.github/skills/azsdk-common-generate-sdk-locally/SKILL.md index a94df960ff4e..4c5a169005f8 100644 --- a/.github/skills/azsdk-common-generate-sdk-locally/SKILL.md +++ b/.github/skills/azsdk-common-generate-sdk-locally/SKILL.md @@ -20,6 +20,8 @@ DO NOT USE FOR: publishing to package registries, CI pipeline configuration, API ## Rules +- This skill generates a **single language** SDK **locally**. If the user asks to "generate SDK for **all languages**", to generate **without a local clone**, or wants SDK **pull requests created** for them (release plan / pipeline flow), do **not** generate locally — call `azure-sdk-mcp:azsdk_run_generate_sdk` instead. It runs the SDK generation pipeline for each language and produces the SDK pull requests. +- Never use `azure-sdk-mcp:azsdk_get_sdk_pull_request_link` or `azure-sdk-mcp:azsdk_get_pull_request` to _generate_ an SDK; those only retrieve links for SDKs that were already generated. - Requires the `azure-sdk-mcp` server for the MCP workflow; without MCP, use `npm exec --prefix eng/common/tsp-client -- tsp-client` CLI. - Verify the target language repo and the correct TypeSpec configuration file before generation. - After generation or customization, run the check and test steps before updating metadata or finalizing changes. @@ -83,6 +85,7 @@ Prerequisites: azure-sdk-mcp server must be running. Without MCP, use `npx tsp-c ## Troubleshooting +- For "generate SDK for all languages", pipeline-based generation, or when no local SDK clone exists, call `azure-sdk-mcp:azsdk_run_generate_sdk` instead of the local generate flow, and never substitute `azure-sdk-mcp:azsdk_get_sdk_pull_request_link` / `azure-sdk-mcp:azsdk_get_pull_request` for generation. - Run `azure-sdk-mcp:azsdk_verify_setup` to confirm MCP and tools. - If build fails with type conflicts, breaking changes, analyzer errors, or customization drift, use `azure-sdk-mcp:azsdk_customized_code_update` to apply customizations. - The customization tool uses a two-phase approach: TypeSpec decorators first (Phase A), then code repairs if needed (Phase B). diff --git a/.github/skills/azsdk-common-generate-sdk-locally/evals/eval.yaml b/.github/skills/azsdk-common-generate-sdk-locally/evals/eval.yaml index 771d472dabbc..2fcd2dfc7dc1 100644 --- a/.github/skills/azsdk-common-generate-sdk-locally/evals/eval.yaml +++ b/.github/skills/azsdk-common-generate-sdk-locally/evals/eval.yaml @@ -8,7 +8,7 @@ tags: area: azsdk-common-generate-sdk-locally type: ci-gate -config: +defaults: runs: 1 timeout: "90s" model: claude-opus-4.6 @@ -20,6 +20,9 @@ scoring: stimuli: - name: sdk-local-basic-001 prompt: "Generate the Python SDK locally from my TypeSpec project at specification/compute/resource-manager/Microsoft.Compute/ComputeRP/tspconfig.yaml. My azure-sdk-for-python repo is at ~/repos/azure-sdk-for-python." + environment: + skills: + - .. graders: - type: tool-calls config: @@ -28,16 +31,28 @@ stimuli: - name: azure-sdk-mcp-azsdk_package_generate_code - name: sdk-local-negative-001 - prompt: "I want to use the Azure DevOps pipeline to generate SDKs automatically and have PRs created for me. I don't want to set up local tools." + prompt: "Run the Azure DevOps SDK generation pipeline for the Python SDK for release plan work item 12345. The TypeSpec project is specification/contosowidgetmanager/Contoso.WidgetManager, the API version is 2024-01-01-preview, the SDK release type is beta, and the service is data plane. I do not have a local SDK clone; create the generated SDK PR." + environment: + skills: + - .. + - ../../azsdk-common-generate-sdk-pipeline graders: + - type: skill-invocation + config: + disallowed: ["azsdk-common-generate-sdk-locally"] - type: tool-calls config: + required: + - name: azure-sdk-mcp-azsdk_run_generate_sdk disallowed: - name: azure-sdk-mcp-azsdk_verify_setup - name: azure-sdk-mcp-azsdk_package_generate_code - name: sdk-local-full-001 - prompt: "Generate the Java SDK locally, build it, run tests, and prepare it for a PR. Use the SDK generation tools to do the full workflow." + prompt: "Generate the Java SDK locally for the compute service, build it, run tests, and prepare it for a PR. My azure-sdk-for-java repo is at ~/repos/azure-sdk-for-java and the package is sdk/compute/azure-resourcemanager-compute. Bump the version to 1.2.0-beta.1 (beta, release date 2026-07-16). Use the SDK generation tools to do the full workflow." + environment: + skills: + - .. graders: - type: tool-calls config: @@ -52,7 +67,10 @@ stimuli: - name: azure-sdk-mcp-azsdk_package_update_version - name: sdk-local-edge-001 - prompt: "I generated the .NET SDK locally but the build is failing with compilation errors. The error says a type name conflicts with a reserved keyword. Use the customization tools to fix it." + prompt: "I generated the .NET SDK locally but the build is failing with compilation errors. The package is sdk/compute/Azure.ResourceManager.Compute in my azure-sdk-for-net repo at ~/repos/azure-sdk-for-net. The error says a type name conflicts with a reserved keyword. Use the customization tools to fix it." + environment: + skills: + - .. graders: - type: output-not-contains config: @@ -63,7 +81,10 @@ stimuli: - name: azure-sdk-mcp-azsdk_customized_code_update - name: sdk-local-breaking-changes-001 - prompt: "After regenerating the Java SDK, the build is failing because the service team renamed 'displayName' to 'name' in the TypeSpec. Our customization class still references getField('displayName'). Use the SDK customization tools to fix this breaking change." + prompt: "After regenerating the Java SDK package at sdk/compute/azure-resourcemanager-compute (~/repos/azure-sdk-for-java), the build is failing because the service team renamed 'displayName' to 'name' in the TypeSpec. Our customization class still references getField('displayName'). Use the SDK customization tools to fix this breaking change." + environment: + skills: + - .. graders: - type: output-not-contains config: @@ -74,7 +95,10 @@ stimuli: - name: azure-sdk-mcp-azsdk_customized_code_update - name: sdk-local-customization-001 - prompt: "The Java SDK build is failing after regeneration. The error says 'variable operationId is already defined in class AnalyzeOperationDetails'. The service team added operationId to the TypeSpec, but our customization already injects that field. Use the SDK customization tools to resolve this duplicate." + prompt: "The Java SDK package at sdk/compute/azure-resourcemanager-compute (~/repos/azure-sdk-for-java) is failing to build after regeneration. The error says 'variable operationId is already defined in class AnalyzeOperationDetails'. The service team added operationId to the TypeSpec, but our customization already injects that field. Use the SDK customization tools to resolve this duplicate." + environment: + skills: + - .. graders: - type: output-not-contains config: @@ -85,7 +109,10 @@ stimuli: - name: azure-sdk-mcp-azsdk_customized_code_update - name: sdk-local-hide-operation-001 - prompt: "Hide the get_create_project_status polling operation from the Python SDK public API. It's an internal operation that shouldn't be exposed to users. Use the SDK customization tools." + prompt: "Hide the get_create_project_status polling operation from the public API of the Python SDK package at sdk/compute/azure-mgmt-compute (~/repos/azure-sdk-for-python). It's an internal operation that shouldn't be exposed to users. Use the SDK customization tools." + environment: + skills: + - .. graders: - type: output-not-contains config: @@ -96,7 +123,10 @@ stimuli: - name: azure-sdk-mcp-azsdk_customized_code_update - name: sdk-local-rename-client-001 - prompt: "Rename the AIProjectConnectionEntraIDCredential model to use 'Id' instead of 'ID' in the .NET SDK. This was flagged in API review. Use the SDK customization tools to apply this rename." + prompt: "Rename the AIProjectConnectionEntraIDCredential model to use 'Id' instead of 'ID' in the .NET SDK package at sdk/compute/Azure.ResourceManager.Compute (~/repos/azure-sdk-for-net). This was flagged in API review. Use the SDK customization tools to apply this rename." + environment: + skills: + - .. graders: - type: output-not-contains config: @@ -107,7 +137,10 @@ stimuli: - name: azure-sdk-mcp-azsdk_customized_code_update - name: sdk-local-update-changelog-001 - prompt: "Update the changelog for the Python SDK package at sdk/compute/azure-mgmt-compute." + prompt: "Update the changelog for the Python SDK package at sdk/compute/azure-mgmt-compute in my azure-sdk-for-python repo at ~/repos/azure-sdk-for-python." + environment: + skills: + - .. graders: - type: output-not-contains config: @@ -118,7 +151,10 @@ stimuli: - name: azure-sdk-mcp-azsdk_package_update_changelog_content - name: sdk-local-update-metadata-001 - prompt: "Update the package metadata for the .NET SDK at sdk/contoso/Azure.ResourceManager.Contoso." + prompt: "Update the package metadata for the .NET SDK package at sdk/compute/Azure.ResourceManager.Compute (~/repos/azure-sdk-for-net)." + environment: + skills: + - .. graders: - type: output-not-contains config: @@ -129,7 +165,10 @@ stimuli: - name: azure-sdk-mcp-azsdk_package_update_metadata - name: sdk-local-update-version-001 - prompt: "Update the package version for the Java SDK at sdk/contoso/azure-resourcemanager-contoso." + prompt: "Update the package version for the Java SDK package at sdk/compute/azure-resourcemanager-compute (~/repos/azure-sdk-for-java) to 1.2.0-beta.1, release type beta, release date 2026-07-16." + environment: + skills: + - .. graders: - type: output-not-contains config: @@ -140,7 +179,10 @@ stimuli: - name: azure-sdk-mcp-azsdk_package_update_version - name: sdk-local-analyzer-errors-001 - prompt: "The .NET SDK build is failing with analyzer errors: AZC0030 says the model name ends with 'Parameters' and AZC0012 says type name 'Tasks' is too generic. Use the SDK customization tools to fix these analyzer errors." + prompt: "The .NET SDK build for sdk/compute/Azure.ResourceManager.Compute (~/repos/azure-sdk-for-net) is failing with analyzer errors: AZC0030 says the model name ends with 'Parameters' and AZC0012 says type name 'Tasks' is too generic. Use the SDK customization tools to fix these analyzer errors." + environment: + skills: + - .. graders: - type: output-not-contains config: diff --git a/.github/skills/azsdk-common-pipeline-analysis/evals/trigger.eval.yaml b/.github/skills/azsdk-common-pipeline-analysis/evals/trigger.eval.yaml index f43b7c37b081..03d1a2fb51f3 100644 --- a/.github/skills/azsdk-common-pipeline-analysis/evals/trigger.eval.yaml +++ b/.github/skills/azsdk-common-pipeline-analysis/evals/trigger.eval.yaml @@ -8,7 +8,7 @@ tags: area: azsdk-common-pipeline-analysis type: ci-gate -config: +defaults: runs: 1 timeout: "90s" model: claude-opus-4.6 @@ -21,30 +21,45 @@ stimuli: # Should trigger - name: trigger-my-ci-pipeline-is-failing prompt: "my Azure SDK CI pipeline build 6447834 is failing, can you help troubleshoot what went wrong?" + environment: + skills: + - .. graders: - type: skill-invocation config: required: ["azsdk-common-pipeline-analysis"] - name: trigger-debug-pipeline-build-error prompt: "debug the build error in my SDK generation pipeline, build 6447834" + environment: + skills: + - .. graders: - type: skill-invocation config: required: ["azsdk-common-pipeline-analysis"] - name: trigger-pipeline-analysis-help - prompt: "I'm stuck on a failing Azure SDK pipeline — can you help me troubleshoot it?" + prompt: "I'm stuck on Azure SDK pipeline build 6447834. Analyze what went wrong and explain the root cause; do not fix it." + environment: + skills: + - .. graders: - type: skill-invocation config: required: ["azsdk-common-pipeline-analysis"] - name: trigger-analyze-failing-sdk-pipeline prompt: "analyze my failing Azure SDK generation pipeline https://dev.azure.com/azure-sdk/internal/_build/results?buildId=6447834 and tell me what went wrong" + environment: + skills: + - .. graders: - type: skill-invocation config: required: ["azsdk-common-pipeline-analysis"] - name: trigger-ci-build-broke-after-my-change prompt: "the SDK CI build broke after my change, help me troubleshoot the pipeline" + environment: + skills: + - .. graders: - type: skill-invocation config: @@ -55,37 +70,65 @@ stimuli: # this analysis-only skill (see each skill's DO NOT USE FOR boundary). - name: anti-trigger-auto-fix-and-commit prompt: "auto-fix the failing tests in my pipeline and commit the changes to my PR" + environment: + skills: + - .. + - ../../azsdk-common-pipeline-fixer graders: - type: skill-invocation config: + required: ["azsdk-common-pipeline-fixer"] disallowed: ["azsdk-common-pipeline-analysis"] - name: anti-trigger-write-a-typespec-definition - prompt: "write a TypeSpec definition" + prompt: "Author Azure TypeSpec by adding an optional color property to the WidgetSuite model in main.tsp." + environment: + skills: + - .. graders: - type: skill-invocation config: disallowed: ["azsdk-common-pipeline-analysis"] - name: anti-trigger-create-a-new-skill prompt: "create a new skill" + environment: + skills: + - .. + - ../../skill-authoring graders: - type: skill-invocation config: + required: ["skill-authoring"] disallowed: ["azsdk-common-pipeline-analysis"] - name: anti-trigger-optimize-markdown-tokens prompt: "optimize markdown tokens" + environment: + skills: + - .. + - ../../markdown-token-optimizer graders: - type: skill-invocation config: + required: ["markdown-token-optimizer"] disallowed: ["azsdk-common-pipeline-analysis"] - name: anti-trigger-how-do-i-release-an-sdk-package prompt: "how do I release an SDK package" + environment: + skills: + - .. + - ../../azsdk-common-sdk-release graders: - type: skill-invocation config: + required: ["azsdk-common-sdk-release"] disallowed: ["azsdk-common-pipeline-analysis"] - name: anti-trigger-resolve-apiview-feedback prompt: "resolve APIView feedback" + environment: + skills: + - .. + - ../../azsdk-common-apiview-feedback-resolution graders: - type: skill-invocation config: + required: ["azsdk-common-apiview-feedback-resolution"] disallowed: ["azsdk-common-pipeline-analysis"] diff --git a/.github/skills/azsdk-common-pipeline-fixer/evals/trigger.eval.yaml b/.github/skills/azsdk-common-pipeline-fixer/evals/trigger.eval.yaml index 16dad26093c7..72c2bf08941c 100644 --- a/.github/skills/azsdk-common-pipeline-fixer/evals/trigger.eval.yaml +++ b/.github/skills/azsdk-common-pipeline-fixer/evals/trigger.eval.yaml @@ -8,7 +8,7 @@ tags: area: azsdk-common-pipeline-fixer type: ci-gate -config: +defaults: runs: 1 # Stimuli with a resolvable build reference (URL or build ID) make the agent # run the full analyze→fix→verify workflow (~90-180s under claude-opus-4.6), @@ -25,30 +25,45 @@ stimuli: # Should trigger - name: trigger-fix-pipeline-failure prompt: "fix the pipeline failure for build 6447834" + environment: + skills: + - .. graders: - type: skill-invocation config: required: ["azsdk-common-pipeline-fixer"] - name: trigger-auto-fix-failing-tests - prompt: "My Azure SDK PR has failing tests in CI: https://dev.azure.com/azure-sdk/internal/_build/results?buildId=6447834. Can you fix them?" + prompt: "Auto-fix the failing tests in my Azure SDK PR for build 6447834." + environment: + skills: + - .. graders: - type: skill-invocation config: required: ["azsdk-common-pipeline-fixer"] - name: trigger-resolve-pipeline-failure prompt: "resolve my Azure SDK pipeline failure" + environment: + skills: + - .. graders: - type: skill-invocation config: required: ["azsdk-common-pipeline-fixer"] - name: trigger-fix-build-error prompt: "fix the build error in my Azure SDK CI" + environment: + skills: + - .. graders: - type: skill-invocation config: required: ["azsdk-common-pipeline-fixer"] - name: trigger-fix-mypy-type-error prompt: "My Python SDK CI is failing with a mypy type error: https://dev.azure.com/azure-sdk/internal/_build/results?buildId=6447834. Can you fix it?" + environment: + skills: + - .. graders: - type: skill-invocation config: @@ -59,31 +74,54 @@ stimuli: # skill applies + verifies fixes and must stay out of pure-diagnosis requests. - name: anti-trigger-analyze-only prompt: "what went wrong in my pipeline build 12345? just tell me, don't fix anything" + environment: + skills: + - .. + - ../../azsdk-common-pipeline-analysis graders: - type: skill-invocation config: + required: ["azsdk-common-pipeline-analysis"] disallowed: ["azsdk-common-pipeline-fixer"] - name: anti-trigger-why-is-ci-red - prompt: "why is my CI red? just diagnose the root cause, do not change any code" + prompt: "Azure SDK pipeline build 6447834 is red. Analyze what went wrong and explain the root cause; do not change any code." + environment: + skills: + - .. + - ../../azsdk-common-pipeline-analysis graders: - type: skill-invocation config: + required: ["azsdk-common-pipeline-analysis"] disallowed: ["azsdk-common-pipeline-fixer"] - name: anti-trigger-write-a-typespec-definition - prompt: "write a TypeSpec definition" + prompt: "Author Azure TypeSpec by adding an optional color property to the WidgetSuite model in main.tsp." + environment: + skills: + - .. graders: - type: skill-invocation config: disallowed: ["azsdk-common-pipeline-fixer"] - name: anti-trigger-release-sdk-package prompt: "how do I release an SDK package" + environment: + skills: + - .. + - ../../azsdk-common-sdk-release graders: - type: skill-invocation config: + required: ["azsdk-common-sdk-release"] disallowed: ["azsdk-common-pipeline-fixer"] - name: anti-trigger-create-a-new-skill prompt: "create a new skill" + environment: + skills: + - .. + - ../../skill-authoring graders: - type: skill-invocation config: + required: ["skill-authoring"] disallowed: ["azsdk-common-pipeline-fixer"] diff --git a/.github/skills/azsdk-common-prepare-release-plan/SKILL.md b/.github/skills/azsdk-common-prepare-release-plan/SKILL.md index 5c4fec7f01a2..31210bc8611f 100644 --- a/.github/skills/azsdk-common-prepare-release-plan/SKILL.md +++ b/.github/skills/azsdk-common-prepare-release-plan/SKILL.md @@ -4,7 +4,7 @@ license: MIT metadata: version: "1.0.0" distribution: shared -description: 'Create, get, update, abandon, and link SDK PRs to release plan work items for Azure SDK releases. **UTILITY SKILL**. USE FOR: "create release plan", "get release plan", "update release plan", "update API spec in release plan", "update SDK details in release plan", "abandon release plan", "link SDK PR to plan", "namespace approval", "check release plan status". DO NOT USE FOR: SDK code generation, pipeline troubleshooting, API review feedback. INVOKES: azure-sdk-mcp:azsdk_create_release_plan, azure-sdk-mcp:azsdk_get_release_plan, azure-sdk-mcp:azsdk_get_release_plan_for_spec_pr, azure-sdk-mcp:azsdk_update_release_plan, azure-sdk-mcp:azsdk_update_api_spec_pull_request_in_release_plan, azure-sdk-mcp:azsdk_update_sdk_details_in_release_plan, azure-sdk-mcp:azsdk_abandon_release_plan, azure-sdk-mcp:azsdk_link_sdk_pull_request_to_release_plan, azure-sdk-mcp:azsdk_link_namespace_approval_issue.' +description: 'Manage Azure SDK release plans and associated SDK PRs. **UTILITY SKILL**. USE FOR: "prepare a release plan", "plan the next SDK release", "create release plan for my package", "get or update release plan", "update API spec or SDK details in release plan", "abandon release plan", "link SDK PR", "namespace approval", "check release plan status". DO NOT USE FOR: SDK generation, pipeline troubleshooting, API review. INVOKES: azure-sdk-mcp:azsdk_create_release_plan, azure-sdk-mcp:azsdk_get_release_plan, azure-sdk-mcp:azsdk_get_release_plan_for_spec_pr, azure-sdk-mcp:azsdk_update_release_plan, azure-sdk-mcp:azsdk_update_api_spec_pull_request_in_release_plan, azure-sdk-mcp:azsdk_update_sdk_details_in_release_plan, azure-sdk-mcp:azsdk_abandon_release_plan, azure-sdk-mcp:azsdk_link_sdk_pull_request_to_release_plan, azure-sdk-mcp:azsdk_link_namespace_approval_issue.' compatibility: "azure-sdk-mcp server, API spec PR in Azure/azure-rest-api-specs" --- @@ -14,8 +14,8 @@ This skill creates, gets, updates, abandons, and links SDK PRs to release plan w ## Triggers -USE FOR: create release plan, get release plan, update release plan, update API spec in release plan, update SDK details in release plan, abandon release plan, link SDK PR to plan, namespace approval, check release plan status -WHEN: "create release plan", "get release plan", "update release plan", "abandon release plan", "link SDK PR to plan", "namespace approval", "check release plan status" +USE FOR: prepare release plan, create release plan, get release plan, update release plan, update API spec in release plan, update SDK details in release plan, abandon release plan, link SDK PR to plan, namespace approval, check release plan status +WHEN: "prepare a release plan", "plan the next SDK release", "create release plan for my package", "create a release plan for my new Azure SDK service", "help me prepare a release plan for my new Azure SDK package", "create release plan", "get release plan", "update release plan", "abandon release plan", "link SDK PR to plan", "namespace approval", "check release plan status" DO NOT USE FOR: SDK code generation, pipeline troubleshooting, API review feedback ## Rules @@ -57,15 +57,12 @@ DO NOT USE FOR: SDK code generation, pipeline troubleshooting, API review feedba 3. **Gather Info** — Collect required details from the user. See [details](references/release-plan-details.md): - Target release month/year (format: "Month YYYY", e.g. "June 2026"). Do NOT use formats like "2026-06" or "06/2026" — these are invalid. - API release type: Value must be one of the following: "Private Preview", "Public Preview", or "GA" - - SDK release type: Value must be "beta" or "stable" — always ask the user explicitly - Spec PR URL (optional) - Service Tree ID (GUID) — optional if previously created - Product Tree ID (GUID) — optional if previously created -4. **Create** — Run `azure-sdk-mcp:azsdk_create_release_plan` with the collected parameters including `sdkReleaseType`. Use `forceCreateReleasePlan: true` only if an existing release plan was found for a different API release type. +4. **Create** — Run `azure-sdk-mcp:azsdk_create_release_plan` with the collected parameters. Use `forceCreateReleasePlan: true` only if an existing release plan was found for a different API release type. 5. **Namespace** — For first management plane releases, link namespace approval issue using `azure-sdk-mcp:azsdk_link_namespace_approval_issue`. -> **IMPORTANT**: Do NOT default the API release type value as the SDK release type. These are separate fields — always ask the user explicitly for the SDK release type. -> > **IMPORTANT**: Do NOT update an existing release plan to change its API release type. If a release plan exists for a different API release type, force-create a new one instead. **Tool**: `azure-sdk-mcp:azsdk_create_release_plan` diff --git a/.github/skills/azsdk-common-prepare-release-plan/evals/trigger.eval.yaml b/.github/skills/azsdk-common-prepare-release-plan/evals/trigger.eval.yaml index c90bb2bcc669..fd278096eccf 100644 --- a/.github/skills/azsdk-common-prepare-release-plan/evals/trigger.eval.yaml +++ b/.github/skills/azsdk-common-prepare-release-plan/evals/trigger.eval.yaml @@ -8,7 +8,7 @@ tags: area: azsdk-common-prepare-release-plan type: ci-gate -config: +defaults: runs: 1 timeout: "90s" model: claude-opus-4.6 @@ -20,30 +20,48 @@ scoring: stimuli: - name: trigger-prepare-a-release-plan prompt: "prepare a release plan" + environment: + skills: + - .. graders: - type: skill-invocation config: required: ["azsdk-common-prepare-release-plan"] - name: trigger-plan-the-next-sdk-release prompt: "plan the next SDK release" + environment: + skills: + - .. graders: - type: skill-invocation config: required: ["azsdk-common-prepare-release-plan"] - name: trigger-create-release-plan-for-my-package prompt: "create release plan for my package" + environment: + skills: + - .. graders: - type: skill-invocation config: required: ["azsdk-common-prepare-release-plan"] - - name: trigger-what-goes-in-a-release-plan - prompt: "create a release plan for my new Azure SDK service" + - name: trigger-link-sdk-pr-to-release-plan + prompt: "Link my Python SDK PR https://github.com/Azure/azure-sdk-for-python/pull/12345 to release plan 67890." + environment: + skills: + - .. graders: - - type: skill-invocation + # The executor can validly invoke this precise MCP operation without + # loading the skill, so verify the observable release-plan route. + - type: tool-calls config: - required: ["azsdk-common-prepare-release-plan"] + required: + - name: azure-sdk-mcp-azsdk_link_sdk_pull_request_to_release_plan - name: trigger-help-me-prepare-for-sdk-release prompt: "help me prepare a release plan for my new Azure SDK package" + environment: + skills: + - .. graders: - type: skill-invocation config: @@ -51,31 +69,50 @@ stimuli: - name: anti-trigger-deploy-my-azure-function prompt: "deploy my Azure Function" + environment: + skills: + - .. graders: - type: skill-invocation config: disallowed: ["azsdk-common-prepare-release-plan"] - name: anti-trigger-fix-my-pipeline-build-error prompt: "fix my pipeline build error" + environment: + skills: + - .. + - ../../azsdk-common-pipeline-fixer graders: - type: skill-invocation config: + required: ["azsdk-common-pipeline-fixer"] disallowed: ["azsdk-common-prepare-release-plan"] - name: anti-trigger-create-a-new-typespec-definition - prompt: "create a new TypeSpec definition" + prompt: "Author Azure TypeSpec by adding an optional color property to the WidgetSuite model in main.tsp." + environment: + skills: + - .. graders: - type: skill-invocation config: disallowed: ["azsdk-common-prepare-release-plan"] - name: anti-trigger-configure-azure-logging prompt: "configure Azure logging" + environment: + skills: + - .. graders: - type: skill-invocation config: disallowed: ["azsdk-common-prepare-release-plan"] - name: anti-trigger-resolve-apiview-feedback prompt: "resolve APIView feedback" + environment: + skills: + - .. + - ../../azsdk-common-apiview-feedback-resolution graders: - type: skill-invocation config: + required: ["azsdk-common-apiview-feedback-resolution"] disallowed: ["azsdk-common-prepare-release-plan"] diff --git a/.github/skills/azsdk-common-sdk-release/SKILL.md b/.github/skills/azsdk-common-sdk-release/SKILL.md index 10fef179d696..04b076f45a4d 100644 --- a/.github/skills/azsdk-common-sdk-release/SKILL.md +++ b/.github/skills/azsdk-common-sdk-release/SKILL.md @@ -4,7 +4,7 @@ license: MIT metadata: version: "1.0.0" distribution: shared -description: 'Check release readiness and trigger the release pipeline for Azure SDK packages. **UTILITY SKILL**. USE FOR: "release SDK", "trigger release", "check release readiness", "release pipeline", "publish package", "ship SDK". DO NOT USE FOR: SDK development, code generation, pipeline debugging, release plan creation. INVOKES: azure-sdk-mcp:azsdk_release_sdk.' +description: 'Check release readiness and trigger the release pipeline for Azure SDK packages. **UTILITY SKILL**. USE FOR: "release SDK", "trigger release", "check release readiness", "release pipeline", "publish package", "ship SDK", "release checklist", "release process", "publish a new SDK version", "how to release/publish an SDK". DO NOT USE FOR: SDK development, code generation, pipeline debugging, release plan creation. INVOKES: azure-sdk-mcp:azsdk_release_sdk.' compatibility: "azure-sdk-mcp server, SDK package merged on release branch. Supports .NET, Java, JavaScript, Python, Go" --- @@ -14,10 +14,12 @@ This skill checks release readiness and triggers the Azure SDK release pipeline ## Triggers -USE FOR: release SDK, trigger release, check release readiness, release pipeline, publish package, ship SDK -WHEN: "release SDK", "trigger release", "check release readiness", "release pipeline", "publish package", "ship SDK" +USE FOR: release SDK, trigger release, check release readiness, release pipeline, publish package, ship SDK, release checklist, release process, publish a new SDK version, how to release/publish an SDK +WHEN: "release SDK", "trigger release", "check release readiness", "release pipeline", "publish package", "ship SDK", "SDK release checklist", "walk me through the SDK release process", "how to publish a new SDK version", "release a new version of azure-sdk" DO NOT USE FOR: SDK development, code generation, pipeline debugging, release plan creation +Also use this skill for informational/how-to questions about releasing or publishing an SDK package (e.g. release checklists, walkthroughs of the release process) — answer using the Steps and Rules below rather than only triggering on direct action requests. + ## Rules - Requires the `azure-sdk-mcp` server; there is no CLI fallback for the release workflow. @@ -41,6 +43,10 @@ DO NOT USE FOR: SDK development, code generation, pipeline debugging, release pl - "Check if azure-storage-blob is ready for release" - "Trigger release for @azure/core-rest-pipeline on JavaScript" +- "What's the SDK release checklist?" +- "Walk me through the SDK release process" +- "How do I publish a new SDK version?" +- "Release a new version of azure-sdk" ## Troubleshooting diff --git a/.github/skills/azsdk-common-sdk-release/evals/trigger.eval.yaml b/.github/skills/azsdk-common-sdk-release/evals/trigger.eval.yaml index c82800b29d53..18ba894618d5 100644 --- a/.github/skills/azsdk-common-sdk-release/evals/trigger.eval.yaml +++ b/.github/skills/azsdk-common-sdk-release/evals/trigger.eval.yaml @@ -8,7 +8,7 @@ tags: area: azsdk-common-sdk-release type: ci-gate -config: +defaults: runs: 1 timeout: "90s" model: claude-opus-4.6 @@ -20,30 +20,45 @@ scoring: stimuli: - name: trigger-release-my-sdk-package prompt: "release my SDK package" + environment: + skills: + - .. graders: - type: skill-invocation config: required: ["azsdk-common-sdk-release"] - name: trigger-how-to-publish-a-new-sdk-version prompt: "how to publish a new SDK version" + environment: + skills: + - .. graders: - type: skill-invocation config: required: ["azsdk-common-sdk-release"] - name: trigger-walk-me-through-sdk-release-process prompt: "walk me through SDK release process" + environment: + skills: + - .. graders: - type: skill-invocation config: required: ["azsdk-common-sdk-release"] - name: trigger-release-a-new-version-of-azure-sdk - prompt: "release a new version of azure-sdk" + prompt: "trigger the Azure SDK release pipeline for a new package version" + environment: + skills: + - .. graders: - type: skill-invocation config: required: ["azsdk-common-sdk-release"] - name: trigger-sdk-release-checklist prompt: "SDK release checklist" + environment: + skills: + - .. graders: - type: skill-invocation config: @@ -51,31 +66,52 @@ stimuli: - name: anti-trigger-write-unit-tests prompt: "write unit tests" + environment: + skills: + - .. graders: - type: skill-invocation config: disallowed: ["azsdk-common-sdk-release"] - name: anti-trigger-fix-my-pipeline-build-error prompt: "fix my pipeline build error" + environment: + skills: + - .. + - ../../azsdk-common-pipeline-fixer graders: - type: skill-invocation config: + required: ["azsdk-common-pipeline-fixer"] disallowed: ["azsdk-common-sdk-release"] - name: anti-trigger-create-a-new-typespec-definition - prompt: "create a new TypeSpec definition" + prompt: "Author Azure TypeSpec by adding an optional color property to the WidgetSuite model in main.tsp." + environment: + skills: + - .. graders: - type: skill-invocation config: disallowed: ["azsdk-common-sdk-release"] - name: anti-trigger-optimize-markdown-tokens prompt: "optimize markdown tokens" + environment: + skills: + - .. + - ../../markdown-token-optimizer graders: - type: skill-invocation config: + required: ["markdown-token-optimizer"] disallowed: ["azsdk-common-sdk-release"] - name: anti-trigger-resolve-apiview-feedback prompt: "resolve APIView feedback" + environment: + skills: + - .. + - ../../azsdk-common-apiview-feedback-resolution graders: - type: skill-invocation config: + required: ["azsdk-common-apiview-feedback-resolution"] disallowed: ["azsdk-common-sdk-release"]