From b30d822920b80bee870889d013a060c91a2f0169 Mon Sep 17 00:00:00 2001 From: Krrish Mittal Date: Mon, 13 Jul 2026 17:09:43 -0700 Subject: [PATCH 1/8] fix(ci): use GITHUB_TOKEN installation auth for Copilot CLI smoke test Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/copilot-test-cli.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/copilot-test-cli.yml b/.github/workflows/copilot-test-cli.yml index 4de5a5b1cf4..9ceeb14923d 100644 --- a/.github/workflows/copilot-test-cli.yml +++ b/.github/workflows/copilot-test-cli.yml @@ -2,14 +2,13 @@ name: "Test Copilot CLI" on: workflow_dispatch: - pull_request: permissions: contents: read copilot-requests: write jobs: - test: + copilot-cli-smoke: runs-on: ubuntu-latest timeout-minutes: 5 steps: @@ -18,16 +17,16 @@ jobs: - name: Install Copilot CLI run: npm install -g @github/copilot@latest - - name: Test CLI invocation + - name: Test CLI invocation (GITHUB_TOKEN installation auth, org-billed) env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - RESULT=$(COPILOT_GITHUB_TOKEN="$GITHUB_TOKEN" timeout 60 copilot \ + RESULT=$(timeout 60 copilot --yolo \ -p "Respond with exactly: COPILOT_CLI_WORKING" \ - -s --no-ask-user --model claude-opus-4.8 --allow-all 2>&1) || true + --model claude-opus-4.8 2>&1) || true echo "Result: $RESULT" if echo "$RESULT" | grep -q "COPILOT_CLI_WORKING"; then - echo "Copilot CLI is working" + echo "Copilot CLI is working (installation auth)" else echo "::error::Copilot CLI did not return expected response" echo "Full output: $RESULT" From ae27d2cf9a9b40a4f618f2cbea6ddcabd27b0b80 Mon Sep 17 00:00:00 2001 From: Krrish Mittal Date: Mon, 13 Jul 2026 18:21:46 -0700 Subject: [PATCH 2/8] test(ci): verify Copilot model-fallback loop (temp bogus primary) --- .github/workflows/copilot-test-cli.yml | 38 +++++++++++++++++++------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/.github/workflows/copilot-test-cli.yml b/.github/workflows/copilot-test-cli.yml index 9ceeb14923d..5383f34a371 100644 --- a/.github/workflows/copilot-test-cli.yml +++ b/.github/workflows/copilot-test-cli.yml @@ -10,25 +10,43 @@ permissions: jobs: copilot-cli-smoke: runs-on: ubuntu-latest - timeout-minutes: 5 + timeout-minutes: 6 + env: + # Ordered model-fallback chain: first model that responds wins. + COPILOT_MODELS: "claude-opus-4.8 claude-opus-4.6 gpt-5.6" steps: - uses: actions/checkout@v4 - name: Install Copilot CLI run: npm install -g @github/copilot@latest - - name: Test CLI invocation (GITHUB_TOKEN installation auth, org-billed) + - name: Test CLI invocation (GITHUB_TOKEN installation auth + model fallback) env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # VERIFY-FALLBACK: bogus model forces the loop to fall through to a real one. + COPILOT_MODELS_OVERRIDE: "definitely-not-a-real-model claude-opus-4.8 claude-opus-4.6" run: | - RESULT=$(timeout 60 copilot --yolo \ - -p "Respond with exactly: COPILOT_CLI_WORKING" \ - --model claude-opus-4.8 2>&1) || true - echo "Result: $RESULT" - if echo "$RESULT" | grep -q "COPILOT_CLI_WORKING"; then - echo "Copilot CLI is working (installation auth)" + MODELS="${COPILOT_MODELS_OVERRIDE:-$COPILOT_MODELS}" + RESULT="" + USED="" + for MODEL in $MODELS; do + echo "::group::Trying model $MODEL" + set +e + OUT=$(timeout 90 copilot --yolo \ + -p "Respond with exactly: COPILOT_CLI_WORKING" \ + --model "$MODEL" 2>&1) + CODE=$? + set -e + echo "$OUT" + echo "::endgroup::" + if [ $CODE -eq 0 ] && echo "$OUT" | grep -q "COPILOT_CLI_WORKING"; then + RESULT="$OUT"; USED="$MODEL"; break + fi + echo "::warning::Model $MODEL failed (exit $CODE) — trying next" + done + if [ -n "$RESULT" ]; then + echo "Copilot CLI is working (installation auth) via model: $USED" else - echo "::error::Copilot CLI did not return expected response" - echo "Full output: $RESULT" + echo "::error::Copilot CLI failed for all models: $MODELS" exit 1 fi From 580f90188c241c7e69aa9da6c148f08216503d65 Mon Sep 17 00:00:00 2001 From: Krrish Mittal Date: Mon, 13 Jul 2026 18:26:51 -0700 Subject: [PATCH 3/8] ci(copilot): finalize smoke test with model-fallback + dispatch input --- .github/workflows/copilot-test-cli.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/copilot-test-cli.yml b/.github/workflows/copilot-test-cli.yml index 5383f34a371..e009e41b771 100644 --- a/.github/workflows/copilot-test-cli.yml +++ b/.github/workflows/copilot-test-cli.yml @@ -2,6 +2,11 @@ name: "Test Copilot CLI" on: workflow_dispatch: + inputs: + models: + description: "Space-separated model fallback chain to test" + required: false + default: "claude-opus-4.8 claude-opus-4.6 gpt-5.6" permissions: contents: read @@ -11,9 +16,6 @@ jobs: copilot-cli-smoke: runs-on: ubuntu-latest timeout-minutes: 6 - env: - # Ordered model-fallback chain: first model that responds wins. - COPILOT_MODELS: "claude-opus-4.8 claude-opus-4.6 gpt-5.6" steps: - uses: actions/checkout@v4 @@ -23,12 +25,10 @@ jobs: - name: Test CLI invocation (GITHUB_TOKEN installation auth + model fallback) env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # VERIFY-FALLBACK: bogus model forces the loop to fall through to a real one. - COPILOT_MODELS_OVERRIDE: "definitely-not-a-real-model claude-opus-4.8 claude-opus-4.6" + COPILOT_MODELS: ${{ github.event.inputs.models }} run: | - MODELS="${COPILOT_MODELS_OVERRIDE:-$COPILOT_MODELS}" - RESULT="" - USED="" + MODELS="${COPILOT_MODELS:-claude-opus-4.8 claude-opus-4.6 gpt-5.6}" + RESULT=""; USED="" for MODEL in $MODELS; do echo "::group::Trying model $MODEL" set +e From 01d718b608e2cdedc1b48c637c573174ad70f35d Mon Sep 17 00:00:00 2001 From: Krrish Mittal Date: Mon, 13 Jul 2026 18:48:46 -0700 Subject: [PATCH 4/8] fix(ci): use GITHUB_TOKEN installation auth + model fallback for all Copilot workflows Switch every direct Copilot CLI workflow from COPILOT_GITHUB_TOKEN (the per-user PAT path, which only authenticates for PR authors who personally hold a Copilot seat) to the plain GITHUB_TOKEN installation path with --yolo. This authenticates as the installation and is metered to the org, so it works for every author, fork, and bot with nothing to rotate. Add an ordered model-fallback loop (claude-opus-4.8 -> claude-opus-4.6 -> gpt-5-mini): the first model that responds wins, so an unavailable primary no longer fails the run. Delete the redundant copilot-pr-validation.yml, whose validate-pr job collided with pr-ai-validation.yml and whose deterministic fallback hard-failed PRs. pr-ai-validation.yml (native Copilot primary) is kept. Rename the smoke-test job test -> copilot-cli-smoke and make it workflow_dispatch-only so it no longer collides with the unit-test check or runs on every PR. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/copilot-design-pass.yml | 18 +- .github/workflows/copilot-flake-review.yml | 12 +- .github/workflows/copilot-investigate.yml | 13 +- .github/workflows/copilot-pr-validation.yml | 324 -------------------- .github/workflows/copilot-test-cli.yml | 4 +- .github/workflows/pr-ai-validation.yml | 15 +- 6 files changed, 43 insertions(+), 343 deletions(-) delete mode 100644 .github/workflows/copilot-pr-validation.yml diff --git a/.github/workflows/copilot-design-pass.yml b/.github/workflows/copilot-design-pass.yml index 1e0f6bbc18a..db1f9613fff 100644 --- a/.github/workflows/copilot-design-pass.yml +++ b/.github/workflows/copilot-design-pass.yml @@ -125,13 +125,17 @@ jobs: printf '%s' "$FULL_PROMPT" > "$RUNNER_TEMP/full-prompt.txt" set +e - RESULT=$(COPILOT_GITHUB_TOKEN="$GITHUB_TOKEN" timeout 840 copilot \ - -f "$RUNNER_TEMP/full-prompt.txt" \ - -s \ - --no-ask-user \ - --model claude-opus-4.8 \ - --allow-all 2>/dev/null) - EXIT_CODE=$? + MODELS="${COPILOT_MODELS:-claude-opus-4.8 claude-opus-4.6 gpt-5-mini}" + RESULT=""; EXIT_CODE=1 + for MODEL in $MODELS; do + RESULT=$(timeout 840 copilot --yolo \ + -f "$RUNNER_TEMP/full-prompt.txt" \ + --model "$MODEL" 2>/dev/null) + EXIT_CODE=$? + [ $EXIT_CODE -eq 0 ] && [ -n "$RESULT" ] && break + echo "::warning::Copilot model $MODEL failed (exit $EXIT_CODE) — trying next" + RESULT=""; EXIT_CODE=1 + done set -e if [ $EXIT_CODE -ne 0 ] || [ -z "$RESULT" ]; then diff --git a/.github/workflows/copilot-flake-review.yml b/.github/workflows/copilot-flake-review.yml index 387707ad292..d60ecc1ea4e 100644 --- a/.github/workflows/copilot-flake-review.yml +++ b/.github/workflows/copilot-flake-review.yml @@ -138,9 +138,15 @@ jobs: TASK="The following tests are flaky (passed and failed across multiple runs). Fix them according to the system prompt guidelines. Flake report: $REPORT" - COPILOT_GITHUB_TOKEN="$GITHUB_TOKEN" copilot \ - -p "$TASK" \ - -s --no-ask-user --model claude-opus-4.8 --allow-all || true + MODELS="${COPILOT_MODELS:-claude-opus-4.8 claude-opus-4.6 gpt-5-mini}" + for MODEL in $MODELS; do + set +e + timeout 600 copilot --yolo -p "$TASK" --model "$MODEL" + CODE=$? + set -e + [ $CODE -eq 0 ] && break + echo "::warning::Copilot model $MODEL failed (exit $CODE) — trying next" + done || true - name: Check for changes and create PR env: diff --git a/.github/workflows/copilot-investigate.yml b/.github/workflows/copilot-investigate.yml index 3e6fbd9a679..9a92ea32793 100644 --- a/.github/workflows/copilot-investigate.yml +++ b/.github/workflows/copilot-investigate.yml @@ -49,9 +49,16 @@ jobs: TASK="Investigate issue #${ISSUE_NUMBER} in repo ${{ github.repository }}. Title: ${ISSUE_TITLE}. Body: ${ISSUE_BODY}. Search the codebase to find relevant files. Check git log for recent changes. Post your investigation notes as a comment on the issue using: gh issue comment ${ISSUE_NUMBER} --body-file /tmp/investigation.md" - COPILOT_GITHUB_TOKEN="$GITHUB_TOKEN" copilot \ - -p "$TASK" \ - -s --no-ask-user --model claude-opus-4.8 --allow-all + # GITHUB_TOKEN installation auth (org-billed) + ordered model fallback. + MODELS="${COPILOT_MODELS:-claude-opus-4.8 claude-opus-4.6 gpt-5-mini}" + for MODEL in $MODELS; do + set +e + timeout 600 copilot --yolo -p "$TASK" --model "$MODEL" + CODE=$? + set -e + [ $CODE -eq 0 ] && break + echo "::warning::Copilot model $MODEL failed (exit $CODE) — trying next" + done - name: Add investigated label env: diff --git a/.github/workflows/copilot-pr-validation.yml b/.github/workflows/copilot-pr-validation.yml deleted file mode 100644 index a98580c008e..00000000000 --- a/.github/workflows/copilot-pr-validation.yml +++ /dev/null @@ -1,324 +0,0 @@ -name: "Copilot: PR Validation" - -# Validates PR title, body, labels, and risk assessment using Copilot CLI. -# This runs ALONGSIDE pr-ai-validation.yml during migration. Once proven reliable, -# the old Logic App-based workflow will be removed in a follow-up PR. -# -# Uses a different comment marker (copilot-pr-validation-v2) so both comments coexist. - -on: - pull_request_target: - types: [opened, edited, synchronize, labeled, unlabeled] - -permissions: - contents: read - pull-requests: write - issues: write - copilot-requests: write - -concurrency: - group: copilot-pr-validation-${{ github.event.pull_request.number }} - cancel-in-progress: true - -jobs: - validate-pr: - runs-on: ubuntu-latest - timeout-minutes: 5 - steps: - - name: Check if contributor is trusted - id: trust-check - uses: actions/github-script@v7 - with: - script: | - if (!context.payload.pull_request) { - core.info('No pull request context. Skipping.'); - core.setOutput('should_run', 'false'); - return; - } - - const authorAssociation = context.payload.pull_request.author_association; - const trustedAssociations = ['COLLABORATOR', 'MEMBER', 'OWNER', 'CONTRIBUTOR']; - - if (trustedAssociations.includes(authorAssociation)) { - core.info('Trusted contributor (' + authorAssociation + '), proceeding.'); - core.setOutput('should_run', 'true'); - return; - } - - const labels = context.payload.pull_request.labels.map(l => l.name); - if (labels.includes('external-approved')) { - core.info('External PR approved by maintainer via label, proceeding.'); - core.setOutput('should_run', 'true'); - return; - } - - core.info('External contributor (' + authorAssociation + ') - skipping validation.'); - core.setOutput('should_run', 'false'); - - - name: Checkout base branch - if: steps.trust-check.outputs.should_run == 'true' - uses: actions/checkout@v4 - - - name: Install Copilot CLI - if: steps.trust-check.outputs.should_run == 'true' - run: npm install -g @github/copilot@latest - - - name: Gather PR data - if: steps.trust-check.outputs.should_run == 'true' - id: gather - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PR_NUMBER: ${{ github.event.pull_request.number }} - run: | - gh pr view "$PR_NUMBER" --json title,body,labels,headRefName,author \ - > "$RUNNER_TEMP/pr.json" - - gh pr diff "$PR_NUMBER" --name-only \ - > "$RUNNER_TEMP/files.txt" 2>/dev/null || true - - gh pr diff "$PR_NUMBER" 2>/dev/null \ - | head -4000 > "$RUNNER_TEMP/diff.txt" || true - - - name: Build agent input - if: steps.trust-check.outputs.should_run == 'true' - run: | - cat > "$RUNNER_TEMP/pr-input.md" << 'ENDINPUT' - # PR Data for Validation - - ## PR Metadata - ENDINPUT - - printf '```json\n' >> "$RUNNER_TEMP/pr-input.md" - cat "$RUNNER_TEMP/pr.json" >> "$RUNNER_TEMP/pr-input.md" - printf '\n```\n\n## Changed Files\n```\n' >> "$RUNNER_TEMP/pr-input.md" - cat "$RUNNER_TEMP/files.txt" >> "$RUNNER_TEMP/pr-input.md" - printf '\n```\n\n## Code Diff (first 4000 lines)\n```diff\n' >> "$RUNNER_TEMP/pr-input.md" - cat "$RUNNER_TEMP/diff.txt" >> "$RUNNER_TEMP/pr-input.md" - printf '\n```\n\nValidate this PR against the rules in your system prompt. Return JSON only.\n' >> "$RUNNER_TEMP/pr-input.md" - - - name: Run Copilot validation - if: steps.trust-check.outputs.should_run == 'true' - id: validate - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - SYSTEM_PROMPT=$(cat prompts/copilot-pr-desc-check.md) - - PR_INPUT=$(cat "$RUNNER_TEMP/pr-input.md") - - FULL_PROMPT="${SYSTEM_PROMPT} - - --- - - ${PR_INPUT}" - - # Write prompt to temp file to avoid shell quoting issues - printf '%s' "$FULL_PROMPT" > "$RUNNER_TEMP/full-prompt.txt" - - set +e - RESULT=$(COPILOT_GITHUB_TOKEN="$GITHUB_TOKEN" timeout 240 copilot \ - -f "$RUNNER_TEMP/full-prompt.txt" \ - -s \ - --no-ask-user \ - --model claude-opus-4.8 \ - --allow-all 2>/dev/null) - EXIT_CODE=$? - set -e - - if [ $EXIT_CODE -ne 0 ] || [ -z "$RESULT" ]; then - echo "agent_success=false" >> "$GITHUB_OUTPUT" - echo "::warning::Copilot CLI returned empty or failed (exit $EXIT_CODE)" - exit 0 - fi - - # Extract JSON from response (strip markdown wrapping if present) - echo "$RESULT" | sed -n '/{/,/}$/p' > "$RUNNER_TEMP/result.json" - - if python3 -c "import json; json.load(open('$RUNNER_TEMP/result.json'))" 2>/dev/null; then - echo "agent_success=true" >> "$GITHUB_OUTPUT" - else - echo "$RESULT" > "$RUNNER_TEMP/result.json" - if python3 -c "import json; json.load(open('$RUNNER_TEMP/result.json'))" 2>/dev/null; then - echo "agent_success=true" >> "$GITHUB_OUTPUT" - else - echo "agent_success=false" >> "$GITHUB_OUTPUT" - echo "::warning::Agent returned invalid JSON" - fi - fi - - - name: Post comment and manage labels - if: steps.trust-check.outputs.should_run == 'true' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PR_NUMBER: ${{ github.event.pull_request.number }} - PR_TITLE: ${{ github.event.pull_request.title }} - HEAD_REF: ${{ github.head_ref }} - PR_AUTHOR: ${{ github.event.pull_request.user.login }} - REPO: ${{ github.repository }} - AGENT_SUCCESS: ${{ steps.validate.outputs.agent_success }} - run: | - MARKER="" - - # Find existing v2 comment - EXISTING=$(gh api "repos/${REPO}/issues/${PR_NUMBER}/comments" \ - --jq '[.[] | select(.body | contains("copilot-pr-validation-v2")) | .id][0] // empty' \ - 2>/dev/null) || EXISTING="" - - post_comment() { - local body_file="$1" - if [ -n "$EXISTING" ]; then - gh api "repos/${REPO}/issues/comments/${EXISTING}" \ - --method PATCH --input "$body_file" >/dev/null 2>&1 - else - gh pr comment "$PR_NUMBER" --body-file "$body_file" >/dev/null 2>&1 - fi - } - - add_label() { - gh api "repos/${REPO}/issues/${PR_NUMBER}/labels" \ - --method POST -f "labels[]=$1" >/dev/null 2>&1 || true - } - - remove_label() { - gh api "repos/${REPO}/issues/${PR_NUMBER}/labels/$1" \ - --method DELETE >/dev/null 2>&1 || true - } - - TIMESTAMP=$(date -u '+%a, %d %b %Y %H:%M:%S GMT') - - if [ "$AGENT_SUCCESS" = "true" ]; then - PASSES=$(python3 -c " - import json - r = json.load(open('$RUNNER_TEMP/result.json')) - print('true' if r.get('passes', False) else 'false') - ") - ADVISED=$(python3 -c " - import json - r = json.load(open('$RUNNER_TEMP/result.json')) - print(r.get('advisedRiskLevel', 'unknown')) - ") - MESSAGE=$(python3 -c " - import json, sys - r = json.load(open('$RUNNER_TEMP/result.json')) - sys.stdout.write(r.get('message', 'No details available.')) - ") - - cat > "$RUNNER_TEMP/comment-body.txt" << ENDCOMMENT - ${MARKER} - ## AI PR Validation Report - - ${MESSAGE} - - --- - **Advised Risk Level:** ${ADVISED} - - *Last updated: ${TIMESTAMP} | Powered by Copilot CLI* - ENDCOMMENT - - post_comment "$RUNNER_TEMP/comment-body.txt" - - if [ "$PASSES" = "true" ]; then - add_label "pr-validated" - remove_label "needs-pr-update" - else - add_label "needs-pr-update" - remove_label "pr-validated" - echo "::error::PR validation failed. Update your PR per the comment feedback." - exit 1 - fi - else - # Copilot unavailable — run deterministic fallback - - # Auto-pass reverts, bots, and loc branches - IS_AUTOPASS="false" - if echo "$PR_TITLE" | grep -qiE '^revert'; then IS_AUTOPASS="true"; fi - if echo "$HEAD_REF" | grep -qE '^(locfiles|loc)/'; then IS_AUTOPASS="true"; fi - if echo "$PR_AUTHOR" | grep -qE '\[bot\]$'; then IS_AUTOPASS="true"; fi - - if [ "$IS_AUTOPASS" = "true" ]; then - cat > "$RUNNER_TEMP/comment-body.txt" << ENDCOMMENT - ${MARKER} - ## AI PR Validation Report - - Auto-pass (${PR_AUTHOR}) - - *Last updated: ${TIMESTAMP} | Copilot unavailable, using deterministic checks* - ENDCOMMENT - post_comment "$RUNNER_TEMP/comment-body.txt" - exit 0 - fi - - # Deterministic validation - BODY=$(gh pr view "$PR_NUMBER" --json body --jq '.body // ""' 2>/dev/null) || BODY="" - LABELS=$(gh pr view "$PR_NUMBER" --json labels \ - --jq '[.labels[].name] | join(",")' 2>/dev/null) || LABELS="" - FAILS="" - - # Title prefix check - if ! echo "$PR_TITLE" | grep -qE '^(feat|fix|chore|meta|refactor|perf|docs|test)(\(.+\))?:'; then - FAILS="${FAILS}- **PR Title** must start with feat:, fix:, chore:, etc. - " - fi - - # Risk label check - if ! echo "$LABELS" | grep -q "risk:"; then - FAILS="${FAILS}- **Risk Label** missing (risk:low, risk:medium, or risk:high) - " - fi - - # Commit Type check - COMMIT_CHECKED=$(echo "$BODY" \ - | sed -n '/^## Commit Type/,/^## /{/^## /d;p}' \ - | grep -c '\[x\]' 2>/dev/null) || COMMIT_CHECKED=0 - if [ "$COMMIT_CHECKED" -eq 0 ]; then - FAILS="${FAILS}- **Commit Type** not selected - " - fi - - # Risk Level section check - RISK_CHECKED=$(echo "$BODY" \ - | sed -n '/^## Risk Level/,/^## /{/^## /d;p}' \ - | grep -c '\[x\]' 2>/dev/null) || RISK_CHECKED=0 - if [ "$RISK_CHECKED" -eq 0 ]; then - FAILS="${FAILS}- **Risk Level** not selected in PR body - " - fi - - # What & Why check - WHAT_WHY=$(echo "$BODY" \ - | sed -n '/^## What & Why/,/^## /{/^## /d;p}' \ - | sed '/^/d' | tr -d '[:space:]') - if [ "${#WHAT_WHY}" -lt 20 ]; then - FAILS="${FAILS}- **What & Why** section is empty or too short - " - fi - - if [ -n "$FAILS" ]; then - cat > "$RUNNER_TEMP/comment-body.txt" << ENDCOMMENT - ${MARKER} - ## AI PR Validation Report - - ### Issues Found - - ${FAILS} - - Please update your PR to match the template. The check re-runs automatically on edit. - - *Last updated: ${TIMESTAMP} | Copilot unavailable, using deterministic checks* - ENDCOMMENT - post_comment "$RUNNER_TEMP/comment-body.txt" - echo "::error::PR validation failed (deterministic). See PR comment." - exit 1 - else - cat > "$RUNNER_TEMP/comment-body.txt" << ENDCOMMENT - ${MARKER} - ## AI PR Validation Report - - All checks pass. - - *Last updated: ${TIMESTAMP} | Copilot unavailable, using deterministic checks* - ENDCOMMENT - post_comment "$RUNNER_TEMP/comment-body.txt" - fi - fi diff --git a/.github/workflows/copilot-test-cli.yml b/.github/workflows/copilot-test-cli.yml index e009e41b771..17a302717a1 100644 --- a/.github/workflows/copilot-test-cli.yml +++ b/.github/workflows/copilot-test-cli.yml @@ -6,7 +6,7 @@ on: models: description: "Space-separated model fallback chain to test" required: false - default: "claude-opus-4.8 claude-opus-4.6 gpt-5.6" + default: "claude-opus-4.8 claude-opus-4.6 gpt-5-mini" permissions: contents: read @@ -27,7 +27,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} COPILOT_MODELS: ${{ github.event.inputs.models }} run: | - MODELS="${COPILOT_MODELS:-claude-opus-4.8 claude-opus-4.6 gpt-5.6}" + MODELS="${COPILOT_MODELS:-claude-opus-4.8 claude-opus-4.6 gpt-5-mini}" RESULT=""; USED="" for MODEL in $MODELS; do echo "::group::Trying model $MODEL" diff --git a/.github/workflows/pr-ai-validation.yml b/.github/workflows/pr-ai-validation.yml index b58ea68afdb..9f4e5e4ad16 100644 --- a/.github/workflows/pr-ai-validation.yml +++ b/.github/workflows/pr-ai-validation.yml @@ -113,10 +113,17 @@ jobs: } > /tmp/full-prompt.txt set +e - RESULT=$(COPILOT_GITHUB_TOKEN="$GITHUB_TOKEN" timeout 90 copilot \ - -p "$(cat /tmp/full-prompt.txt)" \ - -s --no-ask-user --model claude-opus-4.8 --allow-all 2>&1) - CLI_EXIT=$? + MODELS="${COPILOT_MODELS:-claude-opus-4.8 claude-opus-4.6 gpt-5-mini}" + RESULT=""; CLI_EXIT=1 + for MODEL in $MODELS; do + RESULT=$(timeout 90 copilot --yolo \ + -p "$(cat /tmp/full-prompt.txt)" \ + --model "$MODEL" 2>&1) + CLI_EXIT=$? + [ $CLI_EXIT -eq 0 ] && [ -n "$RESULT" ] && break + echo "::warning::Copilot model $MODEL failed (exit $CLI_EXIT) — trying next" + RESULT=""; CLI_EXIT=1 + done set -e if [ $CLI_EXIT -ne 0 ] || [ -z "$RESULT" ]; then From 1b2c3b483c3ed37a1b57ee8a6bf1f8162e4f8adf Mon Sep 17 00:00:00 2001 From: Krrish Mittal Date: Mon, 13 Jul 2026 18:53:47 -0700 Subject: [PATCH 5/8] chore(ci): remove Copilot CLI smoke test after verification The dispatch-only smoke workflow existed only to verify Copilot CLI auth, org billing policy, and model availability in Actions. That verification is complete; the workflow has no ongoing gating purpose, so remove it. This also eliminates the original 'test' check-name collision at the source (rather than renaming/neutralizing it). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/copilot-test-cli.yml | 52 -------------------------- 1 file changed, 52 deletions(-) delete mode 100644 .github/workflows/copilot-test-cli.yml diff --git a/.github/workflows/copilot-test-cli.yml b/.github/workflows/copilot-test-cli.yml deleted file mode 100644 index 17a302717a1..00000000000 --- a/.github/workflows/copilot-test-cli.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: "Test Copilot CLI" - -on: - workflow_dispatch: - inputs: - models: - description: "Space-separated model fallback chain to test" - required: false - default: "claude-opus-4.8 claude-opus-4.6 gpt-5-mini" - -permissions: - contents: read - copilot-requests: write - -jobs: - copilot-cli-smoke: - runs-on: ubuntu-latest - timeout-minutes: 6 - steps: - - uses: actions/checkout@v4 - - - name: Install Copilot CLI - run: npm install -g @github/copilot@latest - - - name: Test CLI invocation (GITHUB_TOKEN installation auth + model fallback) - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - COPILOT_MODELS: ${{ github.event.inputs.models }} - run: | - MODELS="${COPILOT_MODELS:-claude-opus-4.8 claude-opus-4.6 gpt-5-mini}" - RESULT=""; USED="" - for MODEL in $MODELS; do - echo "::group::Trying model $MODEL" - set +e - OUT=$(timeout 90 copilot --yolo \ - -p "Respond with exactly: COPILOT_CLI_WORKING" \ - --model "$MODEL" 2>&1) - CODE=$? - set -e - echo "$OUT" - echo "::endgroup::" - if [ $CODE -eq 0 ] && echo "$OUT" | grep -q "COPILOT_CLI_WORKING"; then - RESULT="$OUT"; USED="$MODEL"; break - fi - echo "::warning::Model $MODEL failed (exit $CODE) — trying next" - done - if [ -n "$RESULT" ]; then - echo "Copilot CLI is working (installation auth) via model: $USED" - else - echo "::error::Copilot CLI failed for all models: $MODELS" - exit 1 - fi From 8551f0d564ef801788aa6d54828d58a84eae009c Mon Sep 17 00:00:00 2001 From: Krrish Mittal Date: Tue, 14 Jul 2026 07:17:56 -0700 Subject: [PATCH 6/8] fix(ci): fail investigate step when all Copilot models fail Prevents the 'investigated' label from being applied when no investigation comment was posted because every fallback model failed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/copilot-investigate.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/copilot-investigate.yml b/.github/workflows/copilot-investigate.yml index 9a92ea32793..cce55dac49f 100644 --- a/.github/workflows/copilot-investigate.yml +++ b/.github/workflows/copilot-investigate.yml @@ -51,14 +51,18 @@ jobs: # GITHUB_TOKEN installation auth (org-billed) + ordered model fallback. MODELS="${COPILOT_MODELS:-claude-opus-4.8 claude-opus-4.6 gpt-5-mini}" + SUCCESS=0 for MODEL in $MODELS; do set +e timeout 600 copilot --yolo -p "$TASK" --model "$MODEL" CODE=$? set -e - [ $CODE -eq 0 ] && break + if [ $CODE -eq 0 ]; then SUCCESS=1; break; fi echo "::warning::Copilot model $MODEL failed (exit $CODE) — trying next" done + # Fail the step if every model failed, so the "investigated" label is not + # applied to an issue that never received an investigation comment. + [ $SUCCESS -eq 1 ] || { echo "::error::All Copilot models failed — no investigation posted"; exit 1; } - name: Add investigated label env: From ba87b905c21d723f97c4fb0abdd4c445c0be7ad4 Mon Sep 17 00:00:00 2001 From: Krrish Mittal Date: Tue, 14 Jul 2026 07:57:34 -0700 Subject: [PATCH 7/8] chore(ci): trigger validation re-run Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> From 2d55e1b89d929cd95ccd83c562ded8d7bbf39d19 Mon Sep 17 00:00:00 2001 From: Krrish Mittal Date: Tue, 14 Jul 2026 08:00:28 -0700 Subject: [PATCH 8/8] chore(ci): re-trigger checks Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>