trigger_eval.yml

name: Trigger Pilo Eval via Repository Dispatch

# Triggers evaluation in pilo-evals-judge via repository_dispatch event
# when pushing to evals/** branches

on:
  push:
    branches:
      - "evals/**"

permissions:
  contents: read

jobs:
  trigger-eval:
    name: Trigger Eval in pilo-evals-judge
    runs-on: ubuntu-latest
    steps:
      - name: Parse eval name from branch
        id: parse
        run: |
          BRANCH="${{ github.ref }}"
          echo "Branch: $BRANCH"

          # Extract eval name from branch: refs/heads/evals/{eval_name}/...
          if [[ "$BRANCH" =~ ^refs/heads/evals/([^/]+)/ ]]; then
            EVAL_NAME="${BASH_REMATCH[1]}"
            echo "eval_name=$EVAL_NAME" >> $GITHUB_OUTPUT
            echo "✅ Parsed eval name: $EVAL_NAME"
          else
            echo "❌ Error: Branch does not match evals/{eval_name}/* pattern"
            echo "   Expected format: evals/single/*, evals/partial/*, evals/full/*"
            exit 1
          fi

      - name: Trigger repository_dispatch
        env:
          COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
        run: |
          echo "Sending repository_dispatch event to pilo-evals-judge..."
          echo ""
          echo "Payload:"
          echo "  eval_name: ${{ steps.parse.outputs.eval_name }}"
          echo "  git_sha: ${{ github.sha }}"
          echo "  git_ref: ${{ github.ref }}"
          echo "  repo: ${{ github.repository }}"
          echo "  commit_timestamp: ${{ github.event.head_commit.timestamp }}"
          echo "  run_id: ${{ github.run_id }}"
          echo ""

          # Escape commit message for JSON (replace newlines, quotes, backslashes)
          COMMIT_MSG=$(echo "$COMMIT_MESSAGE" | jq -Rs .)

          # Generate current timestamp
          TRIGGER_TIMESTAMP=$(date -u +%Y-%m-%dT%H:%M:%SZ)

          # Create payload file with proper JSON escaping
          cat > payload.json <<EOF
          {
            "event_type": "pilo_eval",
            "client_payload": {
              "eval_name": "${{ steps.parse.outputs.eval_name }}",
              "git_sha": "${{ github.sha }}",
              "git_ref": "${{ github.ref }}",
              "repo": "${{ github.repository }}",
              "commit_timestamp": "${{ github.event.head_commit.timestamp }}",
              "commit_message": $COMMIT_MSG,
              "trigger_timestamp": "$TRIGGER_TIMESTAMP",
              "workflow_name": "${{ github.workflow }}",
              "run_id": "${{ github.run_id }}",
              "actor": "${{ github.actor }}"
            }
          }
          EOF

          # Send repository_dispatch event
          HTTP_STATUS=$(curl -s -o response.json -w "%{http_code}" \
            -X POST \
            -H "Accept: application/vnd.github+json" \
            -H "Authorization: Bearer ${{ secrets.PILO_EVALS_DISPATCH_TOKEN }}" \
            -H "X-GitHub-Api-Version: 2022-11-28" \
            https://api.github.com/repos/Mozilla-Ocho/pilo-evals-judge/dispatches \
            --data @payload.json)

          echo ""
          echo "GitHub API HTTP status: $HTTP_STATUS"

          # Check HTTP status code
          if [[ "$HTTP_STATUS" -lt 200 || "$HTTP_STATUS" -ge 300 ]]; then
            echo "❌ Error: Failed to send repository_dispatch event (HTTP $HTTP_STATUS)"
            if [[ -s response.json ]]; then
              echo "Response body:"
              cat response.json
            fi
            exit 1
          fi

          # Even with 200 OK, check for error in response body
          if [[ -s response.json ]]; then
            if grep -q '"message"' response.json || grep -q '"error"' response.json; then
              echo "⚠️  Warning: Response contains error/message field:"
              cat response.json
              # GitHub API sometimes returns errors with 200 OK
              if grep -qi '"Not Found"' response.json || grep -qi '"Bad credentials"' response.json; then
                echo "❌ Error: API call failed despite 200 OK status"
                exit 1
              fi
            fi
          fi

          echo ""
          echo "✅ Repository dispatch event sent"
          echo ""
          echo "Monitor the eval at:"
          echo "  https://github.com/Mozilla-Ocho/pilo-evals-judge/actions"
          echo ""
          echo "Check commit status at:"
          echo "  https://github.com/${{ github.repository }}/commit/${{ github.sha }}"