rps-ab.yml

# Same-job paired A/B for Titanium RPS work (workflow_dispatch only; Linux only).
#
# Why: two rps-saturation dispatches land on two different VMs and runner variance (~5%) is larger than
# the gains left on the Linux gap list. This job checks out a baseline and a candidate product ref on ONE
# ubuntu-latest VM, builds the SAME probe harness (tools/RpsLoadProbe from the dispatch ref) against each,
# and runs them alternately baseline, candidate, baseline, candidate ... for >= 5 pairs with a cool-down
# between runs. Paired deltas, a 95% confidence interval and a keep gate are written to the job summary.
#
# Dispatch the workflow on the harness ref (usually the candidate branch):
#   gh workflow run rps-ab.yml --ref <branch-or-sha> \
#     -f baseline_ref=develop -f candidate_ref=<branch-or-sha> \
#     -f mode=compare-ceiling -f arm_contains=twp-reverse-http1 -f pairs=5
#
# Only product code (everything outside tools/RpsLoadProbe) differs between the two sides.
# Peers (nginx / HAProxy / Envoy) are not installed here; keep arm_contains on Titanium (twp-) or bare- arms.

name: RPS paired A/B

on:
  workflow_dispatch:
    inputs:
      baseline_ref:
        description: 'Baseline product ref (branch, tag or SHA)'
        required: true
        default: develop
      candidate_ref:
        description: 'Candidate product ref (branch, tag or SHA)'
        required: false
        default: ''
      mode:
        description: 'Probe ramp mode (see rps-saturation.yml for the list)'
        required: true
        default: compare-ceiling
      arm_contains:
        description: 'Arm-name substring filter (applied to the arms of the mode)'
        required: true
        default: 'twp-reverse-http1'
      concurrency:
        description: 'Concurrency (one value; paired runs use a fixed step)'
        required: true
        default: '32'
      duration_sec:
        description: 'Measure seconds'
        required: true
        default: '8'
      pairs:
        description: 'Number of baseline/candidate pairs (>= 5 for a keep decision)'
        required: true
        default: '5'
      cooldown_sec:
        description: 'Idle seconds between consecutive runs'
        required: true
        default: '15'
      order:
        description: 'alternate = A,B,A,B ; counterbalance = A,B,B,A,A,B (cancels linear drift)'
        required: true
        default: alternate
        type: choice
        options:
          - alternate
          - counterbalance
      probe_args:
        description: 'Extra probe arguments, space separated; later flags win (e.g. "--response-bytes 65536 --warmup-sec 3 --arm-shard 1/3")'
        required: false
        default: ''

permissions:
  contents: read

jobs:
  ab:
    runs-on: ubuntu-latest
    timeout-minutes: 300
    env:
      AB_BASELINE_REF: ${{ inputs.baseline_ref }}
      AB_CANDIDATE_REF: ${{ inputs.candidate_ref }}
      AB_MODE: ${{ inputs.mode }}
      AB_ARM_CONTAINS: ${{ inputs.arm_contains }}
      AB_CONCURRENCY: ${{ inputs.concurrency }}
      AB_DURATION: ${{ inputs.duration_sec }}
      AB_PAIRS: ${{ inputs.pairs }}
      AB_COOLDOWN: ${{ inputs.cooldown_sec }}
      AB_ORDER: ${{ inputs.order }}
      AB_PROBE_ARGS: ${{ inputs.probe_args }}
    steps:
      - uses: actions/checkout@v7
        with:
          fetch-depth: 0

      - uses: actions/setup-dotnet@v6
        with:
          dotnet-version: |
            10.0.x

      - name: Log runner shape
        run: |
          set -euo pipefail
          echo "os=$(uname -s) $(uname -r)"
          echo "nproc=$(nproc)"
          lscpu | sed -n '1,20p'

      - name: Install libmsquic (HTTP/3 arms)
        run: |
          set -euo pipefail
          . /etc/os-release
          curl --fail --silent --show-error --location --proto '=https' --tlsv1.2 \
            "https://packages.microsoft.com/config/${ID}/${VERSION_ID}/packages-microsoft-prod.deb" \
            -o packages-microsoft-prod.deb
          sudo dpkg -i packages-microsoft-prod.deb
          rm -f packages-microsoft-prod.deb
          sudo find /etc/apt -type f \( -iname '*chrome*' -o -iname '*google*chrome*' \) -print -delete || true
          sudo apt-get update || true
          sudo apt-get install -y libmsquic

      - name: Check out baseline and candidate worktrees
        run: |
          set -euo pipefail
          git fetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' || true
          resolve() {
            git rev-parse --verify --quiet "$1^{commit}" \
              || git rev-parse --verify --quiet "origin/$1^{commit}"
          }
          base_sha="$(resolve "$AB_BASELINE_REF")" || { echo "cannot resolve baseline_ref=$AB_BASELINE_REF" >&2; exit 1; }
          cand_ref="${AB_CANDIDATE_REF:-$GITHUB_SHA}"
          cand_sha="$(resolve "$cand_ref")" || { echo "cannot resolve candidate_ref=$cand_ref" >&2; exit 1; }
          echo "baseline  $AB_BASELINE_REF -> $base_sha"
          echo "candidate $cand_ref -> $cand_sha"
          echo "harness   $GITHUB_SHA (tools/RpsLoadProbe overlaid on both sides)"
          git worktree add --detach "$RUNNER_TEMP/ab/baseline" "$base_sha"
          git worktree add --detach "$RUNNER_TEMP/ab/candidate" "$cand_sha"
          {
            echo "AB_BASELINE_SHA=$base_sha"
            echo "AB_CANDIDATE_SHA=$cand_sha"
          } >> "$GITHUB_ENV"

      - name: Build the identical probe against each product ref
        run: |
          set -euo pipefail
          for side in baseline candidate; do
            wt="$RUNNER_TEMP/ab/$side"
            rm -rf "$wt/tools/RpsLoadProbe"
            cp -r tools/RpsLoadProbe "$wt/tools/RpsLoadProbe"
            dotnet build -c Release "$wt/tools/RpsLoadProbe/RpsLoadProbe.csproj" --warnaserror
            mkdir -p "$RUNNER_TEMP/ab-bin"
            cp -r "$wt/tools/RpsLoadProbe/bin/Release/net10.0" "$RUNNER_TEMP/ab-bin/$side"
          done
          ls -la "$RUNNER_TEMP/ab-bin/baseline/RpsLoadProbe" "$RUNNER_TEMP/ab-bin/candidate/RpsLoadProbe"

      - name: Paired A/B ramp
        run: |
          set -euo pipefail
          ulimit -n 65535 || true
          extra=()
          if [ -n "${AB_PROBE_ARGS:-}" ]; then
            # shellcheck disable=SC2206
            for a in $AB_PROBE_ARGS; do extra+=(--probe-arg "$a"); done
          fi
          python3 tools/RpsLoadProbe/rps-ab-run.py \
            --baseline-probe "$RUNNER_TEMP/ab-bin/baseline/RpsLoadProbe" \
            --candidate-probe "$RUNNER_TEMP/ab-bin/candidate/RpsLoadProbe" \
            --mode "$AB_MODE" --arm-contains "$AB_ARM_CONTAINS" \
            --concurrency "$AB_CONCURRENCY" --duration-sec "$AB_DURATION" \
            --pairs "$AB_PAIRS" --cooldown-sec "$AB_COOLDOWN" --order "$AB_ORDER" \
            "${extra[@]}" --out "$RUNNER_TEMP/ab-out"

      - name: Analyze paired deltas
        if: always()
        run: |
          set -euo pipefail
          test -f "$RUNNER_TEMP/ab-out/manifest.json" || exit 0
          {
            echo "baseline \`$AB_BASELINE_REF\` = \`$AB_BASELINE_SHA\`  "
            echo "candidate \`${AB_CANDIDATE_REF:-$GITHUB_SHA}\` = \`$AB_CANDIDATE_SHA\`  "
            echo "harness (identical on both sides) = \`$GITHUB_SHA\`"
            echo
          } > "$RUNNER_TEMP/ab-out/summary.md"
          python3 tools/RpsLoadProbe/rps-ab-analyze.py "$RUNNER_TEMP/ab-out" --markdown "$RUNNER_TEMP/ab-out/analysis.md" > /dev/null
          cat "$RUNNER_TEMP/ab-out/analysis.md" >> "$RUNNER_TEMP/ab-out/summary.md"
          cat "$RUNNER_TEMP/ab-out/summary.md" >> "$GITHUB_STEP_SUMMARY"
          cat "$RUNNER_TEMP/ab-out/summary.md"

      - name: Upload A/B artifacts
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: rps-ab-${{ github.run_id }}
          path: |
            ${{ runner.temp }}/ab-out/**/*.csv
            ${{ runner.temp }}/ab-out/**/*.tsv
            ${{ runner.temp }}/ab-out/**/run.log
            ${{ runner.temp }}/ab-out/manifest.json
            ${{ runner.temp }}/ab-out/summary.md
          if-no-files-found: error