ci.yml

#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# This workflow is triggered for each push event to the master branch or pull request.
# It contains two jobs:
# - "Main Build" job runs all unit tests jor two LTS java versions and the latest Java version.
# - "Run checkstyle and generate protobufs" job builds Drill and checks its license,
# builds Drill native client and regenerates protobufs to ensure that committed files are up-to-date.
name: Github CI

on: [push, pull_request]

jobs:
  build:
    name: Main Build (java ${{ matrix.java }}, shard ${{ matrix.shard }})
    runs-on: ubuntu-latest
    if: github.repository == 'apache/drill'
    timeout-minutes: 150
    strategy:
      matrix:
        # Java versions to run unit tests (Jetty 12 requires Java 17+)
        java: [ '17', '21', '25' ]
        profile: ['default-hadoop']
        # exec/java-exec holds ~31 minutes of the suite in 577 classes, which dominates
        # the build. Splitting it three ways by class name brings the longest shard to
        # about 11 minutes. Boundaries were chosen from measured per-class times; the
        # patterns are exhaustive, so a newly added test always lands in exactly one
        # shard rather than being silently skipped.
        shard: [ '1', '2', '3' ]
      fail-fast: false
    steps:
      - name: Checkout
        uses: actions/checkout@v7
      - name: Setup java
        uses: actions/setup-java@v6
        with:
          distribution: 'temurin'
          java-version: ${{ matrix.java }}
          cache: 'maven'
      - name: Set up swap space
        # Linux Action Runners come with 7GB of RAM which isn't quite enoough
        # to run the test suite. Two workarounds are used below: a swap file is
        # added to the Runner and memory hungry tests are run separately.
        # dd rather than fallocate: swapon rejects fallocated files on some runner
        # filesystems ("Invalid argument"). A missing swap file only costs headroom,
        # so warn instead of failing the job.
        run: |
          sudo sh -c "
            dd if=/dev/zero of=/drill-swapfile bs=1M count=2048 status=none &&
            chmod 0600 /drill-swapfile &&
            mkswap /drill-swapfile &&
            swapon /drill-swapfile
          " || echo "::warning::Could not add swap space; continuing without it"
      # Which slice of the suite this job runs. The patterns are spelled out one prefix
      # at a time because surefire does NOT support character-class globs:
      # -Dtest='Test[A-F]*' silently matches nothing and the build passes green having
      # run no tests. Only '*' wildcards and comma separated lists work, so each letter
      # gets its own entry, in both cases (TestforBaseTestInheritance has a lowercase f).
      #
      # Shard 3 repeats surefire's other default includes (*Test, *Tests, *TestCase) to
      # pick up classes not named Test*. Verified exhaustive against every test
      # class in the repo. TestFrameworkTest matches shards 1 and 3 and runs twice —
      # harmless, and far safer than a gap that skips tests without saying so.
      - name: Select test shard
        id: shard
        run: |
          case "${{ matrix.shard }}" in
            1) echo 'pattern=TestA*,Testa*,TestB*,Testb*,TestC*,Testc*,TestD*,Testd*,TestE*,Teste*,TestF*,Testf*' >> "$GITHUB_OUTPUT" ;;
            2) echo 'pattern=TestG*,Testg*,TestH*,Testh*,TestI*,Testi*,TestJ*,Testj*,TestK*,Testk*,TestL*,Testl*,TestM*,Testm*,TestN*,Testn*,TestO*,Testo*,TestP*,Testp*' >> "$GITHUB_OUTPUT" ;;
            3) echo 'pattern=TestQ*,Testq*,TestR*,Testr*,TestS*,Tests*,TestT*,Testt*,TestU*,Testu*,TestV*,Testv*,TestW*,Testw*,TestX*,Testx*,TestY*,Testy*,TestZ*,Testz*,*Test,*Tests,*TestCase,Test0*,Test1*,Test2*,Test3*,Test4*,Test5*,Test6*,Test7*,Test8*,Test9*,Test_*' >> "$GITHUB_OUTPUT" ;;
            *) echo "::error::Unknown shard '${{ matrix.shard }}'; expected 1, 2 or 3"; exit 1 ;;
          esac

      # -DforkCount=2: several of the slowest classes are wall-clock bound rather than
      # CPU bound — TestTimedCallable spends 105 seconds asleep, and the RPC, SSL and
      # graceful-shutdown tests wait on timeouts — so a second fork overlaps that idle
      # time at little extra cost. Raise further only with memory headroom: the vector
      # tests fail with OutOfMemoryException under pressure.
      - name: Build and test
        run: |
          mvn -P${{ matrix.profile }} install --batch-mode --no-transfer-progress \
          -DforkCount=2 \
          -Dsurefire.failIfNoSpecifiedTests=false \
          -Dtest='${{ steps.shard.outputs.pattern }}' \
          -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120
      - name: Remove swap space
        run : |
          free -h
          sudo swapoff /drill-swapfile 2>/dev/null || true
          sudo rm -f /drill-swapfile
  checkstyle_protobuf:
    name: Run checkstyle and generate protobufs
    runs-on: ubuntu-22.04
    if: github.repository == 'apache/drill'
    steps:
      - name: Checkout
        uses: actions/checkout@v7
      - name: Setup java
        uses: actions/setup-java@v6
        with:
          distribution: 'temurin'
          java-version: '17'
          cache: 'maven'
      # Caches built protobuf library
      - name: Cache protobufs
        uses: actions/cache@v6
        with:
          path: ~/protobuf
          key: ${{ runner.os }}-protobuf
      # Install libraries required for protobuf generation
      - name: Install dependencies
        run: |
          sudo apt update -y && sudo apt install -y libboost-all-dev libzookeeper-mt-dev libsasl2-dev cmake libcppunit-dev checkinstall && \
          pushd .. && \
          if [ -f $HOME/protobuf/protobuf_3.16.3* ]; then \
            sudo dpkg -i $HOME/protobuf/protobuf_3.16.3*; \
          else \
            wget https://github.com/protocolbuffers/protobuf/releases/download/v3.16.3/protobuf-java-3.16.3.zip && \
            unzip protobuf-java-3.16.3.zip && pushd protobuf-3.16.3 && \
            ./configure && sudo make && sudo checkinstall -y && \
            if [ ! -d $HOME/protobuf ]; then \
              mkdir -p $HOME/protobuf; \
            fi && \
            mv protobuf_3.16.3* $HOME/protobuf/ && popd; \
          fi && \
          sudo ldconfig && popd; \
      # Builds Drill project, performs license checkstyle goal and regenerates java and C++ protobuf files
      - name: Build
        run: |
          MAVEN_OPTS="-Xms1G -Xmx1G" mvn install -Drat.skip=false -Dlicense.skip=false --batch-mode --no-transfer-progress -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -DskipTests=true -Dmaven.javadoc.skip=true -Dmaven.source.skip=true && \
          pushd protocol && mvn process-sources -P proto-compile && popd && \
          mkdir contrib/native/client/build && pushd contrib/native/client/build && cmake -G "Unix Makefiles" .. && make cpProtobufs && popd; \
      # Checks whether project files weren't changed after regenerating protobufs
      - name: Check protobufs
        run: |
          if [ "$(git status -s | grep -c "")" -gt 0 ]; then \
            echo "The following changes are found in files after regenerating protobufs (output may be used as a patchto apply):" >&2 &&
            echo "$(git diff --color)" && \
            exit 1; \
          else
            echo "All checks are passed!";
          fi