Merge pull request #428 from wudidapaopao/cross_compile

Support cross-compilation for macOS

Author: auxten

.github/workflows/build_linux_arm64_wheels-gh.yml

@@ -23,7 +23,7 @@ on:
 
 jobs:
   build_universal_wheel:
-    name: Build Universal Wheel (Linux ARM64)
+    name: Build Universal Wheel (Linux arm64)
     runs-on: GH-Linux-ARM64
     if: ${{ !github.event.pull_request.draft }}
     steps:

.github/workflows/build_linux_x86_wheels.yml

@@ -1,4 +1,4 @@
-name: Build Linux X86
+name: Build Linux x86_64
 
 on:
   workflow_dispatch:

.github/workflows/build_macos_arm64_wheels.yml

@@ -21,10 +21,119 @@ on:
       - '**/*.md'
 
 jobs:
-  build_universal_wheel:
-    name: Build Universal Wheel (macOS ARM64)
+  build_universal_wheel_on_linux:
+    name: Build on Linux (cross-compile for macOS arm64)
+    runs-on: GH-Linux-ARM64
+    if: ${{ !github.event.pull_request.draft }}
+    timeout-minutes: 600
+    steps:
+      - name: Install Python build dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y make build-essential libssl-dev zlib1g-dev \
+            libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \
+            libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev \
+            libffi-dev liblzma-dev p7zip-full
+      - name: Upgrade Rust toolchain
+        run: |
+          rustup toolchain install nightly-2025-07-07
+          rustup default nightly-2025-07-07
+          rustup component add rust-src
+          rustc --version
+          cargo --version
+      - name: Install clang++ for Ubuntu
+        run: |
+          pwd
+          uname -a
+          wget https://apt.llvm.org/llvm.sh
+          chmod +x llvm.sh
+          sudo ./llvm.sh 19
+          which clang++-19
+          clang++-19 --version
+          sudo apt-get install -y make cmake ccache ninja-build yasm gawk wget
+          # Install WebAssembly linker (wasm-ld)
+          sudo apt-get install -y lld-19
+          # Create symlink for wasm-ld
+          if ! command -v wasm-ld &> /dev/null; then
+            sudo ln -sf /usr/bin/wasm-ld-19 /usr/bin/wasm-ld || true
+          fi
+          which wasm-ld || echo "wasm-ld not found in PATH"
+          ccache -s
+      - name: Update git
+        run: |
+          sudo add-apt-repository ppa:git-core/ppa -y
+          sudo apt-get update
+          sudo apt-get install -y git
+          git --version
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Update submodules
+        run: |
+          git submodule update --init --recursive --jobs 4
+      - name: ccache
+        uses: hendrikmuhs/[email protected]
+        with:
+          key: ubuntu-24.04-aarch64-cross-compile
+          max-size: 5G
+          append-timestamp: true
+      - name: remove old clang and link clang-19 to clang
+        run: |
+          sudo rm -f /usr/bin/clang || true
+          sudo ln -s /usr/bin/clang-19 /usr/bin/clang
+          sudo rm -f /usr/bin/clang++ || true
+          sudo ln -s /usr/bin/clang++-19 /usr/bin/clang++
+          which clang++
+          clang++ --version
+      - name: Run chdb/build_mac_on_linux.sh
+        timeout-minutes: 600
+        run: |
+          source ~/.cargo/env
+          bash ./chdb/build_mac_on_linux.sh arm64
+        continue-on-error: false
+      - name: Run chdb/build/build_static_lib_mac_on_linux.sh
+        timeout-minutes: 600
+        run: |
+          source ~/.cargo/env
+          bash ./chdb/build/build_static_lib_mac_on_linux.sh arm64
+        continue-on-error: false
+      - name: Check ccache statistics
+        run: |
+          ccache -s
+          ls -lh chdb
+          df -h
+      - name: Keep killall ccache and wait for ccache to finish
+        if: always()
+        run: |
+          sleep 60
+          while ps -ef | grep ccache | grep -v grep; do \
+            killall ccache; \
+            sleep 10; \
+          done
+      - name: Upload build artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: macos-arm64-build-artifacts
+          path: |
+            ./libchdb.so
+            ./libchdb.a
+            ./chdb/_chdb.abi3.so
+            ./chdb/libpybind11nonlimitedapi_stubs.dylib
+            ./chdb/libpybind11nonlimitedapi_chdb_3.8.dylib
+            ./chdb/libpybind11nonlimitedapi_chdb_3.9.dylib
+            ./chdb/libpybind11nonlimitedapi_chdb_3.10.dylib
+            ./chdb/libpybind11nonlimitedapi_chdb_3.11.dylib
+            ./chdb/libpybind11nonlimitedapi_chdb_3.12.dylib
+            ./chdb/libpybind11nonlimitedapi_chdb_3.13.dylib
+            ./chdb/libpybind11nonlimitedapi_chdb_3.14.dylib
+          retention-days: 1
+
+  test_on_macos:
+    name: Test on macOS arm64
     runs-on: macos-14-xlarge
+    needs: build_universal_wheel_on_linux
     if: ${{ !github.event.pull_request.draft }}
+    timeout-minutes: 600
     steps:
       - name: Check machine architecture
         run: |
@@ -39,17 +148,6 @@ jobs:
           else
             echo "This is an x86_64 (Intel) machine"
           fi
-      - name: Free up disk space (Initial)
-        run: |
-          # Clean Homebrew cache
-          brew cleanup -s 2>/dev/null || true
-          rm -rf "$(brew --cache)" 2>/dev/null || true
-          sudo rm -rf ~/Library/Developer/Xcode/DerivedData 2>/dev/null || true
-          sudo rm -rf ~/Library/Caches/com.apple.dt.Xcode 2>/dev/null || true
-          sudo rm -rf /Users/runner/Library/Android 2>/dev/null || true
-          sudo rm -rf /tmp/* 2>/dev/null || true
-          echo "=== Disk usage after cleanup ==="
-          df -h
       - name: Setup pyenv
         run: |
           curl https://pyenv.run | bash
@@ -97,39 +195,14 @@ jobs:
       - name: Remove /usr/local/bin/python3
         run: |
           sudo rm -f /usr/local/bin/python3
-      - name: Install clang++ for macOS
+      - name: Install go for macOS
         run: |
-          pwd
-          uname -a
-          export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1
           brew update
-          brew install ca-certificates lz4 mpdecimal readline sqlite xz z3 zstd
-          brew install openssl@3 || echo "OpenSSL install failed, continuing..."
-          brew install --ignore-dependencies llvm@19
-          brew install git ninja libtool gettext binutils grep findutils nasm lld@19 libiconv
-          brew install ccache || echo "ccache installation failed, continuing without it"
           brew install go
-          cd /usr/local/opt/ && sudo rm -f llvm && sudo ln -sf llvm@19 llvm
-          export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:$PATH
-          which clang++
-          clang++ --version
-          which wasm-ld || echo "wasm-ld not found in PATH"
-          which go
           go version
-          ccache -s | echo "ccache not available yet"
-      - name: Upgrade Rust toolchain
-        run: |
-          rustup toolchain install nightly-2025-07-07
-          rustup default nightly-2025-07-07
-          rustup component add rust-src
-          rustc --version
-          cargo --version
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
-      - name: Update submodules
-        run: |
-          git submodule update --init --recursive --jobs 4
       - name: Update version for release
         if: startsWith(github.ref, 'refs/tags/v')
         run: |
@@ -143,89 +216,34 @@ jobs:
           bump-my-version replace --new-version $TAG_NAME
           echo "Version files updated to $TAG_NAME"
           pyenv shell --unset
-      - name: Free up disk space (Before compilation)
-        run: |
-          echo "=== Disk usage before compilation cleanup ==="
-          df -h
-          brew cleanup -s 2>/dev/null || true
-          rm -rf "$(brew --cache)" 2>/dev/null || true
-          rm -rf ~/.cache/pip 2>/dev/null || true
-          rm -rf ~/.pyenv/.cache 2>/dev/null || true
-          rm -rf ~/.cargo/registry/cache 2>/dev/null || true
-          echo "=== Disk usage after cleanup ==="
-          df -h
-      - name: ccache
-        uses: hendrikmuhs/[email protected]
+      - name: Download build artifacts
+        uses: actions/download-artifact@v4
         with:
-          key: ${{ runner.os }}-arm64-ccache
-          max-size: 5G
-          append-timestamp: true
-      - name: Run chdb/build.sh
+          name: macos-arm64-build-artifacts
+          path: ./artifacts
+      - name: Restore artifacts to original paths
+        run: |
+          mv ./artifacts/libchdb.so ./
+          mv ./artifacts/libchdb.a ./
+          mv ./artifacts/chdb/_chdb.abi3.so ./chdb/
+          mv ./artifacts/chdb/libpybind11nonlimitedapi_stubs.dylib ./chdb/
+          for v in 8 9 10 11 12 13 14; do
+            mv ./artifacts/chdb/libpybind11nonlimitedapi_chdb_3.${v}.dylib ./chdb/
+          done
+          ls -lh ./libchdb.so ./libchdb.a
+          ls -lh ./chdb/*.so ./chdb/*.dylib
+      - name: Run chdb/test_smoke.sh
         timeout-minutes: 600
         run: |
           export PATH="$HOME/.pyenv/bin:$PATH"
           eval "$(pyenv init -)"
-          source ~/.cargo/env
           pyenv shell 3.8
-          export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin
-          export CC=$(brew --prefix llvm@19)/bin/clang
-          export CXX=$(brew --prefix llvm@19)/bin/clang++
           bash gen_manifest.sh
-          bash ./chdb/build.sh
-          pyenv shell 3.8
-          bash -x ./chdb/test_smoke.sh
-      - name: Run chdb/build/build_static_lib.sh
+          bash -x ./chdb/test_smoke.sh cross-compile
+      - name: Run chdb/build/test_go_example.sh
         timeout-minutes: 600
         run: |
-          export PATH=$HOME/.pyenv/bin:$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin
-          export CC=$(brew --prefix llvm@19)/bin/clang
-          export CXX=$(brew --prefix llvm@19)/bin/clang++
-          source ~/.cargo/env
-          eval "$(pyenv init -)"
-          pyenv shell 3.8
-          bash ./chdb/build/build_static_lib.sh
-          pyenv shell --unset
-        continue-on-error: false
-      - name: Debug libchdb
-        run: |
-          ls -lh
-          llvm-nm libchdb.so | grep query_stable || true
-          echo "Global Symbol in libchdb.so:"
-          llvm-nm -g libchdb.so || true
-          echo "Global Symbol in libclickhouse-local-chdb.a:"
-          llvm-nm -g buildlib/programs/local/libclickhouse-local-chdb.a || true
-          echo "Global Symbol in libclickhouse-local-lib.a:"
-          llvm-nm -g buildlib/programs/local/libclickhouse-local-lib.a || true
-          echo "pychdb_cmd.sh:"
-          cat buildlib/pychdb_cmd.sh
-          echo "libchdb_cmd.sh:"
-          cat buildlib/libchdb_cmd.sh
-      - name: Scan chdb libraries with grype
-        run: |
-          echo "Scanning chdb libraries for vulnerabilities..."
-          # Files to scan
-          FILES_TO_SCAN=""
-          [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so"
-          [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a"
-          FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.dylib" -o -name "*.so" \) 2>/dev/null || true)"
-          SQLITE_VULNERABILITIES_FOUND=false
-          for file in $FILES_TO_SCAN; do
-            if [ -f "$file" ]; then
-              echo "=== Scanning $file ==="
-              SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true)
-              echo "$SCAN_OUTPUT"
-              if echo "$SCAN_OUTPUT" | grep -qi sqlite; then
-                echo "❌ SQLite vulnerability found in $file"
-                SQLITE_VULNERABILITIES_FOUND=true
-              fi
-            fi
-          done
-          if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then
-            echo "❌ SQLite vulnerabilities detected in chdb libraries!"
-            exit 1
-          else
-            echo "✅ No SQLite vulnerabilities found in chdb libraries"
-          fi
+          bash ./chdb/build/test_go_example.sh ${{ github.workspace }}/libchdb.a
         continue-on-error: false
       - name: Run libchdb stub in examples dir
         run: |
@@ -236,9 +254,6 @@ jobs:
           rm -rf chdb/build/
           export PATH="$HOME/.pyenv/bin:$PATH"
           eval "$(pyenv init -)"
-          export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin
-          export CC=$(brew --prefix llvm@19)/bin/clang
-          export CXX=$(brew --prefix llvm@19)/bin/clang++
           pyenv shell 3.8
           make wheel
       - name: Fix wheel platform tag
@@ -257,12 +272,6 @@ jobs:
           sudo sysctl kern.corefile=$PWD/tmp/core/core.%P
           sudo sysctl kern.coredump=1
           ulimit -c unlimited
-      - name: Free up disk space
-        run: |
-          # Clean more build artifacts
-          rm -rf buildlib/contrib 2>/dev/null || true
-          rm -rf buildlib/base 2>/dev/null || true
-          rm -rf buildlib/src 2>/dev/null || true
       - name: Test wheel on all Python versions
         run: |
           ulimit -c unlimited
@@ -282,7 +291,7 @@ jobs:
         run: |
           export PATH="$HOME/.pyenv/bin:$PATH"
           eval "$(pyenv init -)"
-          pyenv shell 3.8
+          pyenv shell 3.9
           python -m pip install dist/*.whl --force-reinstall
           jupyter nbconvert --to notebook --execute tests/test_data_insertion.ipynb --output test_data_insertion_output.ipynb
           pyenv shell --unset
@@ -300,14 +309,6 @@ jobs:
             echo "No core files found in tmp/core"
           fi
         continue-on-error: true
-      - name: Keep killall ccache and wait for ccache to finish
-        if: always()
-        run: |
-          sleep 60
-          while ps -ef | grep ccache | grep -v grep; do \
-            killall ccache; \
-            sleep 10; \
-          done
       - name: Upload core files artifact
         if: always() && env.CORE_FILES_FOUND == 'true'
         uses: actions/upload-artifact@v4