diff --git a/.github/scripts/build-musllinux-arm64.sh b/.github/scripts/build-musllinux-arm64.sh new file mode 100644 index 00000000000..a32392aeeb3 --- /dev/null +++ b/.github/scripts/build-musllinux-arm64.sh @@ -0,0 +1,282 @@ +#!/bin/sh +set -e + +cd /workspace + +# Configure git safe directory in container +apk update +apk add --no-cache git python3 py3-pip py3-setuptools +echo "=== Configure git safe directory ===" +git config --global --add safe.directory /workspace +git describe --tags +python3 -c "import sys; sys.path.append('.'); from setup import get_latest_git_tag; print('version:', get_latest_git_tag())" + +# 1. Check system info +echo "=== Container System Info ===" +echo "System: $(uname -m) $(cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2)" +if [ -f /lib/ld-musl-aarch64.so.1 ]; then + echo "musl libc aarch64" +elif [ -f /lib/libc.musl-aarch64.so.1 ]; then + echo "musl libc aarch64" +else + echo "Not musl libc" +fi +echo "Workspace mounted at: /workspace" +ls -la /workspace + +# 2. Install build dependencies +echo "=== Installing build dependencies ===" +apk add --no-cache make build-base openssl-dev zlib-dev \ + bzip2-dev readline-dev sqlite-dev wget curl llvm \ + ncurses-dev xz-dev tk-dev libxml2-dev \ + libffi-dev linux-headers +apk add --no-cache make cmake ccache ninja yasm gawk +apk add --no-cache clang20 clang20-dev llvm20 llvm20-dev lld20 + +# 3. Scan SQLite vulnerabilities +echo "=== Scanning SQLite vulnerabilities ===" +# Install grype +curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin +grype db update + +# Check SQLite vulnerabilities +echo "Scanning SQLite packages for vulnerabilities..." +GRYPE_RAW_OUTPUT=$(grype dir:/lib/apk/db --scope all-layers 2>/dev/null || true) +echo "Raw grype output:" +echo "$GRYPE_RAW_OUTPUT" + +SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) +if [ -n "$SQLITE_SCAN_OUTPUT" ]; then + echo "SQLite vulnerabilities found in packages! Build should be reviewed." + echo "SQLite vulnerability details:" + echo "$SQLITE_SCAN_OUTPUT" +else + echo "No SQLite vulnerabilities found" +fi + +# 4. Setup Python environments +echo "=== Setting up Python environments ===" +# Setup pyenv +curl https://pyenv.run | bash +export PATH="$HOME/.pyenv/bin:$PATH" +eval "$(pyenv init -)" + +# Install Python versions +for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do + echo "Installing Python $version" + pyenv install $version:latest +done +pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 3.14 + +# Verify installations +echo "Installed versions:" +pyenv versions +for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do + if ! pyenv versions --bare | grep -q "^$version"; then + echo "ERROR: Python $version is not installed!" + exit 1 + fi + echo "Python $version is installed" +done +echo "All Python versions verified successfully!" + +# Install Rust +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable +source $HOME/.cargo/env +rustup toolchain install nightly-2025-07-07 +rustup component add --toolchain nightly-2025-07-07 rust-src + +# Install Python dependencies +for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do + echo "Installing dependencies for Python $version" + pyenv shell $version + python -m pip install --upgrade pip + if [ "$version" = "3.8" ]; then + python -m pip install setuptools tox twine psutil wheel + else + python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel + fi + pyenv shell --unset +done + +# Update version for release (if triggered by tag) +echo "GITHUB_REF: '$GITHUB_REF'" +if [ "${GITHUB_REF#refs/tags/v}" != "$GITHUB_REF" ]; then + pyenv shell 3.9 + + # Install bump-my-version + python -m pip install bump-my-version + TAG_NAME=${GITHUB_REF#refs/tags/v} + bump-my-version replace --new-version $TAG_NAME + echo "Version files updated to $TAG_NAME" + pyenv shell --unset +fi + +# 5. Build chdb +echo "=== Building chdb ===" +echo "Timestamp: $(date)" +echo "Current directory: $(pwd)" +echo "Available disk space: $(df -h .)" + +# Setup clang +echo "Setting up clang compiler..." +ln -sf /usr/bin/clang-20 /usr/bin/clang +ln -sf /usr/bin/clang++-20 /usr/bin/clang++ +export CC=/usr/bin/clang +export CXX=/usr/bin/clang++ +echo "Compiler versions:" +$CC --version +$CXX --version + +# Build +echo "Starting chdb build with Python 3.8..." +pyenv shell 3.8 +python --version +echo "Build start time: $(date)" +bash ./chdb/build-musl.sh +echo "Build end time: $(date)" + +# Test +echo "Running smoke test with Python 3.9..." +pyenv shell 3.9 +python --version +echo "Test start time: $(date)" +bash -x ./chdb/test_smoke.sh +echo "Test end time: $(date)" + +# Check build results +echo "Build results summary:" +ccache -s +echo "chdb directory contents:" +ls -lh chdb +echo "Build artifacts size:" +du -sh chdb + +# 6. Create and audit wheels +echo "=== Creating and auditing wheels ===" +echo "Wheel creation start time: $(date)" +echo "Available disk space before wheel build: $(df -h .)" + +# Build wheels +echo "Building wheels with Python 3.8..." +pyenv shell 3.8 +python --version +echo "Running make wheel..." +make wheel +echo "Wheel build completed at: $(date)" +echo "Initial wheel files:" +ls -lh dist/ || echo "No dist directory yet" + +# Install patchelf +echo "Installing patchelf for wheel auditing..." +wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-aarch64.tar.gz -O patchelf.tar.gz +tar -xvf patchelf.tar.gz +cp bin/patchelf /usr/bin/ +chmod +x /usr/bin/patchelf +echo "patchelf version: $(patchelf --version)" + +# Audit wheels +echo "Auditing wheels with Python 3.13..." +pyenv shell 3.13 +python --version +python -m pip install auditwheel +echo "auditwheel version: $(auditwheel --version)" +echo "Starting wheel audit at: $(date)" +auditwheel -v repair -w dist/ --plat musllinux_1_2_aarch64 dist/*.whl +echo "Wheel audit completed at: $(date)" + +# Clean up non-musllinux wheels +echo "Cleaning up non-musllinux wheels..." +echo "Before cleanup:" +ls -lh dist/ +rm -f dist/*-linux_aarch64.whl +echo "After cleanup:" +ls -lh dist/ +echo "Final wheel sizes:" +du -sh dist/* + +# 7. Test wheels +echo "=== Testing wheels ===" +echo "Wheel testing start time: $(date)" +echo "Available wheels for testing:" +ls -lh dist/*.whl +echo "Wheel file details:" +file dist/*.whl + +TOTAL_TESTS=6 +CURRENT_TEST=0 +TEST_FAILED=false + +for version in 3.9 3.10 3.11 3.12 3.13 3.14; do + CURRENT_TEST=$((CURRENT_TEST + 1)) + echo "=== Test $CURRENT_TEST/$TOTAL_TESTS: Python $version ===" + echo "Test start time: $(date)" + + echo "Switching to Python $version..." + pyenv shell $version + python --version + echo "pip version: $(python -m pip --version)" + + echo "Installing chdb wheel..." + python -m pip install dist/*.whl --force-reinstall + echo "Installation completed at: $(date)" + + echo "Running basic query test..." + python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" + + echo "Running full test suite..." + if make test; then + echo "Test suite PASSED for Python $version at: $(date)" + else + echo "Test suite FAILED for Python $version at: $(date)" + TEST_FAILED=true + break + fi + + pyenv shell --unset + echo "Test $CURRENT_TEST/$TOTAL_TESTS completed successfully" + echo "" +done + +echo "All wheel tests completed at: $(date)" + +# Check if any tests failed +if [ "$TEST_FAILED" = true ]; then + echo "ERROR: One or more test suites failed!" + echo "Test failure detected - aborting build process" + exit 1 +fi + +# Create test success marker file only if all tests passed +echo "All tests passed successfully!" +echo "Creating test success marker..." +touch /workspace/.test_success_marker +echo "Test success marker created at: $(date)" + +# 8. Scan chdb libraries +echo "=== Scanning chdb libraries ===" +FILES_TO_SCAN="$(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" +SQLITE_VULNERABILITIES_FOUND=false + +for file in $FILES_TO_SCAN; do + if [ -f "$file" ]; then + echo "=== Scanning $file ===" + SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) + echo "$SCAN_OUTPUT" + + if echo "$SCAN_OUTPUT" | grep -qi sqlite; then + echo "SQLite vulnerability found in $file" + SQLITE_VULNERABILITIES_FOUND=true + fi + fi +done + +if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then + echo "SQLite vulnerabilities detected in chdb libraries!" +else + echo "No SQLite vulnerabilities found in chdb libraries" +fi + +# Show final results +echo "=== Final wheel files ===" +ls -la ./dist/ \ No newline at end of file diff --git a/.github/workflows/build_linux_arm64_wheels-gh.yml b/.github/workflows/build_linux_arm64_wheels-gh.yml index 7fa9fdf5fc2..77eecba9212 100644 --- a/.github/workflows/build_linux_arm64_wheels-gh.yml +++ b/.github/workflows/build_linux_arm64_wheels-gh.yml @@ -24,7 +24,7 @@ on: jobs: build_universal_wheel: name: Build Universal Wheel (Linux ARM64) - runs-on: GH-Linux-ARM64 + runs-on: [self-hosted, linux, arm64, ubuntu-latest] if: ${{ !github.event.pull_request.draft }} steps: - name: Install Python build dependencies @@ -33,11 +33,24 @@ jobs: sudo apt-get install -y make build-essential libssl-dev zlib1g-dev \ libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \ libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev \ - libffi-dev liblzma-dev + libffi-dev liblzma-dev golang-go + - name: Install GitHub CLI + run: | + wget https://github.com/cli/cli/releases/download/v2.82.1/gh_2.82.1_linux_arm64.tar.gz -O gh.tar.gz + tar -xf gh.tar.gz + sudo cp gh_*/bin/gh /usr/local/bin/ + sudo chmod +x /usr/local/bin/gh + if ! gh --version; then + echo "ERROR: GitHub CLI installation failed!" + exit 1 + fi + echo "GitHub CLI installed successfully" - name: Scan SQLite vulnerabilities with grype run: | # Install grype and required tools - curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin + mkdir -p $HOME/.local/bin + curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b $HOME/.local/bin + echo "$HOME/.local/bin" >> $GITHUB_PATH sudo apt-get update && sudo apt-get install -y jq lsb-release # Detect OS distribution info @@ -46,11 +59,11 @@ jobs: echo "Detected OS: $DISTRO_ID:$DISTRO_VERSION" # Update grype vulnerability database - grype db update + $HOME/.local/bin/grype db update # Check SQLite vulnerabilities in installed packages echo "Scanning SQLite packages for vulnerabilities..." - GRYPE_RAW_OUTPUT=$(grype dir:/var/lib/dpkg --distro "$DISTRO_ID:$DISTRO_VERSION" --scope all-layers 2>/dev/null || true) + GRYPE_RAW_OUTPUT=$($HOME/.local/bin/grype dir:/var/lib/dpkg --distro "$DISTRO_ID:$DISTRO_VERSION" --scope all-layers 2>/dev/null || true) echo "Raw grype output:" echo "$GRYPE_RAW_OUTPUT" @@ -67,6 +80,8 @@ jobs: continue-on-error: true - name: Setup pyenv run: | + # Remove existing pyenv installation if present + rm -rf $HOME/.pyenv curl https://pyenv.run | bash export PATH="$HOME/.pyenv/bin:$PATH" eval "$(pyenv init -)" @@ -213,7 +228,7 @@ jobs: for file in $FILES_TO_SCAN; do if [ -f "$file" ]; then echo "=== Scanning $file ===" - SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) + SCAN_OUTPUT=$($HOME/.local/bin/grype "$file" 2>/dev/null || true) echo "$SCAN_OUTPUT" if echo "$SCAN_OUTPUT" | grep -qi sqlite; then @@ -256,7 +271,10 @@ jobs: patchelf --version - name: Audit wheels run: | - python3 -m pip install auditwheel + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + pyenv shell 3.9 + python -m pip install auditwheel auditwheel -v repair -w dist/ --plat manylinux_2_17_aarch64 dist/*.whl continue-on-error: false - name: Show files @@ -287,7 +305,7 @@ jobs: run: | export PATH="$HOME/.pyenv/bin:$PATH" eval "$(pyenv init -)" - pyenv shell 3.8 + pyenv shell 3.9 python -m pip install dist/*.whl --force-reinstall jupyter nbconvert --to notebook --execute tests/test_data_insertion.ipynb --output test_data_insertion_output.ipynb pyenv shell --unset diff --git a/.github/workflows/build_linux_x86_wheels.yml b/.github/workflows/build_linux_x86_wheels.yml index 744ef0e0795..5c630f44922 100644 --- a/.github/workflows/build_linux_x86_wheels.yml +++ b/.github/workflows/build_linux_x86_wheels.yml @@ -286,7 +286,7 @@ jobs: run: | export PATH="$HOME/.pyenv/bin:$PATH" eval "$(pyenv init -)" - pyenv shell 3.8 + pyenv shell 3.9 python -m pip install dist/*.whl --force-reinstall jupyter nbconvert --to notebook --execute tests/test_data_insertion.ipynb --output test_data_insertion_output.ipynb pyenv shell --unset diff --git a/.github/workflows/build_macos_arm64_wheels.yml b/.github/workflows/build_macos_arm64_wheels.yml index 04cf1b2e4b6..78f9e6a9bdb 100644 --- a/.github/workflows/build_macos_arm64_wheels.yml +++ b/.github/workflows/build_macos_arm64_wheels.yml @@ -282,7 +282,7 @@ jobs: run: | export PATH="$HOME/.pyenv/bin:$PATH" eval "$(pyenv init -)" - pyenv shell 3.8 + pyenv shell 3.9 python -m pip install dist/*.whl --force-reinstall jupyter nbconvert --to notebook --execute tests/test_data_insertion.ipynb --output test_data_insertion_output.ipynb pyenv shell --unset diff --git a/.github/workflows/build_musllinux_arm64_wheels.yml b/.github/workflows/build_musllinux_arm64_wheels.yml index 08e03e0eea5..6a7fee05e36 100644 --- a/.github/workflows/build_musllinux_arm64_wheels.yml +++ b/.github/workflows/build_musllinux_arm64_wheels.yml @@ -23,9 +23,53 @@ on: jobs: build_musllinux_wheels: name: Build musllinux wheels (Alpine Linux aarch64) - runs-on: GH-Linux-ARM64 + runs-on: [self-hosted, linux, arm64, ubuntu-latest] if: ${{ !github.event.pull_request.draft }} steps: + - name: Setup Python and install twine for PyPI upload + run: | + echo "=== Setting up Python for PyPI upload ===" + # Install pip if not available + sudo apt-get update + sudo apt-get install -y python3-pip + python3 --version + python3 -m pip --version + echo "=== Installing twine ===" + python3 -m pip install --upgrade pip --break-system-packages + python3 -m pip install twine --break-system-packages + if ! python3 -m twine --version; then + echo "ERROR: Twine installation failed!" + exit 1 + fi + echo "Twine installed successfully" + - name: Install GitHub CLI + run: | + wget https://github.com/cli/cli/releases/download/v2.82.1/gh_2.82.1_linux_arm64.tar.gz -O gh.tar.gz + tar -xf gh.tar.gz + sudo cp gh_*/bin/gh /usr/local/bin/ + sudo chmod +x /usr/local/bin/gh + if ! gh --version; then + echo "ERROR: GitHub CLI installation failed!" + exit 1 + fi + echo "GitHub CLI installed successfully" + + - name: Setup Docker permissions + run: | + # Ensure Docker is running + sudo systemctl start docker + sudo systemctl enable docker + + # Add current user to docker group + sudo usermod -aG docker $USER + + # Set proper permissions on docker socket + sudo chmod 666 /var/run/docker.sock + + # Verify Docker is working + docker --version + docker info + - uses: actions/checkout@v3 with: fetch-depth: 0 @@ -39,289 +83,9 @@ jobs: git submodule update --init --recursive --jobs 4 - name: Build chdb wheels in container - uses: addnab/docker-run-action@v3 - with: - image: quay.io/pypa/musllinux_1_2_aarch64 - options: -v ${{ github.workspace }}:/workspace --privileged -e GITHUB_REF=${{ github.ref }} - run: | - cd /workspace - - # Configure git safe directory in container - apk update - apk add --no-cache git python3 py3-pip py3-setuptools - echo "=== Configure git safe directory ===" - git config --global --add safe.directory /workspace - git describe --tags - python3 -c "import sys; sys.path.append('.'); from setup import get_latest_git_tag; print('version:', get_latest_git_tag())" - - # 1. Check system info - echo "=== Container System Info ===" - echo "System: $(uname -m) $(cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2)" - if [ -f /lib/ld-musl-aarch64.so.1 ]; then - echo "musl libc aarch64" - elif [ -f /lib/libc.musl-aarch64.so.1 ]; then - echo "musl libc aarch64" - else - echo "Not musl libc" - fi - echo "Workspace mounted at: /workspace" - ls -la /workspace - - # 2. Install build dependencies - echo "=== Installing build dependencies ===" - apk add --no-cache make build-base openssl-dev zlib-dev \ - bzip2-dev readline-dev sqlite-dev wget curl llvm \ - ncurses-dev xz-dev tk-dev libxml2-dev \ - libffi-dev linux-headers - apk add --no-cache make cmake ccache ninja yasm gawk - apk add --no-cache clang20 clang20-dev llvm20 llvm20-dev lld20 - - # 3. Scan SQLite vulnerabilities - echo "=== Scanning SQLite vulnerabilities ===" - # Install grype - curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin - grype db update - - # Check SQLite vulnerabilities - echo "Scanning SQLite packages for vulnerabilities..." - GRYPE_RAW_OUTPUT=$(grype dir:/lib/apk/db --scope all-layers 2>/dev/null || true) - echo "Raw grype output:" - echo "$GRYPE_RAW_OUTPUT" - - SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) - if [ -n "$SQLITE_SCAN_OUTPUT" ]; then - echo "SQLite vulnerabilities found in packages! Build should be reviewed." - echo "SQLite vulnerability details:" - echo "$SQLITE_SCAN_OUTPUT" - else - echo "No SQLite vulnerabilities found" - fi - - # 4. Setup Python environments - echo "=== Setting up Python environments ===" - # Setup pyenv - curl https://pyenv.run | bash - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - - # Install Python versions - for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do - echo "Installing Python $version" - pyenv install $version:latest - done - pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 3.14 - - # Verify installations - echo "Installed versions:" - pyenv versions - for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do - if ! pyenv versions --bare | grep -q "^$version"; then - echo "ERROR: Python $version is not installed!" - exit 1 - fi - echo "Python $version is installed" - done - echo "All Python versions verified successfully!" - - # Install Rust - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable - source $HOME/.cargo/env - rustup toolchain install nightly-2025-07-07 - rustup component add --toolchain nightly-2025-07-07 rust-src - - # Install Python dependencies - for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do - echo "Installing dependencies for Python $version" - pyenv shell $version - python -m pip install --upgrade pip - if [ "$version" = "3.8" ]; then - python -m pip install setuptools tox twine psutil wheel - else - python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel - fi - pyenv shell --unset - done - - # Update version for release (if triggered by tag) - if [ "${GITHUB_REF#refs/tags/v}" != "$GITHUB_REF" ]; then - pyenv shell 3.9 - - # Install bump-my-version - python -m pip install bump-my-version - TAG_NAME=${GITHUB_REF#refs/tags/v} - bump-my-version replace --new-version $TAG_NAME - echo "Version files updated to $TAG_NAME" - pyenv shell --unset - fi - - # 5. Build chdb - echo "=== Building chdb ===" - echo "Timestamp: $(date)" - echo "Current directory: $(pwd)" - echo "Available disk space: $(df -h .)" - - # Setup clang - echo "Setting up clang compiler..." - ln -sf /usr/bin/clang-20 /usr/bin/clang - ln -sf /usr/bin/clang++-20 /usr/bin/clang++ - export CC=/usr/bin/clang - export CXX=/usr/bin/clang++ - echo "Compiler versions:" - $CC --version - $CXX --version - - # Build - echo "Starting chdb build with Python 3.8..." - pyenv shell 3.8 - python --version - echo "Build start time: $(date)" - bash ./chdb/build-musl.sh - echo "Build end time: $(date)" - - # Test - echo "Running smoke test with Python 3.9..." - pyenv shell 3.9 - python --version - echo "Test start time: $(date)" - bash -x ./chdb/test_smoke.sh - echo "Test end time: $(date)" - - # Check build results - echo "Build results summary:" - ccache -s - echo "chdb directory contents:" - ls -lh chdb - echo "Build artifacts size:" - du -sh chdb - - # 6. Create and audit wheels - echo "=== Creating and auditing wheels ===" - echo "Wheel creation start time: $(date)" - echo "Available disk space before wheel build: $(df -h .)" - - # Build wheels - echo "Building wheels with Python 3.8..." - pyenv shell 3.8 - python --version - echo "Running make wheel..." - make wheel - echo "Wheel build completed at: $(date)" - echo "Initial wheel files:" - ls -lh dist/ || echo "No dist directory yet" - - # Install patchelf - echo "Installing patchelf for wheel auditing..." - wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-aarch64.tar.gz -O patchelf.tar.gz - tar -xvf patchelf.tar.gz - cp bin/patchelf /usr/bin/ - chmod +x /usr/bin/patchelf - echo "patchelf version: $(patchelf --version)" - - # Audit wheels - echo "Auditing wheels with Python 3.13..." - pyenv shell 3.13 - python --version - python -m pip install auditwheel - echo "auditwheel version: $(auditwheel --version)" - echo "Starting wheel audit at: $(date)" - auditwheel -v repair -w dist/ --plat musllinux_1_2_aarch64 dist/*.whl - echo "Wheel audit completed at: $(date)" - - # Clean up non-musllinux wheels - echo "Cleaning up non-musllinux wheels..." - echo "Before cleanup:" - ls -lh dist/ - rm -f dist/*-linux_aarch64.whl - echo "After cleanup:" - ls -lh dist/ - echo "Final wheel sizes:" - du -sh dist/* - - # 7. Test wheels - echo "=== Testing wheels ===" - echo "Wheel testing start time: $(date)" - echo "Available wheels for testing:" - ls -lh dist/*.whl - echo "Wheel file details:" - file dist/*.whl - - TOTAL_TESTS=5 - CURRENT_TEST=0 - TEST_FAILED=false - - for version in 3.9 3.10 3.11 3.12 3.13 3.14; do - CURRENT_TEST=$((CURRENT_TEST + 1)) - echo "=== Test $CURRENT_TEST/$TOTAL_TESTS: Python $version ===" - echo "Test start time: $(date)" - - echo "Switching to Python $version..." - pyenv shell $version - python --version - echo "pip version: $(python -m pip --version)" - - echo "Installing chdb wheel..." - python -m pip install dist/*.whl --force-reinstall - echo "Installation completed at: $(date)" - - echo "Running basic query test..." - python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" - - echo "Running full test suite..." - if make test; then - echo "Test suite PASSED for Python $version at: $(date)" - else - echo "Test suite FAILED for Python $version at: $(date)" - TEST_FAILED=true - break - fi - - pyenv shell --unset - echo "Test $CURRENT_TEST/$TOTAL_TESTS completed successfully" - echo "" - done - - echo "All wheel tests completed at: $(date)" - - # Check if any tests failed - if [ "$TEST_FAILED" = true ]; then - echo "ERROR: One or more test suites failed!" - echo "Test failure detected - aborting build process" - exit 1 - fi - - # Create test success marker file only if all tests passed - echo "All tests passed successfully!" - echo "Creating test success marker..." - touch /workspace/.test_success_marker - echo "Test success marker created at: $(date)" - - # 8. Scan chdb libraries - echo "=== Scanning chdb libraries ===" - FILES_TO_SCAN="$(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" - SQLITE_VULNERABILITIES_FOUND=false - - for file in $FILES_TO_SCAN; do - if [ -f "$file" ]; then - echo "=== Scanning $file ===" - SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) - echo "$SCAN_OUTPUT" - - if echo "$SCAN_OUTPUT" | grep -qi sqlite; then - echo "SQLite vulnerability found in $file" - SQLITE_VULNERABILITIES_FOUND=true - fi - fi - done - - if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then - echo "SQLite vulnerabilities detected in chdb libraries!" - else - echo "No SQLite vulnerabilities found in chdb libraries" - fi - - # Show final results - echo "=== Final wheel files ===" - ls -la ./dist/ + run: | + docker run --rm -v ${{ github.workspace }}:/workspace --privileged -e GITHUB_REF=${{ github.ref }} \ + quay.io/pypa/musllinux_1_2_aarch64 /bin/sh /workspace/.github/scripts/build-musllinux-arm64.sh continue-on-error: false # Check test success before upload - name: Verify test completion @@ -335,6 +99,7 @@ jobs: fi echo "Test success marker found. All tests completed successfully." echo "Proceeding with wheel upload..." + ls -la ./dist/ continue-on-error: false # Upload wheels to release - name: Upload wheels to release @@ -347,21 +112,6 @@ jobs: GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} continue-on-error: true - - name: Setup Python and install twine for PyPI upload - run: | - echo "=== Setting up Python for PyPI upload ===" - python3 --version - python3 -m pip --version - echo "=== Installing twine ===" - python3 -m pip install --upgrade pip - python3 -m pip install twine - if ! python3 -m twine --version; then - echo "ERROR: Twine installation failed!" - exit 1 - fi - echo "Twine installed successfully" - ls -la ./dist/ - # Upload to PyPI - name: Upload pypi if: startsWith(github.ref, 'refs/tags/v') diff --git a/chdb/build_pybind11.sh b/chdb/build_pybind11.sh index 33066b916fe..6095f4747e5 100755 --- a/chdb/build_pybind11.sh +++ b/chdb/build_pybind11.sh @@ -33,8 +33,33 @@ build_pybind11_nonlimitedapi() { local py_version=$1 echo "Building pybind11 nonlimitedapi library for Python ${py_version}..." + local full_py_version="" + local custom_python_path="" + + if command -v pyenv >/dev/null 2>&1; then + full_py_version=$(pyenv versions --bare | grep "^${py_version}\." | head -n 1) + + if [ -n "$full_py_version" ]; then + custom_python_path="$HOME/.pyenv/versions/${full_py_version}/bin/python3" + if [ -f "$custom_python_path" ]; then + echo "Found pyenv Python ${full_py_version} at: $custom_python_path" + else + echo "Warning: pyenv Python ${full_py_version} not found at expected path" + custom_python_path="" + fi + else + echo "Warning: Python ${py_version}.x not found in pyenv versions" + fi + else + echo "Warning: pyenv not found" + fi + local py_cmake_args="${CMAKE_ARGS} -DPYBIND11_NONLIMITEDAPI_PYTHON_HEADERS_VERSION=${py_version}" + if [ -n "$custom_python_path" ]; then + py_cmake_args="${py_cmake_args} -DCHDB_CUSTOM_PYTHON_EXECUTABLE=${custom_python_path}" + fi + cmake ${py_cmake_args} -DENABLE_PYTHON=1 .. # Build only the pybind11 targets diff --git a/contrib/pybind11-cmake/CMakeLists.txt b/contrib/pybind11-cmake/CMakeLists.txt index 0427cf47b71..3a34a169049 100644 --- a/contrib/pybind11-cmake/CMakeLists.txt +++ b/contrib/pybind11-cmake/CMakeLists.txt @@ -6,6 +6,13 @@ endif() string(REPLACE "-Wl,-z,defs" "" CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS}") +# Use custom Python path if provided +if (DEFINED CHDB_CUSTOM_PYTHON_EXECUTABLE) + set(Python_EXECUTABLE "${CHDB_CUSTOM_PYTHON_EXECUTABLE}") + set(Python3_EXECUTABLE "${CHDB_CUSTOM_PYTHON_EXECUTABLE}") + message(STATUS "Using custom Python executable: ${CHDB_CUSTOM_PYTHON_EXECUTABLE}") +endif() + if (DEFINED PYBIND11_NONLIMITEDAPI_PYTHON_HEADERS_VERSION) find_package(Python ${PYBIND11_NONLIMITEDAPI_PYTHON_HEADERS_VERSION} EXACT REQUIRED COMPONENTS Interpreter Development) else()