build: add binary build and transpile script, create different binaries for different os in integration delivery workflow

Author: sassanh

.github/workflows/integration_delivery.yml

@@ -132,6 +132,88 @@ jobs:
           fail_ci_if_error: false
           token: ${{ secrets.CODECOV_TOKEN }}
 
+  transpile:
+    name: Transpile to C
+    needs: dependencies
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        name: Checkout
+
+      - uses: actions/setup-python@v5
+        name: Setup Python
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
+
+      - name: Create virtualenv
+        run: |
+          uv venv --system-site-packages
+
+      - name: Transpile
+        run: uv run --frozen poe transpile
+
+      - name: Collect C Source
+        run: |
+          mkdir -p c_source
+          find redux -name "*.c" -exec cp --parents {} c_source \;
+          # Extract version for binary build
+          git fetch --prune --unshallow
+          git fetch --depth=1 origin +refs/tags/*:refs/tags/*
+          uvx hatch version > c_source/VERSION
+
+      - name: Upload C Source
+        uses: actions/upload-artifact@v4
+        with:
+          name: c-source
+          path: c_source
+          include-hidden-files: true
+
+  build_wheels:
+    name: Build Wheels
+    needs: transpile
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    steps:
+      - uses: actions/checkout@v4
+        name: Checkout
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: c-source
+          path: .
+
+      - name: Swap Build System
+        run: |
+          mv pyproject_binary.toml pyproject.toml
+          mv setup_binary.py setup.py
+        shell: bash
+
+      - name: Read Version
+        id: version
+        shell: bash
+        run: |
+          ls -R
+          echo "VERSION=$(cat VERSION)" >> "$GITHUB_ENV"
+          echo "Read version: $(cat VERSION)"
+
+      - name: Build wheels
+        uses: pypa/[email protected]
+        env:
+          CIBW_BUILD: cp311-* cp312-* cp313-* cp314-*
+          CIBW_ENVIRONMENT: PRETEND_VERSION=${{ env.VERSION }}
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: wheels-${{ matrix.os }}
+          path: ./wheelhouse/*.whl
+
   build:
     name: Build
     needs:
@@ -215,6 +297,7 @@ jobs:
       - lint
       - test
       - build
+      - build_wheels
     runs-on: ubuntu-latest
     environment:
       name: pypi
@@ -234,6 +317,12 @@ jobs:
           name: binary
           path: dist
 
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: wheels-*
+          path: dist
+          merge-multiple: true
+
       - name: Publish to PyPI
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
@@ -272,6 +361,13 @@ jobs:
           name: binary
           path: artifacts
 
+      - name: Procure Wheels
+        uses: actions/download-artifact@v4
+        with:
+          pattern: wheels-*
+          path: artifacts
+          merge-multiple: true
+
       - name: Extract Changelog
         id: changelog
         run: |

.gitignore

@@ -95,3 +95,9 @@ redux/_version.py
 
 # mypy
 .mypy_cache/
+
+# Cython
+*.c
+*.so
+*.pyd
+c_source/

pyproject.toml

@@ -62,6 +62,7 @@ todo_demo = "todo_demo:main"
 
 [tool.poe.tasks]
 lint = "ruff check . --unsafe-fixes"
+transpile = "python scripts/transpile.py"
 typecheck = "pyright -p pyproject.toml ."
 test = "pytest --cov --cov-report=term-missing --cov-report=html --cov-report=xml"
 sanity = ["typecheck", "lint", "test"]

pyproject_binary.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools>=68", "wheel", "cython>=3.0", "hatch-vcs"]
+build-backend = "setuptools.build_meta"

scripts/transpile.py

@@ -0,0 +1,38 @@
+# ruff: noqa: T201
+"""Transpile all python files in redux/ to C files using Cython."""
+
+from pathlib import Path
+
+from Cython.Build import cythonize
+
+
+def transpile() -> None:
+    """Found all python files in redux/ and transpile them to C files using Cython."""
+    root = Path(__file__).parent.parent
+    redux_dir = root / 'redux'
+
+    # Find all .py files
+    py_files = list(redux_dir.rglob('*.py'))
+
+    source_files = []
+
+    print(f'Found {len(py_files)} Python files to transpile.')
+
+    for py_file in py_files:
+        # Skip _version.py as it is often generated/problematic
+        if py_file.name == '_version.py':
+            continue
+
+        source_files.append(str(py_file))
+
+    # Cythonize in place to produce .c files next to .py files
+    # We use language_level=3 for Python 3
+    # keep_path=True is implied by passing full paths often, but good to ensure
+    # directory structure is respected in modules if we were building them.
+    # Here we just generate sources.
+    cythonize(source_files, language_level='3', force=True)
+    print('Transpilation complete.')
+
+
+if __name__ == '__main__':
+    transpile()

setup_binary.py

@@ -0,0 +1,50 @@
+# ruff: noqa: T201
+"""Setup script for building the binary extension of python-redux."""
+
+import os
+import sys
+from pathlib import Path
+
+from setuptools import Extension, find_packages, setup
+
+# Add root to path to import version
+root = Path(__file__).parent
+sys.path.insert(0, str(root))
+
+try:
+    from version import get_version
+
+    version = get_version()
+except ImportError:
+    # If version.py or dependencies fail, fallback or fail.
+    # In CI, dependencies should be installed.
+    print('Warning: Could not import get_version from version.py')
+    version = '0.0.0.dev0'
+
+
+def get_extensions() -> list[Extension]:
+    """Dynamically find all C extensions in the redux/ directory."""
+    extensions: list[Extension] = []
+    # Find all .c files in redux/
+    for path in (root / 'redux').rglob('*.c'):
+        # Construct module name: redux/store.c -> redux.store
+        # relatives_to root
+        rel_path = path.relative_to(root)
+        module_name = str(rel_path.with_suffix('')).replace(os.sep, '.')
+
+        extensions.append(Extension(module_name, [str(rel_path)]))
+    return extensions
+
+
+setup(
+    name='python-redux',
+    version=version,
+    description='Redux implementation for Python (Binary Extension)',
+    packages=find_packages(include=['redux', 'redux.*']),
+    ext_modules=get_extensions(),
+    include_package_data=True,
+    # Add other metadata as needed matching pyproject.toml
+    author='Sassan Haradji',
+    author_email='[email protected]',
+    url='https://github.com/sassanh/python-redux/',
+)