TileDB-SOMA python sdist & wheels #936
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Workflow to build distribution, and push to PyPI or TestPyPI | |
| name: TileDB-SOMA python sdist & wheels | |
| on: | |
| # Trigger publication to TestPyPI via user workflow request: | |
| # gh workflow run python-packaging.yml --ref branch | |
| # Specify the branch or other ref as required, allowing testing | |
| # of PRs/branches. | |
| workflow_dispatch: | |
| # Trigger publication to PyPi via new release event. | |
| release: | |
| types: [published] | |
| # Schedule wheel-build and smoke-test on a regular schedule. | |
| # | |
| # This will not publish to either PyPI or TestPyPI (see the conditionals | |
| # guarding those steps below). | |
| schedule: | |
| - cron: "42 9 * * *" | |
| jobs: | |
| sdist: | |
| name: Build source distribution | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout TileDB-SOMA | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # ensure we get all tags to inform package version determination | |
| - name: Install dependencies | |
| run: pip install --upgrade 'setuptools>=70.1' importlib-metadata # cf. https://github.com/single-cell-data/TileDB-SOMA/pull/3076 | |
| - run: pip list | |
| - name: Build sdist | |
| run: python setup.py sdist | |
| working-directory: ./apis/python | |
| - name: Upload sdist artifact to GitHub Actions storage | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: sdist | |
| path: apis/python/dist/ | |
| # This step builds wheels and uploads them to GitHub Actions storage. | |
| # See also https://github.com/single-cell-data/TileDB-SOMA/issues/700. | |
| # See also https://github.com/single-cell-data/TileDB-SOMA/wiki/PyPI-packaging-WIP | |
| # for important transitional context. | |
| wheels: | |
| # Note: tries all supported Python versions as specified in apis/python/setup.py | |
| name: Build ${{ matrix.python-version }}-${{ matrix.wheel-name }} wheel | |
| needs: sdist | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| python-version: [ '39', '310', '311', '312' ] | |
| cibw_build: [ manylinux_x86_64, macosx_x86_64, macosx_arm64 ] | |
| include: | |
| - cibw_build: manylinux_x86_64 | |
| os: ubuntu-latest | |
| wheel-name: manylinux2014 | |
| - cibw_build: macosx_x86_64 | |
| os: macos-latest | |
| cibw_archs_macos: x86_64 | |
| wheel-name: macos-x86_64 | |
| - cibw_build: macosx_arm64 | |
| os: macos-14 # Note: macos-14 is arm64 only | |
| cibw_archs_macos: arm64 | |
| wheel-name: macos-arm64 | |
| steps: | |
| - name: Download sdist artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: sdist | |
| - name: Rename sdist | |
| run: cp tiledbsoma-*.tar.gz tiledbsoma.tar.gz && ls -lh | |
| # This is crucial for ongoing debug (do not remove it) as this shows the | |
| # OS version as used by `pip install` to find wheel names. Importantly, | |
| # macos 12 and macos14 self-report as `macosx-10.9-universal2` via | |
| # `distutil.util.get_platform()`. | |
| - name: Show self-reported platform | |
| run: | | |
| echo "python --version"; python --version | |
| echo matrix.platform: ${{ matrix.platform }} | |
| echo matrix.arch: ${{ matrix.arch }} | |
| # This bit is crucial since it's used to match up to a component of the wheel-file name | |
| pip install setuptools | |
| python -c 'from distutils import util; print("distutil.util.get_platform:", util.get_platform())' | |
| python -c 'import platform; print("platform.platform()", platform.platform())' | |
| - name: Build wheels | |
| uses: pypa/[email protected] | |
| with: | |
| package-dir: tiledbsoma.tar.gz | |
| only: cp${{ matrix.python-version }}-${{ matrix.cibw_build }} | |
| env: | |
| CIBW_BUILD: ${{ matrix.cibw_build }} | |
| CIBW_BUILD_VERBOSITY: 1 | |
| CIBW_BEFORE_BUILD: bash -x -c 'rm -rf tiledbsoma*/dist_links/dist/lib*' | |
| # ^ Delete lib folder that apis/python/setup.py:find_or_build() looks for in deciding to | |
| # run CMake build or not. Otherwise it'll keep reusing the library file built in the | |
| # first iteration of cibuildwheel's outer loop, resulting in wheels with the library | |
| # built for the wrong python version. | |
| CIBW_ARCHS_MACOS: ${{ matrix.cibw_archs_macos }} | |
| CIBW_TEST_SKIP: "*_arm64" | |
| CMAKE_OSX_ARCHITECTURES: ${{ matrix.cibw_archs_macos }} | |
| MACOSX_DEPLOYMENT_TARGET: "11.0" | |
| - name: Upload wheel-${{ matrix.wheel-name }}-${{ matrix.python-version }} to GitHub Actions storage | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: wheel-${{ matrix.wheel-name }}-${{ matrix.python-version }} | |
| path: ./wheelhouse/*.whl | |
| # This step locally tries out the built wheels, without publishing to PyPI | |
| smoke-test: | |
| name: "Smoke test ${{ matrix.python.undotted-version }}-${{ matrix.wheel-name }} wheel" | |
| needs: wheels | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| python: | |
| - undotted-version: '310' | |
| dotted-version: '3.10' | |
| - undotted-version: '311' | |
| dotted-version: '3.11' | |
| - undotted-version: '312' | |
| dotted-version: '3.12' | |
| wheel-name: | |
| - manylinux2014 | |
| - macos-x86_64 | |
| - macos-arm64 | |
| include: | |
| - wheel-name: manylinux2014 | |
| os: ubuntu-latest | |
| arch: x86_64 | |
| cc: gcc-11 | |
| cxx: g++-11 | |
| - wheel-name: macos-x86_64 | |
| os: macos-13 | |
| arch: x86_64 | |
| cc: clang | |
| cxx: clang++ | |
| - wheel-name: macos-arm64 | |
| os: macos-14 | |
| arch: arm64 | |
| cc: clang | |
| cxx: clang++ | |
| fail-fast: false | |
| steps: | |
| - name: Set up Python ${{ matrix.python.dotted-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python.dotted-version }} | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: wheel-${{ matrix.wheel-name }}-${{ matrix.python.undotted-version }} | |
| - name: Install wheel | |
| run: | | |
| set -x | |
| ls -lR | |
| WHL=$(find . -name 'tiledbsoma-*-cp${{ matrix.python.undotted-version }}-cp${{ matrix.python.undotted-version }}-*_${{ matrix.arch }}.whl') | |
| echo "WHL=$WHL" | |
| if [ -z "$WHL" ]; then echo "No wheel found"; exit 1; fi | |
| unzip -l $WHL | |
| pip install wheel | |
| pip install $WHL | |
| echo "WHL=$WHL" >> $GITHUB_ENV | |
| - name: Smoke test ${{ matrix.os }} | |
| run: python -c 'import tiledbsoma; print(tiledbsoma.pytiledbsoma.__file__); tiledbsoma.show_package_versions()' | |
| # TODO: more thorough local smoke test | |
| - name: Smoke test in docker | |
| if: ${{ matrix.os == 'ubuntu-latest' }} | |
| run: | | |
| docker run -v $(pwd):/mnt python:${{ matrix.python.dotted-version }} bash -ec " | |
| apt-get -qq update && apt-get install -y python3-pip python3-wheel | |
| pip3 install /mnt/$WHL | |
| python3 -c 'import tiledbsoma; print(tiledbsoma.pytiledbsoma.__file__); tiledbsoma.show_package_versions()' | |
| " | |
| # Publish to TestPyPI upon user workflow request | |
| publish-to-test-pypi: | |
| name: Publish package to TestPyPI | |
| needs: smoke-test | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'workflow_dispatch' | |
| steps: | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v4 | |
| - name: Create dist | |
| run: | | |
| set -x | |
| mkdir dist | |
| cp sdist/tiledbsoma-*.tar.gz wheel-*/*.whl dist | |
| ls -l dist | |
| - name: Publish packages to TestPyPI | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| continue-on-error: true | |
| with: | |
| password: ${{ secrets.TEST_PYPI_TOKEN }} | |
| repository-url: https://test.pypi.org/legacy/ | |
| packages_dir: dist | |
| verbose: true | |
| # Publish to PyPI upon release. | |
| publish-to-pypi: | |
| name: Publish package to PyPI | |
| needs: smoke-test | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'release' | |
| steps: | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v4 | |
| - name: Create dist | |
| run: | | |
| set -x | |
| mkdir dist | |
| cp sdist/tiledbsoma-*.tar.gz wheel-*/*.whl dist | |
| ls -l dist | |
| - name: Publish packages to PyPI | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| password: ${{ secrets.PYPI_TOKEN }} | |
| packages_dir: dist | |
| verbose: true | |
| # File a bug report if anything fails, but don't file tickets for manual runs | |
| # -- only for scheduled ones. | |
| create_issue_on_fail: | |
| runs-on: ubuntu-latest | |
| needs: [smoke-test, publish-to-test-pypi, publish-to-pypi] | |
| if: (failure() || cancelled()) && github.event_name != 'workflow_dispatch' | |
| steps: | |
| - name: Checkout TileDB-SOMA `main` | |
| uses: actions/checkout@v2 | |
| - name: Create Issue if Build Fails | |
| uses: JasonEtco/create-an-issue@v2 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| filename: .github/workflows/daily-test-build-issue-template.md |