[python/c++] COO to CSX conversion optimization #1038

Workflow file for this run

.github/workflows/python-ci-packaging.yml at 6242085

	# This CI confirms the Python package continues to install properly in the many
	# different required contexts

	name: TileDB-SOMA Python CI (Packaging)

	on:
	push:
	paths:
	- '.github/workflows/python-ci-packaging.yml'
	- 'apis/python/MANIFEST.in'
	- 'apis/python/pyproject.toml'
	- 'apis/python/setup.py'
	- 'apis/python/version.py'
	- 'apis/python/src/tiledbsoma/__init__.py'
	- 'libtiledbsoma/cmake/inputs/Config.cmake.in'
	- 'libtiledbsoma/cmake/inputs/tiledbsoma.pc.in'
	- 'libtiledbsoma/cmake/Modules/FindTileDB_EP.cmake'
	- 'libtiledbsoma/cmake/Modules/TileDBCommon.cmake'
	- 'libtiledbsoma/cmake/Superbuild.cmake'
	- 'libtiledbsoma/CMakeLists.txt'
	pull_request:
	paths:
	- '.github/workflows/python-ci-packaging.yml'
	- 'apis/python/MANIFEST.in'
	- 'apis/python/pyproject.toml'
	- 'apis/python/setup.py'
	- 'apis/python/version.py'
	- 'apis/python/src/tiledbsoma/__init__.py'
	- 'libtiledbsoma/cmake/inputs/Config.cmake.in'
	- 'libtiledbsoma/cmake/inputs/tiledbsoma.pc.in'
	- 'libtiledbsoma/cmake/Modules/FindTileDB_EP.cmake'
	- 'libtiledbsoma/cmake/Modules/TileDBCommon.cmake'
	- 'libtiledbsoma/cmake/Superbuild.cmake'
	- 'libtiledbsoma/CMakeLists.txt'
	workflow_dispatch:

	jobs:

	# Confirm shared object copying when building the Python package
	# https://github.com/single-cell-data/TileDB-SOMA/pull/1937
	docker:
	runs-on: ubuntu-latest
	name: "SO copying (docker) TILEDB_EXISTS: ${{ matrix.TILEDB_EXISTS }} TILEDBSOMA_EXISTS: ${{ matrix.TILEDBSOMA_EXISTS }}"
	strategy:
	fail-fast: false
	matrix:
	TILEDB_EXISTS: ["no", "yes"]
	TILEDBSOMA_EXISTS: ["no", "yes"]
	exclude:
	- TILEDB_EXISTS: "no"
	TILEDBSOMA_EXISTS: "yes"
	container:
	image: ubuntu:latest
	defaults:
	run:
	shell: bash
	steps:
	- name: Docker image info
	run: \|
	uname -a
	cat /etc/lsb-release
	- name: Setup
	run: \|
	apt-get update
	apt-get install --yes cmake git python-is-python3 python3 python3-pip python3-venv unzip wget
	- uses: actions/checkout@v4
	with:
	fetch-depth: 0 # for setuptools-scm
	- name: Configure Git
	run: \|
	# This is a permissions quirk due to running Git as root inside of a Docker container
	git config --global --add safe.directory $(pwd)
	git branch
	- name: Install pre-built libtiledb
	if: ${{ matrix.TILEDB_EXISTS == 'yes' }}
	run: \|
	mkdir -p external
	# Please do not edit manually -- let scripts/update-tiledb-version.py update this
	wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.26.2/tiledb-linux-x86_64-2.26.2-30fc114.tar.gz
	tar -C external -xzf tiledb-linux-x86_64-*.tar.gz
	ls external/lib/
	echo "LD_LIBRARY_PATH=$(pwd)/external/lib" >> $GITHUB_ENV
	echo "PKG_CONFIG_PATH=$(pwd)/external/lib/pkgconfig" >> $GITHUB_ENV
	echo "TILEDB_PATH=$(pwd)/external" >> $GITHUB_ENV
	- name: Build and install libtiledbsoma
	if: ${{ matrix.TILEDBSOMA_EXISTS == 'yes' }}
	run: \|
	cmake -S libtiledbsoma -B build-libtiledbsoma \
	-D CMAKE_BUILD_TYPE=Release \
	-D CMAKE_PREFIX_PATH=$(pwd)/external/ \
	-D CMAKE_INSTALL_PREFIX:PATH=$(pwd)/external/ \
	-D OVERRIDE_INSTALL_PREFIX=OFF \
	-D DOWNLOAD_TILEDB_PREBUILT=OFF \
	-D TILEDB_REMOVE_DEPRECATIONS=ON \
	-D TILEDBSOMA_ENABLE_WERROR=ON \
	-D FORCE_BUILD_TILEDB=OFF
	cmake --build build-libtiledbsoma -j $(nproc) --verbose
	cmake --build build-libtiledbsoma --target install-libtiledbsoma
	ls external/lib/
	echo "TILEDBSOMA_PATH=$(pwd)/external" >> $GITHUB_ENV
	- name: Setup Python
	run: \|
	python --version
	python -m venv ./venv-soma
	./venv-soma/bin/pip install --prefer-binary pybind11-global typeguard sparse wheel 'setuptools>=70.1'
	./venv-soma/bin/pip list
	- name: Build wheel
	run: \|
	echo env vars: $LD_LIBRARY_PATH $PKG_CONFIG_PATH $TILEDB_PATH $TILEDBSOMA_PATH
	cd apis/python
	../../venv-soma/bin/python setup.py bdist_wheel
	- name: Inspect wheel
	run: unzip -l apis/python/dist/tiledbsoma-*.whl \| grep '\.so'
	- name: Confirm libtiledb.so is copied
	if: ${{ matrix.TILEDB_EXISTS == 'no' }}
	run: unzip -l apis/python/dist/tiledbsoma-*.whl \| grep -q libtiledb.so
	- name: Confirm libtiledb.so is not copied when using external shared object
	if: ${{ matrix.TILEDB_EXISTS == 'yes' }}
	run: \|
	if unzip -l apis/python/dist/tiledbsoma-*.whl \| grep -q libtiledb.so
	then
	echo "libtiledb.so was copied into the wheel when it was built against an external shared object"
	exit 1
	fi
	- name: Confirm libtiledbsoma.so is copied
	if: ${{ matrix.TILEDBSOMA_EXISTS == 'no' }}
	run: unzip -l apis/python/dist/tiledbsoma-*.whl \| grep -q libtiledbsoma.so
	- name: Confirm libtiledbsoma.so is not copied when using external shared object
	if: ${{ matrix.TILEDBSOMA_EXISTS == 'yes' }}
	run: \|
	if unzip -l apis/python/dist/tiledbsoma-*.whl \| grep -q libtiledbsoma.so
	then
	echo "libtiledbsoma.so was copied into the wheel when it was built against an external shared object"
	exit 1
	fi
	- name: Install wheel
	run: ./venv-soma/bin/pip install --prefer-binary apis/python/dist/tiledbsoma-*.whl
	- name: Check linking and RPATH
	run: \|
	ldd ./venv-soma/lib/python/site-packages/tiledbsoma/pytiledbsoma..so
	readelf -d ./venv-soma/lib/python/site-packages/tiledbsoma/pytiledbsoma..so \| grep R*PATH
	- name: Runtime test
	run: ./venv-soma/bin/python -c "import tiledbsoma; print(tiledbsoma.pytiledbsoma.version())"
	- name: Confirm linking to installed shared objects
	run: \|
	rm -fr build/ build-libtiledbsoma/ dist/ apis/python/build apis/python/src/tiledbsoma/tile.so*
	# should only show shared objects installed in virtual env or in ./external/
	find . -name 'tile.so*'
	./venv-soma/bin/python -c "import tiledbsoma; print(tiledbsoma.pytiledbsoma.version())"

	# Confirm shared object copying when building the Python package
	# https://github.com/single-cell-data/TileDB-SOMA/pull/1937
	# Same as job above, but running on macOS instead of in a Docker container
	macos:
	runs-on: macos-13
	name: "SO copying (macos) TILEDB_EXISTS: ${{ matrix.TILEDB_EXISTS }} TILEDBSOMA_EXISTS: ${{ matrix.TILEDBSOMA_EXISTS }}"
	strategy:
	fail-fast: false
	matrix:
	TILEDB_EXISTS: ["no", "yes"]
	TILEDBSOMA_EXISTS: ["no", "yes"]
	exclude:
	- TILEDB_EXISTS: "no"
	TILEDBSOMA_EXISTS: "yes"
	steps:
	- uses: actions/checkout@v4
	with:
	fetch-depth: 0 # for setuptools-scm
	- name: Check if System Integrity Protection (SIP) is enabled
	run: csrutil status
	- name: Install pre-built libtiledb
	if: ${{ matrix.TILEDB_EXISTS == 'yes' }}
	run: \|
	mkdir -p external
	# Please do not edit manually -- let scripts/update-tiledb-version.py update this
	if [ `uname -m` == "arm64" ]; then
	wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.26.2/tiledb-macos-arm64-2.26.2-30fc114.tar.gz
	tar -C external -xzf tiledb-macos-arm64-*.tar.gz
	else
	wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.26.2/tiledb-macos-x86_64-2.26.2-30fc114.tar.gz
	tar -C external -xzf tiledb-macos-x86_64-*.tar.gz
	fi
	ls external/lib/
	echo "DYLD_LIBRARY_PATH=$(pwd)/external/lib" >> $GITHUB_ENV
	echo "PKG_CONFIG_PATH=$(pwd)/external/lib/pkgconfig" >> $GITHUB_ENV
	echo "TILEDB_PATH=$(pwd)/external" >> $GITHUB_ENV
	- name: Build and install libtiledbsoma
	if: ${{ matrix.TILEDBSOMA_EXISTS == 'yes' }}
	run: \|
	cmake -S libtiledbsoma -B build-libtiledbsoma \
	-D CMAKE_BUILD_TYPE=Release \
	-D CMAKE_PREFIX_PATH=$(pwd)/external/ \
	-D CMAKE_INSTALL_PREFIX:PATH=$(pwd)/external/ \
	-D OVERRIDE_INSTALL_PREFIX=OFF \
	-D DOWNLOAD_TILEDB_PREBUILT=OFF \
	-D TILEDB_REMOVE_DEPRECATIONS=ON \
	-D FORCE_BUILD_TILEDB=OFF
	cmake --build build-libtiledbsoma -j $(nproc)
	cmake --build build-libtiledbsoma --target install-libtiledbsoma
	ls external/lib/
	echo "TILEDBSOMA_PATH=$(pwd)/external" >> $GITHUB_ENV
	- name: Setup Python
	uses: actions/setup-python@v5
	with:
	python-version: "3.11"
	- name: Setup Python virtual env
	run: \|
	python --version
	python -m venv ./venv-soma
	./venv-soma/bin/pip install --prefer-binary pybind11-global typeguard sparse wheel 'setuptools>=70.1'
	./venv-soma/bin/pip list
	- name: Build wheel
	run: \|
	echo env vars: $DYLD_LIBRARY_PATH $PKG_CONFIG_PATH $TILEDB_PATH $TILEDBSOMA_PATH
	cd apis/python
	../../venv-soma/bin/python setup.py bdist_wheel
	- name: Inspect wheel
	run: unzip -l apis/python/dist/tiledbsoma-*.whl \| grep -e '\.dylib' -e '\.so'
	- name: Confirm libtiledb.dylib is copied
	if: ${{ matrix.TILEDB_EXISTS == 'no' }}
	run: unzip -l apis/python/dist/tiledbsoma-*.whl \| grep -q libtiledb.dylib
	- name: Confirm libtiledb.dylib is not copied when using external shared object
	if: ${{ matrix.TILEDB_EXISTS == 'yes' }}
	run: \|
	if unzip -l apis/python/dist/tiledbsoma-*.whl \| grep -q libtiledb.dylib
	then
	echo "libtiledb.dylib was copied into the wheel when it was built against an external shared object"
	exit 1
	fi
	- name: Confirm libtiledbsoma.dylib is copied
	if: ${{ matrix.TILEDBSOMA_EXISTS == 'no' }}
	run: unzip -l apis/python/dist/tiledbsoma-*.whl \| grep -q libtiledbsoma.dylib
	- name: Confirm libtiledbsoma.dylib is not copied when using external shared object
	if: ${{ matrix.TILEDBSOMA_EXISTS == 'yes' }}
	run: \|
	if unzip -l apis/python/dist/tiledbsoma-*.whl \| grep -q libtiledbsoma.dylib
	then
	echo "libtiledbsoma.dylib was copied into the wheel when it was built against an external shared object"
	exit 1
	fi
	- name: Install wheel
	run: ./venv-soma/bin/pip install --prefer-binary apis/python/dist/tiledbsoma-*.whl
	- name: Check linking and RPATH
	run: otool -L ./venv-soma/lib/python/site-packages/tiledbsoma/pytiledbsoma..so
	- name: Runtime test
	run: ./venv-soma/bin/python -c "import tiledbsoma; print(tiledbsoma.pytiledbsoma.version())"
	- name: Confirm linking to installed shared objects
	run: \|
	rm -fr build/ build-libtiledbsoma/ dist/ apis/python/build apis/python/src/tiledbsoma/tile.dylib*
	# should only show shared objects installed in virtual env or in ./external/
	find . -name 'tile.so*'
	find . -name 'tile.dylib*'
	./venv-soma/bin/python -c "import tiledbsoma; print(tiledbsoma.pytiledbsoma.version())"

	# Tests that the --libtiledbsoma flag to setup.py continues working
	setuptools:
	runs-on: ${{ matrix.os }}
	name: "setuptools (${{ matrix.os }})"
	strategy:
	fail-fast: false
	matrix:
	os: ["ubuntu-latest", "macos-latest"]
	steps:
	- uses: actions/checkout@v4
	with:
	fetch-depth: 0 # for setuptools-scm
	- name: Install pre-built libtiledb
	run: \|
	mkdir -p external
	if [ `uname -s` == "Darwin" ]; then
	if [ `uname -m` == "arm64" ]; then
	# Please do not edit manually -- let scripts/update-tiledb-version.py update this
	wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.26.2/tiledb-macos-arm64-2.26.2-30fc114.tar.gz
	else
	# Please do not edit manually -- let scripts/update-tiledb-version.py update this
	wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.26.2/tiledb-macos-x86_64-2.26.2-30fc114.tar.gz
	fi
	else
	# Please do not edit manually -- let scripts/update-tiledb-version.py update this
	wget --quiet https://github.com/TileDB-Inc/TileDB/releases/download/2.26.2/tiledb-linux-x86_64-2.26.2-30fc114.tar.gz
	fi
	tar -C external -xzf tiledb-*.tar.gz
	ls external/lib/
	- name: Build and install libtiledbsoma
	run: \|
	cmake -S libtiledbsoma -B build-libtiledbsoma \
	-D CMAKE_BUILD_TYPE=Release \
	-D CMAKE_PREFIX_PATH=$(pwd)/external/ \
	-D CMAKE_INSTALL_PREFIX:PATH=$(pwd)/external/ \
	-D OVERRIDE_INSTALL_PREFIX=OFF \
	-D DOWNLOAD_TILEDB_PREBUILT=OFF \
	-D TILEDB_REMOVE_DEPRECATIONS=ON \
	-D FORCE_BUILD_TILEDB=OFF
	cmake --build build-libtiledbsoma -j 2
	cmake --build build-libtiledbsoma --target install-libtiledbsoma
	ls external/lib/
	# Delete all cmake executables from the runner. This will ensure that
	# tiledbsoma-py has to use the cli flags to find the external
	# libtiledbsoma.so and not build it from source by shelling out to cmake
	- name: Delete cmake
	run: \|
	echo before
	which -a cmake
	which -a cmake \| xargs sudo rm -f
	echo after
	which -a cmake \|\| echo cmake removed
	- name: Setup Python
	uses: actions/setup-python@v5
	with:
	python-version: "3.11"
	- name: Setup Python virtual env
	run: \|
	python --version
	python -m venv ./venv-soma
	./venv-soma/bin/pip install --prefer-binary pybind11-global typeguard sparse wheel 'setuptools>=70.1'
	./venv-soma/bin/pip list
	- name: Install TileDB-SOMA-Py with setuptools and --libtiledbsoma
	run: \|
	cd apis/python
	../../venv-soma/bin/python setup.py install \
	--single-version-externally-managed \
	--record record.txt \
	--tiledb=$GITHUB_WORKSPACE/external/ \
	--libtiledbsoma=$GITHUB_WORKSPACE/external/
	- name: Check linking and RPATH (Linux)
	if: runner.os == 'Linux'
	run: \|
	ldd ./venv-soma/lib/python/site-packages/tiledbsoma/pytiledbsoma..so
	readelf -d ./venv-soma/lib/python/site-packages/tiledbsoma/pytiledbsoma..so \| grep R*PATH
	- name: Check linking and RPATH (macOS)
	if: runner.os == 'macOS'
	run: \|
	otool -L ./venv-soma/lib/python/site-packages/tiledbsoma/pytiledbsoma..so
	otool -l ./venv-soma/lib/python/site-packages/tiledbsoma/pytiledbsoma..so
	- name: Install runtime dependencies
	run: \|
	grep -v '^\[' apis/python/src/tiledbsoma.egg-info/requires.txt > runtime-reqs.txt
	./venv-soma/bin/pip install --prefer-binary -r runtime-reqs.txt
	- name: Runtime test
	run: ./venv-soma/bin/python -c "import tiledbsoma; print(tiledbsoma.pytiledbsoma.version())"

	# Build a wheel from a source tarball (confirms all required files are
	# distributed in the sdist). This is in preparation for the official building of
	# the PyPI wheels in python-packaging.yml.
	# https://github.com/single-cell-data/TileDB-SOMA/pull/2506
	#
	#
	# Differences to python-packaging.yml:
	#
	# * Uses transparent python commands instead of the opaque action
	# `pypa/cibuildwheel`, and therefore easier to debug
	# * Only builds two wheels total (python 3.11 on Ubuntu and macOS), so it is
	# much faster. It's not creating the wheels for distribution, but instead just
	# providing a quick check that all the required files are distributed in the
	# source tarball
	# * Runs whenever an installation-related file is modified instead of only
	# after a release
	#
	# In summary, the goal is to identify any potential problems with building
	# the PyPI wheels when a PR is submitted, and not at release time.
	sdist:
	runs-on: ${{ matrix.os }}
	name: "Wheel from sdist (${{ matrix.os }})"
	strategy:
	fail-fast: false
	matrix:
	os: ["ubuntu-latest", "macos-latest"]
	steps:
	- uses: actions/checkout@v4
	with:
	path: TileDB-SOMA
	fetch-depth: 0 # for setuptools-scm
	- name: Setup Python
	uses: actions/setup-python@v5
	with:
	python-version: "3.11"
	- name: Install dependencies
	run: pip install --prefer-binary pybind11 'setuptools>=70.1' wheel
	- name: Build source tarball (sdist)
	run: \|
	cd TileDB-SOMA/apis/python
	python setup.py sdist
	- name: Extract source tarball
	run: \|
	tar --list -f TileDB-SOMA/apis/python/dist/tiledbsoma-*.tar.gz
	tar -xzf TileDB-SOMA/apis/python/dist/tiledbsoma-*.tar.gz
	- name: Build wheel
	run: \|
	cd tiledbsoma-*/
	python setup.py bdist_wheel
	- name: Install wheel
	run: \|
	pip install --prefer-binary tiledbsoma-/dist/tiledbsoma-.whl
	python -c "import tiledbsoma; print(tiledbsoma.pytiledbsoma.version())"

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

[python/c++] COO to CSX conversion optimization #1038

Workflow file

[python/c++] COO to CSX conversion optimization #1038

Jobs

Run details

Workflow file for this run