From 42864201c4a68335aee316a4ff0742faaeac48db Mon Sep 17 00:00:00 2001 From: Maxime Desroches Date: Fri, 19 Jul 2024 21:05:14 -0700 Subject: [PATCH] better --- .github/workflows/tests.yml | 2 +- .github/workflows/wheels.yml | 51 +++++++++++++++++++++++++++ pyproject.toml | 4 +-- requirements.txt | 50 ++++++++++++++++++++++++++ scripts/build_wheel.sh | 9 ----- scripts/install_dependencies.sh | 58 ------------------------------- scripts/macos_dependencies.sh | 6 ++++ scripts/manylinux_dependencies.sh | 10 ++++++ scripts/ubuntu_dependencies.sh | 12 +++++++ setup.py | 15 +++++--- 10 files changed, 143 insertions(+), 74 deletions(-) create mode 100644 .github/workflows/wheels.yml create mode 100644 requirements.txt delete mode 100755 scripts/build_wheel.sh delete mode 100755 scripts/install_dependencies.sh create mode 100755 scripts/macos_dependencies.sh create mode 100755 scripts/manylinux_dependencies.sh create mode 100755 scripts/ubuntu_dependencies.sh diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 876a17e4..d684c048 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,7 +10,7 @@ jobs: matrix: flags: ['', '--asan', '--ubsan'] backend: ['MSGQ', 'ZMQ'] - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.11", "3.12"] steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml new file mode 100644 index 00000000..b96a2e51 --- /dev/null +++ b/.github/workflows/wheels.yml @@ -0,0 +1,51 @@ +name: wheels + +on: [push, pull_request] + +jobs: + wheels: + runs-on: ${{ matrix.platform.os }} + strategy: + fail-fast: true + matrix: + platform: [ {os: "ubuntu-latest", target: "manylinux_x86_64", arch: "x86_64"}, + {os: "ubuntu-latest", target: "manylinux_aarch64", arch: "aarch64"}, + {os: "macos-14", target: "macosx_arm64", arch: "arm64"} ] + python: [ {cp: "cp311", py: "3.11"}, {cp: "cp312", py: "3.12"} ] + + steps: + - uses: actions/checkout@v4 + + - name: Set up QEMU + if: runner.os == 'Linux' + uses: docker/setup-qemu-action@v3 + with: + platforms: all + + - name: Building wheel + uses: pypa/cibuildwheel@v2.19.2 + env: + CIBW_BUILD: "${{ matrix.python.cp }}-${{ matrix.platform.target }}" + CIBW_ARCHS: "${{ matrix.platform.arch }}" + CIBW_BEFORE_ALL_LINUX: "bash {project}/scripts/manylinux_dependencies.sh" + CIBW_BEFORE_BUILD_MACOS: "bash {project}/scripts/macos_dependencies.sh" + CIBW_MANYLINUX_X86_64_IMAGE: "manylinux_2_28" + CIBW_MANYLINUX_AARCH64_IMAGE: "manylinux_2_28" + CIBW_TEST_COMMAND: pip install -r {project}/requirements.txt && pytest {package} + + + - uses: actions/setup-python@v5 + if: ${{ matrix.platform.arch }} != 'aarch64' + with: + python-version: ${{ matrix.python.py }} + + - name: Installing the wheel + if: ${{ matrix.platform.arch }} != 'aarch64' + run: | + pip install --break-system-packages ./wheelhouse/*.whl + + - name: Saving wheel + uses: actions/upload-artifact@v4 + with: + name: cibw-wheels-${{ matrix.platform.target }}-${{ matrix.python.cp }} + path: ./wheelhouse/*.whl diff --git a/pyproject.toml b/pyproject.toml index d3c64e6e..bd7b187c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,17 +1,17 @@ [project] name = "msgq" -requires-python = ">= 3.8" +requires-python = ">= 3.11" version = "0.1.0" dependencies = [ "numpy", "Cython", "pycapnp", + "setuptools", ] [project.optional-dependencies] dev = [ - "pyyaml", "scons", "pre-commit", "ruff", diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..e179397e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,50 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --all-extras -o requirements.txt pyproject.toml +cfgv==3.4.0 + # via pre-commit +coverage==7.6.0 + # via + # msgq (pyproject.toml) + # pytest-cov +cython==3.0.10 + # via msgq (pyproject.toml) +distlib==0.3.8 + # via virtualenv +filelock==3.15.4 + # via virtualenv +identify==2.6.0 + # via pre-commit +iniconfig==2.0.0 + # via pytest +nodeenv==1.9.1 + # via pre-commit +numpy==2.0.0 + # via msgq (pyproject.toml) +packaging==24.1 + # via pytest +parameterized==0.9.0 + # via msgq (pyproject.toml) +platformdirs==4.2.2 + # via virtualenv +pluggy==1.5.0 + # via pytest +pre-commit==3.7.1 + # via msgq (pyproject.toml) +pycapnp==2.0.0 + # via msgq (pyproject.toml) +pytest==8.2.2 + # via + # msgq (pyproject.toml) + # pytest-cov +pytest-cov==5.0.0 + # via msgq (pyproject.toml) +pyyaml==6.0.1 + # via pre-commit +ruff==0.5.3 + # via msgq (pyproject.toml) +scons==4.8.0 + # via msgq (pyproject.toml) +setuptools==71.0.4 + # via msgq (pyproject.toml) +virtualenv==20.26.3 + # via pre-commit diff --git a/scripts/build_wheel.sh b/scripts/build_wheel.sh deleted file mode 100755 index 846ae369..00000000 --- a/scripts/build_wheel.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -set -e - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" -cd $DIR - -./install_dependencies.sh -cd .. -python3 -m build diff --git a/scripts/install_dependencies.sh b/scripts/install_dependencies.sh deleted file mode 100755 index 2acdda92..00000000 --- a/scripts/install_dependencies.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - -cd $DIR/../ - -SUDO="" - -# Use sudo if not root -if [[ ! $(id -u) -eq 0 ]]; then - if [[ -z $(which sudo) ]]; then - echo "Please install sudo or run as root" - exit 1 - fi - SUDO="sudo" -fi - -$SUDO apt-get update -$SUDO apt-get install -y --no-install-recommends \ - autoconf \ - build-essential \ - ca-certificates \ - capnproto \ - clang \ - cppcheck \ - curl \ - git \ - libbz2-dev \ - libcapnp-dev \ - libclang-rt-dev \ - libffi-dev \ - liblzma-dev \ - libncurses5-dev \ - libncursesw5-dev \ - libreadline-dev \ - libsqlite3-dev \ - libssl-dev \ - libtool \ - libzmq3-dev \ - llvm \ - make \ - cmake \ - ocl-icd-opencl-dev \ - opencl-headers \ - python3-dev \ - python3-pip \ - tk-dev \ - wget \ - xz-utils \ - zlib1g-dev - -if [[ -n "$BUILD_TESTS" ]]; then - git clone -b v2.x --depth 1 https://github.com/catchorg/Catch2.git - cd Catch2 - mv single_include/* ../ - cd .. - rm -rf Catch2 -fi diff --git a/scripts/macos_dependencies.sh b/scripts/macos_dependencies.sh new file mode 100755 index 00000000..f0e9dc59 --- /dev/null +++ b/scripts/macos_dependencies.sh @@ -0,0 +1,6 @@ +#!/bin/bash +brew bundle --file=- <<-EOS +brew "zeromq" +cask "gcc-arm-embedded" +brew "gcc@13" +EOS diff --git a/scripts/manylinux_dependencies.sh b/scripts/manylinux_dependencies.sh new file mode 100755 index 00000000..9bd69863 --- /dev/null +++ b/scripts/manylinux_dependencies.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +dnf install -y clang opencl-headers ocl-icd-devel cppcheck wget + +wget https://github.com/zeromq/libzmq/releases/download/v4.3.5/zeromq-4.3.5.tar.gz +tar -xvf zeromq-4.3.5.tar.gz +cd zeromq-4.3.5 +./configure +make -j$(nproc) +make install diff --git a/scripts/ubuntu_dependencies.sh b/scripts/ubuntu_dependencies.sh new file mode 100755 index 00000000..88314f59 --- /dev/null +++ b/scripts/ubuntu_dependencies.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +if [[ ! $(id -u) -eq 0 ]]; then + if [[ -z $(which sudo) ]]; then + echo "Please install sudo or run as root" + exit 1 + fi + SUDO="sudo" +fi + +$SUDO apt-get update +$SUDO apt-get install -y --no-install-recommends clang opencl-headers libzmq3-dev ocl-icd-opencl-dev cppcheck diff --git a/setup.py b/setup.py index 0741723c..d3198fb5 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,13 @@ -from setuptools import Command, setup +from setuptools import Command, setup, Distribution from setuptools.command.build import build import subprocess import os +class BinaryDistribution(Distribution): + def has_ext_modules(self): + return True + + class SconsBuild(Command): def initialize_options(self) -> None: pass @@ -12,14 +17,16 @@ def finalize_options(self) -> None: def run(self) -> None: scons_flags = '' if 'BUILD_TESTS' in os.environ else '--minimal' - subprocess.run([f"scons {scons_flags} -j$(nproc)"], shell=True).check_returncode() + subprocess.run([f"scons {scons_flags} -j$(nproc || sysctl -n hw.logicalcpu)"], shell=True).check_returncode() + class CustomBuild(build): sub_commands = [('scons_build', None)] + build.sub_commands setup( packages = ["msgq", "msgq.visionipc"], - package_data={'msgq': ['**/*.cc', '**/*.h', '**/*.pxd', '**/*.pyx', '**/*.so']}, + package_data={'': ['**/*.cc', '**/*.c', '**/*.h', '**/*.pxd', '**/*.pyx', '**/*.py', '**/*.so', '**/*.npy']}, include_package_data=True, - cmdclass={'build': CustomBuild, 'scons_build': SconsBuild} + cmdclass={'build': CustomBuild, 'scons_build': SconsBuild}, + distclass=BinaryDistribution, )