From 1d33214120dbb235cf4a314e6a12ed4b346d1579 Mon Sep 17 00:00:00 2001 From: Xiao Liang Date: Tue, 22 Jun 2021 01:32:26 +0800 Subject: [PATCH 01/11] ci+cd.feat(wheel & pypi): improve and simplify the process and support more platforms This changes the GitHub action to use cibuildwheel tool to build the wheels, was using manylinux docker instead. By moving to the more generic cibuildwheel, the concrete platform-specific steps to build wheels are encapsulated. This introduction allows building wheels for more os a lot easier (support is TODO). As a side effect, pypy wheels are now built. Second part is to use the GitHub action for publishing to pypi from pypa with simplification to the checking logic. --- .github/workflows/ci.yml | 40 +++++++++++----------------------------- 1 file changed, 11 insertions(+), 29 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 21ba2fbae..f1cc102b1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -127,40 +127,22 @@ jobs: wheels: name: Build and upload wheels - runs-on: ubuntu-latest strategy: matrix: - DOCKER_IMAGE: - - quay.io/pypa/manylinux2014_x86_64 - # Disable i686 builds for now: no binary wheels for cryptography, - # source build fails, e.g. https://github.com/inducer/pyopencl/pull/421/checks?check_run_id=1781071632 - # - quay.io/pypa/manylinux2014_i686 + os: [ubuntu-latest] + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 - - name: "Main Script" - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} - DOCKER_IMAGE: ${{ matrix.DOCKER_IMAGE }} - + - name: "build wheel" run: | - pwd - ls -la - - # Only perform upload for tag builds, otherwise unset TWINE_USERNAME to prevent - if ! [[ $GITHUB_REF == refs/tags* ]]; then - echo "Not a tag build, GITHUB_REF is '$GITHUB_REF'. Unsetting TWINE_USERNAME" - unset TWINE_USERNAME - fi - - if [[ $DOCKER_IMAGE == *i686* ]]; then - PRE_CMD=linux32 - else - PRE_CMD="" - fi - - docker run --rm -v `pwd`:/io -e TWINE_USERNAME -e TWINE_PASSWORD $DOCKER_IMAGE $PRE_CMD /io/scripts/build-wheels.sh - ls wheelhouse/ + pipx run 'cibuildwheel==1.11.1.post1' --output-dir dist + ls -l ./dist/ + - name: "upload wheel" + uses: pypa/gh-action-pypi-publish@v1.4.2 + if: github.event_name == 'release' && github.event.action == 'created' + with: + user: __token__ + password: ${{ secrets.TWINE_PASSWORD }} downstream_tests: strategy: From 2e4ea5f510b1aae72417dfb3f2c0ed4262aa5bd7 Mon Sep 17 00:00:00 2001 From: Xiao Liang Date: Tue, 22 Jun 2021 01:57:39 +0800 Subject: [PATCH 02/11] ci(wheel): fix invocation cmd --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f1cc102b1..e6331d3d5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -135,7 +135,7 @@ jobs: - uses: actions/checkout@v2 - name: "build wheel" run: | - pipx run 'cibuildwheel==1.11.1.post1' --output-dir dist + pipx run --spec='cibuildwheel==1.11.1.post1' cibuildwheel --output-dir dist ls -l ./dist/ - name: "upload wheel" uses: pypa/gh-action-pypi-publish@v1.4.2 From 86b9ed6cae815ef3e8e4b13207c119e02f8aed8b Mon Sep 17 00:00:00 2001 From: Xiao Liang Date: Tue, 22 Jun 2021 02:04:35 +0800 Subject: [PATCH 03/11] ci(wheel): skip unsupported python versions --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e6331d3d5..b00823202 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -134,6 +134,8 @@ jobs: steps: - uses: actions/checkout@v2 - name: "build wheel" + env: + CIBW_SKIP: "cp27-* cp35-* pp27-*" run: | pipx run --spec='cibuildwheel==1.11.1.post1' cibuildwheel --output-dir dist ls -l ./dist/ From aa91b68ec8ff8c19a50a2820e857ba4af70a3073 Mon Sep 17 00:00:00 2001 From: Xiao Liang Date: Tue, 22 Jun 2021 20:06:55 +0800 Subject: [PATCH 04/11] Update ci.yml --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b00823202..3798a33ee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -135,7 +135,9 @@ jobs: - uses: actions/checkout@v2 - name: "build wheel" env: - CIBW_SKIP: "cp27-* cp35-* pp27-*" + CIBW_PROJECT_REQUIRES_PYTHON: "~=3.6" + CIBW_BEFORE_BUILD: "./scripts/prepare-build-wheels.sh" + CIBW_REPAIR_WHEEL_COMMAND: "auditwheel repair {wheel} -w {dest_dir} --lib-sdir=/.libs && python ./scripts/fix-wheel.py {wheel} /deps/ocl-icd/COPYING" run: | pipx run --spec='cibuildwheel==1.11.1.post1' cibuildwheel --output-dir dist ls -l ./dist/ From f9fa31522d7015ec67c9a0733a0049024955c8ed Mon Sep 17 00:00:00 2001 From: Xiao Liang Date: Tue, 22 Jun 2021 20:22:33 +0800 Subject: [PATCH 05/11] Update and rename build-wheels.sh to prepare-build-wheels.sh --- scripts/build-wheels.sh | 100 -------------------------------- scripts/prepare-build-wheels.sh | 80 +++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 100 deletions(-) delete mode 100755 scripts/build-wheels.sh create mode 100755 scripts/prepare-build-wheels.sh diff --git a/scripts/build-wheels.sh b/scripts/build-wheels.sh deleted file mode 100755 index 4b08edc5b..000000000 --- a/scripts/build-wheels.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/bin/bash -set -e -x - -mkdir -p /deps -cd /deps - -function start_spinner { - if [ -n "$SPINNER_PID" ]; then - return - fi - - >&2 echo "Building libraries..." - # Start a process that runs as a keep-alive - # to avoid travis quitting if there is no output - (while true; do - sleep 60 - >&2 echo "Still building..." - done) & - SPINNER_PID=$! - disown -} - -function stop_spinner { - if [ ! -n "$SPINNER_PID" ]; then - return - fi - - kill $SPINNER_PID - unset SPINNER_PID - - >&2 echo "Building libraries finished." -} - -#start_spinner - -curl https://tiker.net/tmp/.tmux.conf -yum install -y git yum openssl-devel ruby - -git clone --branch v2.3.0 https://github.com/OCL-dev/ocl-icd -cd ocl-icd -curl -L -O https://raw.githubusercontent.com/conda-forge/ocl-icd-feedstock/e2c03e3ddb1ff86630ccf80dc7b87a81640025ea/recipe/install-headers.patch -git apply install-headers.patch -curl -L -O https://github.com/isuruf/ocl-icd/commit/3862386b51930f95d9ad1089f7157a98165d5a6b.patch -git apply 3862386b51930f95d9ad1089f7157a98165d5a6b.patch -autoreconf -i -chmod +x configure -./configure --prefix=/usr -make -j4 -make install -cd .. - -# Compile wheels -for PYBIN in /opt/python/*/bin; do - if [[ "${PYBIN}" == *cp36* ]]; then - NUMPY_VERSION="1.11.3" - elif [[ "${PYBIN}" == *cp37* ]]; then - NUMPY_VERSION="1.14.5" - elif [[ "${PYBIN}" == *cp38* ]]; then - NUMPY_VERSION="1.17.3" - elif [[ "${PYBIN}" == *cp39* ]]; then - NUMPY_VERSION="1.19.5" - else - continue - fi - # Build with the oldest numpy available to be compatible with newer ones - "${PYBIN}/pip" install "numpy==${NUMPY_VERSION}" pybind11 mako - "${PYBIN}/pip" wheel /io/ -w wheelhouse/ --no-deps -done - -# Bundle external shared libraries into the wheels -for whl in wheelhouse/pyopencl*.whl; do - auditwheel repair "$whl" -w /io/wheelhouse/ --lib-sdir=/.libs -done - -# Bundle license files - -/opt/python/cp39-cp39/bin/pip install delocate -/opt/python/cp39-cp39/bin/python /io/scripts/fix-wheel.py /deps/ocl-icd/COPYING - -if [[ "${TWINE_USERNAME}" == "" ]]; then - echo "TWINE_USERNAME not set. Skipping uploading wheels" - exit 0 -fi - -/opt/python/cp39-cp39/bin/pip install twine -for WHEEL in /io/wheelhouse/pyopencl*.whl; do - # dev - # /opt/python/cp39-cp39/bin/twine upload \ - # --skip-existing \ - # --repository-url https://test.pypi.org/legacy/ \ - # -u "${TWINE_USERNAME}" -p "${TWINE_PASSWORD}" \ - # "${WHEEL}" - # prod - /opt/python/cp39-cp39/bin/twine upload \ - --skip-existing \ - -u "${TWINE_USERNAME}" -p "${TWINE_PASSWORD}" \ - "${WHEEL}" -done - -#stop_spinner diff --git a/scripts/prepare-build-wheels.sh b/scripts/prepare-build-wheels.sh new file mode 100755 index 000000000..66dffe2f5 --- /dev/null +++ b/scripts/prepare-build-wheels.sh @@ -0,0 +1,80 @@ +#!/bin/bash +set -e -x + +function start_spinner { + if [ -n "$SPINNER_PID" ]; then + return + fi + + >&2 echo "Building libraries..." + # Start a process that runs as a keep-alive + # to avoid travis quitting if there is no output + (while true; do + sleep 60 + >&2 echo "Still building..." + done) & + SPINNER_PID=$! + disown +} + +function stop_spinner { + if [ ! -n "$SPINNER_PID" ]; then + return + fi + + kill $SPINNER_PID + unset SPINNER_PID + + >&2 echo "Building libraries finished." +} + +#start_spinner + +if [[ ! -d ~/deps ]]; then + + # dependencies need to be setup + + mkdir -p ~/deps + pushd ~/deps + + curl https://tiker.net/tmp/.tmux.conf + yum install -y git yum openssl-devel ruby + + git clone --branch v2.3.0 https://github.com/OCL-dev/ocl-icd + cd ocl-icd + curl -L -O https://raw.githubusercontent.com/conda-forge/ocl-icd-feedstock/e2c03e3ddb1ff86630ccf80dc7b87a81640025ea/recipe/install-headers.patch + git apply install-headers.patch + curl -L -O https://github.com/isuruf/ocl-icd/commit/3862386b51930f95d9ad1089f7157a98165d5a6b.patch + git apply 3862386b51930f95d9ad1089f7157a98165d5a6b.patch + autoreconf -i + chmod +x configure + ./configure --prefix=/usr + make -j$(nproc) + make install + + popd +fi + +# Compile wheels +PYTHON_VERSION=$(python -c 'import platform; print(platform.python_version())') +NUMPY_VERSION_SPEC='==' +if [[ "${PYTHON_VERSION}" == '3.6'* ]]; then + NUMPY_VERSION_SPEC="${NUMPY_VERSION_SPEC}1.11.3" +elif [[ "${PYTHON_VERSION}" == '3.7'* ]]; then + NUMPY_VERSION_SPEC="${NUMPY_VERSION_SPEC}1.14.5" +elif [[ "${PYTHON_VERSION}" == '3.8'* ]]; then + NUMPY_VERSION_SPEC="${NUMPY_VERSION_SPEC}1.17.3" +elif [[ "${PYTHON_VERSION}" == '3.9'* ]]; then + NUMPY_VERSION_SPEC="${NUMPY_VERSION_SPEC}1.19.5" +else + # Unknown python version, let it unpinned instead + NUMPY_VERSION_SPEC='' +fi +# TODO: declear dependencies except numpy as build-time dependency as per PEP518 +# Build with the oldest numpy available to be compatible with newer ones +pip install "numpy${NUMPY_VERSION_SPEC}" pybind11 mako + +# For bundling license files +pip install delocate + +#stop_spinner From bbe01daf9144a3d8db8c516124598fe6051b9847 Mon Sep 17 00:00:00 2001 From: Xiao Liang Date: Tue, 22 Jun 2021 20:25:02 +0800 Subject: [PATCH 06/11] Update fix-wheel.py --- scripts/fix-wheel.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/scripts/fix-wheel.py b/scripts/fix-wheel.py index df8f29e2c..f1a65b2f1 100644 --- a/scripts/fix-wheel.py +++ b/scripts/fix-wheel.py @@ -5,20 +5,18 @@ from delocate import wheeltools -def add_library(paths): - wheel_fnames = glob('/io/wheelhouse/pyopencl*.whl') - for fname in wheel_fnames: - print('Processing', fname) - with wheeltools.InWheel(fname, fname): - libs_dir = os.path.join('pyopencl', '.libs') - os.makedirs(libs_dir, exist_ok=True) - for lib_path in paths: - shutil.copy2(lib_path, libs_dir) +def add_library(fname, paths): + print('Processing', fname) + with wheeltools.InWheel(fname, fname): + libs_dir = os.path.join('pyopencl', '.libs') + os.makedirs(libs_dir, exist_ok=True) + for lib_path in paths: + shutil.copy2(lib_path, libs_dir) def main(): - args = list(sys.argv) - args.pop(0) - add_library(args) + fname = sys.argv[1] + paths = sys.argv[2:] + add_library(fname, paths) if __name__ == '__main__': main() From d5c2c2bce8d417310c4c71f6314bcc8f3bf29ddd Mon Sep 17 00:00:00 2001 From: Xiao Liang Date: Tue, 22 Jun 2021 20:38:31 +0800 Subject: [PATCH 07/11] Update ci.yml --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3798a33ee..ed652957c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,6 +8,8 @@ on: pull_request: schedule: - cron: '17 3 * * 0' + release: + types: [created] jobs: flake8: @@ -138,6 +140,7 @@ jobs: CIBW_PROJECT_REQUIRES_PYTHON: "~=3.6" CIBW_BEFORE_BUILD: "./scripts/prepare-build-wheels.sh" CIBW_REPAIR_WHEEL_COMMAND: "auditwheel repair {wheel} -w {dest_dir} --lib-sdir=/.libs && python ./scripts/fix-wheel.py {wheel} /deps/ocl-icd/COPYING" + CIBW_MANYLINUX_X86_64_IMAGE: "manylinux2014" run: | pipx run --spec='cibuildwheel==1.11.1.post1' cibuildwheel --output-dir dist ls -l ./dist/ From 472144baa354ec67f7871dd43d0d50e65f0d5c45 Mon Sep 17 00:00:00 2001 From: Xiao Liang Date: Tue, 22 Jun 2021 20:45:23 +0800 Subject: [PATCH 08/11] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ed652957c..6363f031c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -139,7 +139,7 @@ jobs: env: CIBW_PROJECT_REQUIRES_PYTHON: "~=3.6" CIBW_BEFORE_BUILD: "./scripts/prepare-build-wheels.sh" - CIBW_REPAIR_WHEEL_COMMAND: "auditwheel repair {wheel} -w {dest_dir} --lib-sdir=/.libs && python ./scripts/fix-wheel.py {wheel} /deps/ocl-icd/COPYING" + CIBW_REPAIR_WHEEL_COMMAND: "auditwheel repair {wheel} -w {dest_dir} --lib-sdir=/.libs && python ./scripts/fix-wheel.py {wheel} ~/deps/ocl-icd/COPYING" CIBW_MANYLINUX_X86_64_IMAGE: "manylinux2014" run: | pipx run --spec='cibuildwheel==1.11.1.post1' cibuildwheel --output-dir dist From 67a44859e5556be45b8ff086c663a7a2b201e58f Mon Sep 17 00:00:00 2001 From: Xiao Liang Date: Tue, 22 Jun 2021 20:55:35 +0800 Subject: [PATCH 09/11] Update ci.yml --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6363f031c..20f4c9e0e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -141,6 +141,7 @@ jobs: CIBW_BEFORE_BUILD: "./scripts/prepare-build-wheels.sh" CIBW_REPAIR_WHEEL_COMMAND: "auditwheel repair {wheel} -w {dest_dir} --lib-sdir=/.libs && python ./scripts/fix-wheel.py {wheel} ~/deps/ocl-icd/COPYING" CIBW_MANYLINUX_X86_64_IMAGE: "manylinux2014" + CIBW_ARCHS: "auto64" # 32bit unsupported for now run: | pipx run --spec='cibuildwheel==1.11.1.post1' cibuildwheel --output-dir dist ls -l ./dist/ From 1e3a298dd8467186ee3c13d647429a7ed4919ae1 Mon Sep 17 00:00:00 2001 From: Xiao Liang Date: Tue, 22 Jun 2021 21:08:57 +0800 Subject: [PATCH 10/11] Update ci.yml --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 20f4c9e0e..8fd93c99d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -138,9 +138,11 @@ jobs: - name: "build wheel" env: CIBW_PROJECT_REQUIRES_PYTHON: "~=3.6" + CIBW_SKIP: "pp36-*" # The docker image using doesn't support pp36 CIBW_BEFORE_BUILD: "./scripts/prepare-build-wheels.sh" CIBW_REPAIR_WHEEL_COMMAND: "auditwheel repair {wheel} -w {dest_dir} --lib-sdir=/.libs && python ./scripts/fix-wheel.py {wheel} ~/deps/ocl-icd/COPYING" CIBW_MANYLINUX_X86_64_IMAGE: "manylinux2014" + CIBW_MANYLINUX_PYPY_X86_64_IMAGE: "quay.io/pypa/manylinux2014_x86_64" CIBW_ARCHS: "auto64" # 32bit unsupported for now run: | pipx run --spec='cibuildwheel==1.11.1.post1' cibuildwheel --output-dir dist From d65dd85fbf7627e4db3c881fa65868e7e84ba07c Mon Sep 17 00:00:00 2001 From: Xiao Liang Date: Tue, 22 Jun 2021 21:25:45 +0800 Subject: [PATCH 11/11] Update prepare-build-wheels.sh --- scripts/prepare-build-wheels.sh | 33 +-------------------------------- 1 file changed, 1 insertion(+), 32 deletions(-) diff --git a/scripts/prepare-build-wheels.sh b/scripts/prepare-build-wheels.sh index 66dffe2f5..0d0004965 100755 --- a/scripts/prepare-build-wheels.sh +++ b/scripts/prepare-build-wheels.sh @@ -1,34 +1,5 @@ #!/bin/bash -set -e -x - -function start_spinner { - if [ -n "$SPINNER_PID" ]; then - return - fi - - >&2 echo "Building libraries..." - # Start a process that runs as a keep-alive - # to avoid travis quitting if there is no output - (while true; do - sleep 60 - >&2 echo "Still building..." - done) & - SPINNER_PID=$! - disown -} - -function stop_spinner { - if [ ! -n "$SPINNER_PID" ]; then - return - fi - - kill $SPINNER_PID - unset SPINNER_PID - - >&2 echo "Building libraries finished." -} - -#start_spinner +set -e -x -o pipefail if [[ ! -d ~/deps ]]; then @@ -76,5 +47,3 @@ pip install "numpy${NUMPY_VERSION_SPEC}" pybind11 mako # For bundling license files pip install delocate - -#stop_spinner