From adf5596437297725b4e6ab2f106fbbc6f1a892da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonatan=20K=C5=82osko?= Date: Mon, 14 Aug 2023 16:08:58 +0200 Subject: [PATCH] Simplify release flow --- ...unless_exists.sh => compile_and_upload.sh} | 14 +- .github/scripts/publish_release.sh | 29 ---- .github/scripts/upload_archives.sh | 35 ----- .github/workflows/release.yml | 133 +++++------------- README.md | 2 +- 5 files changed, 50 insertions(+), 163 deletions(-) rename .github/scripts/{compile_unless_exists.sh => compile_and_upload.sh} (51%) delete mode 100755 .github/scripts/publish_release.sh delete mode 100755 .github/scripts/upload_archives.sh diff --git a/.github/scripts/compile_unless_exists.sh b/.github/scripts/compile_and_upload.sh similarity index 51% rename from .github/scripts/compile_unless_exists.sh rename to .github/scripts/compile_and_upload.sh index 12ae740..8734f17 100755 --- a/.github/scripts/compile_unless_exists.sh +++ b/.github/scripts/compile_and_upload.sh @@ -4,18 +4,28 @@ set -ex cd "$(dirname "$0")/../.." +tag=$1 + # Ensure tasks are compiled mix compile -tag=$(mix xla.info release_tag) - if gh release list | grep $tag; then archive_filename=$(mix xla.info archive_filename) + build_archive_dir=$(mix xla.info build_archive_dir) if gh release view $tag | grep $archive_filename; then echo "Found $archive_filename in $tag release artifacts, skipping compilation" else XLA_BUILD=true mix compile + + # Uploading is the final action after several hour long build, + # so in case of any temporary network failures we want to retry + # a number of times + for i in {1..10}; do + gh release upload --clobber $tag "$build_archive_dir/$archive_filename" && break + echo "Upload failed, retrying in 30s" + sleep 30 + done fi else echo "::error::Release $tag not found" diff --git a/.github/scripts/publish_release.sh b/.github/scripts/publish_release.sh deleted file mode 100755 index 844cb1f..0000000 --- a/.github/scripts/publish_release.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -set -e - -cd "$(dirname "$0")/../.." - -# Ensure tasks are compiled -mix compile - -tag=$(mix xla.info release_tag) - -if gh release list | grep -q $tag; then - echo "Release $tag already exists, make sure to bump the version in mix.exs" - exit 1 -fi - -if [[ $(git diff @ @{upstream} | wc -c) -ne 0 ]]; then - echo "Your git working directory is not up to date with remote, make sure to push the changes first" - exit 1 -fi - -read -p "This will publish a new release $tag and trigger the release build. Do you want to continue? [y/N] " -if [[ ! $REPLY =~ ^[yY]$ ]]; then - exit 0 -fi - -gh release create $tag --notes "" - -echo "Successfully created release $tag. Remember to wait for the release build to finish before publishing the Hex package." diff --git a/.github/scripts/upload_archives.sh b/.github/scripts/upload_archives.sh deleted file mode 100755 index 86572cb..0000000 --- a/.github/scripts/upload_archives.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -set -ex - -cd "$(dirname "$0")/../.." - -# Ensure tasks are compiled -mix compile - -tag=$(mix xla.info release_tag) -build_archive_dir=$(mix xla.info build_archive_dir) -upload_dir=tmp/upload - -if [[ -d $build_archive_dir ]]; then - # Copy the archives into directory within this repo, - # so that gh cli works within its context - mkdir -p $upload_dir - rm -f $upload_dir/* - cp $build_archive_dir/* $upload_dir - cd $upload_dir -else - echo "Build directory not found" - exit 0 -fi - -for file in *.tar.gz; do - # Uploading is the final action after several hour long build, - # so in case of any temporary network failures we want to retry - # a number of times - for i in {1..10}; do - gh release upload --clobber $tag $file && break - echo "Upload failed, retrying in 30s" - sleep 30 - done -done diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0c3a118..18841b4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,11 +1,28 @@ name: Release on: - release: - types: [published] - workflow_dispatch: + push: + tags: + - "v*.*.*" + jobs: - # linux x86_64 cpu/tpu + create_draft_release: + if: github.ref_type == 'tag' + permissions: + contents: write + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v3 + - name: Create draft release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + if ! gh release list | grep -q ${{ github.ref_name }}; then + gh release create --title ${{ github.ref_name }} --draft ${{ github.ref_name }} + fi + linux: + name: "x86_64-linux-gnu-{cpu,tpu}" + needs: [create_draft_release] # We intentionally build on ubuntu 20 to compile against # an older version of glibc runs-on: ubuntu-20.04 @@ -25,67 +42,20 @@ jobs: - run: python -m pip install --upgrade pip numpy # Build and upload the archives - run: mix deps.get - - run: .github/scripts/compile_unless_exists.sh + - run: .github/scripts/compile_and_upload.sh ${{ github.ref_name }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} XLA_TARGET: cpu CC: gcc-9 - - run: .github/scripts/compile_unless_exists.sh + - run: .github/scripts/compile_and_upload.sh ${{ github.ref_name }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} XLA_TARGET: tpu CC: gcc-9 - - run: .github/scripts/upload_archives.sh - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - # # linux x86_64 cpu musl - # alpine: - # runs-on: ubuntu-20.04 - # strategy: - # fail-fast: false - # container: hexpm/elixir:1.13.4-erlang-25.0.2-alpine-3.16.0 - # env: - # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # XLA_TARGET: cpu - # BAZEL_VERSION: "5.3.0" - # BAZEL_SHA256SUM: "ee801491ff0ec3a562422322a033c9afe8809b64199e4a94c7433d4e14e6b921 bazel-5.3.0-dist.zip" - # JAVA_HOME: "/usr/lib/jvm/default-jvm" - # steps: - # - name: Install system packages - # run: | - # apk update && apk upgrade && \ - # apk add --no-cache python3 py3-pip python3-dev py3-numpy && \ - # apk add --no-cache libstdc++ openjdk11 && \ - # apk add --no-cache bash curl git wget && \ - # apk add --no-cache musl-dev make libexecinfo libexecinfo-dev && \ - # apk add --no-cache coreutils gcc g++ linux-headers unzip zip && \ - # apk add --no-cache automake gcc subversion && \ - # apk add --no-cache github-cli && \ - # DIR=$(mktemp -d) && cd ${DIR} && \ - # curl -sLO https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-dist.zip && \ - # echo ${BAZEL_SHA256SUM} | sha256sum --check && \ - # unzip bazel-${BAZEL_VERSION}-dist.zip && \ - # EXTRA_BAZEL_ARGS="--tool_java_runtime_version=local_jdk" bash ./compile.sh && \ - # cp ${DIR}/output/bazel /usr/local/bin/ && \ - # rm -rf ${DIR} - # - name: Install hex - # run: mix local.hex --force && mix local.rebar --force - # # Prevent git from checking the repository owner and erroring with "dubious ownership" - # - run: git config --global --add safe.directory '*' - # - uses: actions/checkout@v3 - # # Build and upload the archive - # - run: mix deps.get - # - run: .github/scripts/compile_unless_exists.sh - # env: - # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # XLA_TARGET: cpu - # - run: .github/scripts/upload_archives.sh - # env: - # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # darwin x86_64 cpu macos: + name: "x86_64-darwin-cpu" + needs: [create_draft_release] runs-on: macos-12 steps: - uses: actions/checkout@v3 @@ -101,17 +71,15 @@ jobs: - run: python -m pip install --upgrade pip numpy # Build and upload the archive - run: mix deps.get - - run: .github/scripts/compile_unless_exists.sh + - run: .github/scripts/compile_and_upload.sh ${{ github.ref_name }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} XLA_TARGET: cpu CC: gcc-9 - - run: .github/scripts/upload_archives.sh - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # darwin aarch64 cpu (cross-compiled) macos_arm: + name: "aarch64-darwin-cpu (cross-compiled)" + needs: [create_draft_release] runs-on: macos-12 steps: - uses: actions/checkout@v3 @@ -127,30 +95,18 @@ jobs: - run: python -m pip install --upgrade pip numpy # Build and upload the archive - run: mix deps.get - - run: .github/scripts/compile_unless_exists.sh + - run: .github/scripts/compile_and_upload.sh ${{ github.ref_name }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} XLA_TARGET: cpu + XLA_TARGET_PLATFORM: "aarch64-darwin" # Explicitly cross-compile for arm64 BUILD_FLAGS: "--config=macos_arm64" CC: gcc-9 - - name: Rename archive - run: | - # Ensure tasks are compiled - mix compile - build_archive_dir=$(mix xla.info build_archive_dir) - if [[ -d $build_archive_dir ]]; then - archive_path=$(find $build_archive_dir -type f) - actual_archive_path="${archive_path/x86_64/aarch64}" - echo "$archive_path -> $actual_archive_path" - mv $archive_path $actual_archive_path - fi; - - run: .github/scripts/upload_archives.sh - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # linux x86_64 cuda linux_cuda: + name: "x86_64-linux-gnu-${{ matrix.xla_target }}" + needs: [create_draft_release] runs-on: ubuntu-20.04 strategy: fail-fast: false @@ -211,16 +167,14 @@ jobs: - run: python -m pip install --upgrade pip numpy # Build and upload the archive - run: mix deps.get - - run: .github/scripts/compile_unless_exists.sh + - run: .github/scripts/compile_and_upload.sh ${{ github.ref_name }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} XLA_TARGET: ${{ matrix.xla_target }} - - run: .github/scripts/upload_archives.sh - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # linux aarch64 cpu (cross-compiled) linux_arm: + name: "aarch64-linux-gnu-cpu (cross-compiled)" + needs: [create_draft_release] # We intentionally build on ubuntu 20 to compile against # an older version of glibc runs-on: ubuntu-20.04 @@ -242,24 +196,11 @@ jobs: - run: mix deps.get # Hide system OpenSSL as suggested in https://github.com/tensorflow/tensorflow/issues/48401#issuecomment-818377995 - run: sudo mv /usr/include/openssl /usr/include/openssl.original - - run: .github/scripts/compile_unless_exists.sh + - run: .github/scripts/compile_and_upload.sh ${{ github.ref_name }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} XLA_TARGET: cpu + XLA_TARGET_PLATFORM: "aarch64-linux-gnu" # Explicitly cross-compile for arm64 BUILD_FLAGS: "--config=elinux_aarch64" CC: gcc-9 - - name: Rename archive - run: | - # Ensure tasks are compiled - mix compile - build_archive_dir=$(mix xla.info build_archive_dir) - if [[ -d $build_archive_dir ]]; then - archive_path=$(find $build_archive_dir -type f) - actual_archive_path="${archive_path/x86_64/aarch64}" - echo "$archive_path -> $actual_archive_path" - mv $archive_path $actual_archive_path - fi; - - run: .github/scripts/upload_archives.sh - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/README.md b/README.md index 934a838..60867c4 100644 --- a/README.md +++ b/README.md @@ -160,7 +160,7 @@ for the list of available flags. To publish a new version of this package: 1. Update version in `mix.exs`. -2. Run `.github/scripts/publish_release.sh`. +2. Create and push a new tag. 3. Wait for the release workflow to build all the binaries. 4. Publish the package to Hex.