From 805b5dbc0b96274e139ece3a9b183d904001d23e Mon Sep 17 00:00:00 2001 From: Francesco Ceccon Date: Wed, 20 Mar 2024 15:20:32 +0100 Subject: [PATCH] Add CD workflow --- .github/workflows/build.yml | 17 ++- .github/workflows/cd-check.yml | 41 +++++ .github/workflows/cd-pipeline.yml | 50 ++++++ .github/workflows/cd-release.yml | 142 ++++++++++++++++++ .github/workflows/{check.yml => ci-check.yml} | 0 .../{pipeline.yml => ci-pipeline.yml} | 15 +- CONTRIBUTING.md | 2 +- nix/crates.nix | 116 ++++++++++---- 8 files changed, 337 insertions(+), 46 deletions(-) create mode 100644 .github/workflows/cd-check.yml create mode 100644 .github/workflows/cd-pipeline.yml create mode 100644 .github/workflows/cd-release.yml rename .github/workflows/{check.yml => ci-check.yml} (100%) rename .github/workflows/{pipeline.yml => ci-pipeline.yml} (72%) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d0cc4bf5..90f97fef 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,13 +6,21 @@ on: os: required: true type: string + target: + required: false + type: string + default: "all-crates" + artifact_name: + required: false + type: string + default: "" secrets: cachix-token: required: true jobs: build: - name: Build all crates + name: "Build ${{ inputs.target }}" runs-on: "${{ inputs.os }}" steps: - name: Set $USER if needed @@ -28,4 +36,9 @@ jobs: with: name: apibara-public authToken: "${{ secrets.cachix-token }}" - - run: nix build .#all-crates -L --accept-flake-config + - run: nix build .#${{ inputs.target }} -L --accept-flake-config + - uses: actions/upload-artifact@v4 + if: ${{ inputs.artifact_name != '' }} + with: + name: ${{ inputs.artifact_name }} + path: result/* diff --git a/.github/workflows/cd-check.yml b/.github/workflows/cd-check.yml new file mode 100644 index 00000000..66680840 --- /dev/null +++ b/.github/workflows/cd-check.yml @@ -0,0 +1,41 @@ +name: CD Check Pipeline + +concurrency: + group: cd-check-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +on: + merge_group: + + pull_request: + branches: + - "release" + - "release/*" + types: + - opened + - synchronize + +jobs: + build-linux-x86_64: + name: "Linux x86_64: Build" + uses: ./.github/workflows/build.yml + with: + os: warp-ubuntu-latest-x64-4x + secrets: + cachix-token: "${{ secrets.CACHIX_AUTH_TOKEN }}" + + build-linux-aarch64: + name: "Linux aarch64: Build" + uses: ./.github/workflows/build.yml + with: + os: warp-ubuntu-latest-arm64-8x + secrets: + cachix-token: "${{ secrets.CACHIX_AUTH_TOKEN }}" + + build-macos-aarch64: + name: "MacOS aarch64: Build" + uses: ./.github/workflows/build.yml + with: + os: warp-macos-latest-arm64-6x + secrets: + cachix-token: "${{ secrets.CACHIX_AUTH_TOKEN }}" diff --git a/.github/workflows/cd-pipeline.yml b/.github/workflows/cd-pipeline.yml new file mode 100644 index 00000000..97c82409 --- /dev/null +++ b/.github/workflows/cd-pipeline.yml @@ -0,0 +1,50 @@ +name: CD Release Pipeline + +concurrency: + group: cd-${{ github.ref }} + cancel-in-progress: true + +on: + push: + branches: ["add-cd"] + tags: + - "*/*" + +jobs: + extract-version: + name: "Extract target and version" + runs-on: ubuntu-latest + outputs: + target: ${{ steps.extract.outputs.target }} + major: ${{ steps.extract.outputs.major }} + minor: ${{ steps.extract.outputs.minor }} + patch: ${{ steps.extract.outputs.patch }} + steps: + - name: Set $USER if needed + run: | + if [ -z "$USER" ]; then + echo "USER=runner" >> "$GITHUB_ENV" + fi + - uses: actions/checkout@v4 + - uses: cachix/install-nix-action@v25 + with: + nix_path: nixpkgs=channel:nixos-23.11 + - uses: cachix/cachix-action@v14 + with: + name: apibara-public + authToken: "${{ secrets.CACHIX_AUTH_TOKEN }}" + - id: extract + run: nix develop .#ci --accept-flake-config -c extract-version-from-tag + + release: + name: "${{ needs.extract-version.outputs.target }}: Release" + needs: extract-version + uses: ./.github/workflows/cd-release.yml + if: ${{ needs.extract-version.outputs.target != '' }} + with: + target: ${{ needs.extract-version.outputs.target }} + major: ${{ needs.extract-version.outputs.major }} + minor: ${{ needs.extract-version.outputs.minor }} + patch: ${{ needs.extract-version.outputs.patch }} + secrets: + cachix-token: "${{ secrets.CACHIX_AUTH_TOKEN }}" diff --git a/.github/workflows/cd-release.yml b/.github/workflows/cd-release.yml new file mode 100644 index 00000000..8249ac7f --- /dev/null +++ b/.github/workflows/cd-release.yml @@ -0,0 +1,142 @@ +name: Release a target + +on: + workflow_call: + inputs: + target: + required: true + type: string + major: + required: true + type: string + minor: + required: true + type: string + patch: + required: true + type: string + secrets: + cachix-token: + required: true + +jobs: + # Step 1: build the target for all platforms + + build-linux-x86_64: + name: "Linux x86_64: Build" + uses: ./.github/workflows/build.yml + with: + os: warp-ubuntu-latest-x64-4x + target: ${{ inputs.target }} + secrets: + cachix-token: "${{ secrets.cachix-token }}" + + build-linux-aarch64: + name: "Linux aarch64: Build" + uses: ./.github/workflows/build.yml + with: + os: warp-ubuntu-latest-arm64-8x + target: ${{ inputs.target }} + secrets: + cachix-token: "${{ secrets.cachix-token }}" + + build-macos-aarch64: + name: "MacOS aarch64: Build" + uses: ./.github/workflows/build.yml + with: + os: warp-macos-latest-arm64-6x + target: ${{ inputs.target }} + secrets: + cachix-token: "${{ secrets.cachix-token }}" + + # Step 2a: create release archive for all platforms + + build-linux-x86_64-archive: + name: "Linux x86_64: Build archive" + needs: build-linux-x86_64 + uses: ./.github/workflows/build.yml + with: + os: warp-ubuntu-latest-x64-4x + target: ${{ inputs.target }}-archive + artifact_name: "${{ inputs.target }}-x86_64-linux" + secrets: + cachix-token: "${{ secrets.cachix-token }}" + + build-linux-aarch64-archive: + name: "Linux aarch64: Build archive" + needs: build-linux-aarch64 + uses: ./.github/workflows/build.yml + with: + os: warp-ubuntu-latest-arm64-8x + target: ${{ inputs.target }}-archive + artifact_name: "${{ inputs.target }}-aarch64-linux" + secrets: + cachix-token: "${{ secrets.cachix-token }}" + + build-macos-aarch64-archive: + name: "MacOS aarch64: Build archive" + needs: build-macos-aarch64 + uses: ./.github/workflows/build.yml + with: + os: warp-macos-latest-arm64-6x + target: ${{ inputs.target }}-archive + artifact_name: "${{ inputs.target }}-aarch64-macos" + secrets: + cachix-token: "${{ secrets.cachix-token }}" + + # Step 2b: create Docker image for all (Linux) platforms. + + build-linux-x86_64-image: + name: "Linux x86_64: Build Docker image" + needs: build-linux-x86_64 + uses: ./.github/workflows/build.yml + with: + os: warp-ubuntu-latest-x64-4x + target: ${{ inputs.target }}-image + artifact_name: "${{ inputs.target }}-x86_64-image" + secrets: + cachix-token: "${{ secrets.cachix-token }}" + + build-linux-aarch64-image: + name: "Linux aarch64: Build Docker image" + needs: build-linux-aarch64 + uses: ./.github/workflows/build.yml + with: + os: warp-ubuntu-latest-arm64-8x + target: ${{ inputs.target }}-image + artifact_name: "${{ inputs.target }}-aarch64-image" + secrets: + cachix-token: "${{ secrets.cachix-token }}" + + # Step 3a: publish docker images + + publish-docker-images: + name: "Publish Docker images" + needs: [build-linux-x86_64-image, build-linux-aarch64-image] + runs-on: ubuntu-latest + steps: + - name: Download Docker image for x86_64 + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.target }}-x86_64-image + - name: Download Docker image for aarch64 + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.target }}-aarch64-image + - name: Publish Docker images + run: | + ls -l $PWD + + # Step 3b: publish binaries + + publish-binaries: + name: "Publish binaries" + needs: + - build-linux-x86_64-archive + - build-linux-aarch64-archive + - build-macos-aarch64-archive + runs-on: ubuntu-latest + steps: + - name: Publish binaries + run: | + echo "Publishing binaries ${{ inputs.target }}:${{ inputs.major }}.${{ inputs.minor }}.${{ inputs.patch }}" diff --git a/.github/workflows/check.yml b/.github/workflows/ci-check.yml similarity index 100% rename from .github/workflows/check.yml rename to .github/workflows/ci-check.yml diff --git a/.github/workflows/pipeline.yml b/.github/workflows/ci-pipeline.yml similarity index 72% rename from .github/workflows/pipeline.yml rename to .github/workflows/ci-pipeline.yml index 206fc54c..c825bc88 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -14,14 +14,10 @@ on: - opened - synchronize - push: - branches: - - "main" - jobs: check: name: "Checks" - uses: ./.github/workflows/check.yml + uses: ./.github/workflows/ci-check.yml with: os: warp-ubuntu-latest-x64-4x secrets: @@ -36,15 +32,6 @@ jobs: secrets: cachix-token: "${{ secrets.CACHIX_AUTH_TOKEN }}" - # build-linux-aarch64: - # needs: check - # name: "Linux aarch64: Build" - # uses: ./.github/workflows/build.yml - # with: - # os: warp-ubuntu-latest-arm64-4x - # secrets: - # cachix-token: "${{ secrets.CACHIX_AUTH_TOKEN }}" - build-macos-aarch64: needs: check name: "MacOS aarch64: Build" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f4ae4253..46f89463 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -174,7 +174,7 @@ We switched to merge commits for PRs for the following reasons: learn more about backporting fixes. - Start by opening a PR from `main` into `release`. This PR should contain no changes other than changes to the CHANGELOGs and version numbers. -- The `release-check.yml` pipeline is executed. This pipeline simply builds the +- The `cd-check.yml` pipeline is executed. This pipeline simply builds the binaries (we follow the ["not rocket science"](https://graydon2.dreamwidth.org/1597.html) rule). - Once the PR is merged, nothing happens. diff --git a/nix/crates.nix b/nix/crates.nix index 0c2043c4..b7d91a8e 100644 --- a/nix/crates.nix +++ b/nix/crates.nix @@ -175,33 +175,6 @@ let }) crates; - # Docker images. - images = - let - crateNames = builtins.attrNames crates; - images = map - (name: - let - def = crates.${name}; - extraBinaries = map (bin: binaries.${bin}.bin) (if def ? extraBinaries then def.extraBinaries else [ ]); - built = dockerizeCrateBin { - description = def.description; - binaryName = if def ? binaryName then def.binaryName else null; - volumes = if def ? volumes then def.volumes else null; - ports = if def ? ports then def.ports else null; - crate = binaries.${name}; - extraBinaries = extraBinaries; - }; - in - { - name = "${name}-image"; - value = built; - } - ) - crateNames; - in - builtins.listToAttrs images; - # Binaries for non-NixOS systems. binariesUniversal = let @@ -240,6 +213,67 @@ let in pkgs.lib.attrsets.mapAttrs' mkUniversal binaries; + binariesArchive = + let + mkArchive = name: value: { + name = "${name}-archive"; + value = pkgs.stdenv.mkDerivation { + name = "${name}-archive"; + buildInputs = [ + value + pkgs.gzip + ]; + phases = [ "installPhase" ]; + installPhase = '' + mkdir -p $out + gzip -c ${value}/bin/* > $out/${name}.gz + ''; + }; + }; + in + pkgs.lib.attrsets.mapAttrs' mkArchive binariesUniversal; + + # Helper to move the docker archive into a tar.gz file. + mkDockerArchive = name: image: pkgs.stdenv.mkDerivation { + name = "${name}"; + buildInputs = [ + pkgs.skopeo + ]; + phases = [ "installPhase" ]; + installPhase = '' + mkdir -p $out + echo '{"default": [{"type": "insecureAcceptAnything"}]}' > /tmp/policy.json + skopeo copy --policy=/tmp/policy.json --tmpdir=/tmp docker-archive:${image} docker-archive:$out/${name}.tar.gz + ''; + }; + + # Docker images. + images = + let + crateNames = builtins.attrNames crates; + images = map + (name: + let + def = crates.${name}; + extraBinaries = map (bin: binaries.${bin}.bin) (if def ? extraBinaries then def.extraBinaries else [ ]); + built = dockerizeCrateBin { + description = def.description; + binaryName = if def ? binaryName then def.binaryName else null; + volumes = if def ? volumes then def.volumes else null; + ports = if def ? ports then def.ports else null; + crate = binaries.${name}; + extraBinaries = extraBinaries; + }; + in + { + name = "${name}-image"; + value = mkDockerArchive name built; + } + ) + crateNames; + in + builtins.listToAttrs images; + # Ubuntu-based Docker images. imagesUbuntu = let @@ -256,7 +290,8 @@ let mkUbuntuImage = name: value: { name = "${name}-image-ubuntu"; - value = pkgs.dockerTools.buildImage { + value = + let image = pkgs.dockerTools.buildImage { name = name; # we're publishing images, so make it less confusing tag = "latest-ubuntu"; @@ -270,6 +305,7 @@ let ]; }; }; + in mkDockerArchive name image; }; in pkgs.lib.attrsets.mapAttrs' mkUbuntuImage binariesUniversal; @@ -309,11 +345,33 @@ in ARCHIVE_PATH = "${integrationTestsArchive}/archive.tar.zst"; }); + + ci = + let + extractVersionFromTag = pkgs.writeScriptBin "extract-version-from-tag" '' + echo "Ref: $GITHUB_REF" + + if [[ "''${GITHUB_REF:-}" != "refs/tags/"* ]]; then + echo "Not a tag" + exit 0 + fi + + echo "target=sink-console" >> "$GITHUB_OUTPUT" + echo "major=1" >> "$GITHUB_OUTPUT" + echo "minor=2" >> "$GITHUB_OUTPUT" + echo "patch=3" >> "$GITHUB_OUTPUT" + ''; + in + pkgs.mkShell { + buildInputs = [ + extractVersionFromTag + ]; + }; }; binaries = builtins.attrNames binariesUniversal; - packages = images // imagesUbuntu // binariesUniversal // { + packages = images // imagesUbuntu // binariesUniversal // binariesArchive // { all-crates = allCrates; unit-tests = unitTests; integration-tests-archive = integrationTestsArchive;