diff --git a/.github/workflows/build-build-tools-image.yml b/.github/workflows/build-build-tools-image.yml index 82b065c52479..1e51401db7b2 100644 --- a/.github/workflows/build-build-tools-image.yml +++ b/.github/workflows/build-build-tools-image.yml @@ -3,17 +3,23 @@ name: Build build-tools image on: workflow_call: inputs: - image-tag: - description: "build-tools image tag" - required: true + archs: + description: "Json array of architectures to build" + # Default values are set in `check-image` job, `set-variables` step + type: string + required: false + debians: + description: "Json array of Debian versions to build" + # Default values are set in `check-image` job, `set-variables` step type: string + required: false outputs: image-tag: description: "build-tools tag" - value: ${{ inputs.image-tag }} + value: ${{ jobs.check-image.outputs.tag }} image: description: "build-tools image" - value: neondatabase/build-tools:${{ inputs.image-tag }} + value: neondatabase/build-tools:${{ jobs.check-image.outputs.tag }} defaults: run: @@ -35,7 +41,48 @@ permissions: {} jobs: check-image: - uses: ./.github/workflows/check-build-tools-image.yml + runs-on: ubuntu-22.04 + outputs: + archs: ${{ steps.set-variables.outputs.archs }} + debians: ${{ steps.set-variables.outputs.debians }} + tag: ${{ steps.set-variables.outputs.image-tag }} + everything: ${{ steps.set-more-variables.outputs.everything }} + found: ${{ steps.set-more-variables.outputs.found }} + + steps: + - uses: actions/checkout@v4 + + - name: Set variables + id: set-variables + env: + ARCHS: ${{ inputs.archs || '["x64","arm64"]' }} + DEBIANS: ${{ inputs.debians || '["bullseye","bookworm"]' }} + IMAGE_TAG: | + ${{ hashFiles('build-tools.Dockerfile', + '.github/workflows/build-build-tools-image.yml') }} + run: | + echo "archs=${ARCHS}" | tee -a ${GITHUB_OUTPUT} + echo "debians=${DEBIANS}" | tee -a ${GITHUB_OUTPUT} + echo "image-tag=${IMAGE_TAG}" | tee -a ${GITHUB_OUTPUT} + + - name: Set more variables + id: set-more-variables + env: + IMAGE_TAG: ${{ steps.set-variables.outputs.image-tag }} + EVERYTHING: | + ${{ contains(fromJson(steps.set-variables.outputs.archs), 'x64') && + contains(fromJson(steps.set-variables.outputs.archs), 'arm64') && + contains(fromJson(steps.set-variables.outputs.debians), 'bullseye') && + contains(fromJson(steps.set-variables.outputs.debians), 'bookworm') }} + run: | + if docker manifest inspect neondatabase/build-tools:${IMAGE_TAG}; then + found=true + else + found=false + fi + + echo "everything=${EVERYTHING}" | tee -a ${GITHUB_OUTPUT} + echo "found=${found}" | tee -a ${GITHUB_OUTPUT} build-image: needs: [ check-image ] @@ -43,25 +90,12 @@ jobs: strategy: matrix: - debian-version: [ bullseye, bookworm ] - arch: [ x64, arm64 ] + arch: ${{ fromJson(needs.check-image.outputs.archs) }} + debian: ${{ fromJson(needs.check-image.outputs.debians) }} runs-on: ${{ fromJson(format('["self-hosted", "{0}"]', matrix.arch == 'arm64' && 'large-arm64' || 'large')) }} - env: - IMAGE_TAG: ${{ inputs.image-tag }} - steps: - - name: Check `input.tag` is correct - env: - INPUTS_IMAGE_TAG: ${{ inputs.image-tag }} - CHECK_IMAGE_TAG : ${{ needs.check-image.outputs.image-tag }} - run: | - if [ "${INPUTS_IMAGE_TAG}" != "${CHECK_IMAGE_TAG}" ]; then - echo "'inputs.image-tag' (${INPUTS_IMAGE_TAG}) does not match the tag of the latest build-tools image 'inputs.image-tag' (${CHECK_IMAGE_TAG})" - exit 1 - fi - - uses: actions/checkout@v4 - uses: neondatabase/dev-actions/set-docker-config-dir@6094485bf440001c94a94a3f9e221e81ff6b6193 @@ -88,14 +122,14 @@ jobs: push: true pull: true build-args: | - DEBIAN_VERSION=${{ matrix.debian-version }} - cache-from: type=registry,ref=cache.neon.build/build-tools:cache-${{ matrix.debian-version }}-${{ matrix.arch }} - cache-to: ${{ github.ref_name == 'main' && format('type=registry,ref=cache.neon.build/build-tools:cache-{0}-{1},mode=max', matrix.debian-version, matrix.arch) || '' }} + DEBIAN_VERSION=${{ matrix.debian }} + cache-from: type=registry,ref=cache.neon.build/build-tools:cache-${{ matrix.debian }}-${{ matrix.arch }} + cache-to: ${{ github.ref_name == 'main' && format('type=registry,ref=cache.neon.build/build-tools:cache-{0}-{1},mode=max', matrix.debian, matrix.arch) || '' }} tags: | - neondatabase/build-tools:${{ inputs.image-tag }}-${{ matrix.debian-version }}-${{ matrix.arch }} + neondatabase/build-tools:${{ needs.check-image.outputs.tag }}-${{ matrix.debian }}-${{ matrix.arch }} merge-images: - needs: [ build-image ] + needs: [ check-image, build-image ] runs-on: ubuntu-22.04 steps: @@ -107,15 +141,21 @@ jobs: - name: Create multi-arch image env: DEFAULT_DEBIAN_VERSION: bullseye - IMAGE_TAG: ${{ inputs.image-tag }} + ARCHS: ${{ join(fromJson(needs.check-image.outputs.archs), ' ') }} + DEBIANS: ${{ join(fromJson(needs.check-image.outputs.debians), ' ') }} + EVERYTHING: ${{ needs.check-image.outputs.everything }} + IMAGE_TAG: ${{ needs.check-image.outputs.tag }} run: | - for debian_version in bullseye bookworm; do - tags=("-t" "neondatabase/build-tools:${IMAGE_TAG}-${debian_version}") - if [ "${debian_version}" == "${DEFAULT_DEBIAN_VERSION}" ]; then + for debian in ${DEBIANS}; do + tags=("-t" "neondatabase/build-tools:${IMAGE_TAG}-${debian}") + + if [ "${EVERYTHING}" == "true" ] && [ "${debian}" == "${DEFAULT_DEBIAN_VERSION}" ]; then tags+=("-t" "neondatabase/build-tools:${IMAGE_TAG}") fi - docker buildx imagetools create "${tags[@]}" \ - neondatabase/build-tools:${IMAGE_TAG}-${debian_version}-x64 \ - neondatabase/build-tools:${IMAGE_TAG}-${debian_version}-arm64 + for arch in ${ARCHS}; do + tags+=("neondatabase/build-tools:${IMAGE_TAG}-${debian}-${arch}") + done + + docker buildx imagetools create "${tags[@]}" done diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index d415e20db834..a0a542cd9212 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -77,15 +77,9 @@ jobs: shell: bash id: build-tag - check-build-tools-image: - needs: [ check-permissions ] - uses: ./.github/workflows/check-build-tools-image.yml - build-build-tools-image: - needs: [ check-build-tools-image ] + needs: [ check-permissions ] uses: ./.github/workflows/build-build-tools-image.yml - with: - image-tag: ${{ needs.check-build-tools-image.outputs.image-tag }} secrets: inherit check-codestyle-python: diff --git a/.github/workflows/check-build-tools-image.yml b/.github/workflows/check-build-tools-image.yml deleted file mode 100644 index a7a15ad58b67..000000000000 --- a/.github/workflows/check-build-tools-image.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: Check build-tools image - -on: - workflow_call: - outputs: - image-tag: - description: "build-tools image tag" - value: ${{ jobs.check-image.outputs.tag }} - found: - description: "Whether the image is found in the registry" - value: ${{ jobs.check-image.outputs.found }} - -defaults: - run: - shell: bash -euo pipefail {0} - -# No permission for GITHUB_TOKEN by default; the **minimal required** set of permissions should be granted in each job. -permissions: {} - -jobs: - check-image: - runs-on: ubuntu-22.04 - outputs: - tag: ${{ steps.get-build-tools-tag.outputs.image-tag }} - found: ${{ steps.check-image.outputs.found }} - - steps: - - uses: actions/checkout@v4 - - - name: Get build-tools image tag for the current commit - id: get-build-tools-tag - env: - IMAGE_TAG: | - ${{ hashFiles('build-tools.Dockerfile', - '.github/workflows/check-build-tools-image.yml', - '.github/workflows/build-build-tools-image.yml') }} - run: | - echo "image-tag=${IMAGE_TAG}" | tee -a $GITHUB_OUTPUT - - - name: Check if such tag found in the registry - id: check-image - env: - IMAGE_TAG: ${{ steps.get-build-tools-tag.outputs.image-tag }} - run: | - if docker manifest inspect neondatabase/build-tools:${IMAGE_TAG}; then - found=true - else - found=false - fi - - echo "found=${found}" | tee -a $GITHUB_OUTPUT diff --git a/.github/workflows/neon_extra_builds.yml b/.github/workflows/neon_extra_builds.yml index 287c9ea281e4..c5609cb39931 100644 --- a/.github/workflows/neon_extra_builds.yml +++ b/.github/workflows/neon_extra_builds.yml @@ -26,15 +26,9 @@ jobs: with: github-event-name: ${{ github.event_name}} - check-build-tools-image: - needs: [ check-permissions ] - uses: ./.github/workflows/check-build-tools-image.yml - build-build-tools-image: - needs: [ check-build-tools-image ] + needs: [ check-permissions ] uses: ./.github/workflows/build-build-tools-image.yml - with: - image-tag: ${{ needs.check-build-tools-image.outputs.image-tag }} secrets: inherit check-macos-build: diff --git a/.github/workflows/pg-clients.yml b/.github/workflows/pg-clients.yml index df40b5bedab3..4f5495cbe2fa 100644 --- a/.github/workflows/pg-clients.yml +++ b/.github/workflows/pg-clients.yml @@ -39,15 +39,9 @@ jobs: with: github-event-name: ${{ github.event_name }} - check-build-tools-image: - needs: [ check-permissions ] - uses: ./.github/workflows/check-build-tools-image.yml - build-build-tools-image: - needs: [ check-build-tools-image ] + needs: [ check-permissions ] uses: ./.github/workflows/build-build-tools-image.yml - with: - image-tag: ${{ needs.check-build-tools-image.outputs.image-tag }} secrets: inherit test-logical-replication: diff --git a/.github/workflows/pre-merge-checks.yml b/.github/workflows/pre-merge-checks.yml index 137faa7abcb2..496e23e3024f 100644 --- a/.github/workflows/pre-merge-checks.yml +++ b/.github/workflows/pre-merge-checks.yml @@ -16,42 +16,57 @@ jobs: get-changed-files: runs-on: ubuntu-22.04 outputs: + common-changed: ${{ steps.common-src.outputs.any_changed }} python-changed: ${{ steps.python-src.outputs.any_changed }} + any-changed: | + ${{ steps.common-src.outputs.any_changed == 'true' || + steps.python-src.outputs.any_changed == 'true' }} steps: - uses: actions/checkout@v4 + - uses: tj-actions/changed-files@4edd678ac3f81e2dc578756871e4d00c19191daf # v45.0.4 - id: python-src + id: common-src with: files: | + .github/workflows/build-build-tools-image.yml .github/workflows/pre-merge-checks.yml + + - uses: tj-actions/changed-files@4edd678ac3f81e2dc578756871e4d00c19191daf # v45.0.4 + id: python-src + with: + files: | + .github/workflows/_check-codestyle-python.yml **/**.py poetry.lock pyproject.toml - name: PRINT ALL CHANGED FILES FOR DEBUG PURPOSES env: + COMMON_CHANGED_FILES: ${{ steps.common-src.outputs.all_changed_files }} PYTHON_CHANGED_FILES: ${{ steps.python-src.outputs.all_changed_files }} run: | + echo "${COMMON_CHANGED_FILES}" echo "${PYTHON_CHANGED_FILES}" - check-build-tools-image: - if: needs.get-changed-files.outputs.python-changed == 'true' - needs: [ get-changed-files ] - uses: ./.github/workflows/check-build-tools-image.yml - build-build-tools-image: - needs: [ check-build-tools-image ] + if: needs.get-changed-files.outputs.any-changed == 'true' + needs: [ get-changed-files ] uses: ./.github/workflows/build-build-tools-image.yml with: - image-tag: ${{ needs.check-build-tools-image.outputs.image-tag }} + # Build only one combination to save time + archs: '["x64"]' + debians: '["bookworm"]' secrets: inherit check-codestyle-python: - if: needs.get-changed-files.outputs.python-changed == 'true' + if: | + needs.get-changed-files.outputs.python-changed == 'true' || + needs.get-changed-files.outputs.common-changed == 'true' needs: [ get-changed-files, build-build-tools-image ] uses: ./.github/workflows/_check-codestyle-python.yml with: - build-tools-image: ${{ needs.build-build-tools-image.outputs.image }}-bookworm + # `-bookworm-x64` suffix should match the combination in `build-build-tools-image` + build-tools-image: ${{ needs.build-build-tools-image.outputs.image }}-bookworm-x64 secrets: inherit # To get items from the merge queue merged into main we need to satisfy "Status checks that are required". diff --git a/.github/workflows/report-workflow-stats.yml b/.github/workflows/report-workflow-stats.yml index 0d135a257c27..15e446bcd73a 100644 --- a/.github/workflows/report-workflow-stats.yml +++ b/.github/workflows/report-workflow-stats.yml @@ -9,7 +9,6 @@ on: - Build and Test Locally - Build build-tools image - Check Permissions - - Check build-tools image - Check neon with extra platform builds - Cloud Regression Test - Create Release Branch