try with irsa #54
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| # desc: build container images, perform static tests then publish | |
| name: build-images | |
| on: | |
| push: | |
| pull_request_target: | |
| jobs: | |
| lint: | |
| name: pre-commit | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v3 | |
| - uses: pre-commit/[email protected] | |
| build-image: | |
| runs-on: ubuntu-latest | |
| # if: ${{ github.event_name == 'push' }} | |
| if: false | |
| # TODO: reneable | |
| outputs: | |
| full_image_name: ${{ steps.compute-image-name-step.outputs.full_image_name }} | |
| # TODO: later, we should be able to build multiple versions of keycloak | |
| # strategy: | |
| # matrix: | |
| # keycloak_version: [keycloak-22, keycloak-23] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install if required common software tooling | |
| uses: camunda/infra-global-github-actions/common-tooling@main | |
| with: | |
| java-enabled: false | |
| yarn-enabled: false | |
| python-version: "3" | |
| - name: Import secrets | |
| uses: hashicorp/vault-action@d1720f055e0635fd932a1d2a48f87a666a57906c # v3.0.0 | |
| id: secrets | |
| with: | |
| url: ${{ secrets.VAULT_ADDR }} | |
| method: approle | |
| roleId: ${{ secrets.VAULT_ROLE_ID }} | |
| secretId: ${{ secrets.VAULT_SECRET_ID }} | |
| secrets: | | |
| secret/data/products/infrastructure-experience/ci/common MACHINE_PWD; | |
| secret/data/products/infrastructure-experience/ci/common MACHINE_USR; | |
| - name: Build image using Camunda docker build | |
| id: build-image-step | |
| uses: camunda/infra-global-github-actions/build-docker-image@feature/container-build-multiarch | |
| with: | |
| registry_host: ${{ vars.CONTAINER_REGISTRY_CI }} | |
| registry_username: ${{ steps.secrets.outputs.MACHINE_USR }} | |
| registry_password: ${{ steps.secrets.outputs.MACHINE_PWD }} | |
| force_push: true | |
| image_name: ${{ vars.CONTAINER_IMAGE_NAME_CI }} | |
| build_context: . | |
| build_platforms: linux/amd64,linux/arm64 | |
| extra_tags: | # the ci- prefix ensures a build context, this image is treated as "temporary" | |
| type=sha,enable=true,priority=1000,prefix=ci-sha-,suffix=,format=short | |
| - name: Compute target built image fully qualified name from metadata | |
| id: compute-image-name-step | |
| run: | | |
| image_metadata='${{ steps.build-image-step.outputs.image_metadata }}' | |
| image_name=$(echo "${image_metadata}" | tr -d '\n' | jq -r '."image.name"' | tr ',' '\n' | head -n 1 | tr -d ' ') | |
| digest=$(echo "${image_metadata}" | tr -d '\n' | jq -r '."containerimage.digest"') | |
| full_image_name="${image_name}@${digest}" | |
| echo "full_image_name=${full_image_name}" >> "$GITHUB_OUTPUT" | |
| echo "$full_image_name" | |
| test-base-image: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - build-image | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install if required common software tooling | |
| uses: camunda/infra-global-github-actions/common-tooling@main | |
| with: | |
| java-enabled: false | |
| yarn-enabled: false | |
| python-version: "3" | |
| - name: Import secrets | |
| uses: hashicorp/vault-action@d1720f055e0635fd932a1d2a48f87a666a57906c # v3.0.0 | |
| id: secrets | |
| with: | |
| url: ${{ secrets.VAULT_ADDR }} | |
| method: approle | |
| roleId: ${{ secrets.VAULT_ROLE_ID }} | |
| secretId: ${{ secrets.VAULT_SECRET_ID }} | |
| secrets: | | |
| secret/data/products/infrastructure-experience/ci/common MACHINE_PWD; | |
| secret/data/products/infrastructure-experience/ci/common MACHINE_USR; | |
| - name: Login to the registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ vars.CONTAINER_REGISTRY_CI }} | |
| username: ${{ steps.secrets.outputs.MACHINE_USR }} | |
| password: ${{ steps.secrets.outputs.MACHINE_PWD }} | |
| - name: KeyCloak Show-Config | |
| run: | | |
| docker run "${{ needs.build-image.outputs.full_image_name }}" show-config >> docker.log | |
| echo "config=$(< docker.log tr '\n' ' ')" >> "$GITHUB_ENV" | |
| - name: Assert Config | |
| env: | |
| CONFIG: ${{ env.config }} | |
| run: python3 ./.github/scripts/build-check/main.py | |
| test-postgres-integ: | |
| strategy: | |
| fail-fast: false # don't propate failing jobs | |
| matrix: | |
| runner_desc: | |
| - {runner: ubuntu-22.04, postgres_replicas: 1, keycloak_jdbc_protocol: "postgresql", keycloak_db_driver: software.amazon.jdbc.Driver, keycloak_db_host_template: "postgres", keycloak_db_jdbc_query: "", keycloak_db_port: "5432" } | |
| - {runner: aws-core-2-default, postgres_replicas: 0, keycloak_jdbc_protocol: "aws-wrapper:postgresql", keycloak_db_driver: software.amazon.jdbc.Driver, keycloak_db_host_template: "camunda-ci-eks-aurora-postgresql-{{ postgres_version }}.cluster-clnwzia8ptad.eu-central-1.rds.amazonaws.com", keycloak_db_jdbc_query: "?wrapperPlugins=iam&ssl=true&sslmode=require&sslrootcert=/certs/rds-ca-2019-root.pem", keycloak_db_port: "5432" } | |
| - {runner: aws-arm-core-2-default, postgres_replicas: 0, keycloak_jdbc_protocol: "aws-wrapper:postgresql", keycloak_db_driver: software.amazon.jdbc.Driver, keycloak_db_host_template: "camunda-ci-eks-aurora-postgresql-{{ postgres_version }}.cluster-clnwzia8ptad.eu-central-1.rds.amazonaws.com", keycloak_db_jdbc_query: "?wrapperPlugins=iam&ssl=true&sslmode=require&sslrootcert=/certs/rds-ca-2019-root.pem", keycloak_db_port: "5432" } | |
| # GCloud SQL is not tested yet as we don't have a dedicated db, fallback on integrated db | |
| - {runner: gcp-core-2-default, postgres_replicas: 1, keycloak_jdbc_protocol: "postgresql", keycloak_db_driver: software.amazon.jdbc.Driver, keycloak_db_host_template: "postgres", keycloak_db_jdbc_query: "", keycloak_db_port: "5432" } | |
| postgres_version: [15] # reference: https://www.keycloak.org/server/db | |
| runs-on: ${{ matrix.runner_desc.runner }} | |
| # TODO: renable | |
| # needs: | |
| # - build-image | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install if required common software tooling | |
| uses: camunda/infra-global-github-actions/common-tooling@main | |
| with: | |
| java-enabled: false | |
| yarn-enabled: false | |
| python-version: "3" | |
| - name: Import secrets | |
| uses: hashicorp/vault-action@d1720f055e0635fd932a1d2a48f87a666a57906c # v3.0.0 | |
| id: secrets | |
| with: | |
| url: ${{ secrets.VAULT_ADDR }} | |
| method: approle | |
| roleId: ${{ secrets.VAULT_ROLE_ID }} | |
| secretId: ${{ secrets.VAULT_SECRET_ID }} | |
| # for the moment, we share the secrets of the web-modeler for aurora | |
| secrets: | | |
| secret/data/products/infrastructure-experience/ci/common MACHINE_PWD; | |
| secret/data/products/infrastructure-experience/ci/common MACHINE_USR; | |
| secret/data/products/web-modeler/ci/common AURORA_POSTGRESQL_PASSWORD; | |
| secret/data/products/web-modeler/ci/common AURORA_POSTGRESQL_USERNAME; | |
| - name: Compute AWS variables | |
| if: startsWith(matrix.runner_desc.runner, 'aws') | |
| run: | | |
| : # aws aurora superuser is only used for bootstrapping a standard user that will auth using irsa | |
| echo "postgres_superuser=${{ steps.secrets.outputs.AURORA_POSTGRESQL_USERNAME }}" >> "$GITHUB_ENV" | |
| echo "postgres_superuser_password=${{ steps.secrets.outputs.AURORA_POSTGRESQL_PASSWORD }}" >> "$GITHUB_ENV" | |
| echo "postgres_superuser=${postgres_superuser}" | |
| : # TODO: revert | |
| : # echo "postgres_user=keycloak-irsa" >> "$GITHUB_ENV" | |
| echo "postgres_user=web-modeler-irsa" >> "$GITHUB_ENV" | |
| echo "postgres_user=${postgres_user}" | |
| echo "postgres_password=" >> "$GITHUB_ENV" | |
| echo "compose_keycloak_volumes=./rds-ca-2019-root.pem:/certs/rds-ca-2019-root.pem" >> "$GITHUB_ENV" | |
| : # export AWS variables | |
| echo "AWS_STS_REGIONAL_ENDPOINTS=$AWS_STS_REGIONAL_ENDPOINTS" >> "$GITHUB_ENV" | |
| echo "AWS_STS_REGIONAL_ENDPOINTS=${AWS_STS_REGIONAL_ENDPOINTS}" | |
| echo "AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION" >> "$GITHUB_ENV" | |
| echo "AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION}" | |
| echo "AWS_REGION=$AWS_REGION" >> "$GITHUB_ENV" | |
| echo "AWS_REGION=${AWS_REGION}" | |
| echo "AWS_ROLE_ARN=$AWS_ROLE_ARN" >> "$GITHUB_ENV" | |
| echo "AWS_ROLE_ARN=${AWS_ROLE_ARN}" | |
| echo "AWS_WEB_IDENTITY_TOKEN_FILE=$AWS_WEB_IDENTITY_TOKEN_FILE" >> "$GITHUB_ENV" | |
| echo "AWS_WEB_IDENTITY_TOKEN_FILE=${AWS_WEB_IDENTITY_TOKEN_FILE}" | |
| - name: Compute Ubuntu variables | |
| if: startsWith(matrix.runner_desc.runner, 'ubuntu') | |
| run: | | |
| : # we use plain values that will be used by the postgres container | |
| echo "postgres_user=keycloak" >> "$GITHUB_ENV" | |
| echo "postgres_password=password" >> "$GITHUB_ENV" | |
| echo "postgres_user=${postgres_user}" | |
| - name: Declare test recipe variables | |
| shell: bash | |
| run: | | |
| if (( "${{ matrix.runner_desc.postgres_replicas }}" < 1 )); then | |
| echo "compose_keycloak_depends_on=" >> "$GITHUB_ENV" | |
| else | |
| echo "compose_keycloak_depends_on=postgres" >> "$GITHUB_ENV" | |
| fi | |
| : # ensure uniqueness of the db name | |
| postgres_database="db-${{ matrix.runner_desc.runner }}-${{ github.sha }}" | |
| echo "postgres_database=${postgres_database}" >> "$GITHUB_ENV" | |
| echo "postgres_database=$postgres_database" | |
| : # apply template on the address | |
| postgres_host=$(echo "${{ matrix.runner_desc.keycloak_db_host_template }}" | sed "s/{{ postgres_version }}/${{ matrix.postgres_version }}/g") | |
| echo "postgres_host=${postgres_host}" >> "$GITHUB_ENV" | |
| echo "postgres_host=${postgres_host}" | |
| : # compute the keycloak db url | |
| test_db_url="jdbc:${{ matrix.runner_desc.keycloak_jdbc_protocol }}://${postgres_host}:${{ matrix.runner_desc.keycloak_db_port }}/${postgres_database}${{ matrix.runner_desc.keycloak_db_jdbc_query }}" | |
| echo "test_db_url=${test_db_url}" >> "$GITHUB_ENV" | |
| echo "test_db_url=$test_db_url" | |
| - name: Login to the registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ vars.CONTAINER_REGISTRY_CI }} | |
| username: ${{ steps.secrets.outputs.MACHINE_USR }} | |
| password: ${{ steps.secrets.outputs.MACHINE_PWD }} | |
| # The self-hosted runner doesn't provide a postgres client and the prerequisites for make, | |
| # so we need to install them manually | |
| - name: Install required packages | |
| run: sudo apt-get update && sudo apt-get install -y build-essential postgresql-client | |
| - name: Tear up Aurora PG (aws only) | |
| if: startsWith(matrix.runner_desc.runner, 'aws') | |
| run: ./.helpers/actions/create-aurora-pg-db.sh | |
| env: | |
| PGDATABASE: "${{ env.postgres_database }}" | |
| PGHOST: ${{ env.postgres_host }} | |
| PGPORT: ${{ matrix.runner_desc.keycloak_db_port }} | |
| PGPASSWORD: ${{ env.postgres_superuser_password }} | |
| PGUSER: ${{ env.postgres_superuser }} | |
| PGUSER_IRSA: ${{ env.postgres_user }} | |
| - name: Test aurora pg connection | |
| if: startsWith(matrix.runner_desc.runner, 'aws') | |
| run: | | |
| curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" | |
| unzip awscliv2.zip | |
| sudo ./aws/install | |
| : # download ca certs | |
| curl https://s3.amazonaws.com/rds-downloads/rds-ca-2019-root.pem -o "rds-ca-2019-root.pem" | |
| : # export RDSHOST="${{ env.postgres_host }}" | |
| : # export PGPASSWORD="$(aws rds generate-db-auth-token --hostname $RDSHOST --port ${{ matrix.runner_desc.keycloak_db_port }} --region ${{ env.AWS_REGION }} --username ${{ env.postgres_user }})" | |
| : # psql -h "${{ env.postgres_host }}" -p "${{ matrix.runner_desc.keycloak_db_port }}" "dbname=${{ env.postgres_database }} user=${{ env.postgres_user }}" -c 'SELECT version();' | |
| - name: Start Test Environment | |
| uses: ./.github/actions/compose | |
| with: | |
| compose_file: ${{ github.workspace }}/docker-compose.yml | |
| project_name: keycloak | |
| env: | |
| POSTGRES_DB: "${{ env.postgres_database }}" | |
| POSTGRES_USER: "${{ env.postgres_user }}" | |
| POSTGRES_PASSWORD: "${{ env.postgres_password }}" | |
| KC_DB_USERNAME: "${{ env.postgres_user }}" | |
| KC_DB_PASSWORD: "${{ env.postgres_password }}" | |
| KC_DB_DRIVER: "${{ matrix.runner_desc.keycloak_db_driver }}" | |
| KC_DB_URL: "${{ env.test_db_url }}" | |
| KEYCLOAK_LOG_LEVEL: "INFO" | |
| COMPOSE_POSTGRES_IMAGE: "postgres:${{ matrix.postgres_version }}" | |
| COMPOSE_POSTGRES_DEPLOY_REPLICAS: "${{ matrix.runner_desc.postgres_replicas }}" | |
| COMPOSE_KEYCLOAK_DEPENDS_ON: "${{ env.compose_keycloak_depends_on }}" | |
| COMPOSE_KEYCLOAK_VOLUMES: "${{ env.compose_keycloak_volumes }}" | |
| # TODO: reverse | |
| # COMPOSE_KEYCLOAK_IMAGE: ${{ needs.build-image.outputs.full_image_name }} | |
| COMPOSE_KEYCLOAK_IMAGE: "registry.camunda.cloud/team-infrastructure-experience/keycloak@sha256:766f627ae1ef0aa16ca9af26989434e9c4f8684e9699b43c55afc0a877193d76" | |
| # AWS specific variables to forward, see https://confluence.camunda.com/pages/viewpage.action?pageId=178590693#IAMRolesforServiceAccountsTesting(IRSA)-EnvironmentVariables | |
| AWS_STS_REGIONAL_ENDPOINTS: "${{ env.AWS_STS_REGIONAL_ENDPOINTS }}" | |
| AWS_DEFAULT_REGION: "${{ env.AWS_DEFAULT_REGION }}" | |
| AWS_REGION: "${{ env.AWS_REGION }}" | |
| AWS_ROLE_ARN: "${{ env.AWS_ROLE_ARN }}" | |
| AWS_WEB_IDENTITY_TOKEN_FILE: "${{ env.AWS_WEB_IDENTITY_TOKEN_FILE }}" | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r ./.github/scripts/integration/requirements.txt | |
| - name: Test Environment | |
| run: python3 ./.github/scripts/integration/main.py | |
| - name: Tear down Aurora PG (aws only) | |
| if: startsWith(matrix.runner_desc.runner, 'aws') | |
| run: ./.helpers/actions/delete-aurora-pg-db.sh | |
| env: | |
| PGDATABASE: "${{ env.postgres_database }}" | |
| PGHOST: ${{ env.postgres_host }} | |
| PGPORT: ${{ matrix.runner_desc.keycloak_db_port }} | |
| PGPASSWORD: ${{ env.postgres_superuser_password }} | |
| PGUSER: ${{ env.postgres_superuser }} | |
| # TODO: add tests (static framework tbd) then push | |
| # TODO: implement | |
| # publish-image: | |
| # release: |