wdl-ci-integration.yml

name: Integration Tests

on:
  push

env:
  LC_ALL: C.UTF-8
  LANG: C.UTF-8
  DEBIAN_FRONTEND: noninteractive

jobs:
  cancel-previous:
    runs-on: [self-hosted, idseq-dev]
    steps:
      - uses: styfle/cancel-workflow-action@0.9.0
        with:
          access_token: ${{ github.token }}

  list-workflow-dirs:
    runs-on: [self-hosted, idseq-dev]
    outputs:
      matrix: ${{steps.list_dirs.outputs.matrix}}
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 2
      - id: list_dirs
        # only run on wdl dirs with diff
        run: echo "matrix=$(./scripts/diff-workflows.sh |jq -cnR '[inputs|select(length>0)]')" >> $GITHUB_OUTPUT

  wdl-ci:
    runs-on: [self-hosted, idseq-dev]
    needs: list-workflow-dirs
    if: ${{ needs['list-workflow-dirs'].outputs.matrix != '[]' && needs['list-workflow-dirs'].outputs.matrix != '' }}
    strategy:
      matrix:
        workflow_dir: ${{fromJson(needs['list-workflow-dirs'].outputs.matrix)}}
    steps:
      - uses: actions/checkout@v2
      - name: docker build + push to ghcr.io
        run: |
          TAG=$(git describe --long --tags --always --dirty)
          IMAGE_NAME=czid-${{ matrix.workflow_dir }}-public
          IMAGE_URI="ghcr.io/${GITHUB_REPOSITORY}/${IMAGE_NAME}"

          CACHE_FROM=""; docker pull "$IMAGE_URI" && CACHE_FROM="--cache-from $IMAGE_URI"
          ./scripts/docker-build.sh "workflows/${{ matrix.workflow_dir }}" --tag "${IMAGE_URI}:${TAG}" $CACHE_FROM \
            || ./scripts/docker-build.sh "workflows/${{ matrix.workflow_dir }}" --tag "${IMAGE_URI}:${TAG}"
            # docker push "${IMAGE_URI}:${TAG}" don't push while testing
          if [[ ${GITHUB_REF##*/} == "main" ]]; then
            docker tag "${IMAGE_URI}:${TAG}" "${IMAGE_URI}:latest"
            # docker push "${IMAGE_URI}:latest" don't push while testing
          fi

          echo "IMAGE_URI=${IMAGE_URI}" >> $GITHUB_ENV
          echo "TAG=${TAG}" >> $GITHUB_ENV
      - name: run tests
        run: |
          [ -d "${{ matrix.workflow_dir }}/integration_test" ] || exit 0
          source /etc/profile
          # explicitly block EC2 IMDS endpoint to work around awscli issue:
          # https://github.com/aws/aws-cli/issues/5234#issuecomment-635459464
          # https://github.com/aws/aws-cli/issues/5262

          # configure miniwdl to auto-delete task working directories, to reduce chance of worker
          # running out of space
          export MINIWDL__FILE_IO__OUTPUT_HARDLINKS=true
          export MINIWDL__FILE_IO__DELETE_WORK=success
          export MINIWDL__DOWNLOAD_CACHE__GET=true
          export MINIWDL__DOWNLOAD_CACHE__PUT=true
          export MINIWDL__DOWNLOAD_CACHE__DIR=/tmp/miniwdl_download_cache_workflows/

          sudo apt-get -qq update
          sudo apt-get -qq install --yes jq moreutils make virtualenv zip unzip httpie git shellcheck
          virtualenv --python=python3.10 .venv
          source .venv/bin/activate
          pip install -r requirements-dev.txt

          export DOCKER_IMAGE_ID="${IMAGE_URI}:${TAG}"

          make integration-test-${{ matrix.workflow_dir }}