kind-e2e.yaml

name: KinD e2e tests

on:
  pull_request:
    branches: [ 'main' ]

defaults:
  run:
    shell: bash
    working-directory: ./src/knative.dev/eventing-kafka

jobs:

  e2e-tests:
    name: e2e tests
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false # Keep running if one leg fails.
      matrix:
        k8s-version:
        - v1.17.11
        - v1.18.8
        - v1.19.1

        test-suite:
        - ./test/e2e
        # TODO(https://github.com/knative/eventing/issues/4275): Enable this.
        # - ./test/conformance

        # Map between K8s and KinD versions.
        # This is attempting to make it a bit clearer what's being tested.
        # See: https://github.com/kubernetes-sigs/kind/releases/tag/v0.9.0
        include:
        - k8s-version: v1.17.11
          kind-version: v0.9.0
          kind-image-sha: sha256:5240a7a2c34bf241afb54ac05669f8a46661912eab05705d660971eeb12f6555
        - k8s-version: v1.18.8
          kind-version: v0.9.0
          kind-image-sha: sha256:f4bcc97a0ad6e7abaf3f643d890add7efe6ee4ab90baeb374b4f41a4c95567eb
        - k8s-version: v1.19.1
          kind-version: v0.9.0
          kind-image-sha: sha256:98cf5288864662e37115e362b23e4369c8c4a408f99cbc06e58ac30ddc721600

        # Add the flags we use for each of these test suites.
        - test-suite: ./test/e2e
          extra-test-flags: -channels=messaging.knative.dev/v1alpha1:KafkaChannel,messaging.knative.dev/v1beta1:KafkaChannel
        # TODO(https://github.com/knative/eventing/issues/4275): Enable this.
        # - test-suite: ./test/conformance
        #   extra-test-flags: -brokers=eventing.knative.dev/v1beta1:MTChannelBasedBroker -channels=messaging.knative.dev/v1beta1:Channel,messaging.knative.dev/v1beta1:InMemoryChannel,messaging.knative.dev/v1:Channel,messaging.knative.dev/v1:InMemoryChannel -sources=sources.knative.dev/v1beta1:ApiServerSource,sources.knative.dev/v1alpha2:ContainerSource,sources.knative.dev/v1beta1:PingSource

    env:
      GOPATH: ${{ github.workspace }}
      GO111MODULE: off
      KO_DOCKER_REPO: kind.local
      SYSTEM_NAMESPACE: knative-eventing
      TEST_EVENTING_NAMESPACE: knative-eventing
      # Use a semi-random cluster suffix, but somewhat predictable
      # so reruns don't just give us a completely new value.
      CLUSTER_SUFFIX: c${{ github.run_id }}.local
      ARTIFACTS: ${{ github.workspace }}/artifacts

    steps:
    - name: Set up Go 1.18.x
      uses: actions/setup-go@v2
      with:
        go-version: 1.18.x

    - name: Install Dependencies
      working-directory: ./
      run: |
        echo '::group:: install ko'
        curl -L https://github.com/google/ko/releases/download/v0.7.0/ko_0.7.0_Linux_x86_64.tar.gz | tar xzf - ko
        chmod +x ./ko
        sudo mv ko /usr/local/bin
        echo '::endgroup::'

    - name: Check out code onto GOPATH
      uses: actions/checkout@v2
      with:
        path: ./src/knative.dev/eventing-kafka

    - name: Install KinD
      run: |
        set -x

        # Disable swap otherwise memory enforcement doesn't work
        # See: https://kubernetes.slack.com/archives/CEKK1KTN2/p1600009955324200
        sudo swapoff -a
        sudo rm -f /swapfile

        curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${{ matrix.kind-version }}/kind-$(uname)-amd64
        chmod +x ./kind
        sudo mv kind /usr/local/bin

    - name: Configure KinD Cluster
      working-directory: ./src/knative.dev/eventing-kafka
      run: |
        set -x

        # KinD configuration.
        cat > kind.yaml <<EOF
        apiVersion: kind.x-k8s.io/v1alpha4
        kind: Cluster

        # This is needed in order to support projected volumes with service account tokens.
        # See: https://kubernetes.slack.com/archives/CEKK1KTN2/p1600268272383600
        kubeadmConfigPatches:
          - |
            apiVersion: kubeadm.k8s.io/v1beta2
            kind: ClusterConfiguration
            metadata:
              name: config
            apiServer:
              extraArgs:
                "service-account-issuer": "kubernetes.default.svc"
                "service-account-signing-key-file": "/etc/kubernetes/pki/sa.key"
            networking:
              dnsDomain: "${CLUSTER_SUFFIX}"
        nodes:
        - role: control-plane
          image: kindest/node:${{ matrix.k8s-version }}@${{ matrix.kind-image-sha }}
        - role: worker
          image: kindest/node:${{ matrix.k8s-version }}@${{ matrix.kind-image-sha }}
        EOF

    - name: Create KinD Cluster
      working-directory: ./src/knative.dev/eventing-kafka
      run: |
        set -x

        kind create cluster --config kind.yaml

    - name: Install Strimzi
      run: |

        strimzi_version=`curl https://github.com/strimzi/strimzi-kafka-operator/releases/latest |  awk -F 'tag/' '{print $2}' | awk -F '"' '{print $1}' 2>/dev/null`

        kubectl create namespace kafka
        kubectl -n kafka apply --selector strimzi.io/crd-install=true -f https://github.com/strimzi/strimzi-kafka-operator/releases/download/${strimzi_version}/strimzi-cluster-operator-${strimzi_version}.yaml
        curl -L "https://github.com/strimzi/strimzi-kafka-operator/releases/download/${strimzi_version}/strimzi-cluster-operator-${strimzi_version}.yaml" \
         | sed 's/namespace: .*/namespace: kafka/' \
         | kubectl -n kafka apply -f -

        kubectl -n kafka apply -f "https://raw.githubusercontent.com/strimzi/strimzi-kafka-operator/${strimzi_version}/examples/kafka/kafka-persistent-single.yaml"
        kubectl wait deployment --all --timeout=-1s --for=condition=Available -n kafka

    - name: Install Knative
      env:
        SERVING_VERSION: v0.18.0
        KOURIER_VERSION: v0.18.0
      run: |
        # Prerequisites
        sudo pip install yq

        # Serving
        kubectl apply --filename https://github.com/knative/serving/releases/download/$SERVING_VERSION/serving-crds.yaml
        sleep 5
        curl -L -s https://github.com/knative/serving/releases/download/$SERVING_VERSION/serving-core.yaml | yq 'del(.spec.template.spec.containers[]?.resources)' -y | yq 'del(.metadata.annotations."knative.dev/example-checksum")' -y | kubectl apply -f -
        sleep 60
        kubectl get pod -n knative-serving

        # Kourier
        kubectl apply --filename https://github.com/knative-sandbox/net-kourier/releases/download/$KOURIER_VERSION/kourier.yaml
        kubectl patch configmap/config-network \
        --namespace knative-serving \
        --type merge \
        --patch '{"data":{"ingress.class":"kourier.ingress.networking.knative.dev"}}'
        sleep 30
        kubectl get pod -n kourier-system

        # Eventing
        kubectl apply --filename https://knative-nightly.storage.googleapis.com/eventing/latest/eventing-crds.yaml
        sleep 5
        curl -L -s https://knative-nightly.storage.googleapis.com/eventing/latest/eventing-core.yaml | yq 'del(.spec.template.spec.containers[]?.resources)' -y | yq 'del(.metadata.annotations."knative.dev/example-checksum")' -y | kubectl apply -f -
        # Eventing channels
        curl -L -s https://knative-nightly.storage.googleapis.com/eventing/latest/in-memory-channel.yaml | yq 'del(.spec.template.spec.containers[]?.resources)' -y | yq 'del(.metadata.annotations."knative.dev/example-checksum")' -y | kubectl apply -f -
        # Eventing broker
        curl -L -s https://knative-nightly.storage.googleapis.com/eventing/latest/mt-channel-broker.yaml | yq 'del(.spec.template.spec.containers[]?.resources)' -y | yq 'del(.metadata.annotations."knative.dev/example-checksum")' -y | kubectl apply -f -
        sleep 30
        kubectl get pod -n knative-eventing

        kubectl get pod --all-namespaces

      - name: Install Test Images

    - name: Install Knative Eventing Kafka
      run: |
        set -o pipefail

        sed -i 's/REPLACE_WITH_CLUSTER_URL/my-cluster-kafka-bootstrap.kafka:9092/' config/channel/consolidated/configmaps/kafka-config.yaml

        # Build and Publish our containers to the docker daemon (including test assets)
        export GO111MODULE=on
        export GOFLAGS=-mod=vendor
        ko apply --platform=all -P \
           `# Install the Kafka Channel` \
           -f config/channel/consolidated/ \
           `# Install the Kafka Source` \
           -f config/source

        # Change the cluster domain to the correct one.
        sed -i "s/svc\.cluster\.local/svc\.${CLUSTER_SUFFIX}/g" test/config/config-tracing.yaml
        ko apply --platform=all -Pf test/config/

    - name: Upload Test Images
      run: |
        # Build and Publish our test images to the docker daemon.
        ./test/upload-test-images.sh

    - name: Wait for things to be up
      run: |
        kubectl wait pod --for=condition=Ready -n ${SYSTEM_NAMESPACE} -l '!job-name'

    - name: Run e2e Tests
      run: |
        # Run the tests tagged as e2e on the KinD cluster.
        go test -race -count=1 -parallel=12 -timeout=30m -tags=e2e \
           ${{ matrix.test-suite }} ${{ matrix.extra-test-flags }}

    - name: Collect system diagnostics
      if: ${{ failure() }}
      run: |
        kubectl -n${SYSTEM_NAMESPACE} get pods

        echo '::group:: describe'
        kubectl -n${SYSTEM_NAMESPACE} describe pods
        echo '::endgroup::'

        for x in $(kubectl get pods -n${SYSTEM_NAMESPACE} -oname); do
          echo "::group:: describe $x"
          kubectl -n${SYSTEM_NAMESPACE} describe $x
          echo '::endgroup::'

          echo "::group:: $x logs"
          kubectl -n${SYSTEM_NAMESPACE} logs $x --all-containers
          echo '::endgroup::'
        done

    - name: Dump Artifacts
      if: ${{ failure() }}
      run: |
        if [[ -d ${{ env.ARTIFACTS }} ]]; then
          cd ${{ env.ARTIFACTS }}
          for x in $(find . -type f); do
            echo "::group:: artifact $x"
            cat $x
            echo '::endgroup::'
          done
        fi