[Ready for review] Fixed e2e tests

.github/workflows/build-images.yml

@@ -196,32 +196,25 @@ jobs:
       image: ${{ steps.legacy_vertica_image.outputs.value }}
     steps:
 
-    - name: set lower case owner name
-      env:
-        OWNER: '${{ github.repository_owner }}'
-      run: |
-        echo "OWNER_LC=${OWNER,,}" >>${GITHUB_ENV}
-
     - name: Pick the name of the image
       uses: spilchen/switch-case-action@v2
       id: legacy_vertica_image
       with:
-        default: ghcr.io/${{ env.OWNER_LC }}/vertica-k8s:${{ github.sha }}-legacy
+        default: docker.io/opentext/vertica-k8s-private:latest-legacy-master
         conditionals-with-values: |
           ${{ inputs.legacy_vertica_image != '' }} => ${{ inputs.legacy_vertica_image }}
-          ${{ github.event_name == 'pull_request' }} => vertica-k8s:kind-legacy
 
     - name: Login to GitHub Container registry for non-PRs
       uses: docker/login-action@v3
-      if: ${{ github.event_name != 'pull_request' && inputs.legacy_vertica_image == '' || startsWith(inputs.legacy_vertica_image, 'ghcr.io') }}
+      if: ${{ github.event_name != 'pull_request' && startsWith(inputs.legacy_vertica_image, 'ghcr.io') }}
       with:
         registry: ghcr.io
         username: ${{ github.repository_owner }}
         password: ${{ secrets.GITHUB_TOKEN }}
 
     - name: Login to Docker Hub
       uses: docker/login-action@v3
-      if: ${{ inputs.legacy_vertica_image != '' && startsWith(inputs.legacy_vertica_image, 'docker.io') }}
+      if: ${{ startsWith(steps.legacy_vertica_image.outputs.value, 'docker.io') }}
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -233,15 +226,18 @@ jobs:
       uses: ./.github/actions/download-rpm
       if: ${{ inputs.legacy_vertica_image == '' }}
 
-    - name: Build and optionally push legacy server image
+    - name: Build legacy server image
       if: ${{ inputs.legacy_vertica_image == '' }}
       run: |
-        export VERTICA_IMG=${{ steps.legacy_vertica_image.outputs.value }}
+        # Do not push the image to remote registry because we
+        # do not want to overwrite the existing images. Here we
+        # just validate if dockerfile works fine.
         make docker-build-vertica
-        if [ $GITHUB_EVENT_NAME != 'pull_request' ]
-        then
-          make docker-push-vertica
-        fi
+
+    - name: Do a local pull of the image if we didn't create it
+      if: ${{ steps.legacy_vertica_image.outputs.value != '' }}
+      run: |
+        docker pull ${{ steps.legacy_vertica_image.outputs.value }}
 
     - name: Save the image for consumption by dependent jobs (PRs only)
       if: ${{ github.event_name == 'pull_request' }}
@@ -254,16 +250,10 @@ jobs:
         name: legacy-vertica-image
         path: legacy-vertica-image.tar
 
-    - name: Do a local pull of the image if we didn't create it
-      if: ${{ inputs.legacy_vertica_image != '' }}
-      run: |
-        docker pull ${{ inputs.legacy_vertica_image }}
-
     - name: Print a summary of the job
       run: |
         echo "Image Name: **${{ steps.legacy_vertica_image.outputs.value }}**" >> $GITHUB_STEP_SUMMARY
         echo "Was Built: ${{ inputs.legacy_vertica_image == '' && '**Yes**' || '**No**' }}" >> $GITHUB_STEP_SUMMARY
-        echo "Was Pushed: ${{ inputs.legacy_vertica_image == '' && github.event_name != 'pull_request' && '**Yes**' || '**No**' }}"
         echo "Was Scanned: **No**" >> $GITHUB_STEP_SUMMARY
         echo "Size: **$(docker inspect --format '{{.Size}}' ${{ steps.legacy_vertica_image.outputs.value }} | numfmt --to=iec)**" >> $GITHUB_STEP_SUMMARY
         echo "Image ID: **$(docker inspect --format '{{.ID}}' ${{ steps.legacy_vertica_image.outputs.value }})**" >> $GITHUB_STEP_SUMMARY

.github/workflows/e2e.yml

@@ -38,6 +38,7 @@ on:
         - admintools leg 2
         - admintools leg 3
         - admintools leg 4
+        - admintools leg 4 vdb-gen
         - admintools leg 5
         - admintools server upgrade
         - admintools operator upgrade
@@ -46,11 +47,14 @@ on:
         - vcluster leg 2
         - vcluster leg 3
         - vcluster leg 4
+        - vcluster leg 4 vdb-gen
         - vcluster leg 5
         - vcluster leg 6
+        - vcluster leg 6 revivedb
         - vcluster leg 7
         - leg 8
         - vcluster leg 9
+        - vcluster leg 10
         - vcluster server upgrade
         - vcluster udx
       run_security_scan:
@@ -331,6 +335,51 @@ jobs:
           # All helm deployments will use cert-manager to create the webhook cert
           helm-overrides: '--set webhook.certSource=cert-manager'
 
+  e2e-leg-4-admintools-vdb-gen:
+    if: ${{ ! contains(github.ref, 'k8s-sync')  && (inputs.e2e_test_suites == 'all' || inputs.e2e_test_suites == 'admintools leg 4 vdb-gen' || inputs.e2e_test_suites == '') }}
+    needs: [build]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Test
+        uses: ./.github/actions/run-e2e-leg
+        with:
+          leg-identifier: 'leg-4-vdb-gen'
+          dockerhub-username: ${{ secrets.DOCKERHUB_USERNAME }}
+          dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
+          vlogger-image: ${{ needs.build.outputs.vlogger-image }}
+          operator-image: ${{ needs.build.outputs.operator-image }}
+          vertica-image: ${{ needs.build.outputs.legacy-vertica-image }}
+          vertica-deployment-method: admintools
+          communal-storage-type: s3
+          vertica-superuser-name: dbadmin
+          # All helm deployments will use cert-manager to create the webhook cert
+          helm-overrides: '--set webhook.certSource=cert-manager'
+          need-legacy-image: true
+
+  e2e-leg-4-vcluster-vdb-gen:
+    if: ${{ ! github.event.pull_request.head.repo.fork  && (inputs.e2e_test_suites == 'all' || inputs.e2e_test_suites == 'vcluster leg 4 vdb-gen' || inputs.e2e_test_suites == '')}}
+    needs: [build]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Test
+        uses: ./.github/actions/run-e2e-leg
+        with:
+          leg-identifier: 'leg-4-vdb-gen'
+          dockerhub-username: ${{ secrets.DOCKERHUB_USERNAME }}
+          dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
+          vlogger-image: ${{ needs.build.outputs.vlogger-image }}
+          operator-image: ${{ needs.build.outputs.operator-image }}
+          vertica-image: ${{ needs.build.outputs.minimal-vertica-image }}
+          vertica-deployment-method: vclusterops
+          communal-storage-type: s3
+          vertica-superuser-name: myadmin
+          # All helm deployments will use cert-manager to create the webhook cert
+          helm-overrides: '--set webhook.certSource=cert-manager'
+
   e2e-leg-5-admintools:
     if: ${{ ! contains(github.ref, 'k8s-sync')  && (inputs.e2e_test_suites == 'all' || inputs.e2e_test_suites == 'admintools leg 5' || inputs.e2e_test_suites == '') }}
     needs: [build]
@@ -401,7 +450,7 @@ jobs:
           minimum-vertica-image: '24.2.0'
 
   e2e-leg-6-vcluster-revivedb:
-    if: ${{ ! github.event.pull_request.head.repo.fork  && (inputs.e2e_test_suites == 'all' || inputs.e2e_test_suites == 'vcluster leg 6' || inputs.e2e_test_suites == '')}}
+    if: ${{ ! github.event.pull_request.head.repo.fork  && (inputs.e2e_test_suites == 'all' || inputs.e2e_test_suites == 'vcluster leg 6 revivedb' || inputs.e2e_test_suites == '')}}
     needs: [build]
     runs-on: ubuntu-latest
     steps:
@@ -508,7 +557,7 @@ jobs:
           need-base-vertica-image: 'true'
 
   e2e-leg-10-vcluster:
-    if: ${{ ! github.event.pull_request.head.repo.fork  && (inputs.e2e_test_suites == 'all' || inputs.e2e_test_suites == 'vcluster leg 9' || inputs.e2e_test_suites == '')}}
+    if: ${{ ! github.event.pull_request.head.repo.fork  && (inputs.e2e_test_suites == 'all' || inputs.e2e_test_suites == 'vcluster leg 10' || inputs.e2e_test_suites == '')}}
     needs: [build]
     runs-on: ubuntu-latest
     steps:
@@ -646,6 +695,7 @@ jobs:
       e2e-leg-3-vclusterops-previous-release,
       e2e-leg-3-vcluster,
       e2e-leg-4-vcluster,
+      e2e-leg-4-vcluster-vdb-gen,
       e2e-leg-5-vcluster,
       e2e-leg-6-vcluster,
       e2e-leg-6-vcluster-revivedb,
@@ -661,6 +711,7 @@ jobs:
       e2e-leg-2-admintools, 
       e2e-leg-3-admintools, 
       e2e-leg-4-admintools, 
+      e2e-leg-4-admintools-vdb-gen, 
       e2e-leg-5-admintools, 
       e2e-server-upgrade-admintools, 
       e2e-udx-admintools
@@ -691,6 +742,7 @@ jobs:
       e2e-leg-3-vclusterops-previous-release,
       e2e-leg-3-vcluster,
       e2e-leg-4-vcluster,
+      e2e-leg-4-vcluster-vdb-gen,
       e2e-leg-5-vcluster,
       e2e-leg-6-vcluster,
       e2e-leg-6-vcluster-revivedb,

docker-vertica-v2/Makefile

@@ -16,7 +16,6 @@ VERTICA_ARM_VERSION?=$(shell rpm --nosignature -qp --queryformat '%{VERSION}-%{R
 
 # Our Github runner where we run the build step is on ubuntu so the rpm query extraction fails
 # We skip the verification of the package mismatch in that pipeline
-ifeq ($(FOR_GITHUB_CI), false)
 ifneq ($(VERTICA_X86_VERSION),)
 $(info X86 RPM Found with Version $(VERTICA_X86_VERSION))
 VERTICA_VERSION:=$(VERTICA_X86_VERSION)
@@ -28,7 +27,6 @@ exit 1
 endif
 endif
 endif
-endif
 
 all: docker-build-vertica
 

kuttl-test.yaml

@@ -25,6 +25,7 @@ testDirs:
   - tests/e2e-leg-3
   - tests/e2e-leg-3-at-only
   - tests/e2e-leg-4
+  - tests/e2e-leg-4-vdb-gen
   - tests/e2e-leg-5
   - tests/e2e-leg-5-at-only
   - tests/e2e-leg-6

scripts/capture-scrutinize.sh

@@ -128,6 +128,9 @@ function scrutinizeForAdmintools() {
     OP_FILE="$ns.$v.scrutinize.tar"
     logInfo "Running scrutinize"
     set -o xtrace
+    # scrutinize needs to be able to ssh to the pod
+    POD_IP=$(kubectl get pod -n $ns $pod -o jsonpath='{.status.podIP}')
+    kubectl exec -t -n $ns $pod -- bash -c "ssh-keyscan $POD_IP >> ~/.ssh/known_hosts"
     # We only need 1 pod because scrutinize will collect for the entire VerticaDB
     kubectl exec -t -n $ns $pod -- /opt/vertica/bin/scrutinize --output_dir $POD_OP_DIR --output_file $OP_FILE --vsql-off 
     scrut_res=$?

scripts/is-image.sh

@@ -60,14 +60,6 @@ then
 fi
 
 logAndRunCommand docker pull $IMG
-# Normally we would rely on the vertica-version label of the image but it is empty
-# for the latest-test-master image. Until we figure out why, we are adding
-# this work around to not skip some tests.
-if [[ "$IMG" == "docker.io/opentext/vertica-k8s-private:latest-test-master" ]]
-then
-    logInfo "$IMG is newer"
-    exit 1
-fi
 IMG_VER=$(determine_image_version $IMG)
 logInfo "Image $IMG has version $IMG_VER"
 logInfo "Checking if $IMG_VER is $COMPARE_TYPE than $COMPARE_VERSION"

tests/e2e-leg-4/vdb-gen/45-run-vdbgen.yaml → tests/e2e-leg-4-vdb-gen/vdb-gen/40-run-vdbgen-and-remove-old-vdb.yaml

@@ -21,8 +21,7 @@ commands:
   - command: kubectl -n $NAMESPACE exec v-vdb-gen-sc1-0 -- chmod +x /tmp/vdb-gen
   - command: kubectl -n $NAMESPACE cp run-vdb-gen.sh v-vdb-gen-sc1-0:/tmp/run-vdb-gen.sh
   - command: kubectl -n $NAMESPACE exec v-vdb-gen-sc1-0 -- chmod +x /tmp/run-vdb-gen.sh
-  - script: sh -c "kubectl -n $NAMESPACE exec v-vdb-gen-sc1-0 -- sh -c \"/tmp/run-vdb-gen.sh $NAMESPACE $(cd ../../.. && make echo-images | grep ^VERTICA_IMG= | cut -d'=' -f2) v-vdb-gen-revive $(kubectl get cm e2e -n $NAMESPACE -o jsonpath='{.data.caFileSecretName}') ${VERTICA_DEPLOYMENT_METHOD:-admintools} ${VERTICA_SUPERUSER_NAME:-dbadmin}\" > /tmp/$NAMESPACE-vdb-gen.yaml"
+  - script: sh -c "kubectl -n $NAMESPACE exec v-vdb-gen-sc1-0 -- sh -c \"/tmp/run-vdb-gen.sh $NAMESPACE $(cd ../../.. && make echo-images | grep ^VERTICA_IMG= | cut -d'=' -f2) v-vdb-gen-revive $(kubectl get cm e2e -n $NAMESPACE -o jsonpath='{.data.caFileSecretName}') ${VERTICA_DEPLOYMENT_METHOD:-admintools} ${VERTICA_SUPERUSER_NAME:-dbadmin} v-vdb-gen-sc2-0 v-vdb-gen\" > /tmp/$NAMESPACE-vdb-gen.yaml"
   - command: cat /tmp/$NAMESPACE-vdb-gen.yaml
-  # Apply the generated CR
-  - command: kubectl -n $NAMESPACE apply -f /tmp/$NAMESPACE-vdb-gen.yaml
-  - command: rm /tmp/$NAMESPACE-vdb-gen.yaml
+  # Remove the old vdb
+  - command: kubectl -n $NAMESPACE delete vdb v-vdb-gen

tests/e2e-leg-4/vdb-gen/70-run-vdbgen.yaml → tests/e2e-leg-4-vdb-gen/vdb-gen/45-revive-db.yaml

@@ -14,9 +14,6 @@
 apiVersion: kuttl.dev/v1beta1
 kind: TestStep
 commands:
-  # We have already copied vdb-gen at step 45. Now we run it again, in the container.
-  - script: sh -c "kubectl -n $NAMESPACE exec v-vdb-gen-sc1-0 -- sh -c \"/tmp/run-vdb-gen.sh $NAMESPACE $(cd ../../.. && make echo-images | grep ^VERTICA_IMG= | cut -d'=' -f2) v-vdb-gen-revive2 $(kubectl get cm e2e -n $NAMESPACE -o jsonpath='{.data.caFileSecretName}') ${VERTICA_DEPLOYMENT_METHOD:-admintools} ${VERTICA_SUPERUSER_NAME:-dbadmin}\" > /tmp/$NAMESPACE-vdb-gen.yaml"
-  - command: cat /tmp/$NAMESPACE-vdb-gen.yaml
   # Apply the generated CR
   - command: kubectl -n $NAMESPACE apply -f /tmp/$NAMESPACE-vdb-gen.yaml
   - command: rm /tmp/$NAMESPACE-vdb-gen.yaml

tests/e2e-leg-4/vdb-gen/65-assert.yaml → tests/e2e-leg-4-vdb-gen/vdb-gen/65-assert.yaml

@@ -14,7 +14,7 @@
 apiVersion: vertica.com/v1beta1
 kind: VerticaDB
 metadata:
-  name: v-vdb-gen
+  name: v-vdb-gen-revive
 status:
   subclusterCount: 2
   upNodeCount: 2

tests/e2e-leg-4/vdb-gen/65-scale-sc1.yaml → tests/e2e-leg-4-vdb-gen/vdb-gen/65-scale-sc1.yaml

@@ -14,7 +14,7 @@
 apiVersion: vertica.com/v1beta1
 kind: VerticaDB
 metadata:
-   name: v-vdb-gen
+   name: v-vdb-gen-revive
 spec:
    subclusters:
       - name: sc1

tests/e2e-leg-4-vdb-gen/vdb-gen/67-run-vdbgen-and-remove-old-vdb.yaml

@@ -0,0 +1,27 @@
+# (c) Copyright [2021-2024] Open Text.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  # Copy vdb-gen into the container.  We run it in the container so that it has
+  # access to k8s network.  Running it outside k8s, it would only have access to
+  # what is exposed.
+  - command: kubectl -n $NAMESPACE cp ../../../bin/vdb-gen v-vdb-gen-revive-sc1-0:/tmp/vdb-gen
+  - command: kubectl -n $NAMESPACE exec v-vdb-gen-revive-sc1-0 -- chmod +x /tmp/vdb-gen
+  - command: kubectl -n $NAMESPACE cp run-vdb-gen.sh v-vdb-gen-revive-sc1-0:/tmp/run-vdb-gen.sh
+  - command: kubectl -n $NAMESPACE exec v-vdb-gen-revive-sc1-0 -- chmod +x /tmp/run-vdb-gen.sh
+  - script: sh -c "kubectl -n $NAMESPACE exec v-vdb-gen-revive-sc1-0 -- sh -c \"/tmp/run-vdb-gen.sh $NAMESPACE $(cd ../../.. && make echo-images | grep ^VERTICA_IMG= | cut -d'=' -f2) v-vdb-gen-revive2 v-vdb-gen-revive-ca-cert ${VERTICA_DEPLOYMENT_METHOD:-admintools} ${VERTICA_SUPERUSER_NAME:-dbadmin} v-vdb-gen-revive-sc2-0 v-vdb-gen-revive\" > /tmp/$NAMESPACE-vdb-gen.yaml"
+  - command: cat /tmp/$NAMESPACE-vdb-gen.yaml
+  # Remove the old vdb
+  - command: kubectl -n $NAMESPACE delete vdb v-vdb-gen-revive

tests/e2e-leg-4-vdb-gen/vdb-gen/70-revive-db.yaml

@@ -0,0 +1,19 @@
+# (c) Copyright [2021-2024] Open Text.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  # Apply the generated CR
+  - command: kubectl -n $NAMESPACE apply -f /tmp/$NAMESPACE-vdb-gen.yaml
+  - command: rm /tmp/$NAMESPACE-vdb-gen.yaml

tests/e2e-leg-4/vdb-gen/75-assert.yaml → tests/e2e-leg-4-vdb-gen/vdb-gen/75-assert.yaml

@@ -11,10 +11,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-apiVersion: kuttl.dev/v1beta1
-kind: TestAssert
-timeout: 900
----
 apiVersion: vertica.com/v1beta1
 kind: VerticaDB
 metadata:

tests/e2e-leg-4/vdb-gen/run-vdb-gen.sh → tests/e2e-leg-4-vdb-gen/vdb-gen/run-vdb-gen.sh

@@ -25,6 +25,8 @@ VDB_NAME=$3
 COMMUNAL_EP_CERT_SECRET=$4
 DEPLOYMENT_METHOD=$5
 VDB_USER=$6
+POD_NAME=$7
+VDB_CONTAINS_POD=$8
 
 # The ca.cert is optional.
 if [ -f "/certs/$COMMUNAL_EP_CERT_SECRET/ca.crt" ]
@@ -42,4 +44,4 @@ fi
     $CA_CERT_OPT \
     -depot-volume EmptyDir \
     -deployment-method $DEPLOYMENT_METHOD \
-    v-vdb-gen-sc2-0.v-vdb-gen.$NAMESPACE vertdb
+    $POD_NAME.$VDB_CONTAINS_POD.$NAMESPACE vertdb

tests/e2e-udx/udx-cpp/70-verify-cpp-source.yaml

@@ -22,12 +22,7 @@ data:
     set -o errexit
 
     POD_NAME=v-udx-cpp-sc1-0
-    container_count=$(kubectl get pod $POD_NAME -o jsonpath='{.spec.containers[*].name}' | wc -w)
-    echo "$container_count"
-    PYTHON_REG_EXP=python2
-    if [ "$container_count" -eq 2 ]; then PYTHON_REG_EXP=\$PYTHON3; fi
-    echo "$PYTHON_REG_EXP"
-    UDX_OP=$(kubectl exec $POD_NAME -i -c server -- bash -c "cd /opt/vertica/sdk/examples && sed -i 's/$PYTHON_REG_EXP/python/g' SourceFunctions.sql && vsql -U dbadmin -f SourceFunctions.sql 2>&1 > /tmp/SourceFunctionsOut.txt; cat /tmp/SourceFunctionsOut.txt")
+    UDX_OP=$(kubectl exec $POD_NAME -i -c server -- bash -c "cd /opt/vertica/sdk/examples && sed -i 's/python2/python/g; s/\$PYTHON3/python/g' SourceFunctions.sql && vsql -U dbadmin -f SourceFunctions.sql 2>&1 > /tmp/SourceFunctionsOut.txt; cat /tmp/SourceFunctionsOut.txt")
     echo "$UDX_OP"
     kubectl exec $POD_NAME -i -c server -- bash -c "diff /tmp/SourceFunctionsOut.txt /opt/vertica/sdk/examples/expected-outputs/SourceFunctionsOut.txt"
 ---