online upgrade with package installation and pending pods

pkg/controllers/vdb/onlineupgrade_reconciler.go

@@ -165,6 +165,8 @@ func (r *OnlineUpgradeReconciler) Reconcile(ctx context.Context, _ *ctrl.Request
 
 	// Functions to perform when the image changes.  Order matters.
 	funcs := []func(context.Context) (ctrl.Result, error){
+		// Requeue if not all nodes are up
+		r.checkUpNodes,
 		// Initiate an upgrade by setting condition and event recording
 		r.startUpgrade,
 		r.logEventIfThisUpgradeWasNotChosen,
@@ -290,6 +292,30 @@ func (r *OnlineUpgradeReconciler) loadUpgradeState(ctx context.Context) (ctrl.Re
 	return ctrl.Result{}, nil
 }
 
+// checkUpNodes will requeue the upgrade process if not all the main cluster nodes are up.
+func (r *OnlineUpgradeReconciler) checkUpNodes(ctx context.Context) (ctrl.Result, error) {
+	// We skip this if we have already added the new subclusters
+	if vmeta.GetOnlineUpgradeStepInx(r.VDB.Annotations) > addSubclustersInx {
+		return ctrl.Result{}, nil
+	}
+
+	mainPFacts := r.PFacts[vapi.MainCluster]
+	// All nodes in the main cluster must be up before running online upgrade
+	if mainPFacts.getUpNodeCount() != len(mainPFacts.Detail) {
+		r.VRec.Eventf(r.VDB, corev1.EventTypeWarning, events.NotAllNodesUp,
+			"Not all nodes (%d/%d) are up, restarting the main cluster. Please check the cluster configuration if this issue continues.",
+			mainPFacts.getUpNodeCount(), len(mainPFacts.Detail))
+
+		// try to restart the main cluster, requeuing online upgrade
+		res, err := r.restartMainCluster(ctx)
+		if verrors.IsReconcileAborted(res, err) {
+			return res, err
+		}
+	}
+
+	return ctrl.Result{}, nil
+}
+
 // assignSubclustersToReplicaGroupA will go through all of the subclusters involved
 // in the upgrade and assign them to the first replica group. The assignment is
 // saved in the status.upgradeState.replicaGroups field.

pkg/controllers/vdb/upgrade.go

@@ -845,8 +845,17 @@ func (i *UpgradeManager) routeClientTraffic(ctx context.Context, pfacts *PodFact
 func (i *UpgradeManager) logEventIfRequestedUpgradeIsDifferent(actualUpgrade vapi.UpgradePolicyType) {
 	if !i.ContinuingUpgrade && i.Vdb.Spec.UpgradePolicy != actualUpgrade && i.Vdb.Spec.UpgradePolicy != vapi.AutoUpgrade {
 		actualUpgradeAsText := strings.ToLower(string(actualUpgrade))
+
 		i.Rec.Eventf(i.Vdb, corev1.EventTypeNormal, events.IncompatibleUpgradeRequested,
 			"Requested upgrade is incompatible with the Vertica deployment. Falling back to %s upgrade.", actualUpgradeAsText)
+
+		if i.Vdb.Spec.UpgradePolicy == "Online" {
+			i.Log.Info("Not all online upgrade prerequisites met. Please make sure: " +
+				"1. Vertica server version is 24.3.0-2 or higher. " +
+				"2. Cluster was deployed using `vclusterops`. " +
+				"3. A license file was applied to allow double the DB nodes. " +
+				"4. No sandbox defined.")
+		}
 	}
 }
 

pkg/events/event.go

@@ -22,6 +22,7 @@ const (
 	AddNodeSucceeded                 = "AddNodeSucceeded"
 	AddNodeLicenseFail               = "AddNodeLicenseFail"
 	AddNodeFailed                    = "AddNodeFailed"
+	NotAllNodesUp                    = "NotAllNodesUp"
 	CreateDBStart                    = "CreateDBStart"
 	CreateDBSucceeded                = "CreateDBSucceeded"
 	CreateDBFailed                   = "CreateDBFailed"

pkg/vadmin/install_packages_at.go

@@ -17,6 +17,7 @@ package vadmin
 
 import (
 	"context"
+	"errors"
 	"regexp"
 	"strings"
 
@@ -34,7 +35,8 @@ func (a *Admintools) InstallPackages(ctx context.Context, opts ...installpackage
 
 	status := genInstallPackageStatus(stdout)
 	if err != nil && len(status.Packages) == 0 {
-		_, logErr := a.logFailure("install_package", events.InstallPackagesFailed, stdout, err)
+		pkgErr := errors.New(err.Error() + "This may due to lack of memory resources.")
+		_, logErr := a.logFailure("install_package", events.InstallPackagesFailed, stdout, pkgErr)
 		a.Log.Error(err, "failed to finish package installation", "installPackageStatus", *status)
 		return status, logErr
 	}

pkg/vadmin/install_packages_vc.go

@@ -17,6 +17,7 @@ package vadmin
 
 import (
 	"context"
+	"errors"
 
 	vops "github.com/vertica/vcluster/vclusterops"
 	"github.com/vertica/vertica-kubernetes/pkg/events"
@@ -41,7 +42,8 @@ func (v *VClusterOps) InstallPackages(_ context.Context, opts ...installpackages
 		status = &vops.InstallPackageStatus{}
 	}
 	if err != nil {
-		_, err = v.logFailure("VInstallPackages", events.InstallPackagesFailed, err)
+		pkgErr := errors.New(err.Error() + "This may due to lack of memory resources.")
+		_, err = v.logFailure("VInstallPackages", events.InstallPackagesFailed, pkgErr)
 		v.Log.Error(err, "failed to finish package installation", "installPackageStatus", *status)
 		return status, err
 	}

tests/e2e-leg-9/online-upgrade-with-package-install/02-assert.yaml

@@ -0,0 +1,22 @@
+# (c) Copyright [2021-2024] Open Text.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: integration-test-role
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: integration-test-rb

tests/e2e-leg-9/online-upgrade-with-package-install/02-rbac.yaml

@@ -0,0 +1,18 @@
+# (c) Copyright [2021-2024] Open Text.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: kubectl apply --namespace $NAMESPACE -f ../../manifests/rbac/base/rbac.yaml
+    ignoreFailure: true

tests/e2e-leg-9/online-upgrade-with-package-install/05-create-secrets.yaml

@@ -0,0 +1,19 @@
+# (c) Copyright [2021-2024] Open Text.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: kustomize build ../../manifests/communal-creds/overlay | kubectl apply -f - --namespace $NAMESPACE
+  - script: kustomize build ../../manifests/priv-container-creds/overlay | kubectl apply -f - --namespace $NAMESPACE
+  - script: kustomize build ../../manifests/vertica-license/overlay | kubectl apply -f - --namespace $NAMESPACE

tests/e2e-leg-9/online-upgrade-with-package-install/10-assert.yaml

@@ -0,0 +1,21 @@
+# (c) Copyright [2021-2024] Open Text.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Pod
+metadata:
+  namespace: verticadb-operator
+  labels:
+    control-plane: verticadb-operator
+status:
+  phase: Running

tests/e2e-leg-9/online-upgrade-with-package-install/10-verify-operator.yaml

@@ -0,0 +1,12 @@
+# (c) Copyright [2021-2024] Open Text.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

tests/e2e-leg-9/online-upgrade-with-package-install/15-assert.yaml

@@ -0,0 +1,45 @@
+# (c) Copyright [2021-2024] Open Text.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: v-base-upgrade-pri1
+status:
+  currentReplicas: 2
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: v-base-upgrade-pri-2
+status:
+  currentReplicas: 1
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: v-base-upgrade-sec1
+status:
+  currentReplicas: 2
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: v-base-upgrade-sec2
+status:
+  currentReplicas: 1
+---
+apiVersion: vertica.com/v1
+kind: VerticaDB
+metadata:
+  name: v-base-upgrade

tests/e2e-leg-9/online-upgrade-with-package-install/15-setup-vdb.yaml

@@ -0,0 +1,17 @@
+# (c) Copyright [2021-2024] Open Text.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - command: bash -c "kustomize build setup-vdb/overlay | kubectl -n $NAMESPACE apply -f - "

tests/e2e-leg-9/online-upgrade-with-package-install/20-assert.yaml

@@ -0,0 +1,58 @@
+# (c) Copyright [2021-2024] Open Text.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: v-base-upgrade-pri1
+status:
+  currentReplicas: 2
+  readyReplicas: 2
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: v-base-upgrade-pri-2
+status:
+  currentReplicas: 1
+  readyReplicas: 1
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: v-base-upgrade-sec1
+status:
+  currentReplicas: 2
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: v-base-upgrade-sec2
+status:
+  currentReplicas: 1
+  readyReplicas: 1
+---
+apiVersion: vertica.com/v1
+kind: VerticaDB
+metadata:
+  name: v-base-upgrade
+status:
+  subclusters:
+    - addedToDBCount: 2
+      upNodeCount: 2
+    - addedToDBCount: 1
+      upNodeCount: 1
+    - addedToDBCount: 2
+      upNodeCount: 2
+    - addedToDBCount: 1
+      upNodeCount: 1

tests/e2e-leg-9/online-upgrade-with-package-install/20-wait-for-createdb.yaml

@@ -0,0 +1 @@
+# Intentionally empty to give this step a name in kuttl

tests/e2e-leg-9/online-upgrade-with-package-install/25-wait-for-steady-state.yaml

@@ -0,0 +1,17 @@
+# (c) Copyright [2021-2024] Open Text.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - command: bash -c "../../../scripts/wait-for-verticadb-steady-state.sh -n verticadb-operator -t 360 $NAMESPACE"

tests/e2e-leg-9/online-upgrade-with-package-install/26-assert.yaml

@@ -0,0 +1,20 @@
+# (c) Copyright [2021-2024] Open Text.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: v-base-upgrade-sec2
+status:
+  availableReplicas: 0
+  replicas: 1

tests/e2e-leg-9/online-upgrade-with-package-install/26-set-pod-pending.yaml

@@ -0,0 +1,33 @@
+# (c) Copyright [2021-2024] Open Text.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: vertica.com/v1
+kind: VerticaDB
+metadata:
+  name: v-base-upgrade
+spec:
+  subclusters:
+    - name: sec1
+      size: 2
+      type: secondary
+    - name: sec2
+      size: 1
+      resources:
+        requests:
+          memory: 1Ti
+      type: secondary
+    - name: pri1
+      size: 2
+    - name: pri_2
+      size: 1
+      type: primary