online upgrade with package installation and pending pods

pkg/controllers/vdb/installpackages_reconciler.go

@@ -130,6 +130,10 @@ func (i *InstallPackagesReconciler) runCmd(ctx context.Context, initiatorName ty
 		"skipped installation package list", categorizedStatus.skippedPackages,
 	)
 
+	if len(categorizedStatus.succeededPackages) == 0 {
+		i.Log.Info("No pacakges was installed. This may due to lack of memory resources or other internal errors.")
+	}
+
 	return err
 }
 

pkg/controllers/vdb/onlineupgrade_reconciler.go

@@ -66,6 +66,7 @@ const (
 // be sure to add a *StatusMsgInx const below.
 var onlineUpgradeStatusMsgs = []string{
 	"Starting online upgrade",
+	"Requeue as not all pods are running",
 	"Create new subclusters to mimic subclusters in the main cluster",
 	fmt.Sprintf("Querying the original value of config parameter %q", ConfigParamDisableNonReplicatableQueries),
 	fmt.Sprintf("Disable non-replicatable queries by setting config parameter %q", ConfigParamDisableNonReplicatableQueries),
@@ -86,6 +87,7 @@ var onlineUpgradeStatusMsgs = []string{
 // Constants for each entry in onlineUpgradeStatusMsgs
 const (
 	startOnlineUpgradeStatusMsgInx = iota
+	requeuePodsNotRunningMsgInx
 	createNewSubclustersStatusMsgInx
 	queryOriginalConfigParamDisableNonReplicatableQueriesMsgInx
 	disableNonReplicatableQueriesMsgInx
@@ -171,6 +173,9 @@ func (r *OnlineUpgradeReconciler) Reconcile(ctx context.Context, _ *ctrl.Request
 		r.startUpgrade,
 		r.logEventIfThisUpgradeWasNotChosen,
 		r.postStartOnlineUpgradeMsg,
+		// Requeue if not all nodes are running
+		r.postRequeuePodsNotRunningMsg,
+		r.requeuePodsNotRunning,
 		// Load up state that is used for the subsequent steps
 		r.loadUpgradeState,
 		// Assign subclusters to upgrade to replica group A
@@ -292,6 +297,37 @@ func (r *OnlineUpgradeReconciler) loadUpgradeState(ctx context.Context) (ctrl.Re
 	return ctrl.Result{}, nil
 }
 
+// postRequeuePodsNotRunningMsg will update the status message to indicate that
+// we are requeuing online upgrade if not all pods are running.
+func (r *OnlineUpgradeReconciler) postRequeuePodsNotRunningMsg(ctx context.Context) (ctrl.Result, error) {
+	return r.postNextStatusMsg(ctx, requeuePodsNotRunningMsgInx)
+}
+
+// requeuePodsNotRunning will requeue the upgrade process if not all pods are running.
+func (r *OnlineUpgradeReconciler) requeuePodsNotRunning(ctx context.Context) (ctrl.Result, error) {
+	// We skip this if we have already added the new subclusters
+	if vmeta.GetOnlineUpgradeStepInx(r.VDB.Annotations) > addSubclustersInx {
+		return ctrl.Result{}, nil
+	}
+
+	// For pods are pending due to lack of resources, we requeue restarting them and wait
+	// for user operation.
+	mainPFacts := r.PFacts[vapi.MainCluster]
+	found, _ := mainPFacts.AnyPodsNotRunning()
+	if found {
+		r.Log.Info("Not all pods are running, requeuing.")
+		return ctrl.Result{Requeue: true}, nil
+	}
+
+	// to restart the main cluster if any down pods found
+	res, err := r.restartMainCluster(ctx)
+	if verrors.IsReconcileAborted(res, err) {
+		return res, err
+	}
+
+	return ctrl.Result{}, nil
+}
+
 // assignSubclustersToReplicaGroupA will go through all of the subclusters involved
 // in the upgrade and assign them to the first replica group. The assignment is
 // saved in the status.upgradeState.replicaGroups field.

pkg/controllers/vdb/upgrade.go

@@ -847,6 +847,15 @@ func (i *UpgradeManager) routeClientTraffic(ctx context.Context, pfacts *podfact
 func (i *UpgradeManager) logEventIfRequestedUpgradeIsDifferent(actualUpgrade vapi.UpgradePolicyType) {
 	if !i.ContinuingUpgrade && i.Vdb.Spec.UpgradePolicy != actualUpgrade && i.Vdb.Spec.UpgradePolicy != vapi.AutoUpgrade {
 		actualUpgradeAsText := strings.ToLower(string(actualUpgrade))
+
+		if i.Vdb.Spec.UpgradePolicy == "Online" {
+			i.Log.Info("Not all online upgrade prerequisites met. Please make sure: " +
+				"1. Vertica server version is 24.3.0-2 or higher. " +
+				"2. Cluster was deployed using `vclusterops`. " +
+				"3. A license file was applied to allow double the DB nodes. " +
+				"4. No sandbox defined.")
+		}
+
 		i.Rec.Eventf(i.Vdb, corev1.EventTypeNormal, events.IncompatibleUpgradeRequested,
 			"Requested upgrade is incompatible with the Vertica deployment. Falling back to %s upgrade.", actualUpgradeAsText)
 	}

pkg/podfacts/podfacts.go

@@ -1387,6 +1387,17 @@ func GenPodNames(pods []*PodFact) string {
 	return strings.Join(podNames, ", ")
 }
 
+// AnyPodsNotRunning returns true if any pod isn't running. It could be still pending due to
+// lack of resources. It will return the name of the first pod that isn't running.
+func (p *PodFacts) AnyPodsNotRunning() (bool, types.NamespacedName) {
+	for _, v := range p.Detail {
+		if !v.isPodRunning {
+			return true, v.name
+		}
+	}
+	return false, types.NamespacedName{}
+}
+
 // AnyInstalledPodsNotRunning returns true if any installed pod isn't running.  It will
 // return the name of the first pod that isn't running.
 func (p *PodFacts) AnyInstalledPodsNotRunning() (bool, types.NamespacedName) {

tests/e2e-leg-9/online-upgrade-pods-pending/02-assert.yaml

@@ -0,0 +1,22 @@
+# (c) Copyright [2021-2024] Open Text.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: integration-test-role
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: integration-test-rb

tests/e2e-leg-9/online-upgrade-pods-pending/02-rbac.yaml

@@ -0,0 +1,18 @@
+# (c) Copyright [2021-2024] Open Text.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: kubectl apply --namespace $NAMESPACE -f ../../manifests/rbac/base/rbac.yaml
+    ignoreFailure: true

tests/e2e-leg-9/online-upgrade-pods-pending/05-create-secrets.yaml

@@ -0,0 +1,19 @@
+# (c) Copyright [2021-2024] Open Text.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: kustomize build ../../manifests/communal-creds/overlay | kubectl apply -f - --namespace $NAMESPACE
+  - script: kustomize build ../../manifests/priv-container-creds/overlay | kubectl apply -f - --namespace $NAMESPACE
+  - script: kustomize build ../../manifests/vertica-license/overlay | kubectl apply -f - --namespace $NAMESPACE

tests/e2e-leg-9/online-upgrade-pods-pending/10-assert.yaml

@@ -0,0 +1,21 @@
+# (c) Copyright [2021-2024] Open Text.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Pod
+metadata:
+  namespace: verticadb-operator
+  labels:
+    control-plane: verticadb-operator
+status:
+  phase: Running

tests/e2e-leg-9/online-upgrade-pods-pending/10-verify-operator.yaml

@@ -0,0 +1,12 @@
+# (c) Copyright [2021-2024] Open Text.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

tests/e2e-leg-9/online-upgrade-pods-pending/15-assert.yaml

@@ -0,0 +1,45 @@
+# (c) Copyright [2021-2024] Open Text.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: v-base-upgrade-pri1
+status:
+  currentReplicas: 2
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: v-base-upgrade-pri-2
+status:
+  currentReplicas: 1
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: v-base-upgrade-sec1
+status:
+  currentReplicas: 2
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: v-base-upgrade-sec2
+status:
+  currentReplicas: 1
+---
+apiVersion: vertica.com/v1
+kind: VerticaDB
+metadata:
+  name: v-base-upgrade

tests/e2e-leg-9/online-upgrade-pods-pending/15-setup-vdb.yaml

@@ -0,0 +1,17 @@
+# (c) Copyright [2021-2024] Open Text.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - command: bash -c "kustomize build setup-vdb/overlay | kubectl -n $NAMESPACE apply -f - "

tests/e2e-leg-9/online-upgrade-pods-pending/20-assert.yaml

@@ -0,0 +1,58 @@
+# (c) Copyright [2021-2024] Open Text.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: v-base-upgrade-pri1
+status:
+  currentReplicas: 2
+  readyReplicas: 2
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: v-base-upgrade-pri-2
+status:
+  currentReplicas: 1
+  readyReplicas: 1
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: v-base-upgrade-sec1
+status:
+  currentReplicas: 2
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: v-base-upgrade-sec2
+status:
+  currentReplicas: 1
+  readyReplicas: 1
+---
+apiVersion: vertica.com/v1
+kind: VerticaDB
+metadata:
+  name: v-base-upgrade
+status:
+  subclusters:
+    - addedToDBCount: 2
+      upNodeCount: 2
+    - addedToDBCount: 1
+      upNodeCount: 1
+    - addedToDBCount: 2
+      upNodeCount: 2
+    - addedToDBCount: 1
+      upNodeCount: 1

tests/e2e-leg-9/online-upgrade-pods-pending/20-wait-for-createdb.yaml

@@ -0,0 +1 @@
+# Intentionally empty to give this step a name in kuttl

tests/e2e-leg-9/online-upgrade-pods-pending/25-wait-for-steady-state.yaml

@@ -0,0 +1,17 @@
+# (c) Copyright [2021-2024] Open Text.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - command: bash -c "../../../scripts/wait-for-verticadb-steady-state.sh -n verticadb-operator -t 360 $NAMESPACE"

tests/e2e-leg-9/online-upgrade-pods-pending/26-assert.yaml

@@ -0,0 +1,19 @@
+# (c) Copyright [2021-2024] Open Text.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: Pod
+metadata:
+  name: v-base-upgrade-sec2-0
+status:
+ phase: Pending