Create vertica client proxy pods in subcluster (#977)

This is the first step to supporting session transfer with client proxy: if the proxy field is defined, a deployment with vertical client pods will be created in each subcluster. For example: ``` spec: subclusters: - name: sc1 proxy: image: vertica-client-proxy:latest replica: 2 (default: 1) log-level: DEBUG (default: INFO) port: 5433 (default: 5433, we don't support using other ports in MVP) tls-certs: xxxx ``` The client proxy configurations are stored in a config map, which is mounted into the proxy pods as /config/config.yaml. --------- Co-authored-by: Fenic Fawkes <[email protected]>
vertica · Nov 9, 2024 · 8bb4112 · 8bb4112
1 parent 265f01a
commit 8bb4112
Show file tree

Hide file tree

Showing 42 changed files with 1,074 additions and 7 deletions.
diff --git a/.github/actions/run-e2e-leg/action.yaml b/.github/actions/run-e2e-leg/action.yaml
@@ -1,6 +1,9 @@
 name: 'Run an e2e leg'
 description: 'Will run all tests under one e2e leg with specific configurations'
 inputs:
+  vproxy-image:
+    description: 'Name of the vertica client proxy image'
+    required: false
   vlogger-image:
     description: 'Name of the vertica logger image'
     required: true
@@ -109,6 +112,7 @@ runs:
         export VERTICA_IMG=${{ inputs.vertica-image }}
         export OPERATOR_IMG=${{ inputs.operator-image }}
         export VLOGGER_IMG=${{ inputs.vlogger-image }}
+        export VPROXY_IMG=${{ inputs.vlogger-image }}
 
         if [[ "${{ inputs.vertica-license }}" != "" ]]; then
           export LICENSE_FILE=/tmp/vertica-license.dat

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
@@ -23,7 +23,7 @@ on:
         type: string
         required: false
       vlogger_image:
-        description: 'Name of an existing vlogger image. Leave blank to build oe with the default name'
+        description: 'Name of an existing vlogger image. Leave blank to build one with the default name'
         type: string
         required: false
       e2e_test_suites:
@@ -579,6 +579,30 @@ jobs:
           vertica-license: ${{ secrets.VERTICA_LICENSE }}
           need-base-vertica-image: 'true'
 
+  e2e-leg-11-vcluster:
+    if: ${{ ! github.event.pull_request.head.repo.fork  && (inputs.e2e_test_suites == 'all' || inputs.e2e_test_suites == 'vcluster leg 11' || inputs.e2e_test_suites == '')}}
+    needs: [build]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Test
+        uses: ./.github/actions/run-e2e-leg
+        with:
+          leg-identifier: 'leg-11'
+          dockerhub-username: ${{ secrets.DOCKERHUB_USERNAME }}
+          dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
+          vlogger-image: ${{ needs.build.outputs.vlogger-image }}
+          vproxy-image: 'opentext/client-proxy:latest'
+          operator-image: ${{ needs.build.outputs.operator-image }}
+          vertica-image: ${{ needs.build.outputs.full-vertica-image }}
+          vertica-deployment-method: vclusterops
+          communal-storage-type: s3
+          minimum-vertica-image: '25.1.0'
+          # Include the vertica license so we can test multiple subclusters.
+          vertica-license: ${{ secrets.VERTICA_LICENSE }}
+          need-base-vertica-image: 'true'
+
   e2e-server-upgrade-admintools:
     if: ${{ ! contains(github.ref, 'k8s-sync')  && (inputs.e2e_test_suites == 'all' || inputs.e2e_test_suites == 'admintools server upgrade' || inputs.e2e_test_suites == '') }}
     needs: [build]

diff --git a/DEVELOPER.md b/DEVELOPER.md
@@ -383,7 +383,10 @@ export BASE_VERTICA_IMG=opentext/vertica-k8s:24.2.0-1
 export VERTICA_IMG=opentext/vertica-k8s:latest
 ```
 > VERTICA_IMG is the vertica image you want to run the test with. For upgrade test cases, BASE_VERTICA_IMG is the base vertica version that will be installed. VERTICA_IMG is the vertica version that the base version will be upgraded to. The version in VERTICA_IMG must be higher than that in BASE_VERTICA_IMG.
-
+```shell
+export VPROXY_IMG=opentext/client-proxy:latest
+```
+> VPROXY_IMG is the vertica client proxy image you want to run the client proxy and session transfer test with.
 
 3. kuttl-test.yaml is the configuration file for e2e test cases. There is a "timeout" field in it. If your server is not fast enough, you may need to increase that value to pass the test cases. There is another field "parallel" that controls the maximum number of tests to run at once. It is set to 2 by default. You can set it to 1 if your server is not fast enough.
 

diff --git a/Makefile b/Makefile
@@ -99,6 +99,9 @@ export BASE_VERTICA_IMG
 # Image URL to use for the logger sidecar
 VLOGGER_IMG ?= $(IMG_REPO)vertica-logger:$(VLOGGER_VERSION)
 export VLOGGER_IMG
+# Image URL to use for the vertica client proxy. This is for testing purposes only.
+VPROXY_IMG ?= opentext/client-proxy:latest
+export VPROXY_IMG
 # If the current leg in the CI tests is leg-9
 LEG9 ?= no
 export LEG9
@@ -557,6 +560,7 @@ echo-images:  ## Print the names of all of the images used
 	@echo "VERTICA_IMG=$(VERTICA_IMG)"
 	@echo "BASE_VERTICA_IMG=$(BASE_VERTICA_IMG)"
 	@echo "VLOGGER_IMG=$(VLOGGER_IMG)"
+	@echo "VPROXY_IMG=$(VPROXY_IMG)"
 	@echo "BUNDLE_IMG=$(BUNDLE_IMG)"
 	@echo "OLM_CATALOG_IMG=$(OLM_CATALOG_IMG)"
 

diff --git a/api/v1/helpers.go b/api/v1/helpers.go
@@ -127,7 +127,13 @@ func MakeVDB() *VerticaDB {
 			DBName:     "db",
 			ShardCount: 12,
 			Subclusters: []Subcluster{
-				{Name: "defaultsubcluster", Size: 3, ServiceType: corev1.ServiceTypeClusterIP, Type: PrimarySubcluster},
+				{
+					Name:        "defaultsubcluster",
+					Size:        3,
+					ServiceType: corev1.ServiceTypeClusterIP,
+					Type:        PrimarySubcluster,
+					Proxy:       Proxy{Image: "vertica-client-proxy:latest"},
+				},
 			},
 		},
 	}
@@ -451,6 +457,16 @@ func (s *Subcluster) GetStsSize(vdb *VerticaDB) int32 {
 	return s.Size
 }
 
+// GetVProxyConfigMapName returns the name of the client proxy config map
+func (s *Subcluster) GetVProxyConfigMapName(vdb *VerticaDB) string {
+	return fmt.Sprintf("%s-%s-proxy-cm", vdb.Name, s.Name)
+}
+
+// GetVProxyDeploymentName returns the name of the client proxy deployment
+func (s *Subcluster) GetVProxyDeploymentName(vdb *VerticaDB) string {
+	return fmt.Sprintf("%s-%s-proxy", vdb.Name, s.Name)
+}
+
 // FindSubclusterForServiceName will find any subclusters that match the given service name
 func (v *VerticaDB) FindSubclusterForServiceName(svcName string) (scs []*Subcluster, totalSize int32) {
 	totalSize = int32(0)

diff --git a/api/v1/verticadb_types.go b/api/v1/verticadb_types.go
@@ -801,6 +801,53 @@ type Subcluster struct {
 	// State to indicate whether the operator must shut down the subcluster
 	// and not try to restart it.
 	Shutdown bool `json:"shutdown,omitempty"`
+
+	// +kubebuilder:validation:Optional
+	// +operator-sdk:csv:customresourcedefinitions:type=spec
+	// Create client proxy pods for the subcluster if defined
+	// All incoming connections to the subclusters will be routed through the proxy pods
+	Proxy Proxy `json:"proxy,omitempty"`
+}
+
+type Proxy struct {
+	// +kubebuilder:default:="opentext/vertica-client-proxy:latest"
+	// +kubebuilder:validation:required
+	// +operator-sdk:csv:customresourcedefinitions:type=spec
+	// The docker image name that contains the Vertica proxy server.
+	Image string `json:"image,omitempty"`
+
+	// +kubebuilder:default:=1
+	// +kubebuilder:validation:Optional
+	// +operator-sdk:csv:customresourcedefinitions:type=spec
+	// The number of replicas that the proxy server will have.
+	Replica int32 `json:"replica,omitempty"`
+
+	// +kubebuilder:default:=5433
+	// +kubebuilder:validation:Optional
+	// +operator-sdk:csv:customresourcedefinitions:type=spec
+	// The port proxy server will listen to. The only supported value is 5433
+	// TODO: hide this option since load-balancer cannot work with other port numbers
+	// Port int32 `json:"port,omitempty"`
+
+	// +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:hidden"
+	// +kubebuilder:default:=""
+	// +kubebuilder:validation:Optional
+	// A secret that contains the TLS credentials to use for Vertica's client
+	// proxy. If this is empty, the operator will create a secret to use and
+	// add the name of the generate secret in this field.
+	// When set, the secret must have the following keys defined: tls.key,
+	// tls.crt and ca.crt. To store this secret outside of Kubernetes, you can
+	// use a secret path reference prefix, such as gsm://. Everything after the
+	// prefix is the name of the secret in the service you are storing.
+	TLSSecret string `json:"tlsSecret,omitempty"`
+
+	// +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:resourceRequirements"
+	// This defines the resource requests and limits for the client proxy pods in the subcluster.
+	// It is advisable that the request and limits match as this ensures the
+	// pods are assigned to the guaranteed QoS class. This will reduces the
+	// chance that pods are chosen by the OOM killer.
+	// More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
+	Resources corev1.ResourceRequirements `json:"resources,omitempty"`
 }
 
 // Affinity is used instead of corev1.Affinity and behaves the same.

diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go
diff --git a/kuttl-test.yaml b/kuttl-test.yaml
@@ -36,6 +36,7 @@ testDirs:
   - tests/e2e-leg-8-online
   - tests/e2e-leg-9
   - tests/e2e-leg-10
+  - tests/e2e-leg-11
   - tests/e2e-udx
   - tests/e2e-server-upgrade
   - tests/e2e-server-upgrade-at-only