diff --git a/.github/actions/run-e2e-leg/action.yaml b/.github/actions/run-e2e-leg/action.yaml index 2e14017ce..4a19d0a52 100644 --- a/.github/actions/run-e2e-leg/action.yaml +++ b/.github/actions/run-e2e-leg/action.yaml @@ -1,6 +1,9 @@ name: 'Run an e2e leg' description: 'Will run all tests under one e2e leg with specific configurations' inputs: + vproxy-image: + description: 'Name of the vertica client proxy image' + required: false vlogger-image: description: 'Name of the vertica logger image' required: true @@ -109,6 +112,7 @@ runs: export VERTICA_IMG=${{ inputs.vertica-image }} export OPERATOR_IMG=${{ inputs.operator-image }} export VLOGGER_IMG=${{ inputs.vlogger-image }} + export VPROXY_IMG=${{ inputs.vlogger-image }} if [[ "${{ inputs.vertica-license }}" != "" ]]; then export LICENSE_FILE=/tmp/vertica-license.dat diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index ac9001c64..25a34dd98 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -23,7 +23,7 @@ on: type: string required: false vlogger_image: - description: 'Name of an existing vlogger image. Leave blank to build oe with the default name' + description: 'Name of an existing vlogger image. Leave blank to build one with the default name' type: string required: false e2e_test_suites: @@ -579,6 +579,30 @@ jobs: vertica-license: ${{ secrets.VERTICA_LICENSE }} need-base-vertica-image: 'true' + e2e-leg-11-vcluster: + if: ${{ ! github.event.pull_request.head.repo.fork && (inputs.e2e_test_suites == 'all' || inputs.e2e_test_suites == 'vcluster leg 11' || inputs.e2e_test_suites == '')}} + needs: [build] + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Test + uses: ./.github/actions/run-e2e-leg + with: + leg-identifier: 'leg-11' + dockerhub-username: ${{ secrets.DOCKERHUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }} + vlogger-image: ${{ needs.build.outputs.vlogger-image }} + vproxy-image: 'opentext/client-proxy:latest' + operator-image: ${{ needs.build.outputs.operator-image }} + vertica-image: ${{ needs.build.outputs.full-vertica-image }} + vertica-deployment-method: vclusterops + communal-storage-type: s3 + minimum-vertica-image: '25.1.0' + # Include the vertica license so we can test multiple subclusters. + vertica-license: ${{ secrets.VERTICA_LICENSE }} + need-base-vertica-image: 'true' + e2e-server-upgrade-admintools: if: ${{ ! contains(github.ref, 'k8s-sync') && (inputs.e2e_test_suites == 'all' || inputs.e2e_test_suites == 'admintools server upgrade' || inputs.e2e_test_suites == '') }} needs: [build] diff --git a/DEVELOPER.md b/DEVELOPER.md index 8aaf1d1d0..073fe175d 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -383,7 +383,10 @@ export BASE_VERTICA_IMG=opentext/vertica-k8s:24.2.0-1 export VERTICA_IMG=opentext/vertica-k8s:latest ``` > VERTICA_IMG is the vertica image you want to run the test with. For upgrade test cases, BASE_VERTICA_IMG is the base vertica version that will be installed. VERTICA_IMG is the vertica version that the base version will be upgraded to. The version in VERTICA_IMG must be higher than that in BASE_VERTICA_IMG. - +```shell +export VPROXY_IMG=opentext/client-proxy:latest +``` +> VPROXY_IMG is the vertica client proxy image you want to run the client proxy and session transfer test with. 3. kuttl-test.yaml is the configuration file for e2e test cases. There is a "timeout" field in it. If your server is not fast enough, you may need to increase that value to pass the test cases. There is another field "parallel" that controls the maximum number of tests to run at once. It is set to 2 by default. You can set it to 1 if your server is not fast enough. diff --git a/Makefile b/Makefile index 2fb859a6d..a4bff09f1 100644 --- a/Makefile +++ b/Makefile @@ -99,6 +99,9 @@ export BASE_VERTICA_IMG # Image URL to use for the logger sidecar VLOGGER_IMG ?= $(IMG_REPO)vertica-logger:$(VLOGGER_VERSION) export VLOGGER_IMG +# Image URL to use for the vertica client proxy. This is for testing purposes only. +VPROXY_IMG ?= opentext/client-proxy:latest +export VPROXY_IMG # If the current leg in the CI tests is leg-9 LEG9 ?= no export LEG9 @@ -557,6 +560,7 @@ echo-images: ## Print the names of all of the images used @echo "VERTICA_IMG=$(VERTICA_IMG)" @echo "BASE_VERTICA_IMG=$(BASE_VERTICA_IMG)" @echo "VLOGGER_IMG=$(VLOGGER_IMG)" + @echo "VPROXY_IMG=$(VPROXY_IMG)" @echo "BUNDLE_IMG=$(BUNDLE_IMG)" @echo "OLM_CATALOG_IMG=$(OLM_CATALOG_IMG)" diff --git a/api/v1/helpers.go b/api/v1/helpers.go index b9f22482f..9fa66be8d 100644 --- a/api/v1/helpers.go +++ b/api/v1/helpers.go @@ -127,7 +127,13 @@ func MakeVDB() *VerticaDB { DBName: "db", ShardCount: 12, Subclusters: []Subcluster{ - {Name: "defaultsubcluster", Size: 3, ServiceType: corev1.ServiceTypeClusterIP, Type: PrimarySubcluster}, + { + Name: "defaultsubcluster", + Size: 3, + ServiceType: corev1.ServiceTypeClusterIP, + Type: PrimarySubcluster, + Proxy: Proxy{Image: "vertica-client-proxy:latest"}, + }, }, }, } @@ -451,6 +457,16 @@ func (s *Subcluster) GetStsSize(vdb *VerticaDB) int32 { return s.Size } +// GetVProxyConfigMapName returns the name of the client proxy config map +func (s *Subcluster) GetVProxyConfigMapName(vdb *VerticaDB) string { + return fmt.Sprintf("%s-%s-proxy-cm", vdb.Name, s.Name) +} + +// GetVProxyDeploymentName returns the name of the client proxy deployment +func (s *Subcluster) GetVProxyDeploymentName(vdb *VerticaDB) string { + return fmt.Sprintf("%s-%s-proxy", vdb.Name, s.Name) +} + // FindSubclusterForServiceName will find any subclusters that match the given service name func (v *VerticaDB) FindSubclusterForServiceName(svcName string) (scs []*Subcluster, totalSize int32) { totalSize = int32(0) diff --git a/api/v1/verticadb_types.go b/api/v1/verticadb_types.go index a64b86fe3..c658360a6 100644 --- a/api/v1/verticadb_types.go +++ b/api/v1/verticadb_types.go @@ -801,6 +801,53 @@ type Subcluster struct { // State to indicate whether the operator must shut down the subcluster // and not try to restart it. Shutdown bool `json:"shutdown,omitempty"` + + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // Create client proxy pods for the subcluster if defined + // All incoming connections to the subclusters will be routed through the proxy pods + Proxy Proxy `json:"proxy,omitempty"` +} + +type Proxy struct { + // +kubebuilder:default:="opentext/vertica-client-proxy:latest" + // +kubebuilder:validation:required + // +operator-sdk:csv:customresourcedefinitions:type=spec + // The docker image name that contains the Vertica proxy server. + Image string `json:"image,omitempty"` + + // +kubebuilder:default:=1 + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // The number of replicas that the proxy server will have. + Replica int32 `json:"replica,omitempty"` + + // +kubebuilder:default:=5433 + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // The port proxy server will listen to. The only supported value is 5433 + // TODO: hide this option since load-balancer cannot work with other port numbers + // Port int32 `json:"port,omitempty"` + + // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:hidden" + // +kubebuilder:default:="" + // +kubebuilder:validation:Optional + // A secret that contains the TLS credentials to use for Vertica's client + // proxy. If this is empty, the operator will create a secret to use and + // add the name of the generate secret in this field. + // When set, the secret must have the following keys defined: tls.key, + // tls.crt and ca.crt. To store this secret outside of Kubernetes, you can + // use a secret path reference prefix, such as gsm://. Everything after the + // prefix is the name of the secret in the service you are storing. + TLSSecret string `json:"tlsSecret,omitempty"` + + // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:resourceRequirements" + // This defines the resource requests and limits for the client proxy pods in the subcluster. + // It is advisable that the request and limits match as this ensures the + // pods are assigned to the guaranteed QoS class. This will reduces the + // chance that pods are chosen by the OOM killer. + // More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + Resources corev1.ResourceRequirements `json:"resources,omitempty"` } // Affinity is used instead of corev1.Affinity and behaves the same. diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index a566822ff..6e0cf00f2 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -109,6 +109,22 @@ func (in *LocalStorage) DeepCopy() *LocalStorage { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Proxy) DeepCopyInto(out *Proxy) { + *out = *in + in.Resources.DeepCopyInto(&out.Resources) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Proxy. +func (in *Proxy) DeepCopy() *Proxy { + if in == nil { + return nil + } + out := new(Proxy) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RestorePointInfo) DeepCopyInto(out *RestorePointInfo) { *out = *in @@ -233,6 +249,7 @@ func (in *Subcluster) DeepCopyInto(out *Subcluster) { (*out)[key] = val } } + in.Proxy.DeepCopyInto(&out.Proxy) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Subcluster. diff --git a/kuttl-test.yaml b/kuttl-test.yaml index 226447fb0..d05c3b78c 100644 --- a/kuttl-test.yaml +++ b/kuttl-test.yaml @@ -36,6 +36,7 @@ testDirs: - tests/e2e-leg-8-online - tests/e2e-leg-9 - tests/e2e-leg-10 + - tests/e2e-leg-11 - tests/e2e-udx - tests/e2e-server-upgrade - tests/e2e-server-upgrade-at-only diff --git a/pkg/builder/builder.go b/pkg/builder/builder.go index a05f89f94..4db5a84d4 100644 --- a/pkg/builder/builder.go +++ b/pkg/builder/builder.go @@ -29,6 +29,7 @@ import ( "github.com/vertica/vertica-kubernetes/pkg/names" "github.com/vertica/vertica-kubernetes/pkg/paths" "github.com/vertica/vertica-kubernetes/pkg/secrets" + "gopkg.in/yaml.v2" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" @@ -61,6 +62,13 @@ const ( VSqlUserEnv = "VSQL_USER" VerticaStartupLogDuplicate = "VERTICA_STARTUP_LOG_DUPLICATE" + // Environment variables that are (optionally) set when deploying client proxy + VProxyRootCAEnv = "VPROXY_TLS_SERVER_CA" + VProxyCertEnv = "VPROXY_TLS_SERVER_CERT" + VProxyKeyEnv = "VPROXY_TLS_SERVER_KEY" + VProxySecretNamespaceEnv = "VPROXY_SECRET_NAMESPACE" + VProxySecretNameEnv = "VPROXY_SECRET_NAME" + // Environment variables that are (optionally) set when deployed with vclusterops NMARootCAEnv = "NMA_ROOTCA_PATH" NMACertEnv = "NMA_CERT_PATH" @@ -90,8 +98,19 @@ const ( // The path to the scrutinize tarball scrutinizeTarball = "SCRUTINIZE_TARBALL" passwordMountName = "password" + + // Client proxy config file name + vProxyConfigFile = "config.yaml" ) +type ProxyData struct { + Listener map[string]interface{} + Database map[string][]string + Log map[string]string + // TODO: to support TLS + // Tls map[string]string +} + // BuildExtSvc creates desired spec for the external service. func BuildExtSvc(nm types.NamespacedName, vdb *vapi.VerticaDB, sc *vapi.Subcluster, selectorLabelCreator func(*vapi.VerticaDB, *vapi.Subcluster) map[string]string) *corev1.Service { @@ -841,6 +860,158 @@ func makeScrutinizeInitContainers(vscr *v1beta1.VerticaScrutinize, vdb *vapi.Ver return cnts } +// BuildVProxyDeployment builds manifest for a subclusters VProxy deployment +func BuildVProxyDeployment(nm types.NamespacedName, vdb *vapi.VerticaDB, sc *vapi.Subcluster) *appsv1.Deployment { + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: nm.Name, + Namespace: nm.Namespace, + Labels: MakeLabelsForVProxyObject(vdb, sc, true), + Annotations: MakeAnnotationsForVProxyObject(vdb), + }, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: MakeDepSelectorLabels(vdb, sc), + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: MakeDepSelectorLabels(vdb, sc), + Annotations: MakeAnnotationsForVProxyObject(vdb), + }, + Spec: buildVProxyPodSpec(vdb, sc), + }, + Replicas: &sc.Proxy.Replica, + }, + } +} + +// buildPodSpec creates a PodSpec for the deployment +func buildVProxyPodSpec(vdb *vapi.VerticaDB, sc *vapi.Subcluster) corev1.PodSpec { + termGracePeriod := int64(0) + return corev1.PodSpec{ + ImagePullSecrets: GetK8sLocalObjectReferenceArray(vdb.Spec.ImagePullSecrets), + Containers: []corev1.Container{makeVProxyContainer(vdb, sc)}, + TerminationGracePeriodSeconds: &termGracePeriod, + ServiceAccountName: vdb.Spec.ServiceAccountName, + SecurityContext: vdb.Spec.PodSecurityContext, + Volumes: []corev1.Volume{ + { + Name: sc.Name, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: sc.GetVProxyConfigMapName(vdb)}, + }, + }, + }, + }, + } +} + +// makeDataForProxyConfigMap generates a configmap data in config.yaml format. +// # Vertica Proxy Configuration File Example +// +// # Proxy's listener network address for clients connections +// listener: +// +// host: "" # Listen on all IP addresses of the local system +// port: 5433 +// +// # The database for proxy connections +// database: +// +// nodes: +// - node1.address.com:5433 +// - node2.address.com:5433 +// - node3.address.com:5433 +// +// # Log level: 0=TRACE|1=DEBUG|2=INFO|3=WARN|4=FATAL|5=NONE (Default INFO) +// log: +// +// level: DEBUG +// # OR +// # level: 1 +// +// tls: +// +// # For proxy-server TLS +// serverca: /Path/to/server_cacert.pem +// hostname: db.example.com +// proxykey: /Path/to/proxy_key.pem +// proxycert: /Path/to/proxy_cert.pem +// +// # For client-proxy TLS +// serverkey: /Path/to/server_key.pem +// servercert: /Path/to/server_cert.pem +// clientca: /Path/to/client_cacert.pem +func makeDataForVProxyConfigMap(vdb *vapi.VerticaDB, sc *vapi.Subcluster) string { + var nodeList []string + port := 5433 + + for i := int32(0); i < sc.Size; i++ { + nodeItem := fmt.Sprintf("%s:%d", names.GenPodName(vdb, sc, i).Name, port) + nodeList = append(nodeList, nodeItem) + } + + proxyData := ProxyData{ + Listener: map[string]interface{}{ + "host": "", + "port": port, + }, + Database: map[string][]string{ + "nodes": nodeList, + }, + Log: map[string]string{ + "level": "INFO", + }, + } + + pData, _ := yaml.Marshal(proxyData) + return string(pData) +} + +// BuildVProxyConfigMap builds a config map for client proxy +func BuildVProxyConfigMap(nm types.NamespacedName, vdb *vapi.VerticaDB, sc *vapi.Subcluster) *corev1.ConfigMap { + immutable := false + proxyData := makeDataForVProxyConfigMap(vdb, sc) + return &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + Kind: "ConfigMap", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: nm.Name, + Namespace: nm.Namespace, + Labels: MakeLabelsForVProxyObject(vdb, sc, false), + Annotations: MakeAnnotationsForVProxyObject(vdb), + OwnerReferences: []metav1.OwnerReference{vdb.GenerateOwnerReference()}, + }, + // the data should not be immutable since the proxy database nodes could be changed + Immutable: &immutable, + Data: map[string]string{ + vProxyConfigFile: proxyData, + }, + } +} + +// makeProxyContainer builds the spec for the client proxy container +func makeVProxyContainer(vdb *vapi.VerticaDB, sc *vapi.Subcluster) corev1.Container { + envVars := buildVProxyTLSCertsEnvVars(vdb, sc) + envVars = append(envVars, buildCommonEnvVars(vdb)...) + return corev1.Container{ + Image: sc.Proxy.Image, + ImagePullPolicy: vdb.Spec.ImagePullPolicy, + Name: names.ProxyContainer, + Env: envVars, + Resources: sc.Proxy.Resources, + Ports: []corev1.ContainerPort{ + {ContainerPort: VerticaClientPort, Name: "vertica"}, + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: sc.Name, MountPath: "/config"}, + }, + } +} + // makeServerContainer builds the spec for the server container func makeServerContainer(vdb *vapi.VerticaDB, sc *vapi.Subcluster) corev1.Container { envVars := translateAnnotationsToEnvVars(vdb) @@ -1598,6 +1769,21 @@ func buildNMATLSCertsEnvVars(vdb *vapi.VerticaDB) []corev1.EnvVar { } } +// buildVProxyTLSCertsEnvVars returns environment variables about proxy certs +func buildVProxyTLSCertsEnvVars(vdb *vapi.VerticaDB, sc *vapi.Subcluster) []corev1.EnvVar { + if vmeta.UseVProxyCertsMount(vdb.Annotations) && secrets.IsK8sSecret(sc.Proxy.TLSSecret) { + return []corev1.EnvVar{ + // TODO: use proxy certs + } + } + return []corev1.EnvVar{ + // The proxy will read the secrets directly from the secret store. + // We provide the secret namespace and name for this reason. + {Name: VProxySecretNamespaceEnv, Value: vdb.ObjectMeta.Namespace}, + {Name: VProxySecretNameEnv, Value: sc.Proxy.TLSSecret}, + } +} + // buildCommonEnvVars returns env vars that are common for the nma and server container. func buildCommonEnvVars(vdb *vapi.VerticaDB) []corev1.EnvVar { return []corev1.EnvVar{ diff --git a/pkg/builder/labels_annotations.go b/pkg/builder/labels_annotations.go index 13118191a..6c8fe0e6c 100644 --- a/pkg/builder/labels_annotations.go +++ b/pkg/builder/labels_annotations.go @@ -110,6 +110,13 @@ func MakeLabelsForSvcObject(vdb *vapi.VerticaDB, sc *vapi.Subcluster, svcType st return labels } +// MakeLabelsForVProxyObject constructs the labels of the client proxy config map and pods +func MakeLabelsForVProxyObject(vdb *vapi.VerticaDB, sc *vapi.Subcluster, forPod bool) map[string]string { + labels := makeLabelsForObject(vdb, sc, forPod) + labels[vmeta.ClientProxyLabel] = vmeta.ClientProxyTrue + return labels +} + // MakeLabelsForSandboxConfigMap constructs the labels of the sandbox config map func MakeLabelsForSandboxConfigMap(vdb *vapi.VerticaDB) map[string]string { labels := makeLabelsForObject(vdb, nil, false) @@ -142,6 +149,16 @@ func MakeAnnotationsForStsObject(vdb *vapi.VerticaDB, sc *vapi.Subcluster) map[s return annotations } +// MakeAnnotationsForVProxyObject builds the list of annotations that are included +// in the proxy config object. +func MakeAnnotationsForVProxyObject(vdb *vapi.VerticaDB) map[string]string { + annotations := MakeAnnotationsForObject(vdb) + if ver, ok := vdb.Annotations[vmeta.VersionAnnotation]; ok { + annotations[vmeta.VersionAnnotation] = ver + } + return annotations +} + // MakeAnnotationsForSandboxConfigMap builds the list of annotations that are included // in the sandbox config map. func MakeAnnotationsForSandboxConfigMap(vdb *vapi.VerticaDB) map[string]string { @@ -196,6 +213,16 @@ func MakeStsSelectorLabels(vdb *vapi.VerticaDB, sc *vapi.Subcluster) map[string] return m } +// MakeDepSelectorLabels will create the selector labels for use within a Deployment +func MakeDepSelectorLabels(vdb *vapi.VerticaDB, sc *vapi.Subcluster) map[string]string { + m := MakeBaseSvcSelectorLabels(vdb) + // Set a special selector to pick only the pods for this porxy deployment. It's + // derived from the deployment name as that stays constant and is unique in + // a namespace. + m[vmeta.DeploymentSelectorLabel] = sc.GetVProxyDeploymentName(vdb) + return m +} + // MakeAnnotationsForSubclusterService returns a map of annotations // for Subcluster sc's service under VerticaDB vdb. func MakeAnnotationsForSubclusterService(vdb *vapi.VerticaDB, sc *vapi.Subcluster) map[string]string { diff --git a/pkg/controllers/vdb/k8s.go b/pkg/controllers/vdb/k8s.go index 0424f7e8b..92948c80e 100644 --- a/pkg/controllers/vdb/k8s.go +++ b/pkg/controllers/vdb/k8s.go @@ -91,3 +91,12 @@ func createSts(ctx context.Context, vrec config.ReconcilerInterface, expSts *app } return vrec.GetClient().Create(ctx, expSts) } + +// createSts will create a new deployment. It assumes the deployment doesn't already exist. +func createDep(ctx context.Context, vrec config.ReconcilerInterface, vpDep *appsv1.Deployment, vdb *vapi.VerticaDB) error { + err := ctrl.SetControllerReference(vdb, vpDep, vrec.GetClient().Scheme()) + if err != nil { + return err + } + return vrec.GetClient().Create(ctx, vpDep) +} diff --git a/pkg/controllers/vdb/obj_reconciler.go b/pkg/controllers/vdb/obj_reconciler.go index 6404dc5a0..6949a4c79 100644 --- a/pkg/controllers/vdb/obj_reconciler.go +++ b/pkg/controllers/vdb/obj_reconciler.go @@ -437,9 +437,60 @@ func (o *ObjReconciler) createService(ctx context.Context, svc *corev1.Service, return o.Rec.GetClient().Create(ctx, svc) } +// checkVProxyConfigMap will create or update a client proxy config map if needed +func (o *ObjReconciler) checkVProxyConfigMap(ctx context.Context, cmName types.NamespacedName, sc *vapi.Subcluster) error { + curCM := &corev1.ConfigMap{} + newCM := builder.BuildVProxyConfigMap(cmName, o.Vdb, sc) + + err := o.Rec.GetClient().Get(ctx, cmName, curCM) + if err != nil && kerrors.IsNotFound(err) { + o.Log.Info("Creating client proxy config map", "Name", cmName) + return o.Rec.GetClient().Create(ctx, newCM) + } + + // TODO: support client proxy update + // if o.updateVProxyConfigMapFields(curCM, newCM) { + // o.Log.Info("Updating client proxy config map", "Name", cmName) + // return o.Rec.GetClient().Update(ctx, newCM) + //} + o.Log.Info("Found an existing client proxy config map with correct content, skip updating it", "Name", cmName) + return nil +} + +// checkVProxyDeployment will create or update the client proxy deployment +func (o *ObjReconciler) checkVProxyDeployment(ctx context.Context, sc *vapi.Subcluster) error { + cmName := names.GenVProxyConfigMapName(o.Vdb, sc) + err := o.checkVProxyConfigMap(ctx, cmName, sc) + if err != nil { + return err + } + + vpName := names.GenVProxyName(o.Vdb, sc) + curDep := &appsv1.Deployment{} + vpDep := builder.BuildVProxyDeployment(vpName, o.Vdb, sc) + vpErr := o.Rec.GetClient().Get(ctx, vpName, curDep) + if vpErr != nil && kerrors.IsNotFound(vpErr) { + o.Log.Info("Creating deployment", "Name", vpName, "Size", vpDep.Spec.Replicas, "Image", vpDep.Spec.Template.Spec.Containers[0].Image) + return createDep(ctx, o.Rec, vpDep, o.Vdb) + } + + // TODO: to update existing deployment + // return o.updateDep(ctx, curDep, vpDep) + return nil +} + // reconcileSts reconciles the statefulset for a particular subcluster. Returns // true if any create/update was done. func (o *ObjReconciler) reconcileSts(ctx context.Context, sc *vapi.Subcluster) (ctrl.Result, error) { + if vmeta.UseVProxy(o.Vdb.Annotations) { + // Create or update the client proxy deployment + vpErr := o.checkVProxyDeployment(ctx, sc) + if vpErr != nil { + return ctrl.Result{}, vpErr + } + } + + // Create or update the statefulset nm := names.GenStsName(o.Vdb, sc) curSts := &appsv1.StatefulSet{} expSts := builder.BuildStsSpec(nm, o.Vdb, sc) diff --git a/pkg/controllers/vdb/verticadb_controller.go b/pkg/controllers/vdb/verticadb_controller.go index 665b36eb2..133b286dd 100644 --- a/pkg/controllers/vdb/verticadb_controller.go +++ b/pkg/controllers/vdb/verticadb_controller.go @@ -61,6 +61,7 @@ type VerticaDBReconciler struct { // +kubebuilder:rbac:groups=core,resources=serviceaccounts,verbs=get;list;watch;create // +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles,verbs=get;list;watch;create // +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=rolebindings,verbs=get;list;watch;create +// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=core,resources=serviceaccounts,verbs=get;list;watch;create // +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch;create;update;delete;patch @@ -80,6 +81,7 @@ func (r *VerticaDBReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&rbacv1.RoleBinding{}). Owns(&corev1.Service{}). Owns(&appsv1.StatefulSet{}). + Owns(&appsv1.Deployment{}). Complete(r) } diff --git a/pkg/meta/annotations.go b/pkg/meta/annotations.go index 83e5889a7..18c27e050 100644 --- a/pkg/meta/annotations.go +++ b/pkg/meta/annotations.go @@ -50,6 +50,26 @@ const ( VClusterOpsAnnotationTrue = "true" VClusterOpsAnnotationFalse = "false" + // This is a feature flag for using vertica client proxy. Set this + // annotation in the VerticaDB that you want to start proxy pods to + // redirect the subcluster connections. The value of this annotation is + // treated as a boolean. + UseVProxyAnnotation = "vertica.com/use-client-proxy" + UseVProxyAnnotationTrue = "true" + UseVProxyAnnotationFalse = "false" + + // This is the log level of the proxy server will have. + // Log level: 0=TRACE|1=DEBUG|2=INFO|3=WARN|4=FATAL|5=NONE + VProxyLogLevelAnnotation = "vertica.com/client-proxy-log-level" + VProxyLogLevelDefaultLevel = "INFO" + + // This is a feature flag for mounting vproxy certs as a secret volume in server containerss. + // When set to true the vproxy reads certs from this mounted volume, + // when set to false it reads certs directly from k8s secret store. + MountVProxyCertsAnnotation = "vertica.com/mount-vproxy-certs" + MountVProxyCertsAnnotationTrue = "true" + MountVProxyCertsAnnotationFalse = "false" + // This is a feature flag for mounting NMA certs as a secret volume in server containers // if deployment method is vclusterops. When set to true the NMA reads certs from this mounted // volume, when set to false it reads certs directly from k8s secret store. @@ -345,6 +365,18 @@ func UseVClusterOps(annotations map[string]string) bool { return lookupBoolAnnotation(annotations, VClusterOpsAnnotation, true /* default value */) } +// UseVProxy returns true if all subcluster connections redirect to the proxy pods +func UseVProxy(annotations map[string]string) bool { + // UseVProxy returns false if the annotation isn't set. + return lookupBoolAnnotation(annotations, UseVProxyAnnotation, false /* default value */) +} + +// UseVProxyCertsMount returns true if the proxy reads certs from the mounted secret +// volume rather than directly from k8s secret store. +func UseVProxyCertsMount(annotations map[string]string) bool { + return lookupBoolAnnotation(annotations, MountVProxyCertsAnnotation, true /* default value */) +} + // UseNMACertsMount returns true if the NMA reads certs from the mounted secret // volume rather than directly from k8s secret store. func UseNMACertsMount(annotations map[string]string) bool { @@ -510,6 +542,11 @@ func GetNMAHealthProbeOverride(annotations map[string]string, probeName, field s return int32(convVal), true //nolint:gosec } +// GetVProxyLogLevel returns scrutinize log age hours +func GetVProxyLogLevel(annotations map[string]string) string { + return lookupStringAnnotation(annotations, VProxyLogLevelAnnotation, "INFO" /* default value */) +} + // GetScrutinizePodTTL returns how long the scrutinize pod will keep running func GetScrutinizePodTTL(annotations map[string]string) int { val := lookupIntAnnotation(annotations, diff --git a/pkg/meta/labels.go b/pkg/meta/labels.go index 1a12acd3f..59d520369 100644 --- a/pkg/meta/labels.go +++ b/pkg/meta/labels.go @@ -107,6 +107,13 @@ const ( // subcluster rename. So, it shouldn't be treated as such. SubclusterSelectorLabel = "vertica.com/subcluster-selector-name" + // This is set in the pods, and is used by deployment as a pod selector. This + // stays constant for the life of the deployment, and is unique across + // deployment for other subclusters in the database. It may appear like it + // is derived from a subcluster name, but is does not get updated for + // subcluster rename. So, it shouldn't be treated as such. + DeploymentSelectorLabel = "vertica.com/deployment-selector-name" + // ConfigMap objects // // This indicates that the object is watched by the sandbox controller. @@ -114,6 +121,10 @@ const ( // that represent sandboxed subclusters WatchedBySandboxLabel = "vertica.com/watched-by-sandbox-controller" WatchedBySandboxTrue = "true" + + // This indicates that the object is used as a client proxy object. + ClientProxyLabel = "vertica.com/client-porxy" + ClientProxyTrue = "true" ) // ProtectedLabels lists all of the internally used label. diff --git a/pkg/names/names.go b/pkg/names/names.go index 12d74bbcf..a94f79fc2 100644 --- a/pkg/names/names.go +++ b/pkg/names/names.go @@ -27,6 +27,7 @@ import ( const ( ServerContainer = "server" NMAContainer = "nma" + ProxyContainer = "proxy" ScrutinizeInitContainer = "scrutinize" ScrutinizeMainContainer = "main" ) @@ -65,6 +66,16 @@ func GenSandboxConfigMapName(vdb *vapi.VerticaDB, sandbox string) types.Namespac return GenNamespacedName(vdb, vdb.Name+"-"+vapi.GenCompatibleFQDNHelper(sandbox)) } +// GenVProxyName returns the name of the client proxy deployment +func GenVProxyName(vdb *vapi.VerticaDB, sc *vapi.Subcluster) types.NamespacedName { + return GenNamespacedName(vdb, sc.GetVProxyDeploymentName(vdb)) +} + +// GenVProxyConfigMapName returns the name of the client proxy configmap +func GenVProxyConfigMapName(vdb *vapi.VerticaDB, sc *vapi.Subcluster) types.NamespacedName { + return GenNamespacedName(vdb, sc.GetVProxyConfigMapName(vdb)) +} + // GenCommunalCredSecretName returns the name of the secret that has the credentials to access s3 func GenCommunalCredSecretName(vdb *vapi.VerticaDB) types.NamespacedName { return GenNamespacedName(vdb, vdb.Spec.Communal.CredentialSecret) diff --git a/scripts/kind.sh b/scripts/kind.sh index e0a64b894..1b349b33d 100755 --- a/scripts/kind.sh +++ b/scripts/kind.sh @@ -22,7 +22,7 @@ TAG=latest KUBEVER=1.23.0 IP_FAMILY=ipv4 LISTEN_ALL_INTERFACES=N -VSQL_PORT=5433 +VSQL_PORT=${VSQL_PORT:-5433} SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" REPO_DIR=$(dirname $SCRIPT_DIR) KIND=$REPO_DIR/bin/kind @@ -32,7 +32,7 @@ HANDLE_REGISTRY=1 LOG_MAX_SIZE='100Mi' EVENT_TTL='24h' -while getopts "ut:k:i:ap:xr:m:" opt +while getopts "ut:k:i:ap:xr:m:l:e:" opt do case $opt in u) UPLOAD_IMAGES=1;; @@ -44,8 +44,8 @@ do r) REG_PORT=$OPTARG;; x) HANDLE_REGISTRY=;; m) MOUNT_PATH=$OPTARG;; - l) LOG_MAX_SIZE=$OPTARG;; - e) EVENT_TTL=$OPTARG;; + l) LOG_MAX_SIZE=$OPTARG;; + e) EVENT_TTL=$OPTARG;; esac done diff --git a/scripts/setup-kustomize.sh b/scripts/setup-kustomize.sh index 33821a5a5..e144e08bd 100755 --- a/scripts/setup-kustomize.sh +++ b/scripts/setup-kustomize.sh @@ -96,6 +96,9 @@ fi if [ -z "${VLOGGER_IMG}" ]; then VLOGGER_IMG=$(cd $REPO_DIR && make echo-images | grep VLOGGER_IMG | cut -d'=' -f2) fi +if [ -z "${VPROXY_IMG}" ]; then + VPROXY_IMG=$(cd $REPO_DIR && make echo-images | grep VPROXY_IMG | cut -d'=' -f2) +fi # Name of the secret that contains the cert to use for communal access # authentication. This is the name of the namespace copy, so it is hard coded @@ -273,6 +276,16 @@ replacements: kind: Job fieldPaths: - spec.template.spec.containers.0.image + - source: + kind: ConfigMap + name: e2e + fieldPath: data.vproxyImage + targets: + - select: + kind: VerticaDB + name: v-client-proxy + fieldPaths: + - spec.subclusters.*.proxy.image - source: kind: ConfigMap name: e2e @@ -534,6 +547,7 @@ data: verticaImage: ${VERTICA_IMG} vloggerImage: ${VLOGGER_IMG} baseVerticaImage: ${BASE_VERTICA_IMG} + vproxyImage: ${VPROXY_IMG} EOF # If a cert was specified for communal endpoint access, include a datapoint diff --git a/tests/e2e-leg-11/client-proxy/02-assert.yaml b/tests/e2e-leg-11/client-proxy/02-assert.yaml new file mode 100644 index 000000000..0bd1a36d6 --- /dev/null +++ b/tests/e2e-leg-11/client-proxy/02-assert.yaml @@ -0,0 +1,22 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: integration-test-role +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: integration-test-rb diff --git a/tests/e2e-leg-11/client-proxy/02-rbac.yaml b/tests/e2e-leg-11/client-proxy/02-rbac.yaml new file mode 100644 index 000000000..c6c5d01da --- /dev/null +++ b/tests/e2e-leg-11/client-proxy/02-rbac.yaml @@ -0,0 +1,18 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl apply --namespace $NAMESPACE -f ../../manifests/rbac/base/rbac.yaml + ignoreFailure: true diff --git a/tests/e2e-leg-11/client-proxy/05-create-secrets.yaml b/tests/e2e-leg-11/client-proxy/05-create-secrets.yaml new file mode 100644 index 000000000..ecced2c12 --- /dev/null +++ b/tests/e2e-leg-11/client-proxy/05-create-secrets.yaml @@ -0,0 +1,19 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kustomize build ../../manifests/communal-creds/overlay | kubectl apply -f - --namespace $NAMESPACE + - script: kustomize build ../../manifests/priv-container-creds/overlay | kubectl apply -f - --namespace $NAMESPACE + - script: kustomize build ../../manifests/vertica-license/overlay | kubectl apply -f - --namespace $NAMESPACE diff --git a/tests/e2e-leg-11/client-proxy/10-assert.yaml b/tests/e2e-leg-11/client-proxy/10-assert.yaml new file mode 100644 index 000000000..9faca75c9 --- /dev/null +++ b/tests/e2e-leg-11/client-proxy/10-assert.yaml @@ -0,0 +1,21 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Pod +metadata: + namespace: verticadb-operator + labels: + control-plane: verticadb-operator +status: + phase: Running diff --git a/tests/e2e-leg-11/client-proxy/10-verify-operator.yaml b/tests/e2e-leg-11/client-proxy/10-verify-operator.yaml new file mode 100644 index 000000000..0cd372046 --- /dev/null +++ b/tests/e2e-leg-11/client-proxy/10-verify-operator.yaml @@ -0,0 +1,12 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/e2e-leg-11/client-proxy/15-assert.yaml b/tests/e2e-leg-11/client-proxy/15-assert.yaml new file mode 100644 index 000000000..0b4fb0cd9 --- /dev/null +++ b/tests/e2e-leg-11/client-proxy/15-assert.yaml @@ -0,0 +1,54 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: v-client-proxy-pri1 +status: + currentReplicas: 3 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: v-client-proxy-pri1-proxy +status: + readyReplicas: 2 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: v-client-proxy-pri1-proxy-cm +data: + config.yaml: | + listener: + host: "" + port: 5433 + database: + nodes: + - v-client-proxy-pri1-0:5433 + - v-client-proxy-pri1-1:5433 + - v-client-proxy-pri1-2:5433 + log: + level: INFO +immutable: false +--- +apiVersion: vertica.com/v1 +kind: VerticaDB +metadata: + name: v-client-proxy +spec: + subclusters: + - name: pri1 + proxy: + replica: 2 diff --git a/tests/e2e-leg-11/client-proxy/15-setup-vdb.yaml b/tests/e2e-leg-11/client-proxy/15-setup-vdb.yaml new file mode 100644 index 000000000..c9ed69089 --- /dev/null +++ b/tests/e2e-leg-11/client-proxy/15-setup-vdb.yaml @@ -0,0 +1,17 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - command: bash -c "kustomize build setup-vdb/overlay | kubectl -n $NAMESPACE apply -f - " diff --git a/tests/e2e-leg-11/client-proxy/20-assert.yaml b/tests/e2e-leg-11/client-proxy/20-assert.yaml new file mode 100644 index 000000000..de5016c0d --- /dev/null +++ b/tests/e2e-leg-11/client-proxy/20-assert.yaml @@ -0,0 +1,58 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: v-client-proxy-pri1 +status: + currentReplicas: 3 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: v-client-proxy-pri1-proxy +status: + readyReplicas: 2 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: v-client-proxy-pri1-proxy-cm +data: + config.yaml: | + listener: + host: "" + port: 5433 + database: + nodes: + - v-client-proxy-pri1-0:5433 + - v-client-proxy-pri1-1:5433 + - v-client-proxy-pri1-2:5433 + log: + level: INFO +immutable: false +--- +apiVersion: vertica.com/v1 +kind: VerticaDB +metadata: + name: v-client-proxy +spec: + subclusters: + - name: pri1 + proxy: + replica: 2 +status: + subclusters: + - addedToDBCount: 3 + upNodeCount: 3 diff --git a/tests/e2e-leg-11/client-proxy/20-wait-for-createdb.yaml b/tests/e2e-leg-11/client-proxy/20-wait-for-createdb.yaml new file mode 100644 index 000000000..bf3726035 --- /dev/null +++ b/tests/e2e-leg-11/client-proxy/20-wait-for-createdb.yaml @@ -0,0 +1 @@ +# Intentionally empty to give this step a name in kuttl \ No newline at end of file diff --git a/tests/e2e-leg-11/client-proxy/25-wait-for-steady-state.yaml b/tests/e2e-leg-11/client-proxy/25-wait-for-steady-state.yaml new file mode 100644 index 000000000..7cd771cbb --- /dev/null +++ b/tests/e2e-leg-11/client-proxy/25-wait-for-steady-state.yaml @@ -0,0 +1,17 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - command: bash -c "../../../scripts/wait-for-verticadb-steady-state.sh -n verticadb-operator -t 360 $NAMESPACE" diff --git a/tests/e2e-leg-11/client-proxy/27-assert.yaml b/tests/e2e-leg-11/client-proxy/27-assert.yaml new file mode 100644 index 000000000..c0a51a30b --- /dev/null +++ b/tests/e2e-leg-11/client-proxy/27-assert.yaml @@ -0,0 +1,23 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Pod +metadata: + name: test-client-proxy-config +status: + containerStatuses: + - name: test + state: + terminated: + exitCode: 0 diff --git a/tests/e2e-leg-11/client-proxy/27-verify-client-proxy-config.yaml b/tests/e2e-leg-11/client-proxy/27-verify-client-proxy-config.yaml new file mode 100644 index 000000000..6b850eb93 --- /dev/null +++ b/tests/e2e-leg-11/client-proxy/27-verify-client-proxy-config.yaml @@ -0,0 +1,59 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: script-client-proxy-config +data: + entrypoint.sh: |- + #!/bin/bash + set -o errexit + set -o xtrace + + # get the proxy pod name + PROXY_PRI1=$(kubectl get pod -l pod-template-hash,vertica.com/deployment-selector-name=v-client-proxy-pri1-proxy --no-headers -o custom-columns=:.metadata.name) + + # get the content of the config.yaml + CONFIG=$(kubectl exec -i $PROXY_PRI1 -- sh -c "cat /config/config.yaml") + + for POD_NAME in v-client-proxy-pri1-0 v-client-proxy-pri1-1 v-client-proxy-pri1-2; do + RESULT=$(echo "$CONFIG" | grep -Pzo "$POD_NAME:5433" | tr -d '\0') + if [ "$POD_NAME:5433" != "$RESULT" ]; then + echo "Assertion failed: expect \"$POD_NAME:5433\" to be defined in config.yaml, got \"$RESULT\"" + exit 1 + fi + done +--- +apiVersion: v1 +kind: Pod +metadata: + name: test-client-proxy-config + labels: + stern: include +spec: + restartPolicy: Never + containers: + - name: test + image: bitnami/kubectl:1.20.4 + command: ["/bin/entrypoint.sh"] + volumeMounts: + - name: entrypoint-volume + mountPath: /bin/entrypoint.sh + readOnly: true + subPath: entrypoint.sh + volumes: + - name: entrypoint-volume + configMap: + defaultMode: 0777 + name: script-client-proxy-config diff --git a/tests/e2e-leg-11/client-proxy/55-assert.yaml b/tests/e2e-leg-11/client-proxy/55-assert.yaml new file mode 100644 index 000000000..a628b5b68 --- /dev/null +++ b/tests/e2e-leg-11/client-proxy/55-assert.yaml @@ -0,0 +1,23 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Pod +metadata: + name: test-kill-client-proxy +status: + containerStatuses: + - name: test + state: + terminated: + exitCode: 0 diff --git a/tests/e2e-leg-11/client-proxy/55-kill-proxy-pod.yaml b/tests/e2e-leg-11/client-proxy/55-kill-proxy-pod.yaml new file mode 100644 index 000000000..00ffc04f9 --- /dev/null +++ b/tests/e2e-leg-11/client-proxy/55-kill-proxy-pod.yaml @@ -0,0 +1,63 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: script-kill-client-proxy +data: + entrypoint.sh: |- + #!/bin/bash + set -o errexit + set -o xtrace + + # get the proxy pod name + PROXY_LABEL=pod-template-hash,vertica.com/deployment-selector-name=v-client-proxy-pri1-proxy + PROXY_PRI1_ORIG=$(kubectl get pod -l $PROXY_LABEL --no-headers -o custom-columns=:.metadata.name) + + # delete the proxy pod and a new proxy pod will be created + kubectl delete pod -l $PROXY_LABEL + PROXY_PRI1_NEW=$(kubectl get pod -l $PROXY_LABEL --no-headers -o custom-columns=:.metadata.name) + + # new proxy pod name should be different to the original one + for pod_new in $PROXY_PRI1_NEW; do + for pod_orig in $PROXY_PRI1_ORIG; do + if [ "$pod_new" == "$pod_orig" ]; then + echo "Assertion failed: expect a new proxy pod to be created, got the original pod $PROXY_PRI1_ORIG" + exit 1 + fi + done + done +--- +apiVersion: v1 +kind: Pod +metadata: + name: test-kill-client-proxy + labels: + stern: include +spec: + restartPolicy: Never + containers: + - name: test + image: bitnami/kubectl:1.20.4 + command: ["/bin/entrypoint.sh"] + volumeMounts: + - name: entrypoint-volume + mountPath: /bin/entrypoint.sh + readOnly: true + subPath: entrypoint.sh + volumes: + - name: entrypoint-volume + configMap: + defaultMode: 0777 + name: script-kill-client-proxy diff --git a/tests/e2e-leg-11/client-proxy/56-assert.yaml b/tests/e2e-leg-11/client-proxy/56-assert.yaml new file mode 100644 index 000000000..62db34a4b --- /dev/null +++ b/tests/e2e-leg-11/client-proxy/56-assert.yaml @@ -0,0 +1,20 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: v-client-proxy-pri1-proxy +status: + availableReplicas: 2 + readyReplicas: 2 diff --git a/tests/e2e-leg-11/client-proxy/56-wait-for-restart.yaml b/tests/e2e-leg-11/client-proxy/56-wait-for-restart.yaml new file mode 100644 index 000000000..8bcd1e149 --- /dev/null +++ b/tests/e2e-leg-11/client-proxy/56-wait-for-restart.yaml @@ -0,0 +1 @@ +# Intentionally empty to give this step a name in kuttl diff --git a/tests/e2e-leg-11/client-proxy/95-delete-cr.yaml b/tests/e2e-leg-11/client-proxy/95-delete-cr.yaml new file mode 100644 index 000000000..96d43c1de --- /dev/null +++ b/tests/e2e-leg-11/client-proxy/95-delete-cr.yaml @@ -0,0 +1,18 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: vertica.com/v1 + kind: VerticaDB diff --git a/tests/e2e-leg-11/client-proxy/95-errors.yaml b/tests/e2e-leg-11/client-proxy/95-errors.yaml new file mode 100644 index 000000000..b36c51e3a --- /dev/null +++ b/tests/e2e-leg-11/client-proxy/95-errors.yaml @@ -0,0 +1,34 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: StatefulSet +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/managed-by: verticadb-operator +--- +apiVersion: vertica.com/v1 +kind: VerticaDB +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: v-client-proxy-pri1-proxy +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: v-client-proxy-pri1-proxy-cm diff --git a/tests/e2e-leg-11/client-proxy/96-assert.yaml b/tests/e2e-leg-11/client-proxy/96-assert.yaml new file mode 100644 index 000000000..861c7dc8f --- /dev/null +++ b/tests/e2e-leg-11/client-proxy/96-assert.yaml @@ -0,0 +1,19 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Pod +metadata: + name: clean-communal +status: + phase: Succeeded diff --git a/tests/e2e-leg-11/client-proxy/96-cleanup-storage.yaml b/tests/e2e-leg-11/client-proxy/96-cleanup-storage.yaml new file mode 100644 index 000000000..dcc6306f3 --- /dev/null +++ b/tests/e2e-leg-11/client-proxy/96-cleanup-storage.yaml @@ -0,0 +1,17 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - command: bash -c "kustomize build clean-communal/overlay | kubectl -n $NAMESPACE apply -f - " diff --git a/tests/e2e-leg-11/client-proxy/96-errors.yaml b/tests/e2e-leg-11/client-proxy/96-errors.yaml new file mode 100644 index 000000000..671be36cf --- /dev/null +++ b/tests/e2e-leg-11/client-proxy/96-errors.yaml @@ -0,0 +1,15 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: PersistentVolumeClaim diff --git a/tests/e2e-leg-11/client-proxy/99-delete-ns.yaml b/tests/e2e-leg-11/client-proxy/99-delete-ns.yaml new file mode 100644 index 000000000..1674b3e8f --- /dev/null +++ b/tests/e2e-leg-11/client-proxy/99-delete-ns.yaml @@ -0,0 +1,17 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - command: kubectl delete ns $NAMESPACE diff --git a/tests/e2e-leg-11/client-proxy/setup-vdb/base/kustomization.yaml b/tests/e2e-leg-11/client-proxy/setup-vdb/base/kustomization.yaml new file mode 100644 index 000000000..681396735 --- /dev/null +++ b/tests/e2e-leg-11/client-proxy/setup-vdb/base/kustomization.yaml @@ -0,0 +1,15 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resources: + - setup-vdb.yaml diff --git a/tests/e2e-leg-11/client-proxy/setup-vdb/base/setup-vdb.yaml b/tests/e2e-leg-11/client-proxy/setup-vdb/base/setup-vdb.yaml new file mode 100644 index 000000000..384b20e3b --- /dev/null +++ b/tests/e2e-leg-11/client-proxy/setup-vdb/base/setup-vdb.yaml @@ -0,0 +1,40 @@ +# (c) Copyright [2021-2024] Open Text. +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: vertica.com/v1 +kind: VerticaDB +metadata: + name: v-client-proxy + annotations: + vertica.com/include-uid-in-path: true + vertica.com/use-client-proxy: true + vertica.com/client-proxy-log-level: INFO +spec: + image: kustomize-vertica-image + initPolicy: CreateSkipPackageInstall + communal: {} + local: + requestSize: 250Mi + # We pick a cluster size that is too big for the CE license. This relies on + # having a license, which will be added by kustomize. + subclusters: + - name: pri1 + size: 3 + type: primary + proxy: + image: kustomize-vproxy-image + replica: 2 + certSecrets: [] + imagePullSecrets: [] + volumes: [] + volumeMounts: []