From 41ae0e30b7af8a6b85f7d5460c841ff45167c30b Mon Sep 17 00:00:00 2001 From: Niall D <4562759+driev@users.noreply.github.com> Date: Wed, 25 Sep 2024 10:01:30 +0100 Subject: [PATCH] feat(charts): adding metadata object to a seldonconfig component (#5918) * adding a metadata object to seldonconfig components * adding labels to the servers helm charts * add metadata to seldonconfig.component * using sed to template the templates * add makefile changes * patching each component with labels and annotations * add a test * missing forward slash * hack labelz * remove test labels * add annotationz to servers * add prometheus annotations to the default seldonconfig * add a comment and a test --- k8s/Makefile | 6 ++- .../templates/seldon-v2-crds.yaml | 8 ++++ .../templates/seldon-v2-servers.yaml | 8 ++++ .../templates/seldon-v2-components.yaml | 29 +++++++++++--- .../seldon-core-v2-setup/values.yaml | 4 ++ .../seldon-core-v2-setup/values.yaml.template | 4 ++ .../patch_dataflow_json6902.yaml | 6 +++ .../patch_envoy_json6902.yaml | 6 +++ .../patch_hodometer_json6902.yaml | 6 +++ .../patch_modelgateway_json6902.yaml | 6 +++ .../patch_pipelinegateway_json6902.yaml | 6 +++ .../patch_scheduler_json6902.yaml | 6 +++ .../helm-servers/patch_mlserver.yaml | 2 + k8s/kustomize/helm-servers/patch_triton.yaml | 2 + k8s/yaml/components.yaml | 38 ++++++++++++++++--- k8s/yaml/crds.yaml | 8 ++++ k8s/yaml/servers.yaml | 8 ++++ .../apis/mlops/v1alpha1/seldonconfig_types.go | 3 ++ .../mlops/v1alpha1/zz_generated.deepcopy.go | 14 +++++++ .../bases/mlops.seldon.io_seldonconfigs.yaml | 8 ++++ operator/config/seldonconfigs/default.yaml | 4 ++ .../seldon/deployment_reconciler.go | 18 +++------ .../seldon/deployment_reconciler_test.go | 4 ++ .../reconcilers/seldon/runtime_reconciler.go | 8 +++- .../seldon/runtime_reconciler_test.go | 16 ++++++-- .../seldon/statefulset_reconciler.go | 4 ++ .../seldon/statefulset_reconciler_test.go | 18 +++++---- 27 files changed, 212 insertions(+), 38 deletions(-) diff --git a/k8s/Makefile b/k8s/Makefile index 9e00bb0935..bb4d80b89a 100644 --- a/k8s/Makefile +++ b/k8s/Makefile @@ -18,8 +18,10 @@ create-helm-charts: kustomize build kustomize/helm-servers/ > ${HELM_SERVERS_BASE}/seldon-v2-servers.yaml sed -i 's/\(.*\)\(imagePullSecrets:\).*/{{- with .Values.imagePullSecrets }}\n\1\2\n\1{{- toYaml . | nindent 8 }}\n{{- end }}/g' ${HELM_COMPONENTS_BASE}/seldon-v2-components.yaml sed -i 's/HACK_REMOVE_ME//' ${HELM_COMPONENTS_BASE}/seldon-v2-components.yaml - sed 's/HACK_REMOVE_ME//' ${HELM_SERVERS_BASE}/seldon-v2-servers.yaml \ - > ${HELM_SERVERS_BASE}/.seldon-v2-servers.yaml + # kustomizse has builtin helper fuctions for meta.labels and metadata.annotations, so the field names are changed to escape kustomize's extra processing + sed 's/labelz:/labels:\n\ \ \ /' ${HELM_SERVERS_BASE}/seldon-v2-servers.yaml > ${HELM_SERVERS_BASE}/.seldon-v2-servers.yaml + sed -i 's/annotationz:/annotations:\n\ \ \ /' ${HELM_SERVERS_BASE}/.seldon-v2-servers.yaml + sed -i 's/HACK_REMOVE_ME//' ${HELM_SERVERS_BASE}/.seldon-v2-servers.yaml sed -zi 's#\(apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole.*\- get\n\-\-\-\)#{{- if .Values.controller.clusterwide -}}\n\1\n{{- end }}#' ${HELM_COMPONENTS_BASE}/seldon-v2-components.yaml sed -zi 's#\(apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding.*ServiceAccount\n name: seldon-v2-controller-manager\n namespace:[^-]\{27\}\n\-\-\-\)#{{- if .Values.controller.clusterwide -}}\n\1\n{{- end }}#' ${HELM_COMPONENTS_BASE}/seldon-v2-components.yaml mv ${HELM_SERVERS_BASE}/.seldon-v2-servers.yaml ${HELM_SERVERS_BASE}/seldon-v2-servers.yaml diff --git a/k8s/helm-charts/seldon-core-v2-crds/templates/seldon-v2-crds.yaml b/k8s/helm-charts/seldon-core-v2-crds/templates/seldon-v2-crds.yaml index cdccc3775f..1441c216ba 100644 --- a/k8s/helm-charts/seldon-core-v2-crds/templates/seldon-v2-crds.yaml +++ b/k8s/helm-charts/seldon-core-v2-crds/templates/seldon-v2-crds.yaml @@ -641,6 +641,14 @@ spec: components: items: properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object name: type: string podSpec: diff --git a/k8s/helm-charts/seldon-core-v2-servers/templates/seldon-v2-servers.yaml b/k8s/helm-charts/seldon-core-v2-servers/templates/seldon-v2-servers.yaml index bbe3954810..66441c058d 100644 --- a/k8s/helm-charts/seldon-core-v2-servers/templates/seldon-v2-servers.yaml +++ b/k8s/helm-charts/seldon-core-v2-servers/templates/seldon-v2-servers.yaml @@ -1,6 +1,10 @@ apiVersion: mlops.seldon.io/v1alpha1 kind: Server metadata: + annotations: + {{ .Values.mlserver.annotations | toYaml }} + labels: + {{ .Values.mlserver.labels | toYaml }} name: mlserver namespace: '{{ .Release.Namespace }}' spec: @@ -11,6 +15,10 @@ spec: apiVersion: mlops.seldon.io/v1alpha1 kind: Server metadata: + annotations: + {{ .Values.triton.annotations | toYaml }} + labels: + {{ .Values.triton.labels | toYaml }} name: triton namespace: '{{ .Release.Namespace }}' spec: diff --git a/k8s/helm-charts/seldon-core-v2-setup/templates/seldon-v2-components.yaml b/k8s/helm-charts/seldon-core-v2-setup/templates/seldon-v2-components.yaml index 54a8622763..d588c10aa4 100644 --- a/k8s/helm-charts/seldon-core-v2-setup/templates/seldon-v2-components.yaml +++ b/k8s/helm-charts/seldon-core-v2-setup/templates/seldon-v2-components.yaml @@ -894,7 +894,10 @@ metadata: namespace: '{{ .Release.Namespace }}' spec: components: - - name: seldon-scheduler + - annotations: {{- toYaml .Values.scheduler.annotations | nindent + 8 }} + labels: {{- toYaml .Values.scheduler.labels | nindent 8 }} + name: seldon-scheduler podSpec: containers: - args: @@ -1012,7 +1015,10 @@ spec: resources: requests: storage: '{{ .Values.scheduler.resources.storage }}' - - name: seldon-pipelinegateway + - annotations: {{- toYaml .Values.pipelinegateway.annotations | nindent + 8 }} + labels: {{- toYaml .Values.pipelinegateway.labels | nindent 8 }} + name: seldon-pipelinegateway podSpec: containers: - args: @@ -1148,7 +1154,10 @@ spec: name: seldon-tracing name: tracing-config-volume replicas: 1 - - name: seldon-modelgateway + - annotations: {{- toYaml .Values.modelgateway.annotations | nindent + 8 }} + labels: {{- toYaml .Values.modelgateway.labels | nindent 8 }} + name: seldon-modelgateway podSpec: containers: - args: @@ -1266,7 +1275,10 @@ spec: name: seldon-tracing name: tracing-config-volume replicas: 1 - - name: hodometer + - annotations: {{- toYaml .Values.hodometer.annotations | nindent + 8 }} + labels: {{- toYaml .Values.hodometer.labels | nindent 8 }} + name: hodometer podSpec: containers: - env: @@ -1322,7 +1334,9 @@ spec: serviceAccountName: hodometer terminationGracePeriodSeconds: 5 replicas: 1 - - name: seldon-envoy + - annotations: {{- toYaml .Values.envoy.annotations | nindent 8 }} + labels: {{- toYaml .Values.envoy.labels | nindent 8 }} + name: seldon-envoy podSpec: containers: - env: @@ -1377,7 +1391,10 @@ spec: 8 }} terminationGracePeriodSeconds: 5 replicas: 1 - - name: seldon-dataflow-engine + - annotations: {{- toYaml .Values.dataflow.annotations | nindent 8 + }} + labels: {{- toYaml .Values.dataflow.labels | nindent 8 }} + name: seldon-dataflow-engine podSpec: containers: - env: diff --git a/k8s/helm-charts/seldon-core-v2-setup/values.yaml b/k8s/helm-charts/seldon-core-v2-setup/values.yaml index af905266c9..190d80e5ce 100644 --- a/k8s/helm-charts/seldon-core-v2-setup/values.yaml +++ b/k8s/helm-charts/seldon-core-v2-setup/values.yaml @@ -180,6 +180,10 @@ controller: runAsNonRoot: true envoy: + annotations: + "prometheus.io/path": "/stats/prometheus" + "prometheus.io/port": "9003" + "prometheus.io/scrape": "true" image: pullPolicy: IfNotPresent registry: docker.io diff --git a/k8s/helm-charts/seldon-core-v2-setup/values.yaml.template b/k8s/helm-charts/seldon-core-v2-setup/values.yaml.template index 55f67ea0ec..2103fc9ca5 100644 --- a/k8s/helm-charts/seldon-core-v2-setup/values.yaml.template +++ b/k8s/helm-charts/seldon-core-v2-setup/values.yaml.template @@ -180,6 +180,10 @@ controller: runAsNonRoot: true envoy: + annotations: + "prometheus.io/path": "/stats/prometheus" + "prometheus.io/port": "9003" + "prometheus.io/scrape": "true" image: pullPolicy: IfNotPresent registry: docker.io diff --git a/k8s/kustomize/helm-components-sc/patch_dataflow_json6902.yaml b/k8s/kustomize/helm-components-sc/patch_dataflow_json6902.yaml index 71d0c5b78c..21713857f9 100644 --- a/k8s/kustomize/helm-components-sc/patch_dataflow_json6902.yaml +++ b/k8s/kustomize/helm-components-sc/patch_dataflow_json6902.yaml @@ -1,3 +1,9 @@ - op: replace path: /spec/components/5/podSpec/securityContext value: HACK_REMOVE_ME{{- toYaml .Values.dataflow.securityContext | nindent 8 }} +- op: replace + path: /spec/components/5/labels + value: HACK_REMOVE_ME{{- toYaml .Values.dataflow.labels | nindent 8 }} +- op: replace + path: /spec/components/5/annotations + value: HACK_REMOVE_ME{{- toYaml .Values.dataflow.annotations | nindent 8 }} diff --git a/k8s/kustomize/helm-components-sc/patch_envoy_json6902.yaml b/k8s/kustomize/helm-components-sc/patch_envoy_json6902.yaml index ff1a02abdd..30283c0401 100644 --- a/k8s/kustomize/helm-components-sc/patch_envoy_json6902.yaml +++ b/k8s/kustomize/helm-components-sc/patch_envoy_json6902.yaml @@ -2,3 +2,9 @@ - op: replace path: /spec/components/4/podSpec/securityContext value: HACK_REMOVE_ME{{- toYaml .Values.envoy.securityContext | nindent 8 }} +- op: replace + path: /spec/components/4/labels + value: HACK_REMOVE_ME{{- toYaml .Values.envoy.labels | nindent 8 }} +- op: replace + path: /spec/components/4/annotations + value: HACK_REMOVE_ME{{- toYaml .Values.envoy.annotations | nindent 8 }} diff --git a/k8s/kustomize/helm-components-sc/patch_hodometer_json6902.yaml b/k8s/kustomize/helm-components-sc/patch_hodometer_json6902.yaml index 8fe0fb4a87..fd4859d083 100644 --- a/k8s/kustomize/helm-components-sc/patch_hodometer_json6902.yaml +++ b/k8s/kustomize/helm-components-sc/patch_hodometer_json6902.yaml @@ -2,3 +2,9 @@ - op: replace path: /spec/components/3/podSpec/securityContext value: HACK_REMOVE_ME{{- toYaml .Values.hodometer.securityContext | nindent 8 }} +- op: replace + path: /spec/components/3/labels + value: HACK_REMOVE_ME{{- toYaml .Values.hodometer.labels | nindent 8 }} +- op: replace + path: /spec/components/3/annotations + value: HACK_REMOVE_ME{{- toYaml .Values.hodometer.annotations | nindent 8 }} diff --git a/k8s/kustomize/helm-components-sc/patch_modelgateway_json6902.yaml b/k8s/kustomize/helm-components-sc/patch_modelgateway_json6902.yaml index c8883500a1..63232a4f0e 100644 --- a/k8s/kustomize/helm-components-sc/patch_modelgateway_json6902.yaml +++ b/k8s/kustomize/helm-components-sc/patch_modelgateway_json6902.yaml @@ -1,3 +1,9 @@ - op: replace path: /spec/components/2/podSpec/securityContext value: HACK_REMOVE_ME{{- toYaml .Values.modelgateway.securityContext | nindent 8 }} +- op: replace + path: /spec/components/2/labels + value: HACK_REMOVE_ME{{- toYaml .Values.modelgateway.labels | nindent 8 }} +- op: replace + path: /spec/components/2/annotations + value: HACK_REMOVE_ME{{- toYaml .Values.modelgateway.annotations | nindent 8 }} diff --git a/k8s/kustomize/helm-components-sc/patch_pipelinegateway_json6902.yaml b/k8s/kustomize/helm-components-sc/patch_pipelinegateway_json6902.yaml index c014d72687..f57f29f328 100644 --- a/k8s/kustomize/helm-components-sc/patch_pipelinegateway_json6902.yaml +++ b/k8s/kustomize/helm-components-sc/patch_pipelinegateway_json6902.yaml @@ -1,3 +1,9 @@ - op: replace path: /spec/components/1/podSpec/securityContext value: HACK_REMOVE_ME{{- toYaml .Values.pipelinegateway.securityContext | nindent 8 }} +- op: replace + path: /spec/components/1/labels + value: HACK_REMOVE_ME{{- toYaml .Values.pipelinegateway.labels | nindent 8 }} +- op: replace + path: /spec/components/1/annotations + value: HACK_REMOVE_ME{{- toYaml .Values.pipelinegateway.annotations | nindent 8 }} diff --git a/k8s/kustomize/helm-components-sc/patch_scheduler_json6902.yaml b/k8s/kustomize/helm-components-sc/patch_scheduler_json6902.yaml index 81f76824ff..df9a52f014 100644 --- a/k8s/kustomize/helm-components-sc/patch_scheduler_json6902.yaml +++ b/k8s/kustomize/helm-components-sc/patch_scheduler_json6902.yaml @@ -1,3 +1,9 @@ - op: replace path: /spec/components/0/podSpec/securityContext value: HACK_REMOVE_ME{{- toYaml .Values.scheduler.securityContext | nindent 8 }} +- op: replace + path: /spec/components/0/labels + value: HACK_REMOVE_ME{{- toYaml .Values.scheduler.labels | nindent 8 }} +- op: replace + path: /spec/components/0/annotations + value: HACK_REMOVE_ME{{- toYaml .Values.scheduler.annotations | nindent 8 }} diff --git a/k8s/kustomize/helm-servers/patch_mlserver.yaml b/k8s/kustomize/helm-servers/patch_mlserver.yaml index 2dfe18f6a8..7729695312 100644 --- a/k8s/kustomize/helm-servers/patch_mlserver.yaml +++ b/k8s/kustomize/helm-servers/patch_mlserver.yaml @@ -2,6 +2,8 @@ apiVersion: mlops.seldon.io/v1alpha1 kind: Server metadata: name: mlserver + labelz: HACK_REMOVE_ME{{ .Values.mlserver.labels | toYaml }} + annotationz: HACK_REMOVE_ME{{ .Values.mlserver.annotations | toYaml }} spec: replicas: HACK_REMOVE_ME{{ .Values.mlserver.replicas }} podSpec: HACK_REMOVE_ME{{ toJson .Values.mlserver.podSpec }} diff --git a/k8s/kustomize/helm-servers/patch_triton.yaml b/k8s/kustomize/helm-servers/patch_triton.yaml index 6b461c387b..f121adc785 100644 --- a/k8s/kustomize/helm-servers/patch_triton.yaml +++ b/k8s/kustomize/helm-servers/patch_triton.yaml @@ -2,6 +2,8 @@ apiVersion: mlops.seldon.io/v1alpha1 kind: Server metadata: name: triton + labelz: HACK_REMOVE_ME{{ .Values.triton.labels | toYaml }} + annotationz: HACK_REMOVE_ME{{ .Values.triton.annotations | toYaml }} spec: replicas: HACK_REMOVE_ME{{ .Values.triton.replicas }} podSpec: HACK_REMOVE_ME{{ toJson .Values.triton.podSpec }} diff --git a/k8s/yaml/components.yaml b/k8s/yaml/components.yaml index 9292bc7bb0..fb9906b90f 100644 --- a/k8s/yaml/components.yaml +++ b/k8s/yaml/components.yaml @@ -538,7 +538,11 @@ metadata: name: default spec: components: - - name: seldon-scheduler + - annotations: + null + labels: + null + name: seldon-scheduler podSpec: containers: - args: @@ -651,7 +655,11 @@ spec: resources: requests: storage: '1Gi' - - name: seldon-pipelinegateway + - annotations: + null + labels: + null + name: seldon-pipelinegateway podSpec: containers: - args: @@ -781,7 +789,11 @@ spec: name: seldon-tracing name: tracing-config-volume replicas: 1 - - name: seldon-modelgateway + - annotations: + null + labels: + null + name: seldon-modelgateway podSpec: containers: - args: @@ -894,7 +906,11 @@ spec: name: seldon-tracing name: tracing-config-volume replicas: 1 - - name: hodometer + - annotations: + null + labels: + null + name: hodometer podSpec: containers: - env: @@ -947,7 +963,13 @@ spec: serviceAccountName: hodometer terminationGracePeriodSeconds: 5 replicas: 1 - - name: seldon-envoy + - annotations: + prometheus.io/path: /stats/prometheus + prometheus.io/port: "9003" + prometheus.io/scrape: "true" + labels: + null + name: seldon-envoy podSpec: containers: - env: @@ -999,7 +1021,11 @@ spec: runAsUser: 1000 terminationGracePeriodSeconds: 5 replicas: 1 - - name: seldon-dataflow-engine + - annotations: + null + labels: + null + name: seldon-dataflow-engine podSpec: containers: - env: diff --git a/k8s/yaml/crds.yaml b/k8s/yaml/crds.yaml index 7e6c5c236e..8357c0ca0c 100644 --- a/k8s/yaml/crds.yaml +++ b/k8s/yaml/crds.yaml @@ -646,6 +646,14 @@ spec: components: items: properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object name: type: string podSpec: diff --git a/k8s/yaml/servers.yaml b/k8s/yaml/servers.yaml index b33488932c..6ca39bd0f5 100644 --- a/k8s/yaml/servers.yaml +++ b/k8s/yaml/servers.yaml @@ -3,6 +3,10 @@ apiVersion: mlops.seldon.io/v1alpha1 kind: Server metadata: + annotations: + null + labels: + null name: mlserver spec: podSpec: null @@ -13,6 +17,10 @@ spec: apiVersion: mlops.seldon.io/v1alpha1 kind: Server metadata: + annotations: + null + labels: + null name: triton spec: podSpec: null diff --git a/operator/apis/mlops/v1alpha1/seldonconfig_types.go b/operator/apis/mlops/v1alpha1/seldonconfig_types.go index 4ecec7b626..d2b77515a1 100644 --- a/operator/apis/mlops/v1alpha1/seldonconfig_types.go +++ b/operator/apis/mlops/v1alpha1/seldonconfig_types.go @@ -80,7 +80,10 @@ type TracingConfig struct { type ComponentDefn struct { // +kubebuilder:validation:Required + Name string `json:"name"` + Labels map[string]string `json:"labels,omitempty"` + Annotations map[string]string `json:"annotations,omitempty"` Replicas *int32 `json:"replicas,omitempty"` PodSpec *v1.PodSpec `json:"podSpec,omitempty"` VolumeClaimTemplates []PersistentVolumeClaim `json:"volumeClaimTemplates,omitempty"` diff --git a/operator/apis/mlops/v1alpha1/zz_generated.deepcopy.go b/operator/apis/mlops/v1alpha1/zz_generated.deepcopy.go index 70c5b1bdd6..a57f1f6cba 100644 --- a/operator/apis/mlops/v1alpha1/zz_generated.deepcopy.go +++ b/operator/apis/mlops/v1alpha1/zz_generated.deepcopy.go @@ -39,6 +39,20 @@ func (in *AgentConfiguration) DeepCopy() *AgentConfiguration { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ComponentDefn) DeepCopyInto(out *ComponentDefn) { *out = *in + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } if in.Replicas != nil { in, out := &in.Replicas, &out.Replicas *out = new(int32) diff --git a/operator/config/crd/bases/mlops.seldon.io_seldonconfigs.yaml b/operator/config/crd/bases/mlops.seldon.io_seldonconfigs.yaml index d03cc9257c..bb06101931 100644 --- a/operator/config/crd/bases/mlops.seldon.io_seldonconfigs.yaml +++ b/operator/config/crd/bases/mlops.seldon.io_seldonconfigs.yaml @@ -38,6 +38,14 @@ spec: components: items: properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object name: type: string podSpec: diff --git a/operator/config/seldonconfigs/default.yaml b/operator/config/seldonconfigs/default.yaml index 73b5cfb24d..7fc4c64721 100644 --- a/operator/config/seldonconfigs/default.yaml +++ b/operator/config/seldonconfigs/default.yaml @@ -45,6 +45,10 @@ spec: terminationGracePeriodSeconds: 5 - name: seldon-envoy replicas: 1 + annotations: + "prometheus.io/path": "/stats/prometheus" + "prometheus.io/port": "9003" + "prometheus.io/scrape": "true" podSpec: containers: - image: seldonio/seldon-envoy:latest diff --git a/operator/controllers/reconcilers/seldon/deployment_reconciler.go b/operator/controllers/reconcilers/seldon/deployment_reconciler.go index 5cce767ec3..3fe708a2dc 100644 --- a/operator/controllers/reconcilers/seldon/deployment_reconciler.go +++ b/operator/controllers/reconcilers/seldon/deployment_reconciler.go @@ -40,12 +40,16 @@ func NewComponentDeploymentReconciler( common common.ReconcilerConfig, meta metav1.ObjectMeta, podSpec *v1.PodSpec, + componentLabels map[string]string, + componentAnnotations map[string]string, override *mlopsv1alpha1.OverrideSpec, seldonConfigMeta metav1.ObjectMeta, annotator *patch.Annotator, ) (*ComponentDeploymentReconciler, error) { labels := utils.MergeMaps(meta.Labels, seldonConfigMeta.Labels) + labels = utils.MergeMaps(componentLabels, labels) annotations := utils.MergeMaps(meta.Annotations, seldonConfigMeta.Annotations) + annotations = utils.MergeMaps(componentAnnotations, annotations) deployment, err := toDeployment(name, meta, podSpec, override, labels, annotations) if err != nil { return nil, err @@ -62,20 +66,14 @@ func (s *ComponentDeploymentReconciler) GetResources() []client.Object { return []client.Object{s.Deployment} } -func addEnvoyAnnotations(annotations map[string]string) map[string]string { - annotations["prometheus.io/path"] = "/stats/prometheus" - annotations["prometheus.io/scrape"] = "true" - annotations["prometheus.io/port"] = "9003" - return annotations -} - func toDeployment( name string, meta metav1.ObjectMeta, podSpec *v1.PodSpec, override *mlopsv1alpha1.OverrideSpec, labels map[string]string, - annotations map[string]string) (*appsv1.Deployment, error) { + annotations map[string]string, +) (*appsv1.Deployment, error) { var replicas int32 if override != nil && override.Replicas != nil { replicas = *override.Replicas @@ -90,10 +88,6 @@ func toDeployment( return nil, err } } - // Envoy annotations - if name == mlopsv1alpha1.EnvoyName { - annotations = addEnvoyAnnotations(annotations) - } metaLabels := utils.MergeMaps(map[string]string{constants.KubernetesNameLabelKey: name}, labels) templateLabels := utils.MergeMaps(map[string]string{constants.KubernetesNameLabelKey: name}, labels) d := &appsv1.Deployment{ diff --git a/operator/controllers/reconcilers/seldon/deployment_reconciler_test.go b/operator/controllers/reconcilers/seldon/deployment_reconciler_test.go index 7311b62c9c..2631a40da4 100644 --- a/operator/controllers/reconcilers/seldon/deployment_reconciler_test.go +++ b/operator/controllers/reconcilers/seldon/deployment_reconciler_test.go @@ -124,6 +124,8 @@ func TestDeploymentReconcile(t *testing.T) { common.ReconcilerConfig{Ctx: context.TODO(), Logger: logger, Client: client}, meta, test.podSpec, + make(map[string]string), + make(map[string]string), test.override, test.seldonConfigMeta, annotator) @@ -225,6 +227,8 @@ func TestDeploymentOverride(t *testing.T) { common.ReconcilerConfig{Ctx: context.TODO(), Logger: logger, Client: client}, meta, test.podSpec, + make(map[string]string), + make(map[string]string), test.override, metav1.ObjectMeta{}, annotator) diff --git a/operator/controllers/reconcilers/seldon/runtime_reconciler.go b/operator/controllers/reconcilers/seldon/runtime_reconciler.go index f64b8a6871..89cf2acf1f 100644 --- a/operator/controllers/reconcilers/seldon/runtime_reconciler.go +++ b/operator/controllers/reconcilers/seldon/runtime_reconciler.go @@ -29,8 +29,8 @@ type SeldonRuntimeReconciler struct { func NewSeldonRuntimeReconciler( runtime *mlopsv1alpha1.SeldonRuntime, - commonConfig common.ReconcilerConfig) (common.Reconciler, error) { - + commonConfig common.ReconcilerConfig, +) (common.Reconciler, error) { var err error seldonConfig, err := mlopsv1alpha1.GetSeldonConfigForSeldonRuntime(runtime.Spec.SeldonConfig, commonConfig.Client) @@ -58,6 +58,8 @@ func NewSeldonRuntimeReconciler( runtime.ObjectMeta, c.PodSpec, c.VolumeClaimTemplates, + c.Labels, + c.Annotations, overrides[c.Name], seldonConfig.ObjectMeta, annotator, @@ -73,6 +75,8 @@ func NewSeldonRuntimeReconciler( commonConfig, runtime.ObjectMeta, c.PodSpec, + c.Labels, + c.Annotations, overrides[c.Name], seldonConfig.ObjectMeta, annotator, diff --git a/operator/controllers/reconcilers/seldon/runtime_reconciler_test.go b/operator/controllers/reconcilers/seldon/runtime_reconciler_test.go index d89bf8fced..55d246bfb1 100644 --- a/operator/controllers/reconcilers/seldon/runtime_reconciler_test.go +++ b/operator/controllers/reconcilers/seldon/runtime_reconciler_test.go @@ -53,7 +53,9 @@ func TestRuntimeReconcile(t *testing.T) { Spec: mlopsv1alpha1.SeldonConfigSpec{ Components: []*mlopsv1alpha1.ComponentDefn{ { - Name: mlopsv1alpha1.SchedulerName, + Name: mlopsv1alpha1.SchedulerName, + Labels: map[string]string{"test-label": "123"}, + Annotations: map[string]string{"test-annotation": "456"}, PodSpec: &v1.PodSpec{ Containers: []v1.Container{ { @@ -94,7 +96,9 @@ func TestRuntimeReconcile(t *testing.T) { Spec: mlopsv1alpha1.SeldonConfigSpec{ Components: []*mlopsv1alpha1.ComponentDefn{ { - Name: mlopsv1alpha1.PipelineGatewayName, + Name: mlopsv1alpha1.PipelineGatewayName, + Labels: map[string]string{"test-label": "123"}, + Annotations: map[string]string{"test-annotation": "456"}, PodSpec: &v1.PodSpec{ Containers: []v1.Container{ { @@ -105,7 +109,9 @@ func TestRuntimeReconcile(t *testing.T) { }, }, { - Name: mlopsv1alpha1.ModelGatewayName, + Name: mlopsv1alpha1.ModelGatewayName, + Labels: map[string]string{"test-label": "123"}, + Annotations: map[string]string{"test-annotation": "456"}, PodSpec: &v1.PodSpec{ Containers: []v1.Container{ { @@ -172,6 +178,8 @@ func TestRuntimeReconcile(t *testing.T) { Namespace: test.runtime.GetNamespace(), }, ss) g.Expect(err).To(BeNil()) + g.Expect(ss.Labels["test-label"]).To(Equal("123")) + g.Expect(ss.Annotations["test-annotation"]).To(Equal("456")) } for _, depName := range test.expectedDeployments { dep := &appsv1.Deployment{} @@ -180,6 +188,8 @@ func TestRuntimeReconcile(t *testing.T) { Namespace: test.runtime.GetNamespace(), }, dep) g.Expect(err).To(BeNil()) + g.Expect(dep.Labels["test-label"]).To(Equal("123")) + g.Expect(dep.Annotations["test-annotation"]).To(Equal("456")) } } }) diff --git a/operator/controllers/reconcilers/seldon/statefulset_reconciler.go b/operator/controllers/reconcilers/seldon/statefulset_reconciler.go index a4b0af56b8..20722f14ea 100644 --- a/operator/controllers/reconcilers/seldon/statefulset_reconciler.go +++ b/operator/controllers/reconcilers/seldon/statefulset_reconciler.go @@ -46,12 +46,16 @@ func NewComponentStatefulSetReconciler( meta metav1.ObjectMeta, podSpec *v1.PodSpec, volumeClaimTemplates []mlopsv1alpha1.PersistentVolumeClaim, + componentLabels map[string]string, + componentAnnotations map[string]string, override *mlopsv1alpha1.OverrideSpec, seldonConfigMeta metav1.ObjectMeta, annotator *patch.Annotator, ) (*ComponentStatefulSetReconciler, error) { labels := utils.MergeMaps(meta.Labels, seldonConfigMeta.Labels) + labels = utils.MergeMaps(componentLabels, labels) annotations := utils.MergeMaps(meta.Annotations, seldonConfigMeta.Annotations) + annotations = utils.MergeMaps(componentAnnotations, annotations) statefulSet, err := toStatefulSet(name, meta, podSpec, volumeClaimTemplates, override, labels, annotations) if err != nil { return nil, err diff --git a/operator/controllers/reconcilers/seldon/statefulset_reconciler_test.go b/operator/controllers/reconcilers/seldon/statefulset_reconciler_test.go index f1ab052df8..d821eec722 100644 --- a/operator/controllers/reconcilers/seldon/statefulset_reconciler_test.go +++ b/operator/controllers/reconcilers/seldon/statefulset_reconciler_test.go @@ -34,13 +34,15 @@ func TestStatefulSetReconcile(t *testing.T) { g := NewGomegaWithT(t) type test struct { - name string - statefulSetName string - podSpec *v1.PodSpec - volumeClaims []mlopsv1alpha1.PersistentVolumeClaim - override *mlopsv1alpha1.OverrideSpec - seldonConfigMeta metav1.ObjectMeta - error bool + name string + statefulSetName string + podSpec *v1.PodSpec + volumeClaims []mlopsv1alpha1.PersistentVolumeClaim + componentLabels map[string]string + componentAnnotations map[string]string + override *mlopsv1alpha1.OverrideSpec + seldonConfigMeta metav1.ObjectMeta + error bool } tests := []test{ @@ -78,6 +80,8 @@ func TestStatefulSetReconcile(t *testing.T) { meta, test.podSpec, test.volumeClaims, + test.componentLabels, + test.componentAnnotations, test.override, test.seldonConfigMeta, annotator)