feat(charts): adding metadata object to a seldonconfig component (#5918)

* adding a metadata object to seldonconfig components * adding labels to the servers helm charts * add metadata to seldonconfig.component * using sed to template the templates * add makefile changes * patching each component with labels and annotations * add a test * missing forward slash * hack labelz * remove test labels * add annotationz to servers * add prometheus annotations to the default seldonconfig * add a comment and a test
SeldonIO · Sep 25, 2024 · 41ae0e3 · 41ae0e3
1 parent 1bd8d0f
commit 41ae0e3
Show file tree

Hide file tree

Showing 27 changed files with 212 additions and 38 deletions.
diff --git a/k8s/Makefile b/k8s/Makefile
@@ -18,8 +18,10 @@ create-helm-charts:
 	kustomize build kustomize/helm-servers/ > ${HELM_SERVERS_BASE}/seldon-v2-servers.yaml
 	sed -i 's/\(.*\)\(imagePullSecrets:\).*/{{- with .Values.imagePullSecrets }}\n\1\2\n\1{{- toYaml . | nindent 8 }}\n{{- end }}/g' ${HELM_COMPONENTS_BASE}/seldon-v2-components.yaml
 	sed -i 's/HACK_REMOVE_ME//' ${HELM_COMPONENTS_BASE}/seldon-v2-components.yaml
-	sed 's/HACK_REMOVE_ME//' ${HELM_SERVERS_BASE}/seldon-v2-servers.yaml \
-		> ${HELM_SERVERS_BASE}/.seldon-v2-servers.yaml
+	# kustomizse has builtin helper fuctions for meta.labels and metadata.annotations, so the field names are changed to escape kustomize's extra processing
+	sed 's/labelz:/labels:\n\ \ \ /' ${HELM_SERVERS_BASE}/seldon-v2-servers.yaml > ${HELM_SERVERS_BASE}/.seldon-v2-servers.yaml
+	sed -i 's/annotationz:/annotations:\n\ \ \ /' ${HELM_SERVERS_BASE}/.seldon-v2-servers.yaml
+	sed -i 's/HACK_REMOVE_ME//' ${HELM_SERVERS_BASE}/.seldon-v2-servers.yaml
 	sed -zi 's#\(apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRole.*\- get\n\-\-\-\)#{{- if .Values.controller.clusterwide -}}\n\1\n{{- end }}#' ${HELM_COMPONENTS_BASE}/seldon-v2-components.yaml
 	sed -zi 's#\(apiVersion: rbac.authorization.k8s.io/v1\nkind: ClusterRoleBinding.*ServiceAccount\n  name: seldon-v2-controller-manager\n  namespace:[^-]\{27\}\n\-\-\-\)#{{- if .Values.controller.clusterwide -}}\n\1\n{{- end }}#' ${HELM_COMPONENTS_BASE}/seldon-v2-components.yaml
 	mv ${HELM_SERVERS_BASE}/.seldon-v2-servers.yaml ${HELM_SERVERS_BASE}/seldon-v2-servers.yaml

diff --git a/k8s/helm-charts/seldon-core-v2-crds/templates/seldon-v2-crds.yaml b/k8s/helm-charts/seldon-core-v2-crds/templates/seldon-v2-crds.yaml
@@ -641,6 +641,14 @@ spec:
               components:
                 items:
                   properties:
+                    annotations:
+                      additionalProperties:
+                        type: string
+                      type: object
+                    labels:
+                      additionalProperties:
+                        type: string
+                      type: object
                     name:
                       type: string
                     podSpec:

diff --git a/k8s/helm-charts/seldon-core-v2-servers/templates/seldon-v2-servers.yaml b/k8s/helm-charts/seldon-core-v2-servers/templates/seldon-v2-servers.yaml
@@ -1,6 +1,10 @@
 apiVersion: mlops.seldon.io/v1alpha1
 kind: Server
 metadata:
+  annotations:
+    {{ .Values.mlserver.annotations | toYaml }}
+  labels:
+    {{ .Values.mlserver.labels | toYaml }}
   name: mlserver
   namespace: '{{ .Release.Namespace }}'
 spec:
@@ -11,6 +15,10 @@ spec:
 apiVersion: mlops.seldon.io/v1alpha1
 kind: Server
 metadata:
+  annotations:
+    {{ .Values.triton.annotations | toYaml }}
+  labels:
+    {{ .Values.triton.labels | toYaml }}
   name: triton
   namespace: '{{ .Release.Namespace }}'
 spec:

diff --git a/k8s/helm-charts/seldon-core-v2-setup/templates/seldon-v2-components.yaml b/k8s/helm-charts/seldon-core-v2-setup/templates/seldon-v2-components.yaml
@@ -894,7 +894,10 @@ metadata:
   namespace: '{{ .Release.Namespace }}'
 spec:
   components:
-  - name: seldon-scheduler
+  - annotations: {{- toYaml .Values.scheduler.annotations | nindent
+      8 }}
+    labels: {{- toYaml .Values.scheduler.labels | nindent 8 }}
+    name: seldon-scheduler
     podSpec:
       containers:
       - args:
@@ -1012,7 +1015,10 @@ spec:
         resources:
           requests:
             storage: '{{ .Values.scheduler.resources.storage }}'
-  - name: seldon-pipelinegateway
+  - annotations: {{- toYaml .Values.pipelinegateway.annotations | nindent
+      8 }}
+    labels: {{- toYaml .Values.pipelinegateway.labels | nindent 8 }}
+    name: seldon-pipelinegateway
     podSpec:
       containers:
       - args:
@@ -1148,7 +1154,10 @@ spec:
           name: seldon-tracing
         name: tracing-config-volume
     replicas: 1
-  - name: seldon-modelgateway
+  - annotations: {{- toYaml .Values.modelgateway.annotations | nindent
+      8 }}
+    labels: {{- toYaml .Values.modelgateway.labels | nindent 8 }}
+    name: seldon-modelgateway
     podSpec:
       containers:
       - args:
@@ -1266,7 +1275,10 @@ spec:
           name: seldon-tracing
         name: tracing-config-volume
     replicas: 1
-  - name: hodometer
+  - annotations: {{- toYaml .Values.hodometer.annotations | nindent
+      8 }}
+    labels: {{- toYaml .Values.hodometer.labels | nindent 8 }}
+    name: hodometer
     podSpec:
       containers:
       - env:
@@ -1322,7 +1334,9 @@ spec:
       serviceAccountName: hodometer
       terminationGracePeriodSeconds: 5
     replicas: 1
-  - name: seldon-envoy
+  - annotations: {{- toYaml .Values.envoy.annotations | nindent 8 }}
+    labels: {{- toYaml .Values.envoy.labels | nindent 8 }}
+    name: seldon-envoy
     podSpec:
       containers:
       - env:
@@ -1377,7 +1391,10 @@ spec:
         8 }}
       terminationGracePeriodSeconds: 5
     replicas: 1
-  - name: seldon-dataflow-engine
+  - annotations: {{- toYaml .Values.dataflow.annotations | nindent 8
+      }}
+    labels: {{- toYaml .Values.dataflow.labels | nindent 8 }}
+    name: seldon-dataflow-engine
     podSpec:
       containers:
       - env:

diff --git a/k8s/helm-charts/seldon-core-v2-setup/values.yaml b/k8s/helm-charts/seldon-core-v2-setup/values.yaml
@@ -180,6 +180,10 @@ controller:
     runAsNonRoot: true
 
 envoy:
+  annotations:
+    "prometheus.io/path": "/stats/prometheus"
+    "prometheus.io/port": "9003"
+    "prometheus.io/scrape": "true"
   image:
     pullPolicy: IfNotPresent
     registry: docker.io

diff --git a/k8s/helm-charts/seldon-core-v2-setup/values.yaml.template b/k8s/helm-charts/seldon-core-v2-setup/values.yaml.template
@@ -180,6 +180,10 @@ controller:
     runAsNonRoot: true
 
 envoy:
+  annotations:
+    "prometheus.io/path": "/stats/prometheus"
+    "prometheus.io/port": "9003"
+    "prometheus.io/scrape": "true"
   image:
     pullPolicy: IfNotPresent
     registry: docker.io

diff --git a/k8s/kustomize/helm-components-sc/patch_dataflow_json6902.yaml b/k8s/kustomize/helm-components-sc/patch_dataflow_json6902.yaml
@@ -1,3 +1,9 @@
 - op: replace
   path: /spec/components/5/podSpec/securityContext
   value: HACK_REMOVE_ME{{- toYaml .Values.dataflow.securityContext | nindent 8 }}
+- op: replace
+  path: /spec/components/5/labels
+  value: HACK_REMOVE_ME{{- toYaml .Values.dataflow.labels | nindent 8 }}
+- op: replace
+  path: /spec/components/5/annotations
+  value: HACK_REMOVE_ME{{- toYaml .Values.dataflow.annotations | nindent 8 }}
diff --git a/k8s/kustomize/helm-components-sc/patch_envoy_json6902.yaml b/k8s/kustomize/helm-components-sc/patch_envoy_json6902.yaml
@@ -2,3 +2,9 @@
 - op: replace
   path: /spec/components/4/podSpec/securityContext
   value: HACK_REMOVE_ME{{- toYaml .Values.envoy.securityContext | nindent 8 }}
+- op: replace
+  path: /spec/components/4/labels
+  value: HACK_REMOVE_ME{{- toYaml .Values.envoy.labels | nindent 8 }}
+- op: replace
+  path: /spec/components/4/annotations
+  value: HACK_REMOVE_ME{{- toYaml .Values.envoy.annotations | nindent 8 }}
diff --git a/k8s/kustomize/helm-components-sc/patch_hodometer_json6902.yaml b/k8s/kustomize/helm-components-sc/patch_hodometer_json6902.yaml
@@ -2,3 +2,9 @@
 - op: replace
   path: /spec/components/3/podSpec/securityContext  
   value: HACK_REMOVE_ME{{- toYaml .Values.hodometer.securityContext | nindent 8 }}
+- op: replace
+  path: /spec/components/3/labels
+  value: HACK_REMOVE_ME{{- toYaml .Values.hodometer.labels | nindent 8 }}
+- op: replace
+  path: /spec/components/3/annotations
+  value: HACK_REMOVE_ME{{- toYaml .Values.hodometer.annotations | nindent 8 }}
diff --git a/k8s/kustomize/helm-components-sc/patch_modelgateway_json6902.yaml b/k8s/kustomize/helm-components-sc/patch_modelgateway_json6902.yaml
@@ -1,3 +1,9 @@
 - op: replace
   path: /spec/components/2/podSpec/securityContext
   value: HACK_REMOVE_ME{{- toYaml .Values.modelgateway.securityContext | nindent 8 }}
+- op: replace
+  path: /spec/components/2/labels
+  value: HACK_REMOVE_ME{{- toYaml .Values.modelgateway.labels | nindent 8 }}
+- op: replace
+  path: /spec/components/2/annotations
+  value: HACK_REMOVE_ME{{- toYaml .Values.modelgateway.annotations | nindent 8 }}
diff --git a/k8s/kustomize/helm-components-sc/patch_pipelinegateway_json6902.yaml b/k8s/kustomize/helm-components-sc/patch_pipelinegateway_json6902.yaml
@@ -1,3 +1,9 @@
 - op: replace
   path: /spec/components/1/podSpec/securityContext  
   value: HACK_REMOVE_ME{{- toYaml .Values.pipelinegateway.securityContext | nindent 8 }}
+- op: replace
+  path: /spec/components/1/labels
+  value: HACK_REMOVE_ME{{- toYaml .Values.pipelinegateway.labels | nindent 8 }}
+- op: replace
+  path: /spec/components/1/annotations
+  value: HACK_REMOVE_ME{{- toYaml .Values.pipelinegateway.annotations | nindent 8 }}
diff --git a/k8s/kustomize/helm-components-sc/patch_scheduler_json6902.yaml b/k8s/kustomize/helm-components-sc/patch_scheduler_json6902.yaml
@@ -1,3 +1,9 @@
 - op: replace
   path: /spec/components/0/podSpec/securityContext
   value: HACK_REMOVE_ME{{- toYaml .Values.scheduler.securityContext | nindent 8 }}
+- op: replace
+  path: /spec/components/0/labels
+  value: HACK_REMOVE_ME{{- toYaml .Values.scheduler.labels | nindent 8 }}
+- op: replace
+  path: /spec/components/0/annotations
+  value: HACK_REMOVE_ME{{- toYaml .Values.scheduler.annotations | nindent 8 }}
diff --git a/k8s/kustomize/helm-servers/patch_mlserver.yaml b/k8s/kustomize/helm-servers/patch_mlserver.yaml
@@ -2,6 +2,8 @@ apiVersion: mlops.seldon.io/v1alpha1
 kind: Server
 metadata:
   name: mlserver
+  labelz: HACK_REMOVE_ME{{ .Values.mlserver.labels | toYaml }}
+  annotationz: HACK_REMOVE_ME{{ .Values.mlserver.annotations | toYaml }}
 spec:
   replicas: HACK_REMOVE_ME{{ .Values.mlserver.replicas }}
   podSpec: HACK_REMOVE_ME{{ toJson .Values.mlserver.podSpec }}
diff --git a/k8s/kustomize/helm-servers/patch_triton.yaml b/k8s/kustomize/helm-servers/patch_triton.yaml
@@ -2,6 +2,8 @@ apiVersion: mlops.seldon.io/v1alpha1
 kind: Server
 metadata:
   name: triton
+  labelz: HACK_REMOVE_ME{{ .Values.triton.labels | toYaml }}
+  annotationz: HACK_REMOVE_ME{{ .Values.triton.annotations | toYaml }}
 spec:
   replicas: HACK_REMOVE_ME{{ .Values.triton.replicas }}
   podSpec: HACK_REMOVE_ME{{ toJson .Values.triton.podSpec }}
diff --git a/k8s/yaml/components.yaml b/k8s/yaml/components.yaml
@@ -538,7 +538,11 @@ metadata:
   name: default
 spec:
   components:
-  - name: seldon-scheduler
+  - annotations:
+        null
+    labels:
+        null
+    name: seldon-scheduler
     podSpec:
       containers:
       - args:
@@ -651,7 +655,11 @@ spec:
         resources:
           requests:
             storage: '1Gi'
-  - name: seldon-pipelinegateway
+  - annotations:
+        null
+    labels:
+        null
+    name: seldon-pipelinegateway
     podSpec:
       containers:
       - args:
@@ -781,7 +789,11 @@ spec:
           name: seldon-tracing
         name: tracing-config-volume
     replicas: 1
-  - name: seldon-modelgateway
+  - annotations:
+        null
+    labels:
+        null
+    name: seldon-modelgateway
     podSpec:
       containers:
       - args:
@@ -894,7 +906,11 @@ spec:
           name: seldon-tracing
         name: tracing-config-volume
     replicas: 1
-  - name: hodometer
+  - annotations:
+        null
+    labels:
+        null
+    name: hodometer
     podSpec:
       containers:
       - env:
@@ -947,7 +963,13 @@ spec:
       serviceAccountName: hodometer
       terminationGracePeriodSeconds: 5
     replicas: 1
-  - name: seldon-envoy
+  - annotations:
+        prometheus.io/path: /stats/prometheus
+        prometheus.io/port: "9003"
+        prometheus.io/scrape: "true"
+    labels:
+        null
+    name: seldon-envoy
     podSpec:
       containers:
       - env:
@@ -999,7 +1021,11 @@ spec:
         runAsUser: 1000
       terminationGracePeriodSeconds: 5
     replicas: 1
-  - name: seldon-dataflow-engine
+  - annotations:
+        null
+    labels:
+        null
+    name: seldon-dataflow-engine
     podSpec:
       containers:
       - env:

diff --git a/k8s/yaml/crds.yaml b/k8s/yaml/crds.yaml
@@ -646,6 +646,14 @@ spec:
               components:
                 items:
                   properties:
+                    annotations:
+                      additionalProperties:
+                        type: string
+                      type: object
+                    labels:
+                      additionalProperties:
+                        type: string
+                      type: object
                     name:
                       type: string
                     podSpec:

diff --git a/k8s/yaml/servers.yaml b/k8s/yaml/servers.yaml
@@ -3,6 +3,10 @@
 apiVersion: mlops.seldon.io/v1alpha1
 kind: Server
 metadata:
+  annotations:
+    null
+  labels:
+    null
   name: mlserver
 spec:
   podSpec: null
@@ -13,6 +17,10 @@ spec:
 apiVersion: mlops.seldon.io/v1alpha1
 kind: Server
 metadata:
+  annotations:
+    null
+  labels:
+    null
   name: triton
 spec:
   podSpec: null

diff --git a/operator/apis/mlops/v1alpha1/seldonconfig_types.go b/operator/apis/mlops/v1alpha1/seldonconfig_types.go
@@ -80,7 +80,10 @@ type TracingConfig struct {
 
 type ComponentDefn struct {
 	// +kubebuilder:validation:Required
+
 	Name                 string                  `json:"name"`
+	Labels               map[string]string       `json:"labels,omitempty"`
+	Annotations          map[string]string       `json:"annotations,omitempty"`
 	Replicas             *int32                  `json:"replicas,omitempty"`
 	PodSpec              *v1.PodSpec             `json:"podSpec,omitempty"`
 	VolumeClaimTemplates []PersistentVolumeClaim `json:"volumeClaimTemplates,omitempty"`

diff --git a/operator/apis/mlops/v1alpha1/zz_generated.deepcopy.go b/operator/apis/mlops/v1alpha1/zz_generated.deepcopy.go
diff --git a/operator/config/crd/bases/mlops.seldon.io_seldonconfigs.yaml b/operator/config/crd/bases/mlops.seldon.io_seldonconfigs.yaml
@@ -38,6 +38,14 @@ spec:
               components:
                 items:
                   properties:
+                    annotations:
+                      additionalProperties:
+                        type: string
+                      type: object
+                    labels:
+                      additionalProperties:
+                        type: string
+                      type: object
                     name:
                       type: string
                     podSpec:

diff --git a/operator/config/seldonconfigs/default.yaml b/operator/config/seldonconfigs/default.yaml
@@ -45,6 +45,10 @@ spec:
       terminationGracePeriodSeconds: 5
   - name: seldon-envoy
     replicas: 1
+    annotations:
+        "prometheus.io/path": "/stats/prometheus"
+        "prometheus.io/port": "9003"
+        "prometheus.io/scrape": "true"
     podSpec:
       containers:
       - image: seldonio/seldon-envoy:latest