skipper: enable manual canary deployment

requires dict function zalando-incubator/cluster-lifecycle-manager#711 Signed-off-by: Alexander Yastrebov <[email protected]>
zalando-incubator · Nov 29, 2023 · 56b32e1 · 56b32e1
1 parent 7391b34
commit 56b32e1
Show file tree

Hide file tree

Showing 3 changed files with 59 additions and 18 deletions.
diff --git a/cluster/config-defaults.yaml b/cluster/config-defaults.yaml
@@ -70,6 +70,13 @@ skipper_ingress_min_replicas: "2"
 {{end}}
 skipper_ingress_cpu: "1000m"
 skipper_ingress_memory: "1500Mi"
+
+skipper_ingress_canary_internal_version: ""
+
+# optional canary arguments separated by "[cf724afc]" to allow whitespaces, e.g.:
+# skipper_ingress_canary_args: "-foo=has a whitespace[cf724afc]-baz=qux"
+skipper_ingress_canary_args: ""
+
 # When set to true (and dedicated node pool for skipper is also true) the
 # daemonset overhead will be subtracted from the cpu settings such
 # that skipper will perfectly fit on the node.

diff --git a/cluster/manifests/deletions.yaml b/cluster/manifests/deletions.yaml
@@ -59,6 +59,12 @@ post_apply:
   kind: ClusterRoleBinding
 {{- end }}
 
+{{ if not .Cluster.ConfigItems.skipper_ingress_canary_internal_version }}
+- name: skipper-ingress-canary
+  namespace: kube-system
+  kind: Deployment
+{{ end }}
+
 {{ if eq .Cluster.ConfigItems.skipper_routesrv_enabled "false" }}
 - name: skipper-ingress-routesrv
   namespace: kube-system

diff --git a/cluster/manifests/skipper/deployment.yaml b/cluster/manifests/skipper/deployment.yaml
@@ -1,35 +1,57 @@
 {{ $internal_version := "v0.18.42-691" }}
 {{ $version := index (split $internal_version "-") 0 }}
 
+{{ template "skipper-ingress" dict
+  "ConfigItems" .ConfigItems
+  "name" "skipper-ingress"
+  "internal_version" $internal_version
+}}
+
+{{ if .ConfigItems.skipper_ingress_canary_internal_version }}
+{{ template "skipper-ingress" dict
+  "ConfigItems" .ConfigItems
+  "name" "skipper-ingress-canary"
+  "internal_version" .ConfigItems.skipper_ingress_canary_internal_version
+  "replicas" 1
+  "args" (index .ConfigItems "skipper_ingress_canary_args")
+}}
+{{ end }}
+
+{{ define "skipper-ingress" }}
+{{ $version := index (split .internal_version "-") 0 }}
+---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: skipper-ingress
+  name: "{{ .name }}"
   namespace: kube-system
   labels:
     application: skipper-ingress
     version: {{ $version }}
     component: ingress
 spec:
+{{ if index . "replicas" }}
+  replicas: {{ .replicas }}
+{{ end }}
   strategy:
     rollingUpdate:
       maxSurge: 1
       maxUnavailable: 0
   selector:
     matchLabels:
-      deployment: skipper-ingress
+      deployment: "{{ .name }}"
   template:
     metadata:
       labels:
-        deployment: skipper-ingress
+        deployment: "{{ .name }}"
         application: skipper-ingress
         version: {{ $version }}
         component: ingress
       annotations:
         kubernetes-log-watcher/scalyr-parser: |
           [{"container": "skipper-ingress", "parser": "skipper-access-log"}]
         config/hash: {{"secret.yaml" | manifestHash}}
-        logging/destination: "{{.Cluster.ConfigItems.log_destination_local}}"
+        logging/destination: "{{ .ConfigItems.log_destination_local }}"
         prometheus.io/path: /metrics
         prometheus.io/port: "9911"
         prometheus.io/scrape: "true"
@@ -48,9 +70,9 @@ spec:
               # kube-system in our admitters, since we've never really had any issues with them.
               parent-resource-hash: 71556441059f2d033fb06b1e73df03598c7ecaa6
 {{- end }}
-      priorityClassName: "{{ .Cluster.ConfigItems.system_priority_class }}"
+      priorityClassName: "{{ .ConfigItems.system_priority_class }}"
       serviceAccountName: skipper-ingress
-      terminationGracePeriodSeconds: {{ .Cluster.ConfigItems.skipper_termination_grace_period }}
+      terminationGracePeriodSeconds: {{ .ConfigItems.skipper_termination_grace_period }}
       dnsPolicy: ClusterFirstWithHostNet
       hostNetwork: true
       containers:
@@ -114,7 +136,7 @@ spec:
           - "run.sh"
           - "skipper"
           - "-validate-query={{ .ConfigItems.skipper_validate_query }}"
-          - "-validate-query-log={{ .Cluster.ConfigItems.skipper_validate_query_log }}"
+          - "-validate-query-log={{ .ConfigItems.skipper_validate_query_log }}"
 {{ if eq .ConfigItems.skipper_routesrv_enabled "exec" }}
           - "-routes-urls=http://skipper-ingress-routesrv.kube-system.svc.cluster.local/routes"
           - "-normalize-host"
@@ -129,7 +151,7 @@ spec:
 {{ end }}
           - "-address=:9999"
           - "-wait-first-route-load"
-          - "-wait-for-healthcheck-interval={{ .Cluster.ConfigItems.skipper_wait_for_healthcheck_interval }}"
+          - "-wait-for-healthcheck-interval={{ .ConfigItems.skipper_wait_for_healthcheck_interval }}"
 {{ if and (ne .ConfigItems.skipper_routesrv_enabled "exec") (eq .ConfigItems.enable_skipper_eastwest "true")}}
           - "-enable-kubernetes-east-west"
           - "-kubernetes-east-west-domain=.ingress.cluster.local"
@@ -187,7 +209,7 @@ spec:
             -opentracing=lightstep
             component-name=skipper-ingress
             token=$(LIGHTSTEP_TOKEN)
-            collector={{ .Cluster.ConfigItems.tracing_collector_host }}:8444
+            collector={{ .ConfigItems.tracing_collector_host }}:8444
             cmd-line=skipper-ingress
             tag=application=skipper-ingress
             tag=account={{ .Cluster.Alias }}
@@ -201,7 +223,7 @@ spec:
             max-log-value-len={{ .ConfigItems.skipper_ingress_lightstep_max_log_value_len }}
             max-logs-per-span={{ .ConfigItems.skipper_ingress_lightstep_max_logs_per_span }}
             propagators={{ .ConfigItems.skipper_ingress_lightstep_propagators }}
-            {{ .Cluster.ConfigItems.skipper_ingress_lightstep_log_events }}
+            {{ .ConfigItems.skipper_ingress_lightstep_log_events }}
           - "-opentracing-excluded-proxy-tags={{ .ConfigItems.skipper_ingress_opentracing_excluded_proxy_tags }}"
 {{ if eq .ConfigItems.skipper_ingress_opentracing_backend_name_tag "true" }}
           - "-opentracing-backend-name-tag"
@@ -271,10 +293,15 @@ spec:
           {{ range $pattern := split .ConfigItems.skipper_ingress_refuse_payload "[cf724afc]" }}
           - "-refuse-payload={{ $pattern }}"
           {{ end }}
+{{ end }}
+{{ if index . "args" }}
+          {{ range $arg := split .args "[cf724afc]" }}
+          - "{{ $arg }}"
+          {{ end }}
 {{ end }}
         resources:
-{{ if and (eq .Cluster.ConfigItems.enable_dedicate_nodepool_skipper "true") (eq .Cluster.ConfigItems.skipper_ingress_binpack "true") }}
-{{ $cpu_requests := sumQuantities .Cluster.ConfigItems.skipper_ingress_cpu (printf "-%s"  .Cluster.ConfigItems.teapot_admission_controller_daemonset_reserved_cpu) (printf "-%s" .Cluster.ConfigItems.kubelet_system_reserved_cpu) (printf "-%s" .Cluster.ConfigItems.kubelet_kube_reserved_cpu) }}
+{{ if and (eq .ConfigItems.enable_dedicate_nodepool_skipper "true") (eq .ConfigItems.skipper_ingress_binpack "true") }}
+{{ $cpu_requests := sumQuantities .ConfigItems.skipper_ingress_cpu (printf "-%s"  .ConfigItems.teapot_admission_controller_daemonset_reserved_cpu) (printf "-%s" .ConfigItems.kubelet_system_reserved_cpu) (printf "-%s" .ConfigItems.kubelet_kube_reserved_cpu) }}
           limits:
             cpu: "{{ $cpu_requests }}"
             memory: "{{ .ConfigItems.skipper_ingress_memory }}"
@@ -397,6 +424,7 @@ spec:
         key: dedicated
         value: skipper-ingress
 {{ end }}
+{{ end }}
 
 {{ if ne .ConfigItems.skipper_routesrv_enabled "false" }}
 ---
@@ -429,7 +457,7 @@ spec:
         config/hash: {{"secret.yaml" | manifestHash}}
         kubernetes-log-watcher/scalyr-parser: |
           [{"container": "routesrv", "parser": "keyValue"}]
-        logging/destination: "{{.Cluster.ConfigItems.log_destination_local}}"
+        logging/destination: "{{ .ConfigItems.log_destination_local }}"
         prometheus.io/path: /metrics
         prometheus.io/port: "9990"
         prometheus.io/scrape: "true"
@@ -448,9 +476,9 @@ spec:
               # kube-system in our admitters, since we've never really had any issues with them.
               parent-resource-hash: abd943226b6885f66785592be28bdf303863fbac
 {{- end }}
-      priorityClassName: "{{ .Cluster.ConfigItems.system_priority_class }}"
+      priorityClassName: "{{ .ConfigItems.system_priority_class }}"
       serviceAccountName: skipper-ingress
-      terminationGracePeriodSeconds: {{ .Cluster.ConfigItems.skipper_termination_grace_period }}
+      terminationGracePeriodSeconds: {{ .ConfigItems.skipper_termination_grace_period }}
 {{- if eq .ConfigItems.skipper_routesrv_node_affinity_enabled "true" }}
       affinity:
        podAntiAffinity:
@@ -485,7 +513,7 @@ spec:
           - "-kubernetes-disable-catchall-routes={{ .ConfigItems.skipper_ingress_disable_catchall_routes }}"
           - "-enable-kubernetes-endpointslices={{ .ConfigItems.skipper_endpointslices_enabled }}"
           - "-address=:9990"
-          - "-wait-for-healthcheck-interval={{ .Cluster.ConfigItems.skipper_wait_for_healthcheck_interval }}"
+          - "-wait-for-healthcheck-interval={{ .ConfigItems.skipper_wait_for_healthcheck_interval }}"
 {{ if eq .ConfigItems.enable_skipper_eastwest "true"}}
           - "-enable-kubernetes-east-west"
           - "-kubernetes-east-west-domain=.ingress.cluster.local"
@@ -517,7 +545,7 @@ spec:
             -opentracing=lightstep
             component-name=routesrv
             token=$(LIGHTSTEP_TOKEN)
-            collector={{ .Cluster.ConfigItems.tracing_collector_host }}:8444
+            collector={{ .ConfigItems.tracing_collector_host }}:8444
             cmd-line=routesrv
             tag=application=skipper-ingress
             tag=component=routesrv
@@ -532,7 +560,7 @@ spec:
             max-log-value-len={{ .ConfigItems.skipper_ingress_lightstep_max_log_value_len }}
             max-logs-per-span={{ .ConfigItems.skipper_ingress_lightstep_max_logs_per_span }}
             propagators={{ .ConfigItems.skipper_ingress_lightstep_propagators }}
-            {{ .Cluster.ConfigItems.skipper_ingress_lightstep_log_events }}
+            {{ .ConfigItems.skipper_ingress_lightstep_log_events }}
         env:
         - name: LIGHTSTEP_TOKEN
           valueFrom: