From 9498c6b704596bf10335ae47fc3371167d21903e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Guilherme=20Vanz?= Date: Tue, 5 Nov 2024 14:29:29 -0300 Subject: [PATCH] feat: extra OPTEL collector configuration. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds additional telemetry configuration fields to allow users to add their custom OpenTelemetry collector configuration together with the Kubewarden configuration. Signed-off-by: José Guilherme Vanz --- .../templates/deployment.yaml | 7 +- .../templates/opentelemetry-collector.yaml | 40 +-- .../tests/deployment_test.yaml | 28 ++ .../tests/telemetry_test.yaml | 241 ++++++++++++++++++ charts/kubewarden-controller/values.yaml | 55 +++- 5 files changed, 320 insertions(+), 51 deletions(-) create mode 100644 charts/kubewarden-controller/tests/deployment_test.yaml create mode 100644 charts/kubewarden-controller/tests/telemetry_test.yaml diff --git a/charts/kubewarden-controller/templates/deployment.yaml b/charts/kubewarden-controller/templates/deployment.yaml index a4f7f059..40f3a34e 100644 --- a/charts/kubewarden-controller/templates/deployment.yaml +++ b/charts/kubewarden-controller/templates/deployment.yaml @@ -18,7 +18,7 @@ spec: {{- range keys .Values.podAnnotations }} {{ . | quote }}: {{ get $.Values.podAnnotations . | quote}} {{- end }} - {{- if or .Values.telemetry.metrics.enabled .Values.telemetry.tracing.enabled}} + {{- if or .Values.telemetry.metrics.enabled .Values.telemetry.tracing.enabled }} "sidecar.opentelemetry.io/inject": "true" {{- end }} {{- include "kubewarden-controller.annotations" . | nindent 8 }} @@ -56,10 +56,9 @@ spec: - --zap-log-level={{ .Values.logLevel }} command: - /manager - {{- if .Values.telemetry.metrics.enabled }} + {{- if .Values.env }} env: - - name: KUBEWARDEN_POLICY_SERVER_SERVICES_METRICS_PORT - value: "{{ .Values.telemetry.metrics.port | default 8080 }}" + {{- toYaml .Values.env | nindent 10 }} {{- end }} image: '{{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }}' imagePullPolicy: {{ .Values.image.pullPolicy }} diff --git a/charts/kubewarden-controller/templates/opentelemetry-collector.yaml b/charts/kubewarden-controller/templates/opentelemetry-collector.yaml index e93bb3f8..739fdfae 100644 --- a/charts/kubewarden-controller/templates/opentelemetry-collector.yaml +++ b/charts/kubewarden-controller/templates/opentelemetry-collector.yaml @@ -1,4 +1,4 @@ -{{ if or .Values.telemetry.metrics.enabled .Values.telemetry.tracing.enabled }} +{{ if or .Values.telemetry.metrics.enabled .Values.telemetry.tracing.enabled }} apiVersion: opentelemetry.io/v1beta1 kind: OpenTelemetryCollector metadata: @@ -9,41 +9,5 @@ metadata: annotations: {{- include "kubewarden-controller.annotations" . | nindent 4 }} spec: - mode: sidecar - config: - receivers: - otlp: - protocols: - grpc: {} - processors: - batch: {} - exporters: - {{- if and .Values.telemetry.tracing.enabled .Values.telemetry.tracing.jaeger.endpoint }} - otlp/jaeger: - endpoint: {{ .Values.telemetry.tracing.jaeger.endpoint }} - {{- if hasKey .Values.telemetry.tracing.jaeger "tls" }} - {{- if .Values.telemetry.tracing.jaeger.tls.insecure }} - tls: - insecure: {{ .Values.telemetry.tracing.jaeger.tls.insecure }} - {{- end }} - {{- end }} - {{- end }} - {{- if and .Values.telemetry.metrics.enabled .Values.telemetry.metrics.port }} - prometheus: - endpoint: ":{{ .Values.telemetry.metrics.port }}" - {{- end }} - service: - pipelines: - {{- if and .Values.telemetry.metrics.enabled .Values.telemetry.metrics.port }} - metrics: - receivers: [otlp] - processors: [] - exporters: [prometheus] - {{- end }} - {{- if and .Values.telemetry.tracing.enabled .Values.telemetry.tracing.jaeger.endpoint }} - traces: - receivers: [otlp] - processors: [batch] - exporters: [otlp/jaeger] - {{- end }} + {{- toYaml .Values.telemetry.otelSpec | nindent 2 }} {{ end }} diff --git a/charts/kubewarden-controller/tests/deployment_test.yaml b/charts/kubewarden-controller/tests/deployment_test.yaml new file mode 100644 index 00000000..1fbf90e9 --- /dev/null +++ b/charts/kubewarden-controller/tests/deployment_test.yaml @@ -0,0 +1,28 @@ +suite: Kubewarden controller deployment test +templates: + - deployment.yaml +release: + namespace: "kubewarden" +tests: + - it: "should not add the environment variable in the controller deployment when it is not set" + asserts: + - notExists: + path: spec.template.spec.containers[0].env + - it: "should not add the environment variable in the controller deployment when it is empty" + set: + env: [] + asserts: + - notExists: + path: spec.template.spec.containers[0].env + - it: "should adds the environment variable in the controller deployment" + set: + env: + - name: KUBEWARDEN_POLICY_SERVER_SERVICES_METRICS_PORT + value: "8080" + asserts: + - isSubset: + path: spec.template.spec.containers[0] + content: + env: + - name: KUBEWARDEN_POLICY_SERVER_SERVICES_METRICS_PORT + value: "8080" diff --git a/charts/kubewarden-controller/tests/telemetry_test.yaml b/charts/kubewarden-controller/tests/telemetry_test.yaml new file mode 100644 index 00000000..9f71e9eb --- /dev/null +++ b/charts/kubewarden-controller/tests/telemetry_test.yaml @@ -0,0 +1,241 @@ +suite: Telemetry tests +templates: + - opentelemetry-collector.yaml + - deployment.yaml +release: + name: "kubewarden-controller" + namespace: "kubewarden" +tests: + - it: "should adds cli flags in the controller deployment when tracing or metrics are enabled" + template: deployment.yaml + set: + telemetry: + tracing: + enabled: true + metrics: + enabled: true + asserts: + - equal: + path: spec.template.spec.containers[0].args[3] + value: --enable-metrics + - equal: + path: spec.template.spec.containers[0].args[4] + value: --enable-tracing + - it: "should not creates the OpenTelemetryCollector when telemetry is disabled" + template: opentelemetry-collector.yaml + set: + telemetry: + tracing: + enabled: false + metrics: + enabled: false + asserts: + - containsDocument: + kind: OpenTelemetryCollector + apiVersion: opentelemetry.io/v1beta1 + name: kubewarden + namespace: kubewarden + not: true + - it: "should creates the OpenTelemetryCollector when metrics is enabled" + template: opentelemetry-collector.yaml + set: + telemetry: + tracing: + enabled: false + metrics: + enabled: true + asserts: + - containsDocument: + kind: OpenTelemetryCollector + apiVersion: opentelemetry.io/v1beta1 + name: kubewarden + namespace: kubewarden + - it: "should creates the OpenTelemetryCollector when tracing is enabled" + template: opentelemetry-collector.yaml + set: + telemetry: + tracing: + enabled: true + metrics: + enabled: false + asserts: + - containsDocument: + kind: OpenTelemetryCollector + apiVersion: opentelemetry.io/v1beta1 + name: kubewarden + namespace: kubewarden + - it: "should use the default OpenTelemetryCollector configuration when telemetry is enabled" + template: opentelemetry-collector.yaml + set: + telemetry: + metrics: + enabled: true + asserts: + - containsDocument: + kind: OpenTelemetryCollector + apiVersion: opentelemetry.io/v1beta1 + name: kubewarden + namespace: kubewarden + - equal: + path: spec + value: + config: + connectors: {} + exporters: + otlp/jaeger: + endpoint: my-open-telemetry-collector.jaeger.svc.cluster.local:4317 + tls: + insecure: true + prometheus: + endpoint: :8080 + extensions: {} + processors: + batch: {} + receivers: + otlp: + protocols: + grpc: {} + service: + extensions: {} + pipelines: + metrics: + exporters: + - prometheus + processors: [] + receivers: + - otlp + traces: + exporters: + - otlp/jaeger + processors: + - batch + receivers: + - otlp + envFrom: {} + mode: sidecar + - it: "should use the user defined OpenTelemetryCollector spec configuration" + template: opentelemetry-collector.yaml + set: + telemetry: + tracing: + enabled: true + otelSpec: + mode: sidecar + envFrom: + - secretRef: + name: open-telemetry-collector + config: + processors: + resource: + attributes: + - key: k8s.cluster.name + action: upsert + value: k3d-kubewarden + - key: service.instance.id + from_attribute: k8s.pod.uid + action: insert + extensions: + bearertokenauth: + scheme: SUSEObservability + token: "${env:API_KEY}" + exporters: + debug: + verbosity: normal + otlphttp/stackstate: + auth: + authenticator: bearertokenauth + endpoint: https://otlp-stackstate.oldfield.arch.nue2.suse.org:443 + tls: + insecure_skip_verify: true + service: + extensions: + - bearertokenauth + pipelines: + traces/stackstate: + receivers: [otlp] + processors: [resource] + exporters: [otlphttp/stackstate] + metrics/stackstate: + receivers: [otlp] + processors: [resource] + exporters: [debug, otlphttp/stackstate] + asserts: + - containsDocument: + kind: OpenTelemetryCollector + apiVersion: opentelemetry.io/v1beta1 + name: kubewarden + namespace: kubewarden + - equal: + path: spec + value: + config: + connectors: {} + exporters: + debug: + verbosity: normal + otlp/jaeger: + endpoint: my-open-telemetry-collector.jaeger.svc.cluster.local:4317 + tls: + insecure: true + otlphttp/stackstate: + auth: + authenticator: bearertokenauth + endpoint: https://otlp-stackstate.oldfield.arch.nue2.suse.org:443 + tls: + insecure_skip_verify: true + prometheus: + endpoint: :8080 + extensions: + bearertokenauth: + scheme: SUSEObservability + token: ${env:API_KEY} + processors: + batch: {} + resource: + attributes: + - action: upsert + key: k8s.cluster.name + value: k3d-kubewarden + - action: insert + from_attribute: k8s.pod.uid + key: service.instance.id + receivers: + otlp: + protocols: + grpc: {} + service: + extensions: + - bearertokenauth + pipelines: + metrics: + exporters: + - prometheus + processors: [] + receivers: + - otlp + metrics/stackstate: + exporters: + - debug + - otlphttp/stackstate + processors: + - resource + receivers: + - otlp + traces: + exporters: + - otlp/jaeger + processors: + - batch + receivers: + - otlp + traces/stackstate: + exporters: + - otlphttp/stackstate + processors: + - resource + receivers: + - otlp + envFrom: + - secretRef: + name: open-telemetry-collector + mode: sidecar diff --git a/charts/kubewarden-controller/values.yaml b/charts/kubewarden-controller/values.yaml index e0734083..77177650 100644 --- a/charts/kubewarden-controller/values.yaml +++ b/charts/kubewarden-controller/values.yaml @@ -135,17 +135,50 @@ preDeleteHook: logLevel: info # open-telemetry options telemetry: - metrics: - enabled: false - # port of the prometheus exporter and PolicyServer metric service - port: 8080 tracing: enabled: false - jaeger: {} - # OTLP/Jaeger endpoint to send traces to - # endpoint: "all-in-one-collector.jaeger.svc.cluster.local:4317" - # tls: - # insecure: true + metrics: + enabled: false + # otelSpec field is used to configure the OpenTelemetry Collector. The values + # added here will be used to populate the OTEL collector spec. Refer the the + # OpenTelemetry Collector documentation for more information on the spec: + # https://github.com/open-telemetry/opentelemetry-operator/blob/main/docs/api.md + # + # The default values are set configure OTEL collector as a sidecar in the + # Kubewarden pods and to send traces to the Jaeger collector and metrics + # to the Prometheus exporter. If you enabled metrics, remember to configure + # the environment variable KUBEWARDEN_POLICY_SERVER_SERVICES_METRICS_PORT in + # env values field in this file. + otelSpec: + mode: sidecar + envFrom: {} + config: + extensions: {} + connectors: {} + receivers: + otlp: + protocols: + grpc: {} + processors: + batch: {} + exporters: + otlp/jaeger: + endpoint: "my-open-telemetry-collector.jaeger.svc.cluster.local:4317" + tls: + insecure: true + prometheus: + endpoint: ":8080" + service: + extensions: {} + pipelines: + metrics: + receivers: [otlp] + processors: [] + exporters: [prometheus] + traces: + receivers: [otlp] + processors: [batch] + exporters: [otlp/jaeger] image: # The registry is defined in the global.cattle.systemDefaultRegistry value # controller image to be used @@ -159,6 +192,10 @@ preDeleteJob: # kubectl image to be used in the pre-delete helm hook repository: "kubewarden/kubectl" tag: v1.31.0 +# kubewarden-controller deployment environment variables +# env: +# - name: KUBEWARDEN_POLICY_SERVER_SERVICES_METRICS_PORT +# value: "8080" # kubewarden-controller deployment settings: podAnnotations: {} nodeSelector: {}