diff --git a/.github/workflows/ct.yaml b/.github/workflows/ct.yaml index 412e150..d4c27da 100644 --- a/.github/workflows/ct.yaml +++ b/.github/workflows/ct.yaml @@ -37,6 +37,7 @@ jobs: run: | helm repo add bitnami https://charts.bitnami.com/bitnami ct lint --chart-dirs sickhub --all + - name: Create kind cluster uses: helm/kind-action@v1.5.0 if: github.ref == 'refs/heads/master' || steps.list-changed.outputs.changed == 'true' diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index a67a2e9..026f5b4 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -33,4 +33,4 @@ jobs: config: .github/cr-config.yaml charts_dir: sickhub env: - CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" \ No newline at end of file diff --git a/sickhub/nginx-phpfpm/Chart.yaml b/sickhub/nginx-phpfpm/Chart.yaml index bafc6d6..a9c74b7 100644 --- a/sickhub/nginx-phpfpm/Chart.yaml +++ b/sickhub/nginx-phpfpm/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: nginx-phpfpm description: A chart for an nginx pod with multiple phpfpm pods type: application -version: 0.0.10 +version: 0.1.0 appVersion: "8-fpm-alpine" home: https://github.com/SickHub icon: https://raw.githubusercontent.com/SickHub/charts/master/sickhub/nginx-phpfpm/icon.png @@ -21,4 +21,4 @@ annotations: - name: Ngix image url: https://hub.docker.com/_/nginx - name: PHP FPM image - url: https://hub.docker.com/_/php?tab=tags&ordering=last_updated&name=alpine&page=1 + url: https://hub.docker.com/_/php/tags?page=1&name=fpm diff --git a/sickhub/nginx-phpfpm/README.md b/sickhub/nginx-phpfpm/README.md index c19e239..80a110e 100644 --- a/sickhub/nginx-phpfpm/README.md +++ b/sickhub/nginx-phpfpm/README.md @@ -111,10 +111,70 @@ you can use `persistence.enabled = true` to * B) provide an `existingVolume` to use with a PVC created by the chart * C) provide an `existingClaim` to use an existing PVC +## Load testing +Install the Chart with `loadtest-values.yaml` which includes a few simple scripts (endpoints), one of which causes +the PHP process to consume all available CPU for a few seconds. +```shell +helm upgrade --install load-test sickhub/nginx-phpfpm --values sickhub/nginx-phpfpm/ci/loadtest-values.yaml +``` + +### Run tests with `wrk` +- `-d 10s` test duration +- `-t 20` number of threads +- `-c 20` number of connections to keep open (must be >= threads) +```shell +wrk -d 10s -t 50 -c 50 http://localhost/healthz.php # 10 seconds, 50 threads, 100 connections +wrk -d 10s -t 5 -c 10 http://localhost/livez.html # static HTML page from Nginx +wrk -d 10s -t 20 -c 40 http://localhost/healthz.php # phpinfo from PHP container +wrk -d 10s -t 5 -c 10 http://localhost/load.php # 100% load for X seconds, then return phpinfo +``` + +Run test suite: +- 3 threads which cause load +- 50 threads requesting PHP +- 10 threads requesting static HTML +```shell +export time=300s +wrk -d $time -t 3 -c 3 http://localhost/load.php & +sleep 1 +wrk -d $time -t 50 -c 100 http://localhost/healthz.php & +sleep 1 +wrk -d $time -t 10 -c 100 http://localhost/livez.html & +``` + +You will notice, that with the right `shutdownDelay` and deployment `strategy`, you can get your installation to do +an update without dropping a single request (100% availability while doing a rollingUpdate). + +While running the test suite, simply apply a small change via helm (memory limits for example) and watch the pods being +created and gracefully terminated without missing a request. + +See also: https://medium.com/inside-personio/graceful-shutdown-of-fpm-and-nginx-in-kubernetes-f362369dff22 + +## Test results +Under load, the deployments get scaled up to 3 replicas each +```shell +NAME CPU(cores) MEMORY(bytes) +loadtest-nginx-phpfpm-nginx-8677d86c6d-6srnm 192m 9Mi +loadtest-nginx-phpfpm-nginx-8677d86c6d-b5gdr 195m 10Mi +loadtest-nginx-phpfpm-nginx-8677d86c6d-kd7gg 216m 6Mi +loadtest-nginx-phpfpm-phpfpm-b7c569b8c-7hl56 1265m 20Mi +loadtest-nginx-phpfpm-phpfpm-b7c569b8c-lw9tj 921m 11Mi +loadtest-nginx-phpfpm-phpfpm-b7c569b8c-tsnvr 792m 20Mi +``` +Once the test is over, after about 5 minutes, the deployments get scaled down again +```shell +NAME CPU(cores) MEMORY(bytes) +loadtest-nginx-phpfpm-nginx-8677d86c6d-b5gdr 1m 10Mi +loadtest-nginx-phpfpm-phpfpm-b7c569b8c-lw9tj 1m 11Mi +``` + + + + ## Missing features - help appreciated * [ ] provide `nginx.conf` through `values.yaml` -> make it configurable * [ ] test scenarios and/or examples - * [ ] test autoscaling + * [x] test autoscaling * [ ] test persistence ## Contribute diff --git a/sickhub/nginx-phpfpm/ci/loadtest-values.yaml b/sickhub/nginx-phpfpm/ci/loadtest-values.yaml new file mode 100644 index 0000000..edd98e1 --- /dev/null +++ b/sickhub/nginx-phpfpm/ci/loadtest-values.yaml @@ -0,0 +1,63 @@ +# values for autoscaling +# test with `siege -d 1 -t 120S -c 5 http://localhost/load.php` + +configMaps: + scripts: + path: / + data: + livez.html: | + OK + healthz.php: | + + load.php: | + + +autoscaling: + apiVersionOverride: autoscaling/v2 + +nginx: + strategy: + rollingUpdate: + maxUnavailable: 0 + maxSurge: 3 + resources: + limits: + cpu: 1 + memory: 100Mi + requests: + cpu: 0.2 + memory: 50Mi + autoscaling: + enabled: true + minReplicas: 1 + maxReplicas: 3 + targetCPUUtilizationPercentage: 80 + +phpfpm: +# image: +# tag: 8-fpm-bullseye + strategy: + rollingUpdate: + maxUnavailable: 0 + maxSurge: 3 + resources: + limits: + cpu: 1.5 + memory: 250Mi + requests: + cpu: 0.5 + memory: 100Mi + autoscaling: + enabled: true + minReplicas: 1 + maxReplicas: 3 + targetCPUUtilizationPercentage: 70 + +ingress: + enabled: true + annotations: + kubernetes.io/ingress.class: nginx + hosts: + - host: localhost + paths: + - path: "/" diff --git a/sickhub/nginx-phpfpm/templates/nginx-deployment.yaml b/sickhub/nginx-phpfpm/templates/nginx-deployment.yaml index d5bcdd3..31d45ac 100644 --- a/sickhub/nginx-phpfpm/templates/nginx-deployment.yaml +++ b/sickhub/nginx-phpfpm/templates/nginx-deployment.yaml @@ -17,6 +17,9 @@ spec: {{- if not .Values.nginx.autoscaling.enabled }} replicas: {{ .Values.nginx.replicaCount }} {{- end }} + {{- with .Values.nginx.strategy }} + strategy: {{ toYaml . | nindent 4 }} + {{- end }} selector: matchLabels: {{- include "nginx-phpfpm.selectorLabels" . | nindent 6 }} @@ -70,6 +73,13 @@ spec: httpGet: path: {{ .Values.nginx.readinessProbe.path }} port: http + lifecycle: + preStop: + exec: + command: + - sh + - '-c' + - 'sleep {{ .Values.nginx.shutdownDelay }} && /usr/sbin/nginx -s quit' resources: {{- toYaml .Values.nginx.resources | nindent 12 }} volumeMounts: diff --git a/sickhub/nginx-phpfpm/templates/nginx-hpa.yaml b/sickhub/nginx-phpfpm/templates/nginx-hpa.yaml new file mode 100644 index 0000000..a21d6a3 --- /dev/null +++ b/sickhub/nginx-phpfpm/templates/nginx-hpa.yaml @@ -0,0 +1,73 @@ +{{- if .Values.nginx.autoscaling.enabled }} +{{- $fullName := include "nginx-phpfpm.fullname" . -}} +{{- $apiVersion := "autoscaling/v2" }} +{{- if .Values.autoscaling.apiVersionOverride -}} + {{- $apiVersion = .Values.autoscaling.apiVersionOverride }} +{{- else if .Capabilities.APIVersions.Has "autoscaling/v2beta1" }} + {{- $apiVersion = "autoscaling/v2beta1" }} +{{- end }} +apiVersion: {{ $apiVersion }} +kind: HorizontalPodAutoscaler +metadata: + name: {{ $fullName }}-nginx + labels: + {{- include "nginx-phpfpm.labels" . | nindent 4 }} + checksum/values: {{ toYaml .Values | sha256sum | trunc 20 | quote }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "nginx-phpfpm.fullname" . }}-nginx + minReplicas: {{ .Values.nginx.autoscaling.minReplicas }} + maxReplicas: {{ .Values.nginx.autoscaling.maxReplicas }} + metrics: + {{- with .Values.nginx.autoscaling.requestsPerSecond }} + - type: Object + object: # https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.26/#objectmetricsource-v2-autoscaling + describedObject: + apiVersion: networking.k8s.io/v1 + kind: Ingress + name: {{ $fullName }}-nginx + metric: + name: {{ .name }} + target: + type: Value + value: {{ .value }} + {{- end }} + {{- with .Values.nginx.autoscaling.packetsPerSecond }} + - type: Pods + pods: + metric: + name: {{ .name }} + target: + type: AverageValue + averageValue: {{ .averageValue }} + {{- end }} + {{- with .Values.nginx.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + {{- if eq $apiVersion "autoscaling/v2beta1" }} + targetAverageUtilization: {{ . }} + {{- else }} + target: + averageUtilization: {{ . }} + type: Utilization + {{- end }} + {{- end }} + {{- with .Values.nginx.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + {{- if eq $apiVersion "autoscaling/v2beta1" }} + targetAverageUtilization: {{ . }} + {{- else }} + target: + averageUtilization: {{ . }} + type: Utilization + {{- end }} + {{- end }} + {{- with .Values.nginx.autoscaling.behaviour }} + behaviour: {{ toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/sickhub/nginx-phpfpm/templates/phpfpm-deployment.yaml b/sickhub/nginx-phpfpm/templates/phpfpm-deployment.yaml index a2781cb..811558e 100644 --- a/sickhub/nginx-phpfpm/templates/phpfpm-deployment.yaml +++ b/sickhub/nginx-phpfpm/templates/phpfpm-deployment.yaml @@ -13,6 +13,9 @@ spec: {{- if not .Values.phpfpm.autoscaling.enabled }} replicas: {{ .Values.phpfpm.replicaCount }} {{- end }} + {{- with .Values.phpfpm.strategy }} + strategy: {{ toYaml . | nindent 4 }} + {{- end }} selector: matchLabels: {{- include "nginx-phpfpm.selectorLabels" . | nindent 6 }} @@ -55,6 +58,13 @@ spec: readinessProbe: tcpSocket: port: phpfpm + lifecycle: + preStop: + exec: + command: + - sh + - '-c' + - 'sleep {{ .Values.phpfpm.shutdownDelay }} && kill -QUIT 1' resources: {{- toYaml .Values.phpfpm.resources | nindent 12 }} volumeMounts: diff --git a/sickhub/nginx-phpfpm/templates/phpfpm-hpa.yaml b/sickhub/nginx-phpfpm/templates/phpfpm-hpa.yaml index d410719..e91abe5 100644 --- a/sickhub/nginx-phpfpm/templates/phpfpm-hpa.yaml +++ b/sickhub/nginx-phpfpm/templates/phpfpm-hpa.yaml @@ -1,5 +1,12 @@ {{- if .Values.phpfpm.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 +{{- $fullName := include "nginx-phpfpm.fullname" . -}} +{{- $apiVersion := "autoscaling/v2" }} +{{- if .Values.autoscaling.apiVersionOverride -}} + {{- $apiVersion = .Values.autoscaling.apiVersionOverride }} +{{- else if .Capabilities.APIVersions.Has "autoscaling/v2beta1" }} + {{- $apiVersion = "autoscaling/v2beta1" }} +{{- end }} +apiVersion: {{ $apiVersion }} kind: HorizontalPodAutoscaler metadata: name: {{ include "nginx-phpfpm.fullname" . }}-phpfpm @@ -10,20 +17,58 @@ spec: scaleTargetRef: apiVersion: apps/v1 kind: Deployment - name: {{ include "nginx-phpfpm.fullname" . }} + name: {{ include "nginx-phpfpm.fullname" . }}-phpfpm minReplicas: {{ .Values.phpfpm.autoscaling.minReplicas }} maxReplicas: {{ .Values.phpfpm.autoscaling.maxReplicas }} metrics: - {{- if .Values.phpfpm.autoscaling.targetCPUUtilizationPercentage }} + {{- with .Values.nginx.autoscaling.requestsPerSecond }} + # https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.26/#objectmetricsource-v2-autoscaling + - type: Object + object: + describedObject: + apiVersion: networking.k8s.io/v1 + kind: Ingress + name: {{ $fullName }}-phpfpm + metric: + name: {{ .name }} + target: + type: Value + value: {{ .value }} + {{- end }} + {{- with .Values.nginx.autoscaling.packetsPerSecond }} + - type: Pods + pods: + metric: + name: {{ .name }} + target: + type: AverageValue + averageValue: {{ .averageValue }} + {{- end }} + {{- with .Values.phpfpm.autoscaling.targetCPUUtilizationPercentage }} - type: Resource resource: name: cpu - targetAverageUtilization: {{ .Values.phpfpm.autoscaling.targetCPUUtilizationPercentage }} + {{- if eq $apiVersion "autoscaling/v2beta1" }} + targetAverageUtilization: {{ . }} + {{- else }} + target: + averageUtilization: {{ . }} + type: Utilization + {{- end }} {{- end }} - {{- if .Values.phpfpm.autoscaling.targetMemoryUtilizationPercentage }} + {{- with .Values.phpfpm.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory - targetAverageUtilization: {{ .Values.phpfpm.autoscaling.targetMemoryUtilizationPercentage }} + {{- if eq $apiVersion "autoscaling/v2beta1" }} + targetAverageUtilization: {{ . }} + {{- else }} + target: + averageUtilization: {{ . }} + type: Utilization + {{- end }} {{- end }} + {{- with .Values.nginx.autoscaling.behaviour }} + behaviour: {{ toYaml . | nindent 4 }} + {{- end }} {{- end }} diff --git a/sickhub/nginx-phpfpm/templates/tests/test-connection.yaml b/sickhub/nginx-phpfpm/templates/tests/test-connection.yaml index 6cf609d..61caccd 100644 --- a/sickhub/nginx-phpfpm/templates/tests/test-connection.yaml +++ b/sickhub/nginx-phpfpm/templates/tests/test-connection.yaml @@ -11,5 +11,5 @@ spec: - name: wget image: busybox command: ['wget'] - args: ['{{ include "nginx-phpfpm.fullname" . }}-nginx:{{ .Values.nginx.service.port }}/health.php'] + args: ['{{ include "nginx-phpfpm.fullname" . }}-nginx:{{ .Values.nginx.service.port }}/healthz.php'] restartPolicy: Never diff --git a/sickhub/nginx-phpfpm/values.yaml b/sickhub/nginx-phpfpm/values.yaml index 4cc4779..24cce5a 100644 --- a/sickhub/nginx-phpfpm/values.yaml +++ b/sickhub/nginx-phpfpm/values.yaml @@ -10,8 +10,17 @@ dockerConfigJson: imagePullSecrets: [] +autoscaling: + # autoscaling/v1, autoscaling/v2beta, autoscaling/v2 + apiVersionOverride: "" + nginx: replicaCount: 1 + shutdownDelay: 10 + strategy: {} + # rollingUpdate: + # maxUnavailable: 0 + # maxSurge: 3 image: registry: repository: nginx @@ -44,17 +53,41 @@ nginx: autoscaling: enabled: false minReplicas: 1 - maxReplicas: 10 + maxReplicas: 3 targetCPUUtilizationPercentage: 80 # targetMemoryUtilizationPercentage: 80 - + behaviour: {} + # scaleDown: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 1 + # periodSeconds: 180 + # scaleUp: + # stabilizationWindowSeconds: 300 + # policies: + # - type: Pods + # value: 2 + # periodSeconds: 60 + ## requires custom.metrics.k8s.io + # requestsPerSecond: + # value: 5 + # name: requests-per-second + # packetsPerSecond: + # averageValue: 30 + # name: packets-per-second livenessProbe: - path: /health.php + path: /livez.html readinessProbe: - path: /health.php + path: /healthz.php phpfpm: replicaCount: 1 + shutdownDelay: 20 + strategy: {} + # rollingUpdate: + # maxUnavailable: 0 + # maxSurge: 3 image: registry: repository: php @@ -90,7 +123,9 @@ configMaps: scripts: path: / data: - health.php: | + livez.html: | + OK + healthz.php: | # configs: # path: /etc