HPA and load test scenario (#29)

- changed health.php to healthz.php
- nginx liveness probe uses static HTML livez.html

.github/workflows/ct.yaml

@@ -37,6 +37,7 @@ jobs:
         run: |
           helm repo add bitnami https://charts.bitnami.com/bitnami
           ct lint --chart-dirs sickhub --all
+
       - name: Create kind cluster
         uses: helm/[email protected]
         if: github.ref == 'refs/heads/master' || steps.list-changed.outputs.changed == 'true'

.github/workflows/release.yaml

@@ -33,4 +33,4 @@ jobs:
           config: .github/cr-config.yaml
           charts_dir: sickhub
         env:
-          CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+          CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"

sickhub/nginx-phpfpm/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: nginx-phpfpm
 description: A chart for an nginx pod with multiple phpfpm pods
 type: application
-version: 0.0.10
+version: 0.1.0
 appVersion: "8-fpm-alpine"
 home: https://github.com/SickHub
 icon: https://raw.githubusercontent.com/SickHub/charts/master/sickhub/nginx-phpfpm/icon.png
@@ -21,4 +21,4 @@ annotations:
     - name: Ngix image
       url: https://hub.docker.com/_/nginx
     - name: PHP FPM image
-      url: https://hub.docker.com/_/php?tab=tags&ordering=last_updated&name=alpine&page=1
+      url: https://hub.docker.com/_/php/tags?page=1&name=fpm

sickhub/nginx-phpfpm/README.md

@@ -111,10 +111,70 @@ you can use `persistence.enabled = true` to
 * B) provide an `existingVolume` to use with a PVC created by the chart 
 * C) provide an `existingClaim` to use an existing PVC
 
+## Load testing
+Install the Chart with `loadtest-values.yaml` which includes a few simple scripts (endpoints), one of which causes
+the PHP process to consume all available CPU for a few seconds.
+```shell
+helm upgrade --install load-test sickhub/nginx-phpfpm --values sickhub/nginx-phpfpm/ci/loadtest-values.yaml
+```
+
+### Run tests with `wrk`
+- `-d 10s` test duration
+- `-t 20` number of threads
+- `-c 20` number of connections to keep open (must be >= threads)
+```shell
+wrk -d 10s -t 50 -c 50 http://localhost/healthz.php # 10 seconds, 50 threads, 100 connections
+wrk -d 10s -t 5 -c 10 http://localhost/livez.html # static HTML page from Nginx
+wrk -d 10s -t 20 -c 40 http://localhost/healthz.php # phpinfo from PHP container
+wrk -d 10s -t 5 -c 10 http://localhost/load.php # 100% load for X seconds, then return phpinfo
+```
+
+Run test suite:
+- 3 threads which cause load
+- 50 threads requesting PHP
+- 10 threads requesting static HTML 
+```shell
+export time=300s
+wrk -d $time -t 3 -c 3 http://localhost/load.php &
+sleep 1
+wrk -d $time -t 50 -c 100 http://localhost/healthz.php &
+sleep 1
+wrk -d $time -t 10 -c 100 http://localhost/livez.html &
+```
+
+You will notice, that with the right `shutdownDelay` and deployment `strategy`, you can get your installation to do
+an update without dropping a single request (100% availability while doing a rollingUpdate).
+
+While running the test suite, simply apply a small change via helm (memory limits for example) and watch the pods being
+created and gracefully terminated without missing a request.
+
+See also: https://medium.com/inside-personio/graceful-shutdown-of-fpm-and-nginx-in-kubernetes-f362369dff22
+
+## Test results
+Under load, the deployments get scaled up to 3 replicas each
+```shell
+NAME                                           CPU(cores)   MEMORY(bytes)
+loadtest-nginx-phpfpm-nginx-8677d86c6d-6srnm   192m         9Mi
+loadtest-nginx-phpfpm-nginx-8677d86c6d-b5gdr   195m         10Mi
+loadtest-nginx-phpfpm-nginx-8677d86c6d-kd7gg   216m         6Mi
+loadtest-nginx-phpfpm-phpfpm-b7c569b8c-7hl56   1265m        20Mi
+loadtest-nginx-phpfpm-phpfpm-b7c569b8c-lw9tj   921m         11Mi
+loadtest-nginx-phpfpm-phpfpm-b7c569b8c-tsnvr   792m         20Mi
+```
+Once the test is over, after about 5 minutes, the deployments get scaled down again
+```shell
+NAME                                           CPU(cores)   MEMORY(bytes)
+loadtest-nginx-phpfpm-nginx-8677d86c6d-b5gdr   1m           10Mi
+loadtest-nginx-phpfpm-phpfpm-b7c569b8c-lw9tj   1m           11Mi
+```
+
+
+
+
 ## Missing features - help appreciated
 * [ ] provide `nginx.conf` through `values.yaml` -> make it configurable
 * [ ] test scenarios and/or examples
-  * [ ] test autoscaling
+  * [x] test autoscaling
   * [ ] test persistence
 
 ## Contribute

sickhub/nginx-phpfpm/ci/loadtest-values.yaml

@@ -0,0 +1,63 @@
+# values for autoscaling
+# test with `siege -d 1 -t 120S -c 5 http://localhost/load.php`
+
+configMaps:
+  scripts:
+    path: /
+    data:
+      livez.html: |
+        <html><body>OK</body></html>
+      healthz.php: |
+        <?php phpinfo(); ?>
+      load.php: |
+        <?php $time=time()+rand(1, 5); for(true;time() < $time;); phpinfo(); ?>
+
+autoscaling:
+  apiVersionOverride: autoscaling/v2
+
+nginx:
+  strategy:
+    rollingUpdate:
+      maxUnavailable: 0
+      maxSurge: 3
+  resources:
+    limits:
+      cpu: 1
+      memory: 100Mi
+    requests:
+      cpu: 0.2
+      memory: 50Mi
+  autoscaling:
+    enabled: true
+    minReplicas: 1
+    maxReplicas: 3
+    targetCPUUtilizationPercentage: 80
+
+phpfpm:
+#  image:
+#    tag: 8-fpm-bullseye
+  strategy:
+    rollingUpdate:
+      maxUnavailable: 0
+      maxSurge: 3
+  resources:
+    limits:
+      cpu: 1.5
+      memory: 250Mi
+    requests:
+      cpu: 0.5
+      memory: 100Mi
+  autoscaling:
+    enabled: true
+    minReplicas: 1
+    maxReplicas: 3
+    targetCPUUtilizationPercentage: 70
+
+ingress:
+  enabled: true
+  annotations:
+    kubernetes.io/ingress.class: nginx
+  hosts:
+    - host: localhost
+      paths:
+        - path: "/"

sickhub/nginx-phpfpm/templates/nginx-deployment.yaml

@@ -17,6 +17,9 @@ spec:
   {{- if not .Values.nginx.autoscaling.enabled }}
   replicas: {{ .Values.nginx.replicaCount }}
   {{- end }}
+  {{- with .Values.nginx.strategy }}
+  strategy: {{ toYaml . | nindent 4 }}
+  {{- end }}
   selector:
     matchLabels:
       {{- include "nginx-phpfpm.selectorLabels" . | nindent 6 }}
@@ -70,6 +73,13 @@ spec:
             httpGet:
               path: {{ .Values.nginx.readinessProbe.path }}
               port: http
+          lifecycle:
+            preStop:
+              exec:
+                command:
+                  - sh
+                  - '-c'
+                  - 'sleep {{ .Values.nginx.shutdownDelay }} && /usr/sbin/nginx -s quit'
           resources:
             {{- toYaml .Values.nginx.resources | nindent 12 }}
           volumeMounts:

sickhub/nginx-phpfpm/templates/nginx-hpa.yaml

@@ -0,0 +1,73 @@
+{{- if .Values.nginx.autoscaling.enabled }}
+{{- $fullName := include "nginx-phpfpm.fullname" . -}}
+{{- $apiVersion := "autoscaling/v2" }}
+{{- if .Values.autoscaling.apiVersionOverride -}}
+  {{- $apiVersion = .Values.autoscaling.apiVersionOverride }}
+{{- else if .Capabilities.APIVersions.Has "autoscaling/v2beta1" }}
+  {{- $apiVersion = "autoscaling/v2beta1" }}
+{{- end }}
+apiVersion: {{ $apiVersion }}
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ $fullName }}-nginx
+  labels:
+    {{- include "nginx-phpfpm.labels" . | nindent 4 }}
+    checksum/values: {{ toYaml .Values | sha256sum | trunc 20 | quote }}
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: {{ include "nginx-phpfpm.fullname" . }}-nginx
+  minReplicas: {{ .Values.nginx.autoscaling.minReplicas }}
+  maxReplicas: {{ .Values.nginx.autoscaling.maxReplicas }}
+  metrics:
+    {{- with .Values.nginx.autoscaling.requestsPerSecond }}
+    - type: Object
+      object: # https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.26/#objectmetricsource-v2-autoscaling
+        describedObject:
+          apiVersion: networking.k8s.io/v1
+          kind: Ingress
+          name: {{ $fullName }}-nginx
+        metric:
+          name: {{ .name }}
+        target:
+          type: Value
+          value: {{ .value }}
+    {{- end }}
+    {{- with .Values.nginx.autoscaling.packetsPerSecond }}
+    - type: Pods
+      pods:
+        metric:
+          name: {{ .name }}
+        target:
+          type: AverageValue
+          averageValue: {{ .averageValue }}
+    {{- end }}
+    {{- with .Values.nginx.autoscaling.targetCPUUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: cpu
+        {{- if eq $apiVersion "autoscaling/v2beta1" }}
+        targetAverageUtilization: {{ . }}
+        {{- else }}
+        target:
+          averageUtilization: {{ . }}
+          type: Utilization
+        {{- end }}
+    {{- end }}
+    {{- with .Values.nginx.autoscaling.targetMemoryUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: memory
+        {{- if eq $apiVersion "autoscaling/v2beta1" }}
+        targetAverageUtilization: {{ . }}
+        {{- else }}
+        target:
+          averageUtilization: {{ . }}
+          type: Utilization
+        {{- end }}
+    {{- end }}
+  {{- with .Values.nginx.autoscaling.behaviour }}
+  behaviour: {{ toYaml . | nindent 4 }}
+  {{- end }}
+{{- end }}

sickhub/nginx-phpfpm/templates/phpfpm-deployment.yaml

@@ -13,6 +13,9 @@ spec:
   {{- if not .Values.phpfpm.autoscaling.enabled }}
   replicas: {{ .Values.phpfpm.replicaCount }}
   {{- end }}
+  {{- with .Values.phpfpm.strategy }}
+  strategy: {{ toYaml . | nindent 4 }}
+  {{- end }}
   selector:
     matchLabels:
       {{- include "nginx-phpfpm.selectorLabels" . | nindent 6 }}
@@ -55,6 +58,13 @@ spec:
           readinessProbe:
             tcpSocket:
               port: phpfpm
+          lifecycle:
+            preStop:
+              exec:
+                command:
+                  - sh
+                  - '-c'
+                  - 'sleep {{ .Values.phpfpm.shutdownDelay }} && kill -QUIT 1'
           resources:
             {{- toYaml .Values.phpfpm.resources | nindent 12 }}
           volumeMounts:

sickhub/nginx-phpfpm/templates/phpfpm-hpa.yaml

@@ -1,5 +1,12 @@
 {{- if .Values.phpfpm.autoscaling.enabled }}
-apiVersion: autoscaling/v2beta1
+{{- $fullName := include "nginx-phpfpm.fullname" . -}}
+{{- $apiVersion := "autoscaling/v2" }}
+{{- if .Values.autoscaling.apiVersionOverride -}}
+  {{- $apiVersion = .Values.autoscaling.apiVersionOverride }}
+{{- else if .Capabilities.APIVersions.Has "autoscaling/v2beta1" }}
+  {{- $apiVersion = "autoscaling/v2beta1" }}
+{{- end }}
+apiVersion: {{ $apiVersion }}
 kind: HorizontalPodAutoscaler
 metadata:
   name: {{ include "nginx-phpfpm.fullname" . }}-phpfpm
@@ -10,20 +17,58 @@ spec:
   scaleTargetRef:
     apiVersion: apps/v1
     kind: Deployment
-    name: {{ include "nginx-phpfpm.fullname" . }}
+    name: {{ include "nginx-phpfpm.fullname" . }}-phpfpm
   minReplicas: {{ .Values.phpfpm.autoscaling.minReplicas }}
   maxReplicas: {{ .Values.phpfpm.autoscaling.maxReplicas }}
   metrics:
-    {{- if .Values.phpfpm.autoscaling.targetCPUUtilizationPercentage }}
+    {{- with .Values.nginx.autoscaling.requestsPerSecond }}
+    # https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.26/#objectmetricsource-v2-autoscaling
+    - type: Object
+      object:
+        describedObject:
+          apiVersion: networking.k8s.io/v1
+          kind: Ingress
+          name: {{ $fullName }}-phpfpm
+        metric:
+          name: {{ .name }}
+        target:
+          type: Value
+          value: {{ .value }}
+    {{- end }}
+    {{- with .Values.nginx.autoscaling.packetsPerSecond }}
+    - type: Pods
+      pods:
+        metric:
+          name: {{ .name }}
+        target:
+          type: AverageValue
+          averageValue: {{ .averageValue }}
+    {{- end }}
+    {{- with .Values.phpfpm.autoscaling.targetCPUUtilizationPercentage }}
     - type: Resource
       resource:
         name: cpu
-        targetAverageUtilization: {{ .Values.phpfpm.autoscaling.targetCPUUtilizationPercentage }}
+        {{- if eq $apiVersion "autoscaling/v2beta1" }}
+        targetAverageUtilization: {{ . }}
+        {{- else }}
+        target:
+          averageUtilization: {{ . }}
+          type: Utilization
+        {{- end }}
     {{- end }}
-    {{- if .Values.phpfpm.autoscaling.targetMemoryUtilizationPercentage }}
+    {{- with .Values.phpfpm.autoscaling.targetMemoryUtilizationPercentage }}
     - type: Resource
       resource:
         name: memory
-        targetAverageUtilization: {{ .Values.phpfpm.autoscaling.targetMemoryUtilizationPercentage }}
+        {{- if eq $apiVersion "autoscaling/v2beta1" }}
+        targetAverageUtilization: {{ . }}
+        {{- else }}
+        target:
+          averageUtilization: {{ . }}
+          type: Utilization
+        {{- end }}
     {{- end }}
+  {{- with .Values.nginx.autoscaling.behaviour }}
+  behaviour: {{ toYaml . | nindent 4 }}
+  {{- end }}
 {{- end }}

sickhub/nginx-phpfpm/templates/tests/test-connection.yaml

@@ -11,5 +11,5 @@ spec:
     - name: wget
       image: busybox
       command: ['wget']
-      args: ['{{ include "nginx-phpfpm.fullname" . }}-nginx:{{ .Values.nginx.service.port }}/health.php']
+      args: ['{{ include "nginx-phpfpm.fullname" . }}-nginx:{{ .Values.nginx.service.port }}/healthz.php']
   restartPolicy: Never