Kubernete: Support for tls/x509 redis session connections

openemr · Feb 5, 2023 · 1cc76ba · 1cc76ba
1 parent b3f9f0a
commit 1cc76ba
Show file tree

Hide file tree

Showing 14 changed files with 253 additions and 23 deletions.
diff --git a/kubernetes/certs/redis-openemr-client.yaml b/kubernetes/certs/redis-openemr-client.yaml
@@ -0,0 +1,25 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: redis-openemr-client
+spec:
+  secretName: redis-openemr-client-certs
+  duration: 87660h # 10y
+  renewBefore: 360h # 15d
+  isCA: false
+  privateKey:
+    size: 2048
+    algorithm: RSA
+    encoding: PKCS1
+  usages:
+    - digital signature
+    - key encipherment
+    - client auth
+  subject:
+    organizations:
+      - openemr
+  commonName: openemr
+  issuerRef:
+    name: ca-issuer
+    kind: Issuer
+    group: cert-manager.io
diff --git a/kubernetes/certs/redis.yaml b/kubernetes/certs/redis.yaml
@@ -0,0 +1,25 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: redis
+spec:
+  secretName: redis-certs
+  duration: 87660h # 10y
+  renewBefore: 360h # 15d
+  isCA: false
+  privateKey:
+    size: 2048
+    algorithm: RSA
+    encoding: PKCS1
+  usages:
+    - digital signature
+    - key encipherment
+    - server auth
+  subject:
+    organizations:
+      - redis
+  commonName: redis
+  issuerRef:
+    name: ca-issuer
+    kind: Issuer
+    group: cert-manager.io
diff --git a/kubernetes/certs/redisproxy.yaml b/kubernetes/certs/redisproxy.yaml
@@ -0,0 +1,25 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: redisproxy
+spec:
+  secretName: redisproxy-certs
+  duration: 87660h # 10y
+  renewBefore: 360h # 15d
+  isCA: false
+  privateKey:
+    size: 2048
+    algorithm: RSA
+    encoding: PKCS1
+  usages:
+    - digital signature
+    - key encipherment
+    - server auth
+  subject:
+    organizations:
+      - redisproxy
+  commonName: redisproxy
+  issuerRef:
+    name: ca-issuer
+    kind: Issuer
+    group: cert-manager.io
diff --git a/kubernetes/certs/sentinel.yaml b/kubernetes/certs/sentinel.yaml
@@ -0,0 +1,25 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: sentinel
+spec:
+  secretName: sentinel-certs
+  duration: 87660h # 10y
+  renewBefore: 360h # 15d
+  isCA: false
+  privateKey:
+    size: 2048
+    algorithm: RSA
+    encoding: PKCS1
+  usages:
+    - digital signature
+    - key encipherment
+    - server auth
+  subject:
+    organizations:
+      - sentinel
+  commonName: sentinel
+  issuerRef:
+    name: ca-issuer
+    kind: Issuer
+    group: cert-manager.io
diff --git a/kubernetes/kub-down b/kubernetes/kub-down
@@ -9,6 +9,10 @@ kubectl delete \
     -f certs/mysql-openemr-client.yaml \
     -f certs/phpmyadmin.yaml \
     -f certs/mysql-phpmyadmin-client.yaml
+    -f certs/redis.yaml \
+    -f certs/redis-openemr-client.yaml \
+    -f certs/sentinel.yaml \
+    -f certs/redisproxy.yaml
 
 kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.11.0/cert-manager.yaml
 
@@ -34,4 +38,4 @@ kubectl delete \
     -f volumes/website.yaml \
     -f openemr/secret.yaml \
     -f openemr/deployment.yaml \
-    -f openemr/service.yaml 
+    -f openemr/service.yaml 
diff --git a/kubernetes/kub-down.bat b/kubernetes/kub-down.bat
@@ -9,6 +9,10 @@ kubectl delete ^
     -f certs/mysql-openemr-client.yaml ^
     -f certs/phpmyadmin.yaml ^
     -f certs/mysql-phpmyadmin-client.yaml
+    -f certs/redis.yaml ^
+    -f certs/redis-openemr-client.yaml ^
+    -f certs/sentinel.yaml ^
+    -f certs/redisproxy.yaml
 
 kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v1.11.0/cert-manager.yaml
 

diff --git a/kubernetes/kub-up b/kubernetes/kub-up
@@ -14,6 +14,10 @@ kubectl apply \
     -f certs/mysql-openemr-client.yaml \
     -f certs/phpmyadmin.yaml \
     -f certs/mysql-phpmyadmin-client.yaml
+    -f certs/redis.yaml \
+    -f certs/redis-openemr-client.yaml \
+    -f certs/sentinel.yaml \
+    -f certs/redisproxy.yaml
 echo "...waiting 15 seconds to ensure certs are created..."
 sleep 15
 
@@ -39,4 +43,4 @@ kubectl apply \
     -f volumes/website.yaml \
     -f openemr/secret.yaml \
     -f openemr/deployment.yaml \
-    -f openemr/service.yaml
+    -f openemr/service.yaml
diff --git a/kubernetes/kub-up.bat b/kubernetes/kub-up.bat
@@ -12,6 +12,10 @@ kubectl apply ^
     -f certs/mysql-openemr-client.yaml ^
     -f certs/phpmyadmin.yaml ^
     -f certs/mysql-phpmyadmin-client.yaml
+    -f certs/redis.yaml ^
+    -f certs/redis-openemr-client.yaml ^
+    -f certs/sentinel.yaml ^
+    -f certs/redisproxy.yaml
 timeout 15
 
 kubectl apply ^

diff --git a/kubernetes/openemr/deployment.yaml b/kubernetes/openemr/deployment.yaml
@@ -47,16 +47,22 @@ spec:
         - name: OE_USER
           value: "admin"
         - name: REDIS_SERVER
-          value: "redisproxy"
+          # TODO - change below back to redisproxy after get the proxy working
+          value: "redis"
         - name: REDIS_PASSWORD
           value: "defaultpassword"
+        - name: REDIS_TLS
+          value: "yes"
+        # uncomment below if using redis x509
+        #- name: REDIS_X509
+        #  value: "yes"
         - name: SWARM_MODE
           value: "yes"
         - name: FORCE_DATABASE_SSL_CONNECT
         # uncomment below line (and comment above line) if forcing mysql x509
         #- name: FORCE_DATABASE_X509_CONNECT
           value: "1"
-        image: openemr/openemr:7.0.0
+        image: openemr/openemr:7.0.1
         name: openemr
         ports:
         - containerPort: 80
@@ -68,6 +74,8 @@ spec:
         volumeMounts:
         - mountPath: /root/certs/mysql/server
           name: mysql-openemr-client-certs
+        - mountPath: /root/certs/redis
+          name: redis-openemr-client-certs
         - mountPath: /var/www/localhost/htdocs/openemr/sites
           name: websitevolume
         - mountPath: /etc/ssl
@@ -87,6 +95,17 @@ spec:
           #  path: mysql-cert
           #- key: tls.key
           #  path: mysql-key
+      - name: redis-openemr-client-certs
+        secret:
+          secretName: redis-openemr-client-certs
+          items:
+          - key: ca.crt
+            path: redis-ca
+          # uncomment below if using redis x509
+          #- key: tls.crt
+          #  path: redis-cert
+          #- key: tls.key
+          #  path: redis-key
       - name: websitevolume
         persistentVolumeClaim:
           claimName: websitevolume

diff --git a/kubernetes/redis/configmap-main.yaml b/kubernetes/redis/configmap-main.yaml
@@ -27,7 +27,17 @@ data:
     # this is the second ConfigMap will be mounted to. it has the list of users needed.
     aclfile /conf/acl/users.acl
     
-    # port, each redis nodes will be used
-    port 6379
+    # tls certs and setting
+    tls-cert-file /certs/tls.crt
+    tls-key-file /certs/tls.key
+    tls-ca-cert-file /certs/ca.crt
+    tls-auth-clients no
+    # uncomment below (and comment line above) if using redis x509
+    # tls-auth-clients yes
+    tls-replication yes
+    
+    # port, each redis nodes will be used (only use tls)
+    port 0
+    tls-port 6379
     # More configurations are optional, if not provided, redis will consider default values ------
     # ------ More details on configuration : https://redis.io/docs/manual/config/ ------
diff --git a/kubernetes/redis/configmap-pipy.yaml b/kubernetes/redis/configmap-pipy.yaml
@@ -7,7 +7,9 @@ data:
     {
         "redisAdminUser" : "admin",
         "redisAdminPass" : "adminpassword",
-        "debug" : false,
+        "caCert" : "certs/ca.crt",
+        "tlsKey" : "certs/tls.key",
+        "debug" : true,
         "port" : 6379,
         "servers" : ["redis-0.redis:6379", "redis-1.redis:6379", "redis-2.redis:6379"],
         "connectTimeout" : "1s",
@@ -40,7 +42,15 @@ data:
             role === 'master' && unhealthy_master.remove(_target)
           ))()
       })
-        .listen(config.port)
+        .listen(config.port)    
+        .acceptTLS({
+          certificate: {
+            cert: new crypto.Certificate(pipy.load(config.caCert)),
+            key: new crypto.PrivateKey(pipy.load(config.tlsKey))
+          }
+        }).to('preconnection')
+
+        .pipeline('preconnection')
         .handleData(
           (data, query, command, master_only) => (
             query = new Data(data).shift(20).toString(),
@@ -63,6 +73,14 @@ data:
             config.debug && console.log(`Sending request to node ${_target}`)
           )
         )
+        .connectTLS({
+          certificate: {
+            cert: new crypto.Certificate(pipy.load(config.caCert)),
+            key: new crypto.PrivateKey(pipy.load(config.tlsKey))
+          }
+        }).to('sendconnection')
+    
+        .pipeline('sendconnection')
         .connect(() => _target,
           {
             connectTimeout: config.connectTimeout,
@@ -85,7 +103,8 @@ data:
               unhealthy_nodes.set(t, true),
               unhealthy_master.set(t, true)
             )),
-            _counter = { n: 0 }
+            _counter = { n: 0 },
+            console.log(`Debug 0`)
           )
         )
         .fork('per-node',
@@ -101,9 +120,18 @@ data:
         .replaceMessage(
           () => (
             _counter.n++,
-            new Message(`AUTH ${config.redisAdminUser} ${config.redisAdminPass}\r\ninfo replication\r\n`)
+            new Message(`AUTH ${config.redisAdminUser} ${config.redisAdminPass}\r\ninfo replication\r\n`),
+            console.log(`Debug 1`)
           )
         )
+        .connectTLS({
+          certificate: {
+            cert: new crypto.Certificate(pipy.load(config.caCert)),
+            key: new crypto.PrivateKey(pipy.load(config.tlsKey))
+          }
+        }).to('healthconnection')
+    
+        .pipeline('healthconnection')
         .connect(
           () => _target,
           {
@@ -112,7 +140,8 @@ data:
           }
         )
         .handleData(
-          data => _check(data)
+          data => _check(data),
+          console.log(`Debug 2`)
         )
         .handleStreamEnd(
           () => _counter.n--

diff --git a/kubernetes/redis/deployment-redisproxy.yaml b/kubernetes/redis/deployment-redisproxy.yaml
@@ -24,21 +24,41 @@ spec:
       - name: proxy-init-redis-wait-3
         image: busybox:1.28
         command: ['sh', '-c', "until nslookup redis-2.redis; do echo waiting for redis-2.redis; sleep 10; done"]
+      - name: proxy-init-sentinel-wait-1
+        image: busybox:1.28
+        command: ['sh', '-c', "until nslookup sentinel-0.sentinel; do echo waiting for sentinel-0.sentinel; sleep 10; done"]
+      - name: proxy-init-sentinel-wait-2
+        image: busybox:1.28
+        command: ['sh', '-c', "until nslookup sentinel-1.sentinel; do echo waiting for sentinel-1.sentinel; sleep 10; done"]
+      - name: proxy-init-sentinel-wait-3
+        image: busybox:1.28
+        command: ['sh', '-c', "until nslookup sentinel-2.sentinel; do echo waiting for sentinel-2.sentinel; sleep 10; done"]
       containers:
       - env:
         - name: PIPY_CONFIG_FILE
           value: /proxy/proxy.js
-        image: naqvis/pipy:0.30.0-23
+        image: naqvis/pipy-worker:0.70.0-2-33-g1164e36
         name: redisproxy
         volumeMounts:
+        - name: redisproxy-certs
+          mountPath: /proxy/certs
         - name: redisproxyconf
-          mountPath: /proxy
+          mountPath: /proxy/proxy.js
+          subPath: proxy.js
+        - name: redisproxyconf
+          mountPath: /proxy/config/config.json
+          subPath: config.json
       volumes:
+      - name: redisproxy-certs
+        secret:
+          secretName: redisproxy-certs
+          items:
+          - key: ca.crt
+            path: ca.crt
+          - key: tls.crt
+            path: tls.crt
+          - key: tls.key
+            path: tls.key
       - name: redisproxyconf
         configMap:
-          name: pipy-config
-          items:
-            - key: config.json
-              path: config/config.json
-            - key: proxy.js
-              path: proxy.js
+          name: pipy-config