feature: VPC Egress Gateway

Signed-off-by: zhangzujian <[email protected]>
kubeovn · Nov 13, 2024 · 181d247 · 181d247
1 parent ad73b1f
commit 181d247
Show file tree

Hide file tree

Showing 45 changed files with 2,746 additions and 155 deletions.
diff --git a/charts/kube-ovn/templates/controller-deploy.yaml b/charts/kube-ovn/templates/controller-deploy.yaml
@@ -139,6 +139,7 @@ spec:
           - --enable-anp={{- .Values.func.ENABLE_ANP }}
           - --ovsdb-con-timeout={{- .Values.func.OVSDB_CON_TIMEOUT }}
           - --ovsdb-inactivity-timeout={{- .Values.func.OVSDB_INACTIVITY_TIMEOUT }}
+          - --image={{ .Values.global.registry.address }}/{{ .Values.global.images.kubeovn.repository }}:{{ .Values.global.images.kubeovn.tag }}
           securityContext:
             runAsUser: {{ include "kubeovn.runAsUser" . }}
             privileged: false

diff --git a/charts/kube-ovn/templates/kube-ovn-crd.yaml b/charts/kube-ovn/templates/kube-ovn-crd.yaml
@@ -823,6 +823,227 @@ spec:
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
+metadata:
+  name: vpc-egress-gateways.kubeovn.io
+spec:
+  group: kubeovn.io
+  names:
+    plural: vpc-egress-gateways
+    singular: vpc-egress-gateway
+    shortNames:
+      - vpc-egress-gw
+      - veg
+    kind: VpcEgressGateway
+    listKind: VpcEgressGatewayList
+  scope: Namespaced
+  versions:
+    - additionalPrinterColumns:
+        - jsonPath: .spec.vpc
+          name: VPC
+          type: string
+        - jsonPath: .spec.replicas
+          name: REPLICAS
+          type: integer
+        - jsonPath: .spec.externalSubnet
+          name: EXTERNALSUBNET
+          type: string
+        - jsonPath: .status.ready
+          name: READY
+          type: boolean
+      name: v1
+      served: true
+      storage: true
+      subresources:
+        status: {}
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            status:
+              properties:
+                conditions:
+                  items:
+                    properties:
+                      lastTransitionTime:
+                        format: date-time
+                        type: string
+                      lastUpdateTime:
+                        format: date-time
+                        type: string
+                      message:
+                        maxLength: 32768
+                        type: string
+                      observedGeneration:
+                        format: int64
+                        minimum: 0
+                        type: integer
+                      reason:
+                        maxLength: 1024
+                        minLength: 1
+                        pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+                        type: string
+                      status:
+                        enum:
+                          - "True"
+                          - "False"
+                          - Unknown
+                        type: string
+                      type:
+                        maxLength: 316
+                        pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                        type: string
+                    required:
+                      - lastTransitionTime
+                      - lastUpdateTime
+                      - observedGeneration
+                      - reason
+                      - status
+                      - type
+                    type: object
+                  type: array
+                  x-kubernetes-list-map-keys:
+                    - type
+                  x-kubernetes-list-type: map
+                internalIPs:
+                  items:
+                    type: string
+                  type: array
+                externalIPs:
+                  items:
+                    type: string
+                  type: array
+                phase:
+                  type: string
+                  default: Pending
+                  enum:
+                    - Pending
+                    - Processing
+                    - Completed
+                ready:
+                  type: boolean
+                  default: false
+                workload:
+                  type: string
+              required:
+                - conditions
+                - phase
+              type: object
+            spec:
+              type: object
+              required:
+                - vpc
+                - externalSubnet
+              properties:
+                replicas:
+                  type: integer
+                  default: 1
+                  minimum: 1
+                vpc:
+                  type: string
+                internalSubnet:
+                  type: string
+                externalSubnet:
+                  type: string
+                image:
+                  type: string
+                bfd:
+                  type: object
+                  properties:
+                    enabled:
+                      type: boolean
+                      default: false
+                    minRX:
+                      type: integer
+                      default: 1000
+                    minTX:
+                      type: integer
+                      default: 1000
+                    multiplier:
+                      type: integer
+                      default: 3
+                policies:
+                  type: array
+                  items:
+                    type: object
+                    properties:
+                      snat:
+                        type: boolean
+                        default: false
+                      ipBlocks:
+                        type: array
+                        x-kubernetes-list-type: set
+                        items:
+                          type: string
+                          anyOf:
+                            - format: ipv4
+                            - format: ipv6
+                            - format: cidr
+                      subnets:
+                        type: array
+                        x-kubernetes-list-type: set
+                        items:
+                          type: string
+                          minLength: 1
+                nodeSelector:
+                  type: array
+                  items:
+                    type: object
+                    properties:
+                      matchLabels:
+                        additionalProperties:
+                          type: string
+                        type: object
+                      matchExpressions:
+                        type: array
+                        items:
+                          type: object
+                          properties:
+                            key:
+                              type: string
+                            operator:
+                              type: string
+                              enum:
+                                - In
+                                - NotIn
+                                - Exists
+                                - DoesNotExist
+                                - Gt
+                                - Lt
+                            values:
+                              type: array
+                              x-kubernetes-list-type: set
+                              items:
+                                type: string
+                          required:
+                            - key
+                            - operator
+                      matchFields:
+                        type: array
+                        items:
+                          type: object
+                          properties:
+                            key:
+                              type: string
+                            operator:
+                              type: string
+                              enum:
+                                - In
+                                - NotIn
+                                - Exists
+                                - DoesNotExist
+                                - Gt
+                                - Lt
+                            values:
+                              type: array
+                              x-kubernetes-list-type: set
+                              items:
+                                type: string
+                          required:
+                            - key
+                            - operator
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
 metadata:
   name: iptables-eips.kubeovn.io
 spec:
@@ -1669,6 +1890,7 @@ spec:
                   properties:
                     enabled:
                       type: boolean
+                      default: false
                     ip:
                       type: string
                       anyOf:
@@ -1770,8 +1992,11 @@ spec:
                   properties:
                     enabled:
                       type: boolean
+                      default: false
                     ip:
                       type: string
+                    name:
+                      type: string
                     nodes:
                       type: array
                       items:

diff --git a/charts/kube-ovn/templates/ovn-CR.yaml b/charts/kube-ovn/templates/ovn-CR.yaml
@@ -13,6 +13,8 @@ rules:
       - vpcs/status
       - vpc-nat-gateways
       - vpc-nat-gateways/status
+      - vpc-egress-gateways
+      - vpc-egress-gateways/status
       - subnets
       - subnets/status
       - ippools
@@ -98,6 +100,17 @@ rules:
       - daemonsets
     verbs:
       - get
+  - apiGroups:
+      - apps
+    resources:
+      - deployments
+      - deployments/scale
+    verbs:
+      - get
+      - list
+      - watch
+      - create
+      - update
   - apiGroups:
       - ""
     resources:
@@ -124,8 +137,6 @@ rules:
       - apps
     resources:
       - statefulsets
-      - deployments
-      - deployments/scale
     verbs:
       - get
       - list

diff --git a/charts/kube-ovn/templates/ovncni-ds.yaml b/charts/kube-ovn/templates/ovncni-ds.yaml
@@ -35,11 +35,7 @@ spec:
         command:
           - sh
           - -xec
-          - {{ if not .Values.DISABLE_MODULES_MANAGEMENT -}}
-            iptables -V
-            {{- else -}}
-            echo "nothing to do"
-            {{- end }}
+          - iptables -V
         securityContext:
           allowPrivilegeEscalation: true
           capabilities:
@@ -128,9 +124,6 @@ spec:
               - NET_RAW
               - SYS_ADMIN
               - SYS_PTRACE
-              {{- if not .Values.DISABLE_MODULES_MANAGEMENT }}
-              - SYS_MODULE
-              {{- end }}
               - SYS_NICE
         env:
           - name: ENABLE_SSL

diff --git a/charts/kube-ovn/templates/ovsovn-ds.yaml b/charts/kube-ovn/templates/ovsovn-ds.yaml
@@ -96,11 +96,11 @@ spec:
               add:
                 - NET_ADMIN
                 - NET_BIND_SERVICE
+                - SYS_ADMIN
                 {{- if not .Values.DISABLE_MODULES_MANAGEMENT }}
                 - SYS_MODULE
                 {{- end }}
                 - SYS_NICE
-                - SYS_ADMIN
           env:
             - name: ENABLE_SSL
               value: "{{ .Values.networking.ENABLE_SSL }}"

diff --git a/dist/images/Dockerfile.base b/dist/images/Dockerfile.base
@@ -142,13 +142,14 @@ RUN apt update && apt upgrade -y && apt install ca-certificates python3 hostname
         setcap CAP_NET_RAW+eip $(readlink -f $(which ndisc6)) && \
         setcap CAP_NET_RAW+eip $(readlink -f $(which tcpdump)) && \
         setcap CAP_NET_ADMIN+eip $(readlink -f $(which ethtool)) && \
+        setcap CAP_NET_ADMIN+eip $(readlink -f $(which conntrack)) && \
         setcap CAP_SYS_ADMIN+eip $(readlink -f $(which nsenter)) && \
+        setcap CAP_SYS_ADMIN+eip $(readlink -f $(which sysctl)) && \
         setcap CAP_SYS_MODULE+eip $(readlink -f $(which modprobe)) && \
-        setcap CAP_NET_ADMIN+eip $(readlink -f $(which conntrack)) && \
-        setcap CAP_NET_RAW,CAP_NET_ADMIN,CAP_SYS_MODULE+eip $(readlink -f $(which ipset)) && \
-        setcap CAP_NET_RAW,CAP_NET_ADMIN,CAP_SYS_MODULE+eip $(readlink -f $(which xtables-legacy-multi)) && \
-        setcap CAP_NET_RAW,CAP_NET_ADMIN,CAP_SYS_MODULE+eip $(readlink -f $(which xtables-nft-multi)) && \
-        setcap CAP_NET_RAW,CAP_NET_ADMIN,CAP_SYS_MODULE,CAP_SYS_ADMIN+eip $(readlink -f $(which ip)) && \
+        setcap CAP_NET_RAW,CAP_NET_ADMIN+eip $(readlink -f $(which ipset)) && \
+        setcap CAP_NET_RAW,CAP_NET_ADMIN+eip $(readlink -f $(which xtables-legacy-multi)) && \
+        setcap CAP_NET_RAW,CAP_NET_ADMIN+eip $(readlink -f $(which xtables-nft-multi)) && \
+        setcap CAP_NET_ADMIN,CAP_SYS_ADMIN+eip $(readlink -f $(which ip)) && \
         rm -rf /var/lib/apt/lists/* && \
         rm -rf /etc/localtime && \
         rm -f /usr/bin/nc && \

diff --git a/dist/images/bfdd-prestart.sh b/dist/images/bfdd-prestart.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -ex
+
+bfdd-control session new set mintx "${BFD_MIN_TX:-1000}"
+bfdd-control session new set minrx "${BFD_MIN_RX:-1000}"
+bfdd-control session new set multi "${BFD_MULTI:-3}"
+
+PEER_IPS=($(echo "${BFD_PEER_IPS:-::}" | tr ',' ' '))
+for ip in ${PEER_IPS[*]}; do
+  bfdd-control allow ${ip}
+done
diff --git a/dist/images/cleanup.sh b/dist/images/cleanup.sh
@@ -125,6 +125,7 @@ kubectl delete --ignore-not-found crd \
   security-groups.kubeovn.io \
   ippools.kubeovn.io \
   vpc-nat-gateways.kubeovn.io \
+  vpc-egress-gateways.kubeovn.io \
   vpcs.kubeovn.io \
   vlans.kubeovn.io \
   provider-networks.kubeovn.io \

diff --git a/dist/images/init-vpc-egress-gateway.sh b/dist/images/init-vpc-egress-gateway.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -ex
+
+INTERNAL_GATEWAY_IPV4=${INTERNAL_GATEWAY_IPV4:-}
+INTERNAL_GATEWAY_IPV6=${INTERNAL_GATEWAY_IPV6:-}
+INTERNAL_ROUTE_DST_IPV4=($(echo "${INTERNAL_ROUTE_DST_IPV4:-}" | tr ',' ' '))
+INTERNAL_ROUTE_DST_IPV6=($(echo "${INTERNAL_ROUTE_DST_IPV6:-}" | tr ',' ' '))
+SNAT_SOURCES_IPV4=($(echo "${SNAT_SOURCES_IPV4:-}" | tr ',' ' '))
+SNAT_SOURCES_IPV6=($(echo "${SNAT_SOURCES_IPV6:-}" | tr ',' ' '))
+
+sysctl -w net.ipv4.ip_forward=1
+sysctl -w net.ipv6.conf.all.forwarding=1
+
+for dst in ${INTERNAL_ROUTE_DST_IPV4[*]}; do 
+  ip route add "${dst}" via "${INTERNAL_GATEWAY_IPV4}"
+done
+
+for dst in ${INTERNAL_ROUTE_DST_IPV6[*]}; do 
+  ip route add "${dst}" via "${INTERNAL_GATEWAY_IPV6}"
+done
+
+for src in ${SNAT_SOURCES_IPV4[*]}; do 
+  iptables -t nat -A POSTROUTING -s "${src}" -j MASQUERADE --random-fully
+done
+
+for src in ${SNAT_SOURCES_IPV6[*]}; do
+  ip6tables -t nat -A POSTROUTING -s "${src}" -j MASQUERADE --random-fully
+done