Merge pull request #8487 from zalando-incubator/eks-fix-cluster-tag

Set correct cluster tag on EKS
zalando-incubator · Nov 1, 2024 · a48afc1 · a48afc1
2 parents 4449551 + a935713
commit a48afc1
Show file tree

Hide file tree

Showing 6 changed files with 15 additions and 83 deletions.
diff --git a/cluster/cluster.yaml b/cluster/cluster.yaml
@@ -5,6 +5,8 @@ Metadata:
     InfrastructureComponent: "true"
     application: "kubernetes"
     cluster-provider: "{{ .Cluster.Provider }}"
+    "kubernetes.io/cluster/{{.Cluster.Name}}": "owned"
+
 Resources:
 {{ if eq .Cluster.Provider "zalando-eks" }}
   EKSClusterRole:
@@ -27,18 +29,11 @@ Resources:
   EKSControlPlaneSecurityGroup:
     Type: AWS::EC2::SecurityGroup
     Properties:
-      # GroupDescription: "{{ .Cluster.Alias }}-eks-control-plane"
       GroupDescription: "{{ .Cluster.Alias }}-control-plane"
-      # TODO:
-      # SecurityGroupIngress:
-      Tags:
-        - Key: 'kubernetes.io/cluster/{{.Cluster.ID}}'
-          Value: owned
       VpcId: "{{.Cluster.ConfigItems.vpc_id}}"
   EKSWorkerSecurityGroup:
     Type: "AWS::EC2::SecurityGroup"
     Properties:
-      # GroupDescription: "{{ .Cluster.ID }}-eks-worker-sg"
       GroupDescription: "{{ .Cluster.ID }}-worker-sg"
       SecurityGroupIngress:
         - CidrIp: 0.0.0.0/0
@@ -99,8 +94,6 @@ Resources:
           IpProtocol: udp
           ToPort: 53
       Tags:
-        - Key: 'kubernetes.io/cluster/{{.Cluster.ID}}'
-          Value: owned
         - Key: 'karpenter.sh/discovery'
           Value: '{{ .Cluster.ID }}/WorkerNodeSecurityGroup'
       VpcId: "{{.Cluster.ConfigItems.vpc_id}}"
@@ -122,7 +115,7 @@ Resources:
   EKSCluster:
     Type: AWS::EKS::Cluster
     Properties:
-      Name: "{{.Cluster.ID | eksID}}"
+      Name: "{{.Cluster.Name}}"
       Version: "1.31"
       RoleArn: !GetAtt EKSClusterRole.Arn
       KubernetesNetworkConfig:
@@ -383,9 +376,6 @@ Resources:
       GroupId: !Ref MasterSecurityGroup
       IpProtocol: tcp
       SourceSecurityGroupId: !Ref MasterSecurityGroup
-      Tags:
-        - Key: 'kubernetes.io/cluster/{{.Cluster.ID}}'
-          Value: owned
       ToPort: 10250
     Type: 'AWS::EC2::SecurityGroupIngress'
   MasterLoadBalancerNLB:
@@ -403,8 +393,6 @@ Resources:
   {{ end }}
   {{ end }}
       Tags:
-        - Key: 'kubernetes.io/cluster/{{.Cluster.ID}}'
-          Value: owned
         - Key: "component"
           Value: "kube-apiserver"
       Type: network
@@ -421,8 +409,6 @@ Resources:
       Port: 8443
       Protocol: TLS
       Tags:
-        - Key: 'kubernetes.io/cluster/{{.Cluster.ID}}'
-          Value: owned
         - Key: "component"
           Value: "kube-apiserver"
       VpcId: "{{.Cluster.ConfigItems.vpc_id}}"
@@ -498,9 +484,6 @@ Resources:
           FromPort: 53
           IpProtocol: udp
           ToPort: 53
-      Tags:
-        - Key: 'kubernetes.io/cluster/{{.Cluster.ID}}'
-          Value: owned
       VpcId: "{{.Cluster.ConfigItems.vpc_id}}"
     Type: 'AWS::EC2::SecurityGroup'
   MasterSecurityGroupIngressFromFlannelToMaster:
@@ -509,9 +492,6 @@ Resources:
       GroupId: !Ref MasterSecurityGroup
       IpProtocol: udp
       SourceSecurityGroupId: !Ref WorkerSecurityGroup
-      Tags:
-        - Key: 'kubernetes.io/cluster/{{.Cluster.ID}}'
-          Value: owned
       ToPort: 8472
     Type: 'AWS::EC2::SecurityGroupIngress'
   MasterSecurityGroupIngressFromMasterFlannelToMaster:
@@ -520,9 +500,6 @@ Resources:
       GroupId: !Ref MasterSecurityGroup
       IpProtocol: udp
       SourceSecurityGroupId: !Ref MasterSecurityGroup
-      Tags:
-        - Key: 'kubernetes.io/cluster/{{.Cluster.ID}}'
-          Value: owned
       ToPort: 8472
     Type: 'AWS::EC2::SecurityGroupIngress'
   MasterSecurityGroupIngressFromMaster:
@@ -531,9 +508,6 @@ Resources:
       GroupId: !Ref MasterSecurityGroup
       IpProtocol: tcp
       SourceSecurityGroupId: !Ref MasterSecurityGroup
-      Tags:
-        - Key: 'kubernetes.io/cluster/{{.Cluster.ID}}'
-          Value: owned
       ToPort: 443
     Type: 'AWS::EC2::SecurityGroupIngress'
   MasterSecurityGroupIngressFromWorker:
@@ -542,9 +516,6 @@ Resources:
       GroupId: !Ref MasterSecurityGroup
       IpProtocol: tcp
       SourceSecurityGroupId: !Ref WorkerSecurityGroup
-      Tags:
-        - Key: 'kubernetes.io/cluster/{{.Cluster.ID}}'
-          Value: owned
       ToPort: 443
     Type: 'AWS::EC2::SecurityGroupIngress'
   MasterSecurityGroupIngressFromWorkerToMasterKubeletAndKubeProxy:
@@ -553,9 +524,6 @@ Resources:
       GroupId: !Ref MasterSecurityGroup
       IpProtocol: tcp
       SourceSecurityGroupId: !Ref WorkerSecurityGroup
-      Tags:
-        - Key: 'kubernetes.io/cluster/{{.Cluster.ID}}'
-          Value: owned
       ToPort: 10250 # Kubelet
     Type: 'AWS::EC2::SecurityGroupIngress'
   MasterSecurityGroupIngressFromWorkerToNodeMonitor:
@@ -565,9 +533,6 @@ Resources:
       GroupId: !Ref MasterSecurityGroup
       IpProtocol: tcp
       SourceSecurityGroupId: !Ref WorkerSecurityGroup
-      Tags:
-        - Key: 'kubernetes.io/cluster/{{.Cluster.ID}}'
-          Value: owned
     Type: 'AWS::EC2::SecurityGroupIngress'
   MasterSecurityGroupIngressFromWorkerToSkipperMetrics:
     Properties:
@@ -576,9 +541,6 @@ Resources:
       GroupId: !Ref MasterSecurityGroup
       IpProtocol: tcp
       SourceSecurityGroupId: !Ref WorkerSecurityGroup
-      Tags:
-        - Key: 'kubernetes.io/cluster/{{.Cluster.ID}}'
-          Value: owned
     Type: 'AWS::EC2::SecurityGroupIngress'
   WorkerSecurityGroup:
     Properties:
@@ -643,8 +605,6 @@ Resources:
           IpProtocol: udp
           ToPort: 53
       Tags:
-        - Key: 'kubernetes.io/cluster/{{.Cluster.ID}}'
-          Value: owned
         - Key: 'karpenter.sh/discovery'
           Value: '{{ .Cluster.ID }}/WorkerNodeSecurityGroup'
       VpcId: "{{.Cluster.ConfigItems.vpc_id}}"
@@ -655,9 +615,6 @@ Resources:
       GroupId: !Ref WorkerSecurityGroup
       IpProtocol: udp
       SourceSecurityGroupId: !Ref MasterSecurityGroup
-      Tags:
-        - Key: 'kubernetes.io/cluster/{{.Cluster.ID}}'
-          Value: owned
       ToPort: 8472
     Type: 'AWS::EC2::SecurityGroupIngress'
   WorkerSecurityGroupIngressFromMasterToKubelet:
@@ -666,9 +623,6 @@ Resources:
       GroupId: !Ref WorkerSecurityGroup
       IpProtocol: tcp
       SourceSecurityGroupId: !Ref MasterSecurityGroup
-      Tags:
-        - Key: 'kubernetes.io/cluster/{{.Cluster.ID}}'
-          Value: owned
       ToPort: 10250
     Type: 'AWS::EC2::SecurityGroupIngress'
   WorkerSecurityGroupIngressFromWorkerToFlannel:
@@ -677,9 +631,6 @@ Resources:
       GroupId: !Ref WorkerSecurityGroup
       IpProtocol: udp
       SourceSecurityGroupId: !Ref WorkerSecurityGroup
-      Tags:
-        - Key: 'kubernetes.io/cluster/{{.Cluster.ID}}'
-          Value: owned
       ToPort: 8472
     Type: 'AWS::EC2::SecurityGroupIngress'
   WorkerSecurityGroupIngressFromWorkerToWorkerKubeletAndKubeProxy:
@@ -688,9 +639,6 @@ Resources:
       GroupId: !Ref WorkerSecurityGroup
       IpProtocol: tcp
       SourceSecurityGroupId: !Ref WorkerSecurityGroup
-      Tags:
-        - Key: 'kubernetes.io/cluster/{{.Cluster.ID}}'
-          Value: owned
       ToPort: 10250 # Kubelet
     Type: 'AWS::EC2::SecurityGroupIngress'
   WorkerSecurityGroupIngressFromWorkerToWorkerSkipperMetrics:
@@ -699,9 +647,6 @@ Resources:
       GroupId: !Ref WorkerSecurityGroup
       IpProtocol: tcp
       SourceSecurityGroupId: !Ref WorkerSecurityGroup
-      Tags:
-        - Key: 'kubernetes.io/cluster/{{.Cluster.ID}}'
-          Value: owned
       ToPort: 9911
     Type: 'AWS::EC2::SecurityGroupIngress'
   WorkerSecurityGroupIngressFromWorkerToWorkerSkipperTokeninfoMetrics:
@@ -710,9 +655,6 @@ Resources:
       GroupId: !Ref WorkerSecurityGroup
       IpProtocol: tcp
       SourceSecurityGroupId: !Ref WorkerSecurityGroup
-      Tags:
-        - Key: 'kubernetes.io/cluster/{{.Cluster.ID}}'
-          Value: owned
       ToPort: 9022
     Type: 'AWS::EC2::SecurityGroupIngress'
   WorkerSecurityGroupIngressFromWorkerToNodeMonitor:
@@ -722,27 +664,18 @@ Resources:
       GroupId: !Ref WorkerSecurityGroup
       IpProtocol: tcp
       SourceSecurityGroupId: !Ref WorkerSecurityGroup
-      Tags:
-        - Key: 'kubernetes.io/cluster/{{.Cluster.ID}}'
-          Value: owned
     Type: 'AWS::EC2::SecurityGroupIngress'
   EFSSecurityGroupIngressFromWorkerSecurityGroup:
     Properties:
       FromPort: 2049
       GroupId: !Ref EFSWorkerSecurityGroup
       IpProtocol: tcp
       SourceSecurityGroupId: !Ref WorkerSecurityGroup
-      Tags:
-        - Key: 'kubernetes.io/cluster/{{.Cluster.ID}}'
-          Value: owned
       ToPort: 2049
     Type: 'AWS::EC2::SecurityGroupIngress'
   EFSWorkerSecurityGroup:
     Properties:
       GroupDescription: worker to EFS sg
-      Tags:
-        - Key: 'kubernetes.io/cluster/{{.Cluster.ID}}'
-          Value: owned
       VpcId: "{{.Cluster.ConfigItems.vpc_id}}"
     Type: 'AWS::EC2::SecurityGroup'
 
@@ -774,8 +707,6 @@ Resources:
           IpProtocol: tcp
           ToPort: 443
       Tags:
-        - Key: 'kubernetes.io/cluster/{{.Cluster.ID}}'
-          Value: owned
         - Key: 'kubernetes:application'
           Value: kube-ingress-aws-controller
       VpcId: "{{.Cluster.ConfigItems.vpc_id}}"
@@ -1020,7 +951,7 @@ Resources:
                   ],
                   "Condition": {
                     "StringEquals": {
-                      "aws:RequestTag/kubernetes.io/cluster/{{.Cluster.ID}}": "owned"
+                      "aws:RequestTag/kubernetes.io/cluster/{{.Cluster.Name}}": "owned"
                     },
                     "StringLike": {
                       "aws:RequestTag/karpenter.sh/nodepool": "*"
@@ -1041,7 +972,7 @@ Resources:
                   "Action": "ec2:CreateTags",
                   "Condition": {
                     "StringEquals": {
-                      "aws:RequestTag/kubernetes.io/cluster/{{.Cluster.ID}}": "owned",
+                      "aws:RequestTag/kubernetes.io/cluster/{{.Cluster.Name}}": "owned",
                       "ec2:CreateAction": [
                         "RunInstances",
                         "CreateFleet",
@@ -1060,7 +991,7 @@ Resources:
                   "Action": "ec2:CreateTags",
                   "Condition": {
                     "StringEquals": {
-                      "aws:ResourceTag/kubernetes.io/cluster/{{.Cluster.ID}}": "owned"
+                      "aws:ResourceTag/kubernetes.io/cluster/{{.Cluster.Name}}": "owned"
                     },
                     "StringLike": {
                       "aws:ResourceTag/karpenter.sh/nodepool": "*"
@@ -1086,7 +1017,7 @@ Resources:
                   ],
                   "Condition": {
                     "StringEquals": {
-                      "aws:ResourceTag/kubernetes.io/cluster/{{.Cluster.ID}}": "owned"
+                      "aws:ResourceTag/kubernetes.io/cluster/{{.Cluster.Name}}": "owned"
                     },
                     "StringLike": {
                       "aws:ResourceTag/karpenter.sh/nodepool": "*"
@@ -1156,7 +1087,7 @@ Resources:
                   ],
                   "Condition": {
                     "StringEquals": {
-                      "aws:RequestTag/kubernetes.io/cluster/{{.Cluster.ID}}": "owned",
+                      "aws:RequestTag/kubernetes.io/cluster/{{.Cluster.Name}}": "owned",
                       "aws:RequestTag/topology.kubernetes.io/region": "${AWS::Region}"
                     },
                     "StringLike": {
@@ -1173,9 +1104,9 @@ Resources:
                   ],
                   "Condition": {
                     "StringEquals": {
-                      "aws:ResourceTag/kubernetes.io/cluster/{{.Cluster.ID}}": "owned",
+                      "aws:ResourceTag/kubernetes.io/cluster/{{.Cluster.Name}}": "owned",
+                      "aws:RequestTag/kubernetes.io/cluster/{{.Cluster.Name}}": "owned",
                       "aws:ResourceTag/topology.kubernetes.io/region": "${AWS::Region}",
-                      "aws:RequestTag/kubernetes.io/cluster/{{.Cluster.ID}}": "owned",
                       "aws:RequestTag/topology.kubernetes.io/region": "${AWS::Region}"
                     },
                     "StringLike": {
@@ -1195,7 +1126,7 @@ Resources:
                   ],
                   "Condition": {
                     "StringEquals": {
-                      "aws:ResourceTag/kubernetes.io/cluster/{{.Cluster.ID}}": "owned",
+                      "aws:ResourceTag/kubernetes.io/cluster/{{.Cluster.Name}}": "owned",
                       "aws:ResourceTag/topology.kubernetes.io/region": "${AWS::Region}"
                     },
                     "StringLike": {

diff --git a/cluster/manifests/ingress-controller/deployment.yaml b/cluster/manifests/ingress-controller/deployment.yaml
@@ -65,7 +65,7 @@ spec:
             # {{ end }}
           env:
             - name: CUSTOM_FILTERS
-              value: "tag:kubernetes.io/cluster/{{ .Cluster.ID }}=owned tag:node.kubernetes.io/role=worker tag:zalando.org/ingress-enabled=true"
+              value: "tag:kubernetes.io/cluster/{{.Cluster.Name}}=owned tag:node.kubernetes.io/role=worker tag:zalando.org/ingress-enabled=true"
             - name: AWS_REGION
               value: "{{ .Cluster.Region }}"
           resources:

diff --git a/cluster/manifests/z-karpenter/deployment.yaml b/cluster/manifests/z-karpenter/deployment.yaml
@@ -96,7 +96,7 @@ spec:
             - name: ASSUME_ROLE_DURATION
               value: "15m"
             - name: CLUSTER_NAME
-              value: "{{.Cluster.ID}}"
+              value: "{{.Cluster.Name }}"
             - name: VM_MEMORY_OVERHEAD_PERCENT
               value: "0.075"
             - name: RESERVED_ENIS

diff --git a/cluster/node-pools/worker-combined/stack.yaml b/cluster/node-pools/worker-combined/stack.yaml
@@ -5,6 +5,7 @@ Metadata:
     InfrastructureComponent: "true"
     application: "kubernetes"
     component: "shared-resource"
+    "kubernetes.io/cluster/{{.Cluster.Name}}": "owned"
 
 Mappings:
   Images:

diff --git a/cluster/node-pools/worker-splitaz/stack.yaml b/cluster/node-pools/worker-splitaz/stack.yaml
@@ -5,6 +5,7 @@ Metadata:
     InfrastructureComponent: "true"
     application: "kubernetes"
     component: "shared-resource"
+    "kubernetes.io/cluster/{{.Cluster.Name}}": "owned"
 
 Mappings:
   Images:

diff --git a/test/e2e/run_e2e.sh b/test/e2e/run_e2e.sh
@@ -205,7 +205,6 @@ if [ "$e2e" = true ]; then
             "Mirror pods should be created for the main Kubernetes components \[Zalando\]"
             "Should audit API calls to create, update, patch, delete pods. \[Audit\] \[Zalando\]"
             "should validate permissions for \[Authorization\] \[RBAC\] \[Zalando\]" # TODO: temporary disabled because feature is missing
-            "Should create DNS entry \[Zalando\]" # TODO: broken because type: LoadBalancer doesn't work
         )
     fi