Support separate subnets for LBs and workers (#827)

* Support separate subnets for LBs and workers Signed-off-by: Mikkel Oscar Lyderik Larsen <[email protected]> * Use kubernetes.io/role/node Signed-off-by: Mikkel Oscar Lyderik Larsen <[email protected]> * Update comment Signed-off-by: Mikkel Oscar Lyderik Larsen <[email protected]> * Rename variable Signed-off-by: Mikkel Oscar Lyderik Larsen <[email protected]> --------- Signed-off-by: Mikkel Oscar Lyderik Larsen <[email protected]>
zalando-incubator · Oct 28, 2024 · 8fa036f · 8fa036f
1 parent fffaa32
commit 8fa036f
Showing 1 changed file with 18 additions and 13 deletions.
diff --git a/provisioner/clusterpy.go b/provisioner/clusterpy.go
@@ -39,6 +39,7 @@ const (
 	defaultNamespace                   = "default"
 	tagNameKubernetesClusterPrefix     = "kubernetes.io/cluster/"
 	subnetELBRoleTagName               = "kubernetes.io/role/elb"
+	subnetNodeRoleTagName              = "kubernetes.io/role/node"
 	resourceLifecycleShared            = "shared"
 	resourceLifecycleOwned             = "owned"
 	mainStackTagKey                    = "cluster-lifecycle-controller.zalando.org/main-stack"
@@ -219,16 +220,19 @@ func (p *clusterpyProvisioner) provision(
 	}
 
 	// find the best subnet for each AZ
-	azInfo := selectSubnetIDs(subnets)
+	azInfoLBs := selectSubnetIDs(subnets, subnetELBRoleTagName)
+	azInfoNodes := selectSubnetIDs(subnets, subnetNodeRoleTagName)
 
 	// if availability zones are defined, filter the subnet list
 	if azNames, ok := cluster.ConfigItems[availabilityZonesConfigItemKey]; ok {
-		azInfo = azInfo.RestrictAZs(strings.Split(azNames, ","))
+		azInfoLBs = azInfoLBs.RestrictAZs(strings.Split(azNames, ","))
+		azInfoNodes = azInfoNodes.RestrictAZs(strings.Split(azNames, ","))
 	}
 
-	// TODO legacy, remove once we switch to Values in all clusters
+	// optional config item to hard-code subnets. This is useful for
+	// migrating between subnets.
 	if _, ok := cluster.ConfigItems[subnetsConfigItemKey]; !ok {
-		cluster.ConfigItems[subnetsConfigItemKey] = azInfo.SubnetsByAZ()[subnetAllAZName]
+		cluster.ConfigItems[subnetsConfigItemKey] = azInfoNodes.SubnetsByAZ()[subnetAllAZName]
 	}
 
 	apiURL, err := url.Parse(cluster.APIServerURL)
@@ -258,9 +262,10 @@ func (p *clusterpyProvisioner) provision(
 	}
 
 	values := map[string]interface{}{
-		subnetsValueKey:             azInfo.SubnetsByAZ(),
-		availabilityZonesValueKey:   azInfo.AvailabilityZones(),
-		subnetIPV6CIDRsKey:          strings.Join(azInfo.SubnetIPv6CIDRs(), ","),
+		subnetsValueKey:             azInfoNodes.SubnetsByAZ(),
+		availabilityZonesValueKey:   azInfoNodes.AvailabilityZones(),
+		subnetIPV6CIDRsKey:          strings.Join(azInfoNodes.SubnetIPv6CIDRs(), ","),
+		"lb_subnets":                azInfoLBs.SubnetsByAZ(),
 		"hosted_zone":               hostedZone,
 		"load_balancer_certificate": loadBalancerCert.ID(),
 		"vpc_ipv4_cidr":             aws.StringValue(vpc.CidrBlock),
@@ -350,7 +355,7 @@ func (p *clusterpyProvisioner) provision(
 			encodeUserData: true,
 			instanceTypes:  instanceTypes,
 		},
-		azInfo: azInfo,
+		azInfo: azInfoNodes,
 	}
 
 	karpenterProvisioner, err := NewKarpenterNodePoolProvisioner(
@@ -618,13 +623,13 @@ func subnetNot(predicate func(*ec2.Subnet) bool) func(*ec2.Subnet) bool {
 	}
 }
 
-// selectSubnetIDs finds the best suiting subnets based on tags for each AZ.
+// selectSubnetIDs finds the best suiting subnets based on tag for each AZ.
 //
 // It follows almost the same logic for finding subnets as the
 // kube-controller-manager when finding subnets for ELBs used for services of
 // type LoadBalancer.
 // https://github.com/kubernetes/kubernetes/blob/65efeee64f772e0f38037e91a677138a335a7570/pkg/cloudprovider/providers/aws/aws.go#L2949-L3027
-func selectSubnetIDs(subnets []*ec2.Subnet) *AZInfo {
+func selectSubnetIDs(subnets []*ec2.Subnet, tag string) *AZInfo {
 	subnetsByAZ := make(map[string]*ec2.Subnet)
 	for _, subnet := range subnets {
 		az := aws.StringValue(subnet.AvailabilityZone)
@@ -635,11 +640,11 @@ func selectSubnetIDs(subnets []*ec2.Subnet) *AZInfo {
 			continue
 		}
 
-		// prefer subnet with an ELB role tag
+		// prefer subnet with a tag
 		existingTags := tagsToMap(existing.Tags)
 		subnetTags := tagsToMap(subnet.Tags)
-		_, existingHasTag := existingTags[subnetELBRoleTagName]
-		_, subnetHasTag := subnetTags[subnetELBRoleTagName]
+		_, existingHasTag := existingTags[tag]
+		_, subnetHasTag := subnetTags[tag]
 
 		if existingHasTag != subnetHasTag {
 			if subnetHasTag {