From 6e4d54306d0bf0b3dd184c167a829e56dca6a3f3 Mon Sep 17 00:00:00 2001 From: Jason Deal Date: Wed, 27 Mar 2024 12:05:53 -0700 Subject: [PATCH] test: migrate E2E tests to AL2023 --- test/pkg/environment/aws/environment.go | 2 +- test/suites/consolidation/suite_test.go | 22 +++++++++++++-- test/suites/drift/suite_test.go | 28 ++++++++++--------- test/suites/integration/ami_test.go | 24 ++++++---------- .../integration/block_device_mappings_test.go | 4 +-- .../integration/extended_resources_test.go | 5 +++- 6 files changed, 50 insertions(+), 35 deletions(-) diff --git a/test/pkg/environment/aws/environment.go b/test/pkg/environment/aws/environment.go index c243c7c07205..af9045ffd56b 100644 --- a/test/pkg/environment/aws/environment.go +++ b/test/pkg/environment/aws/environment.go @@ -122,7 +122,7 @@ func GetTimeStreamAPI(session *session.Session) timestreamwriteiface.TimestreamW func (env *Environment) DefaultEC2NodeClass() *v1beta1.EC2NodeClass { nodeClass := test.EC2NodeClass() - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 + nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2023 nodeClass.Spec.Tags = map[string]string{ "testing/cluster": env.ClusterName, } diff --git a/test/suites/consolidation/suite_test.go b/test/suites/consolidation/suite_test.go index 5c8284712b9f..78ff842da802 100644 --- a/test/suites/consolidation/suite_test.go +++ b/test/suites/consolidation/suite_test.go @@ -426,9 +426,9 @@ var _ = Describe("Consolidation", func() { NodeSelectorRequirement: v1.NodeSelectorRequirement{ Key: v1beta1.LabelInstanceFamily, Operator: v1.NodeSelectorOpNotIn, - // remove some cheap burstable and the odd c1 instance types so we have + // remove some cheap burstable and the odd c1 / a1 instance types so we have // more control over what gets provisioned - Values: []string{"t2", "t3", "c1", "t3a", "t4g"}, + Values: []string{"t2", "t3", "c1", "t3a", "t4g", "a1"}, }, }, }, @@ -500,6 +500,15 @@ var _ = Describe("Consolidation", func() { Values: []string{"large", "2xlarge"}, }, }, + { + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceFamily, + Operator: v1.NodeSelectorOpNotIn, + // remove some cheap burstable and the odd c1 / a1 instance types so we have + // more control over what gets provisioned + Values: []string{"t2", "t3", "c1", "t3a", "t4g", "a1"}, + }, + }, }, NodeClassRef: &corev1beta1.NodeClassReference{Name: nodeClass.Name}, }, @@ -627,6 +636,15 @@ var _ = Describe("Consolidation", func() { Values: []string{"large"}, }, }, + { + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceFamily, + Operator: v1.NodeSelectorOpNotIn, + // remove some cheap burstable and the odd c1 / a1 instance types so we have + // more control over what gets provisioned + Values: []string{"t2", "t3", "c1", "t3a", "t4g", "a1"}, + }, + }, }, NodeClassRef: &corev1beta1.NodeClassReference{Name: nodeClass.Name}, }, diff --git a/test/suites/drift/suite_test.go b/test/suites/drift/suite_test.go index 2011d29fcf9e..49ceb84a55d0 100644 --- a/test/suites/drift/suite_test.go +++ b/test/suites/drift/suite_test.go @@ -80,7 +80,7 @@ var _ = Describe("Drift", func() { var selector labels.Selector var numPods int BeforeEach(func() { - amdAMI = env.GetCustomAMI("/aws/service/eks/optimized-ami/%s/amazon-linux-2/recommended/image_id", 1) + amdAMI = env.GetCustomAMI("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/x86_64/standard/recommended/image_id", 1) numPods = 1 // Add pods with a do-not-disrupt annotation so that we can check node metadata before we disrupt dep = coretest.Deployment(coretest.DeploymentOptions{ @@ -399,15 +399,14 @@ var _ = Describe("Drift", func() { It("should disrupt nodes that have drifted due to AMIs", func() { // Choose and old, static image. The 1.23 image is incompatible with EKS 1.29 so fallback to a newer image. parameterName := lo.Ternary(lo.Must(strconv.Atoi(strings.Split(env.GetK8sVersion(0), ".")[1])) >= 29, - "/aws/service/eks/optimized-ami/1.27/amazon-linux-2/amazon-eks-node-1.27-v20240129/image_id", - "/aws/service/eks/optimized-ami/1.23/amazon-linux-2/amazon-eks-node-1.23-v20230322/image_id", + "/aws/service/eks/optimized-ami/1.27/amazon-linux-2023/x86_64/standard/amazon-eks-node-al2023-x86_64-standard-1.27-v20240307/image_id", + "/aws/service/eks/optimized-ami/1.23/amazon-linux-2023/arm64/standard/amazon-eks-node-al2023-arm64-standard-1.23-v20240307/image_id", ) parameter, err := env.SSMAPI.GetParameter(&ssm.GetParameterInput{Name: awssdk.String(parameterName)}) Expect(err).To(BeNil()) oldCustomAMI := *parameter.Parameter.Value - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyCustom + nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2023 nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ID: oldCustomAMI}} - nodeClass.Spec.UserData = awssdk.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", env.ClusterName)) env.ExpectCreated(dep, nodeClass, nodePool) pod := env.EventuallyExpectHealthyPodCount(selector, numPods)[0] @@ -428,11 +427,11 @@ var _ = Describe("Drift", func() { It("should return drifted if the AMI no longer matches the existing NodeClaims instance type", func() { version := env.GetK8sVersion(1) armParameter, err := env.SSMAPI.GetParameter(&ssm.GetParameterInput{ - Name: awssdk.String(fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2-arm64/recommended/image_id", version)), + Name: awssdk.String(fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/arm64/standard/recommended/image_id", version)), }) Expect(err).To(BeNil()) armAMI := *armParameter.Parameter.Value - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 + nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2023 nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ID: armAMI}} env.ExpectCreated(dep, nodeClass, nodePool) @@ -454,15 +453,18 @@ var _ = Describe("Drift", func() { It("should not disrupt nodes that have drifted without the featureGate enabled", func() { version := env.GetK8sVersion(1) env.ExpectSettingsOverridden(v1.EnvVar{Name: "FEATURE_GATES", Value: "Drift=false"}) - // choose an old static image + // Choose an old static image (AL2023 AMIs don't exist for 1.22) + parameterName := lo.Ternary(lo.Must(strconv.Atoi(strings.Split(env.GetK8sVersion(0), ".")[1])) == 23, + "/aws/service/eks/optimized-ami/1.23/amazon-linux-2023/arm64/standard/amazon-eks-node-al2023-arm64-standard-1.23-v20240307/image_id", + fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/arm64/standard/recommended/image_id", version), + ) parameter, err := env.SSMAPI.GetParameter(&ssm.GetParameterInput{ - Name: awssdk.String(fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2-arm64/recommended/image_id", version)), + Name: awssdk.String(parameterName), }) Expect(err).To(BeNil()) oldCustomAMI := *parameter.Parameter.Value - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyCustom + nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2023 nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ID: oldCustomAMI}} - nodeClass.Spec.UserData = awssdk.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", env.ClusterName)) env.ExpectCreated(dep, nodeClass, nodePool) env.EventuallyExpectHealthyPodCount(selector, numPods) @@ -674,7 +676,7 @@ var _ = Describe("Drift", func() { env.EventuallyExpectNotFound(pod, node) env.EventuallyExpectHealthyPodCount(selector, numPods) }, - Entry("UserData", v1beta1.EC2NodeClassSpec{UserData: awssdk.String("#!/bin/bash\n/etc/eks/bootstrap.sh")}), + Entry("UserData", v1beta1.EC2NodeClassSpec{UserData: awssdk.String("#!/bin/bash\necho \"Hello, AL2023\"")}), Entry("Tags", v1beta1.EC2NodeClassSpec{Tags: map[string]string{"keyTag-test-3": "valueTag-test-3"}}), Entry("MetadataOptions", v1beta1.EC2NodeClassSpec{MetadataOptions: &v1beta1.MetadataOptions{HTTPTokens: awssdk.String("required"), HTTPPutResponseHopLimit: awssdk.Int64(10)}}), Entry("BlockDeviceMappings", v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{ @@ -737,7 +739,7 @@ var _ = Describe("Drift", func() { node := env.ExpectCreatedNodeCount("==", 1)[0] nodeClass.Spec.BlockDeviceMappings[0].EBS.VolumeSize = resources.Quantity("100Gi") - env.ExpectCreatedOrUpdated() + env.ExpectCreatedOrUpdated(nodeClass) By("validating the drifted status condition has propagated") Eventually(func(g Gomega) { diff --git a/test/suites/integration/ami_test.go b/test/suites/integration/ami_test.go index 3e7ed437cd6c..96c3bdc57a1c 100644 --- a/test/suites/integration/ami_test.go +++ b/test/suites/integration/ami_test.go @@ -43,7 +43,7 @@ import ( var _ = Describe("AMI", func() { var customAMI string BeforeEach(func() { - customAMI = env.GetCustomAMI("/aws/service/eks/optimized-ami/%s/amazon-linux-2/recommended/image_id", 1) + customAMI = env.GetCustomAMI("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/x86_64/standard/recommended/image_id", 1) }) It("should use the AMI defined by the AMI Selector Terms", func() { @@ -61,12 +61,7 @@ var _ = Describe("AMI", func() { }) It("should use the most recent AMI when discovering multiple", func() { // choose an old static image - parameter, err := env.SSMAPI.GetParameter(&ssm.GetParameterInput{ - Name: aws.String("/aws/service/eks/optimized-ami/1.23/amazon-linux-2/amazon-eks-node-1.23-v20230322/image_id"), - }) - Expect(err).To(BeNil()) - oldCustomAMI := *parameter.Parameter.Value - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyCustom + oldCustomAMI := env.GetCustomAMI("/aws/service/eks/optimized-ami/%[1]s/amazon-linux-2023/x86_64/standard/amazon-eks-node-al2023-x86_64-standard-%[1]s-v20240307/image_id", 1) nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ { ID: customAMI, @@ -75,7 +70,6 @@ var _ = Describe("AMI", func() { ID: oldCustomAMI, }, } - nodeClass.Spec.UserData = aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", env.ClusterName)) pod := coretest.Pod() env.ExpectCreated(pod, nodeClass, nodePool) @@ -90,14 +84,12 @@ var _ = Describe("AMI", func() { }) Expect(err).To(BeNil()) Expect(output.Images).To(HaveLen(1)) - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyCustom nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ { Name: *output.Images[0].Name, Owner: "fakeOwnerValue", }, } - nodeClass.Spec.UserData = aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", env.ClusterName)) pod := coretest.Pod() env.ExpectCreated(pod, nodeClass, nodePool) @@ -111,13 +103,11 @@ var _ = Describe("AMI", func() { Expect(err).To(BeNil()) Expect(output.Images).To(HaveLen(1)) - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyCustom nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ { Name: *output.Images[0].Name, }, } - nodeClass.Spec.UserData = aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", env.ClusterName)) pod := coretest.Pod() env.ExpectCreated(pod, nodeClass, nodePool) @@ -127,13 +117,11 @@ var _ = Describe("AMI", func() { env.ExpectInstance(pod.Spec.NodeName).To(HaveField("ImageId", HaveValue(Equal(customAMI)))) }) It("should support ami selector ids", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyCustom nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ { ID: customAMI, }, } - nodeClass.Spec.UserData = aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", env.ClusterName)) pod := coretest.Pod() env.ExpectCreated(pod, nodeClass, nodePool) @@ -146,6 +134,7 @@ var _ = Describe("AMI", func() { Context("AMIFamily", func() { It("should provision a node using the AL2 family", func() { pod := coretest.Pod() + nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 env.ExpectCreated(nodeClass, nodePool, pod) env.EventuallyExpectHealthy(pod) env.ExpectCreatedNodeCount("==", 1) @@ -195,9 +184,10 @@ var _ = Describe("AMI", func() { }) It("should support Custom AMIFamily with AMI Selectors", func() { nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyCustom + al2AMI := env.GetCustomAMI("/aws/service/eks/optimized-ami/%s/amazon-linux-2/recommended/image_id", 1) nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ { - ID: customAMI, + ID: al2AMI, }, } nodeClass.Spec.UserData = aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", env.ClusterName)) @@ -207,7 +197,7 @@ var _ = Describe("AMI", func() { env.EventuallyExpectHealthy(pod) env.ExpectCreatedNodeCount("==", 1) - env.ExpectInstance(pod.Spec.NodeName).To(HaveField("ImageId", HaveValue(Equal(customAMI)))) + env.ExpectInstance(pod.Spec.NodeName).To(HaveField("ImageId", HaveValue(Equal(al2AMI)))) }) It("should have the EC2NodeClass status for AMIs using wildcard", func() { nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ @@ -236,6 +226,7 @@ var _ = Describe("AMI", func() { It("should merge UserData contents for AL2 AMIFamily", func() { content, err := os.ReadFile("testdata/al2_userdata_input.sh") Expect(err).ToNot(HaveOccurred()) + nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 nodeClass.Spec.UserData = aws.String(string(content)) nodePool.Spec.Template.Spec.Taints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoExecute"}} nodePool.Spec.Template.Spec.StartupTaints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoSchedule"}} @@ -256,6 +247,7 @@ var _ = Describe("AMI", func() { It("should merge non-MIME UserData contents for AL2 AMIFamily", func() { content, err := os.ReadFile("testdata/al2_no_mime_userdata_input.sh") Expect(err).ToNot(HaveOccurred()) + nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 nodeClass.Spec.UserData = aws.String(string(content)) nodePool.Spec.Template.Spec.Taints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoExecute"}} nodePool.Spec.Template.Spec.StartupTaints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoSchedule"}} diff --git a/test/suites/integration/block_device_mappings_test.go b/test/suites/integration/block_device_mappings_test.go index f5336a0a090b..56e0643eeaa0 100644 --- a/test/suites/integration/block_device_mappings_test.go +++ b/test/suites/integration/block_device_mappings_test.go @@ -31,7 +31,7 @@ var _ = Describe("BlockDeviceMappings", func() { { DeviceName: aws.String("/dev/xvda"), EBS: &v1beta1.BlockDevice{ - VolumeSize: resources.Quantity("10G"), + VolumeSize: resources.Quantity("20Gi"), VolumeType: aws.String("io2"), IOPS: aws.Int64(1000), Encrypted: aws.Bool(true), @@ -51,7 +51,7 @@ var _ = Describe("BlockDeviceMappings", func() { Expect(instance.BlockDeviceMappings[0].Ebs).To(HaveField("DeleteOnTermination", HaveValue(BeTrue()))) volume := env.GetVolume(instance.BlockDeviceMappings[0].Ebs.VolumeId) Expect(volume).To(HaveField("Encrypted", HaveValue(BeTrue()))) - Expect(volume).To(HaveField("Size", HaveValue(Equal(int64(10))))) // Convert G -> Gib (rounded up) + Expect(volume).To(HaveField("Size", HaveValue(Equal(int64(20))))) Expect(volume).To(HaveField("Iops", HaveValue(Equal(int64(1000))))) Expect(volume).To(HaveField("VolumeType", HaveValue(Equal("io2")))) }) diff --git a/test/suites/integration/extended_resources_test.go b/test/suites/integration/extended_resources_test.go index 1701567b45a3..4491d77df62f 100644 --- a/test/suites/integration/extended_resources_test.go +++ b/test/suites/integration/extended_resources_test.go @@ -40,7 +40,8 @@ import ( var _ = Describe("Extended Resources", func() { It("should provision nodes for a deployment that requests nvidia.com/gpu", func() { ExpectNvidiaDevicePluginCreated() - + // TODO: jmdeal@ remove AL2 pin once AL2023 accelerated AMIs are available + nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 numPods := 1 dep := test.Deployment(test.DeploymentOptions{ Replicas: int32(numPods), @@ -232,6 +233,8 @@ var _ = Describe("Extended Resources", func() { } // Only select private subnets since instances with multiple network instances at launch won't get a public IP. nodeClass.Spec.SubnetSelectorTerms[0].Tags["Name"] = "*Private*" + // TODO: jmdeal@ remove AL2 pin once AL2023 accelerated AMIs are available + nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 numPods := 1 dep := test.Deployment(test.DeploymentOptions{