From d8be5e1d8edc81da567af174fa78b564139df700 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ram=C3=B3n=20Lobillo?= Date: Wed, 17 Jan 2024 09:25:41 -0500 Subject: [PATCH 1/2] Adapt egressIP test to dualstack SOSQE-1388 SOSQE-217 - Adapting existing ipv4 test to run in all type of dualstack clusters. - Including test to cover OCPBUGS-27222 --- test/extended/openstack/egressip.go | 242 +++++++++++++----- .../generated/zz_generated.annotations.go | 2 + 2 files changed, 184 insertions(+), 60 deletions(-) diff --git a/test/extended/openstack/egressip.go b/test/extended/openstack/egressip.go index bffc0b824..8c2c78da7 100644 --- a/test/extended/openstack/egressip.go +++ b/test/extended/openstack/egressip.go @@ -5,7 +5,9 @@ import ( "encoding/json" "fmt" "net" + "net/netip" "os" + "strings" "github.com/gophercloud/gophercloud" "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/floatingips" @@ -36,7 +38,6 @@ var _ = g.Describe("[sig-installer][Suite:openshift/openstack][egressip] An egre var networkClient *gophercloud.ServiceClient var clientSet *kubernetes.Clientset var err error - var infraID string var workerNodeList *corev1.NodeList var cloudNetworkClientset cloudnetwork.Interface oc := exutil.NewCLI("openstack") @@ -60,10 +61,6 @@ var _ = g.Describe("[sig-installer][Suite:openshift/openstack][egressip] An egre e2eskipper.Skipf("Test not applicable for proxy setup") } - infrastructure, err := oc.AdminConfigClient().ConfigV1().Infrastructures().Get(ctx, "cluster", metav1.GetOptions{}) - o.Expect(err).NotTo(o.HaveOccurred()) - infraID = infrastructure.Status.InfrastructureName - g.By("Getting the worker node list") workerNodeList, err = clientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{ LabelSelector: "node-role.kubernetes.io/worker", @@ -102,13 +99,15 @@ var _ = g.Describe("[sig-installer][Suite:openshift/openstack][egressip] An egre defer node.RemoveLabelOffNode(clientSet, primaryWorker.Name, egressAssignableLabelKey) g.By(fmt.Sprintf("Getting the EgressIP network from the '%s' annotation", egressIPConfigAnnotationKey)) - egressIPNetCidrStr, err := getEgressIPNetwork(primaryWorker) + egressIPNetCidrStr, egressPortId, err := getEgressNetworkInfo(primaryWorker, "ipv4") o.Expect(err).NotTo(o.HaveOccurred()) o.Expect(egressIPNetCidrStr).NotTo(o.BeEmpty(), "Could not get the EgressIP network from the '%s' annotation", egressIPConfigAnnotationKey) e2e.Logf("Found the EgressIP network: %s", egressIPNetCidrStr) + o.Expect(egressPortId).NotTo(o.BeEmpty(), "Could not get the Egress openstack portId from the '%s' annotation", egressPortId) + e2e.Logf("Found the Egress PortID: %s", egressPortId) - g.By("Obtaining a not in use IP address from the EgressIP network (machineNetwork cidr)") - machineNetworkID, err := getNetworkIdFromSubnetCidr(networkClient, egressIPNetCidrStr, infraID) + g.By("Finding the openstack network ID from the egressPortId defined in openshift node annotation") + machineNetworkID, err := getNetworkIdFromPortId(networkClient, egressPortId) o.Expect(err).NotTo(o.HaveOccurred()) o.Expect(machineNetworkID).NotTo(o.BeEmpty(), "Could not get the EgressIP network ID in openstack for '%s' subnet CIDR", egressIPNetCidrStr) e2e.Logf("Found the EgressIP network ID '%s' in Openstack for the EgressIP CIDR '%s'", machineNetworkID, egressIPNetCidrStr) @@ -124,29 +123,9 @@ var _ = g.Describe("[sig-installer][Suite:openshift/openstack][egressip] An egre e2e.Logf("Created '%s' temporary directory", egressIPTempDir) defer os.RemoveAll(egressIPTempDir) - g.By("Creating an EgressIP yaml file") - var egressIPname = "egress-ip" - var egressIPYamlFileName = "egressip.yaml" - var egressIPYamlFilePath = egressIPTempDir + "/" + egressIPYamlFileName - var egressIPYamlTemplate = `apiVersion: k8s.ovn.org/v1 -kind: EgressIP -metadata: - name: %s -spec: - egressIPs: - - %s - namespaceSelector: - matchLabels: - %s` - - egressIPYaml := fmt.Sprintf(egressIPYamlTemplate, egressIPname, egressIPAddrStr, "app: egress") - e2e.Logf("egressIPYaml: %s", egressIPYaml) - - err = os.WriteFile(egressIPYamlFilePath, []byte(egressIPYaml), 0644) - o.Expect(err).NotTo(o.HaveOccurred()) - - g.By(fmt.Sprintf("Creating an EgressIP object from '%s'", egressIPYamlFilePath)) - err = oc.AsAdmin().Run("create").Args("-f", egressIPYamlFilePath).Execute() + g.By("Create egressIP resource in openshift") + egressIPname := "egress-ip" + err = createEgressIpResource(oc, egressIPname, egressIPAddrStr, "app: egress") o.Expect(err).NotTo(o.HaveOccurred()) defer oc.AsAdmin().Run("delete").Args("egressip", egressIPname).Execute() @@ -228,6 +207,85 @@ spec: e2e.Logf("Found the expected Fixed IP (%s) for the FIP '%s'", egressIPAddrStr, fip.FloatingIP) }) + + // https://issues.redhat.com/browse/OCPBUGS-27222 + g.It("with IPv6 format should be created on dualstack cluster with OVN-Kubernetes NetworkType and dhcpv6-stateful mode", func() { + + networks, err := oc.AdminConfigClient().ConfigV1().Networks().Get(ctx, "cluster", metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + dualstack, err := isDualStackCluster(networks.Status.ClusterNetwork) + o.Expect(err).NotTo(o.HaveOccurred()) + if !dualstack { + e2eskipper.Skipf("Test only applicable for dualstack clusters") + } + + g.By("Getting the network type") + networkType, err := getNetworkType(ctx, oc) + o.Expect(err).NotTo(o.HaveOccurred()) + if networkType != NetworkTypeOVNKubernetes { + e2eskipper.Skipf("Test not applicable for '%s' NetworkType (only valid for '%s')", networkType, NetworkTypeOVNKubernetes) + } + + e2e.Logf("Getting the worker for the EgressIP") + worker := workerNodeList.Items[0] + + g.By(fmt.Sprintf("Getting the EgressIP network from the '%s' annotation", egressIPConfigAnnotationKey)) + egressIPNetCidrStr, egressPortId, err := getEgressNetworkInfo(worker, "ipv6") + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(egressIPNetCidrStr).NotTo(o.BeEmpty(), "Could not get the EgressIP network from the '%s' annotation", egressIPConfigAnnotationKey) + e2e.Logf("Found the EgressIP network: %s", egressIPNetCidrStr) + o.Expect(egressPortId).NotTo(o.BeEmpty(), "Could not get the Egress openstack portId from the '%s' annotation", egressPortId) + e2e.Logf("Found the Egress PortID: %s", egressPortId) + + g.By("Finding the openstack network ID from the egressPortId defined in openshift node annotation") + machineNetworkID, err := getNetworkIdFromPortId(networkClient, egressPortId) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(machineNetworkID).NotTo(o.BeEmpty(), "Could not get the EgressIP network ID in openstack for '%s' subnet CIDR", egressIPNetCidrStr) + e2e.Logf("Found the EgressIP network ID '%s' in Openstack for the EgressIP CIDR '%s'", machineNetworkID, egressIPNetCidrStr) + + g.By("Discovering the ipv6 mode configured in the subnet") + ipv6mode, err := getipv6ModeFromSubnetCidr(networkClient, egressIPNetCidrStr, machineNetworkID) + o.Expect(err).NotTo(o.HaveOccurred()) + if ipv6mode != "dhcpv6-stateful" { + e2eskipper.Skipf("Test not applicable for '%s' ipv6mode (only valid for '%s')", ipv6mode, "dhcpv6-stateful") + } + + // Label the worker node as EgressIP assignable node + g.By(fmt.Sprintf("Labeling the primary node '%s' with '%s'", worker.Name, egressAssignableLabelKey)) + node.AddOrUpdateLabelOnNode(clientSet, worker.Name, egressAssignableLabelKey, "dummy") + defer node.RemoveLabelOffNode(clientSet, worker.Name, egressAssignableLabelKey) + + g.By("Looking for a free IP in the subnet to use for the egressIP object in openshift") + egressIPAddrStr, err := getNotInUseEgressIP(networkClient, egressIPNetCidrStr, machineNetworkID) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(egressIPAddrStr).NotTo(o.BeEmpty(), "Couldn't find a free IP address in '%s' network in Openstack", egressIPNetCidrStr) + o.Expect(isIpv6(egressIPAddrStr)).To(o.BeTrue(), "egressIP should be IPv6 but it's %q", egressIPAddrStr) + e2e.Logf("Found '%s' free IP address in the EgressIP network in Openstack", egressIPAddrStr) + + g.By("Create egressIP resource in openshift") + egressIPname := "egress-ip" + err = createEgressIpResource(oc, egressIPname, egressIPAddrStr, "app: egress") + o.Expect(err).NotTo(o.HaveOccurred()) + defer oc.AsAdmin().Run("delete").Args("egressip", egressIPname).Execute() + + g.By("Waiting until CloudPrivateIPConfig is created and assigned to the primary worker node") + cloudNetworkClientset, err = cloudnetwork.NewForConfig(oc.AdminConfig()) + o.Expect(err).NotTo(o.HaveOccurred()) + egressIPAddr, err := netip.ParseAddr(egressIPAddrStr) + o.Expect(err).NotTo(o.HaveOccurred()) + waitOk, err := waitCloudPrivateIPConfigAssignedNode(ctx, cloudNetworkClientset, strings.ReplaceAll(egressIPAddr.StringExpanded(), ":", "."), worker.Name) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(waitOk).To(o.BeTrue(), "Not found the expected assigned node '%s' in '%s' CloudPrivateIPConfig", worker.Name, egressIPAddrStr) + e2e.Logf("Found the expected assigned node '%s' in '%s' CloudPrivateIPConfig", worker.Name, egressIPAddrStr) + + g.By("Checking that the port exists from openstack perspective") + egressNetInUseIPs, err := getInUseIPs(networkClient, machineNetworkID) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(egressNetInUseIPs).To(o.ContainElement(egressIPAddrStr)) + + g.By("Checking that the allowed_addresses_pairs are properly updated for the worker in the Openstack") + checkAllowedAddressesPairs(networkClient, worker, corev1.Node{}, egressIPAddrStr, machineNetworkID) + }) }) // getNotInUseEgressIP returns a not in use IP address from the EgressIP network CIDR @@ -248,8 +306,8 @@ func getNotInUseEgressIP(client *gophercloud.ServiceClient, egressCidr string, e return freeIP, nil } -// getEgressIPNetwork returns the IP address from the node egress-ipconfig annotation -func getEgressIPNetwork(node corev1.Node) (string, error) { +// getEgressNetworkInfo returns the IP address CIDR and openstack portId from the node egress-ipconfig annotation +func getEgressNetworkInfo(node corev1.Node, ipVersion string) (string, string, error) { type ifAddr struct { IPv4 string `json:"ipv4,omitempty"` IPv6 string `json:"ipv6,omitempty"` @@ -268,27 +326,47 @@ func getEgressIPNetwork(node corev1.Node) (string, error) { annotation, ok := node.Annotations[egressIPConfigAnnotationKey] if !ok { e2e.Logf("Annotation '%s' not found in '%s' node", egressIPConfigAnnotationKey, node.Name) - return "", nil + return "", "", nil } e2e.Logf("Found '%s' annotation in '%s': %s", egressIPConfigAnnotationKey, node.Name, annotation) var nodeEgressIPConfigs []*NodeEgressIPConfiguration err := json.Unmarshal([]byte(annotation), &nodeEgressIPConfigs) if err != nil { - return "", err + return "", "", err } - egressIPNetStr := nodeEgressIPConfigs[0].IFAddr.IPv4 - if egressIPNetStr == "" { - e2e.Logf("Empty ifaddr.ipv4 in the '%s' annotation", egressIPConfigAnnotationKey) - return "", nil + egressIPNetStr := "" + if ipVersion == "ipv6" { + egressIPNetStr = nodeEgressIPConfigs[0].IFAddr.IPv6 + if egressIPNetStr == "" { + e2e.Logf("Empty ifaddr.ipv6 in the '%s' annotation", egressIPConfigAnnotationKey) + return "", "", nil + } + } else if ipVersion == "ipv4" { + egressIPNetStr = nodeEgressIPConfigs[0].IFAddr.IPv4 + if egressIPNetStr == "" { + e2e.Logf("Empty ifaddr.ipv4 in the '%s' annotation", egressIPConfigAnnotationKey) + return "", "", nil + } + } else { + return "", "", fmt.Errorf("Unknown ipVersion. Only ipv4 and ipv6 supported") } - return egressIPNetStr, nil + + return egressIPNetStr, nodeEgressIPConfigs[0].Interface, nil } -// getNetworkIdFromSubnetCidr returns the Openstack network ID for a given Openstack subnet CIDR -func getNetworkIdFromSubnetCidr(client *gophercloud.ServiceClient, subnetCidr string, infraID string) (string, error) { +// getNetworkIdFromPortId returns the Openstack network ID for a given Openstack port +func getNetworkIdFromPortId(client *gophercloud.ServiceClient, portId string) (string, error) { + port, err := ports.Get(client, portId).Extract() + if err != nil { + return "", fmt.Errorf("failed to get port") + } + return port.NetworkID, nil +} + +func getipv6ModeFromSubnetCidr(client *gophercloud.ServiceClient, subnetCidr string, networkId string) (string, error) { listOpts := subnets.ListOpts{ - CIDR: subnetCidr, - Tags: "openshiftClusterID=" + infraID, + CIDR: subnetCidr, + NetworkID: networkId, } allPages, err := subnets.List(client, listOpts).AllPages() if err != nil { @@ -301,7 +379,7 @@ func getNetworkIdFromSubnetCidr(client *gophercloud.ServiceClient, subnetCidr st if len(allSubnets) != 1 { return "", fmt.Errorf("unexpected number of subnets found with '%s' CIDR: %d subnets", subnetCidr, len(allSubnets)) } - return allSubnets[0].NetworkID, nil + return allSubnets[0].IPv6AddressMode, nil } // getInUseIPs returns the in use IPs in a given network ID in Openstack @@ -445,13 +523,13 @@ func waitCloudPrivateIPConfigAssignedNode(ctx context.Context, cloudNetClientset func getAllowedIPsFromNode(client *gophercloud.ServiceClient, node corev1.Node, machineNetwork string) ([]string, error) { result := []string{} - ip := node.GetAnnotations()["alpha.kubernetes.io/provided-node-ip"] + ip := strings.Split(node.GetAnnotations()["alpha.kubernetes.io/provided-node-ip"], ",")[0] nodePorts, err := getPortsByIP(client, ip, machineNetwork) if err != nil { return nil, err } if len(nodePorts) != 1 { - return nil, fmt.Errorf("unexpected number of openstack ports for IP %s", ip) + return nil, fmt.Errorf("unexpected number of openstack ports (%d) for IP %s", len(nodePorts), ip) } for _, addressPair := range nodePorts[0].AllowedAddressPairs { result = append(result, addressPair.IPAddress) @@ -465,7 +543,7 @@ func checkAllowedAddressesPairs(client *gophercloud.ServiceClient, nodeHoldingEg o.Eventually(func() bool { allowedIpList, err := getAllowedIPsFromNode(client, nodeHoldingEgressIp, networkID) if err != nil { - e2e.Logf("error obtaining allowedIpList") + e2e.Logf("error obtaining allowedIpList: %q", err) return false } else if !contains(allowedIpList, egressIp) { e2e.Logf("egressIP %s still not found on obtained allowedIpList (%s) for node %s", egressIp, allowedIpList, nodeHoldingEgressIp.Name) @@ -475,16 +553,60 @@ func checkAllowedAddressesPairs(client *gophercloud.ServiceClient, nodeHoldingEg }, "10s", "1s").Should(o.BeTrue(), "Timed out checking allowed address pairs for node %s", nodeHoldingEgressIp.Name) e2e.Logf("egressIp %s correctly included on the node allowed-address-pairs for %s", egressIp, nodeHoldingEgressIp.Name) - o.Eventually(func() bool { - allowedIpList, err := getAllowedIPsFromNode(client, nodeNotHoldingEgressIp, networkID) - if err != nil { - e2e.Logf("error obtaining allowedIpList") - return false - } else if contains(allowedIpList, egressIp) { - e2e.Logf("egressIP %s still found on obtained allowedIpList (%s) for node %s", egressIp, allowedIpList, nodeNotHoldingEgressIp.Name) - return false - } - return true - }, "10s", "1s").Should(o.BeTrue(), "Timed out checking allowed address pairs for node %s", nodeNotHoldingEgressIp.Name) - e2e.Logf("egressIp %s correctly not included on the node allowed-address-pairs for %s", egressIp, nodeNotHoldingEgressIp.Name) + if nodeNotHoldingEgressIp.Name != "" { + o.Eventually(func() bool { + allowedIpList, err := getAllowedIPsFromNode(client, nodeNotHoldingEgressIp, networkID) + if err != nil { + e2e.Logf("error obtaining allowedIpList") + return false + } else if contains(allowedIpList, egressIp) { + e2e.Logf("egressIP %s still found on obtained allowedIpList (%s) for node %s", egressIp, allowedIpList, nodeNotHoldingEgressIp.Name) + return false + } + return true + }, "10s", "1s").Should(o.BeTrue(), "Timed out checking allowed address pairs for node %s", nodeNotHoldingEgressIp.Name) + e2e.Logf("egressIp %s correctly not included on the node allowed-address-pairs for %s", egressIp, nodeNotHoldingEgressIp.Name) + } else { + e2e.Logf("Skipping check for worker not holding the egressIP") + } +} + +func createEgressIpResource(oc *exutil.CLI, egressIPname string, egressIPAddrStr string, labels string) error { + + g.By("Creating a temp directory") + egressIPTempDir, err := os.MkdirTemp("", "egressip-e2e") + if err != nil { + return err + } + e2e.Logf("Created '%s' temporary directory", egressIPTempDir) + defer os.RemoveAll(egressIPTempDir) + + g.By("Creating an EgressIP yaml file") + var egressIPYamlFileName = "egressip.yaml" + var egressIPYamlFilePath = egressIPTempDir + "/" + egressIPYamlFileName + var egressIPYamlTemplate = `apiVersion: k8s.ovn.org/v1 +kind: EgressIP +metadata: + name: %s +spec: + egressIPs: + - %s + namespaceSelector: + matchLabels: + %s` + + egressIPYaml := fmt.Sprintf(egressIPYamlTemplate, egressIPname, egressIPAddrStr, labels) + e2e.Logf("egressIPYaml: %s", egressIPYaml) + + err = os.WriteFile(egressIPYamlFilePath, []byte(egressIPYaml), 0644) + if err != nil { + return err + } + + g.By(fmt.Sprintf("Creating an EgressIP object from '%s'", egressIPYamlFilePath)) + err = oc.AsAdmin().Run("create").Args("-f", egressIPYamlFilePath).Execute() + if err != nil { + return err + } + return nil } diff --git a/test/extended/util/annotate/generated/zz_generated.annotations.go b/test/extended/util/annotate/generated/zz_generated.annotations.go index 693561be8..21a409052 100644 --- a/test/extended/util/annotate/generated/zz_generated.annotations.go +++ b/test/extended/util/annotate/generated/zz_generated.annotations.go @@ -45,6 +45,8 @@ var Annotations = map[string]string{ "[sig-installer][Suite:openshift/openstack][egressip] An egressIP attached to a floating IP should be kept after EgressIP node failover with OVN-Kubernetes NetworkType": "", + "[sig-installer][Suite:openshift/openstack][egressip] An egressIP with IPv6 format should be created on dualstack cluster with OVN-Kubernetes NetworkType and dhcpv6-stateful mode": "", + "[sig-installer][Suite:openshift/openstack][lb][Serial] The Openstack platform should apply lb-method on TCP Amphora LoadBalancer when a TCP svc with monitors and ETP:Local is created on Openshift": "", "[sig-installer][Suite:openshift/openstack][lb][Serial] The Openstack platform should apply lb-method on TCP OVN LoadBalancer when a TCP svc with monitors and ETP:Local is created on Openshift": "", From a7b10e053f7ffb745fa7756486bb6258e1fb6d0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ram=C3=B3n=20Lobillo?= Date: Thu, 30 Nov 2023 11:10:20 -0500 Subject: [PATCH 2/2] Adapt tests to ipv6primary dualstack cluster KURYRQE-1414 While running openstack-test in ipv6primary dualstack cluster, there are some tests that are not passing. This patch performs the needed actions to make those tests available in both ipv4 and ipv6 primary dualstack cluster. --- test/extended/openstack/egressip.go | 6 ++ test/extended/openstack/loadbalancers.go | 36 +++++++++--- test/extended/openstack/utils.go | 72 ++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 7 deletions(-) diff --git a/test/extended/openstack/egressip.go b/test/extended/openstack/egressip.go index 8c2c78da7..4030c7a2c 100644 --- a/test/extended/openstack/egressip.go +++ b/test/extended/openstack/egressip.go @@ -70,6 +70,12 @@ var _ = g.Describe("[sig-installer][Suite:openshift/openstack][egressip] An egre g.It("attached to a floating IP should be kept after EgressIP node failover with OVN-Kubernetes NetworkType", func() { + dualstackIpv6Primary, err := isIpv6primaryDualStackCluster(ctx, oc) + o.Expect(err).NotTo(o.HaveOccurred()) + if dualstackIpv6Primary { //This test is covering and scenario that has no sense with ipv6 as there is no FIP/egressIP association. + e2eskipper.Skipf("Test not applicable for ipv6primary dualstack environments") + } + g.By("Getting the network type") networkType, err := getNetworkType(ctx, oc) o.Expect(err).NotTo(o.HaveOccurred()) diff --git a/test/extended/openstack/loadbalancers.go b/test/extended/openstack/loadbalancers.go index 9d72d61de..506bf75d4 100644 --- a/test/extended/openstack/loadbalancers.go +++ b/test/extended/openstack/loadbalancers.go @@ -131,7 +131,7 @@ var _ = g.Describe("[sig-installer][Suite:openshift/openstack][lb][Serial] The O o.Expect(loadBalancerId).ShouldNot(o.BeEmpty(), "load-balancer-id annotation missing") o.Expect(svc.Status.LoadBalancer.Ingress).ShouldNot(o.BeEmpty(), "svc.Status.LoadBalancer.Ingress should not be empty") svcIp := svc.Status.LoadBalancer.Ingress[0].IP - o.Expect(svcIp).ShouldNot(o.BeEmpty(), "FIP missing on svc Status") + o.Expect(svcIp).ShouldNot(o.BeEmpty(), "Ingress IP missing on svc Status") o.Expect(svc.Spec.ExternalTrafficPolicy).Should(o.Equal(v1.ServiceExternalTrafficPolicyTypeCluster), "Unexpected ExternalTrafficPolicy on svc specs") @@ -139,10 +139,11 @@ var _ = g.Describe("[sig-installer][Suite:openshift/openstack][lb][Serial] The O lb, err := octavialoadbalancers.Get(loadBalancerClient, loadBalancerId).Extract() o.Expect(err).NotTo(o.HaveOccurred()) o.Expect(lb.Provider).Should(o.Equal(strings.ToLower(lbProviderUnderTest)), "Unexpected provider in the Openstack LoadBalancer") - - fip, err := getFipbyFixedIP(networkClient, lb.VipAddress) - o.Expect(err).NotTo(o.HaveOccurred()) - o.Expect(fip.FloatingIP).Should(o.Equal(svcIp), "Unexpected floatingIp in the Openstack LoadBalancer") + if isIpv4(lb.VipAddress) { // No FIP assignment on ipv6 + fip, err := getFipbyFixedIP(networkClient, lb.VipAddress) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(fip.FloatingIP).Should(o.Equal(svcIp), "Unexpected floatingIp in the Openstack LoadBalancer") + } o.Expect(lb.Pools).Should(o.HaveLen(1), "Unexpected number of pools on Openstack LoadBalancer %q", lb.Name) pool, err := pools.Get(loadBalancerClient, lb.Pools[0].ID).Extract() @@ -387,7 +388,7 @@ var _ = g.Describe("[sig-installer][Suite:openshift/openstack][lb][Serial] The O loadBalancerId := svc.GetAnnotations()["loadbalancer.openstack.org/load-balancer-id"] o.Expect(loadBalancerId).ShouldNot(o.BeEmpty(), "load-balancer-id annotation missing") svcIp := svc.Status.LoadBalancer.Ingress[0].IP - o.Expect(svcIp).ShouldNot(o.BeEmpty(), "FIP missing on svc Status") + o.Expect(svcIp).ShouldNot(o.BeEmpty(), "Ingress IP missing on svc Status") o.Expect(svc.Spec.ExternalTrafficPolicy).Should(o.Equal(v1.ServiceExternalTrafficPolicyTypeLocal), "Unexpected ExternalTrafficPolicy on svc specs") @@ -442,6 +443,13 @@ var _ = g.Describe("[sig-installer][Suite:openshift/openstack][lb][Serial] The O skipIfNotLbProvider(lbProviderUnderTest, cloudProviderConfig) + var err error + dualstackIpv6Primary, err := isIpv6primaryDualStackCluster(ctx, oc) + o.Expect(err).NotTo(o.HaveOccurred()) + if dualstackIpv6Primary { //This test is covering and scenario that has no sense with ipv6 as there is no FIP/VIP association. + e2eskipper.Skipf("Test not applicable for ipv6primary dualstack environments") + } + g.By("Create FIP to be used on the subsequent LoadBalancer Service") var fip *floatingips.FloatingIP foundNetworkId, err := GetFloatingNetworkID(networkClient, cloudProviderConfig) @@ -521,7 +529,7 @@ var _ = g.Describe("[sig-installer][Suite:openshift/openstack][lb][Serial] The O o.Expect(loadBalancerId).ShouldNot(o.BeEmpty(), "load-balancer-id annotation missing") o.Expect(svc.Status.LoadBalancer.Ingress).ShouldNot(o.BeEmpty(), "svc.Status.LoadBalancer.Ingress should not be empty") svcIp := svc.Status.LoadBalancer.Ingress[0].IP - o.Expect(svcIp).ShouldNot(o.BeEmpty(), "FIP missing on svc Status") + o.Expect(svcIp).ShouldNot(o.BeEmpty(), "Ingress IP missing on svc Status") o.Expect(svc.Spec.ExternalTrafficPolicy).Should(o.Equal(v1.ServiceExternalTrafficPolicyTypeLocal), "Unexpected ExternalTrafficPolicy on svc specs") e2e.Logf("Service with LoadBalancerType exists with public ip %q and it is pointing to openstack loadbalancer with id %q", svcIp, loadBalancerId) @@ -735,6 +743,13 @@ func getFipbyFixedIP(client *gophercloud.ServiceClient, vip string) (floatingips func getPodNameThroughLb(ip string, port string, protocol v1.Protocol, message string) (string, error) { + if net.ParseIP(ip) == nil { + return "", fmt.Errorf("invalid ip %q", ip) + } + if isIpv6(ip) { + ip = "[" + ip + "]" + } + if protocol == v1.ProtocolUDP { // Send message on provided ip and UDP port and return the answer. // Error if there is no answer or cannot access the port after 5 seconds. @@ -980,6 +995,13 @@ func waitForIngressControllerCondition(oc *exutil.CLI, timeout time.Duration, na // Create https GET towards a given domain using a given IP on transport level. Returns the response or error. func httpsGetWithCustomLookup(url string, ip string) (*http.Response, error) { + if net.ParseIP(ip) == nil { + return nil, fmt.Errorf("invalid ip %q", ip) + } + if isIpv6(ip) { + ip = "[" + ip + "]" + } + // Create a custom resolver that overrides DNS lookups resolver := &net.Resolver{ PreferGo: true, diff --git a/test/extended/openstack/utils.go b/test/extended/openstack/utils.go index ae9429f87..fbc622309 100644 --- a/test/extended/openstack/utils.go +++ b/test/extended/openstack/utils.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "math/rand" + "net" "reflect" "strconv" "strings" @@ -16,6 +17,7 @@ import ( "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/external" "github.com/gophercloud/gophercloud/openstack/networking/v2/networks" "github.com/gophercloud/gophercloud/openstack/networking/v2/subnets" + configv1 "github.com/openshift/api/config/v1" machinev1 "github.com/openshift/api/machine/v1beta1" operatorv1 "github.com/openshift/api/operator/v1" framework "github.com/openshift/cluster-api-actuator-pkg/pkg/framework" @@ -216,6 +218,54 @@ func getNetworkType(ctx context.Context, oc *exutil.CLI) (string, error) { return networkType, nil } +// Check the cluster networks and returns true if it finds one ipv4 and one ipv6 network there. +func isDualStackCluster(clusterNetwork []configv1.ClusterNetworkEntry) (bool, error) { + ipv4Found := false + ipv6Found := false + + for _, network := range clusterNetwork { + ip, _, err := net.ParseCIDR(network.CIDR) + if err != nil { + return false, err + } + e2e.Logf("Detected cluster network: %q", ip.String()) + if !ipv4Found { + ipv4Found = isIpv4(ip.String()) + } + if !ipv6Found { + ipv6Found = isIpv6(ip.String()) + } + } + return (ipv4Found && ipv6Found), nil +} + +// Check if it is a dualstack cluster and the first cluster network technology. Returns true if it is ipv6. +func isIpv6primaryDualStackCluster(ctx context.Context, oc *exutil.CLI) (bool, error) { + + networks, err := oc.AdminConfigClient().ConfigV1().Networks().Get(ctx, "cluster", metav1.GetOptions{}) + if err != nil { + return false, err + } + dualstack, err := isDualStackCluster(networks.Status.ClusterNetwork) + if err != nil { + return false, err + } + if dualstack { + if err != nil { + return false, err + } + primaryNetwork := networks.Status.ClusterNetwork[0] + ip, _, err := net.ParseCIDR(primaryNetwork.CIDR) + if err != nil { + return false, err + } + if isIpv6(ip.String()) { + return true, nil + } + } + return false, nil +} + func getMaxOctaviaAPIVersion(client *gophercloud.ServiceClient) (*semver.Version, error) { allPages, err := apiversions.List(client).AllPages() if err != nil { @@ -356,3 +406,25 @@ func GetClusterLoadBalancerSetting(setting string, config *ini.File) (string, er } return strings.ToLower(result), nil } + +// isIpv6 returns true if the ip is ipv6 +func isIpv6(ip string) bool { + ipv6 := false + + netIP := net.ParseIP(ip) + if netIP != nil && netIP.To4() == nil { + ipv6 = true + } + return ipv6 +} + +// isIpv4 returns true if the ip is ipv4 +func isIpv4(ip string) bool { + ipv4 := false + + netIP := net.ParseIP(ip) + if netIP != nil && netIP.To4() != nil { + ipv4 = true + } + return ipv4 +}