diff --git a/pkg/agent/run.go b/pkg/agent/run.go index 98711cf1..6161b143 100644 --- a/pkg/agent/run.go +++ b/pkg/agent/run.go @@ -90,6 +90,7 @@ func run(ctx context.Context, cfg cmds.Agent, proxy proxy.Proxy) error { return fmt.Errorf("dual-stack or IPv6 are not supported on Windows node") } + syssetup.Configure(enableIPv6) nodeConfig.AgentConfig.EnableIPv4 = enableIPv4 nodeConfig.AgentConfig.EnableIPv6 = enableIPv6 @@ -402,11 +403,22 @@ func updateAddressAnnotations(nodeConfig *daemonconfig.Node, nodeAnnotations map result[cp.ExternalIPKey] = util.JoinIPs(agentConfig.NodeExternalIPs) } + if len(agentConfig.NodeInternalDNSs) > 0 { + result[cp.InternalDNSKey] = strings.Join(agentConfig.NodeInternalDNSs, ",") + } else { + delete(result, cp.InternalDNSKey) + } + if len(agentConfig.NodeExternalDNSs) > 0 { + result[cp.ExternalDNSKey] = strings.Join(agentConfig.NodeExternalDNSs, ",") + } else { + delete(result, cp.ExternalDNSKey) + } + result = labels.Merge(nodeAnnotations, result) return result, !equality.Semantic.DeepEqual(nodeAnnotations, result) } -// setupTunnelAndRunAgent should start the setup tunnel before starting kubelet +// setupTunnelAndRunAgent should start the setup tunnel before starting kubelet and kubeproxy // there are special case for etcd agents, it will wait until it can find the apiaddress from // the address channel and update the proxy with the servers addresses, if in rke2 we need to // start the agent before the tunnel is setup to allow kubelet to start first and start the pods diff --git a/pkg/cloudprovider/instances.go b/pkg/cloudprovider/instances.go index cc7de43e..9567fb15 100644 --- a/pkg/cloudprovider/instances.go +++ b/pkg/cloudprovider/instances.go @@ -9,14 +9,15 @@ import ( "github.com/sirupsen/logrus" "github.com/xiaods/k8e/pkg/version" corev1 "k8s.io/api/core/v1" - v1 "k8s.io/api/core/v1" cloudprovider "k8s.io/cloud-provider" ) var ( - InternalIPKey = version.Program + ".io/internal-ip" - ExternalIPKey = version.Program + ".io/external-ip" - HostnameKey = version.Program + ".io/hostname" + InternalIPKey = version.Program + ".io/internal-ip" + ExternalIPKey = version.Program + ".io/external-ip" + InternalDNSKey = version.Program + ".io/internal-dns" + ExternalDNSKey = version.Program + ".io/external-dns" + HostnameKey = version.Program + ".io/hostname" ) var _ cloudprovider.InstancesV2 = &k8e{} @@ -29,23 +30,44 @@ func (k *k8e) InstanceExists(ctx context.Context, node *corev1.Node) (bool, erro // InstanceShutdown returns true if the instance is shutdown according to the cloud provider. // K8e nodes are never shutdown. -func (k *k8e) InstanceShutdown(ctx context.Context, node *v1.Node) (bool, error) { +func (k *k8e) InstanceShutdown(ctx context.Context, node *corev1.Node) (bool, error) { return false, nil } // InstanceMetadata returns the instance's metadata. -func (k *k8e) InstanceMetadata(ctx context.Context, node *v1.Node) (*cloudprovider.InstanceMetadata, error) { +func (k *k8e) InstanceMetadata(ctx context.Context, node *corev1.Node) (*cloudprovider.InstanceMetadata, error) { if (node.Annotations[InternalIPKey] == "") && (node.Labels[InternalIPKey] == "") { return nil, errors.New("address annotations not yet set") } - addresses := []corev1.NodeAddress{} + + metadata := &cloudprovider.InstanceMetadata{ + ProviderID: fmt.Sprintf("%s://%s", version.Program, node.Name), + InstanceType: version.Program, + } + + if node.Spec.ProviderID != "" { + metadata.ProviderID = node.Spec.ProviderID + } + + if instanceType := node.Labels[corev1.LabelInstanceTypeStable]; instanceType != "" { + metadata.InstanceType = instanceType + } + + if region := node.Labels[corev1.LabelTopologyRegion]; region != "" { + metadata.Region = region + } + + if zone := node.Labels[corev1.LabelTopologyZone]; zone != "" { + metadata.Zone = zone + } + // check internal address if address := node.Annotations[InternalIPKey]; address != "" { for _, v := range strings.Split(address, ",") { - addresses = append(addresses, corev1.NodeAddress{Type: corev1.NodeInternalIP, Address: v}) + metadata.NodeAddresses = append(metadata.NodeAddresses, corev1.NodeAddress{Type: corev1.NodeInternalIP, Address: v}) } } else if address = node.Labels[InternalIPKey]; address != "" { - addresses = append(addresses, corev1.NodeAddress{Type: corev1.NodeInternalIP, Address: address}) + metadata.NodeAddresses = append(metadata.NodeAddresses, corev1.NodeAddress{Type: corev1.NodeInternalIP, Address: address}) } else { logrus.Infof("Couldn't find node internal ip annotation or label on node %s", node.Name) } @@ -53,26 +75,34 @@ func (k *k8e) InstanceMetadata(ctx context.Context, node *v1.Node) (*cloudprovid // check external address if address := node.Annotations[ExternalIPKey]; address != "" { for _, v := range strings.Split(address, ",") { - addresses = append(addresses, corev1.NodeAddress{Type: corev1.NodeExternalIP, Address: v}) + metadata.NodeAddresses = append(metadata.NodeAddresses, corev1.NodeAddress{Type: corev1.NodeExternalIP, Address: v}) } } else if address = node.Labels[ExternalIPKey]; address != "" { - addresses = append(addresses, corev1.NodeAddress{Type: corev1.NodeExternalIP, Address: address}) + metadata.NodeAddresses = append(metadata.NodeAddresses, corev1.NodeAddress{Type: corev1.NodeExternalIP, Address: address}) + } + + // check internal dns + if address := node.Annotations[InternalDNSKey]; address != "" { + for _, v := range strings.Split(address, ",") { + metadata.NodeAddresses = append(metadata.NodeAddresses, corev1.NodeAddress{Type: corev1.NodeInternalDNS, Address: v}) + } + } + + // check external dns + if address := node.Annotations[ExternalDNSKey]; address != "" { + for _, v := range strings.Split(address, ",") { + metadata.NodeAddresses = append(metadata.NodeAddresses, corev1.NodeAddress{Type: corev1.NodeExternalDNS, Address: v}) + } } // check hostname if address := node.Annotations[HostnameKey]; address != "" { - addresses = append(addresses, corev1.NodeAddress{Type: corev1.NodeHostName, Address: address}) + metadata.NodeAddresses = append(metadata.NodeAddresses, corev1.NodeAddress{Type: corev1.NodeHostName, Address: address}) } else if address = node.Labels[HostnameKey]; address != "" { - addresses = append(addresses, corev1.NodeAddress{Type: corev1.NodeHostName, Address: address}) + metadata.NodeAddresses = append(metadata.NodeAddresses, corev1.NodeAddress{Type: corev1.NodeHostName, Address: address}) } else { logrus.Infof("Couldn't find node hostname annotation or label on node %s", node.Name) } - return &cloudprovider.InstanceMetadata{ - ProviderID: fmt.Sprintf("%s://%s", version.Program, node.Name), - InstanceType: version.Program, - NodeAddresses: addresses, - Zone: "", - Region: "", - }, nil + return metadata, nil } diff --git a/pkg/daemons/config/types.go b/pkg/daemons/config/types.go index b7366c5c..6691a069 100644 --- a/pkg/daemons/config/types.go +++ b/pkg/daemons/config/types.go @@ -9,11 +9,11 @@ import ( "strings" "sync" + "github.com/xiaods/k8e/pkg/generated/controllers/k8e.cattle.io" "github.com/k3s-io/kine/pkg/endpoint" "github.com/rancher/wharfie/pkg/registries" "github.com/rancher/wrangler/v3/pkg/generated/controllers/core" "github.com/rancher/wrangler/v3/pkg/leader" - "github.com/xiaods/k8e/pkg/generated/controllers/k8e.cattle.io" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/apiserver/pkg/authentication/authenticator" @@ -22,12 +22,12 @@ import ( ) const ( - EgressSelectorModeAgent = "agent" - EgressSelectorModeCluster = "cluster" - EgressSelectorModeDisabled = "disabled" - EgressSelectorModePod = "pod" - CertificateRenewDays = 90 - StreamServerPort = "10010" + EgressSelectorModeAgent = "agent" + EgressSelectorModeCluster = "cluster" + EgressSelectorModeDisabled = "disabled" + EgressSelectorModePod = "pod" + CertificateRenewDays = 90 + StreamServerPort = "10010" ) type Node struct { @@ -105,11 +105,14 @@ type Agent struct { ResolvConf string RootDir string KubeConfigKubelet string + KubeConfigKubeProxy string KubeConfigK8eController string NodeIP string NodeIPs []net.IP NodeExternalIP string NodeExternalIPs []net.IP + NodeInternalDNSs []string + NodeExternalDNSs []string RuntimeSocket string ImageServiceSocket string ListenAddress string @@ -117,6 +120,7 @@ type Agent struct { CNIBinDir string CNIConfDir string ExtraKubeletArgs []string + ExtraKubeProxyArgs []string PauseImage string Snapshotter string Systemd bool @@ -130,10 +134,12 @@ type Agent struct { SystemDefaultRegistry string AirgapExtraRegistry []string DisableCCM bool + DisableNPC bool MinTLSVersion string CipherSuites []string Rootless bool ProtectKernelDefaults bool + DisableServiceLB bool EnableIPv4 bool EnableIPv6 bool VLevel int @@ -328,6 +334,8 @@ type ControlRuntime struct { ClientControllerKey string ClientSchedulerCert string ClientSchedulerKey string + ClientKubeProxyCert string + ClientKubeProxyKey string ClientKubeletKey string ClientCloudControllerCert string ClientCloudControllerKey string @@ -442,4 +450,4 @@ func GetArgs(initialArgs map[string]string, extraArgs []string) []string { } return args -} +} \ No newline at end of file