Skip to content

Commit

Permalink
fix: update agent/run
Browse files Browse the repository at this point in the history
  • Loading branch information
@xiaods
xiaods committed Oct 22, 2024
1 parent e1a0b52 commit 99cfdea
Show file tree
Hide file tree
Showing 3 changed files with 79 additions and 29 deletions.
14 changes: 13 additions & 1 deletion pkg/agent/run.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ func run(ctx context.Context, cfg cmds.Agent, proxy proxy.Proxy) error {
return fmt.Errorf("dual-stack or IPv6 are not supported on Windows node")
}

syssetup.Configure(enableIPv6)
nodeConfig.AgentConfig.EnableIPv4 = enableIPv4
nodeConfig.AgentConfig.EnableIPv6 = enableIPv6

Expand Down Expand Up @@ -402,11 +403,22 @@ func updateAddressAnnotations(nodeConfig *daemonconfig.Node, nodeAnnotations map
result[cp.ExternalIPKey] = util.JoinIPs(agentConfig.NodeExternalIPs)
}

if len(agentConfig.NodeInternalDNSs) > 0 {
result[cp.InternalDNSKey] = strings.Join(agentConfig.NodeInternalDNSs, ",")
} else {
delete(result, cp.InternalDNSKey)
}
if len(agentConfig.NodeExternalDNSs) > 0 {
result[cp.ExternalDNSKey] = strings.Join(agentConfig.NodeExternalDNSs, ",")
} else {
delete(result, cp.ExternalDNSKey)
}

result = labels.Merge(nodeAnnotations, result)
return result, !equality.Semantic.DeepEqual(nodeAnnotations, result)
}

// setupTunnelAndRunAgent should start the setup tunnel before starting kubelet
// setupTunnelAndRunAgent should start the setup tunnel before starting kubelet and kubeproxy
// there are special case for etcd agents, it will wait until it can find the apiaddress from
// the address channel and update the proxy with the servers addresses, if in rke2 we need to
// start the agent before the tunnel is setup to allow kubelet to start first and start the pods
Expand Down
70 changes: 50 additions & 20 deletions pkg/cloudprovider/instances.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,15 @@ import (
"github.com/sirupsen/logrus"
"github.com/xiaods/k8e/pkg/version"
corev1 "k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
cloudprovider "k8s.io/cloud-provider"
)

var (
InternalIPKey = version.Program + ".io/internal-ip"
ExternalIPKey = version.Program + ".io/external-ip"
HostnameKey = version.Program + ".io/hostname"
InternalIPKey = version.Program + ".io/internal-ip"
ExternalIPKey = version.Program + ".io/external-ip"
InternalDNSKey = version.Program + ".io/internal-dns"
ExternalDNSKey = version.Program + ".io/external-dns"
HostnameKey = version.Program + ".io/hostname"
)

var _ cloudprovider.InstancesV2 = &k8e{}
Expand All @@ -29,50 +30,79 @@ func (k *k8e) InstanceExists(ctx context.Context, node *corev1.Node) (bool, erro

// InstanceShutdown returns true if the instance is shutdown according to the cloud provider.
// K8e nodes are never shutdown.
func (k *k8e) InstanceShutdown(ctx context.Context, node *v1.Node) (bool, error) {
func (k *k8e) InstanceShutdown(ctx context.Context, node *corev1.Node) (bool, error) {
return false, nil
}

// InstanceMetadata returns the instance's metadata.
func (k *k8e) InstanceMetadata(ctx context.Context, node *v1.Node) (*cloudprovider.InstanceMetadata, error) {
func (k *k8e) InstanceMetadata(ctx context.Context, node *corev1.Node) (*cloudprovider.InstanceMetadata, error) {
if (node.Annotations[InternalIPKey] == "") && (node.Labels[InternalIPKey] == "") {
return nil, errors.New("address annotations not yet set")
}
addresses := []corev1.NodeAddress{}

metadata := &cloudprovider.InstanceMetadata{
ProviderID: fmt.Sprintf("%s://%s", version.Program, node.Name),
InstanceType: version.Program,
}

if node.Spec.ProviderID != "" {
metadata.ProviderID = node.Spec.ProviderID
}

if instanceType := node.Labels[corev1.LabelInstanceTypeStable]; instanceType != "" {
metadata.InstanceType = instanceType
}

if region := node.Labels[corev1.LabelTopologyRegion]; region != "" {
metadata.Region = region
}

if zone := node.Labels[corev1.LabelTopologyZone]; zone != "" {
metadata.Zone = zone
}

// check internal address
if address := node.Annotations[InternalIPKey]; address != "" {
for _, v := range strings.Split(address, ",") {
addresses = append(addresses, corev1.NodeAddress{Type: corev1.NodeInternalIP, Address: v})
metadata.NodeAddresses = append(metadata.NodeAddresses, corev1.NodeAddress{Type: corev1.NodeInternalIP, Address: v})
}
} else if address = node.Labels[InternalIPKey]; address != "" {
addresses = append(addresses, corev1.NodeAddress{Type: corev1.NodeInternalIP, Address: address})
metadata.NodeAddresses = append(metadata.NodeAddresses, corev1.NodeAddress{Type: corev1.NodeInternalIP, Address: address})
} else {
logrus.Infof("Couldn't find node internal ip annotation or label on node %s", node.Name)
}

// check external address
if address := node.Annotations[ExternalIPKey]; address != "" {
for _, v := range strings.Split(address, ",") {
addresses = append(addresses, corev1.NodeAddress{Type: corev1.NodeExternalIP, Address: v})
metadata.NodeAddresses = append(metadata.NodeAddresses, corev1.NodeAddress{Type: corev1.NodeExternalIP, Address: v})
}
} else if address = node.Labels[ExternalIPKey]; address != "" {
addresses = append(addresses, corev1.NodeAddress{Type: corev1.NodeExternalIP, Address: address})
metadata.NodeAddresses = append(metadata.NodeAddresses, corev1.NodeAddress{Type: corev1.NodeExternalIP, Address: address})
}

// check internal dns
if address := node.Annotations[InternalDNSKey]; address != "" {
for _, v := range strings.Split(address, ",") {
metadata.NodeAddresses = append(metadata.NodeAddresses, corev1.NodeAddress{Type: corev1.NodeInternalDNS, Address: v})
}
}

// check external dns
if address := node.Annotations[ExternalDNSKey]; address != "" {
for _, v := range strings.Split(address, ",") {
metadata.NodeAddresses = append(metadata.NodeAddresses, corev1.NodeAddress{Type: corev1.NodeExternalDNS, Address: v})
}
}

// check hostname
if address := node.Annotations[HostnameKey]; address != "" {
addresses = append(addresses, corev1.NodeAddress{Type: corev1.NodeHostName, Address: address})
metadata.NodeAddresses = append(metadata.NodeAddresses, corev1.NodeAddress{Type: corev1.NodeHostName, Address: address})
} else if address = node.Labels[HostnameKey]; address != "" {
addresses = append(addresses, corev1.NodeAddress{Type: corev1.NodeHostName, Address: address})
metadata.NodeAddresses = append(metadata.NodeAddresses, corev1.NodeAddress{Type: corev1.NodeHostName, Address: address})
} else {
logrus.Infof("Couldn't find node hostname annotation or label on node %s", node.Name)
}

return &cloudprovider.InstanceMetadata{
ProviderID: fmt.Sprintf("%s://%s", version.Program, node.Name),
InstanceType: version.Program,
NodeAddresses: addresses,
Zone: "",
Region: "",
}, nil
return metadata, nil
}
24 changes: 16 additions & 8 deletions pkg/daemons/config/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,11 @@ import (
"strings"
"sync"

"github.com/xiaods/k8e/pkg/generated/controllers/k8e.cattle.io"
"github.com/k3s-io/kine/pkg/endpoint"
"github.com/rancher/wharfie/pkg/registries"
"github.com/rancher/wrangler/v3/pkg/generated/controllers/core"
"github.com/rancher/wrangler/v3/pkg/leader"
"github.com/xiaods/k8e/pkg/generated/controllers/k8e.cattle.io"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
utilnet "k8s.io/apimachinery/pkg/util/net"
"k8s.io/apiserver/pkg/authentication/authenticator"
Expand All @@ -22,12 +22,12 @@ import (
)

const (
EgressSelectorModeAgent = "agent"
EgressSelectorModeCluster = "cluster"
EgressSelectorModeDisabled = "disabled"
EgressSelectorModePod = "pod"
CertificateRenewDays = 90
StreamServerPort = "10010"
EgressSelectorModeAgent = "agent"
EgressSelectorModeCluster = "cluster"
EgressSelectorModeDisabled = "disabled"
EgressSelectorModePod = "pod"
CertificateRenewDays = 90
StreamServerPort = "10010"
)

type Node struct {
Expand Down Expand Up @@ -105,18 +105,22 @@ type Agent struct {
ResolvConf string
RootDir string
KubeConfigKubelet string
KubeConfigKubeProxy string
KubeConfigK8eController string
NodeIP string
NodeIPs []net.IP
NodeExternalIP string
NodeExternalIPs []net.IP
NodeInternalDNSs []string
NodeExternalDNSs []string
RuntimeSocket string
ImageServiceSocket string
ListenAddress string
ClientCA string
CNIBinDir string
CNIConfDir string
ExtraKubeletArgs []string
ExtraKubeProxyArgs []string
PauseImage string
Snapshotter string
Systemd bool
Expand All @@ -130,10 +134,12 @@ type Agent struct {
SystemDefaultRegistry string
AirgapExtraRegistry []string
DisableCCM bool
DisableNPC bool
MinTLSVersion string
CipherSuites []string
Rootless bool
ProtectKernelDefaults bool
DisableServiceLB bool
EnableIPv4 bool
EnableIPv6 bool
VLevel int
Expand Down Expand Up @@ -328,6 +334,8 @@ type ControlRuntime struct {
ClientControllerKey string
ClientSchedulerCert string
ClientSchedulerKey string
ClientKubeProxyCert string
ClientKubeProxyKey string
ClientKubeletKey string
ClientCloudControllerCert string
ClientCloudControllerKey string
Expand Down Expand Up @@ -442,4 +450,4 @@ func GetArgs(initialArgs map[string]string, extraArgs []string) []string {
}

return args
}
}

0 comments on commit 99cfdea

Please sign in to comment.