Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: update agent/run #364

Merged
merged 1 commit into from
Oct 22, 2024
Merged

fix: update agent/run #364

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion pkg/agent/run.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ func run(ctx context.Context, cfg cmds.Agent, proxy proxy.Proxy) error {
return fmt.Errorf("dual-stack or IPv6 are not supported on Windows node")
}

syssetup.Configure(enableIPv6)
nodeConfig.AgentConfig.EnableIPv4 = enableIPv4
nodeConfig.AgentConfig.EnableIPv6 = enableIPv6

Expand Down Expand Up @@ -402,11 +403,22 @@ func updateAddressAnnotations(nodeConfig *daemonconfig.Node, nodeAnnotations map
result[cp.ExternalIPKey] = util.JoinIPs(agentConfig.NodeExternalIPs)
}

if len(agentConfig.NodeInternalDNSs) > 0 {
result[cp.InternalDNSKey] = strings.Join(agentConfig.NodeInternalDNSs, ",")
} else {
delete(result, cp.InternalDNSKey)
}
if len(agentConfig.NodeExternalDNSs) > 0 {
result[cp.ExternalDNSKey] = strings.Join(agentConfig.NodeExternalDNSs, ",")
} else {
delete(result, cp.ExternalDNSKey)
}

result = labels.Merge(nodeAnnotations, result)
return result, !equality.Semantic.DeepEqual(nodeAnnotations, result)
}

// setupTunnelAndRunAgent should start the setup tunnel before starting kubelet
// setupTunnelAndRunAgent should start the setup tunnel before starting kubelet and kubeproxy
// there are special case for etcd agents, it will wait until it can find the apiaddress from
// the address channel and update the proxy with the servers addresses, if in rke2 we need to
// start the agent before the tunnel is setup to allow kubelet to start first and start the pods
Expand Down
70 changes: 50 additions & 20 deletions pkg/cloudprovider/instances.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,15 @@ import (
"github.com/sirupsen/logrus"
"github.com/xiaods/k8e/pkg/version"
corev1 "k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
cloudprovider "k8s.io/cloud-provider"
)

var (
InternalIPKey = version.Program + ".io/internal-ip"
ExternalIPKey = version.Program + ".io/external-ip"
HostnameKey = version.Program + ".io/hostname"
InternalIPKey = version.Program + ".io/internal-ip"
ExternalIPKey = version.Program + ".io/external-ip"
InternalDNSKey = version.Program + ".io/internal-dns"
ExternalDNSKey = version.Program + ".io/external-dns"
HostnameKey = version.Program + ".io/hostname"
)

var _ cloudprovider.InstancesV2 = &k8e{}
Expand All @@ -29,50 +30,79 @@ func (k *k8e) InstanceExists(ctx context.Context, node *corev1.Node) (bool, erro

// InstanceShutdown returns true if the instance is shutdown according to the cloud provider.
// K8e nodes are never shutdown.
func (k *k8e) InstanceShutdown(ctx context.Context, node *v1.Node) (bool, error) {
func (k *k8e) InstanceShutdown(ctx context.Context, node *corev1.Node) (bool, error) {
return false, nil
}

// InstanceMetadata returns the instance's metadata.
func (k *k8e) InstanceMetadata(ctx context.Context, node *v1.Node) (*cloudprovider.InstanceMetadata, error) {
func (k *k8e) InstanceMetadata(ctx context.Context, node *corev1.Node) (*cloudprovider.InstanceMetadata, error) {
if (node.Annotations[InternalIPKey] == "") && (node.Labels[InternalIPKey] == "") {
return nil, errors.New("address annotations not yet set")
}
addresses := []corev1.NodeAddress{}

metadata := &cloudprovider.InstanceMetadata{
ProviderID: fmt.Sprintf("%s://%s", version.Program, node.Name),
InstanceType: version.Program,
}

if node.Spec.ProviderID != "" {
metadata.ProviderID = node.Spec.ProviderID
}

if instanceType := node.Labels[corev1.LabelInstanceTypeStable]; instanceType != "" {
metadata.InstanceType = instanceType
}

if region := node.Labels[corev1.LabelTopologyRegion]; region != "" {
metadata.Region = region
}

if zone := node.Labels[corev1.LabelTopologyZone]; zone != "" {
metadata.Zone = zone
}

// check internal address
if address := node.Annotations[InternalIPKey]; address != "" {
for _, v := range strings.Split(address, ",") {
addresses = append(addresses, corev1.NodeAddress{Type: corev1.NodeInternalIP, Address: v})
metadata.NodeAddresses = append(metadata.NodeAddresses, corev1.NodeAddress{Type: corev1.NodeInternalIP, Address: v})
}
} else if address = node.Labels[InternalIPKey]; address != "" {
addresses = append(addresses, corev1.NodeAddress{Type: corev1.NodeInternalIP, Address: address})
metadata.NodeAddresses = append(metadata.NodeAddresses, corev1.NodeAddress{Type: corev1.NodeInternalIP, Address: address})
} else {
logrus.Infof("Couldn't find node internal ip annotation or label on node %s", node.Name)
}

// check external address
if address := node.Annotations[ExternalIPKey]; address != "" {
for _, v := range strings.Split(address, ",") {
addresses = append(addresses, corev1.NodeAddress{Type: corev1.NodeExternalIP, Address: v})
metadata.NodeAddresses = append(metadata.NodeAddresses, corev1.NodeAddress{Type: corev1.NodeExternalIP, Address: v})
}
} else if address = node.Labels[ExternalIPKey]; address != "" {
addresses = append(addresses, corev1.NodeAddress{Type: corev1.NodeExternalIP, Address: address})
metadata.NodeAddresses = append(metadata.NodeAddresses, corev1.NodeAddress{Type: corev1.NodeExternalIP, Address: address})
}

// check internal dns
if address := node.Annotations[InternalDNSKey]; address != "" {
for _, v := range strings.Split(address, ",") {
metadata.NodeAddresses = append(metadata.NodeAddresses, corev1.NodeAddress{Type: corev1.NodeInternalDNS, Address: v})
}
}

// check external dns
if address := node.Annotations[ExternalDNSKey]; address != "" {
for _, v := range strings.Split(address, ",") {
metadata.NodeAddresses = append(metadata.NodeAddresses, corev1.NodeAddress{Type: corev1.NodeExternalDNS, Address: v})
}
}

// check hostname
if address := node.Annotations[HostnameKey]; address != "" {
addresses = append(addresses, corev1.NodeAddress{Type: corev1.NodeHostName, Address: address})
metadata.NodeAddresses = append(metadata.NodeAddresses, corev1.NodeAddress{Type: corev1.NodeHostName, Address: address})
} else if address = node.Labels[HostnameKey]; address != "" {
addresses = append(addresses, corev1.NodeAddress{Type: corev1.NodeHostName, Address: address})
metadata.NodeAddresses = append(metadata.NodeAddresses, corev1.NodeAddress{Type: corev1.NodeHostName, Address: address})
} else {
logrus.Infof("Couldn't find node hostname annotation or label on node %s", node.Name)
}

return &cloudprovider.InstanceMetadata{
ProviderID: fmt.Sprintf("%s://%s", version.Program, node.Name),
InstanceType: version.Program,
NodeAddresses: addresses,
Zone: "",
Region: "",
}, nil
return metadata, nil
}
24 changes: 16 additions & 8 deletions pkg/daemons/config/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,11 @@ import (
"strings"
"sync"

"github.com/xiaods/k8e/pkg/generated/controllers/k8e.cattle.io"
"github.com/k3s-io/kine/pkg/endpoint"
"github.com/rancher/wharfie/pkg/registries"
"github.com/rancher/wrangler/v3/pkg/generated/controllers/core"
"github.com/rancher/wrangler/v3/pkg/leader"
"github.com/xiaods/k8e/pkg/generated/controllers/k8e.cattle.io"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
utilnet "k8s.io/apimachinery/pkg/util/net"
"k8s.io/apiserver/pkg/authentication/authenticator"
Expand All @@ -22,12 +22,12 @@ import (
)

const (
EgressSelectorModeAgent = "agent"
EgressSelectorModeCluster = "cluster"
EgressSelectorModeDisabled = "disabled"
EgressSelectorModePod = "pod"
CertificateRenewDays = 90
StreamServerPort = "10010"
EgressSelectorModeAgent = "agent"
EgressSelectorModeCluster = "cluster"
EgressSelectorModeDisabled = "disabled"
EgressSelectorModePod = "pod"
CertificateRenewDays = 90
StreamServerPort = "10010"
)

type Node struct {
Expand Down Expand Up @@ -105,18 +105,22 @@ type Agent struct {
ResolvConf string
RootDir string
KubeConfigKubelet string
KubeConfigKubeProxy string
KubeConfigK8eController string
NodeIP string
NodeIPs []net.IP
NodeExternalIP string
NodeExternalIPs []net.IP
NodeInternalDNSs []string
NodeExternalDNSs []string
RuntimeSocket string
ImageServiceSocket string
ListenAddress string
ClientCA string
CNIBinDir string
CNIConfDir string
ExtraKubeletArgs []string
ExtraKubeProxyArgs []string
PauseImage string
Snapshotter string
Systemd bool
Expand All @@ -130,10 +134,12 @@ type Agent struct {
SystemDefaultRegistry string
AirgapExtraRegistry []string
DisableCCM bool
DisableNPC bool
MinTLSVersion string
CipherSuites []string
Rootless bool
ProtectKernelDefaults bool
DisableServiceLB bool
EnableIPv4 bool
EnableIPv6 bool
VLevel int
Expand Down Expand Up @@ -328,6 +334,8 @@ type ControlRuntime struct {
ClientControllerKey string
ClientSchedulerCert string
ClientSchedulerKey string
ClientKubeProxyCert string
ClientKubeProxyKey string
ClientKubeletKey string
ClientCloudControllerCert string
ClientCloudControllerKey string
Expand Down Expand Up @@ -442,4 +450,4 @@ func GetArgs(initialArgs map[string]string, extraArgs []string) []string {
}

return args
}
}
Loading