Fix ipv6 endpoint address selection for on-demand snapshots

Signed-off-by: Brad Davidson <[email protected]>

pkg/cli/etcdsnapshot/etcd_snapshot.go

@@ -3,8 +3,8 @@ package etcdsnapshot
 import (
 	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
+	"net"
 	"os"
 	"path/filepath"
 	"sort"
@@ -17,7 +17,9 @@ import (
 	daemonconfig "github.com/k3s-io/k3s/pkg/daemons/config"
 	"github.com/k3s-io/k3s/pkg/etcd"
 	"github.com/k3s-io/k3s/pkg/server"
+	"github.com/k3s-io/k3s/pkg/util"
 	util2 "github.com/k3s-io/k3s/pkg/util"
+	"github.com/pkg/errors"
 	"github.com/rancher/wrangler/pkg/signals"
 	"github.com/urfave/cli"
 	"gopkg.in/yaml.v2"
@@ -52,6 +54,7 @@ func commandSetup(app *cli.Context, cfg *cmds.Server, config *server.Config) (*e
 
 	config.DisableAgent = true
 	config.ControlConfig.DataDir = dataDir
+	config.ControlConfig.BindAddress = cfg.BindAddress
 	config.ControlConfig.EtcdSnapshotName = cfg.EtcdSnapshotName
 	config.ControlConfig.EtcdSnapshotDir = cfg.EtcdSnapshotDir
 	config.ControlConfig.EtcdSnapshotCompress = cfg.EtcdSnapshotCompress
@@ -73,6 +76,46 @@ func commandSetup(app *cli.Context, cfg *cmds.Server, config *server.Config) (*e
 	config.ControlConfig.Runtime.ClientETCDKey = filepath.Join(dataDir, "tls", "etcd", "client.key")
 	config.ControlConfig.Runtime.KubeConfigAdmin = filepath.Join(dataDir, "cred", "admin.kubeconfig")
 
+	// We need to go through defaulting of cluster addresses to ensure that the etcd config for the standalone
+	// command uses the same endpoint selection logic as it does when starting up the full server. Specifically,
+	// we need to set an IPv6 service CIDR on IPv6-only or IPv6-first nodes, as the etcd default endpoints check
+	// the service CIDR primary addresss family to determine what loopback address to use.
+	_, nodeIPs, err := util.GetHostnameAndIPs(cmds.AgentConfig.NodeName, cmds.AgentConfig.NodeIP)
+	if err != nil {
+		return nil, err
+	}
+
+	// configure ClusterIPRanges. Use default 10.42.0.0/16 or fd00:42::/56 if user did not set it
+	_, defaultClusterCIDR, defaultServiceCIDR, _ := util.GetDefaultAddresses(nodeIPs[0])
+	if len(cfg.ClusterCIDR) == 0 {
+		cfg.ClusterCIDR.Set(defaultClusterCIDR)
+	}
+	for _, cidr := range util.SplitStringSlice(cfg.ClusterCIDR) {
+		_, parsed, err := net.ParseCIDR(cidr)
+		if err != nil {
+			return nil, errors.Wrapf(err, "invalid cluster-cidr %s", cidr)
+		}
+		config.ControlConfig.ClusterIPRanges = append(config.ControlConfig.ClusterIPRanges, parsed)
+	}
+
+	// set ClusterIPRange to the first address (first defined IPFamily is preferred)
+	config.ControlConfig.ClusterIPRange = config.ControlConfig.ClusterIPRanges[0]
+
+	// configure ServiceIPRanges. Use default 10.43.0.0/16 or fd00:43::/112 if user did not set it
+	if len(cfg.ServiceCIDR) == 0 {
+		cfg.ServiceCIDR.Set(defaultServiceCIDR)
+	}
+	for _, cidr := range util.SplitStringSlice(cfg.ServiceCIDR) {
+		_, parsed, err := net.ParseCIDR(cidr)
+		if err != nil {
+			return nil, errors.Wrapf(err, "invalid service-cidr %s", cidr)
+		}
+		config.ControlConfig.ServiceIPRanges = append(config.ControlConfig.ServiceIPRanges, parsed)
+	}
+
+	// set ServiceIPRange to the first address (first defined IPFamily is preferred)
+	config.ControlConfig.ServiceIPRange = config.ControlConfig.ServiceIPRanges[0]
+
 	e := etcd.NewETCD()
 	if err := e.SetControlConfig(&config.ControlConfig); err != nil {
 		return nil, err

pkg/cli/server/server.go

@@ -439,6 +439,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
 		serverConfig.ControlConfig.DisableControllerManager = true
 		serverConfig.ControlConfig.DisableScheduler = true
 		serverConfig.ControlConfig.DisableCCM = true
+		serverConfig.ControlConfig.DisableServiceLB = true
 
 		// If the supervisor and apiserver are on the same port, everything is running embedded
 		// and we don't need the kubelet or containerd up to perform a cluster reset.

pkg/etcd/etcd.go

@@ -399,6 +399,7 @@ func (e *ETCD) Reset(ctx context.Context, rebootstrap func() error) error {
 	if err := os.WriteFile(e.ResetFile(), []byte{}, 0600); err != nil {
 		return err
 	}
+
 	return e.newCluster(ctx, true)
 }
 
@@ -756,7 +757,7 @@ func getAdvertiseAddress(advertiseIP string) (string, error) {
 
 // newCluster returns options to set up etcd for a new cluster
 func (e *ETCD) newCluster(ctx context.Context, reset bool) error {
-	logrus.Infof("Starting etcd for new cluster")
+	logrus.Infof("Starting etcd for new cluster, cluster-reset=%v", reset)
 	err := e.cluster(ctx, reset, executor.InitialOptions{
 		AdvertisePeerURL: e.peerURL(),
 		Cluster:          fmt.Sprintf("%s=%s", e.name, e.peerURL()),
@@ -765,8 +766,10 @@ func (e *ETCD) newCluster(ctx context.Context, reset bool) error {
 	if err != nil {
 		return err
 	}
-	if err := e.migrateFromSQLite(ctx); err != nil {
-		return fmt.Errorf("failed to migrate content from sqlite to etcd: %w", err)
+	if !reset {
+		if err := e.migrateFromSQLite(ctx); err != nil {
+			return fmt.Errorf("failed to migrate content from sqlite to etcd: %w", err)
+		}
 	}
 	return nil
 }
@@ -847,7 +850,7 @@ func (e *ETCD) clientURL() string {
 // on other nodes connect mid-process.
 func (e *ETCD) advertiseClientURLs(reset bool) string {
 	if reset {
-		return fmt.Sprintf("https://%s", net.JoinHostPort(e.config.Loopback(true), "2379"))
+		return fmt.Sprintf("https://%s:2379", e.config.Loopback(true))
 	}
 	return e.clientURL()
 }

pkg/etcd/etcd_test.go

@@ -47,7 +47,7 @@ func generateTestConfig() *config.Control {
 		EtcdSnapshotRetention: 5,
 		EtcdS3Endpoint:        "s3.amazonaws.com",
 		EtcdS3Region:          "us-east-1",
-		SANs:                  []string{"127.0.0.1"},
+		SANs:                  []string{"127.0.0.1", mustGetAddress()},
 		CriticalControlArgs:   criticalControlArgs,
 	}
 }