Merge pull request #362 from xiaods/dev

update cmds

cmd/cert/main.go

@@ -5,17 +5,18 @@ import (
 	"errors"
 	"os"
 
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli"
 	"github.com/xiaods/k8e/pkg/cli/cert"
 	"github.com/xiaods/k8e/pkg/cli/cmds"
 	"github.com/xiaods/k8e/pkg/configfilearg"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
 )
 
 func main() {
 	app := cmds.NewApp()
 	app.Commands = []cli.Command{
 		cmds.NewCertCommands(
+			cert.Check,
 			cert.Rotate,
 			cert.RotateCA,
 		),
@@ -24,4 +25,4 @@ func main() {
 	if err := app.Run(configfilearg.MustParse(os.Args)); err != nil && !errors.Is(err, context.Canceled) {
 		logrus.Fatal(err)
 	}
-}
+}

cmd/k8e/main.go

@@ -4,14 +4,15 @@ import (
 	"bytes"
 	"context"
 	"io"
+	"io/fs"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strconv"
 	"strings"
 
 	"github.com/pkg/errors"
-	"github.com/rancher/wrangler/pkg/resolvehome"
+	"github.com/rancher/wrangler/v3/pkg/resolvehome"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/pflag"
 	"github.com/urfave/cli"
@@ -29,6 +30,10 @@ var criDefaultConfigPath = "/etc/crictl.yaml"
 
 // main entrypoint for the k8e multicall binary
 func main() {
+	if findDebug(os.Args) {
+		logrus.SetLevel(logrus.DebugLevel)
+	}
+
 	dataDir := findDataDir(os.Args)
 
 	// Handle direct invocation via symlink alias (multicall binary behavior)
@@ -51,33 +56,32 @@ func main() {
 		cmds.NewCRICTL(externalCLIAction("crictl", dataDir)),
 		cmds.NewCtrCommand(externalCLIAction("ctr", dataDir)),
 		cmds.NewCheckConfigCommand(externalCLIAction("check-config", dataDir)),
-		cmds.NewInitOSConfigCommand(externalCLIAction("init-os-config", dataDir)),
 		cmds.NewTokenCommands(
 			tokenCommand,
 			tokenCommand,
 			tokenCommand,
 			tokenCommand,
 			tokenCommand,
 		),
-		cmds.NewEtcdSnapshotCommand(etcdsnapshotCommand,
-			cmds.NewEtcdSnapshotSubcommands(
-				etcdsnapshotCommand,
-				etcdsnapshotCommand,
-				etcdsnapshotCommand,
-				etcdsnapshotCommand),
+		cmds.NewEtcdSnapshotCommands(
+			etcdsnapshotCommand,
+			etcdsnapshotCommand,
+			etcdsnapshotCommand,
+			etcdsnapshotCommand,
 		),
-		cmds.NewSecretsEncryptCommand(secretsencryptCommand,
-			cmds.NewSecretsEncryptSubcommands(
-				secretsencryptCommand,
-				secretsencryptCommand,
-				secretsencryptCommand,
-				secretsencryptCommand,
-				secretsencryptCommand,
-				secretsencryptCommand),
+		cmds.NewSecretsEncryptCommands(
+			secretsencryptCommand,
+			secretsencryptCommand,
+			secretsencryptCommand,
+			secretsencryptCommand,
+			secretsencryptCommand,
+			secretsencryptCommand,
+			secretsencryptCommand,
 		),
 		cmds.NewCertCommands(
 			certCommand,
 			certCommand,
+			certCommand,
 		),
 		cmds.NewCompletionCommand(internalCLIAction(version.Program+"-completion", dataDir, os.Args)),
 	}
@@ -87,11 +91,32 @@ func main() {
 	}
 }
 
-// findDataDir reads data-dir settings from the CLI args and config file.
+// findDebug reads debug settings from the environment, CLI args, and config file.
+func findDebug(args []string) bool {
+	debug, _ := strconv.ParseBool(os.Getenv(version.ProgramUpper + "_DEBUG"))
+	if debug {
+		return debug
+	}
+	fs := pflag.NewFlagSet("debug-set", pflag.ContinueOnError)
+	fs.ParseErrorsWhitelist.UnknownFlags = true
+	fs.SetOutput(io.Discard)
+	fs.BoolVarP(&debug, "debug", "", false, "(logging) Turn on debug logs")
+	fs.Parse(args)
+	if debug {
+		return debug
+	}
+	debug, _ = strconv.ParseBool(configfilearg.MustFindString(args, "debug"))
+	return debug
+}
+
+// findDataDir reads data-dir settings from the environment, CLI args, and config file.
 // If not found, the default will be used, which varies depending on whether
 // k8e is being run as root or not.
 func findDataDir(args []string) string {
-	var dataDir string
+	dataDir := os.Getenv(version.ProgramUpper + "_DATA_DIR")
+	if dataDir != "" {
+		return dataDir
+	}
 	fs := pflag.NewFlagSet("data-dir-set", pflag.ContinueOnError)
 	fs.ParseErrorsWhitelist.UnknownFlags = true
 	fs.SetOutput(io.Discard)
@@ -161,7 +186,7 @@ func externalCLI(cli, dataDir string, args []string) error {
 	return stageAndRun(dataDir, cli, append([]string{cli}, args...), false)
 }
 
-// internalCLIAction returns a function that will call a k8e internal command, be used as the Action of a cli.Command.
+// internalCLIAction returns a function that will call a K8e internal command, be used as the Action of a cli.Command.
 func internalCLIAction(cmd, dataDir string, args []string) func(ctx *cli.Context) error {
 	return func(ctx *cli.Context) error {
 		// We don't want the Info logs seen when printing the autocomplete script
@@ -185,16 +210,24 @@ func stageAndRun(dataDir, cmd string, args []string, calledAsInternal bool) erro
 	}
 	logrus.Debugf("Asset dir %s", dir)
 
-	var pathEnv string
+	pathList := []string{
+		filepath.Clean(filepath.Join(dir, "..", "cni")),
+		filepath.Join(dir, "bin"),
+	}
 	if findPreferBundledBin(args) {
-		pathEnv = filepath.Join(dir, "bin") + string(os.PathListSeparator) + filepath.Join(dir, "bin/aux") + string(os.PathListSeparator) + os.Getenv("PATH")
+		pathList = append(
+			pathList,
+			filepath.Join(dir, "bin", "aux"),
+			os.Getenv("PATH"),
+		)
 	} else {
-		pathEnv = filepath.Join(dir, "bin") + string(os.PathListSeparator) + os.Getenv("PATH") + string(os.PathListSeparator) + filepath.Join(dir, "bin/aux")
-	}
-	if err := os.Setenv("PATH", pathEnv); err != nil {
-		return err
+		pathList = append(
+			pathList,
+			os.Getenv("PATH"),
+			filepath.Join(dir, "bin", "aux"),
+		)
 	}
-	if err := os.Setenv(version.ProgramUpper+"_DATA_DIR", dir); err != nil {
+	if err := os.Setenv("PATH", strings.Join(pathList, string(os.PathListSeparator))); err != nil {
 		return err
 	}
 
@@ -269,6 +302,53 @@ func extract(dataDir string) (string, error) {
 		return "", err
 	}
 
+	// Create a stable CNI bin dir and place it first in the path so that users have a
+	// consistent location to drop their own CNI plugin binaries.
+	cniPath := filepath.Join(dataDir, "data", "cni")
+	cniBin := filepath.Join(dir, "bin", "cni")
+	if err := os.MkdirAll(cniPath, 0755); err != nil {
+		return "", err
+	}
+	// Create symlink that points at the cni multicall binary itself
+	logrus.Debugf("Creating symlink %s -> %s", filepath.Join(cniPath, "cni"), cniBin)
+	os.Remove(filepath.Join(cniPath, "cni"))
+	if err := os.Symlink(cniBin, filepath.Join(cniPath, "cni")); err != nil {
+		return "", err
+	}
+
+	// Find symlinks that point to the cni multicall binary, and clone them in the stable CNI bin dir.
+	// Non-symlink plugins in the stable CNI bin dir will not be overwritten, to allow users to replace our
+	// CNI plugins with their own versions if they want. Note that the cni multicall binary itself is always
+	// symlinked into the stable bin dir and should not be replaced.
+	ents, err := os.ReadDir(filepath.Join(tempDest, "bin"))
+	if err != nil {
+		return "", err
+	}
+	for _, ent := range ents {
+		if info, err := ent.Info(); err == nil && info.Mode()&fs.ModeSymlink != 0 {
+			if target, err := os.Readlink(filepath.Join(tempDest, "bin", ent.Name())); err == nil && target == "cni" {
+				src := filepath.Join(cniPath, ent.Name())
+				// Check if plugin already exists in stable CNI bin dir
+				if info, err := os.Lstat(src); err == nil {
+					if info.Mode()&fs.ModeSymlink != 0 {
+						// Exists and is a symlink, remove it so we can create a new symlink for the new bin.
+						os.Remove(src)
+					} else {
+						// Not a symlink, leave it alone
+						logrus.Debugf("Not replacing non-symlink CNI plugin %s with mode %O", src, info.Mode())
+						continue
+					}
+				}
+				logrus.Debugf("Creating symlink %s -> %s", src, cniBin)
+				if err := os.Symlink(cniBin, src); err != nil {
+					return "", err
+				}
+			}
+		}
+	}
+
+	// Rotate 'current' symlink into 'previous', and create a new 'current' that points
+	// at the new directory.
 	currentSymLink := filepath.Join(dataDir, "data", "current")
 	previousSymLink := filepath.Join(dataDir, "data", "previous")
 	if _, err := os.Lstat(currentSymLink); err == nil {
@@ -279,7 +359,14 @@ func extract(dataDir string) (string, error) {
 	if err := os.Symlink(dir, currentSymLink); err != nil {
 		return "", err
 	}
-	return dir, os.Rename(tempDest, dir)
+
+	// Rename the new directory into place after updating symlinks, so that the k8e binary check at the start
+	// of this function only succeeds if everything else has been completed successfully.
+	if err := os.Rename(tempDest, dir); err != nil {
+		return "", err
+	}
+
+	return dir, nil
 }
 
 // findCriConfig returns the path to crictl.yaml

pkg/cli/cmds/agent.go

@@ -4,8 +4,8 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/xiaods/k8e/pkg/version"
 	"github.com/urfave/cli"
+	"github.com/xiaods/k8e/pkg/version"
 )
 
 type Agent struct {

pkg/cli/cmds/certs.go

@@ -23,23 +23,19 @@ var (
 		DataDirFlag,
 		&cli.StringSliceFlag{
 			Name:  "service,s",
-			Usage: "List of services to rotate certificates for. Options include (admin, api-server, controller-manager, scheduler, " + version.Program + "-controller, " + version.Program + "-server, cloud-controller, etcd, auth-proxy, kubelet)",
+			Usage: "List of services to manage certificates for. Options include (admin, api-server, controller-manager, scheduler, supervisor, " + version.Program + "-controller, " + version.Program + "-server, cloud-controller, etcd, auth-proxy, kubelet, kube-proxy)",
 			Value: &ServicesList,
 		},
 	}
 	CertRotateCACommandFlags = []cli.Flag{
+		DataDirFlag,
 		cli.StringFlag{
 			Name:        "server,s",
 			Usage:       "(cluster) Server to connect to",
 			EnvVar:      version.ProgramUpper + "_URL",
 			Value:       "https://127.0.0.1:6443",
 			Destination: &ServerConfig.ServerURL,
 		},
-		cli.StringFlag{
-			Name:        "data-dir,d",
-			Usage:       "(data) Folder to hold state default /var/lib/" + version.Program + " or ${HOME}/." + version.Program + " if not root",
-			Destination: &ServerConfig.DataDir,
-		},
 		cli.StringFlag{
 			Name:        "path",
 			Usage:       "Path to directory containing new CA certificates",
@@ -54,13 +50,21 @@ var (
 	}
 )
 
-func NewCertCommands(rotate, rotateCA func(ctx *cli.Context) error) cli.Command {
+func NewCertCommands(check, rotate, rotateCA func(ctx *cli.Context) error) cli.Command {
 	return cli.Command{
 		Name:            CertCommand,
 		Usage:           "Manage K8e certificates",
 		SkipFlagParsing: false,
 		SkipArgReorder:  true,
 		Subcommands: []cli.Command{
+			{
+				Name:            "check",
+				Usage:           "Check " + version.Program + " component certificates on disk",
+				SkipFlagParsing: false,
+				SkipArgReorder:  true,
+				Action:          check,
+				Flags:           CertRotateCommandFlags,
+			},
 			{
 				Name:            "rotate",
 				Usage:           "Rotate " + version.Program + " component certificates on disk",