Fixes squashfs images creation (#2230)

* Fixes squashfs images creation (bsc#1233289) If upgrading from a container including the root of the host mounted in /host the upgrade process does not exclude the /host path and other stateful paths such as /run. This commit sets the default excludes used in rsync calls to also apply for mksquashfs. Signed-off-by: David Cassany <[email protected]> * Define static method for default exclude relative paths This commit defines static methods for excluded paths in sync operations. One method for relative paths and another one with the same result rooted to a given path. It also uses wildcards to only exclude certain directories content while keeping the directory itself. Signed-off-by: David Cassany <[email protected]> * Add some additional unit tests for rsync wrappers Signed-off-by: David Cassany <[email protected]> --------- Signed-off-by: David Cassany <[email protected]>
rancher · Nov 13, 2024 · 527c12d · 527c12d
1 parent 5f996b5
commit 527c12d
Show file tree

Hide file tree

Showing 4 changed files with 111 additions and 47 deletions.
diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go
@@ -205,24 +205,35 @@ const (
 	ActionBuildDisk       = "build-disk"
 )
 
-// GetDefaultSystemEcludes returns a list of transient paths
+// GetDefaultSystemExcludes returns a list of paths
 // that are commonly present in an Elemental based running system.
-// Those paths are not needed or wanted in order to replicate the
-// root-tree as they are generated at runtime.
-func GetDefaultSystemExcludes(rootDir string) []string {
+// Those paths are not needed or wanted in order to replicate the root-tree.
+func GetDefaultSystemExcludes() []string {
 	return []string{
-		filepath.Join(rootDir, "/.snapshots"),
-		filepath.Join(rootDir, "/mnt"),
-		filepath.Join(rootDir, "/proc"),
-		filepath.Join(rootDir, "/sys"),
-		filepath.Join(rootDir, "/dev"),
-		filepath.Join(rootDir, "/tmp"),
-		filepath.Join(rootDir, "/run"),
-		filepath.Join(rootDir, "/host"),
-		filepath.Join(rootDir, "/etc/resolv.conf"),
+		".snapshots",
+		"mnt/*",
+		"proc/*",
+		"sys/*",
+		"dev/*",
+		"tmp/*",
+		"run/*",
+		"host",
+		"etc/resolv.conf",
 	}
 }
 
+// GetDefaultSystemExcludes returns a list of transient paths
+// that are commonly present in an Elemental based running system.
+// Paths are rooted to the given rootDir. Those paths are not
+// needed or wanted in order to replicate the root-tree.
+func GetDefaultSystemRootedExcludes(rootDir string) []string {
+	var list []string
+	for _, path := range GetDefaultSystemExcludes() {
+		list = append(list, filepath.Join(rootDir, path))
+	}
+	return list
+}
+
 func GetKernelPatterns() []string {
 	return []string{
 		"/boot/uImage*",

diff --git a/pkg/elemental/elemental.go b/pkg/elemental/elemental.go
@@ -369,13 +369,14 @@ func CreateImageFromTree(c types.Config, img *types.Image, rootDir string, prelo
 			c.Logger.Warnf("failed SELinux labelling at %s: %v", rootDir, err)
 		}
 
-		err = utils.CreateSquashFS(c.Runner, c.Logger, rootDir, img.File, c.SquashFsCompressionConfig)
+		excludes := cnst.GetDefaultSystemExcludes()
+		err = utils.CreateSquashFS(c.Runner, c.Logger, rootDir, img.File, c.SquashFsCompressionConfig, excludes...)
 		if err != nil {
 			c.Logger.Errorf("failed creating squashfs image for %s: %v", img.File, err)
 			return err
 		}
 	} else {
-		excludes := cnst.GetDefaultSystemExcludes(rootDir)
+		excludes := cnst.GetDefaultSystemRootedExcludes(rootDir)
 		err = CreateFileSystemImage(c, img, rootDir, preload, excludes...)
 		if err != nil {
 			c.Logger.Errorf("failed creating filesystem image: %v", err)
@@ -585,7 +586,7 @@ func DumpSource(
 		}
 		imgSrc.SetDigest(digest)
 	} else if imgSrc.IsDir() {
-		excludes := cnst.GetDefaultSystemExcludes(imgSrc.Value())
+		excludes := cnst.GetDefaultSystemRootedExcludes(imgSrc.Value())
 		err = syncFunc(c.Logger, c.Runner, c.Fs, imgSrc.Value(), target, excludes...)
 		if err != nil {
 			return err

diff --git a/pkg/utils/common.go b/pkg/utils/common.go
@@ -279,17 +279,25 @@ func CosignVerify(fs types.FS, runner types.Runner, image string, publicKey stri
 }
 
 // CreateSquashFS creates a squash file at destination from a source, with options
-// TODO: Check validity of source maybe?
-func CreateSquashFS(runner types.Runner, logger types.Logger, source string, destination string, options []string) error {
+func CreateSquashFS(runner types.Runner, logger types.Logger, source string, destination string, options []string, excludes ...string) error {
 	// create args
 	args := []string{source, destination}
 	// append options passed to args in order to have the correct order
 	// protect against options passed together in the same string , i.e. "-x add" instead of "-x", "add"
 	var optionsExpanded []string
 	for _, op := range options {
-		optionsExpanded = append(optionsExpanded, strings.Split(op, " ")...)
+		opExpanded := strings.Split(op, " ")
+		if opExpanded[0] == "-e" {
+			logger.Warnf("Ignoring option '%s', exclude directories must be passed as excludes argument", op)
+			continue
+		}
+		optionsExpanded = append(optionsExpanded, opExpanded...)
 	}
 	args = append(args, optionsExpanded...)
+	if len(excludes) >= 0 {
+		excludesOpt := append([]string{"-wildcards", "-e"}, excludes...)
+		args = append(args, excludesOpt...)
+	}
 	out, err := runner.Run("mksquashfs", args...)
 	if err != nil {
 		logger.Debugf("Error running squashfs creation, stdout: %s", out)

diff --git a/pkg/utils/utils_test.go b/pkg/utils/utils_test.go
@@ -452,13 +452,18 @@ var _ = Describe("Utils", Label("utils"), func() {
 			Expect(utils.CreateDirStructure(fs, "/my/root")).NotTo(BeNil())
 		})
 	})
-	Describe("SyncData", Label("SyncData"), func() {
-		It("Copies all files from source to target", func() {
-			sourceDir, err := utils.TempDir(fs, "", "elementalsource")
+	Describe("Rsync tests", Label("rsync"), func() {
+		var sourceDir, destDir string
+		var err error
+
+		BeforeEach(func() {
+			sourceDir, err = utils.TempDir(fs, "", "elementalsource")
 			Expect(err).ShouldNot(HaveOccurred())
-			destDir, err := utils.TempDir(fs, "", "elementaltarget")
+			destDir, err = utils.TempDir(fs, "", "elementaltarget")
 			Expect(err).ShouldNot(HaveOccurred())
+		})
 
+		It("Copies all files from source to target", func() {
 			for i := 0; i < 5; i++ {
 				_, _ = utils.TempFile(fs, sourceDir, "file*")
 			}
@@ -479,11 +484,6 @@ var _ = Describe("Utils", Label("utils"), func() {
 		})
 
 		It("Copies all files from source to target respecting excludes", func() {
-			sourceDir, err := utils.TempDir(fs, "", "elementalsource")
-			Expect(err).ShouldNot(HaveOccurred())
-			destDir, err := utils.TempDir(fs, "", "elementaltarget")
-			Expect(err).ShouldNot(HaveOccurred())
-
 			utils.MkdirAll(fs, filepath.Join(sourceDir, "host"), constants.DirPerm)
 			utils.MkdirAll(fs, filepath.Join(sourceDir, "run"), constants.DirPerm)
 
@@ -522,11 +522,6 @@ var _ = Describe("Utils", Label("utils"), func() {
 		})
 
 		It("Copies all files from source to target respecting excludes with '/' prefix", func() {
-			sourceDir, err := utils.TempDir(fs, "", "elementalsource")
-			Expect(err).ShouldNot(HaveOccurred())
-			destDir, err := utils.TempDir(fs, "", "elementaltarget")
-			Expect(err).ShouldNot(HaveOccurred())
-
 			utils.MkdirAll(fs, filepath.Join(sourceDir, "host"), constants.DirPerm)
 			utils.MkdirAll(fs, filepath.Join(sourceDir, "run"), constants.DirPerm)
 			utils.MkdirAll(fs, filepath.Join(sourceDir, "var", "run"), constants.DirPerm)
@@ -536,40 +531,65 @@ var _ = Describe("Utils", Label("utils"), func() {
 
 			filesDest, err := fs.ReadDir(destDir)
 			Expect(err).To(BeNil())
-
 			destNames := getNamesFromListFiles(filesDest)
 
 			filesSource, err := fs.ReadDir(sourceDir)
 			Expect(err).To(BeNil())
-
-			SourceNames := getNamesFromListFiles(filesSource)
+			sourceNames := getNamesFromListFiles(filesSource)
 
 			// Shouldn't be the same
-			Expect(destNames).ToNot(Equal(SourceNames))
+			Expect(destNames).ToNot(Equal(sourceNames))
 
 			Expect(utils.Exists(fs, filepath.Join(destDir, "var", "run"))).To(BeTrue())
 			Expect(utils.Exists(fs, filepath.Join(destDir, "tmp", "host"))).To(BeTrue())
 			Expect(utils.Exists(fs, filepath.Join(destDir, "host"))).To(BeFalse())
 			Expect(utils.Exists(fs, filepath.Join(destDir, "run"))).To(BeFalse())
 		})
 
+		It("Copies all files from source to target respecting excludes with wildcards", func() {
+			utils.MkdirAll(fs, filepath.Join(sourceDir, "run"), constants.DirPerm)
+			utils.MkdirAll(fs, filepath.Join(sourceDir, "var", "run"), constants.DirPerm)
+			Expect(fs.WriteFile(filepath.Join(sourceDir, "run", "testfile"), []byte{}, constants.DirPerm)).To(Succeed())
+
+			Expect(utils.SyncData(logger, realRunner, fs, sourceDir, destDir, "/run/*")).To(BeNil())
+
+			Expect(utils.Exists(fs, filepath.Join(destDir, "var", "run"))).To(BeTrue())
+			Expect(utils.Exists(fs, filepath.Join(destDir, "run"))).To(BeTrue())
+			Expect(utils.Exists(fs, filepath.Join(destDir, "run", "testfile"))).To(BeFalse())
+		})
+
+		It("Mirrors all files from source to destination deleting pre-existing files in destination if needed", func() {
+			utils.MkdirAll(fs, filepath.Join(sourceDir, "run"), constants.DirPerm)
+			utils.MkdirAll(fs, filepath.Join(sourceDir, "var", "run"), constants.DirPerm)
+			Expect(fs.WriteFile(filepath.Join(sourceDir, "run", "testfile"), []byte{}, constants.DirPerm)).To(Succeed())
+			Expect(fs.WriteFile(filepath.Join(destDir, "testfile"), []byte{}, constants.DirPerm)).To(Succeed())
+
+			Expect(utils.MirrorData(logger, realRunner, fs, sourceDir, destDir)).To(BeNil())
+
+			filesDest, err := fs.ReadDir(destDir)
+			Expect(err).To(BeNil())
+			destNames := getNamesFromListFiles(filesDest)
+
+			filesSource, err := fs.ReadDir(sourceDir)
+			Expect(err).To(BeNil())
+			sourceNames := getNamesFromListFiles(filesSource)
+
+			// Should be the same
+			Expect(destNames).To(Equal(sourceNames))
+
+			// pre-exising file in destination deleted if this is not part of source
+			Expect(utils.Exists(fs, filepath.Join(destDir, "testfile"))).To(BeFalse())
+		})
+
 		It("should not fail if dirs are empty", func() {
-			sourceDir, err := utils.TempDir(fs, "", "elementalsource")
-			Expect(err).ShouldNot(HaveOccurred())
-			destDir, err := utils.TempDir(fs, "", "elementaltarget")
-			Expect(err).ShouldNot(HaveOccurred())
 			Expect(utils.SyncData(logger, realRunner, fs, sourceDir, destDir)).To(BeNil())
 		})
 		It("should fail if destination does not exist", func() {
-			sourceDir, err := os.MkdirTemp("", "elemental")
-			Expect(err).To(BeNil())
-			defer os.RemoveAll(sourceDir)
+			fs.RemoveAll(destDir)
 			Expect(utils.SyncData(logger, realRunner, nil, sourceDir, "/welp")).NotTo(BeNil())
 		})
 		It("should fail if source does not exist", func() {
-			destDir, err := os.MkdirTemp("", "elemental")
-			Expect(err).To(BeNil())
-			defer os.RemoveAll(destDir)
+			fs.RemoveAll(sourceDir)
 			Expect(utils.SyncData(logger, realRunner, nil, "/welp", destDir)).NotTo(BeNil())
 		})
 	})
@@ -948,6 +968,30 @@ var _ = Describe("Utils", Label("utils"), func() {
 			})).To(BeNil())
 			Expect(err).ToNot(HaveOccurred())
 		})
+		It("ignores any '-e' option", func() {
+			args := append(constants.GetDefaultSquashfsCompressionOptions(), "-e /some/path")
+			err := utils.CreateSquashFS(runner, logger, "source", "dest", args)
+			cmd := []string{"mksquashfs", "source", "dest"}
+			cmd = append(cmd, constants.GetDefaultSquashfsCompressionOptions()...)
+			Expect(runner.IncludesCmds([][]string{
+				cmd,
+			})).To(BeNil())
+			Expect(err).ToNot(HaveOccurred())
+		})
+		It("excludes given paths", func() {
+
+			err := utils.CreateSquashFS(
+				runner, logger, "source", "dest", constants.GetDefaultSquashfsCompressionOptions(),
+				"some/path", "another/path",
+			)
+			cmd := []string{"mksquashfs", "source", "dest"}
+			cmd = append(cmd, constants.GetDefaultSquashfsCompressionOptions()...)
+			cmd = append(cmd, "-wildcards", "-e", "some/path", "another/path")
+			Expect(runner.IncludesCmds([][]string{
+				cmd,
+			})).To(Succeed())
+			Expect(err).ToNot(HaveOccurred())
+		})
 		It("returns an error if it fails", func() {
 			runner.ReturnError = errors.New("error")
 			err := utils.CreateSquashFS(runner, logger, "source", "dest", []string{})