From 527c12d007dcf9e51c15bf03dcf2d94ea599ffd4 Mon Sep 17 00:00:00 2001 From: David Cassany Viladomat Date: Wed, 13 Nov 2024 11:03:25 +0100 Subject: [PATCH] Fixes squashfs images creation (#2230) * Fixes squashfs images creation (bsc#1233289) If upgrading from a container including the root of the host mounted in /host the upgrade process does not exclude the /host path and other stateful paths such as /run. This commit sets the default excludes used in rsync calls to also apply for mksquashfs. Signed-off-by: David Cassany * Define static method for default exclude relative paths This commit defines static methods for excluded paths in sync operations. One method for relative paths and another one with the same result rooted to a given path. It also uses wildcards to only exclude certain directories content while keeping the directory itself. Signed-off-by: David Cassany * Add some additional unit tests for rsync wrappers Signed-off-by: David Cassany --------- Signed-off-by: David Cassany --- pkg/constants/constants.go | 37 +++++++++----- pkg/elemental/elemental.go | 7 +-- pkg/utils/common.go | 14 ++++-- pkg/utils/utils_test.go | 100 ++++++++++++++++++++++++++----------- 4 files changed, 111 insertions(+), 47 deletions(-) diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index d615ffd186f..06b387f9f54 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -205,24 +205,35 @@ const ( ActionBuildDisk = "build-disk" ) -// GetDefaultSystemEcludes returns a list of transient paths +// GetDefaultSystemExcludes returns a list of paths // that are commonly present in an Elemental based running system. -// Those paths are not needed or wanted in order to replicate the -// root-tree as they are generated at runtime. -func GetDefaultSystemExcludes(rootDir string) []string { +// Those paths are not needed or wanted in order to replicate the root-tree. +func GetDefaultSystemExcludes() []string { return []string{ - filepath.Join(rootDir, "/.snapshots"), - filepath.Join(rootDir, "/mnt"), - filepath.Join(rootDir, "/proc"), - filepath.Join(rootDir, "/sys"), - filepath.Join(rootDir, "/dev"), - filepath.Join(rootDir, "/tmp"), - filepath.Join(rootDir, "/run"), - filepath.Join(rootDir, "/host"), - filepath.Join(rootDir, "/etc/resolv.conf"), + ".snapshots", + "mnt/*", + "proc/*", + "sys/*", + "dev/*", + "tmp/*", + "run/*", + "host", + "etc/resolv.conf", } } +// GetDefaultSystemExcludes returns a list of transient paths +// that are commonly present in an Elemental based running system. +// Paths are rooted to the given rootDir. Those paths are not +// needed or wanted in order to replicate the root-tree. +func GetDefaultSystemRootedExcludes(rootDir string) []string { + var list []string + for _, path := range GetDefaultSystemExcludes() { + list = append(list, filepath.Join(rootDir, path)) + } + return list +} + func GetKernelPatterns() []string { return []string{ "/boot/uImage*", diff --git a/pkg/elemental/elemental.go b/pkg/elemental/elemental.go index cef5628fe78..d229d7826ff 100644 --- a/pkg/elemental/elemental.go +++ b/pkg/elemental/elemental.go @@ -369,13 +369,14 @@ func CreateImageFromTree(c types.Config, img *types.Image, rootDir string, prelo c.Logger.Warnf("failed SELinux labelling at %s: %v", rootDir, err) } - err = utils.CreateSquashFS(c.Runner, c.Logger, rootDir, img.File, c.SquashFsCompressionConfig) + excludes := cnst.GetDefaultSystemExcludes() + err = utils.CreateSquashFS(c.Runner, c.Logger, rootDir, img.File, c.SquashFsCompressionConfig, excludes...) if err != nil { c.Logger.Errorf("failed creating squashfs image for %s: %v", img.File, err) return err } } else { - excludes := cnst.GetDefaultSystemExcludes(rootDir) + excludes := cnst.GetDefaultSystemRootedExcludes(rootDir) err = CreateFileSystemImage(c, img, rootDir, preload, excludes...) if err != nil { c.Logger.Errorf("failed creating filesystem image: %v", err) @@ -585,7 +586,7 @@ func DumpSource( } imgSrc.SetDigest(digest) } else if imgSrc.IsDir() { - excludes := cnst.GetDefaultSystemExcludes(imgSrc.Value()) + excludes := cnst.GetDefaultSystemRootedExcludes(imgSrc.Value()) err = syncFunc(c.Logger, c.Runner, c.Fs, imgSrc.Value(), target, excludes...) if err != nil { return err diff --git a/pkg/utils/common.go b/pkg/utils/common.go index b42c58bced9..b597b04a233 100644 --- a/pkg/utils/common.go +++ b/pkg/utils/common.go @@ -279,17 +279,25 @@ func CosignVerify(fs types.FS, runner types.Runner, image string, publicKey stri } // CreateSquashFS creates a squash file at destination from a source, with options -// TODO: Check validity of source maybe? -func CreateSquashFS(runner types.Runner, logger types.Logger, source string, destination string, options []string) error { +func CreateSquashFS(runner types.Runner, logger types.Logger, source string, destination string, options []string, excludes ...string) error { // create args args := []string{source, destination} // append options passed to args in order to have the correct order // protect against options passed together in the same string , i.e. "-x add" instead of "-x", "add" var optionsExpanded []string for _, op := range options { - optionsExpanded = append(optionsExpanded, strings.Split(op, " ")...) + opExpanded := strings.Split(op, " ") + if opExpanded[0] == "-e" { + logger.Warnf("Ignoring option '%s', exclude directories must be passed as excludes argument", op) + continue + } + optionsExpanded = append(optionsExpanded, opExpanded...) } args = append(args, optionsExpanded...) + if len(excludes) >= 0 { + excludesOpt := append([]string{"-wildcards", "-e"}, excludes...) + args = append(args, excludesOpt...) + } out, err := runner.Run("mksquashfs", args...) if err != nil { logger.Debugf("Error running squashfs creation, stdout: %s", out) diff --git a/pkg/utils/utils_test.go b/pkg/utils/utils_test.go index c64f497349a..5af5a604fcb 100644 --- a/pkg/utils/utils_test.go +++ b/pkg/utils/utils_test.go @@ -452,13 +452,18 @@ var _ = Describe("Utils", Label("utils"), func() { Expect(utils.CreateDirStructure(fs, "/my/root")).NotTo(BeNil()) }) }) - Describe("SyncData", Label("SyncData"), func() { - It("Copies all files from source to target", func() { - sourceDir, err := utils.TempDir(fs, "", "elementalsource") + Describe("Rsync tests", Label("rsync"), func() { + var sourceDir, destDir string + var err error + + BeforeEach(func() { + sourceDir, err = utils.TempDir(fs, "", "elementalsource") Expect(err).ShouldNot(HaveOccurred()) - destDir, err := utils.TempDir(fs, "", "elementaltarget") + destDir, err = utils.TempDir(fs, "", "elementaltarget") Expect(err).ShouldNot(HaveOccurred()) + }) + It("Copies all files from source to target", func() { for i := 0; i < 5; i++ { _, _ = utils.TempFile(fs, sourceDir, "file*") } @@ -479,11 +484,6 @@ var _ = Describe("Utils", Label("utils"), func() { }) It("Copies all files from source to target respecting excludes", func() { - sourceDir, err := utils.TempDir(fs, "", "elementalsource") - Expect(err).ShouldNot(HaveOccurred()) - destDir, err := utils.TempDir(fs, "", "elementaltarget") - Expect(err).ShouldNot(HaveOccurred()) - utils.MkdirAll(fs, filepath.Join(sourceDir, "host"), constants.DirPerm) utils.MkdirAll(fs, filepath.Join(sourceDir, "run"), constants.DirPerm) @@ -522,11 +522,6 @@ var _ = Describe("Utils", Label("utils"), func() { }) It("Copies all files from source to target respecting excludes with '/' prefix", func() { - sourceDir, err := utils.TempDir(fs, "", "elementalsource") - Expect(err).ShouldNot(HaveOccurred()) - destDir, err := utils.TempDir(fs, "", "elementaltarget") - Expect(err).ShouldNot(HaveOccurred()) - utils.MkdirAll(fs, filepath.Join(sourceDir, "host"), constants.DirPerm) utils.MkdirAll(fs, filepath.Join(sourceDir, "run"), constants.DirPerm) utils.MkdirAll(fs, filepath.Join(sourceDir, "var", "run"), constants.DirPerm) @@ -536,16 +531,14 @@ var _ = Describe("Utils", Label("utils"), func() { filesDest, err := fs.ReadDir(destDir) Expect(err).To(BeNil()) - destNames := getNamesFromListFiles(filesDest) filesSource, err := fs.ReadDir(sourceDir) Expect(err).To(BeNil()) - - SourceNames := getNamesFromListFiles(filesSource) + sourceNames := getNamesFromListFiles(filesSource) // Shouldn't be the same - Expect(destNames).ToNot(Equal(SourceNames)) + Expect(destNames).ToNot(Equal(sourceNames)) Expect(utils.Exists(fs, filepath.Join(destDir, "var", "run"))).To(BeTrue()) Expect(utils.Exists(fs, filepath.Join(destDir, "tmp", "host"))).To(BeTrue()) @@ -553,23 +546,50 @@ var _ = Describe("Utils", Label("utils"), func() { Expect(utils.Exists(fs, filepath.Join(destDir, "run"))).To(BeFalse()) }) + It("Copies all files from source to target respecting excludes with wildcards", func() { + utils.MkdirAll(fs, filepath.Join(sourceDir, "run"), constants.DirPerm) + utils.MkdirAll(fs, filepath.Join(sourceDir, "var", "run"), constants.DirPerm) + Expect(fs.WriteFile(filepath.Join(sourceDir, "run", "testfile"), []byte{}, constants.DirPerm)).To(Succeed()) + + Expect(utils.SyncData(logger, realRunner, fs, sourceDir, destDir, "/run/*")).To(BeNil()) + + Expect(utils.Exists(fs, filepath.Join(destDir, "var", "run"))).To(BeTrue()) + Expect(utils.Exists(fs, filepath.Join(destDir, "run"))).To(BeTrue()) + Expect(utils.Exists(fs, filepath.Join(destDir, "run", "testfile"))).To(BeFalse()) + }) + + It("Mirrors all files from source to destination deleting pre-existing files in destination if needed", func() { + utils.MkdirAll(fs, filepath.Join(sourceDir, "run"), constants.DirPerm) + utils.MkdirAll(fs, filepath.Join(sourceDir, "var", "run"), constants.DirPerm) + Expect(fs.WriteFile(filepath.Join(sourceDir, "run", "testfile"), []byte{}, constants.DirPerm)).To(Succeed()) + Expect(fs.WriteFile(filepath.Join(destDir, "testfile"), []byte{}, constants.DirPerm)).To(Succeed()) + + Expect(utils.MirrorData(logger, realRunner, fs, sourceDir, destDir)).To(BeNil()) + + filesDest, err := fs.ReadDir(destDir) + Expect(err).To(BeNil()) + destNames := getNamesFromListFiles(filesDest) + + filesSource, err := fs.ReadDir(sourceDir) + Expect(err).To(BeNil()) + sourceNames := getNamesFromListFiles(filesSource) + + // Should be the same + Expect(destNames).To(Equal(sourceNames)) + + // pre-exising file in destination deleted if this is not part of source + Expect(utils.Exists(fs, filepath.Join(destDir, "testfile"))).To(BeFalse()) + }) + It("should not fail if dirs are empty", func() { - sourceDir, err := utils.TempDir(fs, "", "elementalsource") - Expect(err).ShouldNot(HaveOccurred()) - destDir, err := utils.TempDir(fs, "", "elementaltarget") - Expect(err).ShouldNot(HaveOccurred()) Expect(utils.SyncData(logger, realRunner, fs, sourceDir, destDir)).To(BeNil()) }) It("should fail if destination does not exist", func() { - sourceDir, err := os.MkdirTemp("", "elemental") - Expect(err).To(BeNil()) - defer os.RemoveAll(sourceDir) + fs.RemoveAll(destDir) Expect(utils.SyncData(logger, realRunner, nil, sourceDir, "/welp")).NotTo(BeNil()) }) It("should fail if source does not exist", func() { - destDir, err := os.MkdirTemp("", "elemental") - Expect(err).To(BeNil()) - defer os.RemoveAll(destDir) + fs.RemoveAll(sourceDir) Expect(utils.SyncData(logger, realRunner, nil, "/welp", destDir)).NotTo(BeNil()) }) }) @@ -948,6 +968,30 @@ var _ = Describe("Utils", Label("utils"), func() { })).To(BeNil()) Expect(err).ToNot(HaveOccurred()) }) + It("ignores any '-e' option", func() { + args := append(constants.GetDefaultSquashfsCompressionOptions(), "-e /some/path") + err := utils.CreateSquashFS(runner, logger, "source", "dest", args) + cmd := []string{"mksquashfs", "source", "dest"} + cmd = append(cmd, constants.GetDefaultSquashfsCompressionOptions()...) + Expect(runner.IncludesCmds([][]string{ + cmd, + })).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) + }) + It("excludes given paths", func() { + + err := utils.CreateSquashFS( + runner, logger, "source", "dest", constants.GetDefaultSquashfsCompressionOptions(), + "some/path", "another/path", + ) + cmd := []string{"mksquashfs", "source", "dest"} + cmd = append(cmd, constants.GetDefaultSquashfsCompressionOptions()...) + cmd = append(cmd, "-wildcards", "-e", "some/path", "another/path") + Expect(runner.IncludesCmds([][]string{ + cmd, + })).To(Succeed()) + Expect(err).ToNot(HaveOccurred()) + }) It("returns an error if it fails", func() { runner.ReturnError = errors.New("error") err := utils.CreateSquashFS(runner, logger, "source", "dest", []string{})