Expand downgrade test by upgrading back to origin after downgrading (#…

…2224)

* Expand downgrade test by upgrading back to origin after downgrading

Signed-off-by: David Cassany <[email protected]>

* Adapt github worflow to handle cache for downgrade tests and import the container images

Signed-off-by: David Cassany <[email protected]>

---------

Signed-off-by: David Cassany <[email protected]>

.github/workflows/build_and_test_x86.yaml

@@ -201,7 +201,7 @@ jobs:
           go-version-file: go.mod
       - run: |
           git fetch --prune --unshallow
-      - if: ${{ matrix.test == 'test-upgrade' }}
+      - if: ${{ matrix.test == 'test-upgrade' || matrix.test == 'test-downgrade' }}
         name: Pull images
         uses: ./.github/actions/fetchimages
         with:

Dockerfile

@@ -57,6 +57,7 @@ RUN ARCH=$(uname -m); \
         gptfdisk \
         patterns-microos-selinux \
         btrfsprogs \
+	snapper \
         lvm2 && \
     zypper cc -a
 

Makefile

@@ -7,7 +7,7 @@ ISO?=$(shell ls $(ROOT_DIR)/build/elemental*.iso 2> /dev/null)
 FLAVOR?=green
 ARCH?=$(shell uname -m)
 UPGRADE_DISK_URL?=https://github.com/rancher/elemental-toolkit/releases/download/v1.1.4/elemental-$(FLAVOR)-v1.1.4.$(ARCH).qcow2
-UPGRADE_DISK?=upgrade-test-elemental-disk.qcow2
+UPGRADE_DISK?=upgrade-test-elemental-disk-$(FLAVOR).qcow2
 UPGRADE_DISK_CHECK?=$(shell ls $(ROOT_DIR)/build/$(UPGRADE_DISK) 2> /dev/null)
 PLATFORM?=linux/$(ARCH)
 IMAGE_SIZE?=20G

make/Makefile.test

@@ -10,16 +10,19 @@ endif
 	@scripts/run_vm.sh start $(DISK)
 	@echo "VM started from $(DISK)"
 
+.PHONY: prepare-local-registry
+prepare-local-registry:
+	@scripts/run_registry.sh start
+	@scripts/run_registry.sh push "$(TOOLKIT_REPO):$(VERSION)" "$(REPO):$(VERSION)"
+
 .PHONY: prepare-upgrade-test
 prepare-upgrade-test: 
 ifeq ("$(UPGRADE_DISK_CHECK)","")
 	mkdir -p $(ROOT_DIR)/build
 	@echo "Downloading disk from $(UPGRADE_DISK_URL) ..."
 	@wget -q --tries=3 $(UPGRADE_DISK_URL) -O $(ROOT_DIR)/build/$(UPGRADE_DISK)
 endif
-	@scripts/run_registry.sh start
 	@scripts/run_vm.sh start $(ROOT_DIR)/build/$(UPGRADE_DISK)
-	@scripts/run_registry.sh push "$(TOOLKIT_REPO):$(VERSION)" "$(REPO):$(VERSION)"
 	@echo "VM started from $(ROOT_DIR)/build/$(UPGRADE_DISK)"
 
 .PHONY: test-active
@@ -70,12 +73,12 @@ test-fsck: test-active
 	VM_PID=$$(scripts/run_vm.sh vmpid) go run $(GINKGO) $(GINKGO_ARGS) ./tests/fsck
 
 .PHONY: test-downgrade
-test-downgrade: test-active
-	VM_PID=$$(scripts/run_vm.sh vmpid) go run $(GINKGO) $(GINKGO_ARGS) ./tests/downgrade -- $(UPGRADE_ARGS)
+test-downgrade: prepare-test prepare-local-registry
+	@scripts/run_upgradetest.sh start $(GINKGO) "$(GINKGO_ARGS)" "./tests/downgrade" "$(TOOLKIT_REPO):$(VERSION)" "$(REPO):$(VERSION)"
 
 .PHONY: test-upgrade
-test-upgrade: prepare-upgrade-test
-	@scripts/run_upgradetest.sh start $(GINKGO) "$(GINKGO_ARGS)" "$(TOOLKIT_REPO):$(VERSION)" "$(REPO):$(VERSION)"
+test-upgrade: prepare-upgrade-test prepare-local-registry
+	@scripts/run_upgradetest.sh start $(GINKGO) "$(GINKGO_ARGS)" "./tests/upgrade" "$(TOOLKIT_REPO):$(VERSION)" "$(REPO):$(VERSION)"
 
 .PHONY: test-cli
 test-cli:

scripts/run_upgradetest.sh

@@ -15,8 +15,9 @@ function _abort {
 function start {
   local ginkgo="$1"
   local ginkgo_args="$2"
-  local toolkit_img="$3"
-  local upgrade_img="$4"
+  local suite_folder="$3"
+  local toolkit_img="$4"
+  local upgrade_img="$5"
   local reg_url
 
   export VM_PID=$(${SCRIPTS_PATH}/run_vm.sh vmpid)
@@ -26,7 +27,7 @@ function start {
 
   pushd "${ROOT_PATH}" > /dev/null
     go run ${ginkgo} ${ginkgo_args} ./tests/wait-active
-    go run ${ginkgo} ${ginkgo_args} ./tests/upgrade -- \
+    go run ${ginkgo} ${ginkgo_args} ${suite_folder} -- \
       --toolkit-image=docker://${reg_url}/${toolkit_img} --upgrade-image=docker://${reg_url}/${upgrade_img} 
   popd > /dev/null
 }
@@ -36,10 +37,10 @@ cmd=$1
 case $cmd in
   start)
     shift
-    if [[ $# -ne 4 ]]; then
+    if [[ $# -ne 5 ]]; then
       _abort "Wrong number of arguments"
     fi
-    start "$1" "$2" "$3" "$4"
+    start "$1" "$2" "$3" "$4" "$5"
     ;;
   *)
     _abort "Unknown command: ${cmd}"

tests/assets/hacks_for_tests.yaml

@@ -0,0 +1,14 @@
+name: "Hacks for testing environment"
+stages:
+   initramfs:
+   # This is needed in case some unit service have been
+   # refactored across distro releases.
+   # Fixes downgrade test (Nov 2024)
+   - name: "Remove broken unit links"
+     commands:
+     - |
+       find /etc/systemd/system/ -type l -print0 | while IFS= read -r -d $'\0' file; do
+         [ -e ${file} ] && continue
+         echo "Removing broken link ${file}"
+         rm -f "${file}"
+       done

tests/downgrade/downgrade_test.go

@@ -42,10 +42,11 @@ var _ = Describe("Elemental Feature tests", func() {
 	})
 
 	Context("After install", func() {
-		It("downgrades to an older image including upgrade and reset hooks", func() {
+		It("downgrades to an older image to then upgrade again to current including upgrade and reset hooks", func() {
 			By("setting /oem/chroot_hooks.yaml")
-			err := s.SendFile("../assets/chroot_hooks.yaml", "/oem/chroot_hooks.yaml", "0770")
-			Expect(err).ToNot(HaveOccurred())
+			Expect(s.SendFile("../assets/chroot_hooks.yaml", "/oem/chroot_hooks.yaml", "0770")).To(Succeed())
+			Expect(s.SendFile("../assets/hacks_for_tests.yaml", "/oem/hacks_for_tests.yaml", "0770")).To(Succeed())
+
 			originalVersion := s.GetOSRelease("TIMESTAMP")
 
 			By(fmt.Sprintf("upgrading to %s", comm.DefaultUpgradeImage))
@@ -56,8 +57,8 @@ var _ = Describe("Elemental Feature tests", func() {
 
 			s.Reboot()
 			s.EventuallyBootedFrom(sut.Active)
-			currentVersion := s.GetOSRelease("TIMESTAMP")
-			Expect(currentVersion).NotTo(Equal(originalVersion))
+			downgradedVersion := s.GetOSRelease("TIMESTAMP")
+			Expect(downgradedVersion).NotTo(Equal(originalVersion))
 
 			_, err = s.Command("cat /after-upgrade-chroot")
 			Expect(err).ToNot(HaveOccurred())
@@ -70,8 +71,10 @@ var _ = Describe("Elemental Feature tests", func() {
 			s.Reboot()
 			s.EventuallyBootedFrom(sut.Passive)
 			passiveVersion := s.GetOSRelease("TIMESTAMP")
-			Expect(currentVersion).NotTo(Equal(passiveVersion))
+			Expect(downgradedVersion).NotTo(Equal(passiveVersion))
+			Expect(originalVersion).To(Equal(passiveVersion))
 
+			// Tests we can upgrade active from passive
 			By(fmt.Sprintf("Upgrading again from passive to %s", comm.UpgradeImage()))
 			out, err = s.Command(s.ElementalCmd("upgrade", "--system", comm.UpgradeImage()))
 			Expect(err).ToNot(HaveOccurred())
@@ -80,6 +83,34 @@ var _ = Describe("Elemental Feature tests", func() {
 			By("Rebooting to active")
 			s.Reboot()
 			s.EventuallyBootedFrom(sut.Active)
+			activeVersion := s.GetOSRelease("TIMESTAMP")
+			Expect(downgradedVersion).NotTo(Equal(activeVersion))
+			Expect(originalVersion).To(Equal(activeVersion))
+
+			By("Rebooting to passive")
+			s.ChangeBootOnce(sut.Passive)
+			s.Reboot()
+			s.EventuallyBootedFrom(sut.Passive)
+			passiveVersion = s.GetOSRelease("TIMESTAMP")
+			Expect(downgradedVersion).To(Equal(passiveVersion))
+			Expect(originalVersion).NotTo(Equal(passiveVersion))
+
+			// Test we can upgrade from the downgraded version back to original
+			By(fmt.Sprintf("Upgrading again from passive to %s", comm.UpgradeImage()))
+			upgradeCmd := s.ElementalCmd("upgrade", "--tls-verify=false", "--bootloader", "--system", comm.UpgradeImage())
+			out, err = s.NewPodmanRunCommand(comm.ToolkitImage(), fmt.Sprintf("-c \"mount --rbind /host/run /run && %s\"", upgradeCmd)).
+				Privileged().
+				NoTLSVerify().
+				WithMount("/", "/host").
+				Run()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(out).Should(ContainSubstring("Upgrade completed"))
+			By("Rebooting to active")
+			s.Reboot()
+			s.EventuallyBootedFrom(sut.Active)
+			activeVersion = s.GetOSRelease("TIMESTAMP")
+			Expect(downgradedVersion).NotTo(Equal(activeVersion))
+			Expect(originalVersion).To(Equal(activeVersion))
 		})
 	})
 })