From 464702942861cc38597b9da0afa1a47212d26c71 Mon Sep 17 00:00:00 2001 From: Jiri Sztuka Date: Fri, 18 Oct 2024 10:59:11 +0200 Subject: [PATCH] feat(STONEINTG-1009): e2e tests for group snapshot Signed-off-by: Jiri Sztuka --- tests/integration-service/const.go | 6 + .../group-snapshots-tests.go | 266 ++++++++++++++++++ tests/integration-service/integration.go | 32 +++ 3 files changed, 304 insertions(+) create mode 100644 tests/integration-service/group-snapshots-tests.go diff --git a/tests/integration-service/const.go b/tests/integration-service/const.go index 730ae00073..f30ffed322 100644 --- a/tests/integration-service/const.go +++ b/tests/integration-service/const.go @@ -20,9 +20,12 @@ const ( componentRepoNameForGeneralIntegration = "konflux-test-integration" componentRepoNameForIntegrationWithEnv = "konflux-test-integration-with-env" componentRepoNameForStatusReporting = "konflux-test-integration-status-report" + multiComponentRepoNameForGroupSnapshotA = "sample-multi-component" + multiComponentRepoNameForGroupSnapshotB = "sample-multi-component" gitlabComponentRepoName = "hacbs-test-project-integration" componentDefaultBranch = "main" componentRevision = "34da5a8f51fba6a8b7ec75a727d3c72ebb5e1274" + componentGroupRevision = "5eb94a431254e52a4c5841694b44aef351ad55de" referenceDoesntExist = "Reference does not exist" checkrunStatusCompleted = "completed" checkrunConclusionSuccess = "success" @@ -35,6 +38,7 @@ const ( snapshotAnnotation = "appstudio.openshift.io/snapshot" scenarioAnnotation = "test.appstudio.openshift.io/scenario" + groupSnapshotAnnotation = "test.appstudio.openshift.io/pr-group" pipelinerunFinalizerByIntegrationService = "test.appstudio.openshift.io/pipelinerun" snapshotRerunLabel = "test.appstudio.openshift.io/run" @@ -45,6 +49,8 @@ var ( componentGitSourceURLForGeneralIntegration = fmt.Sprintf("https://github.com/%s/%s", utils.GetEnv(constants.GITHUB_E2E_ORGANIZATION_ENV, "redhat-appstudio-qe"), componentRepoNameForGeneralIntegration) componentGitSourceURLForIntegrationWithEnv = fmt.Sprintf("https://github.com/%s/%s", utils.GetEnv(constants.GITHUB_E2E_ORGANIZATION_ENV, "redhat-appstudio-qe"), componentRepoNameForIntegrationWithEnv) componentGitSourceURLForStatusReporting = fmt.Sprintf("https://github.com/%s/%s", utils.GetEnv(constants.GITHUB_E2E_ORGANIZATION_ENV, "redhat-appstudio-qe"), componentRepoNameForStatusReporting) + multiComponentGitSourceURLForGroupSnapshotA = fmt.Sprintf("https://github.com/%s/%s", utils.GetEnv(constants.GITHUB_E2E_ORGANIZATION_ENV, "redhat-appstudio-qe"), multiComponentRepoNameForGroupSnapshotA) + multiComponentGitSourceURLForGroupSnapshotB = fmt.Sprintf("https://github.com/%s/%s", utils.GetEnv(constants.GITHUB_E2E_ORGANIZATION_ENV, "redhat-appstudio-qe"), multiComponentRepoNameForGroupSnapshotB) gitlabOrg = utils.GetEnv(constants.GITLAB_QE_ORG_ENV, constants.DefaultGitLabQEOrg) gitlabProjectIDForStatusReporting = fmt.Sprintf("%s/%s", gitlabOrg, gitlabComponentRepoName) gitlabComponentGitSourceURLForStatusReporting = fmt.Sprintf("https://gitlab.com/%s/%s", gitlabOrg, gitlabComponentRepoName) diff --git a/tests/integration-service/group-snapshots-tests.go b/tests/integration-service/group-snapshots-tests.go new file mode 100644 index 0000000000..081c9e8865 --- /dev/null +++ b/tests/integration-service/group-snapshots-tests.go @@ -0,0 +1,266 @@ +package integration + +import ( + "fmt" + "os" + "time" + + "github.com/konflux-ci/e2e-tests/pkg/clients/has" + "github.com/konflux-ci/e2e-tests/pkg/constants" + "github.com/konflux-ci/e2e-tests/pkg/framework" + "github.com/konflux-ci/e2e-tests/pkg/utils" + + appstudioApi "github.com/konflux-ci/application-api/api/v1alpha1" + integrationv1beta2 "github.com/konflux-ci/integration-service/api/v1beta2" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + pipeline "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1" +) + +var _ = framework.IntegrationServiceSuiteDescribe("Creation of group snapshots for monorepo and multiple repos", Label("integration-service", "group-snapshot-creation"), func() { + defer GinkgoRecover() + + var f *framework.Framework + var err error + + var prNumber int + var timeout, interval time.Duration + var prHeadSha string + var snapshot *appstudioApi.Snapshot + var componentA *appstudioApi.Component + var componentB *appstudioApi.Component + var pipelineRun, testPipelinerun *pipeline.PipelineRun + var integrationTestScenarioPass *integrationv1beta2.IntegrationTestScenario + var applicationName, componentAName, componentBName, componentBaseBranchName, fromBranchNameA, fromBranchNameB, testNamespace, toBranch string + + AfterEach(framework.ReportFailure(&f)) + + Describe("with status reporting of Integration tests in CheckRuns", Ordered, func() { + BeforeAll(func() { + if os.Getenv(constants.SKIP_PAC_TESTS_ENV) == "true" { + Skip("Skipping this test due to configuration issue with Spray proxy") + } + + f, err = framework.NewFramework(utils.GetGeneratedNamespace("group")) + Expect(err).NotTo(HaveOccurred()) + testNamespace = f.UserNamespace + + if utils.IsPrivateHostname(f.OpenshiftConsoleHost) { + Skip("Using private cluster (not reachable from Github), skipping...") + } + toBranch := "test-group-branch" + contextDirA := "go-component/docker" + contextDirB := "python-component/docker" + applicationName = createApp(*f, testNamespace) + componentA, componentAName, fromBranchNameA, componentBaseBranchName = createComponentWithCustomBranch(*f, testNamespace, applicationName, multiComponentRepoNameForGroupSnapshotA, multiComponentGitSourceURLForGroupSnapshotA, toBranch, contextDirA) + componentB, componentBName, fromBranchNameB, componentBaseBranchName = createComponentWithCustomBranch(*f, testNamespace, applicationName, multiComponentRepoNameForGroupSnapshotB, multiComponentGitSourceURLForGroupSnapshotB, toBranch, contextDirB) + + integrationTestScenarioPass, err = f.AsKubeAdmin.IntegrationController.CreateIntegrationTestScenario("", applicationName, testNamespace, gitURL, revision, pathInRepoPass, []string{}) + Expect(err).ShouldNot(HaveOccurred()) + }) + + AfterAll(func() { + if !CurrentSpecReport().Failed() { + cleanup(*f, testNamespace, applicationName, componentAName) + cleanup(*f, testNamespace, applicationName, componentBName) + } + + // Delete new branches created by PaC and a testing branch used as a component's A base branch + err = f.AsKubeAdmin.CommonController.Github.DeleteRef(multiComponentRepoNameForGroupSnapshotA, fromBranchNameA) + if err != nil { + Expect(err.Error()).To(ContainSubstring(referenceDoesntExist)) + } + err = f.AsKubeAdmin.CommonController.Github.DeleteRef(multiComponentRepoNameForGroupSnapshotA, componentBaseBranchName) + if err != nil { + Expect(err.Error()).To(ContainSubstring(referenceDoesntExist)) + } + + // Delete new branches created by PaC and a testing branch used as a component's B base branch + err = f.AsKubeAdmin.CommonController.Github.DeleteRef(multiComponentRepoNameForGroupSnapshotB, fromBranchNameB) + if err != nil { + Expect(err.Error()).To(ContainSubstring(referenceDoesntExist)) + } + err = f.AsKubeAdmin.CommonController.Github.DeleteRef(multiComponentRepoNameForGroupSnapshotB, componentBaseBranchName) + if err != nil { + Expect(err.Error()).To(ContainSubstring(referenceDoesntExist)) + } + }) + /* /\ + / \ + / /\ \ + / ____ \ + /_/ \_\ */ + When("a new Component A with specified custom branch(same as component B) is created", Label("custom-branch"), func() { + It("triggers a Build PipelineRun", func() { + timeout = time.Second * 600 + interval = time.Second * 1 + Eventually(func() error { + pipelineRun, err = f.AsKubeAdmin.HasController.GetComponentPipelineRun(componentAName, applicationName, testNamespace, "") + if err != nil { + GinkgoWriter.Printf("Build PipelineRun has not been created yet for the componentA %s/%s\n", testNamespace, componentAName) + return err + } + if !pipelineRun.HasStarted() { + return fmt.Errorf("build pipelinerun %s/%s hasn't started yet", pipelineRun.GetNamespace(), pipelineRun.GetName()) + } + return nil + }, timeout, constants.PipelineRunPollingInterval).Should(Succeed(), fmt.Sprintf("timed out when waiting for the build PipelineRun to start for the componentA %s/%s", testNamespace, componentAName)) + }) + + It("does not contain an annotation with a Snapshot Name", func() { + Expect(pipelineRun.Annotations[snapshotAnnotation]).To(Equal("")) + }) + + It("should lead to build PipelineRun finishing successfully", func() { + Expect(f.AsKubeAdmin.HasController.WaitForComponentPipelineToBeFinished(componentA, + "", f.AsKubeAdmin.TektonController, &has.RetryOptions{Retries: 2, Always: true}, pipelineRun)).To(Succeed()) + }) + + It("should have a related PaC init PR created", func() { + timeout = time.Second * 300 + interval = time.Second * 1 + + Eventually(func() bool { + prs, err := f.AsKubeAdmin.CommonController.Github.ListPullRequests(multiComponentRepoNameForGroupSnapshotA) + Expect(err).ShouldNot(HaveOccurred()) + + for _, pr := range prs { + if pr.Head.GetRef() == fromBranchNameA { + prNumber = pr.GetNumber() + prHeadSha = pr.Head.GetSHA() + return true + } + } + return false + }, timeout, interval).Should(BeTrue(), fmt.Sprintf("timed out when waiting for init PaC PR (branch name '%s') to be created in %s repository", fromBranchNameA, multiComponentRepoNameForGroupSnapshotA)) + + // in case the first pipelineRun attempt has failed and was retried, we need to update the value of pipelineRun variable + pipelineRun, err = f.AsKubeAdmin.HasController.GetComponentPipelineRun(componentAName, applicationName, testNamespace, prHeadSha) + Expect(err).ShouldNot(HaveOccurred()) + }) + + It("eventually leads to the build PipelineRun's status reported at Checks tab", func() { + expectedCheckRunName := fmt.Sprintf("%s-%s", componentAName, "on-pull-request") + Expect(f.AsKubeAdmin.CommonController.Github.GetCheckRunConclusion(expectedCheckRunName, multiComponentRepoNameForGroupSnapshotA, prHeadSha, prNumber)).To(Equal(constants.CheckrunConclusionSuccess)) + }) + }) + + When("the Snapshot was created", func() { + It("should find both the related Integration PipelineRuns", func() { + testPipelinerun, err = f.AsKubeDeveloper.IntegrationController.WaitForIntegrationPipelineToGetStarted(integrationTestScenarioPass.Name, snapshot.Name, testNamespace) + Expect(err).ToNot(HaveOccurred()) + Expect(testPipelinerun.Labels[snapshotAnnotation]).To(ContainSubstring(snapshot.Name)) + Expect(testPipelinerun.Labels[scenarioAnnotation]).To(ContainSubstring(integrationTestScenarioPass.Name)) + }) + }) + + /*____ + | _ \ + | |_) | + | _ < + | |_) | + |____/ */ + When("a new Component B with specified custom branch(same as component A) is created", Label("custom-branch"), func() { + It("triggers a Build PipelineRun", func() { + timeout = time.Second * 600 + interval = time.Second * 1 + Eventually(func() error { + pipelineRun, err = f.AsKubeAdmin.HasController.GetComponentPipelineRun(componentBName, applicationName, testNamespace, "") + if err != nil { + GinkgoWriter.Printf("Build PipelineRun has not been created yet for the componentB %s/%s\n", testNamespace, componentBName) + return err + } + if !pipelineRun.HasStarted() { + return fmt.Errorf("build pipelinerun %s/%s hasn't started yet", pipelineRun.GetNamespace(), pipelineRun.GetName()) + } + return nil + }, timeout, constants.PipelineRunPollingInterval).Should(Succeed(), fmt.Sprintf("timed out when waiting for the build PipelineRun to start for the componentB %s/%s", testNamespace, componentBName)) + }) + + It("does not contain an annotation with a Snapshot Name", func() { + Expect(pipelineRun.Annotations[snapshotAnnotation]).To(Equal("")) + }) + + It("should lead to build PipelineRun finishing successfully", func() { + Expect(f.AsKubeAdmin.HasController.WaitForComponentPipelineToBeFinished(componentB, + "", f.AsKubeAdmin.TektonController, &has.RetryOptions{Retries: 2, Always: true}, pipelineRun)).To(Succeed()) + }) + + // It("should have a related PaC init PR created", func() { + // timeout = time.Second * 300 + // interval = time.Second * 1 + + // Eventually(func() bool { + // prs, err := f.AsKubeAdmin.CommonController.Github.ListPullRequests(multiComponentRepoNameForGroupSnapshotB) + // Expect(err).ShouldNot(HaveOccurred()) + + // for _, pr := range prs { + // if pr.Head.GetRef() == pacBranchNameB { + // prNumber = pr.GetNumber() + // prHeadSha = pr.Head.GetSHA() + // return true + // } + // } + // return false + // }, timeout, interval).Should(BeTrue(), fmt.Sprintf("timed out when waiting for init PaC PR (branch name '%s') to be created in %s repository", pacBranchNameB, multiComponentRepoNameForGroupSnapshotB)) + + // // in case the first pipelineRun attempt has failed and was retried, we need to update the value of pipelineRun variable + // pipelineRun, err = f.AsKubeAdmin.HasController.GetComponentPipelineRun(componentAName, applicationName, testNamespace, prHeadSha) + // Expect(err).ShouldNot(HaveOccurred()) + // }) + + // It("eventually leads to the build PipelineRun's status reported at Checks tab", func() { + // expectedCheckRunName := fmt.Sprintf("%s-%s", componentAName, "on-pull-request") + // Expect(f.AsKubeAdmin.CommonController.Github.GetCheckRunConclusion(expectedCheckRunName, multiComponentRepoNameForGroupSnapshotB, prHeadSha, prNumber)).To(Equal(constants.CheckrunConclusionSuccess)) + // }) + }) + + When("the PaC build pipelineRun run succeeded", func() { + It("checks if the BuildPipelineRun have the annotation of chains signed", func() { + Expect(f.AsKubeDeveloper.IntegrationController.WaitForBuildPipelineRunToGetAnnotated(testNamespace, applicationName, componentAName, chainsSignedAnnotation)).To(Succeed()) + }) + + It("checks if the Snapshot is created", func() { + snapshot, err = f.AsKubeDeveloper.IntegrationController.WaitForSnapshotToGetCreated("", "", componentAName, testNamespace) + Expect(err).ToNot(HaveOccurred()) + }) + + It("checks if the Build PipelineRun got annotated with Snapshot name", func() { + Expect(f.AsKubeDeveloper.IntegrationController.WaitForBuildPipelineRunToGetAnnotated(testNamespace, applicationName, componentAName, snapshotAnnotation)).To(Succeed()) + }) + }) + + When("the Snapshot was created", func() { + It("should find both the related Integration PipelineRuns", func() { + testPipelinerun, err = f.AsKubeDeveloper.IntegrationController.WaitForIntegrationPipelineToGetStarted(integrationTestScenarioPass.Name, snapshot.Name, testNamespace) + Expect(err).ToNot(HaveOccurred()) + Expect(testPipelinerun.Labels[snapshotAnnotation]).To(ContainSubstring(snapshot.Name)) + Expect(testPipelinerun.Labels[scenarioAnnotation]).To(ContainSubstring(integrationTestScenarioPass.Name)) + + }) + }) + + When("Integration PipelineRuns are created", func() { + It("should eventually complete successfully", func() { + Expect(f.AsKubeAdmin.IntegrationController.WaitForIntegrationPipelineToBeFinished(integrationTestScenarioPass, snapshot, testNamespace)).To(Succeed(), fmt.Sprintf("Error when waiting for an integration pipelinerun for snapshot %s/%s to finish", testNamespace, snapshot.GetName())) + }) + }) + + When("Integration PipelineRuns completes successfully", func() { + It("should lead to Snapshot CR being marked as success", FlakeAttempts(3), func() { + // Snapshot marked as Success, this could be the place where we check for group snapshot + Eventually(func() bool { + snapshot, err = f.AsKubeAdmin.IntegrationController.GetSnapshot("", pipelineRun.Name, "", testNamespace) + return err == nil && !f.AsKubeAdmin.CommonController.HaveTestsSucceeded(snapshot) + }, time.Minute*3, time.Second*5).Should(BeTrue(), fmt.Sprintf("Timed out waiting for Snapshot to be marked as success %s/%s", snapshot.GetNamespace(), snapshot.GetName())) + }) + It("snapshot should contain group-pr annnotation", func() { + Expect(snapshot.Annotations[groupSnapshotAnnotation]).To(ContainSubstring(toBranch)) + }) + + It("eventually leads to the status reported at Checks tab for the successful Integration PipelineRun", func() { + Expect(f.AsKubeAdmin.CommonController.Github.GetCheckRunConclusion(integrationTestScenarioPass.Name, componentRepoNameForStatusReporting, prHeadSha, prNumber)).To(Equal(constants.CheckrunConclusionSuccess)) + }) + }) + }) +}) diff --git a/tests/integration-service/integration.go b/tests/integration-service/integration.go index 635eebf56b..191baece7b 100644 --- a/tests/integration-service/integration.go +++ b/tests/integration-service/integration.go @@ -465,6 +465,38 @@ func createComponent(f framework.Framework, testNamespace, applicationName, comp return originalComponent, componentName, pacBranchName, componentBaseBranchName } +func createComponentWithCustomBranch(f framework.Framework, testNamespace, applicationName, componentRepoName, componentRepoURL string, componentBaseBranchName string, contextDir string) (*appstudioApi.Component, string, string, string) { + componentName := fmt.Sprintf("%s-%s", "test-component-pac", util.GenerateRandomString(6)) + fromBranchName := "love-triangle" + + if contextDir == "go-component/docker" { + err := f.AsKubeAdmin.CommonController.Github.CreateRef(componentRepoName, componentDefaultBranch, componentGroupRevision, componentBaseBranchName) + Expect(err).ShouldNot(HaveOccurred()) + } + + // get the build pipeline bundle annotation + buildPipelineAnnotation := build.GetDockerBuildPipelineBundle() + + componentObj := appstudioApi.ComponentSpec{ + ComponentName: componentName, + Application: applicationName, + Source: appstudioApi.ComponentSource{ + ComponentSourceUnion: appstudioApi.ComponentSourceUnion{ + GitSource: &appstudioApi.GitSource{ + URL: componentRepoURL, + Revision: componentBaseBranchName, + Context: contextDir, + }, + }, + }, + } + + originalComponent, err := f.AsKubeAdmin.HasController.CreateComponent(componentObj, testNamespace, "", "", applicationName, false, utils.MergeMaps(utils.MergeMaps(constants.ComponentPaCRequestAnnotation, constants.ImageControllerAnnotationRequestPublicRepo), buildPipelineAnnotation)) + Expect(err).NotTo(HaveOccurred()) + + return originalComponent, componentName, fromBranchName, componentBaseBranchName +} + func cleanup(f framework.Framework, testNamespace, applicationName, componentName string) { if !CurrentSpecReport().Failed() { Expect(f.AsKubeAdmin.HasController.DeleteApplication(applicationName, testNamespace, false)).To(Succeed())