From f3bc4664fdd52a2cae0ae323daf13af9b7d12093 Mon Sep 17 00:00:00 2001 From: Nandhukumar Date: Tue, 17 Sep 2024 15:33:21 +0530 Subject: [PATCH] MOSIP-35770 Signed-off-by: Nandhukumar --- .../apirig/report/EmailableReport.java | 142 ++++++++++++++---- .../apirig/testrunner/ExtractResource.java | 2 + .../testrig/apirig/utils/ConfigManager.java | 11 ++ .../testrig/apirig/utils/GlobalConstants.java | 3 + .../apirig/utils/SkipTestCaseHandler.java | 62 ++++++++ 5 files changed, 191 insertions(+), 29 deletions(-) create mode 100644 apitest-commons/src/main/java/io/mosip/testrig/apirig/utils/SkipTestCaseHandler.java diff --git a/apitest-commons/src/main/java/io/mosip/testrig/apirig/report/EmailableReport.java b/apitest-commons/src/main/java/io/mosip/testrig/apirig/report/EmailableReport.java index c000ea1ee5..4ae90c3a07 100644 --- a/apitest-commons/src/main/java/io/mosip/testrig/apirig/report/EmailableReport.java +++ b/apitest-commons/src/main/java/io/mosip/testrig/apirig/report/EmailableReport.java @@ -47,6 +47,9 @@ public class EmailableReport implements IReporter { protected PrintWriter writer; protected final List suiteResults = Lists.newArrayList(); + + protected final boolean reportIgnoredTestCases = ConfigManager.reportIgnoredTestCases(); + protected final boolean reportKnownIssueTestCases = ConfigManager.reportKnownIssueTestCases(); // Reusable buffer private final StringBuilder buffer = new StringBuilder(); @@ -58,6 +61,7 @@ public class EmailableReport implements IReporter { private int totalPassedTests = 0; private int totalSkippedTests = 0; private int totalIgnoredTests = 0; + private int totalKnownIssueTests = 0; private int totalFailedTests = 0; private long totalDuration = 0; @@ -89,20 +93,31 @@ public void generateReport(List xmlSuites, List suites, String writer.close(); int totalTestCases = 0; - if (ConfigManager.reportIgnoredTestCases()) { - totalTestCases = totalPassedTests + totalSkippedTests + totalFailedTests + totalIgnoredTests; - }else { - totalTestCases = totalPassedTests + totalSkippedTests + totalFailedTests; + + totalTestCases = totalPassedTests + totalSkippedTests + totalFailedTests; + + if (reportIgnoredTestCases) { + totalTestCases = totalTestCases + totalIgnoredTests; + } + if (reportKnownIssueTestCases) { + totalTestCases = totalTestCases + totalKnownIssueTests; } String oldString = System.getProperty(GlobalConstants.EMAILABLEREPORT2NAME); String temp = ""; - if (ConfigManager.reportIgnoredTestCases()) { + + temp = "-report_T-" + totalTestCases + "_P-" + totalPassedTests + "_S-" + totalSkippedTests + "_F-" + + totalFailedTests; + + if (reportIgnoredTestCases && reportKnownIssueTestCases) { + temp = "-report_T-" + totalTestCases + "_P-" + totalPassedTests + "_S-" + totalSkippedTests + "_F-" + + totalFailedTests + "_I-" + totalIgnoredTests + "_KI-" + totalKnownIssueTests; + } else if (reportIgnoredTestCases && !(reportKnownIssueTestCases)) { temp = "-report_T-" + totalTestCases + "_P-" + totalPassedTests + "_S-" + totalSkippedTests + "_F-" + totalFailedTests + "_I-" + totalIgnoredTests; - }else { + } else if (reportKnownIssueTestCases && !(reportIgnoredTestCases)) { temp = "-report_T-" + totalTestCases + "_P-" + totalPassedTests + "_S-" + totalSkippedTests + "_F-" - + totalFailedTests; + + totalFailedTests + "_KI-" + totalKnownIssueTests; } String newString = oldString.replace("-report", temp); @@ -194,6 +209,7 @@ protected void writeStylesheet() { writer.print(".num {text-align:center}"); writer.print(".orange-bg {background-color: #FFA500}"); writer.print(".grey-bg {background-color: #808080}"); + writer.print(".thich-orange-bg {background-color: #CC5500}"); writer.print(".green-bg {background-color: #0A0}"); writer.print(".attn {background-color: #D00}"); writer.print(".passedodd td {background-color: #3F3}"); @@ -204,6 +220,8 @@ protected void writeStylesheet() { writer.print(".failedeven td,.stripe {background-color: #D00}"); writer.print(".ignoredodd td {background-color: #808080}"); writer.print(".ignoredeven td {background-color: #808080}"); + writer.print(".known_issuesodd td {background-color: #CC5500}"); + writer.print(".known_issueseven td {background-color: #CC5500}"); writer.print(".stacktrace {white-space:pre;font-family:monospace}"); writer.print(".totop {font-size:85%;text-align:center;border-bottom:2px solid #000}"); writer.print(""); @@ -244,6 +262,7 @@ protected void writeSuiteSummary() { totalPassedTests = 0; totalSkippedTests = 0; totalIgnoredTests = 0; + totalKnownIssueTests = 0; totalFailedTests = 0; totalDuration = 0; @@ -251,7 +270,7 @@ protected void writeSuiteSummary() { int testIndex = 0; for (SuiteResult suiteResult : suiteResults) { - writer.print(""); + writer.print(""); writer.print(Utils.escapeHtml(suiteResult.getSuiteName() + " ---- " + "Report Date: " + formattedDate + " ---- " + "Tested Environment: " + System.getProperty("env.endpoint").replaceAll(".*?\\.([^\\.]+)\\..*", "$1") + " ---- " @@ -259,32 +278,32 @@ protected void writeSuiteSummary() { writer.print(GlobalConstants.TRTR); // Left column: "Tested Component Details" with central alignment - writer.print("
");
+			writer.print("
");
 			writer.print(Utils.escapeHtml("Server Component Details"));
 			writer.print("");
 
 			// Right column: Details from AdminTestUtil.getServerComponentsDetails() without bold formatting
-			writer.print("
");
+			writer.print("
");
 			writer.print(Utils.escapeHtml(AdminTestUtil.getServerComponentsDetails()));
 			writer.print("
"); writer.print(GlobalConstants.TRTR); // Left column: "Tested Component Details" with central alignment - writer.print("
");
+			writer.print("
");
 			writer.print(Utils.escapeHtml("End Points used"));
 			writer.print("");
 
 			// Right column: Details from AdminTestUtil.getServerComponentsDetails() without bold formatting
-			writer.print("
");
+			writer.print("
");
 			writer.print(Utils.escapeHtml(GlobalMethods.getComponentDetails()));
 			writer.print("
"); writer.print(GlobalConstants.TRTR); if (GlobalMethods.getServerErrors().equals("No server errors")) { - writer.print("
");
+				writer.print("
");
 			} else {
 				writer.print(
-						"
");
+						"
");
 			}
 			writer.print(Utils.escapeHtml("Server Errors " + "\n" + GlobalMethods.getServerErrors()));
 			writer.print("
"); @@ -297,9 +316,13 @@ protected void writeSuiteSummary() { writer.print("# Passed"); writer.print("# Skipped"); writer.print("# Failed"); - if (ConfigManager.reportIgnoredTestCases()) { + if (reportIgnoredTestCases) { writer.print("# Ignored"); } + + if (reportKnownIssueTestCases) { + writer.print("# Known_Issues"); + } writer.print("Execution Time (ms)"); // writer.print("Included Groups"); // writer.print("Excluded Groups"); @@ -308,13 +331,20 @@ protected void writeSuiteSummary() { for (TestResult testResult : suiteResult.getTestResults()) { int passedTests = testResult.getPassedTestCount(); int ignoredTests = testResult.getIgnoredTestCount(); + int knownIssueTests = testResult.getKnownIssueTestCount(); int skippedTests = testResult.getSkippedTestCount(); int failedTests = testResult.getFailedTestCount(); long duration = testResult.getDuration(); int totalTests = 0; - totalTests = ConfigManager.reportIgnoredTestCases() - ? (passedTests + skippedTests + failedTests + ignoredTests) - : (passedTests + skippedTests + failedTests); + + totalTests = passedTests + skippedTests + failedTests; + + if (reportIgnoredTestCases) { + totalTests = totalTests + ignoredTests; + } + if (reportKnownIssueTestCases) { + totalTests = totalTests + knownIssueTests; + } // All test cases are ignored. Hence don't print anything in the report. if (totalTests < 1) continue; @@ -339,9 +369,13 @@ protected void writeSuiteSummary() { writeTableData(integerFormat.format(skippedTests), (skippedTests > 0 ? "num orange-bg" : "num")); writeTableData(integerFormat.format(failedTests), (failedTests > 0 ? GlobalConstants.NUMATTN : "num")); // print the ignored column based on the flag - if (ConfigManager.reportIgnoredTestCases()) { + if (reportIgnoredTestCases) { writeTableData(integerFormat.format(ignoredTests), (ignoredTests > 0 ? "num grey-bg" : "num")); } + + if (reportKnownIssueTestCases) { + writeTableData(integerFormat.format(knownIssueTests), (knownIssueTests > 0 ? "num thich-orange-bg" : "num")); + } writeTableData(decimalFormat.format(duration), "num"); // writeTableData(testResult.getIncludedGroups()); // writeTableData(testResult.getExcludedGroups()); @@ -352,6 +386,7 @@ protected void writeSuiteSummary() { totalSkippedTests += skippedTests; totalFailedTests += failedTests; totalIgnoredTests += ignoredTests; + totalKnownIssueTests += knownIssueTests; totalDuration += duration; testIndex++; @@ -362,9 +397,16 @@ protected void writeSuiteSummary() { writer.print(""); writer.print("Total"); writeTableHeader("", "num"); - if (ConfigManager.reportIgnoredTestCases()) { - writeTableHeader(integerFormat - .format(totalPassedTests + totalSkippedTests + totalFailedTests + totalIgnoredTests), "num"); + + if (reportIgnoredTestCases && reportKnownIssueTestCases) { + writeTableHeader(integerFormat.format(totalPassedTests + totalSkippedTests + totalFailedTests + + totalIgnoredTests + totalKnownIssueTests), "num"); + } else if (reportIgnoredTestCases && !(reportKnownIssueTestCases)) { + writeTableHeader(integerFormat.format(totalPassedTests + totalSkippedTests + totalFailedTests + + totalIgnoredTests), "num"); + } else if (reportKnownIssueTestCases && !(reportIgnoredTestCases)) { + writeTableHeader(integerFormat.format(totalPassedTests + totalSkippedTests + totalFailedTests + +totalKnownIssueTests), "num"); } else { writeTableHeader(integerFormat.format(totalPassedTests + totalSkippedTests + totalFailedTests), "num"); } @@ -373,10 +415,14 @@ protected void writeSuiteSummary() { (totalSkippedTests > 0 ? "num orange-bg" : "num")); writeTableHeader(integerFormat.format(totalFailedTests), (totalFailedTests > 0 ? GlobalConstants.NUMATTN : "num")); - if (ConfigManager.reportIgnoredTestCases()) { + if (reportIgnoredTestCases) { writeTableHeader(integerFormat.format(totalIgnoredTests), (totalIgnoredTests > 0 ? "num grey-bg" : "num")); } + if (reportKnownIssueTestCases) { + writeTableHeader(integerFormat.format(totalKnownIssueTests), + (totalKnownIssueTests > 0 ? "num thich-orange-bg" : "num")); + } writeTableHeader(decimalFormat.format(totalDuration), "num"); writer.print(GlobalConstants.TR); } @@ -401,9 +447,17 @@ protected static Set getResultsSubSet(Set resultsSet, } else { // Skip the test result } + } else if (subSetString.contains(GlobalConstants.KNOWN_ISSUES_STRING)) { + if (containsAny(throwable.getMessage(), subSetString)) { + // Add only results which are skipped due to feature not supported + testResultsSubList.add(result); + } else { + // Skip the test result + } } else { // Service not deployed. Hence skipping the testcase // skipped if (!throwable.getMessage().contains(GlobalConstants.FEATURE_NOT_SUPPORTED) - && !throwable.getMessage().contains(GlobalConstants.SERVICE_NOT_DEPLOYED)) { + && !throwable.getMessage().contains(GlobalConstants.SERVICE_NOT_DEPLOYED) + && !throwable.getMessage().contains(GlobalConstants.KNOWN_ISSUES)) { // Add only results which are not skipped due to feature not supported testResultsSubList.add(result); } else { @@ -451,12 +505,19 @@ protected void writeScenarioSummary() { for (TestResult testResult : suiteResult.getTestResults()) { int passedTests = testResult.getPassedTestCount(); int ignoredTests = testResult.getIgnoredTestCount(); + int knownIssueTests = testResult.getKnownIssueTestCount(); int skippedTests = testResult.getSkippedTestCount(); int failedTests = testResult.getFailedTestCount(); int totalTests = 0; - totalTests = ConfigManager.reportIgnoredTestCases() - ? (passedTests + skippedTests + failedTests + ignoredTests) - : (passedTests + skippedTests + failedTests); + + totalTests = passedTests + skippedTests + failedTests; + + if (reportIgnoredTestCases) { + totalTests = totalTests + ignoredTests; + } + if (reportKnownIssueTestCases) { + totalTests = totalTests + knownIssueTests; + } // All test cases are ignored. Hence don't print anything in the report. if (totalTests < 1) continue; @@ -466,10 +527,16 @@ protected void writeScenarioSummary() { String testName = Utils.escapeHtml(testResult.getTestName()); - if (ConfigManager.reportIgnoredTestCases()) { + if (reportIgnoredTestCases) { scenarioIndex += writeScenarioSummary(testName + " — Ignored", testResult.getIgnoredTestResults(), "ignored", scenarioIndex); } + + if (reportKnownIssueTestCases) { + scenarioIndex += writeScenarioSummary(testName + " — known_issues", + testResult.getKnownIssueTestResults(), "known_issues", scenarioIndex); + } + scenarioIndex += writeScenarioSummary(testName + " — Failed", testResult.getFailedTestResults(), "failed", scenarioIndex); scenarioIndex += writeScenarioSummary(testName + " — Skipped", testResult.getSkippedTestResults(), @@ -582,9 +649,13 @@ protected void writeScenarioDetails() { writer.print(Utils.escapeHtml(testResult.getTestName())); writer.print(""); - if (ConfigManager.reportIgnoredTestCases()) { + if (reportIgnoredTestCases) { scenarioIndex += writeScenarioDetails(testResult.getIgnoredTestResults(), scenarioIndex); } + + if (reportKnownIssueTestCases) { + scenarioIndex += writeScenarioDetails(testResult.getKnownIssueTestResults(), scenarioIndex); + } scenarioIndex += writeScenarioDetails(testResult.getFailedConfigurationResults(), scenarioIndex); scenarioIndex += writeScenarioDetails(testResult.getFailedTestResults(), scenarioIndex); scenarioIndex += writeScenarioDetails(testResult.getSkippedConfigurationResults(), scenarioIndex); @@ -836,7 +907,9 @@ public int compare(ITestResult o1, ITestResult o2) { private final List skippedTestResults; private final List passedTestResults; private final List ignoredTestResults; + private final List knownIssueTestResults; private final int ignoredTestCount; + private final int knownIssueTestCount; private final int failedTestCount; private final int skippedTestCount; private final int passedTestCount; @@ -855,6 +928,7 @@ public TestResult(ITestContext context) { // Set skippedTests = context.getSkippedTests().getAllResults(); Set skippedTests = getResultsSubSet(context.getSkippedTests().getAllResults(), GlobalConstants.SKIPPED); Set ignoredTests = getResultsSubSet(context.getSkippedTests().getAllResults(), GlobalConstants.IGNORED_SUBSET_STRING); + Set knownIssueTests = getResultsSubSet(context.getSkippedTests().getAllResults(), GlobalConstants.KNOWN_ISSUE_SUBSET_STRING); Set passedTests = context.getPassedTests().getAllResults(); failedConfigurationResults = groupResults(failedConfigurations); @@ -862,12 +936,14 @@ public TestResult(ITestContext context) { skippedConfigurationResults = groupResults(skippedConfigurations); skippedTestResults = groupResults(skippedTests); ignoredTestResults = groupResults(ignoredTests); + knownIssueTestResults = groupResults(knownIssueTests); passedTestResults = groupResults(passedTests); failedTestCount = failedTests.size(); skippedTestCount = skippedTests.size(); passedTestCount = passedTests.size(); ignoredTestCount = ignoredTests.size(); + knownIssueTestCount = knownIssueTests.size(); duration = context.getEndDate().getTime() - context.getStartDate().getTime(); @@ -954,6 +1030,10 @@ public List getFailedTestResults() { public List getIgnoredTestResults() { return ignoredTestResults; } + + public List getKnownIssueTestResults() { + return knownIssueTestResults; + } /** * @return the results for skipped configurations (possibly empty) @@ -991,6 +1071,10 @@ public int getPassedTestCount() { public int getIgnoredTestCount() { return ignoredTestCount; } + + public int getKnownIssueTestCount() { + return knownIssueTestCount; + } public long getDuration() { return duration; diff --git a/apitest-commons/src/main/java/io/mosip/testrig/apirig/testrunner/ExtractResource.java b/apitest-commons/src/main/java/io/mosip/testrig/apirig/testrunner/ExtractResource.java index cbdee0a36e..92379bfe50 100644 --- a/apitest-commons/src/main/java/io/mosip/testrig/apirig/testrunner/ExtractResource.java +++ b/apitest-commons/src/main/java/io/mosip/testrig/apirig/testrunner/ExtractResource.java @@ -21,6 +21,7 @@ public static void extractCommonResourceFromJar() { getListOfFilesFromJarAndCopyToExternalResource("spring.properties"); getListOfFilesFromJarAndCopyToExternalResource("validations.properties"); getListOfFilesFromJarAndCopyToExternalResource("dbFiles/"); + getListOfFilesFromJarAndCopyToExternalResource("testCaseSkippedList.txt"); } public static void copyCommonResources(){ @@ -31,6 +32,7 @@ public static void copyCommonResources(){ copyCommonResources("spring.properties"); copyCommonResources("validations.properties"); copyCommonResources("dbFiles/"); + copyCommonResources("testCaseSkippedList.txt"); } public static void copyCommonResources(String moduleName){ diff --git a/apitest-commons/src/main/java/io/mosip/testrig/apirig/utils/ConfigManager.java b/apitest-commons/src/main/java/io/mosip/testrig/apirig/utils/ConfigManager.java index 964ec364cf..b12b363c4a 100644 --- a/apitest-commons/src/main/java/io/mosip/testrig/apirig/utils/ConfigManager.java +++ b/apitest-commons/src/main/java/io/mosip/testrig/apirig/utils/ConfigManager.java @@ -70,6 +70,7 @@ public class ConfigManager { private static String PUSH_TO_S3 = "push-reports-to-s3"; private static String ENABLE_DEBUG = "enableDebug"; private static String REPORT_IGNORED_TEST_CASES = "reportIgnoredTestCases"; + private static String REPORT_KNOWN_ISSUE_TEST_CASES = "reportKnownIssueTestCases"; private static String THREAD_COUNT = "threadCount"; private static String LANG_SELECT = "langselect"; @@ -201,6 +202,7 @@ public class ConfigManager { private static String push_reports_to_s3; private static String enableDebug; private static String reportIgnoredTestCases; + private static String reportKnownIssueTestCases; private static String threadCount; private static String langselect; private static String usePreConfiguredOtp; @@ -396,6 +398,11 @@ public static void init() { ? propsKernel.getProperty(REPORT_IGNORED_TEST_CASES) : System.getenv(REPORT_IGNORED_TEST_CASES); propsKernel.setProperty(REPORT_IGNORED_TEST_CASES, reportIgnoredTestCases); + + reportKnownIssueTestCases = System.getenv(REPORT_KNOWN_ISSUE_TEST_CASES) == null + ? propsKernel.getProperty(REPORT_KNOWN_ISSUE_TEST_CASES) + : System.getenv(REPORT_KNOWN_ISSUE_TEST_CASES); + propsKernel.setProperty(REPORT_KNOWN_ISSUE_TEST_CASES, reportKnownIssueTestCases); threadCount = System.getenv(THREAD_COUNT) == null ? propsKernel.getProperty(THREAD_COUNT) : System.getenv(THREAD_COUNT); @@ -575,6 +582,10 @@ public static Boolean IsDebugEnabled() { public static Boolean reportIgnoredTestCases() { return reportIgnoredTestCases.equalsIgnoreCase("yes"); } + + public static Boolean reportKnownIssueTestCases() { + return reportKnownIssueTestCases.equalsIgnoreCase("yes"); + } public static String getReportExpirationInDays() { return reportExpirationInDays; diff --git a/apitest-commons/src/main/java/io/mosip/testrig/apirig/utils/GlobalConstants.java b/apitest-commons/src/main/java/io/mosip/testrig/apirig/utils/GlobalConstants.java index 57b3bf2e48..ea7f726ca6 100644 --- a/apitest-commons/src/main/java/io/mosip/testrig/apirig/utils/GlobalConstants.java +++ b/apitest-commons/src/main/java/io/mosip/testrig/apirig/utils/GlobalConstants.java @@ -217,7 +217,10 @@ public class GlobalConstants { public static final String SERVICE_NOT_DEPLOYED_MESSAGE = "Service not deployed. Hence skipping the testcase"; public static final String FEATURE_NOT_SUPPORTED = "feature not supported"; public static final String SERVICE_NOT_DEPLOYED = "Service not deployed"; + public static final String KNOWN_ISSUES_STRING = "known issue"; + public static final String KNOWN_ISSUES = "known issue. Hence skipping the testcase"; public static final String IGNORED_SUBSET_STRING = "feature not supported;Service not deployed"; + public static final String KNOWN_ISSUE_SUBSET_STRING = "known issue;"; public static final String SKIPPED = "Skipped"; public static final String TARGET_ENV_HEALTH_CHECK_FAILED = "Target env health check failed "; public static final String HOTLIST = "hotlist"; diff --git a/apitest-commons/src/main/java/io/mosip/testrig/apirig/utils/SkipTestCaseHandler.java b/apitest-commons/src/main/java/io/mosip/testrig/apirig/utils/SkipTestCaseHandler.java new file mode 100644 index 0000000000..7d48904e26 --- /dev/null +++ b/apitest-commons/src/main/java/io/mosip/testrig/apirig/utils/SkipTestCaseHandler.java @@ -0,0 +1,62 @@ +package io.mosip.testrig.apirig.utils; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.log4j.Logger; + +import io.mosip.testrig.apirig.testrunner.MosipTestRunner; + +public class SkipTestCaseHandler { + private static final Logger logger = Logger.getLogger(SkipTestCaseHandler.class); + private static List testcaseToBeSkippedList = new ArrayList<>(); + + public static void main(String[] args) { + String filePath = "C:\\Users\\sivan\\Downloads\\SynData\\testCaseSkippedList.txt"; // Replace with your file + // path + + loadTestcaseToBeSkippedList(filePath); + + // Example usage of the checkStringInList method + String searchString = "IdRepository_UpdateIdentity_handle_value_value_withupdatevalues"; + boolean exists = isTestCaseInSkippedList(searchString); + logger.info("Does the string exist? " + exists); + + searchString = "IdRepository_UpdateIdentity_handle_value_value_withupdatevalues"; + exists = isTestCaseInSkippedList(searchString); + logger.info("Does the string exist? " + exists); + + } + + // load test cases to be skipped in the execution in the list + public static void loadTestcaseToBeSkippedList(String fileName) { + try (BufferedReader br = new BufferedReader( + new FileReader(MosipTestRunner.getGlobalResourcePath() + "/" + fileName))) { + String line; + while ((line = br.readLine()) != null) { + // Ignore lines starting with # as it is commented line + if (line.startsWith("#")) { + continue; + } + + // Split the line by "------" and store the second part + if (line.contains("------")) { + String[] parts = line.split("------"); + if (parts.length > 1) { + testcaseToBeSkippedList.add(parts[1].trim()); + } + } + } + } catch (IOException e) { + e.printStackTrace(); + } + } + + // Method to check if a given test case exists in the list + public static boolean isTestCaseInSkippedList(String strTestCase) { + return testcaseToBeSkippedList.contains(strTestCase); + } +} \ No newline at end of file