Add smoke tests to the test manifest schema (#5127)

Signed-off-by: Zelin Hao <[email protected]>

src/manifests/test_manifest.py

@@ -36,6 +36,8 @@ class TestManifest(ComponentManifest['TestManifest', 'TestComponents']):
               test-configs:
                 - with-security
                 - without-security
+            smoke-test:
+              test-spec: spec.yml
     """
 
     SCHEMA = {
@@ -88,6 +90,12 @@ class TestManifest(ComponentManifest['TestManifest', 'TestComponents']):
                             "test-configs": {"type": "list", "allowed": ["with-security", "without-security", "with-less-security"]},
                         },
                     },
+                    "smoke-test": {
+                        "type": "dict",
+                        "schema": {
+                            "test-spec": {"type": "string"},
+                        },
+                    },
                 },
             },
         },
@@ -160,6 +168,7 @@ def __init__(self, data: dict) -> None:
         self.working_directory = data.get("working-directory", None)
         self.integ_test = data.get("integ-test", None)
         self.bwc_test = data.get("bwc-test", None)
+        self.smoke_test = data.get("smoke-test", None)
         self.topology = TestComponentTopology(self.integ_test.get("topology", None)) if self.integ_test is not None else TestComponentTopology(None)
         self.components = TestComponents(data.get("components", []))  # type: ignore[assignment]
 
@@ -168,7 +177,8 @@ def __to_dict__(self) -> dict:
             "name": self.name,
             "working-directory": self.working_directory,
             "integ-test": self.integ_test,
-            "bwc-test": self.bwc_test
+            "bwc-test": self.bwc_test,
+            "smoke-test": self.smoke_test
         }
 
 

tests/tests_manifests/data/opensearch-2.18.0-test.yml

@@ -0,0 +1,151 @@
+---
+schema-version: '1.0'
+name: OpenSearch
+ci:
+  image:
+    name: opensearchstaging/ci-runner:ci-runner-al2-opensearch-build-v1
+    args: -e JAVA_HOME=/opt/java/openjdk-21
+components:
+  - name: opensearch
+    smoke-test:
+      test-spec: opensearch.yml
+  - name: alerting
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+      additional-cluster-configs:
+        plugins.destination.host.deny_list:
+          - 10.0.0.0/8
+          - 127.0.0.1
+    bwc-test:
+      test-configs:
+        - with-security
+  - name: anomaly-detection
+    integ-test:
+      build-dependencies:
+        - job-scheduler
+      test-configs:
+        - with-security
+        - without-security
+    bwc-test:
+      test-configs:
+        - with-security
+  - name: flow-framework
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+  - name: asynchronous-search
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+    bwc-test:
+      test-configs:
+        - with-security
+  - name: cross-cluster-replication
+    integ-test:
+      topology:
+        - cluster_name: leader
+          data_nodes: 2
+        - cluster_name: follower
+          data_nodes: 2
+      test-configs:
+        - with-security
+        - without-security
+  - name: geospatial
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+  - name: index-management
+    integ-test:
+      build-dependencies:
+        - job-scheduler
+      test-configs:
+        - with-security
+        - without-security
+      additional-cluster-configs:
+        path.repo:
+          - /tmp
+    bwc-test:
+      test-configs:
+        - with-security
+  - name: k-NN
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+  - name: ml-commons
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+  - name: neural-search
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+  - name: notifications
+    working-directory: notifications
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+    bwc-test:
+      test-configs:
+        - with-security
+  - name: opensearch-observability
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+    bwc-test:
+      test-configs:
+        - with-security
+  - name: opensearch-reports
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+  - name: security
+    integ-test:
+      test-configs:
+        - with-security
+  - name: security-analytics
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+  - name: sql
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+      additional-cluster-configs:
+        script.context.field.max_compilations_rate: 1000/1m
+        plugins.query.datasources.encryption.masterkey: 4fc8fee6a3fd7d6ca01772e5
+    bwc-test:
+      test-configs:
+        - with-security
+  - name: custom-codecs
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+  - name: skills
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+  - name: query-insights
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+  - name: opensearch-system-templates
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security

tests/tests_manifests/test_test_manifest.py

@@ -17,21 +17,24 @@ class TestTestManifest(unittest.TestCase):
     def setUp(self) -> None:
         self.maxDiff = None
         self.data_path = os.path.realpath(os.path.join(os.path.dirname(__file__), "data"))
-        self.manifest_filename = os.path.join(self.data_path, "opensearch-test-1.1.0.yml")
+        self.manifest_filename = os.path.join(self.data_path, "opensearch-2.18.0-test.yml")
         self.manifest = TestManifest.from_path(self.manifest_filename)
 
     def test_component(self) -> None:
-        component = self.manifest.components["index-management"]
-        self.assertEqual(component.name, "index-management")
-        self.assertEqual(component.integ_test, {"test-configs": ["with-security", "without-security"]})
-        self.assertEqual(component.bwc_test, {"test-configs": ["with-security", "without-security"]})
+        component_as = self.manifest.components["asynchronous-search"]
+        self.assertEqual(component_as.name, "asynchronous-search")
+        self.assertEqual(component_as.integ_test, {"test-configs": ["with-security", "without-security"]})
+        self.assertEqual(component_as.bwc_test, {"test-configs": ["with-security"]})
+        component_os = self.manifest.components["opensearch"]
+        self.assertEqual(component_os.name, "opensearch")
+        self.assertEqual(component_os.smoke_test, {"test-spec": "opensearch.yml"})
 
     def test_component_with_working_directory(self) -> None:
-        component = self.manifest.components["dashboards-reports"]
-        self.assertEqual(component.name, "dashboards-reports")
-        self.assertEqual(component.working_directory, "reports-scheduler")
-        self.assertEqual(component.integ_test, {"test-configs": ["without-security"]})
-        self.assertEqual(component.bwc_test, {"test-configs": ["without-security"]})
+        component = self.manifest.components["notifications"]
+        self.assertEqual(component.name, "notifications")
+        self.assertEqual(component.working_directory, "notifications")
+        self.assertEqual(component.integ_test, {"test-configs": ["with-security", "without-security"]})
+        self.assertEqual(component.bwc_test, {"test-configs": ["with-security"]})
 
     def test_to_dict(self) -> None:
         data = self.manifest.to_dict()