index MC metrics during creation phase

libs/platforms/platform.py

@@ -63,12 +63,13 @@ def __init__(self, arguments, logging, utils, es):
         self.environment['load'] = {}
         if arguments['enable_workload']:
             self.environment['load']['enabled'] = "true"
-            self.environment['load']["workload"] = arguments["workload"]
-            self.environment['load']["repo"] = arguments["workload_repo"]
-            self.environment['load']["script_path"] = arguments["workload_script_path"]
-            self.environment['load']["executor"] = arguments["workload_executor"]
-            self.environment['load']['duration'] = arguments['workload_duration']
-            self.environment['load']['jobs'] = arguments['workload_jobs']
+
+        self.environment['load']["workload"] = arguments["workload"]
+        self.environment['load']["repo"] = arguments["workload_repo"]
+        self.environment['load']["script_path"] = arguments["workload_script_path"]
+        self.environment['load']["executor"] = arguments["workload_executor"]
+        self.environment['load']['duration'] = arguments['workload_duration']
+        self.environment['load']['jobs'] = arguments['workload_jobs']
 
         if arguments["static_cluster_name"]:
             self.environment["cluster_name_seed"] = arguments["static_cluster_name"]

libs/platforms/rosa/hypershift/hypershift.py

@@ -498,6 +498,7 @@ def create_cluster(self, platform, cluster_name):
             self.environment["mc_kubeconfig"] = self.download_kubeconfig(mgmt_cluster_name, self.environment["path"])
             mc_namespace = executor.submit(self._namespace_wait, platform.environment["mc_kubeconfig"], cluster_info["metadata"]["cluster_id"], cluster_name, "Management") if platform.environment["mc_kubeconfig"] != "" else 0
             cluster_info["mc_namespace_timing"] = mc_namespace.result() - cluster_start_time if platform.environment["mc_kubeconfig"] != "" else None
+        cluster_start_time_on_mc = mc_namespace.result()
         watch_code, watch_out, watch_err = self.utils.subprocess_exec("rosa logs install -c " + cluster_name + " --watch", cluster_info["path"] + "/installation.log", {'preexec_fn': self.utils.disable_signals})
         if watch_code != 0:
             cluster_info['status'] = "not ready"
@@ -557,6 +558,10 @@ def create_cluster(self, platform, cluster_name):
             if self.es is not None:
                 cluster_info["timestamp"] = datetime.datetime.utcnow().isoformat()
                 self.es.index_metadata(cluster_info)
+                self.logging.info("Indexing Management cluster stats")
+                os.environ["START_TIME"] = f"{cluster_start_time_on_mc}"  # excludes pre-flight durations
+                os.environ["END_TIME"] = f"{cluster_end_time}"
+                self.utils.cluster_load(platform, cluster_name, load="index")
             # if cluster_load:
                 #     with all_clusters_installed:
                 #         logging.info('Waiting for all clusters to be installed to start e2e-benchmarking execution on %s' % cluster_name)

libs/utils.py

@@ -201,24 +201,26 @@ def install_scheduler(self, platform):
             self.logging.error("Thread creation failed")
         return cluster_thread_list
 
-    def cluster_load(self, platform, cluster_name):
+    def cluster_load(self, platform, cluster_name, load=""):
         load_env = os.environ.copy()
         my_path = platform.environment['clusters'][cluster_name]['path']
         load_env["KUBECONFIG"] = platform.environment.get('clusters', {}).get(cluster_name, {}).get('kubeconfig', "")
         load_env["MC_KUBECONFIG"] = platform.environment.get("mc_kubeconfig", "")
-        self.logging.info(f"Cloning workload repo {platform.environment['load']['repo']} on {my_path}/workload")
-        Repo.clone_from(platform.environment['load']['repo'], my_path + '/workload')
+        if not os.path.exists(my_path + '/workload'):
+            self.logging.info(f"Cloning workload repo {platform.environment['load']['repo']} on {my_path}/workload")
+            Repo.clone_from(platform.environment['load']['repo'], my_path + '/workload')
         # Copy executor to the local folder because we shaw in the past that we cannot use kube-burner with multiple executions at the same time
         shutil.copy2(platform.environment['load']['executor'], my_path)
         load_env["ITERATIONS"] = str(platform.environment['clusters'][cluster_name]['workers'] * platform.environment['load']['jobs'])
         load_env["EXTRA_FLAGS"] = "--churn-duration=" + platform.environment['load']['duration'] + " --churn-percent=10 --churn-delay=30s --timeout=24h"
         # if es_url is not None:
         #     load_env["ES_SERVER"] = es_url
         load_env["LOG_LEVEL"] = "debug"
-        load_env["WORKLOAD"] = platform.environment['load']['workload']
+        load_env["WORKLOAD"] = load if load != "" else platform.environment['load']['workload']
+        log_file = load if load != "" else platform.environment['load']['workload']
         load_env["KUBE_DIR"] = my_path
         if not self.force_terminate:
-            load_code, load_out, load_err = self.subprocess_exec('./run.sh', my_path + '/cluster_load.log', extra_params={'cwd': my_path + "/workload/" + platform.environment['load']['script_path'], 'env': load_env})
+            load_code, load_out, load_err = self.subprocess_exec('./run.sh', my_path + '/' + log_file + '.log', extra_params={'cwd': my_path + "/workload/" + platform.environment['load']['script_path'], 'env': load_env})
             if load_code != 0:
                 self.logging.error(f"Failed to execute workload {platform.environment['load']['script_path'] + '/run.sh'} on {cluster_name}")
         else: