Merge remote-tracking branch 'origin/develop' into feature-gromacs

bin/benchpark

@@ -125,17 +125,19 @@ def benchpark_check_system(arg_str):
 
 def benchpark_setup(subparsers, actions_dict):
     create_parser = subparsers.add_parser(
-        "setup", help="Create a benchmark and prepare it to build/run"
+        "setup", help="Set up an experiment and prepare it to build/run"
     )
 
     create_parser.add_argument(
-        "benchmark", type=str, help="The benchmark/experiment to run"
+        "benchmark", type=str, help="The experiment (benchmark/ProgrammingModel) to run"
     )
-    create_parser.add_argument("system", type=str, help="The system we are on")
     create_parser.add_argument(
-        "workspace_root",
+        "system", type=str, help="The system on which to run the experiment"
+    )
+    create_parser.add_argument(
+        "experiments_root",
         type=str,
-        help="Where to install packages, store results, etc. (should be empty/nonexistent)",
+        help="Where to install packages and store results for the experiments. Benchpark expects to manage this directory, and it should be empty/nonexistent the first time you run benchpark setup experiments.",
     )
 
     actions_dict["setup"] = benchpark_setup_handler
@@ -173,23 +175,23 @@ def symlink_tree(src, dst):
 
 def benchpark_setup_handler(args):
     """
-    workspace_root/
+    experiments_root/
+        spack/
+        ramble/
         <benchmark>/
             <system>/
                 workspace/
                     configs/
                         (everything from source/configs/<system>)
                         (everything from source/experiments/<benchmark>)
-        spack/
-        ramble/
     """
 
     benchmark = args.benchmark
     system = args.system
-    workspace_root = pathlib.Path(os.path.abspath(args.workspace_root))
+    experiments_root = pathlib.Path(os.path.abspath(args.experiments_root))
     source_dir = source_location()
     debug_print(f"source_dir = {source_dir}")
-    debug_print(f"specified benchmark = {benchmark}")
+    debug_print(f"specified benchmark/ProgrammingModel = {benchmark}")
     valid_benchmark = benchpark_check_benchmark(benchmark)
     debug_print(f"specified system = {system}")
     valid_system = benchpark_check_system(system)
@@ -200,7 +202,7 @@ def benchpark_setup_handler(args):
             )
         )
 
-    workspace_dir = workspace_root / str(benchmark) / str(system)
+    workspace_dir = experiments_root / str(benchmark) / str(system)
 
     if workspace_dir.exists():
         if workspace_dir.is_dir():
@@ -226,8 +228,8 @@ def benchpark_setup_handler(args):
     symlink_tree(configs_src_dir, ramble_configs_dir)
     symlink_tree(experiment_src_dir, ramble_configs_dir)
 
-    spack_location = workspace_root / "spack"
-    ramble_location = workspace_root / "ramble"
+    spack_location = experiments_root / "spack"
+    ramble_location = experiments_root / "ramble"
 
     spack_exe = spack_location / "bin" / "spack"
     ramble_exe = ramble_location / "bin" / "ramble"
@@ -261,14 +263,12 @@ def benchpark_setup_handler(args):
     instructions = f"""\
 To complete the benchpark setup, do the following:
 
-    cd {workspace_dir}/workspace
-
     . {spack_location}/share/spack/setup-env.sh
     . {ramble_location}/share/ramble/setup-env.sh
 
     export SPACK_DISABLE_LOCAL_CONFIG=1
 
-Further steps are needed to build the experiments (ramble -P -D . workspace setup) and run them (ramble -P -D . on)
+Further steps are needed to build the experiments (ramble -P -D {ramble_workspace_dir} workspace setup) and run them (ramble -P -D {ramble_workspace_dir} on)
 """
     print(instructions)
 

docs/4-benchpark-setup.rst

@@ -3,37 +3,35 @@ Benchpark Setup
 ===============
 
 Select a benchmark experiment to run, along with the programming model to use, and a system to run them on.
-Also choose the workspace for your experiment::
+Also choose a directory for your experiment::
 
-    benchpark setup benchmark/ProgrammingModel system /output/path/to/workspace_root
+    benchpark setup benchmark/ProgrammingModel system /output/path/to/experiments_root
 
 where:
 
-- ``benchmark/ProgrammingModel``: amg2023/openmp | amg2023/cuda | saxpy/openmp (available choices in benchpark/experiments)
-- ``system``: ats2 | ats4 | cts1 (available choices in benchpark/configs)
+- ``benchmark/ProgrammingModel``: amg2023/openmp | amg2023/cuda | saxpy/openmp (available choices in ``benchpark/experiments``)
+- ``system``: ats2 | ats4 | cts1 (available choices in ``benchpark/configs``)
 
-This command will assemble a Ramble workspace 
-with a configuration for the specified benchmark and system 
+This command will assemble a Ramble workspace per experiment
+with a configuration for the specified benchmark and system
 with the following directory structure::
 
-    workspace_root/
+    experiments_root/
+        ramble/
+        spack/
         <benchmark>/
             <ProgrammingModel>/
                 <system>/
-                    ramble/
-                    spack/
                     workspace/
                         configs/
                             (everything from source/configs/<system>)
                             (everything from source/experiments/<benchmark>)
 
 ``benchpark setup`` will output instructions to follow::
 
-   cd <workspace_root>/<benchmark/ProgrammingModel>/<system>/workspace
-
-   . <workspace_root>/<benchmark/ProgrammingModel>/<system>/spack/share/spack/setup-env.sh
-   . <workspace_root>/<benchmark/ProgrammingModel>/<system>/ramble/share/ramble/setup-env.sh
+   . <experiments_root>/spack/share/spack/setup-env.sh
+   . <experiments_root>/ramble/share/ramble/setup-env.sh
 
    export SPACK_DISABLE_LOCAL_CONFIG=1
 
-Now you are ready to build your experiments as described in :doc:`5-build-experiment`.
+Now you are ready to compile your experiments as described in :doc:`5-build-experiment`.

docs/5-build-experiment.rst

@@ -2,21 +2,21 @@
 Build experiment
 ================
 
-``benchpark setup`` has set up the directory structure for your experiment.  
+``benchpark setup`` has set up the directory structure for your experiment.
 The next step is setting up the Ramble workspace and building the code::
 
-   cd <workspace_root>/<benchmark/ProgrammingModel>/<system>/workspace
-   ramble -P -D . workspace setup  
+   cd <experiments_root>/<benchmark/ProgrammingModel>/<system>/workspace
+   ramble -P -D . workspace setup
 
 
 Ramble will build the source code and set up the following workspace directory structure::
 
-    workspace_root/
+    experiments_root/
+        ramble/
+        spack/
         <benchmark>/
             <ProgrammingModel>/
                 <system>/
-                    ramble/
-                    spack/
                     workspace/
                         configs/
                             (everything from source/configs/<system>)
@@ -27,6 +27,5 @@ Ramble will build the source code and set up the following workspace directory s
                                     <benchmark>_<ProgrammingModel>_<problem>
                                         execute_experiment
 
-
 If you edit any of the files, see :doc:`FAQ-what-to-rerun` to determine
 whether you need to re-do any of the previous steps.

docs/experiments_root_structure.rst

@@ -0,0 +1,19 @@
+For each ``experiment`` (``benchmark`` x ``ProgrammingModel`` x ),
+Ramble sets up the following ``workspace`` directory structure
+to build and run the experiment::
+
+    experiments_root/
+        ramble/
+        spack/
+        <benchmark>/
+            <ProgrammingModel>/
+                <system>/
+                    workspace/
+                        configs/
+                            (everything from source/configs/<system>)
+                            (everything from source/experiments/<benchmark>)
+                        experiments/
+                            <benchmark>/
+                                <problem>/
+                                    <benchmark>_<ProgrammingModel>_<problem>
+                                        execute_experiment

docs/index.rst

@@ -21,3 +21,9 @@
    add-a-benchmark
    add-an-experiment
    add-a-system-config
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Benchmark Suites
+
+   set-of-experiments

docs/set-of-experiments.rst

@@ -0,0 +1,41 @@
+==============
+Working with a set of experiments
+==============
+
+You may want to use the same experiments_root directory when setting up multiple experiments:
+
+* Benchpark only stores one copy of Spack/Ramble per experiments directory,
+  and does not repeatedly download them when reusing the experiments directory.
+* Since each experiment in the experiments directory shares the Spack/Ramble instances,
+  the edits/updates you make to Spack packages will apply to all experiments
+  in your experiments directory.
+
+To use the same experiments directory when setting up multiple experiments,
+instruct benchpark to set them up in the same experiments_root::
+
+    benchpark setup benchmark1/ProgrammingModel1 system1 /output/path/to/experiments_root
+    benchpark setup benchmark1/ProgrammingModel2 system2 /output/path/to/experiments_root
+    benchpark setup benchmark2/ProgrammingModel2 system1 /output/path/to/experiments_root
+
+This will result in the following directory structure::
+
+    experiments_root/
+        ramble/
+        spack/
+        benchmark1/
+            ProgrammingModel1/
+                system1/
+                    workspace/
+            ProgrammingModel2/
+                system2/
+                    workspace/
+        benchmark2/
+            ProgrammingModel2/
+                system1/
+                    workspace/
+
+Note that there is a single clone of Ramble, and a single clone of Spack,
+which all of the ``experiments`` use.
+Each ``experiment`` (``benchmark/ProgrammingModel`` x ``system`` combination)
+has its own ``Ramble workspace``, where this specific ``experiment``
+will be compiled and run.