Merge branch 'experiment/ior' of github.com:LLNL/benchpark into exper…

…iment/ior

.github/workflows/run.yml

@@ -219,22 +219,23 @@ jobs:
             --disable-logger \
             workspace setup --dry-run
 
-      - name: Dry run laghos/mpi-only on LLNL-Magma-Penguin-icelake-OmniPath with allocation modifier
+      - name: Dry run dynamic laghos/mpi-only on LLNL-Magma-Penguin-icelake-OmniPath with allocation modifier
         run: |
-          ./bin/benchpark setup laghos/mpi-only LLNL-Magma-Penguin-icelake-OmniPath workspace/
+          ./bin/benchpark experiment init --dest=laghos-mpi-only laghos
+          ./bin/benchpark setup ./laghos-mpi-only LLNL-Magma-Penguin-icelake-OmniPath workspace/
           . workspace/setup.sh
           ramble \
-            --workspace-dir workspace/laghos/mpi-only/LLNL-Magma-Penguin-icelake-OmniPath/workspace \
+            --workspace-dir workspace/laghos-mpi-only/LLNL-Magma-Penguin-icelake-OmniPath/workspace \
             --disable-progress-bar \
             --disable-logger \
             workspace setup --dry-run
 
       - name: Dry run laghos/mpi-only on LLNL-Ruby-icelake-OmniPath with allocation modifier
         run: |
-          ./bin/benchpark setup laghos/mpi-only LLNL-Ruby-icelake-OmniPath workspace/
+          ./bin/benchpark setup laghos-mpi-only LLNL-Ruby-icelake-OmniPath workspace/
           . workspace/setup.sh
           ramble \
-            --workspace-dir workspace/laghos/mpi-only/LLNL-Ruby-icelake-OmniPath/workspace \
+            --workspace-dir workspace/laghos-mpi-only/LLNL-Ruby-icelake-OmniPath/workspace \
             --disable-progress-bar \
             --disable-logger \
             workspace setup --dry-run

.gitlab/ci/test.yml

@@ -7,13 +7,19 @@
         ARCHCONFIG: LLNL-Dane-DELL-sapphirerapids-OmniPath
 
 test_run:
+  resource_group: $HOST
   stage: test
   tags:
     - $HOST
-    - shell
+    - batch
   <<: *test_clusters
+  variables:
+    SCHEDULER_PARAMETERS: -N 1 -t 1h
   rules:
-    - changes:
+    - if: |
+        $CI_PIPELINE_SOURCE == "merge_request_event" ||
+        $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+      changes:
         - .gitlab-ci.yml
         - .gitlab/ci/*
         - experiments/**
@@ -32,4 +38,5 @@ test_run:
     - cd ./workspace/saxpy/openmp/$ARCHCONFIG/workspace/
     - ramble --workspace-dir . --disable-progress-bar --disable-logger workspace setup
     # Run Saxpy Experiments
-    - ramble --workspace-dir . --disable-progress-bar --disable-logger on
+    - ramble --workspace-dir . --disable-progress-bar --disable-logger
+      on --executor '{execute_experiment}' --where '{n_nodes} == 1'

docs/_static/css/custom.css

@@ -89,3 +89,7 @@ a:visited {
 html.writer-html4 .rst-content dl:not(.docutils) > dt, html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.citation):not(.glossary):not(.simple) > dt {
     color: #7fa866
 }
+
+.rst-content div[class^="highlight"] pre {
+    max-height: 600px;
+}

docs/add-a-dryrun.rst

@@ -25,7 +25,7 @@ For example, if you are contributing a system called foo you would test it with
   - name: Dry run dynamic saxpy on dynamic foo
     run: |
       benchpark system init --dest=foo-system foo
-      benchpark experiment init --dest=saxpy-openmp saxpy
+      benchpark experiment init --dest=saxpy-openmp saxpy openmp=oui
       benchpark setup ./saxpy ./foo-system workspace/
       . workspace/setup.sh
       ramble \

docs/add-a-system-config.rst

@@ -219,7 +219,7 @@ Validating the System
 To manually validate your new system, you should initialize it and run an existing experiment such as saxpy. For example::
 
   benchpark system init --dest=modifiedx86-system modifiedx86
-  benchpark experiment init --dest=saxpy saxpy
+  benchpark experiment init --dest=saxpy saxpy openmp=oui
   benchpark setup ./saxpy ./modifiedx86-system workspace/
 
 Then you can run the commands provided by the output, the experiments should be built and run successfully without any errors. 

docs/add-an-experiment.rst

@@ -34,25 +34,28 @@ Some or all of the functions in the Experiment base class can be overridden to d
   This will change and need updates
 
 Variants of the experiment can be added to utilize different *ProgrammingModels* used for on-node parallelization,
-e.g., ``benchpark/experiments/amg2023/experiment.py`` has variant ``programming_model``, which can be
+e.g., ``benchpark/experiments/amg2023/experiment.py`` can be updated to inherit from different experiments to , which can be
 set to ``cuda`` for an experiment using CUDA (on an NVIDIA GPU),
 or ``openmp`` for an experiment using OpenMP (on a CPU).::
 
-    class Amg2023(Experiment):
-      variant(
-          "programming_model",
-          default="openmp",
-          values=("openmp", "cuda", "rocm"),
-          description="on-node parallelism model",
-      )
+    class Amg2023(
+      Experiment,
+      OpenMPExperiment,
+      CudaExperiment,
+      ROCmExperiment,
+      StrongScaling,
+      WeakScaling,
+      ThroughputScaling,
+      Caliper,
+    ):
 
 Multiple types of experiments can be created using variants as well (e.g., strong scaling, weak scaling). See AMG2023 or Kripke for examples.
 
 Once an experiment class has been written, an experiment is initialized with the following command, with any variants that have been defined in your experiment.py passed in as key-value pairs: 
-``benchpark experiment init --dest {path/to/dest} experiment={experiment_variant} programming_model={prog_model_variant}``
+``benchpark experiment init --dest {path/to/dest} {benchmark_name} {variant}={oui/non/value}``
 
 For example, to run the AMG2023 strong scaling experiment for problem 1, using CUDA the command would be:
-``benchpark experiment init --dest amg2023 programming_model=cuda workload=problem1 experiment=strong``
+``benchpark experiment init --dest amg2023_experiment amg2023 cuda=oui workload=problem1 strong=oui``
 
 Initializing an experiment generates the following yaml files:
 
@@ -71,7 +74,7 @@ To manually validate your new experiments work, you should initialize an existin
 For example if you just created a benchmark *baz* with OpenMP and strong scaling variants it may look like this:::
 
   benchpark system init --dest=genericx86-system genericx86 
-  benchpark experiment init --dest=baz-benchmark baz programming_model=openmp scaling=strong
+  benchpark experiment init --dest=baz-benchmark baz openmp=oui strong=oui
   benchpark setup ./baz-benchmark ./x86 workspace/
 
 

docs/benchpark-setup.rst

@@ -11,8 +11,8 @@ To setup an experiment workspace you must first initialize both an experiment an
 Any system or experiment variants are specified at the end of the command as shown below.
 The order of the two init commands does not matter, but they both need to be run before the setup command.::
 
-benchpark system init --dest=</output/path/to/system_def_dir> <SystemName> compiler=<Compiler>
-benchpark experiment init --dest=</output/path/to/experiment_def_dir> <Benchmark> experiment=<Experiment> programming_model=<ProgrammingModel>
+    benchpark system init --dest=</output/path/to/system_def_dir> <SystemName> compiler=<Compiler>
+    benchpark experiment init --dest=</output/path/to/experiment_def_dir> <Benchmark> <Variant>=<oui/non/value>
 
 
 Once you have a benchmark experiment to run, along with the programming model to use, and a system to run them on.

docs/conf.py

@@ -7,6 +7,8 @@
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
 
 import subprocess
+import sys
+import os
 
 subprocess.call(
     [
@@ -32,8 +34,12 @@
 extensions = [
     "sphinx_rtd_theme",
     "sphinxcontrib.programoutput",
+    "sphinx.ext.autodoc",
+    "sphinx.ext.napoleon",
 ]
 
+sys.path.insert(0, os.path.abspath("../lib"))
+
 exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", ".spack-env"]
 
 # -- Options for HTML output -------------------------------------------------

docs/index.rst

@@ -48,6 +48,12 @@
    add-an-experiment
    add-a-dryrun
 
+.. toctree::
+   :maxdepth: 2
+   :caption: API Docs
+
+   Benchpark API Docs <source/benchpark>
+
 .. toctree::
    :maxdepth: 1
    :caption: Legacy Workflow

docs/llnl-tutorial.rst

@@ -14,6 +14,13 @@ Running on an LLNL System
 This tutorial will guide you through the process of using Benchpark on LLNL
 systems. 
 
+To run Benchpark, you will need to install its requirements: go to the
+Benchpark root directory::
+
+    python -m venv my-env
+    . my-env/bin/activate
+    pip install -r requirements.txt
+
 ------------------------
 CTS (Ruby, Dane, Magma)
 ------------------------
@@ -26,9 +33,9 @@ system specification in Benchpark::
 
     benchpark system init --dest=ruby-system cts cluster=ruby
 
-To run the cuda version of the AMG20223 benchmark, initialize it for experiments::
+To run the openmp, strong scaling version of the AMG20223 benchmark, initialize it for experiments::
 
-    benchpark experiment init --dest=amg2023-benchmark amg2023 programming_model=openmp
+    benchpark experiment init --dest=amg2023-benchmark amg2023 openmp=oui
 
 Then setup the workspace directory for the system and experiment together::
 
@@ -44,22 +51,22 @@ Then setup the Ramble experiment workspace, this builds all software and may tak
     cd ./workspace/amg2023-benchmark/Cts-6d48f81/workspace/
     ramble --workspace-dir . --disable-progress-bar workspace setup
 
-Next, we run the Saxpy experiments, which will launch jobs through the
-scheduler on Tioga::
+Next, we run the AMG2023 experiments, which will launch jobs through the
+scheduler on the CTS system::
 
     ramble --workspace-dir . --disable-progress-bar on
 
 ------
 Tioga
 ------
 
-This second tutorial will guide you through the process of using the cuda 
+This second tutorial will guide you through the process of using the ROCm
 version of the Saxpy benchmark on Tioga. 
 The parameters for initializing the system are slightly different due to the 
 different variants defined for the system. For example, the variant ``~gtl`` turns off gtl-enabled MPI, ``+gtl`` turns it on::
 
     benchpark system init --dest=tioga-system tioga ~gtl
-    benchpark experiment init --dest=saxpy-benchmark saxpy programming_model=cuda
+    benchpark experiment init --dest=saxpy-benchmark saxpy rocm=oui
     benchpark setup ./saxpy-benchmark ./tioga-system workspace/
     . workspace/setup.sh
     cd ./workspace/saxpy-benchmark/Tioga-975af3c/workspace/

docs/modifiers.rst

@@ -99,7 +99,7 @@ setup step::
 
     benchpark experiment init --dest=</path/to/experiment_root> caliper=<caliper_modifier> </output/path/to/workspace> 
 
-Valid values for ``<caliper>`` are found in the **Caliper Variant**
+Valid values for ``<caliper_variant>`` are found in the **Caliper Variant**
 column of the table below.  Benchpark will link the experiment to Caliper,
 and inject appropriate Caliper configuration at runtime.  After the experiments
 in the workspace have completed running, a ``.cali`` file

experiments/laghos/experiment.py

@@ -0,0 +1,80 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from benchpark.error import BenchparkError
+from benchpark.directives import variant
+from benchpark.experiment import Experiment
+from benchpark.scaling import StrongScaling
+from benchpark.expr.builtin.caliper import Caliper
+
+
+class Laghos(
+    Experiment,
+    StrongScaling,
+    Caliper,
+):
+
+    variant(
+        "workload",
+        default="triplept",
+        description="triplept or other problem",
+    )
+
+    variant(
+        "version",
+        default="develop",
+        description="app version",
+    )
+
+    def compute_applications_section(self):
+        # TODO: Replace with conflicts clause
+        scaling_modes = {
+            "strong": self.spec.satisfies("strong=oui"),
+            "single_node": self.spec.satisfies("single_node=oui"),
+        }
+
+        scaling_mode_enabled = [key for key, value in scaling_modes.items() if value]
+        if len(scaling_mode_enabled) != 1:
+            raise BenchparkError(
+                f"Only one type of scaling per experiment is allowed for application package {self.name}"
+            )
+
+        # Number of initial nodes
+        num_nodes = {"n_nodes": 1}
+
+        if self.spec.satisfies("single_node=oui"):
+            for pk, pv in num_nodes.items():
+                self.add_experiment_variable(pk, pv, True)
+        elif self.spec.satisfies("strong=oui"):
+            scaled_variables = self.generate_strong_scaling_params(
+                {tuple(num_nodes.keys()): list(num_nodes.values())},
+                int(self.spec.variants["scaling-factor"][0]),
+                int(self.spec.variants["scaling-iterations"][0]),
+            )
+            for pk, pv in scaled_variables.items():
+                self.add_experiment_variable(pk, pv, True)
+
+        self.add_experiment_variable(
+            "n_ranks", "{sys_cores_per_node} * {n_nodes}", True
+        )
+
+    def compute_spack_section(self):
+        # get package version
+        app_version = self.spec.variants["version"][0]
+
+        # get system config options
+        # TODO: Get compiler/mpi/package handles directly from system.py
+        system_specs = {}
+        system_specs["compiler"] = "default-compiler"
+        system_specs["mpi"] = "default-mpi"
+
+        # set package spack specs
+        # empty package_specs value implies external package
+        self.add_spack_spec(system_specs["mpi"])
+        # self.add_spack_spec(system_specs["blas"])
+
+        self.add_spack_spec(
+            self.name, [f"laghos@{app_version} +metis", system_specs["compiler"]]
+        )

lib/benchpark/cmd/audit.py

@@ -11,7 +11,7 @@
 import benchpark.paths
 import benchpark.repo
 
-# import benchpark.system as system
+import benchpark.system as system
 from benchpark.runtime import RuntimeResources
 
 bootstrapper = RuntimeResources(benchpark.paths.benchpark_home)  # noqa
@@ -31,6 +31,10 @@ def setup_parser(subparser):
 def audit_experiment(exp_cls):
     required_methods = ["compute_applications_section", "compute_spack_section"]
 
+    if exp_cls.__name__ == "Caliper":
+        # Caliper is an abstract class, and is never directly instantiated
+        return []
+
     errors = list()
 
     for method in required_methods:
@@ -76,17 +80,12 @@ def audit_system(sys_cls):
     externals = basedir / "externals"
     if externals.exists():
         for f in _find_yaml_files(externals):
-            pass
-            # TODO: this fails for reasons I don't understand, even though
-            # this duplicates logic from system.py
-            # cfg.read_config_file(f, system.packages_schema)
+            cfg.read_config_file(f, system.packages_schema.schema)
 
     compilers = basedir / "compilers"
     if compilers.exists():
         for f in _find_yaml_files(compilers):
-            pass
-            # cfg.read_config_file(f, system.compilers_schema)
-            # TODO: same problem as prior loop
+            cfg.read_config_file(f, system.compilers_schema.schema)
     return errors
 
 

lib/benchpark/directives.py

@@ -116,8 +116,10 @@ def variant(
     sticky: bool = False,
 ):
     """Define a variant.
+
     Can specify a default value as well as a text description.
-    Args:
+
+    Arguments:
         name: Name of the variant
         default: Default value for the variant, if not specified otherwise the default will be
             False for a boolean variant and 'nothing' for a multi-valued variant
@@ -130,6 +132,7 @@ def variant(
             if the group doesn't meet the additional constraints
         when: Optional condition on which the variant applies
         sticky: The variant should not be changed by the concretizer to find a valid concrete spec
+
     Raises:
         DirectiveError: If arguments passed to the directive are invalid
     """