Initial commit of MPI-only Laghos with Sedov blast wave experiment (#199

) * Initial commit of MPI-only Laghos with Sedov blast wave experiment. * Adding custom package.py for Laghos * Updating ramble.yaml * Temporary copyright update (will change back when upstream to spack) * Update Laghos package.py to install data directory. * Fix FOM extraction regex * Customizing package.py to pull and install the Caliper variant of Laghos * Update sspack recipe * Force the use of lib instead of zlib-ng * Use original spack recipe * Adding configs for LLNL Ruby and AWS HPC6a.48xlarge. Replacing Sedov test problem with 3D triple-point strong scaling problem. Adding package.py to pull custom version of Laghos with Caliper support. * Adding Ruby and HPC6a configs. Replacing Sedov experiment with 3D triple-point strong scaling experiment. --------- Co-authored-by: Daryl Hawkins <[email protected]> Co-authored-by: pearce8 <[email protected]> Co-authored-by: Riyaz Haque <[email protected]>
LLNL · Jun 10, 2024 · c6183ee · c6183ee
1 parent 26a4ef1
commit c6183ee
Show file tree

Hide file tree

Showing 15 changed files with 468 additions and 0 deletions.
diff --git a/configs/LLNL-Ruby-icelake-OmniPath/auxiliary_software_files/compilers.yaml b/configs/LLNL-Ruby-icelake-OmniPath/auxiliary_software_files/compilers.yaml
@@ -0,0 +1,32 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+compilers:
+- compiler:
+    spec: [email protected]
+    paths:
+      cc: /usr/tce/packages/gcc/gcc-12.1.1/bin/gcc
+      cxx: /usr/tce/packages/gcc/gcc-12.1.1/bin/g++
+      f77: /usr/tce/packages/gcc/gcc-12.1.1/bin/gfortran
+      fc: /usr/tce/packages/gcc/gcc-12.1.1/bin/gfortran
+    flags: {}
+    operating_system: rhel8
+    target: x86_64
+    modules: []
+    environment: {}
+    extra_rpaths: []
+- compiler:
+    spec: [email protected]
+    paths:
+      cc: /usr/tce/packages/intel-classic/intel-classic-2021.6.0/bin/icc
+      cxx: /usr/tce/packages/intel-classic/intel-classic-2021.6.0/bin/icpc
+      f77: /usr/tce/packages/intel-classic/intel-classic-2021.6.0/bin/ifort
+      fc: /usr/tce/packages/intel-classic/intel-classic-2021.6.0/bin/ifort
+    flags: {}
+    operating_system: rhel8
+    target: x86_64
+    modules: []
+    environment: {}
+    extra_rpaths: []
diff --git a/configs/LLNL-Ruby-icelake-OmniPath/auxiliary_software_files/packages.yaml b/configs/LLNL-Ruby-icelake-OmniPath/auxiliary_software_files/packages.yaml
@@ -0,0 +1,62 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+packages:
+  papi:
+    externals:
+    - spec: [email protected]
+      prefix: /usr/tce/packages/papi/papi-6.0.0.1
+      buildable: false
+  tar:
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+      buildable: false
+  cmake:
+    externals:
+    - spec: [email protected]
+      prefix: /usr/tce/packages/cmake/cmake-3.23.1
+      buildable: false
+  gmake:
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+  blas:
+    externals:
+    - spec: [email protected]
+      prefix: /usr/tce/backend/installations/linux-rhel8-x86_64/intel-19.0.4/intel-oneapi-mkl-2022.1.0-sksz67twjxftvwchnagedk36gf7plkrp
+      buildable: false
+  lapack:
+    externals:
+    - spec: [email protected]
+      prefix: /usr/tce/backend/installations/linux-rhel8-x86_64/intel-19.0.4/intel-oneapi-mkl-2022.1.0-sksz67twjxftvwchnagedk36gf7plkrp
+      buildable: false
+  python:
+    externals:
+    - spec: [email protected]
+      prefix: /usr/tce/packages/python/python-3.9.12/
+      buildable: false
+  hwloc:
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+      buildable: false
+  fftw:
+    externals:
+    - spec: [email protected]
+      prefix: /usr/tce/packages/fftw/fftw-3.3.10
+      buildable: false
+  mpi:
+    externals:
+    - spec: [email protected]
+      prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.7-gcc-12.1.1
+      buildable: false
+      extra_attributes:
+        ldflags: "-L/usr/tce/packages/mvapich2/mvapich2-2.3.7-gcc-12.1.1/lib -lmpi"
+    - spec: [email protected]
+      prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.7-intel-classic-2021.6.0
+      buildable: false
+      extra_attributes:
+        ldflags: "-L/usr/tce/packages/mvapich2/mvapich2-2.3.7-intel-classic-2021.6.0/lib -lmpi"
diff --git a/configs/LLNL-Ruby-icelake-OmniPath/spack.yaml b/configs/LLNL-Ruby-icelake-OmniPath/spack.yaml
@@ -0,0 +1,23 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+spack:
+  packages:
+    default-compiler:
+      spack_spec: [email protected]
+    default-mpi:
+      spack_spec: [email protected]
+    compiler-gcc:
+      spack_spec: [email protected]
+    compiler-intel:
+      spack_spec: [email protected]
+    blas:
+      spack_spec: [email protected]
+    lapack:
+      spack_spec: [email protected]
+    mpi-gcc:
+      spack_spec: [email protected]
+    mpi-intel:
+      spack_spec: [email protected]
diff --git a/configs/LLNL-Ruby-icelake-OmniPath/system_definition.yaml b/configs/LLNL-Ruby-icelake-OmniPath/system_definition.yaml
@@ -0,0 +1,32 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+system_definition:
+  name: Ruby
+  site: LLNL
+  system: Penguin-icelake-OmniPath
+  integrator:
+    vendor: Supermicro
+    name:
+  processor:
+    vendor: Intel
+    name: XeonPlatinum8276L
+    ISA: x86_64
+    uArch: icelake
+  accelerator:
+    vendor:
+    name:
+    ISA:
+    uArch:
+  interconnect:
+    vendor: Cornelis
+    name: OmniPath
+  system-tested:
+    site: LLNL
+    name: ruby 
+    installation-year: 2020
+    description: top500 <https://www.top500.org/system/179866>
+  top500-system-instances:
+    - Ruby (LLNL)
diff --git a/configs/LLNL-Ruby-icelake-OmniPath/variables.yaml b/configs/LLNL-Ruby-icelake-OmniPath/variables.yaml
@@ -0,0 +1,14 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+variables:
+  batch_time: '02:00'
+  mpi_command: 'srun -N {n_nodes} -n {n_ranks}'
+  batch_submit: 'sbatch {execute_experiment}'
+  batch_nodes: '#SBATCH -N {n_nodes}'
+  batch_ranks: '#SBATCH -n {n_ranks}'
+  batch_timeout: '#SBATCH -t {batch_time}:00'
+  sys_cpus_per_node: 56
+  sys_gpus_per_node: 0
diff --git a/configs/nosite-AWS_PCluster_Hpc6a-zen3-EFA/README.txt b/configs/nosite-AWS_PCluster_Hpc6a-zen3-EFA/README.txt
@@ -0,0 +1,34 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+AWS x86 ParallelCluster 3.7.2
+-----------------------------
+
+This config should work on any AWS x86 ParallelCluster 3.7.2 instance with the
+following caveats:
+
+1) All compute instances must be x86 and EFA enabled. Supported instance types
+   can be found here:
+
+   https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/efa.html#efa-instance-types
+
+2) Any OS supported by ParallelCluster 3.7.2 should work, but only Amazon
+   Linux 2 has been tested
+
+3) ParallelCluster does not install optimized versions of BLAS/LAPACK. This
+   config uses the generic versions installed via:
+
+   sudo yum install lapack
+
+3) OpenMPI is the only supported MPI flavor. IntelMPI is not yet supported.
+
+   OpenMPI is running in verbose mode so the user than confirm that EFA is
+   being used when running experiments. A line similar to the following in
+   slurm-NNN.out confirms EFA is being used:
+
+   mtl_ofi_component.c:362: mtl:ofi:provider: rdmap0s6-rdm 
+
+   This debugging output can be silenced by removing the env variable
+   'OMPI_MCA_mtl_base_verbose=100' from the srun line in variables.yaml. 
diff --git a/configs/nosite-AWS_PCluster_Hpc6a-zen3-EFA/auxiliary_software_files/compilers.yaml b/configs/nosite-AWS_PCluster_Hpc6a-zen3-EFA/auxiliary_software_files/compilers.yaml
@@ -0,0 +1,19 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+compilers:
+- compiler:
+    spec: [email protected]
+    paths:
+      cc: /usr/bin/gcc
+      cxx: /usr/bin/g++
+      f77: /usr/bin/gfortran
+      fc: /usr/bin/gfortran
+    flags: {}
+    operating_system: alinux2
+    target: x86_64
+    modules: []
+    environment: {}
+    extra_rpaths: []
diff --git a/configs/nosite-AWS_PCluster_Hpc6a-zen3-EFA/auxiliary_software_files/packages.yaml b/configs/nosite-AWS_PCluster_Hpc6a-zen3-EFA/auxiliary_software_files/packages.yaml
@@ -0,0 +1,32 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+packages:
+  tar:
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+      buildable: false
+  gmake:
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+  blas:
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+      buildable: false
+  lapack:
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+      buildable: false
+  mpi:
+    externals:
+    - spec: [email protected]
+      prefix: /opt/amazon/openmpi
+      buildable: false
+      extra_attributes:
+        ldflags: "-L/opt/amazon/openmpi/lib -lmpi"
diff --git a/configs/nosite-AWS_PCluster_Hpc6a-zen3-EFA/spack.yaml b/configs/nosite-AWS_PCluster_Hpc6a-zen3-EFA/spack.yaml
@@ -0,0 +1,17 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+spack:
+  packages:
+    default-compiler:
+      spack_spec: [email protected]
+    default-mpi:
+      spack_spec: [email protected]
+    compiler-gcc:
+      spack_spec: [email protected]
+    lapack:
+      spack_spec: [email protected]
+    mpi-gcc:
+      spack_spec: [email protected]
diff --git a/configs/nosite-AWS_PCluster_Hpc6a-zen3-EFA/system_definition.yaml b/configs/nosite-AWS_PCluster_Hpc6a-zen3-EFA/system_definition.yaml
@@ -0,0 +1,32 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+system_definition:
+  name: AWS_PCluster_Hpc7a-zen4-EFA
+  site:
+  system: AWS_PCluster_Hpc7a-zen4-EFA
+  integrator:
+    vendor: AWS
+    name: ParallelCluster3.7.2-Hpc7a
+  processor:
+    vendor: AMD
+    name: EPYC-Zen4
+    ISA: x86_64
+    uArch: zen4
+  accelerator:
+    vendor:
+    name:
+    ISA:
+    uArch:
+  interconnect:
+    vendor: AWS
+    name: EFA
+  system-tested:
+    site: AWS
+    name:
+    installation-year:
+    description: aws/hpc7a <https://aws.amazon.com/ec2/instance-types/hpc7a>
+  top500-system-instances:
+    -
diff --git a/configs/nosite-AWS_PCluster_Hpc6a-zen3-EFA/variables.yaml b/configs/nosite-AWS_PCluster_Hpc6a-zen3-EFA/variables.yaml
@@ -0,0 +1,14 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+variables:
+  batch_time: '02:00'
+  mpi_command: 'srun -N {n_nodes} -n {n_ranks} --mpi=pmix --export=ALL,FI_EFA_USE_DEVICE_RDMA=1,FI_PROVIDER="efa",OMPI_MCA_mtl_base_verbose=100'
+  batch_submit: 'sbatch {execute_experiment}'
+  batch_nodes: '#SBATCH -N {n_nodes}'
+  batch_ranks: '#SBATCH -n {n_ranks}'
+  batch_timeout: '#SBATCH -t {batch_time}:00'
+  sys_cpus_per_node: 96
+  sys_gpus_per_node: 0
diff --git a/experiments/laghos/mpi/execute_experiment.tpl b/experiments/laghos/mpi/execute_experiment.tpl
@@ -0,0 +1,13 @@
+#!/bin/bash
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+{batch_nodes}
+{batch_ranks}
+{batch_timeout}
+
+cd {experiment_run_dir}
+
+{command}
diff --git a/experiments/laghos/mpi/ramble.yaml b/experiments/laghos/mpi/ramble.yaml
@@ -0,0 +1,39 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+ramble:
+  include:
+  - ./configs/spack.yaml
+  - ./configs/variables.yaml
+
+  config:
+    deprecated: true
+    spack_flags:
+      install: '--add --keep-stage'
+      concretize: '-U -f'
+
+  applications:
+    laghos:
+      workloads:
+        problem:
+          env_vars:
+          variables:
+            n_nodes: ['1', '2', '4', '8', '16', '32', '64', '128']
+            n_ranks: '{sys_cpus_per_node} * {n_nodes}'
+          experiments:
+            laghos_mpi_triplept_{n_nodes}_{n_ranks}:
+              variables:
+  spack:
+    concretized: true
+    packages:
+      laghos:
+        spack_spec: laghos@develop +metis ^zlib
+        compiler: default-compiler
+    environments:
+      laghos:
+        packages:
+        - default-mpi
+        - blas
+        - laghos