diff --git a/configs/LLNL-Ruby-icelake-OmniPath/auxiliary_software_files/compilers.yaml b/configs/LLNL-Ruby-icelake-OmniPath/auxiliary_software_files/compilers.yaml new file mode 100644 index 000000000..2e71df790 --- /dev/null +++ b/configs/LLNL-Ruby-icelake-OmniPath/auxiliary_software_files/compilers.yaml @@ -0,0 +1,32 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +compilers: +- compiler: + spec: gcc@12.1.1 + paths: + cc: /usr/tce/packages/gcc/gcc-12.1.1/bin/gcc + cxx: /usr/tce/packages/gcc/gcc-12.1.1/bin/g++ + f77: /usr/tce/packages/gcc/gcc-12.1.1/bin/gfortran + fc: /usr/tce/packages/gcc/gcc-12.1.1/bin/gfortran + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] +- compiler: + spec: intel@2021.6.0-classic + paths: + cc: /usr/tce/packages/intel-classic/intel-classic-2021.6.0/bin/icc + cxx: /usr/tce/packages/intel-classic/intel-classic-2021.6.0/bin/icpc + f77: /usr/tce/packages/intel-classic/intel-classic-2021.6.0/bin/ifort + fc: /usr/tce/packages/intel-classic/intel-classic-2021.6.0/bin/ifort + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] diff --git a/configs/LLNL-Ruby-icelake-OmniPath/auxiliary_software_files/packages.yaml b/configs/LLNL-Ruby-icelake-OmniPath/auxiliary_software_files/packages.yaml new file mode 100644 index 000000000..0cb4fc3b5 --- /dev/null +++ b/configs/LLNL-Ruby-icelake-OmniPath/auxiliary_software_files/packages.yaml @@ -0,0 +1,62 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +packages: + papi: + externals: + - spec: papi@6.0.0.1 + prefix: /usr/tce/packages/papi/papi-6.0.0.1 + buildable: false + tar: + externals: + - spec: tar@1.30 + prefix: /usr + buildable: false + cmake: + externals: + - spec: cmake@3.23.1 + prefix: /usr/tce/packages/cmake/cmake-3.23.1 + buildable: false + gmake: + externals: + - spec: gmake@4.2.1 + prefix: /usr + blas: + externals: + - spec: intel-oneapi-mkl@2022.1.0 + prefix: /usr/tce/backend/installations/linux-rhel8-x86_64/intel-19.0.4/intel-oneapi-mkl-2022.1.0-sksz67twjxftvwchnagedk36gf7plkrp + buildable: false + lapack: + externals: + - spec: intel-oneapi-mkl@2022.1.0 + prefix: /usr/tce/backend/installations/linux-rhel8-x86_64/intel-19.0.4/intel-oneapi-mkl-2022.1.0-sksz67twjxftvwchnagedk36gf7plkrp + buildable: false + python: + externals: + - spec: python@3.9.12 + prefix: /usr/tce/packages/python/python-3.9.12/ + buildable: false + hwloc: + externals: + - spec: hwloc@2.9.1 + prefix: /usr + buildable: false + fftw: + externals: + - spec: fftw@3.3.10 + prefix: /usr/tce/packages/fftw/fftw-3.3.10 + buildable: false + mpi: + externals: + - spec: mvapich2@2.3.7-gcc1211 + prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.7-gcc-12.1.1 + buildable: false + extra_attributes: + ldflags: "-L/usr/tce/packages/mvapich2/mvapich2-2.3.7-gcc-12.1.1/lib -lmpi" + - spec: mvapich2@2.3.7-intel202160classic + prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.7-intel-classic-2021.6.0 + buildable: false + extra_attributes: + ldflags: "-L/usr/tce/packages/mvapich2/mvapich2-2.3.7-intel-classic-2021.6.0/lib -lmpi" diff --git a/configs/LLNL-Ruby-icelake-OmniPath/spack.yaml b/configs/LLNL-Ruby-icelake-OmniPath/spack.yaml new file mode 100644 index 000000000..5668e42f7 --- /dev/null +++ b/configs/LLNL-Ruby-icelake-OmniPath/spack.yaml @@ -0,0 +1,23 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +spack: + packages: + default-compiler: + spack_spec: gcc@12.1.1 + default-mpi: + spack_spec: mvapich2@2.3.7-gcc1211 + compiler-gcc: + spack_spec: gcc@12.1.1 + compiler-intel: + spack_spec: intel@2021.6.0-classic + blas: + spack_spec: intel-oneapi-mkl@2022.1.0 + lapack: + spack_spec: intel-oneapi-mkl@2022.1.0 + mpi-gcc: + spack_spec: mvapich2@2.3.7-gcc1211 + mpi-intel: + spack_spec: mvapich2@2.3.7-intel202160classic diff --git a/configs/LLNL-Ruby-icelake-OmniPath/system_definition.yaml b/configs/LLNL-Ruby-icelake-OmniPath/system_definition.yaml new file mode 100644 index 000000000..ca0113a03 --- /dev/null +++ b/configs/LLNL-Ruby-icelake-OmniPath/system_definition.yaml @@ -0,0 +1,32 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +system_definition: + name: Ruby + site: LLNL + system: Penguin-icelake-OmniPath + integrator: + vendor: Supermicro + name: + processor: + vendor: Intel + name: XeonPlatinum8276L + ISA: x86_64 + uArch: icelake + accelerator: + vendor: + name: + ISA: + uArch: + interconnect: + vendor: Cornelis + name: OmniPath + system-tested: + site: LLNL + name: ruby + installation-year: 2020 + description: top500 + top500-system-instances: + - Ruby (LLNL) diff --git a/configs/LLNL-Ruby-icelake-OmniPath/variables.yaml b/configs/LLNL-Ruby-icelake-OmniPath/variables.yaml new file mode 100644 index 000000000..4c9210e1f --- /dev/null +++ b/configs/LLNL-Ruby-icelake-OmniPath/variables.yaml @@ -0,0 +1,14 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +variables: + batch_time: '02:00' + mpi_command: 'srun -N {n_nodes} -n {n_ranks}' + batch_submit: 'sbatch {execute_experiment}' + batch_nodes: '#SBATCH -N {n_nodes}' + batch_ranks: '#SBATCH -n {n_ranks}' + batch_timeout: '#SBATCH -t {batch_time}:00' + sys_cpus_per_node: 56 + sys_gpus_per_node: 0 diff --git a/configs/nosite-AWS_PCluster_Hpc6a-zen3-EFA/README.txt b/configs/nosite-AWS_PCluster_Hpc6a-zen3-EFA/README.txt new file mode 100644 index 000000000..6864ab012 --- /dev/null +++ b/configs/nosite-AWS_PCluster_Hpc6a-zen3-EFA/README.txt @@ -0,0 +1,34 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +AWS x86 ParallelCluster 3.7.2 +----------------------------- + +This config should work on any AWS x86 ParallelCluster 3.7.2 instance with the +following caveats: + +1) All compute instances must be x86 and EFA enabled. Supported instance types + can be found here: + + https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/efa.html#efa-instance-types + +2) Any OS supported by ParallelCluster 3.7.2 should work, but only Amazon + Linux 2 has been tested + +3) ParallelCluster does not install optimized versions of BLAS/LAPACK. This + config uses the generic versions installed via: + + sudo yum install lapack + +3) OpenMPI is the only supported MPI flavor. IntelMPI is not yet supported. + + OpenMPI is running in verbose mode so the user than confirm that EFA is + being used when running experiments. A line similar to the following in + slurm-NNN.out confirms EFA is being used: + + mtl_ofi_component.c:362: mtl:ofi:provider: rdmap0s6-rdm + + This debugging output can be silenced by removing the env variable + 'OMPI_MCA_mtl_base_verbose=100' from the srun line in variables.yaml. diff --git a/configs/nosite-AWS_PCluster_Hpc6a-zen3-EFA/auxiliary_software_files/compilers.yaml b/configs/nosite-AWS_PCluster_Hpc6a-zen3-EFA/auxiliary_software_files/compilers.yaml new file mode 100644 index 000000000..31758ea83 --- /dev/null +++ b/configs/nosite-AWS_PCluster_Hpc6a-zen3-EFA/auxiliary_software_files/compilers.yaml @@ -0,0 +1,19 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +compilers: +- compiler: + spec: gcc@7.3.1 + paths: + cc: /usr/bin/gcc + cxx: /usr/bin/g++ + f77: /usr/bin/gfortran + fc: /usr/bin/gfortran + flags: {} + operating_system: alinux2 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] diff --git a/configs/nosite-AWS_PCluster_Hpc6a-zen3-EFA/auxiliary_software_files/packages.yaml b/configs/nosite-AWS_PCluster_Hpc6a-zen3-EFA/auxiliary_software_files/packages.yaml new file mode 100644 index 000000000..60e9df4dc --- /dev/null +++ b/configs/nosite-AWS_PCluster_Hpc6a-zen3-EFA/auxiliary_software_files/packages.yaml @@ -0,0 +1,32 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +packages: + tar: + externals: + - spec: tar@1.26 + prefix: /usr + buildable: false + gmake: + externals: + - spec: gmake@3.8.2 + prefix: /usr + blas: + externals: + - spec: blas@3.4.2 + prefix: /usr + buildable: false + lapack: + externals: + - spec: lapack@3.4.2 + prefix: /usr + buildable: false + mpi: + externals: + - spec: openmpi@4.1.5-gcc731 + prefix: /opt/amazon/openmpi + buildable: false + extra_attributes: + ldflags: "-L/opt/amazon/openmpi/lib -lmpi" diff --git a/configs/nosite-AWS_PCluster_Hpc6a-zen3-EFA/spack.yaml b/configs/nosite-AWS_PCluster_Hpc6a-zen3-EFA/spack.yaml new file mode 100644 index 000000000..15ddc2bbf --- /dev/null +++ b/configs/nosite-AWS_PCluster_Hpc6a-zen3-EFA/spack.yaml @@ -0,0 +1,17 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +spack: + packages: + default-compiler: + spack_spec: gcc@7.3.1 + default-mpi: + spack_spec: openmpi@4.1.5-gcc731 + compiler-gcc: + spack_spec: gcc@7.3.1 + lapack: + spack_spec: lapack@3.4.2 + mpi-gcc: + spack_spec: openmpi@4.1.5-gcc731 diff --git a/configs/nosite-AWS_PCluster_Hpc6a-zen3-EFA/system_definition.yaml b/configs/nosite-AWS_PCluster_Hpc6a-zen3-EFA/system_definition.yaml new file mode 100644 index 000000000..2d8f3dc1d --- /dev/null +++ b/configs/nosite-AWS_PCluster_Hpc6a-zen3-EFA/system_definition.yaml @@ -0,0 +1,32 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +system_definition: + name: AWS_PCluster_Hpc7a-zen4-EFA + site: + system: AWS_PCluster_Hpc7a-zen4-EFA + integrator: + vendor: AWS + name: ParallelCluster3.7.2-Hpc7a + processor: + vendor: AMD + name: EPYC-Zen4 + ISA: x86_64 + uArch: zen4 + accelerator: + vendor: + name: + ISA: + uArch: + interconnect: + vendor: AWS + name: EFA + system-tested: + site: AWS + name: + installation-year: + description: aws/hpc7a + top500-system-instances: + - diff --git a/configs/nosite-AWS_PCluster_Hpc6a-zen3-EFA/variables.yaml b/configs/nosite-AWS_PCluster_Hpc6a-zen3-EFA/variables.yaml new file mode 100644 index 000000000..f5db177a1 --- /dev/null +++ b/configs/nosite-AWS_PCluster_Hpc6a-zen3-EFA/variables.yaml @@ -0,0 +1,14 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +variables: + batch_time: '02:00' + mpi_command: 'srun -N {n_nodes} -n {n_ranks} --mpi=pmix --export=ALL,FI_EFA_USE_DEVICE_RDMA=1,FI_PROVIDER="efa",OMPI_MCA_mtl_base_verbose=100' + batch_submit: 'sbatch {execute_experiment}' + batch_nodes: '#SBATCH -N {n_nodes}' + batch_ranks: '#SBATCH -n {n_ranks}' + batch_timeout: '#SBATCH -t {batch_time}:00' + sys_cpus_per_node: 96 + sys_gpus_per_node: 0 diff --git a/experiments/laghos/mpi/execute_experiment.tpl b/experiments/laghos/mpi/execute_experiment.tpl new file mode 100755 index 000000000..89e73cf49 --- /dev/null +++ b/experiments/laghos/mpi/execute_experiment.tpl @@ -0,0 +1,13 @@ +#!/bin/bash +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +{batch_nodes} +{batch_ranks} +{batch_timeout} + +cd {experiment_run_dir} + +{command} diff --git a/experiments/laghos/mpi/ramble.yaml b/experiments/laghos/mpi/ramble.yaml new file mode 100644 index 000000000..786fb0b44 --- /dev/null +++ b/experiments/laghos/mpi/ramble.yaml @@ -0,0 +1,39 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +ramble: + include: + - ./configs/spack.yaml + - ./configs/variables.yaml + + config: + deprecated: true + spack_flags: + install: '--add --keep-stage' + concretize: '-U -f' + + applications: + laghos: + workloads: + problem: + env_vars: + variables: + n_nodes: ['1', '2', '4', '8', '16', '32', '64', '128'] + n_ranks: '{sys_cpus_per_node} * {n_nodes}' + experiments: + laghos_mpi_triplept_{n_nodes}_{n_ranks}: + variables: + spack: + concretized: true + packages: + laghos: + spack_spec: laghos@develop +metis ^zlib + compiler: default-compiler + environments: + laghos: + packages: + - default-mpi + - blas + - laghos diff --git a/repo/laghos/application.py b/repo/laghos/application.py new file mode 100644 index 000000000..405d0cc1d --- /dev/null +++ b/repo/laghos/application.py @@ -0,0 +1,28 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +import sys + +from ramble.appkit import * + + +class Laghos(SpackApplication): + """Laghos benchmark""" + name = "laghos" + + tags = ['asc','engineering','hypre','solver','cfd','large-scale', + 'multi-node','single-node','mpi','network-latency-bound', + 'network-collectives','unstructured-grid'] + + executable('p', 'laghos -p 3 -m {laghos}/data/box01_hex.mesh -rs 5 -ms 500', use_mpi=True) + + workload('problem', executables=['p']) + + figure_of_merit('Major kernels total time', + log_file='{experiment_run_dir}/{experiment_name}.out', + fom_regex=r'Major kernels total time \(seconds\):\s+(?P[0-9]+\.[0-9]*(e^[0-9]*)?)', + group_name='fom', units='seconds') + + success_criteria('pass', mode='string', match=r'Major kernels total time', file='{experiment_run_dir}/{experiment_name}.out') diff --git a/repo/laghos/package.py b/repo/laghos/package.py new file mode 100644 index 000000000..d30819bd9 --- /dev/null +++ b/repo/laghos/package.py @@ -0,0 +1,77 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +from spack.package import * + + +class Laghos(MakefilePackage): + """Laghos (LAGrangian High-Order Solver) is a CEED miniapp that solves the + time-dependent Euler equations of compressible gas dynamics in a moving + Lagrangian frame using unstructured high-order finite element spatial + discretization and explicit high-order time-stepping. + """ + + tags = ["proxy-app", "ecp-proxy-app"] + + homepage = "https://github.com/wdhawkins/laghos" + git = "https://github.com/wdhawkins/Laghos.git" + + maintainers("wdhawkins") + + license("BSD-2-Clause") + + version("develop", branch="caliper") + + variant("metis", default=True, description="Enable/disable METIS support") + variant("caliper", default=False, description="Enable/disable Caliper support") + variant("ofast", default=False, description="Enable gcc optimization flags") + + depends_on("mfem+mpi+metis", when="+metis") + depends_on("mfem+mpi~metis", when="~metis") + depends_on("caliper", when="+caliper") + depends_on("adiak", when="+caliper") + + depends_on("mfem@develop", when="@develop") + depends_on("mfem@4.2.0:", when="@3.1") + depends_on("mfem@4.1.0:4.1", when="@3.0") + # Recommended mfem version for laghos v2.0 is: ^mfem@3.4.1-laghos-v2.0 + depends_on("mfem@3.4.1-laghos-v2.0", when="@2.0") + # Recommended mfem version for laghos v1.x is: ^mfem@3.3.1-laghos-v1.0 + depends_on("mfem@3.3.1-laghos-v1.0", when="@1.0,1.1") + + # Replace MPI_Session + patch( + "https://github.com/CEED/Laghos/commit/c800883ab2741c8c3b99486e7d8ddd8e53a7cb95.patch?full_index=1", + sha256="e783a71c3cb36886eb539c0f7ac622883ed5caf7ccae597d545d48eaf051d15d", + when="@3.1 ^mfem@4.4:", + ) + + @property + def build_targets(self): + targets = [] + spec = self.spec + + targets.append("MFEM_DIR=%s" % spec["mfem"].prefix) + targets.append("CONFIG_MK=%s" % spec["mfem"].package.config_mk) + targets.append("TEST_MK=%s" % spec["mfem"].package.test_mk) + if "+caliper" in self.spec: + targets.append("CALIPER_DIR=%s" % spec["caliper"].prefix) + if spec.satisfies("@:2.0"): + targets.append("CXX=%s" % spec["mpi"].mpicxx) + if "+ofast %gcc" in self.spec: + targets.append("CXXFLAGS = -Ofast -finline-functions") + return targets + + # See lib/spack/spack/build_systems/makefile.py + def check(self): + with working_dir(self.build_directory): + make("test", *self.build_targets) + + def install(self, spec, prefix): + mkdirp(prefix.bin) + install("laghos", prefix.bin) + install_tree("data", prefix.data) + + install_time_test_callbacks = [] # type: List[str]