Starting a LLNL-Pascal config (#119)

* Starting a LLNL-Pascal config

* initial config for pascal

* add gcc (for raja-perf blt build); cuda10 components + def cuda10_version

* fix indentation in compilers.yaml; add needed variables (referenced by amg2023 experiment application)

* rocm_arch not needed; cuda_arch is needed

* what I thought was an intel-mkl install was in fact an intel-oneapi-install; also, the config depends on cublas being defined (although I don't think it can be used given that intel-oneapi-mkl is the blas provider as well as the lapack provider)

* wrong variable name

* use slurm scheduler

* get rid of cuda10 references; replace intel compiler with gcc

* adjust arch and force cuda_arch for amg2023

---------

Co-authored-by: Peter Josef Scheibel <[email protected]>

configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/compilers.yaml

@@ -0,0 +1,19 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+compilers:
+- compiler:
+    spec: gcc@=11.2.1
+    paths:
+      cc: /usr/tce/packages/gcc/gcc-11.2.1/bin/gcc
+      cxx: /usr/tce/packages/gcc/gcc-11.2.1/bin/g++
+      f77: /usr/tce/packages/gcc/gcc-11.2.1/bin/gfortran
+      fc: /usr/tce/packages/gcc/gcc-11.2.1/bin/gfortran
+    flags: {}
+    operating_system: rhel8
+    target: x86_64
+    modules: []
+    environment: {}
+    extra_rpaths: []

configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/packages.yaml

@@ -0,0 +1,64 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+packages:
+  tar:
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+    buildable: false
+  cmake:
+    externals:
+    - spec: [email protected]
+      prefix: /usr/tce/packages/cmake/cmake-3.23.1
+    buildable: false
+  gmake:
+    externals:
+    - spec: [email protected]
+      prefix: /usr/tcetmp/packages/gmake/gmake-4.2.1
+    buildable: false
+  automake:
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+  autoconf:
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+  mpi:
+    buildable: false
+  mvapich2:
+    buildable: false
+    externals:
+    - spec: [email protected]%[email protected]
+      prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.7-intel-classic-2021.6.0/
+  cuda:
+    buildable: false
+    externals:
+    - spec: [email protected]
+      prefix: /usr/tce/packages/cuda/cuda-11.8.0/
+  curand:
+    externals:
+    - spec: [email protected]
+      prefix: /usr/tce/packages/cuda/cuda-11.8.0
+    buildable: false
+  cusparse:
+    externals:
+    - spec: [email protected]
+      prefix: /usr/tce/packages/cuda/cuda-11.8.0
+    buildable: false
+  cublas:
+    externals:
+    - spec: [email protected]
+      prefix: /usr/tce/packages/cuda/cuda-11.8.0
+    buildable: false
+  blas:
+    buildable: false
+  lapack:
+    buildable: false
+  intel-oneapi-mkl:
+    externals:
+    - spec: [email protected]
+      prefix: /usr/tce/packages/mkl/mkl-2022.1.0/

configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/spack.yaml

@@ -0,0 +1,17 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+spack:
+  packages:
+    default-compiler:
+      spack_spec: gcc@=11.2.1
+    default-mpi:
+      spack_spec: [email protected]
+    compiler-gcc:
+      spack_spec: gcc@=11.2.1
+    cublas-cuda:
+      spack_spec: cublas@{default_cuda_version}
+    lapack:
+      spack_spec: [email protected]

configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/system_definition.yaml

@@ -0,0 +1,32 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+system_definition:
+  name: Pascal
+  site: LLNL
+  system: Penguin-broadwell-P100-OmniPath
+  integrator:
+    vendor: Penguin
+    name:
+  processor:
+    vendor: Intel
+    name: Xeon_E5-2695_v4
+    ISA: x86_64
+    uArch: broadwell
+  accelerator:
+    vendor: NVIDIA
+    name: P100
+    ISA: PTX
+    uArch: sm_56
+  interconnect:
+    vendor: Cornelis
+    name: OmniPath
+  system-tested:
+    site: LLNL
+    name: pascal
+    installation-year: 2018
+    description:
+  top500-system-instances:
+    -

configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/variables.yaml

@@ -0,0 +1,15 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+variables:
+  gtl_flag: ''  # to be overwritten by tests that need GTL
+  cuda_arch: '60'
+  default_cuda_version: '11.8.0'
+  batch_time: '02:00'
+  mpi_command: 'srun -N {n_nodes} -n {n_ranks}'
+  batch_submit: 'sbatch {execute_experiment}'
+  batch_nodes: '#SBATCH -N {n_nodes}'
+  batch_ranks: '#SBATCH -n {n_ranks}'
+  batch_timeout: '#SBATCH -t {batch_time}:00'

experiments/amg2023/cuda/ramble.yaml

@@ -46,7 +46,7 @@ ramble:
         spack_spec: [email protected] +mpi+cuda+mixedint cuda_arch=={cuda_arch} ^cuda@{default_cuda_version}
         compiler: default-compiler
       amg2023:
-        spack_spec: amg2023@develop +mpi+cuda ^cuda@{default_cuda_version}
+        spack_spec: amg2023@develop +mpi+cuda cuda_arch=={cuda_arch} ^cuda@{default_cuda_version}
         compiler: default-compiler
     environments:
       amg2023: