From d97f257acf0143209220b1a6903a7843f72ea191 Mon Sep 17 00:00:00 2001 From: pearce8 Date: Thu, 22 Feb 2024 19:20:50 +1100 Subject: [PATCH] Adding a Fugaku system config (#118) * Adding a Fugaku system config * license * Update system_definition.yaml * adding site * fix code to increase content width of sphinx docs * Initial compilers for Fugaku * thread counts for fugaku * adding PJM scheduler instructions https://matsci.dev/blog/2023/fugaku/ * add updated table of system definitions with fugaku * renaming RIKEN to RCCS * Removing extra dir * initial draft for fugaku config (#142) * initial draft for fugaku config * Add integrator name * Removing extra dir --------- Co-authored-by: domke <673751-domke@users.noreply.gitlab.com> Co-authored-by: pearce8 * license * column width * license * updating table for docs --------- Co-authored-by: Stephanie Brink Co-authored-by: jdomke <28772296+jdomke@users.noreply.github.com> Co-authored-by: domke <673751-domke@users.noreply.gitlab.com> --- .../auxiliary_software_files/compilers.yaml | 136 ++++ .../auxiliary_software_files/packages.yaml | 586 ++++++++++++++++++ .../spack.yaml | 23 + .../system_definition.yaml | 32 + .../variables.yaml | 13 + docs/tables/current-system-definitions.csv | 40 +- experiments/amg2023/openmp/ramble.yaml | 2 +- 7 files changed, 811 insertions(+), 21 deletions(-) create mode 100644 configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/auxiliary_software_files/compilers.yaml create mode 100644 configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/auxiliary_software_files/packages.yaml create mode 100644 configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/spack.yaml create mode 100644 configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/system_definition.yaml create mode 100644 configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml diff --git a/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/auxiliary_software_files/compilers.yaml b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/auxiliary_software_files/compilers.yaml new file mode 100644 index 000000000..25129f761 --- /dev/null +++ b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/auxiliary_software_files/compilers.yaml @@ -0,0 +1,136 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +compilers: +- compiler: + spec: fj@4.10.0 + modules: [] + paths: + cc: /opt/FJSVxtclanga/tcsds-1.2.38/bin/fcc + cxx: /opt/FJSVxtclanga/tcsds-1.2.38/bin/FCC + f77: /opt/FJSVxtclanga/tcsds-1.2.38/bin/frt + fc: /opt/FJSVxtclanga/tcsds-1.2.38/bin/frt + flags: {} + operating_system: rhel8 + target: aarch64 + environment: + set: + fcc_ENV: -Nclang + FCC_ENV: -Nclang + prepend_path: + PATH: /opt/FJSVxtclanga/tcsds-1.2.38/bin + LD_LIBRARY_PATH: /opt/FJSVxtclanga/tcsds-1.2.38/lib64 + extra_rpaths: [] +- compiler: + spec: fj@4.8.1 + modules: [] + paths: + cc: /opt/FJSVxtclanga/tcsds-1.2.36/bin/fcc + cxx: /opt/FJSVxtclanga/tcsds-1.2.36/bin/FCC + f77: /opt/FJSVxtclanga/tcsds-1.2.36/bin/frt + fc: /opt/FJSVxtclanga/tcsds-1.2.36/bin/frt + flags: {} + operating_system: rhel8 + target: aarch64 + environment: + set: + fcc_ENV: -Nclang + FCC_ENV: -Nclang + prepend_path: + PATH: /opt/FJSVxtclanga/tcsds-1.2.36/bin + LD_LIBRARY_PATH: /opt/FJSVxtclanga/tcsds-1.2.36/lib64 + extra_rpaths: [] +- compiler: + spec: fj@4.8.0 + modules: [] + paths: + cc: /opt/FJSVxtclanga/tcsds-1.2.35/bin/fcc + cxx: /opt/FJSVxtclanga/tcsds-1.2.35/bin/FCC + f77: /opt/FJSVxtclanga/tcsds-1.2.35/bin/frt + fc: /opt/FJSVxtclanga/tcsds-1.2.35/bin/frt + flags: {} + operating_system: rhel8 + target: aarch64 + environment: + set: + fcc_ENV: -Nclang + FCC_ENV: -Nclang + prepend_path: + PATH: /opt/FJSVxtclanga/tcsds-1.2.35/bin + LD_LIBRARY_PATH: /opt/FJSVxtclanga/tcsds-1.2.35/lib64 + extra_rpaths: [] +- compiler: + spec: fj@4.7.0 + modules: [] + paths: + cc: /opt/FJSVxtclanga/tcsds-1.2.34/bin/fcc + cxx: /opt/FJSVxtclanga/tcsds-1.2.34/bin/FCC + f77: /opt/FJSVxtclanga/tcsds-1.2.34/bin/frt + fc: /opt/FJSVxtclanga/tcsds-1.2.34/bin/frt + flags: {} + operating_system: rhel8 + target: aarch64 + environment: + set: + fcc_ENV: -Nclang + FCC_ENV: -Nclang + prepend_path: + PATH: /opt/FJSVxtclanga/tcsds-1.2.34/bin + LD_LIBRARY_PATH: /opt/FJSVxtclanga/tcsds-1.2.34/lib64 + extra_rpaths: [] +- compiler: + paths: + cc: /usr/bin/gcc + cxx: /usr/bin/g++ + f77: /usr/bin/gfortran + fc: /usr/bin/gfortran + operating_system: rhel8 + target: aarch64 + modules: [] + environment: + unset: [] + extra_rpaths: [] + flags: {} + spec: gcc@8.5.0 +- compiler: + paths: + cc: /usr/bin/gcc + cxx: /usr/bin/g++ + f77: /usr/bin/gfortran + fc: /usr/bin/gfortran + operating_system: rhel8 + target: x86_64 + modules: [] + environment: + unset: [] + extra_rpaths: [] + flags: {} + spec: gcc@8.4.1 +- compiler: + spec: gcc@12.2.0 + paths: + cc: /vol0004/apps/oss/spack-v0.19/opt/spack/linux-rhel8-skylake_avx512/gcc-8.4.1/gcc-12.2.0-bjidm56oz6mejysl3wvjr3mbn7sqk3ec/bin/gcc + cxx: /vol0004/apps/oss/spack-v0.19/opt/spack/linux-rhel8-skylake_avx512/gcc-8.4.1/gcc-12.2.0-bjidm56oz6mejysl3wvjr3mbn7sqk3ec/bin//g++ + f77: /vol0004/apps/oss/spack-v0.19/opt/spack/linux-rhel8-skylake_avx512/gcc-8.4.1/gcc-12.2.0-bjidm56oz6mejysl3wvjr3mbn7sqk3ec/bin/gfortran + fc: /vol0004/apps/oss/spack-v0.19/opt/spack/linux-rhel8-skylake_avx512/gcc-8.4.1/gcc-12.2.0-bjidm56oz6mejysl3wvjr3mbn7sqk3ec/bin/gfortran + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] +- compiler: + spec: gcc@12.2.0 + paths: + cc: /vol0004/apps/oss/spack-v0.19/opt/spack/linux-rhel8-a64fx/gcc-8.5.0/gcc-12.2.0-sxcx7kmt3qiktffgzzvrj2wmup3g32bc/bin/gcc + cxx: /vol0004/apps/oss/spack-v0.19/opt/spack/linux-rhel8-a64fx/gcc-8.5.0/gcc-12.2.0-sxcx7kmt3qiktffgzzvrj2wmup3g32bc/bin/g++ + f77: /vol0004/apps/oss/spack-v0.19/opt/spack/linux-rhel8-a64fx/gcc-8.5.0/gcc-12.2.0-sxcx7kmt3qiktffgzzvrj2wmup3g32bc/bin/gfortran + fc: /vol0004/apps/oss/spack-v0.19/opt/spack/linux-rhel8-a64fx/gcc-8.5.0/gcc-12.2.0-sxcx7kmt3qiktffgzzvrj2wmup3g32bc/bin/gfortran + flags: {} + operating_system: rhel8 + target: aarch64 + modules: [] + environment: {} + extra_rpaths: [] diff --git a/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/auxiliary_software_files/packages.yaml b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/auxiliary_software_files/packages.yaml new file mode 100644 index 000000000..931188040 --- /dev/null +++ b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/auxiliary_software_files/packages.yaml @@ -0,0 +1,586 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +packages: + all: + compiler: [fj, gcc] + providers: + mpi: [fujitsu-mpi, openmpi, mpich] + blas: [fujitsu-ssl2, openblas] + lapack: [fujitsu-ssl2, openblas] + scalapack: [fujitsu-ssl2, netlib-scalapack] + fftw-api: [fujitsu-fftw, fftw, rist-fftw] + permissions: + write: group + htslib: + version: [1.12] + python: + externals: + - spec: "python@3.10.8%fj +ssl arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/oss/spack-v0.19/opt/spack/linux-rhel8-a64fx/fj-4.8.1/python-3.10.8-7q66snjvhvy7im57hncbgpirmddrb5sk + - spec: "python@3.10.8%fj +ssl+tkinter arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/oss/spack-v0.19/opt/spack/linux-rhel8-a64fx/fj-4.8.1/python-3.10.8-5q3ncyl2my7oomopsmukduqo36u6pnkg + - spec: "python@3.10.8%gcc +ssl arch=linux-rhel8-cascadelake" + prefix: /vol0004/apps/oss/spack-v0.19/opt/spack/linux-rhel8-cascadelake/gcc-12.2.0/python-3.10.8-yt6afcnywa36aebxovs3ldscknyzlva3 + - spec: "python@3.10.8%gcc +ssl+tkinter arch=linux-rhel8-cascadelake" + prefix: /vol0004/apps/oss/spack-v0.19/opt/spack/linux-rhel8-cascadelake/gcc-12.2.0/python-3.10.8-a5u7ucknpzdipu2dd3d7hf2ybkdyprly + openssh: + permissions: + write: user + fujitsu-mpi: + version: [head, 4.8.1, 4.8.0, 4.7.0] + externals: + - spec: "fujitsu-mpi@head%fj arch=linux-rhel8-a64fx" + prefix: /opt/FJSVxtclanga/tcsds-mpi-latest + - spec: "fujitsu-mpi@4.8.1%fj arch=linux-rhel8-a64fx" + prefix: /opt/FJSVxtclanga/tcsds-mpi-1.2.36 + - spec: "fujitsu-mpi@4.8.0%fj arch=linux-rhel8-a64fx" + prefix: /opt/FJSVxtclanga/tcsds-mpi-1.2.35 + - spec: "fujitsu-mpi@4.7.0%fj arch=linux-rhel8-a64fx" + prefix: /opt/FJSVxtclanga/tcsds-1.2.34 + - spec: "fujitsu-mpi@head%gcc@12.2.0 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/oss/mpigcc/fjmpi-gcc12 + - spec: "fujitsu-mpi@head%gcc@10.4.0 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/oss/mpigcc/fjmpi-gcc10 + - spec: "fujitsu-mpi@head%gcc@8.5.0 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/oss/mpigcc/fjmpi-gcc8 + buildable: False + fujitsu-ssl2: + version: [head, 4.8.1, 4.8.0, 4.7.0] + externals: + - spec: "fujitsu-ssl2@head%fj arch=linux-rhel8-a64fx" + prefix: /opt/FJSVxtclanga/tcsds-ssl2-latest + - spec: "fujitsu-ssl2@4.8.1%fj arch=linux-rhel8-a64fx" + prefix: /opt/FJSVxtclanga/tcsds-ssl2-1.2.36 + - spec: "fujitsu-ssl2@4.8.0%fj arch=linux-rhel8-a64fx" + prefix: /opt/FJSVxtclanga/tcsds-ssl2-1.2.35 + - spec: "fujitsu-ssl2@4.7.0%fj arch=linux-rhel8-a64fx" + prefix: /opt/FJSVxtclanga/tcsds-1.2.34 + buildable: False + rist-fftw: + externals: + - spec: "rist-fftw@3.3.9-272-g63d6bd70 arch=linux-rhel8-a64fx" + prefix: /vol0004/share/rist/fftw/gcc-10.3.0/3.3.9-272-g63d6bd70 + buildable: False + ntchem: + externals: + - spec: "ntchem@13.0.0%fj@4.8.1 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/ntchem/aarch64/NTChem2013/13.0.0/tcsds-1.2.36/release_mpiomp + - spec: "ntchem@12.2.0%fj arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/ntchem/aarch64/NTChem2013/12.2.0/tcsds-1.2.34/release_mpiomp + buildable: False + abinitmp: + externals: + - spec: "abinitmp@1-22%fj@4.8.1 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/SPACK-Feb2023-ABINIT-MP-VER1-REV22 + - spec: "abinitmp@2-4%fj@4.8.1 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/SPACK-Feb2023-ABINIT-MP-VER2-REV4 + - spec: "abinitmp@2-8%fj@4.8.1 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/SPACK-Jan2024-ABINIT-MP-VER2-REV8 + buildable: False + fds: + externals: + - spec: "fds@6.7.7%fj@4.8.0 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/FDS/fds-FDS6.7.7/Build/mpi_fugaku + - spec: "fds@6.7.9%fj@4.8.1 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/FDS/fds-FDS6.7.9/Build/mpi_fugaku + buildable: False + ffvhc-ace: + externals: + - spec: "ffvhc-ace@0.1%fj@4.8.1 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/FFVHC-ACE/0.1/bin + buildable: False + quantum-espresso: + externals: + - spec: "quantum-espresso@6.5%fj@4.10.0 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/qe-6.5/bin + - spec: "quantum-espresso@6.6%fj@4.10.0 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/qe-6.6/bin + - spec: "quantum-espresso@6.7%fj@4.10.0 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/qe-6.7/bin + - spec: "quantum-espresso@6.8%fj@4.10.0 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/qe-6.8/bin + - spec: "quantum-espresso@7.0%fj@4.10.0 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/qe-7.0/bin + - spec: "quantum-espresso@7.1%fj@4.10.0 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/qe-7.1/bin + - spec: "quantum-espresso@7.2%fj@4.10.0 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/qe-7.2/bin + - spec: "quantum-espresso@7.3%fj@4.10.0 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/qe-7.3/bin + buildable: False + autoconf: + externals: + - spec: "autoconf@2.69 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "autoconf@2.69 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "autoconf@2.69 arch=linux-rhel8-skylake_avx512" + prefix: /usr + automake: + externals: + - spec: "automake@1.16.1 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "automake@1.16.1 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "automake@1.16.1 arch=linux-rhel8-skylake_avx512" + prefix: /usr + binutils: + externals: + - spec: "binutils@2.30 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "binutils@2.30 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "binutils@2.30 arch=linux-rhel8-skylake_avx512" + prefix: /usr + bzip2: + externals: + - spec: "bzip2@1.0.6 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "bzip2@1.0.6 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "bzip2@1.0.6 arch=linux-rhel8-skylake_avx512" + prefix: /usr + curl: + externals: + - spec: "curl@7.61.1 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "curl@7.61.1 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "curl@7.61.1 arch=linux-rhel8-skylake_avx512" + prefix: /usr + dbus: + externals: + - spec: "dbus@1.12.8 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "dbus@1.12.8 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "dbus@1.12.8 arch=linux-rhel8-skylake_avx512" + prefix: /usr + elfutils: + externals: + - spec: "elfutils@0.186 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "elfutils@0.182 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "elfutils@0.182 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "elfutils@0.182 arch=linux-rhel8-skylake_avx512" + prefix: /usr + expat: + externals: + - spec: "expat@2.2.5 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "expat@2.2.5 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "expat@2.2.5 arch=linux-rhel8-skylake_avx512" + prefix: /usr + findutils: + externals: + - spec: "findutils@4.6.0 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "findutils@4.6.0 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "findutils@4.6.0 arch=linux-rhel8-skylake_avx512" + prefix: /usr + fontconfig: + externals: + - spec: "fontconfig@2.13.1 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "fontconfig@2.13.1 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "fontconfig@2.13.1 arch=linux-rhel8-skylake_avx512" + prefix: /usr + freetype: + externals: + - spec: "freetype@2.9.1 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "freetype@2.9.1 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "freetype@2.9.1 arch=linux-rhel8-skylake_avx512" + prefix: /usr + gmake: + externals: + - spec: "gmake@4.2.1 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "gmake@4.2.1 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "gmake@4.2.1 arch=linux-rhel8-skylake_avx512" + prefix: /usr + gdbm: + externals: + - spec: "gdbm@1.18 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "gdbm@1.18 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "gdbm@1.18 arch=linux-rhel8-skylake_avx512" + prefix: /usr + gettext: + externals: + - spec: "gettext@0.19.8.1 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "gettext@0.19.8.1 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "gettext@0.19.8.1 arch=linux-rhel8-skylake_avx512" + prefix: /usr + gmp: + externals: + - spec: "gmp@6.1.2 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "gmp@6.1.2 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "gmp@6.1.2 arch=linux-rhel8-skylake_avx512" + prefix: /usr + gnutls: + externals: + - spec: "gnutls@3.6.16 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "gnutls@3.6.14 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "gnutls@3.6.14 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "gnutls@3.6.14 arch=linux-rhel8-skylake_avx512" + prefix: /usr + hwloc: + externals: + - spec: "hwloc@2.2.0 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "hwloc@2.2.0 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "hwloc@2.2.0 arch=linux-rhel8-skylake_avx512" + prefix: /usr + jansson: + externals: + - spec: "jansson@2.14 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "jansson@2.11 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "jansson@2.11 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libaio: + externals: + - spec: "libaio@0.3.112 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libaio@0.3.112 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libaio@0.3.112 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libcap: + externals: + - spec: "libcap@2.48 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libcap@2.26 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libcap@2.26 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libdrm: + externals: + - spec: "libdrm@2.4.108 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libdrm@2.4.103 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libdrm@2.4.103 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libdrm@2.4.103 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libedit: + externals: + - spec: "libedit@3.1 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libedit@3.1 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libedit@3.1 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libevent: + externals: + - spec: "libevent@2.1.8 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libevent@2.1.8 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libevent@2.1.8 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libfabric: + externals: + - spec: "libfabric@1.14.0 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libfabric@1.11.2 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libfabric@1.11.2 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libffi: + externals: + - spec: "libffi@3.1 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libffi@3.1 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libffi@3.1 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libglvnd: + externals: + - spec: "libglvnd@1.3.4 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libglvnd@1.3.2 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libglvnd@1.3.2 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libibumad: + externals: + - spec: "libibumad@37.2 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libibumad@32.0 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libibumad@54mlnx1 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libibumad@54mlnx1 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libpciaccess: + externals: + - spec: "libpciaccess@0.14 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libpciaccess@0.14 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libpciaccess@0.14 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libpng: + externals: + - spec: "libpng@1.6.34 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libpng@1.6.34 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libpng@1.6.34 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libtasn1: + externals: + - spec: "libtasn1@4.13 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libtasn1@4.13 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libtasn1@4.13 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libtirpc: + externals: + - spec: "libtirpc@1.1.4 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libtirpc@1.1.4 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libtirpc@1.1.4 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libtool: + externals: + - spec: "libtool@2.4.6 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libtool@2.4.6 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libtool@2.4.6 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libuuid: + externals: + - spec: "libuuid@2.32.1 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libuuid@2.32.1 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libuuid@2.32.1 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libxcb: + externals: + - spec: "libxcb@1.13.1 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libxcb@1.13.1 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libxcb@1.13.1 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libxkbcommon: + externals: + - spec: "libxkbcommon@0.9.1 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libxkbcommon@0.9.1 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libxkbcommon@0.9.1 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libxml2: + externals: + - spec: "libxml2@2.9.7 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libxml2@2.9.7 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libxml2@2.9.7 arch=linux-rhel8-skylake_avx512" + prefix: /usr + lz4: + externals: + - spec: "lz4@1.8.3 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "lz4@1.8.3 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "lz4@1.8.3 arch=linux-rhel8-skylake_avx512" + prefix: /usr + m4: + externals: + - spec: "m4@1.4.18 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "m4@1.4.18 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "m4@1.4.18 arch=linux-rhel8-skylake_avx512" + prefix: /usr + ncurses: + externals: + - spec: "ncurses@6.1 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "ncurses@6.1 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "ncurses@6.1 arch=linux-rhel8-skylake_avx512" + prefix: /usr + nettle: + externals: + - spec: "nettle@3.4.1 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "nettle@3.4.1 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "nettle@3.4.1 arch=linux-rhel8-skylake_avx512" + prefix: /usr + nspr: + externals: + - spec: "nspr@4.32.0 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "nspr@4.25.0 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "nspr@4.25.0 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "nspr@4.25.0 arch=linux-rhel8-skylake_avx512" + prefix: /usr + numactl: + externals: + - spec: "numactl@2.0.12 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "numactl@2.0.12 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "numactl@2.0.12 arch=linux-rhel8-skylake_avx512" + prefix: /usr + opengl: + buildable: False + externals: + - spec: opengl@4.5.0 + prefix: /usr + openssl: + buildable: False + externals: + - spec: "openssl@1.1.1k arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "openssl@1.1.1g arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "openssl@1.1.1k arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "openssl@1.1.1k arch=linux-rhel8-skylake_avx512" + prefix: /usr + papi: + externals: + - spec: "papi@5.6.0 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "papi@5.6.0 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "papi@5.6.0 arch=linux-rhel8-skylake_avx512" + prefix: /usr + pcre: + externals: + - spec: "pcre@8.42 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "pcre@8.42 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "pcre@8.42 arch=linux-rhel8-skylake_avx512" + prefix: /usr + perl: + externals: + - spec: "perl@5.26.3 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "perl@5.26.3 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "perl@5.26.3 arch=linux-rhel8-skylake_avx512" + prefix: /usr + pkgconf: + externals: + - spec: "pkgconf@1.4.2 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "pkgconf@1.4.2 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "pkgconf@1.4.2 arch=linux-rhel8-skylake_avx512" + prefix: /usr + popt: + externals: + - spec: "popt@1.18 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "popt@1.18 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "popt@1.18 arch=linux-rhel8-skylake_avx512" + prefix: /usr + readline: + externals: + - spec: "readline@7.0 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "readline@7.0 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "readline@7.0 arch=linux-rhel8-skylake_avx512" + prefix: /usr + sqlite: + externals: + - spec: "sqlite@3.26.0 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "sqlite@3.26.0 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "sqlite@3.26.0 arch=linux-rhel8-skylake_avx512" + prefix: /usr + tcl: + externals: + - spec: "tcl@8.6.8 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "tcl@8.6.8 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "tcl@8.6.8 arch=linux-rhel8-skylake_avx512" + prefix: /usr + ucx: + externals: + - spec: "ucx@1.11.2 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "ucx@1.9.0 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "ucx@1.11.0 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "ucx@1.11.0 arch=linux-rhel8-skylake_avx512" + prefix: /usr + valgrind: + externals: + - spec: "valgrind@3.18.1 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "valgrind@3.16.0 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "valgrind@3.16.0 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "valgrind@3.16.0 arch=linux-rhel8-skylake_avx512" + prefix: /usr + xz: + externals: + - spec: "xz@5.2.4 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "xz@5.2.4 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "xz@5.2.4 arch=linux-rhel8-skylake_avx512" + prefix: /usr + zlib: + externals: + - spec: "zlib@1.2.11 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "zlib@1.2.11 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "zlib@1.2.11 arch=linux-rhel8-skylake_avx512" + prefix: /usr + buildable: False + # pmlib: had problems with spack. so far binary packages only. 2023/3/20 mikami + pmlib: + externals: + - spec: "pmlib@9.0-clang-precise arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/oss/pmlib-v9.0/9.0-clang-precise + - spec: "pmlib@9.0-clang-power arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/oss/pmlib-v9.0/9.0-clang-power + - spec: "pmlib@9.0-trad-power arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/oss/pmlib-v9.0/9.0-trad-power + buildable: False diff --git a/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/spack.yaml b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/spack.yaml new file mode 100644 index 000000000..bf2cf20c8 --- /dev/null +++ b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/spack.yaml @@ -0,0 +1,23 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +spack: + packages: + default-compiler: + spack_spec: fj@{default_fujitsu_version} + default-mpi: + spack_spec: fujitsu-mpi@head%fj@{default_fujitsu_version} + compiler-gcc: + spack_spec: gcc@12.2.0 + compiler-fujitsu: + spack_spec: fj@{default_fujitsu_version} + blas: + spack_spec: fujitsu-ssl2@head%fj@{default_fujitsu_version} + lapack: + spack_spec: fujitsu-ssl2@head%fj@{default_fujitsu_version} + mpi-gcc: + spack_spec: fujitsu-mpi@head%gcc@12.2.0 + mpi-fujitsu: + spack_spec: fujitsu-mpi@head%fj@{default_fujitsu_version} \ No newline at end of file diff --git a/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/system_definition.yaml b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/system_definition.yaml new file mode 100644 index 000000000..afdbd6d63 --- /dev/null +++ b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/system_definition.yaml @@ -0,0 +1,32 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +system_definition: + name: Fugaku + site: RIKEN Center for Computational Science + system: Fujitsu-A64FX-TofuD + integrator: + vendor: Fujitsu + name: FX1000 + processor: + vendor: Fujitsu + name: A64FX + ISA: Armv8.2-A-SVE + uArch: aarch64 + accelerator: + vendor: + name: + ISA: + uArch: + interconnect: + vendor: Fujitsu + name: TofuInterconnectD + system-tested: + site: R-CCS + name: Fugaku + installation-year: 2020 + description: top500 + top500-system-instances: + - Fugaku (R-CCS) \ No newline at end of file diff --git a/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml new file mode 100644 index 000000000..046ce5155 --- /dev/null +++ b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml @@ -0,0 +1,13 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +variables: + batch_time: '02:00' + mpi_command: 'mpiexec' + batch_submit: 'pjsub {execute_experiment}' + batch_nodes: '#PJM -L "node={n_nodes}"' + batch_ranks: '#PJM --mpi proc={n_ranks}' + batch_timeout: '#PJM -L "elapse={batch_time}:00"' + default_fujitsu_version: '4.8.1' \ No newline at end of file diff --git a/docs/tables/current-system-definitions.csv b/docs/tables/current-system-definitions.csv index 990480ba7..e891916b6 100644 --- a/docs/tables/current-system-definitions.csv +++ b/docs/tables/current-system-definitions.csv @@ -1,20 +1,20 @@ -,Sierra,AWS_PCluster_Hpc7a-zen4-EFA,HPECray-zen3-MI250X-Slingshot,Magma,Tioga,x86_64 -**site**,LLNL,,,LLNL,LLNL, -**system**,IBM-power9-V100-Infiniband,AWS_PCluster_Hpc7a-zen4-EFA,HPECray-zen3-MI250X-Slingshot,Penguin-icelake-OmniPath,HPECray-zen3-MI250X-Slingshot,x86_64 -**integrator.vendor**,IBM,AWS,HPECray,PenguinComputing,HPECray, -**integrator.name**,AC922,ParallelCluster3.7.2-Hpc7a,EX235a,RelionCluster,EX235a, -**processor.vendor**,IBM,AMD,AMD,Intel,AMD, -**processor.name**,POWER9,EPYC-Zen4,EPYC-Zen3,XeonPlatinum924248C,EPYC-Zen3, -**processor.ISA**,ppc64le,x86_64,x86_64,x86_64,x86_64,x86_64 -**processor.uArch**,power9,zen4,zen3,icelake,zen3, -**accelerator.vendor**,NVIDIA,,AMD,,AMD, -**accelerator.name**,V100,,MI250X,,MI250X, -**accelerator.ISA**,PTX,,GCN,,GCN, -**accelerator.uArch**,sm_70,,gfx90a,,gfx90a, -**interconnect.vendor**,Mellanox,AWS,HPECray,Intel,HPECray, -**interconnect.name**,EDR-Infiniband,EFA,Slingshot11,OmniPath,Slingshot11, -**system-tested.site**,LLNL,AWS,LLNL,LLNL,LLNL, -**system-tested.name**,lassen,,tioga,magma,tioga, -**system-tested.installation-year**,2018,,2022,2019,2022, -**system-tested.description**,`top500 `_,`aws/hpc7a `_,`top500 `_,`top500 `_,`top500 `_, -**top500-system-instances**,Sierra (LLNL),,"Frontier (ORNL), Lumi (CSC), Tioga (LLNL)",Magma (LLNL),"Frontier (ORNL), Lumi (CSC), Tioga (LLNL)", +,Sierra,AWS_PCluster_Hpc7a-zen4-EFA,HPECray-zen3-MI250X-Slingshot,Magma,Fugaku,Pascal,Tioga,x86_64 +**site**,LLNL,,,LLNL,RIKEN Center for Computational Science,LLNL,LLNL, +**system**,IBM-power9-V100-Infiniband,AWS_PCluster_Hpc7a-zen4-EFA,HPECray-zen3-MI250X-Slingshot,Penguin-icelake-OmniPath,Fujitsu-A64FX-TofuD,Penguin-broadwell-P100-OmniPath,HPECray-zen3-MI250X-Slingshot,x86_64 +**integrator.vendor**,IBM,AWS,HPECray,PenguinComputing,Fujitsu,Penguin,HPECray, +**integrator.name**,AC922,ParallelCluster3.7.2-Hpc7a,EX235a,RelionCluster,FX1000,,EX235a, +**processor.vendor**,IBM,AMD,AMD,Intel,Fujitsu,Intel,AMD, +**processor.name**,POWER9,EPYC-Zen4,EPYC-Zen3,XeonPlatinum924248C,A64FX,Xeon_E5-2695_v4,EPYC-Zen3, +**processor.ISA**,ppc64le,x86_64,x86_64,x86_64,Armv8.2-A-SVE,x86_64,x86_64,x86_64 +**processor.uArch**,power9,zen4,zen3,icelake,aarch64,broadwell,zen3, +**accelerator.vendor**,NVIDIA,,AMD,,,NVIDIA,AMD, +**accelerator.name**,V100,,MI250X,,,P100,MI250X, +**accelerator.ISA**,PTX,,GCN,,,PTX,GCN, +**accelerator.uArch**,sm_70,,gfx90a,,,sm_56,gfx90a, +**interconnect.vendor**,Mellanox,AWS,HPECray,Intel,Fujitsu,Cornelis,HPECray, +**interconnect.name**,EDR-Infiniband,EFA,Slingshot11,OmniPath,TofuInterconnectD,OmniPath,Slingshot11, +**system-tested.site**,LLNL,AWS,LLNL,LLNL,R-CCS,LLNL,LLNL, +**system-tested.name**,lassen,,tioga,magma,Fugaku,pascal,tioga, +**system-tested.installation-year**,2018,,2022,2019,2020,2018,2022, +**system-tested.description**,`top500 `_,`aws/hpc7a `_,`top500 `_,`top500 `_,`top500 `_,,`top500 `_, +**top500-system-instances**,Sierra (LLNL),,"Frontier (ORNL), Lumi (CSC), Tioga (LLNL)",Magma (LLNL),Fugaku (R-CCS),,"Frontier (ORNL), Lumi (CSC), Tioga (LLNL)", diff --git a/experiments/amg2023/openmp/ramble.yaml b/experiments/amg2023/openmp/ramble.yaml index 23782fe50..ddd9ac9cb 100644 --- a/experiments/amg2023/openmp/ramble.yaml +++ b/experiments/amg2023/openmp/ramble.yaml @@ -33,7 +33,7 @@ ramble: nz: '{n}' processes_per_node: ['8', '4'] n_nodes: ['1', '2'] - threads_per_node_core: ['8', '10', '13'] #TODO: Specify n_threads according to available n_nodes and n_ranks + threads_per_node_core: ['4', '6', '12'] #TODO: Specify n_threads according to available n_nodes and n_ranks omp_num_threads: '{threads_per_node_core} * {n_nodes}' experiments: amg2023_omp_problem1_{n_nodes}_{omp_num_threads}_{px}_{py}_{pz}_{nx}_{ny}_{nz}: