Nab atlas gpu field

README.md

@@ -99,7 +99,11 @@ Balthasar Reuter ([email protected])
 - **dwarf-cloudsc-fortran-atlas**: A version of **dwarf-cloudsc-fortran** which uses the [Atlas library](https://github.com/ecmwf/atlas) 
   and its Field and FieldSet data stuctures. There are two storage settings for variables. If the environment variable
   CLOUDSC_ATLAS_MULTIFIELD is "0", "OFF", or "FALSE", the variables are managed as atlas::FieldSet, which is an array of atlas::Fields. For other values of CLOUDSC_ATLAS_MULTIFIELD, a batching of variables is used as (BLK_IDX, LEV, VAR_ID, BLK_ID).
-
+- **dwarf-cloudsc-gpu-scc-atlas**: A GPU-enabled version which uses the [Atlas library](https://github.com/ecmwf/atlas) 
+  and its Field and FieldSet data stuctures with device offload capabilities. There are two storage settings for variables,
+  managed through the environment variable CLOUDSC_ATLAS_MULTIFIELD as before.
+- **dwarf-cloudsc-gpu-scc-hoist-atlas**: An optimised version of **dwarf-cloudsc-gpu-scc-atlas** which uses host side
+  hoisted temporary local variables.
 
 ## Download and Installation
 

arch/ecmwf/gh/env.sh

@@ -0,0 +1,54 @@
+#! /usr/bin/env bash
+# (C) Copyright 1988- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+# Source me to get the correct configure/build/run environment
+
+# Store tracing and disable (module is *way* too verbose)
+{ tracing_=${-//[^x]/}; set +x; } 2>/dev/null
+
+module_load() {
+  echo "+ module load $1"
+  module load $1
+}
+module_unload() {
+  echo "+ module unload $1"
+  module unload $1
+}
+
+# Unload all modules to be certain
+module purge
+
+# Load modules
+module use /opt/eb_packages/modules/all
+
+#module_load CMake/3.27.6-NVHPC-24.1-CUDA-12.3.0
+module_load Python/3.11.3-GCCcore-12.3.0
+module_load OpenMPI/4.1.6-GCC-13.2.0
+module_load NVHPC/24.1-CUDA-12.3.0
+module_load HDF5/1.14.3-NVHPC-24.1-CUDA-12.3.0
+module_load Boost/1.82.0-GCC-12.3.0
+
+export PATH=/perm/nawd/software/gh/cmake/3.30.3/bin:$PATH
+
+# module_load CMake/3.27.6-GCCcore-13.2.0
+# module_load GCC/12.3.0 
+export CC=nvc
+export CXX=nvc++
+#export CUDA_HOST_COMPILER=gcc-11
+export CUDAHOSTCXX=nvc++
+export FC=nvfortran
+export NVHPC_CUDA_HOME=/opt/eb_packages/software/NVHPC/24.1-CUDA-12.3.0/Linux_aarch64/24.1/cuda/12.3
+
+# Increase stack size to maximum
+ulimit -S -s unlimited
+
+set -x
+
+# Restore tracing to stored setting
+{ if [[ -n "$tracing_" ]]; then set -x; else set +x; fi } 2>/dev/null

arch/ecmwf/gh/toolchain.cmake

@@ -0,0 +1,68 @@
+# (C) Copyright 1988- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+####################################################################
+# COMPILER
+####################################################################
+
+set( ECBUILD_FIND_MPI ON )
+set( CMAKE_CUDA_COMPILER "nvcc" CACHE STRING "" )
+
+####################################################################
+# OpenMP FLAGS
+####################################################################
+
+# Note: OpenMP_Fortran_FLAGS gets overwritten by the FindOpenMP module
+# unless its stored as a cache variable
+set( OpenMP_Fortran_FLAGS   "-mp -mp=gpu,bind,allcores,numa" CACHE STRING "" )
+
+# Note: OpenMP_C_FLAGS and OpenMP_C_LIB_NAMES have to be provided _both_ to
+# keep FindOpenMP from overwriting the FLAGS variable (the cache entry alone
+# doesn't have any effect here as the module uses FORCE to overwrite the
+# existing value)
+#set( OpenMP_C_FLAGS         "-mp -mp=bind,allcores,numa" CACHE STRING "" )
+#set( OpenMP_C_LIB_NAMES     "acchost" CACHE STRING "")
+
+####################################################################
+# OpenAcc FLAGS
+####################################################################
+
+# Importantly, enable `gvmode` to remove the limit of 32 vector threads
+# per thread block
+set( OpenACC_Fortran_FLAGS "-acc=gpu -gpu=cc90,lineinfo,fastmath,gvmode" CACHE STRING "" )
+# Enable this to get more detailed compiler output
+# set( OpenACC_Fortran_FLAGS "${OpenACC_Fortran_FLAGS} -Minfo" )
+
+####################################################################
+# CUDA FLAGS
+####################################################################
+
+set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -allow-unsupported-compiler" CACHE STRING "")
+
+if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
+  set(CMAKE_CUDA_ARCHITECTURES 90)
+endif()
+
+####################################################################
+# COMMON FLAGS
+####################################################################
+
+set(ECBUILD_Fortran_FLAGS "-fpic")
+set(ECBUILD_Fortran_FLAGS "${ECBUILD_Fortran_FLAGS} -Mframe")
+set(ECBUILD_Fortran_FLAGS "${ECBUILD_Fortran_FLAGS} -Mbyteswapio")
+set(ECBUILD_Fortran_FLAGS "${ECBUILD_Fortran_FLAGS} -Mstack_arrays")
+set(ECBUILD_Fortran_FLAGS "${ECBUILD_Fortran_FLAGS} -Mrecursive")
+set(ECBUILD_Fortran_FLAGS "${ECBUILD_Fortran_FLAGS} -Ktrap=fp")
+set(ECBUILD_Fortran_FLAGS "${ECBUILD_Fortran_FLAGS} -Kieee")
+set(ECBUILD_Fortran_FLAGS "${ECBUILD_Fortran_FLAGS} -Mdaz")
+
+set( ECBUILD_Fortran_FLAGS_BIT "-O2 -gopt" )
+
+set( ECBUILD_C_FLAGS "-O2 -gopt -traceback" )
+
+set( ECBUILD_CXX_FLAGS "-O2 -gopt" )

arch/ecmwf/hpc2020/nvhpc/22.1/env.sh

@@ -36,6 +36,7 @@ module_load prgenv/nvidia
 module_load nvidia/22.1
 module_load hpcx-openmpi/2.10.0
 # module_load boost/1.71.0
+module_load eigen/3.4.0
 module_load hdf5/1.10.6
 module_load cmake/3.20.2
 module_load python3/3.8.8-01

arch/ecmwf/hpc2020/nvhpc/22.11/env.sh

@@ -32,12 +32,13 @@ module_unload python3
 module_unload java
 
 # Load modules
+module_load cmake/3.25.2
 module_load prgenv/nvidia
 module_load nvidia/22.11
-module_load hpcx-openmpi/2.10.0
+module_load hpcx-openmpi/2.14.0-cuda
 # module_load boost/1.71.0
+module_load eigen/3.4.0
 module_load hdf5/1.10.6
-module_load cmake/3.25.2
 module_load python3/3.10.10-01
 module_load java/11.0.6
 

arch/eurohpc/lumi/cray-gpu/17.0.1/env.sh

@@ -0,0 +1,120 @@
+# Source me to get the correct configure/build/run environment
+
+# Warning: with rocm/6.2.2 there are following liner errors for the executables. With rocm/6.0.3 they are gone
+#/opt/cray/pe/cce/17.0.1/binutils/x86_64/x86_64-pc-linux-gnu/bin/ld.bfd: ../../../lib/libatlas.so.0.39: undefined reference to `void atlas::array::ArrayView<long, 1>::assign<true, (int*)0>(long const&)'
+#/opt/cray/pe/cce/17.0.1/binutils/x86_64/x86_64-pc-linux-gnu/bin/ld.bfd: ../../../lib/libatlas.so.0.39: undefined reference to `void atlas::array::ArrayView<float, 2>::assign<true, (int*)0>(float const&)'
+#/opt/cray/pe/cce/17.0.1/binutils/x86_64/x86_64-pc-linux-gnu/bin/ld.bfd: ../../../lib/libatlas.so.0.39: undefined reference to `void atlas::array::ArrayView<float, 1>::assign<true, (int*)0>(float const&)'
+#/opt/cray/pe/cce/17.0.1/binutils/x86_64/x86_64-pc-linux-gnu/bin/ld.bfd: ../../../lib/libatlas.so.0.39: undefined reference to `void atlas::array::ArrayView<double, 1>::assign<true, (int*)0>(double const&)'
+#/opt/cray/pe/cce/17.0.1/binutils/x86_64/x86_64-pc-linux-gnu/bin/ld.bfd: ../../../lib/libatlas.so.0.39: undefined reference to `void atlas::array::ArrayView<int, 2>::assign<true, (int*)0>(atlas::array::ArrayView<int, 2> const&)'
+#/opt/cray/pe/cce/17.0.1/binutils/x86_64/x86_64-pc-linux-gnu/bin/ld.bfd: ../../../lib/libatlas.so.0.39: undefined reference to `void atlas::array::ArrayView<float, 3>::assign<true, (int*)0>(float const&)'
+#/opt/cray/pe/cce/17.0.1/binutils/x86_64/x86_64-pc-linux-gnu/bin/ld.bfd: ../../../lib/libatlas.so.0.39: undefined reference to `void atlas::array::ArrayView<double, 3>::assign<true, (int*)0>(double const&)'
+#/opt/cray/pe/cce/17.0.1/binutils/x86_64/x86_64-pc-linux-gnu/bin/ld.bfd: ../../../lib/libatlas.so.0.39: undefined reference to `void atlas::array::ArrayView<int, 3>::assign<true, (int*)0>(int const&)'
+#/opt/cray/pe/cce/17.0.1/binutils/x86_64/x86_64-pc-linux-gnu/bin/ld.bfd: ../../../lib/libatlas.so.0.39: undefined reference to `void atlas::array::ArrayView<int, 1>::assign<true, (int*)0>(atlas::array::ArrayView<int, 1> const&)'
+#/opt/cray/pe/cce/17.0.1/binutils/x86_64/x86_64-pc-linux-gnu/bin/ld.bfd: ../../../lib/libatlas.so.0.39: undefined reference to `void atlas::array::ArrayView<long, 1>::assign<true, (int*)0>(atlas::array::ArrayView<long, 1> const&)'
+#/opt/cray/pe/cce/17.0.1/binutils/x86_64/x86_64-pc-linux-gnu/bin/ld.bfd: ../../../lib/libatlas.so.0.39: undefined reference to `void atlas::array::ArrayView<double, 2>::assign<true, (int*)0>(double const&)'
+#/opt/cray/pe/cce/17.0.1/binutils/x86_64/x86_64-pc-linux-gnu/bin/ld.bfd: ../../../lib/libatlas.so.0.39: undefined reference to `void atlas::array::ArrayView<double, 2>::assign<true, (int*)0>(atlas::array::ArrayView<double, 2> const&)'
+#/opt/cray/pe/cce/17.0.1/binutils/x86_64/x86_64-pc-linux-gnu/bin/ld.bfd: ../../../lib/libatlas.so.0.39: undefined reference to `void atlas::array::ArrayView<int, 1>::assign<true, (int*)0>(int const&)'
+
+# Store tracing and disable (module is *way* too verbose)
+{ tracing_=${-//[^x]/}; set +x; } 2>/dev/null
+
+module_load() {
+  echo "+ module load $1"
+  if [ "${2:-""}" == "ECBUNDLE_CONFIGURE_ONLY" ]; then
+    if [ -n "${ECBUNDLE_CONFIGURE:-""}" ]; then
+      module load $1
+    else
+      echo " WARNING: Module $1 not loaded (only during configuration)"
+    fi
+  else
+    module load $1
+  fi
+}
+module_unload() {
+  echo "+ module unload $1"
+  module unload $1
+}
+
+# Unload to be certain
+module --force purge
+
+# Load modules
+module_load CrayEnv
+module_load PrgEnv-cray/8.5.0
+module_load cce/17.0.1
+module_load craype-x86-trento
+module_load craype-accel-amd-gfx90a
+module_load rocm/6.0.3
+module_load cray-mpich/8.1.29
+module_load craype-network-ofi
+module_load cray-python
+module_load Eigen/3.4.0
+#module_load cray-dsmml
+module_load buildtools/24.03
+#export AEC_DIR=/scratch/project_465000454/ifs-deps/libaec-1.1.2
+
+### Handling of "magic" cray modules
+# 1) Load the cray modules
+module_load cray-libsci/24.03
+module_load cray-fftw/3.3.10.7
+module_load cray-hdf5
+#module_load cray-netcdf
+# 2) Store variables to locate the packages
+export CRAY_LIBSCI=${CRAY_LIBSCI_PREFIX_DIR}/lib/libsci_cray.so
+_FFTW_ROOT=${FFTW_ROOT}
+_HDF5_ROOT=${CRAY_HDF5_PREFIX}
+#_NETCDF_ROOT=${CRAY_NETCDF_PREFIX}
+_MPI_HOME=${MPICH_DIR}
+# 3) Unload the cray modules in reverse order, removing all the magic
+#module_unload cray-netcdf
+module_unload cray-hdf5
+module_unload cray-fftw
+module_unload cray-libsci
+#module_unload cray-mpich
+# 4) Define variables that CMake introspects
+export FFTW_ROOT=${_FFTW_ROOT}
+export HDF5_ROOT=${_HDF5_ROOT}
+#export NETCDF_ROOT=${_NETCDF_ROOT}
+#export NETCDF_DIR=${_NETCDF_ROOT}
+
+# Export environment variable3s
+export MPI_HOME=${_MPI_HOME}
+#export CMAKE_TOOLCHAIN_FILE=$PWD/toolchain.cmake
+export CC=cc
+export CXX=CC
+export FC=ftn
+export HIPCXX=$(hipconfig --hipclangpath)/clang++
+export CRAY_ADD_RPATH=yes
+export LIBSCI_ARCH_OVERRIDE=broadwell
+  # This is required to work around SIGSEGV in ectrans' SGEMM calls, which
+  # occur when "rome" or "milan" are used (backtrace points to openblas_sgemm__naples)
+
+export LDFLAGS="-fuse-ld=bfd"
+
+#. env-rocm.sh
+
+export LD_LIBRARY_PATH=$CRAY_LD_LIBRARY_PATH:$LD_LIBRARY_PATH
+
+
+
+### Print some exported variables
+echo "+ export FFTW_ROOT=${FFTW_ROOT}"
+#echo "+ export HDF5_ROOT=${HDF5_ROOT}"
+#echo "+ export NETCDF_ROOT=${NETCDF_ROOT}"
+echo "+ export MPI_HOME=${MPI_HOME}"
+echo "+ export CMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}"
+echo "+ export CC=${CC}"
+echo "+ export CXX=${CXX}"
+echo "+ export FC=${FC}"
+echo "+ export HIPCXX=${HIPCXX}"
+echo "+ export CRAY_ADD_RPATH=${CRAY_ADD_RPATH}"
+echo "+ export LIBSCI_ARCH_OVERRIDE=${LIBSCI_ARCH_OVERRIDE}"
+
+module list 2>&1
+
+set -x
+ulimit -S -s unlimited
+
+# Restore tracing to stored setting
+{ if [[ -n "$tracing_" ]]; then set -x; else set +x; fi } 2>/dev/null
+

arch/eurohpc/lumi/cray-gpu/17.0.1/toolchain.cmake

@@ -0,0 +1,46 @@
+####################################################################
+# HIP 
+####################################################################
+
+set(CMAKE_HIP_ARCHITECTURES gfx90a)
+
+####################################################################
+# OpenMP
+####################################################################
+
+set( OpenMP_C_FLAGS           "-fopenmp" )
+set( OpenMP_CXX_FLAGS         "-fopenmp" )
+set( OpenMP_Fortran_FLAGS     "-fopenmp" )
+set( OpenMP_C_LIB_NAMES       "craymp" )
+set( OpenMP_CXX_LIB_NAMES     "craymp" )
+set( OpenMP_Fortran_LIB_NAMES "craymp" )
+set( OpenMP_craymp_LIBRARY    "craymp" )
+
+####################################################################
+# OpenACC
+####################################################################
+
+set( OpenACC_C_FLAGS       "-hacc" )
+set( OpenACC_CXX_FLAGS     "-hacc" )
+#set( OpenACC_Fortran_FLAGS "-hacc -hacc_model=auto_async_kernel:no_fast_addr:deep_copy" )
+set( OpenACC_Fortran_FLAGS "-hacc" )
+
+####################################################################
+# General Flags (add to default)
+####################################################################
+
+set( ECBUILD_Fortran_FLAGS "${ECBUILD_Fortran_FLAGS} -M2260" )
+
+###################################################################
+# Libraries
+###################################################################
+
+set( BLAS_LIBRARIES   "$ENV{CRAY_LIBSCI}" CACHE STRING "BLAS_LIBRARIES" FORCE )
+set( LAPACK_LIBRARIES "$ENV{CRAY_LIBSCI}" CACHE STRING "LAPACK_LIBRARIES" FORCE )
+
+#set ( CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -craype-verbose" )
+#set ( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -craype-verbose -fopenmp" )
+#set ( CMAKE_EXE_LINKER_FLAGS  "${CMAKE_EXE_LINKER_FLAGS} -Wl, --as-needed" )
+#set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=bfd")
+#-set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-relax -lhugetlbfs")
+#set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-relax -lhugetlbfs -fuse-ld=bfd -craype-verbose")

arch/toolchains/ecmwf-hpc2020-nvhpc.cmake

@@ -36,6 +36,8 @@ set( OpenMP_C_LIB_NAMES     "acchost" CACHE STRING "")
 # NB: We have to add `-mp` again to avoid undefined symbols during linking
 # (smells like an Nvidia bug)
 set( OpenACC_Fortran_FLAGS "-acc=gpu -mp=gpu -gpu=cc80,lineinfo,fastmath,gvmode" CACHE STRING "" )
+set( OpenACC_C_FLAGS "${OpenACC_Fortran_FLAGS}")
+
 # Enable this to get more detailed compiler output
 # set( OpenACC_Fortran_FLAGS "${OpenACC_Fortran_FLAGS} -Minfo" )
 

bundle.yml

@@ -18,7 +18,7 @@ projects :
 
     - ecbuild :
         git     : https://github.com/ecmwf/ecbuild
-        version : 3.8.0
+        version : master
         bundle  : false
 
     - serialbox :
@@ -43,7 +43,7 @@ projects :
 
     - eckit :
         git     : https://github.com/ecmwf/eckit
-        version : 1.24.4
+        version : master
         require : ecbuild
         cmake   : >
             ECKIT_ENABLE_TESTS=OFF
@@ -53,20 +53,22 @@ projects :
     - field_api :
         git     : https://github.com/ecmwf-ifs/field_api.git
         version : v0.3.0
+        optional: true
         require : ecbuild
         cmake   : >
             UTIL_MODULE_PATH=${CMAKE_SOURCE_DIR}/cloudsc-dwarf/src/common/module
 
     - fckit :
         git     : https://github.com/ecmwf/fckit
-        version : 0.11.0
+        version : master
         require : ecbuild eckit
         cmake   : >
             FCKIT_ENABLE_TESTS=OFF
 
     - atlas :
         git     : https://github.com/ecmwf/atlas
-        version : feature/MultiField
+        version : feature/pluto
+        optional: true
         require : ecbuild eckit fckit
         cmake   : >
             ATLAS_ENABLE_TESTS=OFF
@@ -99,10 +101,13 @@ options :
             ENABLE_CLOUDSC_GPU_SCC_HOIST=ON
             ENABLE_CLOUDSC_GPU_SCC_K_CACHING=ON
             ENABLE_CLOUDSC_GPU_OMP_SCC_HOIST=ON
+            ATLAS_ENABLE_ACC=ON
+            ATLAS_ENABLE_CUDA=ON
 
     - with-cuda :
         help  : Enable GPU kernel variants based on CUDA and CUDA-Fortran
         cmake : >
+            ATLAS_ENABLE_CUDA=ON
             ENABLE_CUDA=ON
             ENABLE_CLOUDSC_GPU_SCC_CUF=ON
             ENABLE_CLOUDSC_GPU_SCC_CUF_K_CACHING=ON