sync develop and main

.gitlab/build_and_bench.sh

@@ -1,83 +1,36 @@
 #!/usr/bin/env bash
 
+# variables
+
 # Build and test first
-source "$BUILD_ROOT/.gitlab/build_and_test.sh" --no-clean
 
-benchmark_dir="${build_dir}/benchmarks"
-ar3d_dir="${benchmark_dir}/advection_reaction_3D"
-d2d_dir="${benchmark_dir}/diffusion_2D"
+source "$BUILD_ROOT/.gitlab/build_and_test.sh" --no-clean
 
+# Embed this into `make benchmark`
 nresg=$((BENCHMARK_NNODES * 4)) # Lassen has 4 GPUs per node
 nresc=$((BENCHMARK_NNODES * 40)) # Lassen has 40 cores per node
 
 echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
 echo "~~~~~ Benchmarking SUNDIALS"
 echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
 
-if [[ ! -d ${benchmark_dir} ]]
-then
-    echo "ERROR: Benchmark directory not found : ${benchmark_dir}" && exit 1
-fi
-
-cd "${benchmark_dir}"
-
 # We use synchronous kernel launches for benchmarking
 # because we are interested in individual region timings.
 export CUDA_LAUNCH_BLOCKING=1
 
-if [[ -d ${ar3d_dir} ]]
-then
-    date
-
-    if [[ -f ${ar3d_dir}/advection_reaction_3D_mpicuda ]]
-    then
-        export CALI_CONFIG="spot(output=${BUILD_ROOT}/ar3d_arkimex_tlnewton.cali),runtime-report(calc.inclusive)"
-        jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${ar3d_dir}/advection_reaction_3D_mpicuda" --method ARK-IMEX --nls tl-newton --tf 10.0 --dont-save
-        export CALI_CONFIG="spot(output=${BUILD_ROOT}/ar3d_arkdirk_newton.cali),runtime-report(calc.inclusive)"
-        jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${ar3d_dir}/advection_reaction_3D_mpicuda" --method ARK-DIRK --nls newton --tf 10.0 --dont-save
-        export CALI_CONFIG="spot(output=${BUILD_ROOT}/ar3d_cvbdf_newton.cali),runtime-report(calc.inclusive)"
-        jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${ar3d_dir}/advection_reaction_3D_mpicuda" --method CV-BDF --nls newton --tf 10.0 --dont-save
-        export CALI_CONFIG="spot(output=${BUILD_ROOT}/ar3d_ida_newton.cali),runtime-report(calc.inclusive)"
-        jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${ar3d_dir}/advection_reaction_3D_mpicuda" --method IDA --nls newton --tf 10.0 --dont-save
-    elif [[ -f ${ar3d_dir}/advection_reaction_3D ]]
-    then
-        export CALI_CONFIG="spot(output=${BUILD_ROOT}/ar3d_mpionly_arkimex_tlnewton.cali),runtime-report(calc.inclusive)"
-        jsrun -n${nresc} -a1 -c1 "${ar3d_dir}/advection_reaction_3D" --method ARK-IMEX --nls tl-newton --tf 2.0 --dont-save
-        export CALI_CONFIG="spot(output=${BUILD_ROOT}/ar3d_mpionly_arkdirk_newton.cali),runtime-report(calc.inclusive)"
-        jsrun -n${nresc} -a1 -c1 "${ar3d_dir}/advection_reaction_3D" --method ARK-DIRK --nls newton --tf 2.0 --dont-save
-        export CALI_CONFIG="spot(output=${BUILD_ROOT}/ar3d_mpionly_cvbdf_newton.cali),runtime-report(calc.inclusive)"
-        jsrun -n${nresc} -a1 -c1 "${ar3d_dir}/advection_reaction_3D" --method CV-BDF --nls newton --tf 2.0 --dont-save
-        export CALI_CONFIG="spot(output=${BUILD_ROOT}/ar3d_mpionly_ida_newton.cali),runtime-report(calc.inclusive)"
-        jsrun -n${nresc} -a1 -c1 "${ar3d_dir}/advection_reaction_3D" --method IDA --nls newton --tf 2.0 --dont-save
-    fi
-
-    date
-fi
-
-
-if [[ -d ${d2d_dir} ]]
-then
-    date
+if [[ -d ${build_dir} ]]
+then 
 
-    if [[ -d ${d2d_dir}/mpi_gpu ]]
-    then
-        export CALI_CONFIG="spot(output=${BUILD_ROOT}/d2d_arkode.cali),runtime-report(calc.inclusive)"
-        jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${d2d_dir}/mpi_gpu/arkode_diffusion_2D_mpicuda"
-        export CALI_CONFIG="spot(output=${BUILD_ROOT}/d2d_cvode.cali),runtime-report(calc.inclusive)"
-        jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${d2d_dir}/mpi_gpu/cvode_diffusion_2D_mpicuda"
-        export CALI_CONFIG="spot(output=${BUILD_ROOT}/d2d_ida.cali),runtime-report(calc.inclusive)"
-        jsrun --smpiargs="-gpu" -n${nresg} -a1 -c1 -g1 "${d2d_dir}/mpi_gpu/ida_diffusion_2D_mpicuda"
-    elif [[ -d ${d2d_dir}/mpi_serial ]]
-    then
-        export CALI_CONFIG="spot(output=${BUILD_ROOT}/d2d_arkode.cali),runtime-report(calc.inclusive)"
-        jsrun -n${nresc} -a1 -c1 "${d2d_dir}/mpi_serial/arkode_diffusion_2D"
-        export CALI_CONFIG="spot(output=${BUILD_ROOT}/d2d_cvode.cali),runtime-report(calc.inclusive)"
-        jsrun -n${nresc} -a1 -c1 "${d2d_dir}/mpi_serial/arkode_diffusion_2D"
-        export CALI_CONFIG="spot(output=${BUILD_ROOT}/d2d_ida.cali),runtime-report(calc.inclusive)"
-        jsrun -n${nresc} -a1 -c1 "${d2d_dir}/mpi_serial/ida_diffusion_2D"
-    fi
+    # configure for multinode
+    $cmake_exe \
+        -C "${hostconfig_path}" \
+        -DSUNDIALS_BENCHMARK_NUM_CPUS=${nresc} \
+        -DSUNDIALS_BENCHMARK_NUM_GPUS=${nresg} \
+        "${project_dir}"
+
+    cd ${build_dir}
+    make benchmark
 
-    date
 fi
 
 echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"

.gitlab/build_and_test.sh

@@ -155,9 +155,9 @@ fi
 # Build Directory
 if [[ -z ${build_root} ]]
 then
-    build_root="/dev/shm$(pwd)"
+    build_root="$(pwd)"
 else
-    build_root="/dev/shm${build_root}"
+    build_root="${build_root}"
 fi
 
 build_dir="${build_root}/build_${job_unique_id}_${hostconfig//.cmake/}"
@@ -192,7 +192,7 @@ then
         -C "${hostconfig_path}" \
         -DCMAKE_INSTALL_PREFIX=${install_dir} \
         "${project_dir}"
-
+    
     # build
     VERBOSE_BUILD=${VERBOSE_BUILD:-"OFF"}
     if [[ "${VERBOSE_BUILD}" == "ON" ]]; then

.gitlab/lassen-jobs.yml

@@ -39,6 +39,6 @@ lassen_gcc_cuda_bench:
       - COMPILER_SPEC: [email protected]
         CUDA_SPEC: [[email protected]]
   variables:
-    SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 build_type=Release precision=double ~int64 +benchmarks+profiling+caliper +mpi+openmp+cuda+raja cuda_arch=70 ^raja+cuda~openmp~examples~exercises cuda_arch=70 ^caliper+cuda cuda_arch=70 ^${CUDA_SPEC}"
+    SPEC: "%${COMPILER_SPEC} cstd=99 cxxstd=14 build_type=Release precision=double scheduler=lsf caliper-dir=/usr/workspace/sundials/califiles ~int64 +benchmarks+profiling+caliper+adiak+mpi+openmp+cuda+raja cuda_arch=70 ^raja+cuda~openmp~examples~exercises cuda_arch=70 ^caliper+adiak+cuda cuda_arch=70 ^${CUDA_SPEC}+allow-unsupported-compilers"
   extends: .lassen_build_and_bench
 

.gitlab/spack_packages/sundials/package.py

@@ -120,6 +120,7 @@ class Sundials(CachedCMakePackage, CudaPackage, ROCmPackage):
     variant("sycl", default=False, when="@5.7.0:", description="Enable SYCL vector")
 
     # External libraries
+    variant("adiak", default=False, when="@6.6.0:", description="Enable Adiak interfaces")
     variant(
         "caliper",
         default=False,
@@ -181,7 +182,25 @@ class Sundials(CachedCMakePackage, CudaPackage, ROCmPackage):
     )
 
     # Scheduler
-    variant("scheduler", default="default", description="Specify which scheduler the system runs on.", values=("flux", "lsf", "slurm", "default"))
+    variant(
+        "scheduler", 
+        default="slurm", 
+        description="Specify which scheduler the system runs on", 
+        values=("flux", "lsf", "slurm")
+    )
+
+    # Benchmarking
+    variant("benchmarks", default=False, description="Build benchmark programs")
+
+    # Profiling examples
+    variant(
+        "profile-examples", 
+        default=False, 
+        when="+adiak +caliper", 
+        description="Build examples with profiling capabilities")
+
+    # Caliper Directory
+    variant("caliper-dir", default="none", description="Specify where to place Caliper files")
 
     # ==========================================================================
     # Dependencies
@@ -203,6 +222,7 @@ class Sundials(CachedCMakePackage, CudaPackage, ROCmPackage):
     depends_on("raja+rocm", when="+raja +rocm")
 
     # External libraries
+    depends_on("adiak", when="+adiak")
     depends_on("caliper", when="+caliper")
     depends_on("[email protected]:", when="+ginkgo")
     depends_on("kokkos", when="+kokkos")
@@ -672,11 +692,11 @@ def initconfig_mpi_entries(self):
                 ]
             )
             if "scheduler=flux" in spec:
-                entries.append(cmake_cache_string("SUNDIALS_TEST_MPIRUN_COMMAND", "flux run"))
+                entries.append(cmake_cache_string("SUNDIALS_SCHEDULER_COMMAND", "flux run"))
             if "scheduler=slurm" in spec:
-                entries.append(cmake_cache_string("SUNDIALS_TEST_MPIRUN_COMMAND", "srun"))
+                entries.append(cmake_cache_string("SUNDIALS_SCHEDULER_COMMAND", "srun"))
             if "scheduler=lsf" in spec:
-                entries.append(cmake_cache_string("SUNDIALS_TEST_MPIRUN_COMMAND", "jsrun"))
+                entries.append(cmake_cache_string("SUNDIALS_SCHEDULER_COMMAND", "jsrun"))
 
 
         return entries
@@ -736,7 +756,15 @@ def initconfig_package_entries(self):
                 self.cache_option_from_variant("SUNDIALS_BUILD_WITH_MONITORING", "monitoring"),
                 # Profiling
                 self.cache_option_from_variant("SUNDIALS_BUILD_WITH_PROFILING", "profiling"),
-                self.cache_option_from_variant("ENABLE_CALIPER", "caliper")
+                self.cache_option_from_variant("ENABLE_CALIPER", "caliper"),
+                self.cache_option_from_variant("ENABLE_ADIAK", "adiak"),
+                # Benchmarking
+                self.cache_option_from_variant("BUILD_BENCHMARKS", "benchmarks"),
+                # Profile examples
+                self.cache_option_from_variant("SUNDIALS_TEST_PROFILE", "profile-examples"),
+                self.cache_option_from_variant("SUNDIALS_TEST_DEVTESTS", "profile-examples"),
+                cmake_cache_string("SPACK_VERSION", ".".join(map(str, spack.spack_version_info)))
+
             ]
         )
 
@@ -753,6 +781,9 @@ def initconfig_package_entries(self):
         # TPLs
         entries.extend(
             [
+                self.cache_option_from_variant("ENABLE_GINKGO", "ginkgo"),
+                self.cache_option_from_variant("ENABLE_KOKKOS_KERNELS", "kokkos-kernels"),
+                self.cache_option_from_variant("ENABLE_KOKKOS", "kokkos"),
                 self.cache_option_from_variant("ENABLE_SYCL", "sycl"),
                 self.cache_option_from_variant("EXAMPLES_INSTALL", "examples-install"),
                 self.cache_option_from_variant("HYPRE_ENABLE", "hypre"),
@@ -764,15 +795,23 @@ def initconfig_package_entries(self):
                 self.cache_option_from_variant("RAJA_ENABLE", "raja"),
                 self.cache_option_from_variant("SUPERLUDIST_ENABLE", "superlu-dist"),
                 self.cache_option_from_variant("SUPERLUMT_ENABLE", "superlu-mt"),
-                self.cache_option_from_variant("Trilinos_ENABLE", "trilinos"),
-                self.cache_option_from_variant("ENABLE_KOKKOS", "kokkos"),
-                self.cache_option_from_variant("ENABLE_KOKKOS_KERNELS", "kokkos-kernels")
+                self.cache_option_from_variant("Trilinos_ENABLE", "trilinos")
             ]
         )
 
+        # Building with Adiak
+        if "+adiak" in spec: 
+            entries.append(cmake_cache_path("adiak_DIR", spec["adiak"].prefix.lib.cmake + "/adiak"))
+
         # Building with Caliper
         if "+caliper" in spec:
             entries.append(cmake_cache_path("CALIPER_DIR", spec["caliper"].prefix))
+            if "+adiak" in spec["caliper"]:
+                entries.append(cmake_cache_path("adiak_DIR", spec["adiak"].prefix.lib.cmake + "/adiak"))
+
+            if not "caliper-dir=none" in spec:
+                entries.append(self.cache_string_from_variant("SUNDIALS_CALIPER_OUTPUT_DIR", "caliper-dir"))
+
 
         # Building with Ginkgo
         if "+ginkgo" in spec:
@@ -787,7 +826,6 @@ def initconfig_package_entries(self):
                 gko_backends.append("DPCPP")
             entries.extend(
                 [
-                    self.cache_option_from_variant("ENABLE_GINKGO", "ginkgo"),
                     cmake_cache_path("Ginkgo_DIR", spec["ginkgo"].prefix),
                     cmake_cache_string("SUNDIALS_GINKGO_BACKENDS", ";".join(gko_backends)),
                 ]
@@ -928,5 +966,4 @@ def initconfig_package_entries(self):
                     cmake_cache_option("F90_ENABLE", "+examples+fcmix" in spec),
                 ]
             )
-
         return entries

CHANGELOG.md

@@ -1,5 +1,17 @@
 # SUNDIALS Changelog
 
+## Changes to SUNDIALS in release 6.6.1
+
+Updated the Tpetra NVector interface to support Trilinos 14.
+
+Fixed a memory leak when destroying a CUDA, HIP, SYCL, or system SUNMemoryHelper
+object.
+
+Fixed a bug in ARKODE, CVODE, CVODES, IDA, and IDAS where the stop time may not
+be cleared when using normal mode if the requested output time is the same as
+the stop time. Additionally, with ARKODE, CVODE, and CVODES an unnecessary
+interpolation of the solution at the stop time may occur in this case.
+
 ## Changes to SUNDIALS in release 6.6.0
 
 A new time-stepping module, `SPRKStep`, was added to ARKODE. This time-stepper

CITATIONS.md

@@ -66,7 +66,7 @@ they are using rather than the combined SUNDIALS online guide:
   author = {Daniel R. Reynolds and David J. Gardner and Carol S. Woodward and Cody J. Balos},
   title  = {User Documentation for ARKODE},
   year   = {2023},
-  note   = {v5.6.0}
+  note   = {v5.6.1}
 }
 ```
 
@@ -75,7 +75,7 @@ they are using rather than the combined SUNDIALS online guide:
   author = {Alan C. Hindmarsh and Radu Serban and Cody J. Balos and David J. Gardner and Daniel R. Reynolds and Carol S. Woodward},
   title  = {User Documentation for CVODE},
   year   = {2023},
-  note   = {v6.6.0}
+  note   = {v6.6.1}
 }
 ```
 
@@ -84,7 +84,7 @@ they are using rather than the combined SUNDIALS online guide:
   author = {Alan C. Hindmarsh and Radu Serban and Cody J. Balos and David J. Gardner and Daniel R. Reynolds and Carol S. Woodward},
   title  = {User Documentation for CVODES},
   year   = {2023},
-  note   = {v6.6.0}
+  note   = {v6.6.1}
 }
 ```
 
@@ -93,7 +93,7 @@ they are using rather than the combined SUNDIALS online guide:
   author = {Alan C. Hindmarsh and Radu Serban and Cody J. Balos and David J. Gardner and Daniel R. Reynolds and Carol S. Woodward},
   title  = {User Documentation for IDA},
   year   = {2023},
-  note   = {v6.6.0}
+  note   = {v6.6.1}
 }
 ```
 
@@ -102,7 +102,7 @@ they are using rather than the combined SUNDIALS online guide:
   author = {Radu Serban and Cosmin Petra and Alan C. Hindmarsh and Cody J. Balos and David J. Gardner and Daniel R. Reynolds and Carol S. Woodward},
   title  = {User Documentation for IDAS},
   year   = {2023},
-  note   = {v5.6.0}
+  note   = {v5.6.1}
 }
 ```
 
@@ -111,6 +111,6 @@ they are using rather than the combined SUNDIALS online guide:
   author = {Alan C. Hindmarsh and Radu Serban and Cody J. Balos and David J. Gardner and Daniel R. Reynolds and Carol S. Woodward},
   title  = {User Documentation for KINSOL},
   year   = {2023},
-  note   = {v6.6.0}
+  note   = {v6.6.1}
 }
 ```

CMakeLists.txt

@@ -42,7 +42,7 @@ include(FindPackageHandleStandardArgs)
 # Set some variables with info on the SUNDIALS project
 set(PACKAGE_BUGREPORT "[email protected]")
 set(PACKAGE_NAME "SUNDIALS")
-set(PACKAGE_STRING "SUNDIALS 6.6.0")
+set(PACKAGE_STRING "SUNDIALS 6.6.1")
 set(PACKAGE_TARNAME "sundials")
 
 # Set SUNDIALS version numbers
@@ -52,7 +52,7 @@ message(STATUS "SUNDIALS_GIT_VERSION: ${SUNDIALS_GIT_VERSION}")
 # (use "" for the version label if none is needed)
 set(PACKAGE_VERSION_MAJOR "6")
 set(PACKAGE_VERSION_MINOR "6")
-set(PACKAGE_VERSION_PATCH "0")
+set(PACKAGE_VERSION_PATCH "1")
 set(PACKAGE_VERSION_LABEL "")
 
 if(PACKAGE_VERSION_LABEL)
@@ -67,37 +67,37 @@ endif()
 
 # Specify the VERSION and SOVERSION for shared libraries
 
-set(arkodelib_VERSION "5.6.0")
+set(arkodelib_VERSION "5.6.1")
 set(arkodelib_SOVERSION "5")
 
-set(cvodelib_VERSION "6.6.0")
+set(cvodelib_VERSION "6.6.1")
 set(cvodelib_SOVERSION "6")
 
-set(cvodeslib_VERSION "6.6.0")
+set(cvodeslib_VERSION "6.6.1")
 set(cvodeslib_SOVERSION "6")
 
-set(idalib_VERSION "6.6.0")
+set(idalib_VERSION "6.6.1")
 set(idalib_SOVERSION "6")
 
-set(idaslib_VERSION "5.6.0")
+set(idaslib_VERSION "5.6.1")
 set(idaslib_SOVERSION "5")
 
-set(kinsollib_VERSION "6.6.0")
+set(kinsollib_VERSION "6.6.1")
 set(kinsollib_SOVERSION "6")
 
 set(cpodeslib_VERSION "0.0.0")
 set(cpodeslib_SOVERSION "0")
 
-set(nveclib_VERSION "6.6.0")
+set(nveclib_VERSION "6.6.1")
 set(nveclib_SOVERSION "6")
 
-set(sunmatrixlib_VERSION "4.6.0")
+set(sunmatrixlib_VERSION "4.6.1")
 set(sunmatrixlib_SOVERSION "4")
 
-set(sunlinsollib_VERSION "4.6.0")
+set(sunlinsollib_VERSION "4.6.1")
 set(sunlinsollib_SOVERSION "4")
 
-set(sunnonlinsollib_VERSION "3.6.0")
+set(sunnonlinsollib_VERSION "3.6.1")
 set(sunnonlinsollib_SOVERSION "3")
 
 set(sundialslib_VERSION

README.md

@@ -1,5 +1,5 @@
 # SUNDIALS: SUite of Nonlinear and DIfferential/ALgebraic equation Solvers #
-### Version 6.6.0 (Jul 2023) ###
+### Version 6.6.1 (Sep 2023) ###
 
 **Center for Applied Scientific Computing, Lawrence Livermore National Laboratory**