Merge branch 'development' into implicit_add_filtering_redo

.azure-pipelines.yml

@@ -9,36 +9,41 @@ pr:
 
 jobs:
 - job:
+  # FIXME remove unused variables
   variables:
     BLASPP_HOME: '/usr/local'
     CEI_SUDO: 'sudo'
     CEI_TMP: '/tmp/cei'
     CMAKE_GENERATOR: 'Ninja'
     FFTW_HOME: '/usr'
     LAPACKPP_HOME: '/usr/local'
-    OMP_NUM_THREADS: 1
     WARPX_CI_CCACHE: 'TRUE'
-    WARPX_CI_NUM_MAKE_JOBS: 2
-    WARPX_CI_OPENPMD: 'TRUE'
-    WARPX_CI_TMP: '/tmp/ci'
+    #WARPX_OPENPMD: 'TRUE'
 
   strategy:
     matrix:
-      cartesian1d:
-        WARPX_CI_REGULAR_CARTESIAN_1D: 'TRUE'
-        WARPX_CI_PSATD: 'FALSE'
-      cartesian2d:
-        WARPX_CI_REGULAR_CARTESIAN_2D: 'TRUE'
-      cartesian3d:
-        WARPX_CI_REGULAR_CARTESIAN_3D: 'TRUE'
-      single_precision:
-        WARPX_CI_SINGLE_PRECISION: 'TRUE'
-      rz_or_nompi:
-        WARPX_CI_RZ_OR_NOMPI: 'TRUE'
-      qed:
-        WARPX_CI_QED: 'TRUE'
-      embedded_boundary:
-        WARPX_CI_EB: 'TRUE'
+      # Cartesian 1D
+      cartesian_1d:
+        WARPX_CMAKE_FLAGS: -DWarpX_DIMS=1 -DWarpX_FFT=ON -DWarpX_PYTHON=ON
+      # Cartesian 2D
+      cartesian_2d:
+        WARPX_CMAKE_FLAGS: -DWarpX_DIMS=2 -DWarpX_FFT=ON -DWarpX_PYTHON=ON
+      # Cartesian 3D
+      cartesian_3d:
+        WARPX_CMAKE_FLAGS: -DWarpX_DIMS=3 -DWarpX_FFT=ON -DWarpX_PYTHON=ON
+        WARPX_HEFFTE: 'TRUE'
+      # Cylindrical RZ
+      cylindrical_rz:
+        WARPX_CMAKE_FLAGS: -DWarpX_DIMS=RZ -DWarpX_FFT=ON -DWarpX_PYTHON=ON
+        WARPX_RZ_FFT: 'TRUE'
+      # embedded boundaries
+      embedded_boundaries:
+        WARPX_CMAKE_FLAGS: -DWarpX_DIMS='1;2;3;RZ' -DWarpX_FFT=ON -DWarpX_PYTHON=ON -DWarpX_EB=ON
+        WARPX_RZ_FFT: 'TRUE'
+      # single precision
+      #single_precision:
+      #  WARPX_CMAKE_FLAGS: -DWarpX_DIMS='1;2;3;RZ' -DWarpX_FFT=ON -DWarpX_PYTHON=ON -DWarpX_PRECISION=SINGLE
+      #  WARPX_RZ_FFT: 'TRUE'
 
   # default: 60; maximum: 360
   timeoutInMinutes: 240
@@ -50,9 +55,8 @@ jobs:
   - task: Cache@2
     continueOnError: true
     inputs:
-      key: 'Ccache | "$(System.JobName)" | .azure-pipelines.yml | cmake/dependencies/AMReX.cmake | run_test.sh'
+      key: 'Ccache | "$(System.JobName)" | .azure-pipelines.yml | cmake/dependencies/AMReX.cmake'
       restoreKeys: |
-         Ccache | "$(System.JobName)" | .azure-pipelines.yml | cmake/dependencies/AMReX.cmake | run_test.sh
          Ccache | "$(System.JobName)" | .azure-pipelines.yml | cmake/dependencies/AMReX.cmake
          Ccache | "$(System.JobName)" | .azure-pipelines.yml
       path: /home/vsts/.ccache
@@ -62,9 +66,8 @@ jobs:
   - task: Cache@2
     continueOnError: true
     inputs:
-      key: 'Python3 | "$(System.JobName)" | .azure-pipelines.yml | run_test.sh'
+      key: 'Python3 | "$(System.JobName)" | .azure-pipelines.yml'
       restoreKeys: |
-         Python3 | "$(System.JobName)" | .azure-pipelines.yml | run_test.sh
          Python3 | "$(System.JobName)" | .azure-pipelines.yml
       path: /home/vsts/.local/lib/python3.8
       cacheHitVar: PYTHON38_CACHE_RESTORED
@@ -82,6 +85,8 @@ jobs:
         python3 python3-pandas python3-pip python3-venv python3-setuptools libblas-dev liblapack-dev
       ccache --set-config=max_size=10.0G
       python3 -m pip install --upgrade pip
+      python3 -m pip install --upgrade build
+      python3 -m pip install --upgrade packaging
       python3 -m pip install --upgrade setuptools
       python3 -m pip install --upgrade wheel
       python3 -m pip install --upgrade virtualenv
@@ -91,32 +96,72 @@ jobs:
       export PATH="$HOME/.local/bin:$PATH"
       sudo curl -L -o /usr/local/bin/cmake-easyinstall https://raw.githubusercontent.com/ax3l/cmake-easyinstall/main/cmake-easyinstall
       sudo chmod a+x /usr/local/bin/cmake-easyinstall
-      if [ "${WARPX_CI_OPENPMD:-FALSE}" == "TRUE" ]; then
-        cmake-easyinstall --prefix=/usr/local                   \
-          git+https://github.com/openPMD/[email protected] \
-          -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache)         \
-          -DCMAKE_VERBOSE_MAKEFILE=ON                           \
-          -DopenPMD_USE_PYTHON=OFF -DBUILD_TESTING=OFF -DBUILD_EXAMPLES=OFF -DBUILD_CLI_TOOLS=OFF
-        python3 -m pip install --upgrade openpmd-api
-      fi
-      if [[ "${WARPX_CI_RZ_OR_NOMPI:-FALSE}" == "TRUE" ]]; then
-        cmake-easyinstall --prefix=/usr/local git+https://github.com/icl-utk-edu/blaspp.git \
+      #if [ "${WARPX_OPENPMD:-FALSE}" == "TRUE" ]; then
+      #  cmake-easyinstall --prefix=/usr/local                   \
+      #    git+https://github.com/openPMD/[email protected] \
+      #    -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache)         \
+      #    -DCMAKE_VERBOSE_MAKEFILE=ON                           \
+      #    -DopenPMD_USE_PYTHON=OFF -DBUILD_TESTING=OFF -DBUILD_EXAMPLES=OFF -DBUILD_CLI_TOOLS=OFF
+      #  #python3 -m pip install --upgrade openpmd-api
+      #fi
+      if [ "${WARPX_RZ_FFT:-FALSE}" == "TRUE" ]; then
+        # BLAS++
+        cmake-easyinstall --prefix=/usr/local           \
+          git+https://github.com/icl-utk-edu/blaspp.git \
           -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \
           -DCMAKE_CXX_STANDARD=17                       \
           -Duse_openmp=OFF -Dbuild_tests=OFF -DCMAKE_VERBOSE_MAKEFILE=ON
-        cmake-easyinstall --prefix=/usr/local git+https://github.com/icl-utk-edu/lapackpp.git \
-          -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \
-          -DCMAKE_CXX_STANDARD=17                       \
+        # LAPACK++
+        cmake-easyinstall --prefix=/usr/local             \
+          git+https://github.com/icl-utk-edu/lapackpp.git \
+          -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache)   \
+          -DCMAKE_CXX_STANDARD=17                         \
           -Duse_cmake_find_lapack=ON -Dbuild_tests=OFF -DCMAKE_VERBOSE_MAKEFILE=ON
       fi
+      if [ "${WARPX_HEFFTE:-FALSE}" == "TRUE" ]; then
+        cmake-easyinstall --prefix=/usr/local git+https://github.com/icl-utk-edu/[email protected] \
+          -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache)          \
+          -DCMAKE_CXX_STANDARD=17 -DHeffte_ENABLE_DOXYGEN=OFF    \
+          -DHeffte_ENABLE_FFTW=ON -DHeffte_ENABLE_TESTING=OFF    \
+          -DHeffte_ENABLE_CUDA=OFF -DHeffte_ENABLE_ROCM=OFF      \
+          -DHeffte_ENABLE_ONEAPI=OFF -DHeffte_ENABLE_MKL=OFF     \
+          -DHeffte_ENABLE_PYTHON=OFF -DHeffte_ENABLE_FORTRAN=OFF \
+          -DHeffte_ENABLE_MAGMA=OFF                              \
+          -DCMAKE_VERBOSE_MAKEFILE=ON
+      fi
+      # Python modules required for test analysis
+      python3 -m pip install --upgrade -r Regression/requirements.txt
+      python3 -m pip cache purge
+      # external repositories required for test analysis
+      cd ..
+      git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git
+      # TODO select only specific datasets?
+      git clone --depth 1 https://github.com/openPMD/openPMD-example-datasets.git
+      cd -
       rm -rf ${CEI_TMP}
       df -h
     displayName: 'Install dependencies'
 
   - bash: |
       set -eu -o pipefail
       df -h
-      ./run_test.sh
-      rm -rf ${WARPX_CI_TMP}
+
+      # configure
+      export AMReX_CMAKE_FLAGS="-DAMReX_ASSERTIONS=ON -DAMReX_TESTING=ON"
+      cmake -S . -B build       \
+        ${AMReX_CMAKE_FLAGS}    \
+        ${WARPX_CMAKE_FLAGS}    \
+        -DWarpX_TEST_CLEANUP=ON \
+        -DWarpX_TEST_FPETRAP=ON
+
+      # build
+      cmake --build build -j 2
       df -h
-    displayName: 'Build & test'
+    displayName: 'Build'
+
+  - bash: |
+      set -eu -o pipefail
+
+      # run tests (exclude pytest.AMReX when running Python tests)
+      ctest --test-dir build --output-on-failure -E AMReX
+    displayName: 'Test'

.github/workflows/clang_sanitizers.yml

@@ -0,0 +1,162 @@
+name: 🧴 clang sanitizers
+
+on: [push, pull_request]
+
+concurrency:
+  group: ${{ github.ref }}-${{ github.head_ref }}-clangsanitizers
+  cancel-in-progress: true
+
+jobs:
+  build_UB_sanitizer:
+    name: Clang UB sanitizer
+    runs-on: ubuntu-22.04
+    container: ubuntu:23.10
+    if: github.event.pull_request.draft == false
+    env:
+      CC: clang
+      CXX: clang++
+      # On CI for this test, Ninja is slower than the default:
+      #CMAKE_GENERATOR: Ninja
+    steps:
+    - uses: actions/checkout@v4
+    - name: install dependencies
+      run: |
+        .github/workflows/dependencies/clang17.sh
+    - name: CCache Cache
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/ccache
+        key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }}
+        restore-keys: |
+             ccache-${{ github.workflow }}-${{ github.job }}-git-
+    - name: build WarpX
+      run: |
+        export CCACHE_COMPRESS=1
+        export CCACHE_COMPRESSLEVEL=10
+        export CCACHE_MAXSIZE=100M
+        ccache -z
+
+        export CXX=$(which clang++-17)
+        export CC=$(which clang-17)
+        export CXXFLAGS="-fsanitize=undefined,address,pointer-compare -fno-sanitize-recover=all"
+
+        cmake -S . -B build                 \
+          -GNinja                           \
+          -DCMAKE_VERBOSE_MAKEFILE=ON       \
+          -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+          -DWarpX_DIMS="RZ;1;2;3"           \
+          -DWarpX_FFT=ON                    \
+          -DWarpX_QED=ON                    \
+          -DWarpX_QED_TABLE_GEN=ON          \
+          -DWarpX_OPENPMD=ON                \
+          -DWarpX_PRECISION=SINGLE          \
+          -DWarpX_PARTICLE_PRECISION=SINGLE
+        cmake --build build -j 4
+
+        ccache -s
+        du -hs ~/.cache/ccache
+
+    - name: run with UB sanitizer
+      run: |
+
+        # We need these two lines because these tests run inside a docker container
+        export OMPI_ALLOW_RUN_AS_ROOT=1
+        export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
+
+        export OMP_NUM_THREADS=2
+
+        #MPI implementations often leak memory
+        export "ASAN_OPTIONS=detect_leaks=0"
+
+        mpirun -n 2 ./build/bin/warpx.rz Examples/Physics_applications/laser_acceleration/inputs_base_rz
+        mpirun -n 2 ./build/bin/warpx.1d Examples/Physics_applications/laser_acceleration/inputs_test_1d_laser_acceleration
+        mpirun -n 2 ./build/bin/warpx.2d Examples/Physics_applications/laser_acceleration/inputs_base_2d
+        mpirun -n 2 ./build/bin/warpx.3d Examples/Physics_applications/laser_acceleration/inputs_base_3d
+
+  build_thread_sanitizer:
+    name: Clang thread sanitizer
+    runs-on: ubuntu-22.04
+    container: ubuntu:23.10
+    if: github.event.pull_request.draft == false
+    env:
+      CC: clang
+      CXX: clang++
+      # On CI for this test, Ninja is slower than the default:
+      #CMAKE_GENERATOR: Ninja
+    steps:
+    - uses: actions/checkout@v4
+    - name: install dependencies
+      run: |
+        .github/workflows/dependencies/clang17.sh
+    - name: CCache Cache
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/ccache
+        key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }}
+        restore-keys: |
+             ccache-${{ github.workflow }}-${{ github.job }}-git-
+    - name: build WarpX
+      run: |
+        export CCACHE_COMPRESS=1
+        export CCACHE_COMPRESSLEVEL=10
+        export CCACHE_MAXSIZE=100M
+        ccache -z
+
+        export CXX=$(which clang++-17)
+        export CC=$(which clang-17)
+        export CXXFLAGS="-fsanitize=thread"
+
+        cmake -S . -B build                 \
+          -GNinja                           \
+          -DCMAKE_VERBOSE_MAKEFILE=ON       \
+          -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+          -DWarpX_DIMS="RZ;1;2;3"           \
+          -DWarpX_FFT=ON                    \
+          -DWarpX_QED=ON                    \
+          -DWarpX_QED_TABLE_GEN=ON          \
+          -DWarpX_OPENPMD=ON                \
+          -DWarpX_EB=OFF                    \
+          -DWarpX_PRECISION=DOUBLE          \
+          -DWarpX_PARTICLE_PRECISION=DOUBLE
+        cmake --build build -j 4
+
+        cmake -S . -B build_EB              \
+          -GNinja                           \
+          -DCMAKE_VERBOSE_MAKEFILE=ON       \
+          -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+          -DWarpX_DIMS="2"                  \
+          -DWarpX_FFT=ON                    \
+          -DWarpX_QED=ON                    \
+          -DWarpX_QED_TABLE_GEN=ON          \
+          -DWarpX_OPENPMD=ON                \
+          -DWarpX_EB=ON                     \
+          -DWarpX_PRECISION=DOUBLE          \
+          -DWarpX_PARTICLE_PRECISION=DOUBLE
+        cmake --build build_EB -j 4
+
+        ccache -s
+        du -hs ~/.cache/ccache
+
+    - name: run with thread sanitizer
+      run: |
+        export PMIX_MCA_gds=hash
+        export TSAN_OPTIONS='ignore_noninstrumented_modules=1'
+        export ARCHER_OPTIONS="verbose=1"
+
+        # We need these two lines because these tests run inside a docker container
+        export OMPI_ALLOW_RUN_AS_ROOT=1
+        export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
+
+        export OMP_NUM_THREADS=2
+
+        mpirun -n 2 ./build/bin/warpx.rz Examples/Physics_applications/laser_acceleration/inputs_base_rz warpx.serialize_initial_conditions = 0
+        mpirun -n 2 ./build/bin/warpx.1d Examples/Physics_applications/laser_acceleration/inputs_test_1d_laser_acceleration warpx.serialize_initial_conditions = 0
+        mpirun -n 2 ./build/bin/warpx.2d Examples/Physics_applications/laser_acceleration/inputs_base_2d warpx.serialize_initial_conditions = 0
+        mpirun -n 2 ./build/bin/warpx.3d Examples/Physics_applications/laser_acceleration/inputs_base_3d warpx.serialize_initial_conditions = 0
+
+        git clone https://github.com/ECP-WarpX/warpx-data ../warpx-data
+        cd Examples/Tests/embedded_circle
+
+        ulimit -c unlimited
+
+        mpirun -n 2 ../../../build_EB/bin/warpx.2d inputs_test_2d_embedded_circle warpx.serialize_initial_conditions = 0

.github/workflows/clang_tidy.yml

@@ -13,6 +13,7 @@ jobs:
         dim: [1, 2, RZ, 3]
     name: clang-tidy-${{ matrix.dim }}D
     runs-on: ubuntu-22.04
+    timeout-minutes: 180
     if: github.event.pull_request.draft == false
     steps:
     - uses: actions/checkout@v4
@@ -59,18 +60,3 @@ jobs:
 
         ccache -s
         du -hs ~/.cache/ccache
-
-  save_pr_number:
-    if: github.event_name == 'pull_request'
-    runs-on: ubuntu-latest
-    steps:
-      - name: Save PR number
-        env:
-          PR_NUMBER: ${{ github.event.number }}
-        run: |
-          echo $PR_NUMBER > pr_number.txt
-      - uses: actions/upload-artifact@v4
-        with:
-          name: pr_number
-          path: pr_number.txt
-          retention-days: 1

.github/workflows/cleanup-cache-postpr.yml

@@ -23,8 +23,11 @@ jobs:
 
           REPO=${{ github.repository }}
 
-          gh run download ${{ github.event.workflow_run.id }} -n pr_number
-          pr_number=`cat pr_number.txt`
+          # For debugging cat ${GITHUB_EVENT_PATH} to see the payload.
+
+          pr_head_sha=${{ github.event.workflow_run.head_sha }}
+          pr_number=$(gh pr list --state all --search $pr_head_sha --json number --jq '.[0].number')
+          echo "Post-PR cache cleanup for PR ${pr_number}"
           BRANCH=refs/pull/${pr_number}/merge
 
           # Setting this to not fail the workflow while deleting cache keys.