split unit tests, parcel smoke tests and multi-process/condensation e…

…xamples into multiple jobs to reduce CI time; introduce pip cache for examples jobs (#1315)
open-atmos · Apr 14, 2024 · c38c81f · c38c81f
1 parent ec7af2e
commit c38c81f
Show file tree

Hide file tree

Showing 48 changed files with 84 additions and 41 deletions.
diff --git a/.github/workflows/tests+artifacts+pypi.yml b/.github/workflows/tests+artifacts+pypi.yml
@@ -126,13 +126,13 @@ jobs:
       matrix:
         platform: [ubuntu-latest, macos-12, windows-latest]
         python-version: ["3.8", "3.11"]
-        test-suite: ["unit_tests", "smoke_tests/no_env", "smoke_tests/box", "smoke_tests/parcel", "smoke_tests/kinematic_1d", "smoke_tests/kinematic_2d", "tutorials_tests"]
+        test-suite: ["unit_tests/!(dynamics)", "unit_tests/dynamics/!(condensation)", "unit_tests/dynamics/condensation", "smoke_tests/no_env", "smoke_tests/box", "smoke_tests/parcel_a", "smoke_tests/parcel_b", "smoke_tests/parcel_c", "smoke_tests/kinematic_1d", "smoke_tests/kinematic_2d", "tutorials_tests"]
         exclude:
           - test-suite: "devops_tests"
             python-version: "3.8"
       fail-fast: false
     runs-on: ${{ matrix.platform }}
-    timeout-minutes: 100
+    timeout-minutes: 40
     steps:
       - uses: actions/[email protected]
         with:
@@ -142,47 +142,80 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - run: |
-          pip install -e .
+      - run: pip install -e .
 
       # sanity check if we do not depend on something from tests extras
-      - run: |
-          python -We -c "import PySDM"
-
-      - run: |
-          pip install -e ./examples
-          pip install -e .[tests]
+      - run: python -We -c "import PySDM"
 
-      # ensure previous pip did not overwrite the package
-      - run: |
-          pip install -e .
+      - if: startsWith(matrix.test-suite, 'smoke_tests') || matrix.test-suite == 'tutorials_tests'
+        run: pip install -e ./examples
 
-      # https://github.com/numba/numba/issues/6350#issuecomment-728174860
-      - if: startsWith(matrix.platform, 'ubuntu-')
-        run: echo NUMBA_THREADING_LAYER=omp >> $GITHUB_ENV
+      - run: pip install -e .[tests] # incl. ensuring that the previous pip did not overwrite the package
 
       # install devops_tests for tutorials_tests
       - if: matrix.test-suite == 'tutorials_tests'
         run: pip install -r tests/devops_tests/requirements.txt 
 
-      - env:
+      # https://github.com/numba/numba/issues/6350#issuecomment-728174860
+      - if: startsWith(matrix.platform, 'ubuntu-')
+        run: echo NUMBA_THREADING_LAYER=omp >> $GITHUB_ENV
+
+      - name: Run pytest
+        env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: pytest --durations=10 --timeout=900 --timeout_method=thread -p no:unraisableexception -We tests/${{ matrix.test-suite }}
+          NUMBA_OPT: 1
+        run: |
+          shopt -s extglob
+          pytest --durations=10 --timeout=900 --timeout_method=thread -p no:unraisableexception -We tests/${{ matrix.test-suite }}
 
 #       - if: ${{ failure() }}
 #         uses: mxschmitt/action-tmate@v3
 #         with:
 #           limit-access-to-actor: true
 
+  examples-setup:
+    strategy:
+      matrix:
+        platform: [ubuntu-22.04, macos-12, windows-latest]
+        python-version: ["3.8", "3.11"]
+    runs-on: ${{ matrix.platform }}
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: recursive
+          fetch-depth: 0
+      - uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: |
+          echo pip_user_site=$(python -c "import sysconfig; print(sysconfig.get_path('purelib'))") >> $GITHUB_ENV
+          echo toml_ci_md5=$(cat pyproject.toml setup.py examples/pyproject.toml examples/setup.py tests/devops_tests/requirements.txt .github/workflows/tests+artifacts+pypi.yml \
+          | python -c "import hashlib;print(hashlib.md5(open(0,'rb').read()).hexdigest())") >> $GITHUB_ENV
+      - uses: actions/cache@v4
+        with:
+          path: ${{ env.pip_user_site }}
+          key: examples_pip_cache-${{ matrix.platform }}-${{ matrix.python-version }}-${{ env.toml_ci_md5 }}
+      - if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          python -m pip install --upgrade --user pip
+          python -m pip install -e . ./examples
+          python -m pip install -r tests/devops_tests/requirements.txt
+      - if: steps.cache.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v4
+        with:
+          path: ${{ env.pip_user_site }}
+          key: examples_pip_cache-${{ matrix.platform }}-${{ matrix.python-version }}-${{ env.toml_ci_md5 }}
+
   examples:
-    needs: [pylint, precommit]
+    needs: [pylint, precommit, examples-setup]
     strategy:
       matrix:
         platform: [ubuntu-22.04, macos-12, windows-latest]
         python-version: ["3.8", "3.11"]
-        test-suite: [ "aqueous_chemistry", "freezing", "condensation", "coagulation", "breakup", "multi-process", "isotopes"]
+        test-suite: [ "chemistry_freezing_isotopes", "condensation_a", "condensation_b", "coagulation", "breakup", "multi-process_a", "multi-process_b"]
       fail-fast: false
     runs-on: ${{ matrix.platform }}
+    timeout-minutes: 45
     steps:
       - uses: actions/[email protected]
         with:
@@ -192,10 +225,17 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
       - run: |
-          python -m pip install --upgrade --user pip
+          echo pip_user_site=$(python -c "import sysconfig; print(sysconfig.get_path('purelib'))") >> $GITHUB_ENV
+          echo toml_ci_md5=$(cat pyproject.toml setup.py examples/pyproject.toml examples/setup.py tests/devops_tests/requirements.txt .github/workflows/tests+artifacts+pypi.yml \
+          | python -c "import hashlib;print(hashlib.md5(open(0,'rb').read()).hexdigest())") >> $GITHUB_ENV
       - run: |
-          python -m pip install -e . ./examples
-          python -m pip install -r tests/devops_tests/requirements.txt
+          rm -rf $pip_user_site
+      - id: cache
+        uses: actions/cache/restore@v4
+        with:
+          fail-on-cache-miss: true
+          path: ${{ env.pip_user_site }}
+          key: examples_pip_cache-${{ matrix.platform }}-${{ matrix.python-version }}-${{ env.toml_ci_md5 }}
 
       # https://github.com/numba/numba/issues/6350#issuecomment-728174860
       - if: startsWith(matrix.platform, 'ubuntu-')

diff --git a/tests/examples_tests/conftest.py b/tests/examples_tests/conftest.py
@@ -16,16 +16,26 @@ def findfiles(path, regex):
 
 
 TEST_SUITES = {
-    "aqueous_chemistry": ["Jaruga_and_Pawlowska_2018", "Kreidenweis_et_al_2003"],
-    "freezing": [
+    "chemistry_freezing_isotopes": [
+        "Jaruga_and_Pawlowska_2018",
+        "Kreidenweis_et_al_2003",
         "Alpert_and_Knopf_2016",
         "Ervens_and_Feingold_2012",
         "Niedermeier_et_al_2014",
+        "Bolot_et_al_2013",
+        "Merlivat_and_Nief_1967",
+        "Van_Hook_1968",
+        "Pierchala_et_al_2022",
+        "Graf_et_al_2019",
+        "Lamb_et_al_2017",
+        # "Rozanski_and_Sonntag_1982",  # TODO #1207
     ],
-    "condensation": [
+    "condensation_a": [
+        "Lowe_et_al_2019",
         "Abdul_Razzak_Ghan_2000",
+    ],
+    "condensation_b": [
         "Arabas_and_Shima_2017",
-        "Lowe_et_al_2019",
         "Pyrcel",
         "Yang_et_al_2018",
         "Singer_Ward",
@@ -34,27 +44,20 @@ def findfiles(path, regex):
     ],
     "coagulation": ["Berry_1967", "Shima_et_al_2009"],
     "breakup": ["Bieli_et_al_2022", "deJong_Mackay_et_al_2023", "Srivastava_1982"],
-    "multi-process": [
-        "Bartman_et_al_2021",
-        "Morrison_and_Grabowski_2007",
+    "multi-process_a": [
+        "deJong_Azimi",
         "Arabas_et_al_2015",
         "Bartman_2020_MasterThesis",
         "Bulenok_2023_MasterThesis",
-        "deJong_Azimi",
+    ],
+    "multi-process_b": [
         "Arabas_et_al_2023",
-        "Szumowski_et_al_1998",
+        "Bartman_et_al_2021",
+        "Morrison_and_Grabowski_2007",
         "Shipway_and_Hill_2012",
+        "Szumowski_et_al_1998",
         "utils",
     ],
-    "isotopes": [
-        "Bolot_et_al_2013",
-        "Merlivat_and_Nief_1967",
-        "Van_Hook_1968",
-        "Pierchala_et_al_2022",
-        "Graf_et_al_2019",
-        "Lamb_et_al_2017",
-        # "Rozanski_and_Sonntag_1982",  # TODO #1207
-    ],
 }
 
 

diff --git a/tests/smoke_tests/parcel/yang_et_al_2018/__init__.py b/tests/smoke_tests/parcel/yang_et_al_2018/__init__.py
diff --git a/tests/smoke_tests/no_env/__init__.py → ...ests/parcel_a/lowe_et_al_2019/__init__.py b/tests/smoke_tests/no_env/__init__.py → ...ests/parcel_a/lowe_et_al_2019/__init__.py
diff --git a/..._tests/parcel/lowe_et_al_2019/conftest.py → ...ests/parcel_a/lowe_et_al_2019/conftest.py b/..._tests/parcel/lowe_et_al_2019/conftest.py → ...ests/parcel_a/lowe_et_al_2019/conftest.py
diff --git a/...el/lowe_et_al_2019/test_dz_sensitivity.py → ..._a/lowe_et_al_2019/test_dz_sensitivity.py b/...el/lowe_et_al_2019/test_dz_sensitivity.py → ..._a/lowe_et_al_2019/test_dz_sensitivity.py
diff --git a/...ests/parcel/lowe_et_al_2019/test_fig_1.py → ...ts/parcel_a/lowe_et_al_2019/test_fig_1.py b/...ests/parcel/lowe_et_al_2019/test_fig_1.py → ...ts/parcel_a/lowe_et_al_2019/test_fig_1.py
diff --git a/...ests/parcel/lowe_et_al_2019/test_fig_2.py → ...ts/parcel_a/lowe_et_al_2019/test_fig_2.py b/...ests/parcel/lowe_et_al_2019/test_fig_2.py → ...ts/parcel_a/lowe_et_al_2019/test_fig_2.py
diff --git a/...sts/parcel/lowe_et_al_2019/test_fig_s2.py → ...s/parcel_a/lowe_et_al_2019/test_fig_s2.py b/...sts/parcel/lowe_et_al_2019/test_fig_s2.py → ...s/parcel_a/lowe_et_al_2019/test_fig_s2.py
diff --git a/...et_al_2019/test_surface_tension_models.py → ...et_al_2019/test_surface_tension_models.py b/...et_al_2019/test_surface_tension_models.py → ...et_al_2019/test_surface_tension_models.py
diff --git a/.../parcel/lowe_et_al_2019/test_zero_forg.py → ...arcel_a/lowe_et_al_2019/test_zero_forg.py b/.../parcel/lowe_et_al_2019/test_zero_forg.py → ...arcel_a/lowe_et_al_2019/test_zero_forg.py
diff --git a/...parcel/abdul_razzak_ghan_2000/__init__.py → ...s/smoke_tests/parcel_a/pyrcel/__init__.py b/...parcel/abdul_razzak_ghan_2000/__init__.py → ...s/smoke_tests/parcel_a/pyrcel/__init__.py
diff --git a/...ests/parcel/pyrcel/test_parcel_example.py → ...ts/parcel_a/pyrcel/test_parcel_example.py b/...ests/parcel/pyrcel/test_parcel_example.py → ...ts/parcel_a/pyrcel/test_parcel_example.py
diff --git a/.../parcel/arabas_and_shima_2017/__init__.py → ...arcel_b/arabas_and_shima_2017/__init__.py b/.../parcel/arabas_and_shima_2017/__init__.py → ...arcel_b/arabas_and_shima_2017/__init__.py
diff --git a/...rabas_and_shima_2017/test_conservation.py → ...rabas_and_shima_2017/test_conservation.py b/...rabas_and_shima_2017/test_conservation.py → ...rabas_and_shima_2017/test_conservation.py
diff --git a/...rabas_and_shima_2017/test_displacement.py → ...rabas_and_shima_2017/test_displacement.py b/...rabas_and_shima_2017/test_displacement.py → ...rabas_and_shima_2017/test_displacement.py
diff --git a/...arabas_and_shima_2017/test_event_rates.py → ...arabas_and_shima_2017/test_event_rates.py b/...arabas_and_shima_2017/test_event_rates.py → ...arabas_and_shima_2017/test_event_rates.py
diff --git a/...bas_and_shima_2017/test_initialisation.py → ...bas_and_shima_2017/test_initialisation.py b/...bas_and_shima_2017/test_initialisation.py → ...bas_and_shima_2017/test_initialisation.py
diff --git a/...el/arabas_and_shima_2017/test_vs_scipy.py → ..._b/arabas_and_shima_2017/test_vs_scipy.py b/...el/arabas_and_shima_2017/test_vs_scipy.py → ..._b/arabas_and_shima_2017/test_vs_scipy.py
diff --git a/.../grabowski_and_pawlowska_2023/__init__.py → ...ests/parcel_b/yang_et_al_2018/__init__.py b/.../grabowski_and_pawlowska_2023/__init__.py → ...ests/parcel_b/yang_et_al_2018/__init__.py
diff --git a/...rcel/yang_et_al_2018/test_displacement.py → ...el_b/yang_et_al_2018/test_displacement.py b/...rcel/yang_et_al_2018/test_displacement.py → ...el_b/yang_et_al_2018/test_displacement.py
diff --git a/...el/yang_et_al_2018/test_initialisation.py → ..._b/yang_et_al_2018/test_initialisation.py b/...el/yang_et_al_2018/test_initialisation.py → ..._b/yang_et_al_2018/test_initialisation.py
diff --git a/...parcel/yang_et_al_2018/test_just_do_it.py → ...rcel_b/yang_et_al_2018/test_just_do_it.py b/...parcel/yang_et_al_2018/test_just_do_it.py → ...rcel_b/yang_et_al_2018/test_just_do_it.py
diff --git a/..._tests/parcel/graf_et_al_2019/__init__.py → ...rcel_c/abdul_razzak_ghan_2000/__init__.py b/..._tests/parcel/graf_et_al_2019/__init__.py → ...rcel_c/abdul_razzak_ghan_2000/__init__.py
diff --git a/...bdul_razzak_ghan_2000/test_ARG_example.py → ...bdul_razzak_ghan_2000/test_ARG_example.py b/...bdul_razzak_ghan_2000/test_ARG_example.py → ...bdul_razzak_ghan_2000/test_ARG_example.py
diff --git a/...abdul_razzak_ghan_2000/test_just_do_it.py → ...abdul_razzak_ghan_2000/test_just_do_it.py b/...abdul_razzak_ghan_2000/test_just_do_it.py → ...abdul_razzak_ghan_2000/test_just_do_it.py
diff --git a/..._2000/test_single_supersaturation_peak.py → ..._2000/test_single_supersaturation_peak.py b/..._2000/test_single_supersaturation_peak.py → ..._2000/test_single_supersaturation_peak.py
diff --git a/...parcel/jensen_and_nugent_2017/__init__.py → .../grabowski_and_pawlowska_2023/__init__.py b/...parcel/jensen_and_nugent_2017/__init__.py → .../grabowski_and_pawlowska_2023/__init__.py
diff --git a/...owska_2023/test_condensation_tolerance.py → ...owska_2023/test_condensation_tolerance.py b/...owska_2023/test_condensation_tolerance.py → ...owska_2023/test_condensation_tolerance.py
diff --git a/...and_pawlowska_2023/test_figure_1_and_2.py → ...and_pawlowska_2023/test_figure_1_and_2.py b/...and_pawlowska_2023/test_figure_1_and_2.py → ...and_pawlowska_2023/test_figure_1_and_2.py
diff --git a/...owski_and_pawlowska_2023/test_figure_3.py → ...owski_and_pawlowska_2023/test_figure_3.py b/...owski_and_pawlowska_2023/test_figure_3.py → ...owski_and_pawlowska_2023/test_figure_3.py
diff --git a/...owski_and_pawlowska_2023/test_figure_4.py → ...owski_and_pawlowska_2023/test_figure_4.py b/...owski_and_pawlowska_2023/test_figure_4.py → ...owski_and_pawlowska_2023/test_figure_4.py
diff --git a/..._and_pawlowska_2023/test_ripening_rate.py → ..._and_pawlowska_2023/test_ripening_rate.py b/..._and_pawlowska_2023/test_ripening_rate.py → ..._and_pawlowska_2023/test_ripening_rate.py
diff --git a/...parcel/kreidenweis_et_al_2003/__init__.py → ...ests/parcel_c/graf_et_al_2019/__init__.py b/...parcel/kreidenweis_et_al_2003/__init__.py → ...ests/parcel_c/graf_et_al_2019/__init__.py
diff --git a/...ests/parcel/graf_et_al_2019/test_fig_4.py → ...ts/parcel_c/graf_et_al_2019/test_fig_4.py b/...ests/parcel/graf_et_al_2019/test_fig_4.py → ...ts/parcel_c/graf_et_al_2019/test_fig_4.py
diff --git a/...ts/parcel/graf_et_al_2019/test_table_1.py → .../parcel_c/graf_et_al_2019/test_table_1.py b/...ts/parcel/graf_et_al_2019/test_table_1.py → .../parcel_c/graf_et_al_2019/test_table_1.py
diff --git a/..._tests/parcel/lowe_et_al_2019/__init__.py → ...rcel_c/jensen_and_nugent_2017/__init__.py b/..._tests/parcel/lowe_et_al_2019/__init__.py → ...rcel_c/jensen_and_nugent_2017/__init__.py
diff --git a/...rcel/jensen_and_nugent_2017/test_fig_1.py → ...el_c/jensen_and_nugent_2017/test_fig_1.py b/...rcel/jensen_and_nugent_2017/test_fig_1.py → ...el_c/jensen_and_nugent_2017/test_fig_1.py
diff --git a/...rcel/jensen_and_nugent_2017/test_fig_3.py → ...el_c/jensen_and_nugent_2017/test_fig_3.py b/...rcel/jensen_and_nugent_2017/test_fig_3.py → ...el_c/jensen_and_nugent_2017/test_fig_3.py
diff --git a/...rcel/jensen_and_nugent_2017/test_fig_4.py → ...el_c/jensen_and_nugent_2017/test_fig_4.py b/...rcel/jensen_and_nugent_2017/test_fig_4.py → ...el_c/jensen_and_nugent_2017/test_fig_4.py
diff --git a/...el/jensen_and_nugent_2017/test_table_3.py → ..._c/jensen_and_nugent_2017/test_table_3.py b/...el/jensen_and_nugent_2017/test_table_3.py → ..._c/jensen_and_nugent_2017/test_table_3.py
diff --git a/...parcel/niedermeier_et_al_2013/__init__.py → ...rcel_c/kreidenweis_et_al_2003/__init__.py b/...parcel/niedermeier_et_al_2013/__init__.py → ...rcel_c/kreidenweis_et_al_2003/__init__.py
diff --git a/...rcel/kreidenweis_et_al_2003/test_fig_1.py → ...el_c/kreidenweis_et_al_2003/test_fig_1.py b/...rcel/kreidenweis_et_al_2003/test_fig_1.py → ...el_c/kreidenweis_et_al_2003/test_fig_1.py
diff --git a/...denweis_et_al_2003/test_ionic_strength.py → ...denweis_et_al_2003/test_ionic_strength.py b/...denweis_et_al_2003/test_ionic_strength.py → ...denweis_et_al_2003/test_ionic_strength.py
diff --git a/...enweis_et_al_2003/test_spectrum_at_t_0.py → ...enweis_et_al_2003/test_spectrum_at_t_0.py b/...enweis_et_al_2003/test_spectrum_at_t_0.py → ...enweis_et_al_2003/test_spectrum_at_t_0.py
diff --git a/...el/kreidenweis_et_al_2003/test_table_3.py → ..._c/kreidenweis_et_al_2003/test_table_3.py b/...el/kreidenweis_et_al_2003/test_table_3.py → ..._c/kreidenweis_et_al_2003/test_table_3.py
diff --git a/tests/smoke_tests/parcel/pyrcel/__init__.py → ...rcel_c/niedermeier_et_al_2013/__init__.py b/tests/smoke_tests/parcel/pyrcel/__init__.py → ...rcel_c/niedermeier_et_al_2013/__init__.py
diff --git a/...er_et_al_2013/test_temperature_profile.py → ...er_et_al_2013/test_temperature_profile.py b/...er_et_al_2013/test_temperature_profile.py → ...er_et_al_2013/test_temperature_profile.py