Updated pr-build.yml

.github/workflows/pr-build.yml

@@ -7,7 +7,30 @@ on:
       - 'main'
 
 jobs:
-  run-pr-checks:
+  changes:
+    runs-on: ubuntu-latest
+    permissions:
+        pull-requests: read
+    outputs:
+      etl: ${{ steps.filter.outputs.etl }}
+      api: ${{ steps.filter.outputs.api }}
+      etl_tests: ${{ steps.filter.outputs.etl_tests }}
+      api_tests: ${{ steps.filter.outputs.api_tests }}
+    steps:
+    - uses: dorny/[email protected]  
+      id: filter
+      with:
+        filters: |
+          etl:
+            - 'air-quality-backend/src/etl/**'
+          api:
+            - 'air-quality-backend/src/api/**'
+          etl_tests:
+            - 'air-quality-backend/tests/etl_tests/**'
+          api_tests:
+            - 'air-quality-backend/tests/api_tests/**'
+
+  run-lint:
     runs-on: ubuntu-latest
     defaults:
       run:
@@ -20,18 +43,65 @@ jobs:
         uses: conda-incubator/setup-miniconda@v3
         with:
           channels: conda-forge
-          environment-file: air-quality-backend/conda/test_environment.yml
+          environment-file: air-quality-backend/conda/linting_environment.yml
           auto-activate-base: false
-          activate-environment: unit-test
+          activate-environment: linting-env
 
       - name: Run lint check
         run: python -m flake8 --append-config air-quality-backend/.flake8 air-quality-backend
 
-      - name: Run unit tests
-        run: python -m pytest --cov=src --cov=scripts --cov-report=xml:air-quality-backend/tests/coverage/test_report.xml air-quality-backend/tests
-
+  etl:
+    needs: changes
+    if: ${{ needs.changes.outputs.etl == 'true' || needs.changes.outputs.etl_tests == 'true' }}
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -el {0}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup miniconda
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          channels: conda-forge
+          environment-file: air-quality-backend/conda/etl_environment.yml
+          auto-activate-base: false
+          activate-environment: etl-dev
+
+      - name: Run etl unit tests
+        run: python -m pytest --cov=src --cov=scripts --cov-report=xml:air-quality-backend/tests/coverage/etl_test_report.xml air-quality-backend/tests/etl_tests
+
       - name: Code Coverage Report
         uses: orgoro/[email protected]
         with:
-            coverageFile: air-quality-backend/tests/coverage/test_report.xml
+            coverageFile: air-quality-backend/tests/coverage/etl_test_report.xml
             token: ${{ secrets.GITHUB_TOKEN }}
+
+  api:
+    needs: changes
+    if: ${{ needs.changes.outputs.api == 'true' || needs.changes.outputs.api_tests == 'true' }}
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -el {0}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup miniconda
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          channels: conda-forge
+          environment-file: air-quality-backend/conda/api_environment.yml
+          auto-activate-base: false
+          activate-environment: api-dev
+
+      - name: Run api unit tests
+        run: python -m pytest --cov=src --cov=scripts --cov-report=xml:air-quality-backend/tests/coverage/api_test_report.xml air-quality-backend/tests/api_tests
+
+      - name: Code Coverage Report
+        uses: orgoro/[email protected]
+        with:
+            coverageFile: air-quality-backend/tests/coverage/api_test_report.xml
+            token: ${{ secrets.GITHUB_TOKEN }}

air-quality-backend/scripts/run_forecast_etl.py

@@ -2,7 +2,8 @@
 from dotenv import load_dotenv
 import logging
 from logging import config
-from src.database import (
+
+from src.database.air_quality_dashboard_dao import (
     get_locations_by_type,
     insert_data_forecast,
 )

air-quality-backend/scripts/run_in_situ_etl.py

@@ -2,7 +2,8 @@
 from dotenv import load_dotenv
 import logging
 from logging import config
-from src.database import (
+
+from src.database.air_quality_dashboard_dao import (
     get_locations_by_type,
     insert_data_openaq,
 )

air-quality-backend/scripts/run_purge_old_data.py

@@ -2,9 +2,8 @@
 from dotenv import load_dotenv
 import logging
 from logging import config
-from src.database import (
-    delete_data_before,
-)
+
+from src.database.air_quality_dashboard_dao import delete_data_before
 
 config.fileConfig("../logging.ini")