ci.yml

name: CI

on:
  push:
    branches:
      - main
  pull_request:
    # run CI only if files in these whitelisted paths are changed
    paths:
      - '.github/workflows/**'
      - 'rdmo/**'
      - 'testing/**'
      - 'webpack/**'
      - .eslintrc.js
      - .nvmrc
      - .pre-commit-config.yaml
      - conftest.py
      - package.json
      - pyproject.toml
    types:
      - opened
      - synchronize
      - reopened
      - ready_for_review # this is needed to trigger checks, when an auto-generated "draft" PR is set for "ready for review".

# Ref: https://docs.github.com/en/actions/using-jobs/using-concurrency
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  PYTHONDONTWRITEBYTECODE: 1
  FORCE_COLOR: 1 # colored output by pytest etc.

jobs:

  lint:
    # Ref: https://github.com/rdmorganiser/.github/blob/main/.github/workflows/_lint.yml
    uses: rdmorganiser/.github/.github/workflows/_lint.yml@main

  build-wheel:
    name: Build python wheel
    runs-on: ubuntu-24.04
    steps:
    - uses: actions/checkout@v4
    # update the version
    - name: Get short commit SHA
      run: |
        if [ "${{ github.event_name }}" = "pull_request" ]; then
          SHA="${{ github.event.pull_request.head.sha }}"
        else
          SHA="${{ github.sha }}"
        fi
        echo "SHA=$(git rev-parse --short $SHA)" >> $GITHUB_ENV
    - name: Get current version (MAJOR.MINOR.PATCH)
      id: current-version
      run: echo "current_version=$(grep -Po '(?<=__version__ = ")[\d\w.]+(?=")' rdmo/__init__.py)" >> $GITHUB_OUTPUT
    - name: Generate new version (current version + SHA)
      id: new-version
      run: echo "new_version=${{ steps.current-version.outputs.current_version }}+$SHA" >> $GITHUB_OUTPUT
    - name: Update version in rdmo/__init__.py
      run: |
        sed -i "s/__version__ = .*/__version__ = \"${{ steps.new-version.outputs.new_version }}\"/" rdmo/__init__.py
    # build the webpack bundle
    - uses: actions/setup-node@v4
      with:
        node-version: 18
        cache: npm
    - run: npm install && npm run build:prod
    # build the wheel
    - uses: actions/setup-python@v5
      with:
        python-version: '3.12'
        cache: pip
    - run: |
        python -m pip install --upgrade pip build[uv] twine
        python -m pip --version
    - name: Build the wheel
      run: python -m build --installer=uv
    - name: Check the metadata of wheel and sdist
      run: python -m twine check --strict dist/*
    - name: Install package from built wheel
      run: python -m pip install --no-compile dist/rdmo*.whl # do not create __pycache__/*.pyc files
    - name: Write info to step summary
      run: |
          {
            echo -e "# ✓ Wheel successfully built (v${{ steps.new-version.outputs.new_version }})\n\n"
            echo '<details><summary>Information about installed wheel</summary>'
            echo -e "\n\`\`\`console"
            echo "$ python -m pip show --files --verbose rdmo"
            python -m pip show --files --verbose rdmo
            echo -e "\`\`\`\n</details>"
          } >> $GITHUB_STEP_SUMMARY
    - name: Upload wheel as artifact
      uses: actions/upload-artifact@v4
      with:
        name: wheel
        path: dist/rdmo*.whl
        if-no-files-found: error
        retention-days: 30

  test:
    name: "Test (Python: ${{ matrix.python-version }}, DB: ${{ matrix.db-backend }})"
    needs: build-wheel
    runs-on: ubuntu-24.04
    strategy:
      matrix:
        python-version: ['3.8', '3.12']
        db-backend: [mysql, postgres]
    steps:
    - uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
        cache: pip
    - name: Download wheel
      uses: actions/download-artifact@v4
      with:
        name: wheel
        path: dist
    - name: Install Dependencies
      run: |
        sudo apt-get update && sudo apt-get install --yes pandoc texlive-xetex librsvg2-bin
        pandoc --version
        python -m pip install --upgrade pip
        python -m pip --version
    - name: Install rdmo[mysql] from wheel and start mysql
      run: |
        python -m pip install "$(ls dist/*.whl)[ci,mysql]"
        sudo systemctl start mysql.service
      if: matrix.db-backend == 'mysql'
    - name: Install rdmo[postgres] from wheel and start postgresql
      run: |
        python -m pip install "$(ls dist/*.whl)[ci,postgres]"
        sudo systemctl start postgresql.service
        pg_isready
        sudo -u postgres psql --command="CREATE USER postgres_user PASSWORD 'postgres_password' CREATEDB"
      if: matrix.db-backend == 'postgres'
    - name: Prepare Env
      run: |
        cp -r testing/media testing/media_root && mkdir testing/log
    - name: Run package status tests first
      run: |
        pytest rdmo/core/tests/test_package_status.py --nomigrations --verbose
      if: matrix.python-version == '3.12' && matrix.db-backend == 'postgres'
    - name: Run Tests
      run: |
        pytest -p randomly -p no:cacheprovider --cov --reuse-db --numprocesses=auto --dist=loadscope
      env:
        GITHUB_DB_BACKEND: ${{ matrix.db-backend }}
    - name: Upload coverage data to coveralls.io
      uses: coverallsapp/github-action@643bc377ffa44ace6394b2b5d0d3950076de9f63 # v2.3.0
      with:
        flag-name: '${{ matrix.db-backend }}: ${{ matrix.python-version }}'
        parallel: true

  test-e2e:
    name: "End-to-end Test (Python: ${{ matrix.python-version }}, DB: ${{ matrix.db-backend }})"
    needs: build-wheel
    runs-on: ubuntu-24.04
    strategy:
      matrix:
        python-version: ['3.12']
        db-backend: [postgres]
    steps:
    - uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
        cache: pip
    - name: Download wheel
      uses: actions/download-artifact@v4
      with:
        name: wheel
        path: dist
    - name: Install Dependencies
      run: |
        sudo apt-get update && sudo apt install --yes pandoc texlive-xetex librsvg2-bin
        python -m pip install --upgrade pip
    - name: Install rdmo[postgres] from wheel and start postgresql
      run: |
        python -m pip install "$(ls dist/*.whl)[ci,postgres]"
        sudo systemctl start postgresql.service
        pg_isready
        sudo -u postgres psql --command="CREATE USER postgres_user PASSWORD 'postgres_password' CREATEDB"
    - name: Prepare Env
      run: |
        cp -r testing/media testing/media_root && mkdir testing/log
    - name: Install e2e tests dependencies
      run: |
        playwright install --with-deps chromium
    - run: mkdir screenshots
    - name: Collect static files into static root (only required if rdmo is installed from wheel)
      run: python testing/manage.py collectstatic --noinput
    - name: Run end-to-end tests
      run: pytest -p randomly -p no:cacheprovider --reuse-db --numprocesses=auto --dist=loadscope -m e2e --nomigrations
      env:
        DJANGO_DEBUG: True
        GITHUB_DB_BACKEND: ${{ matrix.db-backend }}
    - uses: actions/upload-artifact@v4
      with:
        name: screenshots
        path: screenshots/*.png

  coveralls:
    name: Indicate completion to coveralls
    needs: test
    if: ${{ always() }}
    runs-on: ubuntu-24.04
    steps:
    - name: Run Coveralls finish
      uses: coverallsapp/github-action@643bc377ffa44ace6394b2b5d0d3950076de9f63 # v2.3.0
      with:
        parallel-finished: true

  dev-setup:
    # Ref: structlog (MIT licensed) <https://github.com/hynek/structlog/blob/main/.github/workflows/ci.yml>
    name: "Test dev setup on ${{ matrix.os }}"
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-latest, windows-latest, macos-latest]
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"
          cache: pip
      - run: python -Im pip install --editable .[dev]
      - run: python -Ic 'import rdmo; print(rdmo.__version__)'

  dependencies:
    name: Test installation of all dependencies
    needs: build-wheel
    runs-on: ubuntu-24.04
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"
          cache: pip
      - name: Download wheel
        uses: actions/download-artifact@v4
        with:
          name: wheel
          path: dist
      - name: Install os requirements for python-ldap
        run: sudo apt-get update && sudo apt-get install --yes libldap2-dev libsasl2-dev
      - run: python -m pip install --upgrade pip
      - name: Install rdmo wheel with all optional dependency groups
        run: python -m pip install --no-compile "$(ls dist/*.whl)[allauth,ci,dev,gunicorn,ldap,mysql,postgres,pytest]"
      - name: Verify installed packages have compatible dependencies
        run: python -m pip check
      - uses: actions/setup-node@v4
        with:
          node-version: 18
          cache: npm
      - run: npm install --include=dev
      - name: Write info to step summary
        run: |
          {
            echo -e "# ✓ All dependency groups successfully installed in combination\n\n"
            echo '<details><summary>Installed Python packages (dependency tree)</summary>'
            echo -e "\n\`\`\`console"
            echo "$ python -m pipdeptree --local-only --exclude=pip,pipdeptree"
            python -m pipdeptree --local-only --exclude=pip,pipdeptree
            echo -e "\`\`\`\n</details>"
            echo '<details><summary>Outdated Python dependencies</summary>'
            echo -e "\n\`\`\`console"
            echo "$ python -m pip list --outdated"
            python -m pip list --outdated
            echo -e "\`\`\`\n</details>"
            echo '<details><summary>Installed JavaScript packages (dependency tree)</summary>'
            echo -e "\n\`\`\`console"
            echo "$ npm list --all"
            npm list --all
            echo -e "\`\`\`\n</details>"
            echo '<details><summary>Outdated JavaScript dependencies</summary>'
            echo -e "\n\`\`\`console"
            echo "$ npm outdated --long"
            npm outdated --long || true
            echo -e "\`\`\`\n</details>"
          } >> $GITHUB_STEP_SUMMARY

  required-checks-pass:
    if: always()
    needs:
      - lint
      - build-wheel
      - test
      - coveralls
      - test-e2e
      - dev-setup
      - dependencies
    runs-on: ubuntu-24.04
    steps:
      - uses: re-actors/alls-green@release/v1
        with:
          jobs: ${{ toJSON(needs) }}