push.yml

name: build

on:
  pull_request:
    types: [opened, synchronize]
  merge_group:
    types: [checks_requested]
  push:
    # Always run on push to main. The build cache can only be reused
    # if it was saved by a run from the repository's default branch.
    # The run result will be identical to that from the merge queue
    # because the commit is identical, yet we need to perform it to
    # seed the build cache.
    branches:
      - main

env:
  HATCH_VERSION: 1.9.1

jobs:
  ci:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Install Python
        uses: actions/setup-python@v5
        with:
          cache: 'pip'
          cache-dependency-path: '**/pyproject.toml'
          python-version: '3.10'

      - name: Install hatch
        run: pip install hatch==$HATCH_VERSION

      - name: Setup Spark Remote
        run: |
          chmod +x $GITHUB_WORKSPACE/.github/scripts/setup_spark_remote.sh
          $GITHUB_WORKSPACE/.github/scripts/setup_spark_remote.sh

      - name: Run unit tests
        run: hatch run test

      - name: Publish test coverage
        uses: codecov/codecov-action@v4
        with:
          codecov_yml_path: codecov.yml
          token: ${{ secrets.CODECOV_TOKEN }}

  fmt-python:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Install Python
        uses: actions/setup-python@v5
        with:
          cache: 'pip'
          cache-dependency-path: '**/pyproject.toml'
          python-version: 3.10.x

      - name: Install hatch
        run: pip install hatch==$HATCH_VERSION

      - name: Reformat code
        run: make fmt-python

      - name: Fail on differences
        run: |
          # Exit with status code 1 if there are differences (i.e. unformatted files)
          git diff --exit-code

  fmt-core:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Set up JDK 11
        uses: actions/setup-java@v4
        with:
          distribution: corretto
          java-version: 11

      # GitHub Action team seems not to have cycles to make the cache work properly, hence this hack
      # See https://github.com/actions/setup-java/issues/255
      # See https://github.com/actions/setup-java/issues/577
      - name: Cache Maven
        uses: actions/cache@v4
        with:
          path: ~/.m2
          key: ${{ github.job }}-${{ hashFiles('**/pom.xml') }}

      - name: Reformat code
        run: make fmt-scala

      - name: Fail on differences
        run: |
          # Exit with status code 1 if there are differences (i.e. unformatted files)
          git diff --exit-code

  no-lint-disabled:
    runs-on: ubuntu-latest
    if: github.event_name == 'pull_request' && (github.event.action == 'opened' || github.event.action == 'synchronize')
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Verify no lint disabled in the new code
        run: |
          NEW_CODE=$(git diff origin/main..$(git branch --show-current) | grep -e '^+')
          CHEAT=$(echo "${NEW_CODE}" | grep '# pylint: disable' | grep -v "CHEAT" | wc -c)
          if [ "${CHEAT}" -ne 0 ]; then
            echo "Do not cheat the linter: ${CHEAT}"
            exit 1
          fi

  test-core:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Set up JDK 11
        uses: actions/setup-java@v4
        with:
          distribution: corretto
          java-version: 11

      # GitHub Action team seems not to have cycles to make the cache work properly, hence this hack.
      # See https://github.com/actions/setup-java/issues/255
      # See https://github.com/actions/setup-java/issues/577
      - name: Cache Maven
        uses: actions/cache@v4
        with:
          path: ~/.m2
          key: ${{ github.job }}-${{ hashFiles('**/pom.xml') }}

      - name: Install Python
        uses: actions/setup-python@v5
        with:
          cache: 'pip'
          cache-dependency-path: '**/pyproject.toml'
          python-version: '3.10'

      - name: Install hatch
        run: pip install hatch==$HATCH_VERSION

      - name: Initialize Python virtual environment for StandardInputPythonSubprocess
        run: make dev

      - name: Run Unit Tests with Maven
        run: mvn --update-snapshots scoverage:report --file pom.xml --fail-at-end

      - name: Upload remorph-core jars as Artifacts
        uses: actions/upload-artifact@v4
        with:
          name: remorph-core-jars
          path: ~/.m2/repository/com/databricks/labs/remorph*

      - name: Publish JUnit report
        uses: EnricoMi/publish-unit-test-result-action@v2
        if: always()
        with:
          files: |
            **/TEST-com.databricks.labs.remorph.coverage*.xml
          comment_title: 'Coverage tests results'
          check_name: 'Coverage Tests Results'
          fail_on: 'nothing'
        continue-on-error: true

      - name: Publish test coverage
        uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}

  coverage-tests-with-make:
    runs-on: ubuntu-latest
    env:
      INPUT_DIR_PARENT: .
      OUTPUT_DIR: ./test-reports
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Install Python
        uses: actions/setup-python@v5
        with:
          cache: 'pip'
          cache-dependency-path: '**/pyproject.toml'
          python-version: 3.10.x

      - name: Install hatch
        run: pip install hatch==$HATCH_VERSION

      - name: Set up JDK 11
        uses: actions/setup-java@v4
        with:
          distribution: corretto
          java-version: 11

      # GitHub Action team seems not to have cycles to make the cache work properly, hence this hack.
      # See https://github.com/actions/setup-java/issues/255
      # See https://github.com/actions/setup-java/issues/577
      - name: Cache Maven
        uses: actions/cache@v4
        with:
          path: ~/.m2
          key: ${{ github.job }}-${{ hashFiles('**/pom.xml') }}

      - name: Install Python
        uses: actions/setup-python@v5
        with:
          cache: 'pip'
          cache-dependency-path: '**/pyproject.toml'
          python-version: '3.10'

      - name: Install hatch
        run: pip install hatch==$HATCH_VERSION

      - name: Install Databricks CLI
        uses: databricks/setup-cli@main

      - name: Initialize Python virtual environment for StandardInputPythonSubprocess
        run: make dev

      - name: Create dummy test file
        run: |
          mkdir $INPUT_DIR_PARENT/snowflake
          mkdir $INPUT_DIR_PARENT/tsql
          echo "SELECT * FROM t;" >> $INPUT_DIR_PARENT/snowflake/dummy_test.sql
          echo "SELECT * FROM t;" >> $INPUT_DIR_PARENT/tsql/dummy_test.sql
        shell: bash

      - name: Dry run coverage tests with make
        run: make dialect_coverage_report
        env: # this is a temporary hack
          DATABRICKS_HOST: any
          DATABRICKS_TOKEN: any

      - name: Verify report file
        if: ${{ hashFiles('./test-reports/') == '' }}
        run: |
          echo "No file produced in tests-reports/"
          exit 1


  antlr-grammar-linting:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Set up JDK 11
        uses: actions/setup-java@v4
        with:
          distribution: corretto
          java-version: 11

      # GitHub Action team seems not to have cycles to make the cache work properly, hence this hack.
      # See https://github.com/actions/setup-java/issues/255
      # See https://github.com/actions/setup-java/issues/577
      - name: Cache Maven
        uses: actions/cache@v4
        with:
          path: ~/.m2
          key: ${{ github.job }}-${{ hashFiles('**/pom.xml') }}

      - name: Run Lint Test with Maven
        run: mvn compile -DskipTests --update-snapshots -B exec:java -pl linter --file pom.xml -Dexec.args="-i core/src/main/antlr4 -o .venv/linter/grammar -c true"
        continue-on-error: true