handling transpiling partial ddls #3875

Workflow file for this run

	name: build

	on:
	pull_request:
	types: [opened, synchronize]
	merge_group:
	types: [checks_requested]
	push:
	# Always run on push to main. The build cache can only be reused
	# if it was saved by a run from the repository's default branch.
	# The run result will be identical to that from the merge queue
	# because the commit is identical, yet we need to perform it to
	# seed the build cache.
	branches:
	- main

	env:
	HATCH_VERSION: 1.9.1

	jobs:
	ci:
	runs-on: ubuntu-latest
	steps:
	- name: Checkout
	uses: actions/checkout@v4

	- name: Install Python
	uses: actions/setup-python@v5
	with:
	cache: 'pip'
	cache-dependency-path: '**/pyproject.toml'
	python-version: '3.10'

	- name: Install hatch
	run: pip install hatch==$HATCH_VERSION

	- name: Setup Spark Remote
	run: \|
	chmod +x $GITHUB_WORKSPACE/.github/scripts/setup_spark_remote.sh
	$GITHUB_WORKSPACE/.github/scripts/setup_spark_remote.sh

	- name: Run unit tests
	run: hatch run test

	- name: Publish test coverage
	uses: codecov/codecov-action@v4
	with:
	codecov_yml_path: codecov.yml
	token: ${{ secrets.CODECOV_TOKEN }}

	fmt-python:
	runs-on: ubuntu-latest
	steps:
	- name: Checkout
	uses: actions/checkout@v4

	- name: Install Python
	uses: actions/setup-python@v5
	with:
	cache: 'pip'
	cache-dependency-path: '**/pyproject.toml'
	python-version: 3.10.x

	- name: Install hatch
	run: pip install hatch==$HATCH_VERSION

	- name: Reformat code
	run: make fmt-python

	- name: Fail on differences
	run: \|
	# Exit with status code 1 if there are differences (i.e. unformatted files)
	git diff --exit-code

	fmt-core:
	runs-on: ubuntu-latest
	steps:
	- name: Checkout
	uses: actions/checkout@v4

	- name: Set up JDK 11
	uses: actions/setup-java@v4
	with:
	distribution: corretto
	java-version: 11

	# GitHub Action team seems not to have cycles to make the cache work properly, hence this hack
	# See https://github.com/actions/setup-java/issues/255
	# See https://github.com/actions/setup-java/issues/577
	- name: Cache Maven
	uses: actions/cache@v4
	with:
	path: ~/.m2
	key: ${{ github.job }}-${{ hashFiles('**/pom.xml') }}

	- name: Reformat code
	run: make fmt-scala

	- name: Fail on differences
	run: \|
	# Exit with status code 1 if there are differences (i.e. unformatted files)
	git diff --exit-code

	no-lint-disabled:
	runs-on: ubuntu-latest
	if: github.event_name == 'pull_request' && (github.event.action == 'opened' \|\| github.event.action == 'synchronize')
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	fetch-depth: 0

	- name: Verify no lint disabled in the new code
	run: \|
	NEW_CODE=$(git diff origin/main..$(git branch --show-current) \| grep -e '^+')
	CHEAT=$(echo "${NEW_CODE}" \| grep '# pylint: disable' \| grep -v "CHEAT" \| wc -c)
	if [ "${CHEAT}" -ne 0 ]; then
	echo "Do not cheat the linter: ${CHEAT}"
	exit 1
	fi

	test-core:
	runs-on: ubuntu-latest
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	fetch-depth: 0

	- name: Set up JDK 11
	uses: actions/setup-java@v4
	with:
	distribution: corretto
	java-version: 11

	# GitHub Action team seems not to have cycles to make the cache work properly, hence this hack.
	# See https://github.com/actions/setup-java/issues/255
	# See https://github.com/actions/setup-java/issues/577
	- name: Cache Maven
	uses: actions/cache@v4
	with:
	path: ~/.m2
	key: ${{ github.job }}-${{ hashFiles('**/pom.xml') }}

	- name: Install Python
	uses: actions/setup-python@v5
	with:
	cache: 'pip'
	cache-dependency-path: '**/pyproject.toml'
	python-version: '3.10'

	- name: Install hatch
	run: pip install hatch==$HATCH_VERSION

	- name: Initialize Python virtual environment for StandardInputPythonSubprocess
	run: make dev

	- name: Run Unit Tests with Maven
	run: mvn --update-snapshots scoverage:report --file pom.xml --fail-at-end

	- name: Upload remorph-core jars as Artifacts
	uses: actions/upload-artifact@v4
	with:
	name: remorph-core-jars
	path: ~/.m2/repository/com/databricks/labs/remorph*

	- name: Publish JUnit report
	uses: EnricoMi/publish-unit-test-result-action@v2
	if: always()
	with:
	files: \|
	*/TEST-com.databricks.labs.remorph.coverage.xml
	comment_title: 'Coverage tests results'
	check_name: 'Coverage Tests Results'
	fail_on: 'nothing'
	continue-on-error: true

	- name: Publish test coverage
	uses: codecov/codecov-action@v4
	with:
	token: ${{ secrets.CODECOV_TOKEN }}

	coverage-tests-with-make:
	runs-on: ubuntu-latest
	env:
	INPUT_DIR_PARENT: .
	OUTPUT_DIR: ./test-reports
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	fetch-depth: 0

	- name: Install Python
	uses: actions/setup-python@v5
	with:
	cache: 'pip'
	cache-dependency-path: '**/pyproject.toml'
	python-version: 3.10.x

	- name: Install hatch
	run: pip install hatch==$HATCH_VERSION

	- name: Set up JDK 11
	uses: actions/setup-java@v4
	with:
	distribution: corretto
	java-version: 11

	# GitHub Action team seems not to have cycles to make the cache work properly, hence this hack.
	# See https://github.com/actions/setup-java/issues/255
	# See https://github.com/actions/setup-java/issues/577
	- name: Cache Maven
	uses: actions/cache@v4
	with:
	path: ~/.m2
	key: ${{ github.job }}-${{ hashFiles('**/pom.xml') }}

	- name: Install Python
	uses: actions/setup-python@v5
	with:
	cache: 'pip'
	cache-dependency-path: '**/pyproject.toml'
	python-version: '3.10'

	- name: Install hatch
	run: pip install hatch==$HATCH_VERSION

	- name: Install Databricks CLI
	uses: databricks/setup-cli@main

	- name: Initialize Python virtual environment for StandardInputPythonSubprocess
	run: make dev

	- name: Create dummy test file
	run: \|
	mkdir $INPUT_DIR_PARENT/snowflake
	mkdir $INPUT_DIR_PARENT/tsql
	echo "SELECT * FROM t;" >> $INPUT_DIR_PARENT/snowflake/dummy_test.sql
	echo "SELECT * FROM t;" >> $INPUT_DIR_PARENT/tsql/dummy_test.sql
	shell: bash

	- name: Dry run coverage tests with make
	run: make dialect_coverage_report
	env: # this is a temporary hack
	DATABRICKS_HOST: any
	DATABRICKS_TOKEN: any

	- name: Verify report file
	if: ${{ hashFiles('./test-reports/') == '' }}
	run: \|
	echo "No file produced in tests-reports/"
	exit 1


	antlr-grammar-linting:
	runs-on: ubuntu-latest
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	with:
	fetch-depth: 0

	- name: Set up JDK 11
	uses: actions/setup-java@v4
	with:
	distribution: corretto
	java-version: 11

	# GitHub Action team seems not to have cycles to make the cache work properly, hence this hack.
	# See https://github.com/actions/setup-java/issues/255
	# See https://github.com/actions/setup-java/issues/577
	- name: Cache Maven
	uses: actions/cache@v4
	with:
	path: ~/.m2
	key: ${{ github.job }}-${{ hashFiles('**/pom.xml') }}

	- name: Run Lint Test with Maven
	run: mvn compile -DskipTests --update-snapshots -B exec:java -pl linter --file pom.xml -Dexec.args="-i core/src/main/antlr4 -o .venv/linter/grammar -c true"
	continue-on-error: true

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

handling transpiling partial ddls #3875

Workflow file

handling transpiling partial ddls #3875

Jobs

Run details

Workflow file for this run