Merge pull request #6777 from NvTimLiu/release-tmp

Merge dev branch branch-22.10 to main

.github/workflows/add-to-project.yml

@@ -0,0 +1,31 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Add new issues to project
+
+on:
+  issues:
+    types:
+      - opened
+
+jobs:
+  add-to-project:
+    if: github.repository == 'NVIDIA/spark-rapids'
+    name: Add new issues to project
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/[email protected]
+        with:
+          project-url: https://github.com/orgs/NVIDIA/projects/4
+          github-token: ${{ secrets.PROJECT_TOKEN }}

.github/workflows/auto-merge.yml

@@ -18,7 +18,7 @@ name: auto-merge HEAD to BASE
 on:
   pull_request_target:
     branches:
-    - branch-22.08
+    - branch-22.10
     types: [closed]
 
 jobs:
@@ -29,13 +29,13 @@ jobs:
     steps:
       - uses: actions/checkout@v2
         with:
-          ref: branch-22.08 # force to fetch from latest upstream instead of PR ref
+          ref: branch-22.10 # force to fetch from latest upstream instead of PR ref
 
       - name: auto-merge job
         uses: ./.github/workflows/auto-merge
         env:
           OWNER: NVIDIA
           REPO_NAME: spark-rapids
-          HEAD: branch-22.08
-          BASE: branch-22.10
+          HEAD: branch-22.10
+          BASE: branch-22.12
           AUTOMERGE_TOKEN: ${{ secrets.AUTOMERGE_TOKEN }} # use to merge PR

.github/workflows/blossom-ci.yml

@@ -65,9 +65,10 @@ jobs:
       NVnavkumar,\
       amahussein,\
       mattahrens,\
-      anthony-chang,\
       sinkinben,\
       thirtiseven,\
+      YanxuanLiu,\
+      cindyyuanjiang,\
       ', format('{0},', github.actor)) && github.event.comment.body == 'build'
     steps:
       - name: Check if comment is issued by authorized person

.github/workflows/mvn-verify-check.yml

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 # A workflow to run mvn verify check
-name: Maven verify checks (Scala style, Compile and Doc-gen w/ base Spark version)
+name: mvn[compile,RAT,scalastyle,docgen]
 
 on:
   pull_request:
@@ -24,7 +24,58 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  mvn-verify-check:
+  get-noSnapshot-versions-from-dist:
+    runs-on: ubuntu-latest
+    outputs:
+      sparkHeadVersion: ${{ steps.noSnapshotVersionsStep.outputs.headVersion }}
+      sparkTailVersions: ${{ steps.noSnapshotVersionsStep.outputs.tailVersions }}
+    steps:
+      - uses: actions/checkout@v2 # refs/pull/:prNumber/merge
+
+      - name: Setup Java and Maven Env
+        uses: actions/setup-java@v3
+        with:
+          distribution: adopt
+          java-version: 8
+
+      - name: all noSnapshot versions
+        id: noSnapshotVersionsStep
+        run: |
+          set -x
+          . jenkins/version-def.sh
+          svArrBody=$(printf ",{\"spark-version\":\"%s\"}" "${SPARK_SHIM_VERSIONS_NOSNAPSHOTS_TAIL[@]}")
+          svArrBody=${svArrBody:1}
+          svJsonStr=$(printf {\"include\":[%s]} $svArrBody)
+          echo ::set-output name=headVersion::$SPARK_BASE_SHIM_VERSION
+          echo ::set-output name=tailVersions::$svJsonStr
+
+  package-aggregator:
+    needs: get-noSnapshot-versions-from-dist
+    strategy:
+      matrix: ${{ fromJSON(needs.get-noSnapshot-versions-from-dist.outputs.sparkTailVersions) }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2 # refs/pull/:prNumber/merge
+
+      - name: Setup Java and Maven Env
+        uses: actions/setup-java@v3
+        with:
+          distribution: adopt
+          java-version: 8
+
+      - name: package aggregator check
+        run: >
+          mvn -Dmaven.wagon.http.retryHandler.count=3 -B package -pl aggregator -am
+          -P 'individual,pre-merge'
+          -Dbuildver=${{ matrix.spark-version }}
+          -DskipTests
+          -Dskip
+          -Dmaven.javadoc.skip
+          -Dmaven.scalastyle.skip=true
+          -Drat.skip=true
+
+  verify-all-modules-with-headSparkVersion:
+    needs: get-noSnapshot-versions-from-dist
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2 # refs/pull/:prNumber/merge
@@ -36,5 +87,11 @@ jobs:
           java-version: 8
 
       # includes RAT, code style and doc-gen checks of default shim
-      - name: mvn verify check
-        run: mvn verify -P 'individual,pre-merge' -pl dist -am -DskipTests -Dskip -Dmaven.javadoc.skip
+      - name: verify all modules with lowest-supported Spark version
+        run: >
+          mvn -Dmaven.wagon.http.retryHandler.count=3 -B verify
+          -P 'individual,pre-merge'
+          -Dbuildver=${{ needs.get-noSnapshot-versions-from-dist.outputs.sparkHeadVersion }}
+          -DskipTests
+          -Dskip
+          -Dmaven.javadoc.skip

.gitignore

@@ -1,7 +1,7 @@
 # keep the lines below sorted
 
 # standard .bloop and version-specific .bloop3xy generated by buildall -gb
-.bloop*/
+.bloop*
 .cache
 .classpath
 .DS_Store