Merge pull request #946 from NvTimLiu/release-tmp

Merge branch 'branch-23.02' into main [skip ci]

.github/workflows/add-to-project.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2022, NVIDIA CORPORATION.
+# Copyright (c) 2022-2023, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -28,8 +28,8 @@ jobs:
     name: Add new issues and pull requests to project
     runs-on: ubuntu-latest
     steps:
-      # TODO: update project version when new release supports node 16 instead of 12
-      - uses: actions/[email protected]
+      - uses: actions/[email protected]
         with:
           project-url: https://github.com/orgs/NVIDIA/projects/4
           github-token: ${{ secrets.PROJECT_TOKEN }}
+

.github/workflows/auto-merge.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2022, NVIDIA CORPORATION.
+# Copyright (c) 2022-2023, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,12 +18,12 @@ name: auto-merge HEAD to BASE
 on:
   pull_request_target:
     branches:
-      - branch-22.12
+      - branch-23.02
     types: [closed]
 
 env:
-  HEAD: branch-22.12
-  BASE: branch-23.02
+  HEAD: branch-23.02
+  BASE: branch-23.04
 
 jobs:
   auto-merge:
@@ -60,3 +60,4 @@ jobs:
           HEAD: bot-auto-merge-${{ env.HEAD }}
           BASE: ${{ env.BASE }}
           TOKEN: ${{ secrets.AUTOMERGE_TOKEN }}
+

.gitmodules

@@ -1,4 +1,4 @@
 [submodule "thirdparty/cudf"]
 	path = thirdparty/cudf
 	url = https://github.com/rapidsai/cudf.git
-	branch = branch-22.12
+	branch = branch-23.02

CONTRIBUTING.md

@@ -144,7 +144,7 @@ $ ./build/build-in-docker install ...
 ```
 
 Now cd to ~/repos/NVIDIA/spark-rapids and build with one of the options from
-[spark-rapids instructions](https://github.com/NVIDIA/spark-rapids/blob/branch-22.12/CONTRIBUTING.md#building-from-source).
+[spark-rapids instructions](https://github.com/NVIDIA/spark-rapids/blob/branch-23.02/CONTRIBUTING.md#building-from-source).
 
 ```bash
 $ ./build/buildall

build/run-in-docker

@@ -24,7 +24,7 @@ set -e
 SCRIPTDIR=$(cd $(dirname $0); pwd)
 REPODIR=$SCRIPTDIR/..
 
-CUDA_VERSION=${CUDA_VERSION:-11.5.0}
+CUDA_VERSION=${CUDA_VERSION:-11.8.0}
 DOCKER_CMD=${DOCKER_CMD:-docker}
 DOCKER_BUILD_EXTRA_ARGS=${DOCKER_BUILD_EXTRA_ARGS:-""}
 DOCKER_RUN_EXTRA_ARGS=${DOCKER_RUN_EXTRA_ARGS:-""}
@@ -47,9 +47,10 @@ fi
 
 if (( $# == 0 )); then
   # no arguments gets an interactive shell
-  DOCKER_ARGS="/bin/bash"
+  DOCKER_OPTS="${DOCKER_OPTS} -it"
+  RUN_CMD="/bin/bash"
 else
-  DOCKER_ARGS="$*"
+  RUN_CMD="$*"
 fi
 
 $DOCKER_CMD run $DOCKER_GPU_OPTS $DOCKER_RUN_EXTRA_ARGS -u $(id -u):$(id -g) --rm \
@@ -71,5 +72,6 @@ $DOCKER_CMD run $DOCKER_GPU_OPTS $DOCKER_RUN_EXTRA_ARGS -u $(id -u):$(id -g) --r
   -e CUDA_VISIBLE_DEVICES \
   -e PARALLEL_LEVEL \
   -e VERBOSE \
+  $DOCKER_OPTS \
   $SPARK_IMAGE_NAME \
-  scl enable devtoolset-9 "$DOCKER_ARGS"
+  scl enable devtoolset-9 "$RUN_CMD"

ci/Dockerfile

@@ -17,11 +17,11 @@
 ###
 # Build the image for spark-rapids-jni development environment.
 #
-# Arguments: CUDA_VERSION=11.5.0
+# Arguments: CUDA_VERSION=11.8.0
 #
 ###
-ARG CUDA_VERSION=11.5.0
-FROM gpuci/cuda:$CUDA_VERSION-devel-centos7
+ARG CUDA_VERSION=11.8.0
+FROM nvidia/cuda:$CUDA_VERSION-devel-centos7
 ARG DEVTOOLSET_VERSION=9
 ### Install basic requirements
 RUN yum install -y centos-release-scl
@@ -68,3 +68,6 @@ RUN cd /usr/local && wget --quiet https://boostorg.jfrog.io/artifactory/main/rel
   ./b2 install --prefix=/usr/local --with-filesystem --with-system && \
    cd /usr/local && \
    rm -rf boost_1_79_0
+
+# disable cuda container constraints to allow running w/ elder drivers on data-center GPUs
+ENV NVIDIA_DISABLE_REQUIRE="true"

ci/Jenkinsfile.premerge

@@ -30,7 +30,7 @@ import ipp.blossom.*
 
 def githubHelper // blossom github helper
 def TEMP_IMAGE_BUILD = true
-def IMAGE_PREMERGE = "${common.ARTIFACTORY_NAME}/sw-spark-docker/plugin-jni:centos7-cuda11.5.0-blossom"
+def IMAGE_PREMERGE = "${common.ARTIFACTORY_NAME}/sw-spark-docker/plugin-jni:centos7-cuda11.8.0-blossom"
 def cpuImage = pod.getCPUYAML(IMAGE_PREMERGE)
 def PREMERGE_DOCKERFILE = 'ci/Dockerfile'
 def PREMERGE_TAG
@@ -149,7 +149,7 @@ pipeline {
                         }
 
                         if (TEMP_IMAGE_BUILD) {
-                            PREMERGE_TAG = "centos7-cuda11.5.0-blossom-dev-${BUILD_TAG}"
+                            PREMERGE_TAG = "centos7-cuda11.8.0-blossom-dev-${BUILD_TAG}"
                             IMAGE_PREMERGE = "${ARTIFACTORY_NAME}/sw-spark-docker-local/plugin-jni:${PREMERGE_TAG}"
                             docker.build(IMAGE_PREMERGE, "-f ${PREMERGE_DOCKERFILE} -t $IMAGE_PREMERGE .")
                             uploadDocker(IMAGE_PREMERGE)

ci/deploy.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -23,25 +23,34 @@
 #   SIGN_FILE: true/false, whether to sign the jar/pom file to de deployed
 #
 # Used environment(s):
+#   OUT_PATH:       The path where jar files are
+#   SIGN_TOOL:      Tools to sign files, e.g., gpg, nvsec, only required when $1 is 'true'
 #   CLASSIFIERS:    The classifier list of the jars to be deployed
 #   SERVER_ID:      The repository id for this deployment.
-#   SERVER_URL:     The url where to deploy artifacts.
-#   GPG_PASSPHRASE: The passphrase used to sign files, only required when <SIGN_FILE> is true.
+#   SERVER_URL:     The URL where to deploy artifacts.
+#   GPG_PASSPHRASE: The passphrase used to sign files, only required when <SIGN_TOOL> is gpg
+#   POM_FILE:       Project pom file to be deployed
+#   MVN_SETTINGS:   Maven configuration file
+#
 ###
 
 set -ex
 
 SIGN_FILE=$1
-OUT_PATH=$WORKSPACE/$OUT
+SIGN_TOOL=${SIGN_TOOL:-"gpg"}
+OUT_PATH=${OUT_PATH:-"target"}
+POM_FILE=${POM_FILE:-"pom.xml"}
+MVN_SETTINGS=${MVN_SETTINGS:-"ci/settings.xml"}
 
-cd $WORKSPACE/
-REL_VERSION=$(mvn exec:exec -q --non-recursive -Dexec.executable=echo -Dexec.args='${project.version}')
+MVN="mvn -Dmaven.wagon.http.retryHandler.count=3 -DretryFailedDeploymentCount=3 -B -s $MVN_SETTINGS"
+REL_ARTIFACTID=$($MVN exec:exec -f $POM_FILE -q --non-recursive -Dexec.executable=echo -Dexec.args='${project.artifactId}')
+REL_VERSION=$($MVN exec:exec -f $POM_FILE -q --non-recursive -Dexec.executable=echo -Dexec.args='${project.version}')
 
 echo "REL_VERSION: $REL_VERSION, OUT_PATH: $OUT_PATH \
-SERVER_URL: $SERVER_URL, SERVER_ID: $SERVER_ID"
+        SERVER_URL: $SERVER_URL, SERVER_ID: $SERVER_ID"
 
 ###### Build types/files from classifiers ######
-FPATH="$OUT_PATH/spark-rapids-jni-$REL_VERSION"
+FPATH="$OUT_PATH/$REL_ARTIFACTID-$REL_VERSION"
 CLASS_TYPES=''
 CLASS_FILES=''
 ORI_IFS="$IFS"
@@ -61,19 +70,32 @@ FIRST_FILE=${CLASS_FILES%%,*}
 cp -f "$FIRST_FILE" "$FPATH.jar"
 
 ###### Build the deploy command ######
-MVN="mvn -Dmaven.wagon.http.retryHandler.count=3 -DretryFailedDeploymentCount=3 -B"
 if [ "$SIGN_FILE" == true ]; then
-    DEPLOY_CMD="$MVN -B gpg:sign-and-deploy-file -s ci/settings.xml \
-      -Dsources=$FPATH-sources.jar -Djavadoc=$FPATH-javadoc.jar -Dgpg.passphrase=$GPG_PASSPHRASE"
+    case $SIGN_TOOL in
+        nvsec)
+            DEPLOY_CMD="$MVN gpg:sign-and-deploy-file -Dgpg.executable=nvsec_sign"
+            ;;
+        gpg)
+            DEPLOY_CMD="$MVN gpg:sign-and-deploy-file -Dgpg.passphrase=$GPG_PASSPHRASE "
+            ;;
+        *)
+            echo "Error unsupported sign type : $SIGN_TYPE !"
+            echo "Please set variable SIGN_TOOL 'nvsec'or 'gpg'"
+            exit -1
+            ;;
+    esac
 else
-    DEPLOY_CMD="$MVN -B deploy:deploy-file -s ci/settings.xml \
-      -Dsources=$FPATH-sources.jar -Djavadoc=$FPATH-javadoc.jar"
+    DEPLOY_CMD="$MVN -B deploy:deploy-file"
 fi
+
+DEPLOY_CMD="$DEPLOY_CMD -Durl=$SERVER_URL -DrepositoryId=$SERVER_ID -DpomFile=$POM_FILE"
 echo "Deploy CMD: $DEPLOY_CMD"
 
 ###### Deploy spark-rapids-jni jar with all its additions ######
-$DEPLOY_CMD -Durl=$SERVER_URL -DrepositoryId=$SERVER_ID \
-            -Dfile=$FPATH.jar -DpomFile=pom.xml \
+$DEPLOY_CMD -Dfile=$FPATH.jar \
+            -DpomFile=pom.xml \
+            -Dsources=$FPATH-sources.jar \
+            -Djavadoc=$FPATH-javadoc.jar \
             -Dfiles=$CLASS_FILES \
             -Dtypes=$CLASS_TYPES \
             -Dclassifiers=$CLASSIFIERS

pom.xml

@@ -21,7 +21,7 @@
 
   <groupId>com.nvidia</groupId>
   <artifactId>spark-rapids-jni</artifactId>
-  <version>22.12.0</version>
+  <version>23.02.0</version>
   <packaging>jar</packaging>
   <name>RAPIDS Accelerator JNI for Apache Spark</name>
   <description>