Merge pull request #2311 from NVIDIA/merge-branch-24.08-to-main

Merge branch-24.08 into main

.github/workflows/auto-merge.yml

@@ -18,12 +18,12 @@ name: auto-merge HEAD to BASE
 on:
   pull_request_target:
     branches:
-      - branch-24.06
+      - branch-24.08
     types: [closed]
 
 env:
-  HEAD: branch-24.06
-  BASE: branch-24.08
+  HEAD: branch-24.08
+  BASE: branch-24.10
 
 jobs:
   auto-merge:

.github/workflows/blossom-ci.yml

@@ -33,40 +33,43 @@ jobs:
       args: ${{ env.args }}
 
     # This job only runs for pull request comments
-    if: contains( '\
-      abellina,\
-      anfeng,\
-      firestarman,\
-      GaryShen2008,\
-      jlowe,\
-      mythrocks,\
-      nartal1,\
-      nvdbaranec,\
-      NvTimLiu,\
-      razajafri,\
-      revans2,\
-      rwlee,\
-      sameerz,\
-      tgravescs,\
-      wbo4958,\
-      wjxiz1992,\
-      sperlingxx,\
-      YanxuanLiu,\
-      hyperbolic2346,\
-      gerashegalov,\
-      ttnghia,\
-      nvliyuan,\
-      res-life,\
-      HaoYang670,\
-      NVnavkumar,\
-      yinqingh,\
-      thirtiseven,\
-      parthosa,\
-      liurenjie1024,\
-      binmahone,\
-      pmattione-nvidia,\
-      Feng-Jiang28,\
-      ', format('{0},', github.actor)) && github.event.comment.body == 'build'
+    if: |
+      github.event.comment.body == 'build' &&
+      (
+        github.actor == 'abellina' ||
+        github.actor == 'anfeng' ||
+        github.actor == 'firestarman' ||
+        github.actor == 'GaryShen2008' ||
+        github.actor == 'jlowe' ||
+        github.actor == 'mythrocks' ||
+        github.actor == 'nartal1' ||
+        github.actor == 'nvdbaranec' ||
+        github.actor == 'NvTimLiu' ||
+        github.actor == 'razajafri' ||
+        github.actor == 'revans2' ||
+        github.actor == 'rwlee' ||
+        github.actor == 'sameerz' ||
+        github.actor == 'tgravescs' ||
+        github.actor == 'wbo4958' ||
+        github.actor == 'wjxiz1992' ||
+        github.actor == 'sperlingxx' ||
+        github.actor == 'YanxuanLiu' ||
+        github.actor == 'hyperbolic2346' ||
+        github.actor == 'gerashegalov' ||
+        github.actor == 'ttnghia' ||
+        github.actor == 'nvliyuan' ||
+        github.actor == 'res-life' ||
+        github.actor == 'HaoYang670' ||
+        github.actor == 'NVnavkumar' ||
+        github.actor == 'yinqingh' ||
+        github.actor == 'thirtiseven' ||
+        github.actor == 'parthosa' ||
+        github.actor == 'liurenjie1024' ||
+        github.actor == 'binmahone' ||
+        github.actor == 'pmattione-nvidia' ||
+        github.actor == 'Feng-Jiang28' ||
+        github.actor == 'pxLi'
+      )
     steps:
       - name: Check if comment is issued by authorized person
         run: blossom-ci
@@ -81,15 +84,15 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: ${{ fromJson(needs.Authorization.outputs.args).repo }}
           ref: ${{ fromJson(needs.Authorization.outputs.args).ref }}
           lfs: 'true'
 
       # repo specific steps
       - name: Setup java
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: adopt
           java-version: 8

.gitmodules

@@ -1,4 +1,4 @@
 [submodule "thirdparty/cudf"]
 	path = thirdparty/cudf
 	url = https://github.com/rapidsai/cudf.git
-	branch = branch-24.06
+	branch = branch-24.08

CONTRIBUTING.md

@@ -61,15 +61,21 @@ the Docker container.
 The script passes all of its arguments onto the Maven command run inside the Docker container,
 so it should be invoked as one would invoke Maven, e.g.: `build/build-in-docker clean package`
 
+#### Using spark-rapids-jni Docker Container with other Repos
+
+Spark RAPIDS project spans multiple repos. Some issues are discovered in
+spark-rapids-jni but they need to be made easily reproducible in the cudf repo
+
+To this end export WORKDIR with the path pointing to a different repo
+
+```
+export WORKDIR=~/gits/rapidsai/cudf
+~/gits/NVIDIA/spark-rapids-jni/build/run-in-docker head README.md
+```
+
 ### cudf Submodule and Build
 
 [RAPIDS cuDF](https://github.com/rapidsai/cudf) is being used as a submodule in this project.
-Due to the lengthy build of libcudf, it is **not cleaned** during a normal Maven clean phase
-unless built using `build/build-in-docker`. `build/build-in-docker` uses `ccache` by default
-unless CCACHE_DISABLE=1 is set in the environment.
-
-`-Dlibcudf.clean.skip=false` can also be specified on the Maven command-line to force
-libcudf to be cleaned during the Maven clean phase.
 
 Currently libcudf is only configured once and the build relies on cmake to re-configure as needed.
 This is because libcudf currently is rebuilding almost entirely when it is configured with the same
@@ -93,7 +99,6 @@ to control aspects of the build:
 | `BUILD_BENCHMARKS`                   | Compile benchmarks                      | OFF     |
 | `BUILD_FAULTINJ`                     | Compile fault injection                 | ON      |
 | `libcudf.build.configure`            | Force libcudf build to configure        | false   |
-| `libcudf.clean.skip`                 | Whether to skip cleaning libcudf build  | true    |
 | `submodule.check.skip`               | Whether to skip checking git submodules | false   |
 
 
@@ -160,7 +165,7 @@ $ ./build/build-in-docker install ...
 ```
 
 Now cd to ~/repos/NVIDIA/spark-rapids and build with one of the options from
-[spark-rapids instructions](https://github.com/NVIDIA/spark-rapids/blob/branch-24.06/CONTRIBUTING.md#building-from-source).
+[spark-rapids instructions](https://github.com/NVIDIA/spark-rapids/blob/branch-24.08/CONTRIBUTING.md#building-from-source).
 
 ```bash
 $ ./build/buildall

build/run-in-docker

@@ -20,9 +20,11 @@
 
 set -e
 
-# Base paths relative to this script's location
-SCRIPTDIR=$(cd $(dirname $0); pwd)
-REPODIR=$SCRIPTDIR/..
+REPODIR_REL=$(git rev-parse --show-toplevel)
+REPODIR=$(realpath "$REPODIR_REL")
+GIT_COMMON_DIR_REL=$(git rev-parse --git-common-dir)
+GIT_COMMON_DIR=$(realpath "$GIT_COMMON_DIR_REL")
+WORKDIR=${WORKDIR:-$REPODIR}
 
 CUDA_VERSION=${CUDA_VERSION:-11.8.0}
 DOCKER_CMD=${DOCKER_CMD:-docker}
@@ -63,11 +65,11 @@ $DOCKER_CMD run $DOCKER_GPU_OPTS $DOCKER_RUN_EXTRA_ARGS -u $(id -u):$(id -g) --r
   -v "/etc/passwd:/etc/passwd:ro" \
   -v "/etc/shadow:/etc/shadow:ro" \
   -v "/etc/sudoers.d:/etc/sudoers.d:ro" \
-  -v "$REPODIR:$REPODIR:rw" \
+  -v "$GIT_COMMON_DIR:$GIT_COMMON_DIR:rw" \
+  -v "$WORKDIR:$WORKDIR:rw" \
   -v "$LOCAL_CCACHE_DIR:$LOCAL_CCACHE_DIR:rw" \
   -v "$LOCAL_MAVEN_REPO:$LOCAL_MAVEN_REPO:rw" \
-  --workdir "$REPODIR" \
-  -e CCACHE_DISABLE \
+  --workdir "$WORKDIR" \
   -e CCACHE_DIR="$LOCAL_CCACHE_DIR" \
   -e CMAKE_C_COMPILER_LAUNCHER="ccache" \
   -e CMAKE_CXX_COMPILER_LAUNCHER="ccache" \

ci/Jenkinsfile.premerge

@@ -57,7 +57,8 @@ pipeline {
     parameters {
         string(name: 'PARALLEL_LEVEL', defaultValue: '18',
             description: 'Parallel build cudf cpp with -DCPP_PARALLEL_LEVEL')
-        string(name: 'REF', defaultValue: '',
+        // Put a default value for REF to avoid error when running the pipeline manually
+        string(name: 'REF', defaultValue: 'main',
             description: 'Merged commit of specific PR')
         string(name: 'GITHUB_DATA', defaultValue: '',
             description: 'Json-formatted github data from upstream blossom-ci')

ci/check-cuda-dependencies.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+#
+# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# common script to help check if packaged *.so files have dynamical link to CUDA Runtime
+
+set -exo pipefail
+
+jar_path=$1
+tmp_path=/tmp/"jni-$(date "+%Y%m%d%H%M%S")"
+unzip -j "${jar_path}" "*64/Linux/*.so" -d "${tmp_path}"
+
+find "$tmp_path" -type f -name "*.so" | while read -r so_file; do
+    # Check if *.so file has a dynamic link to CUDA Runtime
+    if objdump -p "$so_file" | grep NEEDED | grep -qi cudart; then
+        echo "Dynamic link to CUDA Runtime found in $so_file..."
+        ldd "$so_file"
+        exit 1
+    else
+        echo "No dynamic link to CUDA Runtime found in $so_file"
+    fi
+done

ci/nightly-build.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -29,19 +29,24 @@ USE_GDS=${USE_GDS:-ON}
 USE_SANITIZER=${USE_SANITIZER:-ON}
 BUILD_FAULTINJ=${BUILD_FAULTINJ:-ON}
 ARM64=${ARM64:-false}
+artifact_suffix="${CUDA_VER}"
 
 profiles="source-javadoc"
 if [ "${ARM64}" == "true" ]; then
   profiles="${profiles},arm64"
   USE_GDS="OFF"
   USE_SANITIZER="ON"
   BUILD_FAULTINJ="OFF"
+  artifact_suffix="${artifact_suffix}-arm64"
 fi
 
 ${MVN} clean package ${MVN_MIRROR}  \
   -P${profiles} \
   -DCPP_PARALLEL_LEVEL=${PARALLEL_LEVEL} \
   -Dlibcudf.build.configure=true \
   -DUSE_GDS=${USE_GDS} -Dtest=*,!CuFileTest,!CudaFatalTest,!ColumnViewNonEmptyNullsTest \
-  -DBUILD_TESTS=ON -DBUILD_FAULTINJ=${BUILD_FAULTINJ} -Dcuda.version=$CUDA_VER \
+  -DBUILD_TESTS=ON -DBUILD_BENCHMARKS=ON -DBUILD_FAULTINJ=${BUILD_FAULTINJ} -Dcuda.version=$CUDA_VER \
   -DUSE_SANITIZER=${USE_SANITIZER}
+
+build_name=$(${MVN} help:evaluate -Dexpression=project.build.finalName -q -DforceStdout)
+. ci/check-cuda-dependencies.sh "target/${build_name}-${artifact_suffix}.jar"

ci/premerge-build.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2022-2024, NVIDIA CORPORATION. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -27,4 +27,8 @@ ${MVN} verify ${MVN_MIRROR} \
   -DCPP_PARALLEL_LEVEL=${PARALLEL_LEVEL} \
   -Dlibcudf.build.configure=true \
   -DUSE_GDS=ON -Dtest=*,!CuFileTest,!CudaFatalTest,!ColumnViewNonEmptyNullsTest \
-  -DBUILD_TESTS=ON
+  -DBUILD_TESTS=ON -DBUILD_BENCHMARKS=ON
+
+build_name=$(${MVN} help:evaluate -Dexpression=project.build.finalName -q -DforceStdout)
+cuda_version=$(${MVN} help:evaluate -Dexpression=cuda.version -q -DforceStdout)
+. ci/check-cuda-dependencies.sh "target/${build_name}-${cuda_version}.jar"

ci/submodule-sync.sh

@@ -88,8 +88,13 @@ else
   echo "Test failed, will update the result"
 fi
 
+build_name=$(${MVN} help:evaluate -Dexpression=project.build.finalName -q -DforceStdout)
+cuda_version=$(${MVN} help:evaluate -Dexpression=cuda.version -q -DforceStdout)
+. ci/check-cuda-dependencies.sh "target/${build_name}-${cuda_version}.jar"
+
+LIBCUDF_BUILD_PATH=$(${MVN} help:evaluate -Dexpression=libcudf.build.path -q -DforceStdout)
 # Extract the rapids-cmake sha1 that we need to pin too
-rapids_cmake_sha=$(git -C thirdparty/cudf/cpp/build/_deps/rapids-cmake-src/ rev-parse HEAD)
+rapids_cmake_sha=$(git -C ${LIBCUDF_BUILD_PATH}/_deps/rapids-cmake-src/ rev-parse HEAD)
 echo "Update rapids-cmake pinned SHA1 to ${rapids_cmake_sha}"
 echo "${rapids_cmake_sha}" > thirdparty/cudf-pins/rapids-cmake.sha