Merge branch 'branch-23.08' into test-libxml2.11

.github/workflows/build-test-publish-images.yml

@@ -6,6 +6,10 @@ on:
       build_type:
         required: true
         type: string
+      run_tests:
+        required: false
+        default: false
+        type: boolean
 
 defaults:
   run:
@@ -56,16 +60,16 @@ jobs:
           TAG_PREFIX=""
           if [ "${{ inputs.build_type }}" = "pull-request" ]; then
             pr_num="${GITHUB_REF_NAME##*/}"
-            BASE_TAG_PREFIX="rapidsai-${pr_num}-"
-            NOTEBOOKS_TAG_PREFIX="rapidsai-notebooks-${pr_num}-"
+            BASE_TAG_PREFIX="docker-${pr_num}-"
+            NOTEBOOKS_TAG_PREFIX="docker-notebooks-${pr_num}-"
           fi
           echo "BASE_TAG_PREFIX=${BASE_TAG_PREFIX}" | tee -a ${GITHUB_OUTPUT}
           echo "NOTEBOOKS_TAG_PREFIX=${NOTEBOOKS_TAG_PREFIX}" | tee -a ${GITHUB_OUTPUT}
       - name: Compute image repo
         id: compute-image-repo
         run: |
-          base_repo="rapidsai"
-          notebooks_repo="rapidsai-notebooks"
+          base_repo="base"
+          notebooks_repo="notebooks"
           if [ "${{ inputs.build_type }}" = "pull-request" ]; then
             base_repo="staging"
             notebooks_repo="staging"
@@ -112,19 +116,18 @@ jobs:
         ${{ needs.compute-matrix.outputs.BASE_TAG_PREFIX }}\
         ${{ needs.compute-matrix.outputs.RAPIDS_VER }}\
         ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-\
-        cuda${{ matrix.CUDA_VER }}-\
+        cuda${{ matrix.CUDA_TAG }}-\
         py${{ matrix.PYTHON_VER }}"
       NOTEBOOKS_TAG:
         "rapidsai/${{ needs.compute-matrix.outputs.NOTEBOOKS_IMAGE_REPO }}:\
         ${{ needs.compute-matrix.outputs.NOTEBOOKS_TAG_PREFIX }}\
         ${{ needs.compute-matrix.outputs.RAPIDS_VER }}\
         ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-\
-        cuda${{ matrix.CUDA_VER }}-\
+        cuda${{ matrix.CUDA_TAG }}-\
         py${{ matrix.PYTHON_VER }}"
   test:
     needs: [build, compute-matrix]
-    # TODO: nightly tests
-    if: inputs.build_type == 'pull-request'
+    if: inputs.run_tests
     strategy:
       matrix: ${{ fromJSON(needs.compute-matrix.outputs.TEST_MATRIX) }}
       fail-fast: false
@@ -171,7 +174,7 @@ jobs:
             ${{ needs.compute-matrix.outputs.BASE_TAG_PREFIX }}
             ${{ needs.compute-matrix.outputs.RAPIDS_VER }}
             ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-
-            cuda${{ matrix.CUDA_VER }}-
+            cuda${{ matrix.CUDA_TAG }}-
             py${{ matrix.PYTHON_VER }}
           )
           base_tag="$(printf %s "${base_tag_array[@]}" $'\n')" # Converts array to string w/o spaces
@@ -181,7 +184,7 @@ jobs:
             ${{ needs.compute-matrix.outputs.NOTEBOOKS_TAG_PREFIX }}
             ${{ needs.compute-matrix.outputs.RAPIDS_VER }}
             ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-
-            cuda${{ matrix.CUDA_VER }}-
+            cuda${{ matrix.CUDA_TAG }}-
             py${{ matrix.PYTHON_VER }}
           )
           notebooks_tag="$(printf %s "${notebooks_tag_array[@]}" $'\n')" # Converts array to string w/o spaces
@@ -197,7 +200,7 @@ jobs:
           docker manifest push ${notebooks_tag}
   delete-temp-images:
     if: always()
-    needs: [compute-matrix, build-multiarch-manifest]
+    needs: [compute-matrix, build, test, build-multiarch-manifest]
     strategy:
       matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
     runs-on: ubuntu-latest
@@ -218,7 +221,7 @@ jobs:
             ${{ needs.compute-matrix.outputs.BASE_TAG_PREFIX }}
             ${{ needs.compute-matrix.outputs.RAPIDS_VER }}
             ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-
-            cuda${{ matrix.CUDA_VER }}-
+            cuda${{ matrix.CUDA_TAG }}-
             py${{ matrix.PYTHON_VER }}
           )
           base_tag="$(printf %s "${base_tag_array[@]}" $'\n')" # Converts array to string w/o spaces
@@ -229,7 +232,7 @@ jobs:
             ${{ needs.compute-matrix.outputs.NOTEBOOKS_TAG_PREFIX }}
             ${{ needs.compute-matrix.outputs.RAPIDS_VER }}
             ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-
-            cuda${{ matrix.CUDA_VER }}-
+            cuda${{ matrix.CUDA_TAG }}-
             py${{ matrix.PYTHON_VER }}
           )
           notebooks_tag="$(printf %s "${notebooks_tag_array[@]}" $'\n')" # Converts array to string w/o spaces

.github/workflows/pr.yml

@@ -14,4 +14,5 @@ jobs:
     uses: ./.github/workflows/build-test-publish-images.yml
     with:
       build_type: pull-request
+      run_tests: true
     secrets: inherit

.github/workflows/publish.yml

@@ -7,6 +7,11 @@ on:
     tags:
       - v[0-9][0-9].[0-9][0-9].[0-9][0-9]
   workflow_dispatch:
+    inputs:
+      run_tests:
+        required: false
+        default: false
+        type: boolean
 
 concurrency:
   group: "${{ github.workflow }} @ ${{ github.ref }}"
@@ -17,4 +22,25 @@ jobs:
     uses: ./.github/workflows/build-test-publish-images.yml
     with:
       build_type: branch
+      run_tests: ${{ inputs.run_tests || false }}
     secrets: inherit
+  readme:
+    runs-on: ubuntu-latest
+    needs: docker
+    if: startsWith(github.ref, 'refs/tags/v')
+    strategy:
+      matrix:
+        repo_name:
+          - rapidsai/base
+          - rapidsai/notebooks
+    steps:
+      - name: checkout code
+        uses: actions/checkout@v3
+
+      - name: Update DockerHub README for ${{ matrix.repo_name }}
+        uses: peter-evans/dockerhub-description@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+          repository: ${{ matrix.repo_name }}
+          readme-filepath: dockerhub-readme.md

.github/workflows/test-notebooks.yml

@@ -44,6 +44,8 @@ permissions:
   security-events: none
   statuses: none
 
+env:
+  GHA_TOOLS_DIR: /home/rapids/.local/bin # Workaround for https://github.com/actions/runner/issues/2411
 
 jobs:
   test:
@@ -65,11 +67,12 @@ jobs:
     steps:
       - name: Install gha-tools
         run: |
-          mkdir -p /tmp/gha-tools
-          curl -s -L 'https://github.com/rapidsai/gha-tools/releases/latest/download/tools.tar.gz' | tar -xz -C /tmp/gha-tools
-          echo "/tmp/gha-tools" >> "${GITHUB_PATH}"
+          mkdir -p "$GHA_TOOLS_DIR"
+          curl -s -L 'https://github.com/rapidsai/gha-tools/releases/latest/download/tools.tar.gz' | tar -xz -C "$GHA_TOOLS_DIR"
       - name: Install git
-        run: rapids-mamba-retry install -n base --freeze-installed git
+        run: |
+          PATH="$PATH:$GHA_TOOLS_DIR"
+          rapids-mamba-retry install -n base --freeze-installed git
       - name: Checkout code
         uses: actions/checkout@v3
         with:
@@ -79,20 +82,25 @@ jobs:
         uses: rapidsai/shared-action-workflows/[email protected]
       - name: Print environment
         run: |
+          PATH="$PATH:$GHA_TOOLS_DIR"
           rapids-print-env
           rapids-logger "nvidia-smi"
           nvidia-smi
       - name: Test notebooks
         run: /home/rapids/test_notebooks.py -i /home/rapids/notebooks -o /home/rapids/notebooks_output
       - name: Install awscli
         if: '!cancelled()'
-        run: rapids-mamba-retry install -n base awscli
-      - uses: aws-actions/configure-aws-credentials@v1-node16
+        run: |
+          PATH="$PATH:$GHA_TOOLS_DIR"
+          rapids-mamba-retry install -n base awscli
+      - uses: aws-actions/configure-aws-credentials@v2
         if: '!cancelled()'
         with:
           role-to-assume: ${{ vars.AWS_ROLE_ARN }}
           aws-region: ${{ vars.AWS_REGION }}
           role-duration-seconds: 1800 # 30m
       - name: Upload notebook test outputs
         if: '!cancelled()'
-        run: rapids-upload-to-s3 test_notebooks_output_${{ inputs.ARCH }}_cuda${{ inputs.CUDA_VER }}_py${{ inputs.PYTHON_VER }}_${{ inputs.GPU }}-${{ inputs.DRIVER }}.tar.gz /home/rapids/notebooks_output
+        run: |
+          PATH="$PATH:$GHA_TOOLS_DIR"
+          rapids-upload-to-s3 test_notebooks_output_${{ inputs.ARCH }}_cuda${{ inputs.CUDA_VER }}_py${{ inputs.PYTHON_VER }}_${{ inputs.GPU }}-${{ inputs.DRIVER }}.tar.gz /home/rapids/notebooks_output

.github/workflows/update-dask-sql.yml

@@ -11,16 +11,18 @@ on:
 jobs:
   update-dask-sql:
     runs-on: ubuntu-latest
+    container:
+      image: rapidsai/ci:latest
     if: github.repository == 'rapidsai/docker'
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Get current dask-sql version
         id: current_version
-        uses: the-coding-turtle/[email protected]
-        with:
-          file: matrix.yaml
+        run: |
+          DASK_SQL_VER="$(yq -r '.DASK_SQL_VER.[0]' matrix.yaml)"
+          echo "DASK_SQL_VER=${DASK_SQL_VER}" | tee -a ${GITHUB_OUTPUT}
 
       - name: Get new dask-sql version
         id: new_version
@@ -35,8 +37,8 @@ jobs:
           FULL_VER: ${{ steps.current_version.outputs.DASK_SQL_VER }}
           FULL_NEW_VER: ${{ steps.new_version.outputs.version }}
         run: |
-          echo SHORT_VER=${FULL_VER::-2} >> $GITHUB_ENV
-          echo SHORT_NEW_VER=${FULL_NEW_VER::-2} >> $GITHUB_ENV
+          echo SHORT_VER=${FULL_VER%.*} >> $GITHUB_ENV
+          echo SHORT_NEW_VER=${FULL_NEW_VER%.*} >> $GITHUB_ENV
 
       - name: Find and replace full dask-sql version
         uses: jacobtomlinson/gha-find-replace@v3
@@ -54,8 +56,8 @@ jobs:
         uses: peter-evans/create-pull-request@v3
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          commit-message: Update `STABLE_DASK_SQL_VERSION` to `${{ steps.new_version.outputs.version }}`
-          title: Update `STABLE_DASK_SQL_VERSION` to `${{ steps.new_version.outputs.version }}`
+          commit-message: Update `DASK_SQL_VERSION` to `${{ steps.new_version.outputs.version }}`
+          title: Update `DASK_SQL_VERSION` to `${{ steps.new_version.outputs.version }}`
           author: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
           branch: "upgrade-dask-sql"
           body: |

Dockerfile

@@ -1,11 +1,11 @@
 # syntax=docker/dockerfile:1
 
-ARG CUDA_VER=11.8.0
+ARG CUDA_VER=12.0.1
 ARG PYTHON_VER=3.10
 ARG LINUX_VER=ubuntu22.04
 
 ARG RAPIDS_VER=23.08
-ARG DASK_SQL_VER=2023.6.0
+ARG DASK_SQL_VER=2023.8.0
 
 # Gather dependency information
 FROM rapidsai/ci:latest AS dependencies
@@ -31,7 +31,7 @@ ARG PYTHON_VER
 ARG RAPIDS_VER
 ARG DASK_SQL_VER
 
-RUN useradd -rm -d /home/rapids -s /bin/bash -g conda -u 1000 rapids
+RUN useradd -rm -d /home/rapids -s /bin/bash -g conda -u 1001 rapids
 
 USER rapids
 
@@ -70,10 +70,11 @@ RUN mamba env update -n base -f test_notebooks_dependencies.yaml \
     && conda clean -afy
 
 RUN mamba install -y -n base \
-        jupyterlab \
+        "jupyterlab=3" \
         dask-labextension \
-        jupyterlab-nvdashboard \
-    && conda clean -afy
+    && pip install jupyterlab-nvdashboard \
+    && conda clean -afy \
+    && pip cache purge
 
 ENV DASK_LABEXTENSION__FACTORY__MODULE="dask_cuda"
 ENV DASK_LABEXTENSION__FACTORY__CLASS="LocalCUDACluster"

ci/compute-matrix.jq

@@ -2,6 +2,7 @@ def compute_arch($x):
   ["amd64"] |
   if
     $x.LINUX_VER != "ubuntu20.04" # Dask-sql arm64 requires glibc >=2.32
+    and $x.CUDA_VER < "12.0" # arm64 packages not available for CUDA 12.0 yet
   then
     . + ["arm64"]
   else
@@ -19,6 +20,9 @@ def compute_ubuntu_version($x):
   end |
   $x + {LINUX_VER: .};
 
+def compute_cuda_tag($x):
+  $x + {CUDA_TAG: $x.CUDA_VER | split(".") | [.[0], .[1]] | join(".") };
+
 # Checks the current entry to see if it matches the given exclude
 def matches($entry; $exclude):
   all($exclude | to_entries | .[]; $entry[.key] == .value);
@@ -41,7 +45,8 @@ def compute_matrix($input):
     combinations |
     lists2dict($matrix_keys; .) |
     compute_ubuntu_version(.) |
+    compute_cuda_tag(.) |
     filter_excludes(.; $excludes) |
     compute_arch(.)
   ] |
-  {include: .};
+  {include: .};

ci/release/update-version.sh

@@ -32,3 +32,7 @@ sed_runner "s/ARG RAPIDS_VER=.*/ARG RAPIDS_VER=${NEXT_SHORT_TAG}/g" Dockerfile
 for FILE in .github/workflows/*.yml; do
   sed_runner "/shared-action-workflows/ s/@.*/@branch-${NEXT_SHORT_TAG}/g" "${FILE}"
 done
+
+sed_runner "s/v[[:digit:]]\+\.[[:digit:]]\+/v${NEXT_SHORT_TAG}/g" dockerhub-readme.md
+sed_runner "s/[[:digit:]]\+\.[[:digit:]]\+-cuda/${NEXT_SHORT_TAG}-cuda/g" dockerhub-readme.md
+sed_runner "s/[[:digit:]]\+\.[[:digit:]]\+a-cuda/${NEXT_SHORT_TAG}a-cuda/g" dockerhub-readme.md

context/notebooks.sh

@@ -8,7 +8,7 @@
 
 set -euo pipefail
 
-NOTEBOOK_REPOS=(cudf cuml cugraph cuxfilter cuspatial cusignal xgboost-conda)
+NOTEBOOK_REPOS=(cudf cuml cugraph cuspatial)
 
 mkdir -p /notebooks /dependencies
 for REPO in "${NOTEBOOK_REPOS[@]}"; do
@@ -26,4 +26,4 @@ for REPO in "${NOTEBOOK_REPOS[@]}"; do
 done
 
 pushd "/dependencies"
-conda-merge ./*.yaml > /test_notebooks_dependencies.yaml
+conda-merge ./*.yaml > /test_notebooks_dependencies.yaml