Add rapids-http-path script

tools/_rapids-download-from-s3

@@ -4,20 +4,20 @@
 # The script can be used locally and in CI.
 # Positional Arguments:
 #   1) path to s3 artifact
-#   2) location to untar it to
+#   2) http path to s3 artifact
+#   3) location to untar it to
 set -euo pipefail
 source rapids-constants
 
 s3_dl_path="$1"
-untar_dest="$2"
+http_dl_path="$2"
+untar_dest="$3"
 
 mkdir -p "${untar_dest}"
 
 if [ "${CI:-false}" = "false" ]; then
-  # shellcheck disable=SC2001
-  s3_dl_path=$(echo "${s3_dl_path}" | sed "s|s3://${RAPIDS_DOWNLOADS_BUCKET}|https://${RAPIDS_DOWNLOADS_DOMAIN}|")
-  rapids-echo-stderr "Downloading and decompressing ${s3_dl_path} into ${untar_dest}"
-  wget -qO- "${s3_dl_path}" | tar xzf - -C "${untar_dest}"
+  rapids-echo-stderr "Downloading and decompressing ${http_dl_path} into ${untar_dest}"
+  wget -qO- "${http_dl_path}" | tar xzf - -C "${untar_dest}"
 else
   rapids-echo-stderr "Downloading and decompressing ${s3_dl_path} into ${untar_dest}"
   aws s3 cp --only-show-errors "${s3_dl_path}" - | tar xzf - -C "${untar_dest}"

tools/_rapids-path

@@ -0,0 +1,43 @@
+#!/bin/bash
+# _rapids-path
+# A utility script to generate the URI path.
+
+set -euo pipefail
+source rapids-constants
+
+repo_name="${RAPIDS_REPOSITORY##*/}"
+
+s3_directory_id=""
+s3_prefix=""
+
+case "${RAPIDS_BUILD_TYPE}" in
+  pull-request)
+    # For PRs, we are using the 'trusted jobs on untrusted forks' paradigm
+    # by copying PRs to branches named pull-request/$prnum
+    # more info:
+    #    CopyPRs plugin in ops-bot: https://github.com/rapidsai/ops-bot#plugins
+    #    https://circleci.com/blog/triggering-trusted-ci-jobs-on-untrusted-forks/
+    s3_directory_id="${RAPIDS_REF_NAME##*/}"
+    s3_prefix="ci"
+    ;;
+  branch)
+    s3_directory_id="${RAPIDS_REF_NAME}"
+    s3_prefix="ci"
+    ;;
+  nightly)
+    s3_directory_id="${RAPIDS_NIGHTLY_DATE}"
+    s3_prefix="nightly"
+    ;;
+  *)
+    rapids-echo-stderr "please pass a valid RAPIDS_BUILD_TYPE"
+    exit 1
+    ;;
+esac
+
+short_hash=${RAPIDS_SHA:0:7}
+
+uri_path="${s3_prefix}/${repo_name}/"
+if [[ "${RAPIDS_BUILD_TYPE}" != "nightly" ]]; then
+  uri_path+="${RAPIDS_BUILD_TYPE}/"
+fi
+uri_path+="${s3_directory_id}/${short_hash}/"

tools/rapids-download-from-s3

@@ -13,6 +13,7 @@ if [ -z "$1" ] || [ -z "$2" ]; then
 fi
 
 s3_dl_path="$(rapids-s3-path)$1"
+http_dl_path="$(rapids-http-path)$1"
 untar_dest="$2"
 
-_rapids-download-from-s3 "$s3_dl_path" "$untar_dest"
+_rapids-download-from-s3 "$s3_dl_path" "$http_dl_path" "$untar_dest"

tools/rapids-get-artifact

@@ -9,6 +9,7 @@ set -euo pipefail
 source rapids-constants
 
 s3_dl_path="s3://${RAPIDS_DOWNLOADS_BUCKET}/${1}"
+http_dl_path="https://${RAPIDS_DOWNLOADS_DOMAIN}/${1}"
 untar_dest=$(mktemp -d)
 
-_rapids-download-from-s3 "$s3_dl_path" "$untar_dest"
+_rapids-download-from-s3 "$s3_dl_path" "$http_dl_path" "$untar_dest"

tools/rapids-http-path

@@ -0,0 +1,28 @@
+#!/bin/bash
+# rapids-http-path
+#
+# Calls _rapids-path and prepends the https://downloads.rapids.ai/ string
+#
+# A utility script that examines environment variables provided
+# by GitHub Actions to print out a HTTP path where the expected artifact
+# should be.
+#
+# The output format should be one of the following:
+#
+## For PR builds:
+## https://downloads.rapids.ai/ci/<REPO_NAME>/pull-request/<PR_NUMBER>/<SHORT_HASH>/
+
+## For branch builds:
+## https://downloads.rapids.ai/ci/<REPO_NAME>/branch/<BRANCH_NAME>/<SHORT_HASH>/
+
+## For nightly builds:
+## https://downloads.rapids.ai/nightly/<REPO_NAME>/<DATE>/<SHORT_HASH>/
+
+source _rapids-path
+export RAPIDS_SCRIPT_NAME="rapids-http-path"
+
+RAPIDS_DOWNLOADS_DOMAIN="${RAPIDS_DOWNLOADS_DOMAIN:-downloads.rapids.ai}"
+
+http_path="https://${RAPIDS_DOWNLOADS_DOMAIN}/${uri_path}"
+
+echo -n "${http_path}"

tools/rapids-s3-path

@@ -1,4 +1,8 @@
 #!/bin/bash
+# rapids-s3-path
+#
+# Calls _rapids-path and prepends the s3://rapids-downloads/ string
+#
 # A utility script that examines environment variables provided
 # by GitHub Actions to print out an S3 path where the expected artifact
 # should be.
@@ -13,45 +17,11 @@
 
 ## For nightly builds:
 ## s3://rapids-downloads/nightly/<REPO_NAME>/<DATE>/<SHORT_HASH>/
-set -euo pipefail
-source rapids-constants
+source _rapids-path
 export RAPIDS_SCRIPT_NAME="rapids-s3-path"
 
-repo_name="${RAPIDS_REPOSITORY##*/}"
-
-s3_directory_id=""
-s3_prefix=""
-
-case "${RAPIDS_BUILD_TYPE}" in
-  pull-request)
-    # For PRs, we are using the 'trusted jobs on untrusted forks' paradigm
-    # by copying PRs to branches named pull-request/$prnum
-    # more info:
-    #    CopyPRs plugin in ops-bot: https://github.com/rapidsai/ops-bot#plugins
-    #    https://circleci.com/blog/triggering-trusted-ci-jobs-on-untrusted-forks/
-    s3_directory_id="${RAPIDS_REF_NAME##*/}"
-    s3_prefix="ci"
-    ;;
-  branch)
-    s3_directory_id="${RAPIDS_REF_NAME}"
-    s3_prefix="ci"
-    ;;
-  nightly)
-    s3_directory_id="${RAPIDS_NIGHTLY_DATE}"
-    s3_prefix="nightly"
-    ;;
-  *)
-    rapids-echo-stderr "please pass a valid RAPIDS_BUILD_TYPE"
-    exit 1
-    ;;
-esac
-
-short_hash=${RAPIDS_SHA:0:7}
+RAPIDS_DOWNLOADS_BUCKET="${RAPIDS_DOWNLOADS_BUCKET:-rapids-downloads}"
 
-s3_path="s3://${RAPIDS_DOWNLOADS_BUCKET}/${s3_prefix}/${repo_name}/"
-if [[ "${RAPIDS_BUILD_TYPE}" != "nightly" ]]; then
-  s3_path+="${RAPIDS_BUILD_TYPE}/"
-fi
-s3_path+="${s3_directory_id}/${short_hash}/"
+s3_path="s3://${RAPIDS_DOWNLOADS_BUCKET}/${uri_path}"
 
 echo -n "${s3_path}"

tools/rapids-upload-artifacts-dir

@@ -29,5 +29,5 @@ else
 fi
 
 echo ""
-ARTIFACTS_URL=$(rapids-s3-path | sed "s|s3://${RAPIDS_DOWNLOADS_BUCKET}|https://${RAPIDS_DOWNLOADS_DOMAIN}|")
+ARTIFACTS_URL=$(rapids-http-path)
 echo "Browse all uploads (NVIDIA Employee VPN Required): ${ARTIFACTS_URL}"