Skip to content

Commit

Permalink
Explicitly select for CUDA-suffixed dependencies in `rapids-make-pip-…
Browse files Browse the repository at this point in the history 
…dependencies` (#365)

* select CUDA-suffixed dependencies for pip env

* change more places

* expand comment

* one more place

* bump version

* move matrix entry to argument

* always keep cuda, arch, and py in the rapids-dependency-file-generator --matrix

* Update features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh

Co-authored-by: Paul Taylor <[email protected]>

* update make-conda-dependencies too

---------

Co-authored-by: Paul Taylor <[email protected]>
  • Loading branch information
@jameslamb @trxcllnt
jameslamb and trxcllnt authored Jul 23, 2024
1 parent 623a7bf commit e414ab9
Show file tree
Hide file tree
Showing 3 changed files with 77 additions and 37 deletions.
2 changes: 1 addition & 1 deletion features/src/rapids-build-utils/devcontainer-feature.json
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "NVIDIA RAPIDS devcontainer build utilities",
"id": "rapids-build-utils",
"version": "24.8.14",
"version": "24.8.15",
"description": "A feature to install the RAPIDS devcontainer build utilities",
"containerEnv": {
"BASH_ENV": "/etc/bash.bash_env"
Expand Down
50 changes: 32 additions & 18 deletions features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@
# -e,--exclude <file> Path(s) to requirement files of packages to exclude.
# -i,--include <file> Path(s) to requirement files of packages to include.
# -k,--key <key> Only include the key(s)
# --matrix-entry <entry> Matrix entries, in the form 'key=value' to be added to the '--matrix' arg
# of rapids-dependency-file-generator.
# (can be passed multiple times)
# @_include_value_options rapids-list-repos -h | tail -n+2 | head -n-3;
# --repo <repo> Only include dependencies for repo(s).
# (default: all repositories)
Expand Down Expand Up @@ -41,6 +44,7 @@ make_conda_dependencies() {
test ${#exclude[@]} -eq 0 && exclude=();
test ${#include[@]} -eq 0 && include=();
test ${#key[@]} -eq 0 && key=(all);
test ${#matrix_entry[@]} -eq 0 && matrix_entry=();

local -a _exclude=();
local exc; for exc in "${exclude[@]}"; do
Expand All @@ -58,6 +62,16 @@ make_conda_dependencies() {
local python_version="${PYTHON_VERSION:-$(python3 --version 2>&1 | cut -d' ' -f2)}";
python_version="$(cut -d'.' -f3 --complement <<< "${python_version}")";

local -a _matrix_selectors=(
arch="$(uname -m)"
cuda="${cuda_version}"
py="${python_version}"
);

# add extra arguments (if there are conflicts, e.g. 'py=3.10;py=3.11', it's fine... the last one will win)
test ${#matrix_entry[@]} -gt 0 && _matrix_selectors+=("${matrix_entry[@]}");
local -r matrix_selectors=$(IFS=";"; echo "${_matrix_selectors[*]}")

local conda_env_yamls=();

local i;
Expand All @@ -78,12 +92,12 @@ make_conda_dependencies() {
for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do
local file="/tmp/${!repo_name}.${repo_keys[$keyi]}.env.yaml";
conda_env_yamls+=("${file}");
generate_env_yaml \
"${file}" \
--file-key "${repo_keys[$keyi]}" \
--output conda \
--config ~/"${!repo_path}/dependencies.yaml" \
--matrix "arch=$(uname -m);cuda=${cuda_version};py=${python_version}" \
generate_env_yaml \
"${file}" \
--file-key "${repo_keys[$keyi]}" \
--output conda \
--config ~/"${!repo_path}/dependencies.yaml" \
--matrix "${matrix_selectors}" \
;
done

Expand All @@ -100,12 +114,12 @@ make_conda_dependencies() {
for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do
local file="/tmp/${!repo_name}.lib${!cpp_name}.${repo_keys[$keyi]}.env.yaml";
conda_env_yamls+=("${file}");
generate_env_yaml \
"${file}" \
--file-key "${repo_keys[$keyi]}" \
--output conda \
--config ~/"${!repo_path}/dependencies.yaml" \
--matrix "arch=$(uname -m);cuda=${cuda_version};py=${python_version}" \
generate_env_yaml \
"${file}" \
--file-key "${repo_keys[$keyi]}" \
--output conda \
--config ~/"${!repo_path}/dependencies.yaml" \
--matrix "${matrix_selectors}" \
;
done
done
Expand All @@ -123,12 +137,12 @@ make_conda_dependencies() {
for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do
local file="/tmp/${!repo_name}.${!py_name}.${repo_keys[$keyi]}.env.yaml";
conda_env_yamls+=("${file}");
generate_env_yaml \
"${file}" \
--file-key "${repo_keys[$keyi]}" \
--output conda \
--config ~/"${!repo_path}/dependencies.yaml" \
--matrix "arch=$(uname -m);cuda=${cuda_version};py=${python_version}" \
generate_env_yaml \
"${file}" \
--file-key "${repo_keys[$keyi]}" \
--output conda \
--config ~/"${!repo_path}/dependencies.yaml" \
--matrix "${matrix_selectors}" \
;
done
done
Expand Down
62 changes: 44 additions & 18 deletions features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@
# -e,--exclude <file> Path(s) to requirement files of packages to exclude.
# -i,--include <file> Path(s) to requirement files of packages to include.
# -k,--key <key> Only include the key(s)
# --matrix-entry <entry> Matrix entries, in the form 'key=value' to be added to the '--matrix' arg
# of rapids-dependency-file-generator.
# (can be passed multiple times)
# @_include_value_options rapids-list-repos -h | tail -n+2 | head -n-3;
# --repo <repo> Only include dependencies for repo(s).
# (default: all repositories)
Expand Down Expand Up @@ -41,6 +44,7 @@ make_pip_dependencies() {
test ${#exclude[@]} -eq 0 && exclude=();
test ${#include[@]} -eq 0 && include=();
test ${#key[@]} -eq 0 && key=(py_build py_run py_test all);
test ${#matrix_entry[@]} -eq 0 && matrix_entry=();
test ${#requirement[@]} -eq 0 && requirement=();

local -a _exclude=();
Expand All @@ -59,6 +63,28 @@ make_pip_dependencies() {
local python_version="${PYTHON_VERSION:-$(python3 --version 2>&1 | cut -d' ' -f2)}";
python_version="$(cut -d'.' -f3 --complement <<< "${python_version}")";

# Why default to cuda_suffixed=true?
#
# Projects that depend on different pip libraries across different CUDA versions
# (e.g. 'cudf' only depending on 'pynvjitlink' from CUDA 12.0 onwards), split up their
# dependency lists with 'cuda_suffixed={true,false}'.
#
# Here we want the suffixed versions (like 'pynvjitlink-cu12').
#
# It's ok for other RAPIDS libraries to end up in this list (like 'rmm-cu12')... in builds
# where those are also being built in the devcontainer, they'll be filtered out via
# inclusion in the 'pip_noinstall' list below.
local -a _matrix_selectors=(
arch="$(uname -m)"
cuda="${cuda_version}"
cuda_suffixed=true
py="${python_version}"
);

# add extra arguments (if there are conflicts, e.g. 'py=3.10;py=3.11', it's fine... the last one will win)
test ${#matrix_entry[@]} -gt 0 && _matrix_selectors+=("${matrix_entry[@]}");
local -r matrix_selectors=$(IFS=";"; echo "${_matrix_selectors[*]}")

local pip_reqs_txts=();

eval "$(rapids-list-repos "${OPTS[@]}")";
Expand All @@ -81,12 +107,12 @@ make_pip_dependencies() {
for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do
local file="/tmp/${!repo_name}.${repo_keys[$keyi]}.requirements.txt";
pip_reqs_txts+=("${file}");
generate_requirements \
"${file}" \
--file-key "${repo_keys[$keyi]}" \
--output requirements \
--config ~/"${!repo_path}/dependencies.yaml" \
--matrix "arch=$(uname -m);cuda=${cuda_version};py=${python_version}" \
generate_requirements \
"${file}" \
--file-key "${repo_keys[$keyi]}" \
--output requirements \
--config ~/"${!repo_path}/dependencies.yaml" \
--matrix "${matrix_selectors}" \
;
done

Expand All @@ -103,12 +129,12 @@ make_pip_dependencies() {
for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do
local file="/tmp/${!repo_name}.lib${!cpp_name}.${repo_keys[$keyi]}.requirements.txt";
pip_reqs_txts+=("${file}");
generate_requirements \
"${file}" \
--file-key "${repo_keys[$keyi]}" \
--output requirements \
--config ~/"${!repo_path}/dependencies.yaml" \
--matrix "arch=$(uname -m);cuda=${cuda_version};py=${python_version}" \
generate_requirements \
"${file}" \
--file-key "${repo_keys[$keyi]}" \
--output requirements \
--config ~/"${!repo_path}/dependencies.yaml" \
--matrix "${matrix_selectors}" \
;
done
done
Expand All @@ -126,12 +152,12 @@ make_pip_dependencies() {
for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do
local file="/tmp/${!repo_name}.${!py_name}.${repo_keys[$keyi]}.requirements.txt";
pip_reqs_txts+=("${file}");
generate_requirements \
"${file}" \
--file-key "${repo_keys[$keyi]}" \
--output requirements \
--config ~/"${!repo_path}/dependencies.yaml" \
--matrix "arch=$(uname -m);cuda=${cuda_version};py=${python_version}" \
generate_requirements \
"${file}" \
--file-key "${repo_keys[$keyi]}" \
--output requirements \
--config ~/"${!repo_path}/dependencies.yaml" \
--matrix "${matrix_selectors}" \
;
done
done
Expand Down

0 comments on commit e414ab9

Please sign in to comment.