From d02317af1551df22071aff267ebd4ca2be77954e Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 16 Oct 2024 16:07:57 -0500 Subject: [PATCH 01/10] some changes --- src/rapids_dependency_file_generator/_cli.py | 8 ++- .../_rapids_dependency_file_generator.py | 59 +++++++++++++++---- 2 files changed, 53 insertions(+), 14 deletions(-) diff --git a/src/rapids_dependency_file_generator/_cli.py b/src/rapids_dependency_file_generator/_cli.py index a79c99a..7a39b07 100644 --- a/src/rapids_dependency_file_generator/_cli.py +++ b/src/rapids_dependency_file_generator/_cli.py @@ -35,7 +35,11 @@ def validate_args(argv): codependent_args = parser.add_argument_group("optional, but codependent") codependent_args.add_argument( "--file-key", - help="The file key from `dependencies.yaml` to generate.", + action='append', + help=( + "The file key from `dependencies.yaml` to generate. " + "If supplied multiple times, dependency lists from all requested file keys will be merged." + ), ) codependent_args.add_argument( "--output", @@ -109,7 +113,7 @@ def main(argv=None) -> None: to_stdout = all([args.file_key, args.output, args.matrix is not None]) if to_stdout: - file_keys = [args.file_key] + file_keys = args.file_key output = {Output(args.output)} else: file_keys = list(parsed_config.files.keys()) diff --git a/src/rapids_dependency_file_generator/_rapids_dependency_file_generator.py b/src/rapids_dependency_file_generator/_rapids_dependency_file_generator.py index b201e36..3bebee5 100644 --- a/src/rapids_dependency_file_generator/_rapids_dependency_file_generator.py +++ b/src/rapids_dependency_file_generator/_rapids_dependency_file_generator.py @@ -101,7 +101,7 @@ def make_dependency_file( conda_channels: list[str], dependencies: typing.Sequence[typing.Union[str, dict[str, list[str]]]], extras: typing.Union[_config.FileExtras, None], -): +) -> str: """Generate the contents of the dependency file. Parameters @@ -360,6 +360,20 @@ def make_dependency_files( If the file is malformed. There are numerous different error cases which are described by the error messages. """ + + # the list of conda channels does not depend on individual file keys + conda_channels=prepend_channels + parsed_config.channels + + # to support merging lists before writing to stdout + stdout_collection = { + "conda_channels": conda_channels, + "dependencies": { + "str_deps": set(), + "dict_deps": dict() + }, + "extras": set() + } + for file_key in file_keys: file_config = parsed_config.files[file_key] file_types_to_generate = file_config.output if output is None else output @@ -436,20 +450,41 @@ def make_dependency_files( config_file_path=parsed_config.path, file_config=file_config, ) - contents = make_dependency_file( - file_type=file_type, - name=full_file_name, - config_file=parsed_config.path, - output_dir=output_dir, - conda_channels=prepend_channels + parsed_config.channels, - dependencies=deduped_deps, - extras=file_config.extras, - ) - + if to_stdout: - print(contents) + for dep in deduped_deps: + if isinstance(dep, dict): + stdout_collection["dependencies"]["dict_deps"].update(dep) + else: + stdout_collection["dependencies"]["str_deps"].add(dep) + #stdout_collection["dependencies"] = stdout_collection["dependencies"].union(set(deduped_deps)) + print("---") + print(deduped_deps) + print("---") else: + contents = make_dependency_file( + file_type=file_type, + name=full_file_name, + config_file=parsed_config.path, + output_dir=output_dir, + conda_channels=conda_channels, + dependencies=deduped_deps, + extras=file_config.extras, + ) os.makedirs(output_dir, exist_ok=True) file_path = os.path.join(output_dir, full_file_name) with open(file_path, "w") as f: f.write(contents) + + # create one unified output from all the file_keys, and print it to stdout + if to_stdout: + contents = make_dependency_file( + file_type=output.pop(), + name="to-stdout", + config_file=parsed_config.path, + output_dir=parsed_config.path, + conda_channels=stdout_collection["conda_channels"], + dependencies=sorted(stdout_collection["dependencies"]["str_deps"]), + extras=None + ) + print(contents) From 8878178b994057690467fd4b0b87f33605a6bd62 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 16 Oct 2024 16:40:53 -0500 Subject: [PATCH 02/10] all the tests passing --- src/rapids_dependency_file_generator/_cli.py | 2 +- .../_rapids_dependency_file_generator.py | 63 +++++++++++-------- 2 files changed, 37 insertions(+), 28 deletions(-) diff --git a/src/rapids_dependency_file_generator/_cli.py b/src/rapids_dependency_file_generator/_cli.py index 7a39b07..1917b26 100644 --- a/src/rapids_dependency_file_generator/_cli.py +++ b/src/rapids_dependency_file_generator/_cli.py @@ -35,7 +35,7 @@ def validate_args(argv): codependent_args = parser.add_argument_group("optional, but codependent") codependent_args.add_argument( "--file-key", - action='append', + action="append", help=( "The file key from `dependencies.yaml` to generate. " "If supplied multiple times, dependency lists from all requested file keys will be merged." diff --git a/src/rapids_dependency_file_generator/_rapids_dependency_file_generator.py b/src/rapids_dependency_file_generator/_rapids_dependency_file_generator.py index 3bebee5..24f35d4 100644 --- a/src/rapids_dependency_file_generator/_rapids_dependency_file_generator.py +++ b/src/rapids_dependency_file_generator/_rapids_dependency_file_generator.py @@ -360,18 +360,17 @@ def make_dependency_files( If the file is malformed. There are numerous different error cases which are described by the error messages. """ + if to_stdout and len(file_keys) > 1: + raise ValueError("Using --file-key multiple times when writing to stdout is not supported.") # the list of conda channels does not depend on individual file keys - conda_channels=prepend_channels + parsed_config.channels + conda_channels = prepend_channels + parsed_config.channels # to support merging lists before writing to stdout stdout_collection = { "conda_channels": conda_channels, - "dependencies": { - "str_deps": set(), - "dict_deps": dict() - }, - "extras": set() + "dependencies": {"str_deps": set(), "dict_deps": dict()}, + "extras": parsed_config.files[file_keys[0]].extras, } for file_key in file_keys: @@ -450,34 +449,44 @@ def make_dependency_files( config_file_path=parsed_config.path, file_config=file_config, ) - + contents = make_dependency_file( + file_type=file_type, + name=full_file_name, + config_file=parsed_config.path, + output_dir=output_dir, + conda_channels=conda_channels, + dependencies=deduped_deps, + extras=file_config.extras, + ) + if to_stdout: - for dep in deduped_deps: - if isinstance(dep, dict): - stdout_collection["dependencies"]["dict_deps"].update(dep) - else: - stdout_collection["dependencies"]["str_deps"].add(dep) - #stdout_collection["dependencies"] = stdout_collection["dependencies"].union(set(deduped_deps)) - print("---") - print(deduped_deps) - print("---") + if len(file_keys) == 1: + print(contents) + else: + for dep in deduped_deps: + if isinstance(dep, dict): + stdout_collection["dependencies"]["dict_deps"].update(dep) + else: + stdout_collection["dependencies"]["str_deps"].add(dep) + # stdout_collection["dependencies"] = stdout_collection["dependencies"].union(set(deduped_deps)) + print("---") + print(deduped_deps) + print("---") else: - contents = make_dependency_file( - file_type=file_type, - name=full_file_name, - config_file=parsed_config.path, - output_dir=output_dir, - conda_channels=conda_channels, - dependencies=deduped_deps, - extras=file_config.extras, - ) os.makedirs(output_dir, exist_ok=True) file_path = os.path.join(output_dir, full_file_name) with open(file_path, "w") as f: f.write(contents) # create one unified output from all the file_keys, and print it to stdout - if to_stdout: + # + # notes: + # + # * 'output' is technically a set because of https://github.com/rapidsai/dependency-file-generator/pull/74, + # but since https://github.com/rapidsai/dependency-file-generator/pull/79 it's only ever one of the following: + # - an exactly-1-item set (stdout=True, or when used with rapids-build-backend) + # - 'None' (stdout=False) + if to_stdout and len(file_keys) > 1: contents = make_dependency_file( file_type=output.pop(), name="to-stdout", @@ -485,6 +494,6 @@ def make_dependency_files( output_dir=parsed_config.path, conda_channels=stdout_collection["conda_channels"], dependencies=sorted(stdout_collection["dependencies"]["str_deps"]), - extras=None + extras=stdout_collection["extras"], ) print(contents) From 675d7782a87740d03e79ddac272ce49e562b4741 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 16 Oct 2024 18:34:11 -0500 Subject: [PATCH 03/10] break up uses of 'name' in make_dependency_file() --- .../_rapids_dependency_file_generator.py | 103 ++++++++++++------ .../test_rapids_dependency_file_generator.py | 6 +- 2 files changed, 72 insertions(+), 37 deletions(-) diff --git a/src/rapids_dependency_file_generator/_rapids_dependency_file_generator.py b/src/rapids_dependency_file_generator/_rapids_dependency_file_generator.py index 24f35d4..044941a 100644 --- a/src/rapids_dependency_file_generator/_rapids_dependency_file_generator.py +++ b/src/rapids_dependency_file_generator/_rapids_dependency_file_generator.py @@ -4,6 +4,7 @@ import textwrap import typing from collections.abc import Generator +from dataclasses import dataclass import tomlkit import yaml @@ -95,7 +96,8 @@ def grid(gridspec: dict[str, list[str]]) -> Generator[dict[str, str], None, None def make_dependency_file( *, file_type: _config.Output, - name: os.PathLike, + conda_env_name: str, + file_name: str, config_file: os.PathLike, output_dir: os.PathLike, conda_channels: list[str], @@ -108,14 +110,18 @@ def make_dependency_file( ---------- file_type : Output An Output value used to determine the file type. - name : PathLike - The name of the file to write. + conda_env_name : str + Name to put in the 'name: ' field when generating conda environment YAML files. + Only used when ``file_type`` is CONDA. + file_name : str + Name of a file in ``output_dir`` to read in. + Only used when ``file_type`` is PYPROJECT. config_file : PathLike The full path to the dependencies.yaml file. output_dir : PathLike The path to the directory where the dependency files will be written. conda_channels : list[str] - The channels to include in the file. Only used when `file_type` is + The channels to include in the file. Only used when ``file_type`` is CONDA. dependencies : Sequence[str | dict[str, list[str]]] The dependencies to include in the file. @@ -137,7 +143,7 @@ def make_dependency_file( if file_type == _config.Output.CONDA: file_contents += yaml.dump( { - "name": os.path.splitext(name)[0], + "name": conda_env_name, "channels": conda_channels, "dependencies": dependencies, } @@ -173,7 +179,7 @@ def make_dependency_file( key = extras.key # This file type needs to be modified in place instead of built from scratch. - with open(os.path.join(output_dir, name)) as f: + with open(os.path.join(output_dir, file_name)) as f: file_contents_toml = tomlkit.load(f) toml_deps = tomlkit.array() @@ -320,6 +326,32 @@ def should_use_specific_entry(matrix_combo: dict[str, str], specific_entry_matri ) +@dataclass +class _DependencyCollection: + str_deps: set[str] + # e.g. {"pip": ["dgl", "pyg"]}, used in conda envs + dict_deps: dict[str, list[str]] + + def update(self, deps: typing.Sequence[typing.Union[str, dict[str, list[str]]]]) -> None: + for dep in deps: + if isinstance(dep, dict): + for k, v in dep.items(): + if k in self.dict_deps: + self.dict_deps[k].extend(v) + self.dict_deps[k] = sorted(set(self.dict_deps[k])) + else: + self.dict_deps[k] = v + else: + self.str_deps.add(dep) + + @property + def deps_list(self) -> typing.Sequence[typing.Union[str, dict[str, list[str]]]]: + if self.dict_deps: + return [*sorted(self.str_deps), self.dict_deps] + + return [*sorted(self.str_deps)] + + def make_dependency_files( *, parsed_config: _config.Config, @@ -360,18 +392,18 @@ def make_dependency_files( If the file is malformed. There are numerous different error cases which are described by the error messages. """ - if to_stdout and len(file_keys) > 1: - raise ValueError("Using --file-key multiple times when writing to stdout is not supported.") + if to_stdout and len(file_keys) > 1 and output is not None and _config.Output.PYPROJECT in output: + raise ValueError( + f"Using --file-key multiple times together with '--output {_config.Output.PYPROJECT}' " + "when writing to stdout is not supported." + ) # the list of conda channels does not depend on individual file keys conda_channels = prepend_channels + parsed_config.channels - # to support merging lists before writing to stdout - stdout_collection = { - "conda_channels": conda_channels, - "dependencies": {"str_deps": set(), "dict_deps": dict()}, - "extras": parsed_config.files[file_keys[0]].extras, - } + # initialize a container for "all dependencies found across all files", to support + # passing multiple files keys and writing a merged result to stdout + all_dependencies = _DependencyCollection(str_deps=set(), dict_deps={}) for file_key in file_keys: file_config = parsed_config.files[file_key] @@ -451,7 +483,8 @@ def make_dependency_files( ) contents = make_dependency_file( file_type=file_type, - name=full_file_name, + conda_env_name=os.path.splitext(full_file_name)[0], + file_name=full_file_name, config_file=parsed_config.path, output_dir=output_dir, conda_channels=conda_channels, @@ -463,15 +496,7 @@ def make_dependency_files( if len(file_keys) == 1: print(contents) else: - for dep in deduped_deps: - if isinstance(dep, dict): - stdout_collection["dependencies"]["dict_deps"].update(dep) - else: - stdout_collection["dependencies"]["str_deps"].add(dep) - # stdout_collection["dependencies"] = stdout_collection["dependencies"].union(set(deduped_deps)) - print("---") - print(deduped_deps) - print("---") + all_dependencies.update(deduped_deps) else: os.makedirs(output_dir, exist_ok=True) file_path = os.path.join(output_dir, full_file_name) @@ -479,21 +504,29 @@ def make_dependency_files( f.write(contents) # create one unified output from all the file_keys, and print it to stdout - # - # notes: - # - # * 'output' is technically a set because of https://github.com/rapidsai/dependency-file-generator/pull/74, - # but since https://github.com/rapidsai/dependency-file-generator/pull/79 it's only ever one of the following: - # - an exactly-1-item set (stdout=True, or when used with rapids-build-backend) - # - 'None' (stdout=False) if to_stdout and len(file_keys) > 1: + # convince mypy that 'output' is not None here + # + # 'output' is technically a set because of https://github.com/rapidsai/dependency-file-generator/pull/74, + # but since https://github.com/rapidsai/dependency-file-generator/pull/79 it's only ever one of the following: + # + # - an exactly-1-item set (stdout=True, or when used by rapids-build-backend) + # - 'None' (stdout=False) + # + err_msg = ( + "Exactly 1 output type should be provided when asking rapids-dependency-file-generator to write to stdout. " + "If you see this, you've found a bug. Please report it at https://github.com/rapidsai/dependency-file-generator/issues." + ) + assert output is not None, err_msg + contents = make_dependency_file( file_type=output.pop(), - name="to-stdout", + conda_env_name="rapids-dfg-combined", + file_name="ignored-because-multiple-pyproject-files-are-not-supported", config_file=parsed_config.path, output_dir=parsed_config.path, - conda_channels=stdout_collection["conda_channels"], - dependencies=sorted(stdout_collection["dependencies"]["str_deps"]), - extras=stdout_collection["extras"], + conda_channels=conda_channels, + dependencies=all_dependencies.deps_list, + extras=None, ) print(contents) diff --git a/tests/test_rapids_dependency_file_generator.py b/tests/test_rapids_dependency_file_generator.py index e8b00e5..4c64acd 100644 --- a/tests/test_rapids_dependency_file_generator.py +++ b/tests/test_rapids_dependency_file_generator.py @@ -54,7 +54,8 @@ def test_make_dependency_file(mock_relpath): """ env = make_dependency_file( file_type=_config.Output.CONDA, - name="tmp_env.yaml", + conda_env_name="tmp_env", + file_name="tmp_env.yaml", config_file="config_file", output_dir="output_path", conda_channels=["rapidsai", "nvidia"], @@ -71,7 +72,8 @@ def test_make_dependency_file(mock_relpath): env = make_dependency_file( file_type=_config.Output.REQUIREMENTS, - name="tmp_env.txt", + conda_env_name="tmp_env", + file_name="tmp_env.txt", config_file="config_file", output_dir="output_path", conda_channels=["rapidsai", "nvidia"], From 2b4ec1f5e2131e838118401c220edc0abd6af076 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 16 Oct 2024 18:48:17 -0500 Subject: [PATCH 04/10] unit tests --- .../_rapids_dependency_file_generator.py | 2 +- tests/test_cli.py | 44 +++++++++++++++++++ .../test_rapids_dependency_file_generator.py | 15 ++++++- 3 files changed, 59 insertions(+), 2 deletions(-) diff --git a/src/rapids_dependency_file_generator/_rapids_dependency_file_generator.py b/src/rapids_dependency_file_generator/_rapids_dependency_file_generator.py index 044941a..2d19224 100644 --- a/src/rapids_dependency_file_generator/_rapids_dependency_file_generator.py +++ b/src/rapids_dependency_file_generator/_rapids_dependency_file_generator.py @@ -394,7 +394,7 @@ def make_dependency_files( """ if to_stdout and len(file_keys) > 1 and output is not None and _config.Output.PYPROJECT in output: raise ValueError( - f"Using --file-key multiple times together with '--output {_config.Output.PYPROJECT}' " + f"Using --file-key multiple times together with '--output {_config.Output.PYPROJECT.value}' " "when writing to stdout is not supported." ) diff --git a/tests/test_cli.py b/tests/test_cli.py index a5c2944..21a05b5 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -130,3 +130,47 @@ def test_validate_args(): "all", ] ) + + # Valid, with 2 files for --output requirements + validate_args( + [ + "--output", + "requirements", + "--matrix", + "cuda=12.5", + "--file-key", + "all", + "--file-key", + "test_python", + ] + ) + + # Valid, with 2 files for --output conda + validate_args( + [ + "--output", + "conda", + "--matrix", + "cuda=12.5", + "--file-key", + "all", + "--file-key", + "test_python", + ] + ) + + # Valid, with 3 files + validate_args( + [ + "--output", + "requirements", + "--matrix", + "cuda=12.5", + "--file-key", + "all", + "--file-key", + "test_python", + "--file-key", + "build_python", + ] + ) diff --git a/tests/test_rapids_dependency_file_generator.py b/tests/test_rapids_dependency_file_generator.py index 4c64acd..bd752b6 100644 --- a/tests/test_rapids_dependency_file_generator.py +++ b/tests/test_rapids_dependency_file_generator.py @@ -83,7 +83,7 @@ def test_make_dependency_file(mock_relpath): assert env == header + "dep1\ndep2\n" -def test_make_dependency_file_should_raise_informative_error_when_extras_is_missing_for_pyproj(): +def test_make_dependency_files_should_raise_informative_error_when_extras_is_missing_for_pyproj(): current_dir = pathlib.Path(__file__).parent with pytest.raises(ValueError, match=r"The 'extras' field must be provided for the 'pyproject' file type"): @@ -97,6 +97,19 @@ def test_make_dependency_file_should_raise_informative_error_when_extras_is_miss ) +def test_make_dependency_files_should_raise_informative_error_when_multiple_files_requested_for_pyproject(): + + current_dir = pathlib.Path(__file__).parent + with pytest.raises(ValueError, match=r"Using \-\-file\-key multiple times together with.*pyproject"): + make_dependency_files( + parsed_config=_config.load_config_from_file(current_dir / "examples" / "integration" / "dependencies.yaml"), + file_keys=["all", "test"], + output={_config.Output.PYPROJECT}, + matrix=None, + prepend_channels=[], + to_stdout=True + ) + def test_make_dependency_files_should_raise_informative_error_on_map_inputs_for_requirements(): current_dir = pathlib.Path(__file__).parent From c2781e5559739c681739bdde188a1a62ac4a2069 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 17 Oct 2024 09:48:57 -0500 Subject: [PATCH 05/10] added example with overlapping dependencies --- .../overlapping-deps/dependencies.yaml | 73 +++++++++++++++++++ .../output/expected/pyproject.toml | 19 +++++ 2 files changed, 92 insertions(+) create mode 100644 tests/examples/overlapping-deps/dependencies.yaml create mode 100644 tests/examples/overlapping-deps/output/expected/pyproject.toml diff --git a/tests/examples/overlapping-deps/dependencies.yaml b/tests/examples/overlapping-deps/dependencies.yaml new file mode 100644 index 0000000..a46f29b --- /dev/null +++ b/tests/examples/overlapping-deps/dependencies.yaml @@ -0,0 +1,73 @@ +files: + build_deps: + output: [pyproject] + pyproject_dir: output/actual + extras: + table: build-system + includes: + - rapids_build_skbuild + - depends_on_numpy + even_more_build_deps: + output: [pyproject] + pyproject_dir: output/actual + extras: + table: tool.rapids-build-backend + key: requires + includes: + - depends_on_numpy + - depends_on_pandas + test_deps: + output: none + includes: + - depends_on_numpy + - depends_on_pandas + even_more_test_deps: + output: none + includes: + - depends_on_numpy + - test_python +channels: + - rapidsai + - conda-forge +dependencies: + depends_on_numpy: + common: + - output_types: [requirements, pyproject] + packages: + - numpy>=2.0 + # using 'pip' intentionally to test handling of that nested list + - output_types: [conda] + packages: + - pip + - pip: + - numpy >=2.0 + depends_on_pandas: + common: + - output_types: [conda, requirements, pyproject] + packages: + - numpy>=2.0 + even_more_test_deps: + common: + - output_types: [conda, requirements, pyproject] + packages: + - matplotlib + - scikit-learn>=1.5 + - output_types: [conda] + packages: + - pip + # intentional overlap (numpy) with depends_on_numpy's pip list, to + # test that pip dependencies don't have duplicates + - pip: + - folium + - numpy >=2.0 + rapids_build_skbuild: + common: + - output_types: [conda, requirements, pyproject] + packages: + - rapids-build-backend>=0.3.1 + - output_types: [requirements, pyproject] + packages: + - scikit-build-core[pyproject]>=0.9.0 + - output_types: [conda] + packages: + - scikit-build-core>=0.9.0 diff --git a/tests/examples/overlapping-deps/output/expected/pyproject.toml b/tests/examples/overlapping-deps/output/expected/pyproject.toml new file mode 100644 index 0000000..e7d9716 --- /dev/null +++ b/tests/examples/overlapping-deps/output/expected/pyproject.toml @@ -0,0 +1,19 @@ +[build-system] +build-backend = "rapids_build_backend.build_meta" +requires = [ + "numpy>=2.0", + "rapids-build-backend>=0.3.1", + "scikit-build-core[pyproject]>=0.9.0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. + +[project] +name = "libbeepboop" +version = "0.1.2" +dependencies = [ + "scipy", +] + +[tool.rapids-build-backend] +requires = [ + "numpy>=2.0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From f9c818aa2d868b42694631536adff7829c5f4fa6 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 17 Oct 2024 09:57:49 -0500 Subject: [PATCH 06/10] added tests on requirements --- .../test_rapids_dependency_file_generator.py | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/test_rapids_dependency_file_generator.py b/tests/test_rapids_dependency_file_generator.py index bd752b6..41842f9 100644 --- a/tests/test_rapids_dependency_file_generator.py +++ b/tests/test_rapids_dependency_file_generator.py @@ -147,6 +147,39 @@ def test_make_dependency_files_should_choose_correct_pyproject_toml(capsys): # and should NOT contain anything from the root-level pyproject.toml assert set(dict(doc).keys()) == {"project"} +def test_make_dependency_files_requirements_to_stdout_with_multiple_file_keys_works(capsys): + + current_dir = pathlib.Path(__file__).parent + make_dependency_files( + parsed_config=_config.load_config_from_file(current_dir / "examples" / "overlapping-deps" / "dependencies.yaml"), + file_keys=["build_deps", "even_more_build_deps"], + output={_config.Output.REQUIREMENTS}, + matrix={"arch": ["x86_64"]}, + prepend_channels=[], + to_stdout=True + ) + captured_stdout = capsys.readouterr().out + reqs_list = [r for r in captured_stdout.split("\n") if not (r.startswith(r"#") or r == "")] + + # should contain exactly the expected dependencies, sorted alphabetically, with no duplicates + assert reqs_list == ["numpy>=2.0", "rapids-build-backend>=0.3.1", "scikit-build-core[pyproject]>=0.9.0"] + +# def test_make_dependency_files_requirements_to_stdout_with_multiple_file_keys_works(capsys): + +# current_dir = pathlib.Path(__file__).parent +# make_dependency_files( +# parsed_config=_config.load_config_from_file(current_dir / "examples" / "overlapping-deps" / "dependencies.yaml"), +# file_keys=["build_deps", "even_more_build_deps"], +# output={_config.Output.REQUIREMENTS}, +# matrix={"arch": ["x86_64"]}, +# prepend_channels=[], +# to_stdout=True +# ) +# captured_stdout = capsys.readouterr().out +# reqs_list = [r for r in captured_stdout.split("\n") if not (r.startswith(r"#") or r == "")] + +# # should contain exactly the expected dependencies, sorted alphabetically, with no duplicates +# assert reqs_list == ["numpy>=2.0", "rapids-build-backend>=0.3.1", "scikit-build-core[pyproject]>=0.9.0"] def test_should_use_specific_entry(): # no match From e504a80a8a10cae067e073d7f59a2640580a8304 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 17 Oct 2024 10:24:12 -0500 Subject: [PATCH 07/10] add unit tests --- .../overlapping-deps/dependencies.yaml | 17 ++++-- .../output/expected/pyproject.toml | 1 + .../test_rapids_dependency_file_generator.py | 61 +++++++++++++------ 3 files changed, 57 insertions(+), 22 deletions(-) diff --git a/tests/examples/overlapping-deps/dependencies.yaml b/tests/examples/overlapping-deps/dependencies.yaml index a46f29b..0b25c02 100644 --- a/tests/examples/overlapping-deps/dependencies.yaml +++ b/tests/examples/overlapping-deps/dependencies.yaml @@ -26,6 +26,10 @@ files: includes: - depends_on_numpy - test_python + test_with_sklearn: + output: none + includes: + - depends_on_scikit_learn channels: - rapidsai - conda-forge @@ -45,21 +49,26 @@ dependencies: common: - output_types: [conda, requirements, pyproject] packages: - - numpy>=2.0 - even_more_test_deps: + - pandas<3.0 + depends_on_scikit_learn: common: - output_types: [conda, requirements, pyproject] packages: - - matplotlib - scikit-learn>=1.5 + test_python: + common: + - output_types: [conda, requirements, pyproject] + packages: + - matplotlib - output_types: [conda] packages: - pip # intentional overlap (numpy) with depends_on_numpy's pip list, to # test that pip dependencies don't have duplicates - pip: - - folium + # intentionally not in alphabetical order - numpy >=2.0 + - folium rapids_build_skbuild: common: - output_types: [conda, requirements, pyproject] diff --git a/tests/examples/overlapping-deps/output/expected/pyproject.toml b/tests/examples/overlapping-deps/output/expected/pyproject.toml index e7d9716..5a5b5ad 100644 --- a/tests/examples/overlapping-deps/output/expected/pyproject.toml +++ b/tests/examples/overlapping-deps/output/expected/pyproject.toml @@ -16,4 +16,5 @@ dependencies = [ [tool.rapids-build-backend] requires = [ "numpy>=2.0", + "pandas<3.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/tests/test_rapids_dependency_file_generator.py b/tests/test_rapids_dependency_file_generator.py index 41842f9..813e2e8 100644 --- a/tests/test_rapids_dependency_file_generator.py +++ b/tests/test_rapids_dependency_file_generator.py @@ -162,24 +162,49 @@ def test_make_dependency_files_requirements_to_stdout_with_multiple_file_keys_wo reqs_list = [r for r in captured_stdout.split("\n") if not (r.startswith(r"#") or r == "")] # should contain exactly the expected dependencies, sorted alphabetically, with no duplicates - assert reqs_list == ["numpy>=2.0", "rapids-build-backend>=0.3.1", "scikit-build-core[pyproject]>=0.9.0"] - -# def test_make_dependency_files_requirements_to_stdout_with_multiple_file_keys_works(capsys): - -# current_dir = pathlib.Path(__file__).parent -# make_dependency_files( -# parsed_config=_config.load_config_from_file(current_dir / "examples" / "overlapping-deps" / "dependencies.yaml"), -# file_keys=["build_deps", "even_more_build_deps"], -# output={_config.Output.REQUIREMENTS}, -# matrix={"arch": ["x86_64"]}, -# prepend_channels=[], -# to_stdout=True -# ) -# captured_stdout = capsys.readouterr().out -# reqs_list = [r for r in captured_stdout.split("\n") if not (r.startswith(r"#") or r == "")] - -# # should contain exactly the expected dependencies, sorted alphabetically, with no duplicates -# assert reqs_list == ["numpy>=2.0", "rapids-build-backend>=0.3.1", "scikit-build-core[pyproject]>=0.9.0"] + assert reqs_list == ["numpy>=2.0", "pandas<3.0", "rapids-build-backend>=0.3.1", "scikit-build-core[pyproject]>=0.9.0"] + +def test_make_dependency_files_conda_to_stdout_with_multiple_file_keys_works(capsys): + + current_dir = pathlib.Path(__file__).parent + make_dependency_files( + parsed_config=_config.load_config_from_file(current_dir / "examples" / "overlapping-deps" / "dependencies.yaml"), + file_keys=["test_with_sklearn", "test_deps", "even_more_test_deps"], + output={_config.Output.CONDA}, + matrix={"py": ["4.7"]}, + prepend_channels=[], + to_stdout=True + ) + captured_stdout = capsys.readouterr().out + env_dict = yaml.safe_load(captured_stdout) + + # should only have the expected keys + assert sorted(env_dict.keys()) == ["channels", "dependencies", "name"] + + # should use preserve the channels from dependencies.yaml, in the order they were supplied + assert env_dict["channels"] == ["rapidsai", "conda-forge"] + + # should use the hard-coded env name + assert env_dict["name"] == "rapids-dfg-combined" + + # dependencies list should: + # + # * be sorted alphabetically (other than "pip:" list at the end) + # * should include the "pip:" subsection + # * should not have any duplicates + # * should contain the union of all dependencies from all requested file keys + # + assert env_dict["dependencies"] == [ + "matplotlib", + "pandas<3.0", + "pip", + "scikit-learn>=1.5", + {"pip": [ + "folium", + "numpy >=2.0", + ]} + ] + def test_should_use_specific_entry(): # no match From cb00f3332f0601551baa1540ff8b6d71177c34a9 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 17 Oct 2024 10:42:35 -0500 Subject: [PATCH 08/10] add docs --- README.md | 12 +++++++++++- tests/test_rapids_dependency_file_generator.py | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0c024df..41ecc66 100644 --- a/README.md +++ b/README.md @@ -322,7 +322,7 @@ ENV_NAME="cudf_test" rapids-dependency-file-generator \ --file-key "test" \ --output "conda" \ - --matrix "cuda=11.5;arch=$(arch)" > env.yaml + --matrix "cuda=12.5;arch=$(arch)" > env.yaml mamba env create --file env.yaml mamba activate "$ENV_NAME" @@ -335,6 +335,16 @@ The `--file-key`, `--output`, and `--matrix` flags must be used together. `--mat Where multiple values for the same key are passed to `--matrix`, e.g. `cuda_suffixed=true;cuda_suffixed=false`, only the last value will be used. +Where `--file-key` is supplied multiple times in the same invocation, the output printed to `stdout` will contain a union (without duplicates) of all of the corresponding dependencies. For example: + +```shell +rapids-dependency-file-generator \ + --file-key "test" \ + --file-key "test_notebooks" \ + --output "conda" \ + --matrix "cuda=12.5;arch=$(arch)" > env.yaml +``` + The `--prepend-channel` argument accepts additional channels to use, like `rapids-dependency-file-generator --prepend-channel my_channel --prepend-channel my_other_channel`. If both `--output` and `--prepend-channel` are provided, the output format must be conda. Prepending channels can be useful for adding local channels with packages to be tested in CI workflows. diff --git a/tests/test_rapids_dependency_file_generator.py b/tests/test_rapids_dependency_file_generator.py index 813e2e8..f9bcbb4 100644 --- a/tests/test_rapids_dependency_file_generator.py +++ b/tests/test_rapids_dependency_file_generator.py @@ -83,7 +83,7 @@ def test_make_dependency_file(mock_relpath): assert env == header + "dep1\ndep2\n" -def test_make_dependency_files_should_raise_informative_error_when_extras_is_missing_for_pyproj(): +def test_make_dependency_file_should_raise_informative_error_when_extras_is_missing_for_pyproj(): current_dir = pathlib.Path(__file__).parent with pytest.raises(ValueError, match=r"The 'extras' field must be provided for the 'pyproject' file type"): From df3848dd98d51c8aabbdad6adfcb6bfba75c1ea7 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 18 Oct 2024 13:01:03 -0500 Subject: [PATCH 09/10] allow conda_env_name to be None, remove spaces in requirements, remove issue page link --- .../_rapids_dependency_file_generator.py | 23 ++++++++++--------- .../overlapping-deps/dependencies.yaml | 4 ++-- .../test_rapids_dependency_file_generator.py | 7 ++---- 3 files changed, 16 insertions(+), 18 deletions(-) diff --git a/src/rapids_dependency_file_generator/_rapids_dependency_file_generator.py b/src/rapids_dependency_file_generator/_rapids_dependency_file_generator.py index 2d19224..a54cf9b 100644 --- a/src/rapids_dependency_file_generator/_rapids_dependency_file_generator.py +++ b/src/rapids_dependency_file_generator/_rapids_dependency_file_generator.py @@ -96,7 +96,7 @@ def grid(gridspec: dict[str, list[str]]) -> Generator[dict[str, str], None, None def make_dependency_file( *, file_type: _config.Output, - conda_env_name: str, + conda_env_name: typing.Union[str, None], file_name: str, config_file: os.PathLike, output_dir: os.PathLike, @@ -110,8 +110,9 @@ def make_dependency_file( ---------- file_type : Output An Output value used to determine the file type. - conda_env_name : str + conda_env_name : str | None Name to put in the 'name: ' field when generating conda environment YAML files. + If ``None``, the generated cond environment file will not have a 'name:' entry. Only used when ``file_type`` is CONDA. file_name : str Name of a file in ``output_dir`` to read in. @@ -141,13 +142,13 @@ def make_dependency_file( """ ) if file_type == _config.Output.CONDA: - file_contents += yaml.dump( - { - "name": conda_env_name, - "channels": conda_channels, - "dependencies": dependencies, - } - ) + env_dict = { + "channels": conda_channels, + "dependencies": dependencies, + } + if conda_env_name is not None: + env_dict["name"] = conda_env_name + file_contents += yaml.dump(env_dict) elif file_type == _config.Output.REQUIREMENTS: for dep in dependencies: if isinstance(dep, dict): @@ -515,13 +516,13 @@ def make_dependency_files( # err_msg = ( "Exactly 1 output type should be provided when asking rapids-dependency-file-generator to write to stdout. " - "If you see this, you've found a bug. Please report it at https://github.com/rapidsai/dependency-file-generator/issues." + "If you see this, you've found a bug. Please report it." ) assert output is not None, err_msg contents = make_dependency_file( file_type=output.pop(), - conda_env_name="rapids-dfg-combined", + conda_env_name=None, file_name="ignored-because-multiple-pyproject-files-are-not-supported", config_file=parsed_config.path, output_dir=parsed_config.path, diff --git a/tests/examples/overlapping-deps/dependencies.yaml b/tests/examples/overlapping-deps/dependencies.yaml index 0b25c02..2d8928c 100644 --- a/tests/examples/overlapping-deps/dependencies.yaml +++ b/tests/examples/overlapping-deps/dependencies.yaml @@ -44,7 +44,7 @@ dependencies: packages: - pip - pip: - - numpy >=2.0 + - numpy>=2.0 depends_on_pandas: common: - output_types: [conda, requirements, pyproject] @@ -67,7 +67,7 @@ dependencies: # test that pip dependencies don't have duplicates - pip: # intentionally not in alphabetical order - - numpy >=2.0 + - numpy>=2.0 - folium rapids_build_skbuild: common: diff --git a/tests/test_rapids_dependency_file_generator.py b/tests/test_rapids_dependency_file_generator.py index f9bcbb4..9bb693e 100644 --- a/tests/test_rapids_dependency_file_generator.py +++ b/tests/test_rapids_dependency_file_generator.py @@ -179,14 +179,11 @@ def test_make_dependency_files_conda_to_stdout_with_multiple_file_keys_works(cap env_dict = yaml.safe_load(captured_stdout) # should only have the expected keys - assert sorted(env_dict.keys()) == ["channels", "dependencies", "name"] + assert sorted(env_dict.keys()) == ["channels", "dependencies"] # should use preserve the channels from dependencies.yaml, in the order they were supplied assert env_dict["channels"] == ["rapidsai", "conda-forge"] - # should use the hard-coded env name - assert env_dict["name"] == "rapids-dfg-combined" - # dependencies list should: # # * be sorted alphabetically (other than "pip:" list at the end) @@ -201,7 +198,7 @@ def test_make_dependency_files_conda_to_stdout_with_multiple_file_keys_works(cap "scikit-learn>=1.5", {"pip": [ "folium", - "numpy >=2.0", + "numpy>=2.0", ]} ] From 8a06818fa2123c025bff575ea3aafbeebcf815da Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 22 Oct 2024 08:17:59 -0500 Subject: [PATCH 10/10] Update src/rapids_dependency_file_generator/_rapids_dependency_file_generator.py Co-authored-by: Bradley Dice --- .../_rapids_dependency_file_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rapids_dependency_file_generator/_rapids_dependency_file_generator.py b/src/rapids_dependency_file_generator/_rapids_dependency_file_generator.py index a54cf9b..5d378a4 100644 --- a/src/rapids_dependency_file_generator/_rapids_dependency_file_generator.py +++ b/src/rapids_dependency_file_generator/_rapids_dependency_file_generator.py @@ -112,7 +112,7 @@ def make_dependency_file( An Output value used to determine the file type. conda_env_name : str | None Name to put in the 'name: ' field when generating conda environment YAML files. - If ``None``, the generated cond environment file will not have a 'name:' entry. + If ``None``, the generated conda environment file will not have a 'name:' entry. Only used when ``file_type`` is CONDA. file_name : str Name of a file in ``output_dir`` to read in.