diff --git a/.github/workflows/build-publish-pypi.yml b/.github/workflows/build-publish-pypi.yml index fbeed09e..b846784b 100644 --- a/.github/workflows/build-publish-pypi.yml +++ b/.github/workflows/build-publish-pypi.yml @@ -36,7 +36,7 @@ jobs: - name: Build and check package run: | - python -m pip install --user hatch twine validate-pyproject[all] + python -m pip install --user -r ./environments/requirements/requirements-packaging.txt python -m validate_pyproject ./pyproject.toml python -m hatch build --clean python -m twine check --strict ./dist/* diff --git a/environments/docker/Dockerfile b/environments/docker/Dockerfile index aa9efa08..9ddbbc5d 100644 --- a/environments/docker/Dockerfile +++ b/environments/docker/Dockerfile @@ -11,7 +11,7 @@ COPY --chmod=0755 ./README.md ./osmnx/ # install and configuration everything in one RUN to keep image tidy RUN conda update --yes -c conda-forge --strict-channel-priority -n base conda mamba && \ - mamba install --update-all --force-reinstall --yes -c conda-forge --strict-channel-priority --file ./osmnx/requirements.txt && \ + mamba install --update-all --force-reinstall --yes -c conda-forge --strict-channel-priority --file ./osmnx/requirements-all.txt && \ python -m pip install --no-cache-dir -e ./osmnx/ && \ python -m ipykernel install --sys-prefix --name ox --display-name "Python (ox)" && \ rm -f -r -v /opt/conda/share/jupyter/kernels/python3 && \ diff --git a/environments/make-env-files.py b/environments/make-env-files.py index d1c8e5fd..97f83fcd 100644 --- a/environments/make-env-files.py +++ b/environments/make-env-files.py @@ -1,21 +1,21 @@ # noqa: INP001 -"""Make conda env.yml and pip requirements.txt files from environments.json data.""" +"""Make conda env.yaml and pip requirements.txt files from environments.json data.""" from __future__ import annotations -import argparse -import itertools -import json +from itertools import chain +from json import load as json_load from pathlib import Path +from typing import Any -import tomllib from packaging.requirements import Requirement +from tomllib import load as tomllib_load # path to package's pyproject and the config json file -pyproject_path = "./pyproject.toml" -environments_config_path = "./environments/requirements/environments.json" +PYPROJECT_PATH = "./pyproject.toml" +ENVS_CONFIG_PATH = "./environments/requirements/environments.json" -# what channels to specify in conda env yml files +# what channels to specify in conda env yaml files CHANNELS = ["conda-forge"] HEADER = ( @@ -25,21 +25,9 @@ ) -def extract_optional_deps() -> list[Requirement]: - """ - Extract a list of the optional dependencies/versions from pyproject.toml. - - Returns - ------- - optional_deps - """ - opts = pyproject["project"]["optional-dependencies"] - return list({Requirement(o) for o in itertools.chain.from_iterable(opts.values())}) - - def make_requirement( requirement: Requirement, - force_pin: bool = False, # noqa: FBT001,FBT002 + pin_exact: bool = False, # noqa: FBT001,FBT002 is_conda: bool = True, # noqa: FBT001,FBT002 ) -> str: """ @@ -51,11 +39,11 @@ def make_requirement( ---------- requirement A requirement object - force_pin + pin_exact If True, pin requirement to version rather than using existing specifier. Allows you to convert minimum versions to pinned versions. is_conda - If True and if `force_pin` is True, format the requirement string to + If True and if `pin_exact` is True, format the requirement string to end with ".*" for conda environment file pinning format compatibility. Returns @@ -63,7 +51,7 @@ def make_requirement( requirement_str """ specifiers = list(requirement.specifier) - if force_pin and len(specifiers) == 1: + if pin_exact and len(specifiers) == 1: spec = f"{requirement.name}=={specifiers[0].version}" if is_conda and not spec.endswith(".*"): spec += ".*" @@ -71,53 +59,48 @@ def make_requirement( return str(requirement) -def make_file(env_name: str) -> None: +def make_file(env: dict[str, Any]) -> None: """ Write a conda environment yaml file or pip requirements.txt file. Parameters ---------- - env_name - An enviroment name among the keys of environments.json. + env + An environment configuration dictionary. Returns ------- None """ - env = envs[env_name] + depends_on = [] + output_path = Path(env["output_path"]) - # it's a conda env file if it ends with ".yml", otherwise it's a pip - # requirements.txt file - is_conda = env["output_path"].endswith(".yml") + # it's conda env if it's a yaml file, otherwise it's pip requirements.txt + is_conda = output_path.suffix in {".yaml", ".yml"} # determine which dependencies to add based on the configuration - depends_on = [] - if env["needs_python"]: - python_dep = Requirement(f"python{pyproject['project']['requires-python']}") - depends_on.append(python_dep) + if is_conda: + depends_on.append(Requirement(f"python{pyproject['project']['requires-python']}")) if env["needs_dependencies"]: - dependencies = [Requirement(d) for d in pyproject["project"]["dependencies"]] - depends_on.extend(dependencies) - if env["needs_optionals"]: - optionals = extract_optional_deps() - depends_on.extend(optionals) + depends_on.extend(Requirement(d) for d in pyproject["project"]["dependencies"]) + optionals = pyproject["project"]["optional-dependencies"].values() + depends_on.extend({Requirement(o) for o in chain.from_iterable(optionals)}) # make the list of requirements - requirements = [ - make_requirement(dep, force_pin=env["force_pin"], is_conda=is_conda) for dep in depends_on - ] + requirements = [make_requirement(dep, env["pin_exact"], is_conda) for dep in depends_on] - # add any extra requirements if provided in the configuration + # inject any additional requirement files if specified by the config if env["extras"] is not None: for extras_filepath in env["extras"]: with Path(extras_filepath).open() as f: requirements += f.read().splitlines() - # convert the requirements to conda env yml or pip requirements.txt + # convert the requirements to conda env yaml or pip requirements text requirements = sorted(requirements) if not is_conda: text = HEADER + "\n".join(requirements) + "\n" else: + env_name = Path(output_path).stem data = {"name": env_name, "channels": CHANNELS, "dependencies": requirements} text = "" for k, v in data.items(): @@ -127,29 +110,20 @@ def make_file(env_name: str) -> None: text += k + ": " + v + "\n" text = HEADER + text - # write the file to disk - with Path(env["output_path"]).open("w") as f: + # write the text to file on disk + with Path(output_path).open("w") as f: f.writelines(text) - print(f"Wrote {len(requirements)} requirements to {env['output_path']!r}") # noqa: T201 + print(f"Wrote {len(requirements)} requirements to {str(output_path)!r}") # noqa: T201 if __name__ == "__main__": # load the pyproject.toml and the environments.json config files - with Path(pyproject_path).open("rb") as f: - pyproject = tomllib.load(f) - with Path(environments_config_path).open("rb") as f: - envs = json.load(f) - - # parse any command-line arguments passed by the user - arg_parser = argparse.ArgumentParser() - arg_parser.add_argument("-n", dest="env_name", type=str) - args = arg_parser.parse_args() - - if args.env_name is not None: - # if user passed -n command line argument, generate only that file - make_file(args.env_name) - else: - # otherwise, make all environment files - for env_name in envs: - make_file(env_name) + with Path(PYPROJECT_PATH).open("rb") as f: + pyproject = tomllib_load(f) + with Path(ENVS_CONFIG_PATH).open("rb") as f: + envs = json_load(f) + + # make each environment/requirements file as configured + for env in envs: + make_file(env) diff --git a/environments/requirements/environments.json b/environments/requirements/environments.json index b66d8ba0..69b6b6cf 100644 --- a/environments/requirements/environments.json +++ b/environments/requirements/environments.json @@ -1,45 +1,47 @@ -{ - "env-ci": { - "output_path": "./environments/tests/env-ci.yml", - "needs_python": true, - "needs_dependencies": true, - "needs_optionals": true, - "force_pin": false, - "extras": ["./environments/requirements/requirements-docs.txt", - "./environments/requirements/requirements-tests.txt"] - }, - "env-test-minimum-deps": { - "output_path": "./environments/tests/env-test-minimum-deps.yml", - "needs_python": true, - "needs_dependencies": true, - "needs_optionals": true, - "force_pin": true, - "extras": ["./environments/requirements/requirements-tests.txt"] - }, - "requirements-test-latest-deps": { - "output_path": "./environments/tests/requirements-test-latest-deps.txt", - "needs_python": false, - "needs_dependencies": true, - "needs_optionals": true, - "force_pin": false, - "extras": ["./environments/requirements/requirements-tests.txt"] - }, - "requirements-rtd": { - "output_path": "./docs/requirements-rtd.txt", - "needs_python": false, - "needs_dependencies": false, - "needs_optionals": false, - "force_pin": false, - "extras": ["./environments/requirements/requirements-docs.txt"] - }, - "requirements-all": { - "output_path": "./environments/requirements/requirements-all.txt", - "needs_python": false, - "needs_dependencies": true, - "needs_optionals": true, - "force_pin": false, - "extras": ["./environments/requirements/requirements-docs.txt", - "./environments/requirements/requirements-extras.txt", - "./environments/requirements/requirements-tests.txt"] - } -} +[ + { + "output_path": "./environments/tests/env-ci.yml", + "needs_dependencies": true, + "pin_exact": false, + "extras": [ + "./environments/requirements/requirements-docs.txt", + "./environments/requirements/requirements-packaging.txt", + "./environments/requirements/requirements-tests.txt" + ] + }, + { + "output_path": "./environments/tests/env-test-minimum-deps.yml", + "needs_dependencies": true, + "pin_exact": true, + "extras": [ + "./environments/requirements/requirements-tests.txt" + ] + }, + { + "output_path": "./environments/tests/requirements-test-latest-deps.txt", + "needs_dependencies": true, + "pin_exact": false, + "extras": [ + "./environments/requirements/requirements-tests.txt" + ] + }, + { + "output_path": "./docs/requirements-rtd.txt", + "needs_dependencies": false, + "pin_exact": false, + "extras": [ + "./environments/requirements/requirements-docs.txt" + ] + }, + { + "output_path": "./environments/requirements/requirements-all.txt", + "needs_dependencies": true, + "pin_exact": false, + "extras": [ + "./environments/requirements/requirements-docs.txt", + "./environments/requirements/requirements-extras.txt", + "./environments/requirements/requirements-packaging.txt", + "./environments/requirements/requirements-tests.txt" + ] + } +] diff --git a/environments/requirements/requirements-all.txt b/environments/requirements/requirements-all.txt index 95f2a017..b520418d 100644 --- a/environments/requirements/requirements-all.txt +++ b/environments/requirements/requirements-all.txt @@ -3,7 +3,6 @@ # /environments/requirements/ and the requirements in pyproject.toml. bottleneck cartopy -conda-smithy folium furo geopandas>=1.0 diff --git a/environments/requirements/requirements-extras.txt b/environments/requirements/requirements-extras.txt index d8204ee5..12ea5efd 100644 --- a/environments/requirements/requirements-extras.txt +++ b/environments/requirements/requirements-extras.txt @@ -1,6 +1,5 @@ bottleneck cartopy -conda-smithy folium jupyterlab nbdime diff --git a/environments/requirements/requirements-packaging.txt b/environments/requirements/requirements-packaging.txt new file mode 100644 index 00000000..1bb70a07 --- /dev/null +++ b/environments/requirements/requirements-packaging.txt @@ -0,0 +1,3 @@ +hatch +twine +validate-pyproject diff --git a/environments/requirements/requirements-tests.txt b/environments/requirements/requirements-tests.txt index 9a03fb1d..209ec5e4 100644 --- a/environments/requirements/requirements-tests.txt +++ b/environments/requirements/requirements-tests.txt @@ -1,9 +1,6 @@ -hatch lxml pip pre-commit pytest pytest-cov -twine typeguard -validate-pyproject diff --git a/environments/tests/env-test-minimum-deps.yml b/environments/tests/env-test-minimum-deps.yml index 3516c4dc..769d6f60 100644 --- a/environments/tests/env-test-minimum-deps.yml +++ b/environments/tests/env-test-minimum-deps.yml @@ -6,7 +6,6 @@ channels: - conda-forge dependencies: - geopandas==1.0.* - - hatch - lxml - matplotlib==3.5.* - networkx==2.5.* @@ -23,6 +22,4 @@ dependencies: - scikit-learn==0.23.* - scipy==1.5.* - shapely==2.0.* - - twine - typeguard - - validate-pyproject diff --git a/environments/tests/requirements-test-latest-deps.txt b/environments/tests/requirements-test-latest-deps.txt index d1675cc2..fc8dc7dd 100644 --- a/environments/tests/requirements-test-latest-deps.txt +++ b/environments/tests/requirements-test-latest-deps.txt @@ -2,7 +2,6 @@ # /environments/make-env-files.py using the environment definition files in # /environments/requirements/ and the requirements in pyproject.toml. geopandas>=1.0 -hatch lxml matplotlib>=3.5 networkx>=2.5 @@ -18,6 +17,4 @@ rio-vrt>=0.3 scikit-learn>=0.23 scipy>=1.5 shapely>=2.0 -twine typeguard -validate-pyproject diff --git a/environments/unix-create-env.sh b/environments/unix-create-env.sh index 8387ad4a..d22724a3 100755 --- a/environments/unix-create-env.sh +++ b/environments/unix-create-env.sh @@ -9,10 +9,10 @@ conda activate base conda env remove --yes -n $ENV || true mamba create --yes -c conda-forge --strict-channel-priority -n $ENV --file ./requirements/requirements-all.txt conda activate $ENV -python -m pip --python $ENV_PATH uninstall $PACKAGE --yes -python -m pip --python $ENV_PATH install -e ../. -python -m ipykernel install --prefix $ENV_PATH --name $ENV --display-name "Python ($ENV)" +python -m pip --python "$ENV_PATH" uninstall $PACKAGE --yes +python -m pip --python "$ENV_PATH" install -e ../. +python -m ipykernel install --prefix "$ENV_PATH" --name $ENV --display-name "Python ($ENV)" conda list -n $ENV -python -m pip --python $ENV_PATH check +python -m pip --python "$ENV_PATH" check jupyter kernelspec list ipython -c "import $PACKAGE; print('$PACKAGE version', $PACKAGE.__version__)" diff --git a/tests/lint_test.sh b/tests/lint_test.sh index f17e0e03..d412c2da 100755 --- a/tests/lint_test.sh +++ b/tests/lint_test.sh @@ -4,6 +4,9 @@ set -euo pipefail # delete temp files and folders rm -r -f ./.coverage* ./.pytest_cache ./.temp ./dist ./docs/build ./*/__pycache__ +# create all the configured environment/requirements files +python ./environments/make-env-files.py + # run the pre-commit hooks for linting/formatting pre-commit run --all-files @@ -12,13 +15,12 @@ python -m validate_pyproject ./pyproject.toml python -m hatch build --clean python -m twine check --strict ./dist/* -# build the docs and test that links are alive -python -m sphinx -E -W --keep-going -b html ./docs/source ./docs/build/html -python -m sphinx -E -W --keep-going -b linkcheck ./docs/source ./docs/build/linkcheck - # run the tests and report the test coverage python -m pytest --verbose --maxfail=1 --typeguard-packages=osmnx --cov=osmnx --cov-report=term-missing:skip-covered +# build the docs and test that links are alive +python -m sphinx -q -a -E -W --keep-going -b html ./docs/source ./docs/build/html +python -m sphinx -q -a -E -W --keep-going -b linkcheck ./docs/source ./docs/build/linkcheck + # delete temp files and folders -sleep 1 rm -r -f ./.coverage* ./.pytest_cache ./.temp ./dist ./docs/build ./*/__pycache__