Merge pull request #1261 from gboeing/reqs

simplify environment/requirements files

.github/workflows/ci.yml

@@ -55,6 +55,12 @@ jobs:
         env:
           SKIP: no-commit-to-branch
 
+      - name: Build and check package
+        run: |
+          python -m validate_pyproject ./pyproject.toml
+          python -m hatch build --clean
+          python -m twine check --strict ./dist/*
+
       - name: Test docs build
         run: python -m sphinx -E -W --keep-going -b html ./docs/source ./docs/build/html
 

docs/requirements-rtd.txt

@@ -1,6 +1,6 @@
 # Do not edit this file. It is automatically generated by the script
-# environments/make-env-files.py using the environment definition data in
-# environments/environments.json and the requirements in pyproject.toml.
+# /environments/make-env-files.py using the environment definition files in
+# /environments/requirements/ and the requirements in pyproject.toml.
 furo
 sphinx-autodoc-typehints
 sphinx>=7

environments/docker/Dockerfile

@@ -4,7 +4,7 @@ LABEL url="https://osmnx.readthedocs.io"
 LABEL description="OSMnx is a Python package to easily download, model, analyze, and visualize street networks and other geospatial features from OpenStreetMap."
 
 # copy the package files needed for installation
-COPY --chmod=0755 ./environments/requirements.txt ./osmnx/
+COPY --chmod=0755 ./environments/requirements/requirements-all.txt ./osmnx/
 COPY --chmod=0755 ./osmnx/ ./osmnx/osmnx/
 COPY --chmod=0755 ./pyproject.toml ./osmnx/
 COPY --chmod=0755 ./README.md ./osmnx/

environments/make-env-files.py

@@ -13,34 +13,28 @@
 
 # path to package's pyproject and the config json file
 pyproject_path = "./pyproject.toml"
-environments_config_path = "./environments/environments.json"
+environments_config_path = "./environments/requirements/environments.json"
 
 # what channels to specify in conda env yml files
 CHANNELS = ["conda-forge"]
 
 HEADER = (
     "# Do not edit this file. It is automatically generated by the script\n"
-    "# environments/make-env-files.py using the environment definition data in\n"
-    "# environments/environments.json and the requirements in pyproject.toml.\n"
+    "# /environments/make-env-files.py using the environment definition files in\n"
+    "# /environments/requirements/ and the requirements in pyproject.toml.\n"
 )
 
 
-def extract_optional_deps(which: list[str] | None = None) -> list[Requirement]:
+def extract_optional_deps() -> list[Requirement]:
     """
     Extract a list of the optional dependencies/versions from pyproject.toml.
 
-    Parameters
-    ----------
-    which
-        Which optional dependencies to extract. If None, extract them all.
-
     Returns
     -------
     optional_deps
     """
     opts = pyproject["project"]["optional-dependencies"]
-    opts = [v for k, v in opts.items() if k in which] if which is not None else opts.values()
-    return list({Requirement(o) for o in itertools.chain.from_iterable(opts)})
+    return list({Requirement(o) for o in itertools.chain.from_iterable(opts.values())})
 
 
 def make_requirement(
@@ -94,7 +88,7 @@ def make_file(env_name: str) -> None:
 
     # it's a conda env file if it ends with ".yml", otherwise it's a pip
     # requirements.txt file
-    is_conda = env["filepath"].endswith(".yml")
+    is_conda = env["output_path"].endswith(".yml")
 
     # determine which dependencies to add based on the configuration
     depends_on = []
@@ -105,33 +99,39 @@ def make_file(env_name: str) -> None:
         dependencies = [Requirement(d) for d in pyproject["project"]["dependencies"]]
         depends_on.extend(dependencies)
     if env["needs_optionals"]:
-        optionals = extract_optional_deps(which=env["which_optionals"])
+        optionals = extract_optional_deps()
         depends_on.extend(optionals)
 
     # make the list of requirements
-    requirements = sorted(
+    requirements = [
         make_requirement(dep, force_pin=env["force_pin"], is_conda=is_conda) for dep in depends_on
-    )
+    ]
 
     # add any extra requirements if provided in the configuration
     if env["extras"] is not None:
-        requirements = sorted(requirements + env["extras"])
-
-    # write the conda env yml or pip requirements.txt file to disk
-    with Path(env["filepath"]).open("w") as f:
-        if is_conda:
-            data = {"name": env_name, "channels": CHANNELS, "dependencies": requirements}
-            text = ""
-            for k, v in data.items():
-                if isinstance(v, list):
-                    text += k + ":\n  - " + "\n  - ".join(v) + "\n"
-                elif isinstance(v, str):
-                    text += k + ": " + v + "\n"
-            f.writelines(HEADER + text)
-        else:
-            f.writelines(HEADER + "\n".join(requirements) + "\n")
-
-    print(f"Wrote {len(requirements)} requirements to {env['filepath']!r}")  # noqa: T201
+        for extras_filepath in env["extras"]:
+            with Path(extras_filepath).open() as f:
+                requirements += f.read().splitlines()
+
+    # convert the requirements to conda env yml or pip requirements.txt
+    requirements = sorted(requirements)
+    if not is_conda:
+        text = HEADER + "\n".join(requirements) + "\n"
+    else:
+        data = {"name": env_name, "channels": CHANNELS, "dependencies": requirements}
+        text = ""
+        for k, v in data.items():
+            if isinstance(v, list):
+                text += k + ":\n  - " + "\n  - ".join(v) + "\n"
+            elif isinstance(v, str):
+                text += k + ": " + v + "\n"
+        text = HEADER + text
+
+    # write the file to disk
+    with Path(env["output_path"]).open("w") as f:
+        f.writelines(text)
+
+    print(f"Wrote {len(requirements)} requirements to {env['output_path']!r}")  # noqa: T201
 
 
 if __name__ == "__main__":

environments/requirements/environments.json

@@ -0,0 +1,45 @@
+{
+ "env-ci": {
+  "output_path": "./environments/tests/env-ci.yml",
+  "needs_python": true,
+  "needs_dependencies": true,
+  "needs_optionals": true,
+  "force_pin": false,
+  "extras": ["./environments/requirements/requirements-docs.txt",
+    "./environments/requirements/requirements-tests.txt"]
+ },
+ "env-test-minimum-deps": {
+  "output_path": "./environments/tests/env-test-minimum-deps.yml",
+  "needs_python": true,
+  "needs_dependencies": true,
+  "needs_optionals": true,
+  "force_pin": true,
+  "extras": ["./environments/requirements/requirements-tests.txt"]
+ },
+ "requirements-test-latest-deps": {
+  "output_path": "./environments/tests/requirements-test-latest-deps.txt",
+  "needs_python": false,
+  "needs_dependencies": true,
+  "needs_optionals": true,
+  "force_pin": false,
+  "extras": ["./environments/requirements/requirements-tests.txt"]
+ },
+ "requirements-rtd": {
+  "output_path": "./docs/requirements-rtd.txt",
+  "needs_python": false,
+  "needs_dependencies": false,
+  "needs_optionals": false,
+  "force_pin": false,
+  "extras": ["./environments/requirements/requirements-docs.txt"]
+ },
+ "requirements-all": {
+  "output_path": "./environments/requirements/requirements-all.txt",
+  "needs_python": false,
+  "needs_dependencies": true,
+  "needs_optionals": true,
+  "force_pin": false,
+  "extras": ["./environments/requirements/requirements-docs.txt",
+    "./environments/requirements/requirements-extras.txt",
+    "./environments/requirements/requirements-tests.txt"]
+ }
+}

environments/requirements.txt → environments/requirements/requirements-all.txt

@@ -1,6 +1,6 @@
 # Do not edit this file. It is automatically generated by the script
-# environments/make-env-files.py using the environment definition data in
-# environments/environments.json and the requirements in pyproject.toml.
+# /environments/make-env-files.py using the environment definition files in
+# /environments/requirements/ and the requirements in pyproject.toml.
 bottleneck
 cartopy
 conda-smithy
@@ -12,7 +12,6 @@ jupyterlab
 lxml
 matplotlib>=3.5
 nbdime
-nbqa
 networkx>=2.5
 numexpr
 numpy>=1.22

environments/requirements/requirements-docs.txt

@@ -0,0 +1,3 @@
+furo
+sphinx-autodoc-typehints
+sphinx>=7

environments/requirements/requirements-extras.txt

@@ -0,0 +1,12 @@
+bottleneck
+cartopy
+conda-smithy
+folium
+jupyterlab
+nbdime
+numexpr
+pillow
+pysal>24
+python-igraph
+seaborn
+statsmodels

environments/requirements/requirements-tests.txt

@@ -0,0 +1,9 @@
+hatch
+lxml
+pip
+pre-commit
+pytest
+pytest-cov
+twine
+typeguard
+validate-pyproject

environments/tests/env-ci.yml

@@ -1,17 +1,19 @@
 # Do not edit this file. It is automatically generated by the script
-# environments/make-env-files.py using the environment definition data in
-# environments/environments.json and the requirements in pyproject.toml.
+# /environments/make-env-files.py using the environment definition files in
+# /environments/requirements/ and the requirements in pyproject.toml.
 name: env-ci
 channels:
   - conda-forge
 dependencies:
   - furo
   - geopandas>=1.0
+  - hatch
   - lxml
   - matplotlib>=3.5
   - networkx>=2.5
   - numpy>=1.22
   - pandas>=1.4
+  - pip
   - pre-commit
   - pytest
   - pytest-cov
@@ -24,4 +26,6 @@ dependencies:
   - shapely>=2.0
   - sphinx-autodoc-typehints
   - sphinx>=7
+  - twine
   - typeguard
+  - validate-pyproject

environments/tests/env-test-minimum-deps.yml

@@ -1,16 +1,18 @@
 # Do not edit this file. It is automatically generated by the script
-# environments/make-env-files.py using the environment definition data in
-# environments/environments.json and the requirements in pyproject.toml.
+# /environments/make-env-files.py using the environment definition files in
+# /environments/requirements/ and the requirements in pyproject.toml.
 name: env-test-minimum-deps
 channels:
   - conda-forge
 dependencies:
   - geopandas==1.0.*
+  - hatch
   - lxml
   - matplotlib==3.5.*
   - networkx==2.5.*
   - numpy==1.22.*
   - pandas==1.4.*
+  - pip
   - pre-commit
   - pytest
   - pytest-cov
@@ -21,4 +23,6 @@ dependencies:
   - scikit-learn==0.23.*
   - scipy==1.5.*
   - shapely==2.0.*
+  - twine
   - typeguard
+  - validate-pyproject

environments/tests/requirements-test-latest-deps.txt

@@ -1,13 +1,14 @@
 # Do not edit this file. It is automatically generated by the script
-# environments/make-env-files.py using the environment definition data in
-# environments/environments.json and the requirements in pyproject.toml.
-furo
+# /environments/make-env-files.py using the environment definition files in
+# /environments/requirements/ and the requirements in pyproject.toml.
 geopandas>=1.0
+hatch
 lxml
 matplotlib>=3.5
 networkx>=2.5
 numpy>=1.22
 pandas>=1.4
+pip
 pre-commit
 pytest
 pytest-cov
@@ -17,6 +18,6 @@ rio-vrt>=0.3
 scikit-learn>=0.23
 scipy>=1.5
 shapely>=2.0
-sphinx-autodoc-typehints
-sphinx>=7
+twine
 typeguard
+validate-pyproject

environments/unix-create-env.sh

@@ -7,7 +7,7 @@ PACKAGE=osmnx
 eval "$(conda shell.bash hook)"
 conda activate base
 conda env remove --yes -n $ENV || true
-mamba create --yes -c conda-forge --strict-channel-priority -n $ENV --file requirements.txt
+mamba create --yes -c conda-forge --strict-channel-priority -n $ENV --file ./requirements/requirements-all.txt
 conda activate $ENV
 python -m pip --python $ENV_PATH uninstall $PACKAGE --yes
 python -m pip --python $ENV_PATH install -e ../.

environments/windows-create-env.bat

@@ -5,7 +5,7 @@ SET PACKAGE=osmnx
 CALL %CONDA_ROOT%\Scripts\activate.bat && ^
 conda activate base && ^
 conda env remove --yes -n %ENV% && ^
-mamba create --yes -c conda-forge --strict-channel-priority -n %ENV% --file requirements.txt && ^
+mamba create --yes -c conda-forge --strict-channel-priority -n %ENV% --file requirements\requirements-all.txt && ^
 conda activate %ENV% && ^
 python -m pip --python %ENV_PATH%\python.exe uninstall %PACKAGE% --yes && ^
 python -m pip --python %ENV_PATH%\python.exe install -e ../. && ^