Merge pull request cylc#5790 from oliver-sanders/5775

replace pkg_resources with importlib_metadata and packaging

.github/workflows/test_fast.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 20
     strategy:
-      fail-fast: false # Don't let a failed MacOS run stop the Ubuntu runs
+      fail-fast: false # don't stop on first failure
       matrix:
         os: ['ubuntu-latest']
         python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
@@ -39,7 +39,7 @@ jobs:
         if: startsWith(matrix.os, 'ubuntu')
         run: |
           sudo apt-get update
-          sudo apt-get install -y shellcheck sqlite3
+          sudo apt-get install -y sqlite3
 
       - name: Install
         run: |
@@ -48,37 +48,10 @@ jobs:
       - name: Configure git  # Needed by the odd test
         uses: cylc/release-actions/configure-git@v1
 
-      - name: Check changelog
-        if: startsWith(matrix.os, 'ubuntu')
-        run: towncrier build --draft
-
-      - name: Style
-        if: startsWith(matrix.os, 'ubuntu')
-        run: |
-          flake8
-          etc/bin/shellchecker
-
-      # note: exclude python 3.10+ from mypy checks as these produce false
-      # positives in installed libraries for python 3.7
-      - name: Typing
-        if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.python-version, 3.9)
-        run: mypy
-
-      - name: Doctests
-        timeout-minutes: 4
-        run: |
-          pytest cylc/flow
-
       - name: Unit Tests
-        timeout-minutes: 4
-        run: |
-          pytest tests/unit
-
-      - name: Bandit
-        if: ${{ matrix.python-version == '3.7' }}
-        # https://github.com/PyCQA/bandit/issues/658
+        timeout-minutes: 5
         run: |
-          bandit -r --ini .bandit cylc/flow
+          pytest cylc/flow tests/unit
 
       - name: Integration Tests
         timeout-minutes: 6
@@ -104,9 +77,47 @@ jobs:
           path: coverage.xml
           retention-days: 7
 
+  lint:
+    runs-on: 'ubuntu-latest'
+    timeout-minutes: 10
+    steps:
+      - name: Apt-Get Install
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y shellcheck
+
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      # note: exclude python 3.10+ from mypy checks as these produce false
+      # positives in installed libraries for python 3.7
+      - name: Configure Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+
+      - name: Install
+        run: |
+          pip install -e ."[tests]"
+
+      - name: Flake8
+        run: flake8
+
+      - name: Bandit
+        run: |
+          bandit -r --ini .bandit cylc/flow
+
+      - name: Shellchecker
+        run: etc/bin/shellchecker
+
+      - name: MyPy
+        run: mypy
+
+      - name: Towncrier
+        run: towncrier build --draft
+
       - name: Linkcheck
-        if: startsWith(matrix.python-version, '3.10')
-        run: pytest -m linkcheck --dist=load tests/unit
+        run: pytest -m linkcheck --dist=load --color=yes -n 10 tests/unit/test_links.py
 
   codecov:
     needs: test

conda-environment.yml

@@ -10,13 +10,13 @@ dependencies:
   # Note: can't pin jinja2 any higher than this until we give up on Cylc 7 back-compat
   - jinja2 >=3.0,<3.1
   - metomi-isodatetime >=1!3.0.0, <1!3.2.0
+  - packaging
   # Constrain protobuf version for compatible Scheduler-UIS comms across hosts
   - protobuf >=4.21.2,<4.22.0
   - psutil >=5.6.0
   - python
   - pyzmq >=22
-  - setuptools >=49,!=67.*
-  - importlib_metadata # [py<3.8]
+  - importlib_metadata >=5.0 # [py<3.12]
   - urwid >=2,<3
   - tomli >=2 # [py<3.11]
 

cylc/flow/__init__.py

@@ -58,11 +58,19 @@ def environ_init():
 
 def iter_entry_points(entry_point_name):
     """Iterate over Cylc entry points."""
-    import pkg_resources
+    import sys
+    if sys.version_info[:2] > (3, 11):
+        from importlib.metadata import entry_points
+    else:
+        # BACK COMPAT: importlib_metadata
+        #   importlib.metadata was added in Python 3.8. The required interfaces
+        #   were completed by 3.12. For lower versions we must use the
+        #   importlib_metadata backport.
+        # FROM: Python 3.7
+        # TO: Python: 3.12
+        from importlib_metadata import entry_points
     yield from (
         entry_point
-        for entry_point in pkg_resources.iter_entry_points(entry_point_name)
-        # Filter out the cylc namespace as it should be empty.
-        # All cylc packages should take the form cylc-<name>
-        if entry_point.dist.key != 'cylc'
+        # for entry_point in entry_points()[entry_point_name]
+        for entry_point in entry_points().select(group=entry_point_name)
     )

cylc/flow/cfgspec/globalcfg.py

@@ -22,7 +22,7 @@
 from typing import List, Optional, Tuple, Any, Union
 
 from contextlib import suppress
-from pkg_resources import parse_version
+from packaging.version import parse as parse_version, Version
 
 from cylc.flow import LOG
 from cylc.flow import __version__ as CYLC_VERSION
@@ -1866,8 +1866,7 @@ def get_version_hierarchy(version: str) -> List[str]:
         ['', '8', '8.0', '8.0.1', '8.0.1a2', '8.0.1a2.dev']
 
     """
-    smart_ver: Any = parse_version(version)
-    # (No type anno. yet for Version in pkg_resources.extern.packaging.version)
+    smart_ver: Version = parse_version(version)
     base = [str(i) for i in smart_ver.release]
     hierarchy = ['']
     hierarchy += ['.'.join(base[:i]) for i in range(1, len(base) + 1)]

cylc/flow/network/scan.py

@@ -50,10 +50,8 @@
 import re
 from typing import AsyncGenerator, Dict, Iterable, List, Optional, Tuple, Union
 
-from pkg_resources import (
-    parse_requirements,
-    parse_version
-)
+from packaging.version import parse as parse_version
+from packaging.specifiers import SpecifierSet
 
 from cylc.flow import LOG
 from cylc.flow.async_util import (
@@ -354,11 +352,7 @@ async def validate_contact_info(flow):
 
 def parse_requirement(requirement_string):
     """Parse a requirement from a requirement string."""
-    # we have to give the requirement a name but what we call it doesn't
-    # actually matter
-    for req in parse_requirements(f'x {requirement_string}'):
-        # there should only be one requirement
-        return (req,), {}
+    return (SpecifierSet(requirement_string),), {}
 
 
 @pipe(preproc=parse_requirement)
@@ -373,7 +367,7 @@ async def cylc_version(flow, requirement):
         flow (dict):
             Flow information dictionary, provided by scan through the pipe.
         requirement (str):
-            Requirement specifier in pkg_resources format e.g. ``> 8, < 9``
+            Requirement specifier in PEP 440 format e.g. ``> 8, < 9``
 
     """
     return parse_version(flow[ContactFileFields.VERSION]) in requirement
@@ -391,7 +385,7 @@ async def api_version(flow, requirement):
         flow (dict):
             Flow information dictionary, provided by scan through the pipe.
         requirement (str):
-            Requirement specifier in pkg_resources format e.g. ``> 8, < 9``
+            Requirement specifier in PEP 440 format e.g. ``> 8, < 9``
 
     """
     return parse_version(flow[ContactFileFields.API]) in requirement

cylc/flow/parsec/fileparse.py

@@ -277,7 +277,7 @@ def process_plugins(fpath, opts):
         try:
             # If you want it to work on sourcedirs you need to get the options
             # to here.
-            plugin_result = entry_point.resolve()(
+            plugin_result = entry_point.load()(
                 srcdir=fpath, opts=opts
             )
         except Exception as exc:

cylc/flow/scheduler_cli.py

@@ -25,7 +25,7 @@
 import sys
 from typing import TYPE_CHECKING
 
-from pkg_resources import parse_version
+from packaging.version import parse as parse_version
 
 from cylc.flow import LOG, __version__
 from cylc.flow.exceptions import (

cylc/flow/scripts/completion_server.py

@@ -46,10 +46,8 @@
 import sys
 import typing as t
 
-from pkg_resources import (
-    parse_requirements,
-    parse_version
-)
+from packaging.version import parse as parse_version
+from packaging.specifiers import SpecifierSet
 
 from cylc.flow.cfgspec.glbl_cfg import glbl_cfg
 from cylc.flow.id import tokenise, IDTokens, Tokens
@@ -74,7 +72,7 @@
 # set the compatibility range for completion scripts with this server
 # I.E. if we change the server interface, change this compatibility range.
 # User's will be presented with an upgrade notice if this happens.
-REQUIRED_SCRIPT_VERSION = 'completion-script >=1.0.0, <2.0.0'
+REQUIRED_SCRIPT_VERSION = '>=1.0.0, <2.0.0'
 
 # register the psudo "help" and "version" commands
 COMMAND_LIST = list(COMMANDS) + ['help', 'version']
@@ -317,7 +315,7 @@ def list_options(command: str) -> t.List[str]:
     if command in COMMAND_OPTION_MAP:
         return COMMAND_OPTION_MAP[command]
     try:
-        entry_point = COMMANDS[command].resolve()
+        entry_point = COMMANDS[command].load()
     except KeyError:
         return []
     parser = entry_point.parser_function()
@@ -637,15 +635,13 @@ def check_completion_script_compatibility(
 
     # check that the installed completion script is compabile with this
     # completion server version
-    for requirement in parse_requirements(REQUIRED_SCRIPT_VERSION):
-        # NOTE: there's only one requirement but we have to iterate to get it
-        if installed_version not in requirement:
-            is_compatible = False
-            print(
-                f'The Cylc {completion_lang} script needs to be updated to'
-                ' work with this version of Cylc.',
-                file=sys.stderr,
-            )
+    if installed_version not in SpecifierSet(REQUIRED_SCRIPT_VERSION):
+        is_compatible = False
+        print(
+            f'The Cylc {completion_lang} script needs to be updated to'
+            ' work with this version of Cylc.',
+            file=sys.stderr,
+        )
 
     # check for completion script updates
     if installed_version < current_version: