Update fixate-requirements.py to cope with PIP 20.3.3

.github/workflows/ci.yaml

@@ -111,10 +111,10 @@ jobs:
             ${{ runner.os }}-${{ matrix.python }}-
       - name: Install apt depedencies
         run: |
-          # install dev dependencies for Python LDAP module
+          # install dev dependencies for Python YAML and LDAP packages
           # https://github.com/StackStorm/st2-auth-ldap
           sudo apt-get -y update
-          sudo apt-get -f -y install libldap2-dev libsasl2-dev libssl-dev ldap-utils
+          sudo apt-get -f -y install libldap2-dev libsasl2-dev libssl-dev libyaml-dev ldap-utils
       - name: Install virtualenv
         run: |
           # Note: Use the verison of virtualenv pinned in fixed-requirements.txt so we

CHANGELOG.rst

@@ -37,6 +37,9 @@ Changed
 
   Contributed by @nmaludy, @winem, and @blag
 
+* Updated fixate-requirements to work with PIP 20.3.3 #512
+  Contributed by Amanda McGuinness (@amanda11 Ammeon Solutions)
+
 Fixed
 ~~~~~~~~~
 * Pin chardet version as newest version was incompatible with pinned requests version #5101

Makefile

@@ -55,7 +55,8 @@ REQUIREMENTS := test-requirements.txt requirements.txt
 
 # Pin common pip version here across all the targets
 # Note! Periodic maintenance pip upgrades are required to be up-to-date with the latest pip security fixes and updates
-PIP_VERSION ?= 20.0.2
+#PIP_VERSION ?= 20.0.2
+PIP_VERSION ?= 20.3.4
 PIP_OPTIONS := $(ST2_PIP_OPTIONS)
 
 ifndef PYLINT_CONCURRENCY
@@ -394,11 +395,7 @@ flake8: requirements .flake8
 	touch $(VIRTUALENV_ST2CLIENT_DIR)/bin/activate
 	chmod +x $(VIRTUALENV_ST2CLIENT_DIR)/bin/activate
 
-	# NOTE We need to upgrade setuptools to avoid bug with dependency resolving in old versions
-	# Setuptools 42 added support for python_requires, which is used by the configparser package,
-	# which is required by the importlib-metadata package
 	$(VIRTUALENV_ST2CLIENT_DIR)/bin/pip install --upgrade "pip==$(PIP_VERSION)"
-	$(VIRTUALENV_ST2CLIENT_DIR)/bin/pip install --upgrade "setuptools==44.1.0"
 
 	$(VIRTUALENV_ST2CLIENT_DIR)/bin/activate; cd st2client ; ../$(VIRTUALENV_ST2CLIENT_DIR)/bin/python setup.py install ; cd ..
 	$(VIRTUALENV_ST2CLIENT_DIR)/bin/st2 --version
@@ -531,9 +528,6 @@ requirements: virtualenv .requirements .sdist-requirements install-runners insta
 	#       only have to update it one place when we change the version
 	$(VIRTUALENV_DIR)/bin/pip install --upgrade $(shell grep "^virtualenv" fixed-requirements.txt)
 
-	# setuptools >= 41.0.1 is required for packs.install in dev envs
-	# setuptools >= 42     is required so setup.py install respects dependencies' python_requires
-	$(VIRTUALENV_DIR)/bin/pip install --upgrade "setuptools==44.1.0"
 	$(VIRTUALENV_DIR)/bin/pip install --upgrade "pbr==5.4.3"  # workaround for pbr issue
 
 	# Fix for Travis CI race

requirements.txt

@@ -12,7 +12,7 @@ argcomplete
 bcrypt==3.1.7
 chardet<3.1.0
 cryptography==3.2
-dnspython<2.0.0,>=1.16.0
+dnspython>=1.16.0,<2.0.0
 eventlet==0.25.1
 flex==6.14.0
 git+https://github.com/StackStorm/logshipper.git@stackstorm_patched#egg=logshipper
@@ -23,7 +23,6 @@ git+https://github.com/StackStorm/st2-rbac-backend.git@master#egg=st2-rbac-backe
 gitpython==2.1.15
 greenlet==0.4.15
 gunicorn==19.9.0
-ipaddr
 jinja2==2.10.3
 jsonpath-rw==1.4.0
 jsonschema==2.6.0
@@ -36,14 +35,14 @@ networkx==1.11
 nose
 nose-parallel==0.3.1
 nose-timer==0.7.5
-oslo.config<1.13,>=1.12.1
+oslo.config>=1.12.1,<1.13
 oslo.utils<5.0,>=4.0.0
 paramiko==2.7.1
 passlib==1.7.1
 prettytable
 prompt-toolkit==1.0.15
 psutil==5.6.6
-pyinotify==0.9.6 ; platform_system == "Linux"
+pyinotify==0.9.6; platform_system=="Linux"
 pymongo==3.10.0
 pyrabbit
 python-dateutil==2.8.0
@@ -68,4 +67,4 @@ unittest2
 webob==1.8.5
 webtest
 zake==0.2.2
-zipp<=1.0.0,>=0.5
+zipp>=0.5,<=1.0.0

scripts/fixate-requirements.py

@@ -70,6 +70,12 @@
         print('Using pip: %s' % (str(pip_version)))
         sys.exit(1)
 
+try:
+    from pip._internal.req.constructors import parse_req_from_line
+except ImportError as e:
+    # Do not error, as will only use on pip >= 20
+    pass
+
 
 def parse_args():
     parser = argparse.ArgumentParser(description='Tool for requirements.txt generation.')
@@ -117,19 +123,34 @@ def merge_source_requirements(sources):
     merged_requirements = []
     for infile_path in (locate_file(p, must_exist=True) for p in sources):
         for req in load_requirements(infile_path):
-            # Requirements starting with project name "project ..."
-            if req.req:
-                # Skip already added project name
-                if req.name in projects:
-                    continue
-                projects.add(req.name)
-                merged_requirements.append(req)
-
-            # Requirements lines like "vcs+proto://url"
-            elif req.link:
-                merged_requirements.append(req)
+            if hasattr(req, "requirement"):
+                # Requirements starting with project name "project ..."
+                parsedreq = parse_req_from_line(req.requirement, req.line_source)
+                if parsedreq.requirement:
+                    # Skip already added project name
+                    if parsedreq.requirement.name in projects:
+                        continue
+                    projects.add(parsedreq.requirement.name)
+                    merged_requirements.append(req)
+
+                    # Requirements lines like "vcs+proto://url"
+                elif parsedreq.link:
+                    merged_requirements.append(req)
+                else:
+                    raise RuntimeError('Unexpected requirement {0}'.format(req))
             else:
-                raise RuntimeError('Unexpected requirement {0}'.format(req))
+                if req.req:
+                    # Skip already added project name
+                    if req.name in projects:
+                        continue
+                    projects.add(req.name)
+                    merged_requirements.append(req)
+
+                    # Requirements lines like "vcs+proto://url"
+                elif req.link:
+                    merged_requirements.append(req)
+                else:
+                    raise RuntimeError('Unexpected requirement {0}'.format(req))
 
     return merged_requirements
 
@@ -147,10 +168,17 @@ def write_requirements(sources=None, fixed_requirements=None, output_file=None,
     # Make sure there are no duplicate / conflicting definitions
     fixedreq_hash = {}
     for req in fixed:
-        project_name = req.name
+        if hasattr(req, "requirement"):
+            parsedreq = parse_req_from_line(req.requirement, req.line_source)
+            project_name = parsedreq.requirement.name
 
-        if not req.req:
-            continue
+            if not req.requirement:
+                continue
+        else:
+            project_name = req.name
+
+            if not req.req:
+                continue
 
         if project_name in fixedreq_hash:
             raise ValueError('Duplicate definition for dependency "%s"' % (project_name))
@@ -160,22 +188,37 @@ def write_requirements(sources=None, fixed_requirements=None, output_file=None,
     lines_to_write = []
     links = set()
     for req in requirements:
-        if req.name in skip:
+        if hasattr(req, "requirement"):
+            parsedreq = parse_req_from_line(req.requirement, req.line_source)
+            project_name = parsedreq.requirement.name
+            linkreq = parsedreq
+        else:
+            project_name = req.name
+            linkreq = req
+        if project_name in skip:
             continue
 
         # we don't have any idea how to process links, so just add them
-        if req.link and req.link not in links:
-            links.add(req.link)
-            rline = str(req.link)
+        if linkreq.link and linkreq.link not in links:
+            links.add(linkreq.link)
+            rline = str(linkreq.link)
 
-            if req.editable:
+            if (hasattr(req, "is_editable") and req.is_editable) or \
+                (hasattr(req, "editable") and req.editable):
                 rline = '-e %s' % (rline)
-        elif req.req:
+        elif hasattr(req, "requirement") and req.requirement:
+            project = parsedreq.requirement.name
+            req_obj = fixedreq_hash.get(project, req)
+
+            rline = str(req_obj.requirement)
+
+            # Markers are included in req_obj.requirement, so no
+            # special processing required
+        elif hasattr(req, "req") and req.req:
             project = req.name
             req_obj = fixedreq_hash.get(project, req)
 
             rline = str(req_obj.req)
-
             # Also write out environment markers
             if req_obj.markers:
                 rline += " ; {}".format(str(req_obj.markers))

st2actions/requirements.txt

@@ -13,9 +13,9 @@ gitpython==2.1.15
 jinja2==2.10.3
 kombu==4.6.6
 lockfile==0.12.2
-oslo.config<1.13,>=1.12.1
+oslo.config>=1.12.1,<1.13
 oslo.utils<5.0,>=4.0.0
-pyinotify==0.9.6 ; platform_system == "Linux"
+pyinotify==0.9.6; platform_system=="Linux"
 python-dateutil==2.8.0
 python-json-logger
 pyyaml==5.1.2

st2api/requirements.txt

@@ -10,7 +10,7 @@ gunicorn==19.9.0
 jsonschema==2.6.0
 kombu==4.6.6
 mongoengine==0.18.2
-oslo.config<1.13,>=1.12.1
+oslo.config>=1.12.1,<1.13
 oslo.utils<5.0,>=4.0.0
 pymongo==3.10.0
 simplejson

st2auth/requirements.txt

@@ -10,7 +10,7 @@ eventlet==0.25.1
 git+https://github.com/StackStorm/st2-auth-backend-flat-file.git@master#egg=st2-auth-backend-flat-file
 git+https://github.com/StackStorm/st2-auth-ldap.git@master#egg=st2-auth-ldap
 gunicorn==19.9.0
-oslo.config<1.13,>=1.12.1
+oslo.config>=1.12.1,<1.13
 passlib==1.7.1
 pymongo==3.10.0
 six==1.13.0

st2client/requirements.txt

@@ -20,4 +20,4 @@ pyyaml==5.1.2
 requests[security]==2.23.0
 six==1.13.0
 sseclient-py==1.7
-zipp<=1.0.0,>=0.5
+zipp>=0.5,<=1.0.0

st2common/in-requirements.txt

@@ -25,7 +25,6 @@ tooz
 # Required by tooz - on new versions of tooz, all the backend dependencies need
 # to be installed manually
 zake
-ipaddr
 routes
 flex
 webob

st2common/requirements.txt

@@ -9,22 +9,21 @@ amqp==2.5.2
 apscheduler==3.6.3
 chardet<3.1.0
 cryptography==3.2
-dnspython<2.0.0,>=1.16.0
+dnspython>=1.16.0,<2.0.0
 eventlet==0.25.1
 flex==6.14.0
 git+https://github.com/StackStorm/[email protected]#egg=orquesta
 git+https://github.com/StackStorm/st2-rbac-backend.git@master#egg=st2-rbac-backend
 gitpython==2.1.15
 greenlet==0.4.15
-ipaddr
 jinja2==2.10.3
 jsonpath-rw==1.4.0
 jsonschema==2.6.0
 kombu==4.6.6
 lockfile==0.12.2
 mongoengine==0.18.2
 networkx==1.11
-oslo.config<1.13,>=1.12.1
+oslo.config>=1.12.1,<1.13
 paramiko==2.7.1
 pymongo==3.10.0
 python-dateutil==2.8.0

st2common/st2common/util/ip_utils.py

@@ -16,7 +16,7 @@
 from __future__ import absolute_import
 import re
 
-import ipaddr
+import ipaddress
 from st2common.log import logging
 
 LOG = logging.getLogger(__name__)
@@ -44,7 +44,7 @@ def is_ipv6(ip_str):
     :rtype: ``bool``
     """
     try:
-        addr = ipaddr.IPAddress(ip_str)
+        addr = ipaddress.ip_address(ip_str)
         return addr.version == 6
     except:
         return False
@@ -60,7 +60,7 @@ def is_ipv4(ip_str):
     :rtype: ``bool``
     """
     try:
-        addr = ipaddr.IPAddress(ip_str)
+        addr = ipaddress.ip_address(ip_str)
         return addr.version == 4
     except:
         return False

st2exporter/requirements.txt

@@ -7,5 +7,5 @@
 # update the component requirements.txt
 eventlet==0.25.1
 kombu==4.6.6
-oslo.config<1.13,>=1.12.1
+oslo.config>=1.12.1,<1.13
 six==1.13.0

st2reactor/requirements.txt

@@ -10,6 +10,6 @@ eventlet==0.25.1
 jsonpath-rw==1.4.0
 jsonschema==2.6.0
 kombu==4.6.6
-oslo.config<1.13,>=1.12.1
+oslo.config>=1.12.1,<1.13
 python-dateutil==2.8.0
 six==1.13.0

st2stream/requirements.txt

@@ -10,7 +10,7 @@ gunicorn==19.9.0
 jsonschema==2.6.0
 kombu==4.6.6
 mongoengine==0.18.2
-oslo.config<1.13,>=1.12.1
+oslo.config>=1.12.1,<1.13
 oslo.utils<5.0,>=4.0.0
 pymongo==3.10.0
 six==1.13.0

st2tests/st2tests/__init__.py

@@ -30,4 +30,4 @@
     'WorkflowTestCase'
 ]
 
-__version__ = '3.3dev'
+__version__ = '3.4dev'

test-requirements.txt

@@ -7,7 +7,7 @@ pylint==2.6.0
 pylint-plugin-utils>=0.4
 bandit==1.5.1
 ipython<6.0.0
-isort<=4.0.0
+isort>=4.2.5,<5
 mock==3.0.5
 nose>=1.3.7
 tabulate