Update fixate-requirements.py to cope with PIP 20.3.3

.github/workflows/ci.yaml

@@ -120,10 +120,10 @@ jobs:
             ${{ runner.os }}-${{ matrix.python }}-
       - name: Install apt depedencies
         run: |
-          # install dev dependencies for Python LDAP module
+          # install dev dependencies for Python YAML and LDAP packages
           # https://github.com/StackStorm/st2-auth-ldap
           sudo apt-get -y update
-          sudo apt-get -f -y install libldap2-dev libsasl2-dev libssl-dev ldap-utils
+          sudo apt-get -f -y install libldap2-dev libsasl2-dev libssl-dev libyaml-dev ldap-utils
       - name: Install virtualenv
         run: |
           # Note: Use the verison of virtualenv pinned in fixed-requirements.txt so we

CHANGELOG.rst

@@ -11,6 +11,9 @@ Changed
   required for all the new code. (#5156)
 
   Contributed by @Kami.
+
+* Updated fixate-requirements to work with PIP 20.3.3 #512
+  Contributed by Amanda McGuinness (@amanda11 Ammeon Solutions)
 
 3.4.0 - March 02, 2021
 ----------------------

Makefile

@@ -55,7 +55,8 @@ REQUIREMENTS := test-requirements.txt requirements.txt
 
 # Pin common pip version here across all the targets
 # Note! Periodic maintenance pip upgrades are required to be up-to-date with the latest pip security fixes and updates
-PIP_VERSION ?= 20.0.2
+PIP_VERSION ?= 20.3.3
+SETUPTOOLS_VERSION ?= 51.3.3
 PIP_OPTIONS := $(ST2_PIP_OPTIONS)
 
 ifndef PYLINT_CONCURRENCY
@@ -461,11 +462,8 @@ flake8: requirements .flake8
 	touch $(VIRTUALENV_ST2CLIENT_DIR)/bin/activate
 	chmod +x $(VIRTUALENV_ST2CLIENT_DIR)/bin/activate
 
-	# NOTE We need to upgrade setuptools to avoid bug with dependency resolving in old versions
-	# Setuptools 42 added support for python_requires, which is used by the configparser package,
-	# which is required by the importlib-metadata package
 	$(VIRTUALENV_ST2CLIENT_DIR)/bin/pip install --upgrade "pip==$(PIP_VERSION)"
-	$(VIRTUALENV_ST2CLIENT_DIR)/bin/pip install --upgrade "setuptools==44.1.0"
+	$(VIRTUALENV_ST2CLIENT_DIR)/bin/pip install --upgrade "setuptools==$(SETUPTOOLS_VERSION)"
 
 	$(VIRTUALENV_ST2CLIENT_DIR)/bin/activate; cd st2client ; ../$(VIRTUALENV_ST2CLIENT_DIR)/bin/python setup.py install ; cd ..
 	$(VIRTUALENV_ST2CLIENT_DIR)/bin/st2 --version
@@ -598,9 +596,7 @@ requirements: virtualenv .requirements .sdist-requirements install-runners insta
 	#       only have to update it one place when we change the version
 	$(VIRTUALENV_DIR)/bin/pip install --upgrade $(shell grep "^virtualenv" fixed-requirements.txt)
 
-	# setuptools >= 41.0.1 is required for packs.install in dev envs
-	# setuptools >= 42     is required so setup.py install respects dependencies' python_requires
-	$(VIRTUALENV_DIR)/bin/pip install --upgrade "setuptools==44.1.0"
+	$(VIRTUALENV_DIR)/bin/pip install --upgrade "setuptools==$(SETUPTOOLS_VERSION)"  # workaround for pbr issue
 	$(VIRTUALENV_DIR)/bin/pip install --upgrade "pbr==5.4.3"  # workaround for pbr issue
 
 	# Fix for Travis CI race

fixed-requirements.txt

@@ -46,9 +46,8 @@ six==1.13.0
 sseclient-py==1.7
 stevedore==1.30.1
 tooz==2.8.0
-# Note: virtualenv embeds the pip wheel, so pinning virtualenv also pins pip
-# virtualenv==20.0.18 has pip==20.0.2
-virtualenv==20.0.18
+# Note: Use version of virtualenv which uses a pip < 21 - 20.4.0 uses pip 20.3.3, wheel 0.36.2 and setuptools 51.3.3
+virtualenv==20.4.0
 webob==1.8.5
 zake==0.2.2
 # test requirements below

requirements.txt

@@ -23,7 +23,6 @@ git+https://github.com/StackStorm/st2-rbac-backend.git@master#egg=st2-rbac-backe
 gitpython==2.1.15
 greenlet==0.4.15
 gunicorn==19.9.0
-ipaddr
 jinja2==2.10.3
 jsonpath-rw==1.4.0
 jsonschema==2.6.0
@@ -36,14 +35,14 @@ networkx==1.11
 nose
 nose-parallel==0.3.1
 nose-timer==0.7.5
-oslo.config<1.13,>=1.12.1
+oslo.config>=1.12.1,<1.13
 oslo.utils<5.0,>=4.0.0
 paramiko==2.7.1
 passlib==1.7.1
 prettytable
 prompt-toolkit==1.0.15
 psutil==5.6.6
-pyinotify==0.9.6 ; platform_system == "Linux"
+pyinotify==0.9.6; platform_system=="Linux"
 pymongo==3.10.0
 pyrabbit
 python-dateutil==2.8.0
@@ -68,4 +67,4 @@ unittest2
 webob==1.8.5
 webtest
 zake==0.2.2
-zipp<=1.0.0,>=0.5
+zipp>=0.5,<=1.0.0

scripts/fixate-requirements.py

@@ -70,6 +70,12 @@
         print("Using pip: %s" % (str(pip_version)))
         sys.exit(1)
 
+try:
+    from pip._internal.req.constructors import parse_req_from_line
+except ImportError:
+    # Do not error, as will only use on pip >= 20
+    pass
+
 
 def parse_args():
     parser = argparse.ArgumentParser(
@@ -138,19 +144,34 @@ def merge_source_requirements(sources):
     merged_requirements = []
     for infile_path in (locate_file(p, must_exist=True) for p in sources):
         for req in load_requirements(infile_path):
-            # Requirements starting with project name "project ..."
-            if req.req:
-                # Skip already added project name
-                if req.name in projects:
-                    continue
-                projects.add(req.name)
-                merged_requirements.append(req)
-
-            # Requirements lines like "vcs+proto://url"
-            elif req.link:
-                merged_requirements.append(req)
+            if hasattr(req, "requirement"):
+                # Requirements starting with project name "project ..."
+                parsedreq = parse_req_from_line(req.requirement, req.line_source)
+                if parsedreq.requirement:
+                    # Skip already added project name
+                    if parsedreq.requirement.name in projects:
+                        continue
+                    projects.add(parsedreq.requirement.name)
+                    merged_requirements.append(req)
+
+                    # Requirements lines like "vcs+proto://url"
+                elif parsedreq.link:
+                    merged_requirements.append(req)
+                else:
+                    raise RuntimeError('Unexpected requirement {0}'.format(req))
             else:
-                raise RuntimeError("Unexpected requirement {0}".format(req))
+                if req.req:
+                    # Skip already added project name
+                    if req.name in projects:
+                        continue
+                    projects.add(req.name)
+                    merged_requirements.append(req)
+
+                    # Requirements lines like "vcs+proto://url"
+                elif req.link:
+                    merged_requirements.append(req)
+                else:
+                    raise RuntimeError('Unexpected requirement {0}'.format(req))
 
     return merged_requirements
 
@@ -169,10 +190,17 @@ def write_requirements(
     # Make sure there are no duplicate / conflicting definitions
     fixedreq_hash = {}
     for req in fixed:
-        project_name = req.name
+        if hasattr(req, "requirement"):
+            parsedreq = parse_req_from_line(req.requirement, req.line_source)
+            project_name = parsedreq.requirement.name
 
-        if not req.req:
-            continue
+            if not req.requirement:
+                continue
+        else:
+            project_name = req.name
+
+            if not req.req:
+                continue
 
         if project_name in fixedreq_hash:
             raise ValueError(
@@ -184,22 +212,37 @@ def write_requirements(
     lines_to_write = []
     links = set()
     for req in requirements:
-        if req.name in skip:
+        if hasattr(req, "requirement"):
+            parsedreq = parse_req_from_line(req.requirement, req.line_source)
+            project_name = parsedreq.requirement.name
+            linkreq = parsedreq
+        else:
+            project_name = req.name
+            linkreq = req
+        if project_name in skip:
             continue
 
         # we don't have any idea how to process links, so just add them
-        if req.link and req.link not in links:
-            links.add(req.link)
-            rline = str(req.link)
+        if linkreq.link and linkreq.link not in links:
+            links.add(linkreq.link)
+            rline = str(linkreq.link)
 
-            if req.editable:
+            if (hasattr(req, "is_editable") and req.is_editable) or \
+               (hasattr(req, "editable") and req.editable):
                 rline = "-e %s" % (rline)
-        elif req.req:
+        elif hasattr(req, "requirement") and req.requirement:
+            project = parsedreq.requirement.name
+            req_obj = fixedreq_hash.get(project, req)
+
+            rline = str(req_obj.requirement)
+
+            # Markers are included in req_obj.requirement, so no
+            # special processing required
+        elif hasattr(req, "req") and req.req:
             project = req.name
             req_obj = fixedreq_hash.get(project, req)
 
             rline = str(req_obj.req)
-
             # Also write out environment markers
             if req_obj.markers:
                 rline += " ; {}".format(str(req_obj.markers))

st2actions/requirements.txt

@@ -13,9 +13,9 @@ gitpython==2.1.15
 jinja2==2.10.3
 kombu==4.6.6
 lockfile==0.12.2
-oslo.config<1.13,>=1.12.1
+oslo.config>=1.12.1,<1.13
 oslo.utils<5.0,>=4.0.0
-pyinotify==0.9.6 ; platform_system == "Linux"
+pyinotify==0.9.6; platform_system=="Linux"
 python-dateutil==2.8.0
 python-json-logger
 pyyaml==5.1.2

st2api/requirements.txt

@@ -10,7 +10,7 @@ gunicorn==19.9.0
 jsonschema==2.6.0
 kombu==4.6.6
 mongoengine==0.18.2
-oslo.config<1.13,>=1.12.1
+oslo.config>=1.12.1,<1.13
 oslo.utils<5.0,>=4.0.0
 pymongo==3.10.0
 simplejson

st2auth/requirements.txt

@@ -10,7 +10,7 @@ eventlet==0.25.1
 git+https://github.com/StackStorm/st2-auth-backend-flat-file.git@master#egg=st2-auth-backend-flat-file
 git+https://github.com/StackStorm/st2-auth-ldap.git@master#egg=st2-auth-ldap
 gunicorn==19.9.0
-oslo.config<1.13,>=1.12.1
+oslo.config>=1.12.1,<1.13
 passlib==1.7.1
 pymongo==3.10.0
 six==1.13.0

st2client/requirements.txt

@@ -20,4 +20,4 @@ pyyaml==5.1.2
 requests[security]==2.23.0
 six==1.13.0
 sseclient-py==1.7
-zipp<=1.0.0,>=0.5
+zipp>=0.5,<=1.0.0

st2common/bin/st2-run-pack-tests

@@ -198,7 +198,7 @@ if [ "${CREATE_VIRTUALENV}" = true ]; then
         activate_virtualenv
 
         # Make sure virtualenv is using latest pip version
-        ${VIRTUALENV_DIR}/bin/pip install --upgrade "pip==20.0.2"
+        ${VIRTUALENV_DIR}/bin/pip install --upgrade "pip==20.3.3"
     fi
 
     if [ ! -d "${VIRTUALENV_DIR}" ]; then

st2common/in-requirements.txt

@@ -25,7 +25,6 @@ tooz
 # Required by tooz - on new versions of tooz, all the backend dependencies need
 # to be installed manually
 zake
-ipaddr
 routes
 flex
 webob

st2common/requirements.txt

@@ -16,15 +16,14 @@ git+https://github.com/StackStorm/[email protected]#egg=orquesta
 git+https://github.com/StackStorm/st2-rbac-backend.git@master#egg=st2-rbac-backend
 gitpython==2.1.15
 greenlet==0.4.15
-ipaddr
 jinja2==2.10.3
 jsonpath-rw==1.4.0
 jsonschema==2.6.0
 kombu==4.6.6
 lockfile==0.12.2
 mongoengine==0.18.2
 networkx==1.11
-oslo.config<1.13,>=1.12.1
+oslo.config>=1.12.1,<1.13
 paramiko==2.7.1
 pymongo==3.10.0
 python-dateutil==2.8.0

st2common/st2common/util/ip_utils.py

@@ -16,7 +16,7 @@
 from __future__ import absolute_import
 import re
 
-import ipaddr
+import ipaddress
 from st2common.log import logging
 
 LOG = logging.getLogger(__name__)
@@ -40,7 +40,7 @@ def is_ipv6(ip_str):
     :rtype: ``bool``
     """
     try:
-        addr = ipaddr.IPAddress(ip_str)
+        addr = ipaddress.ip_address(ip_str)
         return addr.version == 6
     except:
         return False
@@ -56,7 +56,7 @@ def is_ipv4(ip_str):
     :rtype: ``bool``
     """
     try:
-        addr = ipaddr.IPAddress(ip_str)
+        addr = ipaddress.ip_address(ip_str)
         return addr.version == 4
     except:
         return False

st2exporter/requirements.txt

@@ -7,5 +7,5 @@
 # update the component requirements.txt
 eventlet==0.25.1
 kombu==4.6.6
-oslo.config<1.13,>=1.12.1
+oslo.config>=1.12.1,<1.13
 six==1.13.0

st2reactor/requirements.txt

@@ -10,6 +10,6 @@ eventlet==0.25.1
 jsonpath-rw==1.4.0
 jsonschema==2.6.0
 kombu==4.6.6
-oslo.config<1.13,>=1.12.1
+oslo.config>=1.12.1,<1.13
 python-dateutil==2.8.0
 six==1.13.0

st2stream/requirements.txt

@@ -10,7 +10,7 @@ gunicorn==19.9.0
 jsonschema==2.6.0
 kombu==4.6.6
 mongoengine==0.18.2
-oslo.config<1.13,>=1.12.1
+oslo.config>=1.12.1,<1.13
 oslo.utils<5.0,>=4.0.0
 pymongo==3.10.0
 six==1.13.0