[To Main] Upgrading the version of python (#2476)

* Upgrading to python 3.12 on cron (#2475) * [To feature] Upgrading the version of python on notify-api (#2473) * Upgrading to python 3.12 on notify * Updating few library and adding TODO * [To feature] Upgrading the version of python on met analytics (#2472) * Upgrading to python 3.12 on analytics * Updating few library and addressing review comments * Updating conftest * [To feature] Upgrading the version of python on met api (#2474) * Upgrading to python 3.12 on met api * Updating few more libraries * Updating change log and changes as per review comments * Updating pagination logic
bcgov · Apr 29, 2024 · 7617137 · 7617137
1 parent d27c367
commit 7617137
Show file tree

Hide file tree

Showing 62 changed files with 324 additions and 274 deletions.
diff --git a/.github/workflows/analytics-api-ci.yml b/.github/workflows/analytics-api-ci.yml
@@ -28,7 +28,7 @@ jobs:
 
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: [3.12]
 
     steps:
       - uses: actions/checkout@v2

diff --git a/.github/workflows/met-api-ci.yml b/.github/workflows/met-api-ci.yml
@@ -31,7 +31,7 @@ jobs:
 
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: [3.12]
 
     steps:
       - uses: actions/checkout@v2

diff --git a/.github/workflows/met-cron-ci.yml b/.github/workflows/met-cron-ci.yml
@@ -29,7 +29,7 @@ jobs:
 
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: [3.12]
 
     steps:
       - uses: actions/checkout@v2

diff --git a/.github/workflows/notify-api-ci.yml b/.github/workflows/notify-api-ci.yml
@@ -28,7 +28,7 @@ jobs:
 
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: [3.12]
 
     steps:
       - uses: actions/checkout@v2

diff --git a/CHANGELOG.MD b/CHANGELOG.MD
@@ -1,3 +1,9 @@
+## April 29, 2024
+- **Task** Upgrade python to 3.12 [DESENG-466](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-466)
+  - Upgrading the version of python to 3.12
+  - Upgrading the dependencies to support the version of python.
+  - Modify code as needed if the newer version does not support the former.
+
 ## April 22, 2024
 
 - **Task** Openshift - Update the buildconfig for the MET-cron job runner [🎟️ DESENG-559](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-559)

diff --git a/analytics-api/Dockerfile b/analytics-api/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.8.5-buster
+FROM python:3.12.2-bullseye
 
 USER root
 

diff --git a/analytics-api/Makefile b/analytics-api/Makefile
@@ -43,15 +43,15 @@ clean-test: ## clean test files
 	rm -fr htmlcov/
 
 build-req: clean ## Upgrade requirements
-	test -f venv/bin/activate || python3.8 -m venv  $(CURRENT_ABS_DIR)/venv ;\
+	test -f venv/bin/activate || python3.12 -m venv  $(CURRENT_ABS_DIR)/venv ;\
 	. venv/bin/activate ;\
 	pip install --upgrade pip ;\
 	pip install -Ur requirements/prod.txt ;\
 	pip freeze | sort > requirements.txt ;\
 	cat requirements/repo-libraries.txt >> requirements.txt ;\
 	pip install -Ur requirements/repo-libraries.txt
 
-install: clean ## Install python virtrual environment
+install: clean ## Install python virtual environment
 	test -f venv/bin/activate || python3 -m venv  $(CURRENT_ABS_DIR)/venv ;\
 	. venv/bin/activate ;\
 	pip install --upgrade pip ;\

diff --git a/analytics-api/requirements.txt b/analytics-api/requirements.txt
@@ -1,55 +1,55 @@
-Flask-Caching==2.0.2
+Flask-Caching==2.1.0
 Flask-Cors==3.0.10
 Flask-Mail==0.9.1
 Flask-Migrate==2.7.0
 Flask-Moment==1.0.5
-Flask-SQLAlchemy==2.5.1
+Flask-SQLAlchemy==3.0.5
 Flask-Script==2.0.5
 Flask==2.2.5
 Jinja2==3.1.3
-Mako==1.2.4
+Mako==1.3.3
 MarkupSafe==2.1.2
 SQLAlchemy-Utils==0.40.0
-SQLAlchemy==1.4.17
+SQLAlchemy==1.4.52
 Werkzeug==2.3.8
-alembic==1.10.3
+alembic==1.13.1
 aniso8601==9.0.1
-attrs==22.2.0
+attrs==23.2.0
 bcrypt==4.0.1
 blinker==1.6.1
 cachelib==0.9.0
 certifi==2023.7.22
 charset-normalizer==3.1.0
-click==8.1.3
+click==8.1.7
 ecdsa==0.18.0
 flask-jwt-oidc==0.3.0
-flask-marshmallow==0.14.0
+flask-marshmallow==1.0.0
 flask-restx==1.1.0
 gunicorn==20.1.0
 idna==3.4
 importlib-metadata==6.3.0
 importlib-resources==5.12.0
 itsdangerous==2.0.1
 jaeger-client==4.8.0
-jsonschema==4.17.3
-marshmallow-sqlalchemy==0.25.0
-marshmallow==3.19.0
+jsonschema==4.21.1
+marshmallow-sqlalchemy==1.0.0
+marshmallow==3.21.1
 opentracing==2.4.0
 packaging==23.0
 pkgutil_resolve_name==1.3.10
-psycopg2-binary==2.9.6
+psycopg2-binary==2.9.9
 pyasn1==0.4.8
 pyrsistent==0.19.3
-python-dotenv==1.0.0
+python-dotenv==1.0.1
 python-jose==3.3.0
-pytz==2023.3
+pytz==2024.1
 requests==2.31.0
 rsa==4.9
 secure==0.3.0
 six==1.16.0
 threadloop==1.0.2
 thrift==0.16.0
 tornado==6.3.2
-typing_extensions==4.5.0
+typing_extensions==4.6.0
 urllib3==1.26.18
 zipp==3.15.0
diff --git a/analytics-api/requirements/dev.txt b/analytics-api/requirements/dev.txt
@@ -2,15 +2,15 @@
 -r prod.txt
 
 coverage
-pylint==v2.13.8
+pylint==3.1.0
 pylint-flask
 pytest
 pytest-cov
 pytest-env
 pytest-dotenv
 pytest-mock
 requests
-flake8==4.0.1
+flake8==7.0.0
 flake8-blind-except
 flake8-debugger
 flake8-docstrings
@@ -21,5 +21,4 @@ autopep8
 pydocstyle
 FreezeGun
 lovely-pytest-docker
-Faker
-
+Faker
diff --git a/analytics-api/requirements/prod.txt b/analytics-api/requirements/prod.txt
@@ -7,11 +7,11 @@ Flask-Moment
 Flask-SQLAlchemy
 SQLAlchemy-Continuum
 flask-restx
-flask-marshmallow==0.11.0
+flask-marshmallow==1.0.0
 flask-jwt-oidc
 python-dotenv
 psycopg2-binary
-marshmallow-sqlalchemy==0.25.0
+marshmallow-sqlalchemy==1.0.0
 marshmallow_enum
 jsonschema
 requests
@@ -26,7 +26,7 @@ sqlalchemy-utils
 Flask-Caching
 asyncio-nats-client
 asyncio-nats-streaming
-sqlalchemy<1.4
+sqlalchemy==1.4.52
 secure
 python-dotenv
 aws-requests-auth

diff --git a/analytics-api/setup.cfg b/analytics-api/setup.cfg
@@ -42,7 +42,7 @@ per-file-ignores =
     */__init__.py:F401
 
 [pycodestyle]
-max_line_length = 120
+max-line-length = 120
 ignore = E501
 docstring-min-length=10
 notes=FIXME,XXX # TODO is ignored
@@ -64,7 +64,7 @@ good-names=
 
 [pylint]
 ignore=migrations,test
-max_line_length=120
+max-line-length=120
 notes=FIXME,XXX,TODO
 ignored-modules=flask_sqlalchemy,sqlalchemy,SQLAlchemy,alembic,scoped_session
 ignored-classes=scoped_session

diff --git a/analytics-api/tests/conftest.py b/analytics-api/tests/conftest.py
@@ -18,6 +18,7 @@
 from analytics_api import create_app
 from analytics_api.models import db as _db
 from flask_migrate import Migrate, upgrade
+from sqlalchemy.orm import sessionmaker
 
 
 @pytest.fixture(scope='session')
@@ -81,8 +82,7 @@ def session(app, db):  # pylint: disable=redefined-outer-name, invalid-name
         conn = db.engine.connect()
         txn = conn.begin()
 
-        options = dict(bind=conn, binds={})
-        sess = db.create_scoped_session(options=options)
+        sess = db.scoped_session(sessionmaker(bind=conn, binds={}))
 
         # establish  a SAVEPOINT just before beginning the test
         # (http://docs.sqlalchemy.org/en/latest/orm/session_transaction.html#using-savepoint)

diff --git a/met-api/Dockerfile b/met-api/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.8.5-buster
+FROM python:3.12.2-bullseye
 
 USER root
 

diff --git a/met-api/Makefile b/met-api/Makefile
@@ -43,15 +43,15 @@ clean-test: ## clean test files
 	rm -fr htmlcov/
 
 build-req: clean ## Upgrade requirements
-	test -f venv/bin/activate || python3.8 -m venv  $(CURRENT_ABS_DIR)/venv ;\
+	test -f venv/bin/activate || python3.12 -m venv  $(CURRENT_ABS_DIR)/venv ;\
 	. venv/bin/activate ;\
 	pip install --upgrade pip ;\
 	pip install -Ur requirements/prod.txt ;\
 	pip freeze | sort > requirements.txt ;\
 	cat requirements/repo-libraries.txt >> requirements.txt ;\
 	pip install -Ur requirements/repo-libraries.txt
 
-install: clean ## Install python virtrual environment
+install: clean ## Install python virtual environment
 	test -f venv/bin/activate || python3 -m venv  $(CURRENT_ABS_DIR)/venv ;\
 	. venv/bin/activate ;\
 	pip install --upgrade pip ;\

diff --git a/met-api/requirements.txt b/met-api/requirements.txt
@@ -1,42 +1,42 @@
-alembic==1.6.5
+alembic==1.13.1
 aniso8601==9.0.1
-apispec==4.5.0
-attrs==21.2.0
-click==8.0.1
+apispec==6.6.1
+attrs==23.2.0
+click==8.1.7
 flasgger==0.9.5
 Flask==2.2.5
-flask-marshmallow==0.14.0
+flask-marshmallow==1.0.0
 Flask-Migrate==2.7.0
 Flask-Script==2.0.5
 flask-restplus==0.13.0
-Flask-SQLAlchemy==2.5.1
-greenlet
+Flask-SQLAlchemy==3.0.5
+greenlet==3.0.3
 itsdangerous==2.0.1
 Jinja2==3.1.3
-jsonschema==3.2.0
-Mako==1.2.2
-MarkupSafe==2.1.1
-marshmallow==3.12.1
-marshmallow-sqlalchemy==0.26.1
+jsonschema==4.21.1
+Mako==1.3.3
+MarkupSafe==2.1.5
+marshmallow==3.21.1
+marshmallow-sqlalchemy==1.0.0
 mistune==2.0.3
-psycopg2-binary==2.9.1
+psycopg2-binary==2.9.9
 PyJWT==2.4.0
 pyrsistent==0.17.3
 python-dateutil==2.8.1
-python-dotenv==0.17.1
+python-dotenv==1.0.1
 python-editor==1.0.4
-pytz==2021.1
+pytz==2024.1
 PyYAML==6.0.1
 six==1.16.0
-SQLAlchemy==1.4.17
+SQLAlchemy==1.4.52
 Werkzeug==2.3.8
 flask_mail==0.9.1
 flask_restx
 secure==0.3.0
 flask-cors==3.0.10
 flask_jwt_oidc==0.3.0
 pyhumps==1.6.1
-pylint==2.13.9
+pylint==3.1.0
 aws-requests-auth
-requests
+requests==2.31.0
 awesome-slugify==1.6.5
diff --git a/met-api/requirements/dev.txt b/met-api/requirements/dev.txt
@@ -2,15 +2,15 @@
 -r prod.txt
 
 coverage
-pylint==v2.13.8
+pylint==3.1.0
 pylint-flask
 pytest
 pytest-cov
 pytest-env
 pytest-dotenv
 pytest-mock
 requests
-flake8==4.0.1
+flake8==7.0.0
 flake8-blind-except
 flake8-debugger
 flake8-docstrings

diff --git a/met-api/requirements/prod.txt b/met-api/requirements/prod.txt
@@ -7,11 +7,11 @@ Flask-Moment
 Flask-SQLAlchemy
 SQLAlchemy-Continuum
 flask-restx
-flask-marshmallow==0.11.0
+flask-marshmallow==1.0.0
 flask-jwt-oidc
 python-dotenv
 psycopg2-binary
-marshmallow-sqlalchemy==0.25.0
+marshmallow-sqlalchemy==1.0.0
 marshmallow_enum
 jsonschema
 requests
@@ -26,7 +26,7 @@ sqlalchemy-utils
 Flask-Caching
 asyncio-nats-client
 asyncio-nats-streaming
-sqlalchemy<1.4
+sqlalchemy==1.4.52
 secure
 python-dotenv
 aws-requests-auth

diff --git a/met-api/setup.cfg b/met-api/setup.cfg
@@ -43,7 +43,7 @@ per-file-ignores =
     */config.py:N802
 
 [pycodestyle]
-max_line_length = 120
+max-line-length = 120
 ignore = E501
 docstring-min-length=10
 notes=FIXME,XXX # TODO is ignored
@@ -65,7 +65,7 @@ good-names=
 
 [pylint]
 ignore=migrations,test
-max_line_length=120
+max-line-length=120
 notes=FIXME,XXX,TODO
 ignored-modules=flask_sqlalchemy,sqlalchemy,SQLAlchemy,alembic,scoped_session
 ignored-classes=scoped_session

diff --git a/met-api/src/met_api/models/comment.py b/met-api/src/met_api/models/comment.py
@@ -208,11 +208,11 @@ def add_all_comments(cls, comments: list, session=None) -> list[Comment]:
     def update(cls, submission_id, comment: CommentSchema, session=None) -> Comment:
         """Update comment text."""
         query = Comment.query.filter_by(id=comment.get('id'), submission_id=submission_id)
-        update_fields = dict(
-            text=comment.get('text', None),
-            updated_by=comment.get('participant_id', None),
-            updated_date=datetime.utcnow(),
-        )
+        update_fields = {
+            'text': comment.get('text', None),
+            'updated_by': comment.get('participant_id', None),
+            'updated_date': datetime.utcnow(),
+        }
 
         query.update(update_fields)
         if session is None: