Merge pull request #384 from ucam-department-of-psychiatry/python-update

Drop support for Python 3.8, support 3.11 and 3.12
ucam-department-of-psychiatry · Jan 9, 2025 · effe00a · effe00a
2 parents 4b8f93a + 25c84df
commit effe00a
Show file tree

Hide file tree

Showing 22 changed files with 69 additions and 66 deletions.
diff --git a/.github/scripts/build_packages.sh b/.github/scripts/build_packages.sh
@@ -2,21 +2,15 @@
 
 # Run from .github/workflows/release.yml and push-to-repository.yml
 
-set -eux -o pipefail
+set -euo pipefail
+
 sudo apt-get -y install alien fakeroot lintian gdebi
 # 429 = Too many requests. Unfortunately wget doesn't read the
 # Retry-after header so just wait 5 minutes
 wget --retry-on-http-error=429 --waitretry=300 --tries=20 https://downloads.sourceforge.net/project/rpmrebuild/rpmrebuild/2.15/rpmrebuild-2.15-1.noarch.rpm
 fakeroot alien --to-deb rpmrebuild-2.15-1.noarch.rpm
 sudo dpkg -i rpmrebuild_2.15-2_all.deb
-python -m venv ${HOME}/venv
-source ${HOME}/venv/bin/activate
-python -VV
-python -m site
-python -m pip install -U pip
-echo dumping pre-installed packages
-python -m pip freeze
-echo installing pip packages
-python -m pip install -e server/.
+
 echo building packages
-server/tools/MAKE_LINUX_PACKAGES.py
+PYTHON=${HOME}/venv/bin/python
+${PYTHON} server/tools/MAKE_LINUX_PACKAGES.py
diff --git a/.github/scripts/python_setup.sh b/.github/scripts/python_setup.sh
@@ -1,13 +1,13 @@
 #!/usr/bin/env bash
 
-set -eux -o pipefail
+set -euo pipefail
 
 python3 -m venv ${HOME}/venv
-source ${HOME}/venv/bin/activate
-python -VV
-python -m site
-python -m pip install -U pip
+PYTHON=${HOME}/venv/bin/python
+${PYTHON} -VV
+${PYTHON} -m site
+${PYTHON} -m pip install -U pip setuptools
 echo dumping pre-installed packages
-python -m pip freeze
+${PYTHON} -m pip freeze
 echo installing pip packages
-python -m pip install -e server/.
+${PYTHON} -m pip install -e server/.
diff --git a/.github/workflows/build-and-install-packages.yml b/.github/workflows/build-and-install-packages.yml
@@ -5,20 +5,25 @@ name: Build and install Linux packages
 on:
     push:
         paths:
+            - 'server/setup.py'
             - 'server/camcops_server/**'
             - .github/workflows/build-and-install-packages.yml
             - .github/scripts/build_packages.sh
             - .github/scripts/change_apt_mirror.sh
+            - .github/scripts/python_setup.sh
+            - server/tools/install_virtualenv.py
             - server/tools/MAKE_LINUX_PACKAGES.py
             - server/requirements-deb.txt
             - server/requirements-rpm.txt
 jobs:
     build-and-install-package:
         strategy:
             matrix:
-                python-version: [3.8, 3.9, "3.10"]
+                # Just one here as the OS will determine the version used
+                # when the package is installed
+                python-version: ["3.10"]
                 # LTS versions
-                os: [ubuntu-20.04, ubuntu-22.04]
+                os: [ubuntu-22.04, ubuntu-24.04]
         runs-on: ${{ matrix.os }}
         steps:
             - uses: actions/checkout@v4
@@ -27,11 +32,12 @@ jobs:
                   python-version: ${{ matrix.python-version }}
             - name: Change apt mirror
               run: |
-                  set -eux -o pipefail
+                  set -euo pipefail
                   ${GITHUB_WORKSPACE}/.github/scripts/change_apt_mirror.sh
             - name: Build and install debian package
               run: |
-                  set -eux -o pipefail
+                  set -euo pipefail
+                  ${GITHUB_WORKSPACE}/.github/scripts/python_setup.sh
                   ${GITHUB_WORKSPACE}/.github/scripts/build_packages.sh
                   echo installing debian package
                   # Possible scope for confusion here: at this point we should

diff --git a/.github/workflows/build-qt.yml b/.github/workflows/build-qt.yml
@@ -34,7 +34,7 @@ jobs:
                   ${GITHUB_WORKSPACE}/.github/scripts/free_up_disk_space.sh
             - uses: actions/setup-python@v5
               with:
-                  python-version: 3.8
+                  python-version: 3.9
             - name: Common Linux prerequisites
               if: runner.os == 'Linux'
               run: |

diff --git a/.github/workflows/clang-format.yml b/.github/workflows/clang-format.yml
@@ -18,7 +18,7 @@ jobs:
             - uses: actions/checkout@v4
             - uses: actions/setup-python@v5
               with:
-                  python-version: 3.8
+                  python-version: 3.9
 
             - name: Install virtualenv
               run: |

diff --git a/.github/workflows/cpp-tests.yml b/.github/workflows/cpp-tests.yml
@@ -112,7 +112,7 @@ jobs:
 
             - uses: actions/setup-python@v5
               with:
-                  python-version: 3.8
+                  python-version: 3.9
 
             - name: C++ tests
               run: |

diff --git a/.github/workflows/database-upgrade-downgrade.yml b/.github/workflows/database-upgrade-downgrade.yml
@@ -17,7 +17,7 @@ jobs:
             - uses: actions/checkout@v4
             - uses: actions/setup-python@v5
               with:
-                  python-version: 3.8
+                  python-version: 3.9
             - name: Set up MySQL
               run: |
                   ${GITHUB_WORKSPACE}/.github/scripts/mysql_local_setup.sh

diff --git a/.github/workflows/missing-migration-check.yml b/.github/workflows/missing-migration-check.yml
@@ -17,7 +17,7 @@ jobs:
             - uses: actions/checkout@v4
             - uses: actions/setup-python@v5
               with:
-                  python-version: 3.8
+                  python-version: 3.9
             - name: Missing migration check
               run: |
                   set -eux -o pipefail

diff --git a/.github/workflows/precommit.yml b/.github/workflows/precommit.yml
@@ -15,7 +15,7 @@ jobs:
             - uses: actions/checkout@v4
             - uses: actions/setup-python@v5
               with:
-                  python-version: 3.8
+                  python-version: 3.9
             - name: Pre-commit checks
               run: |
                   set -euxo pipefail

diff --git a/.github/workflows/python-package-checks.yml b/.github/workflows/python-package-checks.yml
@@ -12,7 +12,7 @@ jobs:
     python-package-checks:
         strategy:
             matrix:
-                python-version: [3.8, 3.9, "3.10"]
+                python-version: [3.9, "3.10", 3.11, 3.12]
                 # LTS versions
                 os: [ubuntu-20.04, ubuntu-22.04]
         runs-on: ${{ matrix.os }}

diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml
@@ -14,7 +14,7 @@ jobs:
     python-tests:
         strategy:
             matrix:
-                python-version: [3.8, 3.9, "3.10"]
+                python-version: [3.9, "3.10", 3.11, 3.12]
                 # LTS versions
                 os: [ubuntu-20.04, ubuntu-22.04]
         runs-on: ${{ matrix.os }}

diff --git a/.readthedocs.yaml b/.readthedocs.yaml
@@ -11,7 +11,7 @@ version: 2
 build:
   os: ubuntu-20.04
   tools:
-    python: "3.8"
+    python: "3.9"
 
 python:
   install:

diff --git a/docs/source/administrator/_demo_apache_config.conf b/docs/source/administrator/_demo_apache_config.conf
@@ -26,15 +26,15 @@
 
         # CHANGE THIS: aim the alias at your own institutional logo.
 
-    Alias /static/logo_local.png /usr/share/camcops/venv/lib/python3.8/site-packages/camcops_server/static/logo_local.png
+    Alias /static/logo_local.png /usr/share/camcops/venv/lib/python3.9/site-packages/camcops_server/static/logo_local.png
 
         # We move from more specific to less specific aliases; the first match
         # takes precedence. (Apache will warn about conflicting aliases if
         # specified in a wrong, less-to-more-specific, order.)
 
-    Alias /static/ /usr/share/camcops/venv/lib/python3.8/site-packages/camcops_server/static/
+    Alias /static/ /usr/share/camcops/venv/lib/python3.9/site-packages/camcops_server/static/
 
-    <Directory /usr/share/camcops/venv/lib/python3.8/site-packages/camcops_server/static>
+    <Directory /usr/share/camcops/venv/lib/python3.9/site-packages/camcops_server/static>
         Require all granted
 
         # ... for old Apache versions (e.g. 2.2), use instead:

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
@@ -3953,3 +3953,4 @@ Current C++/SQLite client, Python/SQLAlchemy server
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 - Update the Docker image to use Debian 11. Debian 10 has now reached end-of-life.
+- **Minimum Python version now Python 3.9.** Python 3.11 and 3.12 supported.
diff --git a/docs/source/developer/versions.rst b/docs/source/developer/versions.rst
@@ -38,11 +38,13 @@ Versions of software etc. used by CamCOPS
 | OpenSSL        | 3.0.x   | 2026-09-07 (LTS);                                 |
 |                |         | https://www.openssl.org/policies/releasestrat.html|
 +----------------+---------+---------------------------------------------------+
-| Python         | 3.8     | 2023-08-24                                        |
+| Python         | 3.9     | 2025-10-05                                        |
 +----------------+---------+---------------------------------------------------+
-|                | 3.9     | 2025-10-05                                        |
+|                | 3.10    | 2026-10-31                                        |
 +----------------+---------+---------------------------------------------------+
-|                | 3.10    | 2026-10-04;                                       |
+|                | 3.11    | 2027-10-31                                        |
++----------------+---------+---------------------------------------------------+
+|                | 3.12    | 2028-10-31;                                       |
 |                |         | https://devguide.python.org/versions/             |
 +----------------+---------+---------------------------------------------------+
 | Qt             | 6.5.x   | 2026-03-31 (LTS) but 6.5.x branch now             |

diff --git a/server/camcops_server/cc_modules/cc_config.py b/server/camcops_server/cc_modules/cc_config.py
@@ -187,7 +187,7 @@
 DEFAULT_LINUX_CAMCOPS_STATIC_DIR = os.path.join(
     DEFAULT_LINUX_CAMCOPS_VENV_DIR,
     "lib",
-    "python3.8",
+    "python3.9",
     "site-packages",
     "camcops_server",
     "static",

diff --git a/server/camcops_server/cc_modules/cc_export.py b/server/camcops_server/cc_modules/cc_export.py
@@ -188,7 +188,7 @@
 from cardinal_pythonlib.sizeformatter import bytes2human
 from cardinal_pythonlib.sqlalchemy.session import get_safe_url_from_engine
 import lockfile
-from pendulum import DateTime as Pendulum, Duration, Period
+from pendulum import DateTime as Pendulum, Duration
 from pyramid.httpexceptions import HTTPBadRequest
 from pyramid.renderers import render_to_response
 from pyramid.response import Response
@@ -230,6 +230,8 @@
 )
 
 if TYPE_CHECKING:
+    from pendulum import Interval
+
     from camcops_server.cc_modules.cc_request import CamcopsRequest
     from camcops_server.cc_modules.cc_taskcollection import TaskCollection
 
@@ -1497,10 +1499,10 @@ def time_left(self) -> Optional[Duration]:
         death = self.when_last_modified + Duration(
             minutes=self.permitted_lifespan_min
         )
-        remaining = death - now  # type: Period
-        # Note that Period is a subclass of Duration, but its __str__()
+        remaining = death - now  # type: Interval
+        # Note that Interval is a subclass of Duration, but its __str__()
         # method is different. Duration maps __str__() to in_words(), but
-        # Period maps __str__() to __repr__().
+        # Interval maps __str__() to __repr__().
         return remaining
 
     @property

diff --git a/server/installer/installer.py b/server/installer/installer.py
@@ -99,7 +99,7 @@ class DockerPath:
 
     VENV_DIR = os.path.join(ROOT_DIR, "venv")
     CAMCOPS_INSTALL_DIR = os.path.join(
-        VENV_DIR, "lib", "python3.8", "site-packages"
+        VENV_DIR, "lib", "python3.9", "site-packages"
     )
 
 

diff --git a/server/installer/installer.sh b/server/installer/installer.sh
@@ -146,10 +146,10 @@ fi
 source "${CAMCOPS_INSTALLER_VENV}/bin/activate"
 
 # Check virtual environment
-PYTHON_VERSION_OK=$(python -c 'import sys; print(sys.version_info.major >=3 and sys.version_info.minor >= 7)')
+PYTHON_VERSION_OK=$(python -c 'import sys; print(sys.version_info.major >=3 and sys.version_info.minor >= 9)')
 if [ "${PYTHON_VERSION_OK}" == "False" ]; then
     python --version
-    echo "You need at least Python 3.7 to run the installer."
+    echo "You need at least Python 3.9 to run the installer."
     exit 1
 fi
 

diff --git a/server/setup.py b/server/setup.py
@@ -68,7 +68,7 @@
     "alembic==1.4.2",  # database migrations
     "asteval==0.9.25",  # safe-ish alternative to eval
     "cardinal_pythonlib==1.1.25",  # RNC libraries
-    "celery==5.2.2",  # background tasks
+    "celery==5.3.6",  # background tasks
     "colander==1.7.0",  # serialization/deserialization from web forms
     "CherryPy==18.6.0",  # web server
     (  # deform: web forms
@@ -87,16 +87,12 @@
     "flower==2.0.1",  # monitor for Celery
     "gunicorn==23.0.0",  # web server (Unix only)
     "hl7==0.3.5",  # For HL7 export
-    # Celery dependency for Python <= 3.7; workaround import error
-    # https://github.com/celery/celery/issues/7783; scheduled to be fixed in
-    # Celery 5.3
-    "importlib-metadata<5.0",
     "lockfile==0.12.2",  # File locking for background tasks
-    "lxml==4.9.1",  # Will speed up openpyxl export [NO LONGER CRITICAL]
-    "matplotlib==3.2.2",  # Used for trackers and some tasks. SLOW INSTALLATION.  # noqa
-    "numpy==1.23.5",  # Used by some tasks. SLOW INSTALLATION.
+    "lxml==4.9.4",  # Will speed up openpyxl export [NO LONGER CRITICAL]
+    "matplotlib==3.9.4",  # Used for trackers and some tasks. SLOW INSTALLATION.  # noqa
+    "numpy==1.26.4",  # Used by some tasks. SLOW INSTALLATION.
     "paginate==0.5.6",  # pagination for web server
-    "pendulum==2.1.2",  # date/time classes
+    "pendulum==3.0.0",  # date/time classes
     "pexpect==4.8.0",  # for open_sqlcipher.py
     "pdfkit==1.0.0",  # wkhtmltopdf interface, for PDF generation from HTML
     "phonenumbers==8.12.30",  # phone number parsing, storing and validating
@@ -113,10 +109,10 @@
     "requests==2.32.2",  # in fetch_snomed_codes.py and cc_sms.py, but also required by something else?  # noqa
     "rich-argparse==0.5.0",  # colourful help
     "sadisplay==0.4.9",  # SQL Alchemy schema display script
-    "scipy==1.10.1",  # used by some tasks. slow installation.
+    "scipy==1.13.1",  # used by some tasks. slow installation.
     "semantic_version==2.8.5",  # semantic versioning; better than semver
     "sqlalchemy==1.4.49",  # database access
-    "statsmodels==0.13.5",  # e.g. logistic regression
+    "statsmodels==0.14.4",  # e.g. logistic regression
     "twilio==7.9.3",  # SMS backend for Multi-factor authentication
     "urllib3==1.26.19",  # dependency, pinned to avoid vulnerabilities
     "Wand==0.6.1",  # ImageMagick binding
@@ -148,23 +144,23 @@
     "beautifulsoup4==4.9.1",
     "colorlog==4.1.0",
     "isodate==0.6.0",
-    "openpyxl==3.0.4",  # also for pyexcel-xlsx
-    "pandas==1.3.4",
+    "openpyxl==3.0.7",  # also for pyexcel-xlsx
+    "pandas==1.4.4",
     "prettytable==0.7.2",
     "psutil==5.7.0",
     "pyparsing==2.4.7",
     "pypdf==3.17.0",  # Used by cardinal_pythonlib.pdf
-    "python-dateutil==2.8.1",  # date/time extensions.
+    "python-dateutil==2.9.0.post0",  # date/time extensions.
     "sqlparse==0.5.0",
     # extra
     "py-bcrypt==0.4",  # used by cardinal_pythonlib.crypto
     # -------------------------------------------------------------------------
     # Dependencies of other things above
     # -------------------------------------------------------------------------
     "alabaster==0.7.12",  # for sphinx
-    "amqp==5.0.6",  # for celery
+    "amqp==5.3.1",  # for celery
     "Chameleon==3.8.1",  # for Deform
-    "tornado==6.4.1",  # for celery
+    "tornado==6.4.2",  # for celery
     "webob==1.8.8",  # for pyramid
 ]
 
@@ -197,9 +193,10 @@
         "Natural Language :: English",
         "Operating System :: OS Independent",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
         "Topic :: Scientific/Engineering :: Medical Science Apps.",
     ],
     keywords="cardinal",

diff --git a/server/tools/MAKE_LINUX_PACKAGES.py b/server/tools/MAKE_LINUX_PACKAGES.py
@@ -248,9 +248,10 @@ def call(cmdargs: List[str], **kwargs) -> None:
     # Use as: $(system_python_executable) ...
 
     python_options=(
+        python3.12 python312
+        python3.11 python311
         python3.10 python310
         python3.9 python39
-        python3.8 python38
         python3
         python
     )
@@ -551,7 +552,7 @@ def get_debian_control() -> str:
 Architecture: all
 Maintainer: Rudolf Cardinal <[email protected]>
 Depends: {DEPENDENCIES}
-X-Python3-Version: >= 3.8, <= 3.10
+X-Python3-Version: >= 3.9, <= 3.12
 Recommends: mysql-workbench
 Description: Cambridge Cognitive and Psychiatric Test Kit (CamCOPS), server
  packages.

diff --git a/server/tools/install_virtualenv.py b/server/tools/install_virtualenv.py
@@ -49,7 +49,7 @@
 except ImportError:
     distro = None
 
-assert sys.version_info >= (3, 8), "Need Python 3.8 or higher"
+assert sys.version_info >= (3, 9), "Need Python 3.9 or higher"
 LINUX = platform.system() == "Linux"
 if distro:
     LINUX_DIST = distro.linux_distribution()[0].lower()
@@ -222,7 +222,7 @@ def main() -> None:
     print("OK")
 
     title("Upgrading pip within virtualenv")
-    check_call([venv_pip, "install", "--upgrade", "pip"])
+    check_call([venv_pip, "install", "--upgrade", "pip", "setuptools"])
 
     title("Checking version of tools within new virtualenv")
     print(venv_python)
Original file line number	Diff line number	Diff line change
Expand Up		@@ -3953,3 +3953,4 @@ Current C++/SQLite client, Python/SQLAlchemy server
		^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

		- Update the Docker image to use Debian 11. Debian 10 has now reached end-of-life.
		- Minimum Python version now Python 3.9. Python 3.11 and 3.12 supported.