From 3b579530d130e472270d9f226be913d490e8139a Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Tue, 7 Jan 2025 16:46:37 +0000 Subject: [PATCH 01/24] Drop support for Python 3.8, add support for Python 3.11 --- .github/scripts/python_setup.sh | 14 +++++++------- .github/workflows/build-and-install-packages.yml | 2 +- .github/workflows/build-qt.yml | 2 +- .github/workflows/clang-format.yml | 2 +- .github/workflows/cpp-tests.yml | 2 +- .github/workflows/database-upgrade-downgrade.yml | 2 +- .github/workflows/missing-migration-check.yml | 2 +- .github/workflows/precommit.yml | 2 +- .github/workflows/python-package-checks.yml | 2 +- .github/workflows/python-tests.yml | 2 +- .readthedocs.yaml | 2 +- docs/source/developer/versions.rst | 6 +++--- server/camcops_server/cc_modules/cc_config.py | 2 +- server/installer/installer.py | 2 +- server/installer/installer.sh | 4 ++-- server/setup.py | 2 +- server/tools/MAKE_LINUX_PACKAGES.py | 4 ++-- server/tools/install_virtualenv.py | 2 +- 18 files changed, 28 insertions(+), 28 deletions(-) diff --git a/.github/scripts/python_setup.sh b/.github/scripts/python_setup.sh index fe1689902..4c3c5d27e 100755 --- a/.github/scripts/python_setup.sh +++ b/.github/scripts/python_setup.sh @@ -1,13 +1,13 @@ #!/usr/bin/env bash -set -eux -o pipefail +set -euo pipefail python3 -m venv ${HOME}/venv -source ${HOME}/venv/bin/activate -python -VV -python -m site -python -m pip install -U pip +PYTHON=${HOME}/venv/bin/python +${PYTHON} -VV +${PYTHON} -m site +${PYTHON} -m pip install -U pip echo dumping pre-installed packages -python -m pip freeze +${PYTHON} -m pip freeze echo installing pip packages -python -m pip install -e server/. +${PYTHON} -m pip install -e server/. diff --git a/.github/workflows/build-and-install-packages.yml b/.github/workflows/build-and-install-packages.yml index 5fc6484be..fcf3d9da3 100644 --- a/.github/workflows/build-and-install-packages.yml +++ b/.github/workflows/build-and-install-packages.yml @@ -16,7 +16,7 @@ jobs: build-and-install-package: strategy: matrix: - python-version: [3.8, 3.9, "3.10"] + python-version: [3.9, "3.10", 3.11] # LTS versions os: [ubuntu-20.04, ubuntu-22.04] runs-on: ${{ matrix.os }} diff --git a/.github/workflows/build-qt.yml b/.github/workflows/build-qt.yml index 3fa05cf67..d76ca585c 100644 --- a/.github/workflows/build-qt.yml +++ b/.github/workflows/build-qt.yml @@ -34,7 +34,7 @@ jobs: ${GITHUB_WORKSPACE}/.github/scripts/free_up_disk_space.sh - uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: 3.9 - name: Common Linux prerequisites if: runner.os == 'Linux' run: | diff --git a/.github/workflows/clang-format.yml b/.github/workflows/clang-format.yml index 2ac4564c1..bca538244 100644 --- a/.github/workflows/clang-format.yml +++ b/.github/workflows/clang-format.yml @@ -18,7 +18,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: 3.9 - name: Install virtualenv run: | diff --git a/.github/workflows/cpp-tests.yml b/.github/workflows/cpp-tests.yml index 2834afe10..156cb65dc 100644 --- a/.github/workflows/cpp-tests.yml +++ b/.github/workflows/cpp-tests.yml @@ -112,7 +112,7 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: 3.9 - name: C++ tests run: | diff --git a/.github/workflows/database-upgrade-downgrade.yml b/.github/workflows/database-upgrade-downgrade.yml index fa143294f..e89ee35ab 100644 --- a/.github/workflows/database-upgrade-downgrade.yml +++ b/.github/workflows/database-upgrade-downgrade.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: 3.9 - name: Set up MySQL run: | ${GITHUB_WORKSPACE}/.github/scripts/mysql_local_setup.sh diff --git a/.github/workflows/missing-migration-check.yml b/.github/workflows/missing-migration-check.yml index 0884cac52..b44189fe1 100644 --- a/.github/workflows/missing-migration-check.yml +++ b/.github/workflows/missing-migration-check.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: 3.9 - name: Missing migration check run: | set -eux -o pipefail diff --git a/.github/workflows/precommit.yml b/.github/workflows/precommit.yml index 148504297..1de953186 100644 --- a/.github/workflows/precommit.yml +++ b/.github/workflows/precommit.yml @@ -15,7 +15,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: 3.9 - name: Pre-commit checks run: | set -euxo pipefail diff --git a/.github/workflows/python-package-checks.yml b/.github/workflows/python-package-checks.yml index 79679e001..83445bbab 100644 --- a/.github/workflows/python-package-checks.yml +++ b/.github/workflows/python-package-checks.yml @@ -12,7 +12,7 @@ jobs: python-package-checks: strategy: matrix: - python-version: [3.8, 3.9, "3.10"] + python-version: [3.9, "3.10", "3.11"] # LTS versions os: [ubuntu-20.04, ubuntu-22.04] runs-on: ${{ matrix.os }} diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index fb9b8f6bf..eaac8d1c2 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -14,7 +14,7 @@ jobs: python-tests: strategy: matrix: - python-version: [3.8, 3.9, "3.10"] + python-version: [3.9, "3.10", 3.11] # LTS versions os: [ubuntu-20.04, ubuntu-22.04] runs-on: ${{ matrix.os }} diff --git a/.readthedocs.yaml b/.readthedocs.yaml index fdcd0ce43..d80b58fba 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -11,7 +11,7 @@ version: 2 build: os: ubuntu-20.04 tools: - python: "3.8" + python: "3.9" python: install: diff --git a/docs/source/developer/versions.rst b/docs/source/developer/versions.rst index 099524e14..d34ccbd0c 100644 --- a/docs/source/developer/versions.rst +++ b/docs/source/developer/versions.rst @@ -38,11 +38,11 @@ Versions of software etc. used by CamCOPS | OpenSSL | 3.0.x | 2026-09-07 (LTS); | | | | https://www.openssl.org/policies/releasestrat.html| +----------------+---------+---------------------------------------------------+ -| Python | 3.8 | 2023-08-24 | +| Python | 3.9 | 2025-10-05 | +----------------+---------+---------------------------------------------------+ -| | 3.9 | 2025-10-05 | +| | 3.10 | 2026-10-31 | +----------------+---------+---------------------------------------------------+ -| | 3.10 | 2026-10-04; | +| | 3.11 | 2027-10-31; | | | | https://devguide.python.org/versions/ | +----------------+---------+---------------------------------------------------+ | Qt | 6.5.x | 2026-03-31 (LTS) but 6.5.x branch now | diff --git a/server/camcops_server/cc_modules/cc_config.py b/server/camcops_server/cc_modules/cc_config.py index 70fec0237..9a6fb53dc 100644 --- a/server/camcops_server/cc_modules/cc_config.py +++ b/server/camcops_server/cc_modules/cc_config.py @@ -187,7 +187,7 @@ DEFAULT_LINUX_CAMCOPS_STATIC_DIR = os.path.join( DEFAULT_LINUX_CAMCOPS_VENV_DIR, "lib", - "python3.8", + "python3.9", "site-packages", "camcops_server", "static", diff --git a/server/installer/installer.py b/server/installer/installer.py index ebcbc55e1..5753fb48c 100755 --- a/server/installer/installer.py +++ b/server/installer/installer.py @@ -99,7 +99,7 @@ class DockerPath: VENV_DIR = os.path.join(ROOT_DIR, "venv") CAMCOPS_INSTALL_DIR = os.path.join( - VENV_DIR, "lib", "python3.8", "site-packages" + VENV_DIR, "lib", "python3.9", "site-packages" ) diff --git a/server/installer/installer.sh b/server/installer/installer.sh index c5e5201b5..38466a432 100755 --- a/server/installer/installer.sh +++ b/server/installer/installer.sh @@ -146,10 +146,10 @@ fi source "${CAMCOPS_INSTALLER_VENV}/bin/activate" # Check virtual environment -PYTHON_VERSION_OK=$(python -c 'import sys; print(sys.version_info.major >=3 and sys.version_info.minor >= 7)') +PYTHON_VERSION_OK=$(python -c 'import sys; print(sys.version_info.major >=3 and sys.version_info.minor >= 9)') if [ "${PYTHON_VERSION_OK}" == "False" ]; then python --version - echo "You need at least Python 3.7 to run the installer." + echo "You need at least Python 3.9 to run the installer." exit 1 fi diff --git a/server/setup.py b/server/setup.py index 4a5463759..9a85fb490 100644 --- a/server/setup.py +++ b/server/setup.py @@ -197,9 +197,9 @@ "Natural Language :: English", "Operating System :: OS Independent", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Topic :: Scientific/Engineering :: Medical Science Apps.", ], keywords="cardinal", diff --git a/server/tools/MAKE_LINUX_PACKAGES.py b/server/tools/MAKE_LINUX_PACKAGES.py index c5e409b6e..404312927 100755 --- a/server/tools/MAKE_LINUX_PACKAGES.py +++ b/server/tools/MAKE_LINUX_PACKAGES.py @@ -248,9 +248,9 @@ def call(cmdargs: List[str], **kwargs) -> None: # Use as: $(system_python_executable) ... python_options=( + python3.11 python311 python3.10 python310 python3.9 python39 - python3.8 python38 python3 python ) @@ -551,7 +551,7 @@ def get_debian_control() -> str: Architecture: all Maintainer: Rudolf Cardinal Depends: {DEPENDENCIES} -X-Python3-Version: >= 3.8, <= 3.10 +X-Python3-Version: >= 3.9, <= 3.11 Recommends: mysql-workbench Description: Cambridge Cognitive and Psychiatric Test Kit (CamCOPS), server packages. diff --git a/server/tools/install_virtualenv.py b/server/tools/install_virtualenv.py index ab68c9a97..bc456ed23 100755 --- a/server/tools/install_virtualenv.py +++ b/server/tools/install_virtualenv.py @@ -49,7 +49,7 @@ except ImportError: distro = None -assert sys.version_info >= (3, 8), "Need Python 3.8 or higher" +assert sys.version_info >= (3, 9), "Need Python 3.9 or higher" LINUX = platform.system() == "Linux" if distro: LINUX_DIST = distro.linux_distribution()[0].lower() From 40d3166c569e277adf2bc36b2673b13f141819c4 Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Tue, 7 Jan 2025 16:47:54 +0000 Subject: [PATCH 02/24] Bump Tornado to fix CVE-2024-52804 --- server/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/setup.py b/server/setup.py index 9a85fb490..d1cb2b1d3 100644 --- a/server/setup.py +++ b/server/setup.py @@ -164,7 +164,7 @@ "alabaster==0.7.12", # for sphinx "amqp==5.0.6", # for celery "Chameleon==3.8.1", # for Deform - "tornado==6.4.1", # for celery + "tornado==6.4.2", # for celery "webob==1.8.8", # for pyramid ] From 9b8f323609941c2ea4eb0b8a468748c65c7b598a Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Tue, 7 Jan 2025 17:04:32 +0000 Subject: [PATCH 03/24] Drop support for Ubuntu 20.04 and add 24.04 in build-and-install-packages.yml --- .github/workflows/build-and-install-packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-install-packages.yml b/.github/workflows/build-and-install-packages.yml index fcf3d9da3..c818d126a 100644 --- a/.github/workflows/build-and-install-packages.yml +++ b/.github/workflows/build-and-install-packages.yml @@ -18,7 +18,7 @@ jobs: matrix: python-version: [3.9, "3.10", 3.11] # LTS versions - os: [ubuntu-20.04, ubuntu-22.04] + os: [ubuntu-22.04, ubuntu-24.04] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 From d8b76fb7d6de57f15cd7aea313e7e134b1159ed1 Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Wed, 8 Jan 2025 10:46:25 +0000 Subject: [PATCH 04/24] Update demo Apache config for Python 3.9 --- docs/source/administrator/_demo_apache_config.conf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/administrator/_demo_apache_config.conf b/docs/source/administrator/_demo_apache_config.conf index 1100ec9a5..dffbbeac2 100644 --- a/docs/source/administrator/_demo_apache_config.conf +++ b/docs/source/administrator/_demo_apache_config.conf @@ -26,15 +26,15 @@ # CHANGE THIS: aim the alias at your own institutional logo. - Alias /static/logo_local.png /usr/share/camcops/venv/lib/python3.8/site-packages/camcops_server/static/logo_local.png + Alias /static/logo_local.png /usr/share/camcops/venv/lib/python3.9/site-packages/camcops_server/static/logo_local.png # We move from more specific to less specific aliases; the first match # takes precedence. (Apache will warn about conflicting aliases if # specified in a wrong, less-to-more-specific, order.) - Alias /static/ /usr/share/camcops/venv/lib/python3.8/site-packages/camcops_server/static/ + Alias /static/ /usr/share/camcops/venv/lib/python3.9/site-packages/camcops_server/static/ - + Require all granted # ... for old Apache versions (e.g. 2.2), use instead: From 9aea24eff43bfb93f955d745c17036eaab3c1133 Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Wed, 8 Jan 2025 10:47:35 +0000 Subject: [PATCH 05/24] Bump matplotlib for later Python compatibility --- server/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/setup.py b/server/setup.py index d1cb2b1d3..65b8d8ffa 100644 --- a/server/setup.py +++ b/server/setup.py @@ -93,7 +93,7 @@ "importlib-metadata<5.0", "lockfile==0.12.2", # File locking for background tasks "lxml==4.9.1", # Will speed up openpyxl export [NO LONGER CRITICAL] - "matplotlib==3.2.2", # Used for trackers and some tasks. SLOW INSTALLATION. # noqa + "matplotlib==3.9.4", # Used for trackers and some tasks. SLOW INSTALLATION. # noqa "numpy==1.23.5", # Used by some tasks. SLOW INSTALLATION. "paginate==0.5.6", # pagination for web server "pendulum==2.1.2", # date/time classes From 687818ce71d53dc2473051c57d23d34f6704753d Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Wed, 8 Jan 2025 10:58:26 +0000 Subject: [PATCH 06/24] Trigger package workflow when setup.py changes --- .github/workflows/build-and-install-packages.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-and-install-packages.yml b/.github/workflows/build-and-install-packages.yml index c818d126a..01597fbf9 100644 --- a/.github/workflows/build-and-install-packages.yml +++ b/.github/workflows/build-and-install-packages.yml @@ -5,6 +5,7 @@ name: Build and install Linux packages on: push: paths: + - 'server/setup.py' - 'server/camcops_server/**' - .github/workflows/build-and-install-packages.yml - .github/scripts/build_packages.sh From 1c92e59be9d6fce46623cc36362305f74e574014 Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Wed, 8 Jan 2025 11:13:05 +0000 Subject: [PATCH 07/24] Minor refactoring of build package script --- .github/scripts/build_packages.sh | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/scripts/build_packages.sh b/.github/scripts/build_packages.sh index f12f04680..bfa0e09de 100755 --- a/.github/scripts/build_packages.sh +++ b/.github/scripts/build_packages.sh @@ -2,21 +2,23 @@ # Run from .github/workflows/release.yml and push-to-repository.yml -set -eux -o pipefail +set -euo pipefail + sudo apt-get -y install alien fakeroot lintian gdebi # 429 = Too many requests. Unfortunately wget doesn't read the # Retry-after header so just wait 5 minutes wget --retry-on-http-error=429 --waitretry=300 --tries=20 https://downloads.sourceforge.net/project/rpmrebuild/rpmrebuild/2.15/rpmrebuild-2.15-1.noarch.rpm fakeroot alien --to-deb rpmrebuild-2.15-1.noarch.rpm sudo dpkg -i rpmrebuild_2.15-2_all.deb -python -m venv ${HOME}/venv -source ${HOME}/venv/bin/activate -python -VV -python -m site -python -m pip install -U pip + +python3 -m venv ${HOME}/venv +PYTHON=${HOME}/venv/bin/python +${PYTHON} -VV +${PYTHON} -m site +${PYTHON} -m pip install -U pip echo dumping pre-installed packages -python -m pip freeze +${PYTHON} -m pip freeze echo installing pip packages -python -m pip install -e server/. +${PYTHON} -m pip install -e server/. echo building packages server/tools/MAKE_LINUX_PACKAGES.py From 7a279049b36687b9e795f08b94f8fa37c93ba8ab Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Wed, 8 Jan 2025 11:14:22 +0000 Subject: [PATCH 08/24] Install setuptools as part of build package workflow --- .github/scripts/build_packages.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/build_packages.sh b/.github/scripts/build_packages.sh index bfa0e09de..dd8921413 100755 --- a/.github/scripts/build_packages.sh +++ b/.github/scripts/build_packages.sh @@ -15,7 +15,7 @@ python3 -m venv ${HOME}/venv PYTHON=${HOME}/venv/bin/python ${PYTHON} -VV ${PYTHON} -m site -${PYTHON} -m pip install -U pip +${PYTHON} -m pip install -U pip setuptools echo dumping pre-installed packages ${PYTHON} -m pip freeze echo installing pip packages From 77e7b1c49b288077d9b1278dde0545b9f1cd4b19 Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Wed, 8 Jan 2025 11:19:11 +0000 Subject: [PATCH 09/24] Simplify build package workflow script, use correct python Also make other bits less noisy --- .github/scripts/build_packages.sh | 12 ++---------- .github/scripts/python_setup.sh | 2 +- .github/workflows/build-and-install-packages.yml | 6 ++++-- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/.github/scripts/build_packages.sh b/.github/scripts/build_packages.sh index dd8921413..6918e9601 100755 --- a/.github/scripts/build_packages.sh +++ b/.github/scripts/build_packages.sh @@ -11,14 +11,6 @@ wget --retry-on-http-error=429 --waitretry=300 --tries=20 https://downloads.sour fakeroot alien --to-deb rpmrebuild-2.15-1.noarch.rpm sudo dpkg -i rpmrebuild_2.15-2_all.deb -python3 -m venv ${HOME}/venv -PYTHON=${HOME}/venv/bin/python -${PYTHON} -VV -${PYTHON} -m site -${PYTHON} -m pip install -U pip setuptools -echo dumping pre-installed packages -${PYTHON} -m pip freeze -echo installing pip packages -${PYTHON} -m pip install -e server/. echo building packages -server/tools/MAKE_LINUX_PACKAGES.py +PYTHON=${HOME}/venv/bin/python +${PYTHON} server/tools/MAKE_LINUX_PACKAGES.py diff --git a/.github/scripts/python_setup.sh b/.github/scripts/python_setup.sh index 4c3c5d27e..bb233b7d3 100755 --- a/.github/scripts/python_setup.sh +++ b/.github/scripts/python_setup.sh @@ -6,7 +6,7 @@ python3 -m venv ${HOME}/venv PYTHON=${HOME}/venv/bin/python ${PYTHON} -VV ${PYTHON} -m site -${PYTHON} -m pip install -U pip +${PYTHON} -m pip install -U pip setuptools echo dumping pre-installed packages ${PYTHON} -m pip freeze echo installing pip packages diff --git a/.github/workflows/build-and-install-packages.yml b/.github/workflows/build-and-install-packages.yml index 01597fbf9..15f55491a 100644 --- a/.github/workflows/build-and-install-packages.yml +++ b/.github/workflows/build-and-install-packages.yml @@ -10,6 +10,7 @@ on: - .github/workflows/build-and-install-packages.yml - .github/scripts/build_packages.sh - .github/scripts/change_apt_mirror.sh + - .github/scripts/python_setup.sh - server/tools/MAKE_LINUX_PACKAGES.py - server/requirements-deb.txt - server/requirements-rpm.txt @@ -28,11 +29,12 @@ jobs: python-version: ${{ matrix.python-version }} - name: Change apt mirror run: | - set -eux -o pipefail + set -euo pipefail ${GITHUB_WORKSPACE}/.github/scripts/change_apt_mirror.sh - name: Build and install debian package run: | - set -eux -o pipefail + set -euo pipefail + ${GITHUB_WORKSPACE}/.github/scripts/python_setup.sh ${GITHUB_WORKSPACE}/.github/scripts/build_packages.sh echo installing debian package # Possible scope for confusion here: at this point we should From 8a5194efc6aa0965901431c0c1513a0f34e49a07 Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Wed, 8 Jan 2025 11:28:07 +0000 Subject: [PATCH 10/24] Install latest pip and setuptools in package install script --- server/tools/MAKE_LINUX_PACKAGES.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server/tools/MAKE_LINUX_PACKAGES.py b/server/tools/MAKE_LINUX_PACKAGES.py index 404312927..734c55fa6 100755 --- a/server/tools/MAKE_LINUX_PACKAGES.py +++ b/server/tools/MAKE_LINUX_PACKAGES.py @@ -651,6 +651,7 @@ def get_postinst(sdist_basefilename: str) -> str: $(system_python_executable) {DSTVENVSCRIPT} {DSTPYTHONVENV} --skippackagechecks echo 'About to install CamCOPS into virtual environment' +{DSTVENVPIP} install -U pip setuptools {DSTVENVPIP} install {dst_sdist_file} #------------------------------------------------------------------------------ From f7f18c59c578ac11c46d27f5b2a61fdec96681f2 Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Wed, 8 Jan 2025 12:08:31 +0000 Subject: [PATCH 11/24] Ensure PIP available in Python 3.12 and check python 3.12 works --- .github/workflows/build-and-install-packages.yml | 4 +++- .github/workflows/python-package-checks.yml | 2 +- .github/workflows/python-tests.yml | 2 +- server/setup.py | 1 + server/tools/MAKE_LINUX_PACKAGES.py | 4 ++-- server/tools/install_virtualenv.py | 3 +++ 6 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-and-install-packages.yml b/.github/workflows/build-and-install-packages.yml index 15f55491a..34f2ddb7d 100644 --- a/.github/workflows/build-and-install-packages.yml +++ b/.github/workflows/build-and-install-packages.yml @@ -18,7 +18,9 @@ jobs: build-and-install-package: strategy: matrix: - python-version: [3.9, "3.10", 3.11] + # Just one here as the OS will determine the version used + # when the package is installed + python-version: ["3.10"] # LTS versions os: [ubuntu-22.04, ubuntu-24.04] runs-on: ${{ matrix.os }} diff --git a/.github/workflows/python-package-checks.yml b/.github/workflows/python-package-checks.yml index 83445bbab..ecde58ea8 100644 --- a/.github/workflows/python-package-checks.yml +++ b/.github/workflows/python-package-checks.yml @@ -12,7 +12,7 @@ jobs: python-package-checks: strategy: matrix: - python-version: [3.9, "3.10", "3.11"] + python-version: [3.9, "3.10", 3.11, 3.12] # LTS versions os: [ubuntu-20.04, ubuntu-22.04] runs-on: ${{ matrix.os }} diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index eaac8d1c2..43601d6ef 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -14,7 +14,7 @@ jobs: python-tests: strategy: matrix: - python-version: [3.9, "3.10", 3.11] + python-version: [3.9, "3.10", 3.11, 3.12] # LTS versions os: [ubuntu-20.04, ubuntu-22.04] runs-on: ${{ matrix.os }} diff --git a/server/setup.py b/server/setup.py index 65b8d8ffa..50ff0d65c 100644 --- a/server/setup.py +++ b/server/setup.py @@ -200,6 +200,7 @@ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Topic :: Scientific/Engineering :: Medical Science Apps.", ], keywords="cardinal", diff --git a/server/tools/MAKE_LINUX_PACKAGES.py b/server/tools/MAKE_LINUX_PACKAGES.py index 734c55fa6..df244fbc0 100755 --- a/server/tools/MAKE_LINUX_PACKAGES.py +++ b/server/tools/MAKE_LINUX_PACKAGES.py @@ -248,6 +248,7 @@ def call(cmdargs: List[str], **kwargs) -> None: # Use as: $(system_python_executable) ... python_options=( + python3.12 python312 python3.11 python311 python3.10 python310 python3.9 python39 @@ -551,7 +552,7 @@ def get_debian_control() -> str: Architecture: all Maintainer: Rudolf Cardinal Depends: {DEPENDENCIES} -X-Python3-Version: >= 3.9, <= 3.11 +X-Python3-Version: >= 3.9, <= 3.12 Recommends: mysql-workbench Description: Cambridge Cognitive and Psychiatric Test Kit (CamCOPS), server packages. @@ -651,7 +652,6 @@ def get_postinst(sdist_basefilename: str) -> str: $(system_python_executable) {DSTVENVSCRIPT} {DSTPYTHONVENV} --skippackagechecks echo 'About to install CamCOPS into virtual environment' -{DSTVENVPIP} install -U pip setuptools {DSTVENVPIP} install {dst_sdist_file} #------------------------------------------------------------------------------ diff --git a/server/tools/install_virtualenv.py b/server/tools/install_virtualenv.py index bc456ed23..0ee9f6f0c 100755 --- a/server/tools/install_virtualenv.py +++ b/server/tools/install_virtualenv.py @@ -218,6 +218,9 @@ def main() -> None: PYTHON, venv_tool, args.virtualenv ) ) + check_call([PYTHON, "-m", "ensurepip"]) + print("OK") + check_call([PYTHON, "-m", venv_tool, args.virtualenv]) print("OK") From 05609c056897fed0fd3e2f1a88fdc28eff8c45c7 Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Wed, 8 Jan 2025 12:24:28 +0000 Subject: [PATCH 12/24] Use later lxml for Python 3.12 support --- server/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/setup.py b/server/setup.py index 50ff0d65c..8f36b9e86 100644 --- a/server/setup.py +++ b/server/setup.py @@ -92,7 +92,7 @@ # Celery 5.3 "importlib-metadata<5.0", "lockfile==0.12.2", # File locking for background tasks - "lxml==4.9.1", # Will speed up openpyxl export [NO LONGER CRITICAL] + "lxml==4.9.4", # Will speed up openpyxl export [NO LONGER CRITICAL] "matplotlib==3.9.4", # Used for trackers and some tasks. SLOW INSTALLATION. # noqa "numpy==1.23.5", # Used by some tasks. SLOW INSTALLATION. "paginate==0.5.6", # pagination for web server From b96540974d1b8a2764fe0161ed2942353fb813f2 Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Wed, 8 Jan 2025 12:29:09 +0000 Subject: [PATCH 13/24] Don't try to ensure PIP Ubuntu says it's disabled for system python. If it's still broken, try installing the python3-pip APT package --- server/tools/install_virtualenv.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/server/tools/install_virtualenv.py b/server/tools/install_virtualenv.py index 0ee9f6f0c..bc456ed23 100755 --- a/server/tools/install_virtualenv.py +++ b/server/tools/install_virtualenv.py @@ -218,9 +218,6 @@ def main() -> None: PYTHON, venv_tool, args.virtualenv ) ) - check_call([PYTHON, "-m", "ensurepip"]) - print("OK") - check_call([PYTHON, "-m", venv_tool, args.virtualenv]) print("OK") From 44a6291c0e74c96a5edf931a750fa5da1bc0527b Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Wed, 8 Jan 2025 14:24:57 +0000 Subject: [PATCH 14/24] Make sure we have setuptools in the Linux package --- server/tools/install_virtualenv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/tools/install_virtualenv.py b/server/tools/install_virtualenv.py index bc456ed23..e3c00a1f4 100755 --- a/server/tools/install_virtualenv.py +++ b/server/tools/install_virtualenv.py @@ -222,7 +222,7 @@ def main() -> None: print("OK") title("Upgrading pip within virtualenv") - check_call([venv_pip, "install", "--upgrade", "pip"]) + check_call([venv_pip, "install", "--upgrade", "pip setuptools"]) title("Checking version of tools within new virtualenv") print(venv_python) From 7b5b12047bbcdbc0bc047eafb39763a9efa32dea Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Wed, 8 Jan 2025 14:44:24 +0000 Subject: [PATCH 15/24] Trigger the Linux package workflow when install_virtualenv.py changes --- .github/workflows/build-and-install-packages.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-and-install-packages.yml b/.github/workflows/build-and-install-packages.yml index 34f2ddb7d..80529763b 100644 --- a/.github/workflows/build-and-install-packages.yml +++ b/.github/workflows/build-and-install-packages.yml @@ -11,6 +11,7 @@ on: - .github/scripts/build_packages.sh - .github/scripts/change_apt_mirror.sh - .github/scripts/python_setup.sh + - server/tools/install_virtualenv.py - server/tools/MAKE_LINUX_PACKAGES.py - server/requirements-deb.txt - server/requirements-rpm.txt From 323546e3cb36c1d14d18f932720c2c2c13b969c5 Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Wed, 8 Jan 2025 16:56:00 +0000 Subject: [PATCH 16/24] Use later numpy and openpyxl for Python 3.12 compatibility --- server/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/setup.py b/server/setup.py index 8f36b9e86..e467e5d37 100644 --- a/server/setup.py +++ b/server/setup.py @@ -94,7 +94,7 @@ "lockfile==0.12.2", # File locking for background tasks "lxml==4.9.4", # Will speed up openpyxl export [NO LONGER CRITICAL] "matplotlib==3.9.4", # Used for trackers and some tasks. SLOW INSTALLATION. # noqa - "numpy==1.23.5", # Used by some tasks. SLOW INSTALLATION. + "numpy==1.26.4", # Used by some tasks. SLOW INSTALLATION. "paginate==0.5.6", # pagination for web server "pendulum==2.1.2", # date/time classes "pexpect==4.8.0", # for open_sqlcipher.py @@ -148,7 +148,7 @@ "beautifulsoup4==4.9.1", "colorlog==4.1.0", "isodate==0.6.0", - "openpyxl==3.0.4", # also for pyexcel-xlsx + "openpyxl==3.0.7", # also for pyexcel-xlsx "pandas==1.3.4", "prettytable==0.7.2", "psutil==5.7.0", From 3fd777ee55e7d922960df0ba5d5034dee4e058e1 Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Wed, 8 Jan 2025 17:35:03 +0000 Subject: [PATCH 17/24] Use later scipy for Python 3.12 compatibility --- server/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/setup.py b/server/setup.py index e467e5d37..8a63f5594 100644 --- a/server/setup.py +++ b/server/setup.py @@ -113,7 +113,7 @@ "requests==2.32.2", # in fetch_snomed_codes.py and cc_sms.py, but also required by something else? # noqa "rich-argparse==0.5.0", # colourful help "sadisplay==0.4.9", # SQL Alchemy schema display script - "scipy==1.10.1", # used by some tasks. slow installation. + "scipy==1.13.1", # used by some tasks. slow installation. "semantic_version==2.8.5", # semantic versioning; better than semver "sqlalchemy==1.4.49", # database access "statsmodels==0.13.5", # e.g. logistic regression From 61d2e97de54fefe57bb2842a0f4518ef739269dd Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Wed, 8 Jan 2025 22:11:58 +0000 Subject: [PATCH 18/24] Use later statsmodels and pandas (needed by PyCap) --- server/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/setup.py b/server/setup.py index 8a63f5594..814c99319 100644 --- a/server/setup.py +++ b/server/setup.py @@ -116,7 +116,7 @@ "scipy==1.13.1", # used by some tasks. slow installation. "semantic_version==2.8.5", # semantic versioning; better than semver "sqlalchemy==1.4.49", # database access - "statsmodels==0.13.5", # e.g. logistic regression + "statsmodels==0.14.4", # e.g. logistic regression "twilio==7.9.3", # SMS backend for Multi-factor authentication "urllib3==1.26.19", # dependency, pinned to avoid vulnerabilities "Wand==0.6.1", # ImageMagick binding @@ -149,7 +149,7 @@ "colorlog==4.1.0", "isodate==0.6.0", "openpyxl==3.0.7", # also for pyexcel-xlsx - "pandas==1.3.4", + "pandas==1.4.4", "prettytable==0.7.2", "psutil==5.7.0", "pyparsing==2.4.7", From 0c23ec2ffaac2865943ef6657475cc656765fd74 Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Wed, 8 Jan 2025 22:19:48 +0000 Subject: [PATCH 19/24] Fix installing pip and setuptools in Linux package script --- server/tools/install_virtualenv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/tools/install_virtualenv.py b/server/tools/install_virtualenv.py index e3c00a1f4..ef2b29280 100755 --- a/server/tools/install_virtualenv.py +++ b/server/tools/install_virtualenv.py @@ -222,7 +222,7 @@ def main() -> None: print("OK") title("Upgrading pip within virtualenv") - check_call([venv_pip, "install", "--upgrade", "pip setuptools"]) + check_call([venv_pip, "install", "--upgrade", "pip", "setuptools"]) title("Checking version of tools within new virtualenv") print(venv_python) From c518d715ecca34e475ba573a25f8c8227bacb1a4 Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Thu, 9 Jan 2025 06:39:40 +0000 Subject: [PATCH 20/24] Update Pendulum for later Python and swap Period for Interval --- server/camcops_server/cc_modules/cc_export.py | 10 ++++++---- server/setup.py | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/server/camcops_server/cc_modules/cc_export.py b/server/camcops_server/cc_modules/cc_export.py index 10ec6e086..d373e8d87 100644 --- a/server/camcops_server/cc_modules/cc_export.py +++ b/server/camcops_server/cc_modules/cc_export.py @@ -188,7 +188,7 @@ from cardinal_pythonlib.sizeformatter import bytes2human from cardinal_pythonlib.sqlalchemy.session import get_safe_url_from_engine import lockfile -from pendulum import DateTime as Pendulum, Duration, Period +from pendulum import DateTime as Pendulum, Duration from pyramid.httpexceptions import HTTPBadRequest from pyramid.renderers import render_to_response from pyramid.response import Response @@ -230,6 +230,8 @@ ) if TYPE_CHECKING: + from pendulum import Interval + from camcops_server.cc_modules.cc_request import CamcopsRequest from camcops_server.cc_modules.cc_taskcollection import TaskCollection @@ -1497,10 +1499,10 @@ def time_left(self) -> Optional[Duration]: death = self.when_last_modified + Duration( minutes=self.permitted_lifespan_min ) - remaining = death - now # type: Period - # Note that Period is a subclass of Duration, but its __str__() + remaining = death - now # type: Interval + # Note that Interval is a subclass of Duration, but its __str__() # method is different. Duration maps __str__() to in_words(), but - # Period maps __str__() to __repr__(). + # Interval maps __str__() to __repr__(). return remaining @property diff --git a/server/setup.py b/server/setup.py index 814c99319..51c471d77 100644 --- a/server/setup.py +++ b/server/setup.py @@ -96,7 +96,7 @@ "matplotlib==3.9.4", # Used for trackers and some tasks. SLOW INSTALLATION. # noqa "numpy==1.26.4", # Used by some tasks. SLOW INSTALLATION. "paginate==0.5.6", # pagination for web server - "pendulum==2.1.2", # date/time classes + "pendulum==3.0.0", # date/time classes "pexpect==4.8.0", # for open_sqlcipher.py "pdfkit==1.0.0", # wkhtmltopdf interface, for PDF generation from HTML "phonenumbers==8.12.30", # phone number parsing, storing and validating From d60656eec6f6ba22a1971abe503b6902452071b7 Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Thu, 9 Jan 2025 15:12:49 +0000 Subject: [PATCH 21/24] Use newer Celery for Python 3.12 compatibility --- server/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/setup.py b/server/setup.py index 51c471d77..7001aee3f 100644 --- a/server/setup.py +++ b/server/setup.py @@ -68,7 +68,7 @@ "alembic==1.4.2", # database migrations "asteval==0.9.25", # safe-ish alternative to eval "cardinal_pythonlib==1.1.25", # RNC libraries - "celery==5.2.2", # background tasks + "celery==5.3.6", # background tasks "colander==1.7.0", # serialization/deserialization from web forms "CherryPy==18.6.0", # web server ( # deform: web forms From f83347e47aea4c008defa624d2df0918d92f6752 Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Thu, 9 Jan 2025 15:33:20 +0000 Subject: [PATCH 22/24] Fix python packages following Celery upgrade --- server/setup.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/server/setup.py b/server/setup.py index 7001aee3f..f0d350ae1 100644 --- a/server/setup.py +++ b/server/setup.py @@ -87,10 +87,6 @@ "flower==2.0.1", # monitor for Celery "gunicorn==23.0.0", # web server (Unix only) "hl7==0.3.5", # For HL7 export - # Celery dependency for Python <= 3.7; workaround import error - # https://github.com/celery/celery/issues/7783; scheduled to be fixed in - # Celery 5.3 - "importlib-metadata<5.0", "lockfile==0.12.2", # File locking for background tasks "lxml==4.9.4", # Will speed up openpyxl export [NO LONGER CRITICAL] "matplotlib==3.9.4", # Used for trackers and some tasks. SLOW INSTALLATION. # noqa @@ -154,7 +150,7 @@ "psutil==5.7.0", "pyparsing==2.4.7", "pypdf==3.17.0", # Used by cardinal_pythonlib.pdf - "python-dateutil==2.8.1", # date/time extensions. + "python-dateutil==2.9.0.post0", # date/time extensions. "sqlparse==0.5.0", # extra "py-bcrypt==0.4", # used by cardinal_pythonlib.crypto @@ -162,7 +158,7 @@ # Dependencies of other things above # ------------------------------------------------------------------------- "alabaster==0.7.12", # for sphinx - "amqp==5.0.6", # for celery + "amqp==5.3.1", # for celery "Chameleon==3.8.1", # for Deform "tornado==6.4.2", # for celery "webob==1.8.8", # for pyramid From eea81703c36c5b41ff6138bb85cc1977e537cb8f Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Thu, 9 Jan 2025 16:03:06 +0000 Subject: [PATCH 23/24] Update changelog --- docs/source/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index 5d7c014a0..170e148dc 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -3953,3 +3953,4 @@ Current C++/SQLite client, Python/SQLAlchemy server ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - Update the Docker image to use Debian 11. Debian 10 has now reached end-of-life. +- **Minimum Python version now Python 3.9.** Python 3.11 and 3.12 supported. From 25c84df928ac2a8aced09f039dc61ee854ac5b34 Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Thu, 9 Jan 2025 16:03:39 +0000 Subject: [PATCH 24/24] Update documentation for supported Python versions --- docs/source/developer/versions.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/source/developer/versions.rst b/docs/source/developer/versions.rst index d34ccbd0c..1a00a8cf0 100644 --- a/docs/source/developer/versions.rst +++ b/docs/source/developer/versions.rst @@ -42,7 +42,9 @@ Versions of software etc. used by CamCOPS +----------------+---------+---------------------------------------------------+ | | 3.10 | 2026-10-31 | +----------------+---------+---------------------------------------------------+ -| | 3.11 | 2027-10-31; | +| | 3.11 | 2027-10-31 | ++----------------+---------+---------------------------------------------------+ +| | 3.12 | 2028-10-31; | | | | https://devguide.python.org/versions/ | +----------------+---------+---------------------------------------------------+ | Qt | 6.5.x | 2026-03-31 (LTS) but 6.5.x branch now |