Merge pull request #996 from sanger/develop

Develop to master

.github/workflows/automated_release.yml

@@ -30,7 +30,7 @@ jobs:
           SA_PASSWORD: MyS3cr3tPassw0rd
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - uses: nelonoel/[email protected]
 

.github/workflows/check_release_version.yml

@@ -10,7 +10,7 @@ jobs:
   check:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Get specific changed files
         id: changed-files-specific

.github/workflows/ci.yml

@@ -11,15 +11,25 @@ on:
       - master
 
 jobs:
+  setup:
+    runs-on: ubuntu-latest
+    outputs:
+      python_version: ${{ steps.read_python_version.outputs.python_version }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Read Python version
+        id: read_python_version
+        run: echo "::set-output name=python_version::$(cat .python-version)"
   black:
     runs-on: ubuntu-latest
+    needs: setup
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.8
-      - uses: actions/cache@v1
+          python-version: ${{ needs.setup.outputs.python_version }}
+      - uses: actions/cache@v4
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('**/Pipfile') }}
@@ -37,13 +47,14 @@ jobs:
           python -m black --check .
   flake8:
     runs-on: ubuntu-latest
+    needs: setup
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.8
-      - uses: actions/cache@v1
+          python-version: ${{ needs.setup.outputs.python_version }}
+      - uses: actions/cache@v4
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('**/Pipfile') }}
@@ -61,13 +72,14 @@ jobs:
           flake8
   mypy:
     runs-on: ubuntu-latest
+    needs: setup
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.8
-      - uses: actions/cache@v1
+          python-version: ${{ needs.setup.outputs.python_version }}
+      - uses: actions/cache@v4
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('**/Pipfile') }}
@@ -84,6 +96,7 @@ jobs:
           python -m mypy .
   test:
     runs-on: ubuntu-latest
+    needs: setup
     services:
       mysql:
         image: mysql:8.0
@@ -100,17 +113,21 @@ jobs:
           ACCEPT_EULA: Y
           SA_PASSWORD: MyS3cr3tPassw0rd
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.8
-      - uses: actions/cache@v1
+          python-version: ${{ needs.setup.outputs.python_version }}
+      - uses: actions/cache@v4
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('**/Pipfile') }}
           restore-keys: |
             ${{ runner.os }}-pip-
+      - name: Install dependencies for building Python wheels (Ubuntu)
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y build-essential python3-dev libffi-dev libssl-dev libpq-dev unixodbc-dev
       - name: Install pipenv
         run: |
           pip install pipenv
@@ -122,12 +139,17 @@ jobs:
         with:
           mongodb-version: 4.2
           mongodb-replica-set: heron_rs
-      - name: Create SQL Server testing database
-        run: |
-          python setup_sqlserver_test_db.py
       - name: Setup the test MLWH and Events databases
         run: |
+          curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add -
+          curl https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list
+          sudo apt-get update
+          sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18
+          sudo apt-get install -y unixodbc-dev
           python setup_test_db.py
+      - name: Create SQL Server testing database
+        run: |
+          python setup_sqlserver_test_db.py
       - name: Test with pytest
         run: |
           python -m pytest -x

.github/workflows/codeql-analysis.yml

@@ -36,7 +36,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

.github/workflows/docker_dispatch.yml

@@ -15,7 +15,7 @@ jobs:
   build_and_publish:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Build the Docker image
         run: >-
           docker build .

.gitignore

@@ -84,9 +84,6 @@ target/
 profile_default/
 ipython_config.py
 
-# pyenv
-.python-version
-
 # pipenv
 #   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
 #   However, in case of collaboration, if having platform-specific dependencies or dependencies
@@ -132,3 +129,6 @@ dmypy.json
 .pyre/
 
 tests/data/reports/*
+
+# PyCharm indexes
+.idea

.python-version

@@ -0,0 +1 @@
+3.13.0

.release-version

@@ -1 +1 @@
-2.36.0
+2.37.0

Dockerfile

@@ -1,5 +1,5 @@
 # Use slim for a smaller image size and install only the required packages
-FROM python:3.8-slim-buster
+FROM python:3.13-slim
 
 # Use the following on M1; for odbc connection to mssql.
 # FROM --platform=linux/amd64 python:3.8-slim-buster
@@ -20,15 +20,11 @@ RUN apt-get update && \
     unixodbc-dev
 
 # Install the Microsoft ODBC driver for SQL Server
-#   https://docs.microsoft.com/en-us/sql/connect/odbc/linux-mac/installing-the-microsoft-odbc-driver-for-sql-server?view=sql-server-ver15#debian17
-#   Debian 10
 RUN curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - && \
     curl https://packages.microsoft.com/config/debian/10/prod.list > /etc/apt/sources.list.d/mssql-release.list
-
-RUN apt-get update && \
-    ACCEPT_EULA=Y apt-get install -y \
-    msodbcsql17 \
-    && rm -rf /var/lib/apt/lists/*
+RUN apt-get update
+RUN ACCEPT_EULA=Y apt-get install -y msodbcsql18
+RUN apt-get install -y unixodbc-dev
 
 # Install the package manager - pipenv
 RUN pip install --upgrade pip && \

Pipfile

@@ -31,9 +31,11 @@ requests = "~=2.32"
 slackclient = "~=2.9"
 sqlalchemy = "~=2.0"
 pymongo = "~=4.8.0"
+setuptools = "*"
+numpy = "*"
 
 [requires]
-python_version = "3.8"
+python_version = "3.13"
 
 [pipenv]
 allow_prereleases = true