Merge pull request #1754 from alan-turing-institute/release-v4.2.0

Release v4.2.0
alan-turing-institute · Mar 28, 2024 · 9f6fe58 · 9f6fe58
2 parents e9f4a1a + 73b9344
commit 9f6fe58
Show file tree

Hide file tree

Showing 252 changed files with 3,968 additions and 8,789 deletions.
diff --git a/.all-contributorsrc b/.all-contributorsrc
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
@@ -2,34 +2,49 @@ ARG VARIANT=3.10
 FROM python:${VARIANT}-buster
 
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-    && apt-get -y install --no-install-recommends libssl-dev git python3-sphinx
-
+    && apt-get -y install --no-install-recommends apt-transport-https \
+    ca-certificates \
+    curl \
+    git \
+    gnupg \
+    libssl-dev \
+    lsb-release \
+    python3-sphinx \
+    sudo
+
 # Set package versions
-ARG AZURE_CLI_VERSION="2.42.0"
-ARG PWSH_VERSION="7.3.6"
-
-# Set up TARGETARCH variable to use to pull the right binaries for the current architecture.
-ARG TARGETARCH
+ARG AZURE_CLI_VERSION="2.58.0"
+ARG PWSH_VERSION="7.4.1"
 
 # Install Azure-CLI 
-# Standard install method currently does not support ARM64 
-# Use pip instead - https://github.com/Azure/azure-cli/issues/22875 
-RUN pip3 install azure-cli==${AZURE_CLI_VERSION} 
+# Get Microsoft signing key
+RUN sudo mkdir -p /etc/apt/keyrings \
+  && curl -sLS https://packages.microsoft.com/keys/microsoft.asc | \
+      gpg --dearmor | \
+      sudo tee /etc/apt/keyrings/microsoft.gpg > /dev/null \
+  && sudo chmod go+r /etc/apt/keyrings/microsoft.gpg \
+  && AZ_DIST=$(lsb_release -cs) \
+  && echo "deb [arch=`dpkg --print-architecture` signed-by=/etc/apt/keyrings/microsoft.gpg] https://packages.microsoft.com/repos/azure-cli/ $AZ_DIST main" | \
+      sudo tee /etc/apt/sources.list.d/azure-cli.list \
+  && sudo apt-get update \
+  && sudo apt-get install azure-cli=$AZURE_CLI_VERSION-1~$AZ_DIST
 
 # Install Powershell
 # Pull different binaries from Github depending on system architecture
 # The standard APT method currently only works for `amd64`
- RUN if [ "${TARGETARCH}" = "arm64" ]; \
-   then \
-     DEBARCH="arm64"; \
-   else \
-     DEBARCH="x86"; \
-   fi; \
-   curl -L -o /tmp/powershell.tar.gz https://github.com/PowerShell/PowerShell/releases/download/v${PWSH_VERSION}/powershell-${PWSH_VERSION}-linux-$DEBARCH.tar.gz \
-   && mkdir -p /opt/microsoft/powershell/7 \
-   && tar zxf /tmp/powershell.tar.gz -C /opt/microsoft/powershell/7 \
-   && chmod +x /opt/microsoft/powershell/7/pwsh \
-   && ln -s /opt/microsoft/powershell/7/pwsh /usr/bin/pwsh
+# Set up TARGETARCH variable to use to pull the right binaries for the current architecture.
+ARG TARGETARCH
+RUN if [ "${TARGETARCH}" = "arm64" ]; \
+  then \
+    DEBARCH="arm64"; \
+  else \
+    DEBARCH="x86"; \
+  fi; \
+  curl -L -o /tmp/powershell.tar.gz https://github.com/PowerShell/PowerShell/releases/download/v${PWSH_VERSION}/powershell-${PWSH_VERSION}-linux-$DEBARCH.tar.gz \
+  && mkdir -p /opt/microsoft/powershell/7 \
+  && tar zxf /tmp/powershell.tar.gz -C /opt/microsoft/powershell/7 \
+  && chmod +x /opt/microsoft/powershell/7/pwsh \
+  && ln -s /opt/microsoft/powershell/7/pwsh /usr/bin/pwsh
 
 # Create non-root user and give them sudo access
 ARG USERNAME=deploydsh
@@ -53,5 +68,5 @@ RUN pip3 install -r /build/requirements.txt
 COPY ./deployment/CheckRequirements.ps1 /build/CheckRequirements.ps1
 COPY ./deployment/common/Logging.psm1 /build/common/Logging.psm1
 RUN pwsh -Command "& {Set-PSRepository -Name PSGallery -InstallationPolicy Trusted}" \
-  && pwsh -File /build/CheckRequirements.ps1 -InstallMissing \
+  && pwsh -File /build/CheckRequirements.ps1 -InstallMissing  -IncludeDev\
   && sudo rm -rf /build/
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -5,13 +5,17 @@
 		"context": "..",
 		"dockerfile": "Dockerfile"
 	},
-	"settings": {
-		"terminal.integrated.defaultProfile.linux": "bash"
+	"customizations": {
+		"vscode": {
+			"settings": {
+				"terminal.integrated.defaultProfile.linux": "bash"
+			},
+			"extensions": [
+				"github.vscode-pull-request-github",
+				"ms-vscode.powershell",
+				"ms-vscode.azure-account"
+			]
+		}
 	},
-	"extensions": [
-		"github.vscode-pull-request-github",
-		"ms-vscode.powershell",
-		"ms-vscode.azure-account"
-	],
 	"remoteUser": "deploydsh"
 }
diff --git a/.github/ISSUE_TEMPLATE/release_checklist.md b/.github/ISSUE_TEMPLATE/release_checklist.md
@@ -0,0 +1,52 @@
+---
+name: Release checklist
+about: Log completion of required actions for release testing
+title: "Release: <version number>"
+labels: "type: release-candidate"
+assignees: ""
+---
+
+## :white_check_mark: Checklist
+
+<!--
+Before reporting a problem please check the following. Replace the empty checkboxes [ ] below with checked ones [x] accordingly.
+-->
+
+Refer to the [Deployment](https://data-safe-haven.readthedocs.io/en/latest/deployment) section of our documentation when completing these steps.
+
+- [ ] Consult the `data-safe-haven/VERSIONING.md` guide and determine the version number of the new release. Record it in the title of this issue.
+- [ ] Create a release branch called e.g. `release-v0.0.1`
+- [ ] Draft a changelog for the release similar to our previous releases, see https://github.com/alan-turing-institute/data-safe-haven/releases
+- [ ] Deploy an SHM from this branch and save a transcript of the deployment logs
+- [ ] Build an SRE compute image (SRD) and save transcripts of the logs
+- Using the new image, deploy a tier 2 and a tier 3 SRE
+    - [ ] Save the transcript of your tier 2 SRE deployment
+    - [ ] Save the transcript of your tier 3 SRE deployment
+- [ ] Complete the [Security evaluation checklist](https://data-safe-haven.readthedocs.io/en/latest/deployment/security_checklist.html) from the deployment documentation
+
+### For major releases
+
+- [ ] Confirm that a third party has carried out a full penetration test evaluating:
+  1. external attack surface
+  1. ability to exfiltrate data from the system
+  1. ability to transfer data between SREs
+  1. ability to escalate privileges on the SRD.
+
+### Update documentation
+
+- [ ] Update supported versions in `SECURITY.md`
+- [ ] Update pen test results in `VERSIONING.md`
+
+## :computer: Release information
+
+- **Version number:** _
+- **SHM ID:** _
+- **T2 SRE ID:** _
+- **T3 SRE ID:** _
+
+## :deciduous_tree: Deployment problems
+
+<!--
+Keep a record in this issue of problems and fixes implemented during the release process. Be sure to update the changelog if any new commits are added to the release branch.
+-->
+
diff --git a/.github/scripts/update_dbeaver_drivers.py b/.github/scripts/update_dbeaver_drivers.py
@@ -4,30 +4,69 @@
 from natsort import natsorted
 import requests
 
-output = {}
-remote_page = requests.get("https://repo1.maven.org/maven2/com/microsoft/sqlserver/mssql-jdbc/", allow_redirects=True)
-root = html.fromstring(remote_page.content)
-output["mssql_jdbc"] = natsorted([v for v in root.xpath("//a[contains(text(), 'jre8/')]/@href") if v != "../"])[-1].replace("/", "")
-
-remote_page = requests.get("https://repo1.maven.org/maven2/org/postgresql/pgjdbc-versions/", allow_redirects=True)
-root = html.fromstring(remote_page.content)
-output["pgjdbc"] = natsorted([v for v in root.xpath("//a[contains(text(), '/')]/@href") if v != "../"])[-1].replace("/", "")
-
-remote_page = requests.get("https://repo1.maven.org/maven2/org/postgresql/postgresql/", allow_redirects=True)
-root = html.fromstring(remote_page.content)
-output["postgresql"] = natsorted([v for v in root.xpath("//a[contains(text(), '/')]/@href") if v != "../"])[-1].replace("/", "")
-
-remote_page = requests.get("https://repo1.maven.org/maven2/net/postgis/postgis-jdbc/", allow_redirects=True)
-root = html.fromstring(remote_page.content)
-postgis_jdbc_versions = natsorted([v for v in root.xpath("//a[contains(text(), '/')]/@href") if v != "../"])
-
-remote_page = requests.get("https://repo1.maven.org/maven2/net/postgis/postgis-geometry/", allow_redirects=True)
-root = html.fromstring(remote_page.content)
-postgis_geometry_versions = natsorted([v for v in root.xpath("//a[contains(text(), '/')]/@href") if v != "../"])
-
-postgis = natsorted(set(postgis_jdbc_versions).intersection(set(postgis_geometry_versions)))[-1].replace("/", "")
-output["postgis_geometry"] = postgis
-output["postgis_jdbc"] = postgis
+
+def get_latest_version(url, search_text):
+    """
+    Get latest version number of a database driver from the Maven repository.
+
+    Fetches the HTML page at the given URL, then converts it to an lxml tree.
+    Numeric strings are then extracted.
+    Note that mostly numeric strings for some drivers contain non-numeric text,
+    as different driver types exist for those drivers, even where the version number is the same.
+    The largest (latest) version number of the driver is then returned.
+
+    Parameters
+    ----------
+    url : str
+        The URL of the Maven repository containing the driver
+    search_text : str
+        Text to search for in the repository, to distinguish the driver from other files
+
+    Returns
+    -------
+    list
+        The latest available version number of the driver
+    """
+
+    remote_page = requests.get(url, allow_redirects=True)
+    root = html.fromstring(remote_page.content)
+    return natsorted([v for v in root.xpath("//a[contains(text(), '" + search_text + "')]/@href") if v != "../"])[-1].replace("/", "")
+
+
+drivers = [
+    {
+        'name': "mssql_jdbc",
+        'url': "https://repo1.maven.org/maven2/com/microsoft/sqlserver/mssql-jdbc/",
+        'search_text': "jre8/"
+    },
+    {
+        'name': "pgjdbc",
+        'url': "https://repo1.maven.org/maven2/org/postgresql/pgjdbc-versions/",
+        'search_text': "/"
+    },
+    {
+        'name': "postgresql",
+        'url': "https://repo1.maven.org/maven2/org/postgresql/postgresql/",
+        'search_text': "/"
+    },
+    {
+        'name': "postgis_geometry",
+        'url': "https://repo1.maven.org/maven2/net/postgis/postgis-geometry/",
+        'search_text': "/"
+    },
+    {
+        'name': "postgis_jdbc",
+        'url': "https://repo1.maven.org/maven2/net/postgis/postgis-jdbc/",
+        'search_text': "/"
+    },
+    {
+        'name': "waffle_jna",
+        'url': "https://repo1.maven.org/maven2/com/github/waffle/waffle-jna/",
+        'search_text': "/"
+    }
+]
+
+output = {driver['name']: get_latest_version(driver['url'], driver['search_text']) for driver in drivers}
 
 with open("deployment/secure_research_desktop/packages/dbeaver-driver-versions.json", "w") as f_out:
     f_out.writelines(json.dumps(output, indent=4, sort_keys=True))
diff --git a/.github/workflows/documentation.yaml b/.github/workflows/documentation.yaml
@@ -4,9 +4,9 @@ name: Documentation
 # Run workflow on pushes to matching branches
 on:  # yamllint disable-line rule:truthy
   push:
-    branches: [develop]
+    branches: [develop, latest]
   pull_request:
-    branches: [develop]
+    branches: [develop, latest]
 
 jobs:
   build:

diff --git a/.github/workflows/lint_code.yaml b/.github/workflows/lint_code.yaml
@@ -4,9 +4,9 @@ name: Lint code
 # Run workflow on pushes to matching branches
 on:  # yamllint disable-line rule:truthy
   push:
-    branches: [develop]
+    branches: [develop, latest]
   pull_request:
-    branches: [develop]
+    branches: [develop, latest]
 
 jobs:
   lint_json:

diff --git a/.github/workflows/test_code.yaml b/.github/workflows/test_code.yaml
@@ -4,9 +4,9 @@ name: Test code
 # Run workflow on pushes to matching branches
 on:  # yamllint disable-line rule:truthy
   push:
-    branches: [develop]
+    branches: [develop, latest]
   pull_request:
-    branches: [develop]
+    branches: [develop, latest]
 
 jobs:
   test_powershell:

diff --git a/.github/workflows/update_package_versions.yaml b/.github/workflows/update_package_versions.yaml
@@ -42,7 +42,7 @@ jobs:
 
       - name: Update DBeaver driver versions
         run: |
-          python .github/scripts/update_rstudio.py
+          python .github/scripts/update_dbeaver_drivers.py
 
       - name: Check for changes
         shell: bash

diff --git a/.lychee.toml b/.lychee.toml
@@ -100,7 +100,8 @@ exclude = [
 # Exclude these filesystem paths from getting checked.
 # exclude_path = ["file/path/to/Ignore", "./other/file/path/to/Ignore"]
 exclude_path = [
-  'docs/build/html/_static/webpack-macros.html'
+  'docs/build/html/_static/webpack-macros.html',
+  'docs/build/html/contributing/index.html'
 ]
 
 # URLs to check (supports regex). Has preference over all excludes.

diff --git a/CODEOWNERS b/CODEOWNERS
@@ -10,5 +10,5 @@
 # We only plan to require code owner review for
 # main and other branches that may be deployed from.
 # Note: /dir/ applies to directory and all subdirectories
-/deployment/      @martintoreilly @jemrobinson @JimMadge
-/docs/            @martintoreilly @jemrobinson @JimMadge @craddm @edwardchalstrey1
+/deployment/      @martintoreilly @jemrobinson @JimMadge @craddm
+/docs/            @martintoreilly @jemrobinson @JimMadge @craddm
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -27,6 +27,7 @@ Jump to the following sections:
     - [Who's involved in the project](#whos-involved-in-the-project)
     - [Make a change with a pull request](#making-a-change-with-a-pull-request)
     - [Make a change to the documentation](#making-a-change-to-the-documentation)
+    - [Adding new contributors](#adding-new-contributors)
 
 ## A DevOps development philosophy
 
@@ -230,44 +231,15 @@ To preview your changes, you can build the docs locally. See [docs/README.md](do
 
 ### Who's involved in the project
 
-As some users have `GitHub` IDs that make it a little difficult to know who they are in real life and it's sometimes nice to know **who** you're working with on the project, hopefully this table will help you put names to faces and IDs :sparkles:
-
-The following people have made one or more commits to the project:
-
-| Name                 | GitHub ID                                                  |
-| -------------------- | ---------------------------------------------------------- |
-| Daniel Allen         | [@sysdan](https://github.com/sysdan)                       |
-| Diego Arenas         | [@darenasc](https://github.com/darenasc)                   |
-| David Beavan         | [@DavidBeavan](https://github.com/DavidBeavan)             |
-| Alvaro Cabrejas Egea | [@ACabrejas](https://github.com/ACabrejas)                 |
-| Ian Carter           | [@getcarter21](https://github.com/getcarter21)             |
-| Ed Chalstrey         | [@edwardchalstrey1](https://github.com/edwardchalstrey1)   |
-| Rob Clarke           | [@RobC-CTL](https://github.com/RobC-CTL)                   |
-| James Cunningham     | [@james-c](https://github.com/james-c)                     |
-| Tom Doel             | [@tomdoel](https://github.com/tomdoel)                     |
-| Chris Edsall         | [@christopheredsall](https://github.com/christopheredsall) |
-| Oliver Forrest       | [@oforrest](https://github.com/oforrest)                   |
-| Oscar Giles          | [@OscartGiles](https://github.com/OscartGiles)             |
-| James Hetherington   | [@jamespjh](https://github.com/jamespjh)                   |
-| Tim Hobson           | [@thobson88](https://github.com/thobson88)                 |
-| George Holmes        | [@ens-george-holmes](https://github.com/ens-george-holmes) |
-| Catherine Lawrence   | [@cathiest](https://github.com/cathiest)                   |
-| Tomas Lazauskas      | [@tomaslaz](https://github.com/tomaslaz)                   |
-| Jim Madge            | [@JimMadge](https://github.com/JimMadge)                   |
-| Jules Manser         | [@JulesMarz](https://github.com/JulesMarz)                 |
-| Miguel Morin         | [@miguelmorin](https://github.com/miguelmorin)             |
-| Federico Nanni       | [@fedenanni](https://github.com/fedenanni)                 |
-| Guillaume Noell      | [@gn5](https://github.com/gn5)                             |
-| Martin O'Reilly      | [@martintoreilly](https://github.com/martintoreilly)       |
-| Jack Roberts         | [@jack89roberts](https://github.com/jack89roberts)         |
-| James Robinson       | [@jemrobinson](https://github.com/jemrobinson)             |
-| Brett Todd           | [@ens-brett-todd](https://github.com/ens-brett-todd)       |
-| Sebastian Vollmer    | [@vollmersj](https://github.com/vollmersj)                 |
-| Ben Walden           | [@bw-faststream](https://github.com/bw-faststream)         |
-| Kirstie Whitaker     | [@KirstieJane](https://github.com/KirstieJane)             |
-| Rachel Winstanley    | [@rwinstanley1](https://github.com/rwinstanley1)           |
-| Warwick Wood         | [@warwick26](https://github.com/warwick26)                 |
-| Kevin Xu             | [@kevinxufs](https://github.com/kevinxufs)                 |
+Take a look at the full list of contributors on our [README](README.md).
+
+### Adding new contributors
+
+We use @all-contributors to maintain the contributor list on the README, however we do not use the GitHub bot.
+
+You should follow the same instructions as above to [make a change with a pull request](#making-a-change-with-a-pull-request) when adding a new contributor.
+
+To add new contributor to the README table, see the [all-contributors CLI documentation](https://allcontributors.org/docs/en/cli/overview) and use the CLI to add the new contributor, then make a pull request with your changes.
 
 ## Get in touch