Merge branch 'sqlite-int-tests' into db-conn-refactor

G-Research · Aug 18, 2023 · b93e91c · b93e91c
2 parents 763f1cd + ea96774
commit b93e91c
Show file tree

Hide file tree

Showing 22 changed files with 780 additions and 172 deletions.
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
@@ -1,20 +1,26 @@
-# [Choice] Go version (use -bullseye variants on local arm64/Apple Silicon): 1, 1.18, 1.17, 1-bullseye, 1.18-bullseye, 1.17-bullseye, 1-buster, 1.18-buster, 1.17-buster
-ARG VARIANT=1-bullseye
-FROM mcr.microsoft.com/devcontainers/go:0-${VARIANT}
-
-# [Choice] Node.js version: none, lts/*, 16, 14, 12, 10
-ARG NODE_VERSION="none"
-RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi
+# [Choice] Go version (use -bookworm, or -bullseye variants on local arm64/Apple Silicon): 1, 1.21, 1.20, 1-bookworm, 1.21-bookworm, 1.20-bookworm, 1-bullseye, 1.21-bullseye, 1.20-bullseye
+ARG VARIANT=1-bookworm
+FROM mcr.microsoft.com/devcontainers/go:1-${VARIANT}
 
 # [Optional] Uncomment this section to install additional OS packages.
-RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-    && apt-get -y install --no-install-recommends libssl-dev postgresql-client python-is-python3 python3-pip python3-dev python3-venv sqlite3 emacs-nox ripgrep fzf \
-    && pip install pipenv
+RUN apt-get update \
+ && DEBIAN_FRONTEND=noninteractive apt-get -y install --no-install-recommends \
+    emacs-nox \
+    fzf \
+    libssl-dev \
+    pipenv \
+    postgresql-client \
+    python-is-python3 \
+    python3-dev \
+    python3-pip \
+    python3-venv \
+    ripgrep \
+    sqlite3
 
 # [Optional] Uncomment the next lines to use go get to install anything else you need
 USER vscode
-RUN go install github.com/vektra/mockery/v2@v2.30.16 \
- && go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.53.3 \
+RUN go install github.com/vektra/mockery/v2@v2.32.4 \
+ && go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.54.1 \
  && go install golang.org/x/tools/cmd/[email protected] \
  && go install mvdan.cc/[email protected]
 USER root
@@ -23,4 +29,8 @@ USER root
 # RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g <your-package-here>" 2>&1
 
 # Install xh
-RUN curl -sfL https://raw.githubusercontent.com/ducaale/xh/master/install.sh | XH_BINDIR=/usr/local/bin sh
+RUN curl -sfL https://raw.githubusercontent.com/ducaale/xh/master/install.sh | XH_BINDIR=/usr/local/bin sh
+
+# Add docker-compose symlink to Docker Compose v2
+# This is a workaround until https://github.com/devcontainers/features/pull/621 lands
+RUN ln -s /usr/libexec/docker/cli-plugins/docker-compose /usr/local/bin/
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -22,7 +22,9 @@
 				"golang.Go",
 				"ms-python.python",
 				"pbkit.vscode-pbkit",
-				"redhat.vscode-yaml"
+				"redhat.vscode-yaml",
+				"ms-python.black-formatter",
+				"ms-python.isort"
 			]
 		}
 	},
@@ -38,10 +40,13 @@
 	"remoteUser": "vscode",
 	"features": {
 		"ghcr.io/devcontainers/features/docker-in-docker:2": {
-			"dockerDashComposeVersion": "v2"
+			"dockerDashComposeVersion": "none"
 		},
 		"ghcr.io/devcontainers/features/git:1": {},
 		"ghcr.io/devcontainers/features/github-cli:1": {},
+		"ghcr.io/devcontainers/features/node:1": {
+			"version": "16"
+		},
 		"ghcr.io/dhoeric/features/k6:1": {},
 		"ghcr.io/christophermacgown/devcontainer-features/minio-client:1": {}
 	},

diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml
@@ -15,9 +15,7 @@ services:
         # [Choice] Go version 1, 1.18, 1.17
         # Append -bullseye or -buster to pin to an OS version.
         # Use -bullseye variants on local arm64/Apple Silicon.
-        VARIANT: 1.20-bullseye
-        # Options
-        NODE_VERSION: "16"
+        VARIANT: 1.21-bookworm
     env_file:
       # Ensure that the variables in .env match the same variables in devcontainer.json
       - .env

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -12,9 +12,9 @@ permissions:
   contents: read
 
 jobs:
-  lint:
+  go-lint:
     if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id
-    name: Lint
+    name: Go Lint
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -23,7 +23,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v4
         with:
-          go-version: "1.20"
+          go-version: "1.21"
 
       - name: Read build tags
         id: tags
@@ -48,7 +48,7 @@ jobs:
           fi
 
       - name: Download mockery
-        run: go install github.com/vektra/mockery/v2@v2.30.16
+        run: go install github.com/vektra/mockery/v2@v2.32.4
 
       - name: Generate mocks
         run: make mocks-generate
@@ -61,9 +61,28 @@ jobs:
           version: "2022.1.3"
           build-tags: ${{ steps.tags.outputs.tags }}
 
-  golang-unit-tests:
+  python-lint:
     if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id
-    name: Golang Unit Tests
+    name: Python Lint
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+
+      - name: Install pipenv
+        run: pip install pipenv==2022.12.19
+
+      - name: Run Python linters
+        run: make python-lint
+
+  go-unit-tests:
+    if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id
+    name: Go Unit Tests
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -72,20 +91,20 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v4
         with:
-          go-version: "1.20"
+          go-version: "1.21"
 
       - name: Download mockery
-        run: go install github.com/vektra/mockery/v2@v2.30.16
+        run: go install github.com/vektra/mockery/v2@v2.32.4
 
       - name: Generate mocks
         run: make mocks-generate
 
-      - name: Run Golang Unit Tests
+      - name: Run Go Unit Tests
         run: make test-go-unit
 
-  golang-integration-tests:
+  go-integration-tests:
     if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id
-    name: Golang Integration Tests
+    name: Go Integration Tests
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -97,12 +116,19 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v4
         with:
-          go-version: "1.20"
+          go-version: "1.21"
 
-      - name: Run Integration Tests
+      - name: Run Integration Tests (Sqlite)
         run: make service-test
         env:
           DOCKER_BUILDKIT: 1
+          FML_DATABASE_URI: sqlite://fasttrack-test.db
+
+      - name: Run Integration Tests (Postgres)
+        run: make service-test
+        env:
+          DOCKER_BUILDKIT: 1
+          FML_DATABASE_URI: postgres://postgres:postgres@postgres/postgres
 
   python-integration-tests:
     if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id
@@ -133,14 +159,14 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v4
         with:
-          go-version: "1.20"
+          go-version: "1.21"
 
       - name: Run ${{ matrix.api }} integration tests
         run: ./tests/integration/python/${{ matrix.api }}/test.sh
 
-  golang-build:
+  build:
     if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id
-    name: Build Go binary for ${{ matrix.os }}/${{ matrix.arch }}
+    name: Build software distribution for ${{ matrix.os }}/${{ matrix.arch }}
     strategy:
       matrix:
         os: [darwin, linux, windows]
@@ -164,6 +190,19 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: Setup Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: "1.21"
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+
+      - name: Install pipenv
+        run: pip install pipenv==2022.12.19
+
       - name: Install arm64 cross-compilation toolchain on Linux
         if: matrix.os == 'linux' && matrix.arch == 'arm64'
         run: |
@@ -178,18 +217,24 @@ jobs:
           sudo apt-get install -y --no-install-recommends gcc-mingw-w64-x86-64-win32
           echo CC=x86_64-w64-mingw32-gcc >> $GITHUB_ENV
 
-      - name: Build and archive Go binary
-        run: make go-dist
+      - name: Build software distribution
+        run: make dist
         env:
           GOOS: ${{ matrix.os }}
           GOARCH: ${{ matrix.arch }}
 
-      - name: Upload artifact
+      - name: Upload binary artifact
         uses: actions/upload-artifact@v3
         with:
           name: fasttrackml-archives
           path: dist/*
 
+      - name: Upload wheels artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: fasttrackml-wheels
+          path: wheelhouse/*.whl
+
   build-image:
     if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id
     name: Build container image
@@ -250,11 +295,12 @@ jobs:
   all-required-checks-done:
     name: All required checks done
     needs:
-      - lint
-      - golang-unit-tests
-      - golang-integration-tests
+      - go-lint
+      - python-lint
+      - go-unit-tests
+      - go-integration-tests
       - python-integration-tests
-      - golang-build
+      - build
       - build-image
     runs-on: ubuntu-latest
     steps:

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -49,6 +49,24 @@ jobs:
             exit 1
           fi
 
+  pypi-publish:
+    name: upload release to PyPI
+    needs: validate
+    if: github.event.workflow_run.head_branch != 'main'
+    runs-on: ubuntu-latest
+    environment: release
+    permissions:
+      id-token: write
+    steps:
+      - name: Download artifact
+        run: gh run download ${{ github.event.workflow_run.id }} --repo ${{ github.event.workflow_run.repository.full_name }} --name fasttrackml-wheels --dir wheelhouse
+        env:
+          GH_TOKEN: ${{ github.token }}
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: wheelhouse/
+
   github-release:
     name: Publish GitHub release
     needs: validate

diff --git a/.gitignore b/.gitignore
@@ -2,6 +2,7 @@
 .idea
 
 # Binaries
+dist/
 fml
 fasttrackml
 *.exe
@@ -25,4 +26,9 @@ mock_*
 
 # Optional workspace
 go.work
-*.code-workspace
+*.code-workspace
+
+# Python wheels
+python/build/
+python/*.egg-info/
+wheelhouse/
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -8,5 +8,19 @@
     "go.lintTool": "golangci-lint",
     "go.lintFlags": [
         "--fast"
+    ],
+    "[python]": {
+        "editor.defaultFormatter": "ms-python.black-formatter",
+        "editor.codeActionsOnSave": {
+            "source.organizeImports": true
+        }
+    },
+    "black-formatter.args": [
+        "--line-length",
+        "120"
+    ],
+    "isort.args": [
+        "--profile",
+        "black"
     ]
 }
diff --git a/Dockerfile b/Dockerfile
@@ -1,5 +1,5 @@
 # Build fml binary
-FROM --platform=$BUILDPLATFORM golang:1.20 AS go-build
+FROM --platform=$BUILDPLATFORM golang:1.21 AS go-build
 
 ARG TARGETARCH
 RUN bash -c "\