WIP Convert CircleCI config to Github Actions

.github/actions/health_check_action.yml

@@ -0,0 +1,16 @@
+name: 'URL Health Check'
+description: 'Performs URL health check with retries'
+inputs:
+  url:
+    description: 'Health check URL'
+    required: true
+runs:
+  using: 'composite'
+  steps:
+    - shell: bash
+      run: |
+        for i in {1..20}; do
+          curl --fail -v "${{ inputs.url }}" && exit 0
+          sleep 5
+        done
+        exit 1

.github/not-on-master.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+if [ "${GITHUB_REF}" == "master" ]; then
+  echo "Skipping this step on master..."
+else
+  exec "$@"
+fi

.github/workflows/build_test_deploy.yml

@@ -1,13 +1,199 @@
 name: CI Pipeline
 
 on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - '*'
   workflow_dispatch:
 
+env:
+  USER_NAME: root
+  USER_UID: 1000
+  USER_GID: 1000
+  DOCKER_USER: ${{ secrets.DOCKER_USER }}
+  DOCKER_PASS: ${{ secrets.DOCKER_PASS }}
+
 jobs:
-  foo:
+  static_frontend_code_checks:
     runs-on: ubuntu-20.04
     steps:
     - name: Checkout code
       uses: actions/checkout@v3
       with:
-        fetch-depth: 5
+        fetch-depth: 5
+
+    - uses: actions/setup-node@v4
+      with:
+          node-version: 18
+
+    - name: Install frontend dependencies
+      run: corepack enable && yarn install --immutable
+
+    - name: Lint frontend code and check formatting
+      run: yarn run check-frontend
+
+    - name: Typecheck frontend code
+      run: yarn typecheck
+
+    - name: Check for cyclic dependencies in frontend
+      run: yarn check-cyclic-dependencies
+
+
+
+
+  build_test_deploy:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 5
+
+      - name: "Custom environment variables"
+        run: |
+          if [[ ${{ github.ref_type }} == "branch" ]]; then
+            NORMALIZED_BRANCH=$(echo ${{ github.ref_name }} | sed 's/[\/-]/_/g')
+            echo "NORMALIZED_BRANCH=$NORMALIZED_BRANCH" >> $GITHUB_ENV
+            DOCKER_TAG="${NORMALIZED_BRANCH}__${{ github.run_number }}"
+            echo "DOCKER_TAG=$DOCKER_TAG" >> $GITHUB_ENV
+          else
+            echo "NORMALIZED_BRANCH=master" >> $GITHUB_ENV
+            echo "DOCKER_TAG=${{ github.ref_name }}" >> $GITHUB_ENV
+          fi
+
+      - name: Set up Docker
+        uses: docker/setup-buildx-action@v2
+
+      - name: Build webknossos-dev docker image
+        run: |
+          docker pull scalableminds/webknossos-dev:$NORMALIZED_BRANCH || true
+          DEV_CACHE=$NORMALIZED_BRANCH docker compose build base
+
+      - name: Prepare dependency folders
+        run: mkdir -p project/target target ~/.ivy2 ~/.cache/coursier
+
+      - name: Install frontend dependencies
+        run: docker compose run -e PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true base yarn install --immutable
+
+      - name: Assert unique evolution numbers
+        run: docker compose run base tools/postgres/dbtool.js assert-unique-evolution-numbers
+
+      - name: Assert schema.sql and evolutions are equal
+        run: |
+          docker compose up -d postgres
+          sleep 3
+          docker compose run compile tools/postgres/dbtool.js check-evolutions-schema
+
+      - name: Build frontend documentation
+        run: |
+          WK_VERSION=${{ github.event.release.tag_name || github.run_number || 'dev' }}
+          docker compose run base yarn run docs --project-version $WK_VERSION
+
+      - name: Build webknossos (webpack)
+        run: docker compose run base yarn build
+
+      - name: Build webknossos (sbt)
+        run: |
+          if [ "${{ github.ref }}" == "refs/heads/master" ]; then
+            docker compose run compile sbt -no-colors clean compile stage
+          else
+            docker compose run compile sbt -no-colors -DfailOnWarning compile stage
+          fi
+
+      - name: Build webknossos-datastore (sbt)
+        run: docker compose run base sbt -no-colors -DfailOnWarning "project webknossosDatastore" copyMessages compile stage
+
+      - name: Build webknossos-tracingstore (sbt)
+        run: docker compose run base sbt -no-colors -DfailOnWarning "project webknossosTracingstore" copyMessages compile stage
+
+      - name: Checksum App Dirs
+        run: find app webknossos-datastore/app webknossos-tracingstore/app -type f -exec md5sum {} \; | sort -k 2 | md5sum > app_checksum.txt
+
+      - name: Build webknossos docker image
+        run: |
+          docker pull scalableminds/webknossos:$NORMALIZED_BRANCH || true
+          DEV_CACHE=$NORMALIZED_BRANCH docker compose build --pull webknossos
+
+      - name: Build webknossos-datastore docker image
+        run: docker compose build --pull webknossos-datastore
+
+      - name: Build webknossos-tracingstore docker image
+        run: docker compose build --pull webknossos-tracingstore
+
+      - name: Run frontend tests
+        run: .github/not-on-master.sh docker compose run base yarn test-verbose
+
+      - name: Lint backend code and check formatting
+        run: .github/not-on-master.sh docker compose run backend-lint-format
+
+      - name: Run backend tests
+        run: .github/not-on-master.sh docker compose run backend-tests
+
+      - name: Run end-to-end tests
+        run: |
+          for i in {1..3}; do # retry
+            .github/not-on-master.sh docker compose run e2e-tests && s=0 && break || s=$?
+          done
+          (exit $s)
+
+      - name: Validate frontend types
+        run: .github/not-on-master.sh docker compose run base yarn typecheck
+
+      - name: Start webknossos, datastore, and tracingstore
+        run: |
+          docker compose up -d webknossos
+          docker compose up -d webknossos-datastore
+          docker compose up -d webknossos-tracingstore
+
+      - name: Run webknossos smoke test
+        uses: ./.github/actions/health_check_action
+        with: 
+          url: http://localhost:9000/api/health
+
+
+      - name: Run webknossos-datastore smoke test
+        uses: ./.github/actions/health_check_action
+        with: 
+          url: http://localhost:9090/data/health
+
+      - name: Run webknossos-tracingstore smoke test
+        uses: ./.github/actions/health_check_action
+        with: 
+          url: http://localhost:9050/tracings/health
+
+      - name: Stop webknossos, datastore, and tracingstore
+        run: docker compose down --volumes --remove-orphans
+
+      - name: Push docker images
+        run: |
+          function retry() {
+            for i in {1..5}; do
+              "$@" && s=0 && break || s=$?
+              sleep 10
+            done
+            return $s
+          }
+          retry docker login -u $DOCKER_USER -p $DOCKER_PASS
+          retry docker compose push webknossos
+          retry docker compose push webknossos-datastore
+          retry docker compose push webknossos-tracingstore
+          if [[ ${{ github.ref_type }} == "branch" ]]; then
+            docker tag scalableminds/webknossos:${DOCKER_TAG} scalableminds/webknossos:${NORMALIZED_BRANCH}
+            retry docker push scalableminds/webknossos:${NORMALIZED_BRANCH}
+            docker tag scalableminds/webknossos-datastore:${DOCKER_TAG} scalableminds/webknossos-datastore:${NORMALIZED_BRANCH}
+            retry docker push scalableminds/webknossos-datastore:${NORMALIZED_BRANCH}
+            docker tag scalableminds/webknossos-tracingstore:${DOCKER_TAG} scalableminds/webknossos-tracingstore:${NORMALIZED_BRANCH}
+            retry docker push scalableminds/webknossos-tracingstore:${NORMALIZED_BRANCH}
+            docker tag scalableminds/webknossos-dev scalableminds/webknossos-dev:${NORMALIZED_BRANCH}
+            retry docker push scalableminds/webknossos-dev:${NORMALIZED_BRANCH}
+          fi
+          docker logout
+
+      - name: Report coverage
+        run: .github/not-on-master.sh docker compose run base yarn coverage || true
+
+      # - name: Send Slack notification (master only)
+      #   run: .circleci/slack-notification.sh

docker-compose.yml

@@ -105,8 +105,8 @@ services:
       - POSTGRES_USER=webknossos_user
       - POSTGRES_PASSWORD=secret_password
       - HOME=/root
-      - CIRCLE_TAG=${CIRCLE_TAG}
-      - CIRCLE_BUILD_NUM=${CIRCLE_BUILD_NUM}
+      - CI_TAG=${GITHUB_REF}
+      - CI_BUILD_NUM=${GITHUB_JOB}
       - NODE_OPTIONS=--max_old_space_size=2048
       # This will be picked up by ava so that the tests fail if a snapshot doesn't exist during CI, see https://github.com/avajs/ava/issues/1585
       - CI

project/BuildInfoSettings.scala

@@ -13,8 +13,8 @@ object BuildInfoSettings {
     }
 
   val ciBuild: String =
-    if (System.getenv().containsKey("CIRCLE_BUILD_NUM")) System.getenv().get("CIRCLE_BUILD_NUM") else ""
-  val ciTag: String = if (System.getenv().containsKey("CIRCLE_TAG")) System.getenv().get("CIRCLE_TAG") else ""
+    if (System.getenv().containsKey("CI_BUILD_NUM")) System.getenv().get("CI_BUILD_NUM") else ""
+  val ciTag: String = if (System.getenv().containsKey("CI_TAG")) System.getenv().get("CI_TAG") else ""
 
   def commitHash: String = getStdoutFromCommand("git rev-parse HEAD", "<getting commit hash failed>")
   def commitDate: String = getStdoutFromCommand("git log -1 --format=%cd ", "<getting git date failed>")