test ci

esek · Nov 7, 2024 · 16823b0 · 16823b0
1 parent 27cbb8f
commit 16823b0
Show file tree

Hide file tree

Showing 4 changed files with 425 additions and 54 deletions.
diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml
@@ -0,0 +1,297 @@
+name: Docker Image CI
+
+# To run this locally use act (https://github.com/nektos/act)
+
+# To use act:
+# Create a .secrets file and set all the needed variables
+# act -j <job>
+# act <event_name>
+# act -e .github/act/<event>.json // For additinal event params
+# act -P self-hosted=ghcr.io/catthehacker/ubuntu:act-latest  // To make it wokr with self hosted
+
+on:
+  push:
+    branches: [test-ci]
+  pull_request:
+    types: [opened, closed, reopened, synchronize]
+  release:
+    types: [released, prereleased]
+
+env:
+  REGISTRY: registry.esek.se
+  DEFAULT_BRANCH: main
+  ONLY_PROD: false
+
+jobs:
+  # This can only contain env vars and not secrets since actions is a fussy about it
+  env-vars:
+    runs-on: self-hosted
+    container: ghcr.io/catthehacker/ubuntu:act-latest
+    steps:
+      - name: set variables
+        id: vars
+        run: |
+          slugify () {
+            iconv -t ascii//TRANSLIT \
+            | tr -d "'" \
+            | sed -E 's/[^a-zA-Z0-9]+/-/g' \
+            | sed -E 's/^-+|-+$//g' \
+            | tr "[:upper:]" "[:lower:]"
+          }
+
+          echo ${{ github.event }}
+
+          export BRANCH=
+          if [ -z $GITHUB_HEAD_REF ]; then
+            BRANCH=$(basename $GITHUB_REF)
+          else
+            BRANCH=$GITHUB_HEAD_REF
+          fi
+          echo "Running on $BRANCH"
+
+          export TAG=ephermal
+
+          if [ "$BRANCH" = "${{ env.DEFAULT_BRANCH }}" ]; then
+            TAG=latest
+          fi
+
+          if ${{ github.ref_type == 'tag' }}; then
+            TAG=$BRANCH
+            BRANCH=${{ env.DEFAULT_BRANCH }}
+          fi
+
+          TAG=$(echo "$TAG" | slugify)
+          export IMAGE_NAME="${{ env.REGISTRY }}/${{ github.repository }}:$TAG"
+
+          echo "::echo::on"
+          echo "::set-output name=image_name::$IMAGE_NAME"
+          echo "::set-output name=tag::$TAG"
+          echo "::set-output name=basename::$(basename ${{ github.repository }} | slugify)"
+          echo "::set-output name=branch::$( echo $BRANCH | slugify)"
+          echo "::echo::off"
+      - name: select environment
+        id: env
+        run: |
+          export ENV=testing
+          export DEPLOY_ENV=testing
+          export PROJ_BASE=${{ steps.vars.outputs.basename }}
+          export TAG=${{ steps.vars.outputs.tag }}
+          export BRANCH=${{ steps.vars.outputs.branch }}
+          export PROJECT_NAME=$PROJ_BASE-testing
+
+          if ${{ (github.event_name == 'release' && github.event.action == 'released') || env.ONLY_PROD == 'true' }}; then
+            ENV=production
+            DEPLOY_ENV=$ENV
+            PROJECT_NAME=$PROJ_BASE-production
+          fi
+          if ${{ github.event_name == 'release' && github.event.action == 'prereleased' }}; then
+            ENV=staging
+            DEPLOY_ENV=$ENV
+            PROJECT_NAME=$PROJ_BASE-staging
+          fi
+          if ${{ github.event_name == 'pull_request' }}; then
+            ENV=review
+            DEPLOY_ENV=$BRANCH-review
+            PROJECT_NAME=$PROJ_BASE-$BRANCH
+          fi
+
+          echo "::echo::on"
+          echo "::set-output name=env::$ENV"
+          echo "::set-output name=project_name::$PROJECT_NAME"
+          echo "::set-output name=deploy_env::$DEPLOY_ENV"
+          echo "::echo::off"
+    outputs:
+      image_name: ${{ steps.vars.outputs.image_name }}
+      tag: ${{ steps.vars.outputs.tag }}
+      basename: ${{ steps.vars.outputs.basename }}
+      branch: ${{ steps.vars.outputs.branch }}
+      env: ${{ steps.env.outputs.env }}
+      project_name: ${{ steps.env.outputs.project_name }}
+      deploy_env: ${{ steps.env.outputs.deploy_env }}
+  build:
+    runs-on: self-hosted
+    needs: env-vars
+    if: ${{ !(github.event_name == 'pull_request' && github.event.action == 'closed') }}
+    container:
+      image: ghcr.io/catthehacker/ubuntu:act-latest
+    steps:
+      - name: import secrets
+        id: s
+        uses: hashicorp/[email protected]
+        with:
+          url: ${{ secrets.VAULT_URL }}
+          token: ${{ secrets.VAULT_TOKEN }}
+          method: token
+          secrets: |
+            kv/data/common registry_username;
+            kv/data/common registry_password;
+      - name: setup buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          install: true
+      - name: checkout repo
+        uses: actions/checkout@v3
+      - name: login to registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ steps.s.outputs.registry_username }}
+          password: ${{ steps.s.outputs.registry_password }}
+      - name: build and push server image
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          file: ./server/Dockerfile
+          push: true
+          tags: ${{ needs.env-vars.outputs.image_name }}-server
+      # - name: build and push web image
+      #   uses: docker/build-push-action@v3
+      #   with:
+      #     file: ./web/Dockerfile
+      #     context: .
+      #     push: true
+      #     tags: ${{ needs.env-vars.outputs.image_name }}-web
+
+  deploy:
+    # To run this locally using act set act to false in event.json
+    if: ${{ !github.event.act || !(github.event_name == 'pull_request' && github.event.action == 'closed') }}
+    runs-on: self-hosted
+    needs:
+      - env-vars
+      - build
+    steps:
+      - name: import secrets
+        id: s
+        uses: hashicorp/[email protected]
+        with:
+          url: ${{ secrets.VAULT_URL }}
+          token: ${{ secrets.VAULT_TOKEN }}
+          method: token
+          exportEnv: true
+          secrets: |
+            kv/data/common registry_username;
+            kv/data/common registry_password;
+            kv/data/common docker_ca;
+            kv/data/common docker_cert;
+            kv/data/common docker_key;
+            kv/data/common docker_host;
+            kv/data/${{ needs.env-vars.outputs.basename }}/${{ needs.env-vars.outputs.env }} DEPLOY_URL;
+            kv/data/${{ needs.env-vars.outputs.basename }}/${{ needs.env-vars.outputs.env }} DB_DATA_LOCATION;
+            kv/data/${{ needs.env-vars.outputs.basename }}/${{ needs.env-vars.outputs.env }} DB_PASSWORD;
+            kv/data/${{ needs.env-vars.outputs.basename }}/${{ needs.env-vars.outputs.env }} DB_USERNAME;
+            kv/data/${{ needs.env-vars.outputs.basename }}/${{ needs.env-vars.outputs.env }} IMMICH_VERSION;
+            kv/data/${{ needs.env-vars.outputs.basename }}/${{ needs.env-vars.outputs.env }} UPLOAD_LOCATION;
+      - name: set variables
+        run: |
+          echo "IMAGE_NAME=${{ needs.env-vars.outputs.image_name }}" >> $GITHUB_ENV
+          echo "TAG=${{ needs.env-vars.outputs.tag }}" >> $GITHUB_ENV
+          echo "DEPLOY_ENV=${{ needs.env-vars.outputs.deploy_env }}" >> $GITHUB_ENV
+          echo "ENV=${{ needs.env-vars.outputs.env }}" >> $GITHUB_ENV
+          echo "PROJECT_NAME=${{ needs.env-vars.outputs.project_name }}" >> $GITHUB_ENV
+          echo "COMPOSE_PROJECT_NAME=${{ needs.env-vars.outputs.project_name }}" >> $GITHUB_ENV
+
+          if [ ${{ needs.env-vars.outputs.env }} = 'review' ]; then
+            echo "DEPLOY_URL=${{ needs.env-vars.outputs.branch }}-${{ needs.env-vars.outputs.basename }}.review.esek.se" >> $GITHUB_ENV
+          fi
+      - name: checkout repo
+        uses: actions/checkout@v3
+      - name: start deployment
+        uses: bobheadxi/deployments@v1
+        id: deployment
+        if: ${{ !env.ACT && github.event_name != 'pull_request' && needs.env-vars.outputs.env != 'review' }}
+        with:
+          step: start
+          token: ${{ secrets.GITHUB_TOKEN }}
+          override: true
+          env: ${{ env.DEPLOY_ENV }}
+      - name: login to host
+        uses: blennster/setup-docker-remote-tls@v4
+        with:
+          tls_key: ${{ steps.s.outputs.docker_key }}
+          tls_ca: ${{ steps.s.outputs.docker_ca }}
+          tls_cert: ${{ steps.s.outputs.docker_cert }}
+          tcp_host: ${{ steps.s.outputs.docker_host }}:2376
+      - name: check connection
+        run: |
+          docker info
+          docker compose version
+      - name: login to registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ steps.s.outputs.registry_username }}
+          password: ${{ steps.s.outputs.registry_password }}
+      - name: pull image(s)
+        run: echo $IMAGE_NAME; docker compose -f docker-compose.ci.yml pull
+      - name: Stop old container
+        run: |
+          docker compose -f docker-compose.ci.yml -p $PROJECT_NAME down
+      - name: deploy to host
+        shell: bash
+        run: |
+          if [[ "$ENV" = "production" || "$ENV" = "staging" ]]; then
+            docker compose -f docker-compose.ci.yml -f docker-compose.prod.yml -p $PROJECT_NAME up -d --force-recreate
+          fi
+          if [[ "$ENV" = "review" || "$ENV" = "testing" ]]; then
+            docker compose -f docker-compose.ci.yml -p $PROJECT_NAME up -d --force-recreate
+            docker compose -f docker-compose.ci.yml -p $PROJECT_NAME exec lager-be sh -c './lager-be seed' 
+          fi
+      - name: prune system
+        run: docker system prune -f 2> /dev/null || true
+      - name: update deployment status
+        uses: bobheadxi/deployments@v1
+        if: ${{ always() && !env.ACT && github.event_name != 'pull_request' && needs.env-vars.outputs.env != 'review' }}
+        with:
+          step: finish
+          token: ${{ secrets.GITHUB_TOKEN }}
+          status: ${{ job.status }}
+          env: ${{ steps.deployment.outputs.env }}
+          env_url: https://${{ env.DEPLOY_URL }}
+          deployment_id: ${{ steps.deployment.outputs.deployment_id }}
+      - if: ${{ github.event.pull_request && github.event.action == 'opened' && !env.ACT }}
+        name: comment deployment url to pr
+        uses: mshick/add-pr-comment@v1
+        with:
+          message: I have deployed this PR to [${{ env.DEPLOY_URL }}](https://${{ env.DEPLOY_URL }}) 🚀
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+  teardown:
+    runs-on: self-hosted
+    if: ${{ github.event_name == 'pull_request' && github.event.action == 'closed' }}
+    needs: env-vars
+    steps:
+      - name: import secrets
+        id: s
+        uses: hashicorp/[email protected]
+        with:
+          url: ${{ secrets.VAULT_URL }}
+          token: ${{ secrets.VAULT_TOKEN }}
+          method: token
+          exportEnv: true
+          secrets: |
+            kv/data/common docker_ca;
+            kv/data/common docker_cert;
+            kv/data/common docker_key;
+            kv/data/common docker_host;
+      - name: login to host
+        uses: blennster/setup-docker-remote-tls@v4
+        with:
+          tls_key: ${{ steps.s.outputs.docker_key }}
+          tls_ca: ${{ steps.s.outputs.docker_ca }}
+          tls_cert: ${{ steps.s.outputs.docker_cert }}
+          tcp_host: ${{ steps.s.outputs.docker_host }}:2376
+      - name: check connection
+        run: |
+          docker info
+          docker compose version
+      - name: checkout repo
+        uses: actions/checkout@v3
+      - name: stop container
+        run: |
+          docker compose -f docker-compose.ci.yml -p ${{ needs.env-vars.outputs.project_name }} down 2> /dev/null || true
+      - if: ${{ !env.ACT }}
+        name: comment deployment url to pr
+        uses: mshick/add-pr-comment@v1
+        with:
+          message: I have removed this deploy now 😇
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/docker.yml → .github/workflows/docker.yml.disable b/.github/workflows/docker.yml → .github/workflows/docker.yml.disable
diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
@@ -0,0 +1,91 @@
+name: immich-prod
+
+networks:
+  traefik:
+    external: true
+
+services:
+  immich-server:
+    container_name: immich_server
+    image: '${IMAGE_NAME}-server'
+    # extends:
+    #   file: hwaccel.transcoding.yml
+    #   service: cpu # set to one of [nvenc, quicksync, rkmpp, vaapi, vaapi-wsl] for accelerated transcoding
+    #build:
+    #  context: ../
+    #  dockerfile: server/Dockerfile
+    volumes:
+      - ${UPLOAD_LOCATION}/photos:/usr/src/app/upload
+      - /etc/localtime:/etc/localtime:ro
+    env_file:
+      - .env
+    ports:
+      - 2283:2283
+    depends_on:
+      - redis
+      - database
+    restart: always
+    healthcheck:
+      disable: false
+
+    labels:
+      - 'traefik.http.routers.$COMPOSE_PROJECT_NAME-server.rule=Host(`$DEPLOY_URL`)'
+
+    networks:
+      - traefik
+      - default
+
+  # immich-machine-learning:
+  #   container_name: immich_machine_learning
+  #   image: immich-machine-learning:latest
+  #   # extends:
+  #   #   file: hwaccel.ml.yml
+  #   #   service: cpu # set to one of [armnn, cuda, openvino, openvino-wsl] for accelerated inference
+  #   build:
+  #     context: ../machine-learning
+  #     dockerfile: Dockerfile
+  #     args:
+  #       - DEVICE=cpu # set to one of [armnn, cuda, openvino, openvino-wsl] for accelerated inference
+  #   ports:
+  #     - 3003:3003
+  #   volumes:
+  #     - model-cache:/cache
+  #   env_file:
+  #     - .env
+  #   restart: always
+  #   healthcheck:
+  #     disable: false
+
+  redis:
+    container_name: immich_redis
+    image: redis:6.2-alpine@sha256:2ba50e1ac3a0ea17b736ce9db2b0a9f6f8b85d4c27d5f5accc6a416d8f42c6d5
+    healthcheck:
+      test: redis-cli ping || exit 1
+    restart: always
+
+  database:
+    container_name: immich_postgres
+    image: tensorchord/pgvecto-rs:pg14-v0.2.0@sha256:90724186f0a3517cf6914295b5ab410db9ce23190a2d9d0b9dd6463e3fa298f0
+    env_file:
+      - .env
+    environment:
+      POSTGRES_PASSWORD: ${DB_PASSWORD}
+      POSTGRES_USER: ${DB_USERNAME}
+      POSTGRES_DB: ${DB_DATABASE_NAME}
+      POSTGRES_INITDB_ARGS: '--data-checksums'
+    volumes:
+      - ${UPLOAD_LOCATION}/postgres:/var/lib/postgresql/data
+    ports:
+      - 5432:5432
+    healthcheck:
+      test: pg_isready --dbname='${DB_DATABASE_NAME}' --username='${DB_USERNAME}' || exit 1; Chksum="$$(psql --dbname='${DB_DATABASE_NAME}' --username='${DB_USERNAME}' --tuples-only --no-align --command='SELECT COALESCE(SUM(checksum_failures), 0) FROM pg_stat_database')"; echo "checksum failure count is $$Chksum"; [ "$$Chksum" = '0' ] || exit 1
+      interval: 5m
+      start_interval: 30s
+      start_period: 5m
+    command: [ 'postgres', '-c', 'shared_preload_libraries=vectors.so', '-c', 'search_path="$$user", public, vectors', '-c', 'logging_collector=on', '-c', 'max_wal_size=2GB', '-c', 'shared_buffers=512MB', '-c', 'wal_compression=on' ]
+    restart: always
+
+volumes:
+  model-cache:
+  prometheus-data:
+  grafana-data: