✨ Add initial Helm chart

Dockerfile.deployment

@@ -30,7 +30,8 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get install -y \
     unit \
     unit-php \
     yarnpkg \
-    zip
+    zip \
+    sudo
 
 WORKDIR /app
 
@@ -41,6 +42,7 @@ COPY package.json yarn.lock ./
 RUN yarnpkg
 
 COPY . .
+RUN cp /app/docker/deployment/sudoers /etc/sudoers
 RUN mkdir -p bootstrap/cache storage/framework/views storage/framework/sessions
 RUN composer dump-autoload
 

deploy.sh

@@ -12,6 +12,7 @@ ln -snf "$outdir" "/deploy/current"
 
 cd "$outdir"
 
+# if debug is enabled
 if [ ! "${APP_DEBUG:-false}" = "true" ]
 then
   php artisan config:cache

docker-compose.deployment.yml

@@ -0,0 +1,95 @@
+version: '3.4'
+
+x-env: &x-env
+  APP_ENV: production
+  BROADCAST_DRIVER: redis
+  CACHE_DRIVER: redis
+  DB_HOST: 172.17.0.1
+  ES_HOST: elasticsearch:9200
+  ES_SCORES_HOST: elasticsearch:9200
+  GITHUB_TOKEN: "${GITHUB_TOKEN}"
+  NOTIFICATION_REDIS_HOST: 172.17.0.1
+  REDIS_HOST: 172.17.0.1
+  CACHE_REDIS_HOST: 172.17.0.1
+  SESSION_DRIVER: redis
+  NOTIFICATION_ENDPOINT: ws://localhost:2345
+
+x-web: &x-web
+  image: pppy/osu-web:latest
+  build:
+    context: .
+    dockerfile: Dockerfile.deployment
+    args:
+      APP_URL: http://localhost:${NGINX_PORT:-8000}
+      DOCS_URL: https://docs.ppy.sh
+      PAYMENT_SANDBOX: "true"
+      SHOPIFY_DOMAIN: notarealdomainortld
+      SHOPIFY_STOREFRONT_TOKEN: notreal
+      GIT_SHA: latest
+  environment:
+    <<: *x-env
+  env_file: .env
+
+services:
+  nginx-unit:
+    <<: *x-web
+    volumes:
+      - ./storage:/app/storage
+      - ./public/uploads:/app/public/uploads
+      - ./public/uploads-avatar:/app/public/uploads-avatar
+    ports:
+      - "${NGINX_PORT:-8000}:8000"
+    command: ['nginx']
+
+  job:
+    <<: *x-web
+    command: ['artisan', 'queue:work', '--queue=beatmap_default,beatmap_high,default,notification,store-notifications', '--tries=3', '--timeout=1000']
+
+  schedule:
+    <<: *x-web
+    command: ['schedule']
+
+  notification-server:
+    image: pppy/osu-notification-server
+    depends_on:
+      - redis
+      - db
+    volumes:
+      - .env:/app/.env
+      - ./storage/oauth-public.key:/app/oauth-public.key
+    ports:
+      - 2345:2345
+    environment:
+      <<: *x-env
+    env_file: .env
+
+  db:
+    image: mysql:8.0
+    volumes:
+      - database:/var/lib/mysql
+      - ./bin/db_setup.sh:/docker-entrypoint-initdb.d/db_setup.sh
+      - ./docker/development/db_user.sql:/docker-entrypoint-initdb.d/db_user.sql
+    environment:
+      <<: *x-env
+      MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
+      MYSQL_ONETIME_PASSWORD: "yes"
+    ports:
+      - "${MYSQL_EXTERNAL_PORT:-3306}:3306"
+    command: --default-authentication-plugin=mysql_native_password
+
+  redis:
+    image: redis:latest
+
+  elasticsearch:
+    # Version must be kept up to date with library defined in: composer.json
+    image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.8.12
+    volumes:
+      - elasticsearch:/usr/share/elasticsearch/data
+    environment:
+      action.auto_create_index: "false"
+      discovery.type: single-node
+      cluster.routing.allocation.disk.threshold_enabled: "false"
+
+volumes:
+  database:
+  elasticsearch:

docker/deployment/entrypoint.sh

@@ -10,6 +10,8 @@ _nginx() {
   # exit on config update fail
   set -e
 
+  echo "starting nginx unit"
+
   # start NGINX Unit
   touch /home/osuweb/unit/unit.pid
   exec unitd \
@@ -20,25 +22,58 @@ _nginx() {
     --no-daemon &
   PID=$!
 
+  echo "clearing laravel cache"
+
+  /app/artisan config:cache
+  /app/artisan route:cache
+
   # wait for NGINX Unit control socket to be open
-  inotifywait -e create,open /home/osuweb/unit
+  if [ ! -d /home/osuweb/unit/control.unit.sock ]; then
+    echo "awaiting for nginx unit socket to be open"
+    inotifywait -e create,open /home/osuweb/unit -t 2 || true
+  fi
 
+  echo "updating nginx unit config"
   # update config
   curl \
+    -s \
     -X PUT \
     --data-binary @/app/docker/deployment/nginx-unit.json \
     --unix-socket /home/osuweb/unit/control.unit.sock \
-    http://localhost/config
+    http://localhost/config 2>&1 > /dev/null
 
+  echo "applying env vars"
   # update environment variables
   jq -n env | curl \
+    -s \
     -X PUT \
     --data-binary @- \
     --unix-socket /home/osuweb/unit/control.unit.sock \
-    http://localhost/config/applications/laravel/environment
+    http://localhost/config/applications/laravel/environment 2>&1 > /dev/null
 
+  echo "spawning config reloader"
+  /app/docker/deployment/entrypoint.sh config_reloader &
+  CONFIG_RELOADER_PID=$!
+
+  echo "init done"
   # keep running this script until NGINX Unit ends
   wait $PID
+
+  echo "nginx exit $?"
+
+  # kill child processes (config reloader)
+  pkill -TERM -P $$
+}
+
+_config_reloader() {
+  while true; do
+    echo "Awaiting config update on .env..."
+    inotifywait -e modify .env
+
+    echo "Reloading config..."
+    /app/artisan config:cache
+    /app/artisan route:cache
+  done
 }
 
 _php() {
@@ -59,5 +94,6 @@ case "$command" in
     nginx) _nginx;;
     php) _php;;
     schedule) _schedule;;
+    config_reloader) _config_reloader;;
     *) exec "$command" "$@";;
 esac

docker/deployment/sudoers

@@ -0,0 +1 @@
+osuweb  ALL=(ALL) NOPASSWD: /app/docker/deployment/symlink-k8s-env.sh

docker/deployment/symlink-k8s-env.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+set +e
+
+ln -s /var/env-secret/.env /app/.env

osu-web-chart/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

osu-web-chart/Chart.yaml

@@ -0,0 +1,24 @@
+apiVersion: v2
+name: osu-web-chart
+description: production osu!web deployment in Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: 0.1.0

osu-web-chart/templates/NOTES.txt

@@ -0,0 +1,22 @@
+1. Get the application URL by running these commands:
+{{- if .Values.ingress.enabled }}
+{{- range $host := .Values.ingress.hosts }}
+  {{- range .paths }}
+  http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }}
+  {{- end }}
+{{- end }}
+{{- else if contains "NodePort" .Values.service.type }}
+  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "osu-web-chart.fullname" . }})
+  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
+  echo http://$NODE_IP:$NODE_PORT
+{{- else if contains "LoadBalancer" .Values.service.type }}
+     NOTE: It may take a few minutes for the LoadBalancer IP to be available.
+           You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "osu-web-chart.fullname" . }}'
+  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "osu-web-chart.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
+  echo http://$SERVICE_IP:{{ .Values.service.port }}
+{{- else if contains "ClusterIP" .Values.service.type }}
+  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "osu-web-chart.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
+  export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
+  echo "Visit http://127.0.0.1:8080 to use your application"
+  kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
+{{- end }}

osu-web-chart/templates/_helpers.tpl

@@ -0,0 +1,62 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "osu-web-chart.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "osu-web-chart.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "osu-web-chart.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "osu-web-chart.labels" -}}
+helm.sh/chart: {{ include "osu-web-chart.chart" . }}
+{{ include "osu-web-chart.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "osu-web-chart.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "osu-web-chart.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "osu-web-chart.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "osu-web-chart.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}