DM-48385: Add times-square database migration job

applications/times-square/Chart.yaml

@@ -8,7 +8,7 @@ sources:
 type: application
 
 # The default version tag of the times-square docker image
-appVersion: "0.13.0"
+appVersion: "0.14.0"
 
 dependencies:
   - name: redis

applications/times-square/README.md

@@ -33,6 +33,7 @@ An API service for managing and rendering parameterized Jupyter notebooks.
 | config.profile | string | `"production"` | Run profile: "production" or "development" |
 | config.redisCacheUrl | string | Points to embedded Redis | URL for Redis html / noteburst job cache database |
 | config.redisQueueUrl | string | Points to embedded Redis | URL for Redis arq queue database |
+| config.updateSchema | bool | false to disable schema upgrades | Whether to run the database migration job |
 | config.worker.enableLivenessCheck | bool | `true` | Enable liveness checks for the arq queue |
 | fullnameOverride | string | `""` | Override the full name for resources (includes the release name) |
 | global.baseUrl | string | Set by times-square Argo CD Application | Base URL for the environment |

applications/times-square/templates/job-schema-update.yaml

@@ -0,0 +1,150 @@
+{{- if .Values.config.updateSchema -}}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: "times-square-schema-update"
+  annotations:
+  annotations:
+    helm.sh/hook: "pre-install,pre-upgrade"
+    helm.sh/hook-delete-policy: "hook-succeeded"
+    helm.sh/hook-weight: "1"
+  labels:
+    {{- include "times-square.labels" . | nindent 4 }}
+spec:
+  template:
+    metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "times-square.selectorLabels" . | nindent 8 }}
+        app.kubernetes.io/component: "schema-update"
+        times-square-redis-client: "true"
+    spec:
+      {{- if .Values.cloudsql.enabled }}
+      serviceAccountName: "times-square"
+      {{- else }}
+      automountServiceAccountToken: false
+      {{- end }}
+      containers:
+        {{- if .Values.cloudsql.enabled }}
+        - name: "cloud-sql-proxy"
+          # Running the sidecar as normal causes it to keep running and thus
+          # the Pod never exits, the Job never finishes, and the hook blocks
+          # the sync. Have the main pod signal the sidecar by writing to a
+          # file on a shared emptyDir file system, and use a simple watcher
+          # loop in shell in the sidecar container to terminate the proxy when
+          # the main container finishes.
+          #
+          # Based on https://stackoverflow.com/questions/41679364/
+          command:
+            - "/bin/sh"
+            - "-c"
+          args:
+            - |
+              /cloud_sql_proxy -ip_address_types=PRIVATE -log_debug_stdout=true -structured_logs=true -instances={{ required "cloudsql.instanceConnectionName must be specified" .Values.cloudsql.instanceConnectionName }}=tcp:5432 &
+              PID=$!
+              while true; do
+                if [[ -f "/lifecycle/main-terminated" ]]; then
+                  kill $PID
+                  exit 0
+                fi
+                sleep 1
+              done
+          image: "{{ .Values.cloudsql.image.repository }}:{{ .Values.cloudsql.image.tag }}{{ .Values.cloudsql.image.schemaUpdateTagSuffix }}"
+          imagePullPolicy: {{ .Values.cloudsql.image.pullPolicy | quote }}
+          {{- with .Values.cloudsql.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - "all"
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+            runAsUser: 65532
+            runAsGroup: 65532
+          volumeMounts:
+            - name: "lifecycle"
+              mountPath: "/lifecycle"
+        {{- end }}
+        - name: "times-square"
+          command:
+            - "/bin/sh"
+            - "-c"
+            - |
+              times-square update-db-schema
+              touch /lifecycle/main-terminated
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy | quote }}
+          {{- with .Values.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - "all"
+            readOnlyRootFilesystem: true
+          envFrom:
+            - configMapRef:
+                name: {{ include "times-square.fullname" . }}
+          env:
+            - name: "TS_GAFAELFAWR_TOKEN"
+              valueFrom:
+                secretKeyRef:
+                  name: {{ template "times-square.fullname" . }}-gafaelfawr-token
+                  key: "token"
+            - name: "TS_DATABASE_PASSWORD"
+              valueFrom:
+                secretKeyRef:
+                  name: {{ template "times-square.fullname" . }}-secret
+                  key: "TS_DATABASE_PASSWORD"
+            - name: "TS_GITHUB_WEBHOOK_SECRET"
+              valueFrom:
+                secretKeyRef:
+                  name: {{ template "times-square.fullname" . }}-secret
+                  key: "TS_GITHUB_WEBHOOK_SECRET"
+            - name: "TS_GITHUB_WEBHOOK_SECRET"
+              valueFrom:
+                secretKeyRef:
+                  name: {{ template "times-square.fullname" . }}-secret
+                  key: "TS_GITHUB_WEBHOOK_SECRET"
+            - name: "TS_GITHUB_APP_PRIVATE_KEY"
+              valueFrom:
+                secretKeyRef:
+                  name: {{ template "times-square.fullname" . }}-secret
+                  key: "TS_GITHUB_APP_PRIVATE_KEY"
+            - name: "TS_SLACK_WEBHOOK_URL"
+              valueFrom:
+                secretKeyRef:
+                  name: {{ template "times-square.fullname" . }}-secret
+                  key: "TS_SLACK_WEBHOOK_URL"
+          volumeMounts:
+            - name: "lifecycle"
+              mountPath: "/lifecycle"
+      restartPolicy: "Never"
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 1000
+        runAsGroup: 1000
+      volumes:
+        - name: "lifecycle"
+          emptyDir: {}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+{{- end }}

applications/times-square/values.yaml

@@ -119,6 +119,10 @@ config:
   # @default -- None, must be set
   databaseUrl: ""
 
+  # -- Whether to run the database migration job
+  # @default -- false to disable schema upgrades
+  updateSchema: false
+
   # -- URL for Redis html / noteburst job cache database
   # @default -- Points to embedded Redis
   redisCacheUrl: "redis://times-square-redis:6379/0"

docs/applications/times-square/db-migrations.rst

@@ -0,0 +1,33 @@
+###################################
+Managing database schema migrations
+###################################
+
+Times Square use Alembic_ to manage its SQLAlchemy-based relational database.
+When a new version of Times Square is released that includes database schema changes, you will need to run the procedure described here.
+Times Square checks if the database schema is consistent with the codebase when it starts up, making it impossible to run the application if the schema is out of date.
+
+Upgrading the database schema
+=============================
+
+1. If Times Square is running in a production environment, schedule and announce downtime for Times Square.
+
+2. Set ``config.updateSchema`` to true in the :file:`values-{environment}.yaml` file for the ``times-square`` application for this environment:
+
+   .. code-block:: yaml
+      :caption: applications/times-square/values-{environment}.yaml
+
+      config:
+        updateSchema: true
+
+   Push these changes to GitHub on the branch that is deployed to the environment.
+
+3. Stop the Times Square deployments by deleting the Kubernetes deployment for both the API service and the worker service in Argo CD.
+
+4. Sync the Times Square application in Argo CD. This takes the follow actions:
+
+   1. The ``times-square-schema-update`` Kubernetes Job runs as a Helm pre-upgrade hook.
+   2. The Times Square API and worker deployments are recreated.
+
+5. In Phalanx, reset the ``config.updateSchema`` value to false in the :file:`values-{environment}.yaml` file for the ``times-square`` application for this environment.
+
+.. _Alembic: https://alembic.sqlalchemy.org/en/latest/

docs/applications/times-square/index.rst

@@ -15,4 +15,5 @@ Guides
 .. toctree::
    :maxdepth: 1
 
+   db-migrations
    values