run (part of) e2e tests on kind cluster

.dockerignore

@@ -5,3 +5,4 @@ static/system
 _data
 docker_dev/
 docker_prod/
+e2e

.gitignore

@@ -41,3 +41,6 @@ _opam/
 /docker_prod/config/prod.yml
 /_data
 .env*
+e2e/_test*
+e2e/_bin
+e2e/.kubeconfig

README.md

@@ -171,3 +171,24 @@ MIT, except for the following files:
     > LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
     > OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
     > SOFTWARE.
+
+## How to run Waq and Mastodon on kind
+
+Install docker beforehand. Then:
+```
+cd e2e
+make start-tmole
+make create-cluster
+make start-waq
+make start-mastodon
+make waq-port-forward &
+make mastodon-port-forward &
+cat _test_waq # Access this domain for Waq.
+cat _test_mastodon # Access this domain for Mastodon.
+```
+
+When shutting down:
+```
+make clean-cluster
+make stop-tmole
+```

e2e/Makefile

@@ -0,0 +1,103 @@
+KIND_VERSION:=0.20.0
+KUBERNETES_VERSION:=1.28.0
+HELM_VERSION:=3.14.0
+WD:=$(dir $(lastword $(MAKEFILE_LIST)))
+BIN:=$(WD)/_bin
+KIND:=$(BIN)/kind-$(KIND_VERSION)
+KIND_CLUSTER_NAME=waq-test-cluster
+KUBECTL:=$(BIN)/kubectl-$(KUBERNETES_VERSION)
+HELM:=$(BIN)/helm-$(HELM_VERSION)
+IMAGE=waq:dev
+
+.PHONY: run-test
+run-test:
+	#env KUBECTL=$(KUBECTL) MANIFESTS=$(WD)/manifests OCAMLRUNPARAM=b E2E_TEST_WAQ_SERVER_NAME=https://$(shell cat $(WD)/_test_waq_domain) dune exec src/main.exe
+	env KUBECTL=$(KUBECTL) MANIFESTS=$(WD)/manifests OCAMLRUNPARAM=b E2E_TEST_WAQ_SERVER_NAME=http://localhost:58080 dune exec src/main.exe
+
+.PHONY: test
+test:
+	$(MAKE) start-waq
+	$(MAKE) run-test
+
+$(BIN):
+	mkdir -p $(BIN)
+
+$(KIND): | $(BIN)
+	wget -O $(KIND) https://github.com/kubernetes-sigs/kind/releases/download/v$(KIND_VERSION)/kind-linux-amd64
+	chmod a+x $(KIND)
+
+$(KUBECTL): | $(BIN)
+	wget -O $(KUBECTL) https://storage.googleapis.com/kubernetes-release/release/v$(KUBERNETES_VERSION)/bin/linux/amd64/kubectl
+	chmod a+x $(KUBECTL)
+
+$(HELM): | $(BIN)
+	curl -sSLf https://get.helm.sh/helm-v$(HELM_VERSION)-linux-amd64.tar.gz \
+		| tar xvz -C $(BIN) --strip-components 1 linux-amd64/helm
+	mv $(BIN)/helm $@
+
+KUBECONFIG := $(shell pwd)/.kubeconfig
+.PHONY: $(KUBECONFIG)
+$(KUBECONFIG): | $(KIND)
+	$(KIND) export kubeconfig --name $(KIND_CLUSTER_NAME) --kubeconfig=$@
+
+.PHONY: setup
+setup: $(KIND) $(KUBECTL) $(HELM)
+
+.PHONY: build-image
+build-image:
+	cd $(WD)/.. && docker build . -t $(IMAGE)
+
+.PHONY: clean-cluster
+clean-cluster:
+	$(KIND) delete cluster -n $(KIND_CLUSTER_NAME)
+
+.PHONY: start-tmole
+start-tmole:
+	systemctl --user reset-failed tmole-waq || true
+	systemctl --user reset-failed tmole-mastodon || true
+	systemd-run --user --unit=tmole-waq --working-directory=$(WD) ./launch-tmole.sh waq 58080
+	systemd-run --user --unit=tmole-mastodon --working-directory=$(WD) ./launch-tmole.sh mastodon 58081
+
+.PHONY: stop-tmole
+stop-tmole:
+	systemctl --user stop tmole-waq || true
+	rm -f _test_waq_domain
+	systemctl --user stop tmole-mastodon || true
+	rm -f _test_mastodon_domain
+
+.PHONY: create-cluster
+create-cluster:
+	$(MAKE) setup
+	$(MAKE) clean-cluster
+	$(KIND) create cluster -n $(KIND_CLUSTER_NAME) --image kindest/node:v$(KUBERNETES_VERSION)
+	$(MAKE) $(KUBECONFIG)
+	$(KUBECTL) create namespace e2e
+	KUBECONFIG=$(KUBECONFIG) $(HELM) install --namespace e2e --repo https://ushitora-anqou.github.io/mahout mahout mahout
+	$(KUBECTL) apply -f $(WD)/manifests/postgres.yaml
+	$(KUBECTL) apply -f $(WD)/manifests/elk.yaml
+
+.PHONY: start-mastodon
+start-mastodon:
+	cat $(WD)/manifests/mastodon.yaml | sed "s/E2E_TEST_MASTODON_SERVER_NAME/$(shell cat $(WD)/_test_mastodon_domain)/" | $(KUBECTL) apply -f -
+
+.PHONY: start-waq
+start-waq:
+	$(MAKE) build-image
+	$(KIND) load docker-image $(IMAGE) -n $(KIND_CLUSTER_NAME)
+	$(KUBECTL) delete -f $(WD)/manifests/waq.yaml || true
+	$(KUBECTL) delete -f $(WD)/manifests/waq-reset-db.yaml || true
+	cat $(WD)/manifests/waq.yaml | sed "s/E2E_TEST_WAQ_SERVER_NAME/$(shell cat $(WD)/_test_waq_domain)/" | $(KUBECTL) apply -f -
+	$(KUBECTL) apply -f $(WD)/manifests/waq-reset-db.yaml
+	$(KUBECTL) wait --for=condition=available deploy/waq-web -n e2e
+
+.PHONY: waq-port-forward
+waq-port-forward:
+	$(KUBECTL) port-forward -n e2e --address 0.0.0.0 svc/waq-web 58080:8000
+
+.PHONY: mastodon-port-forward
+mastodon-port-forward:
+	$(KUBECTL) port-forward -n e2e --address 0.0.0.0 svc/mastodon-gateway 58081:80
+
+.PHONY: elk-port-forward
+elk-port-forward:
+	$(KUBECTL) port-forward -n e2e --address 0.0.0.0 svc/elk 58082:5314

e2e/launch-tmole.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/bash -eu
+
+set -o pipefail
+
+NAME=$1
+PORT=$2
+
+PROG="
+cd /home/node
+npm i tunnelmole
+cat <<EOS > tmole.js
+const fs = require('node:fs');
+const tunnelmole = require('tunnelmole/cjs');
+(async () => {
+  const url = await tunnelmole({ port: ${PORT} });
+  const domain = new URL(url).hostname;
+  console.log('OUTPUT:'+domain);
+})()
+EOS
+node tmole.js"
+
+docker run --init --rm --add-host=localhost:host-gateway --entrypoint 'bash' node -c "${PROG}" | grep --line-buffered OUTPUT | stdbuf -oL cut -d ':' -f2 | tee _test_${NAME}_domain

e2e/manifests/elk.yaml

@@ -0,0 +1,34 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: elk
+  namespace: e2e
+spec:
+  selector:
+    app: elk
+  ports:
+    - port: 5314
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: elk
+  namespace: e2e
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: elk
+  template:
+    metadata:
+      labels:
+        app: elk
+    spec:
+      containers:
+        - name: elk
+          image: ghcr.io/elk-zone/elk:v0.12.1
+          ports:
+            - name: http
+              containerPort: 5314
+              protocol: TCP

e2e/manifests/mastodon.yaml

@@ -0,0 +1,82 @@
+---
+apiVersion: mahout.anqou.net/v1alpha1
+kind: Mastodon
+metadata:
+  name: mastodon
+  namespace: e2e
+spec:
+  serverName: "mastodon.test"
+  image: "ghcr.io/mastodon/mastodon:v4.2.0"
+  envFrom:
+    - secretRef:
+        name: mastodon-secret-env
+  gateway:
+    replicas: 1
+    image: "nginx:1.25.4"
+  sidekiq:
+    replicas: 1
+  streaming:
+    replicas: 1
+  web:
+    replicas: 1
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: redis
+  namespace: e2e
+  labels:
+    app: redis
+spec:
+  ports:
+    - port: 6379
+      name: redis
+  clusterIP: None
+  selector:
+    app: redis
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: redis
+  namespace: e2e
+spec:
+  selector:
+    matchLabels:
+      app: redis
+  serviceName: "redis"
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: redis
+    spec:
+      terminationGracePeriodSeconds: 10
+      containers:
+        - name: redis
+          image: redis:7.2.3-alpine
+          ports:
+            - containerPort: 6379
+              name: redis
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: mastodon-secret-env
+  namespace: e2e
+type: Opaque
+stringData:
+  LOCAL_DOMAIN: "E2E_TEST_MASTODON_SERVER_NAME"
+  REDIS_HOST: redis.e2e.svc
+  REDIS_PORT: "6379"
+  DB_HOST: postgres.e2e.svc
+  DB_USER: mastodon
+  DB_PASS: password
+  DB_NAME: mastodon_production
+  DB_PORT: "5432"
+  IP_RETENTION_PERIOD: "31556952"
+  SESSION_RETENTION_PERIOD: "31556952"
+  SECRET_KEY_BASE: 928dab5fdf3cfd4a16e89ab92e343d9d3d96e232ee77bf9d0a18ea510d4231f0fb1425c51f3ca8cfb8f031ca6b7bc25a04e2aa2230e11b614b8215974bff23b6
+  OTP_SECRET: a2679b530056aa9a5aa2c5956684bdab64480f69bd5252e5c9f5eb64c6ea1dbcece44195d7f9236f5bfccad421413fd606996ec53c082147303736921e4ad07b
+  VAPID_PRIVATE_KEY: YMFYOwrat5ZQxRQmXka0oaHr56TpctygcO7XtYfuwCA=
+  VAPID_PUBLIC_KEY: BNQoJF_o0Jk-soeZuqdGIx-8vjUfk6bH7ezFw3JtGl29iTUAz8OtjZl6wkb2Zz2I_ekokAk4lI-dyLiUpHLV6gA=

e2e/manifests/postgres.yaml

@@ -0,0 +1,82 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: postgres
+  namespace: e2e
+  labels:
+    app: postgres
+spec:
+  ports:
+    - port: 5432
+      name: postgres
+  clusterIP: None
+  selector:
+    app: postgres
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: postgres
+  namespace: e2e
+spec:
+  selector:
+    matchLabels:
+      app: postgres
+  serviceName: "postgres"
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: postgres
+    spec:
+      terminationGracePeriodSeconds: 10
+      containers:
+        - name: postgres
+          image: postgres
+          ports:
+            - containerPort: 5432
+              name: postgres
+          env:
+            - name: POSTGRES_HOST_AUTH_METHOD
+              value: trust
+          livenessProbe:
+            exec:
+              command:
+                - pg_isready
+                - -U
+                - postgres
+          readinessProbe:
+            exec:
+              command:
+                - pg_isready
+                - -U
+                - postgres
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: create-postgres-database
+  namespace: e2e
+spec:
+  template:
+    spec:
+      restartPolicy: Never
+      containers:
+        - name: postgres
+          image: postgres
+          command:
+            - bash
+            - -ce
+            - |
+              cat <<EOS | psql -U postgres -h postgres.e2e.svc
+                CREATE USER mastodon CREATEDB;
+                ALTER ROLE mastodon WITH PASSWORD 'password';
+              EOS
+              cat <<EOS | psql -U postgres -h postgres.e2e.svc
+                CREATE USER waq CREATEDB;
+                ALTER ROLE waq WITH PASSWORD 'password';
+              EOS
+              cat <<EOS | psql -U waq -h postgres.e2e.svc -d postgres
+                CREATE DATABASE waq_prod;
+              EOS

e2e/manifests/waq-reset-db.yaml

@@ -0,0 +1,21 @@
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: reset-waq-database
+  namespace: e2e
+spec:
+  template:
+    spec:
+      restartPolicy: Never
+      containers:
+        - name: reset-waq-database
+          image: waq:dev
+          command:
+            - bash
+            - -ce
+            - |
+              /waq/waq db:reset
+          envFrom:
+            - secretRef:
+                name: waq-secret-env