diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml new file mode 100644 index 0000000..3633716 --- /dev/null +++ b/.github/workflows/integration-test.yml @@ -0,0 +1,65 @@ +name: Integration Test + +on: + pull_request: + +jobs: + + k8s: + runs-on: ubuntu-latest + strategy: + matrix: + cluster: + - elasticsearch + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Kind + uses: helm/kind-action@v1.10.0 + with: + install_only: true + + - name: Start Kind + run: | + kind create cluster --config integration-test/cluster.yaml + kubectl config set-context --current --namespace=openedx-harmony + sudo echo "127.0.0.1 harmony.test" | sudo tee -a /etc/hosts + + - name: Helm dependency add + run: | + helm dependency list charts/harmony-chart 2> /dev/null | tail +2 | awk '{ print "helm" " repo add " $1 " " $3 }' | while read cmd; do $cmd || true; done + kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.10.1/cert-manager.crds.yaml --namespace=harmony + + - name: Helm dependency build + run: | + helm dependency update charts/harmony-chart + helm dependency build charts/harmony-chart + + - name: Helm install + run: | + helm install harmony --namespace openedx-harmony --create-namespace -f integration-test/${{matrix.cluster}}/values.yaml charts/harmony-chart + kubectl rollout status deployment --timeout 300s + + - name: Healthcheck + run: | + curl http://harmony.test/cluster-echo-test + + - name: Boostrap cluster + run: | + bash integration-test/${{matrix.cluster}}/cluster.sh + + - name: setup python + uses: actions/setup-python@v5 + with: + python-version: 3.12 + + - name: Install openedx + run: | + export INSTALLATIONS=$(ls -d integration-test/${{matrix.cluster}}/*/) + export CI_ROOT=$(pwd) + for installation in $(ls -d integration-test/${{matrix.cluster}}/*/) + do + bash integration-test/environment.sh $installation + done diff --git a/.gitignore b/.gitignore index 50e4b93..908a29c 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,5 @@ infra-*/secrets.auto.tfvars .terraform *secrets.auto.tfvars my-notes +**/env/ +**/venv/ diff --git a/integration-test/cluster.yaml b/integration-test/cluster.yaml new file mode 100644 index 0000000..8d6f0fc --- /dev/null +++ b/integration-test/cluster.yaml @@ -0,0 +1,20 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane + kubeadmConfigPatches: + - | + kind: InitConfiguration + nodeRegistration: + kubeletExtraArgs: + node-labels: "ingress-ready=true" + extraPortMappings: + - containerPort: 80 + hostPort: 80 + listenAddress: 127.0.0.1 # omit for 0.0.0.0 + protocol: TCP + - containerPort: 443 + hostPort: 443 + listenAddress: 127.0.0.1 # omit for 0.0.0.0 + protocol: TCP + diff --git a/integration-test/elasticsearch/cluster.sh b/integration-test/elasticsearch/cluster.sh new file mode 100644 index 0000000..e69de29 diff --git a/integration-test/elasticsearch/openedx-01/config.yml b/integration-test/elasticsearch/openedx-01/config.yml new file mode 100644 index 0000000..6e0439e --- /dev/null +++ b/integration-test/elasticsearch/openedx-01/config.yml @@ -0,0 +1,80 @@ +CMS_OAUTH2_SECRET: kWYPh15ecORZ75TvEqtzfghk +DISCOVERY_INDEX_OVERRIDES: + course_discovery.apps.course_metadata.search_indexes.documents.course: openedx-01-owrj-course + course_discovery.apps.course_metadata.search_indexes.documents.course_run: openedx-01-owrj-course_run + course_discovery.apps.course_metadata.search_indexes.documents.learner_pathway: openedx-01-owrj-learner_pathway + course_discovery.apps.course_metadata.search_indexes.documents.person: openedx-01-owrj-person + course_discovery.apps.course_metadata.search_indexes.documents.program: openedx-01-owrj-program +ELASTICSEARCH_HOST: harmony-search-cluster.openedx-harmony.svc.cluster.local +ELASTICSEARCH_SCHEME: https +ENABLE_HTTPS: true +ENABLE_WEB_PROXY: false +ID: K3vsruy7oOdTvOcKmhiCybNz +JWT_RSA_PRIVATE_KEY: '-----BEGIN RSA PRIVATE KEY----- + + MIIEowIBAAKCAQEA0IiZ4AR3Iz3PPzdjvCwG+sQT2Fqiq5/Z+SBudvJfhwoMKj6R + + KNdmUcqsF+XotuQ6RRWaqmLv1INdlBjWlrhg7YdcJvJstIXE5Hs22M9R6Lth0jyB + + 159SEATsy9HuZBd10M7gG61LVoeamesn9U5Dd/0Fqfnb6PT2blYQ+SX24RYWV00K + + +y1NujnK3NpVeEZGBNUk3xIxLT+W9EYarnGB1PdxQq+IQ56TTNDBWsuliTUkiXZN + + CdDckefirDcUSbSh8VPlxuWNdIPe7CEPslUru9V93Vcs0NPIbXUSpEb5X5bq6gz9 + + ZfSn0bW0O1WwDR6GSiU9+KbdHxukAXGAqXIyiwIDAQABAoIBABEyoCS/ntxbAxtZ + + Y+dyEWtJ6MG9i8/aoYfbBbEtQPU/6LaX2AcwaHb+Ch5NxC5HzonBrofhxpoAnCLp + + aifI8e7KQdkmbLz15pcvQDE2O9uVOeE0Q0MMl8shai6kVA7mI73CP0JDvZmtGkdv + + rikcDhK4XRNiZfaliSiaqXtasaxorILp6xXbWpFwpqiIhtvNgJjbgcKm05L4Ia0e + + YWZ7LKfrM201oX0FyfgYxKk0lP6ZU74i961gBN1OLleW+5ikf6JXgJcRDmPg9+4E + + JQTnj7EzlYg9+l39lltw/6yZaLRPU/o3GiXuLQWxJ7oYxaOFfWc00IjDWuXaCcni + + Ftlv5CUCgYEA1TKaPR2/GE8EdgRZZ/nMPWgi+fptwGQfI4BtQkxZMOEQiVgjDdp/ + + t0l1dV3+HuG0VHyqfgI3OuO0SoIajZMaQEK4+DKQAIjTP5GR+vL+PltItZEYPl0y + + ZULhJY5h6Rn5OtTaE/khQ4kGyu8VAIBhqSPYjIhxGpyLLKfys+APLQUCgYEA+mZJ + + WpjZTfSqwKXMargsgn8uwXF2WgND9b/tqUkFAiwjkGvmXhpOzRUvq1mumFYphscs + + 5mdk+13WNRBmiHvJeRR9H3WcbSZTnVNfJ2QgY8b4uV7L8JetZs/jeF2X9Gkzi4Xa + + UGs1N/DUGsZD1xPnTHCxGnt678oAbjIyLxVndk8CgYAON0uVm7AsXMh9XhjgGAYI + + ajM62/oJhCOXaG7gptqjqb5Cij/8In+1pw5P/y58vKwbnhx4PomUQa658Sb4HcD4 + + W/qdgjGb94Ouj3ReeWl+mvOkN8KYFfrQSb9dk7rvoi2+QIcyIcCYEkEDsyGBYQ4i + + CyPbaxWvopSdK63BVMVuSQKBgQDY8P3krEC5NZ0qw3rTizWk6iPMHU14txOOOvbh + + kA3yz+MQISu5o/lZ3V0ri8uk2CGqT64rALGQxr/rwYXUAcuQR5wQQYmBV49h4vWx + + 3yHg0GZ11BfFHuRxCvP9OLmKXAr4esOmoPu+t7xMZjIoiWn/5DLMyua8EbtFvd+U + + w2qcKQKBgBmTe8NzvMxKstgDcElCnpqFv1e2JVLofXIsrW+07r8qhPywfXIiloR9 + + JRaR1CjsA4ZsU5n+2fISx8RvZqnrYvsSLFrkW9U6iSkJJwSR0gTJb3sNaxpfv1uT + + /GdzMnP1gM/V41wm28/LP6s9Z95tiwXjkHEBhIjYGbWi6DRz22jd + + -----END RSA PRIVATE KEY-----' +K8S_HARMONY_NAMESPACE: openedx-harmony +K8S_HARMONY_SEARCH_CLUSTER_HTTP_AUTH: openedx-01:gXkYL5RNzzP3Zc6BHL1oJrKy +K8S_HARMONY_SEARCH_CLUSTER_INDEX_PREFIX: openedx-01-owrj- +K8S_NAMESPACE: openedx-01 +LMS_HOST: local.openedx.io +MYSQL_ROOT_PASSWORD: c7SVvBJl +OPENEDX_MYSQL_PASSWORD: M4CBFYO9 +OPENEDX_SECRET_KEY: dskQ9UXPKcsSi2molXmggxID +PLUGINS: +- indigo +- k8s_harmony +- mfe +PLUGIN_INDEXES: +- https://overhang.io/tutor/main +RUN_ELASTICSEARCH: false diff --git a/integration-test/elasticsearch/openedx-01/post-installation.sh b/integration-test/elasticsearch/openedx-01/post-installation.sh new file mode 100644 index 0000000..5b23663 --- /dev/null +++ b/integration-test/elasticsearch/openedx-01/post-installation.sh @@ -0,0 +1,2 @@ +# Run any arbitrary commands necessary to verify the installation is working +echo "Make sure to change this script to verify your installation is tested correctly" diff --git a/integration-test/elasticsearch/openedx-01/pre-init.sh b/integration-test/elasticsearch/openedx-01/pre-init.sh new file mode 100644 index 0000000..0cbe464 --- /dev/null +++ b/integration-test/elasticsearch/openedx-01/pre-init.sh @@ -0,0 +1,8 @@ +export HARMONY_NAMESPACE=$(tutor config printvalue K8S_HARMONY_NAMESPACE) +export INSTANCE_NAMESPACE=$(tutor config printvalue K8S_NAMESPACE) +tutor harmony create-elasticsearch-user + +kubectl get secret "search-cluster-certificates-elasticsearch" -n "$HARMONY_NAMESPACE" -o "yaml" | \ + grep -v '^\s*namespace:\s' | \ + sed s/-elasticsearch//g |\ + kubectl apply -n "$INSTANCE_NAMESPACE" --force -f - diff --git a/integration-test/elasticsearch/openedx-01/requirements.txt b/integration-test/elasticsearch/openedx-01/requirements.txt new file mode 100644 index 0000000..c77e537 --- /dev/null +++ b/integration-test/elasticsearch/openedx-01/requirements.txt @@ -0,0 +1,2 @@ +../../../tutor-contrib-harmony-plugin +tutor[full]<19 diff --git a/integration-test/elasticsearch/values.yaml b/integration-test/elasticsearch/values.yaml new file mode 100644 index 0000000..9307116 --- /dev/null +++ b/integration-test/elasticsearch/values.yaml @@ -0,0 +1,50 @@ +# Disable HTTPS cert provisioning for testing with minikube +cert-manager: + enabled: false + +ingress-nginx: + # Use ingress-nginx as a default controller. + enabled: true + controller: + # All these needed for local development + service: + type: NodePort + hostPort: + enabled: true + publishService: + enabled: false + extraArgs: + publish-status-address: localhost + +clusterDomain: harmony.test + +elasticsearch: + enabled: true + + # TODO: move this to a separate PR + # Permit co-located instances for solitary minikube virtual machines. + antiAffinity: "soft" + + volumeClaimTemplate: + resources: + requests: + storage: 8Gi + replicas: 1 + +opensearch: + enabled: false + + # Permit co-located instances for solitary minikube virtual machines. + antiAffinity: "soft" + + persistence: + size: 8Gi + +prometheusstack: + enabled: true + +k8sdashboard: + enabled: false + +openfaas: + enabled: false diff --git a/integration-test/environment.sh b/integration-test/environment.sh new file mode 100644 index 0000000..6d82e90 --- /dev/null +++ b/integration-test/environment.sh @@ -0,0 +1,29 @@ +cd $1 + +# Setup environment +export TUTOR_PLUGINS_ROOT=$(pwd)/plugins/ export TUTOR_ROOT=$(pwd) +python3 -m venv venv && source venv/bin/activate +pip install -r $TUTOR_ROOT/requirements.txt + +# Start/init installation +tutor config save +tutor plugins enable k8s_harmony + +tutor k8s start + +echo "########################################" +echo "# Pre init #" +echo "########################################" + +bash ./pre-init.sh + +tutor k8s init + +echo "########################################" +echo "# Post init #" +echo "########################################" +bash ./post-installation.sh + +# Go back +deactivate +cd $CI_ROOT diff --git a/integration-test/template/cluster.sh b/integration-test/template/cluster.sh new file mode 100644 index 0000000..e69de29 diff --git a/integration-test/template/openedx/config.yml b/integration-test/template/openedx/config.yml new file mode 100644 index 0000000..eb37775 --- /dev/null +++ b/integration-test/template/openedx/config.yml @@ -0,0 +1,9 @@ +K8S_HARMONY_NAMESPACE: openedx-harmony +K8S_NAMESPACE: openedx-01 +LMS_HOST: local.openedx.io +PLUGINS: +- indigo +- k8s_harmony +- mfe +PLUGIN_INDEXES: +- https://overhang.io/tutor/main diff --git a/integration-test/template/openedx/post-installation.sh b/integration-test/template/openedx/post-installation.sh new file mode 100644 index 0000000..5b23663 --- /dev/null +++ b/integration-test/template/openedx/post-installation.sh @@ -0,0 +1,2 @@ +# Run any arbitrary commands necessary to verify the installation is working +echo "Make sure to change this script to verify your installation is tested correctly" diff --git a/integration-test/template/openedx/pre-init.sh b/integration-test/template/openedx/pre-init.sh new file mode 100644 index 0000000..c10e7c6 --- /dev/null +++ b/integration-test/template/openedx/pre-init.sh @@ -0,0 +1,2 @@ +# Run any arbitrary commands necessary to setup the installation +echo "Make sure to change this script to setup your installation" diff --git a/integration-test/template/openedx/requirements.txt b/integration-test/template/openedx/requirements.txt new file mode 100644 index 0000000..c77e537 --- /dev/null +++ b/integration-test/template/openedx/requirements.txt @@ -0,0 +1,2 @@ +../../../tutor-contrib-harmony-plugin +tutor[full]<19 diff --git a/integration-test/template/values.yaml b/integration-test/template/values.yaml new file mode 100644 index 0000000..97f4bbb --- /dev/null +++ b/integration-test/template/values.yaml @@ -0,0 +1,50 @@ +clusterDomain: harmony.test + +# Disable HTTPS cert provisioning for testing with minikube +cert-manager: + enabled: false + +ingress-nginx: + # Use ingress-nginx as a default controller. + enabled: true + controller: + # All these needed for local development + service: + type: NodePort + hostPort: + enabled: true + publishService: + enabled: false + extraArgs: + publish-status-address: localhost + +elasticsearch: + enabled: false + + # TODO: move this to a separate PR + # Permit co-located instances for solitary minikube virtual machines. + antiAffinity: "soft" + + volumeClaimTemplate: + resources: + requests: + storage: 8Gi + replicas: 1 + +opensearch: + enabled: false + + # Permit co-located instances for solitary minikube virtual machines. + antiAffinity: "soft" + + persistence: + size: 8Gi + +prometheusstack: + enabled: false + +k8sdashboard: + enabled: false + +openfaas: + enabled: false diff --git a/temp.txt b/temp.txt new file mode 100644 index 0000000..e895f94 --- /dev/null +++ b/temp.txt @@ -0,0 +1,34 @@ +# ingress nginx +# cert manager +# elastichsearch - openedx +# opensearch - openedx +# prometheusstack + + + +# steps +# 1. Create a key for a CA +openssl genrsa -des3 -out harmony.key 2048 +# 2. Generate a root certificate +openssl req -x509 -new -nodes -key harmony.key -sha256 -days 1825 -out harmony.pem +# 3. Install Certificate +sudo cp harmony.pem /usr/local/share/ca-certificates/harmony.crt +sudo update-ca-certificates +# 4. Creating CA-signed certificates for dev site +openssl genrsa -out harmony.test.key 2048 +# 4.1 Create CSR: +openssl req -new -key harmony.test.key -out harmony.test.csr +# 4.2 Sign certifcate +openssl x509 -req -in harmony.test.csr -CA harmony.pem -CAkey harmony.key -CAcreateserial -out harmony.test.crt -days 825 -sha256 -extfile harmony.test.ext + +# Setup certificates +sudo mkdir -p /etc/systemd/resolved.conf.d +sudo tee /etc/systemd/resolved.conf.d/minikube.conf << EOF +[Resolve] +DNS=$(minikube ip) +Domains=~harmony.test +EOF +sudo systemctl restart systemd-resolved + +# Enable certificate on minikube +cp harmony.pem $HOME/.minikube/certs/harmony.pem \ No newline at end of file diff --git a/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/plugin.py b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/plugin.py index d85ab36..1de3152 100644 --- a/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/plugin.py +++ b/tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/plugin.py @@ -37,7 +37,7 @@ def is_plugin_loaded(plugin_name: str) -> bool: # instance. "ADDITIONAL_INGRESS_HOST_LIST": [], "ENABLE_SHARED_SEARCH_CLUSTER": False, - "DEPLOYMENT_REVISION_HISTORY_LIMIT": 10, + "DEPLOYMENT_REVISION_HISTORY_LIMIT": 10 }, "overrides": { # Don't use Caddy as a per-instance external web proxy, but do still use it diff --git a/values-minikube.yaml b/values-minikube.yaml index 8509816..26b25f9 100644 --- a/values-minikube.yaml +++ b/values-minikube.yaml @@ -1,11 +1,11 @@ # Disable HTTPS cert provisioning for testing with minikube cert-manager: - enabled: false + enabled: true -clusterDomain: "example.local" +clusterDomain: harmony.test elasticsearch: - enabled: false + enabled: true # TODO: move this to a separate PR # Permit co-located instances for solitary minikube virtual machines. @@ -26,7 +26,7 @@ opensearch: size: 8Gi prometheusstack: - enabled: false + enabled: true k8sdashboard: enabled: false