From 0538606d6ec913553c9216879bb01e4068d15668 Mon Sep 17 00:00:00 2001 From: Hussein Martinez Date: Fri, 9 Aug 2024 06:17:03 +0300 Subject: [PATCH 1/8] chore: removed deprecated files --- .github/workflows/deploy-do-web.yml | 24 ------------------------ kb-deployment.yaml | 22 ---------------------- kb-service.yaml | 17 ----------------- 3 files changed, 63 deletions(-) delete mode 100644 .github/workflows/deploy-do-web.yml delete mode 100644 kb-deployment.yaml delete mode 100644 kb-service.yaml diff --git a/.github/workflows/deploy-do-web.yml b/.github/workflows/deploy-do-web.yml deleted file mode 100644 index 8735046a..00000000 --- a/.github/workflows/deploy-do-web.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: Deploy DO Web - -on: workflow_dispatch - -env: - FILENAME: Dockerfile.web - IMAGE_NAME: gitcoinco/indexer-web - IMAGE_TAG: ${{ github.sha }} - -jobs: - build: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Build the Docker image - run: docker build -f "$FILENAME" -t "$IMAGE_NAME:$IMAGE_TAG" . # build the Docker image using envs defined above - - # login to dockerhub then push the image to the dockerhub repo - - name: Push Docker image - run: |- - echo ${{secrets.DOCKERHUB_PASS}} | docker login -u ${{secrets.DOCKERHUB_USER}} --password-stdin - docker push "$IMAGE_NAME:$IMAGE_TAG" diff --git a/kb-deployment.yaml b/kb-deployment.yaml deleted file mode 100644 index 997b4423..00000000 --- a/kb-deployment.yaml +++ /dev/null @@ -1,22 +0,0 @@ -kind: Deployment -apiVersion: apps/v1 -metadata: - name: indexer-web -spec: - replicas: 2 - selector: - matchLabels: - app: indexer-web - template: - metadata: - labels: - app: indexer-web - spec: - containers: - - name: indexer-web - image: "gitcoinco/indexer-web:latest" - ports: - - containerPort: 80 - envFrom: - - configMapRef: - name: indexer-web-config diff --git a/kb-service.yaml b/kb-service.yaml deleted file mode 100644 index 6cde5a7c..00000000 --- a/kb-service.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: load-balancer - labels: - app: indexer-web -spec: - type: LoadBalancer - ports: - - name: http - port: 80 - targetPort: 8080 - - name: https - port: 443 - targetPort: 8080 - selector: - app: indexer-web From b733b4a9692ffec4cd8e65a71b471ceb35cd8c27 Mon Sep 17 00:00:00 2001 From: Hussein Martinez Date: Fri, 9 Aug 2024 06:18:02 +0300 Subject: [PATCH 2/8] feat: added kubernetes config files --- k8s/cert-issuer.yaml | 15 ++++++++++++ k8s/cert-manager-values.yaml | 3 +++ k8s/nginx-values.yaml | 19 ++++++++++++++++ k8s/web-deployment.yaml | 44 ++++++++++++++++++++++++++++++++++++ k8s/web-hpa-autoscaling.yaml | 20 ++++++++++++++++ k8s/web-ingress.yaml | 24 ++++++++++++++++++++ 6 files changed, 125 insertions(+) create mode 100644 k8s/cert-issuer.yaml create mode 100644 k8s/cert-manager-values.yaml create mode 100644 k8s/nginx-values.yaml create mode 100644 k8s/web-deployment.yaml create mode 100644 k8s/web-hpa-autoscaling.yaml create mode 100644 k8s/web-ingress.yaml diff --git a/k8s/cert-issuer.yaml b/k8s/cert-issuer.yaml new file mode 100644 index 00000000..01df8e9b --- /dev/null +++ b/k8s/cert-issuer.yaml @@ -0,0 +1,15 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: letsencrypt-nginx + namespace: backend-web +spec: + acme: + server: https://acme-v02.api.letsencrypt.org/directory + email: engineering@gitcoin.co + privateKeySecretRef: + name: letsencrypt-nginx-private-key + solvers: + - http01: + ingress: + class: nginx diff --git a/k8s/cert-manager-values.yaml b/k8s/cert-manager-values.yaml new file mode 100644 index 00000000..a3b577f2 --- /dev/null +++ b/k8s/cert-manager-values.yaml @@ -0,0 +1,3 @@ +installCRDs: false +prometheus: + enables: false diff --git a/k8s/nginx-values.yaml b/k8s/nginx-values.yaml new file mode 100644 index 00000000..d85fa28e --- /dev/null +++ b/k8s/nginx-values.yaml @@ -0,0 +1,19 @@ +## Ref: https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/nginx-configuration/index.md + +controller: + replicaCount: 2 + resources: + requests: + cpu: 100m + memory: 90Mi + service: + annotations: + service.beta.kubernetes.io/do-loadbalancer-enable-proxy-protocol: "true" + service.beta.kubernetes.io/do-loadbalancer-tls-passthrough: "true" + config: + use-proxy-protocol: "true" + keep-alive-requests: "10000" + upstream-keep-alive: "1000" + worker-processes: "auto" + max-worker-connections: "65535" + use-gzip: "true" diff --git a/k8s/web-deployment.yaml b/k8s/web-deployment.yaml new file mode 100644 index 00000000..57414077 --- /dev/null +++ b/k8s/web-deployment.yaml @@ -0,0 +1,44 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: indexer-web + namespace: backend-web +spec: + replicas: 1 + selector: + matchLabels: + app: indexer-web + template: + metadata: + labels: + app: indexer-web + spec: + containers: + - name: backend-web + image: "gitcoinco/indexer-web:latest" + ports: + - name: http + containerPort: 8080 + envFrom: + - configMapRef: + name: indexer-web-envs + resources: + requests: + memory: "1Gi" # Request 1GB of memory + cpu: "500m" # Request 0.5 vCPU + limits: + memory: "2Gi" # Limit to 2GB of memory + cpu: "1000m" # Limit to 1 vCPU +--- +apiVersion: v1 +kind: Service +metadata: + name: indexer-web + namespace: backend-web +spec: + ports: + - name: http + port: 80 + targetPort: 8080 + selector: + app: indexer-web diff --git a/k8s/web-hpa-autoscaling.yaml b/k8s/web-hpa-autoscaling.yaml new file mode 100644 index 00000000..baf29066 --- /dev/null +++ b/k8s/web-hpa-autoscaling.yaml @@ -0,0 +1,20 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: indexer-web-hpa + namespace: backend-web + +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: indexer-web + minReplicas: 1 + maxReplicas: 3 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 # percentage diff --git a/k8s/web-ingress.yaml b/k8s/web-ingress.yaml new file mode 100644 index 00000000..25ca356f --- /dev/null +++ b/k8s/web-ingress.yaml @@ -0,0 +1,24 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ingress-web + namespace: backend-web + annotations: #### commented before cert-manager was installed + cert-manager.io/issuer: letsencrypt-nginx +spec: + tls: #### commented before cert-manager was installed + - hosts: + - do.gitcoin-indexer.xyz + secretName: letsencrypt-nginx + rules: + - host: do.gitcoin-indexer.xyz + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: indexer-web + port: + number: 8080 + ingressClassName: nginx From 6b203923dfb10072f7bcc79d6e1d40b876ed0ee8 Mon Sep 17 00:00:00 2001 From: Hussein Martinez Date: Fri, 9 Aug 2024 06:18:21 +0300 Subject: [PATCH 3/8] feat: updated docker readme --- READ_DOCKER_KB.md | 226 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 185 insertions(+), 41 deletions(-) diff --git a/READ_DOCKER_KB.md b/READ_DOCKER_KB.md index 2d0e1484..79837cf4 100644 --- a/READ_DOCKER_KB.md +++ b/READ_DOCKER_KB.md @@ -1,70 +1,214 @@ -We followed this tutorial: https://www.digitalocean.com/community/tutorials/deploying-an-express-application-on-a-kubernetes-cluster +## TLDR -- To build the docker images: +### Manual WebServer Deployment to DigitalOcean: - ``` - docker build -f Dockerfile.index -t indexer/index . - docker build -f Dockerfile.web -t indexer/web . - ``` +``` +# build docker image +docker build -f Dockerfile.web -t gitcoinco/indexer-index:latest . -- To push the images to docker hub: +# login into docker (engineering account - 1Password) +docker login - ``` - docker tag indexer/index:latest gitcoinco/indexer-index:latest - docker push gitcoinco/indexer-index:latest +# push the image to DockerHub +docker push gitcoinco/indexer-web:latest - docker tag indexer/web:latest gitcoinco/indexer-web:latest - docker push gitcoinco/indexer-web:latest - ``` +# if you didn't do it before, generate a token https://cloud.digitalocean.com/account/api/tokens +# and authenticate with it in doctl +doctl auth init -t {token} -### SOLUTION FOR ENVIRONMENT VARIABLES (kubernetes doesn't support .env file): +# save locally kubernetes cluster config to use kubectl +doctl kubernetes cluster kubeconfig save k8s-indexer-web -- Created a config map: +# create config map with the env file (production envs) +# to update it, delete it and create a new one with that name +kubectl create configmap indexer-web-envs --from-env-file=.env - ``` - kubectl create configmap indexer-web-config --from-env-file=.env - ``` +# deploy to kubernetes cluster +kubectl apply -f ./k8s/web-deployment.yaml +``` -- Reference the config map in kb-deployment.yml with: +### Force restart and pull of latest image version - ``` - envFrom: - - configMapRef: - name: indexer-web-config - ``` +``` +kubectl rollout restart deployment indexer-web +``` -- NOT DONE - For sensitive data we could have creaded a secret with: +### Getting data and logs from the cluster examples - ``` - kubectl create secret generic indexer-web-secrets --from-env-file=.env - ``` +- `kubectl get nodes` + +- `kubectl get deployment indexer-web` + +- `kubectl get hpa` + +- `kubectl get pods` + +- `kubectl get svc` + +- `kubectl logs {podname}` + +- `helm repo list` + +- `kubectl get all -n ingress-nginx` + +- `kubectl get all -n backend-web` + +- `kubectl get secrets -n backend-web` -- NOT DONE - And referenced the secret in kb-deployment.yml with: +- `kubectl get crds` + +- `kubectl get crds | grep cert-manager` + + - `kubectl get order -n backend-web` + + - `kubectl get certificaterequest -n backend-web` + + - `kubectl get certificate -n backend-web` + +# How did we do it? + +We followed this tutorials as reference: + +- https://www.digitalocean.com/community/tutorials/deploying-an-express-application-on-a-kubernetes-cluster +- https://www.digitalocean.com/community/tech-talks/securing-your-kubernetes-ingress-with-lets-encrypt + +## Requirements + +- docker +- kubectl (https://kubernetes.io/docs/tasks/tools) +- doctl (https://docs.digitalocean.com/reference/doctl/how-to/install) +- helm (https://helm.sh/docs/intro/install) + +## Building Docker image and pushing to DockerHub + +This step is needed everytime we make changes in the repo + +### Build docker image of Dockerfile.web + +``` +docker build -f Dockerfile.web -t gitcoinco/indexer-index:latest . +``` + +### Login into Dockerhub with Engineering account + +``` +docker login +``` + +### Push the image to DockerHub + +``` +docker push gitcoinco/indexer-web:latest +``` + +## Kubernetes Cluster Setup + +### Create the cluster in DigitalOcean with `doctl` + +Run: + +``` +doctl kubernetes cluster create k8s-indexer-web \ + --region nyc1 \ + --tag indexer-do,indexer-do-web \ + --node-pool "name=pool-indexer-web;size=s-2vcpu-4gb;count=2;auto-scale=true;min-nodes=2;max-nodes=5;tag=indexer-do;tag=indexer-do-web" +``` + +This commands creates a kubernetes cluster in DigitalOcean with this specs: + +- name: `k8s-indexer-web` +- region: New York - 1 +- tags: `indexer-do` & `indexer-do-web` +- a pool of 2 nodes initially with auto-scaling set to minimum 2 nodes and maximum 5 +- each node runs in a vm machine type `s-2vcpu-4gb` +- each node with tag values `indexer-do` & `indexer-do-web` + +### Save the created cluster config in your local kubectl + +After the creation of the cluster has finished, run: + +``` +doctl kubernetes cluster kubeconfig save k8s-indexer-web +``` + +### Create a config map with env values (kubernetes doesn't support .env file): + +- Create a config map (with .env having production values): ``` - envFrom: - - secretRef: - name: indexer-web-secrets + kubectl create configmap indexer-web-envs --from-env-file=.env ``` -### Logs +- `indexer-web-envs` is referenced in `k8s/web-deployment.yaml` and **needed for deployment** -Get the current instances of server: +### Deploy web server ``` -kubectl get pods +# Create namespace backend-web +kubectl create ns backend-web + +# Deploy web server +kubectl apply -f ./k8s/web-deployment.yaml + +# Deploy horizontal pod auto-scaling +kubectl apply -f ./k8s/web-hpa-autoscaling.yaml ``` -Possible response: +### SSL, Ingress-nginx, LetsEncrypt + +#### Install ingress-nginx ``` -NAME READY STATUS RESTARTS AGE -indexer-web-554574455d-lg98l 1/1 Running 0 41m -indexer-web-554574455d-t4tgj 1/1 Running 0 41m +helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx + +helm repo update + +helm install ingress-nginx ingress-nginx/ingress-nginx --namespace ingress-nginx --create-namespace +``` + +#### Apply web-ingress config + +web-ingress.yml should have `annotations` and `tls` sections commented at this point + +``` +kubectl apply -f k8s/web-ingress.yaml +``` + +#### Install cert-manager + +``` +kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.15.2/cert-manager.crds.yaml + +helm repo add jetstack https://charts.jetstack.io --force-update + +helm repo update jetstack + +helm install cert-manager jetstack/cert-manager --version v1.15.2 -f ./k8s/cert-manager-values.yaml --namespace cert-manager --create-namespace +``` + +#### Apply cert-issuer config + +``` +kubectl apply -f ./k8s/cert-issuer.yaml +``` + +#### Uncomment annotations and tls sections in web-ingress config and apply the changes + +``` +kubectl apply -f ./k8s/web-ingress.yaml +``` + +#### Wait until the certificate has been emited (READY = True) + +``` +✗ kubectl get certificate -n backend-web + +NAME READY SECRET AGE +letsencrypt-nginx True letsencrypt-nginx 80m ``` -Then to see the logs of the first instance: +#### Upgrade ingress-nginx with nginx-values config file ``` -kubectl logs indexer-web-554574455d-lg98l +helm upgrade ingress-nginx ingress-nginx/ingress-nginx -f ./k8s/nginx-values.yaml -n ingress-nginx ``` From 1fd5ba46b8f0983c1d574189cefe514a6ec5b0ee Mon Sep 17 00:00:00 2001 From: Hussein Martinez Date: Fri, 9 Aug 2024 06:19:40 +0300 Subject: [PATCH 4/8] chore: renamed READ_DOCKER_KB -> README_KUBERNETES --- READ_DOCKER_KB.md => README_KUBERNETES.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename READ_DOCKER_KB.md => README_KUBERNETES.md (100%) diff --git a/READ_DOCKER_KB.md b/README_KUBERNETES.md similarity index 100% rename from READ_DOCKER_KB.md rename to README_KUBERNETES.md From b1393c12efecdf03c4bbbef4813a1006470bca78 Mon Sep 17 00:00:00 2001 From: Hussein Martinez Date: Mon, 12 Aug 2024 10:10:30 +0300 Subject: [PATCH 5/8] chore: update README_KUBERNETES --- README_KUBERNETES.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README_KUBERNETES.md b/README_KUBERNETES.md index 79837cf4..9b70f55a 100644 --- a/README_KUBERNETES.md +++ b/README_KUBERNETES.md @@ -43,11 +43,9 @@ kubectl rollout restart deployment indexer-web - `kubectl get pods` -- `kubectl get svc` - -- `kubectl logs {podname}` + - `kubectl logs {podname}` -- `helm repo list` +- `kubectl get svc` - `kubectl get all -n ingress-nginx` @@ -57,7 +55,7 @@ kubectl rollout restart deployment indexer-web - `kubectl get crds` -- `kubectl get crds | grep cert-manager` + - `kubectl get crds | grep cert-manager` - `kubectl get order -n backend-web` @@ -65,6 +63,8 @@ kubectl rollout restart deployment indexer-web - `kubectl get certificate -n backend-web` +- `helm repo list` + # How did we do it? We followed this tutorials as reference: @@ -111,6 +111,7 @@ Run: doctl kubernetes cluster create k8s-indexer-web \ --region nyc1 \ --tag indexer-do,indexer-do-web \ + --project indexer-do --node-pool "name=pool-indexer-web;size=s-2vcpu-4gb;count=2;auto-scale=true;min-nodes=2;max-nodes=5;tag=indexer-do;tag=indexer-do-web" ``` From 5ae04858f5f759d9dc3ec3f53c5080b4f4dff03f Mon Sep 17 00:00:00 2001 From: Hussein Martinez Date: Tue, 13 Aug 2024 12:22:28 +0300 Subject: [PATCH 6/8] chore: update README_KUBERNETES --- README_KUBERNETES.md | 40 +++++++++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/README_KUBERNETES.md b/README_KUBERNETES.md index 9b70f55a..32c8c708 100644 --- a/README_KUBERNETES.md +++ b/README_KUBERNETES.md @@ -30,20 +30,21 @@ kubectl apply -f ./k8s/web-deployment.yaml ### Force restart and pull of latest image version ``` -kubectl rollout restart deployment indexer-web +kubectl rollout restart deployment indexer-web -n backend-web ``` ### Getting data and logs from the cluster examples - `kubectl get nodes` -- `kubectl get deployment indexer-web` +- `kubectl get deployment indexer-web -n backend-web` -- `kubectl get hpa` +- `kubectl get hpa -n backend-web` -- `kubectl get pods` +- `kubectl get pods -A` +- `kubectl get pods -n backend-web` - - `kubectl logs {podname}` + - `kubectl logs {podname} -n backend-web` - `kubectl get svc` @@ -111,7 +112,6 @@ Run: doctl kubernetes cluster create k8s-indexer-web \ --region nyc1 \ --tag indexer-do,indexer-do-web \ - --project indexer-do --node-pool "name=pool-indexer-web;size=s-2vcpu-4gb;count=2;auto-scale=true;min-nodes=2;max-nodes=5;tag=indexer-do;tag=indexer-do-web" ``` @@ -137,7 +137,13 @@ doctl kubernetes cluster kubeconfig save k8s-indexer-web - Create a config map (with .env having production values): ``` - kubectl create configmap indexer-web-envs --from-env-file=.env + # Check nodes + kubectl get nodes + + # Create namespace backend-web + kubectl create ns backend-web + + kubectl create configmap indexer-web-envs --from-env-file=.env -n backend-web ``` - `indexer-web-envs` is referenced in `k8s/web-deployment.yaml` and **needed for deployment** @@ -145,14 +151,14 @@ doctl kubernetes cluster kubeconfig save k8s-indexer-web ### Deploy web server ``` -# Create namespace backend-web -kubectl create ns backend-web - # Deploy web server kubectl apply -f ./k8s/web-deployment.yaml # Deploy horizontal pod auto-scaling kubectl apply -f ./k8s/web-hpa-autoscaling.yaml + +# Check pods +kubectl get pods -n backend-web ``` ### SSL, Ingress-nginx, LetsEncrypt @@ -160,13 +166,25 @@ kubectl apply -f ./k8s/web-hpa-autoscaling.yaml #### Install ingress-nginx ``` +# Create namespace ingress-nginx +kubectl create ns ingress-nginx + helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx helm repo update -helm install ingress-nginx ingress-nginx/ingress-nginx --namespace ingress-nginx --create-namespace +helm install ingress-nginx ingress-nginx/ingress-nginx --namespace ingress-nginx + +# upgrade with nginx-values with commented service and config +helm upgrade ingress-nginx ingress-nginx/ingress-nginx -f ./k8s/nginx-values.yaml -n ingress-nginx + +# Check resources in namespace ingress-nginx +# Check until the load balancer has external ip +kubectl get all -n ingress-nginx ``` +Point the domain to the load balancer from DO UI -> networking -> domains -> create A record pointing to the load balancer + #### Apply web-ingress config web-ingress.yml should have `annotations` and `tls` sections commented at this point From ec3f219af92ede63d471a8fc66c1c3de2d918d65 Mon Sep 17 00:00:00 2001 From: Hussein Martinez Date: Tue, 13 Aug 2024 12:23:15 +0300 Subject: [PATCH 7/8] chore: update nginx-values --- k8s/nginx-values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/k8s/nginx-values.yaml b/k8s/nginx-values.yaml index d85fa28e..f60231b3 100644 --- a/k8s/nginx-values.yaml +++ b/k8s/nginx-values.yaml @@ -6,11 +6,11 @@ controller: requests: cpu: 100m memory: 90Mi - service: + service: # commented before certificate annotations: service.beta.kubernetes.io/do-loadbalancer-enable-proxy-protocol: "true" service.beta.kubernetes.io/do-loadbalancer-tls-passthrough: "true" - config: + config: # commented before certificate use-proxy-protocol: "true" keep-alive-requests: "10000" upstream-keep-alive: "1000" From 45480997e6742d06c39c53e8d283719b8f51a503 Mon Sep 17 00:00:00 2001 From: Hussein Martinez Date: Tue, 13 Aug 2024 12:23:31 +0300 Subject: [PATCH 8/8] fix: typo enabled --- k8s/cert-manager-values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/cert-manager-values.yaml b/k8s/cert-manager-values.yaml index a3b577f2..b316a527 100644 --- a/k8s/cert-manager-values.yaml +++ b/k8s/cert-manager-values.yaml @@ -1,3 +1,3 @@ installCRDs: false prometheus: - enables: false + enabled: false