From c0a0fa15b2b0af2128ede26e23cc40b0c1a56ff3 Mon Sep 17 00:00:00 2001 From: saurabh3460 Date: Wed, 13 Dec 2023 19:57:16 +0530 Subject: [PATCH 1/4] update grafana.ini with psql creds --- monitoring/chart-values/prometheus-values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monitoring/chart-values/prometheus-values.yaml b/monitoring/chart-values/prometheus-values.yaml index fbab9cc..b9b8346 100644 --- a/monitoring/chart-values/prometheus-values.yaml +++ b/monitoring/chart-values/prometheus-values.yaml @@ -82,8 +82,8 @@ grafana: grafana.ini: database: type: postgres - host: postgres - user: postgres + host: grafanapsql.cn9m6m4s8zo0.us-west-2.rds.amazonaws.com + user: grafana password: postgres enabled: true env: From 52dc343cb4e10d92007e7b35a9237d632f696fce Mon Sep 17 00:00:00 2001 From: saurabh3460 Date: Wed, 13 Dec 2023 20:09:52 +0530 Subject: [PATCH 2/4] update create-grafana-database with psql endpoint --- monitoring/grafana-postgres/job.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monitoring/grafana-postgres/job.yaml b/monitoring/grafana-postgres/job.yaml index 2b6a763..ddedbe8 100644 --- a/monitoring/grafana-postgres/job.yaml +++ b/monitoring/grafana-postgres/job.yaml @@ -11,5 +11,5 @@ spec: containers: - name: create-grafana-database image: postgres:16.1-bullseye # Use the same PostgreSQL image - command: ["/bin/bash", "-c", "PGPASSWORD=postgres psql -h postgres -U postgres -c 'CREATE DATABASE grafana;'"] + command: ["/bin/bash", "-c", "PGPASSWORD=postgres psql -h grafanapsql.cn9m6m4s8zo0.us-west-2.rds.amazonaws.com -U grafana -d postgres -c 'CREATE DATABASE grafana;'"] restartPolicy: Never From 3a3e774bdddc2dc30ec4391faa454f723c8f8df1 Mon Sep 17 00:00:00 2001 From: saurabh3460 Date: Wed, 13 Dec 2023 20:16:10 +0530 Subject: [PATCH 3/4] add rds psql create and destroy scripts --- infra/scripts/dbs/rds/psql/create.sh | 43 +++++++++++++++++++++++++++ infra/scripts/dbs/rds/psql/destroy.sh | 7 +++++ 2 files changed, 50 insertions(+) create mode 100755 infra/scripts/dbs/rds/psql/create.sh create mode 100755 infra/scripts/dbs/rds/psql/destroy.sh diff --git a/infra/scripts/dbs/rds/psql/create.sh b/infra/scripts/dbs/rds/psql/create.sh new file mode 100755 index 0000000..0e4a5ef --- /dev/null +++ b/infra/scripts/dbs/rds/psql/create.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +GIT_TLD=`git rev-parse --show-toplevel` +source ${GIT_TLD}/infra/scripts/dbs/rds/common.sh + +echo "\nAdd security psql security group rules" +aws ec2 authorize-security-group-ingress \ + --group-id ${RDS_VPC_SECURITY_GROUP_ID}\ + --protocol tcp \ + --port 5432 \ + --source-group ${CLUSTER_SG} \ + --region ${AWS_REGION} \ + --no-cli-pager + +## Create subnet group +echo "\nCreate PSQL DB subnet group..." +aws rds create-db-subnet-group --cli-input-json "{\"DBSubnetGroupName\":\"grafana-psql-subnet-group\",\"DBSubnetGroupDescription\":\"grafana psql subnet group\",\"SubnetIds\":$SUBNET}" --region ${AWS_REGION} --no-cli-pager + +######## Create psql db +echo "\nCreate PSQL DB..." +aws rds create-db-instance \ + --db-name grafanapsql \ + --db-instance-identifier grafanapsql \ + --allocated-storage 10 \ + --db-instance-class db.t3.micro \ + --engine postgres \ + --engine-version "16.1" \ + --master-username grafana \ + --master-user-password postgres \ + --no-publicly-accessible \ + --vpc-security-group-ids ${RDS_VPC_SECURITY_GROUP_ID} \ + --db-subnet-group-name "grafana-psql-subnet-group" \ + --backup-retention-period 0 \ + --region ${AWS_REGION} \ + --tags Key=name,Value=grafanapsql \ + --port 5432 \ + --no-cli-pager + +echo "\n Wait for PSQL DB..." +aws rds wait db-instance-available --db-instance-identifier grafanapsql --region ${AWS_REGION} + +## GET DB endpoint +export PSQL_HOST=$(aws rds describe-db-instances --db-instance-identifier grafanapsql --region ${AWS_REGION} --query 'DBInstances[*].Endpoint.Address' --output text) diff --git a/infra/scripts/dbs/rds/psql/destroy.sh b/infra/scripts/dbs/rds/psql/destroy.sh new file mode 100755 index 0000000..bca0beb --- /dev/null +++ b/infra/scripts/dbs/rds/psql/destroy.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +AWS_REGION=us-west-2 + +aws rds delete-db-instance --db-instance-identifier grafanapsql --skip-final-snapshot --region ${AWS_REGION} --no-cli-pager +aws rds wait db-instance-deleted --db-instance-identifier grafanapsql --region ${AWS_REGION} --no-cli-pager +aws rds delete-db-subnet-group --db-subnet-group-name grafana-psql-subnet-group --region ${AWS_REGION} --no-cli-pager From 297a0869b302632dc4837b14d48e80ca6e8b1c68 Mon Sep 17 00:00:00 2001 From: saurabh3460 Date: Wed, 13 Dec 2023 20:17:56 +0530 Subject: [PATCH 4/4] makefile: add setup and destroy rds psql steps --- makefile | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/makefile b/makefile index 8996798..a19b96d 100644 --- a/makefile +++ b/makefile @@ -15,7 +15,7 @@ help: @echo " Clenaup cluster via: make cleanup-cluster" @echo " Clenaup all via: make cleanup" -setup: setup-cluster setup-cluster-autoscaler setup-istio setup-psql setup-observability setup-dbs-rds setup-rabbitmq-operator setup-app setup-gateway setup-keda setup-loadgen +setup: setup-cluster setup-cluster-autoscaler setup-istio setup-rds-psql setup-observability setup-dbs-rds setup-rabbitmq-operator setup-app setup-gateway setup-keda setup-loadgen cleanup: destroy-istio-gateway destroy-dbs-rds cleanup-cluster destroy-loadgen @@ -90,10 +90,9 @@ setup-loadgen: kubectl create ns loadgen --dry-run=client -o yaml | kubectl apply -f - kubectl apply -f scenarios/load-gen/load.yaml -setup-psql: +setup-rds-psql: + ./infra/scripts/dbs/rds/psql/create.sh kubectl create ns monitoring --dry-run=client -o yaml | kubectl apply -f - - kubectl apply -f monitoring/grafana-postgres/statefulset.yaml - kubectl wait --for=condition=ready pod -l app=postgresql --timeout=300s -n monitoring kubectl apply -f monitoring/grafana-postgres/job.yaml destroy-db-rds-mysql: @@ -108,11 +107,14 @@ destroy-db-rds-sg: destroy-istio-gateway: helm uninstall istio-ingressgateway -n istio-system -destroy-dbs-rds: destroy-db-rds-mysql destroy-db-rds-documentdb destroy-db-rds-sg +destroy-dbs-rds: destroy-db-rds-mysql destroy-rds-psql destroy-db-rds-sg destroy-loadgen: kubectl delete -f scenarios/load-gen/load.yaml +destroy-rds-psql: + ./infra/scripts/dbs/rds/psql/destroy.sh + cleanup-cluster: eksctl delete cluster --region=us-east-1 --name=prod-eks-cluster --wait aws iam delete-policy --policy-arn arn:aws:iam::813864300626:policy/k8s-asg-policy