Enable Druid to use GCS Emulator for local setup & to run e2e tests (#…

…741)

* Enable Druid to use GCS Emulator for local setup & to run e2e tests

* Create gcp secret to use emulator for druid local setup

* Add documentation for usage of fake-gcs-server for e2e tests

* Add documentation to setup etcd-druid locally using fake-gcs-server

* Remove cluster.yaml file & instead embed the port-forward values in kind-up.sh

Makefile

@@ -141,6 +141,10 @@ ci-e2e-kind: $(GINKGO) $(YQ) $(KIND)
 ci-e2e-kind-azure: $(GINKGO)
 	@BUCKET_NAME=$(BUCKET_NAME) $(HACK_DIR)/ci-e2e-kind-azure.sh
 
+.PHONY: ci-e2e-kind-gcs
+ci-e2e-kind-gcs: $(GINKGO)
+	@BUCKET_NAME=$(BUCKET_NAME) $(HACK_DIR)/ci-e2e-kind-gcs.sh
+
 # Rules related to binary build, Docker image build and release
 # -------------------------------------------------------------------------
 # Build manager binary
@@ -172,7 +176,7 @@ docker-clean:
 
 # Rules for locale/remote environment
 # -------------------------------------------------------------------------
-kind-up kind-down ci-e2e-kind ci-e2e-kind-azure deploy-localstack deploy-azurite test-e2e deploy deploy-dev deploy-debug undeploy: export KUBECONFIG = $(KUBECONFIG_PATH)
+kind-up kind-down ci-e2e-kind ci-e2e-kind-azure ci-e2e-kind-gcs deploy-localstack deploy-fakegcs deploy-azurite test-e2e deploy deploy-dev deploy-debug undeploy: export KUBECONFIG = $(KUBECONFIG_PATH)
 
 .PHONY: kind-up
 kind-up: $(KIND)
@@ -218,4 +222,8 @@ deploy-localstack: $(KUBECTL)
 
 .PHONY: deploy-azurite
 deploy-azurite: $(KUBECTL)
-	./hack/deploy-azurite.sh
+	@$(HACK_DIR)/deploy-azurite.sh
+
+.PHONY: deploy-fakegcs
+deploy-fakegcs: $(KUBECTL)
+	@$(HACK_DIR)/deploy-fakegcs.sh

config/samples/druid_v1alpha1_etcd_fakegcs.yaml

@@ -0,0 +1,78 @@
+apiVersion: druid.gardener.cloud/v1alpha1
+kind: Etcd
+metadata:
+  name: etcd-test
+  labels:
+    app: etcd-statefulset
+    gardener.cloud/role: controlplane
+    role: test
+spec:
+  selector:
+    matchLabels:
+      app: etcd-statefulset
+      gardener.cloud/role: controlplane
+      role: test
+  annotations:
+    app: etcd-statefulset
+    gardener.cloud/role: controlplane
+    # networking.gardener.cloud/to-dns: allowed
+    # networking.gardener.cloud/to-private-networks: allowed
+    # networking.gardener.cloud/to-public-networks: allowed
+    role: test
+  labels:
+    app: etcd-statefulset
+    gardener.cloud/role: controlplane
+    # networking.gardener.cloud/to-dns: allowed
+    # networking.gardener.cloud/to-private-networks: allowed
+    # networking.gardener.cloud/to-public-networks: allowed
+    role: test
+  etcd:
+    metrics: basic
+    defragmentationSchedule: "0 */24 * * *"
+    resources:
+      limits: { cpu: 500m, memory: 1Gi }
+      requests: { cpu: 100m, memory: 200Mi }
+    clientPort: 2379
+    serverPort: 2380
+    quota: 8Gi
+    # heartbeatDuration: 10s
+  backup:
+    port: 8080
+    fullSnapshotSchedule: "0 */24 * * *"
+    resources:
+      limits: { cpu: 200m, memory: 1Gi }
+      requests: { cpu: 23m, memory: 128Mi }
+    garbageCollectionPolicy: Exponential
+    garbageCollectionPeriod: 43200s
+    deltaSnapshotPeriod: 300s
+    deltaSnapshotMemoryLimit: 1Gi
+    store:
+      container: etcd-bucket
+      prefix: etcd-test
+      provider: gcp
+      secretRef:
+        name: etcd-backup-gcp
+    compression:
+      enabled: false
+      policy: "gzip"
+    leaderElection:
+      reelectionPeriod: 5s
+      etcdConnectionTimeout: 5s
+
+  sharedConfig:
+    autoCompactionMode: periodic
+    autoCompactionRetention: "30m"
+  # schedulingConstraints:
+    # affinity: {}
+    # topologySpreadConstraints:
+    # - maxSkew: 1
+    #   topologyKey: topology.kubernetes.io/zone
+    #   whenUnsatisfiable: DoNotSchedule
+    #   labelSelector:
+    #     matchLabels:
+    #       app: etcd-statefulset
+
+  replicas: 3
+  # priorityClassName: priority-class-name
+  # storageClass: default
+  # storageCapacity: 10Gi

config/samples/etcd-secret-fakegcs.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+data:
+  bucketName: ZXRjZC1idWNrZXQ=
+  serviceaccount.json: ZHVtbXk= #dummy
+  storageAPIEndpoint: aHR0cDovL2Zha2UtZ2NzLmRlZmF1bHQ6ODAwMC9zdG9yYWdlL3YxLw== #http://fake-gcs.default:8000/storage/v1/
+  emulatorEnabled: dHJ1ZQ== #true
+kind: Secret
+metadata:
+  labels:
+    app: etcd-statefulset
+  name: etcd-backup-gcp
+type: Opaque

docs/deployment/getting-started-locally/getting-started-locally.md

@@ -8,13 +8,13 @@ Before we can setup `etcd-druid` and use it to provision `Etcd` clusters, we nee
 
 ## 01-Setting up KIND cluster
 
-`etcd-druid` uses [kind](https://kind.sigs.k8s.io/) as it's local Kubernetes engine. The local setup is configured for kind due to its convenience only. Any other Kubernetes setup would also work. 
+`etcd-druid` uses [kind](https://kind.sigs.k8s.io/) as it's local Kubernetes engine. The local setup is configured for kind due to its convenience only. Any other Kubernetes setup would also work.
 
 ```bash
 make kind-up
 ```
 
-This command sets up a new Kind cluster and stores the kubeconfig at `./hack/kind/kubeconfig`.  Additionally, this command also deploys a local container registry as a docker container. This ensures faster image push/pull times. The local registry can be accessed as `localhost:5001` for pushing and pulling images. 
+This command sets up a new Kind cluster and stores the kubeconfig at `./hack/kind/kubeconfig`.  Additionally, this command also deploys a local container registry as a docker container. This ensures faster image push/pull times. The local registry can be accessed as `localhost:5001` for pushing and pulling images.
 
 To target this newly created cluster, set the `KUBECONFIG` environment variable to the kubeconfig file.
 
@@ -70,7 +70,7 @@ make deploy-debug
 
 This is similar to `make deploy-dev` but additionally configures containers in pods for debugging as required for each container's runtime technology. The associated debugging ports are exposed and labelled so that they can be port-forwarded to the local machine. Skaffold disables automatic image rebuilding and syncing when using the `debug` mode as compared to `dev` mode.
 
-Go debugging uses [Delve](https://github.com/go-delve/delve). Please see the [skaffold debugging documentation](https://skaffold.dev/docs/workflows/debug/) how to setup your IDE accordingly. 
+Go debugging uses [Delve](https://github.com/go-delve/delve). Please see the [skaffold debugging documentation](https://skaffold.dev/docs/workflows/debug/) how to setup your IDE accordingly.
 
 !!! note
     Resuming or stopping only a single goroutine (Go Issue [25578](https://github.com/golang/go/issues/25578), [31132](https://github.com/golang/go/issues/31132)) is currently not supported, so the action will cause all the goroutines to get activated or paused.
@@ -84,12 +84,13 @@ This means that when a goroutine is paused on a breakpoint, then all the other g
 !!! info
     This section is ***Optional*** and is only meant to describe steps to deploy a local object store which can be used for testing and development. If you either do not wish to enable backups or you wish to use remote (infra-provider-specific) object store then this section can be skipped.
 
-An `Etcd` cluster provisioned via etcd-druid provides a capability to take regular delta and full snapshots which are stored in an object store. You can enable this functionality by ensuring that you fill in [spec.backup.store](https://github.com/gardener/etcd-druid/blob/master/config/samples/druid_v1alpha1_etcd.yaml#L49-L54) section of the `Etcd` CR. 
+An `Etcd` cluster provisioned via etcd-druid provides a capability to take regular delta and full snapshots which are stored in an object store. You can enable this functionality by ensuring that you fill in [spec.backup.store](https://github.com/gardener/etcd-druid/blob/master/config/samples/druid_v1alpha1_etcd.yaml#L49-L54) section of the `Etcd` CR.
 
 | Backup Store Variant          | Setup Guide                                                |
 | ----------------------------- | ---------------------------------------------------------- |
 | Azure Object Storage Emulator | [Manage Azurite](manage-azurite-emulator.md) (Steps 00-03) |
 | S3 Object Store Emulator      | [Manage LocalStack](manage-s3-emulator.md) (Steps 00-03)   |
+| GCS Object Store Emulator     | [Manage GCS Emulator](manage-gcs-emulator.md) (Steps 00-03)|
 
 ### Setting up Cloud Provider Object Store Secret
 
@@ -103,6 +104,7 @@ A Kubernetes [Secret](https://kubernetes.io/docs/concepts/configuration/secret/)
     * All the values in the data field of the secret YAML should in `base64` encoded format.
 
 To apply the secret run:
+
 ```bash
 kubectl apply -f <path/to/secret>
 ```
@@ -136,13 +138,15 @@ Brief explanation of the keys:
 ## 05-Applying Etcd CR
 
 Create the Etcd CR (Custom Resource) by applying the Etcd yaml to the cluster
+
 ```bash
 kubectl apply -f <path-to-etcd-cr-yaml>
 ```
 
 ## 06-Verify the Etcd Cluster
 
 To obtain information on the etcd cluster you can invoke the following command:
+
 ```bash
 kubectl get etcd -o=wide
 ```
@@ -151,7 +155,7 @@ We adhere to a naming convention for all resources that are provisioned for an `
 
 ### Verify Etcd Pods' Functionality
 
-`etcd-wrapper` uses a [distroless](https://github.com/GoogleContainerTools/distroless) image, which lacks a shell. To interact with etcd, use an [Ephemeral container](https://kubernetes.io/docs/concepts/workloads/pods/ephemeral-containers/) as a debug container. Refer to this [documentation](https://github.com/gardener/etcd-wrapper/blob/master/docs/deployment/ops.md#operations--debugging) for building and using an ephemeral container which gets attached to the `etcd-wrapper` pod. 
+`etcd-wrapper` uses a [distroless](https://github.com/GoogleContainerTools/distroless) image, which lacks a shell. To interact with etcd, use an [Ephemeral container](https://kubernetes.io/docs/concepts/workloads/pods/ephemeral-containers/) as a debug container. Refer to this [documentation](https://github.com/gardener/etcd-wrapper/blob/master/docs/deployment/ops.md#operations--debugging) for building and using an ephemeral container which gets attached to the `etcd-wrapper` pod.
 
 ```bash
 # Put a key-value pair into the etcd 
@@ -169,11 +173,13 @@ For a multi-node etcd cluster, insert the key-value pair using the `etcd` contai
 ## 08-Cleaning up the setup
 
 If you wish to only delete the `Etcd` cluster then you can use the following command:
+
 ```bash
 kubectl delete etcd <etcd-name>
 ```
 
-This will add the `deletionTimestamp` to the `Etcd` resource.  At the time the creation of the `Etcd` cluster, etcd-druid will add a finalizer to ensure that it cleans up all `Etcd` cluster resources before the CR is removed. 
+This will add the `deletionTimestamp` to the `Etcd` resource.  At the time the creation of the `Etcd` cluster, etcd-druid will add a finalizer to ensure that it cleans up all `Etcd` cluster resources before the CR is removed.
+
 ```yaml
   finalizers:
   - druid.gardener.cloud/etcd-druid
@@ -182,11 +188,13 @@ This will add the `deletionTimestamp` to the `Etcd` resource.  At the time the c
 etcd-druid will automatically pick up the deletion event and attempt clean up `Etcd` cluster resources. It will only remove the finaliser once all resources have been cleaned up.
 
 If you only wish to remove `etcd-druid` but retain the kind cluster then you can use the following make target:
+
 ```bash
 make undeploy
 ```
 
 If you wish to delete the kind cluster then you can use the following make target:
+
 ```bash
 make kind-down
 ```

docs/deployment/getting-started-locally/manage-azurite-emulator.md

@@ -50,8 +50,9 @@ kubectl apply -f config/samples/etcd-secret-azurite.yaml
 
 ## 04-Cleanup
 
-In addition to the kind cluster cleanup you should also unset the environment variable set in step-03 above.
+To clean the setup, unset the environment variable set in step-03 above and delete the Azurite deployment:
+
 ```bash
 unset AZURE_STORAGE_CONNECTION_STRING
+kubectl delete -f ./hack/e2e-test/infrastructure/azurite/azurite.yaml
 ```
-

docs/deployment/getting-started-locally/manage-gcs-emulator.md

@@ -0,0 +1,47 @@
+# Manage GCS Emulator
+
+This document is a step-by-step guide on how to configure, deploy and cleanup [GCS Emulator](https://github.com/fsouza/fake-gcs-server), within a [kind](https://kind.sigs.k8s.io/) cluster. GCS Emulator emulates Google Cloud Storage locally, which allows the `Etcd` cluster to interact with GCS. This setup is ideal for local development and testing.
+
+## 00-Prerequisites
+
+Ensure that you have setup the development environment as per the [documentation](../../development/prepare-dev-environment.md).
+
+> **Note:** It is assumed that you have already created kind cluster and the `KUBECONFIG` is pointing to this Kubernetes cluster.
+
+### Installing gsutil
+
+To interact with `GCS Emulator` you must also install the [gsutil](https://cloud.google.com/storage/docs/gsutil) utility. Follow the instructions [here](https://cloud.google.com/storage/docs/gsutil_install) to install `gsutil`.
+
+## 01-Deploy FakeGCS
+
+Deploy FakeGCS onto the Kubernetes cluster using the command below:
+
+```bash
+make deploy-fakegcs
+```
+
+## 02-Setup GCS Bucket
+
+To create a GCS bucket for `Etcd` backup purposes, execute the following command:
+
+```bash
+gsutil -o "Credentials:gs_json_host=127.0.0.1" -o "Credentials:gs_json_port=4443" -o "Boto:https_validate_certificates=False" mb "gs://etcd-bucket"
+```
+
+## 03-Configure Secret
+
+Connection details for a GCS Object Store are put into a Kubernetes [Secret](https://kubernetes.io/docs/concepts/configuration/secret/). Apply the Kubernetes Secret manifest through:
+
+```bash
+kubectl apply -f config/samples/etcd-secret-fakegcs.yaml
+```
+
+> **Note:** The secret created should be referred to in the `Etcd` CR in `spec.backup.store.secretRef`.
+
+## 04-Cleanup
+
+To clean the setup, delete the FakeGCS deployment:
+
+```bash
+kubectl delete -f ./hack/e2e-test/infrastructure/fake-gcs-server/fake-gcs-server.yaml
+```

docs/deployment/getting-started-locally/manage-s3-emulator.md

@@ -39,11 +39,13 @@ export AWS_DEFAULT_REGION=us-east-2
 ```
 
 Create a S3 bucket using the following command:
+
 ```bash
 aws s3api create-bucket --bucket <bucket-name> --region <region> --create-bucket-configuration LocationConstraint=<region> --acl private
 ```
 
 To verify if the bucket has been created, you can use the following command:
+
 ```bash
 aws s3api head-bucket --bucket <bucket-name>
 ```
@@ -60,7 +62,9 @@ kubectl apply -f config/samples/etcd-secret-localstack.yaml
 
 ## 04-Cleanup
 
-In addition to the kind cluster cleanup you should also unset the environment variable set in step-03 above.
+To clean the setup,, unset the environment variable set in step-03 above and delete the LocalStack deployment:
+
 ```bash
 unset AWS_ENDPOINT_URL_S3 AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_DEFAULT_REGION
-```
+kubectl delete -f ./hack/e2e-test/infrastructure/localstack/localstack.yaml
+```

docs/development/running-e2e-tests.md

@@ -25,7 +25,7 @@ as described in this document.
 It's expected that especially the `deploy` step is run against a Kubernetes cluster which doesn't contain an Druid deployment or any left-overs like `druid.gardener.cloud` CRDs.
 The `deploy` step will likely fail in such scenarios.
 
-> Tip: Create a fresh [KinD](https://kind.sigs.k8s.io/) cluster or a similar one with a small footprint before executing the tests. 
+> Tip: Create a fresh [KinD](https://kind.sigs.k8s.io/) cluster or a similar one with a small footprint before executing the tests.
 
 ## Providers
 
@@ -140,22 +140,56 @@ make \
 test-e2e
 ```
 
-## e2e test with localstack
+## e2e test with local storage emulators [AWS, GCP, AZURE]
 
-The above-mentioned e2e tests need storage from real cloud providers to be setup. But there is a tool named [localstack](https://docs.localstack.cloud/user-guide/aws/s3/) that enables to run e2e test with mock AWS storage. We can also provision KIND cluster for e2e tests. So, together with localstack and KIND cluster, we don't need to depend on any actual cloud provider infrastructure to be setup to run e2e tests.
+The above-mentioned e2e tests need storage from real cloud providers to be setup. But there are tools such as [localstack](https://docs.localstack.cloud/user-guide/aws/s3/), [fake-gcs-server](https://github.com/fsouza/fake-gcs-server) and [azurite](https://github.com/Azure/Azurite) that enable developers to run e2e tests with emulators for AWS S3, Google GCS and Azure ABS respectively. We can provision a KIND cluster for running the e2e tests against. Thus, by using cloud storage emulators alongside the KIND cluster, we eliminate the need for any actual cloud provider infrastructure to be set up for running e2e tests.
 
-### How are the KIND cluster and localstack set up
+### How are the KIND cluster and Emulators set up
 
 KIND or Kubernetes-In-Docker is a kubernetes cluster that is set up inside a docker container. This cluster is with limited capability as it does not have much compute power. But this cluster can easily be setup inside a container and can be tear down easily just by removing a container. That's why KIND cluster is very easy to use for e2e tests. `Makefile` command helps to spin up a KIND cluster and use the cluster to run e2e tests.
 
+#### Localstack setup
+
 There is a docker image for localstack. The image is deployed as pod inside the KIND cluster through `hack/e2e-test/infrastructure/localstack/localstack.yaml`. `Makefile` takes care of deploying the yaml file in a KIND cluster.
 
 The developer needs to run `make ci-e2e-kind` command. This command in turn runs `hack/ci-e2e-kind.sh` which spin up the KIND cluster and deploy localstack in it and then run the e2e tests using localstack as mock AWS storage provider. e2e tests are actually run on host machine but deploy the druid controller inside KIND cluster. Druid controller spawns multinode etcd clusters inside KIND cluster. e2e tests verify whether the druid controller performs its jobs correctly or not. Mock localstack storage is cleaned up after every e2e tests. That's why the e2e tests need to access the localstack pod running inside KIND cluster. The network traffic between host machine and localstack pod is resolved via mapping localstack pod port to host port while setting up the KIND cluster via `hack/e2e-test/infrastructure/kind/cluster.yaml`
 
-### How to execute e2e tests with localstack and KIND cluster
+##### How to execute e2e tests with localstack and KIND cluster
 
 Run the following `make` command to spin up a KinD cluster, deploy localstack and run the e2e tests with provider `aws`:
 
 ```bash
 make ci-e2e-kind
 ```
+
+#### Fake-GCS-Server setup
+
+[Fake-gcs-server](https://github.com/fsouza/fake-gcs-server) is run inside a pod using this [docker image](https://hub.docker.com/r/fsouza/fake-gcs-server) in a KIND cluster.
+
+The user needs to run `make ci-e2e-kind-gcs` to start the e2e tests for druid with GCS emulator as the object storage for etcd backups. The above command internally runs the script `hack/ci-e2e-kind-gcs.sh` which initializes the setup with required steps before going on to create a KIND cluster and deploy fakegcs in it and use that emulator to run e2e tests.
+
+The `fake-gcs-server` running inside the pod serves HTTP requests at port-8000 and HTTPS requests at port-4443. As the e2e tests runs on the host machine while the emulator runs on KIND, both ports i.e 8000 & 4443 needs to be port-forwarded from the host machine to fake-gcs service running inside the KIND cluster. The port forwardings is defined in the `hack/kind-up.sh` file which is used to setup the KIND cluster.
+
+##### How to execute e2e tests with fake-gcs-server and KIND cluster
+
+Run the following `make` command to spin up a KinD cluster, deploy fakegcs and run the e2e tests with provider `gcp`:
+
+```bash
+make ci-e2e-kind-gcs
+```
+
+#### Azurite setup
+
+[Azurite](https://github.com/Azure/Azurite) is run inside a pod using this [docker image](mcr.microsoft.com/azure-storage/azurite:latest) in a KIND cluster.
+
+The user needs to run `make ci-e2e-kind-azure` to start the e2e tests for druid with Azurite as the object storage for etcd backups. The above command internally runs the script `hack/ci-e2e-kind-azure.sh` which initializes the setup with required steps before going on to create a KIND cluster and deploy Azurite in it and use that emulator to run e2e tests.
+
+The `azurite` running inside the pod serves HTTP requests at port 10000. As the e2e tests runs on the host machine while the emulator runs on KIND cluster, the port 10000 needs to be port-forwarded from the host machine to azurite service running inside the KIND cluster. The port forwardings is defined in the `hack/kind-up.sh` file which is used to setup the KIND cluster.
+
+##### How to execute e2e tests with Azurite and KIND cluster
+
+Run the following `make` command to spin up a KinD cluster, deploy Azurite and run the e2e tests with provider `azure`:
+
+```bash
+make ci-e2e-kind-azure
+```