feat: Add vault tests (#43)

* feat: Add vault tests
- Added vault deployment and bootstrap to Minikube
- Added Vault private key test case
- refactor tests
- Increase coverage to 74%

.github/workflows/tests.yaml

@@ -29,11 +29,27 @@ jobs:
       # kustomize for each test but has no permission to do so
       - name: Remove pre-installed kustomize
         run: sudo rm -f /usr/local/bin/kustomize
+      # Install vault to minikube cluster to test vault case with kubernetes auth
+      - name: Install Vault
+        env:
+            GITHUB_PRIVATE_KEY: ${{ secrets.GH_TEST_APP_PK }}
+        run: |
+          cd scripts
+          chmod +x install_and_setup_vault_k8s.sh
+          ./install_and_setup_vault_k8s.sh
       - name: Perform the test
         run: |
           export "GITHUB_PRIVATE_KEY=${{ secrets.GH_TEST_APP_PK }}"
           export "GH_APP_ID=${{ secrets.GH_APP_ID }}"
           export "GH_INSTALL_ID=${{ secrets.GH_INSTALL_ID }}"
+          export "VAULT_ADDRESS=http://localhost:8200"
+          export "VAULT_ROLE_AUDIENCE=githubapp"
+          export "VAULT_ROLE=githubapp"
+
+          # Run vault port forward in background
+          kubectl port-forward vault-0 8200:8200 &
+          
+          # Run tests
           USE_EXISTING_CLUSTER=true make test
       - name: Report failure
         uses: nashmaniac/[email protected]

internal/controller/githubapp_controller_test.go

@@ -70,36 +70,49 @@ var _ = Describe("GithubApp controller", Ordered, func() {
 		_, _ = utils.Run(cmd)
 	})
 
-	Context("When requesting a token from token request api", func() {
-		It("Should create a valid JWT", func() {
-			if os.Getenv("USE_EXISTING_CLUSTER") != existingClusterValue {
-				fmt.Println("Skipping token request api test case as not a real cluster...")
-				return // Skip the test case since requires a real cluster
-			}
+	// Requires vault to be running on cluster and configured.
+	// from ./scripts directory run ./install_and_setup_vault_k8s.sh
+	// kubectl port-forward vault-0 8200:8200 in another terminal
+	// export VAULT_ADDRESS=http://localhost:8200
+	// then run the tests
+	Context("When creating a GithubApp with VaultPrivateKey spec", func() {
+		if os.Getenv("USE_EXISTING_CLUSTER") != existingClusterValue {
+			fmt.Println("Skipping test case as not a real cluster...")
+			return // Skip the test case since requires a real cluster
+		}
+		It("Should create GithubApp custom resources", func() {
 			ctx := context.Background()
 
-			By("Creating a new token via token request")
+			// namespace0 is already created in suite_test.go
 
-			controllerReconciler := &GithubAppReconciler{
-				Client:    k8sClient,
-				Scheme:    k8sClient.Scheme(),
-				K8sClient: k8sClientset,
+			By("Creating a GithubApp custom resource in the namespace0 with VaultPrivateKey spec")
+			vaultPrivateKeySpec := &githubappv1.VaultPrivateKeySpec{
+				MountPath:  "secret",
+				SecretPath: "githubapp/test",
+				SecretKey:  "privateKey",
 			}
-			fmt.Println("Got k8sClientset:", k8sClientset)
-
-			vaultAudience := "githubapp"
-
-			token, err := controllerReconciler.RequestToken(ctx, vaultAudience, namespace0, "default")
+			test_helpers.CreateGitHubAppAndWait(ctx, k8sClient, namespace0, githubAppName, nil, vaultPrivateKeySpec)
+		})
 
-			fmt.Println("Got a JWT from Kubernetes api:", token)
+		It("should successfully reconcile the resource", func() {
+			ctx := context.Background()
 
-			// Verify if reconciliation was successful
-			Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Token request failed: %v", err))
+			By("Waiting for the access token secret to be created")
+			test_helpers.WaitForAccessTokenSecret(ctx, k8sClient, namespace0)
 
+			By("Waiting for the correct event to be recorded")
+			test_helpers.CheckEvent(
+				ctx,
+				k8sClient,
+				githubAppName,
+				namespace0,
+				"Normal",
+				"Created",
+				fmt.Sprintf("Created access token secret %s/github-app-access-token-", namespace0))
 		})
 	})
 
-	Context("When setting up the test environment", func() {
+	Context("When creating a GithubApp with PrivateKeySecret spec", func() {
 		It("Should create GithubApp custom resources", func() {
 			ctx := context.Background()
 
@@ -110,7 +123,7 @@ var _ = Describe("GithubApp controller", Ordered, func() {
 			test_helpers.CreatePrivateKeySecret(ctx, k8sClient, namespace1, "privateKey")
 
 			By("Creating a first GithubApp custom resource in the namespace1")
-			test_helpers.CreateGitHubAppAndWait(ctx, k8sClient, namespace1, githubAppName, nil)
+			test_helpers.CreateGitHubAppAndWait(ctx, k8sClient, namespace1, githubAppName, nil, nil)
 		})
 	})
 
@@ -229,7 +242,7 @@ var _ = Describe("GithubApp controller", Ordered, func() {
 				},
 			}
 			// Create a GithubApp instance with the RolloutDeployment field initialized
-			test_helpers.CreateGitHubAppAndWait(ctx, k8sClient, namespace2, githubAppName2, rolloutDeploymentSpec)
+			test_helpers.CreateGitHubAppAndWait(ctx, k8sClient, namespace2, githubAppName2, rolloutDeploymentSpec, nil)
 
 			By("Waiting for pod1 with the label 'foo: bar' to be deleted")
 			// Wait for the pod to be deleted by the reconcile loop
@@ -273,7 +286,7 @@ var _ = Describe("GithubApp controller", Ordered, func() {
 			test_helpers.CreatePrivateKeySecret(ctx, k8sClient, namespace4, "foo")
 
 			By("Creating a GithubApp without creating the privateKeySecret with 'privateKey' field")
-			test_helpers.CreateGitHubAppAndWait(ctx, k8sClient, namespace4, githubAppName4, nil)
+			test_helpers.CreateGitHubAppAndWait(ctx, k8sClient, namespace4, githubAppName4, nil, nil)
 
 			By("Checking the githubApp `status.error` value is as expected")
 			test_helpers.CheckGithubAppStatusError(
@@ -307,7 +320,7 @@ var _ = Describe("GithubApp controller", Ordered, func() {
 			test_helpers.CreateNamespace(ctx, k8sClient, namespace3)
 
 			By("Creating a GithubApp without creating the privateKeySecret")
-			test_helpers.CreateGitHubAppAndWait(ctx, k8sClient, namespace3, githubAppName3, nil)
+			test_helpers.CreateGitHubAppAndWait(ctx, k8sClient, namespace3, githubAppName3, nil, nil)
 
 			By("Checking the githubApp `status.error` value is as expected")
 			test_helpers.CheckGithubAppStatusError(

internal/controller/suite_test.go

@@ -121,6 +121,13 @@ var _ = BeforeSuite(func() {
 
 	var token string
 	if os.Getenv("USE_EXISTING_CLUSTER") == "true" {
+		// Initialise vault client with VAULT_ADDRESS env var
+		vaultAddress := os.Getenv("VAULT_ADDRESS") // Vault server fqdn
+		vaultClient, err = vault.NewClient(&vault.Config{
+			Address: vaultAddress,
+		})
+		Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Vault client initialisation failed: %v", err))
+
 		// Initialise K8s client
 		defer func() {
 			if r := recover(); r != nil {
@@ -130,6 +137,8 @@ var _ = BeforeSuite(func() {
 		k8sClientset = kubernetes.NewForConfigOrDie(ctrlConfig.GetConfigOrDie())
 		fmt.Println("Got main k8sClientset:", k8sClientset)
 
+		// Create a valid service account token to initialise the controller SetupWithManager with
+		// This will be used in Token Request API
 		By("Creating a new namespace")
 		test_helpers.CreateNamespace(ctx, k8sClient, "namespace0")
 		time.Sleep(5 * time.Second)
@@ -147,7 +156,18 @@ var _ = BeforeSuite(func() {
 		// Verify if reconciliation was successful
 		Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Token request failed: %v", err))
 	} else {
-		token = "eyJhbGciOiJSUzI1NiIsImtpZCI6Ik5ieTJyVUk2ZzlQZ0k0anNGclRvTkJDM0FsUjJjLUJDVUhzNU9mVG9lcEUifQ.eyJhdWQiOlsiZ2l0aHViYXBwIl0sImV4cCI6MTcxMzEyNjIxMiwiaWF0IjoxNzEzMTI1NjEyLCJpc3MiOiJodHRwczovL2t1YmVybmV0ZXMuZGVmYXVsdC5zdmMuY2x1c3Rlci5sb2NhbCIsImt1YmVybmV0ZXMuaW8iOnsibmFtZXNwYWNlIjoibmFtZXNwYWNlMCIsInNlcnZpY2VhY2NvdW50Ijp7Im5hbWUiOiJkZWZhdWx0IiwidWlkIjoiNDY3ZTA4MGMtYWZhNy00OTc4LWFkYzMtYWI5NmFkMWJjOTQzIn19LCJuYmYiOjE3MTMxMjU2MTIsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDpuYW1lc3BhY2UwOmRlZmF1bHQifQ.ftFKNIwM_qi-6W7rvMyjNC2xAbNfFsrRgPQnjEsDw84_fpn9I1LFnQPQzA5HeTtJyIBzjrdHsEGgcCTxsYLgErLkIJ9MfWxwyP3FsNeuQgNoBr4Pmo9lRnayzERU9YZwEb9QCoZXGkCrcv16q15hB_J_ik9lcwlLJ6PYDW58AA39VUDsfqin-8D23ghmAumv8vods6v-WVNeMKAP0oO7oqElLop9r5h8hf9ApaAZ2zRbTnQ-X1HpAFwOzRTGvcPli1hLZ7rgDAw6yJOOnExPgMZ44umiaXVhnow2Vxol2G7yb0mFToWOwpiHZPsL4kZccGK33nk1Kcfcawqqd2IGBA"
+		// Set a dummy token just to satisfy the SetupWithManager fucntion
+		// which will read the token and get the service account and and namespace.
+		// This token is for 'default' service account in the 'namespace0' namespace
+		token = `eyJhbGciOiJSUzI1NiIsImtpZCI6Ik5ieTJyVUk2ZzlQZ0k0anNGclRvTkJDM0FsUjJjLUJDVUhzNU9mVG9lcEUifQ.
+		eyJhdWQiOlsiZ2l0aHViYXBwIl0sImV4cCI6MTcxMzEyNjIxMiwiaWF0IjoxNzEzMTI1NjEyLCJpc3MiOiJodHRwczovL2
+		t1YmVybmV0ZXMuZGVmYXVsdC5zdmMuY2x1c3Rlci5sb2NhbCIsImt1YmVybmV0ZXMuaW8iOnsibmFtZXNwYWNlIjoibmFtZ
+		XNwYWNlMCIsInNlcnZpY2VhY2NvdW50Ijp7Im5hbWUiOiJkZWZhdWx0IiwidWlkIjoiNDY3ZTA4MGMtYWZhNy00OTc4LWFkY
+		zMtYWI5NmFkMWJjOTQzIn19LCJuYmYiOjE3MTMxMjU2MTIsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDpuYW1lc3BhY2
+		UwOmRlZmF1bHQifQ.ftFKNIwM_qi-6W7rvMyjNC2xAbNfFsrRgPQnjEsDw84_fpn9I1LFnQPQzA5HeTtJyIBzjrdHsEGgcCTx
+		sYLgErLkIJ9MfWxwyP3FsNeuQgNoBr4Pmo9lRnayzERU9YZwEb9QCoZXGkCrcv16q15hB_J_ik9lcwlLJ6PYDW58AA39VUDs
+		fqin-8D23ghmAumv8vods6v-WVNeMKAP0oO7oqElLop9r5h8hf9ApaAZ2zRbTnQ-X1HpAFwOzRTGvcPli1hLZ7rgDAw6yJOOn
+		ExPgMZ44umiaXVhnow2Vxol2G7yb0mFToWOwpiHZPsL4kZccGK33nk1Kcfcawqqd2IGBA`
 	}
 
 	var file *os.File

internal/controller/test_helpers/test_helpers.go

@@ -131,6 +131,7 @@ func CreateGitHubAppAndWait(
 	namespace,
 	name string,
 	rolloutDeploymentSpec *githubappv1.RolloutDeploymentSpec,
+	vaultPrivateKeySpec *githubappv1.VaultPrivateKeySpec,
 ) {
 	// create the GitHubApp
 	githubApp := githubappv1.GithubApp{
@@ -143,6 +144,7 @@ func CreateGitHubAppAndWait(
 			InstallId:         installId,
 			PrivateKeySecret:  privateKeySecret,
 			RolloutDeployment: rolloutDeploymentSpec, // Optionally pass rolloutDeployment
+			VaultPrivateKey:   vaultPrivateKeySpec,   // Optionally pass vaultPrivateKeySpec
 		},
 	}
 	gomega.Expect(k8sClient.Create(ctx, &githubApp)).Should(gomega.Succeed())

scripts/delete_vault.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# Run this script to delete the vault setup in kubernetes
+
+helm delete vault
+kubectl delete pvc data-vault-0

scripts/helm-vault-raft-values.yml

@@ -0,0 +1,19 @@
+server:
+   affinity: ""
+   ha:
+      enabled: false
+      raft: 
+         enabled: true
+         setNodeId: true
+         config: |
+            cluster_name = "vault-integrated-storage"
+            storage "raft" {
+               path    = "/vault/data/"
+            }
+
+            listener "tcp" {
+               address = "[::]:8200"
+               cluster_address = "[::]:8201"
+               tls_disable = "true"
+            }
+            service_registration "kubernetes" {}

scripts/install_and_setup_vault_k8s.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+
+# Run this script to setup vault on kubernetes with a simple role, policy and k8s auth
+# Export your github app private key then run the script
+# export GITHUB_PRIVATE_KEY=<YOUR GITHUB APP PRIVATE KEY>
+
+helm repo add hashicorp https://helm.releases.hashicorp.com
+helm repo update
+
+# install vault with single node
+helm install vault hashicorp/vault --values helm-vault-raft-values.yml
+kubectl get pods
+
+# wait for vault to run
+until kubectl get pod vault-0 -o=jsonpath='{.status.phase}' | grep -q "Running"; do sleep 5; done
+
+# get cluster keys
+kubectl exec vault-0 -- vault operator init \
+  -key-shares=1 \
+  -key-threshold=1 \
+  -format=json > cluster-keys.json
+
+# set unseal key
+VAULT_UNSEAL_KEY=$(jq -r ".unseal_keys_b64[]" cluster-keys.json)
+
+# unseal vault
+kubectl exec vault-0 -- vault operator unseal ${VAULT_UNSEAL_KEY}
+
+# wait for vault to be ready
+kubectl wait --for=condition=ready pod/vault-0 --timeout=300s
+
+# get root token
+VAULT_ROOT_TOKEN=$(jq -r ".root_token" cluster-keys.json)
+
+# login
+kubectl exec -i vault-0 -- vault login -non-interactive ${VAULT_ROOT_TOKEN}
+
+# enable kv-v2
+kubectl exec -i vault-0 -- vault secrets enable -path=secret kv-v2
+
+# write github app secret
+kubectl exec -i vault-0 -- vault kv put secret/githubapp/test privateKey="${GITHUB_PRIVATE_KEY}"
+
+# enable k8s auth
+kubectl exec -i vault-0 -- vault auth enable kubernetes
+
+# get k8s host
+KUBERNETES_HOST=$(kubectl exec -i vault-0 -- sh -c 'echo $KUBERNETES_SERVICE_HOST')
+
+# write k8s host
+kubectl exec -i vault-0 -- vault write auth/kubernetes/config kubernetes_host="https://$KUBERNETES_HOST:443"
+
+# write vault policy
+kubectl exec -i vault-0 -- sh -c 'vault policy write githubapp - <<EOF
+path "secret/data/githubapp/test" {
+  capabilities = ["read"]
+}
+EOF'
+
+# write vault role
+kubectl exec -i vault-0 -- sh -c 'vault write auth/kubernetes/role/githubapp \
+  bound_service_account_names="default" \
+  bound_service_account_namespaces="namespace0" \
+  policies=githubapp \
+  ttl=24h'