fix: Use Keycloak as a data source instead of APPUiO Control API

.github/workflows/dev.yml

@@ -17,15 +17,23 @@ jobs:
     - env:
         NEWLINE: |2+
 
-      name: Generate Deployment
+      name: k8ify preview
       run: |
         jq -r 'try to_entries | map("\(.key)=\(.value|tostring|tojson)") | .[]' > .composeenv <<'EOF' ${{ env.NEWLINE }}${{ toJson(vars) }}${{ env.NEWLINE }}EOF
         jq -r 'try to_entries | map("\(.key)=\(.value|tostring|tojson)") | .[]' >> .composeenv <<'EOF' ${{ env.NEWLINE }}${{ toJson(secrets) }}${{ env.NEWLINE }}EOF
         docker run -v "${PWD}:/data" -w /data --env-file <(env) ghcr.io/vshn/k8ify-appcat:latest /bin/k8ify dev ${CI_COMMIT_REF_SLUG} --shell-env-file .composeenv --modified-image '${{ env.NAMESPACE }}/${{ env.CI_PROJECT_NAME }}:${{ env.CI_COMMIT_REF_SLUG }}'
         rm .composeenv
         kubectl diff -f manifests/ || true
-    - name: Deploy
-      run: kubectl apply -f manifests/ || true
+    - env:
+        NEWLINE: |2+
+
+      name: k8ify deploy
+      run: |
+        jq -r 'try to_entries | map("\(.key)=\(.value|tostring|tojson)") | .[]' > .composeenv <<'EOF' ${{ env.NEWLINE }}${{ toJson(vars) }}${{ env.NEWLINE }}EOF
+        jq -r 'try to_entries | map("\(.key)=\(.value|tostring|tojson)") | .[]' >> .composeenv <<'EOF' ${{ env.NEWLINE }}${{ toJson(secrets) }}${{ env.NEWLINE }}EOF
+        docker run -v "${PWD}:/data" -w /data --env-file <(env) ghcr.io/vshn/k8ify-appcat:latest /bin/k8ify dev ${CI_COMMIT_REF_SLUG} --shell-env-file .composeenv --modified-image '${{ env.NAMESPACE }}/${{ env.CI_PROJECT_NAME }}:${{ env.CI_COMMIT_REF_SLUG }}'
+        rm .composeenv
+        kubectl apply -f manifests/
 name: Build and deploy to dev
 "on":
   push:

.gitignore

@@ -1,2 +1,3 @@
 grafana-organizations-operator
 env
+.idea/

README.md

@@ -1,6 +1,49 @@
 # Grafana Organizations Operator
 
-Automatically set up Grafana organizations based on information from the [APPUiO Control API](https://kb.vshn.ch/appuio-cloud/references/architecture/control-api.html).
+Automatically set up Grafana organizations based on information in Keycloak.
+
+## Design
+
+### Data in Keycloak
+
+Keycloak holds the APPUiO Cloud organization and user data.
+
+* APPUiO Cloud organizations are represented as groups with the group path `/organizations/[ORGNAME]`
+* Teams within organizations are represented as groups with the path `/organizations/[ORGNAME]/[TEAMNAME]`.
+* APPUiO Cloud users are represented as normal Keycloak users. All Keycloak users are potential APPUiO Cloud users.
+* User permissions are represented as regular Keycloak group memberships. A user can be a member of an organization or of a team, and can have multiple partially overlapping memberships.
+* All members of the group configured via `KEYCLOAK_ADMIN_GROUP_PATH` are considered to be admins and have "Admin" permissions on all organizations.
+
+The Grafana Organizations Operator does not differentiate between organization and team membership, and does not support team-specific permissions.
+
+This information is translated into Grafana organizations, users and organization users ("permissions" a user has on an organization).
+
+The Grafana Organizations Operator only cares about organizations present in Keycloak. This allows other Organizations in Grafana to exist without being touched; but it also means that it will not clean up organizations in Grafana which have been deleted in Keycloak.
+
+### Issues with the Keycloak API
+
+* The Keycloak LDAP integration is [buggy when it comes to listing members of a group](https://github.com/keycloak/keycloak/issues/10348) (API endpoint `/auth/admin/realms/[REALM]/groups/[ID]/members`). This would not be an issue for the organization users but it is an issue for the `KEYCLOAK_ADMIN_GROUP_PATH` which may come from LDAP. As a workaround we fetch the group memberships of users instead, however this means one HTTP call per user (easily 1000s of HTTP calls).
+* The operator assumes that all Grafana organizations are in the root group `/organizations`
+
+### Issues with Grafana
+
+* When a new user is created in Grafana (either by logging in via Keycloak or explicitly created via API), Grafana's `auto_assign_org` "feature" automatically gives the user permission to some organization (whichever is configured). This is almost never what we want. To work around this:
+  * It would be possible to disable `auto_assign_org`, but then Grafana would create a new organization for every new user, which would be even worse.
+  * We can't let Grafana create users on demand when they first log in (as that would give the user permission to see the `auto_assign_org`), thus we proactively create Grafana users for all known user, thereby avoiding any situations where Grafana would create a user on demand.
+  * When we create a Grafana user via API we immediately list its group memberships and remove all of them. This is a bit dangerous because it has the potential for race conditions or could leave unwanted permissions behind in case operator fails between the Grafana API calls, but it's the easiest way to handle this.
+* Because the `grafana-api-golang-client` implementation is incomplete we are wrapping it in the GrafanaClient type and add some functionality.
+* The Grafana API often ignores the OrgID JSON field. The only workaround for this is to set the HTTP header `x-grafana-org-id`. The GrafanaClient wrapper takes care of this.
+
+## Development Environment Setup
+
+In order to develop the operator you need:
+
+* Read access to Keycloak for all users and groups
+* A [Grafana test instance](https://operator-dev-grafana.apps.cloudscale-lpg-2.appuio.cloud/) to write to
+
+You can run the `gen-dev-env.sh` to set up an environment file (`env`) with the required configuration.
+
+Once that's done you can source the env file (`. ./env`) and run the operator on your local machine using `go run .`.
 
 ## License
 

gen-dev-env.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+set -e -o pipefail
+SECRET="$(kubectl --as cluster-admin -n vshn-appuio-grafana get secret grafana-organizations-operator -ojson)" || (>&2 echo "You must be logged in to 'APPUiO Cloud LPG 2' with cluster-admin permissions for this to work" && exit 1)
+echo -n "" > env
+for VAL in KEYCLOAK_ADMIN_GROUP_PATH KEYCLOAK_CLIENT_ID KEYCLOAK_PASSWORD KEYCLOAK_REALM KEYCLOAK_URL KEYCLOAK_USERNAME; do
+	echo -n "export ${VAL}=\"" >> env
+	echo "${SECRET}" | jq -r ".data.${VAL}" | base64 -d >> env
+	echo "\"" >> env
+done
+echo "export GRAFANA_URL=\"https://operator-dev-grafana.apps.cloudscale-lpg-2.appuio.cloud\"" >> env
+echo "export GRAFANA_USERNAME=\"admin\"" >> env
+echo -n "export GRAFANA_PASSWORD=\"" >> env
+kubectl --as cluster-admin -n vshn-grafana-organizations-operator-dev get secret grafana-env -ojsonpath='{.data.GF_SECURITY_ADMIN_PASSWORD}' | base64 -d >> env
+echo "\"" >> env

go.mod

@@ -4,7 +4,8 @@ go 1.20
 
 require (
 	github.com/appuio/control-api v0.26.0
-	github.com/grafana/grafana-api-golang-client v0.21.0
+	github.com/grafana/grafana-api-golang-client v0.23.0
+	github.com/hashicorp/go-cleanhttp v0.5.2
 	k8s.io/apimachinery v0.26.2
 	k8s.io/client-go v0.26.2
 	k8s.io/klog/v2 v2.90.1
@@ -20,7 +21,6 @@ require (
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/uuid v1.3.0 // indirect
-	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect

go.sum

@@ -18,6 +18,7 @@ github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKf
 github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U=
 github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk=
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
+github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
 github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
 github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
@@ -47,6 +48,8 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/grafana/grafana-api-golang-client v0.21.0 h1:PQ2Wfo9jMMiftC4VRMlJxbUNvYCXMV1YFDKm7Ny3SaM=
 github.com/grafana/grafana-api-golang-client v0.21.0/go.mod h1:24W29gPe9yl0/3A9X624TPkAOR8DpHno490cPwnkv8E=
+github.com/grafana/grafana-api-golang-client v0.23.0 h1:Uta0dSkxWYf1D83/E7MRLCG69387FiUc+k9U/35nMhY=
+github.com/grafana/grafana-api-golang-client v0.23.0/go.mod h1:24W29gPe9yl0/3A9X624TPkAOR8DpHno490cPwnkv8E=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.0 h1:1JYBfzqrWPcCclBwxFCPAou9n+q86mfnu7NAeHfte7A=
@@ -56,8 +59,10 @@ github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk=
 github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
@@ -71,9 +76,11 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/onsi/ginkgo/v2 v2.6.0 h1:9t9b9vRUbFq3C4qKFCGkVuq/fIHji802N1nrtkh1mNc=
 github.com/onsi/gomega v1.24.1 h1:KORJXNNTzJXzu4ScJWssJfJMnJ+2QJqhoQSRwNlze9E=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v1.15.0 h1:5fCgGYogn0hFdhyhLbw7hEsWxufKtY9klyvdNfFlFhM=

main.go

@@ -2,14 +2,9 @@ package main
 
 import (
 	"context"
-	orgs "github.com/appuio/control-api/apis/organization/v1"
-	controlapi "github.com/appuio/control-api/apis/v1"
 	controller "github.com/appuio/grafana-organizations-operator/pkg"
 	grafana "github.com/grafana/grafana-api-golang-client"
-	"k8s.io/apimachinery/pkg/runtime/serializer"
 	"k8s.io/apimachinery/pkg/util/json"
-	"k8s.io/client-go/kubernetes/scheme"
-	"k8s.io/client-go/rest"
 	"k8s.io/klog/v2"
 	"net/http"
 	"net/url"
@@ -20,63 +15,67 @@ import (
 )
 
 var (
-	ControlApiToken string
-	ControlApiUrl   string
-	GrafanaUrl      string
-	GrafanaUsername string
-	GrafanaPassword string
+	GrafanaUrl             string
+	GrafanaUsername        string
+	GrafanaPassword        string
+	KeycloakUrl            string
+	KeycloakRealm          string
+	KeycloakUsername       string
+	KeycloakPassword       string
+	KeycloakClientId       string
+	KeycloakAdminGroupPath string
 )
 
 func main() {
-	ControlApiUrl = os.Getenv("CONTROL_API_URL")
-	ControlApiToken = os.Getenv("CONTROL_API_TOKEN")
 	GrafanaUrl = os.Getenv("GRAFANA_URL")
 	GrafanaUsername = os.Getenv("GRAFANA_USERNAME")
 	if GrafanaUsername == "" {
 		GrafanaUsername = os.Getenv("admin-user") // env variable name used by Grafana Helm chart. And yes using '-' is stupid because of compatibility issues with some shells.
 	}
 	GrafanaPassword = os.Getenv("GRAFANA_PASSWORD")
+	GrafanaPasswordHidden := ""
 	if GrafanaPassword == "" {
 		GrafanaPassword = os.Getenv("admin-password") // env variable name used by Grafana Helm chart. And yes using '-' is stupid because of compatibility issues with some shells.
 	}
+	if GrafanaPassword != "" {
+		GrafanaPasswordHidden = "***hidden***"
+	}
 
-	klog.Infof("CONTROL_API_URL: %s\n", ControlApiUrl)
-	klog.Infof("GRAFANA_URL: %s\n", GrafanaUrl)
-	klog.Infof("GRAFANA_USERNAME: %s\n", GrafanaUsername)
-
-	// Because of the strange design of the k8s client we actually need two client objects, which both internally use the same httpClient.
-	// To make this work we also need three (!) config objects, a common one for the httpClient and one for each k8s client.
-	commonConfig := &rest.Config{}
-	commonConfig.BearerToken = ControlApiToken
-	httpClient, err := rest.HTTPClientFor(commonConfig)
-	if err != nil {
-		klog.Errorf("Could not create Control API httpClient: %v\n", err)
-		os.Exit(1)
+	KeycloakUrl = os.Getenv("KEYCLOAK_URL")
+	KeycloakRealm = os.Getenv("KEYCLOAK_REALM")
+	KeycloakUsername = os.Getenv("KEYCLOAK_USERNAME")
+	KeycloakPassword = os.Getenv("KEYCLOAK_PASSWORD")
+	KeycloakClientId = os.Getenv("KEYCLOAK_CLIENT_ID")
+	KeycloakPasswordHidden := ""
+	if KeycloakPassword != "" {
+		KeycloakPasswordHidden = "***hidden***"
 	}
+	KeycloakAdminGroupPath = os.Getenv("KEYCLOAK_ADMIN_GROUP_PATH")
 
-	organizationAppuioIoConfig := &rest.Config{}
-	organizationAppuioIoConfig.Host = ControlApiUrl
-	organizationAppuioIoConfig.APIPath = "/apis"
-	organizationAppuioIoConfig.GroupVersion = &orgs.GroupVersion
-	organizationAppuioIoConfig.NegotiatedSerializer = serializer.NewCodecFactory(scheme.Scheme)
-	organizationAppuioIoClient, err := rest.RESTClientForConfigAndClient(organizationAppuioIoConfig, httpClient)
+	klog.Infof("GRAFANA_URL:               %s\n", GrafanaUrl)
+	klog.Infof("GRAFANA_USERNAME:          %s\n", GrafanaUsername)
+	klog.Infof("GRAFANA_PASSWORD:          %s\n", GrafanaPasswordHidden)
+	klog.Infof("KEYCLOAK_URL:              %s\n", KeycloakUrl)
+	klog.Infof("KEYCLOAK_REALM:            %s\n", KeycloakRealm)
+	klog.Infof("KEYCLOAK_USERNAME:         %s\n", KeycloakUsername)
+	klog.Infof("KEYCLOAK_PASSWORD:         %s\n", KeycloakPasswordHidden)
+	klog.Infof("KEYCLOAK_CLIENT_ID:        %s\n", KeycloakClientId)
+	klog.Infof("KEYCLOAK_ADMIN_GROUP_PATH: %s\n", KeycloakAdminGroupPath)
+
+	keycloakClient, err := controller.NewKeycloakClient(KeycloakUrl, KeycloakRealm, KeycloakUsername, KeycloakPassword, KeycloakClientId, KeycloakAdminGroupPath)
 	if err != nil {
-		klog.Errorf("Could not create Control API client for organization.appuio.io: %v\n", err)
+		klog.Errorf("Could not create keycloakClient client: %v\n", err)
 		os.Exit(1)
 	}
+	defer keycloakClient.CloseIdleConnections()
 
-	appuioIoConfig := &rest.Config{}
-	appuioIoConfig.Host = ControlApiUrl
-	appuioIoConfig.APIPath = "/apis"
-	appuioIoConfig.GroupVersion = &controlapi.GroupVersion
-	appuioIoConfig.NegotiatedSerializer = serializer.NewCodecFactory(scheme.Scheme)
-	appuioIoClient, err := rest.RESTClientForConfigAndClient(appuioIoConfig, httpClient)
+	grafanaConfig := grafana.Config{Client: http.DefaultClient, BasicAuth: url.UserPassword(GrafanaUsername, GrafanaPassword)}
+	grafanaClient, err := controller.NewGrafanaClient(GrafanaUrl, grafanaConfig)
 	if err != nil {
-		klog.Errorf("Could not connect Control API client for appuio.io: %v\n", err)
+		klog.Errorf("Could not create Grafana client: %v\n", err)
 		os.Exit(1)
 	}
-
-	grafanaConfig := grafana.Config{Client: http.DefaultClient, BasicAuth: url.UserPassword(GrafanaUsername, GrafanaPassword)}
+	defer grafanaClient.CloseIdleConnections()
 
 	// ctx will be passed to controller to signal termination
 	ctx, cancel := context.WithCancel(context.Background())
@@ -100,19 +99,19 @@ func main() {
 	json.Unmarshal(db, &dashboard)
 
 	klog.Info("Starting initial sync...")
-	err = controller.ReconcileAllOrgs(ctx, organizationAppuioIoClient, appuioIoClient, grafanaConfig, GrafanaUrl, dashboard)
+	err = controller.Reconcile(ctx, keycloakClient, grafanaClient, dashboard)
 	if err != nil {
 		klog.Errorf("Could not do initial reconciliation: %v\n", err)
 		os.Exit(1)
 	}
 
 	for {
+		err = controller.Reconcile(ctx, keycloakClient, grafanaClient, dashboard)
 		select {
 		case <-time.After(10 * time.Second):
 		case <-ctx.Done():
 			os.Exit(0)
 		}
-		err = controller.ReconcileAllOrgs(ctx, organizationAppuioIoClient, appuioIoClient, grafanaConfig, GrafanaUrl, dashboard)
 		if err != nil {
 			klog.Errorf("Could not reconcile (will retry): %v\n", err)
 		}

pkg/controlApi.go

@@ -4,9 +4,7 @@ import (
 	"context"
 	orgs "github.com/appuio/control-api/apis/organization/v1"
 	controlapi "github.com/appuio/control-api/apis/v1"
-	grafana "github.com/grafana/grafana-api-golang-client"
 	"k8s.io/client-go/rest"
-	"k8s.io/klog/v2"
 )
 
 // Generate list of control API organizations
@@ -36,25 +34,3 @@ func getControlApiUsersMap(ctx context.Context, appuioIoClient *rest.RESTClient)
 	}
 	return appuioControlApiUsersMap, nil
 }
-
-// Generate map containing all Grafana users grouped by organization. Key is the organization ID, value is an array of users.
-func getControlApiOrganizationUsersMap(ctx context.Context, grafanaUsersMap map[string]grafana.User, appuioIoClient *rest.RESTClient) (map[string][]grafana.User, error) {
-	appuioControlApiOrganizationMembers := controlapi.OrganizationMembersList{}
-	err := appuioIoClient.Get().Resource("OrganizationMembers").Do(ctx).Into(&appuioControlApiOrganizationMembers)
-	if err != nil {
-		return nil, err
-	}
-	controlApiOrganizationUsersMap := make(map[string][]grafana.User)
-	for _, memberlist := range appuioControlApiOrganizationMembers.Items {
-		users := []grafana.User{}
-		for _, userRef := range memberlist.Spec.UserRefs {
-			if grafanaUser, ok := grafanaUsersMap[userRef.Name]; ok {
-				users = append(users, grafanaUser)
-			} else {
-				klog.Warningf("Organization '%s' should have user %s but the user wasn't synced to Grafana, ignoring", memberlist.Namespace, userRef.Name)
-			}
-		}
-		controlApiOrganizationUsersMap[memberlist.Namespace] = users
-	}
-	return controlApiOrganizationUsersMap, nil
-}