diff --git a/README.md b/README.md index b66af67..230ce9e 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,8 @@ There are a bunch of issues this operator needs to overcome. These issues have a * The Control API has no notion of "admin" users, i.e. users which should have access to everything. The configuration option `CONTROL_API_ADMIN_ORG` allows to specify a Control API organization, and all users who have access to that will be treated as admins. * The `Organization` and `User` types use different GroupVersions. It seems that this cannot be implemented using a single k8s.io client; instead we need two client instances for the Control API. -* The `OrganizationMember` field `spec.userRefs` contains a lot of invalid values, including even values with incorrect syntax. These must be filtered out. +* The `OrganizationMember` field `spec.userRefs` contains a lot of invalid values, including even values with incorrect syntax. These must be filtered out. Bug report created. +* The value of the `User` field `status.email` is not unique. Since Grafana requires unique email addresses this leads to sync errors. Bug report created. ### Issues with Grafana diff --git a/main.go b/main.go index 495fe6c..b126098 100644 --- a/main.go +++ b/main.go @@ -2,14 +2,9 @@ package main import ( "context" - orgs "github.com/appuio/control-api/apis/organization/v1" - controlapi "github.com/appuio/control-api/apis/v1" controller "github.com/appuio/grafana-organizations-operator/pkg" grafana "github.com/grafana/grafana-api-golang-client" - "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/util/json" - "k8s.io/client-go/kubernetes/scheme" - "k8s.io/client-go/rest" "k8s.io/klog/v2" "net/http" "net/url" @@ -20,13 +15,19 @@ import ( ) var ( - ControlApiToken string - ControlApiUrl string - ControlApiAdminOrg string - GrafanaUrl string - GrafanaUsername string - GrafanaPassword string - AdminExtraOrgs = [1]int{1} + ControlApiToken string + ControlApiUrl string + ControlApiAdminOrg string + GrafanaUrl string + GrafanaUsername string + GrafanaPassword string + AdminExtraOrgs = [1]int{1} + KeycloakUrl string + KeycloakRealm string + KeycloakUsername string + KeycloakPassword string + KeycloakClientId string + KeycloakAdminGroupPath string ) func main() { @@ -52,44 +53,45 @@ func main() { GrafanaPasswordHidden = "***hidden***" } - klog.Infof("CONTROL_API_URL: %s\n", ControlApiUrl) - klog.Infof("CONTROL_API_ADMIN_ORG: %s\n", ControlApiAdminOrg) - klog.Infof("CONTROL_API_TOKEN: %s\n", ControlApiTokenHidden) - klog.Infof("GRAFANA_URL: %s\n", GrafanaUrl) - klog.Infof("GRAFANA_USERNAME: %s\n", GrafanaUsername) - klog.Infof("GRAFANA_PASSWORD: %s\n", GrafanaPasswordHidden) - - // Because of the strange design of the k8s client we actually need two client objects, which both internally use the same httpClient. - // To make this work we also need three (!) config objects, a common one for the httpClient and one for each k8s client. - commonConfig := &rest.Config{} - commonConfig.BearerToken = ControlApiToken - httpClient, err := rest.HTTPClientFor(commonConfig) - if err != nil { - klog.Errorf("Could not create Control API httpClient: %v\n", err) - os.Exit(1) + KeycloakUrl = os.Getenv("KEYCLOAK_URL") + KeycloakRealm = os.Getenv("KEYCLOAK_REALM") + KeycloakUsername = os.Getenv("KEYCLOAK_USERNAME") + KeycloakPassword = os.Getenv("KEYCLOAK_PASSWORD") + KeycloakClientId = os.Getenv("KEYCLOAK_CLIENT_ID") + KeycloakPasswordHidden := "" + if KeycloakPassword != "" { + KeycloakPasswordHidden = "***hidden***" } + KeycloakAdminGroupPath = os.Getenv("KEYCLOAK_ADMIN_GROUP_PATH") + + klog.Infof("CONTROL_API_URL: %s\n", ControlApiUrl) + klog.Infof("CONTROL_API_ADMIN_ORG: %s\n", ControlApiAdminOrg) + klog.Infof("CONTROL_API_TOKEN: %s\n", ControlApiTokenHidden) + klog.Infof("GRAFANA_URL: %s\n", GrafanaUrl) + klog.Infof("GRAFANA_USERNAME: %s\n", GrafanaUsername) + klog.Infof("GRAFANA_PASSWORD: %s\n", GrafanaPasswordHidden) + klog.Infof("KEYCLOAK_URL: %s\n", KeycloakUrl) + klog.Infof("KEYCLOAK_REALM: %s\n", KeycloakRealm) + klog.Infof("KEYCLOAK_USERNAME: %s\n", KeycloakUsername) + klog.Infof("KEYCLOAK_PASSWORD: %s\n", KeycloakPasswordHidden) + klog.Infof("KEYCLOAK_CLIENT_ID: %s\n", KeycloakClientId) + klog.Infof("KEYCLOAK_ADMIN_GROUP_PATH: %s\n", KeycloakAdminGroupPath) - organizationAppuioIoConfig := &rest.Config{} - organizationAppuioIoConfig.Host = ControlApiUrl - organizationAppuioIoConfig.APIPath = "/apis" - organizationAppuioIoConfig.GroupVersion = &orgs.GroupVersion - organizationAppuioIoConfig.NegotiatedSerializer = serializer.NewCodecFactory(scheme.Scheme) - organizationAppuioIoClient, err := rest.RESTClientForConfigAndClient(organizationAppuioIoConfig, httpClient) + keycloak, err := controller.NewKeycloakClient(KeycloakUrl, KeycloakRealm, KeycloakUsername, KeycloakPassword, KeycloakClientId, KeycloakAdminGroupPath) if err != nil { - klog.Errorf("Could not create Control API client for organization.appuio.io: %v\n", err) + klog.Errorf("Could not create keycloak client: %v\n", err) os.Exit(1) } - appuioIoConfig := &rest.Config{} - appuioIoConfig.Host = ControlApiUrl - appuioIoConfig.APIPath = "/apis" - appuioIoConfig.GroupVersion = &controlapi.GroupVersion - appuioIoConfig.NegotiatedSerializer = serializer.NewCodecFactory(scheme.Scheme) - appuioIoClient, err := rest.RESTClientForConfigAndClient(appuioIoConfig, httpClient) + /*keycloakAdmins, err := keycloak.GetAdminUsers(keycloakToken) if err != nil { - klog.Errorf("Could not connect Control API client for appuio.io: %v\n", err) + klog.Errorf("Could not get keycloak admin users: %v\n", err) os.Exit(1) } + fmt.Printf("Admins:\n") + for _, keycloakUser := range keycloakAdmins { + fmt.Printf("%s\n", keycloakUser.Email) + }*/ grafanaConfig := grafana.Config{Client: http.DefaultClient, BasicAuth: url.UserPassword(GrafanaUsername, GrafanaPassword)} @@ -115,7 +117,7 @@ func main() { json.Unmarshal(db, &dashboard) klog.Info("Starting initial sync...") - err = controller.Reconcile(ctx, organizationAppuioIoClient, appuioIoClient, ControlApiAdminOrg, grafanaConfig, GrafanaUrl, dashboard) + err = controller.Reconcile(ctx, keycloak, ControlApiAdminOrg, grafanaConfig, GrafanaUrl, dashboard) if err != nil { klog.Errorf("Could not do initial reconciliation: %v\n", err) os.Exit(1) @@ -127,7 +129,7 @@ func main() { case <-ctx.Done(): os.Exit(0) } - err = controller.Reconcile(ctx, organizationAppuioIoClient, appuioIoClient, ControlApiAdminOrg, grafanaConfig, GrafanaUrl, dashboard) + err = controller.Reconcile(ctx, keycloak, ControlApiAdminOrg, grafanaConfig, GrafanaUrl, dashboard) if err != nil { klog.Errorf("Could not reconcile (will retry): %v\n", err) } diff --git a/pkg/keycloakClient.go b/pkg/keycloakClient.go new file mode 100644 index 0000000..561e43e --- /dev/null +++ b/pkg/keycloakClient.go @@ -0,0 +1,311 @@ +package controller + +import ( + "encoding/json" + "errors" + "fmt" + "github.com/hashicorp/go-cleanhttp" + "io" + "k8s.io/klog/v2" + "net/http" + "net/url" + "strings" + "sync" +) + +type KeycloakClient struct { + baseURL url.URL + username string + password string + clientId string + realm string + adminGroupPath string + country string + adminGroup *KeycloakGroup + client *http.Client +} + +type KeycloakUser struct { + Id string `json:"id"` + FirstName string `json:"firstName"` + LastName string `json:"lastName"` + Email string `json:"email"` + Username string `json:"username"` +} + +type KeycloakGroup struct { + Id string `json:"id"` + Name string `json:"name"` + Path string `json:"path"` + SubGroups []*KeycloakGroup `json:"subGroups"` + Attributes *map[string][]string `json:"attributes"` +} + +func (this *KeycloakGroup) GetDisplayNameAttribute() string { + if this.Attributes != nil { + displayNames, ok := (*this.Attributes)["displayName"] + if ok && len(displayNames) > 0 { + return displayNames[0] + } + } + return "" +} + +func NewKeycloakClient(baseURL string, realm string, username string, password string, clientId string, adminGroupPath string) (*KeycloakClient, error) { + u, err := url.Parse(baseURL) + if err != nil { + return nil, err + } + + cli := cleanhttp.DefaultClient() + if err != nil { + return nil, err + } + + return &KeycloakClient{ + baseURL: *u, + client: cli, + realm: realm, + username: username, + password: password, + clientId: clientId, + adminGroupPath: adminGroupPath, + }, nil +} + +func (this *KeycloakClient) GetToken() (string, error) { + req, err := http.NewRequest("POST", fmt.Sprintf("%s/auth/realms/%s/protocol/openid-connect/token", this.baseURL.String(), this.realm), nil) + if err != nil { + return "", err + } + + req.Header["Accept"] = []string{"application/json"} + req.Header["Content-Type"] = []string{"application/x-www-form-urlencoded"} + req.Header["cache-control"] = []string{"no-cache"} + + data := fmt.Sprintf("grant_type=password&username=%s&password=%s&client_id=%s", url.QueryEscape(this.username), url.QueryEscape(this.password), url.QueryEscape(this.clientId)) + req.Body = io.NopCloser(strings.NewReader(data)) + + r, err := this.client.Do(req) + if err != nil { + return "", err + } + defer r.Body.Close() + + body, err := io.ReadAll(r.Body) + if err != nil { + return "", err + } + + var objmap map[string]interface{} + json.Unmarshal(body, &objmap) + + accessToken, ok := objmap["access_token"] + if !ok { + return "", errors.New("access_token not found in JSON response") + } + return fmt.Sprintf("%s", accessToken), nil +} + +func (this *KeycloakClient) GetUsers(token string) ([]*KeycloakUser, error) { + req, err := http.NewRequest("GET", fmt.Sprintf("%s/auth/admin/realms/%s/users?max=100000", this.baseURL.String(), this.realm), nil) + if err != nil { + return nil, err + } + + req.Header["Authorization"] = []string{"Bearer " + token} + req.Header["cache-control"] = []string{"no-cache"} + + r, err := this.client.Do(req) + if err != nil { + return nil, err + } + defer r.Body.Close() + + body, err := io.ReadAll(r.Body) + if err != nil { + return nil, err + } + + keycloakUsers := make([]*KeycloakUser, 0) + err = json.Unmarshal(body, &keycloakUsers) + if err != nil { + return nil, err + } + return keycloakUsers, nil +} + +func (this *KeycloakClient) GetGroups(token string) ([]*KeycloakGroup, error) { + req, err := http.NewRequest("GET", fmt.Sprintf("%s/auth/admin/realms/%s/groups?max=100000&briefRepresentation=false", this.baseURL.String(), this.realm), nil) + if err != nil { + return nil, err + } + + req.Header["Authorization"] = []string{"Bearer " + token} + req.Header["cache-control"] = []string{"no-cache"} + + r, err := this.client.Do(req) + if err != nil { + return nil, err + } + defer r.Body.Close() + + body, err := io.ReadAll(r.Body) + if err != nil { + return nil, err + } + + keycloakGroups := make([]*KeycloakGroup, 0) + fmt.Printf("%s", body) + err = json.Unmarshal(body, &keycloakGroups) + if err != nil { + return nil, err + } + return keycloakGroups, nil +} + +func (this *KeycloakClient) GetOrganizations(token string) ([]*KeycloakGroup, error) { + allGroups, err := this.GetGroups(token) + if err != nil { + return nil, err + } + + for _, group := range allGroups { + if group.Path == "/organizations" { + return group.SubGroups, nil + } + } + + return []*KeycloakGroup{}, nil +} + +// FIXME doesn't work properly due to https://github.com/keycloak/keycloak/issues/10348 +func (this *KeycloakClient) GetMembers(token string, group *KeycloakGroup) ([]*KeycloakUser, error) { + req, err := http.NewRequest("GET", fmt.Sprintf("%s/auth/admin/realms/%s/groups/%s/members?max=100000", this.baseURL.String(), this.realm, group.Id), nil) + fmt.Printf("%s/auth/admin/realms/%s/groups/%s/members?max=100000\n", this.baseURL.String(), this.realm, group.Id) + if err != nil { + return nil, err + } + + req.Header["Authorization"] = []string{"Bearer " + token} + req.Header["cache-control"] = []string{"no-cache"} + + r, err := this.client.Do(req) + if err != nil { + return nil, err + } + defer r.Body.Close() + + body, err := io.ReadAll(r.Body) + if err != nil { + return nil, err + } + + keycloakUsers := make([]*KeycloakUser, 0) + err = json.Unmarshal(body, &keycloakUsers) + if err != nil { + return nil, err + } + return keycloakUsers, nil +} + +// FIXME doesn't work properly due to https://github.com/keycloak/keycloak/issues/10348 +// Instead we iterate over all users to fetch their group memberships (see GetGroupMemberships function) +func (this *KeycloakClient) GetAdminUsers(token string) ([]*KeycloakUser, error) { + if this.adminGroup == nil { + groups, err := this.GetGroups(token) + if err != nil { + return nil, err + } + this.findSubgroup(groups) + } + if this.adminGroup == nil { + return nil, errors.New("AdminGroupPath invalid") + } + adminUsers, err := this.GetMembers(token, this.adminGroup) + if err != nil { + return nil, err + } + return adminUsers, nil +} + +func (this *KeycloakClient) findSubgroup(groups []*KeycloakGroup) { + for _, group := range groups { + if group.Path == this.adminGroupPath { + this.adminGroup = group + break + } + this.findSubgroup(group.SubGroups) + if this.adminGroup != nil { + break + } + } +} + +func (this *KeycloakClient) GetGroupMembership(token string, user *KeycloakUser) ([]*KeycloakGroup, error) { + req, err := http.NewRequest("GET", fmt.Sprintf("%s/auth/admin/realms/%s/users/%s/groups", this.baseURL.String(), this.realm, user.Id), nil) + if err != nil { + return nil, err + } + + req.Header["Authorization"] = []string{"Bearer " + token} + req.Header["cache-control"] = []string{"no-cache"} + + response, err := this.client.Do(req) + if err != nil { + return nil, err + } + defer response.Body.Close() + + body, err := io.ReadAll(response.Body) + if err != nil { + return nil, err + } + + groups := make([]*KeycloakGroup, 0) + err = json.Unmarshal(body, &groups) + if err != nil { + return nil, err + } + return groups, nil +} + +func (this *KeycloakClient) worker(token string, userChan chan *KeycloakUser, results *sync.Map, wg *sync.WaitGroup) { + defer wg.Done() + + for user := range userChan { + groups, err := this.GetGroupMembership(token, user) + if err != nil { + klog.Error(err) + } + results.Store(user, groups) + } +} + +func (this *KeycloakClient) GetGroupMemberships(token string, users []*KeycloakUser) (map[*KeycloakUser][]*KeycloakGroup, error) { + results := sync.Map{} + userChan := make(chan *KeycloakUser) + wg := new(sync.WaitGroup) + + // creating workers + for i := 0; i < 5; i++ { + wg.Add(1) + go this.worker(token, userChan, &results, wg) + } + + // sending users to workers + for _, user := range users { + userChan <- user + } + + close(userChan) + wg.Wait() + + userGroups := make(map[*KeycloakUser][]*KeycloakGroup) + results.Range(func(k, v interface{}) bool { + userGroups[k.(*KeycloakUser)] = v.([]*KeycloakGroup) + return true + }) + + return userGroups, nil +} diff --git a/pkg/reconcile.go b/pkg/reconcile.go index 07b5140..21efaaa 100644 --- a/pkg/reconcile.go +++ b/pkg/reconcile.go @@ -3,23 +3,56 @@ package controller import ( "context" "errors" + "fmt" controlapi "github.com/appuio/control-api/apis/v1" grafana "github.com/grafana/grafana-api-golang-client" "k8s.io/client-go/rest" + "k8s.io/klog/v2" ) var ( interruptedError = errors.New("interrupted") ) -func Reconcile(ctx context.Context, organizationAppuioIoClient *rest.RESTClient, appuioIoClient *rest.RESTClient, adminOrg string, grafanaConfig grafana.Config, grafanaUrl string, dashboard map[string]interface{}) error { +func Reconcile(ctx context.Context, keycloakClient *KeycloakClient, adminOrg string, grafanaConfig grafana.Config, grafanaUrl string, dashboard map[string]interface{}) error { + klog.Infof("Fetching Keycloak access token...") + keycloakToken, err := keycloakClient.GetToken() + if err != nil { + return err + } + + klog.Infof("Fetching users from Keycloak...") + keycloakUsers, err := keycloakClient.GetUsers(keycloakToken) + if err != nil { + return err + } + klog.Infof("Found %d users", len(keycloakUsers)) + + klog.Infof("Fetching group memberships from Keycloak...") + keycloakUserGroups, err := keycloakClient.GetGroupMemberships(keycloakToken, keycloakUsers) + if err != nil { + return err + } + memberships := 0 + for _, groups := range keycloakUserGroups { + memberships += len(groups) + } + klog.Infof("Found %d group memberships", memberships) + + klog.Infof("Fetching organizations from Keycloak...") + keycloakOrganizations, err := keycloakClient.GetOrganizations(keycloakToken) + for _, org := range keycloakOrganizations { + fmt.Printf("ORG %s - %s\n", org.Name, org.GetDisplayNameAttribute()) + } + klog.Infof("Found %d organizations", len(keycloakOrganizations)) + grafanaClient, err := NewGrafanaClient(grafanaUrl, grafanaConfig) if err != nil { return err } defer grafanaClient.CloseIdleConnections() - grafanaOrgsMap, err := reconcileAllOrgs(ctx, organizationAppuioIoClient, grafanaClient, dashboard) + /*grafanaOrgsMap, err := reconcileAllOrgs(ctx, organizationAppuioIoClient, grafanaClient, dashboard) if err != nil { return err } @@ -39,7 +72,7 @@ func Reconcile(ctx context.Context, organizationAppuioIoClient *rest.RESTClient, if err != nil { return err } - + */ return nil } diff --git a/pkg/reconcilePermissions.go b/pkg/reconcilePermissions.go index 48040bc..74d1f1e 100644 --- a/pkg/reconcilePermissions.go +++ b/pkg/reconcilePermissions.go @@ -2,6 +2,7 @@ package controller import ( "context" + "fmt" grafana "github.com/grafana/grafana-api-golang-client" "k8s.io/klog/v2" "k8s.io/utils/strings/slices" @@ -38,6 +39,23 @@ func reconcilePermissions(ctx context.Context, grafanaOrgsMap map[string]*grafan desiredOrgUsers = make([]string, 0) } + // debug + fmt.Printf("===== syncing org %s ======\n", org.Name) + fmt.Printf("initial org users: ") + for _, initialOrgUser := range initialOrgUsers { + if initialOrgUser.Role == "Admin" { + continue + } + fmt.Printf("%s, ", initialOrgUser.Login) + } + fmt.Printf("\n") + + fmt.Printf("desired org users: ") + for _, desiredOrgUser := range desiredOrgUsers { + fmt.Printf("%s, ", desiredOrgUser) + } + fmt.Printf("\n") + // Check and if necessary add desired users for _, desiredOrgUser := range desiredOrgUsers { if initialOrgUser, ok := initialOrgUsersMap[desiredOrgUser]; ok {