From 08991d119bc9e6bb9947dd87739a7822ce487a5b Mon Sep 17 00:00:00 2001 From: Sara Lambert Date: Wed, 21 Sep 2022 10:50:08 -0500 Subject: [PATCH 1/3] Add error logging during setupAccount (don't fail) --- apiserver/cmd/server/server.go | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/apiserver/cmd/server/server.go b/apiserver/cmd/server/server.go index 988d1dba..081f72fc 100644 --- a/apiserver/cmd/server/server.go +++ b/apiserver/cmd/server/server.go @@ -898,12 +898,18 @@ func (s *Server) createLMABasicAuthSecret() error { } func (s *Server) setupAccount(account *api.Account) error { - s.kube.CreateNamespace(account.Namespace) - + err := s.kube.CreateNamespace(account.Namespace) + if err != nil { + glog.Error(err) + } + // Create a PVC for this user's data storageClass := s.Config.Kubernetes.StorageClass claimName := account.Namespace + s.Config.HomePvcSuffix - s.kube.CreatePersistentVolumeClaim(account.Namespace, claimName, storageClass) + err = s.kube.CreatePersistentVolumeClaim(account.Namespace, claimName, storageClass) + if err != nil { + glog.Error(err) + } if account.ResourceLimits == (api.AccountResourceLimits{}) { glog.Warningf("No resource limits specified for account %s, using defaults\n", account.Name) @@ -915,12 +921,18 @@ func (s *Server) setupAccount(account *api.Account) error { StorageQuota: s.Config.DefaultLimits.StorageDefault, } } - s.kube.CreateResourceQuota(account.Namespace, + err = s.kube.CreateResourceQuota(account.Namespace, account.ResourceLimits.CPUMax, account.ResourceLimits.MemoryMax) - s.kube.CreateLimitRange(account.Namespace, + if err != nil { + glog.Error(err) + } + err = s.kube.CreateLimitRange(account.Namespace, account.ResourceLimits.CPUDefault, account.ResourceLimits.MemoryDefault) + if err != nil { + glog.Error(err) + } return nil } From 7adaffe9a3c3f148c1b06f7490698afe66ace80f Mon Sep 17 00:00:00 2001 From: Sara Lambert Date: Wed, 21 Sep 2022 13:00:15 -0500 Subject: [PATCH 2/3] fix: better error logging for setupAccount --- apiserver/cmd/server/server.go | 174 ++++++++++++++++----------------- apiserver/pkg/kube/kube.go | 37 ++++--- 2 files changed, 111 insertions(+), 100 deletions(-) diff --git a/apiserver/cmd/server/server.go b/apiserver/cmd/server/server.go index 081f72fc..2c8146f5 100644 --- a/apiserver/cmd/server/server.go +++ b/apiserver/cmd/server/server.go @@ -11,11 +11,11 @@ import ( "net/http" "os" "os/signal" + "regexp" "strconv" "strings" "syscall" "time" - "regexp" "github.com/ndslabs/apiserver/pkg/config" "github.com/ndslabs/apiserver/pkg/email" @@ -538,7 +538,7 @@ func (s *Server) ValidateOAuth(w rest.ResponseWriter, r *rest.Request) { return } - oauth_host := "https://www." + s.Config.Domain + oauth_host := "https://www." + s.Config.Domain oauth_url := oauth_host + "/oauth2/userinfo" glog.Infof("Validating OAuth2 cookie: %s", oauth_url) req, err := http.NewRequest("GET", oauth_url, nil) @@ -548,7 +548,7 @@ func (s *Server) ValidateOAuth(w rest.ResponseWriter, r *rest.Request) { return } - req.Header.Add("Host", "www." + s.Config.Domain) + req.Header.Add("Host", "www."+s.Config.Domain) req.AddCookie(oauth_cookie) client := &http.Client{} @@ -574,7 +574,7 @@ func (s *Server) ValidateOAuth(w rest.ResponseWriter, r *rest.Request) { var oauth_fields map[string]string err = json.Unmarshal(body_bytes, &oauth_fields) - if err != nil { + if err != nil { glog.Errorf("Failed to deserialize JSON: %s\n", oauth_fields) w.WriteHeader(http.StatusUnauthorized) return @@ -587,21 +587,20 @@ func (s *Server) ValidateOAuth(w rest.ResponseWriter, r *rest.Request) { return } - // Fallback to Email prefix if username not available oauth_user := strings.Split(oauth_fields["preferredUsername"], "@")[0] if oauth_user == "" { oauth_user = strings.Split(oauth_email, "@")[0] } - // Make a Regex to say we only want letters and numbers - reg, err := regexp.Compile("[^a-zA-Z0-9]+") - if err != nil { - glog.Fatal(err) - w.WriteHeader(http.StatusUnauthorized) - return - } - oauth_user = reg.ReplaceAllString(oauth_user, "") + // Make a Regex to say we only want letters and numbers + reg, err := regexp.Compile("[^a-zA-Z0-9]+") + if err != nil { + glog.Fatal(err) + w.WriteHeader(http.StatusUnauthorized) + return + } + oauth_user = reg.ReplaceAllString(oauth_user, "") // Fallback to Username if full name not available oauth_name := oauth_fields["name"] @@ -614,78 +613,77 @@ func (s *Server) ValidateOAuth(w rest.ResponseWriter, r *rest.Request) { // TODO: Wire up otherTokens. This is needed for (at least) the MDF Forge Notebook // Assign other tokens, if presented -/* oauth_otherTokenStr := oauth_fields["otherTokens"] - if oauth_otherTokensStr != "" { - tokens := make(map[string]string) - otherTokens := strings.Split(otherTokenStr, " ") - for _, kvpair := range otherTokens { - kv := strings.Split(kvpair, "=") - tokens[kv[0]] = kv[1] - } + /* oauth_otherTokenStr := oauth_fields["otherTokens"] + if oauth_otherTokensStr != "" { + tokens := make(map[string]string) + otherTokens := strings.Split(otherTokenStr, " ") + for _, kvpair := range otherTokens { + kv := strings.Split(kvpair, "=") + tokens[kv[0]] = kv[1] + } - // Write token(s) to user's home directory - err := s.writeAuthPayload(user, tokens) - if err != nil { - glog.Error(err) - w.WriteHeader(http.StatusInternalServerError) - return + // Write token(s) to user's home directory + err := s.writeAuthPayload(user, tokens) + if err != nil { + glog.Error(err) + w.WriteHeader(http.StatusInternalServerError) + return + } } - } -*/ - + */ + // OAuth2 token is valid and contains everything we need, register account if necessary glog.Infof("Creating/updating account for %s %s %s\n", oauth_user, oauth_email, oauth_name) //, oauth_accessToken) -// glog.V(4).Infof("Other tokens %s\n", otherTokens) - + // glog.V(4).Infof("Other tokens %s\n", otherTokens) oauth_account := s.getAccountByEmail(oauth_email) if oauth_account == nil { act := api.Account{ - Name: oauth_name, - Description: "Oauth shadow account", // Fetch this from other OAuth scope info? - Namespace: oauth_user, - EmailAddress: oauth_email, - Password: s.kube.RandomString(10), - Organization: "", // Fetch this from other OAuth scope info? - Created: time.Now().Unix(), - LastLogin: time.Now().Unix(), + Name: oauth_name, + Description: "Oauth shadow account", // Fetch this from other OAuth scope info? + Namespace: oauth_user, + EmailAddress: oauth_email, + Password: s.kube.RandomString(10), + Organization: "", // Fetch this from other OAuth scope info? + Created: time.Now().Unix(), + LastLogin: time.Now().Unix(), InactiveTimeout: s.Config.DefaultLimits.InactiveTimeout, - NextURL: "", // TODO: rd, - } - act.Status = api.AccountStatusApproved - - err := s.etcd.PutAccount(act.Namespace, &act, true) - if err != nil { - glog.Error(err) - w.WriteHeader(http.StatusInternalServerError) - return - } - - err = s.setupAccount(&act) - if err != nil { - glog.Error(err) - w.WriteHeader(http.StatusInternalServerError) - return - } - } else { - oauth_account.LastLogin = time.Now().Unix() - oauth_account.NextURL = "" // TODO: rd - - err := s.etcd.PutAccount(oauth_account.Namespace, oauth_account, true) - if err != nil { - glog.Error(err) - w.WriteHeader(http.StatusInternalServerError) - return - } - } - + NextURL: "", // TODO: rd, + } + act.Status = api.AccountStatusApproved + + err := s.etcd.PutAccount(act.Namespace, &act, true) + if err != nil { + glog.Error(err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + err = s.setupAccount(&act) + if err != nil { + glog.Error(err) + w.WriteHeader(http.StatusInternalServerError) + return + } + } else { + oauth_account.LastLogin = time.Now().Unix() + oauth_account.NextURL = "" // TODO: rd + + err := s.etcd.PutAccount(oauth_account.Namespace, oauth_account, true) + if err != nil { + glog.Error(err) + w.WriteHeader(http.StatusInternalServerError) + return + } + } + // Issue JWT - token, err := s.getTemporaryToken(oauth_user) - if err != nil { - glog.Error(err) - w.WriteHeader(http.StatusUnauthorized) - return - } + token, err := s.getTemporaryToken(oauth_user) + if err != nil { + glog.Error(err) + w.WriteHeader(http.StatusUnauthorized) + return + } w.WriteJson(&token) return @@ -898,15 +896,15 @@ func (s *Server) createLMABasicAuthSecret() error { } func (s *Server) setupAccount(account *api.Account) error { - err := s.kube.CreateNamespace(account.Namespace) + _, err := s.kube.CreateNamespace(account.Namespace) if err != nil { glog.Error(err) } - + // Create a PVC for this user's data storageClass := s.Config.Kubernetes.StorageClass claimName := account.Namespace + s.Config.HomePvcSuffix - err = s.kube.CreatePersistentVolumeClaim(account.Namespace, claimName, storageClass) + _, err = s.kube.CreatePersistentVolumeClaim(account.Namespace, claimName, storageClass) if err != nil { glog.Error(err) } @@ -921,13 +919,13 @@ func (s *Server) setupAccount(account *api.Account) error { StorageQuota: s.Config.DefaultLimits.StorageDefault, } } - err = s.kube.CreateResourceQuota(account.Namespace, + _, err = s.kube.CreateResourceQuota(account.Namespace, account.ResourceLimits.CPUMax, account.ResourceLimits.MemoryMax) if err != nil { glog.Error(err) } - err = s.kube.CreateLimitRange(account.Namespace, + _, err = s.kube.CreateLimitRange(account.Namespace, account.ResourceLimits.CPUDefault, account.ResourceLimits.MemoryDefault) if err != nil { @@ -1770,11 +1768,11 @@ func (s *Server) createIngressRule(userId string, svc *v1.Service, stack *api.St return err } glog.V(4).Infof("Started ingress for service %s (secure=%t)\n", svc.Name, stack.Secure) - if clusterIssuer != "" { - glog.Infof("Using TLS clsuter issuer: %s\n", clusterIssuer) - } else if issuer != "" { - glog.Infof("Using TLS issuer: %s\n", issuer) - } + if clusterIssuer != "" { + glog.Infof("Using TLS clsuter issuer: %s\n", clusterIssuer) + } else if issuer != "" { + glog.Infof("Using TLS issuer: %s\n", issuer) + } return nil } @@ -2426,14 +2424,12 @@ func (s *Server) startStack(userId string, stack *api.Stack) (*api.Stack, error) time.Sleep(time.Second * 3) } - - // To overcome the 503 error on ingress, wait 5 seconds before returning the endpoint time.Sleep(time.Second * 5) stack, err := s.getStackWithStatus(userId, sid) if err != nil { - glog.Errorf("Failed to get stack with status: %s %s\n", userId, sid) - return stack, err + glog.Errorf("Failed to get stack with status: %s %s\n", userId, sid) + return stack, err } stack.Status = "started" for _, stackService := range stack.Services { diff --git a/apiserver/pkg/kube/kube.go b/apiserver/pkg/kube/kube.go index 7b51e9e1..557e22c9 100644 --- a/apiserver/pkg/kube/kube.go +++ b/apiserver/pkg/kube/kube.go @@ -122,7 +122,12 @@ func (k *KubeHelper) CreateNamespace(pid string) (*v1.Namespace, error) { ns := v1.Namespace{} ns.SetName(pid) - return k.kubeGo.CoreV1().Namespaces().Create(&ns) + namespace, err := k.kubeGo.CoreV1().Namespaces().Create(&ns) + if err != nil { + glog.Errorf("Error creating Namespace %s: %s\n", pid, err) + } + + return namespace, err } func (k *KubeHelper) CreateResourceQuota(pid string, cpu int, mem int) (*v1.ResourceQuota, error) { @@ -136,7 +141,12 @@ func (k *KubeHelper) CreateResourceQuota(pid string, cpu int, mem int) (*v1.Reso }, } - return k.kubeGo.CoreV1().ResourceQuotas(pid).Create(&resourceQuota) + quota, err := k.kubeGo.CoreV1().ResourceQuotas(pid).Create(&resourceQuota) + if err != nil { + glog.Errorf("Error creating ResourceQuota %s with mem=%s and cpu=%s: %s\n", pid, mem, cpu, err) + } + + return quota, err } func (k *KubeHelper) CreateLimitRange(pid string, cpu int, mem int) (*v1.LimitRange, error) { @@ -155,7 +165,12 @@ func (k *KubeHelper) CreateLimitRange(pid string, cpu int, mem int) (*v1.LimitRa }, } - return k.kubeGo.CoreV1().LimitRanges(pid).Create(&limitRange) + limrange, err := k.kubeGo.CoreV1().LimitRanges(pid).Create(&limitRange) + if err != nil { + glog.Errorf("Error creating LimitRange %s with mem=%s and cpu=%s: %s\n", pid, mem, cpu, err) + } + + return limrange, err } func (k *KubeHelper) GetNamespace(pid string) (*v1.Namespace, error) { @@ -479,7 +494,7 @@ func (k *KubeHelper) DeleteNetworkPolicy(ns string, name string) error { return k.kubeGo.NetworkingV1().NetworkPolicies(ns).Delete(name, &deleteOptions) } -func (k *KubeHelper) CreatePersistentVolumeClaim(ns string, name string, storageClass string) *v1.PersistentVolumeClaim { +func (k *KubeHelper) CreatePersistentVolumeClaim(ns string, name string, storageClass string) (*v1.PersistentVolumeClaim, error) { k8pvc := v1.PersistentVolumeClaim{} // PersistentVolumeClaim @@ -520,7 +535,7 @@ func (k *KubeHelper) CreatePersistentVolumeClaim(ns string, name string, storage glog.Errorf("Error creating PVC %s in namespace %s: %s\n", name, ns, err) } - return &k8pvc + return &k8pvc, err } func (k *KubeHelper) DeletePersistentVolumeClaim(pid string, name string) error { @@ -913,16 +928,16 @@ func (k *KubeHelper) CreateIngress(pid string, domain string, service string, po } annotations := map[string]string{} - + // TODO: Support configurable ingress class? Applicable for cluster with multiple ingress controllers // annotations["kubernetes.io/ingress.class"] = "nginx" - + if clusterIssuer != "" { - // Check for cert-manager.io/cluster-issuer - annotations["cert-manager.io/cluster-issuer"] = clusterIssuer + // Check for cert-manager.io/cluster-issuer + annotations["cert-manager.io/cluster-issuer"] = clusterIssuer } else if issuer != "" { - // Check for cert-manager.io/issuer - annotations["cert-manager.io/issuer"] = issuer + // Check for cert-manager.io/issuer + annotations["cert-manager.io/issuer"] = issuer } if enableAuth { annotations["ingress.kubernetes.io/auth-signin"] = k.authSignInURL From 3d8726bbc9702c17b33f1b71a238c742b951c832 Mon Sep 17 00:00:00 2001 From: Sara Lambert Date: Fri, 23 Sep 2022 15:05:16 -0500 Subject: [PATCH 3/3] Roll versions forward to 1.3.3 --- apiserver/build.sh | 2 +- gui/ConfigModule.js | 2 +- gui/Dockerfile | 2 +- gui/package.json | 2 +- gui/swagger.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apiserver/build.sh b/apiserver/build.sh index 79abdf10..3388b584 100755 --- a/apiserver/build.sh +++ b/apiserver/build.sh @@ -2,7 +2,7 @@ BUILD_DATE=`date +%Y-%m-%d\ %H:%M` VERSIONFILE="pkg/version/version.go" -VERSION="1.3.2" +VERSION="1.3.3" set -e diff --git a/gui/ConfigModule.js b/gui/ConfigModule.js index 411a9508..003ba688 100644 --- a/gui/ConfigModule.js +++ b/gui/ConfigModule.js @@ -82,7 +82,7 @@ angular /** * The version/revision of this GUI */ -.constant('BuildVersion', '1.3.2-devel') +.constant('BuildVersion', '1.3.3-devel') .constant('BuildDate', '') /** diff --git a/gui/Dockerfile b/gui/Dockerfile index fa518a2c..794d35a8 100644 --- a/gui/Dockerfile +++ b/gui/Dockerfile @@ -4,7 +4,7 @@ FROM ndslabs/ng-base:focal-erbium # Set build information here before building (or at build time with --build-args version=X.X.X) -ARG version="1.3.2" +ARG version="1.3.3" # Set up necessary environment variables ENV DEBIAN_FRONTEND="noninteractive" \ diff --git a/gui/package.json b/gui/package.json index f0f8d95c..16c0e2e2 100644 --- a/gui/package.json +++ b/gui/package.json @@ -1,7 +1,7 @@ { "name": "angular-ndslabs", "private": true, - "version": "1.3.2", + "version": "1.3.3", "description": "AngularJS Client GUI for the NDS Labs API", "repository": "https://github.com/nds-org/ndslabs", "license": "MIT", diff --git a/gui/swagger.yaml b/gui/swagger.yaml index 78bc29b6..4147bae1 100644 --- a/gui/swagger.yaml +++ b/gui/swagger.yaml @@ -6,7 +6,7 @@ info: email: ndslabs-support@nationaldataservice.org name: NDS Labs Support url: http://www.nationaldataservice.org/projects/labs.html - version: 1.3.2 + version: 1.3.3 basePath: /api schemes: - https