From b3d9e09dd4cff3bc93469952328e0544121dc31b Mon Sep 17 00:00:00 2001 From: Marius Date: Thu, 20 May 2021 11:26:26 +0200 Subject: [PATCH 1/9] added mongodb based backend --- .kube/dev-deployment.yaml | 16 +-- config/config.yaml | 6 +- database/handler.go | 263 +++++++++++++++++++------------------- database/handler_test.go | 2 +- database/models.go | 17 ++- endpoints/job.go | 10 +- go.mod | 1 + go.sum | 63 +++++++++ 8 files changed, 216 insertions(+), 162 deletions(-) diff --git a/.kube/dev-deployment.yaml b/.kube/dev-deployment.yaml index 6b5100a..b946236 100644 --- a/.kube/dev-deployment.yaml +++ b/.kube/dev-deployment.yaml @@ -28,27 +28,17 @@ spec: value: harbor.computational.bio.uni-giessen.de/bakta/bakta-web-job:0.1.0-beta.10 - name: BaseKey value: data/dev - - name: DBName - value: bakta - - name: DBPort - value: "3306" - - name: DBUser - value: bakta - - name: DataBucket - value: baktadata - - name: DatabaseHost - value: bakta-mysql.bakta-dev.svc.cluster.local - name: GRPC_GO_LOG_SEVERITY_LEVEL value: debug - name: GRPC_GO_LOG_VERBOSITY_LEVEL value: "2" - name: InCluster value: "true" - - name: DBPassword + - name: MongoPassword valueFrom: secretKeyRef: - key: mysql-password - name: bakta-mysql + key: mongodb-password + name: bakta-mongo-mongodb - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: diff --git a/config/config.yaml b/config/config.yaml index 066c166..dacfdd1 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -4,7 +4,11 @@ Objectstorage: DBBucket: baktadbs BaseKey: data/dev Database: - Backend: Postgres + MongoHost: bakta-mongo-mongodb + MongoDBName: bakta + MongoUser: bakta + MongoUserAuthSource: admin + MongoPort: 27017 K8sNamespace: "bakta-dev" UpdateService: Name: baktawebbackend diff --git a/database/handler.go b/database/handler.go index d27c3a3..5e8c713 100644 --- a/database/handler.go +++ b/database/handler.go @@ -1,12 +1,12 @@ package database import ( + "context" "crypto/rand" "crypto/sha256" "encoding/base64" "errors" "fmt" - "io/ioutil" "log" "os" "path" @@ -15,25 +15,19 @@ import ( "github.com/ag-computational-bio/bakta-web-api-go/api" "github.com/google/uuid" "github.com/spf13/viper" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "go.mongodb.org/mongo-driver/mongo/readpref" "gorm.io/driver/mysql" - "gorm.io/driver/postgres" - "gorm.io/driver/sqlite" "gorm.io/gorm" ) // BackendType The type of the database backend to use type BackendType string -const ( - //SQLite Use an sqlite database in the backend - SQLite BackendType = "SQLite" - //Postgres User a postgres database in the backend - Postgres BackendType = "Postgres" - //Mysql Use a myswql database in the backend - Mysql BackendType = "MySQl" -) - //UploadFileType type of file to upload type UploadFileType string @@ -48,10 +42,12 @@ const ( ) const resultFileName = "results.tar.gz" +const COLLECTIONNAME = "jobs" //Handler Wraps the database with convinence methods type Handler struct { - DB *gorm.DB + DB *mongo.Client + Collection *mongo.Collection BaseKey string UserDataBucket string DBBucket string @@ -60,41 +56,42 @@ type Handler struct { // InitDatabaseHandler Initializes the database to store the Job func InitDatabaseHandler() (*Handler, error) { - - var db *gorm.DB - var err error - - databaseType := viper.GetString("Database.Backend") - - switch databaseType { - case string(SQLite): - db, err = createSQLiteDatabase() - case string(Postgres): - db, err = createPostgresSQL() - case string(Mysql): - db, err = createMySQL() - default: - db, err = createSQLiteDatabase() - } - + host := viper.GetString("MongoHost") + dbName := viper.GetString("MongoDBName") + dbUser := viper.GetString("MongoUser") + dbAuthSource := viper.GetString("MongoUserAuthSource") + dbPassword := getEnvOrPanic("MongoPassword") + dbPort := viper.GetString("MongoPort") + + ctx, _ := context.WithTimeout(context.Background(), 10*time.Second) + client, err := mongo.Connect(ctx, options.Client().ApplyURI(fmt.Sprintf("mongodb://%v:%v", host, dbPort)).SetAuth( + options.Credential{ + AuthSource: dbAuthSource, + Username: dbUser, + Password: dbPassword, + }, + )) if err != nil { log.Println(err.Error()) return nil, err } - err = db.AutoMigrate(&Job{}) + err = client.Ping(ctx, readpref.Primary()) if err != nil { log.Println(err.Error()) return nil, err } + collection := client.Database(dbName).Collection(COLLECTIONNAME) + userBucket := viper.GetString("Objectstorage.S3.UserBucket") dbBucket := viper.GetString("Objectstorage.S3.DBBucket") baseKey := viper.GetString("Objectstorage.S3.BaseKey") expiryTime := viper.GetInt64("ExpiryTime") dbHandler := Handler{ - DB: db, + DB: client, + Collection: collection, UserDataBucket: userBucket, DBBucket: dbBucket, BaseKey: baseKey, @@ -104,21 +101,6 @@ func InitDatabaseHandler() (*Handler, error) { return &dbHandler, nil } -func createSQLiteDatabase() (*gorm.DB, error) { - tmpDir, err := ioutil.TempDir("", "baktadb-*") - if err != nil { - log.Println(err.Error()) - return nil, err - } - db, err := gorm.Open(sqlite.Open(path.Join(tmpDir, "baktadb.db")), &gorm.Config{}) - if err != nil { - log.Println(err.Error()) - return nil, err - } - - return db, nil -} - func createMySQL() (*gorm.DB, error) { host := getEnvOrPanic("DatabaseHost") dbName := getEnvOrPanic("DBName") @@ -130,23 +112,6 @@ func createMySQL() (*gorm.DB, error) { return gorm.Open(mysql.Open(dsn), &gorm.Config{}) } -func createPostgresSQL() (*gorm.DB, error) { - host := getEnvOrPanic("DatabaseHost") - dbName := getEnvOrPanic("DBName") - dbUser := getEnvOrPanic("DBUser") - dbPassword := getEnvOrPanic("DBPassword") - dbPort := getEnvOrPanic("DBPort") - - dsn := fmt.Sprintf("host=%v user=%v password=%v dbname=%v port=%v sslmode=disable TimeZone=Europe/Berlin", host, dbUser, dbPassword, dbName, dbPort) - db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{}) - if err != nil { - log.Println(err.Error()) - return nil, err - } - - return db, nil -} - //CreateJob Creates a new bakta job in init mode func (handler *Handler) CreateJob(repliconTypeAPI api.RepliconTableType) (*Job, string, error) { jobID := uuid.New() @@ -177,59 +142,91 @@ func (handler *Handler) CreateJob(repliconTypeAPI api.RepliconTableType) (*Job, RepliconKey: handler.createUploadStoreKey(jobID.String(), repliconType), ResultKey: handler.createResultStoreKey(jobID.String()), Status: api.JobStatusEnum_INIT.String(), - ExpiryDate: time.Now().AddDate(0, 0, 10), + ExpiryDate: primitive.Timestamp{T: uint32(time.Now().AddDate(0, 0, 10).Unix())}, + Created: primitive.Timestamp{T: uint32(time.Now().Unix())}, + Updated: primitive.Timestamp{T: uint32(time.Now().Unix())}, } - result := handler.DB.Create(&job) - if result.Error != nil { - log.Println(result.Error) + ctx, _ := context.WithTimeout(context.Background(), 1*time.Second) + + inserted, err := handler.Collection.InsertOne(ctx, job) + if err != nil { + log.Println(err.Error()) return nil, "", err } - getDataResult := handler.DB.First(&job) - if getDataResult.Error != nil { - log.Println(getDataResult.Error) + inserted_query := bson.M{ + "_id": inserted.InsertedID, + } + + result := handler.Collection.FindOne(ctx, inserted_query) + if result.Err() != nil { + log.Println(result.Err().Error()) + return nil, "", result.Err() + } + + inserted_Job := Job{} + + err = result.Decode(&inserted_Job) + if err != nil { + log.Println(err.Error()) return nil, "", err } - return &job, secretID, nil + return &inserted_Job, secretID, nil } //UpdateK8s Updates a job with its k8s id func (handler *Handler) UpdateK8s(id string, k8s string, conf string) error { - job := Job{ - JobID: id, - ConfString: conf, + ctx, _ := context.WithTimeout(context.Background(), 1*time.Second) + + update_filter := bson.M{ + "JobID": id, } - getDataResult := handler.DB.First(&job) - if getDataResult.Error != nil { - log.Println(getDataResult.Error) - return getDataResult.Error + update := bson.M{ + "K8sID": k8s, + "Status": api.JobStatusEnum_RUNNING.String(), } - job.K8sID = k8s - job.Status = api.JobStatusEnum_RUNNING.String() + result, err := handler.Collection.UpdateOne(ctx, update_filter, update) + if err != nil { + log.Println(err.Error()) + return err + } - handler.DB.Save(&job) + if result.ModifiedCount != 1 { + err := fmt.Errorf("wrong number of updated job entries found when updating job: %v", id) + log.Println(err.Error()) + return err + } return nil } //UpdateStatus Updates the status of a job func (handler *Handler) UpdateStatus(id string, status api.JobStatusEnum, errorMsg string, isDeleted bool) error { - job := Job{ - JobID: id, + ctx, _ := context.WithTimeout(context.Background(), 1*time.Second) + + update_filter := bson.M{ + "JobID": id, + } + + update := bson.M{ + "Error": errorMsg, + "Status": api.JobStatusEnum_RUNNING.String(), } - if isDeleted { - job.IsDeleted = true + result, err := handler.Collection.UpdateOne(ctx, update_filter, update) + if err != nil { + log.Println(err.Error()) + return err } - updateResult := handler.DB.Model(&job).Updates(Job{Status: status.String(), Error: errorMsg}) - if updateResult.Error != nil { - log.Println(updateResult.Error) - return updateResult.Error + if result.ModifiedCount != 1 { + err := fmt.Errorf("wrong number of updated job entries found when updating job: %v", id) + log.Println(err.Error()) + return err } return nil @@ -237,13 +234,24 @@ func (handler *Handler) UpdateStatus(id string, status api.JobStatusEnum, errorM //GetJob Returns the stored config of a job func (handler *Handler) GetJob(id string) (*Job, error) { + ctx, _ := context.WithTimeout(context.Background(), 1*time.Second) + + find_query := bson.M{ + "JobID": id, + } + + result := handler.Collection.FindOne(ctx, find_query) + if result.Err() != nil { + log.Println(result.Err().Error()) + return nil, result.Err() + } + job := Job{} - job.JobID = id - result := handler.DB.First(&job) - if result.Error != nil { - log.Println(result.Error) - return nil, result.Error + err := result.Decode(&job) + if err != nil { + log.Println(err.Error()) + return nil, err } return &job, nil @@ -251,14 +259,10 @@ func (handler *Handler) GetJob(id string) (*Job, error) { //CheckSecret Compares the provided secret/JobID with a job in the database func (handler *Handler) CheckSecret(id string, secretKey string) error { - job := Job{ - JobID: id, - } - - getDataResult := handler.DB.First(&job) - if getDataResult.Error != nil { - log.Println(getDataResult.Error) - return getDataResult.Error + job, err := handler.GetJob(id) + if err != nil { + log.Println(err.Error()) + return err } secretSHA := sha256.Sum256([]byte(secretKey)) @@ -272,15 +276,25 @@ func (handler *Handler) CheckSecret(id string, secretKey string) error { } // GetJobsStatus Returns the status of a list of jobs -func (handler *Handler) GetJobsStatus(jobIDs []string) ([]Job, error) { +func (handler *Handler) GetJobs(jobIDs []string) ([]Job, error) { var jobs []Job + ctx, _ := context.WithTimeout(context.Background(), 1*time.Second) - current_time := time.Now().Unix() - handler.ExpiryTime + find_query := bson.M{ + "JobID": bson.M{ + "$in": jobIDs, + }, + } + + csr, err := handler.Collection.Find(ctx, find_query) + if err != nil { + log.Println(err.Error()) + return nil, err + } - connection := handler.DB.Where("job_id IN ? AND created > ?", jobIDs, current_time).Find(&jobs) - if connection.Error != nil { - log.Println(connection.Error) - return nil, connection.Error + if err = csr.All(ctx, &jobs); err != nil { + log.Println(err.Error()) + return nil, err } return jobs, nil @@ -288,34 +302,13 @@ func (handler *Handler) GetJobsStatus(jobIDs []string) ([]Job, error) { // GetJobStatus Returns the status of an individual job func (handler *Handler) GetJobStatus(jobID string) (*Job, error) { - job := Job{} - - current_time := time.Now().Unix() - handler.ExpiryTime - - connection := handler.DB.Where("job_id=? AND created > ?", jobID, current_time).Find(&job) - if connection.Error != nil { - log.Println(connection.Error) - return nil, connection.Error - } - - return &job, nil -} - -func (handler *Handler) GetRunningJobs() ([]*Job, error) { - running_status := []string{ - api.JobStatusEnum_RUNNING.String(), - api.JobStatusEnum_INIT.String(), - } - - jobs := make([]*Job, 0) - - connection := handler.DB.Where("status in ?", running_status).Find(jobs) - if connection.Error != nil { - log.Println(connection.Error) - return nil, connection.Error + job, err := handler.GetJob(jobID) + if err != nil { + log.Println(err.Error()) + return nil, err } - return jobs, nil + return job, nil } func (handler *Handler) createUploadStoreKey(id string, uploadFileType UploadFileType) string { diff --git a/database/handler_test.go b/database/handler_test.go index 36ee855..80fdc04 100644 --- a/database/handler_test.go +++ b/database/handler_test.go @@ -26,7 +26,7 @@ func TestDatabaseHandler(t *testing.T) { jobIDs = append(jobIDs, job1.JobID) jobIDs = append(jobIDs, job2.JobID) - _, err = databaseHandler.GetJobsStatus(jobIDs) + _, err = databaseHandler.GetJobs(jobIDs) if err != nil { t.Errorf(err.Error()) } diff --git a/database/models.go b/database/models.go index 189b1a2..7b9f617 100644 --- a/database/models.go +++ b/database/models.go @@ -1,22 +1,25 @@ package database -import "time" +import ( + "go.mongodb.org/mongo-driver/bson/primitive" +) // Job The database model for a bakta job type Job struct { - JobID string `gorm:"primaryKey"` + _id primitive.ObjectID + JobID string Secret string - K8sID string `gorm:"index"` - Updated int64 `gorm:"autoUpdateTime"` // Use unix milli seconds as updating time - Created int64 `gorm:"autoCreateTime"` // Use unix seconds as creating time - Status string `gorm:"index"` + K8sID string + Updated primitive.Timestamp + Created primitive.Timestamp + Status string DataBucket string FastaKey string ProdigalKey string RepliconKey string ResultKey string Error string - ExpiryDate time.Time + ExpiryDate primitive.Timestamp ConfString string IsDeleted bool } diff --git a/endpoints/job.go b/endpoints/job.go index 0c0abaa..4002c16 100644 --- a/endpoints/job.go +++ b/endpoints/job.go @@ -166,7 +166,7 @@ func (apiHandler *BaktaJobAPI) GetJobsStatus(ctx context.Context, request *api.J } } - jobs, err := apiHandler.dbHandler.GetJobsStatus(jobIDs) + jobs, err := apiHandler.dbHandler.GetJobs(jobIDs) if err != nil { log.Println(err.Error()) return nil, err @@ -182,8 +182,8 @@ func (apiHandler *BaktaJobAPI) GetJobsStatus(ctx context.Context, request *api.J statusEnum := api.JobStatusEnum(statusNumber) - created_time := timestamppb.New(time.Unix(job.Created, 0)) - updated_time := timestamppb.New(time.Unix(job.Updated, 0)) + created_time := timestamppb.New(time.Unix(int64(job.Created.T), 0)) + updated_time := timestamppb.New(time.Unix(int64(job.Updated.T), 0)) statusResponse := api.JobStatusResponse{ JobID: job.JobID, @@ -236,8 +236,8 @@ func (apiHandler *BaktaJobAPI) GetJobResult(ctx context.Context, request *api.Jo return nil, err } - created_time := timestamppb.New(time.Unix(job.Created, 0)) - updated_time := timestamppb.New(time.Unix(job.Updated, 0)) + created_time := timestamppb.New(time.Unix(int64(job.Created.T), 0)) + updated_time := timestamppb.New(time.Unix(int64(job.Updated.T), 0)) jobResponse := api.JobResultResponse{ JobID: job.JobID, diff --git a/go.mod b/go.mod index 1c0ff95..0ef1681 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/spf13/cast v1.3.1 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/viper v1.7.1 + go.mongodb.org/mongo-driver v1.5.2 // indirect golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a // indirect golang.org/x/net v0.0.0-20210510120150-4163338589ed // indirect golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c // indirect diff --git a/go.sum b/go.sum index 2dd717e..0e3ab8d 100644 --- a/go.sum +++ b/go.sum @@ -72,6 +72,8 @@ github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6l github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.34.28 h1:sscPpn/Ns3i0F4HPEWAVcwdIRaZZCuL7llJ2/60yPIk= +github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aws/aws-sdk-go-v2 v1.4.0 h1:Ryh4fNebT9SwLyCKPSk83dyEZj+KB6KzDyb1gXii7EI= github.com/aws/aws-sdk-go-v2 v1.4.0/go.mod h1:tI4KhsR5VkzlUa2DZAdwx7wCAYGwkZZ1H31PYrBFx1w= @@ -200,9 +202,35 @@ github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8 github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= +github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= +github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= +github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= +github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= +github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= +github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= +github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= +github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= +github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= +github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= +github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= +github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= +github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= +github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= +github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/gofrs/flock v0.8.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= @@ -245,6 +273,8 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -396,8 +426,10 @@ github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/ github.com/jinzhu/now v1.1.2 h1:eVKgfIdy9b6zbWBMgFpfDPoAMifwSZagU9HmEU6zgiI= github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -410,9 +442,13 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= +github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.11.7 h1:0hzRabrMN4tSTvMfnL3SCv1ZGeAP23ynzodBgaHeMeg= github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -440,6 +476,8 @@ github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaW github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= +github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= @@ -474,6 +512,7 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= @@ -511,6 +550,7 @@ github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIw github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/pelletier/go-toml v1.9.1 h1:a6qW1EVNZWH9WGI6CsYdD8WAylkoXBS5yv0XHlh17Tc= github.com/pelletier/go-toml v1.9.1/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= @@ -551,6 +591,8 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= @@ -565,6 +607,7 @@ github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc h1:jUIKcSPO9MoM github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= @@ -610,12 +653,21 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/twitchtv/twirp v7.1.0+incompatible/go.mod h1:RRJoFSAmTEh2weEqWtpPE3vFK5YBhA6bqp2l1kfCC5A= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.0.2 h1:akYIkZ28e6A96dkWNJQu3nmCzH3YfwMPQExUYDaRv7w= +github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= +github.com/xdg-go/stringprep v1.0.2 h1:6iq84/ryjjeRmMJwxutI51F2GIPlP5BfTvXHeYjyhBc= +github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -624,6 +676,8 @@ github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxt go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= +go.mongodb.org/mongo-driver v1.5.2 h1:AsxOLoJTgP6YNM0fXWw4OjdluYmWzQYp+lFJL7xu9fU= +go.mongodb.org/mongo-driver v1.5.2/go.mod h1:gRXCHX4Jo7J0IJ1oDQyUxF7jfy19UfxniMS4xxMmUqw= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= @@ -650,6 +704,7 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -657,6 +712,7 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -754,6 +810,7 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -774,9 +831,11 @@ golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -844,10 +903,14 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= From a795bf082e41d871eea2812a87dedd9c99b97df5 Mon Sep 17 00:00:00 2001 From: Marius Date: Thu, 20 May 2021 11:54:18 +0200 Subject: [PATCH 2/9] changed to lowercase queries --- .kube/dev-deployment.yaml | 4 ++-- database/handler.go | 30 ++++++++---------------------- 2 files changed, 10 insertions(+), 24 deletions(-) diff --git a/.kube/dev-deployment.yaml b/.kube/dev-deployment.yaml index b946236..328073f 100644 --- a/.kube/dev-deployment.yaml +++ b/.kube/dev-deployment.yaml @@ -25,7 +25,7 @@ spec: containers: - env: - name: JobContainer - value: harbor.computational.bio.uni-giessen.de/bakta/bakta-web-job:0.1.0-beta.10 + value: harbor.computational.bio.uni-giessen.de/bakta/bakta-web-job:0.1.0-beta.11 - name: BaseKey value: data/dev - name: GRPC_GO_LOG_SEVERITY_LEVEL @@ -54,7 +54,7 @@ spec: secretKeyRef: key: Token name: authhandler - image: harbor.computational.bio.uni-giessen.de/bakta/bakta-web-backend:0.2.9-beta.17 + image: harbor.computational.bio.uni-giessen.de/bakta/bakta-web-backend:0.2.9-beta.19 imagePullPolicy: Always name: baktabackend ports: diff --git a/database/handler.go b/database/handler.go index 5e8c713..307e77a 100644 --- a/database/handler.go +++ b/database/handler.go @@ -20,9 +20,6 @@ import ( "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" "go.mongodb.org/mongo-driver/mongo/readpref" - - "gorm.io/driver/mysql" - "gorm.io/gorm" ) // BackendType The type of the database backend to use @@ -101,17 +98,6 @@ func InitDatabaseHandler() (*Handler, error) { return &dbHandler, nil } -func createMySQL() (*gorm.DB, error) { - host := getEnvOrPanic("DatabaseHost") - dbName := getEnvOrPanic("DBName") - dbUser := getEnvOrPanic("DBUser") - dbPassword := getEnvOrPanic("DBPassword") - dbPort := getEnvOrPanic("DBPort") - - dsn := fmt.Sprintf("%v:%v@tcp(%v:%v)/%v?charset=utf8mb4&parseTime=True&loc=Local", dbUser, dbPassword, host, dbPort, dbName) - return gorm.Open(mysql.Open(dsn), &gorm.Config{}) -} - //CreateJob Creates a new bakta job in init mode func (handler *Handler) CreateJob(repliconTypeAPI api.RepliconTableType) (*Job, string, error) { jobID := uuid.New() @@ -181,12 +167,12 @@ func (handler *Handler) UpdateK8s(id string, k8s string, conf string) error { ctx, _ := context.WithTimeout(context.Background(), 1*time.Second) update_filter := bson.M{ - "JobID": id, + "jobid": id, } update := bson.M{ - "K8sID": k8s, - "Status": api.JobStatusEnum_RUNNING.String(), + "k8sid": k8s, + "status": api.JobStatusEnum_RUNNING.String(), } result, err := handler.Collection.UpdateOne(ctx, update_filter, update) @@ -209,12 +195,12 @@ func (handler *Handler) UpdateStatus(id string, status api.JobStatusEnum, errorM ctx, _ := context.WithTimeout(context.Background(), 1*time.Second) update_filter := bson.M{ - "JobID": id, + "jobid": id, } update := bson.M{ - "Error": errorMsg, - "Status": api.JobStatusEnum_RUNNING.String(), + "error": errorMsg, + "status": api.JobStatusEnum_RUNNING.String(), } result, err := handler.Collection.UpdateOne(ctx, update_filter, update) @@ -237,7 +223,7 @@ func (handler *Handler) GetJob(id string) (*Job, error) { ctx, _ := context.WithTimeout(context.Background(), 1*time.Second) find_query := bson.M{ - "JobID": id, + "jobid": id, } result := handler.Collection.FindOne(ctx, find_query) @@ -281,7 +267,7 @@ func (handler *Handler) GetJobs(jobIDs []string) ([]Job, error) { ctx, _ := context.WithTimeout(context.Background(), 1*time.Second) find_query := bson.M{ - "JobID": bson.M{ + "jobid": bson.M{ "$in": jobIDs, }, } From 33a071273b40cba0bd728279443d701a24f37a23 Mon Sep 17 00:00:00 2001 From: Marius Date: Thu, 20 May 2021 12:07:44 +0200 Subject: [PATCH 3/9] fixed update query --- .kube/dev-deployment.yaml | 2 +- database/handler.go | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.kube/dev-deployment.yaml b/.kube/dev-deployment.yaml index 328073f..117bc9d 100644 --- a/.kube/dev-deployment.yaml +++ b/.kube/dev-deployment.yaml @@ -54,7 +54,7 @@ spec: secretKeyRef: key: Token name: authhandler - image: harbor.computational.bio.uni-giessen.de/bakta/bakta-web-backend:0.2.9-beta.19 + image: harbor.computational.bio.uni-giessen.de/bakta/bakta-web-backend:0.2.9-beta.21 imagePullPolicy: Always name: baktabackend ports: diff --git a/database/handler.go b/database/handler.go index 307e77a..a4dc2dd 100644 --- a/database/handler.go +++ b/database/handler.go @@ -171,8 +171,10 @@ func (handler *Handler) UpdateK8s(id string, k8s string, conf string) error { } update := bson.M{ - "k8sid": k8s, - "status": api.JobStatusEnum_RUNNING.String(), + "$set": bson.M{ + "k8sid": k8s, + "status": api.JobStatusEnum_RUNNING.String(), + }, } result, err := handler.Collection.UpdateOne(ctx, update_filter, update) @@ -199,8 +201,10 @@ func (handler *Handler) UpdateStatus(id string, status api.JobStatusEnum, errorM } update := bson.M{ - "error": errorMsg, - "status": api.JobStatusEnum_RUNNING.String(), + "$set": bson.M{ + "error": errorMsg, + "status": api.JobStatusEnum_RUNNING.String(), + }, } result, err := handler.Collection.UpdateOne(ctx, update_filter, update) From 105a6212f04a67031d2547bb98a695e33435116c Mon Sep 17 00:00:00 2001 From: Marius Date: Thu, 20 May 2021 12:19:20 +0200 Subject: [PATCH 4/9] changed check of update status update --- .kube/dev-deployment.yaml | 2 +- database/handler.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.kube/dev-deployment.yaml b/.kube/dev-deployment.yaml index 117bc9d..fce8bf1 100644 --- a/.kube/dev-deployment.yaml +++ b/.kube/dev-deployment.yaml @@ -54,7 +54,7 @@ spec: secretKeyRef: key: Token name: authhandler - image: harbor.computational.bio.uni-giessen.de/bakta/bakta-web-backend:0.2.9-beta.21 + image: harbor.computational.bio.uni-giessen.de/bakta/bakta-web-backend:0.2.9-beta.23 imagePullPolicy: Always name: baktabackend ports: diff --git a/database/handler.go b/database/handler.go index a4dc2dd..0b7760d 100644 --- a/database/handler.go +++ b/database/handler.go @@ -203,7 +203,7 @@ func (handler *Handler) UpdateStatus(id string, status api.JobStatusEnum, errorM update := bson.M{ "$set": bson.M{ "error": errorMsg, - "status": api.JobStatusEnum_RUNNING.String(), + "status": status.String(), }, } @@ -213,7 +213,7 @@ func (handler *Handler) UpdateStatus(id string, status api.JobStatusEnum, errorM return err } - if result.ModifiedCount != 1 { + if result.MatchedCount != 1 { err := fmt.Errorf("wrong number of updated job entries found when updating job: %v", id) log.Println(err.Error()) return err From b347fdadeb0c3814a448170225b292bfd6cee495 Mon Sep 17 00:00:00 2001 From: Marius Date: Thu, 20 May 2021 13:00:52 +0200 Subject: [PATCH 5/9] missing update check fixed --- .kube/dev-deployment.yaml | 2 +- database/handler.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.kube/dev-deployment.yaml b/.kube/dev-deployment.yaml index fce8bf1..f7706c6 100644 --- a/.kube/dev-deployment.yaml +++ b/.kube/dev-deployment.yaml @@ -54,7 +54,7 @@ spec: secretKeyRef: key: Token name: authhandler - image: harbor.computational.bio.uni-giessen.de/bakta/bakta-web-backend:0.2.9-beta.23 + image: harbor.computational.bio.uni-giessen.de/bakta/bakta-web-backend:0.2.9-beta.26 imagePullPolicy: Always name: baktabackend ports: diff --git a/database/handler.go b/database/handler.go index 0b7760d..12a9fbc 100644 --- a/database/handler.go +++ b/database/handler.go @@ -183,7 +183,7 @@ func (handler *Handler) UpdateK8s(id string, k8s string, conf string) error { return err } - if result.ModifiedCount != 1 { + if result.MatchedCount != 1 { err := fmt.Errorf("wrong number of updated job entries found when updating job: %v", id) log.Println(err.Error()) return err From 1c3304ca5bd8671128107af1e4d9700455243971 Mon Sep 17 00:00:00 2001 From: Marius Date: Thu, 20 May 2021 16:09:27 +0200 Subject: [PATCH 6/9] improved monitoring code performance --- .kube/dev-deployment.yaml | 2 +- config/config.yaml | 11 ++++--- database/handler.go | 18 +++++++---- database/handler_test.go | 4 --- endpoints/job.go | 63 +-------------------------------------- 5 files changed, 19 insertions(+), 79 deletions(-) diff --git a/.kube/dev-deployment.yaml b/.kube/dev-deployment.yaml index f7706c6..0dee26a 100644 --- a/.kube/dev-deployment.yaml +++ b/.kube/dev-deployment.yaml @@ -54,7 +54,7 @@ spec: secretKeyRef: key: Token name: authhandler - image: harbor.computational.bio.uni-giessen.de/bakta/bakta-web-backend:0.2.9-beta.26 + image: harbor.computational.bio.uni-giessen.de/bakta/bakta-web-backend:0.2.9-beta.27 imagePullPolicy: Always name: baktabackend ports: diff --git a/config/config.yaml b/config/config.yaml index dacfdd1..a574908 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -3,12 +3,11 @@ Objectstorage: UserBucket: baktauserdata DBBucket: baktadbs BaseKey: data/dev -Database: - MongoHost: bakta-mongo-mongodb - MongoDBName: bakta - MongoUser: bakta - MongoUserAuthSource: admin - MongoPort: 27017 +MongoHost: localhost +MongoDBName: bakta +MongoUser: bakta +MongoUserAuthSource: bakta +MongoPort: 27017 K8sNamespace: "bakta-dev" UpdateService: Name: baktawebbackend diff --git a/database/handler.go b/database/handler.go index 12a9fbc..329afbe 100644 --- a/database/handler.go +++ b/database/handler.go @@ -60,7 +60,7 @@ func InitDatabaseHandler() (*Handler, error) { dbPassword := getEnvOrPanic("MongoPassword") dbPort := viper.GetString("MongoPort") - ctx, _ := context.WithTimeout(context.Background(), 10*time.Second) + ctx, _ := context.WithTimeout(context.Background(), 1*time.Second) client, err := mongo.Connect(ctx, options.Client().ApplyURI(fmt.Sprintf("mongodb://%v:%v", host, dbPort)).SetAuth( options.Credential{ AuthSource: dbAuthSource, @@ -134,7 +134,6 @@ func (handler *Handler) CreateJob(repliconTypeAPI api.RepliconTableType) (*Job, } ctx, _ := context.WithTimeout(context.Background(), 1*time.Second) - inserted, err := handler.Collection.InsertOne(ctx, job) if err != nil { log.Println(err.Error()) @@ -266,14 +265,21 @@ func (handler *Handler) CheckSecret(id string, secretKey string) error { } // GetJobsStatus Returns the status of a list of jobs -func (handler *Handler) GetJobs(jobIDs []string) ([]Job, error) { +func (handler *Handler) GetJobs(jobIDs []*api.JobAuth) ([]Job, error) { var jobs []Job ctx, _ := context.WithTimeout(context.Background(), 1*time.Second) + var jobRequests []bson.M + + for _, jobRequest := range jobIDs { + jobRequests = append(jobRequests, bson.M{ + "jobid": jobRequest.JobID, + "secret": jobRequest.Secret, + }) + } + find_query := bson.M{ - "jobid": bson.M{ - "$in": jobIDs, - }, + "$or": jobRequests, } csr, err := handler.Collection.Find(ctx, find_query) diff --git a/database/handler_test.go b/database/handler_test.go index 80fdc04..6f5d53d 100644 --- a/database/handler_test.go +++ b/database/handler_test.go @@ -26,10 +26,6 @@ func TestDatabaseHandler(t *testing.T) { jobIDs = append(jobIDs, job1.JobID) jobIDs = append(jobIDs, job2.JobID) - _, err = databaseHandler.GetJobs(jobIDs) - if err != nil { - t.Errorf(err.Error()) - } } func TestRndBytes(t *testing.T) { diff --git a/endpoints/job.go b/endpoints/job.go index 4002c16..682ee01 100644 --- a/endpoints/job.go +++ b/endpoints/job.go @@ -13,7 +13,6 @@ import ( "github.com/ag-computational-bio/bakta-web-backend/objectStorage" "google.golang.org/protobuf/types/known/structpb" "google.golang.org/protobuf/types/known/timestamppb" - "k8s.io/apimachinery/pkg/api/errors" "github.com/ag-computational-bio/bakta-web-backend/database" "github.com/ag-computational-bio/bakta-web-backend/scheduler" @@ -104,69 +103,9 @@ func (apiHandler *BaktaJobAPI) StartJob(ctx context.Context, request *api.StartJ //GetJobsStatus Get the job status of the provided list of jobs func (apiHandler *BaktaJobAPI) GetJobsStatus(ctx context.Context, request *api.JobStatusRequestList) (*api.JobStatusReponseList, error) { - var jobIDs []string - var failedJobs []*api.FailedJob - for _, jobID := range request.GetJobs() { - isDeleted := false - - err := apiHandler.dbHandler.CheckSecret(jobID.GetJobID(), jobID.GetSecret()) - if err != nil { - failedJob := api.FailedJob{ - JobID: jobID.JobID, - JobStatus: api.JobFailedStatus_UNAUTHORIZED, - } - failedJobs = append(failedJobs, &failedJob) - continue - } - - job, err := apiHandler.dbHandler.GetJob(jobID.GetJobID()) - if err != nil { - failedJob := api.FailedJob{ - JobID: jobID.JobID, - JobStatus: api.JobFailedStatus_NOT_FOUND, - } - failedJobs = append(failedJobs, &failedJob) - continue - } - - jobIDs = append(jobIDs, jobID.JobID) - - if job.IsDeleted { - continue - } - - newStatus, err := apiHandler.monitor.GetJobStatus(jobID.GetJobID()) - if err != nil && errors.IsNotFound(err) { - continue - } - - if err != nil { - err = fmt.Errorf("could not get updated job status") - return nil, err - } - - if newStatus.Status == api.JobStatusEnum_SUCCESSFULL || newStatus.Status == api.JobStatusEnum_ERROR { - err := apiHandler.scheduler.DeleteJob(job.JobID) - if err != nil { - err = fmt.Errorf("could not get updated job status") - return nil, err - } - - isDeleted = true - } - - if job.Status != newStatus.Status.String() { - err = apiHandler.dbHandler.UpdateStatus(jobID.GetJobID(), newStatus.Status, newStatus.ErrorMsg, isDeleted) - if err != nil { - err = fmt.Errorf("could not update job status") - return nil, err - } - } - } - - jobs, err := apiHandler.dbHandler.GetJobs(jobIDs) + jobs, err := apiHandler.dbHandler.GetJobs(request.GetJobs()) if err != nil { log.Println(err.Error()) return nil, err From 5ba166638222b16ddb358c4e27ffdd86a4beacd7 Mon Sep 17 00:00:00 2001 From: Marius Date: Thu, 20 May 2021 16:28:17 +0200 Subject: [PATCH 7/9] fixed wrong id handover in update call --- .kube/dev-deployment.yaml | 2 +- endpoints/update.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.kube/dev-deployment.yaml b/.kube/dev-deployment.yaml index 0dee26a..9c5a404 100644 --- a/.kube/dev-deployment.yaml +++ b/.kube/dev-deployment.yaml @@ -54,7 +54,7 @@ spec: secretKeyRef: key: Token name: authhandler - image: harbor.computational.bio.uni-giessen.de/bakta/bakta-web-backend:0.2.9-beta.27 + image: harbor.computational.bio.uni-giessen.de/bakta/bakta-web-backend:0.2.9-beta.28 imagePullPolicy: Always name: baktabackend ports: diff --git a/endpoints/update.go b/endpoints/update.go index d12c80b..6381a07 100644 --- a/endpoints/update.go +++ b/endpoints/update.go @@ -31,7 +31,7 @@ func (apiHandler *BaktaUpdateAPI) UpdateStatus(ctx context.Context, request *api return } - status, err := apiHandler.updateMonitor.GetJobStatus(request.GetJobID()) + status, err := apiHandler.updateMonitor.GetJobStatus(job.K8sID) if err != nil { log.Println(err.Error()) return @@ -41,7 +41,7 @@ func (apiHandler *BaktaUpdateAPI) UpdateStatus(ctx context.Context, request *api if status.Status == api.JobStatusEnum_SUCCESSFULL || status.Status == api.JobStatusEnum_ERROR { if !job.IsDeleted { - err = apiHandler.scheduler.DeleteJob(job.JobID) + err = apiHandler.scheduler.DeleteJob(job.K8sID) if err != nil { log.Println(err.Error()) return From e15643ed5f1b8ee0cd24c2e33d487564079f56ed Mon Sep 17 00:00:00 2001 From: Marius Date: Thu, 20 May 2021 16:43:22 +0200 Subject: [PATCH 8/9] changed job status check logic to avoid race conditions --- .kube/dev-deployment.yaml | 2 +- endpoints/update.go | 19 +++++++++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/.kube/dev-deployment.yaml b/.kube/dev-deployment.yaml index 9c5a404..d098e10 100644 --- a/.kube/dev-deployment.yaml +++ b/.kube/dev-deployment.yaml @@ -54,7 +54,7 @@ spec: secretKeyRef: key: Token name: authhandler - image: harbor.computational.bio.uni-giessen.de/bakta/bakta-web-backend:0.2.9-beta.28 + image: harbor.computational.bio.uni-giessen.de/bakta/bakta-web-backend:0.2.9-beta.29 imagePullPolicy: Always name: baktabackend ports: diff --git a/endpoints/update.go b/endpoints/update.go index 6381a07..4763f59 100644 --- a/endpoints/update.go +++ b/endpoints/update.go @@ -3,6 +3,7 @@ package endpoints import ( "context" "log" + "time" "github.com/ag-computational-bio/bakta-web-api-go/api" "github.com/ag-computational-bio/bakta-web-backend/database" @@ -31,7 +32,7 @@ func (apiHandler *BaktaUpdateAPI) UpdateStatus(ctx context.Context, request *api return } - status, err := apiHandler.updateMonitor.GetJobStatus(job.K8sID) + status, err := apiHandler.updateMonitor.GetJobStatus(job.JobID) if err != nil { log.Println(err.Error()) return @@ -39,9 +40,23 @@ func (apiHandler *BaktaUpdateAPI) UpdateStatus(ctx context.Context, request *api isDeleted := false + for { + status, err = apiHandler.updateMonitor.GetJobStatus(job.JobID) + if err != nil { + log.Println(err.Error()) + return + } + + if status.Status != api.JobStatusEnum_RUNNING { + break + } + + time.Sleep(500 * time.Millisecond) + } + if status.Status == api.JobStatusEnum_SUCCESSFULL || status.Status == api.JobStatusEnum_ERROR { if !job.IsDeleted { - err = apiHandler.scheduler.DeleteJob(job.K8sID) + err = apiHandler.scheduler.DeleteJob(job.JobID) if err != nil { log.Println(err.Error()) return From 23a2816240022569168730a6f69db802fa6f485c Mon Sep 17 00:00:00 2001 From: Marius Date: Thu, 20 May 2021 19:43:22 +0200 Subject: [PATCH 9/9] fixed job retrival --- .kube/dev-deployment.yaml | 2 +- database/handler.go | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.kube/dev-deployment.yaml b/.kube/dev-deployment.yaml index d098e10..57b3b77 100644 --- a/.kube/dev-deployment.yaml +++ b/.kube/dev-deployment.yaml @@ -54,7 +54,7 @@ spec: secretKeyRef: key: Token name: authhandler - image: harbor.computational.bio.uni-giessen.de/bakta/bakta-web-backend:0.2.9-beta.29 + image: harbor.computational.bio.uni-giessen.de/bakta/bakta-web-backend:0.2.9-beta.32 imagePullPolicy: Always name: baktabackend ports: diff --git a/database/handler.go b/database/handler.go index 329afbe..9f115dc 100644 --- a/database/handler.go +++ b/database/handler.go @@ -272,9 +272,12 @@ func (handler *Handler) GetJobs(jobIDs []*api.JobAuth) ([]Job, error) { var jobRequests []bson.M for _, jobRequest := range jobIDs { + secretSHA := sha256.Sum256([]byte(jobRequest.Secret)) + secretSHABase64 := base64.StdEncoding.EncodeToString(secretSHA[:]) + jobRequests = append(jobRequests, bson.M{ "jobid": jobRequest.JobID, - "secret": jobRequest.Secret, + "secret": secretSHABase64, }) }