From 80b9fa4ce7ff9cdc55bc0416c9e22b24e9504e97 Mon Sep 17 00:00:00 2001 From: Software Developer <7852635+dsuhinin@users.noreply.github.com> Date: Mon, 8 Apr 2024 16:47:59 +0200 Subject: [PATCH] [release/0.6] Performance improvement for GET /projects/params endpoint. (#1094) --- Pipfile.lock | 12 ++++++------ go.mod | 6 +++--- go.sum | 12 ++++++------ pkg/api/aim2/dao/repositories/metric.go | 14 ++++++++------ pkg/api/aim2/services/project/service.go | 2 +- 5 files changed, 24 insertions(+), 22 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 04091e298..a8492ebc3 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -834,20 +834,20 @@ }, "boto3": { "hashes": [ - "sha256:ba5d2104bba4370766036d64ad9021eb6289d154265852a2a821ec6a5e816faa", - "sha256:eaec72fda124084105a31bcd67eafa1355b34df6da70cadae0c0f262d8a4294f" + "sha256:139dd2d94eaa0e3213ff37ba7cf4cb2e3823269178fe8f3e33c965f680a9ddde", + "sha256:265b0b4865e8c07e27abb32a31d2bd9129bb009b1d89ca0783776ec084886123" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==1.34.75" + "version": "==1.34.79" }, "botocore": { "hashes": [ - "sha256:06113ee2587e6160211a6bd797e135efa6aa21b5bde97bf455c02f7dff40203c", - "sha256:1d7f683d99eba65076dfb9af3b42fa967c64f11111d9699b65757420902aa002" + "sha256:6b59b0f7de219d383a2a633f6718c2600642ebcb707749dc6c67a6a436474b7a", + "sha256:a42a014d3dbaa9ef123810592af69f9e55b456c5be3ac9efc037325685519e83" ], "markers": "python_version >= '3.8'", - "version": "==1.34.75" + "version": "==1.34.79" }, "cachetools": { "hashes": [ diff --git a/go.mod b/go.mod index c95f20df4..5b8662fa5 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/PuerkitoBio/goquery v1.9.1 github.com/apache/arrow/go/v14 v14.0.2 github.com/aws/aws-sdk-go-v2 v1.26.1 - github.com/aws/aws-sdk-go-v2/config v1.27.10 + github.com/aws/aws-sdk-go-v2/config v1.27.11 github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1 github.com/go-python/gpython v0.2.0 github.com/gofiber/fiber/v2 v2.52.4 @@ -52,7 +52,7 @@ require ( github.com/andybalholm/brotli v1.0.5 // indirect github.com/andybalholm/cascadia v1.3.2 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.17.10 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.11 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect @@ -62,7 +62,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.20.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect github.com/aws/smithy-go v1.20.2 // indirect diff --git a/go.sum b/go.sum index 8391ab86e..8fa20c668 100644 --- a/go.sum +++ b/go.sum @@ -42,10 +42,10 @@ github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+ github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 h1:x6xsQXGSmW6frevwDA+vi/wqhp1ct18mVXYN08/93to= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2/go.mod h1:lPprDr1e6cJdyYeGXnRaJoP4Md+cDBvi2eOj00BlGmg= -github.com/aws/aws-sdk-go-v2/config v1.27.10 h1:PS+65jThT0T/snC5WjyfHHyUgG+eBoupSDV+f838cro= -github.com/aws/aws-sdk-go-v2/config v1.27.10/go.mod h1:BePM7Vo4OBpHreKRUMuDXX+/+JWP38FLkzl5m27/Jjs= -github.com/aws/aws-sdk-go-v2/credentials v1.17.10 h1:qDZ3EA2lv1KangvQB6y258OssCHD0xvaGiEDkG4X/10= -github.com/aws/aws-sdk-go-v2/credentials v1.17.10/go.mod h1:6t3sucOaYDwDssHQa0ojH1RpmVmF5/jArkye1b2FKMI= +github.com/aws/aws-sdk-go-v2/config v1.27.11 h1:f47rANd2LQEYHda2ddSCKYId18/8BhSRM4BULGmfgNA= +github.com/aws/aws-sdk-go-v2/config v1.27.11/go.mod h1:SMsV78RIOYdve1vf36z8LmnszlRWkwMQtomCAI0/mIE= +github.com/aws/aws-sdk-go-v2/credentials v1.17.11 h1:YuIB1dJNf1Re822rriUOTxopaHHvIq0l/pX3fwO+Tzs= +github.com/aws/aws-sdk-go-v2/credentials v1.17.11/go.mod h1:AQtFPsDH9bI2O+71anW6EKL+NcD7LG3dpKGMV4SShgo= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 h1:FVJ0r5XTHSmIHJV6KuDmdYhEpvlHpiSd38RQWhut5J4= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1/go.mod h1:zusuAeqezXzAB24LGuzuekqMAEgWkVYukBec3kr3jUg= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 h1:aw39xVGeRWlWx9EzGVnhOR4yOjQDHPQ6o6NmBlscyQg= @@ -66,8 +66,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 h1:f9RyWNtS8oH7cZ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5/go.mod h1:h5CoMZV2VF297/VLhRhO1WF+XYWOzXo+4HsObA4HjBQ= github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1 h1:6cnno47Me9bRykw9AEv9zkXE+5or7jz8TsskTTccbgc= github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1/go.mod h1:qmdkIIAC+GCLASF7R2whgNrJADz0QZPX+Seiw/i4S3o= -github.com/aws/aws-sdk-go-v2/service/sso v1.20.4 h1:WzFol5Cd+yDxPAdnzTA5LmpHYSWinhmSj4rQChV0ee8= -github.com/aws/aws-sdk-go-v2/service/sso v1.20.4/go.mod h1:qGzynb/msuZIE8I75DVRCUXw3o3ZyBmUvMwQ2t/BrGM= +github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 h1:vN8hEbpRnL7+Hopy9dzmRle1xmDc7o8tmY0klsr175w= +github.com/aws/aws-sdk-go-v2/service/sso v1.20.5/go.mod h1:qGzynb/msuZIE8I75DVRCUXw3o3ZyBmUvMwQ2t/BrGM= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 h1:Jux+gDDyi1Lruk+KHF91tK2KCuY61kzoCpvtvJJBtOE= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4/go.mod h1:mUYPBhaF2lGiukDEjJX2BLRRKTmoUSitGDUgM4tRxak= github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 h1:cwIxeBttqPN3qkaAjcEcsh8NYr8n2HZPkcKgPAi1phU= diff --git a/pkg/api/aim2/dao/repositories/metric.go b/pkg/api/aim2/dao/repositories/metric.go index 6fcc07434..f4844dc01 100644 --- a/pkg/api/aim2/dao/repositories/metric.go +++ b/pkg/api/aim2/dao/repositories/metric.go @@ -29,8 +29,8 @@ type SearchResultMap = map[string]SearchResult // MetricRepositoryProvider provides an interface to work with models.Metric entity. type MetricRepositoryProvider interface { BaseRepositoryProvider - // GetLatestMetricsByExperiments returns latest metrics by provided experiments. - GetLatestMetricsByExperiments( + // GetMetricKeysAndContextsByExperiments returns metric keys and contexts by provided experiments. + GetMetricKeysAndContextsByExperiments( ctx context.Context, namespaceID uint, experiments []int, ) ([]models.LatestMetric, error) // SearchMetrics returns a sql.Rows cursor for streaming the metrics matching the request. @@ -57,18 +57,20 @@ func NewMetricRepository(db *gorm.DB) *MetricRepository { } } -// GetLatestMetricsByExperiments returns latest metrics by provided experiments. -func (r MetricRepository) GetLatestMetricsByExperiments( +// GetMetricKeysAndContextsByExperiments returns metric keys and contexts by provided experiments. +func (r MetricRepository) GetMetricKeysAndContextsByExperiments( ctx context.Context, namespaceID uint, experiments []int, ) ([]models.LatestMetric, error) { - query := r.db.WithContext(ctx).Distinct().Model( + query := r.db.WithContext(ctx).Distinct().Select( + "key", "context_id", + ).Model( &models.LatestMetric{}, ).Joins( "JOIN runs USING(run_uuid)", ).Joins( "INNER JOIN experiments ON experiments.experiment_id = runs.experiment_id AND experiments.namespace_id = ?", namespaceID, - ).Joins( + ).Preload( "Context", ).Where( "runs.lifecycle_stage = ?", models.LifecycleStageActive, diff --git a/pkg/api/aim2/services/project/service.go b/pkg/api/aim2/services/project/service.go index cff82de4d..50a74bfda 100644 --- a/pkg/api/aim2/services/project/service.go +++ b/pkg/api/aim2/services/project/service.go @@ -104,7 +104,7 @@ func (s Service) GetProjectParams( } if slices.Contains(req.Sequences, "metric") { - metrics, err := s.metricRepository.GetLatestMetricsByExperiments(ctx, namespaceID, req.Experiments) + metrics, err := s.metricRepository.GetMetricKeysAndContextsByExperiments(ctx, namespaceID, req.Experiments) if err != nil { return nil, api.NewInternalError("error getting metrics: %s", err) }