Skip to content

Commit

Permalink
Merge pull request #71 from Chengxuan/metrics-customization
Browse files Browse the repository at this point in the history 
support metrics customization in transaction handler
  • Loading branch information
@Chengxuan
Chengxuan authored Mar 21, 2023
2 parents 0b79fa5 + d69a00e commit 6c0c0d1
Show file tree
Hide file tree
Showing 26 changed files with 551 additions and 263 deletions.
24 changes: 12 additions & 12 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,18 +31,18 @@ mocks-$(strip $(1))-$(strip $(2)): ${MOCKERY}
${MOCKERY} --case underscore --dir $(1) --name $(2) --outpkg $(3) --output mocks/$(strip $(3))
endef

$(eval $(call makemock, pkg/ffcapi, API, ffcapimocks))
$(eval $(call makemock, pkg/txhandler, TransactionHandler, txhandlermocks))
$(eval $(call makemock, pkg/txhandler, ManagedTxEventHandler, txhandlermocks))
$(eval $(call makemock, internal/metrics, Metrics, metricsmocks))
$(eval $(call makemock, pkg/txhistory, Manager, txhistorymocks))
$(eval $(call makemock, internal/confirmations, Manager, confirmationsmocks))
$(eval $(call makemock, internal/persistence, Persistence, persistencemocks))
$(eval $(call makemock, internal/persistence, TransactionPersistence, persistencemocks))
$(eval $(call makemock, internal/ws, WebSocketChannels, wsmocks))
$(eval $(call makemock, internal/ws, WebSocketServer, wsmocks))
$(eval $(call makemock, internal/events, Stream, eventsmocks))
$(eval $(call makemock, internal/apiclient, FFTMClient, apiclientmocks))
$(eval $(call makemock, pkg/ffcapi, API, ffcapimocks))
$(eval $(call makemock, pkg/txhandler, TransactionHandler, txhandlermocks))
$(eval $(call makemock, pkg/txhandler, ManagedTxEventHandler, txhandlermocks))
$(eval $(call makemock, internal/metrics, TransactionHandlerMetrics, metricsmocks))
$(eval $(call makemock, pkg/txhistory, Manager, txhistorymocks))
$(eval $(call makemock, internal/confirmations, Manager, confirmationsmocks))
$(eval $(call makemock, internal/persistence, Persistence, persistencemocks))
$(eval $(call makemock, internal/persistence, TransactionPersistence, persistencemocks))
$(eval $(call makemock, internal/ws, WebSocketChannels, wsmocks))
$(eval $(call makemock, internal/ws, WebSocketServer, wsmocks))
$(eval $(call makemock, internal/events, Stream, eventsmocks))
$(eval $(call makemock, internal/apiclient, FFTMClient, apiclientmocks))

go-mod-tidy: .ALWAYS
$(VGO) mod tidy
Expand Down
16 changes: 9 additions & 7 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,18 +9,20 @@ require (
github.com/go-resty/resty/v2 v2.7.0
github.com/gorilla/mux v1.8.0
github.com/gorilla/websocket v1.5.0
github.com/hyperledger/firefly-common v1.2.1
github.com/hyperledger/firefly-common v1.2.6
github.com/oklog/ulid/v2 v2.1.0
github.com/prometheus/client_golang v1.13.0
github.com/sirupsen/logrus v1.9.0
github.com/spf13/viper v1.14.0
github.com/stretchr/testify v1.8.1
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
gitlab.com/hfuss/mux-prometheus v0.0.4
golang.org/x/text v0.4.0
golang.org/x/text v0.7.0
)

require github.com/inconshreveable/mousetrap v1.0.1 // indirect
require (
github.com/inconshreveable/mousetrap v1.0.1 // indirect
gitlab.com/hfuss/mux-prometheus v0.0.4 // indirect
)

require (
github.com/Masterminds/goutils v1.1.1 // indirect
Expand Down Expand Up @@ -71,9 +73,9 @@ require (
github.com/subosito/gotenv v1.4.1 // indirect
github.com/x-cray/logrus-prefixed-formatter v0.5.2 // indirect
golang.org/x/crypto v0.1.0 // indirect
golang.org/x/net v0.1.0 // indirect
golang.org/x/sys v0.1.0 // indirect
golang.org/x/term v0.1.0 // indirect
golang.org/x/net v0.7.0 // indirect
golang.org/x/sys v0.5.0 // indirect
golang.org/x/term v0.5.0 // indirect
google.golang.org/protobuf v1.28.1 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
Expand Down
20 changes: 10 additions & 10 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,8 +184,8 @@ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/huandu/xstrings v1.3.1 h1:4jgBlKK6tLKFvO8u5pmYjG91cqytmDCDvGh7ECVFfFs=
github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
github.com/hyperledger/firefly-common v1.2.1 h1:h35T5Ask/ixlW70KZxHpPsO5QfdnKBFEyX+7L10lwoo=
github.com/hyperledger/firefly-common v1.2.1/go.mod h1:aqaq2ZUzSlkwVC/TF+pwUkHxGDLYSE2DzdozZ1MIMeM=
github.com/hyperledger/firefly-common v1.2.6 h1:NDf42a2iH6QSt3p7KkdT1pTDoW/NljcT+ClEfZRh61c=
github.com/hyperledger/firefly-common v1.2.6/go.mod h1:uYO8DMgT3OGoF2R4Bew1Ql4BS3udYGzPs59sJH1V3E4=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
Expand Down Expand Up @@ -441,8 +441,8 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
Expand Down Expand Up @@ -518,12 +518,12 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
Expand All @@ -532,8 +532,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
Expand Down
169 changes: 155 additions & 14 deletions internal/metrics/metrics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,41 +18,182 @@ package metrics

import (
"context"
"fmt"
"net/http"
"time"

"github.com/hyperledger/firefly-common/pkg/config"
"github.com/hyperledger/firefly-common/pkg/metric"
"github.com/hyperledger/firefly-transaction-manager/internal/tmconfig"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
)

const metricsTransactionManagerPrefix = "tm"

// REST api-server and transaction handler are sub-subsystem
var metricsTransactionHandlerSubsystemName = fmt.Sprintf("%s_%s", metricsTransactionManagerPrefix, "th")
var metricsRESTAPIServerSubSystemName = fmt.Sprintf("%s_%s", metricsTransactionManagerPrefix, "api_server_rest")

type metricsManager struct {
ctx context.Context
metricsEnabled bool
timeMap map[string]time.Time
ctx context.Context
metricsEnabled bool
metricsRegistry metric.MetricsRegistry
txHandlerMetricsManager metric.MetricsManager
timeMap map[string]time.Time
}

func NewMetricsManager(ctx context.Context) Metrics {
metricsRegistry := metric.NewPrometheusMetricsRegistry()
txHandlerMetricsManager, _ := metricsRegistry.NewMetricsManagerForSubsystem(ctx, metricsTransactionHandlerSubsystemName)
_ = metricsRegistry.NewHTTPMetricsInstrumentationsForSubsystem(
ctx,
metricsRESTAPIServerSubSystemName,
true,
prometheus.DefBuckets,
map[string]string{},
)
mm := &metricsManager{
ctx: ctx,
metricsEnabled: config.GetBool(tmconfig.MetricsEnabled),
timeMap: make(map[string]time.Time),
ctx: ctx,
metricsEnabled: config.GetBool(tmconfig.MetricsEnabled),
timeMap: make(map[string]time.Time),
metricsRegistry: metricsRegistry,
txHandlerMetricsManager: txHandlerMetricsManager,
}

return mm
}

func (mm *metricsManager) TransactionSubmissionError() {
TransactionSubmissionError.Inc()
func (mm *metricsManager) IsMetricsEnabled() bool {
return mm.metricsEnabled
}
func (mm *metricsManager) TransactionsInFlightSet(count float64) {
TransactionsInFlight.Set(count)

func (mm *metricsManager) HTTPHandler() http.Handler {
httpHandler, _ := mm.metricsRegistry.HTTPHandler(mm.ctx, promhttp.HandlerOpts{})
return httpHandler
}

func (mm *metricsManager) IsMetricsEnabled() bool {
return mm.metricsEnabled
func (mm *metricsManager) GetAPIServerRESTHTTPMiddleware() func(next http.Handler) http.Handler {
httpMiddleware, _ := mm.metricsRegistry.GetHTTPMetricsInstrumentationsMiddlewareForSubsystem(mm.ctx, metricsRESTAPIServerSubSystemName)
return httpMiddleware
}

type Metrics interface {
TransactionSubmissionError()
TransactionsInFlightSet(count float64)
IsMetricsEnabled() bool

// HTTPHandler returns the HTTP handler of this metrics registry
HTTPHandler() http.Handler

GetAPIServerRESTHTTPMiddleware() func(next http.Handler) http.Handler

// functions for transaction handler to define and emit metrics
TransactionHandlerMetrics
}

// Transaction handler metrics are defined and emitted by transaction handlers
type TransactionHandlerMetrics interface {
// functions for declaring new metrics
InitTxHandlerCounterMetric(ctx context.Context, metricName string, helpText string, withDefaultLabels bool)
InitTxHandlerCounterMetricWithLabels(ctx context.Context, metricName string, helpText string, labelNames []string, withDefaultLabels bool)
InitTxHandlerGaugeMetric(ctx context.Context, metricName string, helpText string, withDefaultLabels bool)
InitTxHandlerGaugeMetricWithLabels(ctx context.Context, metricName string, helpText string, labelNames []string, withDefaultLabels bool)
InitTxHandlerHistogramMetric(ctx context.Context, metricName string, helpText string, buckets []float64, withDefaultLabels bool)
InitTxHandlerHistogramMetricWithLabels(ctx context.Context, metricName string, helpText string, buckets []float64, labelNames []string, withDefaultLabels bool)
InitTxHandlerSummaryMetric(ctx context.Context, metricName string, helpText string, withDefaultLabels bool)
InitTxHandlerSummaryMetricWithLabels(ctx context.Context, metricName string, helpText string, labelNames []string, withDefaultLabels bool)

// functions for use existing metrics
SetTxHandlerGaugeMetric(ctx context.Context, metricName string, number float64, defaultLabels *metric.FireflyDefaultLabels)
SetTxHandlerGaugeMetricWithLabels(ctx context.Context, metricName string, number float64, labels map[string]string, defaultLabels *metric.FireflyDefaultLabels)
IncTxHandlerCounterMetric(ctx context.Context, metricName string, defaultLabels *metric.FireflyDefaultLabels)
IncTxHandlerCounterMetricWithLabels(ctx context.Context, metricName string, labels map[string]string, defaultLabels *metric.FireflyDefaultLabels)
ObserveTxHandlerHistogramMetric(ctx context.Context, metricName string, number float64, defaultLabels *metric.FireflyDefaultLabels)
ObserveTxHandlerHistogramMetricWithLabels(ctx context.Context, metricName string, number float64, labels map[string]string, defaultLabels *metric.FireflyDefaultLabels)
ObserveTxHandlerSummaryMetric(ctx context.Context, metricName string, number float64, defaultLabels *metric.FireflyDefaultLabels)
ObserveTxHandlerSummaryMetricWithLabels(ctx context.Context, metricName string, number float64, labels map[string]string, defaultLabels *metric.FireflyDefaultLabels)
}

func (mm *metricsManager) InitTxHandlerCounterMetric(ctx context.Context, metricName string, helpText string, withDefaultLabels bool) {
if mm.metricsEnabled {
mm.txHandlerMetricsManager.NewCounterMetric(ctx, metricName, helpText, withDefaultLabels)
}
}
func (mm *metricsManager) InitTxHandlerCounterMetricWithLabels(ctx context.Context, metricName string, helpText string, labelNames []string, withDefaultLabels bool) {
if mm.metricsEnabled {
mm.txHandlerMetricsManager.NewCounterMetricWithLabels(ctx, metricName, helpText, labelNames, withDefaultLabels)
}
}
func (mm *metricsManager) InitTxHandlerGaugeMetric(ctx context.Context, metricName string, helpText string, withDefaultLabels bool) {
if mm.metricsEnabled {
mm.txHandlerMetricsManager.NewGaugeMetric(ctx, metricName, helpText, withDefaultLabels)
}
}
func (mm *metricsManager) InitTxHandlerGaugeMetricWithLabels(ctx context.Context, metricName string, helpText string, labelNames []string, withDefaultLabels bool) {
if mm.metricsEnabled {
mm.txHandlerMetricsManager.NewGaugeMetricWithLabels(ctx, metricName, helpText, labelNames, withDefaultLabels)
}
}
func (mm *metricsManager) InitTxHandlerHistogramMetric(ctx context.Context, metricName string, helpText string, buckets []float64, withDefaultLabels bool) {
if mm.metricsEnabled {
mm.txHandlerMetricsManager.NewHistogramMetric(ctx, metricName, helpText, buckets, withDefaultLabels)
}
}
func (mm *metricsManager) InitTxHandlerHistogramMetricWithLabels(ctx context.Context, metricName string, helpText string, buckets []float64, labelNames []string, withDefaultLabels bool) {
if mm.metricsEnabled {
mm.txHandlerMetricsManager.NewHistogramMetricWithLabels(ctx, metricName, helpText, buckets, labelNames, withDefaultLabels)
}
}
func (mm *metricsManager) InitTxHandlerSummaryMetric(ctx context.Context, metricName string, helpText string, withDefaultLabels bool) {
if mm.metricsEnabled {
mm.txHandlerMetricsManager.NewSummaryMetric(ctx, metricName, helpText, withDefaultLabels)
}
}
func (mm *metricsManager) InitTxHandlerSummaryMetricWithLabels(ctx context.Context, metricName string, helpText string, labelNames []string, withDefaultLabels bool) {
if mm.metricsEnabled {
mm.txHandlerMetricsManager.NewSummaryMetricWithLabels(ctx, metricName, helpText, labelNames, withDefaultLabels)
}
}

// functions for use existing metrics
func (mm *metricsManager) SetTxHandlerGaugeMetric(ctx context.Context, metricName string, number float64, defaultLabels *metric.FireflyDefaultLabels) {
if mm.metricsEnabled {
mm.txHandlerMetricsManager.SetGaugeMetric(ctx, metricName, number, defaultLabels)
}
}
func (mm *metricsManager) SetTxHandlerGaugeMetricWithLabels(ctx context.Context, metricName string, number float64, labels map[string]string, defaultLabels *metric.FireflyDefaultLabels) {
if mm.metricsEnabled {
mm.txHandlerMetricsManager.SetGaugeMetricWithLabels(ctx, metricName, number, labels, defaultLabels)
}
}

func (mm *metricsManager) IncTxHandlerCounterMetric(ctx context.Context, metricName string, defaultLabels *metric.FireflyDefaultLabels) {
if mm.metricsEnabled {
mm.txHandlerMetricsManager.IncCounterMetric(ctx, metricName, defaultLabels)
}
}
func (mm *metricsManager) IncTxHandlerCounterMetricWithLabels(ctx context.Context, metricName string, labels map[string]string, defaultLabels *metric.FireflyDefaultLabels) {
if mm.metricsEnabled {
mm.txHandlerMetricsManager.IncCounterMetricWithLabels(ctx, metricName, labels, defaultLabels)
}
}
func (mm *metricsManager) ObserveTxHandlerHistogramMetric(ctx context.Context, metricName string, number float64, defaultLabels *metric.FireflyDefaultLabels) {
if mm.metricsEnabled {
mm.txHandlerMetricsManager.ObserveHistogramMetric(ctx, metricName, number, defaultLabels)
}
}
func (mm *metricsManager) ObserveTxHandlerHistogramMetricWithLabels(ctx context.Context, metricName string, number float64, labels map[string]string, defaultLabels *metric.FireflyDefaultLabels) {
if mm.metricsEnabled {
mm.txHandlerMetricsManager.ObserveHistogramMetricWithLabels(ctx, metricName, number, labels, defaultLabels)
}
}

func (mm *metricsManager) ObserveTxHandlerSummaryMetric(ctx context.Context, metricName string, number float64, defaultLabels *metric.FireflyDefaultLabels) {
if mm.metricsEnabled {
mm.txHandlerMetricsManager.ObserveSummaryMetric(ctx, metricName, number, defaultLabels)
}
}
func (mm *metricsManager) ObserveTxHandlerSummaryMetricWithLabels(ctx context.Context, metricName string, number float64, labels map[string]string, defaultLabels *metric.FireflyDefaultLabels) {
if mm.metricsEnabled {
mm.txHandlerMetricsManager.ObserveSummaryMetricWithLabels(ctx, metricName, number, labels, defaultLabels)
}
}
Loading

0 comments on commit 6c0c0d1

Please sign in to comment.