Skip to content

Commit

Permalink
add statsd reporter support (#85)
Browse files Browse the repository at this point in the history
  • Loading branch information
ImNumber4 authored Jan 21, 2024
1 parent d8af4ef commit ad42980
Show file tree
Hide file tree
Showing 9 changed files with 180 additions and 16 deletions.
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ require (
github.com/pkg/errors v0.9.1
github.com/robfig/cron/v3 v3.0.1
github.com/smallnest/weighted v0.0.0-20230419055410-36b780e40a7a
github.com/smira/go-statsd v1.3.3
github.com/spf13/cobra v1.8.0
github.com/spf13/viper v1.18.2
github.com/stretchr/testify v1.8.4
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -703,6 +703,8 @@ github.com/smallnest/weighted v0.0.0-20230419055410-36b780e40a7a h1:eieNTZmrnPzI
github.com/smallnest/weighted v0.0.0-20230419055410-36b780e40a7a/go.mod h1:xc9CoZ+ZBGwajnWto5Aqw/wWg8euy4HtOr6K9Fxp9iw=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/smira/go-statsd v1.3.3 h1:WnMlmGTyMpzto+HvOJWRPoLaLlk5EGfzsnlQBcvj4yI=
github.com/smira/go-statsd v1.3.3/go.mod h1:RjdsESPgDODtg1VpVVf9MJrEW2Hw0wtRNbmB1CAhu6A=
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
Expand Down
6 changes: 6 additions & 0 deletions internal/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ type (
Cron CronConfig `mapstructure:"cron"`
SLA SLAConfig `mapstructure:"sla"`
FunctionalTest FunctionalTestConfig `mapstructure:"functional_test"`
StatsD *StatsDConfig `mapstructure:"statsd"`

namespace string
env Env
Expand Down Expand Up @@ -394,6 +395,11 @@ type (
PerClientRPS int `mapstructure:"per_client_rps"`
}

StatsDConfig struct {
Address string `mapstructure:"address" validate:"required"`
Prefix string `mapstructure:"prefix"`
}

ConfigOption func(options *configOptions)

Env string
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"testing"

"github.com/stretchr/testify/suite"
"go.uber.org/fx"
"google.golang.org/protobuf/proto"

Expand All @@ -13,7 +14,6 @@ import (
"github.com/coinbase/chainstorage/internal/utils/testutil"
"github.com/coinbase/chainstorage/protos/coinbase/c3/common"
api "github.com/coinbase/chainstorage/protos/coinbase/chainstorage"
"github.com/stretchr/testify/suite"
)

type gcpBlobStorageTestSuite struct {
Expand Down
2 changes: 1 addition & 1 deletion internal/storage/blobstorage/module.go
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
package blobstorage

import (
"github.com/coinbase/chainstorage/internal/storage/blobstorage/gcs"
"go.uber.org/fx"

"github.com/coinbase/chainstorage/internal/storage/blobstorage/gcs"
"github.com/coinbase/chainstorage/internal/storage/blobstorage/internal"
"github.com/coinbase/chainstorage/internal/storage/blobstorage/s3"
)
Expand Down
1 change: 1 addition & 0 deletions internal/tally/module.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,5 @@ import (

var Module = fx.Options(
fx.Provide(NewRootScope),
fx.Provide(NewStatsReporter),
)
115 changes: 115 additions & 0 deletions internal/tally/stats_reporter.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
package tally

import (
"context"
"time"

smirastatsd "github.com/smira/go-statsd"
"github.com/uber-go/tally/v4"
"go.uber.org/fx"
"go.uber.org/zap"

"github.com/coinbase/chainstorage/internal/config"
)

type (
StatsReporterParams struct {
fx.In
Lifecycle fx.Lifecycle
Logger *zap.Logger
Config *config.Config
}

reporter struct {
client *smirastatsd.Client
}
)

const (
reportingInterval = time.Second
)

var (
// hardcoding this to be datadog format
// we need think about whats the best way to set it up in config such that
// when we switch reporter impl, config will still be backward compatible
tagFormat = smirastatsd.TagFormatDatadog
)

func NewStatsReporter(params StatsReporterParams) tally.StatsReporter {
if params.Config.StatsD == nil {
return tally.NullStatsReporter
}
cfg := params.Config.StatsD
client := smirastatsd.NewClient(
cfg.Address,
smirastatsd.MetricPrefix(cfg.Prefix),
smirastatsd.TagStyle(tagFormat),
smirastatsd.ReportInterval(reportingInterval),
)
params.Logger.Info("initialized statsd client")
params.Lifecycle.Append(fx.Hook{
OnStop: func(ctx context.Context) error {
return client.Close()
},
})
return &reporter{
client: client,
}
}

func convertTags(tagsMap map[string]string) []smirastatsd.Tag {
tags := make([]smirastatsd.Tag, 0, len(tagsMap))
for key, value := range tagsMap {
tags = append(tags, smirastatsd.StringTag(key, value))
}
return tags
}

func (r *reporter) ReportCounter(name string, tags map[string]string, value int64) {
r.client.Incr(name, value, convertTags(tags)...)
}

func (r *reporter) ReportGauge(name string, tags map[string]string, value float64) {
r.client.FGauge(name, value, convertTags(tags)...)
}

func (r *reporter) ReportTimer(name string, tags map[string]string, value time.Duration) {
r.client.PrecisionTiming(name, value, convertTags(tags)...)
}

func (r *reporter) ReportHistogramValueSamples(
name string,
tags map[string]string,
buckets tally.Buckets,
bucketLowerBound,
bucketUpperBound float64,
samples int64) {
panic("no implemented")
}

func (r *reporter) ReportHistogramDurationSamples(
name string,
tags map[string]string,
buckets tally.Buckets,
bucketLowerBound,
bucketUpperBound time.Duration,
samples int64) {
panic("no implemented")
}

func (r *reporter) Capabilities() tally.Capabilities {
return r
}

func (r *reporter) Reporting() bool {
return true
}

func (r *reporter) Tagging() bool {
return true
}

func (r *reporter) Flush() {
// no-op
}
49 changes: 49 additions & 0 deletions internal/tally/stats_reporter_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
package tally

import (
"testing"

"github.com/uber-go/tally/v4"
"go.uber.org/fx"

"github.com/coinbase/chainstorage/internal/config"
"github.com/coinbase/chainstorage/internal/utils/testapp"
"github.com/coinbase/chainstorage/internal/utils/testutil"
)

func TestNewReporterDefaultNoStatsD(t *testing.T) {
testapp.TestAllConfigs(t, func(t *testing.T, cfg *config.Config) {
require := testutil.Require(t)

var reporter tally.StatsReporter
testapp.New(
t,
testapp.WithConfig(cfg),
fx.Provide(NewStatsReporter),
fx.Populate(&reporter),
)

require.Equal(tally.NullStatsReporter, reporter)
require.Equal(false, reporter.Capabilities().Reporting())
require.Equal(false, reporter.Capabilities().Tagging())
})
}

func TestNewReporterDefaultWithStatsD(t *testing.T) {
testapp.TestAllConfigs(t, func(t *testing.T, cfg *config.Config) {
require := testutil.Require(t)
cfg.StatsD = &config.StatsDConfig{
Address: "localhost:8125",
}
var reporter tally.StatsReporter
testapp.New(
t,
testapp.WithConfig(cfg),
fx.Provide(NewStatsReporter),
fx.Populate(&reporter),
)
require.NotEqual(tally.NullStatsReporter, reporter)
require.Equal(true, reporter.Capabilities().Reporting())
require.Equal(true, reporter.Capabilities().Tagging())
})
}
18 changes: 4 additions & 14 deletions internal/tally/tally.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package tally

import (
"context"
"time"

"github.com/uber-go/tally/v4"
"go.uber.org/fx"
Expand All @@ -16,27 +15,18 @@ type (
fx.In
Lifecycle fx.Lifecycle
Config *config.Config
Reporter tally.StatsReporter `optional:"true"`
Reporter tally.StatsReporter
}
)

const (
reportingInterval = time.Second
)

func NewRootScope(params MetricParams) tally.Scope {
// XXX: Inject your own reporter here.
reporter := params.Reporter
if reporter == nil {
reporter = tally.NullStatsReporter
}

opts := tally.ScopeOptions{
Prefix: consts.ServiceName,
Reporter: reporter,
Reporter: params.Reporter,
Tags: params.Config.GetCommonTags(),
}
scope, closer := tally.NewRootScope(opts, reportingInterval)
//report interval will be set on reporter
scope, closer := tally.NewRootScope(opts, 0)
params.Lifecycle.Append(fx.Hook{
OnStop: func(ctx context.Context) error {
return closer.Close()
Expand Down

0 comments on commit ad42980

Please sign in to comment.