Merge branch 'master' into dependabot/go_modules/github.com/aws/aws-s…

…dk-go-1.44.307

.golangci.yml

@@ -1,3 +1,6 @@
+run:
+  timeout: 5m
+
 output:
   format: line-number
 
@@ -28,5 +31,8 @@ linters:
     - unused
 
 linters-settings:
+  errcheck:
+    exclude-functions:
+      - (github.com/go-kit/log.Logger).Log
   goimports:
     local-prefixes: "github.com/nerdswords/yet-another-cloudwatch-exporter"

README.md

@@ -26,7 +26,7 @@ Only the latest version gets security updates. We won't support older versions.
 ## Features
 
 * Stop worrying about your AWS IDs - Auto discovery of resources via tags
-* Structured JSON logging
+* Structured logging (json and logfmt)
 * Filter monitored resources via regex
 * Automatic adding of tag labels to metrics
 * Automatic adding of dimension labels to metrics
@@ -128,7 +128,7 @@ Refer to the [installation guide](docs/installation.md).
 
 The exporter will need to be running in an environment which has access to AWS. The exporter uses the [AWS SDK for Go](https://aws.github.io/aws-sdk-go-v2/docs/getting-started/) and supports providing authentication via [AWS's default credential chain](https://aws.github.io/aws-sdk-go-v2/docs/configuring-sdk/#specifying-credentials). Regardless of the method used to acquire the credentials, some permissions are needed for the exporter to work.
 
-As a quick start, the following IAM policy can be used to grant the all permissions required by YACE 
+As a quick start, the following IAM policy can be used to grant the all permissions required by YACE
 ```json
 {
   "Version": "2012-10-17",

cmd/yace/main.go

@@ -8,7 +8,6 @@ import (
 	"os"
 	"strings"
 
-	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
 	"golang.org/x/sync/semaphore"
 
@@ -41,6 +40,7 @@ var (
 	addr                  string
 	configFile            string
 	debug                 bool
+	logFormat             string
 	fips                  bool
 	cloudwatchConcurrency int
 	tagConcurrency        int
@@ -95,6 +95,21 @@ func NewYACEApp() *cli.App {
 			Destination: &debug,
 			EnvVars:     []string{"debug"},
 		},
+		&cli.StringFlag{
+			Name:        "log.format",
+			Value:       "json",
+			Usage:       "Output format of log messages. One of: [logfmt, json]. Default: [json].",
+			Destination: &logFormat,
+			Action: func(ctx *cli.Context, s string) error {
+				switch s {
+				case "logfmt", "json":
+					break
+				default:
+					return fmt.Errorf("unrecognized log format %q", s)
+				}
+				return nil
+			},
+		},
 		&cli.BoolFlag{
 			Name:        "fips",
 			Value:       false,
@@ -145,18 +160,16 @@ func NewYACEApp() *cli.App {
 		},
 	}
 
-	yace.Before = func(ctx *cli.Context) error {
-		logger = newLogger(debug)
-		return nil
-	}
-
 	yace.Commands = []*cli.Command{
 		{
-			Name: "verify-config", Aliases: []string{"vc"}, Usage: "Loads and attempts to parse config file, then exits. Useful for CI/CD validation",
+			Name:    "verify-config",
+			Aliases: []string{"vc"},
+			Usage:   "Loads and attempts to parse config file, then exits. Useful for CI/CD validation",
 			Flags: []cli.Flag{
 				&cli.StringFlag{Name: "config.file", Value: "config.yml", Usage: "Path to configuration file.", Destination: &configFile},
 			},
 			Action: func(c *cli.Context) error {
+				logger = logging.NewLogger(logFormat, debug, "version", version)
 				logger.Info("Parsing config")
 				if err := cfg.Load(configFile, logger); err != nil {
 					logger.Error(err, "Couldn't read config file", "path", configFile)
@@ -168,7 +181,9 @@ func NewYACEApp() *cli.App {
 			},
 		},
 		{
-			Name: "version", Aliases: []string{"v"}, Usage: "prints current yace version.",
+			Name:    "version",
+			Aliases: []string{"v"},
+			Usage:   "prints current yace version.",
 			Action: func(c *cli.Context) error {
 				fmt.Println(version)
 				os.Exit(0)
@@ -183,6 +198,8 @@ func NewYACEApp() *cli.App {
 }
 
 func startScraper(c *cli.Context) error {
+	logger = logging.NewLogger(logFormat, debug, "version", version)
+
 	logger.Info("Parsing config")
 	if err := cfg.Load(configFile, logger); err != nil {
 		return fmt.Errorf("Couldn't read %s: %w", configFile, err)
@@ -267,17 +284,3 @@ func startScraper(c *cli.Context) error {
 	srv := &http.Server{Addr: addr, Handler: mux}
 	return srv.ListenAndServe()
 }
-
-func newLogger(debug bool) logging.Logger {
-	l := logrus.New()
-	l.SetFormatter(&logrus.JSONFormatter{})
-	l.SetOutput(os.Stdout)
-
-	if debug {
-		l.SetLevel(logrus.DebugLevel)
-	} else {
-		l.SetLevel(logrus.InfoLevel)
-	}
-
-	return logging.NewLogger(l)
-}

docs/configuration.md

@@ -17,6 +17,7 @@ All flags may be prefixed with either one hypen or two (i.e., both `-config.file
 | --- | --- | --- |
 | `-listen-address` | Network address to listen to | `127.0.0.1:5000` |
 | `-config.file` | Path to the configuration file | `config.yml` |
+| `-log.format` | Output format of log messages. One of: [logfmt, json] | `json` |
 | `-debug` | Log at debug level | `false` |
 | `-fips` | Use FIPS compliant AWS API | `false` |
 | `-cloudwatch-concurrency` | Maximum number of concurrent requests to CloudWatch API | `5` |

go.mod

@@ -19,10 +19,10 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/storagegateway v1.18.16
 	github.com/aws/aws-sdk-go-v2/service/sts v1.19.3
 	github.com/aws/smithy-go v1.13.5
+	github.com/go-kit/log v0.2.1
 	github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db
 	github.com/prometheus/client_golang v1.16.0
 	github.com/prometheus/common v0.44.0
-	github.com/sirupsen/logrus v1.9.3
 	github.com/stretchr/testify v1.8.4
 	github.com/urfave/cli/v2 v2.25.7
 	golang.org/x/exp v0.0.0-20230321023759-10a507213a29
@@ -42,6 +42,7 @@ require (
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/go-logfmt/logfmt v0.5.1 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/kr/text v0.2.0 // indirect

go.sum

@@ -54,6 +54,10 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU=
+github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
+github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA=
+github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
@@ -86,10 +90,7 @@ github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPH
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
-github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs=
@@ -115,7 +116,6 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
@@ -141,6 +141,5 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

pkg/config/config.go

@@ -6,7 +6,6 @@ import (
 	"os"
 
 	"github.com/aws/aws-sdk-go/aws"
-	log "github.com/sirupsen/logrus"
 	"gopkg.in/yaml.v2"
 
 	"github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/logging"
@@ -333,9 +332,10 @@ func (m *Metric) validateMetric(metricIdx int, parent string, discovery *JobLeve
 	}
 
 	if mLength < mPeriod {
-		log.Warningf(
+		return fmt.Errorf(
 			"Metric [%s/%d] in %v: length(%d) is smaller than period(%d). This can cause that the data requested is not ready and generate data gaps",
-			m.Name, metricIdx, parent, mLength, mPeriod)
+			m.Name, metricIdx, parent, mLength, mPeriod,
+		)
 	}
 	m.Length = mLength
 	m.Period = mPeriod

pkg/logging/logger.go

@@ -1,14 +1,10 @@
 package logging
 
 import (
-	"encoding"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io"
-	"reflect"
+	"os"
 
-	"github.com/sirupsen/logrus"
+	"github.com/go-kit/log"
+	"github.com/go-kit/log/level"
 )
 
 type Logger interface {
@@ -20,113 +16,69 @@ type Logger interface {
 	IsDebugEnabled() bool
 }
 
-type logrusLogger struct {
-	entry *logrus.Entry
+type gokitLogger struct {
+	logger       log.Logger
+	debugEnabled bool
 }
 
-func (l logrusLogger) Info(message string, keyvals ...interface{}) {
-	l.entry.WithFields(toFields(keyvals...)).Info(message)
-}
+func NewLogger(format string, debugEnabled bool, keyvals ...interface{}) Logger {
+	var logger log.Logger
+	if format == "json" {
+		logger = log.NewJSONLogger(log.NewSyncWriter(os.Stderr))
+	} else {
+		logger = log.NewLogfmtLogger(log.NewSyncWriter(os.Stderr))
+	}
 
-func (l logrusLogger) Debug(message string, keyvals ...interface{}) {
-	l.entry.WithFields(toFields(keyvals...)).Debug(message)
-}
+	if debugEnabled {
+		logger = level.NewFilter(logger, level.AllowDebug())
+	} else {
+		logger = level.NewFilter(logger, level.AllowInfo())
+	}
 
-func (l logrusLogger) Error(err error, message string, keyvals ...interface{}) {
-	l.entry.WithFields(toFields(keyvals...)).WithError(err).Error(message)
-}
+	logger = log.With(logger, "ts", log.DefaultTimestampUTC, "caller", log.Caller(4))
+	logger = log.With(logger, keyvals...)
 
-func (l logrusLogger) Warn(message string, keyvals ...interface{}) {
-	l.entry.WithFields(toFields(keyvals...)).Warn(message)
+	return gokitLogger{
+		logger:       logger,
+		debugEnabled: debugEnabled,
+	}
 }
 
-func (l logrusLogger) With(keyvals ...interface{}) Logger {
-	return logrusLogger{l.entry.WithFields(toFields(keyvals...))}
+func NewNopLogger() Logger {
+	return gokitLogger{logger: log.NewNopLogger()}
 }
 
-func (l logrusLogger) IsDebugEnabled() bool {
-	return l.entry.Logger.IsLevelEnabled(logrus.DebugLevel)
+func (g gokitLogger) Debug(message string, keyvals ...interface{}) {
+	kv := []interface{}{"msg", message}
+	kv = append(kv, keyvals...)
+	level.Debug(g.logger).Log(kv...)
 }
 
-func NewLogger(l *logrus.Logger) logrusLogger { //nolint:revive
-	return logrusLogger{logrus.NewEntry(l)}
+func (g gokitLogger) Info(message string, keyvals ...interface{}) {
+	kv := []interface{}{"msg", message}
+	kv = append(kv, keyvals...)
+	level.Info(g.logger).Log(kv...)
 }
 
-func NewNopLogger() logrusLogger { //nolint:revive
-	l := logrus.New()
-	l.Out = io.Discard
-	return logrusLogger{logrus.NewEntry(l)}
+func (g gokitLogger) Error(err error, message string, keyvals ...interface{}) {
+	kv := []interface{}{"msg", message, "err", err}
+	kv = append(kv, keyvals...)
+	level.Error(g.logger).Log(kv...)
 }
 
-var ErrMissingValue = errors.New("(MISSING)")
-
-// This code is from https://github.com/go-kit/log/blob/main/json_logger.go#L23-L91 which safely handles odd keyvals
-// lengths, and safely converting interface{} -> string
-func toFields(keyvals ...interface{}) logrus.Fields {
-	n := (len(keyvals) + 1) / 2 // +1 to handle case when len is odd
-	m := make(map[string]interface{}, n)
-	for i := 0; i < len(keyvals); i += 2 {
-		k := keyvals[i]
-		var v interface{} = ErrMissingValue
-		if i+1 < len(keyvals) {
-			v = keyvals[i+1]
-		}
-		merge(m, k, v)
-	}
-
-	return m
+func (g gokitLogger) Warn(message string, keyvals ...interface{}) {
+	kv := []interface{}{"msg", message}
+	kv = append(kv, keyvals...)
+	level.Warn(g.logger).Log(kv...)
 }
 
-func merge(dst map[string]interface{}, k, v interface{}) {
-	var key string
-	switch x := k.(type) {
-	case string:
-		key = x
-	case fmt.Stringer:
-		key = safeString(x)
-	default:
-		key = fmt.Sprint(x)
-	}
-
-	// We want json.Marshaler and encoding.TextMarshaller to take priority over
-	// err.Error() and v.String(). But json.Marshall (called later) does that by
-	// default so we force a no-op if it's one of those 2 case.
-	switch x := v.(type) {
-	case json.Marshaler:
-	case encoding.TextMarshaler:
-	case error:
-		v = safeError(x)
-	case fmt.Stringer:
-		v = safeString(x)
+func (g gokitLogger) With(keyvals ...interface{}) Logger {
+	return gokitLogger{
+		logger:       log.With(g.logger, keyvals...),
+		debugEnabled: g.debugEnabled,
 	}
-
-	dst[key] = v
-}
-
-func safeString(str fmt.Stringer) (s string) { //nolint:nonamedreturns
-	defer func() {
-		if panicVal := recover(); panicVal != nil {
-			if v := reflect.ValueOf(str); v.Kind() == reflect.Ptr && v.IsNil() {
-				s = "NULL"
-			} else {
-				panic(panicVal)
-			}
-		}
-	}()
-	s = str.String()
-	return
 }
 
-func safeError(err error) (s interface{}) { //nolint:nonamedreturns
-	defer func() {
-		if panicVal := recover(); panicVal != nil {
-			if v := reflect.ValueOf(err); v.Kind() == reflect.Ptr && v.IsNil() {
-				s = nil
-			} else {
-				panic(panicVal)
-			}
-		}
-	}()
-	s = err.Error()
-	return
+func (g gokitLogger) IsDebugEnabled() bool {
+	return g.debugEnabled
 }