From 9c9c96f266a01d145b9f1d2dcfba9e725f860e82 Mon Sep 17 00:00:00 2001 From: Roman Lomonosov Date: Sat, 13 Jul 2019 20:42:27 +0300 Subject: [PATCH] grafana dashboard in access log and user agent --- graphite-clickhouse.go | 55 +++++++----------------- helper/clickhouse/clickhouse.go | 3 +- pkg/scope/http_request.go | 47 ++++++++++++++++++++ pkg/scope/key.go | 14 +++++- pkg/scope/logger.go | 19 +++++--- {helper/version => pkg/scope}/version.go | 2 +- 6 files changed, 92 insertions(+), 48 deletions(-) create mode 100644 pkg/scope/http_request.go rename {helper/version => pkg/scope}/version.go (55%) diff --git a/graphite-clickhouse.go b/graphite-clickhouse.go index ec791f1f3..455396278 100644 --- a/graphite-clickhouse.go +++ b/graphite-clickhouse.go @@ -1,21 +1,18 @@ package main import ( - "encoding/binary" "encoding/json" "flag" "fmt" "log" "math/rand" "net/http" - "regexp" "runtime" "time" "github.com/lomik/graphite-clickhouse/autocomplete" "github.com/lomik/graphite-clickhouse/config" "github.com/lomik/graphite-clickhouse/find" - "github.com/lomik/graphite-clickhouse/helper/version" "github.com/lomik/graphite-clickhouse/index" "github.com/lomik/graphite-clickhouse/pkg/scope" "github.com/lomik/graphite-clickhouse/prometheus" @@ -31,7 +28,7 @@ import ( const Version = "0.11.5" func init() { - version.Version = Version + scope.Version = Version } type LogResponseWriter struct { @@ -58,43 +55,23 @@ func WrapResponseWriter(w http.ResponseWriter) *LogResponseWriter { return &LogResponseWriter{ResponseWriter: w} } -var requestIdRegexp *regexp.Regexp = regexp.MustCompile("^[a-zA-Z0-9_.-]+$") -var passHeaders = []string{ - "X-Dashboard-Id", - "X-Grafana-Org-Id", - "X-Panel-Id", -} - -func Handler(logger *zap.Logger, handler http.Handler) http.Handler { +func Handler(handler http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { writer := WrapResponseWriter(w) - requestID := r.Header.Get("X-Request-Id") - if requestID == "" || !requestIdRegexp.MatchString(requestID) { - var b [16]byte - binary.LittleEndian.PutUint64(b[:], rand.Uint64()) - binary.LittleEndian.PutUint64(b[8:], rand.Uint64()) - requestID = fmt.Sprintf("%x", b) - } + r = scope.HttpRequest(r) - logger := logger.With(zap.String("request_id", requestID)) + start := time.Now() + handler.ServeHTTP(writer, r) + d := time.Since(start) - ctx := r.Context() - ctx = scope.WithLogger(ctx, logger) - ctx = scope.WithRequestID(ctx, requestID) + logger := scope.Logger(r.Context()) - for _, h := range passHeaders { - hv := r.Header.Get(h) - if hv != "" { - ctx = scope.With(ctx, h, hv) - } + grafana := scope.Grafana(r.Context()) + if grafana != "" { + logger = logger.With(zap.String("grafana", grafana)) } - r = r.WithContext(ctx) - - start := time.Now() - handler.ServeHTTP(writer, r) - d := time.Since(start) logger.Info("access", zap.Duration("time", d), zap.String("method", r.Method), @@ -166,11 +143,11 @@ func main() { /* CONSOLE COMMANDS end */ - http.Handle("/metrics/find/", Handler(zapwriter.Default(), find.NewHandler(cfg))) - http.Handle("/metrics/index.json", Handler(zapwriter.Default(), index.NewHandler(cfg))) - http.Handle("/render/", Handler(zapwriter.Default(), render.NewHandler(cfg))) - http.Handle("/tags/autoComplete/tags", Handler(zapwriter.Default(), autocomplete.NewTags(cfg))) - http.Handle("/tags/autoComplete/values", Handler(zapwriter.Default(), autocomplete.NewValues(cfg))) + http.Handle("/metrics/find/", Handler(find.NewHandler(cfg))) + http.Handle("/metrics/index.json", Handler(index.NewHandler(cfg))) + http.Handle("/render/", Handler(render.NewHandler(cfg))) + http.Handle("/tags/autoComplete/tags", Handler(autocomplete.NewTags(cfg))) + http.Handle("/tags/autoComplete/values", Handler(autocomplete.NewValues(cfg))) http.HandleFunc("/debug/config", func(w http.ResponseWriter, r *http.Request) { b, err := json.MarshalIndent(cfg, "", " ") if err != nil { @@ -180,7 +157,7 @@ func main() { w.Write(b) }) - http.Handle("/", Handler(zapwriter.Default(), prometheus.NewHandler(cfg))) + http.Handle("/", Handler(prometheus.NewHandler(cfg))) log.Fatal(http.ListenAndServe(cfg.Common.Listen, nil)) } diff --git a/helper/clickhouse/clickhouse.go b/helper/clickhouse/clickhouse.go index e9b881159..34585c6c0 100644 --- a/helper/clickhouse/clickhouse.go +++ b/helper/clickhouse/clickhouse.go @@ -14,7 +14,6 @@ import ( "strings" "time" - "github.com/lomik/graphite-clickhouse/helper/version" "github.com/lomik/graphite-clickhouse/pkg/dry" "github.com/lomik/graphite-clickhouse/pkg/scope" "github.com/lomik/zapwriter" @@ -196,7 +195,7 @@ func reader(ctx context.Context, dsn string, query string, postBody io.Reader, g return } - req.Header.Add("User-Agent", fmt.Sprintf("graphite-clickhouse/%s (table:%s)", version.Version, scope.Table(ctx))) + req.Header.Add("User-Agent", scope.ClickhouseUserAgent(ctx)) if gzip { req.Header.Add("Content-Encoding", "gzip") diff --git a/pkg/scope/http_request.go b/pkg/scope/http_request.go new file mode 100644 index 000000000..dcad9bfda --- /dev/null +++ b/pkg/scope/http_request.go @@ -0,0 +1,47 @@ +package scope + +import ( + "context" + "encoding/binary" + "fmt" + "math/rand" + "net/http" + "regexp" +) + +var requestIdRegexp *regexp.Regexp = regexp.MustCompile("^[a-zA-Z0-9_.-]+$") +var passHeaders = []string{ + "X-Dashboard-Id", + "X-Grafana-Org-Id", + "X-Panel-Id", +} + +func HttpRequest(r *http.Request) *http.Request { + requestID := r.Header.Get("X-Request-Id") + if requestID == "" || !requestIdRegexp.MatchString(requestID) { + var b [16]byte + binary.LittleEndian.PutUint64(b[:], rand.Uint64()) + binary.LittleEndian.PutUint64(b[8:], rand.Uint64()) + requestID = fmt.Sprintf("%x", b) + } + + ctx := r.Context() + ctx = WithRequestID(ctx, requestID) + + for _, h := range passHeaders { + hv := r.Header.Get(h) + if hv != "" { + ctx = With(ctx, h, hv) + } + } + + return r.WithContext(ctx) +} + +func Grafana(ctx context.Context) string { + o, d, p := String(ctx, "X-Grafana-Org-Id"), String(ctx, "X-Dashboard-Id"), String(ctx, "X-Panel-Id") + if o != "" || d != "" || p != "" { + return fmt.Sprintf("Org:%s; Dashboard:%s; Panel:%s", o, d, p) + } + return "" +} diff --git a/pkg/scope/key.go b/pkg/scope/key.go index 6edceb817..84e85e173 100644 --- a/pkg/scope/key.go +++ b/pkg/scope/key.go @@ -1,6 +1,9 @@ package scope -import "context" +import ( + "context" + "fmt" +) // key is type for context.Value keys type scopeKey string @@ -37,3 +40,12 @@ func WithTable(ctx context.Context, table string) context.Context { func Table(ctx context.Context) string { return String(ctx, "table") } + +// ClickhouseUserAgent ... +func ClickhouseUserAgent(ctx context.Context) string { + grafana := Grafana(ctx) + if grafana != "" { + return fmt.Sprintf("Graphite-Clickhouse/%s (table:%s) Grafana(%s)", Version, Table(ctx), grafana) + } + return fmt.Sprintf("Graphite-Clickhouse/%s (table:%s)", Version, Table(ctx)) +} diff --git a/pkg/scope/logger.go b/pkg/scope/logger.go index 1aa9dbe97..076ae3773 100644 --- a/pkg/scope/logger.go +++ b/pkg/scope/logger.go @@ -3,20 +3,29 @@ package scope import ( "context" + "github.com/lomik/zapwriter" "go.uber.org/zap" ) // Logger returns zap.Logger instance func Logger(ctx context.Context) *zap.Logger { logger := ctx.Value(scopeKey("logger")) - if logger == nil { - return zap.NewNop() + var zapLogger *zap.Logger + if logger != nil { + if zl, ok := logger.(*zap.Logger); ok { + zapLogger = zl + } } - if zapLogger, ok := logger.(*zap.Logger); ok { - return zapLogger + if zapLogger == nil { + zapLogger = zapwriter.Default() } - return zap.NewNop() + requestId := RequestID(ctx) + if requestId != "" { + zapLogger = zapLogger.With(zap.String("request_id", requestId)) + } + + return zapLogger } // WithLogger ... diff --git a/helper/version/version.go b/pkg/scope/version.go similarity index 55% rename from helper/version/version.go rename to pkg/scope/version.go index 7f6928ef4..d90fef75e 100644 --- a/helper/version/version.go +++ b/pkg/scope/version.go @@ -1,3 +1,3 @@ -package version +package scope var Version string