diff --git a/cmd/pd-server/main.go b/cmd/pd-server/main.go index d45d2f641d2..0250a36d4be 100644 --- a/cmd/pd-server/main.go +++ b/cmd/pd-server/main.go @@ -227,6 +227,7 @@ func start(cmd *cobra.Command, args []string, services ...string) { grpcprometheus.EnableHandlingTimeHistogram() metricutil.Push(&cfg.Metric) + metricutil.EnablePyroscope() err = join.PrepareJoinCluster(cfg) if err != nil { diff --git a/go.mod b/go.mod index bd5e04f42c0..4263c4dbb6a 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( github.com/gogo/protobuf v1.3.2 github.com/google/btree v1.1.2 github.com/gorilla/mux v1.7.4 + github.com/grafana/pyroscope-go v1.2.0 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/joho/godotenv v1.4.0 github.com/mailru/easyjson v0.7.6 @@ -126,6 +127,7 @@ require ( github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.5.1 // indirect + github.com/grafana/pyroscope-go/godeltaprof v0.1.8 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect diff --git a/go.sum b/go.sum index f9a3a3f820a..7988b94deb0 100644 --- a/go.sum +++ b/go.sum @@ -245,6 +245,10 @@ github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB7 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= +github.com/grafana/pyroscope-go v1.2.0 h1:aILLKjTj8CS8f/24OPMGPewQSYlhmdQMBmol1d3KGj8= +github.com/grafana/pyroscope-go v1.2.0/go.mod h1:2GHr28Nr05bg2pElS+dDsc98f3JTUh2f6Fz1hWXrqwk= +github.com/grafana/pyroscope-go/godeltaprof v0.1.8 h1:iwOtYXeeVSAeYefJNaxDytgjKtUuKQbJqgAIjlnicKg= +github.com/grafana/pyroscope-go/godeltaprof v0.1.8/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= diff --git a/pkg/utils/metricutil/metricutil.go b/pkg/utils/metricutil/metricutil.go index 32583248d5b..8e636c20b87 100644 --- a/pkg/utils/metricutil/metricutil.go +++ b/pkg/utils/metricutil/metricutil.go @@ -16,11 +16,14 @@ package metricutil import ( "os" + "runtime" "time" "unicode" + "github.com/grafana/pyroscope-go" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/push" + "go.uber.org/zap" "github.com/pingcap/log" @@ -102,3 +105,28 @@ func instanceName() string { } return hostname } + +// EnablePyroscope enables pyroscope if pyroscope is enabled. +func EnablePyroscope() { + if os.Getenv("PYROSCOPE_SERVER_ADDRESS") != "" { + runtime.SetMutexProfileFraction(5) + runtime.SetBlockProfileRate(5) + _, err := pyroscope.Start(pyroscope.Config{ + ApplicationName: "pd", + ServerAddress: os.Getenv("PYROSCOPE_SERVER_ADDRESS"), + Logger: pyroscope.StandardLogger, + AuthToken: os.Getenv("PYROSCOPE_AUTH_TOKEN"), + TenantID: os.Getenv("PYROSCOPE_TENANT_ID"), + BasicAuthUser: os.Getenv("PYROSCOPE_BASIC_AUTH_USER"), + BasicAuthPassword: os.Getenv("PYROSCOPE_BASIC_AUTH_PASSWORD"), + ProfileTypes: []pyroscope.ProfileType{ + pyroscope.ProfileCPU, + pyroscope.ProfileAllocSpace, + }, + UploadRate: 30 * time.Second, + }) + if err != nil { + log.Fatal("fail to start pyroscope", zap.Error(err)) + } + } +} diff --git a/tests/integrations/go.mod b/tests/integrations/go.mod index 7eb698f204c..a8884794f88 100644 --- a/tests/integrations/go.mod +++ b/tests/integrations/go.mod @@ -101,6 +101,8 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/gorilla/mux v1.7.4 // indirect github.com/gorilla/websocket v1.5.1 // indirect + github.com/grafana/pyroscope-go v1.2.0 // indirect + github.com/grafana/pyroscope-go/godeltaprof v0.1.8 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect diff --git a/tests/integrations/go.sum b/tests/integrations/go.sum index a892229f801..bf2b01a737a 100644 --- a/tests/integrations/go.sum +++ b/tests/integrations/go.sum @@ -237,6 +237,10 @@ github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB7 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= +github.com/grafana/pyroscope-go v1.2.0 h1:aILLKjTj8CS8f/24OPMGPewQSYlhmdQMBmol1d3KGj8= +github.com/grafana/pyroscope-go v1.2.0/go.mod h1:2GHr28Nr05bg2pElS+dDsc98f3JTUh2f6Fz1hWXrqwk= +github.com/grafana/pyroscope-go/godeltaprof v0.1.8 h1:iwOtYXeeVSAeYefJNaxDytgjKtUuKQbJqgAIjlnicKg= +github.com/grafana/pyroscope-go/godeltaprof v0.1.8/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= diff --git a/tools/go.mod b/tools/go.mod index d24b1699ddc..9f877cddf2e 100644 --- a/tools/go.mod +++ b/tools/go.mod @@ -106,6 +106,8 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/gorilla/mux v1.7.4 // indirect github.com/gorilla/websocket v1.5.1 // indirect + github.com/grafana/pyroscope-go v1.2.0 // indirect + github.com/grafana/pyroscope-go/godeltaprof v0.1.8 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect diff --git a/tools/go.sum b/tools/go.sum index faed947d81a..43cb66c64a8 100644 --- a/tools/go.sum +++ b/tools/go.sum @@ -236,6 +236,10 @@ github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB7 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= +github.com/grafana/pyroscope-go v1.2.0 h1:aILLKjTj8CS8f/24OPMGPewQSYlhmdQMBmol1d3KGj8= +github.com/grafana/pyroscope-go v1.2.0/go.mod h1:2GHr28Nr05bg2pElS+dDsc98f3JTUh2f6Fz1hWXrqwk= +github.com/grafana/pyroscope-go/godeltaprof v0.1.8 h1:iwOtYXeeVSAeYefJNaxDytgjKtUuKQbJqgAIjlnicKg= +github.com/grafana/pyroscope-go/godeltaprof v0.1.8/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho=