diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d50c3cacf..c3699d14bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ * [CHANGE] Change default value of `-blocks-storage.bucket-store.index-cache.multilevel.max-async-concurrency` from `50` to `3` #6265 * [CHANGE] Enable Compactor and Alertmanager in target all. #6204 * [CHANGE] Update the `cortex_ingester_inflight_push_requests` metric to represent the maximum number of inflight requests recorded in the last minute. #6437 +* [CHANGE] gRPC Client: Expose connection timeout and set default to value to 5s. #6523 * [FEATURE] Ruler: Add an experimental flag `-ruler.query-response-format` to retrieve query response as a proto format. #6345 * [FEATURE] Ruler: Pagination support for List Rules API. #6299 * [FEATURE] Query Frontend/Querier: Add protobuf codec `-api.querier-default-codec` and the option to choose response compression type `-querier.response-compression`. #5527 diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 0b977b3aff..69d6713e93 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -266,6 +266,11 @@ query_scheduler: # CLI flag: -query-scheduler.grpc-client-config.tls-insecure-skip-verify [tls_insecure_skip_verify: | default = false] + # The maximum amount of time to establish a connection. A value of 0 means + # using default gRPC client connect timeout 20s. + # CLI flag: -query-scheduler.grpc-client-config.connect-timeout + [connect_timeout: | default = 5s] + # The tracing_config configures backends cortex uses. [tracing: ] ``` @@ -2972,6 +2977,11 @@ grpc_client_config: # Skip validating server certificate. # CLI flag: -querier.frontend-client.tls-insecure-skip-verify [tls_insecure_skip_verify: | default = false] + + # The maximum amount of time to establish a connection. A value of 0 means + # using default gRPC client connect timeout 20s. + # CLI flag: -querier.frontend-client.connect-timeout + [connect_timeout: | default = 5s] ``` ### `ingester_config` @@ -3282,6 +3292,11 @@ grpc_client_config: # CLI flag: -ingester.client.tls-insecure-skip-verify [tls_insecure_skip_verify: | default = false] + # The maximum amount of time to establish a connection. A value of 0 means + # using default gRPC client connect timeout 20s. + # CLI flag: -ingester.client.connect-timeout + [connect_timeout: | default = 5s] + # EXPERIMENTAL: If enabled, gRPC clients perform health checks for each target # and fail the request if the target is marked as unhealthy. healthcheck_config: @@ -4200,6 +4215,11 @@ grpc_client_config: # CLI flag: -frontend.grpc-client-config.tls-insecure-skip-verify [tls_insecure_skip_verify: | default = false] + # The maximum amount of time to establish a connection. A value of 0 means + # using default gRPC client connect timeout 20s. + # CLI flag: -frontend.grpc-client-config.connect-timeout + [connect_timeout: | default = 5s] + # When multiple query-schedulers are available, re-enqueue queries that were # rejected due to too many outstanding requests. # CLI flag: -frontend.retry-on-too-many-outstanding-requests @@ -4426,6 +4446,11 @@ frontend_client: # CLI flag: -ruler.frontendClient.tls-insecure-skip-verify [tls_insecure_skip_verify: | default = false] + # The maximum amount of time to establish a connection. A value of 0 means + # using default gRPC client connect timeout 20s. + # CLI flag: -ruler.frontendClient.connect-timeout + [connect_timeout: | default = 5s] + # URL of alerts return path. # CLI flag: -ruler.external.url [external_url: | default = ] @@ -4501,6 +4526,11 @@ ruler_client: # CLI flag: -ruler.client.tls-insecure-skip-verify [tls_insecure_skip_verify: | default = false] + # The maximum amount of time to establish a connection. A value of 0 means + # using default gRPC client connect timeout 20s. + # CLI flag: -ruler.client.connect-timeout + [connect_timeout: | default = 5s] + # Timeout for downstream rulers. # CLI flag: -ruler.client.remote-timeout [remote_timeout: | default = 2m] diff --git a/pkg/util/grpcclient/grpcclient.go b/pkg/util/grpcclient/grpcclient.go index 9816b7ab6c..6adb8139a2 100644 --- a/pkg/util/grpcclient/grpcclient.go +++ b/pkg/util/grpcclient/grpcclient.go @@ -8,6 +8,7 @@ import ( middleware "github.com/grpc-ecosystem/go-grpc-middleware" "github.com/pkg/errors" "google.golang.org/grpc" + grpcbackoff "google.golang.org/grpc/backoff" "google.golang.org/grpc/encoding/gzip" "google.golang.org/grpc/keepalive" @@ -32,6 +33,8 @@ type Config struct { TLSEnabled bool `yaml:"tls_enabled"` TLS tls.ClientConfig `yaml:",inline"` SignWriteRequestsEnabled bool `yaml:"-"` + + ConnectTimeout time.Duration `yaml:"connect_timeout"` } type ConfigWithHealthCheck struct { @@ -58,6 +61,7 @@ func (cfg *Config) RegisterFlagsWithPrefix(prefix, defaultGrpcCompression string f.IntVar(&cfg.RateLimitBurst, prefix+".grpc-client-rate-limit-burst", 0, "Rate limit burst for gRPC client.") f.BoolVar(&cfg.BackoffOnRatelimits, prefix+".backoff-on-ratelimits", false, "Enable backoff and retry when we hit ratelimits.") f.BoolVar(&cfg.TLSEnabled, prefix+".tls-enabled", cfg.TLSEnabled, "Enable TLS in the GRPC client. This flag needs to be enabled when any other TLS flag is set. If set to false, insecure connection to gRPC server will be used.") + f.DurationVar(&cfg.ConnectTimeout, prefix+".connect-timeout", 5*time.Second, "The maximum amount of time to establish a connection. A value of 0 means using default gRPC client connect timeout 20s.") cfg.BackoffConfig.RegisterFlagsWithPrefix(prefix, f) @@ -111,6 +115,16 @@ func (cfg *Config) DialOption(unaryClientInterceptors []grpc.UnaryClientIntercep unaryClientInterceptors = append([]grpc.UnaryClientInterceptor{NewRateLimiter(cfg)}, unaryClientInterceptors...) } + if cfg.ConnectTimeout > 0 { + opts = append( + opts, + grpc.WithConnectParams(grpc.ConnectParams{ + Backoff: grpcbackoff.DefaultConfig, + MinConnectTimeout: cfg.ConnectTimeout, + }), + ) + } + if cfg.SignWriteRequestsEnabled { unaryClientInterceptors = append(unaryClientInterceptors, UnarySigningClientInterceptor) }