diff --git a/agent/cmd/agent/main.go b/agent/cmd/agent/main.go index 79eab275e..4d1846a11 100644 --- a/agent/cmd/agent/main.go +++ b/agent/cmd/agent/main.go @@ -90,6 +90,10 @@ func doMain(ctx context.Context, restConfig *rest.Config, agentConfig *config.Ag return 1 } + if agentConfig.EnablePprof { + go utils.StartDefaultPprofServer() + } + mgr, err := createManager(ctx, restConfig, agentConfig) if err != nil { setupLog.Error(err, "failed to create manager") @@ -171,6 +175,7 @@ func parseFlags() *config.AgentConfig { "QPS for the multicluster global hub agent") pflag.IntVar(&agentConfig.Burst, "burst", 300, "Burst for the multicluster global hub agent") + pflag.BoolVar(&agentConfig.EnablePprof, "enable-pprof", false, "Enable the pprof tool.") pflag.Parse() // set zap logger diff --git a/agent/pkg/config/agent_config.go b/agent/pkg/config/agent_config.go index 7d4225d91..00ae8a6ca 100644 --- a/agent/pkg/config/agent_config.go +++ b/agent/pkg/config/agent_config.go @@ -18,4 +18,5 @@ type AgentConfig struct { EnableGlobalResource bool QPS float32 Burst int + EnablePprof bool } diff --git a/manager/cmd/manager/main.go b/manager/cmd/manager/main.go index 48d2c7c19..7672009d6 100644 --- a/manager/cmd/manager/main.go +++ b/manager/cmd/manager/main.go @@ -167,7 +167,7 @@ func parseFlags() *managerconfig.ManagerConfig { "data retention indicates how many months the expired data will kept in the database") pflag.BoolVar(&managerConfig.EnableGlobalResource, "enable-global-resource", false, "enable the global resource feature.") - + pflag.BoolVar(&managerConfig.EnablePprof, "enable-pprof", false, "Enable the pprof tool.") pflag.Parse() // set zap logger ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) @@ -299,6 +299,11 @@ func doMain(ctx context.Context, restConfig *rest.Config) int { setupLog.Error(err, "failed to complete configuration") return 1 } + + if managerConfig.EnablePprof { + go utils.StartDefaultPprofServer() + } + utils.PrintVersion(setupLog) databaseConfig := &database.DatabaseConfig{ URL: managerConfig.DatabaseConfig.ProcessDatabaseURL, diff --git a/manager/pkg/config/manager_config.go b/manager/pkg/config/manager_config.go index 787eb7548..e14091305 100644 --- a/manager/pkg/config/manager_config.go +++ b/manager/pkg/config/manager_config.go @@ -24,6 +24,7 @@ type ManagerConfig struct { ElectionConfig *commonobjects.LeaderElectionConfig EnableGlobalResource bool LaunchJobNames string + EnablePprof bool } type SyncerConfig struct { diff --git a/operator/main.go b/operator/main.go index 242e0d683..69d0098f0 100644 --- a/operator/main.go +++ b/operator/main.go @@ -46,6 +46,10 @@ func doMain(ctx context.Context, cfg *rest.Config) int { operatorConfig := parseFlags() utils.PrintVersion(setupLog) + if operatorConfig.EnablePprof { + go utils.StartDefaultPprofServer() + } + kubeClient, err := kubernetes.NewForConfig(cfg) if err != nil { setupLog.Error(err, "failed to create kube client") @@ -106,6 +110,7 @@ func parseFlags() *config.OperatorConfig { "Enable leader election for controller manager. ") pflag.BoolVar(&config.GlobalResourceEnabled, "global-resource-enabled", false, "Enable the global resource. It is expermental feature. Do not support upgrade.") + pflag.BoolVar(&config.EnablePprof, "enable-pprof", false, "Enable the pprof tool.") pflag.Parse() config.LogLevel = "info" diff --git a/operator/pkg/config/operator_config.go b/operator/pkg/config/operator_config.go index 2a691b4eb..ad01cf0b5 100644 --- a/operator/pkg/config/operator_config.go +++ b/operator/pkg/config/operator_config.go @@ -6,5 +6,6 @@ type OperatorConfig struct { PodNamespace string LeaderElection bool GlobalResourceEnabled bool + EnablePprof bool LogLevel string } diff --git a/operator/pkg/controllers/addon/addon_controller.go b/operator/pkg/controllers/addon/addon_controller.go index 06c4554cf..a97a77767 100644 --- a/operator/pkg/controllers/addon/addon_controller.go +++ b/operator/pkg/controllers/addon/addon_controller.go @@ -41,12 +41,11 @@ import ( // +kubebuilder:rbac:groups=packages.operators.coreos.com,resources=packagemanifests,verbs=get;list;watch type AddonController struct { - addonManager addonmanager.AddonManager - kubeConfig *rest.Config - client client.Client - log logr.Logger - EnableGlobalResource bool - LogLevel string + addonManager addonmanager.AddonManager + kubeConfig *rest.Config + client client.Client + log logr.Logger + operatorConfig *config.OperatorConfig } // used to create addon manager @@ -59,12 +58,11 @@ func NewAddonController(kubeConfig *rest.Config, client client.Client, operatorC return nil, err } return &AddonController{ - kubeConfig: kubeConfig, - client: client, - log: log, - addonManager: addonMgr, - EnableGlobalResource: operatorConfig.GlobalResourceEnabled, - LogLevel: operatorConfig.LogLevel, + kubeConfig: kubeConfig, + client: client, + log: log, + addonManager: addonMgr, + operatorConfig: operatorConfig, }, nil } @@ -93,13 +91,12 @@ func (a *AddonController) Start(ctx context.Context) error { } hohAgentAddon := HohAgentAddon{ - ctx: ctx, - kubeClient: kubeClient, - client: a.client, - dynamicClient: dynamicClient, - log: a.log.WithName("values"), - EnableGlobalResource: a.EnableGlobalResource, - LogLevel: a.LogLevel, + ctx: ctx, + kubeClient: kubeClient, + client: a.client, + dynamicClient: dynamicClient, + log: a.log.WithName("values"), + operatorConfig: a.operatorConfig, } _, err = utils.WaitGlobalHubReady(ctx, a.client, 5*time.Second) if err != nil { diff --git a/operator/pkg/controllers/addon/addon_controller_manifests.go b/operator/pkg/controllers/addon/addon_controller_manifests.go index 2af8b219e..d796fbba5 100644 --- a/operator/pkg/controllers/addon/addon_controller_manifests.go +++ b/operator/pkg/controllers/addon/addon_controller_manifests.go @@ -68,6 +68,7 @@ type ManifestsConfig struct { AgentQPS float32 AgentBurst int LogLevel string + EnablePprof bool // cannot use *corev1.ResourceRequirements, addonfactory.StructToValues removes the real value Resources *Resources } @@ -81,13 +82,12 @@ type Resources struct { } type HohAgentAddon struct { - ctx context.Context - client client.Client - kubeClient kubernetes.Interface - dynamicClient dynamic.Interface - log logr.Logger - EnableGlobalResource bool - LogLevel string + ctx context.Context + client client.Client + kubeClient kubernetes.Interface + dynamicClient dynamic.Interface + log logr.Logger + operatorConfig *config.OperatorConfig } func (a *HohAgentAddon) getMulticlusterGlobalHub() (*globalhubv1alpha4.MulticlusterGlobalHub, error) { @@ -227,10 +227,11 @@ func (a *HohAgentAddon) GetValues(cluster *clusterv1.ManagedCluster, RetryPeriod: strconv.Itoa(electionConfig.RetryPeriod), KlusterletNamespace: "open-cluster-management-agent", KlusterletWorkSA: "klusterlet-work-sa", - EnableGlobalResource: a.EnableGlobalResource, + EnableGlobalResource: a.operatorConfig.GlobalResourceEnabled, AgentQPS: agentQPS, AgentBurst: agentBurst, - LogLevel: a.LogLevel, + LogLevel: a.operatorConfig.LogLevel, + EnablePprof: a.operatorConfig.EnablePprof, Resources: agentRes, } diff --git a/operator/pkg/controllers/addon/addon_suite_test.go b/operator/pkg/controllers/addon/addon_suite_test.go index 8d05d1842..e5bd2aa0d 100644 --- a/operator/pkg/controllers/addon/addon_suite_test.go +++ b/operator/pkg/controllers/addon/addon_suite_test.go @@ -114,6 +114,7 @@ var _ = BeforeSuite(func() { addonController, err := addon.NewAddonController(k8sManager.GetConfig(), k8sClient, &config.OperatorConfig{ GlobalResourceEnabled: true, LogLevel: "info", + EnablePprof: false, }) Expect(err).ToNot(HaveOccurred()) err = k8sManager.Add(addonController) diff --git a/operator/pkg/controllers/addon/manifests/templates/agent/multicluster-global-hub-agent-deployment.yaml b/operator/pkg/controllers/addon/manifests/templates/agent/multicluster-global-hub-agent-deployment.yaml index 5faafeb1b..dd377034c 100644 --- a/operator/pkg/controllers/addon/manifests/templates/agent/multicluster-global-hub-agent-deployment.yaml +++ b/operator/pkg/controllers/addon/manifests/templates/agent/multicluster-global-hub-agent-deployment.yaml @@ -55,6 +55,7 @@ spec: - --enable-global-resource={{.EnableGlobalResource}} - --qps={{.AgentQPS}} - --burst={{.AgentBurst}} + - --enable-pprof={{.EnablePprof}} env: - name: POD_NAMESPACE valueFrom: diff --git a/operator/pkg/controllers/hubofhubs/globalhub_controller.go b/operator/pkg/controllers/hubofhubs/globalhub_controller.go index 070cf8c24..f4ece2b31 100644 --- a/operator/pkg/controllers/hubofhubs/globalhub_controller.go +++ b/operator/pkg/controllers/hubofhubs/globalhub_controller.go @@ -85,31 +85,29 @@ var watchedConfigmap = sets.NewString( type MulticlusterGlobalHubReconciler struct { manager.Manager client.Client - Scheme *runtime.Scheme - addonMgr addonmanager.AddonManager - KubeClient kubernetes.Interface - Log logr.Logger - LogLevel string - MiddlewareConfig *MiddlewareConfig - EnableGlobalResource bool - upgradeOnce sync.Once - KafkaInit bool - KafkaController *KafkaController + Scheme *runtime.Scheme + addonMgr addonmanager.AddonManager + KubeClient kubernetes.Interface + Log logr.Logger + MiddlewareConfig *MiddlewareConfig + OperatorConfig *config.OperatorConfig + upgradeOnce sync.Once + KafkaInit bool + KafkaController *KafkaController } func NewMulticlusterGlobalHubReconciler(mgr ctrl.Manager, addonMgr addonmanager.AddonManager, kubeClient kubernetes.Interface, operatorConfig *config.OperatorConfig, ) *MulticlusterGlobalHubReconciler { return &MulticlusterGlobalHubReconciler{ - Manager: mgr, - Client: mgr.GetClient(), - addonMgr: addonMgr, - KubeClient: kubeClient, - Scheme: mgr.GetScheme(), - Log: ctrl.Log.WithName("global-hub-reconciler"), - MiddlewareConfig: &MiddlewareConfig{}, - EnableGlobalResource: operatorConfig.GlobalResourceEnabled, - LogLevel: operatorConfig.LogLevel, + Manager: mgr, + Client: mgr.GetClient(), + addonMgr: addonMgr, + KubeClient: kubeClient, + Scheme: mgr.GetScheme(), + Log: ctrl.Log.WithName("global-hub-reconciler"), + MiddlewareConfig: &MiddlewareConfig{}, + OperatorConfig: operatorConfig, } } diff --git a/operator/pkg/controllers/hubofhubs/globalhub_database.go b/operator/pkg/controllers/hubofhubs/globalhub_database.go index 900b96df3..8735e86d1 100644 --- a/operator/pkg/controllers/hubofhubs/globalhub_database.go +++ b/operator/pkg/controllers/hubofhubs/globalhub_database.go @@ -97,7 +97,7 @@ func (r *MulticlusterGlobalHubReconciler) ReconcileDatabase(ctx context.Context, return err } - if r.EnableGlobalResource { + if r.OperatorConfig.GlobalResourceEnabled { if err := applySQL(ctx, conn, databaseOldFS, "database.old", readonlyUsername); err != nil { return err } diff --git a/operator/pkg/controllers/hubofhubs/globalhub_manager.go b/operator/pkg/controllers/hubofhubs/globalhub_manager.go index f370d42b5..4fdc4c740 100644 --- a/operator/pkg/controllers/hubofhubs/globalhub_manager.go +++ b/operator/pkg/controllers/hubofhubs/globalhub_manager.go @@ -140,8 +140,9 @@ func (r *MulticlusterGlobalHubReconciler) reconcileManager(ctx context.Context, Tolerations: mgh.Spec.Tolerations, RetentionMonth: months, StatisticLogInterval: config.GetStatisticLogInterval(), - EnableGlobalResource: r.EnableGlobalResource, - LogLevel: r.LogLevel, + EnableGlobalResource: r.OperatorConfig.GlobalResourceEnabled, + EnablePprof: r.OperatorConfig.EnablePprof, + LogLevel: r.OperatorConfig.LogLevel, Resources: utils.GetResources(operatorconstants.Manager, mgh.Spec.AdvancedConfig), }, nil }) @@ -257,6 +258,7 @@ type ManagerVariables struct { RetentionMonth int StatisticLogInterval string EnableGlobalResource bool + EnablePprof bool LogLevel string Resources *corev1.ResourceRequirements } diff --git a/operator/pkg/controllers/hubofhubs/integration_test.go b/operator/pkg/controllers/hubofhubs/integration_test.go index c3f5fdd26..4e32258d5 100644 --- a/operator/pkg/controllers/hubofhubs/integration_test.go +++ b/operator/pkg/controllers/hubofhubs/integration_test.go @@ -378,6 +378,7 @@ var _ = Describe("MulticlusterGlobalHub controller", Ordered, func() { StatisticLogInterval: config.GetStatisticLogInterval(), EnableGlobalResource: true, LaunchJobNames: config.GetLaunchJobNames(mgh), + EnablePprof: false, LogLevel: "info", Resources: operatorutils.GetResources(operatorconstants.Manager, mgh.Spec.AdvancedConfig), }, nil diff --git a/operator/pkg/controllers/hubofhubs/manifests/manager/deployment.yaml b/operator/pkg/controllers/hubofhubs/manifests/manager/deployment.yaml index 5bc829329..705eac83e 100644 --- a/operator/pkg/controllers/hubofhubs/manifests/manager/deployment.yaml +++ b/operator/pkg/controllers/hubofhubs/manifests/manager/deployment.yaml @@ -59,6 +59,7 @@ spec: {{- end}} - --data-retention={{.RetentionMonth}} - --statistics-log-interval={{.StatisticLogInterval}} + - --enable-pprof={{.EnablePprof}} {{- if eq .SkipAuth true}} - --cluster-api-url= {{- end}} diff --git a/operator/pkg/controllers/hubofhubs/suite_test.go b/operator/pkg/controllers/hubofhubs/suite_test.go index 988c31be7..60240300a 100644 --- a/operator/pkg/controllers/hubofhubs/suite_test.go +++ b/operator/pkg/controllers/hubofhubs/suite_test.go @@ -157,14 +157,17 @@ var _ = BeforeSuite(func() { Expect(err).Should(Succeed()) mghReconciler = &hubofhubscontroller.MulticlusterGlobalHubReconciler{ - Manager: k8sManager, - Client: k8sManager.GetClient(), - KubeClient: kubeClient, - Scheme: k8sManager.GetScheme(), - Log: ctrl.Log.WithName("multicluster-global-hub-reconciler"), - LogLevel: "info", - MiddlewareConfig: &hubofhubscontroller.MiddlewareConfig{}, - EnableGlobalResource: true, + Manager: k8sManager, + Client: k8sManager.GetClient(), + KubeClient: kubeClient, + Scheme: k8sManager.GetScheme(), + Log: ctrl.Log.WithName("multicluster-global-hub-reconciler"), + MiddlewareConfig: &hubofhubscontroller.MiddlewareConfig{}, + OperatorConfig: &config.OperatorConfig{ + LogLevel: "info", + EnablePprof: false, + GlobalResourceEnabled: true, + }, } Expect(mghReconciler.SetupWithManager(k8sManager)).ToNot(HaveOccurred()) diff --git a/pkg/utils/pprof.go b/pkg/utils/pprof.go new file mode 100644 index 000000000..4138d88d1 --- /dev/null +++ b/pkg/utils/pprof.go @@ -0,0 +1,19 @@ +package utils + +import ( + "net/http" + _ "net/http/pprof" // #nosec G108 + "time" + + "k8s.io/klog" +) + +func StartDefaultPprofServer() { + server := &http.Server{ + Addr: ":6060", + ReadHeaderTimeout: 10 * time.Second, + } + if err := server.ListenAndServe(); err != nil { + klog.Error(err, "failed to start the pprof server") + } +}