diff --git a/.gitignore b/.gitignore index 017b3207..56a4f0a5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .vscode/* .idea/* +.test/* kubernetes-ingress dist/ .code-generator/ diff --git a/deploy/tests/config/3.ingress-controller.yaml b/deploy/tests/config/3.ingress-controller.yaml index b6da51e9..a2e287cb 100644 --- a/deploy/tests/config/3.ingress-controller.yaml +++ b/deploy/tests/config/3.ingress-controller.yaml @@ -32,6 +32,8 @@ spec: - --configmap-patternfiles=$(POD_NAMESPACE)/patternfiles - --ingress.class=haproxy - --sync-period=1s + - --pprof + - --prometheus securityContext: runAsUser: 1000 runAsGroup: 1000 diff --git a/deploy/tests/e2e/admin-port/pprof_test.go b/deploy/tests/e2e/admin-port/pprof_test.go new file mode 100644 index 00000000..d5a61774 --- /dev/null +++ b/deploy/tests/e2e/admin-port/pprof_test.go @@ -0,0 +1,50 @@ +// Copyright 2019 HAProxy Technologies LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build e2e_parallel + +package adminport + +import ( + "net/http" + + "github.com/haproxytech/kubernetes-ingress/deploy/tests/e2e" +) + +func (suite *AdminPortSuite) Test_Pprof() { + suite.Run("OK", func() { + suite.Eventually(func() bool { + suite.client.Path = "/debug/pprof/cmdline" + res, cls, err := suite.client.Do() + if err != nil { + suite.T().Logf("Connection ERROR: %s", err.Error()) + return false + } + defer cls() + return res.StatusCode == http.StatusOK + }, e2e.WaitDuration, e2e.TickDuration) + }) + suite.Run("OK", func() { + suite.Eventually(func() bool { + suite.client.Path = "/debug/pprof/symbol" + res, cls, err := suite.client.Do() + if err != nil { + suite.T().Logf("Connection ERROR: %s", err.Error()) + return false + } + defer cls() + return res.StatusCode == http.StatusOK + }, e2e.WaitDuration, e2e.TickDuration) + }) +} diff --git a/deploy/tests/e2e/admin-port/suite_test.go b/deploy/tests/e2e/admin-port/suite_test.go new file mode 100644 index 00000000..f852a07c --- /dev/null +++ b/deploy/tests/e2e/admin-port/suite_test.go @@ -0,0 +1,56 @@ +// Copyright 2019 HAProxy Technologies LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build e2e_parallel + +package adminport + +import ( + "testing" + + "github.com/stretchr/testify/suite" + + "github.com/haproxytech/kubernetes-ingress/deploy/tests/e2e" +) + +type AdminPortSuite struct { + suite.Suite + test e2e.Test + client *e2e.Client + tmplData tmplData +} + +type tmplData struct { + Host string +} + +func (suite *AdminPortSuite) SetupSuite() { + var err error + suite.test, err = e2e.NewTest() + suite.NoError(err) + suite.tmplData = tmplData{Host: suite.test.GetNS() + ".test"} + suite.client, err = e2e.NewHTTPClient(suite.tmplData.Host) + suite.NoError(err) +} + +func (suite *AdminPortSuite) TearDownSuite() { + err := suite.test.TearDown() + if err != nil { + suite.T().Error(err) + } +} + +func TestAdminPortSuite(t *testing.T) { + suite.Run(t, new(AdminPortSuite)) +} diff --git a/deploy/tests/e2e/map-updates/update_test.go b/deploy/tests/e2e/map-updates/update_test.go index 7fb05058..8d13898b 100644 --- a/deploy/tests/e2e/map-updates/update_test.go +++ b/deploy/tests/e2e/map-updates/update_test.go @@ -36,8 +36,8 @@ func (suite *MapUpdateSuite) Test_Update() { suite.NoError(err) count, err := e2e.GetHAProxyMapCount("path-prefix") suite.NoError(err) - suite.T().Log(count) - return oldInfo.Pid == newInfo.Pid && count == 700 + suite.T().Logf("oldInfo.Pid(%s) == newInfo.Pid(%s) && count(%d) == 701", oldInfo.Pid, newInfo.Pid, count) + return oldInfo.Pid == newInfo.Pid && count == 701 // 700 + pprof }, e2e.WaitDuration, e2e.TickDuration) }) } diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 8141572c..7fc84181 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -49,6 +49,7 @@ type HAProxyController struct { podPrefix string chShutdown chan struct{} updatePublishServiceFunc func(ingresses []*ingress.Ingress, publishServiceAddresses []string) + beforeUpdateHandlers []UpdateHandler } // Wrapping a Native-Client transaction and commit it. @@ -111,6 +112,13 @@ func (c *HAProxyController) updateHAProxy() { } ingresses := []*ingress.Ingress{} + + for _, handler := range c.beforeUpdateHandlers { + reload, err = handler.Update(c.store, c.haproxy, c.annotations) + logger.Error(err) + c.reload = c.reload || reload + } + for _, namespace := range c.store.Namespaces { if !namespace.Relevant { continue diff --git a/pkg/controller/handler.go b/pkg/controller/handler.go index 816d949f..000fab64 100644 --- a/pkg/controller/handler.go +++ b/pkg/controller/handler.go @@ -54,6 +54,12 @@ func (c *HAProxyController) initHandlers() { } c.updateHandlers = append(c.updateHandlers, handler.Refresh{}) + + c.beforeUpdateHandlers = []UpdateHandler{} + // Need to be before Refresh. If after, maps are refreshed without pprof content + if c.osArgs.PprofEnabled { + c.beforeUpdateHandlers = append(c.beforeUpdateHandlers, handler.Pprof{}) + } } func (c *HAProxyController) startupHandlers() error { @@ -70,9 +76,7 @@ func (c *HAProxyController) startupHandlers() error { IPv6Addr: c.osArgs.IPV6BindAddr, }, } - if c.osArgs.PprofEnabled { - handlers = append(handlers, handler.Pprof{}) - } + for _, handler := range handlers { _, err := handler.Update(c.store, c.haproxy, c.annotations) if err != nil { diff --git a/pkg/handler/pprof.go b/pkg/handler/pprof.go index c8d66117..62c251e0 100644 --- a/pkg/handler/pprof.go +++ b/pkg/handler/pprof.go @@ -15,6 +15,8 @@ package handler import ( + "fmt" + "github.com/haproxytech/client-native/v3/models" "github.com/haproxytech/kubernetes-ingress/pkg/annotations" @@ -38,8 +40,8 @@ func (handler Pprof) Update(k store.K8s, h haproxy.HAProxy, a annotations.Annota return } err = h.BackendServerCreate(pprofBackend, models.Server{ - Name: "pprof", - Address: "127.0.0.1:6060", + Name: pprofBackend, + Address: fmt.Sprintf("127.0.0.1:%d", h.Env.ControllerPort), }) if err != nil { return @@ -56,6 +58,6 @@ func (handler Pprof) Update(k store.K8s, h haproxy.HAProxy, a annotations.Annota if err != nil { return } - reload = true + // instance.Reload("pprof backend created") return } diff --git a/pkg/haproxy/env/main.go b/pkg/haproxy/env/main.go index c5a72259..6b9c61b8 100644 --- a/pkg/haproxy/env/main.go +++ b/pkg/haproxy/env/main.go @@ -41,6 +41,7 @@ type Env struct { MapsDir string Certs certs.Env Proxies + ControllerPort int } // Proxies contains names of the main proxies of haproxy config @@ -85,7 +86,7 @@ func (env *Env) Init(osArgs utils.OSArgs) (err error) { env.MapsDir = filepath.Join(env.CfgDir, "maps") env.PatternDir = filepath.Join(env.CfgDir, "patterns") env.ErrFileDir = filepath.Join(env.CfgDir, "errorfiles") - + env.ControllerPort = osArgs.ControllerPort for _, d := range []string{ env.Certs.MainDir, env.Certs.FrontendDir, diff --git a/pkg/utils/flags.go b/pkg/utils/flags.go index 63c9fdf1..961ac6d8 100644 --- a/pkg/utils/flags.go +++ b/pkg/utils/flags.go @@ -102,7 +102,7 @@ type OSArgs struct { //nolint:maligned RuntimeDir string `long:"runtime-dir" description:"path to HAProxy runtime directory. NOTE: works only in External mode"` DisableServiceExternalName bool `long:"disable-service-external-name" description:"disable forwarding to ExternalName Services due to CVE-2021-25740"` UseWiths6Overlay bool `long:"with-s6-overlay" description:"use s6 overlay to start/stpop/reload HAProxy"` - ControllerPort int `long:"controller-port" description:"port to listen on for controller data: prometheus, pprof"` + ControllerPort int `long:"controller-port" description:"port to listen on for controller data: prometheus, pprof" default:"6060"` PprofEnabled bool `long:"pprof" short:"p" description:"enable pprof"` PrometheusEnabled bool `long:"prometheus" description:"enable prometheus of IC data"` ChannelSize int64 `long:"channel-size" description:"sets the size of controller buffers used to receive and send k8s events.NOTE: increase the value to accommodate large number of resources "`