From cdbed04a6c06402ab3ea118c5e9e5c933a4e6a3f Mon Sep 17 00:00:00 2001 From: Paulo Sousa Date: Tue, 8 Aug 2023 17:04:34 -0300 Subject: [PATCH] (feat) add rpaas debug (#136) * api/web/client: add initial instance debug support * api: proper attach and wait for status for debug * api: add tests for debug // return debug container logs when unable to attach to it * integration: add integration tests for debug plugin cmd * integration: bump rpaas-api chart * k8s: minor refactor on debug and add tests * k8s: fix debugPodWithContainerStatus signature and add debug test with no pod defined * Update internal/pkg/rpaas/k8s.go Co-authored-by: Claudio Netto * k8s: minor refactor on patchEphemeralContainers Co-authored-by: Claudio Netto * api: add missing license header to transport * plugin: fix typo for debug/exec cmds --------- Co-authored-by: Claudio Netto --- Makefile | 2 +- cmd/plugin/rpaasv2/cmd/app.go | 1 + cmd/plugin/rpaasv2/cmd/debug.go | 138 +++++++++++++++ cmd/plugin/rpaasv2/cmd/debug_test.go | 91 ++++++++++ cmd/plugin/rpaasv2/cmd/exec.go | 2 +- go.mod | 23 ++- go.sum | 54 +++--- internal/config/config.go | 2 + internal/pkg/rpaas/fake/manager.go | 8 + internal/pkg/rpaas/k8s.go | 219 +++++++++++++++++++++--- internal/pkg/rpaas/k8s_test.go | 241 +++++++++++++++++++++++++++ internal/pkg/rpaas/manager.go | 29 +++- pkg/rpaas/client/client.go | 22 +++ pkg/rpaas/client/debug.go | 65 ++++++++ pkg/rpaas/client/debug_test.go | 80 +++++++++ pkg/rpaas/client/exec.go | 8 - pkg/rpaas/client/fake/client.go | 9 + pkg/web/api.go | 1 + pkg/web/debug.go | 61 +++++++ pkg/web/debug_test.go | 223 +++++++++++++++++++++++++ pkg/web/exec.go | 182 +++----------------- pkg/web/exec_test.go | 6 +- pkg/web/transport.go | 169 +++++++++++++++++++ scripts/localkube-integration.sh | 2 +- test/integration_test.go | 45 +++++ 25 files changed, 1468 insertions(+), 215 deletions(-) create mode 100644 cmd/plugin/rpaasv2/cmd/debug.go create mode 100644 cmd/plugin/rpaasv2/cmd/debug_test.go create mode 100644 pkg/rpaas/client/debug.go create mode 100644 pkg/rpaas/client/debug_test.go create mode 100644 pkg/web/debug.go create mode 100644 pkg/web/debug_test.go create mode 100644 pkg/web/transport.go diff --git a/Makefile b/Makefile index bbf373def..f517c4be7 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ build: build/api build/manager build/plugin/rpaasv2 build/purger .PHONY: build/api build/api: build-dirs - go build -o $(GO_BUILD_DIR)/ ./cmd/api + CGO_ENABLED=0 go build -o $(GO_BUILD_DIR)/ ./cmd/api .PHONY: build/manager build/manager: manager diff --git a/cmd/plugin/rpaasv2/cmd/app.go b/cmd/plugin/rpaasv2/cmd/app.go index 363aaff06..676a856a6 100644 --- a/cmd/plugin/rpaasv2/cmd/app.go +++ b/cmd/plugin/rpaasv2/cmd/app.go @@ -40,6 +40,7 @@ func NewApp(o, e io.Writer, client rpaasclient.Client) (app *cli.App) { NewCmdRoutes(), NewCmdInfo(), NewCmdAutoscale(), + NewCmdDebug(), NewCmdExec(), NewCmdShell(), NewCmdLogs(), diff --git a/cmd/plugin/rpaasv2/cmd/debug.go b/cmd/plugin/rpaasv2/cmd/debug.go new file mode 100644 index 000000000..d3637064b --- /dev/null +++ b/cmd/plugin/rpaasv2/cmd/debug.go @@ -0,0 +1,138 @@ +// Copyright 2023 tsuru authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmd + +import ( + "fmt" + "os" + + "github.com/gorilla/websocket" + "github.com/urfave/cli/v2" + "k8s.io/kubectl/pkg/util/term" + + rpaasclient "github.com/tsuru/rpaas-operator/pkg/rpaas/client" +) + +func NewCmdDebug() *cli.Command { + return &cli.Command{ + Name: "debug", + Usage: "Run debug in an pod from instance", + ArgsUsage: "[-p POD] [-c CONTAINER] [--debug-image image] [--] COMMAND [args...]", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "service", + Aliases: []string{"tsuru-service", "s"}, + Usage: "the Tsuru service name", + }, + &cli.StringFlag{ + Name: "instance", + Aliases: []string{"tsuru-service-instance", "i"}, + Usage: "the reverse proxy instance name", + Required: true, + }, + &cli.StringFlag{ + Name: "pod", + Aliases: []string{"p"}, + Usage: "pod name - if omitted, the first pod will be chosen", + }, + &cli.StringFlag{ + Name: "container", + Aliases: []string{"c"}, + Usage: "container name - if omitted, the \"nginx\" container will be chosen", + }, + &cli.StringFlag{ + Name: "debug-image", + Aliases: []string{"d"}, + Usage: "debug image name - if omitted, service default defined debug image will be chosen", + }, + &cli.BoolFlag{ + Name: "interactive", + Aliases: []string{"I", "stdin"}, + Usage: "pass STDIN to container", + }, + &cli.BoolFlag{ + Name: "tty", + Aliases: []string{"t"}, + Usage: "allocate a pseudo-TTY", + }, + }, + Before: setupClient, + Action: runDebug, + } +} + +func runDebug(c *cli.Context) error { + client, err := getClient(c) + if err != nil { + return err + } + + var width, height uint16 + if ts := term.GetSize(os.Stdin.Fd()); ts != nil { + width, height = ts.Width, ts.Height + } + + args := rpaasclient.DebugArgs{ + Command: c.Args().Slice(), + Instance: c.String("instance"), + Pod: c.String("pod"), + Container: c.String("container"), + Interactive: c.Bool("interactive"), + Image: c.String("debug-image"), + TTY: c.Bool("tty"), + TerminalWidth: width, + TerminalHeight: height, + } + + if args.Interactive { + args.In = os.Stdin + } + + tty := &term.TTY{ + In: args.In, + Out: c.App.Writer, + Raw: args.TTY, + } + return tty.Safe(func() error { + conn, err := client.Debug(c.Context, args) + if err != nil { + return err + } + defer conn.Close() + + done := make(chan error, 1) + go func() { + defer close(done) + for { + mtype, message, nerr := conn.ReadMessage() + if nerr != nil { + closeErr, ok := nerr.(*websocket.CloseError) + if !ok { + done <- fmt.Errorf("ERROR: received an unexpected error while reading messages: %w", err) + return + } + + switch closeErr.Code { + case websocket.CloseNormalClosure: + case websocket.CloseInternalServerErr: + done <- fmt.Errorf("ERROR: the command may not be executed as expected - reason: %s", closeErr.Text) + default: + done <- fmt.Errorf("ERROR: unexpected close error: %s", closeErr.Error()) + } + + return + } + + switch mtype { + case websocket.TextMessage, websocket.BinaryMessage: + c.App.Writer.Write(message) + } + } + }() + err = <-done + conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")) + return err + }) +} diff --git a/cmd/plugin/rpaasv2/cmd/debug_test.go b/cmd/plugin/rpaasv2/cmd/debug_test.go new file mode 100644 index 000000000..d32e8a9c8 --- /dev/null +++ b/cmd/plugin/rpaasv2/cmd/debug_test.go @@ -0,0 +1,91 @@ +// Copyright 2023 tsuru authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmd + +import ( + "bytes" + "context" + "fmt" + "os" + "testing" + + "github.com/gorilla/websocket" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/tsuru/rpaas-operator/pkg/rpaas/client" + "github.com/tsuru/rpaas-operator/pkg/rpaas/client/fake" +) + +func TestDebug(t *testing.T) { + var called bool + + tests := []struct { + name string + args []string + expected string + expectedError string + expectedCalled bool + client client.Client + }{ + { + name: "with command and arguments", + args: []string{"rpaasv2", "debug", "-s", "rpaasv2", "-i", "my-instance", "--", "my-command", "-arg1", "--arg2"}, + client: &fake.FakeClient{ + FakeDebug: func(ctx context.Context, args client.DebugArgs) (*websocket.Conn, error) { + called = true + expected := client.DebugArgs{ + Command: []string{"my-command", "-arg1", "--arg2"}, + Instance: "my-instance", + } + assert.Equal(t, expected, args) + return nil, fmt.Errorf("some error") + }, + }, + expectedCalled: true, + expectedError: "some error", + }, + { + name: "with all options activated", + args: []string{"rpaasv2", "debug", "-s", "rpaasv2", "-i", "my-instance", "--tty", "--interactive", "-p", "pod-1", "-c", "container-1", "--", "my-shell"}, + client: &fake.FakeClient{ + FakeDebug: func(ctx context.Context, args client.DebugArgs) (*websocket.Conn, error) { + called = true + expected := client.DebugArgs{ + In: os.Stdin, + Command: []string{"my-shell"}, + Instance: "my-instance", + Pod: "pod-1", + Container: "container-1", + TTY: true, + Interactive: true, + } + assert.Equal(t, expected, args) + return nil, fmt.Errorf("another error") + }, + }, + expectedCalled: true, + expectedError: "another error", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + called = false + stdout := &bytes.Buffer{} + stderr := &bytes.Buffer{} + app := NewApp(stdout, stderr, tt.client) + err := app.Run(tt.args) + if tt.expectedError != "" { + assert.EqualError(t, err, tt.expectedError) + return + } + require.NoError(t, err) + assert.Equal(t, tt.expectedCalled, called) + assert.Equal(t, tt.expected, stdout.String()) + assert.Empty(t, stderr.String()) + }) + } +} diff --git a/cmd/plugin/rpaasv2/cmd/exec.go b/cmd/plugin/rpaasv2/cmd/exec.go index 80fae9826..6de4fcbed 100644 --- a/cmd/plugin/rpaasv2/cmd/exec.go +++ b/cmd/plugin/rpaasv2/cmd/exec.go @@ -104,7 +104,7 @@ func runExec(c *cli.Context) error { if nerr != nil { closeErr, ok := nerr.(*websocket.CloseError) if !ok { - done <- fmt.Errorf("ERROR: receveid an unexpected error while reading messages: %w", err) + done <- fmt.Errorf("ERROR: received an unexpected error while reading messages: %w", err) return } diff --git a/go.mod b/go.mod index 7c8d30f4f..179d14cfe 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( k8s.io/api v0.26.2 k8s.io/apimachinery v0.26.2 k8s.io/client-go v0.26.2 - k8s.io/kubectl v0.24.2 + k8s.io/kubectl v0.26.2 k8s.io/metrics v0.26.2 sigs.k8s.io/controller-runtime v0.14.5 sigs.k8s.io/go-open-service-broker-client/v2 v2.0.0-20200925085050-ae25e62aaf10 @@ -49,18 +49,24 @@ require ( github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c // indirect github.com/HdrHistogram/hdrhistogram-go v1.0.0 // indirect + github.com/MakeNowJust/heredoc v1.0.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver/v3 v3.1.1 // indirect github.com/antihax/optional v1.0.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect + github.com/chai2010/gettext-go v1.0.2 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/docker/docker v20.10.20+incompatible // indirect github.com/docker/go-units v0.5.0 // indirect github.com/emicklei/go-restful/v3 v3.10.1 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect + github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect + github.com/fatih/camelcase v1.0.0 // indirect + github.com/fvbommel/sortorder v1.0.1 // indirect github.com/go-asn1-ber/asn1-ber v1.5.1 // indirect + github.com/go-errors/errors v1.0.1 // indirect github.com/go-ldap/ldap/v3 v3.4.2 // indirect github.com/go-logr/zapr v1.2.3 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect @@ -70,18 +76,23 @@ require ( github.com/golang-jwt/jwt v3.2.2+incompatible // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect + github.com/google/btree v1.0.1 // indirect github.com/google/gnostic v0.6.9 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/uuid v1.3.0 // indirect + github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-version v0.0.0-20180716215031-270f2f71b1ee // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/howeyc/fsnotify v0.9.0 // indirect github.com/huandu/xstrings v1.3.2 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/labstack/gommon v0.4.0 // indirect + github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect github.com/magiconair/properties v1.8.5 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect @@ -95,9 +106,11 @@ require ( github.com/moby/term v0.0.0-20221105221325-4eb28fa6025c // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pelletier/go-toml v1.9.4 // indirect + github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect @@ -109,6 +122,7 @@ require ( github.com/shopspring/decimal v1.2.0 // indirect github.com/spf13/afero v1.6.0 // indirect github.com/spf13/cast v1.4.1 // indirect + github.com/spf13/cobra v1.6.1 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/subosito/gotenv v1.2.0 // indirect github.com/tsuru/config v0.0.0-20201023175036-375aaee8b560 // indirect @@ -118,6 +132,8 @@ require ( github.com/uber/jaeger-lib v2.4.0+incompatible // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.2 // indirect + github.com/xlab/treeprint v1.1.0 // indirect + go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.9.0 // indirect go.uber.org/zap v1.24.0 // indirect @@ -134,6 +150,7 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.26.2 // indirect + k8s.io/cli-runtime v0.26.2 // indirect k8s.io/component-base v0.26.2 // indirect k8s.io/klog/v2 v2.90.1 // indirect k8s.io/kube-aggregator v0.24.2 // indirect @@ -142,6 +159,8 @@ require ( knative.dev/pkg v0.0.0-20230306194819-b77a78c6c0ad // indirect sigs.k8s.io/gateway-api v0.4.3 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/kustomize/api v0.12.1 // indirect + sigs.k8s.io/kustomize/kyaml v0.13.9 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/go.sum b/go.sum index a40e86a74..05cc2a666 100644 --- a/go.sum +++ b/go.sum @@ -57,7 +57,8 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/Djarvur/go-err113 v0.0.0-20200511133814-5174e21577d5/go.mod h1:4UJr5HIiMZrwgkSPdsjy2uOQExX/WEILpIrO9UPGuXs= github.com/HdrHistogram/hdrhistogram-go v1.0.0 h1:jivTvI9tBw5B8wW9Qd0uoQ2qaajb29y4TPhYTgh8Lb0= github.com/HdrHistogram/hdrhistogram-go v1.0.0/go.mod h1:YzE1EgsuAz8q9lfGdlxBZo2Ma655+PfKp2mlzcAqIFw= -github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E= +github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= +github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= @@ -116,7 +117,8 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw= +github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk= +github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -142,8 +144,9 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= @@ -152,11 +155,9 @@ github.com/daixiang0/gci v0.2.4/go.mod h1:+AV8KmHTGxxwp/pY84TLQfFKp2vuKXXJVzF3kD github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE= github.com/denis-tingajkin/go-header v0.3.1/go.mod h1:sq/2IxMhaZX+RRcgHfCRx/m0M5na0fBt4/CRe7Lrji0= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v20.10.20+incompatible h1:kH9tx6XO+359d+iAkumyKDc5Q1kOwPuAUaeri48nD6E= github.com/docker/docker v20.10.20+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -186,7 +187,9 @@ github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCv github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= +github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= +github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8= github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= @@ -201,6 +204,7 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/fvbommel/sortorder v1.0.1 h1:dSnXLt4mJYH25uDDGa3biZNQsozaUWDSWeKJ0qqFfzE= github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0= github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= @@ -210,6 +214,7 @@ github.com/globocom/echo-prometheus v0.1.2/go.mod h1:3oQLuoG5ZI5nufWK0ILpMl4vmw1 github.com/go-asn1-ber/asn1-ber v1.5.1 h1:pDbRAunXzIUXfx4CB2QJFv5IuPiuoW+sWvr/Us009o8= github.com/go-asn1-ber/asn1-ber v1.5.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= github.com/go-critic/go-critic v0.5.2/go.mod h1:cc0+HvdE3lFpqLecgqMaJcvWWH77sLdBp+wLGPM1Yyo= +github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -328,9 +333,9 @@ github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlS github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI= github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4= github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ= -github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/gnostic v0.6.9 h1:ZK/5VhkoX835RikCHpSUJV9a+S3e1zLh59YnyWeBW+0= @@ -371,6 +376,7 @@ github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20230228050547-1710fef4ab10 h1:CqYfpuYIjnlNxM3msdyPRKabhXZWbKjf3Q8BWROFBso= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -391,6 +397,7 @@ github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0U github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= github.com/gostaticanalysis/analysisutil v0.0.3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -441,6 +448,9 @@ github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jingyugao/rowserrcheck v0.0.0-20191204022205-72ab7603b68a/go.mod h1:xRskid8CManxVta/ALEhJha/pweKBaVG6fWgc0yH25s= github.com/jirfag/go-printf-func-name v0.0.0-20191110105641-45db9963cdd3/go.mod h1:HEWGJkRDzjJY2sqdDwxccsGicWEf9BQOZsq2tV+xzM0= @@ -492,6 +502,7 @@ github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL github.com/labstack/gommon v0.4.0 h1:y7cvthEAEbU0yHOf4axH8ZG2NH8knB9iNSoTO8dyIk8= github.com/labstack/gommon v0.4.0/go.mod h1:uW6kP17uPlLJsD3ijUYn3/M5bAxtlZhMI6m3MFxTMTM= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= @@ -527,7 +538,6 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= @@ -571,6 +581,7 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= @@ -589,7 +600,6 @@ github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -607,7 +617,6 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.14.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= github.com/onsi/gomega v1.27.2 h1:SKU0CXeKE/WVgIV1T61kSa3+IRE8Ekrv9rdXDwwTqnY= -github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opentracing-contrib/go-stdlib v1.0.1-0.20201028152118-adbfc141dfc2 h1:Fy4bjXz8oBZwuPQ0NgTjOG068OSuBlv2BPWtH6+Yuv8= github.com/opentracing-contrib/go-stdlib v1.0.1-0.20201028152118-adbfc141dfc2/go.mod h1:qtI1ogk+2JhVPIXVc6q+NHziSmy2W5GbdQZFUHADCBU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= @@ -619,6 +628,7 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9 github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d/go.mod h1:3OzsM7FXDQlpCiw2j81fOmAwQLnZnLGXVKUzeKQXIAw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -678,7 +688,6 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.0/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -690,7 +699,7 @@ github.com/sajari/fuzzy v1.0.0/go.mod h1:OjYR6KxoWOe9+dOlXeiCJd4dIbED4Oo8wpS89o0 github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/securego/gosec/v2 v2.4.0/go.mod h1:0/Q4cjmlFDfDUj1+Fib61sc+U5IQb2w+Iv9/C3wPVko= -github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c/go.mod h1:/PevMnwAxekIXwN8qQyfc5gl2NlkB3CQlkizAbOkeBs= github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc= github.com/shirou/gopsutil v2.20.4+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= @@ -728,6 +737,8 @@ github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJ github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= +github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= +github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= @@ -809,8 +820,9 @@ github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xlab/treeprint v1.0.0/go.mod h1:IoImgRak9i3zJyuxOKUP1v4UZd1tMoKkq/Cimt1uhCg= +github.com/xlab/treeprint v1.1.0 h1:G/1DjNkPpfZCFt9CSh6b5/nY4VimlbHF3Rh4obvtzDk= +github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -852,6 +864,7 @@ go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 h1:+FNtrFTmVw0YZGpBGX56XDee331t6JAXeK2bcyhLOOc= go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -1380,7 +1393,8 @@ k8s.io/apimachinery v0.26.2 h1:da1u3D5wfR5u2RpLhE/ZtZS2P7QvDgLZTi9wrNZl/tQ= k8s.io/apimachinery v0.26.2/go.mod h1:ats7nN1LExKHvJ9TmwootT00Yz05MuYqPXEXaVeOy5I= k8s.io/apiserver v0.21.3/go.mod h1:eDPWlZG6/cCCMj/JBcEpDoK+I+6i3r9GsChYBHSbAzU= k8s.io/apiserver v0.24.2/go.mod h1:pSuKzr3zV+L+MWqsEo0kHHYwCo77AT5qXbFXP2jbvFI= -k8s.io/cli-runtime v0.24.2/go.mod h1:1LIhKL2RblkhfG4v5lZEt7FtgFG5mVb8wqv5lE9m5qY= +k8s.io/cli-runtime v0.26.2 h1:6XcIQOYW1RGNwFgRwejvyUyAojhToPmJLGr0JBMC5jw= +k8s.io/cli-runtime v0.26.2/go.mod h1:U7sIXX7n6ZB+MmYQsyJratzPeJwgITqrSlpr1a5wM5I= k8s.io/client-go v0.21.3/go.mod h1:+VPhCgTsaFmGILxR/7E1N0S+ryO010QBeNCv5JwRGYU= k8s.io/client-go v0.22.1/go.mod h1:BquC5A4UOo4qVDUtoc04/+Nxp1MeHcVc1HJm1KmG8kk= k8s.io/client-go v0.24.2/go.mod h1:zg4Xaoo+umDsfCWr4fCnmLEtQXyCNXCvJuSsglNcV30= @@ -1393,7 +1407,6 @@ k8s.io/component-base v0.21.3/go.mod h1:kkuhtfEHeZM6LkX0saqSK8PbdO7A0HigUngmhhrw k8s.io/component-base v0.24.2/go.mod h1:ucHwW76dajvQ9B7+zecZAP3BVqvrHoOxm8olHEg0nmM= k8s.io/component-base v0.26.2 h1:IfWgCGUDzrD6wLLgXEstJKYZKAFS2kO+rBRi0p3LqcI= k8s.io/component-base v0.26.2/go.mod h1:DxbuIe9M3IZPRxPIzhch2m1eT7uFrSBJUBuVCQEBivs= -k8s.io/component-helpers v0.24.2/go.mod h1:TRQPBQKfmqkmV6c0HAmUs8cXVNYYYLsXy4zu8eODi9g= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20201203183100-97869a43a9d9/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= @@ -1415,9 +1428,8 @@ k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2R k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdiUkI+v/ImEGAvu3WatcZl3lPMR4Rk= k8s.io/kube-openapi v0.0.0-20230303024457-afdc3dddf62d h1:VcFq5n7wCJB2FQMCIHfC+f+jNcGgNMar1uKd6rVlifU= k8s.io/kube-openapi v0.0.0-20230303024457-afdc3dddf62d/go.mod h1:y5VtZWM9sHHc2ZodIH/6SHzXj+TPU5USoA8lcIeKEKY= -k8s.io/kubectl v0.24.2 h1:+RfQVhth8akUmIc2Ge8krMl/pt66V7210ka3RE/p0J4= -k8s.io/kubectl v0.24.2/go.mod h1:+HIFJc0bA6Tzu5O/YcuUt45APAxnNL8LeMuXwoiGsPg= -k8s.io/metrics v0.24.2/go.mod h1:5NWURxZ6Lz5gj8TFU83+vdWIVASx7W8lwPpHYCqopMo= +k8s.io/kubectl v0.26.2 h1:SMPB4j48eVFxsYluBq3VLyqXtE6b72YnszkbTAtFye4= +k8s.io/kubectl v0.26.2/go.mod h1:KYWOXSwp2BrDn3kPeoU/uKzKtdqvhK1dgZGd0+no4cM= k8s.io/metrics v0.26.2 h1:2gUvUWWnHPdE2tyA5DvyHC8HGryr+izhY9i5dzLP06s= k8s.io/metrics v0.26.2/go.mod h1:PX1wm9REV9hSGuw9GcXTFNDgab1KRXck3mNeiLYbRho= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= @@ -1451,10 +1463,10 @@ sigs.k8s.io/go-open-service-broker-client/v2 v2.0.0-20200925085050-ae25e62aaf10/ sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/kustomize/api v0.11.4/go.mod h1:k+8RsqYbgpkIrJ4p9jcdPqe8DprLxFUUO0yNOq8C+xI= -sigs.k8s.io/kustomize/cmd/config v0.10.6/go.mod h1:/S4A4nUANUa4bZJ/Edt7ZQTyKOY9WCER0uBS1SW2Rco= -sigs.k8s.io/kustomize/kustomize/v4 v4.5.4/go.mod h1:Zo/Xc5FKD6sHl0lilbrieeGeZHVYCA4BzxeAaLI05Bg= -sigs.k8s.io/kustomize/kyaml v0.13.6/go.mod h1:yHP031rn1QX1lr/Xd934Ri/xdVNG8BE2ECa78Ht/kEg= +sigs.k8s.io/kustomize/api v0.12.1 h1:7YM7gW3kYBwtKvoY216ZzY+8hM+lV53LUayghNRJ0vM= +sigs.k8s.io/kustomize/api v0.12.1/go.mod h1:y3JUhimkZkR6sbLNwfJHxvo1TCLwuwm14sCYnkH6S1s= +sigs.k8s.io/kustomize/kyaml v0.13.9 h1:Qz53EAaFFANyNgyOEJbT/yoIHygK40/ZcvU3rgry2Tk= +sigs.k8s.io/kustomize/kyaml v0.13.9/go.mod h1:QsRbD0/KcU+wdk0/L0fIp2KLnohkVzs6fQ85/nOXac4= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= diff --git a/internal/config/config.go b/internal/config/config.go index 90c36bfb4..1148f9a1b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -54,6 +54,7 @@ type RpaasConfig struct { EnableCertManager bool `json:"enable-cert-manager"` NewInstanceReplicas int `json:"new-instance-replicas"` ForbiddenAnnotationsPrefixes []string `json:"forbidden-annotations-prefixes"` + DebugImage string `json:"debug-image"` } type ClusterConfig struct { @@ -112,6 +113,7 @@ func Init() error { viper.SetDefault("enable-cert-manager", false) viper.SetDefault("new-instance-replicas", 1) viper.SetDefault("forbidden-annotations-prefixes", []string{"rpaas.extensions.tsuru.io", "afh.tsuru.io"}) + viper.SetDefault("debug-image", "") viper.AutomaticEnv() err := readConfig() if err != nil { diff --git a/internal/pkg/rpaas/fake/manager.go b/internal/pkg/rpaas/fake/manager.go index 9268c0a67..73d1b5c7d 100644 --- a/internal/pkg/rpaas/fake/manager.go +++ b/internal/pkg/rpaas/fake/manager.go @@ -50,6 +50,7 @@ type RpaasManager struct { FakeDeleteAutoscale func(name string) error FakeGetInstanceInfo func(instanceName string) (*clientTypes.InstanceInfo, error) FakeExec func(instanceName string, args rpaas.ExecArgs) error + FakeDebug func(instanceName string, args rpaas.DebugArgs) error FakeLog func(instanceName string, args rpaas.LogArgs) error FakeAddUpstream func(instanceName string, upstream v1alpha1.AllowedUpstream) error FakeGetUpstreams func(instanceName string) ([]v1alpha1.AllowedUpstream, error) @@ -284,6 +285,13 @@ func (m *RpaasManager) Exec(ctx context.Context, instanceName string, args rpaas return nil } +func (m *RpaasManager) Debug(ctx context.Context, instanceName string, args rpaas.DebugArgs) error { + if m.FakeDebug != nil { + return m.FakeDebug(instanceName, args) + } + return nil +} + func (m *RpaasManager) AddUpstream(ctx context.Context, instanceName string, upstream v1alpha1.AllowedUpstream) error { if m.FakeAddUpstream != nil { return m.FakeAddUpstream(instanceName, upstream) diff --git a/internal/pkg/rpaas/k8s.go b/internal/pkg/rpaas/k8s.go index f4ef197fc..99db0d906 100644 --- a/internal/pkg/rpaas/k8s.go +++ b/internal/pkg/rpaas/k8s.go @@ -38,13 +38,21 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/httpstream/spdy" + utilrand "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/apimachinery/pkg/util/strategicpatch" "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/remotecommand" + watchtools "k8s.io/client-go/tools/watch" + "k8s.io/kubectl/pkg/cmd/logs" "k8s.io/kubectl/pkg/scheme" + "k8s.io/kubectl/pkg/util/interrupt" metricsv1beta1 "k8s.io/metrics/pkg/apis/metrics/v1beta1" "sigs.k8s.io/controller-runtime/pkg/client" osb "sigs.k8s.io/go-open-service-broker-client/v2" @@ -68,6 +76,7 @@ const ( ) var _ RpaasManager = &k8sRpaasManager{} +var nameSuffixFunc = utilrand.String var podAllowedReasonsToFail = map[string]bool{ "shutdown": true, @@ -133,44 +142,166 @@ func (q *fixedSizeQueue) Next() *remotecommand.TerminalSize { return q.sz } -func (m *k8sRpaasManager) Exec(ctx context.Context, instanceName string, args ExecArgs) error { - instance, err := m.GetInstance(ctx, instanceName) +func (m *k8sRpaasManager) Debug(ctx context.Context, instanceName string, args DebugArgs) error { + instance, debugContainerName, status, err := m.debugPodWithContainerStatus(ctx, &args, instanceName) if err != nil { return err } + if status.State.Terminated != nil { + req := m.kcs.CoreV1().Pods(instance.Namespace).GetLogs(args.Pod, &corev1.PodLogOptions{Container: debugContainerName}) + return logs.DefaultConsumeRequest(req, args.Stdout) - nginx, err := m.getNginx(ctx, instance) + } + req := m.kcs. + CoreV1(). + RESTClient(). + Post(). + Resource("pods"). + Name(args.Pod). + Namespace(instance.Namespace). + SubResource("attach"). + VersionedParams(&corev1.PodAttachOptions{ + Container: debugContainerName, + Stdin: args.Stdin != nil, + Stdout: true, + Stderr: true, + TTY: args.TTY, + }, scheme.ParameterCodec) + + executor, err := keepAliveSpdyExecutor(m.restConfig, "POST", req.URL()) if err != nil { return err } + return executorStream(args.CommonTerminalArgs, executor, ctx) +} - podsInfo, err := m.getPodStatuses(ctx, nginx) +func (m *k8sRpaasManager) debugPodWithContainerStatus(ctx context.Context, args *DebugArgs, instanceName string) (*v1alpha1.RpaasInstance, string, *v1.ContainerStatus, error) { + if args.Image == "" && config.Get().DebugImage == "" { + return nil, "", nil, ValidationError{Msg: "Debug image not set and no default image configured"} + } + if args.Image == "" { + args.Image = config.Get().DebugImage + } + instance, err := m.checkPodOnInstance(ctx, instanceName, &args.CommonTerminalArgs) + if err != nil { + return nil, "", nil, err + } + debugContainerName, err := m.generateDebugContainer(ctx, args, instance) + if err != nil { + return nil, "", nil, err + } + pod, err := m.waitForContainer(ctx, instance.Namespace, args.Pod, debugContainerName) + if err != nil { + return nil, "", nil, err + } + status := getContainerStatusByName(pod, debugContainerName) + if status == nil { + return nil, "", nil, fmt.Errorf("error getting container status of container name %q: %+v", debugContainerName, err) + } + return instance, debugContainerName, status, nil +} + +func (m *k8sRpaasManager) generateDebugContainer(ctx context.Context, args *DebugArgs, instance *v1alpha1.RpaasInstance) (string, error) { + instancePod := corev1.Pod{} + err := m.cli.Get(ctx, types.NamespacedName{Name: args.Pod, Namespace: instance.Namespace}, &instancePod) + if err != nil { + return "", err + } + debugContainerName := fmt.Sprintf("debugger-%s", nameSuffixFunc(5)) + debugContainer := &corev1.EphemeralContainer{ + EphemeralContainerCommon: corev1.EphemeralContainerCommon{ + Name: debugContainerName, + Command: args.Command, + Image: args.Image, + ImagePullPolicy: corev1.PullIfNotPresent, + Stdin: args.Stdin != nil, + TTY: args.TTY, + }, TargetContainerName: args.Container, + } + instancePodWithDebug := instancePod.DeepCopy() + instancePodWithDebug.Spec.EphemeralContainers = append(instancePod.Spec.EphemeralContainers, *debugContainer) + err = m.patchEphemeralContainers(ctx, instancePodWithDebug, instancePod, debugContainer) + return debugContainerName, err +} + +func (m *k8sRpaasManager) patchEphemeralContainers(ctx context.Context, instancePodWithDebug *v1.Pod, instancePod v1.Pod, debugContainer *v1.EphemeralContainer) error { + podJS, err := json.Marshal(instancePod) if err != nil { return err } + podWithDebugJS, err := json.Marshal(instancePodWithDebug) + if err != nil { + return err + } + debugPatch, err := strategicpatch.CreateTwoWayMergePatch(podJS, podWithDebugJS, instancePod) + if err != nil { + return err + } + return m.cli.SubResource("ephemeralcontainers").Patch(ctx, &instancePod, client.RawPatch(types.StrategicMergePatchType, debugPatch)) +} - if args.Pod == "" { - for _, ps := range podsInfo { - if strings.EqualFold(ps.Status, "Running") { - args.Pod = ps.Name +func (m *k8sRpaasManager) waitForContainer(ctx context.Context, ns, podName, containerName string) (*corev1.Pod, error) { + ctx, cancel := watchtools.ContextWithOptionalTimeout(ctx, 0*time.Second) + defer cancel() + + fieldSelector := fields.OneTermEqualSelector("metadata.name", podName).String() + lw := &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + options.FieldSelector = fieldSelector + return m.kcs.CoreV1().Pods(ns).List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + options.FieldSelector = fieldSelector + return m.kcs.CoreV1().Pods(ns).Watch(ctx, options) + }, + } + intr := interrupt.New(nil, cancel) + var result *corev1.Pod + err := intr.Run(func() error { + ev, err := watchtools.UntilWithSync(ctx, lw, &corev1.Pod{}, nil, func(ev watch.Event) (bool, error) { + switch ev.Type { + case watch.Deleted: + return false, fmt.Errorf("pod %q was deleted", podName) } - } - } else { - var podFound bool - for _, ps := range podsInfo { - if ps.Name == args.Pod { - podFound = true - break + + p, ok := ev.Object.(*corev1.Pod) + if !ok { + return false, fmt.Errorf("watch did not return a pod: %v", ev.Object) } + + s := getContainerStatusByName(p, containerName) + if s == nil { + return false, nil + } + if s.State.Running != nil || s.State.Terminated != nil { + return true, nil + } + return false, nil + }) + if ev != nil { + result = ev.Object.(*corev1.Pod) } + return err + }) + return result, err +} - if !podFound { - return fmt.Errorf("no such pod %s in instance %s", args.Pod, instanceName) +func getContainerStatusByName(pod *corev1.Pod, containerName string) *corev1.ContainerStatus { + allContainerStatus := [][]corev1.ContainerStatus{pod.Status.InitContainerStatuses, pod.Status.ContainerStatuses, pod.Status.EphemeralContainerStatuses} + for _, statusSlice := range allContainerStatus { + for i := range statusSlice { + if statusSlice[i].Name == containerName { + return &statusSlice[i] + } } } + return nil +} - if args.Container == "" { - args.Container = "nginx" +func (m *k8sRpaasManager) Exec(ctx context.Context, instanceName string, args ExecArgs) error { + instance, err := m.checkPodOnInstance(ctx, instanceName, &args.CommonTerminalArgs) + if err != nil { + return err } req := m.kcs. @@ -195,6 +326,10 @@ func (m *k8sRpaasManager) Exec(ctx context.Context, instanceName string, args Ex return err } + return executorStream(args.CommonTerminalArgs, executor, ctx) +} + +func executorStream(args CommonTerminalArgs, executor remotecommand.Executor, ctx context.Context) error { var tsq remotecommand.TerminalSizeQueue if args.TerminalWidth != uint16(0) && args.TerminalHeight != uint16(0) { tsq = &fixedSizeQueue{ @@ -214,6 +349,52 @@ func (m *k8sRpaasManager) Exec(ctx context.Context, instanceName string, args Ex }) } +func (m *k8sRpaasManager) checkPodOnInstance(ctx context.Context, instanceName string, args *CommonTerminalArgs) (*v1alpha1.RpaasInstance, error) { + instance, err := m.GetInstance(ctx, instanceName) + if err != nil { + return nil, err + } + + nginx, err := m.getNginx(ctx, instance) + if err != nil { + return nil, err + } + + podsInfo, err := m.getPodStatuses(ctx, nginx) + if err != nil { + return nil, err + } + + if args.Pod == "" { + for _, ps := range podsInfo { + if strings.EqualFold(ps.Status, "Running") { + args.Pod = ps.Name + } + } + } else { + var podFound bool + for _, ps := range podsInfo { + if ps.Name == args.Pod { + podFound = true + break + } + } + + if !podFound { + return nil, fmt.Errorf("no such pod %s in instance %s", args.Pod, instanceName) + } + } + + if args.Pod == "" { + return nil, fmt.Errorf("no pod running found in instance %s", instanceName) + } + + if args.Container == "" { + args.Container = "nginx" + } + return instance, nil +} + func (m *k8sRpaasManager) DeleteInstance(ctx context.Context, name string) error { instance, err := m.GetInstance(ctx, name) if err != nil { diff --git a/internal/pkg/rpaas/k8s_test.go b/internal/pkg/rpaas/k8s_test.go index cd1268f61..078ee519c 100644 --- a/internal/pkg/rpaas/k8s_test.go +++ b/internal/pkg/rpaas/k8s_test.go @@ -5,11 +5,14 @@ package rpaas import ( + "bytes" "context" "crypto/tls" "fmt" + "io" "net/http" "regexp" + "sync" "testing" "time" @@ -25,6 +28,9 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/apimachinery/pkg/watch" + k8sclient "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" metricsv1beta1 "k8s.io/metrics/pkg/apis/metrics/v1beta1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -4728,3 +4734,238 @@ func Test_k8sRpaasManager_GetAccessControlList(t *testing.T) { }) } } + +func Test_k8sRpaasManager_Debug(t *testing.T) { + defer func(old func(int) string) { nameSuffixFunc = old }(nameSuffixFunc) + var suffixCounter int + nameSuffixFunc = func(int) string { + suffixCounter++ + return fmt.Sprint(suffixCounter) + } + + instance1 := newEmptyRpaasInstance() + instance1.ObjectMeta.Name = "instance1" + instance2 := newEmptyRpaasInstance() + instance2.ObjectMeta.Name = "instance2" + instance3 := newEmptyRpaasInstance() + instance3.ObjectMeta.Name = "instance3" + instance4 := newEmptyRpaasInstance() + instance4.ObjectMeta.Name = "instance4" + instance5 := newEmptyRpaasInstance() + instance5.ObjectMeta.Name = "instance5" + + nginx1 := &nginxv1alpha1.Nginx{ + ObjectMeta: instance1.ObjectMeta, + Status: nginxv1alpha1.NginxStatus{ + PodSelector: "nginx.tsuru.io/app=nginx,nginx.tsuru.io/resource-name=instance1", + }, + } + nginx2 := &nginxv1alpha1.Nginx{ + ObjectMeta: instance2.ObjectMeta, + Status: nginxv1alpha1.NginxStatus{ + PodSelector: "nginx.tsuru.io/app=nginx,nginx.tsuru.io/resource-name=instance2", + }, + } + nginx3 := &nginxv1alpha1.Nginx{ + ObjectMeta: instance3.ObjectMeta, + Status: nginxv1alpha1.NginxStatus{ + PodSelector: "nginx.tsuru.io/app=nginx,nginx.tsuru.io/resource-name=instance3", + }, + } + nginx4 := &nginxv1alpha1.Nginx{ + ObjectMeta: instance5.ObjectMeta, + Status: nginxv1alpha1.NginxStatus{ + PodSelector: "nginx.tsuru.io/app=nginx,nginx.tsuru.io/resource-name=instance5", + }, + } + + pod1 := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod1", + Namespace: instance2.Namespace, + Labels: map[string]string{ + "nginx.tsuru.io/app": "nginx", + "nginx.tsuru.io/resource-name": "instance2", + }, + UID: types.UID("pod1-uid"), + }, + Status: corev1.PodStatus{ + PodIP: "10.0.0.1", + Phase: corev1.PodRunning, + ContainerStatuses: []corev1.ContainerStatus{ + { + Name: "nginx", + Ready: true, + }, + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "nginx"}, + }, + }, + } + pod2 := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod2", + Namespace: instance2.Namespace, + Labels: map[string]string{ + "nginx.tsuru.io/app": "nginx", + "nginx.tsuru.io/resource-name": "instance2", + }, + UID: types.UID("pod2-uid"), + }, + Status: corev1.PodStatus{ + PodIP: "10.0.0.2", + Phase: corev1.PodPending, + ContainerStatuses: []corev1.ContainerStatus{ + { + Name: "nginx", + Ready: false, + }, + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "nginx"}, + }, + }, + } + pod4 := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod4", + Namespace: instance5.Namespace, + Labels: map[string]string{ + "nginx.tsuru.io/app": "nginx", + "nginx.tsuru.io/resource-name": "instance5", + }, + UID: types.UID("pod4-uid"), + }, + Status: corev1.PodStatus{ + PodIP: "10.0.0.9", + ContainerStatuses: []corev1.ContainerStatus{ + { + Ready: false, + }, + }, + }, + } + + resources := []runtime.Object{instance1, instance2, instance3, instance4, instance5, nginx1, nginx2, nginx3, nginx4, pod1, pod2, pod4} + staticTimeNow := metav1.Now() + + testCases := []struct { + name string + instance string + args DebugArgs + assertion func(*testing.T, error, *k8sRpaasManager, *v1alpha1.RpaasInstance, string, *corev1.ContainerStatus) + pods func() []corev1.Pod + }{ + { + name: "no debug image configured", + instance: "instance1", + pods: func() []corev1.Pod { return []corev1.Pod{} }, + assertion: func(t *testing.T, err error, m *k8sRpaasManager, instance *v1alpha1.RpaasInstance, debugContainerName string, debugContainerStatus *corev1.ContainerStatus) { + assert.Error(t, err) + assert.EqualError(t, err, "Debug image not set and no default image configured") + }, + }, + { + name: "no pod running for debug", + instance: "instance1", + pods: func() []corev1.Pod { return []corev1.Pod{} }, + assertion: func(t *testing.T, err error, m *k8sRpaasManager, instance *v1alpha1.RpaasInstance, debugContainerName string, debugContainerStatus *corev1.ContainerStatus) { + assert.Error(t, err) + assert.EqualError(t, err, "no pod running found in instance instance1") + }, + }, + { + name: "debug on invalid pod", + instance: "instance1", + args: DebugArgs{CommonTerminalArgs: CommonTerminalArgs{Pod: "pod1"}}, + pods: func() []corev1.Pod { return []corev1.Pod{} }, + assertion: func(t *testing.T, err error, m *k8sRpaasManager, instance *v1alpha1.RpaasInstance, debugContainerName string, debugContainerStatus *corev1.ContainerStatus) { + assert.Error(t, err) + assert.EqualError(t, err, "no such pod pod1 in instance instance1") + }, + }, + { + name: "run debug on pod1 with default image", + instance: "instance2", + args: DebugArgs{CommonTerminalArgs: CommonTerminalArgs{Pod: "pod1", Stdin: &bytes.Buffer{}, Stdout: io.Discard, Stderr: io.Discard}}, + pods: func() []corev1.Pod { + pod1Debug := pod1.DeepCopy() + pod1Debug.Spec.EphemeralContainers = append(pod1Debug.Spec.EphemeralContainers, corev1.EphemeralContainer{EphemeralContainerCommon: corev1.EphemeralContainerCommon{Name: "debugger-1"}, TargetContainerName: "nginx"}) + pod1Debug.Status.ContainerStatuses = append(pod1Debug.Status.EphemeralContainerStatuses, corev1.ContainerStatus{Name: "debugger-1", Ready: true, State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{StartedAt: staticTimeNow}}}) + return []corev1.Pod{*pod1Debug} + }, + assertion: func(t *testing.T, err error, m *k8sRpaasManager, instance *v1alpha1.RpaasInstance, debugContainerName string, debugContainerStatus *corev1.ContainerStatus) { + assert.NoError(t, err) + assert.Equal(t, "debugger-1", debugContainerName) + assert.Equal(t, corev1.ContainerStatus{Name: "debugger-1", Ready: true, State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{StartedAt: staticTimeNow}}}, *debugContainerStatus) + instancePod := corev1.Pod{} + err = m.cli.Get(context.Background(), types.NamespacedName{Name: "pod1", Namespace: instance2.Namespace}, &instancePod) + require.NoError(t, err) + expectedEphemerals := []corev1.EphemeralContainer{{EphemeralContainerCommon: corev1.EphemeralContainerCommon{Name: "debugger-1", Image: "tsuru/debug-image", ImagePullPolicy: corev1.PullIfNotPresent, Stdin: true}, TargetContainerName: "nginx"}} + assert.Equal(t, expectedEphemerals, instancePod.Spec.EphemeralContainers) + }, + }, + { + name: "run debug on random pod with running status", + instance: "instance2", + args: DebugArgs{CommonTerminalArgs: CommonTerminalArgs{Stdin: &bytes.Buffer{}, Stdout: io.Discard, Stderr: io.Discard}}, + pods: func() []corev1.Pod { + pod1Debug := pod1.DeepCopy() + pod1Debug.Spec.EphemeralContainers = append(pod1Debug.Spec.EphemeralContainers, corev1.EphemeralContainer{EphemeralContainerCommon: corev1.EphemeralContainerCommon{Name: "debugger-2"}, TargetContainerName: "nginx"}) + pod1Debug.Status.ContainerStatuses = append(pod1Debug.Status.EphemeralContainerStatuses, corev1.ContainerStatus{Name: "debugger-2", Ready: true, State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{StartedAt: staticTimeNow}}}) + return []corev1.Pod{*pod1Debug} + }, + assertion: func(t *testing.T, err error, m *k8sRpaasManager, instance *v1alpha1.RpaasInstance, debugContainerName string, debugContainerStatus *corev1.ContainerStatus) { + assert.NoError(t, err) + assert.Equal(t, "debugger-2", debugContainerName) + assert.Equal(t, corev1.ContainerStatus{Name: "debugger-2", Ready: true, State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{StartedAt: staticTimeNow}}}, *debugContainerStatus) + instancePod := corev1.Pod{} + err = m.cli.Get(context.Background(), types.NamespacedName{Name: "pod1", Namespace: instance2.Namespace}, &instancePod) + require.NoError(t, err) + expectedEphemerals := []corev1.EphemeralContainer{{EphemeralContainerCommon: corev1.EphemeralContainerCommon{Name: "debugger-2", Image: "tsuru/debug-image", ImagePullPolicy: corev1.PullIfNotPresent, Stdin: true}, TargetContainerName: "nginx"}} + assert.Equal(t, expectedEphemerals, instancePod.Spec.EphemeralContainers) + }, + }, + } + + for _, tt := range testCases { + cfg := config.Get() + defer func() { config.Set(cfg) }() + config.Set(config.RpaasConfig{DebugImage: "tsuru/debug-image"}) + t.Run(tt.name, func(t *testing.T) { + var wg sync.WaitGroup + manager := &k8sRpaasManager{cli: fake.NewClientBuilder().WithScheme(newScheme()).WithRuntimeObjects(resources...).Build()} + watcher := watch.NewFake() + kcs := k8sclient.NewSimpleClientset() + kcs.PrependWatchReactor("pods", k8stesting.DefaultWatchReactor(watcher, nil)) + manager.kcs = kcs + if tt.name == "no debug image configured" { + config.Set(config.RpaasConfig{}) + } + ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) + defer cancel() + wg.Add(1) + go func() { + defer wg.Done() + defer watcher.Stop() + for _, pod := range tt.pods() { + watcher.Modify(&pod) + time.Sleep(1000 * time.Millisecond) + } + }() + rpaasInstance, debugContainerName, debugContainerStatus, err := manager.debugPodWithContainerStatus(ctx, &tt.args, tt.instance) + wg.Wait() + if err != nil { + tt.assertion(t, err, manager, nil, "", nil) + } else { + tt.assertion(t, err, manager, rpaasInstance, debugContainerName, debugContainerStatus) + } + }) + } + +} diff --git a/internal/pkg/rpaas/manager.go b/internal/pkg/rpaas/manager.go index 28b0a6806..67970d380 100644 --- a/internal/pkg/rpaas/manager.go +++ b/internal/pkg/rpaas/manager.go @@ -196,7 +196,17 @@ type AutoscaleHandler interface { } type ExecArgs struct { - Command []string + Command []string + CommonTerminalArgs +} + +type DebugArgs struct { + Command []string + Image string + CommonTerminalArgs +} + +type CommonTerminalArgs struct { Pod string Container string TerminalWidth uint16 @@ -209,6 +219,22 @@ type ExecArgs struct { Stderr io.Writer } +func (c *CommonTerminalArgs) SetStdin(r io.Reader) { + c.Stdin = r +} + +func (c *CommonTerminalArgs) SetStdout(w io.Writer) { + c.Stdout = w +} + +func (c *CommonTerminalArgs) SetStderr(w io.Writer) { + c.Stderr = w +} + +func (c *CommonTerminalArgs) GetInteractive() bool { + return c.Interactive +} + type LogArgs struct { Stderr io.Writer Stdout io.Writer @@ -245,6 +271,7 @@ type RpaasManager interface { PurgeCache(ctx context.Context, instanceName string, args PurgeCacheArgs) (int, error) GetInstanceInfo(ctx context.Context, instanceName string) (*clientTypes.InstanceInfo, error) Exec(ctx context.Context, instanceName string, args ExecArgs) error + Debug(ctx context.Context, instanceName string, args DebugArgs) error Log(ctx context.Context, intanceName string, args LogArgs) error AddUpstream(ctx context.Context, instanceName string, upstream v1alpha1.AllowedUpstream) error diff --git a/pkg/rpaas/client/client.go b/pkg/rpaas/client/client.go index e3301b635..46ba35ab1 100644 --- a/pkg/rpaas/client/client.go +++ b/pkg/rpaas/client/client.go @@ -119,6 +119,19 @@ type ExecArgs struct { TTY bool } +type DebugArgs struct { + In io.Reader + Command []string + Instance string + Pod string + Container string + Image string + TerminalWidth uint16 + TerminalHeight uint16 + TTY bool + Interactive bool +} + type LogArgs struct { Out io.Writer Instance string @@ -149,6 +162,7 @@ type Client interface { ListRoutes(ctx context.Context, args ListRoutesArgs) ([]types.Route, error) UpdateRoute(ctx context.Context, args UpdateRouteArgs) error Exec(ctx context.Context, args ExecArgs) (*websocket.Conn, error) + Debug(ctx context.Context, args DebugArgs) (*websocket.Conn, error) Log(ctx context.Context, args LogArgs) error AddExtraFiles(ctx context.Context, args ExtraFilesArgs) error UpdateExtraFiles(ctx context.Context, args ExtraFilesArgs) error @@ -166,3 +180,11 @@ type Client interface { UpdateCertManager(ctx context.Context, args UpdateCertManagerArgs) error DeleteCertManager(ctx context.Context, instance, issuer string) error } + +type wsWriter struct { + *websocket.Conn +} + +func (w *wsWriter) Write(p []byte) (int, error) { + return len(p), w.Conn.WriteMessage(websocket.BinaryMessage, p) +} diff --git a/pkg/rpaas/client/debug.go b/pkg/rpaas/client/debug.go new file mode 100644 index 000000000..63b89984c --- /dev/null +++ b/pkg/rpaas/client/debug.go @@ -0,0 +1,65 @@ +// Copyright 2023 tsuru authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package client + +import ( + "context" + "fmt" + "io" + "net/url" + "strconv" + + "github.com/gorilla/websocket" +) + +func (args DebugArgs) Validate() error { + if args.Instance == "" { + return ErrMissingInstance + } + + return nil +} + +func (c *client) Debug(ctx context.Context, args DebugArgs) (*websocket.Conn, error) { + if err := args.Validate(); err != nil { + return nil, err + } + + serverAddress := c.formatURL(fmt.Sprintf("/resources/%s/debug", args.Instance), args.Instance) + u, err := url.Parse(serverAddress) + if err != nil { + return nil, err + } + + if u.Scheme == "https" { + u.Scheme = "wss" + } else { + u.Scheme = "ws" + } + + qs := u.Query() + qs.Set("ws", "true") + qs["command"] = args.Command + qs.Set("pod", args.Pod) + qs.Set("container", args.Container) + qs.Set("interactive", strconv.FormatBool(args.Interactive)) + qs.Set("image", args.Image) + qs.Set("tty", strconv.FormatBool(args.TTY)) + qs.Set("width", strconv.FormatInt(int64(args.TerminalWidth), 10)) + qs.Set("height", strconv.FormatInt(int64(args.TerminalHeight), 10)) + + u.RawQuery = qs.Encode() + + conn, _, err := c.ws.DialContext(ctx, u.String(), c.baseAuthHeader(nil)) + if err != nil { + return nil, err + } + + if args.In != nil { + go io.Copy(&wsWriter{conn}, args.In) + } + + return conn, nil +} diff --git a/pkg/rpaas/client/debug_test.go b/pkg/rpaas/client/debug_test.go new file mode 100644 index 000000000..39fde0a8a --- /dev/null +++ b/pkg/rpaas/client/debug_test.go @@ -0,0 +1,80 @@ +// Copyright 2023 tsuru authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package client + +import ( + "context" + "net/http" + "net/url" + "testing" + + "github.com/gorilla/websocket" + "github.com/stretchr/testify/assert" +) + +func TestClientThroughTsuru_Debug(t *testing.T) { + var called bool + tests := []struct { + name string + args DebugArgs + expectedError string + expectedCalled bool + handler http.HandlerFunc + }{ + { + name: "when instance is empty", + expectedError: "rpaasv2: instance cannot be empty", + }, + { + name: "when all options are set", + args: DebugArgs{ + Instance: "my-instance", + Command: []string{"bash"}, + Pod: "pod-1", + Container: "nginx", + Interactive: true, + TTY: true, + Image: "my-image/test", + TerminalWidth: uint16(80), + TerminalHeight: uint16(24), + }, + handler: func(w http.ResponseWriter, r *http.Request) { + called = true + assert.True(t, websocket.IsWebSocketUpgrade(r)) + assert.Equal(t, "Bearer f4k3t0k3n", r.Header.Get("Authorization")) + expectedQS := url.Values{} + expectedQS.Set("callback", "/resources/my-instance/debug") + expectedQS.Set("ws", "true") + expectedQS.Set("command", "bash") + expectedQS.Set("pod", "pod-1") + expectedQS.Set("container", "nginx") + expectedQS.Set("width", "80") + expectedQS.Set("height", "24") + expectedQS.Set("tty", "true") + expectedQS.Set("interactive", "true") + expectedQS.Set("image", "my-image/test") + assert.Equal(t, expectedQS, r.URL.Query()) + w.WriteHeader(http.StatusBadRequest) + }, + expectedCalled: true, + expectedError: "websocket: bad handshake", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + called = false + client, server := newClientThroughTsuru(t, tt.handler) + defer server.Close() + _, err := client.Debug(context.TODO(), tt.args) + if tt.expectedError != "" { + assert.EqualError(t, err, tt.expectedError) + return + } + assert.NoError(t, err) + assert.Equal(t, tt.expectedCalled, called) + }) + } +} diff --git a/pkg/rpaas/client/exec.go b/pkg/rpaas/client/exec.go index f7e075893..a42d23baf 100644 --- a/pkg/rpaas/client/exec.go +++ b/pkg/rpaas/client/exec.go @@ -66,11 +66,3 @@ func (c *client) Exec(ctx context.Context, args ExecArgs) (*websocket.Conn, erro return conn, nil } - -type wsWriter struct { - *websocket.Conn -} - -func (w *wsWriter) Write(p []byte) (int, error) { - return len(p), w.Conn.WriteMessage(websocket.BinaryMessage, p) -} diff --git a/pkg/rpaas/client/fake/client.go b/pkg/rpaas/client/fake/client.go index 3b8878a93..1668e8b3d 100644 --- a/pkg/rpaas/client/fake/client.go +++ b/pkg/rpaas/client/fake/client.go @@ -29,6 +29,7 @@ type FakeClient struct { FakeUpdateRoute func(args client.UpdateRouteArgs) error FakeInfo func(args client.InfoArgs) (*types.InstanceInfo, error) FakeExec func(ctx context.Context, args client.ExecArgs) (*websocket.Conn, error) + FakeDebug func(ctx context.Context, args client.DebugArgs) (*websocket.Conn, error) FakeAddAccessControlList func(instance, host string, port int) error FakeListAccessControlList func(instance string) ([]types.AllowedUpstream, error) FakeRemoveAccessControlList func(instance, host string, port int) error @@ -148,6 +149,14 @@ func (f *FakeClient) Exec(ctx context.Context, args client.ExecArgs) (*websocket return nil, nil } +func (f *FakeClient) Debug(ctx context.Context, args client.DebugArgs) (*websocket.Conn, error) { + if f.FakeDebug != nil { + return f.FakeDebug(ctx, args) + } + + return nil, nil +} + func (f *FakeClient) AddAccessControlList(ctx context.Context, instance, host string, port int) error { if f.FakeAddAccessControlList != nil { return f.FakeAddAccessControlList(instance, host, port) diff --git a/pkg/web/api.go b/pkg/web/api.go index 12f011e03..a7b7d6da7 100644 --- a/pkg/web/api.go +++ b/pkg/web/api.go @@ -249,6 +249,7 @@ func newEcho(targetFactory target.Factory) *echo.Echo { group.POST("/:instance/purge", cachePurge) group.POST("/:instance/purge/bulk", cachePurgeBulk) group.Any("/:instance/exec", exec) + group.Any("/:instance/debug", debug) group.GET("/:instance/acl", getUpstreams) group.POST("/:instance/acl", addUpstream) group.DELETE("/:instance/acl", deleteUpstream) diff --git a/pkg/web/debug.go b/pkg/web/debug.go new file mode 100644 index 000000000..ed8ea5966 --- /dev/null +++ b/pkg/web/debug.go @@ -0,0 +1,61 @@ +// Copyright 2023 tsuru authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package web + +import ( + "net/url" + "strconv" + + "github.com/gorilla/websocket" + "github.com/labstack/echo/v4" + + "github.com/tsuru/rpaas-operator/internal/config" + "github.com/tsuru/rpaas-operator/internal/pkg/rpaas" +) + +func debug(c echo.Context) error { + var wsUpgrader websocket.Upgrader = websocket.Upgrader{ + HandshakeTimeout: config.Get().WebSocketHandshakeTimeout, + ReadBufferSize: config.Get().WebSocketReadBufferSize, + WriteBufferSize: config.Get().WebSocketWriteBufferSize, + CheckOrigin: checkOrigin, + } + useWebSocket, _ := strconv.ParseBool(c.QueryParam("ws")) + if useWebSocket { + ws := &wsTransport{extractArgs: extractDebugArgs, command: debugCommandOnInstance} + return ws.Run(c, &wsUpgrader) + } + http := &http2Transport{extractArgs: extractDebugArgs, command: debugCommandOnInstance} + return http.Run(c) +} + +func debugCommandOnInstance(c echo.Context, args commonArgs) error { + ctx := c.Request().Context() + manager, err := getManager(ctx) + if err != nil { + return err + } + debugArgs := args.(*rpaas.DebugArgs) + return manager.Debug(ctx, c.Param("instance"), *debugArgs) +} + +func extractDebugArgs(r url.Values) commonArgs { + tty, _ := strconv.ParseBool(r.Get("tty")) + interactive, _ := strconv.ParseBool(r.Get("interactive")) + width, _ := strconv.ParseUint(r.Get("width"), 10, 16) + height, _ := strconv.ParseUint(r.Get("height"), 10, 16) + return &rpaas.DebugArgs{ + Image: r.Get("image"), + Command: r["command"], + CommonTerminalArgs: rpaas.CommonTerminalArgs{ + Pod: r.Get("pod"), + Container: r.Get("container"), + TTY: tty, + Interactive: interactive, + TerminalWidth: uint16(width), + TerminalHeight: uint16(height), + }, + } +} diff --git a/pkg/web/debug_test.go b/pkg/web/debug_test.go new file mode 100644 index 000000000..1ce1c4838 --- /dev/null +++ b/pkg/web/debug_test.go @@ -0,0 +1,223 @@ +// Copyright 2023 tsuru authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package web + +import ( + "bufio" + "crypto/tls" + "fmt" + "io" + "net" + "net/http" + "strings" + "testing" + + "github.com/gorilla/websocket" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/net/http2" + + "github.com/tsuru/rpaas-operator/internal/config" + "github.com/tsuru/rpaas-operator/internal/pkg/rpaas" + "github.com/tsuru/rpaas-operator/internal/pkg/rpaas/fake" +) + +var debugH2cClient = &http.Client{ + Transport: &http2.Transport{ + AllowHTTP: true, + DialTLS: func(network, addr string, cfg *tls.Config) (net.Conn, error) { + return net.Dial(network, addr) + }, + }, +} + +func Test_Debug(t *testing.T) { + require.NoError(t, config.Init()) + originalConfig := config.Get() + defer config.Set(originalConfig) + + var called bool + var clientCh, serverCh chan bool + + tests := []struct { + name string + manager *fake.RpaasManager + request func(t *testing.T, serverURL string) + expectedCalled bool + }{ + { + name: "over websocket from a not allowed origin", + request: func(t *testing.T, serverURL string) { + uri := fmt.Sprintf("ws://%s/resources/my-instance/debug?ws=true&command=bash&tty=true&width=80&height=24", strings.TrimPrefix(serverURL, "http://")) + _, response, err := websocket.DefaultDialer.Dial(uri, http.Header{"origin": {"evil.test"}}) + require.Error(t, err) + assert.Equal(t, http.StatusForbidden, response.StatusCode) + assert.Equal(t, "Forbidden", bodyContent(response)) + }, + }, + { + name: "upgrading to WebSocket via HTTP/1.1", + request: func(t *testing.T, serverURL string) { + uri := fmt.Sprintf("ws://%s/resources/my-instance/debug?ws=true&interactive=true&command=bash&tty=true&width=80&height=24&image=tsuru/debug", strings.TrimPrefix(serverURL, "http://")) + conn, _, err := websocket.DefaultDialer.Dial(uri, nil) + require.NoError(t, err) + defer conn.Close() + + <-clientCh + mtype, b, err := conn.ReadMessage() + require.NoError(t, err) + assert.Equal(t, mtype, websocket.TextMessage) + assert.Equal(t, string(b), "root@some-hostname # ") + + conn.WriteMessage(websocket.TextMessage, []byte("my-interactive-command\n")) + serverCh <- true + + <-clientCh + mtype, b, err = conn.ReadMessage() + require.NoError(t, err) + assert.Equal(t, mtype, websocket.TextMessage) + assert.Equal(t, string(b), "some error :/\n") + }, + manager: &fake.RpaasManager{ + FakeDebug: func(instance string, args rpaas.DebugArgs) error { + called = true + assert.Equal(t, instance, "my-instance") + assert.Equal(t, args.Command, []string{"bash"}) + assert.Equal(t, args.TTY, true) + assert.Equal(t, args.TerminalWidth, uint16(80)) + assert.Equal(t, args.TerminalHeight, uint16(24)) + assert.Equal(t, args.Image, "tsuru/debug") + assert.NotNil(t, args.Stdin) + assert.NotNil(t, args.Stderr) + assert.NotNil(t, args.Stdout) + + fmt.Fprintf(args.Stdout, "root@some-hostname # ") + clientCh <- true + + <-serverCh + r := bufio.NewReader(args.Stdin) + + body, _, err := r.ReadLine() + assert.NoError(t, err) + assert.Equal(t, "my-interactive-command", string(body)) + + fmt.Fprintf(args.Stderr, "some error :/\n") + clientCh <- true + return nil + }, + }, + expectedCalled: true, + }, + { + name: "using HTTP/2 directly (h2c)", + request: func(t *testing.T, serverURL string) { + pr, pw := io.Pipe() + uri := fmt.Sprintf("%s/resources/my-instance/debug?ws=false&interactive=true&command=/bin/sh&tty=true&width=124&height=80&image=tsuru/debug", serverURL) + request, err := http.NewRequest("POST", uri, pr) + require.NoError(t, err) + + response, err := debugH2cClient.Do(request) + require.NoError(t, err) + defer response.Body.Close() + assert.NotNil(t, response) + require.Equal(t, http.StatusOK, response.StatusCode) + + <-clientCh + r := bufio.NewReader(response.Body) + body, _, err := r.ReadLine() + require.NoError(t, err) + assert.Equal(t, "root@some-hostname # ", string(body)) + + fmt.Fprintf(pw, "./my-command.sh -abcde\r\n") + serverCh <- true + + <-clientCh + body, _, err = r.ReadLine() + require.NoError(t, err) + assert.Equal(t, "some command error", string(body)) + }, + manager: &fake.RpaasManager{ + FakeDebug: func(instance string, args rpaas.DebugArgs) error { + called = true + assert.Equal(t, instance, "my-instance") + assert.Equal(t, args.Command, []string{"/bin/sh"}) + assert.Equal(t, args.TTY, true) + assert.Equal(t, args.TerminalWidth, uint16(124)) + assert.Equal(t, args.TerminalHeight, uint16(80)) + assert.Equal(t, args.Image, "tsuru/debug") + assert.NotNil(t, args.Stdin) + assert.NotNil(t, args.Stderr) + assert.NotNil(t, args.Stdout) + + fmt.Fprintf(args.Stdout, "root@some-hostname # \r\n") + clientCh <- true + + <-serverCh + r := bufio.NewReader(args.Stdin) + body, _, err := r.ReadLine() + require.NoError(t, err) + assert.Equal(t, "./my-command.sh -abcde", string(body)) + + fmt.Fprintf(args.Stderr, "some command error\r\n") + clientCh <- true + return nil + }, + }, + expectedCalled: true, + }, + { + name: "trying execute over HTTP/1.x", + request: func(t *testing.T, serverURL string) { + uri := fmt.Sprintf("%s/resources/my-instance/debug?command=some-command", serverURL) + request, err := http.NewRequest("POST", uri, nil) + require.NoError(t, err) + + response, err := http.DefaultClient.Do(request) + require.NoError(t, err) + assert.Equal(t, http.StatusHTTPVersionNotSupported, response.StatusCode) + assert.Equal(t, bodyContent(response), "this endpoint only works over HTTP/2") + }, + }, + { + name: "using HTTP/2 with method different than POST", + request: func(t *testing.T, serverURL string) { + uri := fmt.Sprintf("%s/resources/my-instance/exec?command=some-command", serverURL) + request, err := http.NewRequest("GET", uri, nil) + require.NoError(t, err) + + response, err := debugH2cClient.Do(request) + require.NoError(t, err) + assert.Equal(t, http.StatusMethodNotAllowed, response.StatusCode) + assert.Equal(t, bodyContent(response), "only POST method is supported") + }, + }, + } + + for _, tt := range tests { + cfg := config.Get() + cfg.WebSocketAllowedOrigins = []string{"rpaasv2.example.com"} + config.Set(cfg) + + t.Run(tt.name, func(t *testing.T) { + called = false + clientCh, serverCh = make(chan bool), make(chan bool) + defer close(clientCh) + defer close(serverCh) + + l, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer l.Close() + + webApi, err := NewWithManager(tt.manager) + require.NoError(t, err) + webApi.e.Listener = l + go webApi.Start() + defer webApi.Stop() + + tt.request(t, fmt.Sprintf("http://%s", l.Addr().String())) + assert.Equal(t, called, tt.expectedCalled) + }) + } +} diff --git a/pkg/web/exec.go b/pkg/web/exec.go index d61c0a773..93288a8ec 100644 --- a/pkg/web/exec.go +++ b/pkg/web/exec.go @@ -5,11 +5,8 @@ package web import ( - "io" - "net/http" "net/url" "strconv" - "time" "github.com/gorilla/websocket" "github.com/labstack/echo/v4" @@ -19,176 +16,45 @@ import ( ) func exec(c echo.Context) error { + var wsUpgrader websocket.Upgrader = websocket.Upgrader{ + HandshakeTimeout: config.Get().WebSocketHandshakeTimeout, + ReadBufferSize: config.Get().WebSocketReadBufferSize, + WriteBufferSize: config.Get().WebSocketWriteBufferSize, + CheckOrigin: checkOrigin, + } useWebSocket, _ := strconv.ParseBool(c.QueryParam("ws")) if useWebSocket { - return wsExec(c) - } - return http2Exec(c) -} - -var wsUpgrader websocket.Upgrader = websocket.Upgrader{ - HandshakeTimeout: config.Get().WebSocketHandshakeTimeout, - ReadBufferSize: config.Get().WebSocketReadBufferSize, - WriteBufferSize: config.Get().WebSocketWriteBufferSize, - CheckOrigin: checkOrigin, -} - -type wsReadWriter struct { - *websocket.Conn -} - -func (r *wsReadWriter) Read(p []byte) (int, error) { - messageType, re, err := r.NextReader() - if err != nil { - return 0, err - } - - if messageType != websocket.TextMessage && messageType != websocket.BinaryMessage { - return 0, nil - } - - return re.Read(p) -} - -func (r *wsReadWriter) Write(p []byte) (int, error) { - return len(p), r.WriteMessage(websocket.TextMessage, p) -} - -func wsExec(c echo.Context) error { - conn, err := wsUpgrader.Upgrade(c.Response(), c.Request(), nil) - if err != nil { - return err - } - - cfg := config.Get() - defer func() { - code, message := websocket.CloseNormalClosure, "" - if err != nil { - // NOTE: logging the error here since we have no guarantees that - // client is going to receive it. - c.Logger().Errorf("failed to run the remote command: %v", err) - code, message = websocket.CloseInternalServerErr, err.Error() - } - - nerr := conn.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(code, message), time.Now().Add(cfg.WebSocketWriteWait)) - if nerr != nil { - c.Logger().Errorf("failed to write the close message to peer %s: %v", conn.RemoteAddr(), nerr) - conn.Close() - } - }() - - quit := make(chan bool) - defer close(quit) - - go func() { - for { - select { - case <-quit: - return - - case <-time.After(cfg.WebSocketPingInterval): - conn.WriteControl(websocket.PingMessage, nil, time.Now().Add(cfg.WebSocketWriteWait)) - } - } - }() - - conn.SetReadDeadline(time.Now().Add(cfg.WebSocketMaxIdleTime)) - conn.SetPongHandler(func(s string) error { - conn.SetReadDeadline(time.Now().Add(cfg.WebSocketMaxIdleTime)) - return nil - }) - - wsRW := &wsReadWriter{conn} - args := extractExecArgs(c.QueryParams()) - if args.Interactive { - args.Stdin = wsRW - } - args.Stdout = wsRW - args.Stderr = wsRW - - err = execCommandOnInstance(c, args) - - // NOTE: avoiding to return error since the connection has already been - // hijacked by websocket at this point. - // - // See: https://github.com/labstack/echo/issues/268 - return nil -} - -func checkOrigin(r *http.Request) bool { - origin := r.Header.Get("Origin") - if origin == "" { - return true - } - - allowedOrigins := config.Get().WebSocketAllowedOrigins - if len(allowedOrigins) == 0 { - return true - } - - for _, ao := range allowedOrigins { - if ao == origin { - return true - } - } - return false -} - -type http2Writer struct { - io.Writer -} - -func (c *http2Writer) Write(arr []byte) (int, error) { - n, err := c.Writer.Write(arr) - if err != nil { - return n, err - } - - if f, ok := c.Writer.(http.Flusher); ok { - f.Flush() - } - return n, nil -} - -func http2Exec(c echo.Context) error { - if c.Request().ProtoMajor != 2 { - return c.String(http.StatusHTTPVersionNotSupported, "this endpoint only works over HTTP/2") - } - - if c.Request().Method != http.MethodPost { - return c.String(http.StatusMethodNotAllowed, "only POST method is supported") - } - - buffer := &http2Writer{c.Response().Writer} - args := extractExecArgs(c.QueryParams()) - if args.Interactive { - args.Stdin = c.Request().Body + ws := &wsTransport{extractArgs: extractExecArgs, command: execCommandOnInstance} + return ws.Run(c, &wsUpgrader) } - args.Stdout, args.Stderr = buffer, buffer - return execCommandOnInstance(c, args) + http := &http2Transport{extractArgs: extractExecArgs, command: execCommandOnInstance} + return http.Run(c) } -func execCommandOnInstance(c echo.Context, args rpaas.ExecArgs) error { +func execCommandOnInstance(c echo.Context, args commonArgs) error { ctx := c.Request().Context() manager, err := getManager(ctx) if err != nil { return err } - return manager.Exec(ctx, c.Param("instance"), args) + execArgs := args.(*rpaas.ExecArgs) + return manager.Exec(ctx, c.Param("instance"), *execArgs) } -func extractExecArgs(r url.Values) rpaas.ExecArgs { +func extractExecArgs(r url.Values) commonArgs { tty, _ := strconv.ParseBool(r.Get("tty")) interactive, _ := strconv.ParseBool(r.Get("interactive")) width, _ := strconv.ParseUint(r.Get("width"), 10, 16) height, _ := strconv.ParseUint(r.Get("height"), 10, 16) - return rpaas.ExecArgs{ - Command: r["command"], - Pod: r.Get("pod"), - Container: r.Get("container"), - TTY: tty, - Interactive: interactive, - TerminalWidth: uint16(width), - TerminalHeight: uint16(height), + return &rpaas.ExecArgs{ + Command: r["command"], + CommonTerminalArgs: rpaas.CommonTerminalArgs{ + Pod: r.Get("pod"), + Container: r.Get("container"), + TTY: tty, + Interactive: interactive, + TerminalWidth: uint16(width), + TerminalHeight: uint16(height), + }, } } diff --git a/pkg/web/exec_test.go b/pkg/web/exec_test.go index e36614edf..5020adfb1 100644 --- a/pkg/web/exec_test.go +++ b/pkg/web/exec_test.go @@ -24,7 +24,7 @@ import ( "github.com/tsuru/rpaas-operator/internal/pkg/rpaas/fake" ) -var h2cClient = &http.Client{ +var execH2cClient = &http.Client{ Transport: &http2.Transport{ AllowHTTP: true, DialTLS: func(network, addr string, cfg *tls.Config) (net.Conn, error) { @@ -117,7 +117,7 @@ func Test_Exec(t *testing.T) { request, err := http.NewRequest("POST", uri, pr) require.NoError(t, err) - response, err := h2cClient.Do(request) + response, err := execH2cClient.Do(request) require.NoError(t, err) defer response.Body.Close() assert.NotNil(t, response) @@ -185,7 +185,7 @@ func Test_Exec(t *testing.T) { request, err := http.NewRequest("GET", uri, nil) require.NoError(t, err) - response, err := h2cClient.Do(request) + response, err := execH2cClient.Do(request) require.NoError(t, err) assert.Equal(t, http.StatusMethodNotAllowed, response.StatusCode) assert.Equal(t, bodyContent(response), "only POST method is supported") diff --git a/pkg/web/transport.go b/pkg/web/transport.go new file mode 100644 index 000000000..0befc8524 --- /dev/null +++ b/pkg/web/transport.go @@ -0,0 +1,169 @@ +// Copyright 2023 tsuru authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package web + +import ( + "io" + "net/http" + "net/url" + "time" + + "github.com/gorilla/websocket" + "github.com/labstack/echo/v4" + + "github.com/tsuru/rpaas-operator/internal/config" +) + +type wsReadWriter struct { + *websocket.Conn +} + +func (r *wsReadWriter) Read(p []byte) (int, error) { + messageType, re, err := r.NextReader() + if err != nil { + return 0, err + } + + if messageType != websocket.TextMessage && messageType != websocket.BinaryMessage { + return 0, nil + } + + return re.Read(p) +} + +func (r *wsReadWriter) Write(p []byte) (int, error) { + return len(p), r.WriteMessage(websocket.TextMessage, p) +} + +type commonArgs interface { + SetStdout(io.Writer) + SetStderr(io.Writer) + SetStdin(io.Reader) + GetInteractive() bool +} + +type wsTransport struct { + extractArgs func(r url.Values) commonArgs + command func(c echo.Context, args commonArgs) error +} + +func (w *wsTransport) Run(c echo.Context, wsUpgrader *websocket.Upgrader) error { + conn, err := wsUpgrader.Upgrade(c.Response(), c.Request(), nil) + if err != nil { + return err + } + + cfg := config.Get() + defer func() { + code, message := websocket.CloseNormalClosure, "" + if err != nil { + // NOTE: logging the error here since we have no guarantees that + // client is going to receive it. + c.Logger().Errorf("failed to run the remote command: %v", err) + code, message = websocket.CloseInternalServerErr, err.Error() + } + + nerr := conn.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(code, message), time.Now().Add(cfg.WebSocketWriteWait)) + if nerr != nil { + c.Logger().Errorf("failed to write the close message to peer %s: %v", conn.RemoteAddr(), nerr) + conn.Close() + } + }() + + quit := make(chan bool) + defer close(quit) + + go func() { + for { + select { + case <-quit: + return + + case <-time.After(cfg.WebSocketPingInterval): + conn.WriteControl(websocket.PingMessage, nil, time.Now().Add(cfg.WebSocketWriteWait)) + } + } + }() + + conn.SetReadDeadline(time.Now().Add(cfg.WebSocketMaxIdleTime)) + conn.SetPongHandler(func(s string) error { + conn.SetReadDeadline(time.Now().Add(cfg.WebSocketMaxIdleTime)) + return nil + }) + + wsRW := &wsReadWriter{conn} + args := w.extractArgs(c.QueryParams()) + if args.GetInteractive() { + args.SetStdin(wsRW) + } + args.SetStdout(wsRW) + args.SetStderr(wsRW) + err = w.command(c, args) + + // NOTE: avoiding to return error since the connection has already been + // hijacked by websocket at this point. + // + // See: https://github.com/labstack/echo/issues/268 + return nil +} + +type http2Writer struct { + io.Writer +} + +func (c *http2Writer) Write(arr []byte) (int, error) { + n, err := c.Writer.Write(arr) + if err != nil { + return n, err + } + + if f, ok := c.Writer.(http.Flusher); ok { + f.Flush() + } + return n, nil +} + +type http2Transport struct { + extractArgs func(r url.Values) commonArgs + command func(c echo.Context, args commonArgs) error +} + +func (h *http2Transport) Run(c echo.Context) error { + if c.Request().ProtoMajor != 2 { + return c.String(http.StatusHTTPVersionNotSupported, "this endpoint only works over HTTP/2") + } + + if c.Request().Method != http.MethodPost { + return c.String(http.StatusMethodNotAllowed, "only POST method is supported") + } + + buffer := &http2Writer{c.Response().Writer} + args := h.extractArgs(c.QueryParams()) + if args.GetInteractive() { + args.SetStdin(c.Request().Body) + } + args.SetStdout(buffer) + args.SetStderr(buffer) + return h.command(c, args) +} + +func checkOrigin(r *http.Request) bool { + origin := r.Header.Get("Origin") + if origin == "" { + return true + } + + allowedOrigins := config.Get().WebSocketAllowedOrigins + if len(allowedOrigins) == 0 { + return true + } + + for _, ao := range allowedOrigins { + if ao == origin { + return true + } + } + return false +} diff --git a/scripts/localkube-integration.sh b/scripts/localkube-integration.sh index 02ebcb59c..a76765c60 100755 --- a/scripts/localkube-integration.sh +++ b/scripts/localkube-integration.sh @@ -20,7 +20,7 @@ readonly NAMESPACE=${NAMESPACE:-rpaasv2-system} readonly INSTALL_CERT_MANAGER=${INSTALL_CERT_MANAGER:-} readonly CHART_VERSION_CERT_MANAGER=${CHART_VERSION_CERT_MANAGER:-1.11.2} readonly CHART_VERSION_RPAAS_OPERATOR=${CHART_VERSION_RPAAS_OPERATOR:-0.11.7} -readonly CHART_VERSION_RPAAS_API=${CHART_VERSION_RPAAS_API:-0.2.1} +readonly CHART_VERSION_RPAAS_API=${CHART_VERSION_RPAAS_API:-0.2.2} function onerror() { echo diff --git a/test/integration_test.go b/test/integration_test.go index 50e4a82d2..04a08d4e9 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -196,6 +196,51 @@ func Test_RpaasApi(t *testing.T) { assert.Contains(t, string(logOut), "--WORKING--", logCmd.String()) }) + t.Run("run debug container and get output", func(t *testing.T) { + instanceName := generateRandomName("my-instance") + teamName := generateRandomName("team-one") + planName := "basic" + + cleanFunc, err := api.createInstance(instanceName, planName, teamName) + require.NoError(t, err) + defer cleanFunc() + + _, err = getReadyNginx(instanceName, namespaceName, ensureNginxPodsAndServices(1, 1)) + require.NoError(t, err) + + debugArgs := []string{"--rpaas-url", apiAddress, "debug", "-i", instanceName, "-d", "nicolaka/netshoot", "--", "/bin/bash", "-c", "whoami && echo -n \"--WORKING--\""} + debugCmd := exec.CommandContext(context.Background(), rpaasv2Bin, debugArgs...) + debugOut, err := debugCmd.CombinedOutput() + require.NoError(t, err) + assert.Contains(t, string(debugOut), "\nroot\n--WORKING--", debugCmd.String()) + }) + + t.Run("interact with debug shell", func(t *testing.T) { + instanceName := generateRandomName("my-instance") + teamName := generateRandomName("team-one") + planName := "basic" + + cleanFunc, err := api.createInstance(instanceName, planName, teamName) + require.NoError(t, err) + defer cleanFunc() + + _, err = getReadyNginx(instanceName, namespaceName, ensureNginxPodsAndServices(1, 1)) + require.NoError(t, err) + + debugArgs := []string{"--rpaas-url", apiAddress, "debug", "-t", "-I", "-i", instanceName, "-d", "nicolaka/netshoot"} + debugCmd := exec.CommandContext(context.Background(), rpaasv2Bin, debugArgs...) + stdin, err := debugCmd.StdinPipe() + require.NoError(t, err) + go func() { + defer stdin.Close() + io.WriteString(stdin, "echo \"i am $(whoami)\"\n") + io.WriteString(stdin, "exit\n") + }() + stdout, err := debugCmd.CombinedOutput() + require.NoError(t, err) + assert.Contains(t, string(stdout), "i am root", debugCmd.String()) + }) + t.Run("creating and deleting an instance", func(t *testing.T) { instanceName := generateRandomName("my-instance") teamName := generateRandomName("team-one")