diff --git a/pkg/kwok/server/debugging_exec_other.go b/pkg/kwok/server/debugging_exec_other.go index 83bdca1dc0..03e791c6e7 100644 --- a/pkg/kwok/server/debugging_exec_other.go +++ b/pkg/kwok/server/debugging_exec_other.go @@ -27,6 +27,7 @@ import ( "sigs.k8s.io/kwok/pkg/log" "sigs.k8s.io/kwok/pkg/utils/exec" + utilsnet "sigs.k8s.io/kwok/pkg/utils/net" ) func (s *Server) execInContainerWithTTY(ctx context.Context, cmd []string, in io.Reader, out io.WriteCloser, resize <-chan clientremotecommand.TerminalSize) error { @@ -54,7 +55,7 @@ func (s *Server) execInContainerWithTTY(ctx context.Context, cmd []string, in io io.Reader io.Writer }{in, out} - err := tunnel(ctx, pty, stm, buf1, buf2) + err := utilsnet.Tunnel(ctx, pty, stm, buf1, buf2) if err != nil { logger.Error("failed to tunnel", err) } diff --git a/pkg/kwok/server/debugging_port_forword.go b/pkg/kwok/server/debugging_port_forword.go index c4841c65e3..73a37880f1 100644 --- a/pkg/kwok/server/debugging_port_forword.go +++ b/pkg/kwok/server/debugging_port_forword.go @@ -32,6 +32,7 @@ import ( "sigs.k8s.io/kwok/pkg/apis/internalversion" "sigs.k8s.io/kwok/pkg/log" "sigs.k8s.io/kwok/pkg/utils/exec" + utilsnet "sigs.k8s.io/kwok/pkg/utils/net" "sigs.k8s.io/kwok/pkg/utils/slices" ) @@ -74,7 +75,7 @@ func (s *Server) PortForward(ctx context.Context, name string, uid types.UID, po s.bufPool.Put(buf1) s.bufPool.Put(buf2) }() - return tunnel(ctx, stream, dial, buf1, buf2) + return utilsnet.Tunnel(ctx, stream, dial, buf1, buf2) } return errors.New("no target or command") @@ -144,37 +145,3 @@ func findPortInForwards(port int32, forwards []internalversion.Forward) (*intern } return defaultForward, defaultForward != nil } - -// tunnel create tunnels for two streams. -func tunnel(ctx context.Context, c1, c2 io.ReadWriter, buf1, buf2 []byte) error { - errCh := make(chan error) - go func() { - _, err := io.CopyBuffer(c2, c1, buf1) - errCh <- err - }() - go func() { - _, err := io.CopyBuffer(c1, c2, buf2) - errCh <- err - }() - select { - case <-ctx.Done(): - // Do nothing - case err1 := <-errCh: - select { - case <-ctx.Done(): - if err1 != nil { - return err1 - } - // Do nothing - case err2 := <-errCh: - if err1 != nil { - return err1 - } - return err2 - } - } - if err := ctx.Err(); err != nil && !errors.Is(err, context.Canceled) { - return err - } - return nil -} diff --git a/pkg/kwokctl/cmd/hack/del/del.go b/pkg/kwokctl/cmd/hack/del/del.go index a17802f89b..d30ca5a869 100644 --- a/pkg/kwokctl/cmd/hack/del/del.go +++ b/pkg/kwokctl/cmd/hack/del/del.go @@ -105,10 +105,11 @@ func runE(ctx context.Context, flags *flagpole, args []string) error { return nil } - etcdclient, err := rt.GetEtcdClient(ctx) + etcdclient, cancel, err := rt.GetEtcdClient(ctx) if err != nil { return err } + defer cancel() var targetGvr schema.GroupVersionResource var targetName string diff --git a/pkg/kwokctl/cmd/hack/get/get.go b/pkg/kwokctl/cmd/hack/get/get.go index 233274b53e..f81498a228 100644 --- a/pkg/kwokctl/cmd/hack/get/get.go +++ b/pkg/kwokctl/cmd/hack/get/get.go @@ -111,10 +111,11 @@ func runE(ctx context.Context, flags *flagpole, args []string) error { return nil } - etcdclient, err := rt.GetEtcdClient(ctx) + etcdclient, cancel, err := rt.GetEtcdClient(ctx) if err != nil { return err } + defer cancel() var targetGvr schema.GroupVersionResource var targetName string diff --git a/pkg/kwokctl/cmd/hack/put/put.go b/pkg/kwokctl/cmd/hack/put/put.go index 360bf19091..014b3af75d 100644 --- a/pkg/kwokctl/cmd/hack/put/put.go +++ b/pkg/kwokctl/cmd/hack/put/put.go @@ -125,10 +125,11 @@ func runE(ctx context.Context, flags *flagpole, args []string) error { } } - etcdclient, err := rt.GetEtcdClient(ctx) + etcdclient, cancel, err := rt.GetEtcdClient(ctx) if err != nil { return err } + defer cancel() kubeconfigPath := rt.GetWorkdirPath(runtime.InHostKubeconfigName) clientset, err := client.NewClientset("", kubeconfigPath) diff --git a/pkg/kwokctl/cmd/logs/logs.go b/pkg/kwokctl/cmd/logs/logs.go index 5157bdcf03..39cbe54b7f 100644 --- a/pkg/kwokctl/cmd/logs/logs.go +++ b/pkg/kwokctl/cmd/logs/logs.go @@ -40,7 +40,7 @@ func NewCommand(ctx context.Context) *cobra.Command { flags := &flagpole{} cmd := &cobra.Command{ - Use: "logs [command]", + Use: "logs [component]", Short: "Logs one of [audit, etcd, kube-apiserver, kube-controller-manager, kube-scheduler, kwok-controller, dashboard, metrics-server, prometheus, jaeger]", RunE: func(cmd *cobra.Command, args []string) error { if len(args) == 0 { diff --git a/pkg/kwokctl/cmd/port_forward/port_forward.go b/pkg/kwokctl/cmd/port_forward/port_forward.go new file mode 100644 index 0000000000..2630a0e7c7 --- /dev/null +++ b/pkg/kwokctl/cmd/port_forward/port_forward.go @@ -0,0 +1,101 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package port_forward implements the `port-forward` command +package port_forward + +import ( + "context" + "errors" + "fmt" + "os" + "path" + "strconv" + "strings" + + "github.com/spf13/cobra" + + "sigs.k8s.io/kwok/pkg/config" + "sigs.k8s.io/kwok/pkg/kwokctl/runtime" + "sigs.k8s.io/kwok/pkg/log" +) + +type flagpole struct { + Name string +} + +// NewCommand returns a new cobra.Command for forwarding the port +func NewCommand(ctx context.Context) *cobra.Command { + flags := &flagpole{} + cmd := &cobra.Command{ + Args: cobra.ExactArgs(2), + Use: "port-forward [component] [local-port]:[port-name]", + Short: "Forward one local ports to a component", + RunE: func(cmd *cobra.Command, args []string) error { + flags.Name = config.DefaultCluster + return runE(ctx, flags, args) + }, + } + return cmd +} + +func runE(ctx context.Context, flags *flagpole, args []string) error { + name := config.ClusterName(flags.Name) + workdir := path.Join(config.ClustersDir, flags.Name) + + logger := log.FromContext(ctx).With("cluster", flags.Name) + ctx = log.NewContext(ctx, logger) + + rt, err := runtime.DefaultRegistry.Load(ctx, name, workdir) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + logger.Warn("Cluster does not exist") + } + return err + } + + hostPort, containerPort, err := splitParts(args[1]) + if err != nil { + return err + } + + port, err := strconv.ParseUint(hostPort, 0, 0) + if err != nil { + return err + } + + cancel, err := rt.PortForward(ctx, args[0], containerPort, uint32(port)) + if err != nil { + return err + } + + defer cancel() + <-ctx.Done() + + return nil +} + +func splitParts(rawport string) (hostPort string, containerPort string, err error) { + parts := strings.Split(rawport, ":") + n := len(parts) + + switch n { + case 2: + return parts[0], parts[1], nil + default: + return "", "", fmt.Errorf("unsupported port format: %s", rawport) + } +} diff --git a/pkg/kwokctl/cmd/root.go b/pkg/kwokctl/cmd/root.go index 303470ccec..01f847ac68 100644 --- a/pkg/kwokctl/cmd/root.go +++ b/pkg/kwokctl/cmd/root.go @@ -32,6 +32,7 @@ import ( "sigs.k8s.io/kwok/pkg/kwokctl/cmd/hack" "sigs.k8s.io/kwok/pkg/kwokctl/cmd/kubectl" "sigs.k8s.io/kwok/pkg/kwokctl/cmd/logs" + "sigs.k8s.io/kwok/pkg/kwokctl/cmd/port_forward" "sigs.k8s.io/kwok/pkg/kwokctl/cmd/scale" "sigs.k8s.io/kwok/pkg/kwokctl/cmd/snapshot" "sigs.k8s.io/kwok/pkg/kwokctl/cmd/start" @@ -72,6 +73,7 @@ func NewCommand(ctx context.Context) *cobra.Command { snapshot.NewCommand(ctx), export.NewCommand(ctx), hack.NewCommand(ctx), + port_forward.NewCommand(ctx), ) return cmd } diff --git a/pkg/kwokctl/cmd/snapshot/record/record.go b/pkg/kwokctl/cmd/snapshot/record/record.go index b615213aa1..b717130dda 100644 --- a/pkg/kwokctl/cmd/snapshot/record/record.go +++ b/pkg/kwokctl/cmd/snapshot/record/record.go @@ -89,10 +89,11 @@ func runE(ctx context.Context, flags *flagpole) error { return err } - etcdclient, err := rt.GetEtcdClient(ctx) + etcdclient, cancel, err := rt.GetEtcdClient(ctx) if err != nil { return err } + defer cancel() clientset, err := rt.GetClientset(ctx) if err != nil { diff --git a/pkg/kwokctl/cmd/snapshot/replay/replay.go b/pkg/kwokctl/cmd/snapshot/replay/replay.go index 5ea229e15b..bf3aea6ec3 100644 --- a/pkg/kwokctl/cmd/snapshot/replay/replay.go +++ b/pkg/kwokctl/cmd/snapshot/replay/replay.go @@ -121,10 +121,11 @@ func runE(ctx context.Context, flags *flagpole) error { } }() - etcdclient, err := rt.GetEtcdClient(ctx) + etcdclient, cancel, err := rt.GetEtcdClient(ctx) if err != nil { return err } + defer cancel() clientset, err := rt.GetClientset(ctx) if err != nil { diff --git a/pkg/kwokctl/components/dashboard.go b/pkg/kwokctl/components/dashboard.go index ed428fc343..7b3d145458 100644 --- a/pkg/kwokctl/components/dashboard.go +++ b/pkg/kwokctl/components/dashboard.go @@ -112,6 +112,14 @@ func BuildDashboardComponent(conf BuildDashboardComponentConfig) (component inte "--kubeconfig="+conf.KubeconfigPath, "--insecure-port="+format.String(conf.Port), ) + ports = append(ports, + internalversion.Port{ + Name: "http", + Port: conf.Port, + HostPort: 0, + Protocol: internalversion.ProtocolTCP, + }, + ) } component = internalversion.Component{ diff --git a/pkg/kwokctl/components/etcd.go b/pkg/kwokctl/components/etcd.go index f1de73dacf..e9cd2f4f15 100644 --- a/pkg/kwokctl/components/etcd.go +++ b/pkg/kwokctl/components/etcd.go @@ -44,17 +44,6 @@ type BuildEtcdComponentConfig struct { // BuildEtcdComponent builds an etcd component. func BuildEtcdComponent(conf BuildEtcdComponentConfig) (component internalversion.Component, err error) { - exposePeerPort := true - if conf.PeerPort == 0 { - conf.PeerPort = 2380 - exposePeerPort = false - } - exposePort := true - if conf.Port == 0 { - conf.Port = 2379 - exposePort = false - } - var volumes []internalversion.Volume var ports []internalversion.Port @@ -78,24 +67,21 @@ func BuildEtcdComponent(conf BuildEtcdComponentConfig) (component internalversio "--data-dir=/etcd-data", ) - if exposePeerPort { - ports = append( - ports, - internalversion.Port{ - HostPort: conf.PeerPort, - Port: 2380, - }, - ) - } - if exposePort { - ports = append( - ports, - internalversion.Port{ - HostPort: conf.Port, - Port: 2379, - }, - ) - } + ports = append( + ports, + internalversion.Port{ + Name: "peer-http", + HostPort: conf.PeerPort, + Port: 2380, + Protocol: internalversion.ProtocolTCP, + }, + internalversion.Port{ + Name: "http", + HostPort: conf.Port, + Port: 2379, + Protocol: internalversion.ProtocolTCP, + }, + ) etcdArgs = append(etcdArgs, "--initial-advertise-peer-urls=http://"+conf.BindAddress+":2380", "--listen-peer-urls=http://"+conf.BindAddress+":2380", @@ -112,6 +98,23 @@ func BuildEtcdComponent(conf BuildEtcdComponentConfig) (component internalversio } else { etcdPeerPortStr := format.String(conf.PeerPort) etcdClientPortStr := format.String(conf.Port) + + ports = append( + ports, + internalversion.Port{ + Name: "peer-http", + HostPort: 0, + Port: conf.PeerPort, + Protocol: internalversion.ProtocolTCP, + }, + internalversion.Port{ + Name: "http", + HostPort: 0, + Port: conf.Port, + Protocol: internalversion.ProtocolTCP, + }, + ) + etcdArgs = append(etcdArgs, "--data-dir="+conf.DataPath, "--initial-advertise-peer-urls=http://"+conf.BindAddress+":"+etcdPeerPortStr, diff --git a/pkg/kwokctl/components/jaeger.go b/pkg/kwokctl/components/jaeger.go index 7c5e7d9382..f06c8a2d2a 100644 --- a/pkg/kwokctl/components/jaeger.go +++ b/pkg/kwokctl/components/jaeger.go @@ -46,16 +46,41 @@ func BuildJaegerComponent(conf BuildJaegerComponentConfig) (component internalve var ports []internalversion.Port if GetRuntimeMode(conf.Runtime) != RuntimeModeNative { - ports = []internalversion.Port{ - { + ports = append( + ports, + internalversion.Port{ + Name: "http", HostPort: conf.Port, Port: 16686, + Protocol: internalversion.ProtocolTCP, }, - } + internalversion.Port{ + Name: "otlp-grpc", + HostPort: conf.OtlpGrpcPort, + Port: 4317, + Protocol: internalversion.ProtocolTCP, + }, + ) jaegerArgs = append(jaegerArgs, "--query.http-server.host-port="+conf.BindAddress+":16686", + "--collector.otlp.grpc.host-port="+net.LocalAddress+":4317", ) } else { + ports = append( + ports, + internalversion.Port{ + Name: "http", + HostPort: 0, + Port: conf.Port, + Protocol: internalversion.ProtocolTCP, + }, + internalversion.Port{ + Name: "otlp-grpc", + HostPort: 0, + Port: conf.OtlpGrpcPort, + Protocol: internalversion.ProtocolTCP, + }, + ) jaegerArgs = append(jaegerArgs, "--query.http-server.host-port="+conf.BindAddress+":"+format.String(conf.Port), "--collector.otlp.grpc.host-port="+net.LocalAddress+":"+format.String(conf.OtlpGrpcPort), diff --git a/pkg/kwokctl/components/kube_apiserver.go b/pkg/kwokctl/components/kube_apiserver.go index b82d348a70..ec7fe4f429 100644 --- a/pkg/kwokctl/components/kube_apiserver.go +++ b/pkg/kwokctl/components/kube_apiserver.go @@ -139,12 +139,15 @@ func BuildKubeApiserverComponent(conf BuildKubeApiserverComponentConfig) (compon } if GetRuntimeMode(conf.Runtime) != RuntimeModeNative { - ports = []internalversion.Port{ - { + ports = append( + ports, + internalversion.Port{ + Name: "https", HostPort: conf.Port, Port: 6443, + Protocol: internalversion.ProtocolTCP, }, - } + ) volumes = append(volumes, internalversion.Volume{ HostPath: conf.CaCertPath, @@ -183,6 +186,15 @@ func BuildKubeApiserverComponent(conf BuildKubeApiserverComponentConfig) (compon InsecureSkipVerify: true, } } else { + ports = append( + ports, + internalversion.Port{ + Name: "https", + HostPort: 0, + Port: conf.Port, + Protocol: internalversion.ProtocolTCP, + }, + ) kubeApiserverArgs = append(kubeApiserverArgs, "--bind-address="+conf.BindAddress, "--secure-port="+format.String(conf.Port), @@ -206,12 +218,15 @@ func BuildKubeApiserverComponent(conf BuildKubeApiserverComponentConfig) (compon } } else { if GetRuntimeMode(conf.Runtime) != RuntimeModeNative { - ports = []internalversion.Port{ - { + ports = append( + ports, + internalversion.Port{ + Name: "http", HostPort: conf.Port, Port: 8080, + Protocol: internalversion.ProtocolTCP, }, - } + ) kubeApiserverArgs = append(kubeApiserverArgs, "--insecure-bind-address="+conf.BindAddress, @@ -223,6 +238,15 @@ func BuildKubeApiserverComponent(conf BuildKubeApiserverComponentConfig) (compon Path: "/metrics", } } else { + ports = append( + ports, + internalversion.Port{ + Name: "http", + HostPort: 0, + Port: conf.Port, + Protocol: internalversion.ProtocolTCP, + }, + ) kubeApiserverArgs = append(kubeApiserverArgs, "--insecure-bind-address="+conf.BindAddress, "--insecure-port="+format.String(conf.Port), diff --git a/pkg/kwokctl/components/kube_controller_manager.go b/pkg/kwokctl/components/kube_controller_manager.go index 09535a9729..0a005df843 100644 --- a/pkg/kwokctl/components/kube_controller_manager.go +++ b/pkg/kwokctl/components/kube_controller_manager.go @@ -120,15 +120,15 @@ func BuildKubeControllerManagerComponent(conf BuildKubeControllerManagerComponen "--bind-address="+conf.BindAddress, "--secure-port=10257", ) - if conf.Port > 0 { - ports = append( - ports, - internalversion.Port{ - HostPort: conf.Port, - Port: 10257, - }, - ) - } + ports = append( + ports, + internalversion.Port{ + Name: "https", + HostPort: conf.Port, + Port: 10257, + Protocol: internalversion.ProtocolTCP, + }, + ) metric = &internalversion.ComponentMetric{ Scheme: "https", Host: conf.ProjectName + "-" + consts.ComponentKubeControllerManager + ":10257", @@ -142,6 +142,15 @@ func BuildKubeControllerManagerComponent(conf BuildKubeControllerManagerComponen "--bind-address="+conf.BindAddress, "--secure-port="+format.String(conf.Port), ) + ports = append( + ports, + internalversion.Port{ + Name: "https", + HostPort: 0, + Port: conf.Port, + Protocol: internalversion.ProtocolTCP, + }, + ) metric = &internalversion.ComponentMetric{ Scheme: "https", Host: net.LocalAddress + ":" + format.String(conf.Port), @@ -162,15 +171,15 @@ func BuildKubeControllerManagerComponent(conf BuildKubeControllerManagerComponen "--address="+conf.BindAddress, "--port=10252", ) - if conf.Port > 0 { - ports = append( - ports, - internalversion.Port{ - HostPort: conf.Port, - Port: 10252, - }, - ) - } + ports = append( + ports, + internalversion.Port{ + Name: "http", + HostPort: conf.Port, + Port: 10252, + Protocol: internalversion.ProtocolTCP, + }, + ) metric = &internalversion.ComponentMetric{ Scheme: "http", Host: conf.ProjectName + "-" + consts.ComponentKubeControllerManager + ":10252", @@ -181,6 +190,15 @@ func BuildKubeControllerManagerComponent(conf BuildKubeControllerManagerComponen "--address="+conf.BindAddress, "--port="+format.String(conf.Port), ) + ports = append( + ports, + internalversion.Port{ + Name: "http", + HostPort: 0, + Port: conf.Port, + Protocol: internalversion.ProtocolTCP, + }, + ) metric = &internalversion.ComponentMetric{ Scheme: "http", Host: net.LocalAddress + ":" + format.String(conf.Port), diff --git a/pkg/kwokctl/components/kube_scheduler.go b/pkg/kwokctl/components/kube_scheduler.go index 07316fdaa7..115ff4d90c 100644 --- a/pkg/kwokctl/components/kube_scheduler.go +++ b/pkg/kwokctl/components/kube_scheduler.go @@ -124,15 +124,15 @@ func BuildKubeSchedulerComponent(conf BuildKubeSchedulerComponentConfig) (compon "--bind-address="+conf.BindAddress, "--secure-port=10259", ) - if conf.Port != 0 { - ports = append( - ports, - internalversion.Port{ - HostPort: conf.Port, - Port: 10259, - }, - ) - } + ports = append( + ports, + internalversion.Port{ + Name: "https", + HostPort: conf.Port, + Port: 10259, + Protocol: internalversion.ProtocolTCP, + }, + ) metric = &internalversion.ComponentMetric{ Scheme: "https", Host: conf.ProjectName + "-" + consts.ComponentKubeScheduler + ":10259", @@ -146,6 +146,15 @@ func BuildKubeSchedulerComponent(conf BuildKubeSchedulerComponentConfig) (compon "--bind-address="+conf.BindAddress, "--secure-port="+format.String(conf.Port), ) + ports = append( + ports, + internalversion.Port{ + Name: "https", + HostPort: 0, + Port: conf.Port, + Protocol: internalversion.ProtocolTCP, + }, + ) metric = &internalversion.ComponentMetric{ Scheme: "https", Host: net.LocalAddress + ":" + format.String(conf.Port), @@ -165,15 +174,15 @@ func BuildKubeSchedulerComponent(conf BuildKubeSchedulerComponentConfig) (compon "--address="+conf.BindAddress, "--port=10251", ) - if conf.Port != 0 { - ports = append( - ports, - internalversion.Port{ - HostPort: conf.Port, - Port: 10251, - }, - ) - } + ports = append( + ports, + internalversion.Port{ + Name: "http", + HostPort: conf.Port, + Port: 10251, + Protocol: internalversion.ProtocolTCP, + }, + ) metric = &internalversion.ComponentMetric{ Scheme: "http", Host: conf.ProjectName + "-" + consts.ComponentKubeScheduler + ":10251", @@ -184,6 +193,15 @@ func BuildKubeSchedulerComponent(conf BuildKubeSchedulerComponentConfig) (compon "--address="+conf.BindAddress, "--port="+format.String(conf.Port), ) + ports = append( + ports, + internalversion.Port{ + Name: "http", + HostPort: 0, + Port: conf.Port, + Protocol: internalversion.ProtocolTCP, + }, + ) metric = &internalversion.ComponentMetric{ Scheme: "http", Host: net.LocalAddress + ":" + format.String(conf.Port), diff --git a/pkg/kwokctl/components/kubectl_proxy.go b/pkg/kwokctl/components/kubectl_proxy.go index 429e7d9c11..e1142d4d87 100644 --- a/pkg/kwokctl/components/kubectl_proxy.go +++ b/pkg/kwokctl/components/kubectl_proxy.go @@ -80,17 +80,29 @@ func BuildKubectlProxyComponent(conf BuildKubectlProxyComponentConfig) (componen "--kubeconfig=/root/.kube/config", "--port=8001", ) - ports = []internalversion.Port{ - { + ports = append( + ports, + internalversion.Port{ + Name: "http", HostPort: conf.Port, Port: 8001, + Protocol: internalversion.ProtocolTCP, }, - } + ) } else { kubectlProxyArgs = append(kubectlProxyArgs, "--kubeconfig="+conf.KubeconfigPath, "--port="+format.String(conf.Port), ) + ports = append( + ports, + internalversion.Port{ + Name: "http", + HostPort: 0, + Port: conf.Port, + Protocol: internalversion.ProtocolTCP, + }, + ) } if conf.Verbosity != log.LevelInfo { diff --git a/pkg/kwokctl/components/kwok_controller.go b/pkg/kwokctl/components/kwok_controller.go index 99d34021ab..db25b1d387 100644 --- a/pkg/kwokctl/components/kwok_controller.go +++ b/pkg/kwokctl/components/kwok_controller.go @@ -96,14 +96,15 @@ func BuildKwokControllerComponent(conf BuildKwokControllerComponentConfig) (comp }, ) - if conf.Port != 0 { - ports = append(ports, - internalversion.Port{ - HostPort: conf.Port, - Port: 10247, - }, - ) - } + ports = append( + ports, + internalversion.Port{ + Name: "http", + HostPort: conf.Port, + Port: 10247, + Protocol: internalversion.ProtocolTCP, + }, + ) kwokControllerArgs = append(kwokControllerArgs, "--kubeconfig=/root/.kube/config", "--config=/root/.kwok/kwok.yaml", @@ -116,6 +117,15 @@ func BuildKwokControllerComponent(conf BuildKwokControllerComponentConfig) (comp "--node-lease-duration-seconds="+format.String(conf.NodeLeaseDurationSeconds), ) } else { + ports = append( + ports, + internalversion.Port{ + Name: "http", + HostPort: 0, + Port: conf.Port, + Protocol: internalversion.ProtocolTCP, + }, + ) kwokControllerArgs = append(kwokControllerArgs, "--kubeconfig="+conf.KubeconfigPath, "--config="+conf.ConfigPath, diff --git a/pkg/kwokctl/components/metrics_server.go b/pkg/kwokctl/components/metrics_server.go index 31a43f7b7a..c2aaec8fab 100644 --- a/pkg/kwokctl/components/metrics_server.go +++ b/pkg/kwokctl/components/metrics_server.go @@ -99,14 +99,15 @@ func BuildMetricsServerComponent(conf BuildMetricsServerComponentConfig) (compon "--tls-cert-file=/etc/kubernetes/pki/admin.crt", "--tls-private-key-file=/etc/kubernetes/pki/admin.key", ) - if conf.Port != 0 { - ports = []internalversion.Port{ - { - HostPort: conf.Port, - Port: 4443, - }, - } - } + ports = append( + ports, + internalversion.Port{ + Name: "https", + HostPort: conf.Port, + Port: 4443, + Protocol: internalversion.ProtocolTCP, + }, + ) metric = &internalversion.ComponentMetric{ Scheme: "https", Host: metricsHost, @@ -126,7 +127,15 @@ func BuildMetricsServerComponent(conf BuildMetricsServerComponentConfig) (compon "--tls-cert-file="+conf.AdminCertPath, "--tls-private-key-file="+conf.AdminKeyPath, ) - + ports = append( + ports, + internalversion.Port{ + Name: "https", + HostPort: 0, + Port: conf.Port, + Protocol: internalversion.ProtocolTCP, + }, + ) metric = &internalversion.ComponentMetric{ Scheme: "https", Host: metricsHost, diff --git a/pkg/kwokctl/components/prometheus.go b/pkg/kwokctl/components/prometheus.go index 31c0a66b0e..93e8093dd7 100644 --- a/pkg/kwokctl/components/prometheus.go +++ b/pkg/kwokctl/components/prometheus.go @@ -66,17 +66,29 @@ func BuildPrometheusComponent(conf BuildPrometheusComponentConfig) (component in ReadOnly: true, }, ) - ports = []internalversion.Port{ - { + ports = append( + ports, + internalversion.Port{ + Name: "http", HostPort: conf.Port, Port: 9090, + Protocol: internalversion.ProtocolTCP, }, - } + ) prometheusArgs = append(prometheusArgs, "--config.file=/etc/prometheus/prometheus.yaml", "--web.listen-address="+conf.BindAddress+":9090", ) } else { + ports = append( + ports, + internalversion.Port{ + Name: "http", + HostPort: 0, + Port: conf.Port, + Protocol: internalversion.ProtocolTCP, + }, + ) prometheusArgs = append(prometheusArgs, "--config.file="+conf.ConfigPath, "--web.listen-address="+conf.BindAddress+":"+format.String(conf.Port), diff --git a/pkg/kwokctl/runtime/binary/cluster_port_forward.go b/pkg/kwokctl/runtime/binary/cluster_port_forward.go new file mode 100644 index 0000000000..e82731a972 --- /dev/null +++ b/pkg/kwokctl/runtime/binary/cluster_port_forward.go @@ -0,0 +1,93 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package binary + +import ( + "context" + "fmt" + "net" + "strconv" + + "sigs.k8s.io/kwok/pkg/apis/internalversion" + "sigs.k8s.io/kwok/pkg/log" + utilsnet "sigs.k8s.io/kwok/pkg/utils/net" + "sigs.k8s.io/kwok/pkg/utils/slices" +) + +// PortForward expose the port of the component +func (c *Cluster) PortForward(ctx context.Context, name string, portOrName string, hostPort uint32) (cancel func(), retErr error) { + targetPort, err := strconv.ParseUint(portOrName, 0, 0) + if err != nil { + component, err := c.GetComponent(ctx, name) + if err != nil { + return nil, err + } + port, ok := slices.Find(component.Ports, func(port internalversion.Port) bool { + return port.Name == portOrName && port.Protocol == internalversion.ProtocolTCP + }) + if !ok { + return nil, fmt.Errorf("port %q not found", portOrName) + } + targetPort = uint64(port.Port) + } + + listener, err := net.Listen("tcp", fmt.Sprintf(":%d", hostPort)) + if err != nil { + return nil, err + } + defer func() { + if retErr != nil { + _ = listener.Close() + } + }() + + logger := log.FromContext(ctx) + cancel = func() { + _ = listener.Close() + } + + go func() { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + for { + conn, err := listener.Accept() + if err != nil { + logger.Error("accepting connection", err) + return + } + + target, err := net.Dial("tcp", fmt.Sprintf(":%d", targetPort)) + if err != nil { + _ = conn.Close() + logger.Error("failed to connect", err, "port", targetPort) + return + } + go func() { + defer func() { + _ = target.Close() + _ = conn.Close() + }() + err = utilsnet.Tunnel(ctx, conn, target, nil, nil) + if err != nil { + logger.Warn("failed tunneling port", "err", err) + } + }() + } + }() + + return cancel, nil +} diff --git a/pkg/kwokctl/runtime/binary/cluster_snapshot.go b/pkg/kwokctl/runtime/binary/cluster_snapshot.go index 93ba405052..d1194d7b8d 100644 --- a/pkg/kwokctl/runtime/binary/cluster_snapshot.go +++ b/pkg/kwokctl/runtime/binary/cluster_snapshot.go @@ -20,8 +20,11 @@ import ( "context" "sigs.k8s.io/kwok/pkg/consts" + "sigs.k8s.io/kwok/pkg/kwokctl/etcd" "sigs.k8s.io/kwok/pkg/kwokctl/runtime" "sigs.k8s.io/kwok/pkg/log" + "sigs.k8s.io/kwok/pkg/utils/format" + "sigs.k8s.io/kwok/pkg/utils/net" "sigs.k8s.io/kwok/pkg/utils/wait" ) @@ -147,3 +150,21 @@ func (c *Cluster) SnapshotRestoreWithYAML(ctx context.Context, path string, conf } return nil } + +// GetEtcdClient returns the etcd client of cluster +func (c *Cluster) GetEtcdClient(ctx context.Context) (etcd.Client, func(), error) { + config, err := c.Config(ctx) + if err != nil { + return nil, nil, err + } + conf := &config.Options + + cli, err := etcd.NewClient(etcd.ClientConfig{ + Endpoints: []string{"http://" + net.LocalAddress + ":" + format.String(conf.EtcdPort)}, + }) + if err != nil { + return nil, nil, err + } + + return cli, func() {}, nil +} diff --git a/pkg/kwokctl/runtime/cluster_snapshot.go b/pkg/kwokctl/runtime/cluster_snapshot.go index 59c32f5357..6f718ad703 100644 --- a/pkg/kwokctl/runtime/cluster_snapshot.go +++ b/pkg/kwokctl/runtime/cluster_snapshot.go @@ -23,12 +23,9 @@ import ( "strings" "sigs.k8s.io/kwok/pkg/kwokctl/dryrun" - "sigs.k8s.io/kwok/pkg/kwokctl/etcd" "sigs.k8s.io/kwok/pkg/kwokctl/snapshot" "sigs.k8s.io/kwok/pkg/log" "sigs.k8s.io/kwok/pkg/utils/client" - "sigs.k8s.io/kwok/pkg/utils/format" - "sigs.k8s.io/kwok/pkg/utils/net" "sigs.k8s.io/kwok/pkg/utils/yaml" ) @@ -137,16 +134,3 @@ func (c *Cluster) SnapshotRestoreWithYAML(ctx context.Context, path string, conf return nil } - -// GetEtcdClient returns the etcd client of cluster -func (c *Cluster) GetEtcdClient(ctx context.Context) (etcd.Client, error) { - config, err := c.Config(ctx) - if err != nil { - return nil, err - } - conf := &config.Options - - return etcd.NewClient(etcd.ClientConfig{ - Endpoints: []string{"http://" + net.LocalAddress + ":" + format.String(conf.EtcdPort)}, - }) -} diff --git a/pkg/kwokctl/runtime/compose/cluster.go b/pkg/kwokctl/runtime/compose/cluster.go index 8ebb7d281b..319366c0bb 100644 --- a/pkg/kwokctl/runtime/compose/cluster.go +++ b/pkg/kwokctl/runtime/compose/cluster.go @@ -283,7 +283,6 @@ func (c *Cluster) Install(ctx context.Context) error { err = c.setupPorts(ctx, env.usedPorts, &env.kwokctlConfig.Options.KubeApiserverPort, - &env.kwokctlConfig.Options.EtcdPort, ) if err != nil { return err @@ -775,13 +774,14 @@ func (c *Cluster) addJaeger(ctx context.Context, env *env) (err error) { } jaegerComponent, err := components.BuildJaegerComponent(components.BuildJaegerComponentConfig{ - Runtime: conf.Runtime, - Workdir: env.workdir, - Image: conf.JaegerImage, - Version: jaegerVersion, - BindAddress: net.PublicAddress, - Port: conf.JaegerPort, - Verbosity: env.verbosity, + Runtime: conf.Runtime, + Workdir: env.workdir, + Image: conf.JaegerImage, + Version: jaegerVersion, + BindAddress: net.PublicAddress, + Port: conf.JaegerPort, + OtlpGrpcPort: conf.JaegerOtlpGrpcPort, + Verbosity: env.verbosity, }) if err != nil { return err diff --git a/pkg/kwokctl/runtime/compose/cluster_port_forward.go b/pkg/kwokctl/runtime/compose/cluster_port_forward.go new file mode 100644 index 0000000000..a9a02bd1ca --- /dev/null +++ b/pkg/kwokctl/runtime/compose/cluster_port_forward.go @@ -0,0 +1,141 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package compose + +import ( + "context" + "fmt" + "io" + "net" + "strconv" + "time" + + "sigs.k8s.io/kwok/pkg/apis/internalversion" + "sigs.k8s.io/kwok/pkg/consts" + "sigs.k8s.io/kwok/pkg/log" + "sigs.k8s.io/kwok/pkg/utils/exec" + "sigs.k8s.io/kwok/pkg/utils/format" + "sigs.k8s.io/kwok/pkg/utils/slices" +) + +// PortForward expose the port of the component +func (c *Cluster) PortForward(ctx context.Context, name string, portOrName string, hostPort uint32) (cancel func(), retErr error) { + kwokController, err := c.GetComponent(ctx, consts.ComponentKwokController) + if err != nil { + return nil, err + } + + targetPort, err := strconv.ParseUint(portOrName, 0, 0) + if err != nil { + component, err := c.GetComponent(ctx, name) + if err != nil { + return nil, err + } + port, ok := slices.Find(component.Ports, func(port internalversion.Port) bool { + return port.Name == portOrName && port.Protocol == internalversion.ProtocolTCP + }) + if !ok { + return nil, fmt.Errorf("port %q not found", portOrName) + } + targetPort = uint64(port.Port) + } + + logger := log.FromContext(ctx) + + tempContainerName := "temp-port-forward-proxy-" + format.String(time.Now().Unix()) + + args := []string{ + "run", + "--rm", + "-i", + "--pull=never", + "--network=" + c.networkName(), + "--name=" + tempContainerName, + "--entrypoint=/bin/sh", + } + args = append(args, c.labelArgs()...) + args = append(args, kwokController.Image) + + r, w := io.Pipe() + command, err := exec.Command(exec.WithWait(exec.WithIOStreams(ctx, exec.IOStreams{In: r}), false), + c.runtime, args...) + if err != nil { + return nil, fmt.Errorf("running command: %w", err) + } + + cleanTempContainer := func() { + _, _ = w.Write([]byte("exit\n")) + _ = command.Cancel() + + if c.runtime == consts.RuntimeTypeNerdctl || + c.runtime == consts.RuntimeTypeLima || + c.runtime == consts.RuntimeTypeFinch { + _, err := exec.Command(context.Background(), c.runtime, "rm", "--force", tempContainerName) + if err != nil { + logger.Error("Remove temporary port-forward container", err) + } + } + } + + defer func() { + if retErr != nil { + cleanTempContainer() + } + }() + + listener, err := net.Listen("tcp", fmt.Sprintf(":%d", hostPort)) + if err != nil { + return nil, err + } + defer func() { + if retErr != nil { + _ = listener.Close() + } + }() + + cancel = func() { + _ = listener.Close() + cleanTempContainer() + } + + go func() { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + for { + conn, err := listener.Accept() + if err != nil { + logger.Error("accepting connection", err) + return + } + + go func() { + defer func() { + _ = conn.Close() + }() + err := c.Exec(exec.WithReadWriter(ctx, conn), + c.runtime, "exec", "-i", tempContainerName, + "nc", c.Name()+"-"+name, format.String(targetPort)) + if err != nil { + logger.Warn("failed tunneling port", "err", err) + } + }() + } + }() + + return cancel, nil +} diff --git a/pkg/kwokctl/runtime/compose/cluster_snapshot.go b/pkg/kwokctl/runtime/compose/cluster_snapshot.go index 0af38a67d0..76db5ac28e 100644 --- a/pkg/kwokctl/runtime/compose/cluster_snapshot.go +++ b/pkg/kwokctl/runtime/compose/cluster_snapshot.go @@ -20,8 +20,11 @@ import ( "context" "sigs.k8s.io/kwok/pkg/consts" + "sigs.k8s.io/kwok/pkg/kwokctl/etcd" "sigs.k8s.io/kwok/pkg/kwokctl/runtime" "sigs.k8s.io/kwok/pkg/log" + "sigs.k8s.io/kwok/pkg/utils/format" + "sigs.k8s.io/kwok/pkg/utils/net" ) // SnapshotSave save the snapshot of cluster @@ -216,3 +219,41 @@ func (c *Cluster) SnapshotRestoreWithYAML(ctx context.Context, path string, conf } return nil } + +// GetEtcdClient returns the etcd client of cluster +func (c *Cluster) GetEtcdClient(ctx context.Context) (etcd.Client, func(), error) { + config, err := c.Config(ctx) + if err != nil { + return nil, nil, err + } + conf := &config.Options + + if conf.EtcdPort == 0 { + unused, err := net.GetUnusedPort(ctx, nil) + if err != nil { + return nil, nil, err + } + + cli, err := etcd.NewClient(etcd.ClientConfig{ + Endpoints: []string{"http://" + net.LocalAddress + ":" + format.String(unused)}, + }) + if err != nil { + return nil, nil, err + } + + cancel, err := c.PortForward(ctx, consts.ComponentEtcd, "http", unused) + if err != nil { + return nil, nil, err + } + return cli, cancel, nil + } + + cli, err := etcd.NewClient(etcd.ClientConfig{ + Endpoints: []string{"http://" + net.LocalAddress + ":" + format.String(conf.EtcdPort)}, + }) + if err != nil { + return nil, nil, err + } + + return cli, func() {}, nil +} diff --git a/pkg/kwokctl/runtime/compose/self_compose.go b/pkg/kwokctl/runtime/compose/self_compose.go index 8638cb8fa9..cf04179106 100644 --- a/pkg/kwokctl/runtime/compose/self_compose.go +++ b/pkg/kwokctl/runtime/compose/self_compose.go @@ -252,6 +252,9 @@ func (c *Cluster) createComponent(ctx context.Context, componentName string) err args = append(args, c.labelArgs()...) for _, port := range component.Ports { + if port.HostPort == 0 { + continue + } protocol := port.Protocol if protocol == "" { protocol = internalversion.ProtocolTCP diff --git a/pkg/kwokctl/runtime/config.go b/pkg/kwokctl/runtime/config.go index d8d0141c1c..c16a94a782 100644 --- a/pkg/kwokctl/runtime/config.go +++ b/pkg/kwokctl/runtime/config.go @@ -73,6 +73,9 @@ type Runtime interface { // InspectComponent inspect the component InspectComponent(ctx context.Context, name string) (ComponentStatus, error) + // PortForward expose the port of the component + PortForward(ctx context.Context, name string, portOrName string, hostPort uint32) (cancel func(), retErr error) + // Ready check the cluster is ready Ready(ctx context.Context) (bool, error) @@ -143,7 +146,7 @@ type Runtime interface { GetClientset(ctx context.Context) (client.Clientset, error) // GetEtcdClient returns the etcd client of cluster - GetEtcdClient(ctx context.Context) (etcd.Client, error) + GetEtcdClient(ctx context.Context) (etcd.Client, func(), error) } type SnapshotSaveWithYAMLConfig struct { diff --git a/pkg/kwokctl/runtime/kind/cluster.go b/pkg/kwokctl/runtime/kind/cluster.go index 87e1e2cc84..0522a27e1e 100644 --- a/pkg/kwokctl/runtime/kind/cluster.go +++ b/pkg/kwokctl/runtime/kind/cluster.go @@ -252,7 +252,6 @@ func (c *Cluster) Install(ctx context.Context) error { err = c.setupPorts(ctx, env.usedPorts, &env.kwokctlConfig.Options.KubeApiserverPort, - &env.kwokctlConfig.Options.EtcdPort, ) if err != nil { return err @@ -889,13 +888,14 @@ func (c *Cluster) addJaeger(ctx context.Context, env *env) (err error) { } jaegerComponent, err := components.BuildJaegerComponent(components.BuildJaegerComponentConfig{ - Runtime: conf.Runtime, - Workdir: env.workdir, - Image: conf.JaegerImage, - Version: jaegerVersion, - BindAddress: net.PublicAddress, - Port: 16686, - Verbosity: env.verbosity, + Runtime: conf.Runtime, + Workdir: env.workdir, + Image: conf.JaegerImage, + Version: jaegerVersion, + BindAddress: net.PublicAddress, + Port: 16686, + OtlpGrpcPort: 4317, + Verbosity: env.verbosity, }) if err != nil { return err diff --git a/pkg/kwokctl/runtime/kind/cluster_port_forward.go b/pkg/kwokctl/runtime/kind/cluster_port_forward.go new file mode 100644 index 0000000000..69cd851bd9 --- /dev/null +++ b/pkg/kwokctl/runtime/kind/cluster_port_forward.go @@ -0,0 +1,90 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kind + +import ( + "context" + "fmt" + "net" + "strconv" + + "sigs.k8s.io/kwok/pkg/apis/internalversion" + "sigs.k8s.io/kwok/pkg/log" + "sigs.k8s.io/kwok/pkg/utils/exec" + "sigs.k8s.io/kwok/pkg/utils/slices" +) + +// PortForward expose the port of the component +func (c *Cluster) PortForward(ctx context.Context, name string, portOrName string, hostPort uint32) (cancel func(), retErr error) { + targetPort, err := strconv.ParseUint(portOrName, 0, 0) + if err != nil { + component, err := c.GetComponent(ctx, name) + if err != nil { + return nil, err + } + port, ok := slices.Find(component.Ports, func(port internalversion.Port) bool { + return port.Name == portOrName && port.Protocol == internalversion.ProtocolTCP + }) + if !ok { + return nil, fmt.Errorf("port %q not found", portOrName) + } + targetPort = uint64(port.Port) + } + + listener, err := net.Listen("tcp", fmt.Sprintf(":%d", hostPort)) + if err != nil { + return nil, err + } + defer func() { + if retErr != nil { + _ = listener.Close() + } + }() + + logger := log.FromContext(ctx) + cancel = func() { + _ = listener.Close() + } + + go func() { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + for { + conn, err := listener.Accept() + if err != nil { + logger.Error("accepting connection", err) + return + } + + go func() { + defer func() { + _ = conn.Close() + }() + + command := fmt.Sprintf(`{ cat <&3 & cat >&3; } 3<> /dev/tcp/127.0.0.1/%d`, targetPort) + err := c.Exec(exec.WithReadWriter(ctx, conn), + c.runtime, "exec", "-i", c.getClusterName(), + "bash", "-c", command) + if err != nil { + logger.Warn("failed tunneling port", "err", err) + } + }() + } + }() + + return cancel, nil +} diff --git a/pkg/kwokctl/runtime/kind/cluster_snapshot.go b/pkg/kwokctl/runtime/kind/cluster_snapshot.go index a0a06f9e45..2a6abf2ef1 100644 --- a/pkg/kwokctl/runtime/kind/cluster_snapshot.go +++ b/pkg/kwokctl/runtime/kind/cluster_snapshot.go @@ -156,21 +156,46 @@ func (c *Cluster) SnapshotRestoreWithYAML(ctx context.Context, path string, conf } // GetEtcdClient returns the etcd client of cluster -func (c *Cluster) GetEtcdClient(ctx context.Context) (etcd.Client, error) { +func (c *Cluster) GetEtcdClient(ctx context.Context) (etcd.Client, func(), error) { + certFile := c.GetWorkdirPath("pki/etcd/server.crt") + keyFile := c.GetWorkdirPath("pki/etcd/server.key") + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return nil, nil, err + } + config, err := c.Config(ctx) if err != nil { - return nil, err + return nil, nil, err } conf := &config.Options - certFile := c.GetWorkdirPath("pki/etcd/server.crt") - keyFile := c.GetWorkdirPath("pki/etcd/server.key") - cert, err := tls.LoadX509KeyPair(certFile, keyFile) - if err != nil { - return nil, err + if conf.EtcdPort == 0 { + unused, err := net.GetUnusedPort(ctx, nil) + if err != nil { + return nil, nil, err + } + + cli, err := etcd.NewClient(etcd.ClientConfig{ + Endpoints: []string{"https://" + net.LocalAddress + ":" + format.String(unused)}, + TLS: &tls.Config{ + Certificates: []tls.Certificate{cert}, + //nolint:gosec + InsecureSkipVerify: true, + }, + }) + if err != nil { + return nil, nil, err + } + + cancel, err := c.PortForward(ctx, consts.ComponentEtcd, "2379", unused) + if err != nil { + return nil, nil, err + } + return cli, cancel, nil } - return etcd.NewClient(etcd.ClientConfig{ + cli, err := etcd.NewClient(etcd.ClientConfig{ Endpoints: []string{"https://" + net.LocalAddress + ":" + format.String(conf.EtcdPort)}, TLS: &tls.Config{ Certificates: []tls.Certificate{cert}, @@ -178,4 +203,8 @@ func (c *Cluster) GetEtcdClient(ctx context.Context) (etcd.Client, error) { InsecureSkipVerify: true, }, }) + if err != nil { + return nil, nil, err + } + return cli, func() {}, nil } diff --git a/pkg/utils/exec/exec.go b/pkg/utils/exec/exec.go index b933c3a57a..d09ed1a31d 100644 --- a/pkg/utils/exec/exec.go +++ b/pkg/utils/exec/exec.go @@ -54,6 +54,8 @@ type Options struct { PipeStdin bool // Fork is true if the command should be forked. Fork bool + // Wait is true if the command wait. + Wait bool } func (e *Options) deepCopy() *Options { @@ -65,6 +67,7 @@ func (e *Options) deepCopy() *Options { IOStreams: e.IOStreams, PipeStdin: e.PipeStdin, Fork: e.Fork, + Wait: e.Wait, } } @@ -152,13 +155,23 @@ func WithAllWriteToErrOut(ctx context.Context) context.Context { func WithFork(ctx context.Context, fork bool) context.Context { ctx, opt := withExecOptions(ctx) opt.Fork = fork + opt.Wait = !fork + return ctx +} + +// WithWait returns a context with the given wait option. +func WithWait(ctx context.Context, wait bool) context.Context { + ctx, opt := withExecOptions(ctx) + opt.Wait = wait return ctx } func withExecOptions(ctx context.Context) (context.Context, *Options) { v := ctx.Value(optCtx(0)) if v == nil { - opt := &Options{} + opt := &Options{ + Wait: true, + } return context.WithValue(ctx, optCtx(0), opt), opt } opt := v.(*Options).deepCopy() @@ -169,7 +182,9 @@ func withExecOptions(ctx context.Context) (context.Context, *Options) { func GetExecOptions(ctx context.Context) *Options { v := ctx.Value(optCtx(0)) if v == nil { - return &Options{} + return &Options{ + Wait: true, + } } return v.(*Options).deepCopy() } @@ -227,7 +242,7 @@ func Command(ctx context.Context, name string, args ...string) (cmd *Cmd, err er return nil, fmt.Errorf("cmd start: %s %s: %w", name, strings.Join(args, " "), err) } - if !opt.Fork { + if opt.Wait { err = cmd.Wait() if err != nil { if buf, ok := cmd.Stderr.(*bytes.Buffer); ok { diff --git a/pkg/utils/net/tunnel.go b/pkg/utils/net/tunnel.go new file mode 100644 index 0000000000..c6f925de3a --- /dev/null +++ b/pkg/utils/net/tunnel.go @@ -0,0 +1,57 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package net + +import ( + "context" + "errors" + "io" +) + +// Tunnel create tunnels for two streams. +func Tunnel(ctx context.Context, c1, c2 io.ReadWriter, buf1, buf2 []byte) error { + errCh := make(chan error) + go func() { + _, err := io.CopyBuffer(c2, c1, buf1) + errCh <- err + }() + go func() { + _, err := io.CopyBuffer(c1, c2, buf2) + errCh <- err + }() + select { + case <-ctx.Done(): + // Do nothing + case err1 := <-errCh: + select { + case <-ctx.Done(): + if err1 != nil { + return err1 + } + // Do nothing + case err2 := <-errCh: + if err1 != nil { + return err1 + } + return err2 + } + } + if err := ctx.Err(); err != nil && !errors.Is(err, context.Canceled) { + return err + } + return nil +} diff --git a/site/content/en/docs/generated/kwokctl.md b/site/content/en/docs/generated/kwokctl.md index 8ed077e400..1a632cf613 100644 --- a/site/content/en/docs/generated/kwokctl.md +++ b/site/content/en/docs/generated/kwokctl.md @@ -27,6 +27,7 @@ kwokctl [command] [flags] * [kwokctl hack](kwokctl_hack.md) - [experimental] Hack [get, put, delete] resources in etcd without apiserver * [kwokctl kubectl](kwokctl_kubectl.md) - kubectl in cluster * [kwokctl logs](kwokctl_logs.md) - Logs one of [audit, etcd, kube-apiserver, kube-controller-manager, kube-scheduler, kwok-controller, dashboard, metrics-server, prometheus, jaeger] +* [kwokctl port-forward](kwokctl_port-forward.md) - Forward one local ports to a component * [kwokctl scale](kwokctl_scale.md) - Scale a resource in cluster * [kwokctl snapshot](kwokctl_snapshot.md) - Snapshot [save, restore, record, replay, export] one of cluster * [kwokctl start](kwokctl_start.md) - Start one of [cluster] diff --git a/site/content/en/docs/generated/kwokctl_logs.md b/site/content/en/docs/generated/kwokctl_logs.md index 2901a8fec7..9a6c726598 100644 --- a/site/content/en/docs/generated/kwokctl_logs.md +++ b/site/content/en/docs/generated/kwokctl_logs.md @@ -3,7 +3,7 @@ Logs one of [audit, etcd, kube-apiserver, kube-controller-manager, kube-scheduler, kwok-controller, dashboard, metrics-server, prometheus, jaeger] ``` -kwokctl logs [command] [flags] +kwokctl logs [component] [flags] ``` ### Options diff --git a/site/content/en/docs/generated/kwokctl_port-forward.md b/site/content/en/docs/generated/kwokctl_port-forward.md new file mode 100644 index 0000000000..44264b3920 --- /dev/null +++ b/site/content/en/docs/generated/kwokctl_port-forward.md @@ -0,0 +1,27 @@ +## kwokctl port-forward + +Forward one local ports to a component + +``` +kwokctl port-forward [component] [local-port]:[port-name] [flags] +``` + +### Options + +``` + -h, --help help for port-forward +``` + +### Options inherited from parent commands + +``` + -c, --config strings config path (default [~/.kwok/kwok.yaml]) + --dry-run Print the command that would be executed, but do not execute it + --name string cluster name (default "kwok") + -v, --v log-level number for the log level verbosity (DEBUG, INFO, WARN, ERROR) or (-4, 0, 4, 8) (default INFO) +``` + +### SEE ALSO + +* [kwokctl](kwokctl.md) - kwokctl is a tool to streamline the creation and management of clusters, with nodes simulated by kwok + diff --git a/test/e2e/kwokctl/binary/kubectl_test.go b/test/e2e/kwokctl/binary/kubectl_test.go index f53413345a..a7928cb580 100644 --- a/test/e2e/kwokctl/binary/kubectl_test.go +++ b/test/e2e/kwokctl/binary/kubectl_test.go @@ -40,6 +40,12 @@ func TestDryrunExportLogs(t *testing.T) { testEnv.Test(t, f0) } +func TestKwokctlPortForward(t *testing.T) { + f0 := e2e.CaseKwokctlPortForward(kwokctlPath, clusterName). + Feature() + testEnv.Test(t, f0) +} + func TestHack(t *testing.T) { f0 := e2e.CaseHack(kwokctlPath, clusterName, envconf.RandomName("node", 16)). Feature() diff --git a/test/e2e/kwokctl/docker/kubectl_test.go b/test/e2e/kwokctl/docker/kubectl_test.go index 4ef4d0569d..222d69b0dc 100644 --- a/test/e2e/kwokctl/docker/kubectl_test.go +++ b/test/e2e/kwokctl/docker/kubectl_test.go @@ -30,6 +30,12 @@ func TestDryrunExportLogs(t *testing.T) { testEnv.Test(t, f0) } +func TestKwokctlPortForward(t *testing.T) { + f0 := e2e.CaseKwokctlPortForward(kwokctlPath, clusterName). + Feature() + testEnv.Test(t, f0) +} + func TestHack(t *testing.T) { f0 := e2e.CaseHack(kwokctlPath, clusterName, envconf.RandomName("node", 16)). Feature() diff --git a/test/e2e/kwokctl/dryrun/testdata/docker/create_cluster.txt b/test/e2e/kwokctl/dryrun/testdata/docker/create_cluster.txt index a4b4991330..285dced0bf 100644 --- a/test/e2e/kwokctl/dryrun/testdata/docker/create_cluster.txt +++ b/test/e2e/kwokctl/dryrun/testdata/docker/create_cluster.txt @@ -52,7 +52,7 @@ users: EOF # Save cluster config to /workdir/clusters//kwok.yaml docker network create kwok- --label=com.docker.compose.project=kwok- -docker create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32765:2379/tcp registry.k8s.io/etcd:3.5.11-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 +docker create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- registry.k8s.io/etcd:3.5.11-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 docker create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --link=kwok--etcd --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-apiserver:v1.31.0 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --proxy-client-key-file=/etc/kubernetes/pki/admin.key --proxy-client-cert-file=/etc/kubernetes/pki/admin.crt docker create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-controller-manager:v1.31.0 --node-monitor-period=25s --node-monitor-grace-period=3m20s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --kube-api-qps=5000 --kube-api-burst=10000 docker create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-scheduler:v1.31.0 --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 diff --git a/test/e2e/kwokctl/dryrun/testdata/docker/create_cluster_with_extra.txt b/test/e2e/kwokctl/dryrun/testdata/docker/create_cluster_with_extra.txt index f874e533f4..835c06a9a4 100644 --- a/test/e2e/kwokctl/dryrun/testdata/docker/create_cluster_with_extra.txt +++ b/test/e2e/kwokctl/dryrun/testdata/docker/create_cluster_with_extra.txt @@ -139,7 +139,7 @@ users: EOF # Save cluster config to /workdir/clusters//kwok.yaml docker network create kwok- --label=com.docker.compose.project=kwok- -docker create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32765:2379/tcp --env=TEST_KEY=TEST_VALUE registry.k8s.io/etcd:3.5.11-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 --log-level=debug +docker create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --env=TEST_KEY=TEST_VALUE registry.k8s.io/etcd:3.5.11-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 --log-level=debug docker create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --link=kwok--etcd --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/test/e2e/kwokctl/dryrun/extras/apiserver:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-apiserver:v1.31.0 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --proxy-client-key-file=/etc/kubernetes/pki/admin.key --proxy-client-cert-file=/etc/kubernetes/pki/admin.crt --v=5 docker create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/test/e2e/kwokctl/dryrun/extras/controller-manager:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-controller-manager:v1.31.0 --node-monitor-period=25s --node-monitor-grace-period=3m20s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 --v=5 docker create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/test/e2e/kwokctl/dryrun/extras/scheduler:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-scheduler:v1.31.0 --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 --v=5 diff --git a/test/e2e/kwokctl/dryrun/testdata/docker/create_cluster_with_verbosity.txt b/test/e2e/kwokctl/dryrun/testdata/docker/create_cluster_with_verbosity.txt index 3dc2dd2bfa..a7d8589009 100644 --- a/test/e2e/kwokctl/dryrun/testdata/docker/create_cluster_with_verbosity.txt +++ b/test/e2e/kwokctl/dryrun/testdata/docker/create_cluster_with_verbosity.txt @@ -171,8 +171,8 @@ users: EOF # Save cluster config to /workdir/clusters//kwok.yaml docker network create kwok- --label=com.docker.compose.project=kwok- -docker create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32765:2379/tcp registry.k8s.io/etcd:3.5.11-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 -docker create --name=kwok--jaeger --pull=never --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=16686:16686/tcp docker.io/jaegertracing/all-in-one:1.58.1 --collector.otlp.enabled=true --query.http-server.host-port=0.0.0.0:16686 +docker create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- registry.k8s.io/etcd:3.5.11-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 +docker create --name=kwok--jaeger --pull=never --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=16686:16686/tcp docker.io/jaegertracing/all-in-one:1.58.1 --collector.otlp.enabled=true --query.http-server.host-port=0.0.0.0:16686 --collector.otlp.grpc.host-port=127.0.0.1:4317 docker create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --link=kwok--etcd --link=kwok--jaeger --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//audit.yaml:/etc/kubernetes/audit-policy.yaml:ro --volume=/workdir/clusters//logs/audit.log:/var/log/kubernetes/audit/audit.log --volume=/workdir/clusters//apiserver-tracing-config.yaml:/etc/kubernetes/apiserver-tracing-config.yaml:ro registry.k8s.io/kube-apiserver:v1.31.0 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --proxy-client-key-file=/etc/kubernetes/pki/admin.key --proxy-client-cert-file=/etc/kubernetes/pki/admin.crt --audit-policy-file=/etc/kubernetes/audit-policy.yaml --audit-log-path=/var/log/kubernetes/audit/audit.log --tracing-config-file=/etc/kubernetes/apiserver-tracing-config.yaml docker create --name=kwok--kube-apiserver-insecure-proxy --pull=never --entrypoint=kubectl --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=6080:8001/tcp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kubectl:v1.31.0 proxy --accept-hosts=^*$ --address=0.0.0.0 --kubeconfig=~/.kube/config --port=8001 docker create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-controller-manager:v1.31.0 --node-monitor-period=25s --node-monitor-grace-period=3m20s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 diff --git a/test/e2e/kwokctl/dryrun/testdata/kind-podman/create_cluster.txt b/test/e2e/kwokctl/dryrun/testdata/kind-podman/create_cluster.txt index efd8953db8..6eb1804921 100644 --- a/test/e2e/kwokctl/dryrun/testdata/kind-podman/create_cluster.txt +++ b/test/e2e/kwokctl/dryrun/testdata/kind-podman/create_cluster.txt @@ -41,10 +41,6 @@ nodes: hostPath: /workdir/clusters//manifests - containerPath: /etc/kubernetes/pki hostPath: /workdir/clusters//pki - extraPortMappings: - - containerPort: 2379 - hostPort: 32765 - protocol: TCP role: control-plane EOF podman pull registry.k8s.io/kwok/kwok:v0.7.0 @@ -79,6 +75,11 @@ spec: image: registry.k8s.io/kwok/kwok:v0.7.0 imagePullPolicy: Never name: kwok-controller + ports: + - containerPort: 10247 + hostPort: 10247 + name: http + protocol: TCP resources: {} volumeMounts: - mountPath: ~/.kube/config diff --git a/test/e2e/kwokctl/dryrun/testdata/kind-podman/create_cluster_with_extra.txt b/test/e2e/kwokctl/dryrun/testdata/kind-podman/create_cluster_with_extra.txt index e1ed404ad1..73e4470ae3 100644 --- a/test/e2e/kwokctl/dryrun/testdata/kind-podman/create_cluster_with_extra.txt +++ b/test/e2e/kwokctl/dryrun/testdata/kind-podman/create_cluster_with_extra.txt @@ -77,9 +77,6 @@ nodes: - containerPort: 9090 hostPort: 9090 protocol: TCP - - containerPort: 2379 - hostPort: 32765 - protocol: TCP role: control-plane EOF podman pull registry.k8s.io/kwok/kwok:v0.7.0 @@ -117,6 +114,11 @@ spec: image: registry.k8s.io/kwok/kwok:v0.7.0 imagePullPolicy: Never name: kwok-controller + ports: + - containerPort: 10247 + hostPort: 10247 + name: http + protocol: TCP resources: {} volumeMounts: - mountPath: ~/.kube/config @@ -188,6 +190,8 @@ spec: ports: - containerPort: 9090 hostPort: 9090 + name: http + protocol: TCP resources: {} volumeMounts: - mountPath: /etc/prometheus/prometheus.yaml diff --git a/test/e2e/kwokctl/dryrun/testdata/kind-podman/create_cluster_with_verbosity.txt b/test/e2e/kwokctl/dryrun/testdata/kind-podman/create_cluster_with_verbosity.txt index 431199ff8a..3540a9d534 100644 --- a/test/e2e/kwokctl/dryrun/testdata/kind-podman/create_cluster_with_verbosity.txt +++ b/test/e2e/kwokctl/dryrun/testdata/kind-podman/create_cluster_with_verbosity.txt @@ -106,9 +106,6 @@ nodes: - containerPort: 16686 hostPort: 16686 protocol: TCP - - containerPort: 2379 - hostPort: 32765 - protocol: TCP role: control-plane EOF podman pull registry.k8s.io/kubectl:v1.31.0 @@ -135,6 +132,8 @@ spec: ports: - containerPort: 8001 hostPort: 8001 + name: http + protocol: TCP resources: {} volumeMounts: - mountPath: ~/.kube/config @@ -201,6 +200,11 @@ spec: image: registry.k8s.io/kwok/kwok:v0.7.0 imagePullPolicy: Never name: kwok-controller + ports: + - containerPort: 10247 + hostPort: 10247 + name: http + protocol: TCP resources: {} volumeMounts: - mountPath: ~/.kube/config @@ -384,6 +388,8 @@ spec: ports: - containerPort: 4443 hostPort: 4443 + name: https + protocol: TCP resources: {} volumeMounts: - mountPath: ~/.kube/config @@ -439,6 +445,8 @@ spec: ports: - containerPort: 9090 hostPort: 9090 + name: http + protocol: TCP resources: {} volumeMounts: - mountPath: /etc/prometheus/prometheus.yaml @@ -492,12 +500,19 @@ spec: - args: - --collector.otlp.enabled=true - --query.http-server.host-port=0.0.0.0:16686 + - --collector.otlp.grpc.host-port=127.0.0.1:4317 image: docker.io/jaegertracing/all-in-one:1.58.1 imagePullPolicy: Never name: jaeger ports: - containerPort: 16686 hostPort: 16686 + name: http + protocol: TCP + - containerPort: 4317 + hostPort: 4317 + name: otlp-grpc + protocol: TCP resources: {} hostNetwork: true restartPolicy: Always diff --git a/test/e2e/kwokctl/dryrun/testdata/kind/create_cluster.txt b/test/e2e/kwokctl/dryrun/testdata/kind/create_cluster.txt index c1056dd702..5f6d4e101d 100644 --- a/test/e2e/kwokctl/dryrun/testdata/kind/create_cluster.txt +++ b/test/e2e/kwokctl/dryrun/testdata/kind/create_cluster.txt @@ -41,10 +41,6 @@ nodes: hostPath: /workdir/clusters//manifests - containerPath: /etc/kubernetes/pki hostPath: /workdir/clusters//pki - extraPortMappings: - - containerPort: 2379 - hostPort: 32765 - protocol: TCP role: control-plane EOF docker pull registry.k8s.io/kwok/kwok:v0.7.0 @@ -79,6 +75,11 @@ spec: image: registry.k8s.io/kwok/kwok:v0.7.0 imagePullPolicy: Never name: kwok-controller + ports: + - containerPort: 10247 + hostPort: 10247 + name: http + protocol: TCP resources: {} volumeMounts: - mountPath: ~/.kube/config diff --git a/test/e2e/kwokctl/dryrun/testdata/kind/create_cluster_with_extra.txt b/test/e2e/kwokctl/dryrun/testdata/kind/create_cluster_with_extra.txt index c75bc64678..4f785ce825 100644 --- a/test/e2e/kwokctl/dryrun/testdata/kind/create_cluster_with_extra.txt +++ b/test/e2e/kwokctl/dryrun/testdata/kind/create_cluster_with_extra.txt @@ -77,9 +77,6 @@ nodes: - containerPort: 9090 hostPort: 9090 protocol: TCP - - containerPort: 2379 - hostPort: 32765 - protocol: TCP role: control-plane EOF docker pull registry.k8s.io/kwok/kwok:v0.7.0 @@ -117,6 +114,11 @@ spec: image: registry.k8s.io/kwok/kwok:v0.7.0 imagePullPolicy: Never name: kwok-controller + ports: + - containerPort: 10247 + hostPort: 10247 + name: http + protocol: TCP resources: {} volumeMounts: - mountPath: ~/.kube/config @@ -188,6 +190,8 @@ spec: ports: - containerPort: 9090 hostPort: 9090 + name: http + protocol: TCP resources: {} volumeMounts: - mountPath: /etc/prometheus/prometheus.yaml diff --git a/test/e2e/kwokctl/dryrun/testdata/kind/create_cluster_with_verbosity.txt b/test/e2e/kwokctl/dryrun/testdata/kind/create_cluster_with_verbosity.txt index eb34662f3b..05b69c9bc9 100644 --- a/test/e2e/kwokctl/dryrun/testdata/kind/create_cluster_with_verbosity.txt +++ b/test/e2e/kwokctl/dryrun/testdata/kind/create_cluster_with_verbosity.txt @@ -106,9 +106,6 @@ nodes: - containerPort: 16686 hostPort: 16686 protocol: TCP - - containerPort: 2379 - hostPort: 32765 - protocol: TCP role: control-plane EOF docker pull registry.k8s.io/kubectl:v1.31.0 @@ -135,6 +132,8 @@ spec: ports: - containerPort: 8001 hostPort: 8001 + name: http + protocol: TCP resources: {} volumeMounts: - mountPath: ~/.kube/config @@ -201,6 +200,11 @@ spec: image: registry.k8s.io/kwok/kwok:v0.7.0 imagePullPolicy: Never name: kwok-controller + ports: + - containerPort: 10247 + hostPort: 10247 + name: http + protocol: TCP resources: {} volumeMounts: - mountPath: ~/.kube/config @@ -384,6 +388,8 @@ spec: ports: - containerPort: 4443 hostPort: 4443 + name: https + protocol: TCP resources: {} volumeMounts: - mountPath: ~/.kube/config @@ -439,6 +445,8 @@ spec: ports: - containerPort: 9090 hostPort: 9090 + name: http + protocol: TCP resources: {} volumeMounts: - mountPath: /etc/prometheus/prometheus.yaml @@ -492,12 +500,19 @@ spec: - args: - --collector.otlp.enabled=true - --query.http-server.host-port=0.0.0.0:16686 + - --collector.otlp.grpc.host-port=127.0.0.1:4317 image: docker.io/jaegertracing/all-in-one:1.58.1 imagePullPolicy: Never name: jaeger ports: - containerPort: 16686 hostPort: 16686 + name: http + protocol: TCP + - containerPort: 4317 + hostPort: 4317 + name: otlp-grpc + protocol: TCP resources: {} hostNetwork: true restartPolicy: Always diff --git a/test/e2e/kwokctl/dryrun/testdata/nerdctl/create_cluster.txt b/test/e2e/kwokctl/dryrun/testdata/nerdctl/create_cluster.txt index b819b3677c..401eba2a4f 100644 --- a/test/e2e/kwokctl/dryrun/testdata/nerdctl/create_cluster.txt +++ b/test/e2e/kwokctl/dryrun/testdata/nerdctl/create_cluster.txt @@ -52,7 +52,7 @@ users: EOF # Save cluster config to /workdir/clusters//kwok.yaml nerdctl network create kwok- --label=com.docker.compose.project=kwok- -nerdctl create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32765:2379/tcp registry.k8s.io/etcd:3.5.11-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 +nerdctl create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- registry.k8s.io/etcd:3.5.11-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 nerdctl create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-apiserver:v1.31.0 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --proxy-client-key-file=/etc/kubernetes/pki/admin.key --proxy-client-cert-file=/etc/kubernetes/pki/admin.crt nerdctl create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-controller-manager:v1.31.0 --node-monitor-period=25s --node-monitor-grace-period=3m20s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --kube-api-qps=5000 --kube-api-burst=10000 nerdctl create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-scheduler:v1.31.0 --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 diff --git a/test/e2e/kwokctl/dryrun/testdata/nerdctl/create_cluster_with_extra.txt b/test/e2e/kwokctl/dryrun/testdata/nerdctl/create_cluster_with_extra.txt index 6247bba78e..4f1c3dcf38 100644 --- a/test/e2e/kwokctl/dryrun/testdata/nerdctl/create_cluster_with_extra.txt +++ b/test/e2e/kwokctl/dryrun/testdata/nerdctl/create_cluster_with_extra.txt @@ -139,7 +139,7 @@ users: EOF # Save cluster config to /workdir/clusters//kwok.yaml nerdctl network create kwok- --label=com.docker.compose.project=kwok- -nerdctl create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32765:2379/tcp --env=TEST_KEY=TEST_VALUE registry.k8s.io/etcd:3.5.11-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 --log-level=debug +nerdctl create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --env=TEST_KEY=TEST_VALUE registry.k8s.io/etcd:3.5.11-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 --log-level=debug nerdctl create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/test/e2e/kwokctl/dryrun/extras/apiserver:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-apiserver:v1.31.0 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --proxy-client-key-file=/etc/kubernetes/pki/admin.key --proxy-client-cert-file=/etc/kubernetes/pki/admin.crt --v=5 nerdctl create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/test/e2e/kwokctl/dryrun/extras/controller-manager:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-controller-manager:v1.31.0 --node-monitor-period=25s --node-monitor-grace-period=3m20s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 --v=5 nerdctl create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/test/e2e/kwokctl/dryrun/extras/scheduler:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-scheduler:v1.31.0 --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 --v=5 diff --git a/test/e2e/kwokctl/dryrun/testdata/nerdctl/create_cluster_with_verbosity.txt b/test/e2e/kwokctl/dryrun/testdata/nerdctl/create_cluster_with_verbosity.txt index d77b60deac..90486935f7 100644 --- a/test/e2e/kwokctl/dryrun/testdata/nerdctl/create_cluster_with_verbosity.txt +++ b/test/e2e/kwokctl/dryrun/testdata/nerdctl/create_cluster_with_verbosity.txt @@ -171,8 +171,8 @@ users: EOF # Save cluster config to /workdir/clusters//kwok.yaml nerdctl network create kwok- --label=com.docker.compose.project=kwok- -nerdctl create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32765:2379/tcp registry.k8s.io/etcd:3.5.11-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 -nerdctl create --name=kwok--jaeger --pull=never --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=16686:16686/tcp docker.io/jaegertracing/all-in-one:1.58.1 --collector.otlp.enabled=true --query.http-server.host-port=0.0.0.0:16686 +nerdctl create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- registry.k8s.io/etcd:3.5.11-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 +nerdctl create --name=kwok--jaeger --pull=never --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=16686:16686/tcp docker.io/jaegertracing/all-in-one:1.58.1 --collector.otlp.enabled=true --query.http-server.host-port=0.0.0.0:16686 --collector.otlp.grpc.host-port=127.0.0.1:4317 nerdctl create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//audit.yaml:/etc/kubernetes/audit-policy.yaml:ro --volume=/workdir/clusters//logs/audit.log:/var/log/kubernetes/audit/audit.log --volume=/workdir/clusters//apiserver-tracing-config.yaml:/etc/kubernetes/apiserver-tracing-config.yaml:ro registry.k8s.io/kube-apiserver:v1.31.0 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --proxy-client-key-file=/etc/kubernetes/pki/admin.key --proxy-client-cert-file=/etc/kubernetes/pki/admin.crt --audit-policy-file=/etc/kubernetes/audit-policy.yaml --audit-log-path=/var/log/kubernetes/audit/audit.log --tracing-config-file=/etc/kubernetes/apiserver-tracing-config.yaml nerdctl create --name=kwok--kube-apiserver-insecure-proxy --pull=never --entrypoint=kubectl --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=6080:8001/tcp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kubectl:v1.31.0 proxy --accept-hosts=^*$ --address=0.0.0.0 --kubeconfig=~/.kube/config --port=8001 nerdctl create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-controller-manager:v1.31.0 --node-monitor-period=25s --node-monitor-grace-period=3m20s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 diff --git a/test/e2e/kwokctl/dryrun/testdata/podman/create_cluster.txt b/test/e2e/kwokctl/dryrun/testdata/podman/create_cluster.txt index 8be8ae4818..605d23a2d1 100644 --- a/test/e2e/kwokctl/dryrun/testdata/podman/create_cluster.txt +++ b/test/e2e/kwokctl/dryrun/testdata/podman/create_cluster.txt @@ -52,7 +52,7 @@ users: EOF # Save cluster config to /workdir/clusters//kwok.yaml podman network create kwok- --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- -podman create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=32765:2379/tcp registry.k8s.io/etcd:3.5.11-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 +podman create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- registry.k8s.io/etcd:3.5.11-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 podman create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --requires=kwok--etcd --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-apiserver:v1.31.0 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --proxy-client-key-file=/etc/kubernetes/pki/admin.key --proxy-client-cert-file=/etc/kubernetes/pki/admin.crt podman create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-controller-manager:v1.31.0 --node-monitor-period=25s --node-monitor-grace-period=3m20s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --kube-api-qps=5000 --kube-api-burst=10000 podman create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-scheduler:v1.31.0 --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 diff --git a/test/e2e/kwokctl/dryrun/testdata/podman/create_cluster_with_extra.txt b/test/e2e/kwokctl/dryrun/testdata/podman/create_cluster_with_extra.txt index c2256fe6c5..2e7b84bec2 100644 --- a/test/e2e/kwokctl/dryrun/testdata/podman/create_cluster_with_extra.txt +++ b/test/e2e/kwokctl/dryrun/testdata/podman/create_cluster_with_extra.txt @@ -139,7 +139,7 @@ users: EOF # Save cluster config to /workdir/clusters//kwok.yaml podman network create kwok- --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- -podman create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=32765:2379/tcp --env=TEST_KEY=TEST_VALUE registry.k8s.io/etcd:3.5.11-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 --log-level=debug +podman create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --env=TEST_KEY=TEST_VALUE registry.k8s.io/etcd:3.5.11-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 --log-level=debug podman create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --requires=kwok--etcd --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/test/e2e/kwokctl/dryrun/extras/apiserver:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-apiserver:v1.31.0 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --proxy-client-key-file=/etc/kubernetes/pki/admin.key --proxy-client-cert-file=/etc/kubernetes/pki/admin.crt --v=5 podman create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/test/e2e/kwokctl/dryrun/extras/controller-manager:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-controller-manager:v1.31.0 --node-monitor-period=25s --node-monitor-grace-period=3m20s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 --v=5 podman create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/test/e2e/kwokctl/dryrun/extras/scheduler:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-scheduler:v1.31.0 --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 --v=5 diff --git a/test/e2e/kwokctl/dryrun/testdata/podman/create_cluster_with_verbosity.txt b/test/e2e/kwokctl/dryrun/testdata/podman/create_cluster_with_verbosity.txt index 72bd0f9a8a..34a2ce08a6 100644 --- a/test/e2e/kwokctl/dryrun/testdata/podman/create_cluster_with_verbosity.txt +++ b/test/e2e/kwokctl/dryrun/testdata/podman/create_cluster_with_verbosity.txt @@ -171,8 +171,8 @@ users: EOF # Save cluster config to /workdir/clusters//kwok.yaml podman network create kwok- --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- -podman create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=32765:2379/tcp registry.k8s.io/etcd:3.5.11-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 -podman create --name=kwok--jaeger --pull=never --network=kwok- --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=16686:16686/tcp docker.io/jaegertracing/all-in-one:1.58.1 --collector.otlp.enabled=true --query.http-server.host-port=0.0.0.0:16686 +podman create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- registry.k8s.io/etcd:3.5.11-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 +podman create --name=kwok--jaeger --pull=never --network=kwok- --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=16686:16686/tcp docker.io/jaegertracing/all-in-one:1.58.1 --collector.otlp.enabled=true --query.http-server.host-port=0.0.0.0:16686 --collector.otlp.grpc.host-port=127.0.0.1:4317 podman create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --requires=kwok--etcd --requires=kwok--jaeger --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//audit.yaml:/etc/kubernetes/audit-policy.yaml:ro --volume=/workdir/clusters//logs/audit.log:/var/log/kubernetes/audit/audit.log --volume=/workdir/clusters//apiserver-tracing-config.yaml:/etc/kubernetes/apiserver-tracing-config.yaml:ro registry.k8s.io/kube-apiserver:v1.31.0 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --proxy-client-key-file=/etc/kubernetes/pki/admin.key --proxy-client-cert-file=/etc/kubernetes/pki/admin.crt --audit-policy-file=/etc/kubernetes/audit-policy.yaml --audit-log-path=/var/log/kubernetes/audit/audit.log --tracing-config-file=/etc/kubernetes/apiserver-tracing-config.yaml podman create --name=kwok--kube-apiserver-insecure-proxy --pull=never --entrypoint=kubectl --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=6080:8001/tcp --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kubectl:v1.31.0 proxy --accept-hosts=^*$ --address=0.0.0.0 --kubeconfig=~/.kube/config --port=8001 podman create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-controller-manager:v1.31.0 --node-monitor-period=25s --node-monitor-grace-period=3m20s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 diff --git a/test/e2e/kwokctl/kind-podman/kubectl_test.go b/test/e2e/kwokctl/kind-podman/kubectl_test.go index e79f7595af..8edda9e8b6 100644 --- a/test/e2e/kwokctl/kind-podman/kubectl_test.go +++ b/test/e2e/kwokctl/kind-podman/kubectl_test.go @@ -31,6 +31,12 @@ func TestDryrunExportLogs(t *testing.T) { testEnv.Test(t, f0) } +func TestKwokctlPortForward(t *testing.T) { + f0 := e2e.CaseKwokctlPortForward(kwokctlPath, clusterName). + Feature() + testEnv.Test(t, f0) +} + func TestHack(t *testing.T) { f0 := e2e.CaseHack(kwokctlPath, clusterName, envconf.RandomName("node", 16)). Feature() diff --git a/test/e2e/kwokctl/kind/kubectl_test.go b/test/e2e/kwokctl/kind/kubectl_test.go index 9ed5509383..34402d4520 100644 --- a/test/e2e/kwokctl/kind/kubectl_test.go +++ b/test/e2e/kwokctl/kind/kubectl_test.go @@ -31,6 +31,12 @@ func TestDryrunExportLogs(t *testing.T) { testEnv.Test(t, f0) } +func TestKwokctlPortForward(t *testing.T) { + f0 := e2e.CaseKwokctlPortForward(kwokctlPath, clusterName). + Feature() + testEnv.Test(t, f0) +} + func TestHack(t *testing.T) { f0 := e2e.CaseHack(kwokctlPath, clusterName, envconf.RandomName("node", 16)). Feature() diff --git a/test/e2e/kwokctl/nerdctl/kubectl_test.go b/test/e2e/kwokctl/nerdctl/kubectl_test.go index d32d275a6f..663b451cfd 100644 --- a/test/e2e/kwokctl/nerdctl/kubectl_test.go +++ b/test/e2e/kwokctl/nerdctl/kubectl_test.go @@ -30,6 +30,12 @@ func TestDryrunExportLogs(t *testing.T) { testEnv.Test(t, f0) } +func TestKwokctlPortForward(t *testing.T) { + f0 := e2e.CaseKwokctlPortForward(kwokctlPath, clusterName). + Feature() + testEnv.Test(t, f0) +} + func TestHack(t *testing.T) { f0 := e2e.CaseHack(kwokctlPath, clusterName, envconf.RandomName("node", 16)). Feature() diff --git a/test/e2e/kwokctl/podman/kubectl_test.go b/test/e2e/kwokctl/podman/kubectl_test.go index 4272d6a31e..a90a277cab 100644 --- a/test/e2e/kwokctl/podman/kubectl_test.go +++ b/test/e2e/kwokctl/podman/kubectl_test.go @@ -30,6 +30,12 @@ func TestDryrunExportLogs(t *testing.T) { testEnv.Test(t, f0) } +func TestKwokctlPortForward(t *testing.T) { + f0 := e2e.CaseKwokctlPortForward(kwokctlPath, clusterName). + Feature() + testEnv.Test(t, f0) +} + func TestHack(t *testing.T) { f0 := e2e.CaseHack(kwokctlPath, clusterName, envconf.RandomName("node", 16)). Feature() diff --git a/test/e2e/kwokctl_port_forward.go b/test/e2e/kwokctl_port_forward.go new file mode 100644 index 0000000000..bc0b13b4f6 --- /dev/null +++ b/test/e2e/kwokctl_port_forward.go @@ -0,0 +1,81 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "io" + "net/http" + "testing" + "time" + + "sigs.k8s.io/e2e-framework/pkg/envconf" + "sigs.k8s.io/e2e-framework/pkg/features" + + "sigs.k8s.io/kwok/pkg/utils/exec" +) + +// CaseKwokctlPortForward creates a feature that tests port forward +func CaseKwokctlPortForward(kwokctlPath, clusterName string) *features.FeatureBuilder { + return features.New("Port Forward"). + Assess("test port forward", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + t.Log("test port forward") + + cmd, err := exec.Command(exec.WithFork(ctx, true), kwokctlPath, "--name", clusterName, "port-forward", "kwok-controller", "8080:http") + if err != nil { + t.Fatal(err) + } + + defer func() { + err = cmd.Process.Kill() + if err != nil { + t.Fatal(err) + } + }() + + var resp *http.Response + for i := 0; i != 30; i++ { + resp, err = http.Get("http://localhost:8080/healthz") + if err == nil { + break + } + time.Sleep(1 * time.Second) + } + if err != nil { + t.Fatal(err) + } + + defer func() { + _ = resp.Body.Close() + }() + + if resp.StatusCode != http.StatusOK { + t.Fatal("port forward failed", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + + if string(body) != "ok" { + t.Fatal("port forward failed", string(body)) + } + + return ctx + }) +}