From 09d8546d9fdc8637f2b64bd17701339dac89f9db Mon Sep 17 00:00:00 2001 From: Jenny Shu <28537278+jenshu@users.noreply.github.com> Date: Fri, 21 Jun 2024 17:21:11 -0400 Subject: [PATCH] make glooctl check work for kube gateways (#9621) * glooctl check * cl * codegen * use const * standardize casing * standardize case * update kubegateway check * fix import * scheme * KUBE2E_TESTS not needed by setup-kind * fix test * Adding changelog file to new location * Deleting changelog file from old location * fix typo * add additional check outputs * exclude kube checks * add docs to exclude * ensure kube resources are not checked if not enabled * add exclude options * add todo * update kubegatewayenabled check * fix newline * use constants * return early * update comment * Update error * address comments * rename kube gateway * update message --------- Co-authored-by: soloio-bulldozer[bot] <48420018+soloio-bulldozer[bot]@users.noreply.github.com> Co-authored-by: changelog-bot Co-authored-by: David Jumani --- .../regression-tests/action.yaml | 1 - changelog/v1.17.0-rc7/glooctl-check.yaml | 6 + ci/kind/setup-kind.sh | 2 - .../federated_configuration.md | 67 +++-- .../operations/debugging_gloo/_index.md | 20 +- .../operations/upgrading/upgrade_steps.md | 16 +- docs/content/reference/cli/glooctl_check.md | 2 +- docs/content/support/support-info.md | 20 +- .../gloo/cli/pkg/cmd/check/gloo_instances.go | 36 +-- .../gloo/cli/pkg/cmd/check/internal/checks.go | 121 +------- .../cmd/check/internal/kubegateway/check.go | 30 -- .../gloo/cli/pkg/cmd/check/kube_gateway.go | 83 ++++++ projects/gloo/cli/pkg/cmd/check/root.go | 179 +++++------- .../pkg/cmd/install/kubegateway/install.go | 27 +- projects/gloo/cli/pkg/constants/constants.go | 21 ++ projects/gloo/cli/pkg/flagutils/metadata.go | 2 +- .../gloo/cli/pkg/kubegatewayutils/detect.go | 79 +++++ projects/gloo/cli/pkg/printers/check.go | 4 +- test/kube2e/README.md | 1 - test/kube2e/glooctl/check_test.go | 276 ++++++++++-------- test/kubernetes/e2e/features/glooctl/check.go | 58 ++-- test/kubernetes/e2e/tests/k8s_gw_tests.go | 10 + 22 files changed, 567 insertions(+), 494 deletions(-) create mode 100644 changelog/v1.17.0-rc7/glooctl-check.yaml delete mode 100644 projects/gloo/cli/pkg/cmd/check/internal/kubegateway/check.go create mode 100644 projects/gloo/cli/pkg/cmd/check/kube_gateway.go create mode 100644 projects/gloo/cli/pkg/kubegatewayutils/detect.go diff --git a/.github/workflows/composite-actions/regression-tests/action.yaml b/.github/workflows/composite-actions/regression-tests/action.yaml index b9e52293db1..449319b8936 100644 --- a/.github/workflows/composite-actions/regression-tests/action.yaml +++ b/.github/workflows/composite-actions/regression-tests/action.yaml @@ -22,7 +22,6 @@ runs: - name: Setup test env shell: bash env: - KUBE2E_TESTS: ${{ matrix.kube-e2e-test-type }} CLUSTER_NAME: 'kind' CLUSTER_NODE_VERSION: ${{ matrix.kube-version.node }} IMAGE_VARIANT: ${{ matrix.image-variant }} diff --git a/changelog/v1.17.0-rc7/glooctl-check.yaml b/changelog/v1.17.0-rc7/glooctl-check.yaml new file mode 100644 index 00000000000..8a634732796 --- /dev/null +++ b/changelog/v1.17.0-rc7/glooctl-check.yaml @@ -0,0 +1,6 @@ +changelog: + - type: FIX + issueLink: https://github.com/solo-io/solo-projects/issues/5741 + resolvesIssue: false + description: >- + Update `glooctl check` to check for Kubernetes Gateway resources. \ No newline at end of file diff --git a/ci/kind/setup-kind.sh b/ci/kind/setup-kind.sh index 962a30a5186..e4256e2f270 100755 --- a/ci/kind/setup-kind.sh +++ b/ci/kind/setup-kind.sh @@ -13,8 +13,6 @@ VERSION="${VERSION:-1.0.0-ci1}" SKIP_DOCKER="${SKIP_DOCKER:-false}" # Stop after creating the kind cluster JUST_KIND="${JUST_KIND:-false}" -# Offer a default value for type of installation -KUBE2E_TESTS="${KUBE2E_TESTS:-gateway}" # If 'KUBE2E_TESTS' not set or null, use 'gateway'. # Set the default image variant to standard IMAGE_VARIANT="${IMAGE_VARIANT:-standard}" # If true, run extra steps to set up k8s gateway api conformance test environment diff --git a/docs/content/guides/gloo_federation/federated_configuration.md b/docs/content/guides/gloo_federation/federated_configuration.md index 56e2648b9c7..0c1e33fce67 100644 --- a/docs/content/guides/gloo_federation/federated_configuration.md +++ b/docs/content/guides/gloo_federation/federated_configuration.md @@ -183,47 +183,52 @@ glooctl check ``` ```shell -Checking deployments... OK -Checking pods... OK -Checking upstreams... OK -Checking upstream groups... OK -Checking auth configs... OK -Checking rate limit configs... OK +Checking Deployments... OK +Checking Pods... OK +Checking Upstreams... OK +Checking UpstreamGroups... OK +Checking AuthConfigs... OK +Checking RateLimitConfigs... OK Checking VirtualHostOptions... OK Checking RouteOptions... OK -Checking secrets... OK -Checking virtual services... OK -Checking gateways... OK -Checking proxies... OK -Checking rate limit server... OK +Checking Secrets... OK +Checking VirtualServices... OK +Checking Gateways... OK +Checking Proxies... OK No problems detected. Detected Gloo Federation! Checking Gloo Instance remote-1-gloo-system... -Checking deployments... OK -Checking pods... OK -Checking settings... OK -Checking upstreams... OK -Checking upstream groups... OK -Checking auth configs... OK -Checking virtual services... OK -Checking route tables... OK -Checking gateways... OK -Checking proxies... OK +Checking Deployments... OK +Checking Pods... OK +Checking Settings... OK +Checking Upstreams... OK +Checking UpstreamGroups... OK +Checking AuthConfigs... OK +Checking RateLimitConfigs... OK +Checking VirtualServices... OK +Checking RouteRables... OK +Checking Gateways... OK +Checking MatchableHttpGateways... OK +Checking MatchableTcpGateways... OK +Checking Proxies... OK Checking Gloo Instance remote-2-gloo-system... -Checking deployments... OK -Checking pods... OK -Checking settings... OK -Checking upstreams... OK -Checking upstream groups... OK -Checking auth configs... OK -Checking virtual services... OK -Checking route tables... OK -Checking gateways... OK -Checking proxies... OK +Checking Deployments... OK +Checking Pods... OK +Checking Settings... OK +Checking Upstreams... OK +Checking UpstreamGroups... OK +Checking AuthConfigs... OK +Checking RateLimitConfigs... OK +Checking VirtualServices... OK +Checking RouteRables... OK +Checking Gateways... OK +Checking MatchableHttpGateways... OK +Checking MatchableTcpGateways... OK +Checking Proxies... OK ``` Note that it is best to have `gloo-edge` installed on this admin cluster. To do so, if needed, you can use the following values and command. diff --git a/docs/content/operations/debugging_gloo/_index.md b/docs/content/operations/debugging_gloo/_index.md index a451b37690a..0556509e8c8 100644 --- a/docs/content/operations/debugging_gloo/_index.md +++ b/docs/content/operations/debugging_gloo/_index.md @@ -34,18 +34,18 @@ glooctl check Example output for a healthy setup: ``` -Checking deployments... OK -Checking pods... OK -Checking upstreams... OK -Checking upstream groups... OK -Checking auth configs... OK -Checking rate limit configs... OK +Checking Deployments... OK +Checking Pods... OK +Checking Upstreams... OK +Checking UpstreamGroups... OK +Checking AuthConfigs... OK +Checking RateLimitConfigs... OK Checking VirtualHostOptions... OK Checking RouteOptions... OK -Checking secrets... OK -Checking virtual services... OK -Checking gateways... OK -Checking proxies... OK +Checking Secrets... OK +Checking VirtualServices... OK +Checking Gateways... OK +Checking Proxies... OK No problems detected. ``` diff --git a/docs/content/operations/upgrading/upgrade_steps.md b/docs/content/operations/upgrading/upgrade_steps.md index c45f64877f3..a427bab3721 100644 --- a/docs/content/operations/upgrading/upgrade_steps.md +++ b/docs/content/operations/upgrading/upgrade_steps.md @@ -122,14 +122,14 @@ Upgrade your Gloo Edge installation. The following example upgrade commands assu ``` Example output: ```bash - Checking deployments... OK - Checking pods... OK - Checking upstreams... OK - Checking upstream groups... OK - Checking secrets... OK - Checking virtual services... OK - Checking gateways... OK - Checking proxies... OK + Checking Deployments... OK + Checking Pods... OK + Checking Upstreams... OK + Checking UpstreamGroups... OK + Checking Secrets... OK + Checking VirtualServices... OK + Checking Gateways... OK + Checking Proxies... OK No problems detected. ``` diff --git a/docs/content/reference/cli/glooctl_check.md b/docs/content/reference/cli/glooctl_check.md index 8301436f8bf..0a614d80f58 100644 --- a/docs/content/reference/cli/glooctl_check.md +++ b/docs/content/reference/cli/glooctl_check.md @@ -17,7 +17,7 @@ glooctl check [flags] ### Options ``` - -x, --exclude strings check to exclude: (deployments, pods, upstreams, upstreamgroup, auth-configs, rate-limit-configs, secrets, virtual-services, gateways, proxies, xds-metrics) + -x, --exclude strings check to exclude: (deployments, pods, upstreams, upstreamgroup, auth-configs, rate-limit-configs, virtual-host-options, route-options, secrets, virtual-services, gateways, proxies, xds-metrics, kube-gateway-classes, kube-gateways, kube-http-routes) -h, --help help for check -n, --namespace string namespace for reading or writing resources (default "gloo-system") -o, --output OutputType output format: (json, table) (default table) diff --git a/docs/content/support/support-info.md b/docs/content/support/support-info.md index 28e4a12457c..56383876910 100644 --- a/docs/content/support/support-info.md +++ b/docs/content/support/support-info.md @@ -92,18 +92,18 @@ Get the Argo CD applications that define the installation of Gloo Edge by using
Typically, the command output indicates any errors in the control plane components or associated resources, such as in the following example. ``` - Checking deployments... 1 Errors! - Checking pods... 2 Errors! - Checking upstreams... OK - Checking upstream groups... OK - Checking auth configs... OK - Checking rate limit configs... OK + Checking Deployments... 1 Errors! + Checking Pods... 2 Errors! + Checking Upstreams... OK + Checking UpstreamGroups... OK + Checking AuthConfigs... OK + Checking RateLimitConfigs... OK Checking VirtualHostOptions... OK Checking RouteOptions... OK - Checking secrets... OK - Checking virtual services... OK - Checking gateways... OK - Checking proxies... Skipping due to an error in checking deployments + Checking Secrets... OK + Checking VirtualServices... OK + Checking Gateways... OK + Checking Proxies... Skipping due to an error in checking deployments Skipping due to an error in checking deployments Error: 5 errors occurred: * Deployment gloo in namespace gloo-system is not available! Message: Deployment does not have minimum availability. diff --git a/projects/gloo/cli/pkg/cmd/check/gloo_instances.go b/projects/gloo/cli/pkg/cmd/check/gloo_instances.go index 5ee3dd64584..10aac6f869b 100644 --- a/projects/gloo/cli/pkg/cmd/check/gloo_instances.go +++ b/projects/gloo/cli/pkg/cmd/check/gloo_instances.go @@ -10,6 +10,7 @@ import ( "github.com/golang/protobuf/jsonpb" "github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/options" + "github.com/solo-io/gloo/projects/gloo/cli/pkg/constants" "github.com/solo-io/gloo/projects/gloo/cli/pkg/helpers" "github.com/solo-io/gloo/projects/gloo/cli/pkg/printers" glooinstancev1 "github.com/solo-io/solo-apis/pkg/api/fed.solo.io/v1" @@ -27,10 +28,10 @@ import ( func CheckMulticlusterResources(ctx context.Context, printer printers.P, opts *options.Options) { // check if the gloo fed deployment exists client := helpers.MustKubeClientWithKubecontext(opts.Top.KubeContext) - _, err := client.AppsV1().Deployments(opts.Metadata.GetNamespace()).Get(ctx, "gloo-fed", metav1.GetOptions{}) + _, err := client.AppsV1().Deployments(opts.Metadata.GetNamespace()).Get(ctx, constants.GlooFedDeploymentName, metav1.GetOptions{}) if err != nil { if apierrors.IsNotFound(err) { - printer.AppendMessage("Skipping Gloo Instance check -- Gloo Federation not detected") + printer.AppendMessage("\nSkipping Gloo Instance check -- Gloo Federation not detected.") } else { fmt.Printf("Warning: could not get Gloo Fed deployment: %v. Skipping Gloo Instance check.\n", err) } @@ -51,7 +52,7 @@ func CheckMulticlusterResources(ctx context.Context, printer printers.P, opts *o glooInstanceList, err := instanceReader.listGlooInstances(ctx) if err != nil { if meta.IsNoMatchError(err) { - printer.AppendMessage("Skipping Gloo Instance check -- Gloo Federation not detected") + printer.AppendMessage("Skipping Gloo Instance check -- Gloo Federation not detected.") return } fmt.Printf("Warning: could not list Gloo Instances: %v\n", err) @@ -60,16 +61,19 @@ func CheckMulticlusterResources(ctx context.Context, printer printers.P, opts *o printer.AppendMessage("\nDetected Gloo Federation!") for _, glooInstance := range glooInstanceList.Items { fmt.Printf("\nChecking Gloo Instance %s... ", glooInstance.GetName()) - printGlooInstanceCheckSummary("deployments", glooInstance.Spec.GetCheck().GetDeployments()) - printGlooInstanceCheckSummary("pods", glooInstance.Spec.GetCheck().GetPods()) - printGlooInstanceCheckSummary("settings", glooInstance.Spec.GetCheck().GetSettings()) - printGlooInstanceCheckSummary("upstreams", glooInstance.Spec.GetCheck().GetUpstreams()) - printGlooInstanceCheckSummary("upstream groups", glooInstance.Spec.GetCheck().GetUpstreamGroups()) - printGlooInstanceCheckSummary("auth configs", glooInstance.Spec.GetCheck().GetAuthConfigs()) - printGlooInstanceCheckSummary("virtual services", glooInstance.Spec.GetCheck().GetVirtualServices()) - printGlooInstanceCheckSummary("route tables", glooInstance.Spec.GetCheck().GetRouteTables()) - printGlooInstanceCheckSummary("gateways", glooInstance.Spec.GetCheck().GetGateways()) - printGlooInstanceCheckSummary("proxies", glooInstance.Spec.GetCheck().GetProxies()) + printGlooInstanceCheckSummary("Deployments", glooInstance.Spec.GetCheck().GetDeployments()) + printGlooInstanceCheckSummary("Pods", glooInstance.Spec.GetCheck().GetPods()) + printGlooInstanceCheckSummary("Settings", glooInstance.Spec.GetCheck().GetSettings()) + printGlooInstanceCheckSummary("Upstreams", glooInstance.Spec.GetCheck().GetUpstreams()) + printGlooInstanceCheckSummary("UpstreamGroups", glooInstance.Spec.GetCheck().GetUpstreamGroups()) + printGlooInstanceCheckSummary("AuthConfigs", glooInstance.Spec.GetCheck().GetAuthConfigs()) + printGlooInstanceCheckSummary("RateLimitConfigs", glooInstance.Spec.GetCheck().GetRateLimitConfigs()) + printGlooInstanceCheckSummary("VirtualServices", glooInstance.Spec.GetCheck().GetVirtualServices()) + printGlooInstanceCheckSummary("RouteTables", glooInstance.Spec.GetCheck().GetRouteTables()) + printGlooInstanceCheckSummary("Gateways", glooInstance.Spec.GetCheck().GetGateways()) + printGlooInstanceCheckSummary("MatchableHttpGateways", glooInstance.Spec.GetCheck().GetMatchableHttpGateways()) + printGlooInstanceCheckSummary("MatchableTcpGateways", glooInstance.Spec.GetCheck().GetMatchableTcpGateways()) + printGlooInstanceCheckSummary("Proxies", glooInstance.Spec.GetCheck().GetProxies()) fmt.Printf("\n\n") } } @@ -101,12 +105,8 @@ type unstructuredGlooInstanceReader struct { } func getUnstructuredGlooInstanceReader(cfg *rest.Config) (*unstructuredGlooInstanceReader, error) { - scheme := scheme.Scheme - if err := glooinstancev1.AddToScheme(scheme); err != nil { - return nil, err - } client, err := client.New(cfg, client.Options{ - Scheme: scheme, + Scheme: scheme.Scheme, }) if err != nil { return nil, err diff --git a/projects/gloo/cli/pkg/cmd/check/internal/checks.go b/projects/gloo/cli/pkg/cmd/check/internal/checks.go index 5594ac2032e..6d36e816584 100644 --- a/projects/gloo/cli/pkg/cmd/check/internal/checks.go +++ b/projects/gloo/cli/pkg/cmd/check/internal/checks.go @@ -6,14 +6,10 @@ import ( "reflect" "github.com/hashicorp/go-multierror" - "github.com/rotisserie/eris" "github.com/solo-io/gloo/projects/gateway2/wellknown" "github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/options" - "github.com/solo-io/gloo/projects/gloo/cli/pkg/helpers" "github.com/solo-io/gloo/projects/gloo/cli/pkg/printers" "golang.org/x/exp/slices" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client/config" @@ -21,105 +17,14 @@ import ( gwclient "sigs.k8s.io/gateway-api/pkg/client/clientset/versioned" ) -// Checks whether the cluster that the kubeconfig points at is available -// The timeout for the kubernetes client is set to a low value to notify the user of the failure -func CheckConnection(ctx context.Context, _ printers.P, opts *options.Options) error { - client, err := helpers.GetKubernetesClient(opts.Top.KubeContext) - if err != nil { - return eris.Wrapf(err, "Could not get kubernetes client") - } - _, err = client.CoreV1().Namespaces().Get(ctx, opts.Metadata.GetNamespace(), metav1.GetOptions{}) - if err != nil { - return eris.Wrapf(err, "Could not communicate with kubernetes cluster") - } - return nil -} - -func CheckDeployments(ctx context.Context, printer printers.P, opts *options.Options) error { - printer.AppendCheck("Checking deployments... ") - client, err := helpers.GetKubernetesClient(opts.Top.KubeContext) - if err != nil { - errMessage := "error getting KubeClient" - fmt.Println(errMessage) - return fmt.Errorf(errMessage+": %v", err) - } - _, err = client.CoreV1().Namespaces().Get(ctx, opts.Metadata.GetNamespace(), metav1.GetOptions{}) - if err != nil { - errMessage := "Gloo namespace does not exist" - fmt.Println(errMessage) - return fmt.Errorf(errMessage) - } - deployments, err := client.AppsV1().Deployments(opts.Metadata.GetNamespace()).List(ctx, metav1.ListOptions{}) - if err != nil { - return err - } - if len(deployments.Items) == 0 { - errMessage := "Gloo is not installed" - fmt.Println(errMessage) - return fmt.Errorf(errMessage) - } - var multiErr *multierror.Error - var message string - setMessage := func(c appsv1.DeploymentCondition) { - if c.Message != "" { - message = fmt.Sprintf(" Message: %s", c.Message) - } - } - - for _, deployment := range deployments.Items { - // possible condition types listed at https://godoc.org/k8s.io/api/apps/v1#DeploymentConditionType - // check for each condition independently because multiple conditions will be True and DeploymentReplicaFailure - // tends to provide the most explicit error message. - for _, condition := range deployment.Status.Conditions { - setMessage(condition) - if condition.Type == appsv1.DeploymentReplicaFailure && condition.Status == corev1.ConditionTrue { - err := fmt.Errorf("Deployment %s in namespace %s failed to create pods!%s", deployment.Name, deployment.Namespace, message) - multiErr = multierror.Append(multiErr, err) - } - } - - for _, condition := range deployment.Status.Conditions { - setMessage(condition) - if condition.Type == appsv1.DeploymentProgressing && condition.Status != corev1.ConditionTrue { - err := fmt.Errorf("Deployment %s in namespace %s is not progressing!%s", deployment.Name, deployment.Namespace, message) - multiErr = multierror.Append(multiErr, err) - } - } - - for _, condition := range deployment.Status.Conditions { - setMessage(condition) - if condition.Type == appsv1.DeploymentAvailable && condition.Status != corev1.ConditionTrue { - err := fmt.Errorf("Deployment %s in namespace %s is not available!%s", deployment.Name, deployment.Namespace, message) - multiErr = multierror.Append(multiErr, err) - } - - } - - for _, condition := range deployment.Status.Conditions { - if condition.Type != appsv1.DeploymentAvailable && - condition.Type != appsv1.DeploymentReplicaFailure && - condition.Type != appsv1.DeploymentProgressing { - err := fmt.Errorf("Deployment %s has an unhandled deployment condition %s", deployment.Name, condition.Type) - multiErr = multierror.Append(multiErr, err) - } - } - } - if multiErr.ErrorOrNil() != nil { - printer.AppendStatus("deployments", fmt.Sprintf("%v Errors!", multiErr.Len())) - return multiErr.ErrorOrNil() - } - printer.AppendStatus("deployments", "OK") - return nil -} - func CheckGatewayClass(ctx context.Context, printer printers.P, opts *options.Options) error { - printer.AppendCheck("Checking GatewayClass... ") + printer.AppendCheck("Checking Kubernetes GatewayClasses... ") cfg := config.GetConfigOrDie() cli := gwclient.NewForConfigOrDie(cfg) - gc, err := cli.GatewayV1().GatewayClasses().Get(ctx, "gloo-gateway", metav1.GetOptions{}) + gc, err := cli.GatewayV1().GatewayClasses().Get(ctx, wellknown.GatewayClassName, metav1.GetOptions{}) if err != nil { - errMessage := "Could not find solo GatewayClass gloo-gateway" + errMessage := fmt.Sprintf("Could not find solo GatewayClass %s", wellknown.GatewayClassName) fmt.Println(errMessage) return fmt.Errorf(errMessage) } @@ -139,15 +44,15 @@ func CheckGatewayClass(ctx context.Context, printer printers.P, opts *options.Op ) if multierr.ErrorOrNil() != nil { - printer.AppendStatus("GatewayClass", fmt.Sprintf("%v Errors!", multierr.Len())) + printer.AppendStatus("Kubernetes GatewayClasses", fmt.Sprintf("%v Errors!", multierr.Len())) return multierr.ErrorOrNil() } - printer.AppendStatus("GatewayClass", "OK") + printer.AppendStatus("Kubernetes GatewayClasses", "OK") return nil } -func CheckGatewys(ctx context.Context, printer printers.P, opts *options.Options) error { - printer.AppendCheck("Checking Gateways... ") +func CheckGateways(ctx context.Context, printer printers.P, opts *options.Options) error { + printer.AppendCheck("Checking Kubernetes Gateways... ") cfg := config.GetConfigOrDie() cli := gwclient.NewForConfigOrDie(cfg) @@ -164,7 +69,7 @@ func CheckGatewys(ctx context.Context, printer printers.P, opts *options.Options // Pike until go 1.22 gw := gw - if gw.Spec.GatewayClassName != "gloo-gateway" { + if gw.Spec.GatewayClassName != wellknown.GatewayClassName { // #not_my_gateway continue } @@ -215,15 +120,15 @@ func CheckGatewys(ctx context.Context, printer printers.P, opts *options.Options } if multierr.ErrorOrNil() != nil { - printer.AppendStatus("Gateway", fmt.Sprintf("%v Errors!", multierr.Len())) + printer.AppendStatus("Kubernetes Gateways", fmt.Sprintf("%v Errors!", multierr.Len())) return multierr.ErrorOrNil() } - printer.AppendStatus("Gateway", "OK") + printer.AppendStatus("Kubernetes Gateways", "OK") return nil } func CheckHTTPRoutes(ctx context.Context, printer printers.P, opts *options.Options) error { - printer.AppendCheck("Checking HTTPRoutes... ") + printer.AppendCheck("Checking Kubernetes HTTPRoutes... ") cfg := config.GetConfigOrDie() cli := gwclient.NewForConfigOrDie(cfg) @@ -272,10 +177,10 @@ func CheckHTTPRoutes(ctx context.Context, printer printers.P, opts *options.Opti } if multierr.ErrorOrNil() != nil { - printer.AppendStatus(wellknown.HTTPRouteKind, fmt.Sprintf("%v Errors!", multierr.Len())) + printer.AppendStatus("Kubernetes HTTPRoutes", fmt.Sprintf("%v Errors!", multierr.Len())) return multierr.ErrorOrNil() } - printer.AppendStatus(wellknown.HTTPRouteKind, "OK") + printer.AppendStatus("Kubernetes HTTPRoutes", "OK") return nil } diff --git a/projects/gloo/cli/pkg/cmd/check/internal/kubegateway/check.go b/projects/gloo/cli/pkg/cmd/check/internal/kubegateway/check.go deleted file mode 100644 index bec6e7441bc..00000000000 --- a/projects/gloo/cli/pkg/cmd/check/internal/kubegateway/check.go +++ /dev/null @@ -1,30 +0,0 @@ -package kubegateway - -import ( - "context" - - "github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/check/internal" - "github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/options" - "github.com/solo-io/gloo/projects/gloo/cli/pkg/printers" -) - -type CheckFunc = func(ctx context.Context, printer printers.P, opts *options.Options) error - -func Check(ctx context.Context, printer printers.P, opts *options.Options) error { - checks := []CheckFunc{ - internal.CheckConnection, - internal.CheckDeployments, - internal.CheckGatewayClass, - internal.CheckGatewys, - internal.CheckHTTPRoutes, - } - - for _, check := range checks { - if err := check(ctx, printer, opts); err != nil { - return err - } - } - - return nil - -} diff --git a/projects/gloo/cli/pkg/cmd/check/kube_gateway.go b/projects/gloo/cli/pkg/cmd/check/kube_gateway.go new file mode 100644 index 00000000000..e516292e5b9 --- /dev/null +++ b/projects/gloo/cli/pkg/cmd/check/kube_gateway.go @@ -0,0 +1,83 @@ +package check + +import ( + "context" + + "github.com/hashicorp/go-multierror" + "github.com/rotisserie/eris" + "github.com/solo-io/gloo/pkg/utils/kubeutils" + "github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/check/internal" + "github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/options" + "github.com/solo-io/gloo/projects/gloo/cli/pkg/constants" + "github.com/solo-io/gloo/projects/gloo/cli/pkg/kubegatewayutils" + "github.com/solo-io/gloo/projects/gloo/cli/pkg/printers" +) + +type CheckFunc = func(ctx context.Context, printer printers.P, opts *options.Options) error + +func CheckKubeGatewayResources(ctx context.Context, printer printers.P, opts *options.Options) error { + var multiErr *multierror.Error + + kubeGatewayEnabled, err := isKubeGatewayEnabled(ctx, opts) + if err != nil { + multiErr = multierror.Append(multiErr, eris.Wrapf(err, "unable to determine if Kubernetes Gateway integration is enabled")) + return multiErr + } + + if !kubeGatewayEnabled { + printer.AppendMessage("\nSkipping Kubernetes Gateway resources check -- Kubernetes Gateway integration not enabled") + return nil + } + + printer.AppendMessage("\nDetected Kubernetes Gateway integration!") + + var checks = []CheckFunc{} + + if included := doesNotContain(opts.Top.CheckName, constants.KubeGatewayClasses); included { + checks = append(checks, internal.CheckGatewayClass) + } + + if included := doesNotContain(opts.Top.CheckName, constants.KubeGateways); included { + checks = append(checks, internal.CheckGateways) + } + + if included := doesNotContain(opts.Top.CheckName, constants.KubeHTTPRoutes); included { + checks = append(checks, internal.CheckHTTPRoutes) + } + + for _, check := range checks { + if err := check(ctx, printer, opts); err != nil { + multiErr = multierror.Append(multiErr, err) + } + } + + return multiErr.ErrorOrNil() +} + +// check if Kubernetes Gateway integration is enabled by checking if the Gateway API CRDs are installed and +// whether the GG_EXPERIMENTAL_K8S_GW_CONTROLLER env var is true in the gloo deployment. +func isKubeGatewayEnabled(ctx context.Context, opts *options.Options) (bool, error) { + cfg, err := kubeutils.GetRestConfigWithKubeContext(opts.Top.KubeContext) + if err != nil { + return false, err + } + + gatewayEnabled, err := kubegatewayutils.DetectKubeGatewayEnabled(ctx, opts) + if err != nil { + return false, eris.Wrapf(err, "unable to determine if Kubernetes Gateway integration is enabled") + } + + if !gatewayEnabled { + return false, nil + } + + hasCRDs, err := kubegatewayutils.DetectKubeGatewayCrds(cfg) + if err != nil { + return false, eris.Wrapf(err, "unable to determine if Kubernetes Gateway CRDs are applied") + } + if !hasCRDs { + return false, eris.New("The Kubernetes Gateway integration is enabled, but the Kubernetes Gateway CRDs are not applied in the cluster. To apply the CRDs, run `kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.0.0/standard-install.yaml`") + } + + return true, nil +} diff --git a/projects/gloo/cli/pkg/cmd/check/root.go b/projects/gloo/cli/pkg/cmd/check/root.go index 1efc7b4ec32..84b0902fe36 100644 --- a/projects/gloo/cli/pkg/cmd/check/root.go +++ b/projects/gloo/cli/pkg/cmd/check/root.go @@ -6,13 +6,12 @@ import ( "errors" "fmt" - "github.com/solo-io/gloo/pkg/utils/kubeutils" - "github.com/solo-io/gloo/projects/gloo/pkg/utils" "github.com/hashicorp/go-multierror" "github.com/rotisserie/eris" gatewayv1 "github.com/solo-io/gloo/projects/gateway/pkg/api/v1" + gatewayv1alpha1 "github.com/solo-io/gloo/projects/gateway2/pkg/api/gateway.gloo.solo.io/v1alpha1" "github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/options" "github.com/solo-io/gloo/projects/gloo/cli/pkg/common" "github.com/solo-io/gloo/projects/gloo/cli/pkg/constants" @@ -23,7 +22,8 @@ import ( v1 "github.com/solo-io/gloo/projects/gloo/pkg/api/v1" rlopts "github.com/solo-io/gloo/projects/gloo/pkg/api/v1/enterprise/options/ratelimit" "github.com/solo-io/go-utils/cliutils" - "github.com/solo-io/solo-apis/pkg/api/ratelimit.solo.io/v1alpha1" + fedv1 "github.com/solo-io/solo-apis/pkg/api/fed.solo.io/v1" + rlv1alpha1 "github.com/solo-io/solo-apis/pkg/api/ratelimit.solo.io/v1alpha1" "github.com/solo-io/solo-kit/pkg/api/v1/clients" "github.com/solo-io/solo-kit/pkg/api/v1/clients/kube/crd" "github.com/solo-io/solo-kit/pkg/api/v1/resources/core" @@ -32,17 +32,25 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/discovery" + "k8s.io/client-go/kubernetes/scheme" ) var ( CrdNotFoundErr = func(crdName string) error { return eris.Errorf("%s CRD has not been registered", crdName) } - - // printer printers.P ) +func init() { + scheme := scheme.Scheme + if err := gatewayv1alpha1.AddToScheme(scheme); err != nil { + panic(err) + } + if err := fedv1.AddToScheme(scheme); err != nil { + panic(err) + } +} + // contains method func doesNotContain(arr []string, str string) bool { for _, a := range arr { @@ -73,30 +81,37 @@ func RootCmd(opts *options.Options, optionsFunc ...cliutils.OptionsFunc) *cobra. printer := printers.P{OutputType: opts.Top.Output} printer.CheckResult = printer.NewCheckResult() - // TODO add back the kubegateway check (probably better to pass in as a flag rather than try to infer it) - // var err error - // call kubegateway check if k8s gateway crds are detected, otherwise call classic check - // isKubeGateway := detectKubeGatewayCrds(opts.Top.KubeContext) - // if isKubeGateway { - // err = kubegateway.Check(ctx, printer, opts) - // } else { + var multiErr *multierror.Error + + // check edge gateway resources err := CheckResources(ctx, printer, opts) - // } + if err != nil { + multiErr = multierror.Append(multiErr, err) + } + // check kubernetes gateway resources + err = CheckKubeGatewayResources(ctx, printer, opts) if err != nil { - // Not returning error here because this shouldn't propagate as a standard CLI error, which prints usage. + multiErr = multierror.Append(multiErr, err) + } + + // check gloo fed resources (TODO this should return errors too) + CheckMulticlusterResources(ctx, printer, opts) + + if multiErr.ErrorOrNil() != nil { + for _, err := range multiErr.Errors { + printer.AppendError(fmt.Sprint(err)) + } + + // when output type is table, the returned errors get printed directly as cli output. if opts.Top.Output.IsTable() { - return err + return multiErr.ErrorOrNil() } } else { printer.AppendMessage("No problems detected.") } - // TODO add back the kubegateway check - // if !isKubeGateway { - CheckMulticlusterResources(ctx, printer, opts) - // } - + // when output type is json, any errors added via printer.AppendError will show up in the output here. if opts.Top.Output.IsJSON() { printer.PrintChecks(new(bytes.Buffer)) } @@ -125,7 +140,7 @@ func CheckResources(ctx context.Context, printer printers.P, opts *options.Optio } var deployments *appsv1.DeploymentList - deploymentsIncluded := doesNotContain(opts.Top.CheckName, "deployments") + deploymentsIncluded := doesNotContain(opts.Top.CheckName, constants.Deployments) if deploymentsIncluded { deployments, err = getAndCheckDeployments(ctx, printer, opts) if err != nil { @@ -133,7 +148,7 @@ func CheckResources(ctx context.Context, printer printers.P, opts *options.Optio } } - if included := doesNotContain(opts.Top.CheckName, "pods"); included { + if included := doesNotContain(opts.Top.CheckName, constants.Pods); included { err := checkPods(ctx, printer, opts) if err != nil { multiErr = multierror.Append(multiErr, err) @@ -168,14 +183,14 @@ func CheckResources(ctx context.Context, printer printers.P, opts *options.Optio } var knownUpstreams []string - if included := doesNotContain(opts.Top.CheckName, "upstreams"); included { + if included := doesNotContain(opts.Top.CheckName, constants.Upstreams); included { knownUpstreams, err = checkUpstreams(ctx, printer, opts, namespaces) if err != nil { multiErr = multierror.Append(multiErr, err) } } - if included := doesNotContain(opts.Top.CheckName, "upstreamgroup"); included { + if included := doesNotContain(opts.Top.CheckName, constants.UpstreamGroup); included { err := checkUpstreamGroups(ctx, printer, opts, namespaces) if err != nil { multiErr = multierror.Append(multiErr, err) @@ -183,7 +198,7 @@ func CheckResources(ctx context.Context, printer printers.P, opts *options.Optio } var knownAuthConfigs []string - if included := doesNotContain(opts.Top.CheckName, "auth-configs"); included { + if included := doesNotContain(opts.Top.CheckName, constants.AuthConfigs); included { knownAuthConfigs, err = checkAuthConfigs(ctx, printer, opts, namespaces) if err != nil { multiErr = multierror.Append(multiErr, err) @@ -191,7 +206,7 @@ func CheckResources(ctx context.Context, printer printers.P, opts *options.Optio } var knownRateLimitConfigs []string - if included := doesNotContain(opts.Top.CheckName, "rate-limit-configs"); included { + if included := doesNotContain(opts.Top.CheckName, constants.RateLimitConfigs); included { knownRateLimitConfigs, err = checkRateLimitConfigs(ctx, printer, opts, namespaces) if err != nil { multiErr = multierror.Append(multiErr, err) @@ -199,7 +214,7 @@ func CheckResources(ctx context.Context, printer printers.P, opts *options.Optio } var knownVirtualHostOptions []string - if included := doesNotContain(opts.Top.CheckName, "virtual-host-options"); included { + if included := doesNotContain(opts.Top.CheckName, constants.VirtualHostOptions); included { knownVirtualHostOptions, err = checkVirtualHostOptions(ctx, printer, opts, namespaces) if err != nil { multiErr = multierror.Append(multiErr, err) @@ -207,59 +222,53 @@ func CheckResources(ctx context.Context, printer printers.P, opts *options.Optio } var knownRouteOptions []string - if included := doesNotContain(opts.Top.CheckName, "route-options"); included { + if included := doesNotContain(opts.Top.CheckName, constants.RouteOptions); included { knownRouteOptions, err = checkRouteOptions(ctx, printer, opts, namespaces) if err != nil { multiErr = multierror.Append(multiErr, err) } } - if included := doesNotContain(opts.Top.CheckName, "secrets"); included { + if included := doesNotContain(opts.Top.CheckName, constants.Secrets); included { err := checkSecrets(ctx, printer, opts, namespaces) if err != nil { multiErr = multierror.Append(multiErr, err) } } - if included := doesNotContain(opts.Top.CheckName, "virtual-services"); included { + if included := doesNotContain(opts.Top.CheckName, constants.VirtualServices); included { err = checkVirtualServices(ctx, printer, opts, namespaces, knownUpstreams, knownAuthConfigs, knownRateLimitConfigs, knownVirtualHostOptions, knownRouteOptions) if err != nil { multiErr = multierror.Append(multiErr, err) } } - if included := doesNotContain(opts.Top.CheckName, "gateways"); included { + if included := doesNotContain(opts.Top.CheckName, constants.Gateways); included { err := checkGateways(ctx, printer, opts, namespaces) if err != nil { multiErr = multierror.Append(multiErr, err) } } - if included := doesNotContain(opts.Top.CheckName, "proxies"); included { + if included := doesNotContain(opts.Top.CheckName, constants.Proxies); included { err := checkProxies(ctx, printer, opts, namespaces, opts.Metadata.GetNamespace(), deployments, deploymentsIncluded, settings) if err != nil { multiErr = multierror.Append(multiErr, err) } } - if included := doesNotContain(opts.Top.CheckName, "xds-metrics"); included { + if included := doesNotContain(opts.Top.CheckName, constants.XDSMetrics); included { err = checkXdsMetrics(ctx, printer, opts, deployments) if err != nil { multiErr = multierror.Append(multiErr, err) } } - if multiErr.ErrorOrNil() != nil { - for _, err := range multiErr.Errors { - printer.AppendError(fmt.Sprint(err)) - } - } - return multiErr.ErrorOrNil() } func getAndCheckDeployments(ctx context.Context, printer printers.P, opts *options.Options) (*appsv1.DeploymentList, error) { - printer.AppendCheck("Checking deployments... ") + printer.AppendCheck("Checking Deployments... ") client, err := helpers.GetKubernetesClient(opts.Top.KubeContext) if err != nil { errMessage := "error getting KubeClient" @@ -328,15 +337,15 @@ func getAndCheckDeployments(ctx context.Context, printer printers.P, opts *optio } } if multiErr != nil { - printer.AppendStatus("deployments", fmt.Sprintf("%v Errors!", multiErr.Len())) + printer.AppendStatus("Deployments", fmt.Sprintf("%v Errors!", multiErr.Len())) return nil, multiErr } - printer.AppendStatus("deployments", "OK") + printer.AppendStatus("Deployments", "OK") return deployments, nil } func checkPods(ctx context.Context, printer printers.P, opts *options.Options) error { - printer.AppendCheck("Checking pods... ") + printer.AppendCheck("Checking Pods... ") client, err := helpers.GetKubernetesClient(opts.Top.KubeContext) if err != nil { return err @@ -385,7 +394,7 @@ func checkPods(ctx context.Context, printer printers.P, opts *options.Options) e case corev1.PodReadyToStartContainers: // This condition was introduced in k8s 1.29. Skip it since completed jobs have Status=False for this condition default: - fmt.Printf("Note: Unhandled pod condition %s", condition.Type) + fmt.Printf("Note: Unhandled pod condition %s\n", condition.Type) } if errorToPrint != "" { @@ -394,13 +403,13 @@ func checkPods(ctx context.Context, printer printers.P, opts *options.Options) e } } if multiErr != nil { - printer.AppendStatus("pods", fmt.Sprintf("%v Errors!", multiErr.Len())) + printer.AppendStatus("Pods", fmt.Sprintf("%v Errors!", multiErr.Len())) return multiErr } if len(pods.Items) == 0 { printer.AppendMessage("Warning: The provided label selector (" + opts.Top.PodSelector + ") applies to no pods") } else { - printer.AppendStatus("pods", "OK") + printer.AppendStatus("Pods", "OK") } return nil } @@ -413,7 +422,7 @@ func getNamespaces(ctx context.Context, settings *v1.Settings) ([]string, error) } func checkUpstreams(ctx context.Context, printer printers.P, _ *options.Options, namespaces []string) ([]string, error) { - printer.AppendCheck("Checking upstreams... ") + printer.AppendCheck("Checking Upstreams... ") var knownUpstreams []string var multiErr *multierror.Error for _, ns := range namespaces { @@ -450,15 +459,15 @@ func checkUpstreams(ctx context.Context, printer printers.P, _ *options.Options, } } if multiErr != nil { - printer.AppendStatus("upstreams", fmt.Sprintf("%v Errors!", multiErr.Len())) + printer.AppendStatus("Upstreams", fmt.Sprintf("%v Errors!", multiErr.Len())) return knownUpstreams, multiErr } - printer.AppendStatus("upstreams", "OK") + printer.AppendStatus("Upstreams", "OK") return knownUpstreams, nil } func checkUpstreamGroups(ctx context.Context, printer printers.P, _ *options.Options, namespaces []string) error { - printer.AppendCheck("Checking upstream groups... ") + printer.AppendCheck("Checking UpstreamGroups... ") var multiErr *multierror.Error for _, ns := range namespaces { upstreamGroupClient, err := helpers.UpstreamGroupClient(ctx, []string{ns}) @@ -496,15 +505,15 @@ func checkUpstreamGroups(ctx context.Context, printer printers.P, _ *options.Opt } } if multiErr != nil { - printer.AppendStatus("upstream groups", fmt.Sprintf("%v Errors!", multiErr.Len())) + printer.AppendStatus("UpstreamGroups", fmt.Sprintf("%v Errors!", multiErr.Len())) return multiErr } - printer.AppendStatus("upstream groups", "OK") + printer.AppendStatus("UpstreamGroups", "OK") return nil } func checkAuthConfigs(ctx context.Context, printer printers.P, _ *options.Options, namespaces []string) ([]string, error) { - printer.AppendCheck("Checking auth configs... ") + printer.AppendCheck("Checking AuthConfigs... ") var knownAuthConfigs []string var multiErr *multierror.Error for _, ns := range namespaces { @@ -541,15 +550,15 @@ func checkAuthConfigs(ctx context.Context, printer printers.P, _ *options.Option } } if multiErr != nil { - printer.AppendStatus("auth configs", fmt.Sprintf("%v Errors!", multiErr.Len())) + printer.AppendStatus("AuthConfigs", fmt.Sprintf("%v Errors!", multiErr.Len())) return knownAuthConfigs, multiErr } - printer.AppendStatus("auth configs", "OK") + printer.AppendStatus("AuthConfigs", "OK") return knownAuthConfigs, nil } func checkRateLimitConfigs(ctx context.Context, printer printers.P, _ *options.Options, namespaces []string) ([]string, error) { - printer.AppendCheck("Checking rate limit configs... ") + printer.AppendCheck("Checking RateLimitConfigs... ") var knownConfigs []string var multiErr *multierror.Error for _, ns := range namespaces { @@ -569,7 +578,7 @@ func checkRateLimitConfigs(ctx context.Context, printer printers.P, _ *options.O return nil, err } for _, config := range configs { - if config.Status.GetState() == v1alpha1.RateLimitConfigStatus_REJECTED { + if config.Status.GetState() == rlv1alpha1.RateLimitConfigStatus_REJECTED { errMessage := fmt.Sprintf("Found rejected rate limit config: %s ", renderMetadata(config.GetMetadata())) errMessage += fmt.Sprintf("(Reason: %s)", config.Status.GetMessage()) multiErr = multierror.Append(multiErr, errors.New(errMessage)) @@ -580,11 +589,11 @@ func checkRateLimitConfigs(ctx context.Context, printer printers.P, _ *options.O } if multiErr != nil { - printer.AppendStatus("rate limit configs", fmt.Sprintf("%v Errors!", multiErr.Len())) + printer.AppendStatus("RateLimitConfigs", fmt.Sprintf("%v Errors!", multiErr.Len())) return knownConfigs, multiErr } - printer.AppendStatus("rate limit configs", "OK") + printer.AppendStatus("RateLimitConfigs", "OK") return knownConfigs, nil } @@ -685,7 +694,7 @@ func checkRouteOptions(ctx context.Context, printer printers.P, _ *options.Optio } func checkVirtualServices(ctx context.Context, printer printers.P, _ *options.Options, namespaces, knownUpstreams, knownAuthConfigs, knownRateLimitConfigs, knownVirtualHostOptions, knownRouteOptions []string) error { - printer.AppendCheck("Checking virtual services... ") + printer.AppendCheck("Checking VirtualServices... ") var multiErr *multierror.Error for _, ns := range namespaces { @@ -818,15 +827,15 @@ func checkVirtualServices(ctx context.Context, printer printers.P, _ *options.Op } if multiErr != nil { - printer.AppendStatus("virtual services", fmt.Sprintf("%v Errors!", multiErr.Len())) + printer.AppendStatus("VirtualServices", fmt.Sprintf("%v Errors!", multiErr.Len())) return multiErr } - printer.AppendStatus("virtual services", "OK") + printer.AppendStatus("VirtualServices", "OK") return nil } func checkGateways(ctx context.Context, printer printers.P, _ *options.Options, namespaces []string) error { - printer.AppendCheck("Checking gateways... ") + printer.AppendCheck("Checking Gateways... ") var multiErr *multierror.Error for _, ns := range namespaces { gatewayClient, err := helpers.GatewayClient(ctx, []string{ns}) @@ -862,16 +871,16 @@ func checkGateways(ctx context.Context, printer printers.P, _ *options.Options, } if multiErr != nil { - printer.AppendStatus("gateways", fmt.Sprintf("%v Errors!", multiErr.Len())) + printer.AppendStatus("Gateways", fmt.Sprintf("%v Errors!", multiErr.Len())) return multiErr } - printer.AppendStatus("gateways", "OK") + printer.AppendStatus("Gateways", "OK") return nil } func checkProxies(ctx context.Context, printer printers.P, opts *options.Options, namespaces []string, glooNamespace string, deployments *appsv1.DeploymentList, deploymentsIncluded bool, settings *v1.Settings) error { - printer.AppendCheck("Checking proxies... ") + printer.AppendCheck("Checking Proxies... ") if !deploymentsIncluded { printer.AppendStatus("proxies", "Skipping proxies because deployments were excluded") return nil @@ -924,10 +933,10 @@ func checkProxies(ctx context.Context, printer printers.P, opts *options.Options } if multiErr != nil { - printer.AppendStatus("proxies", fmt.Sprintf("%v Errors!", multiErr.Len())) + printer.AppendStatus("Proxies", fmt.Sprintf("%v Errors!", multiErr.Len())) return multiErr } - printer.AppendStatus("proxies", "OK") + printer.AppendStatus("Proxies", "OK") if warnings != nil && warnings.Len() != 0 { for _, warning := range warnings.Errors { printer.AppendMessage(warning.Error()) @@ -937,7 +946,7 @@ func checkProxies(ctx context.Context, printer printers.P, opts *options.Options } func checkSecrets(ctx context.Context, printer printers.P, _ *options.Options, namespaces []string) error { - printer.AppendCheck("Checking secrets... ") + printer.AppendCheck("Checking Secrets... ") var multiErr *multierror.Error client, err := helpers.GetSecretClient(ctx, namespaces) if err != nil { @@ -954,10 +963,10 @@ func checkSecrets(ctx context.Context, printer printers.P, _ *options.Options, n // currently this would only find syntax errors } if multiErr != nil { - printer.AppendStatus("secrets", fmt.Sprintf("%v Errors!", multiErr.Len())) + printer.AppendStatus("Secrets", fmt.Sprintf("%v Errors!", multiErr.Len())) return multiErr } - printer.AppendStatus("secrets", "OK") + printer.AppendStatus("Secrets", "OK") return nil } @@ -1006,29 +1015,3 @@ func isCrdNotFoundErr(crd crd.Crd, err error) bool { return false } } - -// copied from projects/gloo/cli/pkg/cmd/install/kubegateway/install.go (TODO: dedupe/clean up) -func detectKubeGatewayCrds(kubeContext string) bool { - cfg, err := kubeutils.GetRestConfigWithKubeContext(kubeContext) - if err != nil { - return false - } - discClient, err := discovery.NewDiscoveryClientForConfig(cfg) - if err != nil { - return false - } - - groups, err := discClient.ServerGroups() - if err != nil { - return false - } - - // Check if gateway group exists - for _, group := range groups.Groups { - if group.Name == "gateway.networking.k8s.io" { - return true - } - } - - return false -} diff --git a/projects/gloo/cli/pkg/cmd/install/kubegateway/install.go b/projects/gloo/cli/pkg/cmd/install/kubegateway/install.go index 70490374063..14e4623db16 100644 --- a/projects/gloo/cli/pkg/cmd/install/kubegateway/install.go +++ b/projects/gloo/cli/pkg/cmd/install/kubegateway/install.go @@ -5,18 +5,16 @@ import ( "fmt" "github.com/solo-io/gloo/pkg/utils/kubeutils" - "github.com/solo-io/gloo/projects/gateway2/crds" "github.com/solo-io/gloo/projects/gateway2/deployer" "github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/options" + "github.com/solo-io/gloo/projects/gloo/cli/pkg/kubegatewayutils" "helm.sh/helm/v3/pkg/cli" "helm.sh/helm/v3/pkg/cli/values" "helm.sh/helm/v3/pkg/getter" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/discovery" - "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" gwv1 "sigs.k8s.io/gateway-api/apis/v1" ) @@ -59,7 +57,7 @@ func install(opts *options.Options, installOpts *Options) error { createNamespace(ctx, cli, installOpts.Namespace) // Check if CRDs already exist - crdsExist, err := detectCrds(cfg) + crdsExist, err := kubegatewayutils.DetectKubeGatewayCrds(cfg) if err != nil { return err } @@ -139,27 +137,6 @@ func install(opts *options.Options, installOpts *Options) error { return nil } -func detectCrds(cfg *rest.Config) (bool, error) { - discClient, err := discovery.NewDiscoveryClientForConfig(cfg) - if err != nil { - return false, err - } - - groups, err := discClient.ServerGroups() - if err != nil { - return false, err - } - - // Check if gateway group exists - for _, group := range groups.Groups { - if group.Name == "gateway.networking.k8s.io" { - return true, nil - } - } - - return false, nil -} - func createNamespace(ctx context.Context, cli client.Client, namespace string) { ns := corev1.Namespace{} err := cli.Get(ctx, client.ObjectKey{Name: namespace}, &ns) diff --git a/projects/gloo/cli/pkg/constants/constants.go b/projects/gloo/cli/pkg/constants/constants.go index 8729284f6c2..241f1ca9f9f 100644 --- a/projects/gloo/cli/pkg/constants/constants.go +++ b/projects/gloo/cli/pkg/constants/constants.go @@ -7,6 +7,27 @@ const ( GlooReleaseName = "gloo" GlooFedReleaseName = "gloo-fed" KnativeServingNamespace = "knative-serving" + + GlooFedDeploymentName = "gloo-fed" + + GlooContainerName = "gloo" + + Deployments = "deployments" + Pods = "pods" + Upstreams = "upstreams" + UpstreamGroup = "upstreamgroup" + AuthConfigs = "auth-configs" + RateLimitConfigs = "rate-limit-configs" + VirtualHostOptions = "virtual-host-options" + RouteOptions = "route-options" + Secrets = "secrets" + VirtualServices = "virtual-services" + Gateways = "gateways" + Proxies = "proxies" + XDSMetrics = "xds-metrics" + KubeGatewayClasses = "kube-gateway-classes" + KubeGateways = "kube-gateways" + KubeHTTPRoutes = "kube-http-routes" ) var ( diff --git a/projects/gloo/cli/pkg/flagutils/metadata.go b/projects/gloo/cli/pkg/flagutils/metadata.go index ef1bfec7880..af69ba2743b 100644 --- a/projects/gloo/cli/pkg/flagutils/metadata.go +++ b/projects/gloo/cli/pkg/flagutils/metadata.go @@ -31,7 +31,7 @@ func AddResourceNamespaceFlag(set *pflag.FlagSet, strptr *[]string) { } func AddExcludeCheckFlag(set *pflag.FlagSet, strarrptr *[]string) { - set.StringSliceVarP(strarrptr, "exclude", "x", []string{}, "check to exclude: (deployments, pods, upstreams, upstreamgroup, auth-configs, rate-limit-configs, secrets, virtual-services, gateways, proxies, xds-metrics)") + set.StringSliceVarP(strarrptr, "exclude", "x", []string{}, "check to exclude: (deployments, pods, upstreams, upstreamgroup, auth-configs, rate-limit-configs, virtual-host-options, route-options, secrets, virtual-services, gateways, proxies, xds-metrics, kube-gateway-classes, kube-gateways, kube-http-routes)") } // AddReadOnlyFlag adds a flag to our flag set that indicates we shouldn't do anything that requires RBAC create permissions diff --git a/projects/gloo/cli/pkg/kubegatewayutils/detect.go b/projects/gloo/cli/pkg/kubegatewayutils/detect.go new file mode 100644 index 00000000000..781cb50f2d1 --- /dev/null +++ b/projects/gloo/cli/pkg/kubegatewayutils/detect.go @@ -0,0 +1,79 @@ +package kubegatewayutils + +import ( + "context" + "strconv" + + "github.com/rotisserie/eris" + "github.com/solo-io/gloo/pkg/utils/kubeutils" + "github.com/solo-io/gloo/projects/gateway2/wellknown" + "github.com/solo-io/gloo/projects/gloo/cli/pkg/cmd/options" + cliconstants "github.com/solo-io/gloo/projects/gloo/cli/pkg/constants" + "github.com/solo-io/gloo/projects/gloo/cli/pkg/helpers" + "github.com/solo-io/gloo/projects/gloo/constants" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/discovery" + "k8s.io/client-go/rest" +) + +// Returns true if Kubernetes Gateway API CRDs are on the cluster. +// Note: this doesn't check for specific CRD names; it returns true if *any* k8s Gateway CRD is detected +func DetectKubeGatewayCrds(cfg *rest.Config) (bool, error) { + discClient, err := discovery.NewDiscoveryClientForConfig(cfg) + if err != nil { + return false, err + } + + groups, err := discClient.ServerGroups() + if err != nil { + return false, err + } + + // Check if gateway group exists + for _, group := range groups.Groups { + if group.Name == wellknown.GatewayGroup { + return true, nil + } + } + + return false, nil +} + +// Returns true if the GG_EXPERIMENTAL_K8S_GW_CONTROLLER env var is true in the gloo deployment. +// Note: This is tied up with the GG implementation and will need to be updated if it changes +func DetectKubeGatewayEnabled(ctx context.Context, opts *options.Options) (bool, error) { + // check if Kubernetes Gateway integration is enabled by checking if the controller env variable is set in the + // gloo deployment + client, err := helpers.GetKubernetesClient(opts.Top.KubeContext) + if err != nil { + return false, eris.Wrapf(err, "could not get kubernetes client") + } + + glooDeployment, err := client.AppsV1().Deployments(opts.Metadata.GetNamespace()).Get(ctx, kubeutils.GlooDeploymentName, metav1.GetOptions{}) + if err != nil { + return false, eris.Wrapf(err, "could not get gloo deployment") + } + + var glooContainer *corev1.Container + for _, container := range glooDeployment.Spec.Template.Spec.Containers { + if container.Name == cliconstants.GlooContainerName { + glooContainer = &container + break + } + } + if glooContainer == nil { + return false, eris.New("could not find gloo container in gloo deployment") + } + + for _, envVar := range glooContainer.Env { + if envVar.Name == constants.GlooGatewayEnableK8sGwControllerEnv { + val, err := strconv.ParseBool(envVar.Value) + if err != nil { + return false, eris.Wrapf(err, "could not parse value of %s env var in gloo deployment", constants.GlooGatewayEnableK8sGwControllerEnv) + } + return val, nil + } + } + return false, nil +} diff --git a/projects/gloo/cli/pkg/printers/check.go b/projects/gloo/cli/pkg/printers/check.go index 87ee9a497f2..3d49819d167 100644 --- a/projects/gloo/cli/pkg/printers/check.go +++ b/projects/gloo/cli/pkg/printers/check.go @@ -89,8 +89,8 @@ func (p P) NewCheckResult() *CheckResult { return nil } -// We must sanitze the name for json formatting because the name comes in as "Checking deployments..." -// and we just require the type "deployments" +// We must sanitize the name for json formatting because the name comes in as "Checking Deployments..." +// and we just require the type "Deployments" func sanitizeName(name string) string { return strings.ReplaceAll(strings.ReplaceAll(name, "Checking ", ""), "... ", "") } diff --git a/test/kube2e/README.md b/test/kube2e/README.md index a4985afdeb4..07b6b8e4f98 100644 --- a/test/kube2e/README.md +++ b/test/kube2e/README.md @@ -35,7 +35,6 @@ It accepts a number of environment variables, to control the creation of a kind | CLUSTER_NAME | kind | The name of the cluster that will be generated | | CLUSTER_NODE_VERSION | v1.28.0 | The version of the [Node Docker image](https://hub.docker.com/r/kindest/node/) to use for booting the cluster | | VERSION | 1.0.0-ci1 | The version used to tag Gloo images that are deployed to the cluster | -| KUBE2E_TESTS | gateway | Name of the test suite to be run. Options: `'gateway', 'gloo', 'ingress', 'helm', 'gloomtls', 'glooctl', 'upgrade', 'istio'` | | SKIP_DOCKER | false | Skip building docker images (used when testing a release version) | | RELEASED_VERSION | '' | Used if you want to test a previously released version. 'LATEST' will find the latest release | diff --git a/test/kube2e/glooctl/check_test.go b/test/kube2e/glooctl/check_test.go index e02fde0e226..5751769b417 100644 --- a/test/kube2e/glooctl/check_test.go +++ b/test/kube2e/glooctl/check_test.go @@ -23,176 +23,194 @@ var _ = Describe("Check", func() { output, err := GlooctlOut("check", "-x", "xds-metrics") Expect(err).NotTo(HaveOccurred()) - Expect(output).To(ContainSubstring("Checking deployments... OK")) - Expect(output).To(ContainSubstring("Checking upstreams... OK")) - Expect(output).To(ContainSubstring("Checking upstream groups... OK")) - Expect(output).To(ContainSubstring("Checking auth configs... OK")) - Expect(output).To(ContainSubstring("Checking rate limit configs... OK")) - Expect(output).To(ContainSubstring("Checking secrets... OK")) - Expect(output).To(ContainSubstring("Checking virtual services... OK")) - Expect(output).To(ContainSubstring("Checking gateways... OK")) - Expect(output).To(ContainSubstring("Checking proxies... OK")) + Expect(output).To(ContainSubstring("Checking Deployments... OK")) + Expect(output).To(ContainSubstring("Checking Upstreams... OK")) + Expect(output).To(ContainSubstring("Checking UpstreamGroups... OK")) + Expect(output).To(ContainSubstring("Checking AuthConfigs... OK")) + Expect(output).To(ContainSubstring("Checking RateLimitConfigs... OK")) + Expect(output).To(ContainSubstring("Checking Secrets... OK")) + Expect(output).To(ContainSubstring("Checking VirtualServices... OK")) + Expect(output).To(ContainSubstring("Checking Gateways... OK")) + Expect(output).To(ContainSubstring("Checking Proxies... OK")) }) It("can exclude deployments", func() { output, err := GlooctlOut("check", "-x", "xds-metrics,deployments") Expect(err).NotTo(HaveOccurred()) - Expect(output).NotTo(ContainSubstring("Checking deployments...")) - Expect(output).To(ContainSubstring("Checking pods... OK")) - Expect(output).To(ContainSubstring("Checking upstreams... OK")) - Expect(output).To(ContainSubstring("Checking upstream groups... OK")) - Expect(output).To(ContainSubstring("Checking auth configs... OK")) - Expect(output).To(ContainSubstring("Checking rate limit configs... OK")) - Expect(output).To(ContainSubstring("Checking secrets... OK")) - Expect(output).To(ContainSubstring("Checking virtual services... OK")) - Expect(output).To(ContainSubstring("Checking gateways... OK")) - Expect(output).To(ContainSubstring("Checking proxies... Skipping proxies because deployments were excluded")) + Expect(output).NotTo(ContainSubstring("Checking Deployments...")) + Expect(output).To(ContainSubstring("Checking Pods... OK")) + Expect(output).To(ContainSubstring("Checking Upstreams... OK")) + Expect(output).To(ContainSubstring("Checking UpstreamGroups... OK")) + Expect(output).To(ContainSubstring("Checking AuthConfigs... OK")) + Expect(output).To(ContainSubstring("Checking RateLimitConfigs... OK")) + Expect(output).To(ContainSubstring("Checking Secrets... OK")) + Expect(output).To(ContainSubstring("Checking VirtualServices... OK")) + Expect(output).To(ContainSubstring("Checking Gateways... OK")) + Expect(output).To(ContainSubstring("Checking Proxies... Skipping proxies because deployments were excluded")) }) It("can exclude pods", func() { output, err := GlooctlOut("check", "-x", "xds-metrics,pods") Expect(err).NotTo(HaveOccurred()) - Expect(output).To(ContainSubstring("Checking deployments... OK")) - Expect(output).NotTo(ContainSubstring("Checking pods...")) - Expect(output).To(ContainSubstring("Checking upstreams... OK")) - Expect(output).To(ContainSubstring("Checking upstream groups... OK")) - Expect(output).To(ContainSubstring("Checking auth configs... OK")) - Expect(output).To(ContainSubstring("Checking rate limit configs... OK")) - Expect(output).To(ContainSubstring("Checking secrets... OK")) - Expect(output).To(ContainSubstring("Checking virtual services... OK")) - Expect(output).To(ContainSubstring("Checking gateways... OK")) - Expect(output).To(ContainSubstring("Checking proxies... OK")) + Expect(output).To(ContainSubstring("Checking Deployments... OK")) + Expect(output).NotTo(ContainSubstring("Checking Pods...")) + Expect(output).To(ContainSubstring("Checking Upstreams... OK")) + Expect(output).To(ContainSubstring("Checking UpstreamGroups... OK")) + Expect(output).To(ContainSubstring("Checking AuthConfigs... OK")) + Expect(output).To(ContainSubstring("Checking RateLimitConfigs... OK")) + Expect(output).To(ContainSubstring("Checking Secrets... OK")) + Expect(output).To(ContainSubstring("Checking VirtualServices... OK")) + Expect(output).To(ContainSubstring("Checking Gateways... OK")) + Expect(output).To(ContainSubstring("Checking Proxies... OK")) }) It("can exclude upstreams", func() { output, err := GlooctlOut("check", "-x", "xds-metrics,upstreams,virtual-services") Expect(err).NotTo(HaveOccurred()) - Expect(output).To(ContainSubstring("Checking deployments... OK")) - Expect(output).To(ContainSubstring("Checking pods... OK")) - Expect(output).NotTo(ContainSubstring("Checking upstreams...")) - Expect(output).To(ContainSubstring("Checking upstream groups... OK")) - Expect(output).To(ContainSubstring("Checking auth configs... OK")) - Expect(output).To(ContainSubstring("Checking rate limit configs... OK")) - Expect(output).To(ContainSubstring("Checking secrets... OK")) - Expect(output).To(ContainSubstring("Checking gateways... OK")) - Expect(output).To(ContainSubstring("Checking proxies... OK")) + Expect(output).To(ContainSubstring("Checking Deployments... OK")) + Expect(output).To(ContainSubstring("Checking Pods... OK")) + Expect(output).NotTo(ContainSubstring("Checking Upstreams...")) + Expect(output).To(ContainSubstring("Checking UpstreamGroups... OK")) + Expect(output).To(ContainSubstring("Checking AuthConfigs... OK")) + Expect(output).To(ContainSubstring("Checking RateLimitConfigs... OK")) + Expect(output).To(ContainSubstring("Checking Secrets... OK")) + Expect(output).To(ContainSubstring("Checking Gateways... OK")) + Expect(output).To(ContainSubstring("Checking Proxies... OK")) }) It("can exclude upstreamgroups", func() { output, err := GlooctlOut("check", "-x", "xds-metrics,upstreamgroup") Expect(err).NotTo(HaveOccurred()) - Expect(output).To(ContainSubstring("Checking deployments... OK")) - Expect(output).To(ContainSubstring("Checking pods... OK")) - Expect(output).To(ContainSubstring("Checking upstreams... OK")) - Expect(output).NotTo(ContainSubstring("Checking upstream groups...")) - Expect(output).To(ContainSubstring("Checking auth configs... OK")) - Expect(output).To(ContainSubstring("Checking rate limit configs... OK")) - Expect(output).To(ContainSubstring("Checking secrets... OK")) - Expect(output).To(ContainSubstring("Checking virtual services... OK")) - Expect(output).To(ContainSubstring("Checking gateways... OK")) - Expect(output).To(ContainSubstring("Checking proxies... OK")) + Expect(output).To(ContainSubstring("Checking Deployments... OK")) + Expect(output).To(ContainSubstring("Checking Pods... OK")) + Expect(output).To(ContainSubstring("Checking Upstreams... OK")) + Expect(output).NotTo(ContainSubstring("Checking UpstreamGroups...")) + Expect(output).To(ContainSubstring("Checking AuthConfigs... OK")) + Expect(output).To(ContainSubstring("Checking RateLimitConfigs... OK")) + Expect(output).To(ContainSubstring("Checking Secrets... OK")) + Expect(output).To(ContainSubstring("Checking VirtualServices... OK")) + Expect(output).To(ContainSubstring("Checking Gateways... OK")) + Expect(output).To(ContainSubstring("Checking Proxies... OK")) }) It("can exclude auth-configs", func() { output, err := GlooctlOut("check", "-x", "xds-metrics,auth-configs") Expect(err).NotTo(HaveOccurred()) - Expect(output).To(ContainSubstring("Checking deployments... OK")) - Expect(output).To(ContainSubstring("Checking pods... OK")) - Expect(output).To(ContainSubstring("Checking upstreams... OK")) - Expect(output).To(ContainSubstring("Checking upstream groups... OK")) - Expect(output).NotTo(ContainSubstring("Checking auth configs...")) - Expect(output).To(ContainSubstring("Checking rate limit configs... OK")) - Expect(output).To(ContainSubstring("Checking secrets... OK")) - Expect(output).To(ContainSubstring("Checking virtual services... OK")) - Expect(output).To(ContainSubstring("Checking gateways... OK")) - Expect(output).To(ContainSubstring("Checking proxies... OK")) + Expect(output).To(ContainSubstring("Checking Deployments... OK")) + Expect(output).To(ContainSubstring("Checking Pods... OK")) + Expect(output).To(ContainSubstring("Checking Upstreams... OK")) + Expect(output).To(ContainSubstring("Checking UpstreamGroups... OK")) + Expect(output).NotTo(ContainSubstring("Checking AuthConfigs...")) + Expect(output).To(ContainSubstring("Checking RateLimitConfigs... OK")) + Expect(output).To(ContainSubstring("Checking Secrets... OK")) + Expect(output).To(ContainSubstring("Checking VirtualServices... OK")) + Expect(output).To(ContainSubstring("Checking Gateways... OK")) + Expect(output).To(ContainSubstring("Checking Proxies... OK")) }) It("can exclude rate-limit-configs", func() { output, err := GlooctlOut("check", "-x", "xds-metrics,rate-limit-configs") Expect(err).NotTo(HaveOccurred()) - Expect(output).To(ContainSubstring("Checking deployments... OK")) - Expect(output).To(ContainSubstring("Checking pods... OK")) - Expect(output).To(ContainSubstring("Checking upstreams... OK")) - Expect(output).To(ContainSubstring("Checking upstream groups... OK")) - Expect(output).To(ContainSubstring("Checking auth configs... OK")) - Expect(output).NotTo(ContainSubstring("Checking rate limit configs...")) - Expect(output).To(ContainSubstring("Checking secrets... OK")) - Expect(output).To(ContainSubstring("Checking virtual services... OK")) - Expect(output).To(ContainSubstring("Checking gateways... OK")) - Expect(output).To(ContainSubstring("Checking proxies... OK")) + Expect(output).To(ContainSubstring("Checking Deployments... OK")) + Expect(output).To(ContainSubstring("Checking Pods... OK")) + Expect(output).To(ContainSubstring("Checking Upstreams... OK")) + Expect(output).To(ContainSubstring("Checking UpstreamGroups... OK")) + Expect(output).To(ContainSubstring("Checking AuthConfigs... OK")) + Expect(output).NotTo(ContainSubstring("Checking RateLimitConfigs...")) + Expect(output).To(ContainSubstring("Checking Secrets... OK")) + Expect(output).To(ContainSubstring("Checking VirtualServices... OK")) + Expect(output).To(ContainSubstring("Checking Gateways... OK")) + Expect(output).To(ContainSubstring("Checking Proxies... OK")) }) It("can exclude secrets", func() { output, err := GlooctlOut("check", "-x", "xds-metrics,secrets") Expect(err).NotTo(HaveOccurred()) - Expect(output).To(ContainSubstring("Checking deployments... OK")) - Expect(output).To(ContainSubstring("Checking pods... OK")) - Expect(output).To(ContainSubstring("Checking upstreams... OK")) - Expect(output).To(ContainSubstring("Checking upstream groups... OK")) - Expect(output).To(ContainSubstring("Checking auth configs... OK")) - Expect(output).To(ContainSubstring("Checking rate limit configs... OK")) - Expect(output).NotTo(ContainSubstring("Checking secrets...")) - Expect(output).To(ContainSubstring("Checking virtual services... OK")) - Expect(output).To(ContainSubstring("Checking gateways... OK")) - Expect(output).To(ContainSubstring("Checking proxies... OK")) + Expect(output).To(ContainSubstring("Checking Deployments... OK")) + Expect(output).To(ContainSubstring("Checking Pods... OK")) + Expect(output).To(ContainSubstring("Checking Upstreams... OK")) + Expect(output).To(ContainSubstring("Checking UpstreamGroups... OK")) + Expect(output).To(ContainSubstring("Checking AuthConfigs... OK")) + Expect(output).To(ContainSubstring("Checking RateLimitConfigs... OK")) + Expect(output).NotTo(ContainSubstring("Checking Secrets...")) + Expect(output).To(ContainSubstring("Checking VirtualServices... OK")) + Expect(output).To(ContainSubstring("Checking Gateways... OK")) + Expect(output).To(ContainSubstring("Checking Proxies... OK")) }) It("can exclude virtual-services", func() { output, err := GlooctlOut("check", "-x", "xds-metrics,virtual-services") Expect(err).NotTo(HaveOccurred()) - Expect(output).To(ContainSubstring("Checking deployments... OK")) - Expect(output).To(ContainSubstring("Checking pods... OK")) - Expect(output).To(ContainSubstring("Checking upstreams... OK")) - Expect(output).To(ContainSubstring("Checking upstream groups... OK")) - Expect(output).To(ContainSubstring("Checking auth configs... OK")) - Expect(output).To(ContainSubstring("Checking rate limit configs... OK")) - Expect(output).To(ContainSubstring("Checking secrets... OK")) - Expect(output).NotTo(ContainSubstring("Checking virtual services...")) - Expect(output).To(ContainSubstring("Checking gateways... OK")) - Expect(output).To(ContainSubstring("Checking proxies... OK")) + Expect(output).To(ContainSubstring("Checking Deployments... OK")) + Expect(output).To(ContainSubstring("Checking Pods... OK")) + Expect(output).To(ContainSubstring("Checking Upstreams... OK")) + Expect(output).To(ContainSubstring("Checking UpstreamGroups... OK")) + Expect(output).To(ContainSubstring("Checking AuthConfigs... OK")) + Expect(output).To(ContainSubstring("Checking RateLimitConfigs... OK")) + Expect(output).To(ContainSubstring("Checking Secrets... OK")) + Expect(output).NotTo(ContainSubstring("Checking VirtualServices...")) + Expect(output).To(ContainSubstring("Checking Gateways... OK")) + Expect(output).To(ContainSubstring("Checking Proxies... OK")) }) It("can exclude gateways", func() { output, err := GlooctlOut("check", "-x", "xds-metrics,gateways") Expect(err).NotTo(HaveOccurred()) - Expect(output).To(ContainSubstring("Checking deployments... OK")) - Expect(output).To(ContainSubstring("Checking pods... OK")) - Expect(output).To(ContainSubstring("Checking upstreams... OK")) - Expect(output).To(ContainSubstring("Checking upstream groups... OK")) - Expect(output).To(ContainSubstring("Checking auth configs... OK")) - Expect(output).To(ContainSubstring("Checking rate limit configs... OK")) - Expect(output).To(ContainSubstring("Checking secrets... OK")) - Expect(output).To(ContainSubstring("Checking virtual services... OK")) - Expect(output).NotTo(ContainSubstring("Checking gateways...")) - Expect(output).To(ContainSubstring("Checking proxies... OK")) + Expect(output).To(ContainSubstring("Checking Deployments... OK")) + Expect(output).To(ContainSubstring("Checking Pods... OK")) + Expect(output).To(ContainSubstring("Checking Upstreams... OK")) + Expect(output).To(ContainSubstring("Checking UpstreamGroups... OK")) + Expect(output).To(ContainSubstring("Checking AuthConfigs... OK")) + Expect(output).To(ContainSubstring("Checking RateLimitConfigs... OK")) + Expect(output).To(ContainSubstring("Checking Secrets... OK")) + Expect(output).To(ContainSubstring("Checking VirtualServices... OK")) + Expect(output).NotTo(ContainSubstring("Checking Gateways...")) + Expect(output).To(ContainSubstring("Checking Proxies... OK")) }) It("can exclude proxies", func() { output, err := GlooctlOut("check", "-x", "xds-metrics,proxies") Expect(err).NotTo(HaveOccurred()) - Expect(output).To(ContainSubstring("Checking deployments... OK")) - Expect(output).To(ContainSubstring("Checking pods... OK")) - Expect(output).To(ContainSubstring("Checking upstreams... OK")) - Expect(output).To(ContainSubstring("Checking upstream groups... OK")) - Expect(output).To(ContainSubstring("Checking auth configs... OK")) - Expect(output).To(ContainSubstring("Checking rate limit configs... OK")) - Expect(output).To(ContainSubstring("Checking secrets... OK")) - Expect(output).To(ContainSubstring("Checking virtual services... OK")) - Expect(output).To(ContainSubstring("Checking gateways... OK")) - Expect(output).NotTo(ContainSubstring("Checking proxies...")) + Expect(output).To(ContainSubstring("Checking Deployments... OK")) + Expect(output).To(ContainSubstring("Checking Pods... OK")) + Expect(output).To(ContainSubstring("Checking Upstreams... OK")) + Expect(output).To(ContainSubstring("Checking UpstreamGroups... OK")) + Expect(output).To(ContainSubstring("Checking AuthConfigs... OK")) + Expect(output).To(ContainSubstring("Checking RateLimitConfigs... OK")) + Expect(output).To(ContainSubstring("Checking Secrets... OK")) + Expect(output).To(ContainSubstring("Checking VirtualServices... OK")) + Expect(output).To(ContainSubstring("Checking Gateways... OK")) + Expect(output).NotTo(ContainSubstring("Checking Proxies...")) }) + It("excludes the Kubernetes Gateway resources when it is disabled", func() { + output, err := GlooctlOut("check", "-x", "xds-metrics") + Expect(err).NotTo(HaveOccurred()) + + Expect(output).To(ContainSubstring("Checking Deployments... OK")) + Expect(output).To(ContainSubstring("Checking Upstreams... OK")) + Expect(output).To(ContainSubstring("Checking UpstreamGroups... OK")) + Expect(output).To(ContainSubstring("Checking AuthConfigs... OK")) + Expect(output).To(ContainSubstring("Checking RateLimitConfigs... OK")) + Expect(output).To(ContainSubstring("Checking Secrets... OK")) + Expect(output).To(ContainSubstring("Checking VirtualServices... OK")) + Expect(output).To(ContainSubstring("Checking Gateways... OK")) + Expect(output).To(ContainSubstring("Checking Proxies... OK")) + + Expect(output).NotTo(ContainSubstring("Checking Kubernetes GatewayClasses...")) + Expect(output).NotTo(ContainSubstring("Checking Kubernetes Gateways...")) + Expect(output).NotTo(ContainSubstring("Checking Kubernetes HTTPRoutes...")) + }) }) Context("timeouts", Ordered, func() { @@ -236,16 +254,16 @@ var _ = Describe("Check", func() { output, err := GlooctlOut("check", "--read-only") Expect(err).NotTo(HaveOccurred()) - Expect(output).To(ContainSubstring("Checking deployments... OK")) - Expect(output).To(ContainSubstring("Checking pods... OK")) - Expect(output).To(ContainSubstring("Checking upstreams... OK")) - Expect(output).To(ContainSubstring("Checking upstream groups... OK")) - Expect(output).To(ContainSubstring("Checking auth configs... OK")) - Expect(output).To(ContainSubstring("Checking rate limit configs... OK")) - Expect(output).To(ContainSubstring("Checking secrets... OK")) - Expect(output).To(ContainSubstring("Checking virtual services... OK")) - Expect(output).To(ContainSubstring("Checking gateways... OK")) - Expect(output).To(ContainSubstring("Checking proxies... OK")) + Expect(output).To(ContainSubstring("Checking Deployments... OK")) + Expect(output).To(ContainSubstring("Checking Pods... OK")) + Expect(output).To(ContainSubstring("Checking Upstreams... OK")) + Expect(output).To(ContainSubstring("Checking UpstreamGroups... OK")) + Expect(output).To(ContainSubstring("Checking AuthConfigs... OK")) + Expect(output).To(ContainSubstring("Checking RateLimitConfigs... OK")) + Expect(output).To(ContainSubstring("Checking Secrets... OK")) + Expect(output).To(ContainSubstring("Checking VirtualServices... OK")) + Expect(output).To(ContainSubstring("Checking Gateways... OK")) + Expect(output).To(ContainSubstring("Checking Proxies... OK")) Expect(output).To(ContainSubstring("Warning: checking proxies with port forwarding is disabled")) Expect(output).To(ContainSubstring("Warning: checking xds with port forwarding is disabled")) }) @@ -262,7 +280,7 @@ var _ = Describe("Check", func() { It("fails when scanning with invalid kubecontext", func() { _, err := GlooctlOut("check", "--kube-context", "not-gloo-context") Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("Could not get kubernetes client: Error retrieving Kubernetes configuration: context \"not-gloo-context\" does not exist")) + Expect(err.Error()).To(ContainSubstring("context \"not-gloo-context\" does not exist")) }) It("succeeds with correct kubecontext", func() { @@ -340,18 +358,18 @@ var _ = Describe("Check", func() { output, err := GlooctlOut("check", "-x", "xds-metrics") Expect(err).ToNot(HaveOccurred()) - Expect(output).To(ContainSubstring("Checking deployments... OK")) - Expect(output).To(ContainSubstring("Checking pods... OK")) - Expect(output).To(ContainSubstring("Checking upstreams... OK")) - Expect(output).To(ContainSubstring("Checking upstream groups... OK")) - Expect(output).To(ContainSubstring("Checking auth configs... OK")) - Expect(output).To(ContainSubstring("Checking rate limit configs... OK")) + Expect(output).To(ContainSubstring("Checking Deployments... OK")) + Expect(output).To(ContainSubstring("Checking Pods... OK")) + Expect(output).To(ContainSubstring("Checking Upstreams... OK")) + Expect(output).To(ContainSubstring("Checking UpstreamGroups... OK")) + Expect(output).To(ContainSubstring("Checking AuthConfigs... OK")) + Expect(output).To(ContainSubstring("Checking RateLimitConfigs... OK")) Expect(output).To(ContainSubstring("Checking VirtualHostOptions... OK")) Expect(output).To(ContainSubstring("Checking RouteOptions... OK")) - Expect(output).To(ContainSubstring("Checking secrets... OK")) - Expect(output).To(ContainSubstring("Checking virtual services... OK")) - Expect(output).To(ContainSubstring("Checking gateways... OK")) - Expect(output).To(ContainSubstring("Checking proxies... OK")) + Expect(output).To(ContainSubstring("Checking Secrets... OK")) + Expect(output).To(ContainSubstring("Checking VirtualServices... OK")) + Expect(output).To(ContainSubstring("Checking Gateways... OK")) + Expect(output).To(ContainSubstring("Checking Proxies... OK")) Expect(output).To(ContainSubstring("Warning: gloo-system:gateway-proxy has zero replicas")) Expect(output).To(ContainSubstring("No problems detected.")) diff --git a/test/kubernetes/e2e/features/glooctl/check.go b/test/kubernetes/e2e/features/glooctl/check.go index c9df0b9205e..1d26f450773 100644 --- a/test/kubernetes/e2e/features/glooctl/check.go +++ b/test/kubernetes/e2e/features/glooctl/check.go @@ -18,31 +18,36 @@ type checkOutput struct { var ( checkOutputByKey = map[string]checkOutput{ "deployments": { - include: ContainSubstring("Checking deployments... OK"), + include: ContainSubstring("Checking Deployments... OK"), exclude: And( - Not(ContainSubstring("Checking deployments...")), - ContainSubstring("Checking proxies... Skipping proxies because deployments were excluded"), + Not(ContainSubstring("Checking Deployments...")), + ContainSubstring("Checking Proxies... Skipping proxies because deployments were excluded"), ), readOnly: gstruct.Ignore(), }, "pods": { - include: ContainSubstring("Checking pods... OK"), - exclude: Not(ContainSubstring("Checking pods...")), + include: ContainSubstring("Checking Pods... OK"), + exclude: Not(ContainSubstring("Checking Pods...")), readOnly: gstruct.Ignore(), }, "upstreams": { - include: ContainSubstring("Checking upstreams... OK"), - exclude: Not(ContainSubstring("Checking upstreams...")), + include: ContainSubstring("Checking Upstreams... OK"), + exclude: Not(ContainSubstring("Checking Upstreams...")), readOnly: gstruct.Ignore(), }, "upstreamgroup": { - include: ContainSubstring("Checking upstream groups... OK"), - exclude: Not(ContainSubstring("Checking upstream groups...")), + include: ContainSubstring("Checking UpstreamGroups... OK"), + exclude: Not(ContainSubstring("Checking UpstreamGroups...")), + readOnly: gstruct.Ignore(), + }, + "auth-configs": { + include: ContainSubstring("Checking AuthConfigs... OK"), + exclude: Not(ContainSubstring("Checking AuthConfigs...")), readOnly: gstruct.Ignore(), }, "rate-limit-configs": { - include: ContainSubstring("Checking rate limit configs... OK"), - exclude: Not(ContainSubstring("Checking rate limit configs...")), + include: ContainSubstring("Checking RateLimitConfigs... OK"), + exclude: Not(ContainSubstring("Checking RateLimitConfigs...")), readOnly: gstruct.Ignore(), }, "virtual-host-options": { @@ -56,13 +61,13 @@ var ( readOnly: gstruct.Ignore(), }, "secrets": { - include: ContainSubstring("Checking secrets... OK"), - exclude: Not(ContainSubstring("Checking secrets...")), + include: ContainSubstring("Checking Secrets... OK"), + exclude: Not(ContainSubstring("Checking Secrets...")), readOnly: gstruct.Ignore(), }, "virtual-services": { - include: ContainSubstring("Checking virtual services... OK"), - exclude: Not(ContainSubstring("Checking virtual services...")), + include: ContainSubstring("Checking VirtualServices... OK"), + exclude: Not(ContainSubstring("Checking VirtualServices...")), readOnly: gstruct.Ignore(), }, "route-tables": { @@ -74,13 +79,13 @@ var ( readOnly: gstruct.Ignore(), }, "gateways": { - include: ContainSubstring("Checking gateways... OK"), - exclude: Not(ContainSubstring("Checking gateways...")), + include: ContainSubstring("Checking Gateways... OK"), + exclude: Not(ContainSubstring("Checking Gateways...")), readOnly: gstruct.Ignore(), }, "proxies": { - include: ContainSubstring("Checking proxies... OK"), - exclude: Not(ContainSubstring("Checking proxies...")), + include: ContainSubstring("Checking Proxies... OK"), + exclude: Not(ContainSubstring("Checking Proxies...")), readOnly: ContainSubstring("Warning: checking proxies with port forwarding is disabled"), }, "xds-metrics": { @@ -88,5 +93,20 @@ var ( exclude: gstruct.Ignore(), // We have not had historical tests for this, it would be good to add readOnly: ContainSubstring("Warning: checking proxies with port forwarding is disabled"), }, + "kube-gateway-classes": { + include: ContainSubstring("Checking Kubernetes GatewayClasses... OK"), + exclude: Not(ContainSubstring("Checking Kubernetes GatewayClasses...")), + readOnly: gstruct.Ignore(), + }, + "kube-gateways": { + include: ContainSubstring("Checking Kubernetes Gateways... OK"), + exclude: Not(ContainSubstring("Checking Kubernetes Gateways...")), + readOnly: gstruct.Ignore(), + }, + "kube-http-routes": { + include: ContainSubstring("Checking Kubernetes HTTPRoutes... OK"), + exclude: Not(ContainSubstring("Checking Kubernetes HTTPRoutes...")), + readOnly: gstruct.Ignore(), + }, } ) diff --git a/test/kubernetes/e2e/tests/k8s_gw_tests.go b/test/kubernetes/e2e/tests/k8s_gw_tests.go index 78a0f7630c7..4c9853e3e24 100644 --- a/test/kubernetes/e2e/tests/k8s_gw_tests.go +++ b/test/kubernetes/e2e/tests/k8s_gw_tests.go @@ -51,7 +51,17 @@ func newGlooctlTestingSuite(ctx context.Context, testInstallation *e2e.TestInsta } func (s *glooctlSuite) TestCheck() { + // To ensure that glooctl check works as expected in an installation with kubeGateway enabled : + // 0. Install Edge with kubeGateway enabled (done when the test begins) + // 1. Verify it checks kubeGateway resources suite.Run(s.T(), glooctl.NewCheckSuite(s.ctx, s.testInstallation)) + // TODO (davidjumani) : + // 2. Upgrade Edge with kubeGateway disabled + // 3. Verify it does not check kubeGateway resources + // 4. Upgrade Edge with kubeGateway enabled + // 5. Verify it checks kubeGateway resources + // This verifies that we are not relying on any logic / resources that can be left behind after an upgrade or when the user switches between gateway modes + // Doing so will also eliminate the need for the kube2e/glooctl/exclude tests } func (s *glooctlSuite) TestDebug() {