Skip to content

Commit

Permalink
Setup controllers dependent on Sveltos CRDs after its provider is ins…
Browse files Browse the repository at this point in the history 
…talled (#877)

* Setup controllers dependent on Sveltos CRDs after its provider is installed

* changes based on review suggestions (to be squashed)
  • Loading branch information
@wahabmk
wahabmk authored Jan 15, 2025
1 parent 75122e9 commit 2007ea1
Show file tree
Hide file tree
Showing 5 changed files with 93 additions and 55 deletions.
27 changes: 0 additions & 27 deletions cmd/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@ import (
apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
"k8s.io/apimachinery/pkg/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/client-go/dynamic"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
_ "k8s.io/client-go/plugin/pkg/client/auth"
capz "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
Expand Down Expand Up @@ -185,12 +184,6 @@ func main() {
os.Exit(1)
}

dc, err := dynamic.NewForConfig(mgr.GetConfig())
if err != nil {
setupLog.Error(err, "failed to create dynamic client")
os.Exit(1)
}

ctx := ctrl.SetupSignalHandler()
if err = kcmv1.SetupIndexers(ctx, mgr); err != nil {
setupLog.Error(err, "unable to setup indexers")
Expand Down Expand Up @@ -228,20 +221,7 @@ func main() {
setupLog.Error(err, "unable to create controller", "controller", "ProviderTemplate")
os.Exit(1)
}
if err = (&controller.ClusterDeploymentReconciler{
Client: mgr.GetClient(),
Config: mgr.GetConfig(),
DynamicClient: dc,
SystemNamespace: currentNamespace,
}).SetupWithManager(mgr); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "ClusterDeployment")
os.Exit(1)
}
if err = (&controller.ManagementReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Config: mgr.GetConfig(),
DynamicClient: dc,
SystemNamespace: currentNamespace,
CreateAccessManagement: createAccessManagement,
}).SetupWithManager(mgr); err != nil {
Expand Down Expand Up @@ -310,13 +290,6 @@ func main() {
os.Exit(1)
}

if err = (&controller.MultiClusterServiceReconciler{
Client: mgr.GetClient(),
SystemNamespace: currentNamespace,
}).SetupWithManager(mgr); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "MultiClusterService")
os.Exit(1)
}
// TODO (zerospiel): disabled until the #605
// if err = (&controller.BackupReconciler{
// Client: mgr.GetClient(),
Expand Down
23 changes: 13 additions & 10 deletions internal/controller/clusterdeployment_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ type helmActor interface {

// ClusterDeploymentReconciler reconciles a ClusterDeployment object
type ClusterDeploymentReconciler struct {
client.Client
Client client.Client
helmActor
Config *rest.Config
DynamicClient *dynamic.DynamicClient
Expand All @@ -80,7 +80,7 @@ func (r *ClusterDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Re
l.Info("Reconciling ClusterDeployment")

clusterDeployment := &kcm.ClusterDeployment{}
if err := r.Get(ctx, req.NamespacedName, clusterDeployment); err != nil {
if err := r.Client.Get(ctx, req.NamespacedName, clusterDeployment); err != nil {
if apierrors.IsNotFound(err) {
l.Info("ClusterDeployment not found, ignoring since object must be deleted")
return ctrl.Result{}, nil
Expand All @@ -98,7 +98,7 @@ func (r *ClusterDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Re
if clusterDeployment.Status.ObservedGeneration == 0 {
mgmt := &kcm.Management{}
mgmtRef := client.ObjectKey{Name: kcm.ManagementName}
if err := r.Get(ctx, mgmtRef, mgmt); err != nil {
if err := r.Client.Get(ctx, mgmtRef, mgmt); err != nil {
l.Error(err, "Failed to get Management object")
return ctrl.Result{}, err
}
Expand Down Expand Up @@ -167,7 +167,7 @@ func (r *ClusterDeploymentReconciler) reconcileUpdate(ctx context.Context, mc *k
err = errors.Join(err, r.updateStatus(ctx, mc, clusterTpl))
}()

if err = r.Get(ctx, client.ObjectKey{Name: mc.Spec.Template, Namespace: mc.Namespace}, clusterTpl); err != nil {
if err = r.Client.Get(ctx, client.ObjectKey{Name: mc.Spec.Template, Namespace: mc.Namespace}, clusterTpl); err != nil {
l.Error(err, "Failed to get Template")
errMsg := fmt.Sprintf("failed to get provided template: %s", err)
if apierrors.IsNotFound(err) {
Expand Down Expand Up @@ -489,7 +489,7 @@ func (r *ClusterDeploymentReconciler) updateServices(ctx context.Context, mc *kc
// will ultimately return the error in servicesErr instead of nil.
profile := sveltosv1beta1.Profile{}
profileRef := client.ObjectKey{Name: mc.Name, Namespace: mc.Namespace}
if servicesErr = r.Get(ctx, profileRef, &profile); servicesErr != nil {
if servicesErr = r.Client.Get(ctx, profileRef, &profile); servicesErr != nil {
servicesErr = fmt.Errorf("failed to get Profile %s to fetch status from its associated ClusterSummary: %w", profileRef.String(), servicesErr)
return ctrl.Result{}, nil
}
Expand All @@ -514,7 +514,7 @@ func (r *ClusterDeploymentReconciler) updateStatus(ctx context.Context, clusterD
return errors.New("failed to set available upgrades")
}

if err := r.Status().Update(ctx, clusterDeployment); err != nil {
if err := r.Client.Status().Update(ctx, clusterDeployment); err != nil {
return fmt.Errorf("failed to update status for clusterDeployment %s/%s: %w", clusterDeployment.Namespace, clusterDeployment.Name, err)
}

Expand All @@ -538,7 +538,7 @@ func (r *ClusterDeploymentReconciler) Delete(ctx context.Context, clusterDeploym

hr := &hcv2.HelmRelease{}

if err := r.Get(ctx, client.ObjectKeyFromObject(clusterDeployment), hr); err != nil {
if err := r.Client.Get(ctx, client.ObjectKeyFromObject(clusterDeployment), hr); err != nil {
if !apierrors.IsNotFound(err) {
return ctrl.Result{}, err
}
Expand Down Expand Up @@ -637,7 +637,7 @@ func (r *ClusterDeploymentReconciler) releaseCluster(ctx context.Context, namesp
func (r *ClusterDeploymentReconciler) getInfraProvidersNames(ctx context.Context, templateNamespace, templateName string) ([]string, error) {
template := &kcm.ClusterTemplate{}
templateRef := client.ObjectKey{Name: templateName, Namespace: templateNamespace}
if err := r.Get(ctx, templateRef, template); err != nil {
if err := r.Client.Get(ctx, templateRef, template); err != nil {
ctrl.LoggerFrom(ctx).Error(err, "Failed to get ClusterTemplate", "template namespace", templateNamespace, "template name", templateName)
return nil, err
}
Expand Down Expand Up @@ -806,7 +806,7 @@ func (r *ClusterDeploymentReconciler) setAvailableUpgrades(ctx context.Context,
return nil
}
chains := &kcm.ClusterTemplateChainList{}
err := r.List(ctx, chains,
err := r.Client.List(ctx, chains,
client.InNamespace(template.Namespace),
client.MatchingFields{kcm.TemplateChainSupportedTemplatesIndexKey: template.GetName()},
)
Expand Down Expand Up @@ -835,7 +835,10 @@ func (r *ClusterDeploymentReconciler) setAvailableUpgrades(ctx context.Context,

// SetupWithManager sets up the controller with the Manager.
func (r *ClusterDeploymentReconciler) SetupWithManager(mgr ctrl.Manager) error {
r.helmActor = helm.NewActor(r.Config, r.RESTMapper())
r.Client = mgr.GetClient()
r.Config = mgr.GetConfig()

r.helmActor = helm.NewActor(r.Config, r.Client.RESTMapper())

return ctrl.NewControllerManagedBy(mgr).
For(&kcm.ClusterDeployment{}).
Expand Down
84 changes: 72 additions & 12 deletions internal/controller/management_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ import (
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
"sigs.k8s.io/controller-runtime/pkg/manager"

kcm "github.com/K0rdent/kcm/api/v1alpha1"
"github.com/K0rdent/kcm/internal/certmanager"
Expand All @@ -49,20 +50,22 @@ import (

// ManagementReconciler reconciles a Management object
type ManagementReconciler struct {
client.Client
Scheme *runtime.Scheme
Config *rest.Config
DynamicClient *dynamic.DynamicClient
SystemNamespace string
CreateAccessManagement bool
Client client.Client
Manager manager.Manager
Scheme *runtime.Scheme
Config *rest.Config
DynamicClient *dynamic.DynamicClient
SystemNamespace string
CreateAccessManagement bool
sveltosDependentControllersStarted bool
}

func (r *ManagementReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
l := ctrl.LoggerFrom(ctx)
l.Info("Reconciling Management")

management := &kcm.Management{}
if err := r.Get(ctx, req.NamespacedName, management); err != nil {
if err := r.Client.Get(ctx, req.NamespacedName, management); err != nil {
if apierrors.IsNotFound(err) {
l.Info("Management not found, ignoring since object must be deleted")
return ctrl.Result{}, nil
Expand Down Expand Up @@ -145,7 +148,7 @@ func (r *ManagementReconciler) Update(ctx context.Context, management *kcm.Manag
continue
}
template := new(kcm.ProviderTemplate)
if err := r.Get(ctx, client.ObjectKey{Name: component.Template}, template); err != nil {
if err := r.Client.Get(ctx, client.ObjectKey{Name: component.Template}, template); err != nil {
errMsg := fmt.Sprintf("Failed to get ProviderTemplate %s: %s", component.Template, err)
updateComponentsStatus(statusAccumulator, component, nil, errMsg)
errs = errors.Join(errs, errors.New(errMsg))
Expand Down Expand Up @@ -196,7 +199,15 @@ func (r *ManagementReconciler) Update(ctx context.Context, management *kcm.Manag
management.Status.ObservedGeneration = management.Generation
management.Status.Release = management.Spec.Release

if err := r.Status().Update(ctx, management); err != nil {
shouldRequeue, err := r.startDependentControllers(ctx, management)
if err != nil {
return ctrl.Result{}, err
}
if shouldRequeue {
requeue = true
}

if err := r.Client.Status().Update(ctx, management); err != nil {
errs = errors.Join(errs, fmt.Errorf("failed to update status for Management %s: %w", management.Name, err))
}

Expand All @@ -211,6 +222,44 @@ func (r *ManagementReconciler) Update(ctx context.Context, management *kcm.Manag
return ctrl.Result{}, nil
}

// startDependentControllers starts controllers that cannot be started
// at process startup because of some dependency like CRDs being present.
func (r *ManagementReconciler) startDependentControllers(ctx context.Context, management *kcm.Management) (requue bool, err error) {
l := ctrl.LoggerFrom(ctx)

if r.sveltosDependentControllersStarted {
// Only need to start controllers once.
return false, nil
}

if !management.Status.Components[kcm.ProviderSveltosName].Success {
l.Info(fmt.Sprintf("Waiting for %s provider to be ready to setup contollers dependent on it", kcm.ProviderSveltosName))
return true, nil
}

currentNamespace := utils.CurrentNamespace()

l.Info(fmt.Sprintf("Provider %s has been successfully installed, so setting up controller for ClusterDeployment", kcm.ProviderSveltosName))
if err = (&ClusterDeploymentReconciler{
DynamicClient: r.DynamicClient,
SystemNamespace: currentNamespace,
}).SetupWithManager(r.Manager); err != nil {
return false, fmt.Errorf("failed to setup controller for ClusterDeployment: %w", err)
}
l.Info("Setup for ClusterDeployment controller successful")

l.Info(fmt.Sprintf("Provider %s has been successfully installed, so setting up controller for MultiClusterService", kcm.ProviderSveltosName))
if err = (&MultiClusterServiceReconciler{
SystemNamespace: currentNamespace,
}).SetupWithManager(r.Manager); err != nil {
return false, fmt.Errorf("failed to setup controller for MultiClusterService: %w", err)
}
l.Info("Setup for MultiClusterService controller successful")

r.sveltosDependentControllersStarted = true
return false, nil
}

func (r *ManagementReconciler) cleanupRemovedComponents(ctx context.Context, management *kcm.Management) error {
var (
errs error
Expand Down Expand Up @@ -270,7 +319,7 @@ func (r *ManagementReconciler) ensureAccessManagement(ctx context.Context, mgmt
},
},
}
err := r.Get(ctx, client.ObjectKey{
err := r.Client.Get(ctx, client.ObjectKey{
Name: kcm.AccessManagementName,
}, amObj)
if err == nil {
Expand All @@ -279,7 +328,7 @@ func (r *ManagementReconciler) ensureAccessManagement(ctx context.Context, mgmt
if !apierrors.IsNotFound(err) {
return fmt.Errorf("failed to get %s AccessManagement object: %w", kcm.AccessManagementName, err)
}
err = r.Create(ctx, amObj)
err = r.Client.Create(ctx, amObj)
if err != nil {
return fmt.Errorf("failed to create %s AccessManagement object: %w", kcm.AccessManagementName, err)
}
Expand All @@ -294,7 +343,7 @@ func (r *ManagementReconciler) ensureAccessManagement(ctx context.Context, mgmt
func (r *ManagementReconciler) checkProviderStatus(ctx context.Context, component component) error {
helmReleaseName := component.helmReleaseName
hr := &fluxv2.HelmRelease{}
err := r.Get(ctx, types.NamespacedName{Namespace: r.SystemNamespace, Name: helmReleaseName}, hr)
err := r.Client.Get(ctx, types.NamespacedName{Namespace: r.SystemNamespace, Name: helmReleaseName}, hr)
if err != nil {
return fmt.Errorf("failed to check provider status: %w", err)
}
Expand Down Expand Up @@ -622,6 +671,17 @@ func updateComponentsStatus(

// SetupWithManager sets up the controller with the Manager.
func (r *ManagementReconciler) SetupWithManager(mgr ctrl.Manager) error {
dc, err := dynamic.NewForConfig(mgr.GetConfig())
if err != nil {
return fmt.Errorf("failed to create dynamic client: %w", err)
}

r.Manager = mgr
r.Client = mgr.GetClient()
r.Scheme = mgr.GetScheme()
r.Config = mgr.GetConfig()
r.DynamicClient = dc

return ctrl.NewControllerManagedBy(mgr).
For(&kcm.Management{}).
Complete(r)
Expand Down
2 changes: 1 addition & 1 deletion internal/controller/management_controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -277,8 +277,8 @@ var _ = Describe("Management Controller", func() {
controllerReconciler := &ManagementReconciler{
Client: k8sClient,
Scheme: k8sClient.Scheme(),
SystemNamespace: utils.DefaultSystemNamespace,
DynamicClient: dynamicClient,
SystemNamespace: utils.DefaultSystemNamespace,
}

_, err := controllerReconciler.Reconcile(ctx, reconcile.Request{
Expand Down
12 changes: 7 additions & 5 deletions internal/controller/multiclusterservice_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ import (

// MultiClusterServiceReconciler reconciles a MultiClusterService object
type MultiClusterServiceReconciler struct {
client.Client
Client client.Client
SystemNamespace string
}

Expand All @@ -52,7 +52,7 @@ func (r *MultiClusterServiceReconciler) Reconcile(ctx context.Context, req ctrl.
l.Info("Reconciling MultiClusterService")

mcs := &kcm.MultiClusterService{}
err := r.Get(ctx, req.NamespacedName, mcs)
err := r.Client.Get(ctx, req.NamespacedName, mcs)
if apierrors.IsNotFound(err) {
l.Info("MultiClusterService not found, ignoring since object must be deleted")
return ctrl.Result{}, nil
Expand All @@ -72,7 +72,7 @@ func (r *MultiClusterServiceReconciler) Reconcile(ctx context.Context, req ctrl.

func (r *MultiClusterServiceReconciler) reconcileUpdate(ctx context.Context, mcs *kcm.MultiClusterService) (_ ctrl.Result, err error) {
if utils.AddLabel(mcs, kcm.GenericComponentLabelName, kcm.GenericComponentLabelValueKCM) {
if err := r.Update(ctx, mcs); err != nil {
if err := r.Client.Update(ctx, mcs); err != nil {
return ctrl.Result{}, fmt.Errorf("failed to update labels: %w", err)
}
return ctrl.Result{Requeue: true}, nil // generation has not changed, need explicit requeue
Expand Down Expand Up @@ -153,7 +153,7 @@ func (r *MultiClusterServiceReconciler) reconcileUpdate(ctx context.Context, mcs
// will ultimately return the error in servicesErr instead of nil.
profile := sveltosv1beta1.ClusterProfile{}
profileRef := client.ObjectKey{Name: mcs.Name}
if servicesErr = r.Get(ctx, profileRef, &profile); servicesErr != nil {
if servicesErr = r.Client.Get(ctx, profileRef, &profile); servicesErr != nil {
servicesErr = fmt.Errorf("failed to get ClusterProfile %s to fetch status from its associated ClusterSummary: %w", profileRef.String(), servicesErr)
return ctrl.Result{}, nil
}
Expand All @@ -173,7 +173,7 @@ func (r *MultiClusterServiceReconciler) updateStatus(ctx context.Context, mcs *k
mcs.Status.ObservedGeneration = mcs.Generation
mcs.Status.Conditions = updateStatusConditions(mcs.Status.Conditions, "MultiClusterService is ready")

if err := r.Status().Update(ctx, mcs); err != nil {
if err := r.Client.Status().Update(ctx, mcs); err != nil {
return fmt.Errorf("failed to update status for MultiClusterService %s/%s: %w", mcs.Namespace, mcs.Name, err)
}

Expand Down Expand Up @@ -311,6 +311,8 @@ func requeueSveltosProfileForClusterSummary(ctx context.Context, obj client.Obje

// SetupWithManager sets up the controller with the Manager.
func (r *MultiClusterServiceReconciler) SetupWithManager(mgr ctrl.Manager) error {
r.Client = mgr.GetClient()

return ctrl.NewControllerManagedBy(mgr).
For(&kcm.MultiClusterService{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})).
Watches(&sveltosv1beta1.ClusterSummary{},
Expand Down

0 comments on commit 2007ea1

Please sign in to comment.