Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow to override options for specific controllers #2030

Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 25 additions & 1 deletion cmd/provider/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"context"
"os"
"path/filepath"
"strings"
"time"

xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1"
Expand All @@ -41,9 +42,14 @@ import (
"github.com/crossplane-contrib/provider-aws/apis/v1alpha1"
"github.com/crossplane-contrib/provider-aws/pkg/controller"
"github.com/crossplane-contrib/provider-aws/pkg/features"
utilscontroller "github.com/crossplane-contrib/provider-aws/pkg/utils/controller"
"github.com/crossplane-contrib/provider-aws/pkg/utils/metrics"
)

// Env prefix for options to configure controllers.
// Example usage: `PROVIDER_AWS_ec2.instance.pollInterval=10m`.
const OPTION_ENV_PREFIX = "PROVIDER_AWS_"

func main() {
var (
app = kingpin.New(filepath.Base(os.Args[0]), "AWS support for Crossplane.").DefaultEnvars()
Expand Down Expand Up @@ -126,8 +132,11 @@ func main() {
log.Info("Alpha feature enabled", "flag", features.EnableAlphaManagementPolicies)
}

optionsWithOverrides := utilscontroller.NewOptions(o)
kingpin.FatalIfError(optionsWithOverrides.AddOverrides(optionsOverridesFromEnv()), "Cannot add overrides")

kingpin.FatalIfError(metrics.SetupMetrics(), "Cannot setup AWS metrics hook")
kingpin.FatalIfError(controller.Setup(mgr, o), "Cannot setup AWS controllers")
kingpin.FatalIfError(controller.Setup(mgr, optionsWithOverrides), "Cannot setup AWS controllers")
kingpin.FatalIfError(mgr.Start(ctrl.SetupSignalHandler()), "Cannot start controller manager")

}
Expand All @@ -138,3 +147,18 @@ func UseISO8601() zap.Opts {
o.TimeEncoder = zapcore.ISO8601TimeEncoder
}
}

// Collects all env variables with the prefix OPTION_ENV_PREFIX and returns them as a map
// with the prefix removed.
func optionsOverridesFromEnv() map[string]string {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Doesn't kingpin support natively loading flags from environment variables?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see it has binding of particular flag to an env variable. This will add ~320 new flags and it'll still require some code to collect these flags into overrides map.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not a maintainer of this provider anymore, so I'll defer to @MisterMX and @chlunde.

That said, I'm a bit concerned about:

  • Adding a feature to only this provider to tune poll interval per resource type
  • Introducing a new path to configure the provider (environment variables)

I think if this is a problem for this provider it's probably a problem for other providers. It would be ideal to agree on the problem, and a standard way to address it. Usually to do that we'd write a one-pager in the crossplane/crossplane repo, that provider owners can adopt.

@ulucinar is the TL of Upbound's extensions team (he owns tools like upjet that generate a lot of the other providers). He might be a good person to review a one-pager if you're interested in writing one.

result := make(map[string]string)
for _, str := range os.Environ() {
if rest, ok := strings.CutPrefix(str, OPTION_ENV_PREFIX); ok {
parts := strings.SplitN(rest, "=", 2)
if len(parts) == 2 {
result[parts[0]] = parts[1]
}
}
}
return result
}
119 changes: 59 additions & 60 deletions pkg/controller/aws.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ limitations under the License.
package controller

import (
"github.com/crossplane/crossplane-runtime/pkg/controller"
ctrl "sigs.k8s.io/controller-runtime"

"github.com/crossplane-contrib/provider-aws/pkg/controller/acm"
Expand Down Expand Up @@ -76,69 +75,69 @@ import (
"github.com/crossplane-contrib/provider-aws/pkg/controller/sns"
"github.com/crossplane-contrib/provider-aws/pkg/controller/sqs"
"github.com/crossplane-contrib/provider-aws/pkg/controller/transfer"
"github.com/crossplane-contrib/provider-aws/pkg/utils/controller"
"github.com/crossplane-contrib/provider-aws/pkg/utils/setup"
)

// Setup creates all AWS controllers with the supplied logger and adds them to
// the supplied manager.
func Setup(mgr ctrl.Manager, o controller.Options) error {
return setup.SetupControllers(
mgr, o,
acm.Setup,
acmpca.Setup,
apigateway.Setup,
apigatewayv2.Setup,
athena.Setup,
autoscaling.Setup,
batch.Setup,
cache.Setup,
cloudfront.Setup,
cloudsearch.Setup,
cloudwatchlogs.Setup,
cognitoidentity.Setup,
cognitoidentityprovider.Setup,
config.Setup,
database.Setup,
dax.Setup,
docdb.Setup,
dynamodb.Setup,
ec2.Setup,
ecr.Setup,
ecs.Setup,
efs.Setup,
eks.Setup,
elasticache.Setup,
elasticloadbalancing.Setup,
elbv2.Setup,
emrcontainers.Setup,
firehose.Setup,
glue.Setup,
globalaccelerator.Setup,
iam.Setup,
iot.Setup,
kafka.Setup,
kinesis.Setup,
kms.Setup,
lambda.Setup,
mq.Setup,
mwaa.Setup,
neptune.Setup,
opensearchservice.Setup,
prometheusservice.Setup,
ram.Setup,
rds.Setup,
redshift.Setup,
route53.Setup,
route53resolver.Setup,
s3.Setup,
s3control.Setup,
secretsmanager.Setup,
servicecatalog.Setup,
servicediscovery.Setup,
sesv2.Setup,
sfn.Setup,
sns.Setup,
sqs.Setup,
transfer.Setup,
)
b := setup.NewBatch(mgr, o, "")
b.AddUnscoped(acm.Setup)
b.AddUnscoped(acmpca.Setup)
b.AddUnscoped(apigateway.Setup)
b.AddUnscoped(apigatewayv2.Setup)
b.AddUnscoped(athena.Setup)
b.AddUnscoped(autoscaling.Setup)
b.AddUnscoped(batch.Setup)
b.AddUnscoped(cache.Setup)
b.AddUnscoped(cloudfront.Setup)
b.AddUnscoped(cloudsearch.Setup)
b.AddUnscoped(cloudwatchlogs.Setup)
b.AddUnscoped(cognitoidentity.Setup)
b.AddUnscoped(cognitoidentityprovider.Setup)
b.AddUnscoped(config.Setup)
b.AddUnscoped(database.Setup)
b.AddUnscoped(dax.Setup)
b.AddUnscoped(docdb.Setup)
b.AddUnscoped(dynamodb.Setup)
b.AddProxy(ec2.Setup)
b.AddUnscoped(ecr.Setup)
b.AddUnscoped(ecs.Setup)
b.AddUnscoped(efs.Setup)
b.AddUnscoped(eks.Setup)
b.AddUnscoped(elasticache.Setup)
b.AddUnscoped(elasticloadbalancing.Setup)
b.AddUnscoped(elbv2.Setup)
b.AddUnscoped(emrcontainers.Setup)
b.AddUnscoped(firehose.Setup)
b.AddUnscoped(glue.Setup)
b.AddUnscoped(globalaccelerator.Setup)
b.AddUnscoped(iam.Setup)
b.AddUnscoped(iot.Setup)
b.AddUnscoped(kafka.Setup)
b.AddUnscoped(kinesis.Setup)
b.AddUnscoped(kms.Setup)
b.AddUnscoped(lambda.Setup)
b.AddUnscoped(mq.Setup)
b.AddUnscoped(mwaa.Setup)
b.AddUnscoped(neptune.Setup)
b.AddUnscoped(opensearchservice.Setup)
b.AddUnscoped(prometheusservice.Setup)
b.AddUnscoped(ram.Setup)
b.AddUnscoped(rds.Setup)
b.AddUnscoped(redshift.Setup)
b.AddProxy(route53.Setup)
b.AddUnscoped(route53resolver.Setup)
b.AddUnscoped(s3.Setup)
b.AddUnscoped(s3control.Setup)
b.AddUnscoped(secretsmanager.Setup)
b.AddUnscoped(servicecatalog.Setup)
b.AddUnscoped(servicediscovery.Setup)
b.AddUnscoped(sesv2.Setup)
b.AddUnscoped(sfn.Setup)
b.AddUnscoped(sns.Setup)
b.AddUnscoped(sqs.Setup)
b.AddUnscoped(transfer.Setup)
return b.Run()
}
51 changes: 25 additions & 26 deletions pkg/controller/ec2/setup.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ limitations under the License.
package ec2

import (
"github.com/crossplane/crossplane-runtime/pkg/controller"
ctrl "sigs.k8s.io/controller-runtime"

"github.com/crossplane-contrib/provider-aws/pkg/controller/ec2/address"
Expand All @@ -42,34 +41,34 @@ import (
"github.com/crossplane-contrib/provider-aws/pkg/controller/ec2/vpcendpoint"
"github.com/crossplane-contrib/provider-aws/pkg/controller/ec2/vpcendpointserviceconfiguration"
"github.com/crossplane-contrib/provider-aws/pkg/controller/ec2/vpcpeeringconnection"
"github.com/crossplane-contrib/provider-aws/pkg/utils/controller"
"github.com/crossplane-contrib/provider-aws/pkg/utils/setup"
)

// Setup ec2 controllers.
func Setup(mgr ctrl.Manager, o controller.Options) error {
return setup.SetupControllers(
mgr, o,
address.SetupAddress,
flowlog.SetupFlowLog,
instance.SetupInstance,
internetgateway.SetupInternetGateway,
launchtemplate.SetupLaunchTemplate,
launchtemplateversion.SetupLaunchTemplateVersion,
natgateway.SetupNatGateway,
route.SetupRoute,
routetable.SetupRouteTable,
securitygroup.SetupSecurityGroup,
securitygrouprule.SetupSecurityGroupRule,
subnet.SetupSubnet,
transitgateway.SetupTransitGateway,
transitgatewayroute.SetupTransitGatewayRoute,
transitgatewayroutetable.SetupTransitGatewayRouteTable,
transitgatewayvpcattachment.SetupTransitGatewayVPCAttachment,
volume.SetupVolume,
vpc.SetupVPC,
vpccidrblock.SetupVPCCIDRBlock,
vpcendpoint.SetupVPCEndpoint,
vpcendpointserviceconfiguration.SetupVPCEndpointServiceConfiguration,
vpcpeeringconnection.SetupVPCPeeringConnection,
)
batch := setup.NewBatch(mgr, o, "ec2")
batch.Add("address", address.SetupAddress)
batch.Add("flowlog", flowlog.SetupFlowLog)
batch.Add("instance", instance.SetupInstance)
batch.Add("internetgateway", internetgateway.SetupInternetGateway)
batch.Add("launchtemplate", launchtemplate.SetupLaunchTemplate)
batch.Add("launchtemplateversion", launchtemplateversion.SetupLaunchTemplateVersion)
batch.Add("natgateway", natgateway.SetupNatGateway)
batch.Add("route", route.SetupRoute)
batch.Add("routetable", routetable.SetupRouteTable)
batch.Add("securitygroup", securitygroup.SetupSecurityGroup)
batch.Add("securitygrouprule", securitygrouprule.SetupSecurityGroupRule)
batch.Add("subnet", subnet.SetupSubnet)
batch.Add("transitgateway", transitgateway.SetupTransitGateway)
batch.Add("transitgatewayroute", transitgatewayroute.SetupTransitGatewayRoute)
batch.Add("transitgatewayroutetable", transitgatewayroutetable.SetupTransitGatewayRouteTable)
batch.Add("transitgatewayvpcattachment", transitgatewayvpcattachment.SetupTransitGatewayVPCAttachment)
batch.Add("volume", volume.SetupVolume)
batch.Add("vpc", vpc.SetupVPC)
batch.Add("vpccidrblock", vpccidrblock.SetupVPCCIDRBlock)
batch.Add("vpcendpoint", vpcendpoint.SetupVPCEndpoint)
batch.Add("vpcendpointserviceconfiguration", vpcendpointserviceconfiguration.SetupVPCEndpointServiceConfiguration)
batch.Add("vpcpeeringconnection", vpcpeeringconnection.SetupVPCPeeringConnection)
return batch.Run()
}
11 changes: 5 additions & 6 deletions pkg/controller/route53/setup.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,19 +17,18 @@ limitations under the License.
package route53

import (
"github.com/crossplane/crossplane-runtime/pkg/controller"
ctrl "sigs.k8s.io/controller-runtime"

"github.com/crossplane-contrib/provider-aws/pkg/controller/route53/hostedzone"
"github.com/crossplane-contrib/provider-aws/pkg/controller/route53/resourcerecordset"
"github.com/crossplane-contrib/provider-aws/pkg/utils/controller"
"github.com/crossplane-contrib/provider-aws/pkg/utils/setup"
)

// Setup route53 controllers.
func Setup(mgr ctrl.Manager, o controller.Options) error {
return setup.SetupControllers(
mgr, o,
hostedzone.SetupHostedZone,
resourcerecordset.SetupResourceRecordSet,
)
batch := setup.NewBatch(mgr, o, "route53")
batch.Add("hostedzone", hostedzone.SetupHostedZone)
batch.Add("resourcerecordset", resourcerecordset.SetupResourceRecordSet)
return batch.Run()
}
101 changes: 101 additions & 0 deletions pkg/utils/controller/options.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
package controller

import (
"fmt"
"strconv"
"strings"
"time"

"github.com/crossplane/crossplane-runtime/pkg/controller"
)

// Options allows to override controller.Options for specific controllers.
type Options struct {
defaultOptions controller.Options
specific map[string]OptionsOverride
}

// OptionsOverride allows to override specific controller.Options properties.
type OptionsOverride struct {
PollInterval *time.Duration
MaxConcurrentReconciles *int
}

func (override OptionsOverride) applyTo(options *controller.Options) {
if override.PollInterval != nil {
options.PollInterval = *override.PollInterval
}

if override.MaxConcurrentReconciles != nil {
options.MaxConcurrentReconciles = *override.MaxConcurrentReconciles
}
}

func NewOptions(defaultOptions controller.Options) Options {
return Options{
defaultOptions: defaultOptions,
specific: map[string]OptionsOverride{},
}
}

// AddOverrides adds overrides for specific controllers from the provided map
// which is similar to ConfigMap data.
// Key format is "<scope>.<property>". Properties without scope or with "default" scope
// owerride default values.
func (options *Options) AddOverrides(values map[string]string) error {
for key, value := range values {
if err := options.addOverride(key, value); err != nil {
return fmt.Errorf("failed to add override for %s: %w", key, err)
}
}
return nil
}

func (options *Options) addOverride(key, value string) error {
propSeparatorIdx := strings.LastIndex(key, ".")
propName := key
scope := "default"
if propSeparatorIdx != -1 {
propName = key[propSeparatorIdx+1:]
scope = key[:propSeparatorIdx]
}
overrides := options.specific[scope]

switch propName {
case "pollInterval":
if duration, err := time.ParseDuration(value); err != nil {
return fmt.Errorf("failed to parse pollInterval value %s: %w", value, err)
} else {
overrides.PollInterval = &duration
}
case "maxConcurrentReconciles":
if maxConcurrentReconciles, err := strconv.Atoi(value); err != nil {
return fmt.Errorf("failed to parse maxConcurrentReconciles value %s: %w", value, err)
} else {
overrides.MaxConcurrentReconciles = &maxConcurrentReconciles
}
default:
return fmt.Errorf("unknown override property %s", propName)
}

if scope == "default" {
overrides.applyTo(&options.defaultOptions)
} else {
options.specific[scope] = overrides
}
return nil
}

// Default returns default controller.Options.
func (options Options) Default() controller.Options {
return options.defaultOptions
}

// Get returns controller.Options for the specific controller.
func (options Options) Get(name string) controller.Options {
result := options.defaultOptions
if override, ok := options.specific[name]; ok {
override.applyTo(&result)
}
return result
}
Loading
Loading