From 42a6833b8ccd8f11509640f75725886275fffc1b Mon Sep 17 00:00:00 2001 From: Kyle Eckhart Date: Tue, 2 Jul 2024 06:18:25 -0400 Subject: [PATCH] Start a unified scraper (#1432) --- go.mod | 3 + go.sum | 8 + pkg/job/cloudwatchrunner/customnamespace.go | 32 ++ pkg/job/cloudwatchrunner/discovery.go | 33 ++ pkg/job/cloudwatchrunner/runner.go | 19 + pkg/job/listmetrics/processor.go | 10 + pkg/job/resourcemetadata/resource.go | 26 + pkg/job/scraper.go | 265 ++++++++++ pkg/job/scraper_test.go | 550 ++++++++++++++++++++ 9 files changed, 946 insertions(+) create mode 100644 pkg/job/cloudwatchrunner/customnamespace.go create mode 100644 pkg/job/cloudwatchrunner/discovery.go create mode 100644 pkg/job/cloudwatchrunner/runner.go create mode 100644 pkg/job/listmetrics/processor.go create mode 100644 pkg/job/resourcemetadata/resource.go create mode 100644 pkg/job/scraper.go create mode 100644 pkg/job/scraper_test.go diff --git a/go.mod b/go.mod index a212a0c9..312bb1f0 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db github.com/prometheus/client_golang v1.19.1 github.com/prometheus/common v0.54.0 + github.com/r3labs/diff/v3 v3.0.1 github.com/stretchr/testify v1.9.0 github.com/urfave/cli/v2 v2.27.2 golang.org/x/sync v0.7.0 @@ -50,6 +51,8 @@ require ( github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect + github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect golang.org/x/sys v0.19.0 // indirect google.golang.org/protobuf v1.34.0 // indirect diff --git a/go.sum b/go.sum index d65f6fa5..e9751e6f 100644 --- a/go.sum +++ b/go.sum @@ -84,15 +84,22 @@ github.com/prometheus/common v0.54.0 h1:ZlZy0BgJhTwVZUn7dLOkwCZHUkrAqd3WYtcFCWnM github.com/prometheus/common v0.54.0/go.mod h1:/TQgMJP5CuVYveyT7n/0Ix8yLNNXy9yRSkhnLTHPDIQ= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/r3labs/diff/v3 v3.0.1 h1:CBKqf3XmNRHXKmdU7mZP1w7TV0pDyVCis1AUHtA4Xtg= +github.com/r3labs/diff/v3 v3.0.1/go.mod h1:f1S9bourRbiM66NskseyUdo0fTmEE0qKrikYJX63dgo= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/urfave/cli/v2 v2.27.2 h1:6e0H+AkS+zDckwPCUrZkKX38mRaau4nL2uipkJpbkcI= github.com/urfave/cli/v2 v2.27.2/go.mod h1:g0+79LmHHATl7DAcHO99smiR/T7uGLw84w8Y42x+4eM= +github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= +github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= +github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= +github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 h1:+qGGcbkzsfDQNPPe9UDgpxAWQrhbbBXOYJFQDq/dtJw= github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913/go.mod h1:4aEEwZQutDLsQv2Deui4iYQ6DWTxR14g6m8Wv88+Xqk= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= @@ -107,5 +114,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/job/cloudwatchrunner/customnamespace.go b/pkg/job/cloudwatchrunner/customnamespace.go new file mode 100644 index 00000000..64a9c894 --- /dev/null +++ b/pkg/job/cloudwatchrunner/customnamespace.go @@ -0,0 +1,32 @@ +package cloudwatchrunner + +import ( + "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/job/listmetrics" + "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/model" +) + +type CustomNamespaceJob struct { + Job model.CustomNamespaceJob +} + +func (c CustomNamespaceJob) Namespace() string { + return c.Job.Namespace +} + +func (c CustomNamespaceJob) listMetricsParams() listmetrics.ProcessingParams { + return listmetrics.ProcessingParams{ + Namespace: c.Job.Namespace, + Metrics: c.Job.Metrics, + RecentlyActiveOnly: c.Job.RecentlyActiveOnly, + DimensionNameRequirements: c.Job.DimensionNameRequirements, + } +} + +func (c CustomNamespaceJob) CustomTags() []model.Tag { + return c.Job.CustomTags +} + +func (c CustomNamespaceJob) resourceEnrichment() ResourceEnrichment { + // TODO add implementation in followup + return nil +} diff --git a/pkg/job/cloudwatchrunner/discovery.go b/pkg/job/cloudwatchrunner/discovery.go new file mode 100644 index 00000000..0b7f5851 --- /dev/null +++ b/pkg/job/cloudwatchrunner/discovery.go @@ -0,0 +1,33 @@ +package cloudwatchrunner + +import ( + "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/job/listmetrics" + "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/model" +) + +type DiscoveryJob struct { + Job model.DiscoveryJob + Resources []*model.TaggedResource +} + +func (d DiscoveryJob) Namespace() string { + return d.Job.Type +} + +func (d DiscoveryJob) CustomTags() []model.Tag { + return d.Job.CustomTags +} + +func (d DiscoveryJob) listMetricsParams() listmetrics.ProcessingParams { + return listmetrics.ProcessingParams{ + Namespace: d.Job.Type, + Metrics: d.Job.Metrics, + RecentlyActiveOnly: d.Job.RecentlyActiveOnly, + DimensionNameRequirements: d.Job.DimensionNameRequirements, + } +} + +func (d DiscoveryJob) resourceEnrichment() ResourceEnrichment { + // TODO add implementation in followup + return nil +} diff --git a/pkg/job/cloudwatchrunner/runner.go b/pkg/job/cloudwatchrunner/runner.go new file mode 100644 index 00000000..27ad1dcd --- /dev/null +++ b/pkg/job/cloudwatchrunner/runner.go @@ -0,0 +1,19 @@ +package cloudwatchrunner + +import ( + "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/job/listmetrics" + "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/job/resourcemetadata" + "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/logging" + "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/model" +) + +type ResourceEnrichment interface { + Create(logger logging.Logger) resourcemetadata.MetricResourceEnricher +} + +type Job interface { + Namespace() string + CustomTags() []model.Tag + listMetricsParams() listmetrics.ProcessingParams + resourceEnrichment() ResourceEnrichment +} diff --git a/pkg/job/listmetrics/processor.go b/pkg/job/listmetrics/processor.go new file mode 100644 index 00000000..9ced0371 --- /dev/null +++ b/pkg/job/listmetrics/processor.go @@ -0,0 +1,10 @@ +package listmetrics + +import "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/model" + +type ProcessingParams struct { + Namespace string + Metrics []*model.MetricConfig + RecentlyActiveOnly bool + DimensionNameRequirements []string +} diff --git a/pkg/job/resourcemetadata/resource.go b/pkg/job/resourcemetadata/resource.go new file mode 100644 index 00000000..66cc07b6 --- /dev/null +++ b/pkg/job/resourcemetadata/resource.go @@ -0,0 +1,26 @@ +package resourcemetadata + +import ( + "context" + + "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/model" +) + +type Resource struct { + // Name is an identifiable value for the resource and is variable dependent on the match made + // It will be the AWS ARN (Amazon Resource Name) if a unique resource was found + // It will be "global" if a unique resource was not found + // CustomNamespaces will have the custom namespace Name + Name string + // Tags is a set of tags associated to the resource + Tags []model.Tag +} + +type Resources struct { + StaticResource *Resource + AssociatedResources []*Resource +} + +type MetricResourceEnricher interface { + Enrich(ctx context.Context, metrics []*model.Metric) ([]*model.Metric, Resources) +} diff --git a/pkg/job/scraper.go b/pkg/job/scraper.go new file mode 100644 index 00000000..f208a827 --- /dev/null +++ b/pkg/job/scraper.go @@ -0,0 +1,265 @@ +package job + +import ( + "context" + "fmt" + "sync" + + "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/job/cloudwatchrunner" + + "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/clients/account" + "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/logging" + "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/model" +) + +type Scraper struct { + jobsCfg model.JobsConfig + logger logging.Logger + runnerFactory runnerFactory +} + +type runnerFactory interface { + GetAccountClient(region string, role model.Role) account.Client + NewResourceMetadataRunner(logger logging.Logger, region string, role model.Role) ResourceMetadataRunner + NewCloudWatchRunner(logger logging.Logger, region string, role model.Role, job cloudwatchrunner.Job) CloudwatchRunner +} + +type ResourceMetadataRunner interface { + Run(ctx context.Context, region string, job model.DiscoveryJob) ([]*model.TaggedResource, error) +} + +type CloudwatchRunner interface { + Run(ctx context.Context) ([]*model.CloudwatchData, error) +} + +func NewScraper(logger logging.Logger, + jobsCfg model.JobsConfig, + runnerFactory runnerFactory, +) *Scraper { + return &Scraper{ + runnerFactory: runnerFactory, + logger: logger, + jobsCfg: jobsCfg, + } +} + +type ErrorType string + +var ( + AccountErr ErrorType = "Account for job was not found" + ResourceMetadataErr ErrorType = "Failed to run resource metadata for job" + CloudWatchCollectionErr ErrorType = "Failed to gather cloudwatch metrics for job" +) + +type Account struct { + ID string + Alias string +} + +func (s Scraper) Scrape(ctx context.Context) ([]model.TaggedResourceResult, []model.CloudwatchMetricResult, []Error) { + // Setup so we only do one GetAccount call per region + role combo when running jobs + roleRegionToAccount := map[model.Role]map[string]func() (Account, error){} + jobConfigVisitor(s.jobsCfg, func(_ any, role model.Role, region string) { + if _, exists := roleRegionToAccount[role]; !exists { + roleRegionToAccount[role] = map[string]func() (Account, error){} + } + roleRegionToAccount[role][region] = sync.OnceValues[Account, error](func() (Account, error) { + client := s.runnerFactory.GetAccountClient(region, role) + accountID, err := client.GetAccount(ctx) + if err != nil { + return Account{}, fmt.Errorf("failed to get Account: %w", err) + } + a := Account{ + ID: accountID, + } + accountAlias, err := client.GetAccountAlias(ctx) + if err != nil { + s.logger.Warn("Failed to get optional account alias from account", "err", err, "account_id", accountID) + } else { + a.Alias = accountAlias + } + return a, nil + }) + }) + + var wg sync.WaitGroup + mux := &sync.Mutex{} + jobErrors := make([]Error, 0) + metricResults := make([]model.CloudwatchMetricResult, 0) + resourceResults := make([]model.TaggedResourceResult, 0) + s.logger.Debug("Starting job runs") + + jobConfigVisitor(s.jobsCfg, func(job any, role model.Role, region string) { + wg.Add(1) + go func() { + defer wg.Done() + + var namespace string + jobAction(s.logger, job, func(job model.DiscoveryJob) { + namespace = job.Type + }, func(job model.CustomNamespaceJob) { + namespace = job.Namespace + }) + jobContext := JobContext{ + Namespace: namespace, + Region: region, + RoleARN: role.RoleArn, + } + jobLogger := s.logger.With("namespace", jobContext.Namespace, "region", jobContext.Region, "arn", jobContext.RoleARN) + + account, err := roleRegionToAccount[role][region]() + if err != nil { + jobError := NewError(jobContext, AccountErr, err) + mux.Lock() + jobErrors = append(jobErrors, jobError) + mux.Unlock() + return + } + jobContext.Account = account + jobLogger = jobLogger.With("account_id", jobContext.Account.ID) + + var jobToRun cloudwatchrunner.Job + jobAction(jobLogger, job, + func(job model.DiscoveryJob) { + jobLogger.Debug("Starting resource discovery") + rmRunner := s.runnerFactory.NewResourceMetadataRunner(jobLogger, region, role) + resources, err := rmRunner.Run(ctx, region, job) + if err != nil { + jobError := NewError(jobContext, ResourceMetadataErr, err) + mux.Lock() + jobErrors = append(jobErrors, jobError) + mux.Unlock() + + return + } + if len(resources) > 0 { + result := model.TaggedResourceResult{ + Context: jobContext.ToScrapeContext(job.CustomTags), + Data: resources, + } + mux.Lock() + resourceResults = append(resourceResults, result) + mux.Unlock() + } else { + jobLogger.Debug("No tagged resources") + } + jobLogger.Debug("Resource discovery finished", "number_of_discovered_resources", len(resources)) + + jobToRun = cloudwatchrunner.DiscoveryJob{Job: job, Resources: resources} + }, func(job model.CustomNamespaceJob) { + jobToRun = cloudwatchrunner.CustomNamespaceJob{Job: job} + }, + ) + if jobToRun == nil { + jobLogger.Debug("Ending job run early due to job error see job errors") + return + } + + jobLogger.Debug("Starting cloudwatch metrics runner") + cwRunner := s.runnerFactory.NewCloudWatchRunner(jobLogger, region, role, jobToRun) + metricResult, err := cwRunner.Run(ctx) + if err != nil { + jobError := NewError(jobContext, CloudWatchCollectionErr, err) + mux.Lock() + jobErrors = append(jobErrors, jobError) + mux.Unlock() + + return + } + + if len(metricResult) == 0 { + jobLogger.Debug("No metrics data found") + return + } + + jobLogger.Debug("Job run finished", "number_of_metrics", len(metricResult)) + + result := model.CloudwatchMetricResult{ + Context: jobContext.ToScrapeContext(jobToRun.CustomTags()), + Data: metricResult, + } + + mux.Lock() + defer mux.Unlock() + metricResults = append(metricResults, result) + }() + }) + wg.Wait() + s.logger.Debug("Finished job runs", "resource_results", len(resourceResults), "metric_results", len(metricResults)) + return resourceResults, metricResults, jobErrors +} + +// Walk through each custom namespace and discovery jobs and take an action +func jobConfigVisitor(jobsCfg model.JobsConfig, action func(job any, role model.Role, region string)) { + for _, job := range jobsCfg.DiscoveryJobs { + for _, role := range job.Roles { + for _, region := range job.Regions { + action(job, role, region) + } + } + } + + for _, job := range jobsCfg.CustomNamespaceJobs { + for _, role := range job.Roles { + for _, region := range job.Regions { + action(job, role, region) + } + } + } +} + +// Take an action depending on the job type, only supports discovery and custom job types +func jobAction(logger logging.Logger, job any, discovery func(job model.DiscoveryJob), custom func(job model.CustomNamespaceJob)) { + // Type switches are free https://stackoverflow.com/a/28027945 + switch typedJob := job.(type) { + case model.DiscoveryJob: + discovery(typedJob) + case model.CustomNamespaceJob: + custom(typedJob) + default: + logger.Error(fmt.Errorf("config type of %T is not supported", typedJob), "Unexpected job type") + return + } +} + +// JobContext exists to track data we want for logging, errors, or other output context that's learned as the job runs +// This makes it easier to track the data additively and morph it to the final shape necessary be it a model.ScrapeContext +// or an Error. It's an exported type for tests but is not part of the public interface +type JobContext struct { //nolint:revive + Account Account + Namespace string + Region string + RoleARN string +} + +func (jc JobContext) ToScrapeContext(customTags []model.Tag) *model.ScrapeContext { + return &model.ScrapeContext{ + AccountID: jc.Account.ID, + Region: jc.Region, + CustomTags: customTags, + AccountAlias: jc.Account.Alias, + } +} + +type Error struct { + JobContext + ErrorType ErrorType + Err error +} + +func NewError(context JobContext, errorType ErrorType, err error) Error { + return Error{ + JobContext: context, + ErrorType: errorType, + Err: err, + } +} + +func (e Error) ToLoggerKeyVals() []interface{} { + return []interface{}{ + "account_id", e.Account.ID, + "namespace", e.Namespace, + "region", e.Region, + "role_arn", e.RoleARN, + } +} diff --git a/pkg/job/scraper_test.go b/pkg/job/scraper_test.go new file mode 100644 index 00000000..82d173d5 --- /dev/null +++ b/pkg/job/scraper_test.go @@ -0,0 +1,550 @@ +package job_test + +import ( + "context" + "errors" + "reflect" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/r3labs/diff/v3" + "github.com/stretchr/testify/assert" + + "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/clients/account" + "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/job" + "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/job/cloudwatchrunner" + "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/logging" + "github.com/nerdswords/yet-another-cloudwatch-exporter/pkg/model" +) + +type testRunnerFactory struct { + GetAccountAliasFunc func() (string, error) + GetAccountFunc func() (string, error) + MetadataRunFunc func(ctx context.Context, region string, job model.DiscoveryJob) ([]*model.TaggedResource, error) + CloudwatchRunFunc func(ctx context.Context, job cloudwatchrunner.Job) ([]*model.CloudwatchData, error) +} + +func (t *testRunnerFactory) GetAccountAlias(context.Context) (string, error) { + return t.GetAccountAliasFunc() +} + +func (t *testRunnerFactory) GetAccount(context.Context) (string, error) { + return t.GetAccountFunc() +} + +func (t *testRunnerFactory) Run(ctx context.Context, region string, job model.DiscoveryJob) ([]*model.TaggedResource, error) { + return t.MetadataRunFunc(ctx, region, job) +} + +func (t *testRunnerFactory) GetAccountClient(string, model.Role) account.Client { + return t +} + +func (t *testRunnerFactory) NewResourceMetadataRunner(logging.Logger, string, model.Role) job.ResourceMetadataRunner { + return &testMetadataRunner{RunFunc: t.MetadataRunFunc} +} + +func (t *testRunnerFactory) NewCloudWatchRunner(_ logging.Logger, _ string, _ model.Role, job cloudwatchrunner.Job) job.CloudwatchRunner { + return &testCloudwatchRunner{Job: job, RunFunc: t.CloudwatchRunFunc} +} + +type testMetadataRunner struct { + RunFunc func(ctx context.Context, region string, job model.DiscoveryJob) ([]*model.TaggedResource, error) +} + +func (t testMetadataRunner) Run(ctx context.Context, region string, job model.DiscoveryJob) ([]*model.TaggedResource, error) { + return t.RunFunc(ctx, region, job) +} + +type testCloudwatchRunner struct { + RunFunc func(ctx context.Context, job cloudwatchrunner.Job) ([]*model.CloudwatchData, error) + Job cloudwatchrunner.Job +} + +func (t testCloudwatchRunner) Run(ctx context.Context) ([]*model.CloudwatchData, error) { + return t.RunFunc(ctx, t.Job) +} + +func TestScrapeRunner_Run(t *testing.T) { + tests := []struct { + name string + jobsCfg model.JobsConfig + getAccountFunc func() (string, error) + getAccountAliasFunc func() (string, error) + metadataRunFunc func(ctx context.Context, region string, job model.DiscoveryJob) ([]*model.TaggedResource, error) + cloudwatchRunFunc func(ctx context.Context, job cloudwatchrunner.Job) ([]*model.CloudwatchData, error) + expectedResources []model.TaggedResourceResult + expectedMetrics []model.CloudwatchMetricResult + expectedErrs []job.Error + }{ + { + name: "can run a discovery job", + jobsCfg: model.JobsConfig{ + DiscoveryJobs: []model.DiscoveryJob{ + { + Regions: []string{"us-east-1"}, + Type: "aws-namespace", + Roles: []model.Role{ + {RoleArn: "aws-arn-1", ExternalID: "external-id-1"}, + }, + }, + }, + }, + getAccountFunc: func() (string, error) { + return "aws-account-1", nil + }, + getAccountAliasFunc: func() (string, error) { + return "my-aws-account", nil + }, + metadataRunFunc: func(_ context.Context, _ string, _ model.DiscoveryJob) ([]*model.TaggedResource, error) { + return []*model.TaggedResource{{ + ARN: "resource-1", Namespace: "aws-namespace", Region: "us-east-1", Tags: []model.Tag{{Key: "tag1", Value: "value1"}}, + }}, nil + }, + cloudwatchRunFunc: func(_ context.Context, _ cloudwatchrunner.Job) ([]*model.CloudwatchData, error) { + return []*model.CloudwatchData{ + { + MetricName: "metric-1", + ResourceName: "resource-1", + Namespace: "aws-namespace", + Tags: []model.Tag{{Key: "tag1", Value: "value1"}}, + Dimensions: []model.Dimension{{Name: "dimension1", Value: "value1"}}, + GetMetricDataResult: &model.GetMetricDataResult{Statistic: "Maximum", Datapoint: aws.Float64(1.0), Timestamp: time.Time{}}, + }, + }, nil + }, + expectedResources: []model.TaggedResourceResult{ + { + Context: &model.ScrapeContext{Region: "us-east-1", AccountID: "aws-account-1", AccountAlias: "my-aws-account"}, + Data: []*model.TaggedResource{ + {ARN: "resource-1", Namespace: "aws-namespace", Region: "us-east-1", Tags: []model.Tag{{Key: "tag1", Value: "value1"}}}, + }, + }, + }, + expectedMetrics: []model.CloudwatchMetricResult{ + { + Context: &model.ScrapeContext{Region: "us-east-1", AccountID: "aws-account-1", AccountAlias: "my-aws-account"}, + Data: []*model.CloudwatchData{ + { + MetricName: "metric-1", + ResourceName: "resource-1", + Namespace: "aws-namespace", + Tags: []model.Tag{{Key: "tag1", Value: "value1"}}, + Dimensions: []model.Dimension{{Name: "dimension1", Value: "value1"}}, + GetMetricDataResult: &model.GetMetricDataResult{Statistic: "Maximum", Datapoint: aws.Float64(1.0), Timestamp: time.Time{}}, + }, + }, + }, + }, + }, + { + name: "can run a custom namespace job", + jobsCfg: model.JobsConfig{ + CustomNamespaceJobs: []model.CustomNamespaceJob{ + { + Regions: []string{"us-east-2"}, + Name: "my-custom-job", + Namespace: "custom-namespace", + Roles: []model.Role{ + {RoleArn: "aws-arn-2", ExternalID: "external-id-2"}, + }, + }, + }, + }, + getAccountFunc: func() (string, error) { + return "aws-account-1", nil + }, + getAccountAliasFunc: func() (string, error) { + return "my-aws-account", nil + }, + cloudwatchRunFunc: func(_ context.Context, _ cloudwatchrunner.Job) ([]*model.CloudwatchData, error) { + return []*model.CloudwatchData{ + { + MetricName: "metric-2", + ResourceName: "resource-2", + Namespace: "custom-namespace", + Dimensions: []model.Dimension{{Name: "dimension2", Value: "value2"}}, + GetMetricDataResult: &model.GetMetricDataResult{Statistic: "Minimum", Datapoint: aws.Float64(2.0), Timestamp: time.Time{}}, + }, + }, nil + }, + expectedMetrics: []model.CloudwatchMetricResult{ + { + Context: &model.ScrapeContext{Region: "us-east-2", AccountID: "aws-account-1", AccountAlias: "my-aws-account"}, + Data: []*model.CloudwatchData{ + { + MetricName: "metric-2", + ResourceName: "resource-2", + Namespace: "custom-namespace", + Dimensions: []model.Dimension{{Name: "dimension2", Value: "value2"}}, + GetMetricDataResult: &model.GetMetricDataResult{Statistic: "Minimum", Datapoint: aws.Float64(2.0), Timestamp: time.Time{}}, + }, + }, + }, + }, + }, + { + name: "can run a discovery and custom namespace job", + jobsCfg: model.JobsConfig{ + DiscoveryJobs: []model.DiscoveryJob{ + { + Regions: []string{"us-east-1"}, + Type: "aws-namespace", + Roles: []model.Role{ + {RoleArn: "aws-arn-1", ExternalID: "external-id-1"}, + }, + }, + }, + CustomNamespaceJobs: []model.CustomNamespaceJob{ + { + Regions: []string{"us-east-2"}, + Name: "my-custom-job", + Namespace: "custom-namespace", + Roles: []model.Role{ + {RoleArn: "aws-arn-2", ExternalID: "external-id-2"}, + }, + }, + }, + }, + getAccountFunc: func() (string, error) { + return "aws-account-1", nil + }, + getAccountAliasFunc: func() (string, error) { + return "my-aws-account", nil + }, + metadataRunFunc: func(_ context.Context, _ string, _ model.DiscoveryJob) ([]*model.TaggedResource, error) { + return []*model.TaggedResource{{ + ARN: "resource-1", Namespace: "aws-namespace", Region: "us-east-1", Tags: []model.Tag{{Key: "tag1", Value: "value1"}}, + }}, nil + }, + cloudwatchRunFunc: func(_ context.Context, job cloudwatchrunner.Job) ([]*model.CloudwatchData, error) { + if job.Namespace() == "custom-namespace" { + return []*model.CloudwatchData{ + { + MetricName: "metric-2", + ResourceName: "resource-2", + Namespace: "custom-namespace", + Dimensions: []model.Dimension{{Name: "dimension2", Value: "value2"}}, + GetMetricDataResult: &model.GetMetricDataResult{Statistic: "Minimum", Datapoint: aws.Float64(2.0), Timestamp: time.Time{}}, + }, + }, nil + } + return []*model.CloudwatchData{ + { + MetricName: "metric-1", + ResourceName: "resource-1", + Namespace: "aws-namespace", + Tags: []model.Tag{{Key: "tag1", Value: "value1"}}, + Dimensions: []model.Dimension{{Name: "dimension1", Value: "value1"}}, + GetMetricDataResult: &model.GetMetricDataResult{Statistic: "Maximum", Datapoint: aws.Float64(1.0), Timestamp: time.Time{}}, + }, + }, nil + }, + expectedResources: []model.TaggedResourceResult{ + { + Context: &model.ScrapeContext{Region: "us-east-1", AccountID: "aws-account-1", AccountAlias: "my-aws-account"}, + Data: []*model.TaggedResource{ + {ARN: "resource-1", Namespace: "aws-namespace", Region: "us-east-1", Tags: []model.Tag{{Key: "tag1", Value: "value1"}}}, + }, + }, + }, + expectedMetrics: []model.CloudwatchMetricResult{ + { + Context: &model.ScrapeContext{Region: "us-east-1", AccountID: "aws-account-1", AccountAlias: "my-aws-account"}, + Data: []*model.CloudwatchData{ + { + MetricName: "metric-1", + ResourceName: "resource-1", + Namespace: "aws-namespace", + Tags: []model.Tag{{Key: "tag1", Value: "value1"}}, + Dimensions: []model.Dimension{{Name: "dimension1", Value: "value1"}}, + GetMetricDataResult: &model.GetMetricDataResult{Statistic: "Maximum", Datapoint: aws.Float64(1.0), Timestamp: time.Time{}}, + }, + }, + }, + { + Context: &model.ScrapeContext{Region: "us-east-2", AccountID: "aws-account-1", AccountAlias: "my-aws-account"}, + Data: []*model.CloudwatchData{ + { + MetricName: "metric-2", + ResourceName: "resource-2", + Namespace: "custom-namespace", + Dimensions: []model.Dimension{{Name: "dimension2", Value: "value2"}}, + GetMetricDataResult: &model.GetMetricDataResult{Statistic: "Minimum", Datapoint: aws.Float64(2.0), Timestamp: time.Time{}}, + }, + }, + }, + }, + }, + { + name: "returns errors from GetAccounts", + jobsCfg: model.JobsConfig{ + DiscoveryJobs: []model.DiscoveryJob{ + { + Regions: []string{"us-east-1"}, + Type: "aws-namespace", + Roles: []model.Role{ + {RoleArn: "aws-arn-1", ExternalID: "external-id-1"}, + }, + }, + }, + CustomNamespaceJobs: []model.CustomNamespaceJob{ + { + Regions: []string{"us-east-2"}, + Name: "my-custom-job", + Namespace: "custom-namespace", + Roles: []model.Role{ + {RoleArn: "aws-arn-2", ExternalID: "external-id-2"}, + }, + }, + }, + }, + getAccountFunc: func() (string, error) { + return "", errors.New("failed to get account") + }, + expectedErrs: []job.Error{ + {JobContext: job.JobContext{Account: job.Account{}, Namespace: "aws-namespace", Region: "us-east-1", RoleARN: "aws-arn-1"}, ErrorType: job.AccountErr}, + {JobContext: job.JobContext{Account: job.Account{}, Namespace: "custom-namespace", Region: "us-east-2", RoleARN: "aws-arn-2"}, ErrorType: job.AccountErr}, + }, + }, + { + name: "ignores errors from GetAccountAlias", + jobsCfg: model.JobsConfig{ + DiscoveryJobs: []model.DiscoveryJob{ + { + Regions: []string{"us-east-1"}, + Type: "aws-namespace", + Roles: []model.Role{ + {RoleArn: "aws-arn-1", ExternalID: "external-id-1"}, + }, + }, + }, + }, + getAccountFunc: func() (string, error) { + return "aws-account-1", nil + }, + getAccountAliasFunc: func() (string, error) { return "", errors.New("No alias here") }, + metadataRunFunc: func(_ context.Context, _ string, _ model.DiscoveryJob) ([]*model.TaggedResource, error) { + return []*model.TaggedResource{{ + ARN: "resource-1", Namespace: "aws-namespace", Region: "us-east-1", Tags: []model.Tag{{Key: "tag1", Value: "value1"}}, + }}, nil + }, + cloudwatchRunFunc: func(_ context.Context, _ cloudwatchrunner.Job) ([]*model.CloudwatchData, error) { + return []*model.CloudwatchData{ + { + MetricName: "metric-1", + ResourceName: "resource-1", + Namespace: "aws-namespace", + Tags: []model.Tag{{Key: "tag1", Value: "value1"}}, + Dimensions: []model.Dimension{{Name: "dimension1", Value: "value1"}}, + GetMetricDataResult: &model.GetMetricDataResult{Statistic: "Maximum", Datapoint: aws.Float64(1.0), Timestamp: time.Time{}}, + }, + }, nil + }, + expectedResources: []model.TaggedResourceResult{ + { + Context: &model.ScrapeContext{Region: "us-east-1", AccountID: "aws-account-1", AccountAlias: ""}, + Data: []*model.TaggedResource{ + {ARN: "resource-1", Namespace: "aws-namespace", Region: "us-east-1", Tags: []model.Tag{{Key: "tag1", Value: "value1"}}}, + }, + }, + }, + expectedMetrics: []model.CloudwatchMetricResult{ + { + Context: &model.ScrapeContext{Region: "us-east-1", AccountID: "aws-account-1", AccountAlias: ""}, + Data: []*model.CloudwatchData{ + { + MetricName: "metric-1", + ResourceName: "resource-1", + Namespace: "aws-namespace", + Tags: []model.Tag{{Key: "tag1", Value: "value1"}}, + Dimensions: []model.Dimension{{Name: "dimension1", Value: "value1"}}, + GetMetricDataResult: &model.GetMetricDataResult{Statistic: "Maximum", Datapoint: aws.Float64(1.0), Timestamp: time.Time{}}, + }, + }, + }, + }, + }, + { + name: "returns errors from resource discovery without failing scrape", + jobsCfg: model.JobsConfig{ + DiscoveryJobs: []model.DiscoveryJob{ + { + Regions: []string{"us-east-1"}, + Type: "aws-namespace", + Roles: []model.Role{ + {RoleArn: "aws-arn-1", ExternalID: "external-id-1"}, + }, + }, + }, + CustomNamespaceJobs: []model.CustomNamespaceJob{ + { + Regions: []string{"us-east-2"}, + Name: "my-custom-job", + Namespace: "custom-namespace", + Roles: []model.Role{ + {RoleArn: "aws-arn-2", ExternalID: "external-id-2"}, + }, + }, + }, + }, + getAccountFunc: func() (string, error) { + return "aws-account-1", nil + }, + getAccountAliasFunc: func() (string, error) { + return "my-aws-account", nil + }, + metadataRunFunc: func(_ context.Context, _ string, _ model.DiscoveryJob) ([]*model.TaggedResource, error) { + return nil, errors.New("I failed you") + }, + cloudwatchRunFunc: func(_ context.Context, _ cloudwatchrunner.Job) ([]*model.CloudwatchData, error) { + return []*model.CloudwatchData{ + { + MetricName: "metric-2", + ResourceName: "resource-2", + Namespace: "custom-namespace", + Dimensions: []model.Dimension{{Name: "dimension2", Value: "value2"}}, + GetMetricDataResult: &model.GetMetricDataResult{Statistic: "Minimum", Datapoint: aws.Float64(2.0), Timestamp: time.Time{}}, + }, + }, nil + }, + expectedMetrics: []model.CloudwatchMetricResult{ + { + Context: &model.ScrapeContext{Region: "us-east-2", AccountID: "aws-account-1", AccountAlias: "my-aws-account"}, + Data: []*model.CloudwatchData{ + { + MetricName: "metric-2", + ResourceName: "resource-2", + Namespace: "custom-namespace", + Dimensions: []model.Dimension{{Name: "dimension2", Value: "value2"}}, + GetMetricDataResult: &model.GetMetricDataResult{Statistic: "Minimum", Datapoint: aws.Float64(2.0), Timestamp: time.Time{}}, + }, + }, + }, + }, + expectedErrs: []job.Error{ + { + JobContext: job.JobContext{ + Account: job.Account{ID: "aws-account-1", Alias: "my-aws-account"}, + Namespace: "aws-namespace", + Region: "us-east-1", + RoleARN: "aws-arn-1", + }, + ErrorType: job.ResourceMetadataErr, + }, + }, + }, + { + name: "returns errors from cloudwatch metrics runner without failing scrape", + jobsCfg: model.JobsConfig{ + DiscoveryJobs: []model.DiscoveryJob{ + { + Regions: []string{"us-east-1"}, + Type: "aws-namespace", + Roles: []model.Role{ + {RoleArn: "aws-arn-1", ExternalID: "external-id-1"}, + }, + }, + }, + CustomNamespaceJobs: []model.CustomNamespaceJob{ + { + Regions: []string{"us-east-2"}, + Name: "my-custom-job", + Namespace: "custom-namespace", + Roles: []model.Role{ + {RoleArn: "aws-arn-2", ExternalID: "external-id-2"}, + }, + }, + }, + }, + getAccountFunc: func() (string, error) { + return "aws-account-1", nil + }, + getAccountAliasFunc: func() (string, error) { + return "my-aws-account", nil + }, + metadataRunFunc: func(_ context.Context, _ string, _ model.DiscoveryJob) ([]*model.TaggedResource, error) { + return []*model.TaggedResource{{ + ARN: "resource-1", Namespace: "aws-namespace", Region: "us-east-1", Tags: []model.Tag{{Key: "tag1", Value: "value1"}}, + }}, nil + }, + cloudwatchRunFunc: func(_ context.Context, job cloudwatchrunner.Job) ([]*model.CloudwatchData, error) { + if job.Namespace() == "custom-namespace" { + return nil, errors.New("I failed you") + } + return []*model.CloudwatchData{ + { + MetricName: "metric-1", + ResourceName: "resource-1", + Namespace: "aws-namespace", + Tags: []model.Tag{{Key: "tag1", Value: "value1"}}, + Dimensions: []model.Dimension{{Name: "dimension1", Value: "value1"}}, + GetMetricDataResult: &model.GetMetricDataResult{Statistic: "Maximum", Datapoint: aws.Float64(1.0), Timestamp: time.Time{}}, + }, + }, nil + }, + expectedResources: []model.TaggedResourceResult{ + { + Context: &model.ScrapeContext{Region: "us-east-1", AccountID: "aws-account-1", AccountAlias: "my-aws-account"}, + Data: []*model.TaggedResource{ + {ARN: "resource-1", Namespace: "aws-namespace", Region: "us-east-1", Tags: []model.Tag{{Key: "tag1", Value: "value1"}}}, + }, + }, + }, + expectedMetrics: []model.CloudwatchMetricResult{ + { + Context: &model.ScrapeContext{Region: "us-east-1", AccountID: "aws-account-1", AccountAlias: "my-aws-account"}, + Data: []*model.CloudwatchData{ + { + MetricName: "metric-1", + ResourceName: "resource-1", + Namespace: "aws-namespace", + Tags: []model.Tag{{Key: "tag1", Value: "value1"}}, + Dimensions: []model.Dimension{{Name: "dimension1", Value: "value1"}}, + GetMetricDataResult: &model.GetMetricDataResult{Statistic: "Maximum", Datapoint: aws.Float64(1.0), Timestamp: time.Time{}}, + }, + }, + }, + }, + expectedErrs: []job.Error{ + { + JobContext: job.JobContext{ + Account: job.Account{ID: "aws-account-1", Alias: "my-aws-account"}, + Namespace: "custom-namespace", + Region: "us-east-2", + RoleARN: "aws-arn-2", + }, + ErrorType: job.CloudWatchCollectionErr, + }, + }, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + rf := testRunnerFactory{ + GetAccountFunc: tc.getAccountFunc, + GetAccountAliasFunc: tc.getAccountAliasFunc, + MetadataRunFunc: tc.metadataRunFunc, + CloudwatchRunFunc: tc.cloudwatchRunFunc, + } + sr := job.NewScraper(logging.NewLogger("", true), tc.jobsCfg, &rf) + resources, metrics, errs := sr.Scrape(context.Background()) + + changelog, err := diff.Diff(tc.expectedResources, resources) + assert.NoError(t, err, "failed to diff resources") + assert.Len(t, changelog, 0, changelog) + + changelog, err = diff.Diff(tc.expectedMetrics, metrics) + assert.NoError(t, err, "failed to diff metrics") + assert.Len(t, changelog, 0, changelog) + + // We don't want to check the exact error just the message + changelog, err = diff.Diff(tc.expectedErrs, errs, diff.Filter(func(_ []string, _ reflect.Type, field reflect.StructField) bool { + return !(field.Name == "Err") + })) + assert.NoError(t, err, "failed to diff errs") + assert.Len(t, changelog, 0, changelog) + }) + } +}