From ab0e490f79916aff8ac0ac795c1582788c46c343 Mon Sep 17 00:00:00 2001 From: Samir Faci Date: Mon, 29 Jan 2024 13:23:12 -0500 Subject: [PATCH] Adding tag filtering Fixes #236 --- cli/backup/dashboard.go | 7 +- internal/service/dashboards.go | 85 ++++++++++++++----- test/dashboard_integration_test.go | 71 ++++++++++++++-- .../General/top-talkers-over-time.json | 2 +- website/content/en/docs/gdg/backup_guide.md | 4 +- website/content/en/docs/releases/gdg_0.5.md | 6 +- 6 files changed, 143 insertions(+), 32 deletions(-) diff --git a/cli/backup/dashboard.go b/cli/backup/dashboard.go index 03e415d1..d5c1b7b7 100644 --- a/cli/backup/dashboard.go +++ b/cli/backup/dashboard.go @@ -37,7 +37,7 @@ func newDashboardCommand() simplecobra.Commander { dashboard.PersistentFlags().BoolVarP(&skipConfirmAction, "skip-confirmation", "", false, "when set to true, bypass confirmation prompts") dashboard.PersistentFlags().StringP("dashboard", "d", "", "filter by dashboard slug") dashboard.PersistentFlags().StringP("folder", "f", "", "Filter by Folder Name (Quotes in names not supported)") - dashboard.PersistentFlags().StringSliceP("tags", "t", []string{}, "Filter by Tags (does not apply on upload)") + dashboard.PersistentFlags().StringSliceP("tags", "t", []string{}, "Filter by list of comma delimited tags") }, CommandsList: []simplecobra.Commander{ newListDashboardsCmd(), @@ -90,7 +90,7 @@ func newUploadDashboardsCmd() simplecobra.Commander { Long: description, CommandsList: []simplecobra.Commander{}, WithCFunc: func(cmd *cobra.Command, r *support.RootCommand) { - cmd.Aliases = []string{"u"} + cmd.Aliases = []string{"u", "up"} }, RunFunc: func(ctx context.Context, cd *simplecobra.Commandeer, rootCmd *support.RootCommand, args []string) error { filter := service.NewDashboardFilter(parseDashboardGlobalFlags(cd.CobraCommand)...) @@ -106,6 +106,7 @@ func newUploadDashboardsCmd() simplecobra.Commander { rootCmd.TableObj.AppendHeader(table.Row{"Title", "id", "folder", "UID"}) boards := rootCmd.GrafanaSvc().ListDashboards(filter) + slog.Info(fmt.Sprintf("%d dashboards have been uploaded", len(boards))) for _, link := range boards { rootCmd.TableObj.AppendRow(table.Row{link.Title, link.ID, link.FolderTitle, link.UID}) @@ -161,7 +162,7 @@ func newListDashboardsCmd() simplecobra.Commander { filters := service.NewDashboardFilter(parseDashboardGlobalFlags(cd.CobraCommand)...) boards := rootCmd.GrafanaSvc().ListDashboards(filters) - slog.Info("Listing dashboards for context", "context", config.Config().GetGDGConfig().GetContext()) + slog.Info("Listing dashboards for context", slog.String("context", config.Config().GetGDGConfig().GetContext()), slog.Any("count", len(boards))) for _, link := range boards { base, err := url.Parse(config.Config().GetDefaultGrafanaConfig().URL) var baseHost string diff --git a/internal/service/dashboards.go b/internal/service/dashboards.go index 81a03684..b0f9a8a0 100644 --- a/internal/service/dashboards.go +++ b/internal/service/dashboards.go @@ -10,11 +10,13 @@ import ( "github.com/grafana/grafana-openapi-client-go/client/search" "github.com/grafana/grafana-openapi-client-go/models" "github.com/tidwall/pretty" + "golang.org/x/exp/maps" "golang.org/x/exp/slices" "log" "log/slog" "path/filepath" "regexp" + "sort" "strings" "github.com/thoas/go-funk" @@ -123,37 +125,56 @@ func (s *DashNGoImpl) ListDashboards(filterReq filters.Filter) []*models.Hit { var boardsList = make([]*models.Hit, 0) var boardLinks = make([]*models.Hit, 0) + var deduplicatedLinks = make(map[int64]*models.Hit) var page uint = 1 var limit uint = 5000 // Upper bound of Grafana API call var tagsParams = make([]string, 0) - if !config.Config().GetDefaultGrafanaConfig().GetFilterOverrides().IgnoreDashboardFilters { - tagsParams = append(tagsParams, filterReq.GetEntity(filters.TagsFilter)...) - } + tagsParams = append(tagsParams, filterReq.GetEntity(filters.TagsFilter)...) - for { - searchParams := search.NewSearchParams() - searchParams.Tag = tagsParams - searchParams.Limit = tools.PtrOf(int64(limit)) - searchParams.Page = tools.PtrOf(int64(page)) - searchParams.Type = tools.PtrOf(searchTypeDashboard) + retrieve := func(tag string) { + for { + searchParams := search.NewSearchParams() + if tag != "" { + searchParams.Tag = []string{tag} + } + searchParams.Limit = tools.PtrOf(int64(limit)) + searchParams.Page = tools.PtrOf(int64(page)) + searchParams.Type = tools.PtrOf(searchTypeDashboard) - pageBoardLinks, err := s.GetClient().Search.Search(searchParams) - if err != nil { - log.Fatal("Failed to retrieve dashboards", err) + pageBoardLinks, err := s.GetClient().Search.Search(searchParams) + if err != nil { + log.Fatal("Failed to retrieve dashboards", err) + } + boardLinks = append(boardLinks, pageBoardLinks.GetPayload()...) + if len(pageBoardLinks.GetPayload()) < int(limit) { + break + } + page += 1 } - boardLinks = append(boardLinks, pageBoardLinks.GetPayload()...) - if len(pageBoardLinks.GetPayload()) < int(limit) { - break + } + if len(tagsParams) == 0 { + retrieve("") + } else { + for _, tag := range tagsParams { + slog.Info("retrieving dashboard for tag", slog.String("tag", tag)) + retrieve(tag) } - page += 1 } folderFilters := filterReq.GetEntity(filters.FolderFilter) var validFolder bool var validUid bool - for _, link := range boardLinks { + for ndx, link := range boardLinks { + link.Slug = updateSlug(link.URI) + _, ok := deduplicatedLinks[link.ID] + if ok { + slog.Debug("duplicate board, skipping ") + continue + } else { + deduplicatedLinks[link.ID] = boardLinks[ndx] + } validFolder = false if config.Config().GetDefaultGrafanaConfig().GetFilterOverrides().IgnoreDashboardFilters { validFolder = true @@ -166,7 +187,6 @@ func (s *DashNGoImpl) ListDashboards(filterReq filters.Filter) []*models.Hit { if !validFolder { continue } - link.Slug = updateSlug(link.URI) validUid = filterReq.GetFilter(filters.DashFilter) == "" || link.Slug == filterReq.GetFilter(filters.DashFilter) if link.FolderID == 0 { @@ -178,7 +198,12 @@ func (s *DashNGoImpl) ListDashboards(filterReq filters.Filter) []*models.Hit { } } - return boardsList + boardLinks = maps.Values(deduplicatedLinks) + sort.Slice(boardLinks, func(i, j int) bool { + return boardLinks[i].ID < boardLinks[j].ID + }) + + return boardLinks } @@ -271,6 +296,26 @@ func (s *DashNGoImpl) UploadDashboards(filterReq filters.Filter) { slog.Warn("Failed to unmarshall file", "filename", file) continue } + //Extract Tags + if filterVal := filterReq.GetFilter(filters.TagsFilter); filterVal != "" { + var boardTags []string + for _, val := range board["tags"].([]interface{}) { + boardTags = append(boardTags, val.(string)) + } + requestedSlices := strings.Split(filterVal, ",") + valid := false + for _, val := range requestedSlices { + if slices.Contains(boardTags, val) { + valid = true + break + } + } + if !valid { + slog.Debug("board fails tag filter, ignoring board", slog.Any("title", board["title"])) + continue + } + + } //Extract Folder Name based on path folderName, err = getFolderFromResourcePath(s.grafanaConf.Storage, file, config.DashboardResource) @@ -345,6 +390,8 @@ func (s *DashNGoImpl) DeleteAllDashboards(filter filters.Filter) []string { _, err := s.GetClient().Dashboards.DeleteDashboardByUID(item.UID) if err == nil { dashboardListing = append(dashboardListing, item.Title) + } else { + slog.Warn("Unable to remove dashboard", slog.String("title", item.Title), slog.String("uid", item.UID)) } } } diff --git a/test/dashboard_integration_test.go b/test/dashboard_integration_test.go index 1aec578f..18152c60 100644 --- a/test/dashboard_integration_test.go +++ b/test/dashboard_integration_test.go @@ -5,6 +5,7 @@ import ( "github.com/esnet/gdg/internal/service" "github.com/esnet/gdg/internal/service/filters" "github.com/grafana/grafana-openapi-client-go/models" + "os" "strings" "testing" @@ -13,16 +14,17 @@ import ( "log/slog" ) -//TODO: with full CRUD. -// - Add single dashboard test -d <> -// - Add Folder dashboard test -f <> - func TestDashboardCRUD(t *testing.T) { if testing.Short() { t.Skip("skipping integration test") } apiClient, _, cleanup := initTest(t, nil) - defer cleanup() + defer func() { + err := cleanup() + if err != nil { + slog.Warn("Unable to clean up after dashboard tests") + } + }() filtersEntity := service.NewDashboardFilter("", "", "") slog.Info("Exporting all dashboards") apiClient.UploadDashboards(filtersEntity) @@ -49,6 +51,15 @@ func TestDashboardCRUD(t *testing.T) { assert.True(t, ignoredSkipped) validateGeneralBoard(t, generalBoard) validateOtherBoard(t, otherBoard) + //Validate filters + + filterFolder := service.NewDashboardFilter("Other", "", "") + boards = apiClient.ListDashboards(filterFolder) + assert.Equal(t, 8, len(boards)) + dashboardFilter := service.NewDashboardFilter("", "flow-information", "") + boards = apiClient.ListDashboards(dashboardFilter) + assert.Equal(t, 1, len(boards)) + //Import Dashboards slog.Info("Importing Dashboards") list := apiClient.DownloadDashboards(filtersEntity) @@ -61,6 +72,56 @@ func TestDashboardCRUD(t *testing.T) { assert.Equal(t, len(boards), 0) } +func TestDashboardCRUDTags(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + apiClient, _, cleanup := initTest(t, nil) + defer func() { + err := cleanup() + if err != nil { + slog.Warn("Unable to clean up after dashboard tests") + } + }() + filtersEntity := service.NewDashboardFilter("", "", "netsage") + slog.Info("Uploading all dashboards, filtered by tags") + apiClient.UploadDashboards(filtersEntity) + slog.Info("Listing all dashboards") + boards := apiClient.ListDashboards(filtersEntity) + slog.Info("Removing all dashboards") + assert.Equal(t, 13, len(boards)) + deleteList := apiClient.DeleteAllDashboards(filtersEntity) + assert.Equal(t, 13, len(deleteList)) + //Multiple Tags behavior + slog.Info("Uploading all dashboards, filtered by tags") + filtersEntity = service.NewDashboardFilter("", "", "flow,netsage") + apiClient.UploadDashboards(filtersEntity) + slog.Info("Listing all dashboards") + boards = apiClient.ListDashboards(filtersEntity) + assert.Equal(t, 8, len(boards)) + slog.Info("Removing all dashboards") + deleteList = apiClient.DeleteAllDashboards(filtersEntity) + assert.Equal(t, 13, len(deleteList)) + // + os.Setenv("GDG_CONTEXTS__TESTING__IGNORE_FILTERS", "true") + defer os.Unsetenv("") + apiClient, _ = createSimpleClient(t, nil) + filterNone := service.NewDashboardFilter("", "", "") + apiClient.UploadDashboards(filterNone) + //Listing with no filter + boards = apiClient.ListDashboards(filterNone) + assert.Equal(t, 14, len(boards)) + + filtersEntity = service.NewDashboardFilter("", "", "netsage") + slog.Info("Listing dashboards by tag") + boards = apiClient.ListDashboards(filtersEntity) + assert.Equal(t, 14, len(deleteList)) + //Listing with + filtersEntity = service.NewDashboardFilter("", "", "netsage,flow") + boards = apiClient.ListDashboards(filtersEntity) + assert.Equal(t, 8, len(deleteList)) +} + func TestDashboardTagsFilter(t *testing.T) { if testing.Short() { t.Skip("skipping integration test") diff --git a/test/data/org_1/dashboards/General/top-talkers-over-time.json b/test/data/org_1/dashboards/General/top-talkers-over-time.json index ca9fc565..dd893526 100644 --- a/test/data/org_1/dashboards/General/top-talkers-over-time.json +++ b/test/data/org_1/dashboards/General/top-talkers-over-time.json @@ -300,7 +300,7 @@ ], "schemaVersion": 26, "style": "dark", - "tags": [], + "tags": ["netsage", "moo", "flow"], "templating": { "list": [ { diff --git a/website/content/en/docs/gdg/backup_guide.md b/website/content/en/docs/gdg/backup_guide.md index 8b3783b1..1d0acbd8 100644 --- a/website/content/en/docs/gdg/backup_guide.md +++ b/website/content/en/docs/gdg/backup_guide.md @@ -57,10 +57,10 @@ You can also use filtering options to list or import your dashboard by folder or ```sh ./bin/gdg backup dash download -f myFolder ./bin/gdg backup dash download -t myTag -./bin/gdg backup dash download -t tagA -t tagB -t tagC +./bin/gdg backup dash download -t tagA,tagB,tagC ``` - +**NOTE**: Starting with v0.5.2 full crud support for tag filtering. You can list,upload,clear,download dashboards using tag filters. ### Folders diff --git a/website/content/en/docs/releases/gdg_0.5.md b/website/content/en/docs/releases/gdg_0.5.md index e790bd07..73286b0c 100644 --- a/website/content/en/docs/releases/gdg_0.5.md +++ b/website/content/en/docs/releases/gdg_0.5.md @@ -10,16 +10,18 @@ toc: true ## Release Notes for v0.5.2 ### Changes - - Replacing Connection Auth with a secure/foobar.json file. Allows for more flexible data to be pushed to grafana. - - *TechDebt* refactored packages, moving cmd-> cli, and created cmd/ to allow for multiple binaries to be generated. +- [#229](https://github.com/esnet/gdg/issues/229) Datasource auth has been moved to a file based configuration under secure/. This allows for any number of secure values to be passed in. Using the wizard for initial config is recommended, or see test data for some examples. - [#168](https://github.com/esnet/gdg/issues/168) Introduced a new tool called gdg-generate which allows for templating of dashboards using go.tmpl syntax. - gdg context has been moved under tools. ie. `gdg tools ctx` instead of `gdg ctx` - [#221](https://github.com/esnet/gdg/issues/221) Version check no longer requires a valid configuration + - [#236](https://github.com/esnet/gdg/issues/236) Dashboard filter by tag support. Allows a user to only list,delete,upload dashboards that match a set of given tags. Previously, it was only supported for list/clear ### Bug Fixes +- [#235](https://github.com/esnet/gdg/issues/235) Fixed a bug that prevented proxy grafana instances from working correctly. ie. someURL/grafana/ would not work since it expected grafana to hosted on slash (/). ### Developer Changes - Migrated to Office Grafana GoLang API + - refactored packages, moving cmd-> cli, and created cmd/ to allow for multiple binaries to be generated. ## Release Notes for v0.5.1