Adding tag filtering

Fixes #236

cli/backup/dashboard.go

@@ -37,7 +37,7 @@ func newDashboardCommand() simplecobra.Commander {
 			dashboard.PersistentFlags().BoolVarP(&skipConfirmAction, "skip-confirmation", "", false, "when set to true, bypass confirmation prompts")
 			dashboard.PersistentFlags().StringP("dashboard", "d", "", "filter by dashboard slug")
 			dashboard.PersistentFlags().StringP("folder", "f", "", "Filter by Folder Name (Quotes in names not supported)")
-			dashboard.PersistentFlags().StringSliceP("tags", "t", []string{}, "Filter by Tags (does not apply on upload)")
+			dashboard.PersistentFlags().StringSliceP("tags", "t", []string{}, "Filter by list of comma delimited tags")
 		},
 		CommandsList: []simplecobra.Commander{
 			newListDashboardsCmd(),
@@ -90,7 +90,7 @@ func newUploadDashboardsCmd() simplecobra.Commander {
 		Long:         description,
 		CommandsList: []simplecobra.Commander{},
 		WithCFunc: func(cmd *cobra.Command, r *support.RootCommand) {
-			cmd.Aliases = []string{"u"}
+			cmd.Aliases = []string{"u", "up"}
 		},
 		RunFunc: func(ctx context.Context, cd *simplecobra.Commandeer, rootCmd *support.RootCommand, args []string) error {
 			filter := service.NewDashboardFilter(parseDashboardGlobalFlags(cd.CobraCommand)...)
@@ -106,6 +106,7 @@ func newUploadDashboardsCmd() simplecobra.Commander {
 			rootCmd.TableObj.AppendHeader(table.Row{"Title", "id", "folder", "UID"})
 			boards := rootCmd.GrafanaSvc().ListDashboards(filter)
 
+			slog.Info(fmt.Sprintf("%d dashboards have been uploaded", len(boards)))
 			for _, link := range boards {
 				rootCmd.TableObj.AppendRow(table.Row{link.Title, link.ID, link.FolderTitle, link.UID})
 
@@ -161,7 +162,7 @@ func newListDashboardsCmd() simplecobra.Commander {
 			filters := service.NewDashboardFilter(parseDashboardGlobalFlags(cd.CobraCommand)...)
 			boards := rootCmd.GrafanaSvc().ListDashboards(filters)
 
-			slog.Info("Listing dashboards for context", "context", config.Config().GetGDGConfig().GetContext())
+			slog.Info("Listing dashboards for context", slog.String("context", config.Config().GetGDGConfig().GetContext()), slog.Any("count", len(boards)))
 			for _, link := range boards {
 				base, err := url.Parse(config.Config().GetDefaultGrafanaConfig().URL)
 				var baseHost string

internal/service/dashboards.go

@@ -10,11 +10,13 @@ import (
 	"github.com/grafana/grafana-openapi-client-go/client/search"
 	"github.com/grafana/grafana-openapi-client-go/models"
 	"github.com/tidwall/pretty"
+	"golang.org/x/exp/maps"
 	"golang.org/x/exp/slices"
 	"log"
 	"log/slog"
 	"path/filepath"
 	"regexp"
+	"sort"
 	"strings"
 
 	"github.com/thoas/go-funk"
@@ -123,37 +125,56 @@ func (s *DashNGoImpl) ListDashboards(filterReq filters.Filter) []*models.Hit {
 
 	var boardsList = make([]*models.Hit, 0)
 	var boardLinks = make([]*models.Hit, 0)
+	var deduplicatedLinks = make(map[int64]*models.Hit)
 
 	var page uint = 1
 	var limit uint = 5000 // Upper bound of Grafana API call
 
 	var tagsParams = make([]string, 0)
-	if !config.Config().GetDefaultGrafanaConfig().GetFilterOverrides().IgnoreDashboardFilters {
-		tagsParams = append(tagsParams, filterReq.GetEntity(filters.TagsFilter)...)
-	}
+	tagsParams = append(tagsParams, filterReq.GetEntity(filters.TagsFilter)...)
 
-	for {
-		searchParams := search.NewSearchParams()
-		searchParams.Tag = tagsParams
-		searchParams.Limit = tools.PtrOf(int64(limit))
-		searchParams.Page = tools.PtrOf(int64(page))
-		searchParams.Type = tools.PtrOf(searchTypeDashboard)
+	retrieve := func(tag string) {
+		for {
+			searchParams := search.NewSearchParams()
+			if tag != "" {
+				searchParams.Tag = []string{tag}
+			}
+			searchParams.Limit = tools.PtrOf(int64(limit))
+			searchParams.Page = tools.PtrOf(int64(page))
+			searchParams.Type = tools.PtrOf(searchTypeDashboard)
 
-		pageBoardLinks, err := s.GetClient().Search.Search(searchParams)
-		if err != nil {
-			log.Fatal("Failed to retrieve dashboards", err)
+			pageBoardLinks, err := s.GetClient().Search.Search(searchParams)
+			if err != nil {
+				log.Fatal("Failed to retrieve dashboards", err)
+			}
+			boardLinks = append(boardLinks, pageBoardLinks.GetPayload()...)
+			if len(pageBoardLinks.GetPayload()) < int(limit) {
+				break
+			}
+			page += 1
 		}
-		boardLinks = append(boardLinks, pageBoardLinks.GetPayload()...)
-		if len(pageBoardLinks.GetPayload()) < int(limit) {
-			break
+	}
+	if len(tagsParams) == 0 {
+		retrieve("")
+	} else {
+		for _, tag := range tagsParams {
+			slog.Info("retrieving dashboard for tag", slog.String("tag", tag))
+			retrieve(tag)
 		}
-		page += 1
 	}
 
 	folderFilters := filterReq.GetEntity(filters.FolderFilter)
 	var validFolder bool
 	var validUid bool
-	for _, link := range boardLinks {
+	for ndx, link := range boardLinks {
+		link.Slug = updateSlug(link.URI)
+		_, ok := deduplicatedLinks[link.ID]
+		if ok {
+			slog.Debug("duplicate board, skipping ")
+			continue
+		} else {
+			deduplicatedLinks[link.ID] = boardLinks[ndx]
+		}
 		validFolder = false
 		if config.Config().GetDefaultGrafanaConfig().GetFilterOverrides().IgnoreDashboardFilters {
 			validFolder = true
@@ -166,7 +187,6 @@ func (s *DashNGoImpl) ListDashboards(filterReq filters.Filter) []*models.Hit {
 		if !validFolder {
 			continue
 		}
-		link.Slug = updateSlug(link.URI)
 		validUid = filterReq.GetFilter(filters.DashFilter) == "" || link.Slug == filterReq.GetFilter(filters.DashFilter)
 		if link.FolderID == 0 {
 
@@ -178,7 +198,12 @@ func (s *DashNGoImpl) ListDashboards(filterReq filters.Filter) []*models.Hit {
 		}
 	}
 
-	return boardsList
+	boardLinks = maps.Values(deduplicatedLinks)
+	sort.Slice(boardLinks, func(i, j int) bool {
+		return boardLinks[i].ID < boardLinks[j].ID
+	})
+
+	return boardLinks
 
 }
 
@@ -271,6 +296,26 @@ func (s *DashNGoImpl) UploadDashboards(filterReq filters.Filter) {
 			slog.Warn("Failed to unmarshall file", "filename", file)
 			continue
 		}
+		//Extract Tags
+		if filterVal := filterReq.GetFilter(filters.TagsFilter); filterVal != "" {
+			var boardTags []string
+			for _, val := range board["tags"].([]interface{}) {
+				boardTags = append(boardTags, val.(string))
+			}
+			requestedSlices := strings.Split(filterVal, ",")
+			valid := false
+			for _, val := range requestedSlices {
+				if slices.Contains(boardTags, val) {
+					valid = true
+					break
+				}
+			}
+			if !valid {
+				slog.Debug("board fails tag filter, ignoring board", slog.Any("title", board["title"]))
+				continue
+			}
+
+		}
 
 		//Extract Folder Name based on path
 		folderName, err = getFolderFromResourcePath(s.grafanaConf.Storage, file, config.DashboardResource)
@@ -345,6 +390,8 @@ func (s *DashNGoImpl) DeleteAllDashboards(filter filters.Filter) []string {
 			_, err := s.GetClient().Dashboards.DeleteDashboardByUID(item.UID)
 			if err == nil {
 				dashboardListing = append(dashboardListing, item.Title)
+			} else {
+				slog.Warn("Unable to remove dashboard", slog.String("title", item.Title), slog.String("uid", item.UID))
 			}
 		}
 	}

test/dashboard_integration_test.go

@@ -5,6 +5,7 @@ import (
 	"github.com/esnet/gdg/internal/service"
 	"github.com/esnet/gdg/internal/service/filters"
 	"github.com/grafana/grafana-openapi-client-go/models"
+	"os"
 	"strings"
 	"testing"
 
@@ -13,16 +14,17 @@ import (
 	"log/slog"
 )
 
-//TODO: with full CRUD.
-// - Add single dashboard test -d <>
-// - Add Folder dashboard test -f <>
-
 func TestDashboardCRUD(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping integration test")
 	}
 	apiClient, _, cleanup := initTest(t, nil)
-	defer cleanup()
+	defer func() {
+		err := cleanup()
+		if err != nil {
+			slog.Warn("Unable to clean up after dashboard tests")
+		}
+	}()
 	filtersEntity := service.NewDashboardFilter("", "", "")
 	slog.Info("Exporting all dashboards")
 	apiClient.UploadDashboards(filtersEntity)
@@ -49,6 +51,15 @@ func TestDashboardCRUD(t *testing.T) {
 	assert.True(t, ignoredSkipped)
 	validateGeneralBoard(t, generalBoard)
 	validateOtherBoard(t, otherBoard)
+	//Validate filters
+
+	filterFolder := service.NewDashboardFilter("Other", "", "")
+	boards = apiClient.ListDashboards(filterFolder)
+	assert.Equal(t, 8, len(boards))
+	dashboardFilter := service.NewDashboardFilter("", "flow-information", "")
+	boards = apiClient.ListDashboards(dashboardFilter)
+	assert.Equal(t, 1, len(boards))
+
 	//Import Dashboards
 	slog.Info("Importing Dashboards")
 	list := apiClient.DownloadDashboards(filtersEntity)
@@ -61,6 +72,56 @@ func TestDashboardCRUD(t *testing.T) {
 	assert.Equal(t, len(boards), 0)
 }
 
+func TestDashboardCRUDTags(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test")
+	}
+	apiClient, _, cleanup := initTest(t, nil)
+	defer func() {
+		err := cleanup()
+		if err != nil {
+			slog.Warn("Unable to clean up after dashboard tests")
+		}
+	}()
+	filtersEntity := service.NewDashboardFilter("", "", "netsage")
+	slog.Info("Uploading all dashboards, filtered by tags")
+	apiClient.UploadDashboards(filtersEntity)
+	slog.Info("Listing all dashboards")
+	boards := apiClient.ListDashboards(filtersEntity)
+	slog.Info("Removing all dashboards")
+	assert.Equal(t, 13, len(boards))
+	deleteList := apiClient.DeleteAllDashboards(filtersEntity)
+	assert.Equal(t, 13, len(deleteList))
+	//Multiple Tags behavior
+	slog.Info("Uploading all dashboards, filtered by tags")
+	filtersEntity = service.NewDashboardFilter("", "", "flow,netsage")
+	apiClient.UploadDashboards(filtersEntity)
+	slog.Info("Listing all dashboards")
+	boards = apiClient.ListDashboards(filtersEntity)
+	assert.Equal(t, 8, len(boards))
+	slog.Info("Removing all dashboards")
+	deleteList = apiClient.DeleteAllDashboards(filtersEntity)
+	assert.Equal(t, 13, len(deleteList))
+	//
+	os.Setenv("GDG_CONTEXTS__TESTING__IGNORE_FILTERS", "true")
+	defer os.Unsetenv("")
+	apiClient, _ = createSimpleClient(t, nil)
+	filterNone := service.NewDashboardFilter("", "", "")
+	apiClient.UploadDashboards(filterNone)
+	//Listing with no filter
+	boards = apiClient.ListDashboards(filterNone)
+	assert.Equal(t, 14, len(boards))
+
+	filtersEntity = service.NewDashboardFilter("", "", "netsage")
+	slog.Info("Listing dashboards by tag")
+	boards = apiClient.ListDashboards(filtersEntity)
+	assert.Equal(t, 14, len(deleteList))
+	//Listing with
+	filtersEntity = service.NewDashboardFilter("", "", "netsage,flow")
+	boards = apiClient.ListDashboards(filtersEntity)
+	assert.Equal(t, 8, len(deleteList))
+}
+
 func TestDashboardTagsFilter(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping integration test")

test/data/org_1/dashboards/General/top-talkers-over-time.json

@@ -300,7 +300,7 @@
   ],
   "schemaVersion": 26,
   "style": "dark",
-  "tags": [],
+  "tags": ["netsage", "moo", "flow"],
   "templating": {
     "list": [
       {

website/content/en/docs/gdg/backup_guide.md

@@ -57,10 +57,10 @@ You can also use filtering options to list or import your dashboard by folder or
 ```sh
 ./bin/gdg backup dash download -f myFolder
 ./bin/gdg backup dash download -t myTag
-./bin/gdg backup dash download -t tagA -t tagB -t tagC
+./bin/gdg backup dash download -t tagA,tagB,tagC
 ```
 
-
+**NOTE**: Starting with v0.5.2 full crud support for tag filtering.  You can list,upload,clear,download dashboards using tag filters.
 
 ### Folders
 

website/content/en/docs/releases/gdg_0.5.md

@@ -10,16 +10,18 @@ toc: true
 
 ##  Release Notes for v0.5.2
 ### Changes
-  - Replacing Connection Auth with a secure/foobar.json file.  Allows for more flexible data to be pushed to grafana.
-  - *TechDebt* refactored packages, moving cmd-> cli, and created cmd/ to allow for multiple binaries to be generated.
+- [#229](https://github.com/esnet/gdg/issues/229) Datasource auth has been moved to a file based configuration under secure/.  This allows for any number of secure values to be passed in.  Using the wizard for initial config is recommended, or see test data for some examples.
   - [#168](https://github.com/esnet/gdg/issues/168) Introduced a new tool called gdg-generate which allows for templating of dashboards using go.tmpl syntax.
   - gdg context has been moved under tools.  ie. `gdg tools ctx` instead of `gdg ctx`
   - [#221](https://github.com/esnet/gdg/issues/221) Version check no longer requires a valid configuration
+  - [#236](https://github.com/esnet/gdg/issues/236) Dashboard filter by tag support.  Allows a user to only list,delete,upload dashboards that match a set of given tags.  Previously, it was only supported for list/clear
 
 ### Bug Fixes
+- [#235](https://github.com/esnet/gdg/issues/235) Fixed a bug that prevented proxy grafana instances from working correctly. ie. someURL/grafana/ would not work since it expected grafana to hosted on slash (/).
 
 ### Developer Changes
   - Migrated to Office Grafana GoLang API
+  - refactored packages, moving cmd-> cli, and created cmd/ to allow for multiple binaries to be generated.
 
 
 ##  Release Notes for v0.5.1