From d8eb7429294e79c691f084b5abd027fabd016130 Mon Sep 17 00:00:00 2001 From: Baruch Odem Date: Sun, 3 Mar 2024 12:36:19 +0200 Subject: [PATCH 1/4] plugin initialization methods remove unused arg --- plugins/confluence.go | 4 ++-- plugins/discord.go | 4 ++-- plugins/filesystem.go | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/plugins/confluence.go b/plugins/confluence.go index 8fd2cf9d..f908455a 100644 --- a/plugins/confluence.go +++ b/plugins/confluence.go @@ -64,7 +64,7 @@ func (p *ConfluencePlugin) DefineCommand(items chan Item, errors chan error) (*c Long: "Scan Confluence server for sensitive information", Args: cobra.MatchAll(cobra.ExactArgs(1), isValidURL), Run: func(cmd *cobra.Command, args []string) { - err := p.initialize(cmd, args[0]) + err := p.initialize(args[0]) if err != nil { errors <- fmt.Errorf("error while initializing confluence plugin: %w", err) } @@ -84,7 +84,7 @@ func (p *ConfluencePlugin) DefineCommand(items chan Item, errors chan error) (*c return confluenceCmd, nil } -func (p *ConfluencePlugin) initialize(cmd *cobra.Command, urlArg string) error { +func (p *ConfluencePlugin) initialize(urlArg string) error { p.URL = strings.TrimRight(urlArg, "/") diff --git a/plugins/discord.go b/plugins/discord.go index ed49f591..018a3756 100644 --- a/plugins/discord.go +++ b/plugins/discord.go @@ -61,7 +61,7 @@ func (p *DiscordPlugin) DefineCommand(items chan Item, errors chan error) (*cobr flags.IntVar(&p.Count, messagesCountFlag, 0, "The number of messages to scan. If not provided, all messages will be scanned until the fromDate flag value.") discordCmd.Run = func(cmd *cobra.Command, args []string) { - err := p.initialize(cmd) + err := p.initialize() if err != nil { errors <- fmt.Errorf("discord plugin initialization failed: %w", err) return @@ -76,7 +76,7 @@ func (p *DiscordPlugin) DefineCommand(items chan Item, errors chan error) (*cobr return discordCmd, nil } -func (p *DiscordPlugin) initialize(cmd *cobra.Command) error { +func (p *DiscordPlugin) initialize() error { if len(p.Channels) == 0 { log.Warn().Msg("discord channels not provided. Will scan all channels") } diff --git a/plugins/filesystem.go b/plugins/filesystem.go index 6556ec13..b32c831e 100644 --- a/plugins/filesystem.go +++ b/plugins/filesystem.go @@ -104,7 +104,7 @@ func (p *FileSystemPlugin) getItems(items chan Item, errs chan error, wg *sync.W wg.Add(1) go func(filePath string) { defer wg.Done() - actualFile, err := p.getItem(wg, filePath) + actualFile, err := p.getItem(filePath) if err != nil { errs <- err return @@ -114,7 +114,7 @@ func (p *FileSystemPlugin) getItems(items chan Item, errs chan error, wg *sync.W } } -func (p *FileSystemPlugin) getItem(wg *sync.WaitGroup, filePath string) (*Item, error) { +func (p *FileSystemPlugin) getItem(filePath string) (*Item, error) { log.Debug().Str("file", filePath).Msg("reading file") b, err := os.ReadFile(filePath) if err != nil { From df98d5b2abc208b0a54cf09cbc80b249862681fe Mon Sep 17 00:00:00 2001 From: Baruch Odem Date: Wed, 6 Mar 2024 16:59:49 +0200 Subject: [PATCH 2/4] Refactor ConfluencePlugin methods --- plugins/confluence.go | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/plugins/confluence.go b/plugins/confluence.go index f908455a..e16d09d9 100644 --- a/plugins/confluence.go +++ b/plugins/confluence.go @@ -69,7 +69,7 @@ func (p *ConfluencePlugin) DefineCommand(items chan Item, errors chan error) (*c errors <- fmt.Errorf("error while initializing confluence plugin: %w", err) } wg := &sync.WaitGroup{} - p.getItems(items, errors, wg) + p.scanConfluence(items, errors, wg) wg.Wait() close(items) }, @@ -96,7 +96,7 @@ func (p *ConfluencePlugin) initialize(urlArg string) error { return nil } -func (p *ConfluencePlugin) getItems(items chan Item, errs chan error, wg *sync.WaitGroup) { +func (p *ConfluencePlugin) scanConfluence(items chan Item, errs chan error, wg *sync.WaitGroup) { spaces, err := p.getSpaces() if err != nil { errs <- err @@ -104,11 +104,11 @@ func (p *ConfluencePlugin) getItems(items chan Item, errs chan error, wg *sync.W for _, space := range spaces { wg.Add(1) - go p.getSpaceItems(items, errs, wg, space) + go p.loopSpaces(items, errs, wg, space) } } -func (p *ConfluencePlugin) getSpaceItems(items chan Item, errs chan error, wg *sync.WaitGroup, space ConfluenceSpaceResult) { +func (p *ConfluencePlugin) loopSpaces(items chan Item, errs chan error, wg *sync.WaitGroup, space ConfluenceSpaceResult) { defer wg.Done() pages, err := p.getPages(space) @@ -121,7 +121,7 @@ func (p *ConfluencePlugin) getSpaceItems(items chan Item, errs chan error, wg *s wg.Add(1) p.Limit <- struct{}{} go func(page ConfluencePage) { - p.getPageItems(items, errs, wg, page, space) + p.pageVersionsToItem(items, errs, wg, page, space) <-p.Limit }(page) } @@ -224,19 +224,18 @@ func (p *ConfluencePlugin) getPagesRequest(space ConfluenceSpaceResult, start in return &response.Results, nil } -func (p *ConfluencePlugin) getPageItems(items chan Item, errs chan error, wg *sync.WaitGroup, page ConfluencePage, space ConfluenceSpaceResult) { +func (p *ConfluencePlugin) pageVersionsToItem(items chan Item, errs chan error, wg *sync.WaitGroup, page ConfluencePage, space ConfluenceSpaceResult) { defer wg.Done() - actualPage, previousVersion, err := p.getItem(page, space, 0) + actualPage, previousVersion, err := p.convertPageToItem(page, space, 0) if err != nil { errs <- err return } items <- *actualPage - // If older versions exist & run history is true for previousVersion > 0 && p.History { - actualPage, previousVersion, err = p.getItem(page, space, previousVersion) + actualPage, previousVersion, err = p.convertPageToItem(page, space, previousVersion) if err != nil { errs <- err return @@ -245,7 +244,7 @@ func (p *ConfluencePlugin) getPageItems(items chan Item, errs chan error, wg *sy } } -func (p *ConfluencePlugin) getItem(page ConfluencePage, space ConfluenceSpaceResult, version int) (*Item, int, error) { +func (p *ConfluencePlugin) convertPageToItem(page ConfluencePage, space ConfluenceSpaceResult, version int) (*Item, int, error) { var url string // If no version given get the latest, else get the specified version From 6af2a9c4122a3cf303ccd4c6bbf9ba868ea6867e Mon Sep 17 00:00:00 2001 From: Baruch Odem Date: Thu, 7 Mar 2024 08:44:24 +0200 Subject: [PATCH 3/4] extract confluence API to interface --- plugins/confluence.go | 186 +++++++++++++++++++++++++----------------- 1 file changed, 112 insertions(+), 74 deletions(-) diff --git a/plugins/confluence.go b/plugins/confluence.go index e16d09d9..f7b28b55 100644 --- a/plugins/confluence.go +++ b/plugins/confluence.go @@ -24,30 +24,22 @@ const ( confluenceMaxRequests = 500 ) +var ( + username string + token string +) + type ConfluencePlugin struct { Plugin - URL string - Token string - Username string - Spaces []string - History bool + Spaces []string + History bool + client IConfluenceClient } func (p *ConfluencePlugin) GetName() string { return "confluence" } -func (p *ConfluencePlugin) GetCredentials() (string, string) { - return p.Username, p.Token -} - -func (p *ConfluencePlugin) GetAuthorizationHeader() string { - if p.Username == "" || p.Token == "" { - return "" - } - return utils.CreateBasicAuthCredentials(p) -} - func isValidURL(cmd *cobra.Command, args []string) error { urlStr := args[0] parsedURL, err := url.Parse(urlStr) @@ -59,10 +51,11 @@ func isValidURL(cmd *cobra.Command, args []string) error { func (p *ConfluencePlugin) DefineCommand(items chan Item, errors chan error) (*cobra.Command, error) { var confluenceCmd = &cobra.Command{ - Use: fmt.Sprintf("%s ", p.GetName()), - Short: "Scan Confluence server", - Long: "Scan Confluence server for sensitive information", - Args: cobra.MatchAll(cobra.ExactArgs(1), isValidURL), + Use: fmt.Sprintf("%s ", p.GetName()), + Short: "Scan Confluence server", + Long: "Scan Confluence server for sensitive information", + Example: fmt.Sprintf(" 2ms %s https://checkmarx.atlassian.net/wiki", p.GetName()), + Args: cobra.MatchAll(cobra.ExactArgs(1), isValidURL), Run: func(cmd *cobra.Command, args []string) { err := p.initialize(args[0]) if err != nil { @@ -77,8 +70,8 @@ func (p *ConfluencePlugin) DefineCommand(items chan Item, errors chan error) (*c flags := confluenceCmd.Flags() flags.StringSliceVar(&p.Spaces, argSpaces, []string{}, "Confluence spaces: The names or IDs of the spaces to scan") - flags.StringVar(&p.Username, argUsername, "", "Confluence user name or email for authentication") - flags.StringVar(&p.Token, argToken, "", "The Confluence API token for authentication") + flags.StringVar(&username, argUsername, "", "Confluence user name or email for authentication") + flags.StringVar(&token, argToken, "", "The Confluence API token for authentication") flags.BoolVar(&p.History, argHistory, false, "Scan pages history") return confluenceCmd, nil @@ -86,11 +79,12 @@ func (p *ConfluencePlugin) DefineCommand(items chan Item, errors chan error) (*c func (p *ConfluencePlugin) initialize(urlArg string) error { - p.URL = strings.TrimRight(urlArg, "/") + url := strings.TrimRight(urlArg, "/") - if p.Username == "" || p.Token == "" { + if username == "" || token == "" { log.Warn().Msg("confluence credentials were not provided. The scan will be made anonymously only for the public pages") } + p.client = newConfluenceClient(url, token, username) p.Limit = make(chan struct{}, confluenceMaxRequests) return nil @@ -104,11 +98,11 @@ func (p *ConfluencePlugin) scanConfluence(items chan Item, errs chan error, wg * for _, space := range spaces { wg.Add(1) - go p.loopSpaces(items, errs, wg, space) + go p.scanConfluenceSpace(items, errs, wg, space) } } -func (p *ConfluencePlugin) loopSpaces(items chan Item, errs chan error, wg *sync.WaitGroup, space ConfluenceSpaceResult) { +func (p *ConfluencePlugin) scanConfluenceSpace(items chan Item, errs chan error, wg *sync.WaitGroup, space ConfluenceSpaceResult) { defer wg.Done() pages, err := p.getPages(space) @@ -128,7 +122,7 @@ func (p *ConfluencePlugin) loopSpaces(items chan Item, errs chan error, wg *sync } func (p *ConfluencePlugin) getSpaces() ([]ConfluenceSpaceResult, error) { - totalSpaces, err := p.getSpacesRequest(0) + totalSpaces, err := p.client.getSpacesRequest(0) if err != nil { return nil, err } @@ -136,7 +130,7 @@ func (p *ConfluencePlugin) getSpaces() ([]ConfluenceSpaceResult, error) { actualSize := totalSpaces.Size for actualSize == confluenceDefaultWindow { - moreSpaces, err := p.getSpacesRequest(totalSpaces.Size) + moreSpaces, err := p.client.getSpacesRequest(totalSpaces.Size) if err != nil { return nil, err } @@ -166,24 +160,8 @@ func (p *ConfluencePlugin) getSpaces() ([]ConfluenceSpaceResult, error) { return filteredSpaces, nil } -func (p *ConfluencePlugin) getSpacesRequest(start int) (*ConfluenceSpaceResponse, error) { - url := fmt.Sprintf("%s/rest/api/space?start=%d", p.URL, start) - body, _, err := utils.HttpRequest(http.MethodGet, url, p, utils.RetrySettings{}) - if err != nil { - return nil, fmt.Errorf("unexpected error creating an http request %w", err) - } - - response := &ConfluenceSpaceResponse{} - jsonErr := json.Unmarshal(body, response) - if jsonErr != nil { - return nil, fmt.Errorf("could not unmarshal response %w", err) - } - - return response, nil -} - func (p *ConfluencePlugin) getPages(space ConfluenceSpaceResult) (*ConfluencePageResult, error) { - totalPages, err := p.getPagesRequest(space, 0) + totalPages, err := p.client.getPagesRequest(space, 0) if err != nil { return nil, fmt.Errorf("unexpected error creating an http request %w", err) @@ -192,7 +170,7 @@ func (p *ConfluencePlugin) getPages(space ConfluenceSpaceResult) (*ConfluencePag actualSize := len(totalPages.Pages) for actualSize == confluenceDefaultWindow { - morePages, err := p.getPagesRequest(space, len(totalPages.Pages)) + morePages, err := p.client.getPagesRequest(space, len(totalPages.Pages)) if err != nil { return nil, fmt.Errorf("unexpected error creating an http request %w", err) @@ -207,23 +185,6 @@ func (p *ConfluencePlugin) getPages(space ConfluenceSpaceResult) (*ConfluencePag return totalPages, nil } -func (p *ConfluencePlugin) getPagesRequest(space ConfluenceSpaceResult, start int) (*ConfluencePageResult, error) { - url := fmt.Sprintf("%s/rest/api/space/%s/content?start=%d", p.URL, space.Key, start) - body, _, err := utils.HttpRequest(http.MethodGet, url, p, utils.RetrySettings{}) - - if err != nil { - return nil, fmt.Errorf("unexpected error creating an http request %w", err) - } - - response := ConfluencePageResponse{} - jsonErr := json.Unmarshal(body, &response) - if jsonErr != nil { - return nil, fmt.Errorf("could not unmarshal response %w", err) - } - - return &response.Results, nil -} - func (p *ConfluencePlugin) pageVersionsToItem(items chan Item, errs chan error, wg *sync.WaitGroup, page ConfluencePage, space ConfluenceSpaceResult) { defer wg.Done() @@ -245,32 +206,109 @@ func (p *ConfluencePlugin) pageVersionsToItem(items chan Item, errs chan error, } func (p *ConfluencePlugin) convertPageToItem(page ConfluencePage, space ConfluenceSpaceResult, version int) (*Item, int, error) { + pageContent, err := p.client.getPageContentRequest(page, version) + if err != nil { + return nil, 0, fmt.Errorf("unexpected error creating an http request %w", err) + } + + item := &Item{ + Content: pageContent.Body.Storage.Value, + ID: fmt.Sprintf("%s-%s-%s", p.GetName(), space.Key, page.ID), + Source: pageContent.Links["base"] + pageContent.Links["webui"], + } + return item, pageContent.History.PreviousVersion.Number, nil +} + +/* + * Confluence client + */ + +type IConfluenceClient interface { + getSpacesRequest(start int) (*ConfluenceSpaceResponse, error) + getPagesRequest(space ConfluenceSpaceResult, start int) (*ConfluencePageResult, error) + getPageContentRequest(page ConfluencePage, version int) (*ConfluencePageContent, error) +} + +type confluenceClient struct { + baseURL string + token string + username string +} + +func newConfluenceClient(baseURL, token, username string) IConfluenceClient { + return &confluenceClient{ + baseURL: baseURL, + token: token, + username: username, + } +} + +func (c *confluenceClient) GetCredentials() (string, string) { + return c.username, c.token +} + +func (c *confluenceClient) GetAuthorizationHeader() string { + if c.username == "" || c.token == "" { + return "" + } + return utils.CreateBasicAuthCredentials(c) +} + +func (c *confluenceClient) getSpacesRequest(start int) (*ConfluenceSpaceResponse, error) { + url := fmt.Sprintf("%s/rest/api/space?start=%d", c.baseURL, start) + body, _, err := utils.HttpRequest(http.MethodGet, url, c, utils.RetrySettings{}) + if err != nil { + return nil, fmt.Errorf("unexpected error creating an http request %w", err) + } + + response := &ConfluenceSpaceResponse{} + jsonErr := json.Unmarshal(body, response) + if jsonErr != nil { + return nil, fmt.Errorf("could not unmarshal response %w", err) + } + + return response, nil +} + +func (c *confluenceClient) getPagesRequest(space ConfluenceSpaceResult, start int) (*ConfluencePageResult, error) { + url := fmt.Sprintf("%s/rest/api/space/%s/content?start=%d", c.baseURL, space.Key, start) + body, _, err := utils.HttpRequest(http.MethodGet, url, c, utils.RetrySettings{}) + + if err != nil { + return nil, fmt.Errorf("unexpected error creating an http request %w", err) + } + + response := ConfluencePageResponse{} + jsonErr := json.Unmarshal(body, &response) + if jsonErr != nil { + return nil, fmt.Errorf("could not unmarshal response %w", err) + } + + return &response.Results, nil +} + +func (c *confluenceClient) getPageContentRequest(page ConfluencePage, version int) (*ConfluencePageContent, error) { var url string // If no version given get the latest, else get the specified version if version == 0 { - url = fmt.Sprintf("%s/rest/api/content/%s?expand=body.storage,version,history.previousVersion", p.URL, page.ID) + url = fmt.Sprintf("%s/rest/api/content/%s?expand=body.storage,version,history.previousVersion", c.baseURL, page.ID) } else { - url = fmt.Sprintf("%s/rest/api/content/%s?status=historical&version=%d&expand=body.storage,version,history.previousVersion", p.URL, page.ID, version) + url = fmt.Sprintf("%s/rest/api/content/%s?status=historical&version=%d&expand=body.storage,version,history.previousVersion", c.baseURL, page.ID, version) } - request, _, err := utils.HttpRequest(http.MethodGet, url, p, utils.RetrySettings{MaxRetries: 3, ErrorCodes: []int{500}}) + request, _, err := utils.HttpRequest(http.MethodGet, url, c, utils.RetrySettings{MaxRetries: 3, ErrorCodes: []int{500}}) if err != nil { - return nil, 0, fmt.Errorf("unexpected error creating an http request %w", err) + return nil, fmt.Errorf("unexpected error creating an http request %w", err) } pageContent := ConfluencePageContent{} jsonErr := json.Unmarshal(request, &pageContent) if jsonErr != nil { - return nil, 0, jsonErr + return nil, jsonErr } - content := &Item{ - Content: pageContent.Body.Storage.Value, - ID: fmt.Sprintf("%s-%s-%s-%s", p.GetName(), p.URL, space.Key, page.ID), - Source: pageContent.Links["base"] + pageContent.Links["webui"], - } - return content, pageContent.History.PreviousVersion.Number, nil + return &pageContent, nil } type ConfluenceSpaceResult struct { From 6c2f40d03b125b3e8326a133fe4ed74235077a71 Mon Sep 17 00:00:00 2001 From: Baruch Odem Date: Thu, 7 Mar 2024 10:04:57 +0200 Subject: [PATCH 4/4] Refactor ConfluencePlugin to use channels for items and errors --- plugins/confluence.go | 87 +++++++++++++++++++++++-------------------- 1 file changed, 46 insertions(+), 41 deletions(-) diff --git a/plugins/confluence.go b/plugins/confluence.go index f7b28b55..84716e35 100644 --- a/plugins/confluence.go +++ b/plugins/confluence.go @@ -34,6 +34,9 @@ type ConfluencePlugin struct { Spaces []string History bool client IConfluenceClient + + itemsChan chan Item + errorsChan chan error } func (p *ConfluencePlugin) GetName() string { @@ -50,6 +53,9 @@ func isValidURL(cmd *cobra.Command, args []string) error { } func (p *ConfluencePlugin) DefineCommand(items chan Item, errors chan error) (*cobra.Command, error) { + p.itemsChan = items + p.errorsChan = errors + var confluenceCmd = &cobra.Command{ Use: fmt.Sprintf("%s ", p.GetName()), Short: "Scan Confluence server", @@ -62,7 +68,7 @@ func (p *ConfluencePlugin) DefineCommand(items chan Item, errors chan error) (*c errors <- fmt.Errorf("error while initializing confluence plugin: %w", err) } wg := &sync.WaitGroup{} - p.scanConfluence(items, errors, wg) + p.scanConfluence(wg) wg.Wait() close(items) }, @@ -90,24 +96,24 @@ func (p *ConfluencePlugin) initialize(urlArg string) error { return nil } -func (p *ConfluencePlugin) scanConfluence(items chan Item, errs chan error, wg *sync.WaitGroup) { +func (p *ConfluencePlugin) scanConfluence(wg *sync.WaitGroup) { spaces, err := p.getSpaces() if err != nil { - errs <- err + p.errorsChan <- err } for _, space := range spaces { wg.Add(1) - go p.scanConfluenceSpace(items, errs, wg, space) + go p.scanConfluenceSpace(wg, space) } } -func (p *ConfluencePlugin) scanConfluenceSpace(items chan Item, errs chan error, wg *sync.WaitGroup, space ConfluenceSpaceResult) { +func (p *ConfluencePlugin) scanConfluenceSpace(wg *sync.WaitGroup, space ConfluenceSpaceResult) { defer wg.Done() pages, err := p.getPages(space) if err != nil { - errs <- err + p.errorsChan <- err return } @@ -115,12 +121,45 @@ func (p *ConfluencePlugin) scanConfluenceSpace(items chan Item, errs chan error, wg.Add(1) p.Limit <- struct{}{} go func(page ConfluencePage) { - p.pageVersionsToItem(items, errs, wg, page, space) + p.scanPageAllVersions(wg, page, space) <-p.Limit }(page) } } +func (p *ConfluencePlugin) scanPageAllVersions(wg *sync.WaitGroup, page ConfluencePage, space ConfluenceSpaceResult) { + defer wg.Done() + + previousVersion := p.scanPageVersion(page, space, 0) + if !p.History { + return + } + + for previousVersion > 0 { + previousVersion = p.scanPageVersion(page, space, previousVersion) + } +} + +func (p *ConfluencePlugin) scanPageVersion(page ConfluencePage, space ConfluenceSpaceResult, version int) int { + pageContent, err := p.client.getPageContentRequest(page, version) + if err != nil { + p.errorsChan <- err + return 0 + } + itemID := fmt.Sprintf("%s-%s-%s", p.GetName(), space.Key, page.ID) + p.itemsChan <- convertPageToItem(pageContent, itemID) + + return pageContent.History.PreviousVersion.Number +} + +func convertPageToItem(pageContent *ConfluencePageContent, itemID string) Item { + return Item{ + Content: pageContent.Body.Storage.Value, + ID: itemID, + Source: pageContent.Links["base"] + pageContent.Links["webui"], + } +} + func (p *ConfluencePlugin) getSpaces() ([]ConfluenceSpaceResult, error) { totalSpaces, err := p.client.getSpacesRequest(0) if err != nil { @@ -185,40 +224,6 @@ func (p *ConfluencePlugin) getPages(space ConfluenceSpaceResult) (*ConfluencePag return totalPages, nil } -func (p *ConfluencePlugin) pageVersionsToItem(items chan Item, errs chan error, wg *sync.WaitGroup, page ConfluencePage, space ConfluenceSpaceResult) { - defer wg.Done() - - actualPage, previousVersion, err := p.convertPageToItem(page, space, 0) - if err != nil { - errs <- err - return - } - items <- *actualPage - - for previousVersion > 0 && p.History { - actualPage, previousVersion, err = p.convertPageToItem(page, space, previousVersion) - if err != nil { - errs <- err - return - } - items <- *actualPage - } -} - -func (p *ConfluencePlugin) convertPageToItem(page ConfluencePage, space ConfluenceSpaceResult, version int) (*Item, int, error) { - pageContent, err := p.client.getPageContentRequest(page, version) - if err != nil { - return nil, 0, fmt.Errorf("unexpected error creating an http request %w", err) - } - - item := &Item{ - Content: pageContent.Body.Storage.Value, - ID: fmt.Sprintf("%s-%s-%s", p.GetName(), space.Key, page.ID), - Source: pageContent.Links["base"] + pageContent.Links["webui"], - } - return item, pageContent.History.PreviousVersion.Number, nil -} - /* * Confluence client */