Skip to content

Commit

Permalink
refactor: confluence plugin methods (#219)
Browse files Browse the repository at this point in the history 
This pull request refactors the initialization methods of the
ConfluencePlugin and DiscordPlugin classes. It removes unused arguments,
extracts the Confluence API to an interface, and updates the
ConfluencePlugin to use channels for items and errors. These changes
improve the code structure and make it easier to maintain and extend.
  • Loading branch information
Baruch Odem (Rothkoff) authored Mar 7, 2024
1 parent 2055905 commit 35f27ed
Show file tree
Hide file tree
Showing 3 changed files with 131 additions and 89 deletions.
212 changes: 127 additions & 85 deletions plugins/confluence.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,30 +24,25 @@ const (
confluenceMaxRequests = 500
)

var (
username string
token string
)

type ConfluencePlugin struct {
Plugin
URL string
Token string
Username string
Spaces []string
History bool
Spaces []string
History bool
client IConfluenceClient

itemsChan chan Item
errorsChan chan error
}

func (p *ConfluencePlugin) GetName() string {
return "confluence"
}

func (p *ConfluencePlugin) GetCredentials() (string, string) {
return p.Username, p.Token
}

func (p *ConfluencePlugin) GetAuthorizationHeader() string {
if p.Username == "" || p.Token == "" {
return ""
}
return utils.CreateBasicAuthCredentials(p)
}

func isValidURL(cmd *cobra.Command, args []string) error {
urlStr := args[0]
parsedURL, err := url.Parse(urlStr)
Expand All @@ -58,85 +53,123 @@ func isValidURL(cmd *cobra.Command, args []string) error {
}

func (p *ConfluencePlugin) DefineCommand(items chan Item, errors chan error) (*cobra.Command, error) {
p.itemsChan = items
p.errorsChan = errors

var confluenceCmd = &cobra.Command{
Use: fmt.Sprintf("%s <URL>", p.GetName()),
Short: "Scan Confluence server",
Long: "Scan Confluence server for sensitive information",
Args: cobra.MatchAll(cobra.ExactArgs(1), isValidURL),
Use: fmt.Sprintf("%s <URL>", p.GetName()),
Short: "Scan Confluence server",
Long: "Scan Confluence server for sensitive information",
Example: fmt.Sprintf(" 2ms %s https://checkmarx.atlassian.net/wiki", p.GetName()),
Args: cobra.MatchAll(cobra.ExactArgs(1), isValidURL),
Run: func(cmd *cobra.Command, args []string) {
err := p.initialize(cmd, args[0])
err := p.initialize(args[0])
if err != nil {
errors <- fmt.Errorf("error while initializing confluence plugin: %w", err)
}
wg := &sync.WaitGroup{}
p.getItems(items, errors, wg)
p.scanConfluence(wg)
wg.Wait()
close(items)
},
}

flags := confluenceCmd.Flags()
flags.StringSliceVar(&p.Spaces, argSpaces, []string{}, "Confluence spaces: The names or IDs of the spaces to scan")
flags.StringVar(&p.Username, argUsername, "", "Confluence user name or email for authentication")
flags.StringVar(&p.Token, argToken, "", "The Confluence API token for authentication")
flags.StringVar(&username, argUsername, "", "Confluence user name or email for authentication")
flags.StringVar(&token, argToken, "", "The Confluence API token for authentication")
flags.BoolVar(&p.History, argHistory, false, "Scan pages history")

return confluenceCmd, nil
}

func (p *ConfluencePlugin) initialize(cmd *cobra.Command, urlArg string) error {
func (p *ConfluencePlugin) initialize(urlArg string) error {

p.URL = strings.TrimRight(urlArg, "/")
url := strings.TrimRight(urlArg, "/")

if p.Username == "" || p.Token == "" {
if username == "" || token == "" {
log.Warn().Msg("confluence credentials were not provided. The scan will be made anonymously only for the public pages")
}
p.client = newConfluenceClient(url, token, username)

p.Limit = make(chan struct{}, confluenceMaxRequests)
return nil
}

func (p *ConfluencePlugin) getItems(items chan Item, errs chan error, wg *sync.WaitGroup) {
func (p *ConfluencePlugin) scanConfluence(wg *sync.WaitGroup) {
spaces, err := p.getSpaces()
if err != nil {
errs <- err
p.errorsChan <- err
}

for _, space := range spaces {
wg.Add(1)
go p.getSpaceItems(items, errs, wg, space)
go p.scanConfluenceSpace(wg, space)
}
}

func (p *ConfluencePlugin) getSpaceItems(items chan Item, errs chan error, wg *sync.WaitGroup, space ConfluenceSpaceResult) {
func (p *ConfluencePlugin) scanConfluenceSpace(wg *sync.WaitGroup, space ConfluenceSpaceResult) {
defer wg.Done()

pages, err := p.getPages(space)
if err != nil {
errs <- err
p.errorsChan <- err
return
}

for _, page := range pages.Pages {
wg.Add(1)
p.Limit <- struct{}{}
go func(page ConfluencePage) {
p.getPageItems(items, errs, wg, page, space)
p.scanPageAllVersions(wg, page, space)
<-p.Limit
}(page)
}
}

func (p *ConfluencePlugin) scanPageAllVersions(wg *sync.WaitGroup, page ConfluencePage, space ConfluenceSpaceResult) {
defer wg.Done()

previousVersion := p.scanPageVersion(page, space, 0)
if !p.History {
return
}

for previousVersion > 0 {
previousVersion = p.scanPageVersion(page, space, previousVersion)
}
}

func (p *ConfluencePlugin) scanPageVersion(page ConfluencePage, space ConfluenceSpaceResult, version int) int {
pageContent, err := p.client.getPageContentRequest(page, version)
if err != nil {
p.errorsChan <- err
return 0
}
itemID := fmt.Sprintf("%s-%s-%s", p.GetName(), space.Key, page.ID)
p.itemsChan <- convertPageToItem(pageContent, itemID)

return pageContent.History.PreviousVersion.Number
}

func convertPageToItem(pageContent *ConfluencePageContent, itemID string) Item {
return Item{
Content: pageContent.Body.Storage.Value,
ID: itemID,
Source: pageContent.Links["base"] + pageContent.Links["webui"],
}
}

func (p *ConfluencePlugin) getSpaces() ([]ConfluenceSpaceResult, error) {
totalSpaces, err := p.getSpacesRequest(0)
totalSpaces, err := p.client.getSpacesRequest(0)
if err != nil {
return nil, err
}

actualSize := totalSpaces.Size

for actualSize == confluenceDefaultWindow {
moreSpaces, err := p.getSpacesRequest(totalSpaces.Size)
moreSpaces, err := p.client.getSpacesRequest(totalSpaces.Size)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -166,24 +199,8 @@ func (p *ConfluencePlugin) getSpaces() ([]ConfluenceSpaceResult, error) {
return filteredSpaces, nil
}

func (p *ConfluencePlugin) getSpacesRequest(start int) (*ConfluenceSpaceResponse, error) {
url := fmt.Sprintf("%s/rest/api/space?start=%d", p.URL, start)
body, _, err := utils.HttpRequest(http.MethodGet, url, p, utils.RetrySettings{})
if err != nil {
return nil, fmt.Errorf("unexpected error creating an http request %w", err)
}

response := &ConfluenceSpaceResponse{}
jsonErr := json.Unmarshal(body, response)
if jsonErr != nil {
return nil, fmt.Errorf("could not unmarshal response %w", err)
}

return response, nil
}

func (p *ConfluencePlugin) getPages(space ConfluenceSpaceResult) (*ConfluencePageResult, error) {
totalPages, err := p.getPagesRequest(space, 0)
totalPages, err := p.client.getPagesRequest(space, 0)

if err != nil {
return nil, fmt.Errorf("unexpected error creating an http request %w", err)
Expand All @@ -192,7 +209,7 @@ func (p *ConfluencePlugin) getPages(space ConfluenceSpaceResult) (*ConfluencePag
actualSize := len(totalPages.Pages)

for actualSize == confluenceDefaultWindow {
morePages, err := p.getPagesRequest(space, len(totalPages.Pages))
morePages, err := p.client.getPagesRequest(space, len(totalPages.Pages))

if err != nil {
return nil, fmt.Errorf("unexpected error creating an http request %w", err)
Expand All @@ -207,71 +224,96 @@ func (p *ConfluencePlugin) getPages(space ConfluenceSpaceResult) (*ConfluencePag
return totalPages, nil
}

func (p *ConfluencePlugin) getPagesRequest(space ConfluenceSpaceResult, start int) (*ConfluencePageResult, error) {
url := fmt.Sprintf("%s/rest/api/space/%s/content?start=%d", p.URL, space.Key, start)
body, _, err := utils.HttpRequest(http.MethodGet, url, p, utils.RetrySettings{})
/*
* Confluence client
*/

type IConfluenceClient interface {
getSpacesRequest(start int) (*ConfluenceSpaceResponse, error)
getPagesRequest(space ConfluenceSpaceResult, start int) (*ConfluencePageResult, error)
getPageContentRequest(page ConfluencePage, version int) (*ConfluencePageContent, error)
}

type confluenceClient struct {
baseURL string
token string
username string
}

func newConfluenceClient(baseURL, token, username string) IConfluenceClient {
return &confluenceClient{
baseURL: baseURL,
token: token,
username: username,
}
}

func (c *confluenceClient) GetCredentials() (string, string) {
return c.username, c.token
}

func (c *confluenceClient) GetAuthorizationHeader() string {
if c.username == "" || c.token == "" {
return ""
}
return utils.CreateBasicAuthCredentials(c)
}

func (c *confluenceClient) getSpacesRequest(start int) (*ConfluenceSpaceResponse, error) {
url := fmt.Sprintf("%s/rest/api/space?start=%d", c.baseURL, start)
body, _, err := utils.HttpRequest(http.MethodGet, url, c, utils.RetrySettings{})
if err != nil {
return nil, fmt.Errorf("unexpected error creating an http request %w", err)
}

response := ConfluencePageResponse{}
jsonErr := json.Unmarshal(body, &response)
response := &ConfluenceSpaceResponse{}
jsonErr := json.Unmarshal(body, response)
if jsonErr != nil {
return nil, fmt.Errorf("could not unmarshal response %w", err)
}

return &response.Results, nil
return response, nil
}

func (p *ConfluencePlugin) getPageItems(items chan Item, errs chan error, wg *sync.WaitGroup, page ConfluencePage, space ConfluenceSpaceResult) {
defer wg.Done()
func (c *confluenceClient) getPagesRequest(space ConfluenceSpaceResult, start int) (*ConfluencePageResult, error) {
url := fmt.Sprintf("%s/rest/api/space/%s/content?start=%d", c.baseURL, space.Key, start)
body, _, err := utils.HttpRequest(http.MethodGet, url, c, utils.RetrySettings{})

actualPage, previousVersion, err := p.getItem(page, space, 0)
if err != nil {
errs <- err
return
return nil, fmt.Errorf("unexpected error creating an http request %w", err)
}
items <- *actualPage

// If older versions exist & run history is true
for previousVersion > 0 && p.History {
actualPage, previousVersion, err = p.getItem(page, space, previousVersion)
if err != nil {
errs <- err
return
}
items <- *actualPage
response := ConfluencePageResponse{}
jsonErr := json.Unmarshal(body, &response)
if jsonErr != nil {
return nil, fmt.Errorf("could not unmarshal response %w", err)
}

return &response.Results, nil
}

func (p *ConfluencePlugin) getItem(page ConfluencePage, space ConfluenceSpaceResult, version int) (*Item, int, error) {
func (c *confluenceClient) getPageContentRequest(page ConfluencePage, version int) (*ConfluencePageContent, error) {
var url string

// If no version given get the latest, else get the specified version
if version == 0 {
url = fmt.Sprintf("%s/rest/api/content/%s?expand=body.storage,version,history.previousVersion", p.URL, page.ID)
url = fmt.Sprintf("%s/rest/api/content/%s?expand=body.storage,version,history.previousVersion", c.baseURL, page.ID)

} else {
url = fmt.Sprintf("%s/rest/api/content/%s?status=historical&version=%d&expand=body.storage,version,history.previousVersion", p.URL, page.ID, version)
url = fmt.Sprintf("%s/rest/api/content/%s?status=historical&version=%d&expand=body.storage,version,history.previousVersion", c.baseURL, page.ID, version)
}

request, _, err := utils.HttpRequest(http.MethodGet, url, p, utils.RetrySettings{MaxRetries: 3, ErrorCodes: []int{500}})
request, _, err := utils.HttpRequest(http.MethodGet, url, c, utils.RetrySettings{MaxRetries: 3, ErrorCodes: []int{500}})
if err != nil {
return nil, 0, fmt.Errorf("unexpected error creating an http request %w", err)
return nil, fmt.Errorf("unexpected error creating an http request %w", err)
}
pageContent := ConfluencePageContent{}
jsonErr := json.Unmarshal(request, &pageContent)
if jsonErr != nil {
return nil, 0, jsonErr
return nil, jsonErr
}

content := &Item{
Content: pageContent.Body.Storage.Value,
ID: fmt.Sprintf("%s-%s-%s-%s", p.GetName(), p.URL, space.Key, page.ID),
Source: pageContent.Links["base"] + pageContent.Links["webui"],
}
return content, pageContent.History.PreviousVersion.Number, nil
return &pageContent, nil
}

type ConfluenceSpaceResult struct {
Expand Down
4 changes: 2 additions & 2 deletions plugins/discord.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ func (p *DiscordPlugin) DefineCommand(items chan Item, errors chan error) (*cobr
flags.IntVar(&p.Count, messagesCountFlag, 0, "The number of messages to scan. If not provided, all messages will be scanned until the fromDate flag value.")

discordCmd.Run = func(cmd *cobra.Command, args []string) {
err := p.initialize(cmd)
err := p.initialize()
if err != nil {
errors <- fmt.Errorf("discord plugin initialization failed: %w", err)
return
Expand All @@ -76,7 +76,7 @@ func (p *DiscordPlugin) DefineCommand(items chan Item, errors chan error) (*cobr
return discordCmd, nil
}

func (p *DiscordPlugin) initialize(cmd *cobra.Command) error {
func (p *DiscordPlugin) initialize() error {
if len(p.Channels) == 0 {
log.Warn().Msg("discord channels not provided. Will scan all channels")
}
Expand Down
4 changes: 2 additions & 2 deletions plugins/filesystem.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ func (p *FileSystemPlugin) getItems(items chan Item, errs chan error, wg *sync.W
wg.Add(1)
go func(filePath string) {
defer wg.Done()
actualFile, err := p.getItem(wg, filePath)
actualFile, err := p.getItem(filePath)
if err != nil {
errs <- err
return
Expand All @@ -114,7 +114,7 @@ func (p *FileSystemPlugin) getItems(items chan Item, errs chan error, wg *sync.W
}
}

func (p *FileSystemPlugin) getItem(wg *sync.WaitGroup, filePath string) (*Item, error) {
func (p *FileSystemPlugin) getItem(filePath string) (*Item, error) {
log.Debug().Str("file", filePath).Msg("reading file")
b, err := os.ReadFile(filePath)
if err != nil {
Expand Down

0 comments on commit 35f27ed

Please sign in to comment.