Feat bitbucket server support (apache#6689)

* feat: initial commit for bitbucket-server * chore: remove bb-cloud code from server Signed-off-by: Marais Van Zyl <[email protected]> * chore: remove more bb cloud references Signed-off-by: Marais Van Zyl <[email protected]> * chore: set line ending to lf Signed-off-by: Marais Van Zyl <[email protected]> * chore(bitbucket-server): spelling mistake * chore(bitbucket-server): update swagger paths * chore(bitbucket-server): fix migration name * chore(bitbucket-server): renames from bitbucket * chore: change test endpoint * refactor: getting scopes to function * chore: update repo/project remote search Signed-off-by: Marais Van Zyl <[email protected]> * chore: update paging * fix(bitbucket-server): remote scopes paging * fix(bitbucket-server): remote scopes * refactor: small changes to account for server apis * test: update raw repository data * test: repo test * chore: refactor fields * several fixes * fix: made prs and pr comments extractable, renamed account to user (better suits bb terminology) * fix: user convert * fix: pr comment convertor * fix: pr_commit_collector and pr_commit_extractor * fix: commit collector/extractor * feat: branch collector, fixes * fix: made collection of commits happen across all branches * fix: deleted build_collector - no use * feat: bitbucket-server in ui + several fixes * fix: renamed bitbucket-server to bitbucket_server * fix: pr base_repo_id issue * feat: grafana dashboard * fix: pr collection issue * fix: pr_collector end * test: e2e for prs * test: e2e for pr_commits * fix: extraction commit author + e2e tests for commits * fix: renamed account_convertor to user_convertor * chore: removed dead comments * fix: rebase issues * fix: test connection * refactor: using dsHelper * fix: scope config crud * feat: implemented transformations, removed previous transformations * refactor: renamed migration * chore: removed dead code * fix: domain types, enabling tasks by default * fix: merge commit sha for pull requests * fix: pr commits pagination issue, remote scopes pagination * fix: pagination fix for all collections in bitbucket_server * fix: recollect issues * fix: merge commit sha through activities api * refactor: removed useless code and bumped up concurrency * refactor: reduced repeating code * fix: renamed bitbucket_server structs and fixed RootPkgPath for both bitbucket and bitbucket_server * fix: remove commit sha parents * fix: pagination issue, removed not used concurrency * fix: conflict BitBucket * fix: cicd detected issues * fix: models * fix: ui issues, linting issues, removed dead code * fix: test issues * fix: branches are not incremental * fix: branches are not incremental * fix: all collectors are now stateless * fix: removed useless tasks * fix: stateful collectors --------- Signed-off-by: Marais Van Zyl <[email protected]> Co-authored-by: Marais Van Zyl <[email protected]> Co-authored-by: Lynwee <[email protected]>
xebia · Feb 16, 2024 · ea0daf4 · ea0daf4
1 parent 27faa6e
commit ea0daf4
Show file tree

Hide file tree

Showing 65 changed files with 5,643 additions and 3 deletions.
diff --git a/.editorconfig b/.editorconfig
@@ -1,5 +1,6 @@
 [*]
 trim_trailing_whitespace = true
+end_of_line = lf
 
 [*.go]
 indent_style = tab

diff --git a/backend/plugins/bitbucket/impl/impl.go b/backend/plugins/bitbucket/impl/impl.go
@@ -180,7 +180,7 @@ func (p Bitbucket) PrepareTaskData(taskCtx plugin.TaskContext, options map[strin
 }
 
 func (p Bitbucket) RootPkgPath() string {
-	return "github.com/apache/incubator-devlake/plugins/bitbucket"
+	return "github.com/apache/incubator-devlake/plugins/bitbucket/" // the "/" fixes an issue where records from "bitbucket_server" are counted as "bitbucket" records and vice versa
 }
 
 func (p Bitbucket) MigrationScripts() []plugin.MigrationScript {

diff --git a/backend/plugins/bitbucket_server/api/blueprint_v200.go b/backend/plugins/bitbucket_server/api/blueprint_v200.go
@@ -0,0 +1,194 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+	"context"
+	"net/url"
+
+	"github.com/apache/incubator-devlake/core/errors"
+	coreModels "github.com/apache/incubator-devlake/core/models"
+	"github.com/apache/incubator-devlake/core/models/domainlayer"
+	"github.com/apache/incubator-devlake/core/models/domainlayer/code"
+	"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
+	"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
+	"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
+	"github.com/apache/incubator-devlake/core/plugin"
+	"github.com/apache/incubator-devlake/core/utils"
+	helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+	"github.com/apache/incubator-devlake/helpers/srvhelper"
+	"github.com/apache/incubator-devlake/plugins/bitbucket_server/models"
+	"github.com/apache/incubator-devlake/plugins/bitbucket_server/tasks"
+)
+
+func MakeDataSourcePipelinePlanV200(
+	subtaskMetas []plugin.SubTaskMeta,
+	connectionId uint64,
+	bpScopes []*coreModels.BlueprintScope,
+) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
+	// get the connection info for url
+	connection, err := dsHelper.ConnSrv.FindByPk(connectionId)
+	if err != nil {
+		return nil, nil, err
+	}
+	scopeDetails, err := dsHelper.ScopeSrv.MapScopeDetails(connectionId, bpScopes)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// needed for the connection to populate its access tokens
+	// if AppKey authentication method is selected
+	_, err = helper.NewApiClientFromConnection(context.TODO(), basicRes, connection)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	plan, err := makeDataSourcePipelinePlanV200(subtaskMetas, scopeDetails, connection)
+	if err != nil {
+		return nil, nil, err
+	}
+	scopes, err := makeScopesV200(scopeDetails, connection)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return plan, scopes, nil
+}
+
+func makeDataSourcePipelinePlanV200(
+	subtaskMetas []plugin.SubTaskMeta,
+	scopeDetails []*srvhelper.ScopeDetail[models.BitbucketServerRepo, models.BitbucketServerScopeConfig],
+	connection *models.BitbucketServerConnection,
+) (coreModels.PipelinePlan, errors.Error) {
+	plan := make(coreModels.PipelinePlan, len(scopeDetails))
+	for i, scopeDetail := range scopeDetails {
+		repo, scopeConfig := scopeDetail.Scope, scopeDetail.ScopeConfig
+		stage := plan[i]
+		if stage == nil {
+			stage = coreModels.PipelineStage{}
+		}
+		// refdiff
+		if scopeConfig != nil && scopeConfig.Refdiff != nil {
+			// add a new task to next stage
+			j := i + 1
+			if j == len(plan) {
+				plan = append(plan, nil)
+			}
+			refdiffOp := scopeConfig.Refdiff
+			refdiffOp["repoId"] = didgen.NewDomainIdGenerator(&models.BitbucketServerRepo{}).Generate(connection.ID, repo.BitbucketId)
+			plan[j] = coreModels.PipelineStage{
+				{
+					Plugin:  "refdiff",
+					Options: refdiffOp,
+				},
+			}
+			scopeConfig.Refdiff = nil
+		}
+
+		// construct task options for bitbucket
+		op := &tasks.BitbucketServerOptions{
+			ConnectionId: repo.ConnectionId,
+			FullName:     repo.BitbucketId,
+		}
+		options, err := tasks.EncodeTaskOptions(op)
+		if err != nil {
+			return nil, err
+		}
+
+		subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas, scopeConfig.Entities)
+		if err != nil {
+			return nil, err
+		}
+		stage = append(stage, &coreModels.PipelineTask{
+			Plugin:   "bitbucket_server",
+			Subtasks: subtasks,
+			Options:  options,
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		// add gitex stage
+		if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CODE) {
+			cloneUrl, err := errors.Convert01(url.Parse(repo.CloneUrl))
+			if err != nil {
+				return nil, err
+			}
+			cloneUrl.User = url.UserPassword(connection.Username, connection.Password)
+			stage = append(stage, &coreModels.PipelineTask{
+				Plugin: "gitextractor",
+				Options: map[string]interface{}{
+					"url":    cloneUrl.String(),
+					"name":   repo.BitbucketId,
+					"repoId": didgen.NewDomainIdGenerator(&models.BitbucketServerRepo{}).Generate(connection.ID, repo.BitbucketId),
+					"proxy":  connection.Proxy,
+				},
+			})
+
+		}
+		plan[i] = stage
+	}
+	return plan, nil
+}
+
+func makeScopesV200(
+	scopeDetails []*srvhelper.ScopeDetail[models.BitbucketServerRepo, models.BitbucketServerScopeConfig],
+	connection *models.BitbucketServerConnection,
+) ([]plugin.Scope, errors.Error) {
+	scopes := make([]plugin.Scope, 0)
+	for _, scopeDetail := range scopeDetails {
+		repo, scopeConfig := scopeDetail.Scope, scopeDetail.ScopeConfig
+		// if no entities specified, use all entities enabled by default
+		if len(scopeConfig.Entities) == 0 {
+			scopeConfig.Entities = plugin.DOMAIN_TYPES
+		}
+		if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CODE_REVIEW) ||
+			utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CODE) ||
+			utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CROSS) {
+			// if we don't need to collect gitex, we need to add repo to scopes here
+			scopeRepo := &code.Repo{
+				DomainEntity: domainlayer.DomainEntity{
+					Id: didgen.NewDomainIdGenerator(&models.BitbucketServerRepo{}).Generate(connection.ID, repo.BitbucketId),
+				},
+				Name: repo.BitbucketId,
+			}
+			scopes = append(scopes, scopeRepo)
+		}
+		// add cicd_scope to scopes
+		if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CICD) {
+			scopeCICD := &devops.CicdScope{
+				DomainEntity: domainlayer.DomainEntity{
+					Id: didgen.NewDomainIdGenerator(&models.BitbucketServerRepo{}).Generate(connection.ID, repo.BitbucketId),
+				},
+				Name: repo.BitbucketId,
+			}
+			scopes = append(scopes, scopeCICD)
+		}
+		// add board to scopes
+		if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_TICKET) {
+			scopeTicket := &ticket.Board{
+				DomainEntity: domainlayer.DomainEntity{
+					Id: didgen.NewDomainIdGenerator(&models.BitbucketServerRepo{}).Generate(connection.ID, repo.BitbucketId),
+				},
+				Name: repo.BitbucketId,
+			}
+			scopes = append(scopes, scopeTicket)
+		}
+	}
+	return scopes, nil
+}
diff --git a/backend/plugins/bitbucket_server/api/connection.go b/backend/plugins/bitbucket_server/api/connection.go
@@ -0,0 +1,163 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+	"context"
+	"net/http"
+
+	"github.com/apache/incubator-devlake/server/api/shared"
+
+	"github.com/apache/incubator-devlake/core/errors"
+	plugin "github.com/apache/incubator-devlake/core/plugin"
+	"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+	"github.com/apache/incubator-devlake/plugins/bitbucket_server/models"
+)
+
+type BitBucketServerTestConnResponse struct {
+	shared.ApiBody
+	Connection *models.BitbucketServerConn
+}
+
+func testConnection(ctx context.Context, connection models.BitbucketServerConn) (*BitBucketServerTestConnResponse, errors.Error) {
+	// test connection
+	apiClient, err := api.NewApiClientFromConnection(context.TODO(), basicRes, &connection)
+	if err != nil {
+		return nil, err
+	}
+	res, err := apiClient.Get("rest/api/1.0/projects", nil, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	if res.StatusCode == http.StatusUnauthorized {
+		return nil, errors.HttpStatus(http.StatusBadRequest).New("StatusUnauthorized error when testing connection")
+	}
+
+	if res.StatusCode != http.StatusOK {
+		return nil, errors.HttpStatus(res.StatusCode).New("unexpected status code when testing connection")
+	}
+	body := BitBucketServerTestConnResponse{}
+	body.Success = true
+	body.Message = "success"
+	body.Connection = &connection
+	// output
+	return &body, nil
+}
+
+// @Summary test bitbucket connection
+// @Description Test bitbucket Connection
+// @Tags plugins/bitbucket_server
+// @Param body body models.BitbucketServerConn true "json body"
+// @Success 200  {object} BitBucketServerTestConnResponse "Success"
+// @Failure 400  {string} errcode.Error "Bad Request"
+// @Failure 500  {string} errcode.Error "Internal Error"
+// @Router /plugins/bitbucket_server/test [POST]
+func TestConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+	// decode
+	var err errors.Error
+	var connection models.BitbucketServerConn
+	if err := api.Decode(input.Body, &connection, vld); err != nil {
+		return nil, errors.BadInput.Wrap(err, "could not decode request parameters")
+	}
+	// test connection
+	result, err := testConnection(context.TODO(), connection)
+	if err != nil {
+		return nil, err
+	}
+	return &plugin.ApiResourceOutput{Body: result, Status: http.StatusOK}, nil
+}
+
+// TestExistingConnection test bitbucket connection
+// @Summary test bitbucket connection
+// @Description Test bitbucket Connection
+// @Tags plugins/bitbucket
+// @Success 200  {object} BitBucketTestConnResponse "Success"
+// @Failure 400  {string} errcode.Error "Bad Request"
+// @Failure 500  {string} errcode.Error "Internal Error"
+// @Router /plugins/bitbucket_server/{connectionId}/test [POST]
+func TestExistingConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+	connection, err := dsHelper.ConnApi.FindByPk(input)
+	if err != nil {
+		return nil, err
+	}
+	// test connection
+	result, err := testConnection(context.TODO(), connection.BitbucketServerConn)
+	if err != nil {
+		return nil, err
+	}
+	return &plugin.ApiResourceOutput{Body: result, Status: http.StatusOK}, nil
+}
+
+// @Summary create bitbucket connection
+// @Description Create bitbucket connection
+// @Tags plugins/bitbucket_server
+// @Param body body models.BitbucketServerConnection true "json body"
+// @Success 200  {object} models.BitbucketServerConnection
+// @Failure 400  {string} errcode.Error "Bad Request"
+// @Failure 500  {string} errcode.Error "Internal Error"
+// @Router /plugins/bitbucket_server/connections [POST]
+func PostConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+	return dsHelper.ConnApi.Post(input)
+}
+
+// @Summary patch bitbucket connection
+// @Description Patch bitbucket connection
+// @Tags plugins/bitbucket_server
+// @Param body body models.BitbucketServerConnection true "json body"
+// @Success 200  {object} models.BitbucketServerConnection
+// @Failure 400  {string} errcode.Error "Bad Request"
+// @Failure 500  {string} errcode.Error "Internal Error"
+// @Router /plugins/bitbucket_server/connections/{connectionId} [PATCH]
+func PatchConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+	return dsHelper.ConnApi.Patch(input)
+}
+
+// @Summary delete a bitbucket connection
+// @Description Delete a bitbucket connection
+// @Tags plugins/bitbucket_server
+// @Success 200  {object} models.BitbucketServerConnection
+// @Failure 400  {string} errcode.Error "Bad Request"
+// @Failure 409  {object} services.BlueprintProjectPairs "References exist to this connection"
+// @Failure 500  {string} errcode.Error "Internal Error"
+// @Router /plugins/bitbucket_server/connections/{connectionId} [DELETE]
+func DeleteConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+	return dsHelper.ConnApi.Delete(input)
+}
+
+// @Summary get all bitbucket connections
+// @Description Get all bitbucket connections
+// @Tags plugins/bitbucket_server
+// @Success 200  {object} []models.BitbucketServerConnection
+// @Failure 400  {string} errcode.Error "Bad Request"
+// @Failure 500  {string} errcode.Error "Internal Error"
+// @Router /plugins/bitbucket_server/connections [GET]
+func ListConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+	return dsHelper.ConnApi.GetAll(input)
+}
+
+// @Summary get bitbucket connection detail
+// @Description Get bitbucket connection detail
+// @Tags plugins/bitbucket_server
+// @Success 200  {object} models.BitbucketServerConnection
+// @Failure 400  {string} errcode.Error "Bad Request"
+// @Failure 500  {string} errcode.Error "Internal Error"
+// @Router /plugins/bitbucket_server/connections/{connectionId} [GET]
+func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+	return dsHelper.ConnApi.GetDetail(input)
+}