Merge branch 'TencentBlueKing:v1.5.0' into feat/mongodb_ticket_add_fi…

…eld/TencentBlueKing#8849

dbm-services/common/db-resource/internal/svr/apply/apply.go

@@ -329,7 +329,7 @@ func (o *SearchContext) MatchLabels(db *gorm.DB) {
 		db.Where(model.JSONQuery("labels").JointOrContains(o.Labels))
 	} else {
 		// 如果请求没有标签, 只能匹配没有标签的资源
-		db.Where(" JSON_TYPE(labels) = 'NULL' OR JSON_LENGTH(labels) < 1 ")
+		db.Where(" JSON_TYPE(labels) is null OR JSON_LENGTH(labels) < 1 ")
 	}
 }
 

dbm-services/common/db-resource/internal/svr/apply/apply_match.go

@@ -186,14 +186,14 @@ func (c *PickerObject) pickerOneByPriority(key string, cross_switch bool) bool {
 const (
 	// PriorityPMax 园区count 最大
 	PriorityPMax = 100000000
-	// PriorityP0 TODO
-	PriorityP0 = 10000
-	// PriorityP1 TODO
-	PriorityP1 = 1000
-	// PriorityP2 TODO
-	PriorityP2 = 100
-	// PriorityP3 TODO
-	PriorityP3 = 10
+	// PriorityP0 priority 0
+	PriorityP0 = 100000
+	// PriorityP1 priority 1
+	PriorityP1 = 10000
+	// PriorityP2 priority 2
+	PriorityP2 = 10
+	// PriorityP3 priority 3
+	PriorityP3 = 1
 )
 
 const (
@@ -214,16 +214,16 @@ func (o *SearchContext) setResourcePriority(ins model.TbRpDetail, ele *Item) {
 	}
 	// 如果请求参数包含规格，如果机器机型匹配,则高优先级
 	if len(o.DeviceClass) > 0 && lo.Contains(o.DeviceClass, ins.DeviceClass) {
-		ele.Priority += PriorityP1
+		ele.Priority += PriorityP0
 	}
 	// 如果请求参数请求了专属业务资源，则标记了专用业务的资源优先级更高
 	if o.IntetionBkBizId > 0 && ins.DedicatedBiz == o.IntetionBkBizId {
-		ele.Priority += PriorityP2
+		ele.Priority += PriorityP0
 	}
 
 	//  如果请求参数请求了专属db类型，机器的资源类型标签只有一个，且等于请求的资源的类中，则优先级更高
 	if lo.IsNotEmpty(o.RsType) && (ins.RsType == o.RsType) {
-		ele.Priority += PriorityP2
+		ele.Priority += PriorityP3
 	}
 	// 如果是匹配的资源是redis资源
 	// 在内存满足的条件下，偏向取cpu核心小的机器

dbm-services/mysql/db-priv/service/clone_client_priv.go

@@ -224,18 +224,45 @@ func (m *CloneClientPrivPara) CloneClientPriv(jsonPara string, ticket string) ([
 							"source ip", m.SourceIp)
 						continue
 					}
-					proxyGrants, err := GetProxyPrivilege(address, matchHosts, item.BkCloudId, m.User)
+					proxyUsers, err := GetProxyPrivilege(address, matchHosts, item.BkCloudId, m.User)
 					if err != nil {
 						slog.Error("msg", "GetProxyPrivilege", err)
 						AddError(&errMsg, address, err)
 					}
-					if len(proxyGrants) == 0 {
+					if len(proxyUsers) == 0 {
 						slog.Info("no match user@host", "instance", address, "user", m.User)
 						continue
 					}
-					proxyGrants = ReplaceHostInProxyGrants(proxyGrants, m.TargetIp)
-					clusterGrant.Sqls = append(clusterGrant.Sqls, InstanceGrantSql{address, proxyGrants})
-					err = ImportProxyPrivileges(proxyGrants, address, item.BkCloudId)
+					proxyUsers = ReplaceHostInProxyGrants(proxyUsers, m.TargetIp)
+
+					var oneBuckUsers []string
+					for _, u := range proxyUsers {
+						oneBuckUsers = append(oneBuckUsers, u)
+						if len(oneBuckUsers) >= 1000 {
+							refreshSql := fmt.Sprintf(
+								"refresh_users('%s', '+')",
+								strings.Join(oneBuckUsers, ","),
+							)
+							clusterGrant.Sqls = append(clusterGrant.Sqls,
+								InstanceGrantSql{address, []string{refreshSql}},
+							)
+							err = ImportProxyPrivileges(
+								[]string{refreshSql},
+								address, item.BkCloudId)
+							if err != nil {
+								AddError(&errMsg, address, err)
+							}
+							oneBuckUsers = []string{}
+						}
+					}
+					refreshSql := fmt.Sprintf(
+						"refresh_users('%s', '+')",
+						strings.Join(oneBuckUsers, ","),
+					)
+					clusterGrant.Sqls = append(clusterGrant.Sqls,
+						InstanceGrantSql{address, []string{refreshSql}},
+					)
+					err = ImportProxyPrivileges([]string{refreshSql}, address, item.BkCloudId)
 					if err != nil {
 						AddError(&errMsg, address, err)
 					}

dbm-services/mysql/db-priv/service/clone_client_priv_base_func.go

@@ -81,8 +81,8 @@ func GetProxyPrivilege(address string, hosts []string, bkCloudId int64, specifie
 	if len(hosts) == 0 {
 		// 实例间克隆
 		for _, user := range usersResult {
-			addUserSQL := fmt.Sprintf("refresh_users('%s','+')", user["user@ip"].(string))
-			grants = append(grants, addUserSQL)
+			//addUserSQL := fmt.Sprintf("refresh_users('%s','+')", user["user@ip"].(string))
+			grants = append(grants, user["user@ip"].(string))
 		}
 	} else {
 		// 客户端克隆
@@ -99,8 +99,9 @@ func GetProxyPrivilege(address string, hosts []string, bkCloudId int64, specifie
 		for _, user := range usersResult {
 			tmpUser := user["user@ip"].(string)
 			if re.MatchString(tmpUser) && !monitorReg.MatchString(tmpUser) {
-				addUserSQL := fmt.Sprintf("refresh_users('%s','+')", tmpUser)
-				grants = append(grants, addUserSQL)
+				//addUserSQL := fmt.Sprintf("refresh_users('%s','+')", tmpUser)
+				//grants = append(grants, addUserSQL)
+				grants = append(grants, tmpUser)
 			}
 		}
 	}

dbm-services/mysql/db-priv/service/clone_instance_priv.go

@@ -1,6 +1,7 @@
 package service
 
 import (
+	"errors"
 	"fmt"
 	"strings"
 	"time"
@@ -93,15 +94,44 @@ func (m *CloneInstancePrivPara) CloneInstancePriv(jsonPara string, ticket string
 		if err != nil {
 			return errno.ClonePrivilegesFail.Add(err.Error())
 		}
-		proxyGrants, err := GetProxyPrivilege(m.Source.Address, nil, *m.BkCloudId, "")
+		proxyUsers, err := GetProxyPrivilege(m.Source.Address, nil, *m.BkCloudId, "")
 		if err != nil {
 			return err
-		} else if len(proxyGrants) == 0 {
+		} else if len(proxyUsers) == 0 {
 			return errno.NoPrivilegesNothingToDo
 		}
-		err = ImportProxyPrivileges(proxyGrants, m.Target.Address, *m.BkCloudId)
+
+		var oneBuckUsers []string
+		var errCollect error
+		for _, u := range proxyUsers {
+			oneBuckUsers = append(oneBuckUsers, u)
+			if len(oneBuckUsers) >= 1000 {
+				refreshSql := fmt.Sprintf(
+					"refresh_users('%s', '+')",
+					strings.Join(oneBuckUsers, ","),
+				)
+
+				err = ImportProxyPrivileges(
+					[]string{refreshSql},
+					m.Target.Address, *m.BkCloudId)
+				if err != nil {
+					errCollect = errors.Join(errCollect, err)
+				}
+				oneBuckUsers = []string{}
+			}
+		}
+		refreshSql := fmt.Sprintf(
+			"refresh_users('%s', '+')",
+			strings.Join(oneBuckUsers, ","),
+		)
+
+		err = ImportProxyPrivileges([]string{refreshSql}, m.Target.Address, *m.BkCloudId)
 		if err != nil {
-			return err
+			errCollect = errors.Join(errCollect, err)
+		}
+
+		if errCollect != nil {
+			return errCollect
 		}
 	}
 	return nil

dbm-services/mysql/db-priv/service/v2/add_priv/prepare.go

@@ -66,7 +66,7 @@ func (c *PrivTaskPara) prepareTenDBHA(targetMetaInfos []*service.Instance) (
 		)
 		if ele.BindTo == internal.MachineTypeProxy {
 			if ele.PaddingProxy {
-				clientIps = append(clientIps, proxyIps...)
+				clientIps = append(c.SourceIPs, proxyIps...)
 			} else {
 				clientIps = proxyIps
 			}

dbm-services/mysql/db-tools/dbactuator/pkg/components/truncate/via_ctl.go

@@ -125,21 +125,6 @@ func (c *ViaCtlComponent) CreateStageTables() error {
 
 // Truncate truncate table 不需要任何处理
 func (c *ViaCtlComponent) Truncate() error {
-	/*
-			v1 版本中, 这一步操作是直接 drop db, 而且写法是 drop db if exists
-			所以可以不用关心 remote 上这个 db 在不在
-			v2 版本优化了 dropSourceDBs, 会先定向删除 db 中表, 能减少超时
-			但是当 remote 的 db 不存在时 (实际确实没了), 会报错
-			所以需要在这里单独把 tcadmin 设置为 0, 不转发 drop 命令
-		    只能在这个函数内设置, 因为这个函数只在最后的 中控清档 步骤执行
-		    如果在前面设置会影响其他步骤
-	*/
-	_, err := c.dbConn.ExecContext(context.Background(), `SET TC_ADMIN=0`)
-	if err != nil {
-		logger.Error("truncate on ctl set tc admin failed: ", err.Error())
-		return err
-	}
-
 	if c.Param.TruncateDataType == "drop_database" {
 		err := c.dropSourceDBs()
 		if err != nil {
@@ -192,11 +177,24 @@ func (c *ViaCtlComponent) dropSourceTables() error {
 // 得循环起来先一个一个把表 drop 了
 func (c *ViaCtlComponent) dropSourceDBs() error {
 	for db := range c.dbTablesMap {
+		// 中控连接设置了 tc_admin = 1
+		// 当清档类型是 drop db 时, remote 的库已经删除了
+		// 为了能清理 spider, tc_admin 必须保持为 1
+		// 所以必须在中控把库临时恢复出来
+		_, err := c.dbConn.ExecContext(
+			context.Background(),
+			fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`", db),
+		)
+		if err != nil {
+			logger.Error("drop source dbs on ctl recreate source db failed: ", err.Error())
+			return err
+		}
+
 		stageDBName := generateStageDBName(c.Param.StageDBHeader, c.Param.FlowTimeStr, db)
 
 		logger.Info(fmt.Sprintf("drop source dbs %v should drop it's tables", c.dbTablesMap[db]))
 
-		err := tpkg.SafeDropSourceTables(c.dbConn, db, stageDBName, c.dbTablesMap[db])
+		err = tpkg.SafeDropSourceTables(c.dbConn, db, stageDBName, c.dbTablesMap[db])
 		if err != nil {
 			logger.Error("drop source tables %v failed: ", c.dbTablesMap[db], err.Error())
 			return err

dbm-services/mysql/db-tools/mysql-monitor/go.mod

@@ -3,6 +3,7 @@ module dbm-services/mysql/db-tools/mysql-monitor
 go 1.21
 
 require (
+	github.com/antlr4-go/antlr/v4 v4.13.0
 	github.com/dlclark/regexp2 v1.10.0
 	github.com/go-playground/validator/v10 v10.15.4
 	github.com/go-sql-driver/mysql v1.6.0
@@ -21,6 +22,8 @@ require (
 	gopkg.in/yaml.v2 v2.4.0
 )
 
+require golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc
+
 require (
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.2 // indirect

dbm-services/mysql/db-tools/mysql-monitor/go.sum

@@ -38,6 +38,8 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI=
+github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
@@ -243,6 +245,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc h1:mCRnTeVUjcrhlRmO0VK8a6k6Rrf6TF9htwo2pJVSjIU=
+golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=