Dev local (#8900)

dbm-services/common/go-pubpkg/errno/code.go

@@ -13,6 +13,8 @@ package errno
 var (
 	// OK TODO
 	OK = Errno{Code: 0, Message: "", CNMessage: ""}
+	// SimulationTaskFailed 语义检查失败
+	SimulationTaskFailed = Errno{Code: 1, Message: "simulation failed", CNMessage: "模拟执行失败"}
 
 	// InternalServerError TODO
 	InternalServerError = Errno{Code: 10001, Message: "Internal server error", CNMessage: "服务器内部错误。"}

dbm-services/common/reverse-api/.gitignore

@@ -0,0 +1,2 @@
+cmd
+cmd/*

dbm-services/common/reverse-api/apis/common/list_nginx_addrs.go

@@ -0,0 +1,23 @@
+package common
+
+import (
+	"dbm-services/common/reverse-api/config"
+	"dbm-services/common/reverse-api/internal"
+	"encoding/json"
+
+	"github.com/pkg/errors"
+)
+
+func ListNginxAddrs(bkCloudId int) ([]string, error) {
+	data, err := internal.ReverseCall(config.ReverseApiCommonListNginxAddrs, bkCloudId)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to call ListNginxAddrs")
+	}
+
+	var addrs []string
+	if err := json.Unmarshal(data, &addrs); err != nil {
+		return nil, errors.Wrap(err, "failed to unmarshal ListNginxAddrs")
+	}
+
+	return addrs, nil
+}

dbm-services/common/reverse-api/apis/mysql/list_instance_info.go

@@ -0,0 +1,56 @@
+package mysql
+
+import (
+	"dbm-services/common/reverse-api/config"
+	"dbm-services/common/reverse-api/internal"
+	"encoding/json"
+
+	"github.com/pkg/errors"
+)
+
+const (
+	AccessLayerStorage string = "storage"
+	AccessLayerProxy   string = "proxy"
+)
+
+type instanceAddr struct {
+	Ip   string `json:"ip"`
+	Port int    `json:"port"`
+}
+
+type commonInstanceInfo struct {
+	instanceAddr
+	ImmuteDomain string `json:"immute_domain"`
+	Phase        string `json:"phase"`
+	Status       string `json:"status"`
+	AccessLayer  string `json:"access_layer"`
+	MachineType  string `json:"machine_type"`
+}
+
+type StorageInstanceInfo struct {
+	commonInstanceInfo
+	IsStandBy         bool           `json:"is_stand_by"`
+	InstanceRole      string         `json:"instance_role"`
+	InstanceInnerRole string         `json:"instance_inner_role"`
+	Receivers         []instanceAddr `json:"receivers"`
+	Ejectors          []instanceAddr `json:"ejectors"`
+}
+
+type ProxyInstanceInfo struct {
+	commonInstanceInfo
+	StorageInstanceList []instanceAddr `json:"storage_instance_list"`
+}
+
+func ListInstanceInfo(bkCloudId int, ports ...int) ([]byte, string, error) {
+	data, err := internal.ReverseCall(config.ReverseApiMySQLListInstanceInfo, bkCloudId, ports...)
+	if err != nil {
+		return nil, "", errors.Wrap(err, "failed to call ListInstanceInfo")
+	}
+	var r []commonInstanceInfo
+	err = json.Unmarshal(data, &r)
+	if err != nil {
+		return nil, "", errors.Wrap(err, "failed to unmarshal ListInstanceInfo")
+	}
+
+	return data, r[0].AccessLayer, nil
+}

dbm-services/common/reverse-api/config/apis.go

@@ -0,0 +1,6 @@
+package config
+
+const (
+	ReverseApiCommonListNginxAddrs  = "common/list_nginx_addrs"
+	ReverseApiMySQLListInstanceInfo = "mysql/list_instance_info"
+)

dbm-services/common/reverse-api/config/init.go

@@ -0,0 +1,11 @@
+package config
+
+const CommonConfigDir = "/home/mysql/common_config"
+const NginxProxyAddrsFileName = "nginx_proxy.list"
+const ReverseApiBase = "apis/proxypass/reverse_api"
+
+type ReverseApiName string
+
+func (c ReverseApiName) String() string {
+	return string(c)
+}

dbm-services/common/reverse-api/go.mod

@@ -0,0 +1,3 @@
+module dbm-services/common/reverse-api
+
+go 1.21.11

dbm-services/common/reverse-api/init.go

@@ -0,0 +1 @@
+package reverse_api

dbm-services/common/reverse-api/internal/init.go

@@ -0,0 +1,11 @@
+package internal
+
+import "encoding/json"
+
+type apiResponse struct {
+	Result  bool            `json:"result"`
+	Code    int             `json:"code"`
+	Message string          `json:"message"`
+	Errors  string          `json:"errors"`
+	Data    json.RawMessage `json:"data"`
+}

dbm-services/common/reverse-api/internal/reverse_call.go

@@ -0,0 +1,103 @@
+package internal
+
+import (
+	"bufio"
+	"dbm-services/common/reverse-api/config"
+	"encoding/json"
+	errs "errors"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strconv"
+
+	"github.com/pkg/errors"
+)
+
+func ReverseCall(api config.ReverseApiName, bkCloudId int, ports ...int) (data []byte, err error) {
+	addrs, err := readNginxProxyAddrs()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to read nginx proxy addresses")
+	}
+
+	var errCollect []error
+	for _, addr := range addrs {
+		apiPath, _ := url.JoinPath(config.ReverseApiBase, api.String(), "/")
+		ep := url.URL{
+			Scheme: "http",
+			Host:   addr,
+			Path:   apiPath,
+		}
+
+		req, err := http.NewRequest(http.MethodGet, ep.String(), nil)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to create request")
+		}
+
+		q := req.URL.Query()
+		q.Add("bk_cloud_id", strconv.Itoa(bkCloudId))
+		for _, port := range ports {
+			q.Add("port", strconv.Itoa(port))
+		}
+		req.URL.RawQuery = q.Encode()
+
+		data, err = do(req)
+		if err == nil {
+			return data, nil
+		}
+		errCollect = append(errCollect, err)
+	}
+
+	return nil, errs.Join(errCollect...)
+}
+
+func do(request *http.Request) (data []byte, err error) {
+	resp, err := http.DefaultClient.Do(request)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to send request")
+	}
+	defer func() {
+		_ = resp.Body.Close()
+	}()
+
+	b, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to read response body")
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, errors.Errorf("unexpected status code: %d, body: %s", resp.StatusCode, string(b))
+	}
+
+	var r apiResponse
+	err = json.Unmarshal(b, &r)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to unmarshal response body")
+	}
+
+	if !r.Result {
+		return nil, errors.Errorf("unexpected status code: %d, body: %s", resp.StatusCode, r.Errors)
+	}
+
+	return r.Data, nil
+}
+
+func readNginxProxyAddrs() (addrs []string, err error) {
+	f, err := os.Open(filepath.Join(config.CommonConfigDir, config.NginxProxyAddrsFileName))
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to open nginx proxy addrs")
+	}
+	defer func() {
+		_ = f.Close()
+	}()
+
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		addrs = append(addrs, scanner.Text())
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, errors.Wrap(err, "failed to read nginx proxy addrs")
+	}
+	return addrs, nil
+}

dbm-services/go.work

@@ -31,6 +31,7 @@ use (
 	mongodb/db-tools/mongo-toolkit-go
 	mongodb/db-tools/dbactuator
 	common/db-dns/dns-api/pkg
+	common/reverse-api
 )
 
 replace github.com/go-sql-driver/mysql => github.com/go-sql-driver/mysql v1.7.1

dbm-services/mysql/db-partition/assests/migrations/000008_init.down.sql

@@ -0,0 +1 @@
+SET NAMES utf8;

dbm-services/mysql/db-partition/assests/migrations/000008_init.up.sql

@@ -0,0 +1,9 @@
+SET NAMES utf8;
+CREATE TABLE `partition_customization_config` (
+    `id` int(11) NOT NULL AUTO_INCREMENT,
+    `bk_biz_id` int(11) NOT NULL,
+    `immute_domain` varchar(255) NOT NULL DEFAULT '',
+    `partition_column` varchar(255) NOT NULL DEFAULT '',
+    PRIMARY KEY (`id`),
+    KEY `bk_biz_id` (`bk_biz_id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;

dbm-services/mysql/db-partition/handler/handler.go

@@ -1,16 +1,17 @@
-// Package handler TODO
+// Package handler TODOG
 package handler
 
 import (
-	"dbm-services/mysql/db-partition/model"
-	"dbm-services/mysql/db-partition/monitor"
 	"errors"
 	"fmt"
 	"log/slog"
 	"net/http"
 	_ "runtime/debug" // debug TODO
 	"time"
 
+	"dbm-services/mysql/db-partition/model"
+	"dbm-services/mysql/db-partition/monitor"
+
 	cron_pkg "github.com/robfig/cron/v3"
 
 	"dbm-services/common/go-pubpkg/errno"
@@ -185,7 +186,7 @@ func DisablePartitionByCluster(r *gin.Context) {
 		SendResponse(r, err, nil)
 		return
 	}
-	slog.Info(fmt.Sprintf("ids: %v, operator: %s", input.Ids, input.Operator))
+	slog.Info(fmt.Sprintf("cluster_ids: %v, operator: %s", input.ClusterIds, input.Operator))
 	err := input.DisablePartitionConfigByCluster()
 	if err != nil {
 		slog.Error(err.Error())

dbm-services/mysql/db-partition/main.go

@@ -1,11 +1,12 @@
 package main
 
 import (
+	"net/http"
+	"os"
+
 	"dbm-services/mysql/db-partition/monitor"
 	"dbm-services/mysql/db-partition/service"
 	"dbm-services/mysql/db-partition/util"
-	"net/http"
-	"os"
 
 	"github.com/gin-gonic/gin"
 	"github.com/golang-migrate/migrate/v4"
@@ -82,4 +83,5 @@ func init() {
 	model.DB.Init()
 	model.InitClient()
 	model.InitBkRepo()
+	model.InitCustimazation()
 }

dbm-services/mysql/db-partition/model/init_custimazation.go

@@ -0,0 +1,46 @@
+/*
+ * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+ * Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+ * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at https://opensource.org/licenses/MIT
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package model
+
+import (
+	"log/slog"
+
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+)
+
+var CustimazationMap map[int64]string
+
+// Custimazation TODO
+type Custimazation struct {
+	id              int    `gorm:"column:id"`
+	BkBizId         int64  `json:"bk_biz_id" gorm:"column:bk_biz_id"`
+	PartitionColumn string `json:"partition_column" gorm:"column:partition_column"`
+	ImmuteDomain    string `json:"immute_domain" gorm:"column:immute_domain"`
+}
+
+func InitCustimazation() {
+	CustimazationMap = make(map[int64]string)
+	custimazations := []Custimazation{}
+	result := DB.Self.Session(&gorm.Session{
+		Logger: logger.Default.LogMode(logger.Info),
+	}).Table("partition_customization_config").Find(&custimazations)
+	if result.Error != nil {
+		slog.Error("定制化配置读取失败！", result.Error)
+	}
+	for _, cs := range custimazations {
+		if cs.ImmuteDomain != "" {
+			CustimazationMap[cs.BkBizId] = cs.ImmuteDomain
+		} else if cs.PartitionColumn != "" {
+			CustimazationMap[cs.BkBizId] = cs.PartitionColumn
+		}
+	}
+}

dbm-services/mysql/db-partition/service/check_partition_base_func.go

@@ -146,16 +146,9 @@ func (config *PartitionConfig) GetDbTableInfo(fromCron bool, host Host) (ptlist
 				slog.Error("GetDbTableInfo", sql, err.Error())
 				return nil, err
 			}
-			// (1)兼容【分区字段为空】的历史问题，对于某些特殊的分区类型，旧系统已经不在页面上支持，所以旧系统有意将分区字段留空，
-			// 		使其无法在页面编辑，避免改变了其他所属分区类别，因此无法核对比较，但不影响新增和删除分区。
-			// (2)兼容web、dnf业务的特殊定制类型，分区字段类型为int，但是系统记录为timestamp，因此无法核对比较，但不影响新增和删除分区。
-			// (3)兼容minigame业务的特殊定制类型，分区类型为0，但是实际定义与分区类型存在差异，因此无法核对比较，但不影响新增和删除分区。
-			webCustomization := config.BkBizId == 159 && config.PartitionColumn == "Fcreate_time"
-			iegamsCustomization := config.BkBizId == 5016839 && config.PartitionColumn == "Fcreate_time"
-			minigameCustomization := config.BkBizId == 121 && config.ImmuteDomain == "gamedb.game-record.minigame.db"
-			dnfCustomization := config.BkBizId == 105 && config.PartitionColumn == "occ_date"
-			if config.PartitionColumn != "" && !webCustomization && !iegamsCustomization &&
-				!minigameCustomization && !dnfCustomization {
+
+			flag := config.CustomizationCheck()
+			if flag {
 				// 分区表至少会有一个分区
 				for _, v := range output.CmdResults[0].TableData {
 					// 如果发现分区字段、分区间隔与规则不符合，需要重新做分区，页面调整了分区规则
@@ -215,6 +208,19 @@ func (config *PartitionConfig) GetDbTableInfo(fromCron bool, host Host) (ptlist
 	return ptlist, nil
 }
 
+func (config *PartitionConfig) CustomizationCheck() (flag bool) {
+	// (1)兼容【分区字段为空】的历史问题，对于某些特殊的分区类型，旧系统已经不在页面上支持，所以旧系统有意将分区字段留空，
+	// 		使其无法在页面编辑，避免改变了其他所属分区类别，因此无法核对比较，但不影响新增和删除分区。
+	// (2)兼容web、dnf业务的特殊定制类型，分区字段类型为int，但是系统记录为timestamp，因此无法核对比较，但不影响新增和删除分区。
+	// (3)兼容minigame业务的特殊定制类型，分区类型为0，但是实际定义与分区类型存在差异，因此无法核对比较，但不影响新增和删除分区。
+	var custFlag bool
+	if val, ok := model.CustimazationMap[config.BkBizId]; ok {
+		custFlag = config.PartitionColumn == val || config.ImmuteDomain == val
+	}
+	flag = config.PartitionColumn != "" && !custFlag
+	return flag
+}
+
 func CheckPartitionExpression(expression, method, column string, partitionType int) (bool, error) {
 	columnWithBackquote := fmt.Sprintf("`%s`", column)
 	switch partitionType {