fix(mysql): mysql-crond wrong error log TencentBlueKing#9286

dbm-services/common/reverse-api/internal/reverse_call.go

@@ -6,11 +6,13 @@ import (
 	"encoding/json"
 	errs "errors"
 	"io"
+	"log/slog"
 	"net/http"
 	"net/url"
 	"os"
 	"path/filepath"
 	"strconv"
+	"strings"
 
 	"github.com/pkg/errors"
 )
@@ -20,19 +22,24 @@ func ReverseCall(api config.ReverseApiName, bkCloudId int, ports ...int) (data [
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to read nginx proxy addresses")
 	}
+	slog.Info("reserve call", slog.String("nginx addrs", strings.Join(addrs, ",")))
 
 	var errCollect []error
 	for _, addr := range addrs {
+		slog.Info("reserve call", slog.String("on addr", addr))
 		apiPath, _ := url.JoinPath(config.ReverseApiBase, api.String(), "/")
 		ep := url.URL{
 			Scheme: "http",
 			Host:   addr,
 			Path:   apiPath,
 		}
+		slog.Info("reserve call", slog.String("endpoint", ep.String()))
 
 		req, err := http.NewRequest(http.MethodGet, ep.String(), nil)
 		if err != nil {
-			return nil, errors.Wrap(err, "failed to create request")
+			slog.Error("reserve call create request", slog.String("error", err.Error()))
+			errCollect = append(errCollect, err)
+			continue
 		}
 
 		q := req.URL.Query()
@@ -44,8 +51,10 @@ func ReverseCall(api config.ReverseApiName, bkCloudId int, ports ...int) (data [
 
 		data, err = do(req)
 		if err == nil {
+			slog.Info("reserve call", slog.String("data len", strconv.Itoa(len(data))))
 			return data, nil
 		}
+		slog.Error("reserve call do request", slog.String("error", err.Error()))
 		errCollect = append(errCollect, err)
 	}
 
@@ -77,7 +86,8 @@ func do(request *http.Request) (data []byte, err error) {
 	}
 
 	if !r.Result {
-		return nil, errors.Errorf("unexpected status code: %d, body: %s", resp.StatusCode, r.Errors)
+		return nil, errors.Errorf("unexpected status code: %d, msg: %s, error: %s", r.Code, r.Message,
+			r.Errors)
 	}
 
 	return r.Data, nil

dbm-services/mysql/db-tools/mysql-crond/pkg/third_party/nginx_updater/init.go

@@ -5,8 +5,11 @@ import (
 	rconfig "dbm-services/common/reverse-api/config"
 	"dbm-services/mysql/db-tools/mysql-crond/pkg/config"
 	"log/slog"
+	"math/rand"
 	"os"
 	"path/filepath"
+	"strings"
+	"time"
 
 	"github.com/pkg/errors"
 	"github.com/robfig/cron/v3"
@@ -37,10 +40,16 @@ func updater() error {
 		return errors.Wrap(err, "can't create config directory")
 	}
 
+	sleepN := time.Second * time.Duration(rand.Intn(120))
+	slog.Info("rand sleep", slog.Float64("seconds", sleepN.Seconds()))
+	time.Sleep(sleepN)
+	slog.Info("rand sleep awake")
+
 	addrs, err := common.ListNginxAddrs(*config.RuntimeConfig.BkCloudID)
 	if err != nil {
 		return errors.Wrap(err, "list nginx addrs failed")
 	}
+	slog.Info("list nginx addrs", slog.String("addrs", strings.Join(addrs, ",")))
 
 	f, err := os.OpenFile(
 		filepath.Join(rconfig.CommonConfigDir, rconfig.NginxProxyAddrsFileName),
@@ -53,12 +62,14 @@ func updater() error {
 	defer func() {
 		_ = f.Close()
 	}()
+	slog.Info("update nginx addrs recreate addr file success")
 
 	for _, addr := range addrs {
 		if _, err := f.WriteString(addr + "\n"); err != nil {
 			return errors.Wrap(err, "write nginx addrs failed")
 		}
 	}
+	slog.Info("update nginx addrs write addr file success")
 
 	return nil
 }

dbm-services/mysql/db-tools/mysql-monitor/items-config.yaml

@@ -281,7 +281,7 @@
   - spider_master
 - name: priv-check
   enable: true
-  schedule: 0 40 9 * * *
+  schedule: 0 40 9 * * 2
   machine_type:
   - spider
   - remote
@@ -290,7 +290,7 @@
   role: []
 - name: proxy-rebind
   enable: true
-  schedule: '@every 10s'
+  schedule: 0 55 9 * * *
   machine_type:
   - proxy
   role: []

dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/privcheck/checker.go

@@ -1,16 +1,19 @@
 package privcheck
 
 import (
+	"bufio"
 	"dbm-services/common/go-pubpkg/cmutil"
 	"dbm-services/common/go-pubpkg/reportlog"
 	"dbm-services/mysql/db-tools/mysql-monitor/pkg/config"
 	"dbm-services/mysql/db-tools/mysql-monitor/pkg/internal/cst"
 	"dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/privcheck/internal/checker"
 	"dbm-services/mysql/db-tools/mysql-monitor/pkg/monitoriteminterface"
 	"fmt"
+	"io/fs"
 	"log/slog"
 	"os"
 	"path/filepath"
+	"regexp"
 	"time"
 
 	"github.com/jmoiron/sqlx"
@@ -36,26 +39,42 @@ type reportType struct {
 }
 
 func (c *Checker) Run() (msg string, err error) {
-	privs, err := c.showAllPrivileges()
+	//privs, err := c.showAllPrivileges()
+	//if err != nil {
+	//	slog.Error("show all privs", slog.String("err", err.Error()))
+	//	return "", err
+	//}
+
+	privs, err := c.readPrivBackupFile()
 	if err != nil {
-		slog.Error("show all privs", slog.String("err", err.Error()))
+		slog.Error("read backup file", slog.String("err", err.Error()))
 		return "", err
 	}
 
+	slog.Info("priv-check read priv success")
+
 	for _, priv := range privs {
 		c.az.AddPrivSQLString(priv)
 	}
 
+	slog.Info("init az finished")
+
 	report := c.az.Check(true)
 
+	slog.Info("check finished")
+
 	privCheckReportBaseDir := filepath.Join(cst.DBAReportBase, "mysql/privcheck")
 	err = os.MkdirAll(privCheckReportBaseDir, os.ModePerm)
 	if err != nil {
 		slog.Error("create priv check report dir", slog.String("err", err.Error()))
 		return "", err
 	}
 
-	resultReport, err := reportlog.NewReporter(privCheckReportBaseDir, "report.log", nil)
+	resultReport, err := reportlog.NewReporter(
+		privCheckReportBaseDir,
+		fmt.Sprintf("report_%d.log", config.MonitorConfig.Port),
+		nil,
+	)
 	if err != nil {
 		slog.Error("create priv check report", slog.String("err", err.Error()))
 		return "", err
@@ -78,6 +97,67 @@ func (c *Checker) Run() (msg string, err error) {
 	return "", nil
 }
 
+func (c *Checker) readPrivBackupFile() (privs []string, err error) {
+	dbBackupBaseDirs := []string{"/data/dbbak", "/data1/dbbak"}
+	filePattern := regexp.MustCompile(
+		fmt.Sprintf(
+			`^.*_%s_%d_%s.*priv$`,
+			config.MonitorConfig.Ip,
+			config.MonitorConfig.Port,
+			time.Now().Format("20060102"),
+		),
+	)
+
+	var latestPrivFileEntry *fs.DirEntry
+	var latestPrivFileInfo *fs.FileInfo
+	var latestPrivFilePath string
+
+	for _, dir := range dbBackupBaseDirs {
+		err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
+			if err != nil {
+				return err
+			}
+
+			if !d.IsDir() && filePattern.MatchString(d.Name()) {
+				info, err := d.Info()
+				if err != nil {
+					return err
+				}
+				if latestPrivFileEntry == nil || info.ModTime().After((*latestPrivFileInfo).ModTime()) {
+					latestPrivFileEntry = &d
+					latestPrivFileInfo = &info
+					latestPrivFilePath = path
+				}
+			}
+
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if latestPrivFileEntry != nil {
+		f, err := os.Open(latestPrivFilePath)
+		if err != nil {
+			return nil, err
+		}
+		defer func() {
+			_ = f.Close()
+		}()
+
+		scanner := bufio.NewScanner(f)
+		for scanner.Scan() {
+			privs = append(privs, scanner.Text())
+		}
+		if err := scanner.Err(); err != nil {
+			return nil, err
+		}
+	}
+
+	return privs, nil
+}
+
 func (c *Checker) showAllPrivileges() (privs []string, err error) {
 	rows, err := c.db.Queryx(`SELECT user, host FROM mysql.user`)
 	if err != nil {

dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/privcheck/internal/checker/add_priv_sql.go

@@ -1,13 +1,19 @@
 package checker
 
 import (
+	"context"
 	"dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/privcheck/internal/listener"
 	"dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/privcheck/internal/parsing"
+	"log/slog"
 
 	"github.com/antlr4-go/antlr/v4"
 )
 
 func (c *Analyzer) AddPrivSQLString(sql string) {
+	_ = limiter.Wait(context.Background())
+
+	slog.Info("adding privSQL string", slog.String("sql", sql))
+
 	in := antlr.NewInputStream(sql)
 	lexer := parsing.NewMySqlLexer(in)
 	stream := antlr.NewCommonTokenStream(lexer, 0)

dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/privcheck/internal/checker/check.go

@@ -1,5 +1,7 @@
 package checker
 
+import "log/slog"
+
 const (
 	PrivErrorHostConflict            = "host_conflict"
 	PrivErrorDBConflict              = "db_conflict"
@@ -8,6 +10,7 @@ const (
 	PrivErrorGrantToDifferentDB      = "grant_to_different_db"
 	PrivErrorGrantToDifferentTable   = "grant_to_different_table"
 	PrivErrorPrivilegesNotMatch      = "privileges_not_match"
+	PrvErrorsTooManyHosts            = "too_many_hosts"
 )
 
 type PrivErrorInfo struct {
@@ -20,6 +23,7 @@ type PrivErrorInfo struct {
 func (c *Analyzer) Check(deep bool) (res []*PrivErrorInfo) {
 	c.deep = deep
 	for userName, userSummary := range c.userPrivSummaries {
+		slog.Info("priv check", slog.String("userName", userName))
 		if !IsSystemUser(userName) {
 			res = append(res, c.checkUser(userSummary)...)
 		}

dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/privcheck/internal/checker/check_hosts.go

@@ -1,7 +1,9 @@
 package checker
 
 import (
+	"context"
 	"fmt"
+	"log/slog"
 	"slices"
 	"strings"
 
@@ -48,6 +50,17 @@ dbname 不同的权限明细对比
   - 权限
 */
 func compareDBSummary(username, host0, host1 string, ds0, ds1 *dbPrivSummary) (ok bool, res []*PrivErrorInfo) {
+	_ = limiter.Wait(context.Background())
+
+	slog.Info(
+		"priv check compare db summary",
+		slog.String("username", username),
+		slog.String("host", host0),
+		slog.String("host1", host1),
+		slog.Any("ds0", ds0),
+		slog.Any("ds1", ds1),
+	)
+
 	if ds0.WithGrantOption != ds1.WithGrantOption {
 		res = append(
 			res,

dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/privcheck/internal/checker/check_user.go

@@ -1,15 +1,34 @@
 package checker
 
 import (
+	"context"
 	"fmt"
+	"log/slog"
 	"slices"
 	"strings"
 
 	"golang.org/x/exp/maps"
 )
 
 func (c *Analyzer) checkUser(userSummary *userPrivSummary) (res []*PrivErrorInfo) {
+	_ = limiter.Wait(context.Background())
+
+	slog.Info("priv check user", slog.Any("summary", userSummary))
+	if len(userSummary.HostPrivSummaries) > 1000 {
+		slog.Info("priv check user too many hosts")
+		res = append(res, &PrivErrorInfo{
+			ErrorType: PrvErrorsTooManyHosts,
+			Msg: fmt.Sprintf(
+				"too many hosts [%d] on user %s",
+				len(userSummary.HostPrivSummaries),
+				userSummary.Username,
+			),
+		})
+		return res
+	}
+
 	conflictHosts := FindPatternCover(maps.Keys(userSummary.HostPrivSummaries))
+	slog.Info("priv check user", slog.Any("conflict hosts", conflictHosts))
 
 	for _, pair := range conflictHosts {
 		if ok, msg := compareHostSummary(
@@ -32,6 +51,15 @@ host 不同的权限明细对吧
 2. db 明细对比
 */
 func compareHostSummary(username string, hs0, hs1 *hostPrivSummary) (ok bool, res []*PrivErrorInfo) {
+	_ = limiter.Wait(context.Background())
+
+	slog.Info(
+		"priv check host summary",
+		slog.String("username", username),
+		slog.Any("hs0", hs0),
+		slog.Any("hs1", hs1),
+	)
+
 	if hs0.Password != hs1.Password {
 		res = append(res, &PrivErrorInfo{
 			ErrorType: PrivErrorPasswordNotMatch,

dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/privcheck/internal/checker/helper.go

@@ -1,6 +1,7 @@
 package checker
 
 import (
+	"context"
 	"fmt"
 	"regexp"
 	"strings"
@@ -22,6 +23,8 @@ func coverEachOther(a, b string) bool {
 }
 
 func pCover(a, b string) bool {
+	_ = limiter.Wait(context.Background())
+
 	a = strings.Replace(a, "*", `\*`, -1)
 	a = strings.Replace(a, ".", `\.`, -1)
 	a = strings.Replace(a, "%", ".*", -1)