fix(mysql): dbbackup处理mydumper中断不退出的问题 TencentBlueKing#9085

dbm-services/mysql/db-tools/mysql-dbbackup/cmd/subcmd_dump.go

@@ -30,7 +30,7 @@ import (
 )
 
 func init() {
-	dumpCmd.Flags().String("backup-type", cst.BackupTypeAuto, "overwrite Public.BackupType")
+	dumpCmd.Flags().StringP("backup-type", "t", cst.BackupTypeAuto, "overwrite Public.BackupType")
 	_ = viper.BindPFlag("Public.BackupType", dumpCmd.Flags().Lookup("backup-type"))
 
 	dumpCmd.PersistentFlags().String("backup-id", "", "overwrite Public.BackupId")
@@ -50,7 +50,7 @@ func init() {
 		"backup root path to save, overwrite Public.BackupDir")
 	dumpCmd.PersistentFlags().String("cluster-domain", "",
 		"cluster domain to report, overwrite Public.ClusterAddress")
-	dumpCmd.PersistentFlags().String("data-schema-grant", "",
+	dumpCmd.PersistentFlags().StringP("data-schema-grant", "g", "",
 		"all|schema|data|grant, overwrite Public.DataSchemaGrant")
 	dumpCmd.PersistentFlags().Int("is-full-backup", 0,
 		"report backup-id as full backup. default 0 means auto judge by backup-type,data-schema-grant")

dbm-services/mysql/db-tools/mysql-dbbackup/cmd/subcmd_dump_logical.go

@@ -45,10 +45,12 @@ func init() {
 	dumpLogicalCmd.Flags().BoolP("no-data", "d", false, "tables to dump, comma separated")
 	dumpLogicalCmd.Flags().BoolP("no-schemas", "m", false, "Do not dump table data")
 	//dumpLogicalCmd.Flags().BoolP("no-views", "W", false, "Do not dump VIEWs")
-	dumpLogicalCmd.Flags().BoolP("triggers", "G", false, "Dump triggers. By default, it do not dump triggers")
-	dumpLogicalCmd.Flags().BoolP("events", "E", false, "Dump stored procedures and functions. "+
-		"By default, it do not dump stored procedures nor functions")
-	dumpLogicalCmd.Flags().BoolP("routines", "R", false, "Dump events. By default, it do not dump events")
+	dumpLogicalCmd.Flags().BoolP("triggers", "G", false,
+		"Dump triggers. By default, it do not dump triggers. work only when data-schema-grant is empty")
+	dumpLogicalCmd.Flags().BoolP("events", "E", false,
+		"Dump events. By default, it do not dump events. work only when data-schema-grant is empty")
+	dumpLogicalCmd.Flags().BoolP("routines", "R", false,
+		"Dump stored procedures and functions. By default, it do not dump. work only when data-schema-grant is empty")
 	viper.BindPFlag("LogicalBackup.NoData", dumpLogicalCmd.Flags().Lookup("no-data"))
 	viper.BindPFlag("LogicalBackup.NoSchemas", dumpLogicalCmd.Flags().Lookup("no-schemas"))
 	//viper.BindPFlag("LogicalBackup.NoViews", dumpLogicalCmd.Flags().Lookup("no-views"))
@@ -102,11 +104,11 @@ var dumpLogicalCmd = &cobra.Command{
 		cnf.PhysicalBackup = config.PhysicalBackup{}
 		cnf.PhysicalLoad = config.PhysicalLoad{}
 		if cnf.Public.IsFullBackup == 0 {
-			cnf.Public.IsFullBackup = -1 // dumplogical command 一律不认为是 full backup，不可用于全库恢复
+			// cnf.Public.IsFullBackup = -1 // dumplogical command 一律不认为是 full backup，不可用于全库恢复
 		}
 		err = backupData(&cnf)
 		if err != nil {
-			logger.Log.Error("dumpbackup logical failed", err.Error())
+			logger.Log.Error("dumpbackup logical failed ", err.Error())
 		}
 		return err
 	},

dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_logical.go

@@ -15,9 +15,11 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
+	"os/signal"
 	"path/filepath"
 	"strconv"
 	"strings"
+	"syscall"
 	"time"
 
 	"github.com/pkg/errors"
@@ -178,18 +180,40 @@ func (l *LogicalDumper) Execute(enableTimeOut bool) error {
 
 	cmd.Stdout = outFile
 	cmd.Stderr = outFile
-	err = cmd.Run()
+
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Setpgid: true,
+	}
+
+	err = cmd.Start()
+
+	sig := make(chan os.Signal)
+	signal.Notify(sig, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) // syscall.SIGKILL
+	go func() {
+		select {
+		case s := <-sig:
+			logger.Log.Warn("dbbackup got signal %v,", s)
+			time.Sleep(500 * time.Millisecond)
+			if cmd.Process != nil {
+				err := cmd.Process.Kill() // syscall.Kill(-cmd.Process.Pid,syscall.SIGKILL)
+				logger.Log.Warnf("kill mydumper %d exit with %v", cmd.Process.Pid, err)
+			}
+		}
+	}()
+
+	err = cmd.Wait()
 	if err != nil {
-		errStrPrefix := fmt.Sprintf("tail 5 error from %s", mydumperLogFile)
+		// mydumper 的错误信息要从头往后看，看最近的日志因为多线程的原因，不准确
+		errStrPrefix := fmt.Sprintf("head 5 error from %s", mydumperLogFile)
 		errStrDetail, _ := util.GrepLinesFromFile(mydumperLogFile, []string{"ERROR", "fatal", "critical"},
-			5, false, true)
+			5, false, false)
 		if len(errStrDetail) > 0 {
 			logger.Log.Info(errStrPrefix)
 			logger.Log.Error(errStrDetail)
 		} else {
 			logger.Log.Warn("tail can not find more detail error message from ", mydumperLogFile)
 		}
-		logger.Log.Error("run logical backup failed", err, l.cnf.Public.MysqlPort)
+		logger.Log.Error("run logical backup failed ", err, l.cnf.Public.MysqlPort)
 		return errors.WithMessagef(err, fmt.Sprintf("%s\n%s", errStrPrefix, errStrDetail))
 	}
 	// check the integrity of backup

dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_logical_mysqldump.go

@@ -14,6 +14,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"io"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -33,6 +34,7 @@ import (
 	"dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/dbareport"
 	"dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/logger"
 	"dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/mysqlconn"
+	"dbm-services/mysql/db-tools/mysql-dbbackup/pkg/util"
 )
 
 // LogicalDumperMysqldump logical dumper using mysqldump tool
@@ -242,6 +244,8 @@ func (l *LogicalDumperMysqldump) Execute(enableTimeOut bool) (err error) {
 	if l.cnf.LogicalBackup.DisableCompress {
 		args = append(args, "-r", outSqlFile)
 	} else {
+		// 注意这里用了管道
+		// bash -c "aaa | bbb" 只要 bbb 命令没报错，返回值一直是 0，所有不能根据 return code 判断命令成功失败，而要从 stderr 判断
 		args = append(args, "|", CmdZstd, "-q", "-f", "-o", outSqlFile+cst.ZstdSuffix)
 	}
 	var cmd *exec.Cmd
@@ -253,18 +257,16 @@ func (l *LogicalDumperMysqldump) Execute(enableTimeOut bool) (err error) {
 		ctx, cancel := context.WithTimeout(context.Background(), (time.Duration(timeDiffUnix))*time.Second)
 		defer cancel()
 
-		cmd = exec.CommandContext(ctx,
-			"bash", "-c",
-			fmt.Sprintf(`%s %s`, binPath, strings.Join(args, " ")))
+		cmd = exec.CommandContext(ctx, "bash", "-c", fmt.Sprintf(`%s %s`, binPath, strings.Join(args, " ")))
 	} else {
-		cmd = exec.Command("bash", "-c",
-			fmt.Sprintf(`%s %s`, binPath, strings.Join(args, " ")))
+		cmd = exec.Command("bash", "-c", fmt.Sprintf(`%s %s`, binPath, strings.Join(args, " ")))
 	}
 
 	logger.Log.Info("logical dump command with mysqldump: ", cmd.String())
 
-	outFile, err := os.Create(filepath.Join(logger.GetLogDir(),
-		fmt.Sprintf("mysqldump_%d.log", int(time.Now().Weekday()))))
+	mysqldumpLogFile := filepath.Join(logger.GetLogDir(),
+		fmt.Sprintf("mysqldump_%d_%d.log", l.cnf.Public.MysqlPort, int(time.Now().Weekday())))
+	outFile, err := os.OpenFile(mysqldumpLogFile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
 	if err != nil {
 		logger.Log.Error("create log file failed: ", err)
 		return err
@@ -273,9 +275,10 @@ func (l *LogicalDumperMysqldump) Execute(enableTimeOut bool) (err error) {
 		_ = outFile.Close()
 	}()
 	cmd.Stdout = outFile
-	//cmd.Stderr = outFile
 	var stderr bytes.Buffer
-	cmd.Stderr = &stderr
+
+	errWriter := io.MultiWriter(outFile, &stderr)
+	cmd.Stderr = errWriter
 
 	mysqldumpBeginTime := time.Now().Format("2006-01-02 15:04:05")
 	l.backupInfo.BackupBeginTime, err = time.ParseInLocation(cst.MydumperTimeLayout, mysqldumpBeginTime, time.Local)
@@ -314,9 +317,20 @@ func (l *LogicalDumperMysqldump) Execute(enableTimeOut bool) (err error) {
 	}
 
 	err = cmd.Run()
-	if err != nil {
-		logger.Log.Error("run logical backup(with mysqldump) failed: ", err, stderr.String())
-		return errors.WithMessage(err, stderr.String())
+	if err != nil || stderr.String() != "" {
+		errStrPrefix := fmt.Sprintf("tail 5 error from %s", mysqldumpLogFile)
+		errStrDetail, _ := util.GrepLinesFromFile(mysqldumpLogFile, nil, 2, false, true)
+		if len(errStrDetail) > 0 {
+			logger.Log.Info(errStrPrefix)
+			logger.Log.Error(errStrDetail)
+		} else {
+			logger.Log.Warn("tail can not find more detail error message from ", mysqldumpLogFile)
+		}
+		logger.Log.Error("run logical(mysqldump) backup failed", err, l.cnf.Public.MysqlPort)
+		if err == nil {
+			return errors.Errorf("%s\n%s", errStrPrefix, errStrDetail)
+		}
+		return errors.WithMessagef(err, fmt.Sprintf("%s\n%s", errStrPrefix, errStrDetail))
 	}
 	mysqldumpEndTime := time.Now().Format("2006-01-02 15:04:05")
 	l.backupInfo.BackupEndTime, err = time.ParseInLocation(cst.MydumperTimeLayout, mysqldumpEndTime, time.Local)

dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/loader_physical.go

@@ -81,6 +81,7 @@ func (p *PhysicalLoader) decompress() error {
 		fmt.Sprintf("--parallel=%d", p.cnf.PhysicalLoad.Threads),
 	}
 	if strings.Compare(p.mysqlVersion, "005007000") < 0 {
+		// xtrabackup <=5.6 没有 removal original 选项
 		args = append(args, p.cnf.PhysicalLoad.MysqlLoadDir)
 	} else {
 		args = append(args, "--remove-original")