feat(mysql): dbbackup支持备份表结构为全备 TencentBlueKing#8990

dbm-services/common/go-pubpkg/reportlog/report.go

@@ -31,10 +31,12 @@ type LoggerOption struct {
 
 func defaultLoggerOpt() *LoggerOption {
 	return &LoggerOption{
-		MaxSize:    5,  // MB
-		MaxBackups: 10, // num
-		MaxAge:     30, // days
-		Compress:   false,
+		MaxSize:    100, // MB
+		MaxBackups: 10,  // num
+		MaxAge:     30,  // days
+		// report Compress 建议开启 compress，因为 lumberjack 默认 rotate 文件名格式是 report-2025-01-09T08-18-41.933.log
+		//  日志采集如果没配置好可能会重复采集 *.log
+		Compress: true,
 	}
 }
 
@@ -75,6 +77,7 @@ func NewReporter(reportDir, filename string, logOpt *LoggerOption) (*Reporter, e
 		MaxBackups: logOpt.MaxBackups,
 		MaxAge:     logOpt.MaxAge,
 		Compress:   logOpt.Compress,
+		LocalTime:  true,
 	}
 	reporter.log.SetOutput(resultLogger)
 	return reporter, nil

dbm-services/mysql/db-tools/mysql-dbbackup/cmd/subcmd_dump.go

@@ -46,13 +46,19 @@ func init() {
 	_ = viper.BindPFlag("BackupClient.Enable", dumpCmd.PersistentFlags().Lookup("backup-client"))
 	_ = viper.BindPFlag("BackupClient.FileTag", dumpCmd.PersistentFlags().Lookup("backup-file-tag"))
 
-	dumpCmd.PersistentFlags().String("data-schema-grant", "", "all|schema|data|grant, overwrite Public.DataSchemaGrant")
-	dumpCmd.PersistentFlags().String("backup-dir", "/data/dbbak", "backup root path to save, overwrite Public.BackupDir")
-	dumpCmd.PersistentFlags().String("cluster-domain", "", "cluster domain to report, overwrite Public.ClusterAddress")
-	viper.BindPFlag("Public.DataSchemaGrant", dumpCmd.PersistentFlags().Lookup("data-schema-grant"))
+	dumpCmd.PersistentFlags().String("backup-dir", "/data/dbbak",
+		"backup root path to save, overwrite Public.BackupDir")
+	dumpCmd.PersistentFlags().String("cluster-domain", "",
+		"cluster domain to report, overwrite Public.ClusterAddress")
+	dumpCmd.PersistentFlags().String("data-schema-grant", "",
+		"all|schema|data|grant, overwrite Public.DataSchemaGrant")
+	dumpCmd.PersistentFlags().Int("is-full-backup", 0,
+		"report backup-id as full backup. default 0 means auto judge by backup-type,data-schema-grant")
+
 	viper.BindPFlag("Public.BackupDir", dumpCmd.PersistentFlags().Lookup("backup-dir"))
 	viper.BindPFlag("Public.ClusterAddress", dumpCmd.PersistentFlags().Lookup("cluster-domain"))
-	//dumpCmd.PersistentFlags().SetAnnotation("backup-type", "Public.BackupType", []string{"logical", "physical"})
+	viper.BindPFlag("Public.DataSchemaGrant", dumpCmd.PersistentFlags().Lookup("data-schema-grant"))
+	viper.BindPFlag("Public.IsFullBackup", dumpCmd.PersistentFlags().Lookup("is-full-backup"))
 
 	// Connection Options
 	dumpCmd.PersistentFlags().StringP("host", "h", "", "The host to connect to, overwrite Public.MysqlHost")

dbm-services/mysql/db-tools/mysql-dbbackup/cmd/subcmd_dump_logical.go

@@ -101,7 +101,9 @@ var dumpLogicalCmd = &cobra.Command{
 
 		cnf.PhysicalBackup = config.PhysicalBackup{}
 		cnf.PhysicalLoad = config.PhysicalLoad{}
-		cnf.Public.SetFlagFullBackup(-1) // dumplogical command 一律不认为是 full backup，不可用于全库恢复
+		if cnf.Public.IsFullBackup == 0 {
+			cnf.Public.IsFullBackup = -1 // dumplogical command 一律不认为是 full backup，不可用于全库恢复
+		}
 		err = backupData(&cnf)
 		if err != nil {
 			logger.Log.Error("dumpbackup logical failed", err.Error())

dbm-services/mysql/db-tools/mysql-dbbackup/docs/faq.md

@@ -123,10 +123,19 @@ mydumper 的处理比较粗暴，表结构，表数据 都是以指定的 `--set
 
 也可以指定为具体的字符集，但最好与表写入的字符集或者定义的字符集相同，否则导出数据可能错乱。也可以指定为 binary，但这也要求表定义的 comment上没有一些乱码等不可识别的字符，否则结果无法导入（数据可以导入）。
 
-### 7. 关于 tendbcluster 集群备份，请参考 [spider](spiderbackup.md)
+### 7. 如果只备份表结构用于重做从库
+`Public.IsFullBackup` or `--is-full-backup` 这个选项默认 0 代表会自动根据备份方式+备份对象 来决定是否将备份上报为全备
 
+某些情况只需要表结构，可以设置此选项强制上报为全备
+```
+./dbbackup dumpbackup -c dbbackup.3306.ini --is-full-backup 1 \
+ --data-schema-grant schema,grant \
+ --backup-type logical
+```
+
+### 8. 关于 tendbcluster 集群备份，请参考 [spider](spiderbackup.md)
 
-### 8. 常见备份失败处理
+### 9. 常见备份失败处理
 
 #### 1. log copying being too slow
 > it looks like InnoDB log has wrapped around before xtrabackup could process all records due to either log copying being too slow, or  log files being too small.
@@ -142,7 +151,7 @@ mydumper 的处理比较粗暴，表结构，表数据 都是以指定的 `--set
 
 mydumper / myloader 依赖 glibc>=2.14, centos 6.x(or tlinux 1.2) 是 glibc 2.12，可能会报如上错误。查看 glibc 版本`ldd --version |grep libc`。
 
-如果必须使用逻辑备份，可以设置
+如果必须使用逻辑备份，可以设置 `UseMysqldump = auto` 则会在 mydumper 不可用时，自动选择 mysqldump 进行备份
 ```
 [LogicalBackup]
 UseMysqldump = auto

dbm-services/mysql/db-tools/mysql-dbbackup/example-dump.3305.ini

@@ -0,0 +1,68 @@
+[Public]
+MysqlHost       =       x.x.x.x
+MysqlPort       =       3305
+MysqlUser       =       xx
+MysqlPasswd     =       xx
+MysqlCharset    =
+MysqlRole       =       slave
+BackupType      =       physical  # physical | logical | auto
+DataSchemaGrant =       grant
+NoCheckDiskSpace        =       false
+OldFileLeftDay  =       2
+BkBizId         =       123
+BkCloudId       =       0
+ClusterId       =       1234
+ClusterAddress  =       xx.xx.xx.db
+ShardValue      =       0
+BackupTimeout   =       09:00:00
+BackupDir       =       /data/dbbak/
+IOLimitMBPerSec =       300
+IOLimitMasterFactor     =       0.5
+TarSizeThreshold        =       8192
+FtwrlWaitTimeout        =       120
+AcquireLockWaitTimeout  =       10
+KillLongQueryTime       =       0
+BillId  =
+BackupId        =
+StatusReportPath        =       /home/mysql/dbareport/mysql/dbbackup/status
+ReportPath      =       /home/mysql/dbareport/mysql/dbbackup
+IsFullBackup    =   0
+
+[PhysicalBackup]
+Threads =       2
+Throttle        =       200
+LockDDL         =       false
+DefaultsFile    =       /etc/my.cnf
+DisableSlaveMultiThread =       true
+MaxMyisamTables =       10
+ExtraOpt        =
+
+[LogicalBackup]
+Regex   =       ^(?=(?:(.*\..*$)))(?!(?:(test\..*$|mysql\..*$|sys\..*$|db_infobase\..*$|information_schema\..*$|performance_schema\..*$)))
+Databases       =       *
+Tables  =       *
+ExcludeDatabases        =       # 默认会排除这些系统库 mysql,sys,test,information_schema,performance_schema,db_infobase
+ChunkFilesize   =       2048
+DisableCompress =       false
+Threads =       4
+FlushRetryCount =       3
+TrxConsistencyOnly      =       true
+DefaultsFile    =
+UseMysqldump    =       no  # auto | no | yes
+ExtraOpt        =
+
+[LogicalBackupMysqldump]
+BinPath =
+ExtraOpt        =
+
+[EncryptOpt]
+EncryptElgo     =
+EncryptPublicKey        =
+EncryptCmd      =       openssl
+EncryptEnable   =       false
+
+[BackupClient]
+FileTag =       MYSQL_FULL_BACKUP
+StorageType     =
+DoChecksum      =       true
+Enable  =       true

dbm-services/mysql/db-tools/mysql-dbbackup/example-load.3305.ini

@@ -0,0 +1,35 @@
+[LogicalLoad]
+MysqlLoadDir           = /data/dbbak/xx/3305/doDr_20250108145134/3306/xx_yy_zz_logical
+IndexFilePath          = /data/dbbak/xx/3305/xx_yy_zz_logical.index
+MysqlHost              = x.x.x.x
+MysqlPort              = 3305
+MysqlUser              = xxuser
+MysqlPasswd            = xxpass
+MysqlCharset           = utf8mb4
+EnableBinlog           = false
+InitCommand            =
+Threads                = 16
+SchemaOnly             = false
+ExtraOpt               =
+DBListDropIfExists     = infodba_schema
+CreateTableIfNotExists = false
+Databases              =
+Tables                 =
+ExcludeDatabases       =
+ExcludeTables          =
+TablesList             =
+Regex                  = ^(?=(?:(.*\..*$)))(?!(?:(mysql\..*$|sys\..*$|db_infobase\..*$|information_schema\..*$|performance_schema\..*$|test\..*$)))
+
+[PhysicalLoad]
+MysqlLoadDir  = /data/dbbak/xx/3305/doDr_20250108175010/3305/xx_yy_zz_physical
+IndexFilePath = /data/dbbak/xx/3305/xx_yy_zz_physical.index
+DefaultsFile  = /etc/my.cnf.3305
+Threads       = 4
+CopyBack      = false
+ExtraOpt      =
+
+[EncryptOpt]
+EncryptElgo     =
+EncryptPublicKey        =
+EncryptCmd      =       openssl
+EncryptEnable   =       false

dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/public.go

@@ -88,11 +88,13 @@ type Public struct {
 	// issue lock to mysqld: default 10s
 	//  lock-ddl-timeout(xtrabackup57), backup-lock-timeout(xtrabackup80) --lock-wait-timeout(mydumper)
 	AcquireLockWaitTimeout int `ini:"AcquireLockWaitTimeout"`
+	// IsFullBackup 1: true, -1: false, 0: auto
+	// 这个选项默认 0 代表会自动根据备份方式+备份对象 来决定是否将备份上报为全备
+	// 某些情况只需要表结构，可以设置此选项强制上报为全备
+	IsFullBackup int `ini:"IsFullBackup"`
 
 	cnfFilename string
 	targetName  string
-	// isFullBackup 1: true, -1: false, 0: unknown
-	isFullBackup int
 }
 
 // GetCnfFileName TODO
@@ -105,17 +107,6 @@ func (c *Public) SetCnfFileName(filename string) {
 	c.cnfFilename = filename
 }
 
-// IsFullBackup TODO
-func (c *Public) IsFullBackup() int {
-	return c.isFullBackup
-}
-
-// SetFlagFullBackup 是否是全备
-// 全备判断是：DataGrantSchema 是 all, 并且备份的是所有 db
-func (c *Public) SetFlagFullBackup(isFullBackup int) {
-	c.isFullBackup = isFullBackup
-}
-
 func (c *Public) splitDataSchemaGrant() []string {
 	pattern := regexp.MustCompile(`\s*,\s*`)
 	return pattern.Split(c.DataSchemaGrant, -1)

dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_logical.go

@@ -237,6 +237,7 @@ func (l *LogicalDumper) PrepareBackupMetaInfo(cnf *config.BackupConfig) (*dbarep
 		}
 	}
 	metaInfo.JudgeIsFullBackup(&cnf.Public)
+
 	return &metaInfo, nil
 }
 

dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_privileges.go

@@ -62,7 +62,7 @@ func (d *DumperGrant) PrepareBackupMetaInfo(cnf *config.BackupConfig) (*dbarepor
 	metaInfo.BackupBeginTime = d.backupStartTime
 	metaInfo.BackupEndTime = d.backupEndTime
 	metaInfo.BackupConsistentTime = d.backupStartTime
-	// metaInfo.IsFullBackup = false
+	// metaInfo.GetIsFullBackup = false
 	return &metaInfo, nil
 }
 

dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/execute_dump.go

@@ -64,6 +64,7 @@ func ExecuteBackup(cnf *config.BackupConfig) (*dbareport.IndexContent, error) {
 	if err != nil {
 		return nil, err
 	}
+
 	metaInfo.BackupTool = BackupTool
 	return metaInfo, nil
 }

dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/dbareport/index_file.go

@@ -131,15 +131,21 @@ type ExtraFields struct {
 // JudgeIsFullBackup 是否是带所有数据的全备
 // 这里比较难判断逻辑备份 Regex 正则是否只包含系统库，所以优先判断如果是库表备份，认为false
 func (i *IndexContent) JudgeIsFullBackup(cnf *config.Public) bool {
-	if cnf.IsFullBackup() < 0 {
+	if cnf.IsFullBackup < 0 {
+		i.IsFullBackup = false
 		return false
-	} else if cnf.IsFullBackup() > 0 {
+	} else if cnf.IsFullBackup > 0 {
+		i.IsFullBackup = true
 		return true
-	} // == 0: unknown
+	}
+
+	// == 0: unknown，自动判断
+	// 库表备份单，false
 	if !cnf.IfBackupAll() || strings.Contains(cnf.BackupDir, "backupDatabaseTable_") {
 		i.IsFullBackup = false
 		return i.IsFullBackup
 	}
+	// 物理备份数据，true
 	if cnf.IfBackupAll() && i.BackupType == cst.BackupPhysical {
 		i.IsFullBackup = true
 	}

dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/dbareport/report_logger.go

@@ -49,9 +49,10 @@ func Report() *ReportLogger {
 // NewLogReporter TODO
 func NewLogReporter(reportDir string) (*ReportLogger, error) {
 	logOpt := reportlog.LoggerOption{
-		MaxSize:    5,
-		MaxBackups: 30,
+		MaxSize:    100,
+		MaxBackups: 10,
 		MaxAge:     60,
+		Compress:   true,
 	}
 	resultReport, err := reportlog.NewReporter(reportDir, "backup_result.log", &logOpt)
 	if err != nil {