From a006ef39382d0b344414bbd9542f1762a441d51e Mon Sep 17 00:00:00 2001 From: seanlook Date: Thu, 9 Jan 2025 11:23:37 +0800 Subject: [PATCH] =?UTF-8?q?feat(mysql):=20dbbackup=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E5=A4=87=E4=BB=BD=E8=A1=A8=E7=BB=93=E6=9E=84=E4=B8=BA=E5=85=A8?= =?UTF-8?q?=E5=A4=87=20#8990?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/go-pubpkg/reportlog/report.go | 11 +- .../mysql-dbbackup/cmd/subcmd_dump.go | 16 ++- .../mysql-dbbackup/cmd/subcmd_dump_logical.go | 4 +- .../mysql/db-tools/mysql-dbbackup/docs/faq.md | 15 ++- .../mysql-dbbackup/example-dump.3305.ini | 68 ++++++++++++ .../mysql-dbbackup/example-load.3305.ini | 35 ++++++ .../mysql-dbbackup/pkg/config/public.go | 17 +-- .../pkg/src/backupexe/dumper_logical.go | 1 + .../pkg/src/backupexe/dumper_privileges.go | 2 +- .../pkg/src/backupexe/execute_dump.go | 1 + .../pkg/src/dbareport/index_file.go | 12 +- .../pkg/src/dbareport/report_logger.go | 5 +- .../db-tools/mysql-dbbackup/test.2000.ini | 91 --------------- .../mysql/db-tools/mysql-monitor/README.md | 48 ++++++++ .../db-tools/mysql-monitor/items-config.yaml | 9 +- .../ibdstatistic/collect_result2.go | 11 +- .../ibdstatistic/ibd_statistic.go | 105 +++++++++++------- .../itemscollect/ibdstatistic/report_log2.go | 27 +++-- 18 files changed, 294 insertions(+), 184 deletions(-) create mode 100644 dbm-services/mysql/db-tools/mysql-dbbackup/example-dump.3305.ini create mode 100644 dbm-services/mysql/db-tools/mysql-dbbackup/example-load.3305.ini delete mode 100644 dbm-services/mysql/db-tools/mysql-dbbackup/test.2000.ini diff --git a/dbm-services/common/go-pubpkg/reportlog/report.go b/dbm-services/common/go-pubpkg/reportlog/report.go index 004a65491c..804ffff075 100644 --- a/dbm-services/common/go-pubpkg/reportlog/report.go +++ b/dbm-services/common/go-pubpkg/reportlog/report.go @@ -31,10 +31,12 @@ type LoggerOption struct { func defaultLoggerOpt() *LoggerOption { return &LoggerOption{ - MaxSize: 5, // MB - MaxBackups: 10, // num - MaxAge: 30, // days - Compress: false, + MaxSize: 100, // MB + MaxBackups: 10, // num + MaxAge: 30, // days + // report Compress 建议开启 compress，因为 lumberjack 默认 rotate 文件名格式是 report-2025-01-09T08-18-41.933.log + // 日志采集如果没配置好可能会重复采集 *.log + Compress: true, } } @@ -75,6 +77,7 @@ func NewReporter(reportDir, filename string, logOpt *LoggerOption) (*Reporter, e MaxBackups: logOpt.MaxBackups, MaxAge: logOpt.MaxAge, Compress: logOpt.Compress, + LocalTime: true, } reporter.log.SetOutput(resultLogger) return reporter, nil diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/cmd/subcmd_dump.go b/dbm-services/mysql/db-tools/mysql-dbbackup/cmd/subcmd_dump.go index e13f3616b0..9394dcf5e9 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/cmd/subcmd_dump.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/cmd/subcmd_dump.go @@ -46,13 +46,19 @@ func init() { _ = viper.BindPFlag("BackupClient.Enable", dumpCmd.PersistentFlags().Lookup("backup-client")) _ = viper.BindPFlag("BackupClient.FileTag", dumpCmd.PersistentFlags().Lookup("backup-file-tag")) - dumpCmd.PersistentFlags().String("data-schema-grant", "", "all|schema|data|grant, overwrite Public.DataSchemaGrant") - dumpCmd.PersistentFlags().String("backup-dir", "/data/dbbak", "backup root path to save, overwrite Public.BackupDir") - dumpCmd.PersistentFlags().String("cluster-domain", "", "cluster domain to report, overwrite Public.ClusterAddress") - viper.BindPFlag("Public.DataSchemaGrant", dumpCmd.PersistentFlags().Lookup("data-schema-grant")) + dumpCmd.PersistentFlags().String("backup-dir", "/data/dbbak", + "backup root path to save, overwrite Public.BackupDir") + dumpCmd.PersistentFlags().String("cluster-domain", "", + "cluster domain to report, overwrite Public.ClusterAddress") + dumpCmd.PersistentFlags().String("data-schema-grant", "", + "all|schema|data|grant, overwrite Public.DataSchemaGrant") + dumpCmd.PersistentFlags().Int("is-full-backup", 0, + "report backup-id as full backup. default 0 means auto judge by backup-type,data-schema-grant") + viper.BindPFlag("Public.BackupDir", dumpCmd.PersistentFlags().Lookup("backup-dir")) viper.BindPFlag("Public.ClusterAddress", dumpCmd.PersistentFlags().Lookup("cluster-domain")) - //dumpCmd.PersistentFlags().SetAnnotation("backup-type", "Public.BackupType", []string{"logical", "physical"}) + viper.BindPFlag("Public.DataSchemaGrant", dumpCmd.PersistentFlags().Lookup("data-schema-grant")) + viper.BindPFlag("Public.IsFullBackup", dumpCmd.PersistentFlags().Lookup("is-full-backup")) // Connection Options dumpCmd.PersistentFlags().StringP("host", "h", "", "The host to connect to, overwrite Public.MysqlHost") diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/cmd/subcmd_dump_logical.go b/dbm-services/mysql/db-tools/mysql-dbbackup/cmd/subcmd_dump_logical.go index 9826ec7f09..4716837576 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/cmd/subcmd_dump_logical.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/cmd/subcmd_dump_logical.go @@ -101,7 +101,9 @@ var dumpLogicalCmd = &cobra.Command{ cnf.PhysicalBackup = config.PhysicalBackup{} cnf.PhysicalLoad = config.PhysicalLoad{} - cnf.Public.SetFlagFullBackup(-1) // dumplogical command 一律不认为是 full backup，不可用于全库恢复 + if cnf.Public.IsFullBackup == 0 { + cnf.Public.IsFullBackup = -1 // dumplogical command 一律不认为是 full backup，不可用于全库恢复 + } err = backupData(&cnf) if err != nil { logger.Log.Error("dumpbackup logical failed", err.Error()) diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/docs/faq.md b/dbm-services/mysql/db-tools/mysql-dbbackup/docs/faq.md index d51241ee11..207e1a107a 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/docs/faq.md +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/docs/faq.md @@ -123,10 +123,19 @@ mydumper 的处理比较粗暴，表结构，表数据 都是以指定的 `--set 也可以指定为具体的字符集，但最好与表写入的字符集或者定义的字符集相同，否则导出数据可能错乱。也可以指定为 binary，但这也要求表定义的 comment上没有一些乱码等不可识别的字符，否则结果无法导入（数据可以导入）。 -### 7. 关于 tendbcluster 集群备份，请参考 [spider](spiderbackup.md) +### 7. 如果只备份表结构用于重做从库 +`Public.IsFullBackup` or `--is-full-backup` 这个选项默认 0 代表会自动根据备份方式+备份对象 来决定是否将备份上报为全备 +某些情况只需要表结构，可以设置此选项强制上报为全备 +``` +./dbbackup dumpbackup -c dbbackup.3306.ini --is-full-backup 1 \ + --data-schema-grant schema,grant \ + --backup-type logical +``` + +### 8. 关于 tendbcluster 集群备份，请参考 [spider](spiderbackup.md) -### 8. 常见备份失败处理 +### 9. 常见备份失败处理 #### 1. log copying being too slow > it looks like InnoDB log has wrapped around before xtrabackup could process all records due to either log copying being too slow, or log files being too small. @@ -142,7 +151,7 @@ mydumper 的处理比较粗暴，表结构，表数据 都是以指定的 `--set mydumper / myloader 依赖 glibc>=2.14, centos 6.x(or tlinux 1.2) 是 glibc 2.12，可能会报如上错误。查看 glibc 版本`ldd --version |grep libc`。 -如果必须使用逻辑备份，可以设置 +如果必须使用逻辑备份，可以设置 `UseMysqldump = auto` 则会在 mydumper 不可用时，自动选择 mysqldump 进行备份 ``` [LogicalBackup] UseMysqldump = auto diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/example-dump.3305.ini b/dbm-services/mysql/db-tools/mysql-dbbackup/example-dump.3305.ini new file mode 100644 index 0000000000..94ec6036e7 --- /dev/null +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/example-dump.3305.ini @@ -0,0 +1,68 @@ +[Public] +MysqlHost = x.x.x.x +MysqlPort = 3305 +MysqlUser = xx +MysqlPasswd = xx +MysqlCharset = +MysqlRole = slave +BackupType = physical # physical | logical | auto +DataSchemaGrant = grant +NoCheckDiskSpace = false +OldFileLeftDay = 2 +BkBizId = 123 +BkCloudId = 0 +ClusterId = 1234 +ClusterAddress = xx.xx.xx.db +ShardValue = 0 +BackupTimeout = 09:00:00 +BackupDir = /data/dbbak/ +IOLimitMBPerSec = 300 +IOLimitMasterFactor = 0.5 +TarSizeThreshold = 8192 +FtwrlWaitTimeout = 120 +AcquireLockWaitTimeout = 10 +KillLongQueryTime = 0 +BillId = +BackupId = +StatusReportPath = /home/mysql/dbareport/mysql/dbbackup/status +ReportPath = /home/mysql/dbareport/mysql/dbbackup +IsFullBackup = 0 + +[PhysicalBackup] +Threads = 2 +Throttle = 200 +LockDDL = false +DefaultsFile = /etc/my.cnf +DisableSlaveMultiThread = true +MaxMyisamTables = 10 +ExtraOpt = + +[LogicalBackup] +Regex = ^(?=(?:(.*\..*$)))(?!(?:(test\..*$|mysql\..*$|sys\..*$|db_infobase\..*$|information_schema\..*$|performance_schema\..*$))) +Databases = * +Tables = * +ExcludeDatabases = # 默认会排除这些系统库 mysql,sys,test,information_schema,performance_schema,db_infobase +ChunkFilesize = 2048 +DisableCompress = false +Threads = 4 +FlushRetryCount = 3 +TrxConsistencyOnly = true +DefaultsFile = +UseMysqldump = no # auto | no | yes +ExtraOpt = + +[LogicalBackupMysqldump] +BinPath = +ExtraOpt = + +[EncryptOpt] +EncryptElgo = +EncryptPublicKey = +EncryptCmd = openssl +EncryptEnable = false + +[BackupClient] +FileTag = MYSQL_FULL_BACKUP +StorageType = +DoChecksum = true +Enable = true \ No newline at end of file diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/example-load.3305.ini b/dbm-services/mysql/db-tools/mysql-dbbackup/example-load.3305.ini new file mode 100644 index 0000000000..c45f48b757 --- /dev/null +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/example-load.3305.ini @@ -0,0 +1,35 @@ +[LogicalLoad] +MysqlLoadDir = /data/dbbak/xx/3305/doDr_20250108145134/3306/xx_yy_zz_logical +IndexFilePath = /data/dbbak/xx/3305/xx_yy_zz_logical.index +MysqlHost = x.x.x.x +MysqlPort = 3305 +MysqlUser = xxuser +MysqlPasswd = xxpass +MysqlCharset = utf8mb4 +EnableBinlog = false +InitCommand = +Threads = 16 +SchemaOnly = false +ExtraOpt = +DBListDropIfExists = infodba_schema +CreateTableIfNotExists = false +Databases = +Tables = +ExcludeDatabases = +ExcludeTables = +TablesList = +Regex = ^(?=(?:(.*\..*$)))(?!(?:(mysql\..*$|sys\..*$|db_infobase\..*$|information_schema\..*$|performance_schema\..*$|test\..*$))) + +[PhysicalLoad] +MysqlLoadDir = /data/dbbak/xx/3305/doDr_20250108175010/3305/xx_yy_zz_physical +IndexFilePath = /data/dbbak/xx/3305/xx_yy_zz_physical.index +DefaultsFile = /etc/my.cnf.3305 +Threads = 4 +CopyBack = false +ExtraOpt = + +[EncryptOpt] +EncryptElgo = +EncryptPublicKey = +EncryptCmd = openssl +EncryptEnable = false diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/public.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/public.go index 93a26e8ed0..d3d82894c6 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/public.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/public.go @@ -88,11 +88,13 @@ type Public struct { // issue lock to mysqld: default 10s // lock-ddl-timeout(xtrabackup57), backup-lock-timeout(xtrabackup80) --lock-wait-timeout(mydumper) AcquireLockWaitTimeout int `ini:"AcquireLockWaitTimeout"` + // IsFullBackup 1: true, -1: false, 0: auto + // 这个选项默认 0 代表会自动根据备份方式+备份对象 来决定是否将备份上报为全备 + // 某些情况只需要表结构，可以设置此选项强制上报为全备 + IsFullBackup int `ini:"IsFullBackup"` cnfFilename string targetName string - // isFullBackup 1: true, -1: false, 0: unknown - isFullBackup int } // GetCnfFileName TODO @@ -105,17 +107,6 @@ func (c *Public) SetCnfFileName(filename string) { c.cnfFilename = filename } -// IsFullBackup TODO -func (c *Public) IsFullBackup() int { - return c.isFullBackup -} - -// SetFlagFullBackup 是否是全备 -// 全备判断是：DataGrantSchema 是 all, 并且备份的是所有 db -func (c *Public) SetFlagFullBackup(isFullBackup int) { - c.isFullBackup = isFullBackup -} - func (c *Public) splitDataSchemaGrant() []string { pattern := regexp.MustCompile(`\s*,\s*`) return pattern.Split(c.DataSchemaGrant, -1) diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_logical.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_logical.go index 92e0f82b40..08c90a8f6d 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_logical.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_logical.go @@ -237,6 +237,7 @@ func (l *LogicalDumper) PrepareBackupMetaInfo(cnf *config.BackupConfig) (*dbarep } } metaInfo.JudgeIsFullBackup(&cnf.Public) + return &metaInfo, nil } diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_privileges.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_privileges.go index e1f91ad01a..21d796230c 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_privileges.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_privileges.go @@ -62,7 +62,7 @@ func (d *DumperGrant) PrepareBackupMetaInfo(cnf *config.BackupConfig) (*dbarepor metaInfo.BackupBeginTime = d.backupStartTime metaInfo.BackupEndTime = d.backupEndTime metaInfo.BackupConsistentTime = d.backupStartTime - // metaInfo.IsFullBackup = false + // metaInfo.GetIsFullBackup = false return &metaInfo, nil } diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/execute_dump.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/execute_dump.go index 55d90ca7b7..f3683ef19d 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/execute_dump.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/execute_dump.go @@ -64,6 +64,7 @@ func ExecuteBackup(cnf *config.BackupConfig) (*dbareport.IndexContent, error) { if err != nil { return nil, err } + metaInfo.BackupTool = BackupTool return metaInfo, nil } diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/dbareport/index_file.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/dbareport/index_file.go index 9127557716..e5839f56c1 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/dbareport/index_file.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/dbareport/index_file.go @@ -131,15 +131,21 @@ type ExtraFields struct { // JudgeIsFullBackup 是否是带所有数据的全备 // 这里比较难判断逻辑备份 Regex 正则是否只包含系统库，所以优先判断如果是库表备份，认为false func (i *IndexContent) JudgeIsFullBackup(cnf *config.Public) bool { - if cnf.IsFullBackup() < 0 { + if cnf.IsFullBackup < 0 { + i.IsFullBackup = false return false - } else if cnf.IsFullBackup() > 0 { + } else if cnf.IsFullBackup > 0 { + i.IsFullBackup = true return true - } // == 0: unknown + } + + // == 0: unknown，自动判断 + // 库表备份单，false if !cnf.IfBackupAll() || strings.Contains(cnf.BackupDir, "backupDatabaseTable_") { i.IsFullBackup = false return i.IsFullBackup } + // 物理备份数据，true if cnf.IfBackupAll() && i.BackupType == cst.BackupPhysical { i.IsFullBackup = true } diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/dbareport/report_logger.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/dbareport/report_logger.go index 64ec56820f..6cf168c62d 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/dbareport/report_logger.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/dbareport/report_logger.go @@ -49,9 +49,10 @@ func Report() *ReportLogger { // NewLogReporter TODO func NewLogReporter(reportDir string) (*ReportLogger, error) { logOpt := reportlog.LoggerOption{ - MaxSize: 5, - MaxBackups: 30, + MaxSize: 100, + MaxBackups: 10, MaxAge: 60, + Compress: true, } resultReport, err := reportlog.NewReporter(reportDir, "backup_result.log", &logOpt) if err != nil { diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/test.2000.ini b/dbm-services/mysql/db-tools/mysql-dbbackup/test.2000.ini deleted file mode 100644 index a5dbca5e3e..0000000000 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/test.2000.ini +++ /dev/null @@ -1,91 +0,0 @@ -[Public] -BkBizId = 1111 -BkCloudId= -BillId= -BackupId= -ClusterId= -ClusterAddress= -MysqlHost= 127.0.0.1 -MysqlPort= 12006 -MysqlUser= tt -MysqlPasswd= 123456 -MysqlCharset= utf8 -DataSchemaGrant= data,schema,grant -BackupDir= /data/git-code/dbbackup/file -MysqlRole= slave -BackupTimeOut= 09:00:00 -MysqlCharset= utf8 -BackupType= logical -UseMysqldump= true -OldFileLeftDay = 0 -TarSizeThreshold = 8192 -IOLimitMBPerSec = 500 -ReportPath=/data/git-code/dbbackup -StatusReportPath=/data/git-code/dbbackup - - -[Public.EncryptOpt] -EncryptEnable = false -EncryptCmd = openssl -EncryptPublicKey = -EncryptElgo = - -[BackupClient] -Enable = false -StorageType = cos -FileTag = MYSQL_FULL_BACKUP -DoChecksum = true -BackupClientBin = /usr/local/backup_client/bin/backup_client -CosInfoFile = /home/mysql/.cosinfo.toml - -[LogicalBackup] -ChunkFilesize = 2048 -DefaultsFile = /data/mydumper.cnf -Threads = 4 -DisableCompress = false -ExtraOpt = --skip-definer -Regex = ^(?=(?:(.*\..*))) -FlushRetryCount = 3 - -[LogicalBackupMysqldump] -BinPath = /usr/local/mysql/bin/mysqldump -ExtraOpt = --databases tt - -[LogicalLoadMysqldump] -BinPath= /usr/local/mysql/bin/mysql -MysqlHost= 127.0.0.1 -MysqlPort= 12006 -MysqlUser= tt -MysqlPasswd= 123456 -MysqlCharset= utf8 -IndexFilePath = /data/git-code/dbbackup/file/1111_0_127.0.0.1_12006_20240425174637_logical.index -MysqlLoadFilePath = /data/git-code/dbbackup/file/1111_0_127.0.0.1_12006_20240425174637_logical/1111_0_127.0.0.1_12006_20240425174637_logical.sql - -[PhysicalBackup] -Threads= 4 -SplitSpeed= 300 #MB/s -Throttle = 50 # 50 * 10 MB/s -DefaultsFile= /data/mysql-test/mysql-5-7-test/my.cnf.12006 -ExtraOpt = --safe-slave-backup-timeout=60 - -[LogicalLoad] -MysqlHost= 127.0.0.1 -MysqlPort= 12006 -MysqlUser= tt -MysqlPasswd= 123456 -MysqlCharset= utf8 -Threads = 4 -Regex = ^(tt\.) -EnableBinlog = false -MysqlLoadDir = /data/git-code/dbbackup/file/1111_VM-165-14-centos_127.0.0.1_12000_20230423_173959_logical -#IndexFilePath = /data/git-code/dbbackup/file/1111_VM-165-14-centos_127.0.0.1_12000_20230423_173959_logical.index -ExtraOpt = -DBListDropIfExists = db1,dbx -CreateTableIfNotExists = false - -[PhysicalLoad] -Threads = 4 -#IndexFilePath = /data/dbbak/xxxx_physical.index -DefaultsFile= /etc/my.cnf.3306 -MysqlLoadDir = /data/dbbak/xxxx_physical -CopyBack = false diff --git a/dbm-services/mysql/db-tools/mysql-monitor/README.md b/dbm-services/mysql/db-tools/mysql-monitor/README.md index 80136f5071..1703dcf565 100644 --- a/dbm-services/mysql/db-tools/mysql-monitor/README.md +++ b/dbm-services/mysql/db-tools/mysql-monitor/README.md @@ -193,6 +193,54 @@ type ConnectionCollect struct { |db-up|@every 10s|backend, proxy| | 致命 |db 连通性. 硬编码, 不可配置, 无需录入配置系统|enable |mysql_monitor_heart_beat|@every 10s|backend, proxy| | 无 |监控心跳. 硬编码, 不可配置, 无需录入配置系统|enable +## ibd-statistic 说明 +- 统计表空间大小，支持自定义库表名合并 +- 支持只上报 top N 的表，避免上报数据过多，其它表会上报到 `__OTHER_.__OTHER_` 表 + +- 示例 1： +我们一般不关注清档备份库的每个表大小，把他们合并，示例： +``` +- name: ibd-statistic + enable: true + schedule: 0 45 23 * * * + ... + options: + topk_num: 0 + disable_merge_partition: false + disable_merge_rules: false + merge_rules: + - from: "(?Pstage_truncate_).+\\..*" + to: "${db}_MERGED._MERGED" + - from: "(?Pbak_20\\d\\d).+\\..*" + to: "${db}_MERGED._MERGED" + - from: "(bak_cbs)_.+\\.(?P.+)" + to: "${1}_MERGED.${table}" +``` + +上面的是内置规则，如果是 spider 集群，rules 稍微不一样，因为要带了 shard 后缀： +``` + merge_rules: + - from: "(?Pstage_truncate)_.+_(?P\\d+)\\..*" + to: "${db}_MERGED_${shard}._MERGED" + - from: "(?Pbak_20\\d\\d).+_(?P\\d+)\\..*" + to: "${db}_MERGED_${shard}._MERGED" + - from: "(bak_cbs)_.+_(\\d+)\\.(?P
.+)" + to: "${1}_MERGED_${2}.${table}" +``` + +- 示例 2： +某些业务自己分表，统计表可以不关注表的分表后缀 +``` + merge_rules: + - from: "(?P.+)\\.(?P
.+)_\\d+" + to: "${db}.${table}" +``` +注意： + - 当配置自己的规则时，如果想内置规则也要生效，也需要把示例1 里的 3 条规则加上 + - 自定义规则是，`\`需要转义 + 比如一般正则 from: `(?P.+)\.(?P
.+)_\d+`，配置到 yaml 里需要写成 `(?P.+)\\.(?P
.+)_\\d+`， + 否则会报错 Error: yaml: line X: found unknown escape character + ## 生成dbconfig 配置 ``` perl config2sql.pl | sed 's/"enable":"1"/"enable":true/g' diff --git a/dbm-services/mysql/db-tools/mysql-monitor/items-config.yaml b/dbm-services/mysql/db-tools/mysql-monitor/items-config.yaml index 265320d253..a1ed888acf 100644 --- a/dbm-services/mysql/db-tools/mysql-monitor/items-config.yaml +++ b/dbm-services/mysql/db-tools/mysql-monitor/items-config.yaml @@ -60,15 +60,16 @@ - slave - orphan options: - merge_partition: true topk_num: 0 + disable_merge_partition: false + disable_merge_rules: false merge_rules: - from: "(?Pstage_truncate_).+\\..*" to: "${db}_MERGED._MERGED" - from: "(?Pbak_20\\d\\d).+\\..*" - to: "${db}._MERGED" - - from: "(bak_cbs)_.+_(\\d+)\\.(?P
.+)" - to: "${1}_X_${2}.${table}" + to: "${db}_MERGED._MERGED" + - from: "(bak_cbs)_.+\\.(?P
.+)" + to: "${1}_MERGED.${table}" - name: master-slave-heartbeat enable: true schedule: '@every 1m' diff --git a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/collect_result2.go b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/collect_result2.go index 81ae603b9a..f01e829a10 100644 --- a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/collect_result2.go +++ b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/collect_result2.go @@ -52,7 +52,7 @@ func (c *ibdStatistic) collectResult2(dataDir string) (map[string]int64, map[str dbName := filepath.Base(dir) tableName := strings.TrimSuffix(d.Name(), ibdExt) - if *c.MergePartition { + if !c.DisableMergePartition { match := partitionPattern.FindStringSubmatch(d.Name()) if match != nil { tableName = match[1] @@ -60,12 +60,13 @@ func (c *ibdStatistic) collectResult2(dataDir string) (map[string]int64, map[str } if len(c.reMergeRulesFrom) > 0 { - newDbTbName := fmt.Sprintf("%s.%s", dbName, tableName) + oldDbTbName := fmt.Sprintf("%s.%s", dbName, tableName) for i, reMergeRule := range c.reMergeRulesFrom { - if reMergeRule.MatchString(newDbTbName) { - newDbTbName = reMergeRule.ReplaceAllString(newDbTbName, c.reMergeRulesTo[i]) + if reMergeRule.MatchString(oldDbTbName) { + newDbTbName := reMergeRule.ReplaceAllString(oldDbTbName, c.reMergeRulesTo[i]) dbName, tableName, err = cmutil.GetDbTableName(newDbTbName) - //fmt.Println("xxxx1", newDbTbName, dbName, tableName) + slog.Debug("merge rule matched", slog.String("rule_to", c.reMergeRulesTo[i]), + slog.String("name_from", oldDbTbName), slog.String("name_to", newDbTbName)) if err != nil { return errors.WithMessagef(err, "using merge rules to %s", c.reMergeRulesTo[i]) } diff --git a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/ibd_statistic.go b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/ibd_statistic.go index 23e0b75015..60d0cc5960 100644 --- a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/ibd_statistic.go +++ b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/ibd_statistic.go @@ -7,6 +7,7 @@ import ( "regexp" "sort" + "dbm-services/mysql/db-tools/mysql-monitor/pkg/config" "dbm-services/mysql/db-tools/mysql-monitor/pkg/internal/cst" "dbm-services/mysql/db-tools/mysql-monitor/pkg/monitoriteminterface" @@ -38,33 +39,13 @@ var systemDBs = []string{ "performance_schema", "test", "db_infobase", + cst.OTHER_DB_NAME, } func init() { ibdExt = ".ibd" partitionPattern = regexp.MustCompile(`^(.*)#[pP]#.*\.ibd`) - defaultMergeRules = []*MergeRuleDef{ - // 规则配在 yaml 里要 \\. 转义 - // 合并转换后的库表明，必须是 dbX.tableY 格式，如果.分割出的 dbName,tableName 为空，会报错 - &MergeRuleDef{ - // "(?Pstage_truncate_).+\\..*" - // 将以 stage_truncate_ 开头的库表 合并成 stage_truncate_MERGED._MERGED - From: `(?Pstage_truncate_).+\..*`, - To: `${db}_MERGED._MERGED`, - }, - &MergeRuleDef{ - // "(?Pbak_20\\d\\d).+\\..*" - // 将 bak_20190218_dbtest.tb1 / bak_20190318_dbtest_1.tb2 合并成 bak_2019._MERGED - From: `(?Pbak_20\d\d).+\..*`, - To: `${db}._MERGED`, - }, - &MergeRuleDef{ - // "(bak_cbs)_.+_(\\d+)\\.(?P
.+)" - // 将 bak_cbs_dbtest_0.tb1 bak_cbs_dbtesta_1.tb2 合并成 bak_cbs_X_0.tb1 bak_cbs_X_1.tb2 - From: `(bak_cbs)_.+_(\d+)\.(?P
.+)`, - To: `${1}_X_${2}.${table}`, - }, - } + } type MergeRuleDef struct { @@ -73,9 +54,12 @@ type MergeRuleDef struct { } type ibdStatistic struct { - // MergePartition 合并分区表 - MergePartition *bool `mapstructure:"merge_partition"` - // MergeRuleRegex 合并表名，比如 db\.test_(\d+) 会合并 db.test_1 db.test_2 成 db.test_X + // DisableMergePartition 是否合并分区表，默认合并 + DisableMergePartition bool `mapstructure:"disable_merge_partition"` + // DisableMergeRules 是否启用库表名合并规则，默认启用，已内置 3 条规则 + DisableMergeRules bool `mapstructure:"disable_merge_rules"` + + // MergeRules 合并表名，比如 db\.test_(\d+) 会合并 db.test_1 db.test_2 成 db.test_X // 提示：这里的替换规则，可能会把 spider remote _ 也去掉，统计时需要注意 MergeRules []*MergeRuleDef `mapstructure:"merge_rules"` // TopkNum 只上报排名前 k 条记录，0 表示全部 @@ -145,8 +129,53 @@ func (c *ibdStatistic) Name() string { return name } -func (c *ibdStatistic) initCustomOptions(opts monitoriteminterface.ItemOptions) error { - return nil +func (c *ibdStatistic) initCustomOptions() { + defaultMergeRules = []*MergeRuleDef{ + // 规则配在 yaml 里要 \\. 转义 + // 合并转换后的库表明，必须是 dbX.tableY 格式，如果.分割出的 dbName,tableName 为空，会报错 + &MergeRuleDef{ + // "(?Pstage_truncate_).+\\..*" + // 将以 stage_truncate_ 开头的库表 合并成 stage_truncate_MERGED._MERGED + From: `(?Pstage_truncate_20\d\d).+\..*`, + To: `${db}_MERGED._MERGED`, + }, + &MergeRuleDef{ + // "(?Pbak_20\\d\\d).+\\..*" + // 将 bak_20190218_dbtest.tb1 / bak_20190318_dbtest_1.tb2 合并成 bak_2019._MERGED + From: `(?Pbak_20\d\d).+\..*`, + To: `${db}_MERGED._MERGED`, + }, + &MergeRuleDef{ + // "(bak_cbs)_.+_(\\d+)\\.(?P
.+)" + // 将 bak_cbs_dbtest.tb1 bak_cbs_dbtesta.tb2 合并成 bak_cbs_X.tb1 bak_cbs_X.tb2 + From: `(bak_cbs)_.+\.(?P
.+)`, + To: `${1}_MERGED.${table}`, + }, + } + if config.MonitorConfig.MachineType == "remote" { // spider 集群 + defaultMergeRules = []*MergeRuleDef{ + // 规则配在 yaml 里要 \\. 转义 + // 合并转换后的库表明，必须是 dbX.tableY 格式，如果.分割出的 dbName,tableName 为空，会报错 + &MergeRuleDef{ + // "(?Pstage_truncate_).+\\..*" + // 将以 stage_truncate_ 开头的库表 合并成 stage_truncate_MERGED._MERGED + From: `(?Pstage_truncate_20\d\d)_.+_(?P\d+)\..*`, + To: `${db}_MERGED_${shard}._MERGED`, + }, + &MergeRuleDef{ + // "(?Pbak_20\\d\\d).+\\..*" + // 将 bak_20190218_dbtest.tb1 / bak_20190318_dbtest_1.tb2 合并成 bak_2019._MERGED + From: `(?Pbak_20\d\d).+_(?P\d+)\..*`, + To: `${db}_MERGED_${shard}._MERGED`, + }, + &MergeRuleDef{ + // "(bak_cbs)_.+_(\\d+)\\.(?P
.+)" + // 将 bak_cbs_dbtest_0.tb1 bak_cbs_dbtesta_1.tb2 合并成 bak_cbs_X_0.tb1 bak_cbs_X_1.tb2 + From: `(bak_cbs)_.+_(\d+)\.(?P
.+)`, + To: `${1}_MERGED_${2}.${table}`, + }, + } + } } // New TODO @@ -159,17 +188,17 @@ func New(cc *monitoriteminterface.ConnectionCollect) monitoriteminterface.Monito itemObj.db = cc.MySqlDB itemObj.optionMap = opts - if len(itemObj.MergeRules) == 0 { - slog.Info("ibd-statistic", slog.String("msg", "use default merge rules"), - slog.Int("count", len(itemObj.MergeRules))) - itemObj.MergeRules = defaultMergeRules - } else { - slog.Info("ibd-statistic", slog.String("msg", "use custom merge rules"), - slog.Int("count", len(itemObj.MergeRules))) - } - if itemObj.MergePartition == nil { - truePtr := true - itemObj.MergePartition = &truePtr + itemObj.initCustomOptions() + + if !itemObj.DisableMergeRules { + if len(itemObj.MergeRules) == 0 { + itemObj.MergeRules = defaultMergeRules + slog.Info("ibd-statistic", slog.String("msg", "use default merge rules"), + slog.Int("count", len(itemObj.MergeRules))) + } else { + slog.Info("ibd-statistic", slog.String("msg", "use custom merge rules"), + slog.Int("count", len(itemObj.MergeRules))) + } } return &itemObj } diff --git a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/report_log2.go b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/report_log2.go index d362c160dd..f038efdd12 100644 --- a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/report_log2.go +++ b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/report_log2.go @@ -13,7 +13,6 @@ import ( "os" "path/filepath" "slices" - "strings" "time" "dbm-services/common/go-pubpkg/cmutil" @@ -31,6 +30,7 @@ func reportLog2(dbTableSize map[string]int64, dbSize map[string]int64) error { slog.Error("failed to create database size reports directory", slog.String("error", err.Error())) return errors.Wrap(err, "failed to create database size reports directory") } + // TODO add port to report_.log resultReport, err := reportlog.NewReporter(dbsizeReportBaseDir, "report.log", nil) if err != nil { return err @@ -38,23 +38,25 @@ func reportLog2(dbTableSize map[string]int64, dbSize map[string]int64) error { reportTs := cmutil.TimeToSecondPrecision(time.Now()) for dbTableName, tableSize := range dbTableSize { var originalDBName, tableName string - if ss := strings.SplitN(dbTableName, ".", 2); len(ss) == 2 { - originalDBName = ss[0] - tableName = ss[1] - if _, ok := dbSize[originalDBName]; !ok { // 这个不应该发生，防止后面 panic 设置默认值 - slog.Error("failed to read database size for db %s", originalDBName) - dbSize[originalDBName] = 0 - } - } else { - return errors.Errorf("fail to get db table name from %s", dbTableName) + if originalDBName, tableName, err = cmutil.GetDbTableName(dbTableName); err != nil { + return err + } + if _, ok := dbSize[originalDBName]; !ok { // 这个不应该发生，防止后面 panic 设置默认值 + slog.Error("failed to read database size for db %s", originalDBName) + dbSize[originalDBName] = 0 } // 根据 dbm 枚举约定, remote 是 tendbcluster 的存储机器类型 + // originalDBName 是 remote/backend 上的真实 db名 + // dbName 是 业务看到的 db 名（去掉 remote shard后缀） dbName := originalDBName if dbTableName == cst.OTHER_DB_TABLE_NAME { dbSize[originalDBName] = tableSize } - if config.MonitorConfig.MachineType == "remote" && slices.Index(systemDBs, originalDBName) < 0 { + if slices.Index(systemDBs, dbName) >= 0 { + continue + } else if config.MonitorConfig.MachineType == "remote" { + // 针对 spider remote 转换 dbName match := tenDBClusterDbNamePattern.FindStringSubmatch(originalDBName) if match == nil { err := errors.Errorf( @@ -68,9 +70,6 @@ func reportLog2(dbTableSize map[string]int64, dbSize map[string]int64) error { dbName = match[1] } } - if slices.Index(systemDBs, dbName) >= 0 { - continue - } oneTableInfo := tableSizeStruct{ BkCloudId: *config.MonitorConfig.BkCloudID,