From 3fd21f4e428d3670e4fe1c56045b665f63386a07 Mon Sep 17 00:00:00 2001 From: seanlook Date: Tue, 7 Jan 2025 12:41:02 +0800 Subject: [PATCH] =?UTF-8?q?feat(mysql):=20=E4=BF=AE=E6=94=B9mysqldump?= =?UTF-8?q?=E5=8E=8B=E7=BC=A9=E8=8E=B7=E5=8F=96=E4=BD=8D=E7=82=B9=E5=A4=B1?= =?UTF-8?q?=E8=B4=A5,mysql-monitor=E8=A1=A8=E5=A4=A7=E5=B0=8F=E7=BB=9F?= =?UTF-8?q?=E8=AE=A1=E5=90=88=E5=B9=B6=20#8926?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../migrations/000013_kafka_data.up.sql | 47 ++ .../000019_tendbcluster_data.up.sql | 14 +- .../db-config/internal/api/apply_config.go | 2 +- .../db-config/internal/api/config_item.go | 3 +- .../db-config/internal/api/config_plat.go | 6 +- .../common/db-config/internal/api/dbha.go | 4 +- .../internal/handler/simple/config_file.go | 3 +- .../internal/handler/simple/config_version.go | 3 +- .../simpleconfig/config_item_format.go | 14 +- .../service/simpleconfig/config_meta.go | 4 +- .../common/db-config/pkg/validate/validate.go | 140 ------ .../check_value.go | 133 +++--- .../check_value_test.go | 5 +- .../pkg/{validate => validatestruct}/const.go | 2 +- dbm-services/common/go-pubpkg/cmutil/mysql.go | 18 + .../common/go-pubpkg/validate/validate.go | 43 +- dbm-services/common/reverse-api/go.mod | 2 + dbm-services/common/reverse-api/go.sum | 2 + dbm-services/go.work | 1 + .../dbactuator/internal/subcmd/subcmd.go | 8 +- .../backup_download/ieg_backupsys.go | 2 +- .../components/mysql/restore/dbbackup_load.go | 2 +- .../components/mysql/restore/mload_restore.go | 2 +- .../dbactuator/pkg/native/dbworker.go | 49 +- .../mysql/db-tools/gobatch-dml/go.mod | 37 ++ .../mysql/db-tools/gobatch-dml/go.sum | 103 ++++ .../db-tools/gobatch-dml/load_data_tosql.go | 448 ++++++++++++++++++ .../db-tools/gobatch-dml/load_processor.go | 208 ++++++++ .../db-tools/mysql-crond/cmd/subcmd_pause.go | 4 +- .../mysql-dbbackup/cmd/subcmd_dump.go | 4 +- .../mysql/db-tools/mysql-dbbackup/docs/faq.md | 46 +- .../db-tools/mysql-dbbackup/docs/readme.md | 96 ++-- .../pkg/src/backupexe/dumper.go | 6 +- .../pkg/src/backupexe/loader.go | 8 +- .../pkg/src/backupexe/prepareinfo.go | 13 +- .../mysql/db-tools/mysql-monitor/go.mod | 6 +- .../db-tools/mysql-monitor/items-config.yaml | 10 + .../mysql-monitor/pkg/config/items_config.go | 10 + .../mysql-monitor/pkg/internal/cst/const.go | 9 + .../ibdstatistic/collect_result.go | 2 +- .../ibdstatistic/collect_result2.go | 102 ++++ .../ibdstatistic/ibd_statistic.go | 99 +++- .../itemscollect/ibdstatistic/report_log2.go | 95 ++++ .../pkg/itemscollect/items_collect.go | 1 + .../mysqlprocesslist/query_kill.go | 16 + .../events_statements_summary_by_digest.go | 9 + .../pkg/itemscollect/perfschema/readme.md | 62 +++ .../mysql-monitor/pkg/mainloop/main_loop.go | 4 +- .../connection_collect.go | 35 ++ .../pkg/backup/backup_ibs.go | 2 +- .../mysql-rotatebinlog/pkg/rotate/main.go | 2 +- 51 files changed, 1612 insertions(+), 334 deletions(-) delete mode 100644 dbm-services/common/db-config/pkg/validate/validate.go rename dbm-services/common/db-config/pkg/{validate => validatestruct}/check_value.go (77%) rename dbm-services/common/db-config/pkg/{validate => validatestruct}/check_value_test.go (99%) rename dbm-services/common/db-config/pkg/{validate => validatestruct}/const.go (98%) create mode 100644 dbm-services/common/reverse-api/go.sum create mode 100644 dbm-services/mysql/db-tools/gobatch-dml/go.mod create mode 100644 dbm-services/mysql/db-tools/gobatch-dml/go.sum create mode 100644 dbm-services/mysql/db-tools/gobatch-dml/load_data_tosql.go create mode 100644 dbm-services/mysql/db-tools/gobatch-dml/load_processor.go create mode 100644 dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/collect_result2.go create mode 100644 dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/report_log2.go create mode 100644 dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/mysqlprocesslist/query_kill.go create mode 100644 dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/perfschema/events_statements_summary_by_digest.go create mode 100644 dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/perfschema/readme.md diff --git a/dbm-services/common/db-config/assets/migrations/000013_kafka_data.up.sql b/dbm-services/common/db-config/assets/migrations/000013_kafka_data.up.sql index a3800b2286..37dbd2c603 100644 --- a/dbm-services/common/db-config/assets/migrations/000013_kafka_data.up.sql +++ b/dbm-services/common/db-config/assets/migrations/000013_kafka_data.up.sql @@ -29,6 +29,7 @@ INSERT INTO `tb_config_file_def` (`id`, `namespace`, `conf_type`, `conf_file`, ` INSERT INTO `tb_config_file_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_type_lc`, `conf_file_lc`, `level_names`, `level_versioned`, `conf_name_validate`, `conf_value_validate`, `value_type_strict`, `namespace_info`, `version_keep_limit`, `version_keep_days`, `conf_name_order`, `description`, `created_at`, `updated_at`, `updated_by`) VALUES (348,'kafka','dbconf','2.1.1','kafka配置','kafka-2.1.1','plat,app,cluster','cluster',1,1,0,'NULL',5,365,0,'kafka配置文件','2022-09-20 15:17:36','2023-06-30 17:27:00',NULL); INSERT INTO `tb_config_file_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_type_lc`, `conf_file_lc`, `level_names`, `level_versioned`, `conf_name_validate`, `conf_value_validate`, `value_type_strict`, `namespace_info`, `version_keep_limit`, `version_keep_days`, `conf_name_order`, `description`, `created_at`, `updated_at`, `updated_by`) VALUES (350,'kafka','dbconf','2.3.1','kafka配置','kafka-2.3.1','plat,app,cluster','cluster',1,1,0,'NULL',5,365,0,'kafka配置文件','2022-09-20 15:17:36','2023-06-30 17:27:00',NULL); INSERT INTO `tb_config_file_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_type_lc`, `conf_file_lc`, `level_names`, `level_versioned`, `conf_name_validate`, `conf_value_validate`, `value_type_strict`, `namespace_info`, `version_keep_limit`, `version_keep_days`, `conf_name_order`, `description`, `created_at`, `updated_at`, `updated_by`) VALUES (84,'kafka','dbconf','2.4.0','kafka配置','kafka-2.4','plat,app,cluster','cluster',1,1,0,NULL,5,365,0,'kafka配置文件','2022-09-20 15:17:36','2023-06-30 17:27:00',''); +INSERT INTO `tb_config_file_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_type_lc`, `conf_file_lc`, `level_names`, `level_versioned`, `conf_name_validate`, `conf_value_validate`, `value_type_strict`, `namespace_info`, `version_keep_limit`, `version_keep_days`, `conf_name_order`, `description`, `created_at`, `updated_at`, `updated_by`) VALUES (366,'kafka','dbconf','2.4.bkbase','kafka配置','kafka-2.4.bkbase','plat,app,cluster','cluster',1,1,0,'NULL',5,365,0,'kafka配置文件','2024-12-27 16:25:25','2024-12-27 16:25:25',NULL); INSERT INTO `tb_config_file_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_type_lc`, `conf_file_lc`, `level_names`, `level_versioned`, `conf_name_validate`, `conf_value_validate`, `value_type_strict`, `namespace_info`, `version_keep_limit`, `version_keep_days`, `conf_name_order`, `description`, `created_at`, `updated_at`, `updated_by`) VALUES (359,'kafka','dbconf','2.8.2','kafka配置','kafka-2.8','plat,app,cluster','cluster',1,1,0,'NULL',5,365,0,'kafka配置文件\n','2022-09-20 15:17:36','2023-06-30 17:27:00',NULL); INSERT INTO `tb_config_file_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_type_lc`, `conf_file_lc`, `level_names`, `level_versioned`, `conf_name_validate`, `conf_value_validate`, `value_type_strict`, `namespace_info`, `version_keep_limit`, `version_keep_days`, `conf_name_order`, `description`, `created_at`, `updated_at`, `updated_by`) VALUES (360,'kafka','dbconf','3.8.0','kafka配置','kafka-3.8','plat,app,cluster','cluster',1,1,0,'NULL',5,365,0,'kafka配置文件','2022-09-20 15:17:36','2023-06-30 17:27:00',NULL); /*!50112 SET @disable_bulk_load = IF (@is_rocksdb_supported, 'SET SESSION rocksdb_bulk_load = @old_rocksdb_bulk_load', 'SET @dummy_rocksdb_bulk_load = 0') */; @@ -266,6 +267,7 @@ INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, ` INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (23803,'kafka','dbconf','2.4.0','log.dirs','STRING','{{.LogDirs}}',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.0','Kafka configuration for log.dirs','2024-08-01 16:37:16','2024-08-01 16:37:16',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (23797,'kafka','dbconf','2.4.0','log.flush.interval.messages','STRING','10000',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.0','Kafka configuration for log.flush.interval.messages','2024-08-01 16:37:16','2024-08-01 16:37:16',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (23798,'kafka','dbconf','2.4.0','log.flush.interval.ms','STRING','1000',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.0','Kafka configuration for log.flush.interval.ms','2024-08-01 16:37:16','2024-08-01 16:37:16',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25917,'kafka','dbconf','2.4.0','log.message.timestamp.type','STRING','CreateTime','NULL','',1,0,0,0,1,'NULL','NULL','NULL',-1,'NULL','默认消息时间戳','2024-12-27 16:23:37','2024-12-27 16:23:37',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (23780,'kafka','dbconf','2.4.0','log.retention.bytes','STRING','{{.LogRetentionBytes}}',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.0','Kafka configuration for log.retention.bytes','2024-08-01 16:37:16','2024-08-01 16:37:16',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (23801,'kafka','dbconf','2.4.0','log.retention.check.interval.ms','STRING','300000',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.0','Kafka configuration for log.retention.check.interval.ms','2024-08-01 16:37:16','2024-08-01 16:37:16',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (23779,'kafka','dbconf','2.4.0','log.retention.hours','STRING','{{.LogRetentionHours}}',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.0','Kafka configuration for log.retention.hours','2024-08-01 16:37:16','2024-08-01 16:37:16',0); @@ -297,6 +299,51 @@ INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, ` INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (23806,'kafka','dbconf','2.4.0','zookeeper.connect','STRING','{{.ZookeeperConnect}}',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.0','Kafka configuration for zookeeper.connect','2024-08-01 16:37:16','2024-08-01 16:37:16',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (23802,'kafka','dbconf','2.4.0','zookeeper.connection.timeout.ms','STRING','6000',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.0','Kafka configuration for zookeeper.connection.timeout.ms','2024-08-01 16:37:16','2024-08-01 16:37:16',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (12928,'kafka','dbconf','2.4.0','zookeeper_conf','STRING','1','NULL','',1,0,0,0,1,'NULL','NULL','NULL',-1,'NULL','kafka配置','2022-11-14 17:41:55','2022-11-18 11:30:20',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25946,'kafka','dbconf','2.4.bkbase','advertised.listeners','STRING','SASL_PLAINTEXT://{{.Listeners}}',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for advertised.listeners','2024-12-27 16:40:28','2024-12-27 16:40:28',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25934,'kafka','dbconf','2.4.bkbase','auto.create.topics.enable','STRING','true',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for auto.create.topics.enable','2024-12-27 16:40:28','2024-12-27 16:40:28',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25933,'kafka','dbconf','2.4.bkbase','auto.leader.rebalance.enable','STRING','true',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for auto.leader.rebalance.enable','2024-12-27 16:40:28','2024-12-27 16:40:28',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25922,'kafka','dbconf','2.4.bkbase','default.replication.factor','STRING','{{.DefaultReplicationFactor}}',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for default.replication.factor','2024-12-27 16:40:28','2024-12-27 16:40:28',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25932,'kafka','dbconf','2.4.bkbase','delete.topic.enable','STRING','true',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for delete.topic.enable','2024-12-27 16:40:28','2024-12-27 16:40:28',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25961,'kafka','dbconf','2.4.bkbase','factor','STRING','1',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for factor','2024-12-27 19:24:02','2024-12-27 19:24:02',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25952,'kafka','dbconf','2.4.bkbase','fetch.message.max.bytes','STRING','1073741824',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for fetch.message.max.bytes','2024-12-27 16:40:28','2024-12-27 16:40:28',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25928,'kafka','dbconf','2.4.bkbase','group.initial.rebalance.delay.ms','STRING','3000',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for group.initial.rebalance.delay.ms','2024-12-27 16:40:28','2024-12-27 16:40:28',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25955,'kafka','dbconf','2.4.bkbase','jmx_port','INT','9999','NULL','',1,0,0,0,1,'NULL','NULL','NULL',-1,'NULL','jmx端口','2024-12-27 17:22:01','2024-12-27 17:22:01',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25945,'kafka','dbconf','2.4.bkbase','listeners','STRING','SASL_PLAINTEXT://{{.Listeners}}',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for listeners','2024-12-27 16:40:28','2024-12-27 16:40:28',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25940,'kafka','dbconf','2.4.bkbase','log.cleanup.policy','STRING','delete',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for log.cleanup.policy','2024-12-27 16:40:28','2024-12-27 16:40:28',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25944,'kafka','dbconf','2.4.bkbase','log.dirs','STRING','{{.LogDirs}}',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for log.dirs','2024-12-27 16:40:28','2024-12-27 16:40:28',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25938,'kafka','dbconf','2.4.bkbase','log.flush.interval.messages','STRING','10000',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for log.flush.interval.messages','2024-12-27 16:40:28','2024-12-27 16:40:28',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25939,'kafka','dbconf','2.4.bkbase','log.flush.interval.ms','STRING','1000',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for log.flush.interval.ms','2024-12-27 16:40:28','2024-12-27 16:40:28',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25954,'kafka','dbconf','2.4.bkbase','log.message.timestamp.type','STRING','LogAppendTime',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for log.message.timestamp.type','2024-12-27 16:40:28','2024-12-27 16:40:28',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25921,'kafka','dbconf','2.4.bkbase','log.retention.bytes','STRING','{{.LogRetentionBytes}}',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for log.retention.bytes','2024-12-27 16:40:28','2024-12-27 16:40:28',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25942,'kafka','dbconf','2.4.bkbase','log.retention.check.interval.ms','STRING','300000',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for log.retention.check.interval.ms','2024-12-27 16:40:28','2024-12-27 16:40:28',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25920,'kafka','dbconf','2.4.bkbase','log.retention.hours','STRING','{{.LogRetentionHours}}',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for log.retention.hours','2024-12-27 16:40:28','2024-12-27 16:40:28',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25941,'kafka','dbconf','2.4.bkbase','log.segment.bytes','STRING','1073741824',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for log.segment.bytes','2024-12-27 16:40:28','2024-12-27 16:40:28',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25953,'kafka','dbconf','2.4.bkbase','max.incremental.fetch.session.cache.slots','STRING','10000',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for max.incremental.fetch.session.cache.slots','2024-12-27 16:40:28','2024-12-27 16:40:28',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25918,'kafka','dbconf','2.4.bkbase','message.max.bytes','STRING','104857600',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for message.max.bytes','2024-12-27 16:40:28','2024-12-27 16:40:28',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25975,'kafka','dbconf','2.4.bkbase','no_security','STRING','0',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for no_security','2024-12-27 19:24:02','2024-12-27 19:24:02',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25929,'kafka','dbconf','2.4.bkbase','num.io.threads','STRING','{{.NumIOThreads}}',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for num.io.threads','2024-12-27 16:40:28','2024-12-27 16:40:28',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25919,'kafka','dbconf','2.4.bkbase','num.network.threads','STRING','{{.NumNetWorkThreads}}',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for num.network.threads','2024-12-27 16:40:28','2024-12-27 16:40:28',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25923,'kafka','dbconf','2.4.bkbase','num.partitions','STRING','{{.NumPartitions}}',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for num.partitions','2024-12-27 16:40:28','2024-12-27 16:40:28',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25924,'kafka','dbconf','2.4.bkbase','num.recovery.threads.per.data.dir','STRING','4',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for num.recovery.threads.per.data.dir','2024-12-27 16:40:28','2024-12-27 16:40:28',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25930,'kafka','dbconf','2.4.bkbase','num.replica.fetchers','STRING','{{.NumReplicaFetchers}}',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for num.replica.fetchers','2024-12-27 16:40:28','2024-12-27 16:40:28',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25925,'kafka','dbconf','2.4.bkbase','offsets.topic.replication.factor','STRING','{{.DefaultReplicationFactor}}',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for offsets.topic.replication.factor','2024-12-27 16:40:28','2024-12-27 16:40:28',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25958,'kafka','dbconf','2.4.bkbase','partition_num','STRING','1',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for partition_num','2024-12-27 19:24:02','2024-12-27 19:24:02',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25956,'kafka','dbconf','2.4.bkbase','port','STRING','9092',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for port','2024-12-27 19:24:02','2024-12-27 19:24:02',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25951,'kafka','dbconf','2.4.bkbase','replica.fetch.max.bytes','STRING','1073741824',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for replica.fetch.max.bytes','2024-12-27 16:40:28','2024-12-27 16:40:28',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25959,'kafka','dbconf','2.4.bkbase','replication_num','STRING','1',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for replication_num','2024-12-27 19:24:02','2024-12-27 19:24:02',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25957,'kafka','dbconf','2.4.bkbase','retention_hours','STRING','1',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for retention_hours','2024-12-27 19:24:02','2024-12-27 19:24:02',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25948,'kafka','dbconf','2.4.bkbase','sasl.enabled.mechanisms','STRING','SCRAM-SHA-512',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for sasl.enabled.mechanisms','2024-12-27 16:40:28','2024-12-27 16:40:28',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25949,'kafka','dbconf','2.4.bkbase','sasl.mechanism.inter.broker.protocol','STRING','SCRAM-SHA-512',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for sasl.mechanism.inter.broker.protocol','2024-12-27 16:40:28','2024-12-27 16:40:28',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25950,'kafka','dbconf','2.4.bkbase','security.inter.broker.protocol','STRING','SASL_PLAINTEXT',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for security.inter.broker.protocol','2024-12-27 16:40:28','2024-12-27 16:40:28',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25936,'kafka','dbconf','2.4.bkbase','socket.receive.buffer.bytes','STRING','102400',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for socket.receive.buffer.bytes','2024-12-27 16:40:28','2024-12-27 16:40:28',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25937,'kafka','dbconf','2.4.bkbase','socket.request.max.bytes','STRING','104857600',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for socket.request.max.bytes','2024-12-27 16:40:28','2024-12-27 16:40:28',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25935,'kafka','dbconf','2.4.bkbase','socket.send.buffer.bytes','STRING','102400',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for socket.send.buffer.bytes','2024-12-27 16:40:28','2024-12-27 16:40:28',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25927,'kafka','dbconf','2.4.bkbase','transaction.state.log.min.isr','STRING','{{.DefaultReplicationFactor}}',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for transaction.state.log.min.isr','2024-12-27 16:40:28','2024-12-27 16:40:28',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25926,'kafka','dbconf','2.4.bkbase','transaction.state.log.replication.factor','STRING','{{.DefaultReplicationFactor}}',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for transaction.state.log.replication.factor','2024-12-27 16:40:28','2024-12-27 16:40:28',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25931,'kafka','dbconf','2.4.bkbase','unclean.leader.election.enable','STRING','true',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for unclean.leader.election.enable','2024-12-27 16:40:28','2024-12-27 16:40:28',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25947,'kafka','dbconf','2.4.bkbase','zookeeper.connect','STRING','{{.ZookeeperConnect}}',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for zookeeper.connect','2024-12-27 16:40:28','2024-12-27 16:40:28',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25943,'kafka','dbconf','2.4.bkbase','zookeeper.connection.timeout.ms','STRING','6000',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for zookeeper.connection.timeout.ms','2024-12-27 16:40:28','2024-12-27 16:40:28',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25960,'kafka','dbconf','2.4.bkbase','zookeeper_conf','STRING','1',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.4.bkbase','Kafka configuration for zookeeper_conf','2024-12-27 19:24:02','2024-12-27 19:24:02',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25465,'kafka','dbconf','2.8.2','advertised.listeners','STRING','SASL_PLAINTEXT://{{.Listeners}}',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.8.2','Kafka configuration for advertised.listeners','2024-09-12 10:31:12','2024-09-12 10:31:12',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25453,'kafka','dbconf','2.8.2','auto.create.topics.enable','STRING','true',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.8.2','Kafka configuration for auto.create.topics.enable','2024-09-12 10:31:12','2024-09-12 10:31:12',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25452,'kafka','dbconf','2.8.2','auto.leader.rebalance.enable','STRING','true',NULL,'',1,0,0,0,1,NULL,NULL,NULL,-1,'2.8.2','Kafka configuration for auto.leader.rebalance.enable','2024-09-12 10:31:12','2024-09-12 10:31:12',0); diff --git a/dbm-services/common/db-config/assets/migrations/000019_tendbcluster_data.up.sql b/dbm-services/common/db-config/assets/migrations/000019_tendbcluster_data.up.sql index eab23b78ae..71010fd43c 100644 --- a/dbm-services/common/db-config/assets/migrations/000019_tendbcluster_data.up.sql +++ b/dbm-services/common/db-config/assets/migrations/000019_tendbcluster_data.up.sql @@ -1040,7 +1040,7 @@ INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, ` INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (15424,'tendbcluster','dbconf','Spider-1','mysqld.auto_increment_increment','INT','1','[1,65535]','RANGE',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'Controls the interval between successive column values.','2023-03-09 17:55:11','2023-03-09 17:55:11',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (15425,'tendbcluster','dbconf','Spider-1','mysqld.auto_increment_offset','INT','1','[1,65535]','RANGE',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'Determines the starting point for the AUTO_INCREMENT column value.','2023-03-09 17:55:11','2023-03-09 17:55:11',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (15426,'tendbcluster','dbconf','Spider-1','mysqld.bind-address','STRING','{{.Mysqld.BindAddress}}',NULL,'',2,0,0,0,1,'{{mysqld.bind-address}}',NULL,NULL,-1,NULL,NULL,'2023-03-09 17:55:11','2023-03-28 18:01:04',0); -INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (15427,'tendbcluster','dbconf','Spider-1','mysqld.binlog_format','STRING','ROW','ROW| MIXED| STATEMENT','ENUM',1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'What form of binary logging the master will use.','2023-03-09 17:55:11','2023-04-19 14:31:26',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (15427,'tendbcluster','dbconf','Spider-1','mysqld.binlog_format','STRING','STATEMENT','MIXED| STATEMENT','ENUM',1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'What form of binary logging the master will use.','2023-03-09 17:55:11','2024-12-13 15:37:16',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (15428,'tendbcluster','dbconf','Spider-1','mysqld.character_set_server','STRING','{{.Mysqld.CharacterSetServer}}','LATIN1| UTF8| GBK| UTF8MB4 |{{mysqld.character_set_server}}','ENUM',2,0,0,0,1,'{{mysqld.character_set_server}}',NULL,NULL,-1,NULL,'Specify default server character set','2023-03-09 17:55:11','2023-03-28 18:01:10',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (15429,'tendbcluster','dbconf','Spider-1','mysqld.collation_server','STRING','',NULL,'',-1,0,0,0,1,'{{mysqld.collation_server}}',NULL,NULL,-1,NULL,NULL,'2023-03-09 17:55:11','2023-03-09 17:55:11',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (15430,'tendbcluster','dbconf','Spider-1','mysqld.concurrent_insert','STRING','AUTO','NEVER| AUTO| ALWAYS ','ENUM',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'If AUTO (the default), MySQL permits INSERT and SELECT statements to run concurrently for MyISAM tables that have no free blocks in the middle of the data file.','2023-03-09 17:55:11','2023-03-09 17:55:11',0); @@ -1153,7 +1153,7 @@ INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, ` INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (15176,'tendbcluster','dbconf','Spider-3','mysqld.bind-address','STRING','{{.Mysqld.BindAddress}}',NULL,'',2,0,0,0,1,'{{mysqld.bind-address}}',NULL,NULL,-1,NULL,NULL,'2023-03-09 17:55:05','2023-03-28 18:01:04',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (15177,'tendbcluster','dbconf','Spider-3','mysqld.binlog_cache_size','INT','2097152','[4096,16777216]','RANGE',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'The size of the cache to hold changes to the binary log during a transaction','2023-03-09 17:55:05','2023-03-09 17:55:05',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (15178,'tendbcluster','dbconf','Spider-3','mysqld.binlog_checksum','STRING','CRC32','NONE| CRC32 ','ENUM',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'Include checksum for log events in the binary log. None indicates only check length.','2023-03-09 17:55:05','2023-03-09 17:55:05',0); -INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (15179,'tendbcluster','dbconf','Spider-3','mysqld.binlog_format','STRING','ROW','ROW| MIXED| STATEMENT','ENUM',1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'What form of binary logging the master will use.','2023-03-09 17:55:05','2023-04-19 14:31:26',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (15179,'tendbcluster','dbconf','Spider-3','mysqld.binlog_format','STRING','STATEMENT','MIXED| STATEMENT','ENUM',1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'What form of binary logging the master will use.','2023-03-09 17:55:05','2024-12-13 15:37:16',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (15180,'tendbcluster','dbconf','Spider-3','mysqld.binlog_order_commits','STRING','ON','ON| OFF ','ENUM',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'whether enable on a master (the default), transactions are externalized in the same order as they are written to the binary log ','2023-03-09 17:55:05','2023-03-09 17:55:05',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (15181,'tendbcluster','dbconf','Spider-3','mysqld.binlog_rows_query_log_events','STRING','OFF','ON| OFF ','ENUM',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'This system variable affects row-based logging only. When enabled, it causes the server to write informational log events such as row query log events into its binary log','2023-03-09 17:55:05','2023-03-09 17:55:05',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (15182,'tendbcluster','dbconf','Spider-3','mysqld.binlog_row_image','STRING','FULL','FULL| MINIMAL ','ENUM',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'Controls what formats that rows should be logged in.','2023-03-09 17:55:05','2023-03-09 17:55:05',0); @@ -1406,7 +1406,7 @@ INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, ` INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25605,'tendbcluster','dbconf','Spider-3.5','mysqld.bind-address','STRING','{{.Mysqld.BindAddress}}',NULL,'',2,0,0,0,1,NULL,NULL,NULL,-1,NULL,NULL,'2024-09-04 10:32:31','2024-09-04 10:32:31',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25606,'tendbcluster','dbconf','Spider-3.5','mysqld.binlog_cache_size','INT','2097152','[4096,16777216]','RANGE',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,NULL,'2024-09-04 10:32:31','2024-09-04 10:32:31',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25607,'tendbcluster','dbconf','Spider-3.5','mysqld.binlog_checksum','STRING','CRC32','NONE| CRC32 ','ENUM',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,NULL,'2024-09-04 10:32:31','2024-09-04 10:32:31',0); -INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25608,'tendbcluster','dbconf','Spider-3.5','mysqld.binlog_format','STRING','ROW','ROW| MIXED| STATEMENT','ENUM',1,0,0,0,0,NULL,NULL,NULL,-1,NULL,NULL,'2024-09-04 10:32:31','2024-09-04 10:32:31',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25608,'tendbcluster','dbconf','Spider-3.5','mysqld.binlog_format','STRING','STATEMENT','MIXED| STATEMENT','ENUM',1,0,0,0,0,NULL,NULL,NULL,-1,NULL,NULL,'2024-09-04 10:32:31','2024-12-13 15:37:16',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25609,'tendbcluster','dbconf','Spider-3.5','mysqld.binlog_order_commits','STRING','ON','ON| OFF ','ENUM',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,NULL,'2024-09-04 10:32:31','2024-09-04 10:32:31',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25610,'tendbcluster','dbconf','Spider-3.5','mysqld.binlog_rows_query_log_events','STRING','OFF','ON| OFF ','ENUM',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,NULL,'2024-09-04 10:32:31','2024-09-04 10:32:31',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25611,'tendbcluster','dbconf','Spider-3.5','mysqld.binlog_row_image','STRING','FULL','FULL| MINIMAL ','ENUM',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,NULL,'2024-09-04 10:32:31','2024-09-04 10:32:31',0); @@ -1659,7 +1659,7 @@ INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, ` INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (24515,'tendbcluster','dbconf','Spider-3.6','mysqld.bind-address','STRING','{{.Mysqld.BindAddress}}',NULL,'',2,0,0,0,1,NULL,NULL,NULL,-1,NULL,NULL,'2024-09-04 10:32:31','2024-09-04 10:32:31',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (24516,'tendbcluster','dbconf','Spider-3.6','mysqld.binlog_cache_size','INT','2097152','[4096,16777216]','RANGE',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,NULL,'2024-09-04 10:32:31','2024-09-04 10:32:31',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (24517,'tendbcluster','dbconf','Spider-3.6','mysqld.binlog_checksum','STRING','CRC32','NONE| CRC32 ','ENUM',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,NULL,'2024-09-04 10:32:31','2024-09-04 10:32:31',0); -INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (24518,'tendbcluster','dbconf','Spider-3.6','mysqld.binlog_format','STRING','ROW','ROW| MIXED| STATEMENT','ENUM',1,0,0,0,0,NULL,NULL,NULL,-1,NULL,NULL,'2024-09-04 10:32:31','2024-09-04 10:32:31',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (24518,'tendbcluster','dbconf','Spider-3.6','mysqld.binlog_format','STRING','STATEMENT','MIXED| STATEMENT','ENUM',1,0,0,0,0,NULL,NULL,NULL,-1,NULL,NULL,'2024-09-04 10:32:31','2024-12-13 15:37:16',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (24519,'tendbcluster','dbconf','Spider-3.6','mysqld.binlog_order_commits','STRING','ON','ON| OFF ','ENUM',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,NULL,'2024-09-04 10:32:31','2024-09-04 10:32:31',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (24520,'tendbcluster','dbconf','Spider-3.6','mysqld.binlog_rows_query_log_events','STRING','OFF','ON| OFF ','ENUM',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,NULL,'2024-09-04 10:32:31','2024-09-04 10:32:31',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (24521,'tendbcluster','dbconf','Spider-3.6','mysqld.binlog_row_image','STRING','FULL','FULL| MINIMAL ','ENUM',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,NULL,'2024-09-04 10:32:31','2024-09-04 10:32:31',0); @@ -1912,7 +1912,7 @@ INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, ` INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (24771,'tendbcluster','dbconf','Spider-3.7','mysqld.bind-address','STRING','{{.Mysqld.BindAddress}}',NULL,'',2,0,0,0,1,NULL,NULL,NULL,-1,NULL,NULL,'2024-09-04 10:32:54','2024-09-04 10:32:54',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (24772,'tendbcluster','dbconf','Spider-3.7','mysqld.binlog_cache_size','INT','2097152','[4096,16777216]','RANGE',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,NULL,'2024-09-04 10:32:54','2024-09-04 10:32:54',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (24773,'tendbcluster','dbconf','Spider-3.7','mysqld.binlog_checksum','STRING','CRC32','NONE| CRC32 ','ENUM',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,NULL,'2024-09-04 10:32:54','2024-09-04 10:32:54',0); -INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (24774,'tendbcluster','dbconf','Spider-3.7','mysqld.binlog_format','STRING','ROW','ROW| MIXED| STATEMENT','ENUM',1,0,0,0,0,NULL,NULL,NULL,-1,NULL,NULL,'2024-09-04 10:32:54','2024-09-04 10:32:54',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (24774,'tendbcluster','dbconf','Spider-3.7','mysqld.binlog_format','STRING','STATEMENT','MIXED| STATEMENT','ENUM',1,0,0,0,0,NULL,NULL,NULL,-1,NULL,NULL,'2024-09-04 10:32:54','2024-12-13 15:37:16',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (24775,'tendbcluster','dbconf','Spider-3.7','mysqld.binlog_order_commits','STRING','ON','ON| OFF ','ENUM',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,NULL,'2024-09-04 10:32:54','2024-09-04 10:32:54',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (24776,'tendbcluster','dbconf','Spider-3.7','mysqld.binlog_rows_query_log_events','STRING','OFF','ON| OFF ','ENUM',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,NULL,'2024-09-04 10:32:54','2024-09-04 10:32:54',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (24777,'tendbcluster','dbconf','Spider-3.7','mysqld.binlog_row_image','STRING','FULL','FULL| MINIMAL ','ENUM',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,NULL,'2024-09-04 10:32:54','2024-09-04 10:32:54',0); @@ -2165,7 +2165,7 @@ INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, ` INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25026,'tendbcluster','dbconf','Spider-3.8','mysqld.bind-address','STRING','{{.Mysqld.BindAddress}}',NULL,'',2,0,0,0,1,NULL,NULL,NULL,-1,NULL,NULL,'2024-09-04 10:33:03','2024-09-04 10:33:03',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25027,'tendbcluster','dbconf','Spider-3.8','mysqld.binlog_cache_size','INT','2097152','[4096,16777216]','RANGE',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,NULL,'2024-09-04 10:33:03','2024-09-04 10:33:03',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25028,'tendbcluster','dbconf','Spider-3.8','mysqld.binlog_checksum','STRING','CRC32','NONE| CRC32 ','ENUM',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,NULL,'2024-09-04 10:33:03','2024-09-04 10:33:03',0); -INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25029,'tendbcluster','dbconf','Spider-3.8','mysqld.binlog_format','STRING','ROW','ROW| MIXED| STATEMENT','ENUM',1,0,0,0,0,NULL,NULL,NULL,-1,NULL,NULL,'2024-09-04 10:33:03','2024-09-04 10:33:03',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25029,'tendbcluster','dbconf','Spider-3.8','mysqld.binlog_format','STRING','STATEMENT','MIXED| STATEMENT','ENUM',1,0,0,0,0,NULL,NULL,NULL,-1,NULL,NULL,'2024-09-04 10:33:03','2024-12-13 15:37:16',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25030,'tendbcluster','dbconf','Spider-3.8','mysqld.binlog_order_commits','STRING','ON','ON| OFF ','ENUM',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,NULL,'2024-09-04 10:33:03','2024-09-04 10:33:03',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25031,'tendbcluster','dbconf','Spider-3.8','mysqld.binlog_rows_query_log_events','STRING','OFF','ON| OFF ','ENUM',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,NULL,'2024-09-04 10:33:03','2024-09-04 10:33:03',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (25032,'tendbcluster','dbconf','Spider-3.8','mysqld.binlog_row_image','STRING','FULL','FULL| MINIMAL ','ENUM',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,NULL,'2024-09-04 10:33:03','2024-09-04 10:33:03',0); @@ -2538,7 +2538,7 @@ INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, ` INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (15681,'tendbcluster','dbconf','Tdbctl','mysqld.innodb_stats_transient_sample_pages','INT','8','[1,4294967296]','RANGE',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'The number of leaf index pages to sample when calculating transient statistics','2023-03-09 17:57:45','2023-05-10 19:35:56',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (15682,'tendbcluster','dbconf','Tdbctl','mysqld.innodb_status_output','STRING','OFF','ON| OFF ','ENUM',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'Enables or disables periodic output for the standard InnoDB Monitor.','2023-03-09 17:57:45','2023-05-10 19:35:56',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (15683,'tendbcluster','dbconf','Tdbctl','mysqld.innodb_status_output_locks','STRING','OFF','ON| OFF ','ENUM',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'Enables or disables the InnoDB Lock Monitor.','2023-03-09 17:57:45','2023-05-10 19:35:56',0); -INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (15684,'tendbcluster','dbconf','Tdbctl','mysqld.innodb_strict_mode','STRING','ON','ON| OFF ','ENUM',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'When innodb_strict_mode is ON, InnoDB returns errors rather than warnings for certain conditions','2023-03-09 17:57:45','2023-05-10 19:35:56',0); +INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (15684,'tendbcluster','dbconf','Tdbctl','mysqld.innodb_strict_mode','INT','0','0 | 1','ENUM',1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'When innodb_strict_mode is ON, InnoDB returns errors rather than warnings for certain conditions','2023-03-09 17:57:45','2024-12-26 18:16:35',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (15685,'tendbcluster','dbconf','Tdbctl','mysqld.innodb_sync_array_size','INT','1','[1,1024]','RANGE',-1,0,0,0,1,NULL,NULL,NULL,-1,NULL,'Defines the size of the mutex/lock wait array.','2023-03-09 17:57:45','2023-05-10 19:35:56',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (15686,'tendbcluster','dbconf','Tdbctl','mysqld.innodb_sync_spin_loops','INT','30','[0,4294967295]','RANGE',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'The number of times a thread waits for an InnoDB mutex to be freed before the thread is suspended.','2023-03-09 17:57:45','2023-05-10 19:35:56',0); INSERT INTO `tb_config_name_def` (`id`, `namespace`, `conf_type`, `conf_file`, `conf_name`, `value_type`, `value_default`, `value_allowed`, `value_type_sub`, `flag_status`, `flag_disable`, `flag_locked`, `flag_encrypt`, `need_restart`, `value_formula`, `extra_info`, `conf_name_lc`, `order_index`, `since_version`, `description`, `created_at`, `updated_at`, `stage`) VALUES (15687,'tendbcluster','dbconf','Tdbctl','mysqld.innodb_table_locks','STRING','ON','ON| OFF ','ENUM',-1,0,0,0,0,NULL,NULL,NULL,-1,NULL,'If autocommit = 0, InnoDB honors LOCK TABLES; MySQL does not return from LOCK TABLES ... WRITE until all other threads have released all their locks to the table. The default value of innodb_table_locks is 1, which means that LOCK TABLES causes InnoDB to ','2023-03-09 17:57:45','2023-05-10 19:35:56',0); diff --git a/dbm-services/common/db-config/internal/api/apply_config.go b/dbm-services/common/db-config/internal/api/apply_config.go index 0a17694c27..241c90466f 100644 --- a/dbm-services/common/db-config/internal/api/apply_config.go +++ b/dbm-services/common/db-config/internal/api/apply_config.go @@ -1,7 +1,7 @@ package api import ( - "bk-dbconfig/pkg/validate" + "dbm-services/common/go-pubpkg/validate" ) // ApplyConfigItem TODO diff --git a/dbm-services/common/db-config/internal/api/config_item.go b/dbm-services/common/db-config/internal/api/config_item.go index 2a283eae14..a9bd49194d 100644 --- a/dbm-services/common/db-config/internal/api/config_item.go +++ b/dbm-services/common/db-config/internal/api/config_item.go @@ -2,7 +2,8 @@ package api import ( "bk-dbconfig/pkg/constvar" - "bk-dbconfig/pkg/validate" + + "dbm-services/common/go-pubpkg/validate" "github.com/pkg/errors" ) diff --git a/dbm-services/common/db-config/internal/api/config_plat.go b/dbm-services/common/db-config/internal/api/config_plat.go index 4b2fef4013..b720e7b4f8 100644 --- a/dbm-services/common/db-config/internal/api/config_plat.go +++ b/dbm-services/common/db-config/internal/api/config_plat.go @@ -1,7 +1,9 @@ package api import ( - "bk-dbconfig/pkg/validate" + "bk-dbconfig/pkg/validatestruct" + + "dbm-services/common/go-pubpkg/validate" ) // UpsertConfFilePlatReq TODO @@ -42,7 +44,7 @@ func (f *UpsertConfFilePlatReq) Validate() error { if err := validate.GoValidateStruct(*c, true); err != nil { return err } - valueTypeSub := validate.ValueTypeDef{ValueType: c.ValueType, ValueTypeSub: c.ValueTypeSub} + valueTypeSub := validatestruct.ValueTypeDef{ValueType: c.ValueType, ValueTypeSub: c.ValueTypeSub} if err := valueTypeSub.Validate(); err != nil { return err } diff --git a/dbm-services/common/db-config/internal/api/dbha.go b/dbm-services/common/db-config/internal/api/dbha.go index b3bb717df7..ce3ad9be0f 100644 --- a/dbm-services/common/db-config/internal/api/dbha.go +++ b/dbm-services/common/db-config/internal/api/dbha.go @@ -1,6 +1,8 @@ package api -import "bk-dbconfig/pkg/validate" +import ( + "dbm-services/common/go-pubpkg/validate" +) // BatchGetConfigItemReq TODO // 批量获取多个对象的某一配置项 diff --git a/dbm-services/common/db-config/internal/handler/simple/config_file.go b/dbm-services/common/db-config/internal/handler/simple/config_file.go index 26c8defac7..c17dc038e7 100644 --- a/dbm-services/common/db-config/internal/handler/simple/config_file.go +++ b/dbm-services/common/db-config/internal/handler/simple/config_file.go @@ -5,7 +5,8 @@ import ( "bk-dbconfig/internal/handler" "bk-dbconfig/internal/service/simpleconfig" "bk-dbconfig/pkg/constvar" - "bk-dbconfig/pkg/validate" + + "dbm-services/common/go-pubpkg/validate" "github.com/gin-gonic/gin" ) diff --git a/dbm-services/common/db-config/internal/handler/simple/config_version.go b/dbm-services/common/db-config/internal/handler/simple/config_version.go index 57d64860c7..3464dcbe4e 100644 --- a/dbm-services/common/db-config/internal/handler/simple/config_version.go +++ b/dbm-services/common/db-config/internal/handler/simple/config_version.go @@ -12,7 +12,8 @@ import ( "bk-dbconfig/pkg/constvar" "bk-dbconfig/pkg/core/logger" "bk-dbconfig/pkg/util" - "bk-dbconfig/pkg/validate" + + "dbm-services/common/go-pubpkg/validate" "github.com/gin-gonic/gin" "github.com/pkg/errors" diff --git a/dbm-services/common/db-config/internal/service/simpleconfig/config_item_format.go b/dbm-services/common/db-config/internal/service/simpleconfig/config_item_format.go index 951af6e7b3..cd22bcfdfd 100644 --- a/dbm-services/common/db-config/internal/service/simpleconfig/config_item_format.go +++ b/dbm-services/common/db-config/internal/service/simpleconfig/config_item_format.go @@ -9,7 +9,7 @@ import ( "bk-dbconfig/pkg/constvar" "bk-dbconfig/pkg/core/logger" "bk-dbconfig/pkg/util" - "bk-dbconfig/pkg/validate" + "bk-dbconfig/pkg/validatestruct" "github.com/pkg/errors" "github.com/spf13/cast" @@ -141,17 +141,17 @@ func CastValueType(confName string, confValue string, f api.BaseConfFileDef, val if valueType == "" { return confValue } - if valueType == validate.DTypeInt { + if valueType == validatestruct.DTypeInt { return cast.ToInt(confValue) - } else if valueType == validate.DTypeFloat || valueType == validate.DTypeNumber { + } else if valueType == validatestruct.DTypeFloat || valueType == validatestruct.DTypeNumber { return cast.ToFloat32(confValue) - } else if valueType == validate.DTypeBool { + } else if valueType == validatestruct.DTypeBool { return cmutil.ToBoolExt(confValue) - } else if valueType == validate.DTypeString { - if valueSubType == validate.DTypeSubList { + } else if valueType == validatestruct.DTypeString { + if valueSubType == validatestruct.DTypeSubList { newValue := util.SplitAnyRuneTrim(confValue, ",") return newValue - } else if valueSubType == validate.DTypeSubMap { + } else if valueSubType == validatestruct.DTypeSubMap { mapI := make(map[string]interface{}) err := json.Unmarshal([]byte(confValue), &mapI) if err != nil { diff --git a/dbm-services/common/db-config/internal/service/simpleconfig/config_meta.go b/dbm-services/common/db-config/internal/service/simpleconfig/config_meta.go index 4471faee7d..02b213a30d 100644 --- a/dbm-services/common/db-config/internal/service/simpleconfig/config_meta.go +++ b/dbm-services/common/db-config/internal/service/simpleconfig/config_meta.go @@ -7,7 +7,7 @@ import ( "bk-dbconfig/internal/pkg/errno" "bk-dbconfig/internal/repository/model" "bk-dbconfig/pkg/util" - "bk-dbconfig/pkg/validate" + "bk-dbconfig/pkg/validatestruct" "github.com/pkg/errors" "gorm.io/gorm" @@ -59,7 +59,7 @@ func CheckConfNameAndValue(c *model.ConfigModel, checkValue bool, valueType, val } cn.ValueDefault = c.ConfValue // 如果不校验 conf_name， 那么 conf_name 可能在 name_def 里没定义，value_type, value_type_sub, value_allowed 都为空 - err := validate.ValidateConfValue(cn.ValueDefault, cn.ValueType, cn.ValueTypeSub, cn.ValueAllowed) + err := validatestruct.ValidateConfValue(cn.ValueDefault, cn.ValueType, cn.ValueTypeSub, cn.ValueAllowed) if err != nil { errors.WithMessage(err, c.ConfName) } diff --git a/dbm-services/common/db-config/pkg/validate/validate.go b/dbm-services/common/db-config/pkg/validate/validate.go deleted file mode 100644 index d86c7e248a..0000000000 --- a/dbm-services/common/db-config/pkg/validate/validate.go +++ /dev/null @@ -1,140 +0,0 @@ -// Package validate TODO -package validate - -import ( - "bk-dbconfig/pkg/util" - "fmt" - "log" - "reflect" - "strings" - - "github.com/go-playground/locales/en" - ut "github.com/go-playground/universal-translator" - "github.com/go-playground/validator/v10" - en_translations "github.com/go-playground/validator/v10/translations/en" - "github.com/pkg/errors" -) - -// ValidateEnums TODO -// make validate tag work with enums tag -// 避免 validate oneof 和 swagger enums 写 2 份重复的校验和文档 -// example: Method string `validate:"required,enums" enums:"post,get" json:"method"` -func ValidateEnums(f validator.FieldLevel) bool { - fieldValue := f.Field().String() - fieldName := f.StructFieldName() - // get StructField - sf, _ := f.Parent().Type().FieldByName(fieldName) - // get tag value from tag_field enums - tagValue := sf.Tag.Get(TagEnum) - enumsValues := strings.Split(tagValue, ",") - if util.StringsHas(enumsValues, fieldValue) { - return true - } else { - return false - } -} - -// GoValidateStructSimple TODO -// 简单校验 struct，不涉及逻辑 -// 如果 struct 上有 tag validate:"enums"，必须启用enum=true校验 -func GoValidateStructSimple(v interface{}, enum bool) error { - validate := validator.New() - if enum { - _ = validate.RegisterValidation("enums", ValidateEnums) - } - if err := validate.Struct(v); err != nil { - return err - } - return nil -} - -// TagEnum TODO -const TagEnum = "enums" - -// GoValidateStruct v 不能是Ptr -func GoValidateStruct(v interface{}, enum bool) error { - validate := validator.New() - uni := ut.New(en.New()) - trans, _ := uni.GetTranslator("en") - // 提示时显示 json 字段的名字 - validate.RegisterTagNameFunc(func(fld reflect.StructField) string { - // name := fld.Tag.Get("json") - name := strings.SplitN(fld.Tag.Get("json"), ",", 2)[0] - if name == "-" { - return "" - } - return name - }) - if err := en_translations.RegisterDefaultTranslations(validate, trans); err != nil { - return err - } - - if enum { - _ = validate.RegisterValidation(TagEnum, ValidateEnums) - } - if err := validate.Struct(v); err != nil { - return translateErr2Msg(v, trans, err) - } - return nil -} - -// translateErr2Msg v 不能是Ptr -func translateErr2Msg(v interface{}, trans ut.Translator, err error) error { - var errStr []string - for _, vErr := range err.(validator.ValidationErrors) { - if vErr.Tag() == TagEnum { - errmsg := "" - // errmsg := customEnumTransFunc(vErr, v) - if vErr.Param() == "" { - sf, _ := reflect.TypeOf(v).FieldByName(vErr.StructField()) - tagValue := sf.Tag.Get(TagEnum) - errmsg = fmt.Sprintf("%s must be one of [%s]", vErr.Field(), tagValue) - } else { - errmsg = vErr.Param() - } - errStr = append(errStr, errmsg) - continue - } - errStr = append(errStr, vErr.Translate(trans)) - } - return errors.New(strings.Join(errStr, " || ")) -} -func customEnumTransFunc(fe validator.FieldError, v interface{}) string { - if fe.Param() == "" { - sf, _ := reflect.TypeOf(v).FieldByName(fe.StructField()) - tagValue := sf.Tag.Get(TagEnum) - errmsg := fmt.Sprintf("%s must be one of [%s]", fe.Field(), tagValue) - return errmsg - } else { - return fe.Param() - } -} - -// registerTranslator 为自定义字段添加翻译功能 -func registerTranslator(tag string, msg string) validator.RegisterTranslationsFunc { - return func(trans ut.Translator) error { - if err := trans.Add(tag, msg, false); err != nil { - return err - } - return nil - } -} - -// customTransFunc TODO -// translate 自定义字段的翻译方法 -func customTransFunc(trans ut.Translator, fe validator.FieldError) string { - msg, err := trans.T(fe.Tag(), fe.Field()) - if err != nil { - panic(fe.(error).Error()) - } - return msg -} - -func translate(ut ut.Translator, fe validator.FieldError) string { - s, err := ut.T(fe.Tag(), fe.Field(), "fe.Param()") - if err != nil { - log.Printf("warning: error translating FieldError: %#v", fe) - return fe.(error).Error() - } - return s -} diff --git a/dbm-services/common/db-config/pkg/validate/check_value.go b/dbm-services/common/db-config/pkg/validatestruct/check_value.go similarity index 77% rename from dbm-services/common/db-config/pkg/validate/check_value.go rename to dbm-services/common/db-config/pkg/validatestruct/check_value.go index d763e245d7..6c6c1361ce 100644 --- a/dbm-services/common/db-config/pkg/validate/check_value.go +++ b/dbm-services/common/db-config/pkg/validatestruct/check_value.go @@ -1,3 +1,13 @@ +/* + * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. + * Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at https://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + package validate import ( @@ -321,70 +331,79 @@ func ValidateConfValue(confValue, valueType, valueTypeSub, valueAllowed string) return invalidErr } } else if util.StringsHas([]string{DTypeInt, DTypeFloat, DTypeNumber}, valueType) { + return validateValueNumber(valueType, valueTypeSub, confValue, valueAllowed) + } else { // STRING + return validateValueString(valueType, valueTypeSub, confValue, valueAllowed) + } +} + +func validateValueNumber(valueType, valueTypeSub, confValue, valueAllowed string) error { + if valueTypeSub == "" { + valueTypeSub = AutoDetectTypeSub(valueAllowed) if valueTypeSub == "" { - valueTypeSub = AutoDetectTypeSub(valueAllowed) - if valueTypeSub == "" { - return errors.Errorf("cannot detect value_type_sub for %s", valueAllowed) - } + return errors.Errorf("cannot detect value_type_sub for %s", valueAllowed) } - switch valueTypeSub { - case DTypeSubEnum: - return CheckInEnums(confValue, valueAllowed, false) - case DTypeSubRange: - return CheckInRange(confValue, valueAllowed) - default: - return invalidErr + } + switch valueTypeSub { + case DTypeSubEnum: + return CheckInEnums(confValue, valueAllowed, false) + case DTypeSubRange: + return CheckInRange(confValue, valueAllowed) + default: + var invalidErr = errors.Errorf("invalid value_type_sub %s for %s", valueTypeSub, valueType) + return invalidErr + } +} + +func validateValueString(valueType, valueTypeSub, confValue, valueAllowed string) error { + if valueAllowed == "" && !(valueTypeSub == DTypeSubJson || valueTypeSub == DTypeSubMap) { + // JSON,MAP 合法性 不依赖 valueAllowed + return nil + } + switch valueTypeSub { + case DTypeSubEnum: + if err := CheckInEnums(confValue, valueAllowed, false); err != nil { + return err } - } else { // STRING - if valueAllowed == "" && !(valueTypeSub == DTypeSubJson || valueTypeSub == DTypeSubMap) { - // JSON,MAP 合法性 不依赖 valueAllowed - return nil + case DTypeSubEnums: + if err := CheckInEnums(confValue, valueAllowed, true); err != nil { + return err } - switch valueTypeSub { - case DTypeSubEnum: - if err := CheckInEnums(confValue, valueAllowed, false); err != nil { - return err - } - case DTypeSubEnums: - if err := CheckInEnums(confValue, valueAllowed, true); err != nil { - return err - } - case DTypeSubRegex: - if err := CheckInRegex(confValue, valueAllowed); err != nil { - return err - } - case DTypeSubBytes: - if err := CheckInSizeRange(confValue, valueAllowed); err != nil { - return err - } - case DTypeSubDuration: - if err := CheckInDuration(confValue, valueAllowed); err != nil { - return err - } - case DTypeSubJson, DTypeSubMap: - if err := CheckInJson(confValue); err != nil { - return err - } - case DTypeSubGovalidate: - if err := CheckGoValidate(confValue, valueAllowed); err != nil { - return err - } - case DTypeSubList: - // 忽略 value_allowed，只用户返回格式化 - return nil - case DTypeSubString, "": - if valueAllowed != "" { - // value_allowed !='' and value_type_sub=''，要求 conf_value 只能一个值即 value_allowed - if confValue != valueAllowed { - return errors.Errorf("value must equal value_allowed:%s", valueAllowed) - } - return nil + case DTypeSubRegex: + if err := CheckInRegex(confValue, valueAllowed); err != nil { + return err + } + case DTypeSubBytes: + if err := CheckInSizeRange(confValue, valueAllowed); err != nil { + return err + } + case DTypeSubDuration: + if err := CheckInDuration(confValue, valueAllowed); err != nil { + return err + } + case DTypeSubJson, DTypeSubMap: + if err := CheckInJson(confValue); err != nil { + return err + } + case DTypeSubGovalidate: + if err := CheckGoValidate(confValue, valueAllowed); err != nil { + return err + } + case DTypeSubList: + // 忽略 value_allowed，只用户返回格式化 + return nil + case DTypeSubString, "": + if valueAllowed != "" { + // value_allowed !='' and value_type_sub=''，要求 conf_value 只能一个值即 value_allowed + if confValue != valueAllowed { + return errors.Errorf("value must equal value_allowed:%s", valueAllowed) } return nil - default: - return invalidErr } + return nil + default: + var invalidErr = errors.Errorf("invalid value_type_sub %s for %s", valueTypeSub, valueType) + return invalidErr } - return nil } diff --git a/dbm-services/common/db-config/pkg/validate/check_value_test.go b/dbm-services/common/db-config/pkg/validatestruct/check_value_test.go similarity index 99% rename from dbm-services/common/db-config/pkg/validate/check_value_test.go rename to dbm-services/common/db-config/pkg/validatestruct/check_value_test.go index f00c7af3b4..e270b600f4 100644 --- a/dbm-services/common/db-config/pkg/validate/check_value_test.go +++ b/dbm-services/common/db-config/pkg/validatestruct/check_value_test.go @@ -1,10 +1,11 @@ -package validate +package validatestruct import ( - "bk-dbconfig/pkg/util" "log" "testing" + "bk-dbconfig/pkg/util" + . "github.com/smartystreets/goconvey/convey" ) diff --git a/dbm-services/common/db-config/pkg/validate/const.go b/dbm-services/common/db-config/pkg/validatestruct/const.go similarity index 98% rename from dbm-services/common/db-config/pkg/validate/const.go rename to dbm-services/common/db-config/pkg/validatestruct/const.go index f8f0c56330..01ffeebbf4 100644 --- a/dbm-services/common/db-config/pkg/validate/const.go +++ b/dbm-services/common/db-config/pkg/validatestruct/const.go @@ -1,4 +1,4 @@ -package validate +package validatestruct import "regexp" diff --git a/dbm-services/common/go-pubpkg/cmutil/mysql.go b/dbm-services/common/go-pubpkg/cmutil/mysql.go index ffba31e567..358f27ee7a 100644 --- a/dbm-services/common/go-pubpkg/cmutil/mysql.go +++ b/dbm-services/common/go-pubpkg/cmutil/mysql.go @@ -4,6 +4,9 @@ package cmutil import ( "regexp" "strconv" + "strings" + + "github.com/pkg/errors" ) // GetMysqlSystemDatabases TODO @@ -130,3 +133,18 @@ func RemovePassword(input string) string { return mysqlPasswordRegex.ReplaceAllString(sub, " -pxxxx") }) } + +// GetDbTableName get dbName, tableName from db1.table1 / `db1`.`table1` +func GetDbTableName(dbTableName string) (dbName string, tableName string, err error) { + ss := strings.SplitN(dbTableName, ".", 2) + if len(ss) == 2 { + dbName = strings.Trim(ss[0], "`") + tableName = strings.Trim(ss[1], "`") + } else { + return "", "", errors.Errorf("failed get db table name spliting with . for %s", dbTableName) + } + if dbName == "" || tableName == "" { + return "", "", errors.Errorf("db table cannot be empty parsing %s", dbTableName) + } + return +} diff --git a/dbm-services/common/go-pubpkg/validate/validate.go b/dbm-services/common/go-pubpkg/validate/validate.go index 38157e10b4..7c89e7b37e 100644 --- a/dbm-services/common/go-pubpkg/validate/validate.go +++ b/dbm-services/common/go-pubpkg/validate/validate.go @@ -8,14 +8,14 @@ import ( "strings" "time" - "dbm-services/common/go-pubpkg/cmutil" - "github.com/go-playground/locales/en" ut "github.com/go-playground/universal-translator" "github.com/go-playground/validator/v10" en_translations "github.com/go-playground/validator/v10/translations/en" "github.com/pkg/errors" "github.com/robfig/cron/v3" + + "dbm-services/common/go-pubpkg/cmutil" ) // ValidateEnums TODO @@ -55,21 +55,31 @@ func GoValidateStructSimple(v interface{}, enum bool) error { const TagEnum = "enums" // GoValidateStruct v 不能是Ptr -func GoValidateStruct(v interface{}, enum bool, charset bool) error { +func GoValidateStruct(v interface{}, enum bool) error { + return GoValidateTransError(v, "json", enum, false) +} + +func GoValidateStructCharset(v interface{}, enum bool, checkCharset bool) error { + return GoValidateTransError(v, "json", enum, checkCharset) +} + +// GoValidateTransError v 不能是Ptr +func GoValidateTransError(v interface{}, tagKey string, enum bool, charset bool) error { + if tagKey == "" { + tagKey = "json" + } validate := validator.New() uni := ut.New(en.New()) trans, _ := uni.GetTranslator("en") // 提示时显示 json 字段的名字 - validate.RegisterTagNameFunc( - func(fld reflect.StructField) string { - // name := fld.Tag.Get("json") - name := strings.SplitN(fld.Tag.Get("json"), ",", 2)[0] - if name == "-" { - return "" - } - return name - }, - ) + validate.RegisterTagNameFunc(func(fld reflect.StructField) string { + // name := fld.Tag.Get("json") + name := strings.SplitN(fld.Tag.Get(tagKey), ",", 2)[0] + if name == "-" { + return "" + } + return name + }) if err := en_translations.RegisterDefaultTranslations(validate, trans); err != nil { return err } @@ -80,8 +90,6 @@ func GoValidateStruct(v interface{}, enum bool, charset bool) error { if charset { _ = validate.RegisterValidation("checkCharset", validCharSet) } - _ = validate.RegisterValidation("crontabexpr", validateCrontabExpr) - _ = validate.RegisterValidation("time", validateTimeStr) if err := validate.Struct(v); err != nil { return translateErr2Msg(v, trans, err) } @@ -91,10 +99,6 @@ func GoValidateStruct(v interface{}, enum bool, charset bool) error { // translateErr2Msg v 不能是Ptr func translateErr2Msg(v interface{}, trans ut.Translator, err error) error { var errStr []string - _, ok := err.(*validator.InvalidValidationError) - if ok { - return fmt.Errorf("param error:%s", err.Error()) - } for _, vErr := range err.(validator.ValidationErrors) { if vErr.Tag() == TagEnum { errmsg := "" @@ -113,7 +117,6 @@ func translateErr2Msg(v interface{}, trans ut.Translator, err error) error { } return errors.New(strings.Join(errStr, " || ")) } - func customEnumTransFunc(fe validator.FieldError, v interface{}) string { if fe.Param() == "" { sf, _ := reflect.TypeOf(v).FieldByName(fe.StructField()) diff --git a/dbm-services/common/reverse-api/go.mod b/dbm-services/common/reverse-api/go.mod index bfee1531b6..5bcf89ef9b 100644 --- a/dbm-services/common/reverse-api/go.mod +++ b/dbm-services/common/reverse-api/go.mod @@ -1,3 +1,5 @@ module dbm-services/common/reverse-api go 1.21.11 + +require github.com/pkg/errors v0.9.1 diff --git a/dbm-services/common/reverse-api/go.sum b/dbm-services/common/reverse-api/go.sum new file mode 100644 index 0000000000..7c401c3f58 --- /dev/null +++ b/dbm-services/common/reverse-api/go.sum @@ -0,0 +1,2 @@ +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/dbm-services/go.work b/dbm-services/go.work index ff08289b98..5921f89030 100644 --- a/dbm-services/go.work +++ b/dbm-services/go.work @@ -32,6 +32,7 @@ use ( mongodb/db-tools/dbactuator common/db-dns/dns-api/pkg common/reverse-api + mysql/db-tools/gobatch-dml ) replace github.com/go-sql-driver/mysql => github.com/go-sql-driver/mysql v1.7.1 diff --git a/dbm-services/mysql/db-tools/dbactuator/internal/subcmd/subcmd.go b/dbm-services/mysql/db-tools/dbactuator/internal/subcmd/subcmd.go index b33e418bae..ecc4b330cc 100644 --- a/dbm-services/mysql/db-tools/dbactuator/internal/subcmd/subcmd.go +++ b/dbm-services/mysql/db-tools/dbactuator/internal/subcmd/subcmd.go @@ -144,7 +144,7 @@ func (b *BaseOptions) DeserializeNonStandard(s interface{}) (err error) { logger.Error("json.Unmarshal failed, %v", s, err) return } - if err = validate.GoValidateStruct(s, false, true); err != nil { + if err = validate.GoValidateStructCharset(s, false, true); err != nil { logger.Error("validate struct failed, %v", s, err) return } @@ -184,7 +184,7 @@ func Deserialize(s interface{}) (p *BaseOptions, err error) { err = errors.WithMessage(err, "参数解析错误") return nil, err } - if err = validate.GoValidateStruct(bip, false, true); err != nil { + if err = validate.GoValidateStructCharset(bip, false, true); err != nil { logger.Error("validate struct failed, %v", s, err) err = errors.WithMessage(err, "参数输入错误") return nil, err @@ -246,7 +246,7 @@ func (b *BaseOptions) Deserialize(s interface{}) (err error) { return err } s = bip.ExtendParam - if err = validate.GoValidateStruct(bip, false, true); err != nil { + if err = validate.GoValidateStructCharset(bip, false, true); err != nil { logger.Error("validate struct failed, %v", s, err) err = errors.WithMessage(err, "参数输入错误") return err @@ -279,7 +279,7 @@ func (b *BaseOptions) DeserializeSimple(s interface{}) (err error) { err = errors.WithMessage(err, "参数解析错误") return } - if err = validate.GoValidateStruct(s, false, true); err != nil { + if err = validate.GoValidateStructCharset(s, false, true); err != nil { logger.Error("validate struct failed, %v", s, err) err = errors.WithMessage(err, "参数输入错误") return diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/components/backup_download/ieg_backupsys.go b/dbm-services/mysql/db-tools/dbactuator/pkg/components/backup_download/ieg_backupsys.go index ef213e0677..204b6d2570 100644 --- a/dbm-services/mysql/db-tools/dbactuator/pkg/components/backup_download/ieg_backupsys.go +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/components/backup_download/ieg_backupsys.go @@ -172,7 +172,7 @@ type IBSRecoverQueryResp struct { // BsQuery 搜索备份系统中的文件 func (r *IBSQueryParam) BsQuery(param IBSQueryReq) (*IBSQueryResp, error) { - if err := validate.GoValidateStruct(param, false, false); err != nil { + if err := validate.GoValidateStruct(param, false); err != nil { return nil, err } url := fmt.Sprintf("%s%s", r.client.Url, "/query") diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_load.go b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_load.go index fbd27fc14c..81261d90fa 100644 --- a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_load.go +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/dbbackup_load.go @@ -111,7 +111,7 @@ func (m *DBLoader) chooseDBBackupLoader() error { } } // logger.Warn("validate dbLoaderUtil: %+v", m.dbLoaderUtil) - if err := validate.GoValidateStruct(m.dbLoaderUtil, false, false); err != nil { + if err := validate.GoValidateStruct(m.dbLoaderUtil, false); err != nil { return err } diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/mload_restore.go b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/mload_restore.go index d587f970e6..041fcde6c7 100644 --- a/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/mload_restore.go +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/components/mysql/restore/mload_restore.go @@ -101,7 +101,7 @@ func (m *MLoad) Start() error { } } - if err := validate.GoValidateStruct(m.mloadUtil, false, false); err != nil { + if err := validate.GoValidateStruct(m.mloadUtil, false); err != nil { return err } logger.Info("mload params %+v", m) diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/native/dbworker.go b/dbm-services/mysql/db-tools/dbactuator/pkg/native/dbworker.go index 9da7ce9838..6d4e40557f 100644 --- a/dbm-services/mysql/db-tools/dbactuator/pkg/native/dbworker.go +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/native/dbworker.go @@ -116,8 +116,13 @@ func (h *DbWorker) ExecWithTimeout(dura time.Duration, query string, args ...int // 会在同一个连接里执行 // 空元素会跳过 func (h *DbWorker) ExecMore(sqls []string) (rowsAffectedCount int64, err error) { + ctx := context.Background() + return h.ExecMoreContext(sqls, ctx) +} + +func (h *DbWorker) ExecMoreContext(sqls []string, ctx context.Context) (rowsAffected int64, err error) { var c int64 - db, err := h.Db.Conn(context.Background()) + db, err := h.Db.Conn(ctx) if err != nil { return 0, err } @@ -126,14 +131,14 @@ func (h *DbWorker) ExecMore(sqls []string) (rowsAffectedCount int64, err error) if strings.TrimSpace(sqlStr) == "" { continue } - ret, err := db.ExecContext(context.Background(), sqlStr) + ret, err := db.ExecContext(ctx, sqlStr) if err != nil { - return rowsAffectedCount, fmt.Errorf("exec %s failed,err:%w", sqlStr, err) + return rowsAffected, fmt.Errorf("exec %s failed,err:%w", sqlStr, err) } if c, err = ret.RowsAffected(); err != nil { - return rowsAffectedCount, fmt.Errorf("exec %s failed,err:%w", sqlStr, err) + return rowsAffected, fmt.Errorf("exec %s failed,err:%w", sqlStr, err) } - rowsAffectedCount += c + rowsAffected += c } return } @@ -935,13 +940,13 @@ func GetTableUniqueKeyBest(uniqKeys map[string][]string) []string { // GetOneTableColumns get table column info // return {columnName1:info, columName2:info} +// info example: +// +// DATA_TYPE:smallint COLUMN_TYPE:smallint(5) unsigned +// DATA_TYPE:varchar COLUMN_TYPE:varchar(64) +// DATA_TYPE:datetime COLUMN_TYPE:datetime(6) func GetOneTableColumns(dbworker *DbWorker, dbName, tblName string) (map[string]TableColumnDef, error) { // tblSchemas = {"dbX.tableY": {"a": {"name":"a", "pos":"1", "type":"int"}}} - /* - queryStr := fmt.Sprintf("SELECT TABLE_SCHEMA,TABLE_NAME,COLUMN_NAME,ORDINAL_POSITION,DATA_TYPE,COLUMN_TYPE " + - "FROM information_schema.COLUMNS WHERE TABLE_SCHEMA =%s AND TABLE_NAME = %s" + - " ORDER BY TABLE_SCHEMA,TABLE_NAME,ORDINAL_POSITION asc", dbName, tblName) - */ queryStr := fmt.Sprintf( "SELECT TABLE_SCHEMA,TABLE_NAME,COLUMN_NAME,ORDINAL_POSITION,DATA_TYPE,COLUMN_TYPE " + "FROM information_schema.COLUMNS WHERE TABLE_SCHEMA =? AND TABLE_NAME = ? " + @@ -969,30 +974,30 @@ func GetOneTableColumns(dbworker *DbWorker, dbName, tblName string) (map[string] return nil, errors.New("table not found") } -// GetTableColumnList TODO -func GetTableColumnList(dbworker *DbWorker, dbName, tblName string) ([]string, error) { - // tblSchemas = {"dbX.tableY": {"a": {"name":"a", "pos":"1", "type":"int"}}} - /* - queryStr := fmt.Sprintf("SELECT TABLE_SCHEMA,TABLE_NAME,COLUMN_NAME,ORDINAL_POSITION,DATA_TYPE,COLUMN_TYPE " + - "FROM information_schema.COLUMNS WHERE TABLE_SCHEMA =%s AND TABLE_NAME = %s" + - " ORDER BY TABLE_SCHEMA,TABLE_NAME,ORDINAL_POSITION asc", dbName, tblName) - */ +// GetTableColumnList get table column names info, order by position ASC +func GetTableColumnList(dbName, tblName string, dbWorker *DbWorker) ([]TableColumnDef, error) { queryStr := fmt.Sprintf( "SELECT TABLE_SCHEMA,TABLE_NAME,COLUMN_NAME,ORDINAL_POSITION,DATA_TYPE,COLUMN_TYPE " + "FROM information_schema.COLUMNS WHERE TABLE_SCHEMA =? AND TABLE_NAME = ? " + "ORDER BY TABLE_SCHEMA,TABLE_NAME,ORDINAL_POSITION asc", ) - result, err := dbworker.QueryWithArgs(queryStr, dbName, tblName) + result, err := dbWorker.QueryWithArgs(queryStr, dbName, tblName) // todo 这里的err没有正确捕捉到，比如sql执行错误 if err != nil { return nil, err } - var columnList []string if len(result) > 0 { + tblColumns := make([]TableColumnDef, 0) + for _, row := range result { - columnList = append(columnList, row["COLUMN_NAME"].(string)) + colDef := TableColumnDef{ + ColName: row["COLUMN_NAME"].(string), + ColPos: cast.ToInt(row["ORDINAL_POSITION"].(string)), + ColType: row["DATA_TYPE"].(string), + } + tblColumns = append(tblColumns, colDef) } - return columnList, nil + return tblColumns, nil } return nil, errors.New("table not found") } diff --git a/dbm-services/mysql/db-tools/gobatch-dml/go.mod b/dbm-services/mysql/db-tools/gobatch-dml/go.mod new file mode 100644 index 0000000000..9af1199920 --- /dev/null +++ b/dbm-services/mysql/db-tools/gobatch-dml/go.mod @@ -0,0 +1,37 @@ +module dbm-services/mysql/db-tools/gobatch-dml + +go 1.21.0 + +toolchain go1.21.11 + +require ( + github.com/doug-martin/goqu/v9 v9.19.0 + github.com/pkg/errors v0.9.1 + github.com/samber/lo v1.47.0 + github.com/spf13/cobra v1.7.0 + github.com/spf13/viper v1.19.0 + golang.org/x/sync v0.7.0 +) + +require ( + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/magiconair/properties v1.8.7 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.6.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + go.uber.org/atomic v1.9.0 // indirect + go.uber.org/multierr v1.9.0 // indirect + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/text v0.16.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/dbm-services/mysql/db-tools/gobatch-dml/go.sum b/dbm-services/mysql/db-tools/gobatch-dml/go.sum new file mode 100644 index 0000000000..efa821fa11 --- /dev/null +++ b/dbm-services/mysql/db-tools/gobatch-dml/go.sum @@ -0,0 +1,103 @@ +github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= +github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/denisenkom/go-mssqldb v0.10.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= +github.com/doug-martin/goqu/v9 v9.19.0 h1:PD7t1X3tRcUiSdc5TEyOFKujZA5gs3VSA7wxSvBx7qo= +github.com/doug-martin/goqu/v9 v9.19.0/go.mod h1:nf0Wc2/hV3gYK9LiyqIrzBEVGlI8qW3GuDCEobC4wBQ= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lib/pq v1.10.1 h1:6VXZrLU0jHBYyAqrSPa+MgPfnSvTPuMgK+k0o5kVFWo= +github.com/lib/pq v1.10.1/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/samber/lo v1.47.0 h1:z7RynLwP5nbyRscyvcD043DWYoOcYRv3mV8lBeqOCLc= +github.com/samber/lo v1.47.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= +github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/dbm-services/mysql/db-tools/gobatch-dml/load_data_tosql.go b/dbm-services/mysql/db-tools/gobatch-dml/load_data_tosql.go new file mode 100644 index 0000000000..b39e8a1b88 --- /dev/null +++ b/dbm-services/mysql/db-tools/gobatch-dml/load_data_tosql.go @@ -0,0 +1,448 @@ +package main + +import ( + "context" + "database/sql" + "encoding/csv" + "fmt" + "io" + "os" + "slices" + "strconv" + "strings" + "sync" + + "github.com/doug-martin/goqu/v9" + _ "github.com/doug-martin/goqu/v9/dialect/mysql" + "github.com/pkg/errors" + "github.com/samber/lo" + "github.com/spf13/cobra" + "github.com/spf13/viper" + "golang.org/x/sync/errgroup" + + "dbm-services/common/go-pubpkg/cmutil" + "dbm-services/common/go-pubpkg/validate" + "dbm-services/mysql/db-tools/dbactuator/pkg/native" +) + +/* +LOAD DATA [low_priority] [local] INFILE 'file_name txt' [REPLACE | IGNORE] +INTO TABLE tbl_name +[fields +[terminated by '\t'] +[OPTIONALLY] enclosed by '"'] +[escaped by '\' ]] +[lines terminated by '\n'] +[ignore number lines] +[(col_name, )] + +--low-priority +--dry-run // 生成一个 batch-size, rollback +--concurrency xx // not work well with auto-commit-rows +--rows-per-second 100000 +--disable-autocommit false +--disable-log-bin false +--insert-mode insert|replace|ignore +--fields-terminated-by ',' +--fields-enclosed-by '"' +--lines-terminated-by '\n' +--ignore-lines N +--column-names col1,col2,col3 +--column-types int,time,string,hex +--table-name dbx.table1 +--batch-size 1000 // 1 +--host xx --port xx --user xx --password xx --charset xx +--init-command 'set sql_mode=""' +--force +--lock-tables +--progress +--control-file xx +--result-file xxx +--load-data + +rows_affected: xx, rows_failed: xx (x statements) +time_cost: xx s + +control-file: +batch-size=xx +concurrency=yy +resume-from=0 +*/ + +var rootCmd = &cobra.Command{ + Use: "load-data-tosql", + Short: "load-data-tosql", + Long: "load-data-tosql replace mysqlbinlog", + Version: "1.0.0", + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) (err error) { + return readCsvFile(viper.GetString("file")) + }, +} + +func init() { + // rootCmd + rootCmd.PersistentFlags().String("file", "", "source csv file name") + rootCmd.PersistentFlags().String("result-file", "", + "output generated sql to file, if --concurrency>1, will use it as prefix") + + rootCmd.PersistentFlags().String("table-name", "", "format dbX.tableY") + rootCmd.PersistentFlags().String("column-names", "", "column names list separated by comma") + rootCmd.PersistentFlags().Bool("column-names-from-header", false, + "get column names from csv header that is the first line") + rootCmd.PersistentFlags().Bool("column-names-from-db", false, + "get column names from table in target db") + + rootCmd.PersistentFlags().Int("ignore-lines", 0, "ignore number lines from beginning. "+ + "ignored lines will include the first line when using --column-names-from-header") + rootCmd.PersistentFlags().Int("batch-size", 2, "rows insert per statement") + rootCmd.PersistentFlags().Int("concurrency", 1, "threads or number of result-file to ingest") + rootCmd.PersistentFlags().Bool("dry-run", false, + "try run, only process one batch, and will not load-data to db") + rootCmd.PersistentFlags().Bool("disable-log-bin", false, "disable sql_log_bin to ingest") + rootCmd.PersistentFlags().Bool("disable-autocommit", false, "autocommit=1 by default, "+ + "set --disable-autocommit to ingest within one transaction") + rootCmd.PersistentFlags().Bool("lock-table", false, + "lock table will block all reads/writes to target table") + rootCmd.PersistentFlags().Bool("load-data", false, "write data to mysql") + rootCmd.PersistentFlags().String("fields-terminated-by", ",", "fields-terminated-by") + + rootCmd.PersistentFlags().StringP("host", "h", "", "connect host") + rootCmd.PersistentFlags().IntP("port", "P", 3306, "connect port") + rootCmd.PersistentFlags().StringP("user", "u", "root", "connect user name") + rootCmd.PersistentFlags().StringP("password", "p", "", "connect password") + rootCmd.PersistentFlags().String("charset", "utf8mb4", "connect default-character-set") + rootCmd.PersistentFlags().StringP("socket", "S", "", "connect socket") + + _ = viper.BindPFlag("file", rootCmd.PersistentFlags().Lookup("file")) + _ = viper.BindPFlag("result-file", rootCmd.PersistentFlags().Lookup("result-file")) + _ = viper.BindPFlag("table-name", rootCmd.PersistentFlags().Lookup("table-name")) + _ = viper.BindPFlag("column-names", rootCmd.PersistentFlags().Lookup("column-names")) + _ = viper.BindPFlag("ignore-lines", rootCmd.PersistentFlags().Lookup("ignore-lines")) + _ = viper.BindPFlag("column-names-from-header", + rootCmd.PersistentFlags().Lookup("column-names-from-header")) + _ = viper.BindPFlag("column-names-from-db", rootCmd.PersistentFlags().Lookup("column-names-from-db")) + _ = viper.BindPFlag("batch-size", rootCmd.PersistentFlags().Lookup("batch-size")) + _ = viper.BindPFlag("concurrency", rootCmd.PersistentFlags().Lookup("concurrency")) + _ = viper.BindPFlag("dry-run", rootCmd.PersistentFlags().Lookup("dry-run")) + _ = viper.BindPFlag("disable-log-bin", rootCmd.PersistentFlags().Lookup("disable-log-bin")) + _ = viper.BindPFlag("disable-autocommit", rootCmd.PersistentFlags().Lookup("disable-autocommit")) + _ = viper.BindPFlag("lock-table", rootCmd.PersistentFlags().Lookup("lock-table")) + _ = viper.BindPFlag("load-data", rootCmd.PersistentFlags().Lookup("load-data")) + _ = viper.BindPFlag("fields-terminated-by", rootCmd.PersistentFlags().Lookup("fields-terminated-by")) + + _ = viper.BindPFlag("host", rootCmd.PersistentFlags().Lookup("host")) + _ = viper.BindPFlag("port", rootCmd.PersistentFlags().Lookup("port")) + _ = viper.BindPFlag("user", rootCmd.PersistentFlags().Lookup("user")) + _ = viper.BindPFlag("password", rootCmd.PersistentFlags().Lookup("password")) + _ = viper.BindPFlag("charset", rootCmd.PersistentFlags().Lookup("charset")) + _ = viper.BindPFlag("socket", rootCmd.PersistentFlags().Lookup("socket")) + + rootCmd.PersistentFlags().BoolP("help", "", false, "help for this command") + _ = rootCmd.MarkFlagRequired("file") + _ = rootCmd.MarkFlagRequired("table-name") + rootCmd.MarkFlagsMutuallyExclusive("column-names", "column-names-from-db", "column-names-from-header") + //rootCmd.MarkFlagsRequiredTogether("") + //rootCmd.MarkFlagsRequiredTogether("host", "port", "user", "password") +} + +type Options struct { + DisableLogBin bool `mapstructure:"disable-log-bin"` + // DisableAutocommit run in one connection + DisableAutocommit bool `mapstructure:"disable-autocommit"` + // LockTable run in one connection + LockTable bool `mapstructure:"lock-table"` + + DryRun bool `mapstructure:"dry-run"` + LoadData bool `mapstructure:"load-data"` + ResultFile string `mapstructure:"result-file"` + + File string `mapstructure:"file" validate:"required"` + TableName string `mapstructure:"table-name" validate:"required"` + ColumnNames string `mapstructure:"column-names"` + ColumnNamesFromHeader bool `mapstructure:"column-names-from-header"` + ColumnNamesFromDb bool `mapstructure:"column-names-from-db"` + + IgnoreLines int `mapstructure:"ignore-lines"` + BatchSize int `mapstructure:"batch-size" validate:"gte=1"` + Concurrency int `mapstructure:"concurrency"` + + FieldsTerminatedBy string `mapstructure:"fields-terminated-by"` + + columnNames []interface{} + fieldsTerminatedBy rune +} + +func main() { + if err := rootCmd.Execute(); err != nil { + fmt.Println(err) + os.Exit(1) + } +} + +func readCsvFile(filePath string) error { + var opt Options + err := viper.Unmarshal(&opt) + if err != nil { + return err + } + if err = validate.GoValidateTransError(opt, "mapstructure", false, false); err != nil { + return err + } + + if opt.DryRun { + opt.DisableAutocommit = true + } + //replace := false + var columnNames []interface{} + tableName := opt.TableName + if opt.ColumnNamesFromHeader && opt.IgnoreLines == 0 { + opt.IgnoreLines = 1 + } + + if opt.DisableAutocommit && opt.Concurrency > 1 { + return errors.Errorf("--disable-autocommit only work with --concurrency=1") + } + if opt.LockTable && opt.Concurrency > 1 { + return errors.Errorf("--lock-table only work with --concurrency=1") + } + + var writer io.WriteCloser + var instance native.InsObject + var dbWorker *native.DbWorker + if opt.LoadData || opt.ColumnNamesFromDb { + instance = native.InsObject{ + Host: viper.GetString("host"), + Port: viper.GetInt("port"), + User: viper.GetString("user"), + Pwd: viper.GetString("password"), + Charset: viper.GetString("charset"), + Socket: viper.GetString("socket"), + } + dbWorker, err = instance.Conn() + if err != nil { + return err + } + } + if resultFile := viper.GetString("result-file"); resultFile == "" { + writer = os.Stdout + } else { + outFile, err := os.OpenFile(resultFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0755) + if err != nil { + return err + } + writer = outFile + } + defer writer.Close() + + if opt.ColumnNames != "" { + columnNames = lo.Map(strings.Split(opt.ColumnNames, ","), func(x string, index int) interface{} { + return interface{}(strings.TrimSpace(x)) + }) + } + + inFile, err := os.Open(filePath) + if err != nil { + return err + } + defer inFile.Close() + + reader := csv.NewReader(inFile) + //reader.InputOffset() + sep := getSeparator(opt.FieldsTerminatedBy) + if !slices.Contains([]rune{',', '\t'}, sep) { + return errors.Errorf("--fields-terminated-by expect one character, bug got %s", + viper.GetString("fields-terminated-by")) + } else { + reader.Comma = sep + } + //reader.LazyQuotes = true + + g, ctx := errgroup.WithContext(context.Background()) + var result sync.Map + + if opt.Concurrency > 0 { + g.SetLimit(opt.Concurrency) + } else { + return errors.Errorf("invalid ingest concurrency value %d", opt.Concurrency) + } + + sqlBuilder := goqu.Dialect("mysql") + //db := goqu.New("mysql", mysqlDB) // db + ds := sqlBuilder.Insert(tableName).Cols(columnNames...) + batchCount := 0 + rowsCurrent := 0 + + var processor OutputProcessor + if opt.LoadData { + processor = &LoadProcessor{ + dbConn: dbWorker.Db, + opt: &opt, + } + } else { + processor = &PrintProcessor{ + writer: writer, + opt: &opt, + } + } + + if err = processor.HandleHeader(ctx); err != nil { + return err + } + var dryRunSql string + var linesIgnored int + var lineNum int64 + + var errChan = make(chan error, 1) + errChan <- nil + for { + line, err := reader.Read() + if err == io.EOF { + break + } + if err != nil { + _ = processor.HandleFooter(err, ctx) + if errors.Is(err, csv.ErrFieldCount) { + return errors.WithMessagef(err, "expect %d but got %d", len(columnNames), len(line)) + } + return err + } + lineNum += 1 + + fieldValues := lo.Map(line, func(x string, index int) interface{} { + return interface{}(x) + }) + + if lineNum == 1 { // the first line + if opt.ColumnNamesFromHeader { + // 把第一行作为列名 + columnNames = fieldValues + reader.FieldsPerRecord = len(columnNames) + } + if opt.ColumnNamesFromDb && len(columnNames) == 0 { + columns, err := getColumnNamesFromDb(tableName, dbWorker.Db) + if err != nil { + return errors.WithMessage(err, "get column names from db") + } + for _, col := range columns { + columnNames = append(columnNames, col.ColName) + } + } + if len(columnNames) == 0 { + return errors.Errorf("not column names givien, use --column-names c1,c2 / " + + "--column-names-from-header / --column-names-from-db as you need") + } + ds = sqlBuilder.Insert(tableName).Cols(columnNames...) + + if len(fieldValues) != len(columnNames) { + return errors.Errorf("--column-names count %d does not match csv header %d", + len(columnNames), len(fieldValues)) + // 只导入部分列 + } + if opt.IgnoreLines > 0 { + linesIgnored += 1 + continue + } + } + if opt.IgnoreLines > linesIgnored { + linesIgnored += 1 + continue + } + ds = ds.Vals(fieldValues) + rowsCurrent += 1 + + if rowsCurrent >= opt.BatchSize { + batchCount += 1 + insertSQL, _, _ := ds.ToSQL() + + if opt.DryRun { // the first batch, dry-run + dryRunSql = insertSQL + break + } + // reset builder + ds = sqlBuilder.Insert(tableName).Cols(columnNames...) + rowsCurrent = 0 + + select { + case e := <-errChan: + if e != nil { + return e + } + //default: + } + + g.Go(func() error { + internalErr := processor.Process(insertSQL, ctx) + errChan <- internalErr + return internalErr + }) + } + } + if opt.DryRun { + err = processor.Process(dryRunSql, ctx) + _ = processor.Process("ROLLBACK", ctx) + return nil + } + if rowsCurrent > 0 { // the last batch + if lastErr := <-errChan; lastErr != nil { + fmt.Println("line", lineNum, "has error") + return lastErr + } + + batchCount += 1 + insertSQL, _, _ := ds.ToSQL() + if err = processor.Process(insertSQL, ctx); err != nil { + return err + } + return processor.HandleFooter(nil, ctx) + } + + // 等待所有 goroutine 完成并返回第一个错误（如果有） + if err := g.Wait(); err != nil { + fmt.Printf("Encountered an error: %v\n", err) + } + + // 所有 goroutine 都执行完成，遍历并打印成功的结果 + result.Range(func(key, value any) bool { + //fmt.Printf("fetch url %s status %s\n", key, value) + return true + }) + + return nil +} + +func getColumnNamesFromDb(dbTable string, db *sql.DB) ([]native.TableColumnDef, error) { + dbWorker := &native.DbWorker{ + Db: db, + } + dbName, tableName, err := cmutil.GetDbTableName(dbTable) + if err != nil { + return nil, err + } + columnsDef, err := native.GetTableColumnList(dbName, tableName, dbWorker) + if err != nil { + return nil, err + } + return columnsDef, nil +} + +func getSeparator(sepString string) (sepRune rune) { + sepString = `'` + sepString + `'` + sepRunes, err := strconv.Unquote(sepString) + if err != nil { + // Single quote was used as separator. No idea why someone would want this, but it doesn't hurt to support it + if err.Error() == "invalid syntax" { + sepString = `"` + sepString + `"` + sepRunes, err = strconv.Unquote(sepString) + if err != nil { + panic(err) + } + + } else { + panic(err) + } + } + sepRune = ([]rune(sepRunes))[0] + + return sepRune +} diff --git a/dbm-services/mysql/db-tools/gobatch-dml/load_processor.go b/dbm-services/mysql/db-tools/gobatch-dml/load_processor.go new file mode 100644 index 0000000000..f0ed230bf2 --- /dev/null +++ b/dbm-services/mysql/db-tools/gobatch-dml/load_processor.go @@ -0,0 +1,208 @@ +// TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +// Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +// Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://opensource.org/licenses/MIT +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package main + +import ( + "context" + "database/sql" + "fmt" + "io" + "sync" + "sync/atomic" + + "github.com/pkg/errors" +) + +type PrintProcessor struct { + headerInited bool + writer io.WriteCloser + + opt *Options +} + +type OutputProcessor interface { + Process(string, context.Context) error + HandleHeader(context.Context) error + HandleFooter(error, context.Context) error +} + +func (p *PrintProcessor) HandleHeader(ctx context.Context) error { + if p.writer == nil { + return errors.New("writer is not inited") + } + if p.headerInited { + return nil + } + var err error + defer func() { + p.headerInited = true + }() + + var sqls []string + if p.opt.DisableLogBin { + sqls = append(sqls, "set session sql_log_bin=0") + } + if p.opt.LockTable { + sqls = append(sqls, fmt.Sprintf("LOCK TABLE %s WRITE", p.opt.TableName)) + } + if p.opt.DisableAutocommit { + sqls = append(sqls, "BEGIN") + } + for _, s := range sqls { + _, err = p.writer.Write([]byte(s + ";\n")) + if err != nil { + return err + } + } + return nil +} + +func (p *PrintProcessor) HandleFooter(internalErr error, ctx context.Context) error { + var err error + if p.opt.DisableAutocommit { + if internalErr != nil { + _, err = p.writer.Write([]byte("ROLLBACK;\n")) + } else { + _, err = p.writer.Write([]byte("COMMIT;\n")) + } + } + if p.opt.LockTable { + _, err = p.writer.Write([]byte("UNLOCK TABLES;\n")) + } + return err +} + +func (p *PrintProcessor) Process(sql string, ctx context.Context) error { + _, err := p.writer.Write([]byte(sql + ";\n")) + if err != nil { + return err + } + return nil +} + +type LoadProcessor struct { + headerInited bool + dbConn *sql.DB + oneConn *sql.Conn + //autoCommit bool + + opt *Options + + mu sync.Mutex + rowsAffectedTotal int64 +} + +func (p *LoadProcessor) HandleHeader(ctx context.Context) error { + if p.dbConn == nil { + return errors.New("db connection is not inited") + } + if p.headerInited { + return nil + } + var err error + defer func() { + p.headerInited = true + }() + + var sqls []string + if p.opt.DisableLogBin { + sqls = append(sqls, "set session sql_log_bin=0") + } + if p.opt.LockTable { + sqls = append(sqls, fmt.Sprintf("LOCK TABLE %s WRITE", p.opt.TableName)) + } + // 正常 / 异常结束，都要 UNLOCK TABLES + + if p.opt.DisableAutocommit || p.opt.LockTable { + if p.oneConn == nil { + p.oneConn, err = p.dbConn.Conn(ctx) + if err != nil { + return err + } + } + } + if p.opt.DisableAutocommit { + // 事务操作 + sqls = append(sqls, "BEGIN") + for _, s := range sqls { + if p.opt.DryRun { + fmt.Println(s, ";") + } + _, err = p.oneConn.ExecContext(ctx, s) + if err != nil { + if p.opt.LockTable { + _, _ = p.oneConn.ExecContext(ctx, "UNLOCK TABLES") + } + return errors.WithMessagef(err, "run sql: %s", s) + } + } + } else { + sqls = append(sqls, "set session autocommit=1") + + for _, s := range sqls { + if p.opt.DryRun { + fmt.Println(s, ";") + } + _, err = p.dbConn.ExecContext(ctx, s) + if err != nil { + if p.opt.LockTable { + _, _ = p.dbConn.ExecContext(ctx, "UNLOCK TABLES") + } + return errors.WithMessagef(err, "run sql: %s", s) + } + } + } + return nil +} +func (p *LoadProcessor) HandleFooter(internalErr error, ctx context.Context) error { + var err error + if p.dbConn == nil { + return errors.New("db connection is not inited") + } + if p.opt.DisableAutocommit { + if internalErr != nil { + _, err = p.oneConn.ExecContext(ctx, "ROLLBACK") + } else { + _, err = p.oneConn.ExecContext(ctx, "COMMIT") + } + } + if p.opt.LockTable { + _, err = p.oneConn.ExecContext(ctx, "UNLOCK TABLES") + } + return err +} + +// Process 如果 开启事务，Process失败后会自动回滚 +func (p *LoadProcessor) Process(query string, ctx context.Context) error { + var err error + var res sql.Result + if p.opt.DisableAutocommit { // 开启事务中 + if p.opt.DryRun { + fmt.Println(query, ";") + } + p.mu.Lock() + res, err = p.oneConn.ExecContext(ctx, query) + if err != nil { + _, _ = p.oneConn.ExecContext(ctx, "ROLLBACK") + } + p.mu.Unlock() + } else { + res, err = p.dbConn.ExecContext(ctx, query) + } + if err != nil { + if p.opt.LockTable { + _, _ = p.oneConn.ExecContext(ctx, "UNLOCK TABLES") + } + return errors.WithMessage(err, query) + } else { + rowsAffected, _ := res.RowsAffected() + atomic.AddInt64(&p.rowsAffectedTotal, rowsAffected) + } + return nil +} diff --git a/dbm-services/mysql/db-tools/mysql-crond/cmd/subcmd_pause.go b/dbm-services/mysql/db-tools/mysql-crond/cmd/subcmd_pause.go index 647aa82410..89d07251a0 100644 --- a/dbm-services/mysql/db-tools/mysql-crond/cmd/subcmd_pause.go +++ b/dbm-services/mysql/db-tools/mysql-crond/cmd/subcmd_pause.go @@ -15,7 +15,6 @@ import ( "os" "time" - "github.com/pkg/errors" "github.com/spf13/cobra" "dbm-services/mysql/db-tools/mysql-crond/api" @@ -36,7 +35,8 @@ var pauseJobCmd = &cobra.Command{ } else if nameMatch, _ := cmd.Flags().GetString("name-match"); nameMatch != "" { entries := listEntries(cmd, api.JobStatusEnabled) if len(entries) == 0 { - return errors.Errorf("no job match %s", nameMatch) + return nil + //return errors.Errorf("no job match %s", nameMatch) } for _, entry := range entries { jobNames = append(jobNames, entry.Job.Name) diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/cmd/subcmd_dump.go b/dbm-services/mysql/db-tools/mysql-dbbackup/cmd/subcmd_dump.go index 77a289581d..e13f3616b0 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/cmd/subcmd_dump.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/cmd/subcmd_dump.go @@ -164,7 +164,7 @@ func dumpExecute(cmd *cobra.Command, args []string) (err error) { func backupData(cnf *config.BackupConfig) (err error) { logger.Log.Infof("Dbbackup begin for %d", cnf.Public.MysqlPort) // validate dumpBackup - if err = validate.GoValidateStruct(cnf.Public, false, false); err != nil { + if err = validate.GoValidateStruct(cnf.Public, false); err != nil { return err } if cnf.Public.EncryptOpt == nil { @@ -224,10 +224,12 @@ func backupData(cnf *config.BackupConfig) (err error) { } // ExecuteBackup 执行备份后，返回备份元数据信息 + logger.Log.Info("backup main run:", cnf.Public.MysqlPort) metaInfo, exeErr := backupexe.ExecuteBackup(cnf) if exeErr != nil { return exeErr } + logger.Log.Info("backup main finish:", cnf.Public.MysqlPort) // tar and split err = logReport.ReportBackupStatus("Tar") diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/docs/faq.md b/dbm-services/mysql/db-tools/mysql-dbbackup/docs/faq.md index 8d9c80b65e..d51241ee11 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/docs/faq.md +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/docs/faq.md @@ -32,11 +32,24 @@ DataSchemaGrant = all 不建议将 `Public.NoCheckDiskSpace` 持久化到配置文件 ### 4. 怎么修改备份开始时间 + +- **tendbha主从高可用集群** 进入 mysql-crond 任务调度程序目录 ``` cd /home/mysql/mysql-crond -vim jobs-config.yaml +./mysql-crond list + +如果需要重启 mysql-crond +./stop.sh && sleep 1 +./start.sh +``` +修改 schedule: +``` + ./mysql-crond change-job --permanent -n "dbbackup-schedule" --schedule "4 3 * * *" +``` +某些旧版本的 mysql-crond 没有 change-job命令，可以直接修改 `jobs-config.yaml`，再重启 mysql-crond +``` - name: dbbackup-schedule enable: true command: /home/mysql/dbbackup-go/dbbackup_main.sh @@ -47,10 +60,37 @@ vim jobs-config.yaml schedule: 3 3 * * * creator: system work_dir: /home/mysql/dbbackup-go - -修改 schedule ``` +- **tendbcluster 集群** +spider 集群的备份由两个任务组成 + - `spiderbackup-schedule` + 发起备份，即调度起一个 backup-id 的备份，会向 spider, remote master 写入备份任务`infodba_schema.global_backup`。 + remote slave 的任务由 remote master 同步过来，如果有主从延迟，可能不会马上在 slave 上看到备份进程 + - `spiderbackup-check` + 每分钟轮训判断本机是否有备份任务，如果有则执行备份 + +修改备份时间时间 schedule： +``` +./mysql-crond change-job --permanent -n "spiderbackup-schedule" --schedule "4 3 * * *" +``` + +某些旧版本的 mysql-crond 没有 change-job 命令，可以直接修改 `jobs-config.yaml`，再重启 mysql-crond +``` + - name: spiderbackup-schedule + enable: true + command: /home/mysql/dbbackup-go/dbbackup + args: + - spiderbackup + - schedule + - --config + - dbbackup.25000.ini + schedule: 3 3 * * * + creator: xxx + work_dir: /home/mysql/dbbackup-go +``` + + ### 5. 怎么调整磁盘 io 限速 限速分为 2 个阶段：导出阶段，打包切分阶段 diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/docs/readme.md b/dbm-services/mysql/db-tools/mysql-dbbackup/docs/readme.md index cd3b75bbd5..32d25d3526 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/docs/readme.md +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/docs/readme.md @@ -30,38 +30,72 @@ Flags: ## 3.1 dumpbackup 生成备份时，即 `dumpbackup`，其配置文件config的格式为ini，配置项如下： ``` -[public] - BkBizId string `ini:"BkBizId"` - BkCloudId string `ini:"BkCloudId"` - BillId string `ini:"BillId"` - ClusterAddress string `ini:"ClusterAddress"` - MysqlHost string `ini:"MysqlHost"` - MysqlPort string `ini:"MysqlPort"` - MysqlUser string `ini:"MysqlUser"` - MysqlPasswd string `ini:"MysqlPasswd"` - DataSchemaGrant string `ini:"DataSchemaGrant"` [data,schema,grant] data表示表内数据，schema表示表结构，grant代表权限信息。例如输入 data 表示只备份表内信息，输入 data, grant表示备份表内数据和权限信息， 输入data, schema, grant表示上述三种信息数据都备份。输入你想备份的信息名称，并用`,`连接起来，输入的信息名称必须是data,schema,grant中的一种，否则视为非法输入。也可单独输入all，相当于data,schema,grant。 - BackupDir string `ini:"BackupDir"` - MysqlRole string `ini:"MysqlRole"` [master|slave] - MysqlCharset string `ini:"MysqlCharset"` - BackupTimeOut string `ini:"BackupTimeout"` //example: 09:00:00 - BackupType string `ini:"BackupType"` [logical|physical] - OldFileLeftDay int `ini:"OldFileLeftDay"` - TarSizeThreshold uint64 `ini:"TarSizeThreshold"` -[BackupClient] - FileTag string `ini:"FileTag"` - StorageType string `ini:"StorageType"` [hdfs|cos] - DoChecksum bool `ini:"DoChecksum"` //默认为true -[LogicalBackup] - PartMaxRows uint64 `ini:"PartMaxRows"` - PartChunkSize uint64 `ini:"PartChunkSize"` - Regex string `ini:"Regex"` - Threads int `ini:"Threads"` - FlushRetryCount int `ini:"FlushRetryCount"` - MydumperDefaultsFile string `ini:"MydumperDefaultsFile"` //暂未启用 +[Public] +MysqlHost = x.x.x.x +MysqlPort = 3306 +MysqlUser = xx +MysqlPasswd = xx +MysqlCharset = +MysqlRole = slave +BackupType = physical # physical | logical | auto +DataSchemaGrant = grant +NoCheckDiskSpace = false +OldFileLeftDay = 2 +BkBizId = 123 +BkCloudId = 0 +ClusterId = 1234 +ClusterAddress = xx.xx.xx.db +ShardValue = 0 +BackupTimeout = 09:00:00 +BackupDir = /data/dbbak/ +IOLimitMBPerSec = 300 +IOLimitMasterFactor = 0.5 +TarSizeThreshold = 8192 +FtwrlWaitTimeout = 120 +AcquireLockWaitTimeout = 10 +KillLongQueryTime = 0 +BillId = +BackupId = +ReportPath = /home/mysql/dbareport/mysql/dbbackup +StatusReportPath = /home/mysql/dbareport/mysql/dbbackup/status + [PhysicalBackup] - Threads int `ini:"Threads"` - SplitSpeed int64 `ini:"SplitSpeed"` // MB/s - MysqlDefaultsFile string `ini:"MysqlDefaultsFile"` +Threads = 2 +Throttle = 200 +DefaultsFile = /etc/my.cnf +DisableSlaveMultiThread = true +MaxMyisamTables = 10 +ExtraOpt = + +[LogicalBackup] +Regex = ^(?=(?:(.*\..*$)))(?!(?:(test\..*$|mysql\..*$|sys\..*$|db_infobase\..*$|information_schema\..*$|performance_schema\..*$))) +Databases = * +Tables = * +ExcludeDatabases = # 默认会排除这些系统库 mysql,sys,test,information_schema,performance_schema,db_infobase +ChunkFilesize = 2048 +DisableCompress = false +Threads = 4 +FlushRetryCount = 3 +TrxConsistencyOnly = true +DefaultsFile = +ExtraOpt = +UseMysqldump = no # auto | no | yes + +[LogicalBackupMysqldump] +BinPath = +ExtraOpt = + +[EncryptOpt] +EncryptElgo = +EncryptPublicKey = +EncryptCmd = openssl +EncryptEnable = false + +[BackupClient] +FileTag = MYSQL_FULL_BACKUP +StorageType = +DoChecksum = true +Enable = true ``` OldFileLeftDay = N ： dbbackup运行时，首先删除距今N天的备份文件。如果发现硬盘空间不足，则删除所有以前的备份文件。 diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper.go index 1980fdd218..80221bc8b5 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper.go @@ -57,14 +57,14 @@ func BuildDumper(cnf *config.BackupConfig, db *sql.DB) (dumper Dumper, err error } } if cnf.LogicalBackup.UseMysqldump == cst.LogicalMysqldumpNo { - if err := validate.GoValidateStruct(cnf.LogicalBackup, false, false); err != nil { + if err := validate.GoValidateStruct(cnf.LogicalBackup, false); err != nil { return nil, err } dumper = &LogicalDumper{ cnf: cnf, } } else if cnf.LogicalBackup.UseMysqldump == cst.LogicalMysqldumpYes { - if err := validate.GoValidateStruct(cnf.LogicalBackupMysqldump, false, false); err != nil { + if err := validate.GoValidateStruct(cnf.LogicalBackupMysqldump, false); err != nil { return nil, err } dumper = &LogicalDumperMysqldump{ @@ -78,7 +78,7 @@ func BuildDumper(cnf *config.BackupConfig, db *sql.DB) (dumper Dumper, err error return nil, err } } else if strings.ToLower(cnf.Public.BackupType) == cst.BackupPhysical { - if err := validate.GoValidateStruct(cnf.PhysicalBackup, false, false); err != nil { + if err := validate.GoValidateStruct(cnf.PhysicalBackup, false); err != nil { return nil, err } diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/loader.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/loader.go index cdeb222dda..4c65c24353 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/loader.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/loader.go @@ -23,17 +23,17 @@ func BuildLoader(cnf *config.BackupConfig, backupType string, backupTool string, if strings.ToLower(backupType) == cst.BackupLogical { if backupTool == cst.ToolMysqldump { // mysqldump 共用 LogicalLoad 参数 - if err := validate.GoValidateStruct(cnf.LogicalLoad, false, false); err != nil { + if err := validate.GoValidateStruct(cnf.LogicalLoad, false); err != nil { return nil, err } - if err := validate.GoValidateStruct(cnf.LogicalLoadMysqldump, false, false); err != nil { + if err := validate.GoValidateStruct(cnf.LogicalLoadMysqldump, false); err != nil { return nil, err } loader = &LogicalLoaderMysqldump{ cnf: cnf, } } else { - if err := validate.GoValidateStruct(cnf.LogicalLoad, false, false); err != nil { + if err := validate.GoValidateStruct(cnf.LogicalLoad, false); err != nil { return nil, err } loader = &LogicalLoader{ @@ -44,7 +44,7 @@ func BuildLoader(cnf *config.BackupConfig, backupType string, backupTool string, return nil, err } } else if strings.ToLower(backupType) == cst.BackupPhysical { - if err := validate.GoValidateStruct(cnf.PhysicalLoad, false, false); err != nil { + if err := validate.GoValidateStruct(cnf.PhysicalLoad, false); err != nil { return nil, err } if cst.StorageEngineRocksdb == storageEngine { diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/prepareinfo.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/prepareinfo.go index b82735b693..e102c8f01d 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/prepareinfo.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/prepareinfo.go @@ -10,6 +10,7 @@ import ( "time" "github.com/pkg/errors" + "github.com/spf13/cast" "dbm-services/common/go-pubpkg/cmutil" "dbm-services/mysql/db-tools/mysql-dbbackup/pkg/cst" @@ -29,7 +30,7 @@ type mydumperMetadata struct { // parseMysqldumpMetadata 从 mysqldump sql 文件里解析 change master / change slave 命令 // 命令被注释，在文件开头的前几行 func parseMysqldumpMetadata(sqlFilePath string) (*mydumperMetadata, error) { - + logger.Log.Infof("start parseMysqldumpMetadata from %s", sqlFilePath) sqlFile, err := os.Open(sqlFilePath) if err != nil { return nil, err @@ -43,15 +44,17 @@ func parseMysqldumpMetadata(sqlFilePath string) (*mydumperMetadata, error) { var bufScanner *bufio.Scanner if strings.HasSuffix(sqlFilePath, cst.ZstdSuffix) { - cmds := []string{"cut", "-b", "-4096", sqlFilePath, "|", CmdZstd, "-d", "-c"} + cmds := []string{"head", "-c", "4096", sqlFilePath, "|", CmdZstd, "-d", "-c"} outBuf, _, err := cmutil.ExecCommandReturnBytes(true, "", cmds[0], cmds[1:]...) if len(outBuf) < 100 { // 返回小于这个长度，肯定非法了，重试一遍 - cmds = []string{"cut", "-b", "-10240", sqlFilePath, "|", CmdZstd, "-d", "-c"} + // https://github.com/facebook/zstd/issues/1358 The maximum block size is indeed a hard limit of 128 KB + zstdMaxBlockSize := cast.ToString(128 * 1024 * 2) + cmds = []string{"head", "-c", zstdMaxBlockSize, sqlFilePath, "|", CmdZstd, "-d", "-c"} outBuf, _, err = cmutil.ExecCommandReturnBytes(true, "", cmds[0], cmds[1:]...) } if err != nil { - logger.Log.Warnf("zstd decode first 2048 bytes failed from %s, err:%s", sqlFilePath, err.Error()) + logger.Log.Warnf("zstd decode first 4096 bytes failed from %s, err:%s", sqlFilePath, err.Error()) } if len(outBuf) < 100 { // 返回小于这个长度，非法报错 return nil, errors.Errorf("failed to get binlog position from zst file %s", sqlFilePath) @@ -87,6 +90,7 @@ func parseMysqldumpMetadata(sqlFilePath string) (*mydumperMetadata, error) { } func parseMydumperMetadata(metadataFile string) (*mydumperMetadata, error) { + logger.Log.Infof("start parseMydumperMetadata %s", metadataFile) metafile, err := os.Open(metadataFile) if err != nil { return nil, err @@ -201,7 +205,6 @@ func parseXtraInfo(qpress string, fileName string, tmpFileName string, metaInfo if err != nil { return err } - scanner := bufio.NewScanner(fileBytes) var startTimeStr, endTimeStr string for scanner.Scan() { diff --git a/dbm-services/mysql/db-tools/mysql-monitor/go.mod b/dbm-services/mysql/db-tools/mysql-monitor/go.mod index 9696d1ee81..58d8d29dbb 100644 --- a/dbm-services/mysql/db-tools/mysql-monitor/go.mod +++ b/dbm-services/mysql/db-tools/mysql-monitor/go.mod @@ -22,7 +22,10 @@ require ( gopkg.in/yaml.v2 v2.4.0 ) -require golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc +require ( + github.com/mitchellh/mapstructure v1.5.0 + golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc +) require ( github.com/fsnotify/fsnotify v1.6.0 // indirect @@ -34,7 +37,6 @@ require ( github.com/leodido/go-urn v1.2.4 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/spf13/afero v1.9.5 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect diff --git a/dbm-services/mysql/db-tools/mysql-monitor/items-config.yaml b/dbm-services/mysql/db-tools/mysql-monitor/items-config.yaml index 80a8b64100..7b3edb39c0 100644 --- a/dbm-services/mysql/db-tools/mysql-monitor/items-config.yaml +++ b/dbm-services/mysql/db-tools/mysql-monitor/items-config.yaml @@ -59,6 +59,16 @@ role: - slave - orphan + options: + merge_partition: true + topk_num: 0 + merge_rules: + - from: "(?Pstage_truncate_).+\\..*" + to: "${db}_MERGED._MERGED" + - from: "(?Pbak_20\\d\\d).+\\..*" + to: "${db}._MERGED" + - from: "(bak_cbs)_.+_(\\d+)\\.(?P.+)" + to: "${1}_X_${2}.${table}" - name: master-slave-heartbeat enable: true schedule: '@every 1m' diff --git a/dbm-services/mysql/db-tools/mysql-monitor/pkg/config/items_config.go b/dbm-services/mysql/db-tools/mysql-monitor/pkg/config/items_config.go index 32b0712455..7bb1dde8b5 100644 --- a/dbm-services/mysql/db-tools/mysql-monitor/pkg/config/items_config.go +++ b/dbm-services/mysql/db-tools/mysql-monitor/pkg/config/items_config.go @@ -9,6 +9,8 @@ type MonitorItem struct { Schedule *string `json:"schedule" yaml:"schedule"` MachineType []string `json:"machine_type" yaml:"machine_type"` Role []string `json:"role" yaml:"role"` + // Options custom options for this item + Options map[string]interface{} `json:"options" yaml:"options"` } // IsEnable 监控项启用 @@ -37,3 +39,11 @@ func (c *MonitorItem) IsMatchRole() bool { return slices.Index(c.Role, *MonitorConfig.Role) >= 0 } + +// HasOptions 是否存在自定义选项 +func (c *MonitorItem) HasOptions() bool { + if c.Options == nil || len(c.Options) == 0 { + return false + } + return true +} diff --git a/dbm-services/mysql/db-tools/mysql-monitor/pkg/internal/cst/const.go b/dbm-services/mysql/db-tools/mysql-monitor/pkg/internal/cst/const.go index f55bf760a2..b47db454d2 100644 --- a/dbm-services/mysql/db-tools/mysql-monitor/pkg/internal/cst/const.go +++ b/dbm-services/mysql/db-tools/mysql-monitor/pkg/internal/cst/const.go @@ -1,5 +1,7 @@ package cst +import "fmt" + const ( // DBASchema TODO DBASchema = "infodba_schema" @@ -8,3 +10,10 @@ const ( // SystemUser replication thread user name SystemUser = "system user" ) + +const ( + OTHER_DB_NAME = "_OTHER_" + OTHER_TABLE_NAME = "_OTHER_" +) + +var OTHER_DB_TABLE_NAME = fmt.Sprintf("%s.%s", OTHER_DB_NAME, OTHER_TABLE_NAME) diff --git a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/collect_result.go b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/collect_result.go index 3e00c4f12f..61771dda1b 100644 --- a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/collect_result.go +++ b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/collect_result.go @@ -16,7 +16,7 @@ import ( "strings" ) -func collectResult(dataDir string) (map[string]map[string]int64, error) { +func (c *ibdStatistic) collectResult(dataDir string) (map[string]map[string]int64, error) { result := make(map[string]map[string]int64) err := filepath.WalkDir( diff --git a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/collect_result2.go b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/collect_result2.go new file mode 100644 index 0000000000..81ae603b9a --- /dev/null +++ b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/collect_result2.go @@ -0,0 +1,102 @@ +// TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +// Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +// Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://opensource.org/licenses/MIT +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package ibdstatistic + +import ( + "fmt" + "io/fs" + "log/slog" + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/pkg/errors" + + "dbm-services/common/go-pubpkg/cmutil" +) + +func (c *ibdStatistic) collectResult2(dataDir string) (map[string]int64, map[string]int64, error) { + var err error + dbSize := make(map[string]int64) + tableSize := make(map[string]int64) + + for _, rule := range c.MergeRules { + if rule == nil { + continue + } else if rule.To == "" { + return nil, nil, errors.Errorf("rule to cannot be empty for %s", rule.From) + } + if reMergeRule, err := regexp.Compile(rule.From); err != nil { + return nil, nil, err + } else { + c.reMergeRulesFrom = append(c.reMergeRulesFrom, reMergeRule) + c.reMergeRulesTo = append(c.reMergeRulesTo, rule.To) + } + } + + err = filepath.WalkDir( + dataDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return fs.SkipDir + } + + if !d.IsDir() && strings.ToLower(filepath.Ext(d.Name())) == ibdExt { + dir := filepath.Dir(path) + dbName := filepath.Base(dir) + tableName := strings.TrimSuffix(d.Name(), ibdExt) + + if *c.MergePartition { + match := partitionPattern.FindStringSubmatch(d.Name()) + if match != nil { + tableName = match[1] + } + } + + if len(c.reMergeRulesFrom) > 0 { + newDbTbName := fmt.Sprintf("%s.%s", dbName, tableName) + for i, reMergeRule := range c.reMergeRulesFrom { + if reMergeRule.MatchString(newDbTbName) { + newDbTbName = reMergeRule.ReplaceAllString(newDbTbName, c.reMergeRulesTo[i]) + dbName, tableName, err = cmutil.GetDbTableName(newDbTbName) + //fmt.Println("xxxx1", newDbTbName, dbName, tableName) + if err != nil { + return errors.WithMessagef(err, "using merge rules to %s", c.reMergeRulesTo[i]) + } + break + } + } + } + dbTableName := fmt.Sprintf("%s.%s", dbName, tableName) + + st, err := os.Stat(path) + if err != nil { + slog.Error("ibd-statistic collect result", slog.String("error", err.Error())) + return err + } + if _, ok := dbSize[dbName]; !ok { + dbSize[dbName] = 0 + } + if _, ok := tableSize[dbTableName]; !ok { + tableSize[dbTableName] = 0 + } + dbSize[dbName] += st.Size() + tableSize[dbTableName] += st.Size() + } + return nil + }, + ) + + if err != nil { + slog.Error("ibd-statistic collect result", slog.String("error", err.Error())) + return nil, nil, err + } + + return tableSize, dbSize, nil +} diff --git a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/ibd_statistic.go b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/ibd_statistic.go index 0a1392a776..23e0b75015 100644 --- a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/ibd_statistic.go +++ b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/ibd_statistic.go @@ -5,10 +5,13 @@ import ( "database/sql" "log/slog" "regexp" + "sort" + "dbm-services/mysql/db-tools/mysql-monitor/pkg/internal/cst" "dbm-services/mysql/db-tools/mysql-monitor/pkg/monitoriteminterface" "github.com/jmoiron/sqlx" + "github.com/mitchellh/mapstructure" "github.com/pkg/errors" ) @@ -26,6 +29,7 @@ var name = "ibd-statistic" var ibdExt string var partitionPattern *regexp.Regexp +var defaultMergeRules []*MergeRuleDef var systemDBs = []string{ "mysql", "sys", @@ -39,11 +43,48 @@ var systemDBs = []string{ func init() { ibdExt = ".ibd" partitionPattern = regexp.MustCompile(`^(.*)#[pP]#.*\.ibd`) + defaultMergeRules = []*MergeRuleDef{ + // 规则配在 yaml 里要 \\. 转义 + // 合并转换后的库表明，必须是 dbX.tableY 格式，如果.分割出的 dbName,tableName 为空，会报错 + &MergeRuleDef{ + // "(?Pstage_truncate_).+\\..*" + // 将以 stage_truncate_ 开头的库表 合并成 stage_truncate_MERGED._MERGED + From: `(?Pstage_truncate_).+\..*`, + To: `${db}_MERGED._MERGED`, + }, + &MergeRuleDef{ + // "(?Pbak_20\\d\\d).+\\..*" + // 将 bak_20190218_dbtest.tb1 / bak_20190318_dbtest_1.tb2 合并成 bak_2019._MERGED + From: `(?Pbak_20\d\d).+\..*`, + To: `${db}._MERGED`, + }, + &MergeRuleDef{ + // "(bak_cbs)_.+_(\\d+)\\.(?P
.+)" + // 将 bak_cbs_dbtest_0.tb1 bak_cbs_dbtesta_1.tb2 合并成 bak_cbs_X_0.tb1 bak_cbs_X_1.tb2 + From: `(bak_cbs)_.+_(\d+)\.(?P
.+)`, + To: `${1}_X_${2}.${table}`, + }, + } +} +type MergeRuleDef struct { + From string `mapstructure:"from"` + To string `mapstructure:"to"` } type ibdStatistic struct { - db *sqlx.DB + // MergePartition 合并分区表 + MergePartition *bool `mapstructure:"merge_partition"` + // MergeRuleRegex 合并表名，比如 db\.test_(\d+) 会合并 db.test_1 db.test_2 成 db.test_X + // 提示：这里的替换规则，可能会把 spider remote _ 也去掉，统计时需要注意 + MergeRules []*MergeRuleDef `mapstructure:"merge_rules"` + // TopkNum 只上报排名前 k 条记录，0 表示全部 + TopkNum int `mapstructure:"topk_num"` + + optionMap monitoriteminterface.ItemOptions + reMergeRulesFrom []*regexp.Regexp + reMergeRulesTo []string + db *sqlx.DB } // Run TODO @@ -61,17 +102,37 @@ func (c *ibdStatistic) Run() (msg string, err error) { return "", err } - result, err := collectResult(dataDir.String) + dbTableSize, dbSize, err := c.collectResult2(dataDir.String) if err != nil { return "", err } - err = reportMetrics(result) - if err != nil { - return "", err + if c.TopkNum > 0 { + type dbTableInfo struct { + dbTableName string + size int64 + } + var dbTableSizeSorted []dbTableInfo + + for k, v := range dbTableSize { + dbTableSizeSorted = append(dbTableSizeSorted, dbTableInfo{dbTableName: k, size: v}) + } + // 降序 + sort.Slice(dbTableSizeSorted, func(i, j int) bool { + return dbTableSizeSorted[i].size > dbTableSizeSorted[j].size + }) + dbTableSize = nil + dbTableSize = make(map[string]int64) // reuse + for i, sz := range dbTableSizeSorted { + if i < c.TopkNum { + dbTableSize[sz.dbTableName] = sz.size + } else { + dbTableSize[cst.OTHER_DB_TABLE_NAME] += sz.size + } + } } - err = reportLog(result) + err = reportLog2(dbTableSize, dbSize) if err != nil { return "", err } @@ -84,9 +145,33 @@ func (c *ibdStatistic) Name() string { return name } +func (c *ibdStatistic) initCustomOptions(opts monitoriteminterface.ItemOptions) error { + return nil +} + // New TODO func New(cc *monitoriteminterface.ConnectionCollect) monitoriteminterface.MonitorItemInterface { - return &ibdStatistic{db: cc.MySqlDB} + opts := cc.GetCustomOptions(name) + var itemObj ibdStatistic + if err := mapstructure.Decode(opts, &itemObj); err != nil { + panic(err) + } + itemObj.db = cc.MySqlDB + itemObj.optionMap = opts + + if len(itemObj.MergeRules) == 0 { + slog.Info("ibd-statistic", slog.String("msg", "use default merge rules"), + slog.Int("count", len(itemObj.MergeRules))) + itemObj.MergeRules = defaultMergeRules + } else { + slog.Info("ibd-statistic", slog.String("msg", "use custom merge rules"), + slog.Int("count", len(itemObj.MergeRules))) + } + if itemObj.MergePartition == nil { + truePtr := true + itemObj.MergePartition = &truePtr + } + return &itemObj } // Register TODO diff --git a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/report_log2.go b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/report_log2.go new file mode 100644 index 0000000000..d362c160dd --- /dev/null +++ b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/ibdstatistic/report_log2.go @@ -0,0 +1,95 @@ +// TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +// Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +// Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://opensource.org/licenses/MIT +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package ibdstatistic + +import ( + "log/slog" + "os" + "path/filepath" + "slices" + "strings" + "time" + + "dbm-services/common/go-pubpkg/cmutil" + "dbm-services/common/go-pubpkg/reportlog" + "dbm-services/mysql/db-tools/mysql-monitor/pkg/config" + "dbm-services/mysql/db-tools/mysql-monitor/pkg/internal/cst" + + "github.com/pkg/errors" +) + +func reportLog2(dbTableSize map[string]int64, dbSize map[string]int64) error { + dbsizeReportBaseDir := filepath.Join(cst.DBAReportBase, "mysql/dbsize") + err := os.MkdirAll(dbsizeReportBaseDir, os.ModePerm) + if err != nil { + slog.Error("failed to create database size reports directory", slog.String("error", err.Error())) + return errors.Wrap(err, "failed to create database size reports directory") + } + resultReport, err := reportlog.NewReporter(dbsizeReportBaseDir, "report.log", nil) + if err != nil { + return err + } + reportTs := cmutil.TimeToSecondPrecision(time.Now()) + for dbTableName, tableSize := range dbTableSize { + var originalDBName, tableName string + if ss := strings.SplitN(dbTableName, ".", 2); len(ss) == 2 { + originalDBName = ss[0] + tableName = ss[1] + if _, ok := dbSize[originalDBName]; !ok { // 这个不应该发生，防止后面 panic 设置默认值 + slog.Error("failed to read database size for db %s", originalDBName) + dbSize[originalDBName] = 0 + } + } else { + return errors.Errorf("fail to get db table name from %s", dbTableName) + } + // 根据 dbm 枚举约定, remote 是 tendbcluster 的存储机器类型 + dbName := originalDBName + if dbTableName == cst.OTHER_DB_TABLE_NAME { + dbSize[originalDBName] = tableSize + } + + if config.MonitorConfig.MachineType == "remote" && slices.Index(systemDBs, originalDBName) < 0 { + match := tenDBClusterDbNamePattern.FindStringSubmatch(originalDBName) + if match == nil { + err := errors.Errorf( + "invalid dbname: '%s' on %s", + originalDBName, config.MonitorConfig.MachineType, + ) + slog.Error("ibd-statistic report", slog.String("error", err.Error())) + // 这里不退出，尽可能上报 dbTableName == "_OTHER_._OTHER_" + //return err + } else { + dbName = match[1] + } + } + if slices.Index(systemDBs, dbName) >= 0 { + continue + } + + oneTableInfo := tableSizeStruct{ + BkCloudId: *config.MonitorConfig.BkCloudID, + BkBizId: config.MonitorConfig.BkBizId, + ImmuteDomain: config.MonitorConfig.ImmuteDomain, + DBModule: *config.MonitorConfig.DBModuleID, + MachineType: config.MonitorConfig.MachineType, + Ip: config.MonitorConfig.Ip, + Port: config.MonitorConfig.Port, + Role: *config.MonitorConfig.Role, + ServiceInstanceId: config.MonitorConfig.BkInstanceId, + OriginalDBName: originalDBName, + DBName: dbName, + DBSize: dbSize[originalDBName], // 每个表都会跟随上报一份 database size + TableName: tableName, + TableSize: tableSize, + ReportTime: reportTs, + } + resultReport.Println(oneTableInfo) + } + return nil +} diff --git a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/items_collect.go b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/items_collect.go index cacab8183b..2f437afc5e 100644 --- a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/items_collect.go +++ b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/items_collect.go @@ -12,6 +12,7 @@ package itemscollect import ( "dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/privcheck" "dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/rotateproxyconnlog" + "fmt" "log/slog" diff --git a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/mysqlprocesslist/query_kill.go b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/mysqlprocesslist/query_kill.go new file mode 100644 index 0000000000..14bc6f7b34 --- /dev/null +++ b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/mysqlprocesslist/query_kill.go @@ -0,0 +1,16 @@ +package mysqlprocesslist + +/* +query-kill + match-db + match-user + match-state + match-host + match-info + busy-time + max-kills-per-round + idle-time-include 0 + ignore-user + kill + print +*/ diff --git a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/perfschema/events_statements_summary_by_digest.go b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/perfschema/events_statements_summary_by_digest.go new file mode 100644 index 0000000000..24f928c461 --- /dev/null +++ b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/perfschema/events_statements_summary_by_digest.go @@ -0,0 +1,9 @@ +// TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +// Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +// Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +// You may obtain a copy of the License at https://opensource.org/licenses/MIT +// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +package perfschema diff --git a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/perfschema/readme.md b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/perfschema/readme.md new file mode 100644 index 0000000000..e02e739196 --- /dev/null +++ b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/perfschema/readme.md @@ -0,0 +1,62 @@ + +example1: +``` +// performance_schema = ON +// performance_schema_digests_size = 10000 +// performance_schema_max_digest_length = 1024 +// performance_schema_max_sql_text_length = 1024 + +mysql> select * from performance_schema.setup_consumers; ++----------------------------------+---------+ +| NAME | ENABLED | ++----------------------------------+---------+ +| events_stages_current | NO | +| events_stages_history | NO | +| events_stages_history_long | NO | +| events_statements_current | YES | +| events_statements_history | YES | +| events_statements_history_long | NO | +| events_transactions_current | NO | +| events_transactions_history | NO | +| events_transactions_history_long | NO | +| events_waits_current | NO | +| events_waits_history | NO | +| events_waits_history_long | NO | +| global_instrumentation | YES | +| thread_instrumentation | YES | +| statements_digest | YES | ++----------------------------------+---------+ +``` + +example2: +``` + SCHEMA_NAME: ceiba_admin_pro + DIGEST: 09ec094394a7af2af3230110f87a5bbd + DIGEST_TEXT: SELECT * FROM `t_data_source` + COUNT_STAR: 133 + SUM_TIMER_WAIT: 30786417000 + MIN_TIMER_WAIT: 120668000 + AVG_TIMER_WAIT: 231476000 + MAX_TIMER_WAIT: 399896000 + SUM_LOCK_TIME: 14579000000 + SUM_ERRORS: 0 + SUM_WARNINGS: 0 + SUM_ROWS_AFFECTED: 0 + SUM_ROWS_SENT: 133 + SUM_ROWS_EXAMINED: 133 +SUM_CREATED_TMP_DISK_TABLES: 0 + SUM_CREATED_TMP_TABLES: 0 + SUM_SELECT_FULL_JOIN: 0 + SUM_SELECT_FULL_RANGE_JOIN: 0 + SUM_SELECT_RANGE: 0 + SUM_SELECT_RANGE_CHECK: 0 + SUM_SELECT_SCAN: 133 + SUM_SORT_MERGE_PASSES: 0 + SUM_SORT_RANGE: 0 + SUM_SORT_ROWS: 0 + SUM_SORT_SCAN: 0 + SUM_NO_INDEX_USED: 133 + SUM_NO_GOOD_INDEX_USED: 0 + FIRST_SEEN: 2022-08-23 11:47:13 + LAST_SEEN: 2024-12-17 17:35:36 +``` \ No newline at end of file diff --git a/dbm-services/mysql/db-tools/mysql-monitor/pkg/mainloop/main_loop.go b/dbm-services/mysql/db-tools/mysql-monitor/pkg/mainloop/main_loop.go index 0e5aa5d518..b03c3faa16 100644 --- a/dbm-services/mysql/db-tools/mysql-monitor/pkg/mainloop/main_loop.go +++ b/dbm-services/mysql/db-tools/mysql-monitor/pkg/mainloop/main_loop.go @@ -10,13 +10,14 @@ package mainloop import ( - "dbm-services/mysql/db-tools/dbactuator/pkg/core/cst" "fmt" "log/slog" "path/filepath" "slices" "strings" + "dbm-services/mysql/db-tools/dbactuator/pkg/core/cst" + "dbm-services/mysql/db-tools/mysql-monitor/pkg/config" "dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect" "dbm-services/mysql/db-tools/mysql-monitor/pkg/monitoriteminterface" @@ -73,6 +74,7 @@ func Run(hardcode bool) error { defer func() { cc.Close() }() + cc.InitItemOptions() // set item custom options to runner slog.Debug("make connection collect", slog.Any("connection collect", cc)) diff --git a/dbm-services/mysql/db-tools/mysql-monitor/pkg/monitoriteminterface/connection_collect.go b/dbm-services/mysql/db-tools/mysql-monitor/pkg/monitoriteminterface/connection_collect.go index 6339cc3bbe..4d809514b9 100644 --- a/dbm-services/mysql/db-tools/mysql-monitor/pkg/monitoriteminterface/connection_collect.go +++ b/dbm-services/mysql/db-tools/mysql-monitor/pkg/monitoriteminterface/connection_collect.go @@ -26,6 +26,8 @@ type ConnectionCollect struct { ProxyDB *sqlx.DB ProxyAdminDB *sqlx.DB CtlDB *sqlx.DB + + itemOptions map[string]ItemOptions } // Close 关闭所有连接 @@ -47,6 +49,39 @@ func (c *ConnectionCollect) Close() { } } +type ItemOptions map[string]interface{} + +func (c *ConnectionCollect) InitItemOptions() map[string]ItemOptions { + opts := make(map[string]ItemOptions) + for _, opt := range config.ItemsConfig { + opts[opt.Name] = opt.Options + } + c.itemOptions = opts + return opts +} +func (c *ConnectionCollect) GetCustomOptions(name string) ItemOptions { + return c.itemOptions[name] +} + +func (o ItemOptions) Get(optionName string, defaultValue interface{}) interface{} { + if val, ok := o[optionName]; ok { + return val + } + return defaultValue +} +func (o ItemOptions) GetInt(optionName string, defaultValue interface{}) int { + return o.Get(optionName, defaultValue).(int) +} +func (o ItemOptions) GetBool(optionName string, defaultValue interface{}) bool { + return o.Get(optionName, defaultValue).(bool) +} +func (o ItemOptions) GetString(optionName string, defaultValue interface{}) string { + return o.Get(optionName, defaultValue).(string) +} +func (o ItemOptions) GetStringSlice(optionName string, defaultValue interface{}) []string { + return o.Get(optionName, defaultValue).([]string) +} + // NewConnectionCollect 新建连接 func NewConnectionCollect() (*ConnectionCollect, error) { switch config.MonitorConfig.MachineType { diff --git a/dbm-services/mysql/db-tools/mysql-rotatebinlog/pkg/backup/backup_ibs.go b/dbm-services/mysql/db-tools/mysql-rotatebinlog/pkg/backup/backup_ibs.go index e269e02f7e..db29e1d78f 100644 --- a/dbm-services/mysql/db-tools/mysql-rotatebinlog/pkg/backup/backup_ibs.go +++ b/dbm-services/mysql/db-tools/mysql-rotatebinlog/pkg/backup/backup_ibs.go @@ -35,7 +35,7 @@ type IBSBackupClient struct { // Init TODO func (o *IBSBackupClient) Init() error { - if err := validate.GoValidateStruct(o, false, false); err != nil { + if err := validate.GoValidateStruct(o, false); err != nil { return err } o.ibsQueryCmd = o.ToolPath diff --git a/dbm-services/mysql/db-tools/mysql-rotatebinlog/pkg/rotate/main.go b/dbm-services/mysql/db-tools/mysql-rotatebinlog/pkg/rotate/main.go index e6b65f2cc4..c456b6968a 100644 --- a/dbm-services/mysql/db-tools/mysql-rotatebinlog/pkg/rotate/main.go +++ b/dbm-services/mysql/db-tools/mysql-rotatebinlog/pkg/rotate/main.go @@ -82,7 +82,7 @@ func (c *RotateBinlogComp) Start() (err error) { Pwd: inst.Password, Socket: inst.Socket, } - if err = validate.GoValidateStruct(inst, true, false); err != nil { + if err = validate.GoValidateStruct(inst, true); err != nil { err = errs.WithMessagef(err, "validate instance %s", inst) logger.Error("%+v", err.Error()) errRet = errors.Join(errRet, err)