From 02c4fbac51e3c342cebe6e94732817923d47b93e Mon Sep 17 00:00:00 2001 From: iSecloud <869820505@qq.com> Date: Wed, 13 Nov 2024 19:30:29 +0800 Subject: [PATCH] =?UTF-8?q?feat(backend):=20=E6=B8=85=E6=A1=A3=E5=8D=95?= =?UTF-8?q?=E6=8D=AE=E4=BC=98=E5=8C=96=20#8739?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../mysql/generate_drop_stage_db_sql.py | 29 +++++-- .../builders/mysql/mysql_delete_clear_db.py | 83 +++++++++++++++++++ .../ticket/builders/mysql/mysql_ha_clear.py | 6 ++ .../tendbcluster/tendb_delete_clear_db.py | 39 +++++++++ dbm-ui/backend/ticket/constants.py | 4 +- 5 files changed, 153 insertions(+), 8 deletions(-) create mode 100644 dbm-ui/backend/ticket/builders/mysql/mysql_delete_clear_db.py create mode 100644 dbm-ui/backend/ticket/builders/tendbcluster/tendb_delete_clear_db.py diff --git a/dbm-ui/backend/flow/plugins/components/collections/mysql/generate_drop_stage_db_sql.py b/dbm-ui/backend/flow/plugins/components/collections/mysql/generate_drop_stage_db_sql.py index 080e8ff5b1..51178eb558 100644 --- a/dbm-ui/backend/flow/plugins/components/collections/mysql/generate_drop_stage_db_sql.py +++ b/dbm-ui/backend/flow/plugins/components/collections/mysql/generate_drop_stage_db_sql.py @@ -8,6 +8,7 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ +from datetime import datetime, timedelta from django.db.transaction import atomic from django.utils.translation import gettext as _ @@ -16,8 +17,9 @@ from backend.db_services.mysql.sql_import.constants import BKREPO_SQLFILE_PATH, SQLCharset, SQLExecuteTicketMode from backend.flow.plugins.components.collections.common.base_service import BaseService from backend.ticket.builders.common.base import fetch_cluster_ids -from backend.ticket.constants import TicketType -from backend.ticket.models import Ticket +from backend.ticket.constants import FlowType, TicketType +from backend.ticket.models import Flow, Ticket +from backend.utils.time import datetime2str class GenerateDropStageDBSqlService(BaseService): @@ -57,28 +59,41 @@ def generate_dropsql_ticket(global_data, trans_data, kwargs): bk_biz_id, cluster_ids = global_data["bk_biz_id"], fetch_cluster_ids(global_data["infos"]) drop_sql_content = ";".join(ticket.details["drop_stage_db_cmds"]) + ticket_mode = ticket.details.get("clear_mode", {"mode": SQLExecuteTicketMode.MANUAL.value}) + if ticket_mode["mode"] == SQLExecuteTicketMode.TIMER: + trigger_time = datetime.now().astimezone() + timedelta(days=ticket_mode["days"]) + ticket_mode["trigger_time"] = datetime2str(trigger_time) + details = { "bk_biz_id": bk_biz_id, "cluster_ids": cluster_ids, "backup": [], "path": BKREPO_SQLFILE_PATH.format(biz=bk_biz_id), "charset": SQLCharset.DEFAULT.value, - "ticket_mode": {"mode": SQLExecuteTicketMode.MANUAL.value}, + "ticket_mode": ticket_mode, "execute_objects": [{"dbnames": ["test"], "ignore_dbnames": [], "sql_content": drop_sql_content}], } if ticket.ticket_type == TicketType.TENDBCLUSTER_TRUNCATE_DATABASE: - ticket_type = TicketType.TENDBCLUSTER_FORCE_IMPORT_SQLFILE.value + ticket_type = TicketType.TENDBCLUSTER_DELETE_CLEAR_DB.value else: - ticket_type = TicketType.MYSQL_FORCE_IMPORT_SQLFILE.value + ticket_type = TicketType.MYSQL_DELETE_CLEAR_DB.value - Ticket.create_ticket( + # 创建删除备份库单据 + clear_ticket = Ticket.create_ticket( ticket_type=ticket_type, creator=global_data["created_by"], bk_biz_id=bk_biz_id, - remark=_("清档自动发起的变更SQL单据\n关联单据：{}").format(ticket.url), + remark=_("清档自动发起的清理单据\n关联单据：{}").format(ticket.url), details=details, ) + # 原单据创建flow，关联清档单据 + Flow.objects.create( + ticket=ticket, + flow_type=FlowType.DELIVERY.value, + details={"related_ticket": clear_ticket.id}, + flow_alias=_("关联删除清档备份库单据"), + ) class GenerateDropStageDBSqlComponent(Component): diff --git a/dbm-ui/backend/ticket/builders/mysql/mysql_delete_clear_db.py b/dbm-ui/backend/ticket/builders/mysql/mysql_delete_clear_db.py new file mode 100644 index 0000000000..df6b3f3ef6 --- /dev/null +++ b/dbm-ui/backend/ticket/builders/mysql/mysql_delete_clear_db.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import logging + +from django.utils.translation import ugettext as _ + +from backend.db_services.mysql.sql_import.constants import SQLExecuteTicketMode +from backend.flow.engine.controller.mysql import MySQLController +from backend.ticket import builders +from backend.ticket.builders.mysql.mysql_force_import_sqlfile import ( + MysqlForceSqlImportDetailSerializer, + MysqlForceSqlImportFlowBuilder, + MysqlForceSqlImportFlowParamBuilder, +) +from backend.ticket.constants import FlowRetryType, FlowType, TicketType +from backend.ticket.models import Flow + +logger = logging.getLogger("root") + + +class MysqlDeleteClearDBDetailSerializer(MysqlForceSqlImportDetailSerializer): + def validate(self, attrs): + return attrs + + +class MysqlDeleteClearDBFlowParamBuilder(MysqlForceSqlImportFlowParamBuilder): + controller = MySQLController.mysql_import_sqlfile_scene + + def format_ticket_data(self): + pass + + +@builders.BuilderFactory.register(TicketType.MYSQL_DELETE_CLEAR_DB, is_sensitive=True) +class MysqlDeleteClearDBFlowBuilder(MysqlForceSqlImportFlowBuilder): + serializer = MysqlDeleteClearDBDetailSerializer + inner_flow_builder = MysqlDeleteClearDBFlowParamBuilder + editable = False + + @property + def need_itsm(self): + return False + + def init_ticket_flows(self): + """ + sql导入根据执行模式可分为三种执行流程： + 手动：手动确认-->(备份)--->sql导入 + 自动：(备份)--->sql导入 + 定时：定时触发-->(备份)--->sql导入 + """ + flows = [] + mode = self.ticket.details["ticket_mode"]["mode"] + + if mode == SQLExecuteTicketMode.MANUAL.value: + flows.append(Flow(ticket=self.ticket, flow_type=FlowType.PAUSE.value, flow_alias=_("人工确认执行"))) + + if mode == SQLExecuteTicketMode.TIMER.value: + flows.append(Flow(ticket=self.ticket, flow_type=FlowType.TIMER.value, flow_alias=_("定时执行"))) + + flows.append( + Flow( + ticket=self.ticket, + flow_type=FlowType.INNER_FLOW.value, + details=self.inner_flow_builder(self.ticket).get_params(), + retry_type=FlowRetryType.MANUAL_RETRY.value, + flow_alias=_("删除清档备份库"), + ) + ) + + Flow.objects.bulk_create(flows) + return list(Flow.objects.filter(ticket=self.ticket)) + + @classmethod + def describe_ticket_flows(cls, flow_config_map): + flow_desc = [_("定时执行/人工执行"), _("变更SQL执行")] + return flow_desc diff --git a/dbm-ui/backend/ticket/builders/mysql/mysql_ha_clear.py b/dbm-ui/backend/ticket/builders/mysql/mysql_ha_clear.py index 5c28753863..b6246e5b80 100644 --- a/dbm-ui/backend/ticket/builders/mysql/mysql_ha_clear.py +++ b/dbm-ui/backend/ticket/builders/mysql/mysql_ha_clear.py @@ -13,6 +13,7 @@ from rest_framework import serializers from backend.db_meta.enums import ClusterType +from backend.db_services.mysql.sql_import.constants import SQLExecuteTicketMode from backend.flow.consts import TruncateDataTypeEnum from backend.flow.engine.controller.mysql import MySQLController from backend.ticket import builders @@ -34,7 +35,12 @@ class TruncateDataInfoSerializer(serializers.Serializer): truncate_data_type = serializers.ChoiceField(help_text=_("清档类型"), choices=TruncateDataTypeEnum.get_choices()) force = serializers.BooleanField(help_text=_("是否强制执行"), default=False) + class ClearImportModeSerializer(serializers.Serializer): + mode = serializers.ChoiceField(help_text=_("单据执行模式"), choices=SQLExecuteTicketMode.get_choices()) + days = serializers.IntegerField(help_text=_("执行删除延迟天数"), required=False, default=15) + infos = serializers.ListSerializer(help_text=_("清档信息列表"), child=TruncateDataInfoSerializer()) + clear_mode = ClearImportModeSerializer(help_text=_("删除备份库模式"), required=False) def validate(self, attrs): """校验库表选择器信息是否正确""" diff --git a/dbm-ui/backend/ticket/builders/tendbcluster/tendb_delete_clear_db.py b/dbm-ui/backend/ticket/builders/tendbcluster/tendb_delete_clear_db.py new file mode 100644 index 0000000000..b15085a2ac --- /dev/null +++ b/dbm-ui/backend/ticket/builders/tendbcluster/tendb_delete_clear_db.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import logging + +from backend.configuration.constants import DBType +from backend.flow.engine.controller.spider import SpiderController +from backend.ticket import builders +from backend.ticket.builders.mysql.mysql_delete_clear_db import ( + MysqlDeleteClearDBDetailSerializer, + MysqlDeleteClearDBFlowBuilder, + MysqlDeleteClearDBFlowParamBuilder, +) +from backend.ticket.constants import TicketType + +logger = logging.getLogger("root") + + +class TendbClusterDeleteClearDBDetailSerializer(MysqlDeleteClearDBDetailSerializer): + pass + + +class TendbClusterDeleteClearDBFlowParamBuilder(MysqlDeleteClearDBFlowParamBuilder): + controller = SpiderController.spider_sql_import_scene + + +@builders.BuilderFactory.register(TicketType.TENDBCLUSTER_DELETE_CLEAR_DB, is_sensitive=True) +class TendbClusterDeleteClearDBFlowBuilder(MysqlDeleteClearDBFlowBuilder): + group = DBType.TenDBCluster + serializer = TendbClusterDeleteClearDBDetailSerializer + inner_flow_builder = TendbClusterDeleteClearDBFlowParamBuilder + editable = False diff --git a/dbm-ui/backend/ticket/constants.py b/dbm-ui/backend/ticket/constants.py index 6a4c2d6508..f638b48505 100644 --- a/dbm-ui/backend/ticket/constants.py +++ b/dbm-ui/backend/ticket/constants.py @@ -227,6 +227,7 @@ def get_approve_mode_by_ticket(cls, ticket_type): MYSQL_HA_APPLY = TicketEnumField("MYSQL_HA_APPLY", _("MySQL 高可用部署"), register_iam=False) MYSQL_IMPORT_SQLFILE = TicketEnumField("MYSQL_IMPORT_SQLFILE", _("MySQL 变更SQL执行"), _("SQL 任务")) MYSQL_FORCE_IMPORT_SQLFILE = TicketEnumField("MYSQL_FORCE_IMPORT_SQLFILE", _("MySQL 强制变更SQL执行"), _("SQL 任务"), register_iam=False) # noqa + MYSQL_DELETE_CLEAR_DB = TicketEnumField("MYSQL_DELETE_CLEAR_DB", _("MySQL 删除清档备份库"), _("数据处理"), register_iam=False) # noqa MYSQL_SEMANTIC_CHECK = TicketEnumField("MYSQL_SEMANTIC_CHECK", _("MySQL 模拟执行"), register_iam=False) MYSQL_PROXY_ADD = TicketEnumField("MYSQL_PROXY_ADD", _("MySQL 添加Proxy"), _("集群维护")) MYSQL_PROXY_SWITCH = TicketEnumField("MYSQL_PROXY_SWITCH", _("MySQL 替换Proxy"), _("集群维护")) @@ -278,7 +279,8 @@ def get_approve_mode_by_ticket(cls, ticket_type): TENDBCLUSTER_MASTER_FAIL_OVER = TicketEnumField("TENDBCLUSTER_MASTER_FAIL_OVER", _("TenDB Cluster 主库故障切换"), _("集群维护")) # noqa TENDBCLUSTER_MASTER_SLAVE_SWITCH = TicketEnumField("TENDBCLUSTER_MASTER_SLAVE_SWITCH", _("TenDB Cluster 主从互切"), _("集群维护")) # noqa TENDBCLUSTER_IMPORT_SQLFILE = TicketEnumField("TENDBCLUSTER_IMPORT_SQLFILE", _("TenDB Cluster 变更SQL执行"), _("SQL 任务")) # noqa - TENDBCLUSTER_FORCE_IMPORT_SQLFILE = TicketEnumField("TENDBCLUSTER_FORCE_IMPORT_SQLFILE", _("TenDB Cluster 强制变更SQL执行"), register_iam=False) # noqa + TENDBCLUSTER_FORCE_IMPORT_SQLFILE = TicketEnumField("TENDBCLUSTER_FORCE_IMPORT_SQLFILE", _("TenDB Cluster 强制变更SQL执行"), _("SQL 任务"), register_iam=False) # noqa + TENDBCLUSTER_DELETE_CLEAR_DB = TicketEnumField("TENDBCLUSTER_DELETE_CLEAR_DB", _("TenDB Cluster 删除清档备份库"), _("数据处理"), register_iam=False) # noqa TENDBCLUSTER_SEMANTIC_CHECK = TicketEnumField("TENDBCLUSTER_SEMANTIC_CHECK", _("TenDB Cluster 模拟执行"), register_iam=False) # noqa TENDBCLUSTER_SPIDER_ADD_NODES = TicketEnumField("TENDBCLUSTER_SPIDER_ADD_NODES", _("TenDB Cluster 扩容接入层"), _("集群维护")) # noqa TENDBCLUSTER_SPIDER_REDUCE_NODES = TicketEnumField("TENDBCLUSTER_SPIDER_REDUCE_NODES", _("TenDB Cluster 缩容接入层"), _("集群维护")) # noqa