Skip to content

Commit

Permalink
feat(backend): 清档单据优化 #8739
Browse files Browse the repository at this point in the history
  • Loading branch information
iSecloud committed Dec 20, 2024
1 parent 769afe7 commit 02c4fba
Show file tree
Hide file tree
Showing 5 changed files with 153 additions and 8 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
specific language governing permissions and limitations under the License.
"""
from datetime import datetime, timedelta

from django.db.transaction import atomic
from django.utils.translation import gettext as _
Expand All @@ -16,8 +17,9 @@
from backend.db_services.mysql.sql_import.constants import BKREPO_SQLFILE_PATH, SQLCharset, SQLExecuteTicketMode
from backend.flow.plugins.components.collections.common.base_service import BaseService
from backend.ticket.builders.common.base import fetch_cluster_ids
from backend.ticket.constants import TicketType
from backend.ticket.models import Ticket
from backend.ticket.constants import FlowType, TicketType
from backend.ticket.models import Flow, Ticket
from backend.utils.time import datetime2str


class GenerateDropStageDBSqlService(BaseService):
Expand Down Expand Up @@ -57,28 +59,41 @@ def generate_dropsql_ticket(global_data, trans_data, kwargs):
bk_biz_id, cluster_ids = global_data["bk_biz_id"], fetch_cluster_ids(global_data["infos"])

drop_sql_content = ";".join(ticket.details["drop_stage_db_cmds"])
ticket_mode = ticket.details.get("clear_mode", {"mode": SQLExecuteTicketMode.MANUAL.value})
if ticket_mode["mode"] == SQLExecuteTicketMode.TIMER:
trigger_time = datetime.now().astimezone() + timedelta(days=ticket_mode["days"])
ticket_mode["trigger_time"] = datetime2str(trigger_time)

details = {
"bk_biz_id": bk_biz_id,
"cluster_ids": cluster_ids,
"backup": [],
"path": BKREPO_SQLFILE_PATH.format(biz=bk_biz_id),
"charset": SQLCharset.DEFAULT.value,
"ticket_mode": {"mode": SQLExecuteTicketMode.MANUAL.value},
"ticket_mode": ticket_mode,
"execute_objects": [{"dbnames": ["test"], "ignore_dbnames": [], "sql_content": drop_sql_content}],
}

if ticket.ticket_type == TicketType.TENDBCLUSTER_TRUNCATE_DATABASE:
ticket_type = TicketType.TENDBCLUSTER_FORCE_IMPORT_SQLFILE.value
ticket_type = TicketType.TENDBCLUSTER_DELETE_CLEAR_DB.value
else:
ticket_type = TicketType.MYSQL_FORCE_IMPORT_SQLFILE.value
ticket_type = TicketType.MYSQL_DELETE_CLEAR_DB.value

Ticket.create_ticket(
# 创建删除备份库单据
clear_ticket = Ticket.create_ticket(
ticket_type=ticket_type,
creator=global_data["created_by"],
bk_biz_id=bk_biz_id,
remark=_("清档自动发起的变更SQL单据\n关联单据:{}").format(ticket.url),
remark=_("清档自动发起的清理单据\n关联单据:{}").format(ticket.url),
details=details,
)
# 原单据创建flow,关联清档单据
Flow.objects.create(
ticket=ticket,
flow_type=FlowType.DELIVERY.value,
details={"related_ticket": clear_ticket.id},
flow_alias=_("关联删除清档备份库单据"),
)


class GenerateDropStageDBSqlComponent(Component):
Expand Down
83 changes: 83 additions & 0 deletions dbm-ui/backend/ticket/builders/mysql/mysql_delete_clear_db.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
# -*- coding: utf-8 -*-
"""
TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at https://opensource.org/licenses/MIT
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
specific language governing permissions and limitations under the License.
"""
import logging

from django.utils.translation import ugettext as _

from backend.db_services.mysql.sql_import.constants import SQLExecuteTicketMode
from backend.flow.engine.controller.mysql import MySQLController
from backend.ticket import builders
from backend.ticket.builders.mysql.mysql_force_import_sqlfile import (
MysqlForceSqlImportDetailSerializer,
MysqlForceSqlImportFlowBuilder,
MysqlForceSqlImportFlowParamBuilder,
)
from backend.ticket.constants import FlowRetryType, FlowType, TicketType
from backend.ticket.models import Flow

logger = logging.getLogger("root")


class MysqlDeleteClearDBDetailSerializer(MysqlForceSqlImportDetailSerializer):
def validate(self, attrs):
return attrs


class MysqlDeleteClearDBFlowParamBuilder(MysqlForceSqlImportFlowParamBuilder):
controller = MySQLController.mysql_import_sqlfile_scene

def format_ticket_data(self):
pass


@builders.BuilderFactory.register(TicketType.MYSQL_DELETE_CLEAR_DB, is_sensitive=True)
class MysqlDeleteClearDBFlowBuilder(MysqlForceSqlImportFlowBuilder):
serializer = MysqlDeleteClearDBDetailSerializer
inner_flow_builder = MysqlDeleteClearDBFlowParamBuilder
editable = False

@property
def need_itsm(self):
return False

def init_ticket_flows(self):
"""
sql导入根据执行模式可分为三种执行流程:
手动:手动确认-->(备份)--->sql导入
自动:(备份)--->sql导入
定时:定时触发-->(备份)--->sql导入
"""
flows = []
mode = self.ticket.details["ticket_mode"]["mode"]

if mode == SQLExecuteTicketMode.MANUAL.value:
flows.append(Flow(ticket=self.ticket, flow_type=FlowType.PAUSE.value, flow_alias=_("人工确认执行")))

if mode == SQLExecuteTicketMode.TIMER.value:
flows.append(Flow(ticket=self.ticket, flow_type=FlowType.TIMER.value, flow_alias=_("定时执行")))

flows.append(
Flow(
ticket=self.ticket,
flow_type=FlowType.INNER_FLOW.value,
details=self.inner_flow_builder(self.ticket).get_params(),
retry_type=FlowRetryType.MANUAL_RETRY.value,
flow_alias=_("删除清档备份库"),
)
)

Flow.objects.bulk_create(flows)
return list(Flow.objects.filter(ticket=self.ticket))

@classmethod
def describe_ticket_flows(cls, flow_config_map):
flow_desc = [_("定时执行/人工执行"), _("变更SQL执行")]
return flow_desc
6 changes: 6 additions & 0 deletions dbm-ui/backend/ticket/builders/mysql/mysql_ha_clear.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
from rest_framework import serializers

from backend.db_meta.enums import ClusterType
from backend.db_services.mysql.sql_import.constants import SQLExecuteTicketMode
from backend.flow.consts import TruncateDataTypeEnum
from backend.flow.engine.controller.mysql import MySQLController
from backend.ticket import builders
Expand All @@ -34,7 +35,12 @@ class TruncateDataInfoSerializer(serializers.Serializer):
truncate_data_type = serializers.ChoiceField(help_text=_("清档类型"), choices=TruncateDataTypeEnum.get_choices())
force = serializers.BooleanField(help_text=_("是否强制执行"), default=False)

class ClearImportModeSerializer(serializers.Serializer):
mode = serializers.ChoiceField(help_text=_("单据执行模式"), choices=SQLExecuteTicketMode.get_choices())
days = serializers.IntegerField(help_text=_("执行删除延迟天数"), required=False, default=15)

infos = serializers.ListSerializer(help_text=_("清档信息列表"), child=TruncateDataInfoSerializer())
clear_mode = ClearImportModeSerializer(help_text=_("删除备份库模式"), required=False)

def validate(self, attrs):
"""校验库表选择器信息是否正确"""
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
# -*- coding: utf-8 -*-
"""
TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at https://opensource.org/licenses/MIT
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
specific language governing permissions and limitations under the License.
"""
import logging

from backend.configuration.constants import DBType
from backend.flow.engine.controller.spider import SpiderController
from backend.ticket import builders
from backend.ticket.builders.mysql.mysql_delete_clear_db import (
MysqlDeleteClearDBDetailSerializer,
MysqlDeleteClearDBFlowBuilder,
MysqlDeleteClearDBFlowParamBuilder,
)
from backend.ticket.constants import TicketType

logger = logging.getLogger("root")


class TendbClusterDeleteClearDBDetailSerializer(MysqlDeleteClearDBDetailSerializer):
pass


class TendbClusterDeleteClearDBFlowParamBuilder(MysqlDeleteClearDBFlowParamBuilder):
controller = SpiderController.spider_sql_import_scene


@builders.BuilderFactory.register(TicketType.TENDBCLUSTER_DELETE_CLEAR_DB, is_sensitive=True)
class TendbClusterDeleteClearDBFlowBuilder(MysqlDeleteClearDBFlowBuilder):
group = DBType.TenDBCluster
serializer = TendbClusterDeleteClearDBDetailSerializer
inner_flow_builder = TendbClusterDeleteClearDBFlowParamBuilder
editable = False
4 changes: 3 additions & 1 deletion dbm-ui/backend/ticket/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,7 @@ def get_approve_mode_by_ticket(cls, ticket_type):
MYSQL_HA_APPLY = TicketEnumField("MYSQL_HA_APPLY", _("MySQL 高可用部署"), register_iam=False)
MYSQL_IMPORT_SQLFILE = TicketEnumField("MYSQL_IMPORT_SQLFILE", _("MySQL 变更SQL执行"), _("SQL 任务"))
MYSQL_FORCE_IMPORT_SQLFILE = TicketEnumField("MYSQL_FORCE_IMPORT_SQLFILE", _("MySQL 强制变更SQL执行"), _("SQL 任务"), register_iam=False) # noqa
MYSQL_DELETE_CLEAR_DB = TicketEnumField("MYSQL_DELETE_CLEAR_DB", _("MySQL 删除清档备份库"), _("数据处理"), register_iam=False) # noqa
MYSQL_SEMANTIC_CHECK = TicketEnumField("MYSQL_SEMANTIC_CHECK", _("MySQL 模拟执行"), register_iam=False)
MYSQL_PROXY_ADD = TicketEnumField("MYSQL_PROXY_ADD", _("MySQL 添加Proxy"), _("集群维护"))
MYSQL_PROXY_SWITCH = TicketEnumField("MYSQL_PROXY_SWITCH", _("MySQL 替换Proxy"), _("集群维护"))
Expand Down Expand Up @@ -278,7 +279,8 @@ def get_approve_mode_by_ticket(cls, ticket_type):
TENDBCLUSTER_MASTER_FAIL_OVER = TicketEnumField("TENDBCLUSTER_MASTER_FAIL_OVER", _("TenDB Cluster 主库故障切换"), _("集群维护")) # noqa
TENDBCLUSTER_MASTER_SLAVE_SWITCH = TicketEnumField("TENDBCLUSTER_MASTER_SLAVE_SWITCH", _("TenDB Cluster 主从互切"), _("集群维护")) # noqa
TENDBCLUSTER_IMPORT_SQLFILE = TicketEnumField("TENDBCLUSTER_IMPORT_SQLFILE", _("TenDB Cluster 变更SQL执行"), _("SQL 任务")) # noqa
TENDBCLUSTER_FORCE_IMPORT_SQLFILE = TicketEnumField("TENDBCLUSTER_FORCE_IMPORT_SQLFILE", _("TenDB Cluster 强制变更SQL执行"), register_iam=False) # noqa
TENDBCLUSTER_FORCE_IMPORT_SQLFILE = TicketEnumField("TENDBCLUSTER_FORCE_IMPORT_SQLFILE", _("TenDB Cluster 强制变更SQL执行"), _("SQL 任务"), register_iam=False) # noqa
TENDBCLUSTER_DELETE_CLEAR_DB = TicketEnumField("TENDBCLUSTER_DELETE_CLEAR_DB", _("TenDB Cluster 删除清档备份库"), _("数据处理"), register_iam=False) # noqa
TENDBCLUSTER_SEMANTIC_CHECK = TicketEnumField("TENDBCLUSTER_SEMANTIC_CHECK", _("TenDB Cluster 模拟执行"), register_iam=False) # noqa
TENDBCLUSTER_SPIDER_ADD_NODES = TicketEnumField("TENDBCLUSTER_SPIDER_ADD_NODES", _("TenDB Cluster 扩容接入层"), _("集群维护")) # noqa
TENDBCLUSTER_SPIDER_REDUCE_NODES = TicketEnumField("TENDBCLUSTER_SPIDER_REDUCE_NODES", _("TenDB Cluster 缩容接入层"), _("集群维护")) # noqa
Expand Down

0 comments on commit 02c4fba

Please sign in to comment.