From ec9498ef8ad0ea6b99d7fea8f7a161e9f06295a5 Mon Sep 17 00:00:00 2001 From: iSecloud <869820505@qq.com> Date: Fri, 6 Sep 2024 20:18:23 +0800 Subject: [PATCH 001/107] =?UTF-8?q?feat(backend):=20=E5=8D=95=E6=8D=AE?= =?UTF-8?q?=E7=8A=B6=E6=80=81=E7=BB=86=E5=8C=96=20#6755?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../db_services/bigdata/resources/query.py | 4 +- .../db_services/mysql/dumper/handlers.py | 4 +- .../dbm_init/json_files/itsm/itsm_dbm.json | 10 +- dbm-ui/backend/dbm_init/services.py | 10 +- .../components/collections/common/pause.py | 25 +- .../tests/mock_data/ticket/ticket_flow.py | 13 + .../ticket/{ => doris}/test_doris_flow.py | 6 +- .../ticket/{ => mongo}/test_mongodb_flow.py | 2 +- .../test_mysql_flow.py} | 0 .../{ => sqlserver}/test_sqlserver_flow.py | 0 .../tests/ticket/test_ticket_revoke.py | 75 ++++ .../builders/mysql/mysql_ha_full_backup.py | 67 ++++ dbm-ui/backend/ticket/constants.py | 87 +++-- dbm-ui/backend/ticket/contexts.py | 6 +- dbm-ui/backend/ticket/filters.py | 66 +++- dbm-ui/backend/ticket/flow_manager/base.py | 28 +- .../backend/ticket/flow_manager/delivery.py | 9 +- dbm-ui/backend/ticket/flow_manager/inner.py | 64 +++- dbm-ui/backend/ticket/flow_manager/itsm.py | 49 ++- dbm-ui/backend/ticket/flow_manager/manager.py | 17 +- dbm-ui/backend/ticket/flow_manager/pause.py | 11 +- .../backend/ticket/flow_manager/resource.py | 16 +- dbm-ui/backend/ticket/flow_manager/timer.py | 20 +- dbm-ui/backend/ticket/handler.py | 136 +++++-- dbm-ui/backend/ticket/models/ticket.py | 23 +- dbm-ui/backend/ticket/models/todo.py | 33 +- dbm-ui/backend/ticket/serializers.py | 75 +++- dbm-ui/backend/ticket/tasks/ticket_tasks.py | 3 +- dbm-ui/backend/ticket/todos/__init__.py | 30 +- dbm-ui/backend/ticket/todos/itsm_todo.py | 71 ++++ dbm-ui/backend/ticket/todos/pause_todo.py | 14 +- dbm-ui/backend/ticket/todos/pipeline_todo.py | 27 +- dbm-ui/backend/ticket/views.py | 362 +++++++++--------- 33 files changed, 942 insertions(+), 421 deletions(-) rename dbm-ui/backend/tests/ticket/{ => doris}/test_doris_flow.py (96%) rename dbm-ui/backend/tests/ticket/{ => mongo}/test_mongodb_flow.py (99%) rename dbm-ui/backend/tests/ticket/{test_ticket_flow.py => mysql/test_mysql_flow.py} (100%) rename dbm-ui/backend/tests/ticket/{ => sqlserver}/test_sqlserver_flow.py (100%) create mode 100644 dbm-ui/backend/tests/ticket/test_ticket_revoke.py create mode 100644 dbm-ui/backend/ticket/builders/mysql/mysql_ha_full_backup.py create mode 100644 dbm-ui/backend/ticket/todos/itsm_todo.py diff --git a/dbm-ui/backend/db_services/bigdata/resources/query.py b/dbm-ui/backend/db_services/bigdata/resources/query.py index 552eb9a5d5..f6cbbcea13 100644 --- a/dbm-ui/backend/db_services/bigdata/resources/query.py +++ b/dbm-ui/backend/db_services/bigdata/resources/query.py @@ -21,7 +21,7 @@ from backend.db_proxy.models import ClusterExtension from backend.db_services.dbbase.resources import query from backend.db_services.ipchooser.query.resource import ResourceQueryHelper -from backend.ticket.constants import TicketFlowStatus +from backend.ticket.constants import TICKET_RUNNING_STATUS from backend.ticket.models import InstanceOperateRecord from backend.utils.time import datetime2str @@ -65,7 +65,7 @@ def _filter_instance_hook(cls, bk_biz_id, query_params, instances, **kwargs): # 获取实例的操作与实例记录 records = InstanceOperateRecord.objects.filter( - instance_id__in=instance_ids, ticket__status=TicketFlowStatus.RUNNING + instance_id__in=instance_ids, ticket__status__in=TICKET_RUNNING_STATUS ) instance_operate_records_map: Dict[int, List] = defaultdict(list) for record in records: diff --git a/dbm-ui/backend/db_services/mysql/dumper/handlers.py b/dbm-ui/backend/db_services/mysql/dumper/handlers.py index 53b1cf9b17..06db016c68 100644 --- a/dbm-ui/backend/db_services/mysql/dumper/handlers.py +++ b/dbm-ui/backend/db_services/mysql/dumper/handlers.py @@ -15,7 +15,7 @@ from backend.db_meta.enums import InstanceInnerRole from backend.db_meta.models import Cluster from backend.db_services.mysql.dumper.models import DumperSubscribeConfig -from backend.ticket.constants import FlowType, TicketFlowStatus, TicketStatus, TicketType +from backend.ticket.constants import TICKET_RUNNING_STATUS, FlowType, TicketFlowStatus, TicketStatus, TicketType from backend.ticket.models import Flow, Ticket @@ -66,7 +66,7 @@ def patch_dumper_list_info(cls, dumper_results: List[Dict], bk_biz_id: int = 0, dumper_ticket_types.remove(TicketType.TBINLOGDUMPER_INSTALL) dumper_ticket_types.extend([TicketType.MYSQL_MASTER_SLAVE_SWITCH, TicketType.MYSQL_MASTER_FAIL_OVER]) active_tickets = Ticket.objects.filter( - bk_biz_id=bk_biz_id, status=TicketStatus.RUNNING, ticket_type__in=dumper_ticket_types + bk_biz_id=bk_biz_id, status__in=TICKET_RUNNING_STATUS, ticket_type__in=dumper_ticket_types ) # 获取每个dumper单据状态与id的映射 dumper_inst_id__ticket: Dict[int, str] = {} diff --git a/dbm-ui/backend/dbm_init/json_files/itsm/itsm_dbm.json b/dbm-ui/backend/dbm_init/json_files/itsm/itsm_dbm.json index c2b0a2a2b2..8420c236d1 100644 --- a/dbm-ui/backend/dbm_init/json_files/itsm/itsm_dbm.json +++ b/dbm-ui/backend/dbm_init/json_files/itsm/itsm_dbm.json @@ -1398,5 +1398,13 @@ "display_role": "", "source": "custom", "project_key": "bk_dbm_dev", - "for_update": false + "for_update": true, + "remark_key": { + "0": "2b188068fc0864e15307933a953ed0b3", + "1": "d33b7919a6805e3e6f9162600b451657" + }, + "approve_key": { + "0": "b58ca8d060692fe1fa91a4e9418d545a", + "1": "be937ddce3ec8435c96a8c313bae4836" + } } \ No newline at end of file diff --git a/dbm-ui/backend/dbm_init/services.py b/dbm-ui/backend/dbm_init/services.py index 850f963b74..4cf4a84d9d 100644 --- a/dbm-ui/backend/dbm_init/services.py +++ b/dbm-ui/backend/dbm_init/services.py @@ -81,7 +81,11 @@ def auto_create_itsm_service() -> int: dbm_service_json["project_key"] = project_key dbm_service_json["catalog_id"] = dbm_catalog_id dbm_service_name = dbm_service_json["name"] + for_update = dbm_service_json.pop("for_update", False) + approve_key = dbm_service_json.pop("approve_key") + remark_key = dbm_service_json.pop("remark_key") + dbm_service_id = 0 try: @@ -100,9 +104,9 @@ def auto_create_itsm_service() -> int: # 更新到系统配置中 if dbm_service_id: - SystemSettings.insert_setting_value( - key=SystemSettingsEnum.BK_ITSM_SERVICE_ID.value, value=str(dbm_service_id) - ) + SystemSettings.insert_setting_value(key=SystemSettingsEnum.BK_ITSM_SERVICE_ID, value=str(dbm_service_id)) + SystemSettings.insert_setting_value(key=SystemSettingsEnum.ITSM_APPROVAL_KEY, value=approve_key) + SystemSettings.insert_setting_value(key=SystemSettingsEnum.ITSM_REMARK_KEY, value=remark_key) logger.info("服务创建/更新成功") else: logger.info("本次更新跳过...") diff --git a/dbm-ui/backend/flow/plugins/components/collections/common/pause.py b/dbm-ui/backend/flow/plugins/components/collections/common/pause.py index 55e157e496..82f99d92e1 100644 --- a/dbm-ui/backend/flow/plugins/components/collections/common/pause.py +++ b/dbm-ui/backend/flow/plugins/components/collections/common/pause.py @@ -15,9 +15,8 @@ from pipeline.core.flow.io import ObjectItemSchema, StringItemSchema from backend.flow.plugins.components.collections.common.base_service import BaseService -from backend.ticket.constants import TodoType -from backend.ticket.models import Ticket, Todo -from backend.ticket.todos.pipeline_todo import PipelineTodoContext +from backend.ticket.models import Ticket +from backend.ticket.todos.pipeline_todo import PipelineTodo logger = logging.getLogger("root") @@ -34,26 +33,14 @@ def _execute(self, data, parent_data): self.log_info("execute PauseService") kwargs = data.get_one_of_inputs("kwargs") global_data = data.get_one_of_inputs("global_data") + + # 获取单据和flow信息 ticket_id = global_data["uid"] ticket = Ticket.objects.get(id=ticket_id) - - # todo：这里假设ticket中不会出现并行的flow flow = ticket.current_flow() - Todo.objects.create( - name=_("【{}】流程待确认,是否继续？").format(ticket.get_ticket_type_display()), - flow=flow, - ticket=ticket, - type=TodoType.INNER_APPROVE, - # todo: 待办人暂定为提单人 - operators=[ticket.creator], - context=PipelineTodoContext( - flow.id, - ticket_id, - self.runtime_attrs.get("root_pipeline_id"), - self.runtime_attrs.get("id"), - ).to_dict(), - ) + # 创建一条代办 + PipelineTodo.create(ticket, flow, self.runtime_attrs.get("root_pipeline_id"), self.runtime_attrs.get("id")) self.log_info("pause kwargs: {}".format(kwargs)) return True diff --git a/dbm-ui/backend/tests/mock_data/ticket/ticket_flow.py b/dbm-ui/backend/tests/mock_data/ticket/ticket_flow.py index 81cdc9d8e9..6806df954a 100644 --- a/dbm-ui/backend/tests/mock_data/ticket/ticket_flow.py +++ b/dbm-ui/backend/tests/mock_data/ticket/ticket_flow.py @@ -40,6 +40,19 @@ "ticket_type": "MYSQL_AUTHORIZE_RULES", } +MYSQL_FULL_BACKUP_TICKET_DATA = { + "bk_biz_id": constant.BK_BIZ_ID, + "details": { + "infos": { + "backup_type": "logical", + "file_tag": "DBFILE1M", + "clusters": [{"cluster_id": 1, "backup_local": "master"}], + } + }, + "remark": "", + "ticket_type": "MYSQL_HA_FULL_BACKUP", +} + MYSQL_PERMISSION_ACCOUNT = { "items": [ { diff --git a/dbm-ui/backend/tests/ticket/test_doris_flow.py b/dbm-ui/backend/tests/ticket/doris/test_doris_flow.py similarity index 96% rename from dbm-ui/backend/tests/ticket/test_doris_flow.py rename to dbm-ui/backend/tests/ticket/doris/test_doris_flow.py index 2f5f4a7a09..affbae034d 100644 --- a/dbm-ui/backend/tests/ticket/test_doris_flow.py +++ b/dbm-ui/backend/tests/ticket/doris/test_doris_flow.py @@ -38,14 +38,14 @@ SCALEUP_POOL_TICKET_DATA, ) from backend.tests.ticket.server_base import TestFlowBase -from backend.ticket.constants import TicketFlowStatus, TicketStatus +from backend.ticket.constants import TicketFlowStatus logger = logging.getLogger("test") pytestmark = pytest.mark.django_db client = APIClient() -INITIAL_FLOW_FINISHED_STATUS = [TicketFlowStatus.SKIPPED, TicketStatus.SUCCEEDED] -CHANGED_MOCK_STATUS = [TicketFlowStatus.SKIPPED, TicketStatus.SUCCEEDED, TicketFlowStatus.RUNNING] +INITIAL_FLOW_FINISHED_STATUS = [TicketFlowStatus.SKIPPED, TicketFlowStatus.SUCCEEDED] +CHANGED_MOCK_STATUS = [TicketFlowStatus.SKIPPED, TicketFlowStatus.SUCCEEDED, TicketFlowStatus.RUNNING] @pytest.fixture(autouse=True) # autouse=True 会自动应用这个fixture到所有的测试中 diff --git a/dbm-ui/backend/tests/ticket/test_mongodb_flow.py b/dbm-ui/backend/tests/ticket/mongo/test_mongodb_flow.py similarity index 99% rename from dbm-ui/backend/tests/ticket/test_mongodb_flow.py rename to dbm-ui/backend/tests/ticket/mongo/test_mongodb_flow.py index ea933a0562..6ecb142fec 100644 --- a/dbm-ui/backend/tests/ticket/test_mongodb_flow.py +++ b/dbm-ui/backend/tests/ticket/mongo/test_mongodb_flow.py @@ -53,7 +53,7 @@ pytestmark = pytest.mark.django_db client = APIClient() -INITIAL_FLOW_FINISHED_STATUS = [TicketFlowStatus.SKIPPED, TicketStatus.SUCCEEDED] +INITIAL_FLOW_FINISHED_STATUS = [TicketFlowStatus.SKIPPED, TicketFlowStatus.SUCCEEDED] CHANGED_MOCK_STATUS = [TicketFlowStatus.SKIPPED, TicketStatus.SUCCEEDED, TicketFlowStatus.RUNNING] diff --git a/dbm-ui/backend/tests/ticket/test_ticket_flow.py b/dbm-ui/backend/tests/ticket/mysql/test_mysql_flow.py similarity index 100% rename from dbm-ui/backend/tests/ticket/test_ticket_flow.py rename to dbm-ui/backend/tests/ticket/mysql/test_mysql_flow.py diff --git a/dbm-ui/backend/tests/ticket/test_sqlserver_flow.py b/dbm-ui/backend/tests/ticket/sqlserver/test_sqlserver_flow.py similarity index 100% rename from dbm-ui/backend/tests/ticket/test_sqlserver_flow.py rename to dbm-ui/backend/tests/ticket/sqlserver/test_sqlserver_flow.py diff --git a/dbm-ui/backend/tests/ticket/test_ticket_revoke.py b/dbm-ui/backend/tests/ticket/test_ticket_revoke.py new file mode 100644 index 0000000000..ee10680937 --- /dev/null +++ b/dbm-ui/backend/tests/ticket/test_ticket_revoke.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import copy +import logging +from unittest.mock import PropertyMock, patch + +import pytest +from django.conf import settings +from rest_framework.permissions import AllowAny +from rest_framework.test import APIClient + +from backend.constants import DEFAULT_SYSTEM_USER +from backend.tests.mock_data.components.cc import CCApiMock +from backend.tests.mock_data.components.itsm import ItsmApiMock +from backend.tests.mock_data.iam_app.permission import PermissionMock +from backend.tests.mock_data.ticket.ticket_flow import MYSQL_FULL_BACKUP_TICKET_DATA, SN +from backend.ticket.builders.mysql.mysql_ha_full_backup import MySQLHaFullBackupDetailSerializer +from backend.ticket.constants import TicketStatus, TodoStatus, TodoType +from backend.ticket.flow_manager.inner import InnerFlow +from backend.ticket.handler import TicketHandler +from backend.ticket.models import Flow, Ticket +from backend.ticket.views import TicketViewSet + +logger = logging.getLogger("test") +pytestmark = pytest.mark.django_db +client = APIClient() + + +@pytest.fixture(autouse=True) # autouse=True 会自动应用这个fixture到所有的测试中 +def set_empty_middleware(): + with patch.object(settings, "MIDDLEWARE", []): + yield + + +class TestTicketRevoke: + """ + 测试单据终止 + """ + + @patch.object(TicketViewSet, "permission_classes") + @patch.object(MySQLHaFullBackupDetailSerializer, "validate") + @patch.object(InnerFlow, "status", new_callable=PropertyMock) + @patch.object(TicketViewSet, "get_permissions", lambda x: []) + @patch("backend.ticket.flow_manager.itsm.ItsmApi", ItsmApiMock()) + @patch("backend.db_services.cmdb.biz.CCApi", CCApiMock()) + @patch("backend.db_services.cmdb.biz.Permission", PermissionMock) + def test_ticket_revoke( + self, mocked_status, mocked_validate, mocked_permission_classes, query_fixture, db, init_app + ): + # 以全库备份为例，测试流程：start --> itsm --> inner --> end + mocked_status.return_value = TicketStatus.SUCCEEDED + mocked_permission_classes.return_value = [AllowAny] + mocked_validate.return_value = MYSQL_FULL_BACKUP_TICKET_DATA + + client.login(username="admin") + # 创建单据 + sql_import_data = copy.deepcopy(MYSQL_FULL_BACKUP_TICKET_DATA) + ticket = client.post("/apis/tickets/", data=sql_import_data).data + + # 在todo流程终止 + current_flow = Flow.objects.filter(flow_obj_id=SN).first() + client.post(f"/apis/tickets/{current_flow.ticket_id}/callback/") + TicketHandler.revoke_ticket(ticket_ids=[ticket["id"]], operator=DEFAULT_SYSTEM_USER) + # 验证单据和todo已经终止 + revoke_ticket = Ticket.objects.get(id=ticket["id"]) + assert revoke_ticket.status == TicketStatus.TERMINATED + assert revoke_ticket.todo_of_ticket.filter(type=TodoType.APPROVE)[0].status == TodoStatus.DONE_FAILED diff --git a/dbm-ui/backend/ticket/builders/mysql/mysql_ha_full_backup.py b/dbm-ui/backend/ticket/builders/mysql/mysql_ha_full_backup.py new file mode 100644 index 0000000000..ae152a99dc --- /dev/null +++ b/dbm-ui/backend/ticket/builders/mysql/mysql_ha_full_backup.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from django.utils.translation import ugettext_lazy as _ +from rest_framework import serializers + +from backend.db_meta.enums import ClusterDBHAStatusFlags, InstanceInnerRole +from backend.db_meta.models import Cluster +from backend.flow.consts import MySQLBackupFileTagEnum, MySQLBackupTypeEnum +from backend.flow.engine.controller.mysql import MySQLController +from backend.ticket import builders +from backend.ticket.builders.common.base import fetch_cluster_ids +from backend.ticket.builders.mysql.base import BaseMySQLHATicketFlowBuilder, MySQLBaseOperateDetailSerializer +from backend.ticket.constants import FlowRetryType, TicketType + + +class MySQLHaFullBackupDetailSerializer(MySQLBaseOperateDetailSerializer): + class FullBackupDataInfoSerializer(serializers.Serializer): + class ClusterDetailSerializer(serializers.Serializer): + cluster_id = serializers.IntegerField(help_text=_("集群ID")) + backup_local = serializers.ChoiceField( + help_text=_("备份位置"), choices=InstanceInnerRole.get_choices(), default=InstanceInnerRole.SLAVE.value + ) + + # 废弃online，暂时不需要传递 + # online = serializers.BooleanField(help_text=_("是否在线备份"), required=False) + backup_type = serializers.ChoiceField(help_text=_("备份类型"), choices=MySQLBackupTypeEnum.get_choices()) + file_tag = serializers.ChoiceField(help_text=_("备份文件tag"), choices=MySQLBackupFileTagEnum.get_choices()) + clusters = serializers.ListSerializer(help_text=_("集群信息"), child=ClusterDetailSerializer()) + + infos = FullBackupDataInfoSerializer() + + def validate(self, attrs): + try: + self.validate_cluster_can_access(attrs) + except serializers.ValidationError as e: + clusters = Cluster.objects.filter(id__in=fetch_cluster_ids(details=attrs)) + id__cluster = {cluster.id: cluster for cluster in clusters} + # 如果备份位置选的是master，但是slave异常，则认为是可以的 + for info in attrs["infos"]["clusters"]: + if info["backup_local"] != InstanceInnerRole.MASTER: + raise serializers.ValidationError(e) + if id__cluster[info["cluster_id"]].status_flag & ClusterDBHAStatusFlags.BackendMasterUnavailable: + raise serializers.ValidationError(e) + + return attrs + + +class MySQLHaFullBackupFlowParamBuilder(builders.FlowParamBuilder): + """MySQL HA 备份执行单据参数""" + + controller = MySQLController.mysql_full_backup_scene + + +@builders.BuilderFactory.register(TicketType.MYSQL_HA_FULL_BACKUP) +class MySQLHaFullBackupFlowBuilder(BaseMySQLHATicketFlowBuilder): + serializer = MySQLHaFullBackupDetailSerializer + inner_flow_builder = MySQLHaFullBackupFlowParamBuilder + retry_type = FlowRetryType.AUTO_RETRY diff --git a/dbm-ui/backend/ticket/constants.py b/dbm-ui/backend/ticket/constants.py index 95acc6a541..407c632c9e 100644 --- a/dbm-ui/backend/ticket/constants.py +++ b/dbm-ui/backend/ticket/constants.py @@ -12,7 +12,7 @@ from django.utils.translation import ugettext_lazy as _ -from backend.configuration.constants import DBType, SystemSettingsEnum +from backend.configuration.constants import DBType from backend.db_meta.exceptions import ClusterExclusiveOperateException from backend.flow.consts import StateType from backend.ticket.exceptions import TicketBaseException @@ -29,7 +29,9 @@ class TodoType(str, StructuredEnum): 待办类型 """ + ITSM = EnumField("ITSM", _("主流程-单据审批")) APPROVE = EnumField("APPROVE", _("主流程-人工确认")) + INNER_FAILED = EnumField("INNER_FAILED", _("主流程-失败后待确认")) INNER_APPROVE = EnumField("INNER_APPROVE", _("自动化流程-人工确认")) RESOURCE_REPLENISH = EnumField("RESOURCE_REPLENISH", _("资源补货")) @@ -39,19 +41,24 @@ class CountType(str, StructuredEnum): 单据计数类型 """ - MY_TODO = EnumField("MY_TODO", _("我的待办")) MY_APPROVE = EnumField("MY_APPROVE", _("我的申请")) + APPROVE = EnumField("APPROVE", _("待我审批")) + TODO = EnumField("TODO", _("待我确认执行")) + INNER_TODO = EnumField("INNER_TODO", _("待我继续")) + RESOURCE_REPLENISH = EnumField("RESOURCE_REPLENISH", _("待我补货")) + FAILED = EnumField("FAILED", _("失败待处理")) + DONE = EnumField("DONE", _("我的已办")) + SELF_MANAGE = EnumField("SELF_MANAGE", _("我负责的业务")) class TodoStatus(str, StructuredEnum): """ 待办状态枚举 - TODO -> (RUNNING，可选) -> DONE_SUCCESS - | -> DONE_FAILED + TODO -> (RUNNING，可选) -> DONE_SUCCESS + | -> DONE_FAILED """ TODO = EnumField("TODO", _("待处理")) - RUNNING = EnumField("RUNNING", _("处理中")) DONE_SUCCESS = EnumField("DONE_SUCCESS", _("已处理")) DONE_FAILED = EnumField("DONE_FAILED", _("已终止")) @@ -68,18 +75,44 @@ class ResourceApplyErrCode(int, StructuredEnum): TODO_DONE_STATUS = [TodoStatus.DONE_SUCCESS, TodoStatus.DONE_FAILED] -TODO_RUNNING_STATUS = [TodoStatus.TODO, TodoStatus.RUNNING] +TODO_RUNNING_STATUS = [TodoStatus.TODO] class TicketStatus(str, StructuredEnum): """单据状态枚举""" PENDING = EnumField("PENDING", _("等待中")) + APPROVE = EnumField("APPROVE", _("待审批")) + RESOURCE_REPLENISH = EnumField("RESOURCE_REPLENISH", _("待补货")) + TODO = EnumField("TODO", _("待执行")) + TIMER = EnumField("TIMER", _("定时中")) RUNNING = EnumField("RUNNING", _("执行中")) - SUCCEEDED = EnumField("SUCCEEDED", _("成功")) + SUCCEEDED = EnumField("SUCCEEDED", _("已完成")) FAILED = EnumField("FAILED", _("失败")) - REVOKED = EnumField("REVOKED", _("撤销")) - TERMINATED = EnumField("TERMINATED", _("终止")) + REVOKED = EnumField("REVOKED", _("已撤销")) + TERMINATED = EnumField("TERMINATED", _("已终止")) + # 仅展示，不参与状态流转，不落地db + INNER_TODO = EnumField("INNER_TODO", _("待继续")) + + +# 单据[正在进行]的状态合集 +TICKET_RUNNING_STATUS = [ + TicketStatus.APPROVE, + TicketStatus.TODO, + TicketStatus.RESOURCE_REPLENISH, + TicketStatus.RUNNING, + TicketStatus.TIMER, +] +# 单据[包含TODO]的状态合集 +TICKET_TODO_STATUS = [ + TicketStatus.APPROVE, + TicketStatus.TODO, + TicketStatus.RESOURCE_REPLENISH, + TicketStatus.FAILED, + TicketStatus.RUNNING, +] +# 单据[失败]的状态合集 +TICKET_FAILED_STATUS = [TicketStatus.REVOKED, TicketStatus.TERMINATED, TicketStatus.FAILED] class TicketFlowStatus(str, StructuredEnum): @@ -94,8 +127,8 @@ class TicketFlowStatus(str, StructuredEnum): SKIPPED = EnumField("SKIPPED", _("跳过")) -FLOW_FINISHED_STATUS = [TicketFlowStatus.SKIPPED, TicketStatus.SUCCEEDED] -FLOW_NOT_EXECUTE_STATUS = [TicketFlowStatus.SKIPPED, TicketStatus.PENDING] +FLOW_FINISHED_STATUS = [TicketFlowStatus.SKIPPED, TicketFlowStatus.SUCCEEDED] +FLOW_NOT_EXECUTE_STATUS = [TicketFlowStatus.SKIPPED, TicketFlowStatus.PENDING] BAMBOO_STATE__TICKET_STATE_MAP = { StateType.FINISHED.value: TicketFlowStatus.SUCCEEDED.value, @@ -167,6 +200,12 @@ def get_cluster_type_by_ticket(cls, ticket_type): raise TicketBaseException(_("无法找到{}关联的集群类型").format(ticket_type)) return builder.cluster_types + @classmethod + def get_approve_mode_by_ticket(cls, ticket_type): + if ticket_type in [cls.MYSQL_ACCOUNT_RULE_CHANGE, cls.TENDBCLUSTER_ACCOUNT_RULE_CHANGE]: + return ItsmApproveMode.CounterSign.value + return ItsmApproveMode.OrSign.value + # fmt: off # MYSQL MYSQL_SINGLE_APPLY = TicketEnumField("MYSQL_SINGLE_APPLY", _("MySQL 单节点部署"), register_iam=False) @@ -216,7 +255,7 @@ def get_cluster_type_by_ticket(cls, ticket_type): MYSQL_SLAVE_MIGRATE_UPGRADE = TicketEnumField("MYSQL_SLAVE_MIGRATE_UPGRADE", _("MySQL Slave 迁移升级"), _("版本升级")) MYSQL_RO_SLAVE_UNINSTALL = TicketEnumField("MYSQL_RO_SLAVE_UNINSTALL", _("MySQL非standby slave下架"), _("集群维护")) MYSQL_PROXY_UPGRADE = TicketEnumField("MYSQL_PROXY_UPGRADE", _("MySQL Proxy升级"), _("版本升级")) - MYSQL_HA_TRANSFER_TO_OTHER_BIZ = TicketEnumField("MYSQL_HA_TRANSFER_TO_OTHER_BIZ", _("TendbHA集群迁移至其他业务"), register_iam=False) # noqa + MYSQL_HA_TRANSFER_TO_OTHER_BIZ = TicketEnumField("MYSQL_HA_TRANSFER_TO_OTHER_BIZ", _("TendbHA集群迁移至其他业务"), register_iam=False) # noqa MYSQL_PUSH_PERIPHERAL_CONFIG = TicketEnumField("MYSQL_PUSH_PERIPHERAL_CONFIG", _("推送周边配置"), register_iam=False) MYSQL_ACCOUNT_RULE_CHANGE = TicketEnumField("MYSQL_ACCOUNT_RULE_CHANGE", _("MySQL 授权规则变更"), register_iam=False) @@ -432,7 +471,7 @@ def get_cluster_type_by_ticket(cls, ticket_type): MONGODB_DESTROY = TicketEnumField("MONGODB_DESTROY", _("MongoDB 集群删除"), _("集群管理")) MONGODB_CUTOFF = TicketEnumField("MONGODB_CUTOFF", _("MongoDB 整机替换"), _("集群维护")) MONGODB_AUTHORIZE_RULES = TicketEnumField("MONGODB_AUTHORIZE_RULES", _("MongoDB 授权"), _("权限管理")) - MONGODB_EXCEL_AUTHORIZE_RULES = TicketEnumField("MONGODB_EXCEL_AUTHORIZE_RULES", _("MongoDB Excel授权"), _("权限管理")) # noqa + MONGODB_EXCEL_AUTHORIZE_RULES = TicketEnumField("MONGODB_EXCEL_AUTHORIZE_RULES", _("MongoDB Excel授权"), _("权限管理")) # noqa MONGODB_IMPORT = TicketEnumField("MONGODB_IMPORT", _("MongoDB 数据导入"), _("集群维护")) MONGODB_RESTORE = TicketEnumField("MONGODB_RESTORE", _("MongoDB 定点回档"), _("集群维护")) MONGODB_TEMPORARY_DESTROY = TicketEnumField("MONGODB_TEMPORARY_DESTROY", _("MongoDB 临时集群销毁"), _("集群维护")) @@ -656,11 +695,7 @@ class OperateNodeActionType(str, StructuredEnum): DISTRIBUTE = EnumField("DISTRIBUTE", _("派单")) DELIVER = EnumField("DELIVER", _("转单")) TERMINATE = EnumField("TERMINATE", _("终止节点和单据")) - - -class ItsmTicketNodeEnum(str, StructuredEnum): - ApprovalOption = EnumField("审批意见", "审批意见") - Remark = EnumField("备注", "备注") + WITHDRAW = EnumField("WITHDRAW", _("撤销单据")) class ItsmApproveMode(int, StructuredEnum): @@ -668,12 +703,6 @@ class ItsmApproveMode(int, StructuredEnum): CounterSign = EnumField(1, _("会签模式")) -ITSM_FIELD_NAME__ITSM_KEY = { - ItsmTicketNodeEnum.ApprovalOption.value: SystemSettingsEnum.ITSM_APPROVAL_KEY, - ItsmTicketNodeEnum.Remark.value: SystemSettingsEnum.ITSM_REMARK_KEY, -} - - class FlowMsgType(str, StructuredEnum): DONE = EnumField(_("完成"), _("完成")) TODO = EnumField(_("待办"), _("待办")) @@ -707,3 +736,13 @@ class TicketExpireType(str, StructuredEnum): FlowType.RESOURCE_APPLY: TicketExpireType.FLOW_TODO, FlowType.RESOURCE_BATCH_APPLY: TicketExpireType.FLOW_TODO, } + +# 根据流程类型来映射单据状态 +RUNNING_FLOW__TICKET_STATUS = { + FlowType.BK_ITSM: TicketStatus.APPROVE, + FlowType.RESOURCE_APPLY: TicketStatus.RESOURCE_REPLENISH, + FlowType.RESOURCE_BATCH_APPLY: TicketStatus.RESOURCE_REPLENISH, + FlowType.PAUSE: TicketStatus.TODO, + FlowType.INNER_FLOW: TicketStatus.RUNNING, + FlowType.TIMER: TicketStatus.TIMER, +} diff --git a/dbm-ui/backend/ticket/contexts.py b/dbm-ui/backend/ticket/contexts.py index 391170d660..d0f78b3f70 100644 --- a/dbm-ui/backend/ticket/contexts.py +++ b/dbm-ui/backend/ticket/contexts.py @@ -22,9 +22,9 @@ def __init__( self.spec_map = get_spec_display_map() self.db_config = {} - bizs = list(AppCache.objects.all()) - self.biz_name_map = {biz.bk_biz_id: biz.bk_biz_name for biz in bizs} - self.app_abbr_map = {biz.bk_biz_id: biz.db_app_abbr for biz in bizs} + bizs = AppCache.get_appcache(key="appcache_dict") + self.biz_name_map = {int(bk_biz_id): biz["bk_biz_name"] for bk_biz_id, biz in bizs.items()} + self.app_abbr_map = {int(bk_biz_id): biz["db_app_abbr"] for bk_biz_id, biz in bizs.items()} db_modules = list(DBModule.objects.all()) self.db_module_map = {module.db_module_id: module.alias_name for module in db_modules} diff --git a/dbm-ui/backend/ticket/filters.py b/dbm-ui/backend/ticket/filters.py index b0961795de..7f783e2ca5 100644 --- a/dbm-ui/backend/ticket/filters.py +++ b/dbm-ui/backend/ticket/filters.py @@ -8,18 +8,22 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ - +from django.db.models import Q from django.utils.translation import ugettext_lazy as _ from django_filters import rest_framework as filters from backend.db_meta.models import Cluster -from backend.ticket.models import ClusterOperateRecord, Ticket +from backend.ticket.constants import TODO_RUNNING_STATUS, TicketStatus +from backend.ticket.models import ClusterOperateRecord, InstanceOperateRecord, Ticket class TicketListFilter(filters.FilterSet): + ids = filters.CharFilter(field_name="ids", method="filter_ids", label=_("单据ID列表")) remark = filters.CharFilter(field_name="remark", lookup_expr="icontains", label=_("备注")) + status = filters.CharFilter(field_name="status", method="filter_status", label=_("单据状态")) cluster = filters.CharFilter(field_name="cluster", method="filter_cluster", label=_("集群域名")) - ids = filters.CharFilter(field_name="ids", method="filter_ids", label=_("单据ID列表")) + todo = filters.CharFilter(field_name="todo", method="filter_todo", label=_("代办状态")) + ordering = filters.CharFilter(field_name="ordering", method="order_ticket", label=_("排序字段")) class Meta: model = Ticket @@ -27,7 +31,6 @@ class Meta: "id": ["exact", "in"], "bk_biz_id": ["exact"], "ticket_type": ["exact", "in"], - "status": ["exact", "in"], "create_at": ["gte", "lte"], "creator": ["exact"], } @@ -40,3 +43,58 @@ def filter_cluster(self, queryset, name, value): def filter_ids(self, queryset, name, value): ids = list(map(int, value.split(","))) return queryset.filter(id__in=ids) + + def filter_todo(self, queryset, name, value): + user = self.request.user.username + if value == "running": + todo_filter = Q(todo_of_ticket__operators__contains=user, todo_of_ticket__status__in=TODO_RUNNING_STATUS) + else: + todo_filter = Q(todo_of_ticket__done_by=user) + return queryset.filter(todo_filter).distinct() + + def filter_status(self, queryset, name, value): + status = value.split(",") + status_filter = Q() + # 如果有待确认，则解析为：running + 包含正在运行的todo + if TicketStatus.INNER_TODO in status: + status_filter |= Q(status=TicketStatus.RUNNING, todo_of_ticket__status__in=TODO_RUNNING_STATUS) + status.remove(TicketStatus.INNER_TODO.value) + # 如果有待执行，则解析为：running + 不包含正在运行的todo + if TicketStatus.RUNNING in status: + status_filter |= Q(status=TicketStatus.RUNNING) & ~Q(todo_of_ticket__status__in=TODO_RUNNING_STATUS) + status.remove(TicketStatus.RUNNING.value) + # 其他状态，直接in即可 + status_filter |= Q(status__in=status) + return queryset.filter(status_filter).distinct() + + def order_ticket(self, queryset, name, value): + return queryset.order_by(value) + + +class OpRecordListFilter(filters.FilterSet): + start_time = filters.DateTimeFilter(field_name="create_at", lookup_expr="gte", label=_("开始时间")) + end_time = filters.DateTimeFilter(field_name="create_at", lookup_expr="lte", label=_("开始时间")) + op_type = filters.CharFilter(field_name="op_type", method="filter_op_type", label=_("操作类型")) + op_status = filters.CharFilter(field_name="op_status", method="filter_op_status", label=_("操作状态")) + + def filter_op_type(self, queryset, name, value): + return queryset.filter(ticket__ticket_type=value) + + def filter_op_status(self, queryset, name, value): + return queryset.filter(ticket__status=value) + + +class ClusterOpRecordListFilter(OpRecordListFilter): + cluster_id = filters.NumberFilter(field_name="cluster_id", lookup_expr="exact", label=_("集群ID")) + + class Meta: + model = ClusterOperateRecord + fields = ["start_time", "end_time", "op_type", "op_status"] + + +class InstanceOpRecordListFilter(OpRecordListFilter): + instance_id = filters.NumberFilter(field_name="instance_id", lookup_expr="exact", label=_("实例ID")) + + class Meta: + model = InstanceOperateRecord + fields = ["start_time", "end_time", "op_type", "op_status"] diff --git a/dbm-ui/backend/ticket/flow_manager/base.py b/dbm-ui/backend/ticket/flow_manager/base.py index 15ce0f58ff..1dc06f2e1f 100644 --- a/dbm-ui/backend/ticket/flow_manager/base.py +++ b/dbm-ui/backend/ticket/flow_manager/base.py @@ -25,12 +25,14 @@ FLOW_FINISHED_STATUS, FLOW_NOT_EXECUTE_STATUS, FLOW_TYPE__EXPIRE_TYPE_CONFIG, + TICKET_EXPIRE_DEFAULT_CONFIG, FlowContext, FlowErrCode, + FlowType, FlowTypeConfig, TicketFlowStatus, ) -from backend.ticket.models import ClusterOperateRecord, Flow, InstanceOperateRecord, TicketFlowsConfig +from backend.ticket.models import ClusterOperateRecord, Flow, InstanceOperateRecord, TicketFlowsConfig, Todo logger = logging.getLogger("root") @@ -55,10 +57,9 @@ def status(self) -> str: if self.flow_obj.err_msg: # 如果flow的状态包含错误信息，则是saas侧出错，当前的flow流程直接返回失败 # 注意：这里不能直接根据flow的状态为失败就进行返回，有可能是pipeline跳过失败的操作 - - # 如果是自动重试互斥错误，则返回RUNNING状态 - if self.flow_obj.err_code == FlowErrCode.AUTO_EXCLUSIVE_ERROR: - return TicketFlowStatus.RUNNING + # 如果是inner_flow，则进入子status处理 + if self.flow_obj.flow_type == FlowType.INNER_FLOW: + return self._status return TicketFlowStatus.FAILED @@ -70,7 +71,7 @@ def status(self) -> str: if not self.flow_obj.flow_obj_id: # 任务流程未创建时未PENDING状态 - return constants.TicketStatus.PENDING + return constants.TicketFlowStatus.PENDING # 其他情况暂时认为在PENDING状态 return TicketFlowStatus.PENDING @@ -144,10 +145,11 @@ def flush_error_status_handler(self): def flush_revoke_status_handler(self, operator): """终止节点，更新相关状态和错误信息""" self.flow_obj.status = TicketFlowStatus.TERMINATED - self.flow_obj.err_code = FlowErrCode.GENERAL_ERROR if operator == DEFAULT_SYSTEM_USER: self.flow_obj.err_code = FlowErrCode.SYSTEM_TERMINATED_ERROR self.flow_obj.context = {FlowContext.EXPIRE_TIME: self.get_current_config_expire_time()} + else: + self.flow_obj.err_code = FlowErrCode.GENERAL_ERROR self.flow_obj.save(update_fields=["status", "err_code", "context", "update_at"]) # 更新操作者 self.ticket.updater = operator @@ -157,8 +159,9 @@ def get_current_config_expire_time(self): """获取当前配置的flow过期时间""" if self.flow_obj.flow_type not in FLOW_TYPE__EXPIRE_TYPE_CONFIG: return -1 - config = TicketFlowsConfig.get_config(ticket_type=self.ticket.ticket_type) - expire_time = config[FlowTypeConfig.EXPIRE_CONFIG][FLOW_TYPE__EXPIRE_TYPE_CONFIG[self.flow_obj.flow_type]] + config = TicketFlowsConfig.get_config(ticket_type=self.ticket.ticket_type).configs + expire_config = config.get(FlowTypeConfig.EXPIRE_CONFIG, TICKET_EXPIRE_DEFAULT_CONFIG) + expire_time = expire_config[FLOW_TYPE__EXPIRE_TYPE_CONFIG[self.flow_obj.flow_type]] return expire_time def create_operate_records(self, object_key, record_model, object_ids): @@ -259,4 +262,11 @@ def _retry(self) -> Any: self.run() def _revoke(self, operator) -> Any: + # 停止相关联的todo + from backend.ticket.todos import ActionType, TodoActorFactory + + todos = Todo.objects.filter(ticket=self.ticket, flow=self.flow_obj) + for todo in todos: + TodoActorFactory.actor(todo).process(operator, ActionType.TERMINATE, params={}) + # 刷新flow和单据状态 --> 终止 self.flush_revoke_status_handler(operator) diff --git a/dbm-ui/backend/ticket/flow_manager/delivery.py b/dbm-ui/backend/ticket/flow_manager/delivery.py index f5ac54089e..c07a453210 100644 --- a/dbm-ui/backend/ticket/flow_manager/delivery.py +++ b/dbm-ui/backend/ticket/flow_manager/delivery.py @@ -84,13 +84,14 @@ def _end_time(self) -> Optional[str]: return datetime2str(self.pre_flow_tree.updated_at) @property - def _summary(self) -> str: + def _summary(self) -> dict: if self.pre_flow_tree.status in FAILED_STATES: - return _("失败后继续提交") + return {"status": TicketFlowStatus.FAILED, "message": _("失败后继续提交")} elif self.pre_flow_tree.status in SUCCEED_STATES: - return _("执行成功") + return {"status": TicketFlowStatus.SUCCEEDED, "message": _("执行成功")} else: - return _("执行{}".format(StateType.get_choice_label(self.pre_flow_tree.status))) + state = StateType.get_choice_label(self.pre_flow_tree.status) + return {"status": TicketFlowStatus.RUNNING, "message": _("执行{}".format(state))} @property def _url(self) -> str: diff --git a/dbm-ui/backend/ticket/flow_manager/inner.py b/dbm-ui/backend/ticket/flow_manager/inner.py index 1b9847232b..b607e0009c 100644 --- a/dbm-ui/backend/ticket/flow_manager/inner.py +++ b/dbm-ui/backend/ticket/flow_manager/inner.py @@ -28,11 +28,15 @@ BAMBOO_STATE__TICKET_STATE_MAP, FlowCallbackType, FlowMsgType, + FlowErrCode, TicketFlowStatus, TicketType, + TodoStatus, + TodoType, ) from backend.ticket.flow_manager.base import BaseTicketFlow -from backend.ticket.models import Flow +from backend.ticket.models import Flow, Todo +from backend.ticket.todos import BaseTodoContext from backend.utils.basic import generate_root_id from backend.utils.time import datetime2str @@ -88,22 +92,47 @@ def _end_time(self) -> Union[str, datetime]: @property def _summary(self) -> str: # TODO 可以给出具体失败的节点和原因 - return _("任务{status_display}").format(status_display=constants.TicketStatus.get_choice_label(self.status)) + return _("任务{status_display}").format(status_display=constants.TicketFlowStatus.get_choice_label(self.status)) @property def _status(self) -> str: - # 如果未找到流程树，则直接取flow_obj的status + # 如果是自动重试互斥错误，则返回RUNNING状态 + if self.flow_obj.err_msg and self.flow_obj.err_code == FlowErrCode.AUTO_EXCLUSIVE_ERROR: + return TicketFlowStatus.RUNNING + + # 查询流程树状态，如果未找到则直接取flow_obj的status if not self.flow_tree: - return self.flow_obj.status + status = self.flow_obj.status + else: + status = BAMBOO_STATE__TICKET_STATE_MAP.get(self.flow_tree.status, constants.TicketFlowStatus.RUNNING) - status = BAMBOO_STATE__TICKET_STATE_MAP.get(self.flow_tree.status, constants.TicketStatus.RUNNING) - self.flow_obj.update_status(status) - return status + todo_status = TodoStatus.TODO if status == TicketFlowStatus.FAILED else TodoStatus.DONE_SUCCESS + fail_todo = self.flow_obj.todo_of_flow.filter(type=TodoType.INNER_FAILED).first() + # 如果任务失败，且不存在todo，则创建一条 + if not fail_todo and todo_status == TodoStatus.TODO: + self.create_failed_todo() + # 变更todo状态 + if fail_todo and fail_todo.status != todo_status: + fail_todo.set_status(self.ticket.creator, todo_status) + + return self.flow_obj.update_status(status) @property def _url(self) -> str: return f"{env.BK_SAAS_HOST}/{self.ticket.bk_biz_id}/task-history/detail/{self.root_id}" + def create_failed_todo(self): + # 创建一条todo失败记录，在失败时变更为TODO状态 + Todo.objects.create( + name=_("【{}】单据任务执行失败，待处理").format(self.ticket.get_ticket_type_display()), + flow=self.flow_obj, + ticket=self.ticket, + type=TodoType.INNER_FAILED, + operators=[self.ticket.creator], + context=BaseTodoContext(self.flow_obj.id, self.ticket.id).to_dict(), + status=TodoStatus.DONE_SUCCESS, + ) + def check_exclusive_operations(self): """判断执行互斥""" # TODO: 目前来说，执行互斥对于同时提单或者同时重试的操作是防不住的。 @@ -128,10 +157,6 @@ def check_exclusive_operations(self): cluster_ids=cluster_ids, ticket_type=ticket_type, exclude_ticket_ids=[self.ticket.id] ) - def handle_exclusive_error(self): - """处理执行互斥后重试的逻辑""" - pass - def callback(self, callback_type: FlowCallbackType) -> None: """ inner节点独有的钩子函数，执行前置/后继流程节点动作 @@ -202,6 +227,15 @@ def _retry(self) -> Any: ) super()._retry() + def _revoke(self, operator) -> Any: + # 终止运行的pipeline + from backend.db_services.taskflow.handlers import TaskFlowHandler + + if FlowTree.objects.filter(root_id=self.flow_obj.flow_obj_id).exists(): + TaskFlowHandler(self.flow_obj.flow_obj_id).revoke_pipeline() + # 流转flow的终止状态 + super()._revoke(operator) + class QuickInnerFlow(InnerFlow): """ @@ -211,7 +245,7 @@ class QuickInnerFlow(InnerFlow): @property def _status(self) -> str: - return constants.TicketStatus.SUCCEEDED + return constants.TicketFlowStatus.SUCCEEDED @property def _summary(self) -> str: @@ -236,7 +270,7 @@ class IgnoreResultInnerFlow(InnerFlow): @property def _summary(self) -> str: return _("(执行结果可忽略)任务状态: {status_display}").format( - status_display=constants.TicketStatus.get_choice_label(self._raw_status) + status_display=constants.TicketFlowStatus.get_choice_label(self._raw_status) ) @property @@ -246,7 +280,7 @@ def _raw_status(self) -> str: @property def _status(self) -> str: status = self._raw_status - if status in [constants.TicketStatus.SUCCEEDED, constants.TicketStatus.REVOKED, constants.TicketStatus.FAILED]: - return constants.TicketStatus.SUCCEEDED + if status in [constants.TicketFlowStatus.SUCCEEDED, *constants.TICKET_FAILED_STATUS]: + return constants.TicketFlowStatus.SUCCEEDED return status diff --git a/dbm-ui/backend/ticket/flow_manager/itsm.py b/dbm-ui/backend/ticket/flow_manager/itsm.py index ff7d0d397a..d96c4b3f8f 100644 --- a/dbm-ui/backend/ticket/flow_manager/itsm.py +++ b/dbm-ui/backend/ticket/flow_manager/itsm.py @@ -16,10 +16,11 @@ from backend.components import ItsmApi from backend.components.itsm.constants import ItsmTicketStatus from backend.exceptions import ApiResultError -from backend.ticket.constants import FlowMsgStatus, FlowMsgType, TicketFlowStatus, TicketStatus +from backend.ticket.constants import FlowMsgStatus, FlowMsgType, TicketFlowStatus, TicketStatus, TodoStatus, TodoType from backend.ticket.flow_manager.base import BaseTicketFlow -from backend.ticket.models import Flow +from backend.ticket.models import Flow, Todo from backend.ticket.tasks.ticket_tasks import send_msg_for_flow +from backend.ticket.todos.itsm_todo import ItsmTodoContext from backend.utils.time import datetime2str, standardized_time_str @@ -56,44 +57,51 @@ def _end_time(self) -> Union[datetime, Any]: return self.flow_obj.update_at @property - def _summary(self) -> str: + def _summary(self) -> dict: try: logs = ItsmApi.get_ticket_logs({"sn": [self.flow_obj.flow_obj_id]}) except ApiResultError: return _("未知单据") + + # 获取单据审批状态 + current_status = self.ticket_approval_result["current_status"] + approve_result = self.ticket_approval_result["approve_result"] + summary = {"status": current_status, "approve_result": approve_result} + # 目前审批流程是固定的，取流程中第三个节点的日志作为概览即可 try: - return logs["logs"][2]["message"] + summary.update(operator=logs["logs"][2]["operator"], message=logs["logs"][2]["message"]) except (IndexError, KeyError): # 异常时根据状态取默认的概览 - status_summary_map = { - TicketStatus.RUNNING.value: _("审批中"), - TicketStatus.SUCCEEDED.value: _("已通过"), - TicketStatus.REVOKED.value: _("已撤销"), - TicketStatus.FAILED.value: _("被拒绝"), - TicketStatus.TERMINATED.value: _("已终止"), - } - return status_summary_map.get(self.status, "") + msg = TicketStatus.get_choice_label(self.status) + summary.update(operator=logs["logs"][-1]["operator"], status=self.status, message=msg) + return summary @property def _status(self) -> str: # 把 ITSM 单据状态映射为本系统内的单据状态 current_status = self.ticket_approval_result["current_status"] approve_result = self.ticket_approval_result["approve_result"] + updater = self.ticket_approval_result["updated_by"] + todo = self.flow_obj.todo_of_flow.first() # 进行中 if current_status == ItsmTicketStatus.RUNNING: return self.flow_obj.update_status(TicketFlowStatus.RUNNING) # 撤单 elif current_status == ItsmTicketStatus.REVOKED: + todo.set_status(username=updater, status=TodoStatus.DONE_FAILED) return self.flow_obj.update_status(TicketFlowStatus.TERMINATED) # 审批通过 elif current_status == ItsmTicketStatus.FINISHED and approve_result: + todo.set_status(username=updater, status=TodoStatus.DONE_SUCCESS) return self.flow_obj.update_status(TicketFlowStatus.SUCCEEDED) # 审批拒绝 elif current_status == ItsmTicketStatus.FINISHED and not approve_result: + todo.set_status(username=updater, status=TodoStatus.DONE_FAILED) return self.flow_obj.update_status(TicketFlowStatus.TERMINATED) # 终止 elif current_status == ItsmTicketStatus.TERMINATED: + todo.set_status(username=updater, status=TodoStatus.DONE_FAILED) return self.flow_obj.update_status(TicketFlowStatus.TERMINATED) @property @@ -104,9 +112,20 @@ def _url(self) -> str: return "" def _run(self) -> str: + itsm_fields = {f["key"]: f["value"] for f in self.flow_obj.details["fields"]} + # 创建审批todo + operators = itsm_fields["approver"].split(",") + Todo.objects.create( + name=_("【{}】单据等待审批").format(self.ticket.get_ticket_type_display()), + flow=self.flow_obj, + ticket=self.ticket, + type=TodoType.ITSM, + operators=operators, + context=ItsmTodoContext(self.flow_obj.id, self.ticket.id).to_dict(), + ) + # 创建单据 data = ItsmApi.create_ticket(self.flow_obj.details) # 异步发送待审批消息 - itsm_fields = {f["key"]: f["value"] for f in self.flow_obj.details["fields"]} send_msg_for_flow.apply_async( kwargs={ "flow_id": self.flow_obj.id, @@ -117,3 +136,7 @@ def _run(self) -> str: } ) return data["sn"] + + def _revoke(self, operator) -> Any: + # 父类通过触发todo的终止可以终止itsm单据 + super()._revoke(operator) diff --git a/dbm-ui/backend/ticket/flow_manager/manager.py b/dbm-ui/backend/ticket/flow_manager/manager.py index c12118d2f5..8ac86ace95 100644 --- a/dbm-ui/backend/ticket/flow_manager/manager.py +++ b/dbm-ui/backend/ticket/flow_manager/manager.py @@ -89,23 +89,26 @@ def run_next_flow(self): def update_ticket_status(self): # 获取流程状态集合 - statuses = { - self.get_ticket_flow_cls(flow_type=flow.flow_type)(flow).status for flow in self.ticket.flows.all() + flow_status_map = { + self.get_ticket_flow_cls(flow_type=flow.flow_type)(flow).status: flow for flow in self.ticket.flows.all() } + statuses = set(flow_status_map.keys()) logger.info(f"update_ticket_status for ticket:{self.ticket.id}, statuses: {statuses}") + # 只要存在其中一个终止，则单据状态为已终止 if constants.TicketFlowStatus.TERMINATED in statuses: - # 只要存在其中一个终止，则单据状态为已终止 target_status = constants.TicketStatus.TERMINATED + # 只要存在其中一个失败，则单据状态为失败态 elif constants.TicketFlowStatus.FAILED in statuses: - # 只要存在其中一个失败，则单据状态为失败态 target_status = constants.TicketStatus.FAILED + # 只要存在其中一个撤销，则单据状态为撤销态 elif constants.TicketFlowStatus.REVOKED in statuses: - # 只要存在其中一个撤销，则单据状态为撤销态 target_status = constants.TicketStatus.REVOKED + # 只要有一个存在running，则需要根据flow的type决定单据的状态 elif constants.TicketFlowStatus.RUNNING in statuses: - target_status = constants.TicketStatus.RUNNING + flow = flow_status_map[constants.TicketFlowStatus.RUNNING] + target_status = constants.RUNNING_FLOW__TICKET_STATUS.get(flow.flow_type, constants.TicketStatus.RUNNING) + # 如果所有flow的状态处于完成态，则单据为成功 elif statuses.issubset(set(FLOW_FINISHED_STATUS)): - # 如果所有flow的状态处于完成态，则单据为成功 target_status = constants.TicketStatus.SUCCEEDED else: # 其他场景下状态未变更，无需更新DB diff --git a/dbm-ui/backend/ticket/flow_manager/pause.py b/dbm-ui/backend/ticket/flow_manager/pause.py index 1c1bb03a80..b643e87b9f 100644 --- a/dbm-ui/backend/ticket/flow_manager/pause.py +++ b/dbm-ui/backend/ticket/flow_manager/pause.py @@ -40,16 +40,15 @@ def _end_time(self) -> Optional[str]: @property def _summary(self) -> str: - return _("暂停状态{status_display}").format(status_display=constants.TicketStatus.get_choice_label(self.status)) + return _("暂停状态{status_display}").format( + status_display=constants.TicketFlowStatus.get_choice_label(self.status) + ) @property def _status(self) -> str: if self.ticket.todo_of_ticket.exist_unfinished(): - self.flow_obj.update_status(constants.TicketFlowStatus.RUNNING) - return constants.TicketStatus.RUNNING.value - - self.flow_obj.update_status(constants.TicketFlowStatus.SUCCEEDED) - return constants.TicketStatus.SUCCEEDED.value + return self.flow_obj.update_status(constants.TicketFlowStatus.RUNNING) + return self.flow_obj.update_status(constants.TicketFlowStatus.SUCCEEDED) @property def _url(self) -> str: diff --git a/dbm-ui/backend/ticket/flow_manager/resource.py b/dbm-ui/backend/ticket/flow_manager/resource.py index ee3b2f6012..bceb1d6651 100644 --- a/dbm-ui/backend/ticket/flow_manager/resource.py +++ b/dbm-ui/backend/ticket/flow_manager/resource.py @@ -58,7 +58,9 @@ def _end_time(self) -> Optional[str]: @property def _summary(self) -> str: - return _("资源申请状态{status_display}").format(status_display=constants.TicketStatus.get_choice_label(self.status)) + return _("资源申请状态{status_display}").format( + status_display=constants.TicketFlowStatus.get_choice_label(self.status) + ) @property def status(self) -> str: @@ -73,24 +75,24 @@ def update_flow_status(self, status): def _status(self) -> str: # 任务流程未创建时未PENDING状态 if not self.flow_obj.flow_obj_id: - return self.update_flow_status(constants.TicketStatus.PENDING.value) + return self.update_flow_status(constants.TicketFlowStatus.PENDING) # 如果资源申请成功，则直接返回success if self.resource_apply_status: - return self.update_flow_status(constants.TicketStatus.SUCCEEDED.value) + return self.update_flow_status(constants.TicketFlowStatus.SUCCEEDED) if self.flow_obj.err_msg: # 如果是其他情况引起的错误，则直接返回fail if not self.flow_obj.todo_of_flow.exists(): - return self.update_flow_status(constants.TicketStatus.FAILED.value) + return self.update_flow_status(constants.TicketFlowStatus.FAILED) # 如果是资源申请的todo状态，则判断todo是否完成 if self.ticket.todo_of_ticket.exist_unfinished(): - return self.update_flow_status(constants.TicketStatus.RUNNING.value) + return self.update_flow_status(constants.TicketFlowStatus.RUNNING) else: - return self.update_flow_status(constants.TicketStatus.SUCCEEDED.value) + return self.flow_obj.status # 其他情况认为还在RUNNING状态 - return self.update_flow_status(constants.TicketStatus.RUNNING.value) + return self.update_flow_status(constants.TicketFlowStatus.RUNNING) @property def _url(self) -> str: diff --git a/dbm-ui/backend/ticket/flow_manager/timer.py b/dbm-ui/backend/ticket/flow_manager/timer.py index 6031cd1f94..8dfdae4d8d 100644 --- a/dbm-ui/backend/ticket/flow_manager/timer.py +++ b/dbm-ui/backend/ticket/flow_manager/timer.py @@ -56,7 +56,7 @@ def _summary(self) -> str: return _("定时时间{}，已超时{}，需手动触发。暂停状态:{}").format( self.trigger_time, countdown2str(run_time - trigger_time), - constants.TicketStatus.get_choice_label(self.status), + constants.TicketFlowStatus.get_choice_label(self.status), ) now = datetime.now(timezone.utc) @@ -68,19 +68,17 @@ def _summary(self) -> str: @property def _status(self) -> str: trigger_time = str2datetime(self.trigger_time) + # 还未到定时节点，返回pending if self.expired_flag is None: - return constants.TicketStatus.PENDING.value - + return constants.TicketFlowStatus.PENDING.value + # 已过期，但是todo未处理，则返回running if self.expired_flag and self.ticket.todo_of_ticket.exist_unfinished(): - self.flow_obj.update_status(constants.TicketStatus.RUNNING.value) - return constants.TicketStatus.RUNNING.value - + return self.flow_obj.update_status(constants.TicketFlowStatus.RUNNING.value) + # 触发时间晚于当前时间，则返回running if trigger_time > datetime.now(timezone.utc): - self.flow_obj.update_status(constants.TicketStatus.RUNNING.value) - return constants.TicketStatus.RUNNING.value - - self.flow_obj.update_status(constants.TicketStatus.SUCCEEDED.value) - return constants.TicketStatus.SUCCEEDED.value + return self.flow_obj.update_status(constants.TicketFlowStatus.RUNNING.value) + # 其他情况说明已触发，返回succeed + return self.flow_obj.update_status(constants.TicketFlowStatus.SUCCEEDED.value) @property def _url(self) -> str: diff --git a/dbm-ui/backend/ticket/handler.py b/dbm-ui/backend/ticket/handler.py index 010f7f8a14..9800c5fb0a 100644 --- a/dbm-ui/backend/ticket/handler.py +++ b/dbm-ui/backend/ticket/handler.py @@ -11,6 +11,7 @@ import itertools import json import logging +import time from typing import Dict, List from django.db import transaction @@ -28,17 +29,22 @@ from backend.ticket.builders.common.base import fetch_cluster_ids, fetch_instance_ids from backend.ticket.constants import ( FLOW_FINISHED_STATUS, - ITSM_FIELD_NAME__ITSM_KEY, + RUNNING_FLOW__TICKET_STATUS, FlowType, FlowTypeConfig, OperateNodeActionType, TicketFlowStatus, + TicketStatus, TicketType, + TodoStatus, + TodoType, ) from backend.ticket.exceptions import TicketFlowsConfigException from backend.ticket.flow_manager.manager import TicketFlowManager from backend.ticket.models import Flow, Ticket, TicketFlowsConfig, Todo -from backend.ticket.todos import ActionType, TodoActorFactory +from backend.ticket.serializers import TodoSerializer +from backend.ticket.todos import BaseTodoContext, TodoActorFactory +from backend.ticket.todos.itsm_todo import ItsmTodoContext logger = logging.getLogger("root") @@ -200,24 +206,20 @@ def ticket_flow_config_init(cls): TicketFlowsConfig.objects.bulk_create(created_configs) @classmethod - def get_itsm_fields(cls, sample_sn=None): + def get_itsm_fields(cls, ticket_type): """获取单据审批需要的itsm字段""" + # 根据单据类型决定审批模式 + approve_mode = str(TicketType.get_approve_mode_by_ticket(ticket_type)) # 预先获取审批接口的field的审批意见和备注的key approval_key = SystemSettings.get_setting_value(key=SystemSettingsEnum.ITSM_APPROVAL_KEY) remark_key = SystemSettings.get_setting_value(key=SystemSettingsEnum.ITSM_REMARK_KEY) - - # 如果未入库，则获取任意一个ticket的信息来初始化key - if not approval_key or not remark_key: - ticket_info_response = ItsmApi.get_ticket_info(params={"sn": sample_sn}) - for field in ticket_info_response["fields"]: - SystemSettings.insert_setting_value(key=ITSM_FIELD_NAME__ITSM_KEY[field["name"]], value=field["key"]) - - return {SystemSettingsEnum.ITSM_APPROVAL_KEY: approval_key, SystemSettingsEnum.ITSM_REMARK_KEY: remark_key} + return approval_key[approve_mode], remark_key[approve_mode] @classmethod def approve_itsm_ticket(cls, ticket_id, action, operator, **kwargs): """审批 / 终止itsm中的单据""" - sn = Flow.objects.get(ticket_id=ticket_id, flow_type="BK_ITSM").flow_obj_id + flow = Flow.objects.get(ticket_id=ticket_id, flow_type="BK_ITSM") + sn = flow.flow_obj_id itsm_info = ItsmApi.get_ticket_info(params={"sn": sn}) # 当前没有正在进行的步骤，退出 @@ -225,16 +227,23 @@ def approve_itsm_ticket(cls, ticket_id, action, operator, **kwargs): return state_id = itsm_info["current_steps"][0]["state_id"] + act_msg_tpl = _("{}对单据{}操作: {}").format(operator, ticket_id, OperateNodeActionType.get_choice_label(action)) + act_msg = kwargs.get("action_message") or act_msg_tpl + # 审批单据 + params = {"action_message": act_msg} if action == OperateNodeActionType.TRANSITION: is_approved = kwargs["is_approved"] - fields = [{"key": field, "value": json.dumps(is_approved)} for field in cls.get_itsm_fields(sn).values()] - params = {"sn": sn, "state_id": state_id, "action_type": action, "operator": operator, "fields": fields} + itsm_fields = cls.get_itsm_fields(flow.ticket.ticket_type) + fields = [ + {"key": itsm_fields[0], "value": json.dumps(is_approved)}, + {"key": itsm_fields[1], "value": act_msg}, + ] + params.update(sn=sn, state_id=state_id, action_type=action, operator=operator, fields=fields) ItsmApi.operate_node(params, use_admin=True) - # 终止单据 - elif action == OperateNodeActionType.TERMINATE: - action_message = _("{} 终止了此单据").format(operator) - params = {"sn": sn, "action_type": action, "operator": operator, "action_message": action_message} + # 终止/撤销单据 + elif action in [OperateNodeActionType.TERMINATE, OperateNodeActionType.WITHDRAW]: + params.update(sn=sn, action_type=action, operator=operator) ItsmApi.operate_ticket(params, use_admin=True) return sn @@ -255,8 +264,8 @@ def revoke_ticket(cls, ticket_ids, operator): - 找到第一个非成功的flow 设置为终止 - 如果有关联正在运行的todos，也设置为终止 """ - # 查询ticket，关联正在运行的flows(这里定义的"运行"指的就是非成功) - finished_status = [*FLOW_FINISHED_STATUS, Flow, TicketFlowStatus.TERMINATED] + # 查询ticket，关联正在运行的flows(这里定义的"运行"指的就是非成功/终止/撤销) + finished_status = [*FLOW_FINISHED_STATUS, TicketFlowStatus.TERMINATED, TicketFlowStatus.REVOKED] running_flows = Flow.objects.filter(ticket__in=ticket_ids).exclude(status__in=finished_status) tickets = Ticket.objects.prefetch_related( Prefetch("flows", queryset=running_flows, to_attr="running_flows") @@ -265,22 +274,28 @@ def revoke_ticket(cls, ticket_ids, operator): # 对每个单据进行终止 for ticket in tickets: if not ticket.running_flows: - logger.info(_("单据[{}]没有需要终止的流程，跳过...").format(ticket.id)) continue - first_running_flow = ticket.running_flows[0] - # 如果有todo，则把所有todo终止 - todos = Todo.objects.filter(ticket=ticket, flow=first_running_flow) - for todo in todos: - TodoActorFactory.actor(todo).process(operator, ActionType.TERMINATE, params={}) + first_running_flow = ticket.running_flows[0] + cls.operate_flow(ticket.id, first_running_flow.id, func="revoke", operator=operator) + logger.info(_("操作人[{}]终止了单据[{}]").format(operator, ticket.id)) - # 如果是处于审批阶段，需要关闭itsm单据 - if first_running_flow.flow_type == FlowType.BK_ITSM: - cls.approve_itsm_ticket(ticket.id, OperateNodeActionType.TERMINATE, "admin", is_approved=False) + @classmethod + def batch_process_todo(cls, user, action, operations): + """ + 批量操作todo + @param user 用户 + @param action 动作 + @param operations: todo列表，每个item包含todo id和params + """ - # 用户终止 / 系统终止flow - logger.info(_("操作人[{}]终止了单据[{}]").format(operator, ticket.id)) - cls.operate_flow(ticket.id, first_running_flow.id, func="revoke", operator=operator) + results = [] + for operation in operations: + todo_id, params = operation["todo_id"], operation["params"] + todo = Todo.objects.get(id=todo_id) + TodoActorFactory.actor(todo).process(user, action, params) + results.append(todo) + return TodoSerializer(results, many=True).data @classmethod def create_ticket_flow_config(cls, bk_biz_id, cluster_ids, ticket_types, configs, operator): @@ -400,3 +415,60 @@ def query_ticket_flows_describe(cls, bk_biz_id, db_type, ticket_types=None): flow_desc_list.append(flow_config_info) return flow_desc_list + + @classmethod + def ticket_status_standardization(cls): + """ + 旧单据状态标准化。TODO: 迁移后此段代码可删除 + """ + batch = 50 + + # 标准化只针对running的单据，其他状态单据不影响 + running_tickets = Ticket.objects.filter(status=TicketStatus.RUNNING) + count = running_tickets.count() + for current in range(0, count, batch): + for ticket in running_tickets[current : current + batch]: + raw_status = ticket.status + ticket.status = RUNNING_FLOW__TICKET_STATUS[ticket.current_flow().flow_type] + ticket.save() + print(f"ticket[{ticket.id}] status {raw_status} ---> {ticket.status}") + time.sleep(1) + + # 失败的单据要增加一条todo关联 + failed_tickets = Ticket.objects.prefetch_related("flows").filter(status=TicketStatus.FAILED) + for current in range(0, count, batch): + for ticket in failed_tickets[current : current + batch]: + inner_flow = ticket.flows.filter(flow_type=FlowType.INNER_FLOW, status=TicketFlowStatus.FAILED).first() + if not inner_flow or inner_flow.todo_of_flow.exists(): + continue + Todo.objects.create( + name=_("【{}】单据任务执行失败，待处理").format(ticket.get_ticket_type_display()), + flow=inner_flow, + ticket=ticket, + type=TodoType.INNER_FAILED, + operators=[ticket.creator], + context=BaseTodoContext(inner_flow.id, ticket.id).to_dict(), + status=TodoStatus.TODO, + ) + print(f"ticket[{ticket.id}] add a failed todo") + time.sleep(1) + + # 待审批的单据要增加一条todo关联 + itsm_tickets = Ticket.objects.prefetch_related("flows").filter(status=TicketStatus.FAILED) + for current in range(0, count, batch): + for ticket in itsm_tickets[current : current + batch]: + itsm_flow = ticket.flows.filter(flow_type=FlowType.BK_ITSM, status=TicketFlowStatus.RUNNING).first() + if not itsm_flow or itsm_flow.todo_of_flow.exists(): + continue + itsm_fields = {f["key"]: f["value"] for f in itsm_flow.details["fields"]} + operators = itsm_fields["approver"].split(",") + Todo.objects.create( + name=_("【{}】单据等待审批").format(ticket.get_ticket_type_display()), + flow=itsm_flow, + ticket=ticket, + type=TodoType.ITSM, + operators=operators, + context=ItsmTodoContext(itsm_flow.id, ticket.id).to_dict(), + ) + print(f"ticket[{ticket.id}] add a itsm todo") + time.sleep(1) diff --git a/dbm-ui/backend/ticket/models/ticket.py b/dbm-ui/backend/ticket/models/ticket.py index 971facaa14..89844179b2 100644 --- a/dbm-ui/backend/ticket/models/ticket.py +++ b/dbm-ui/backend/ticket/models/ticket.py @@ -25,6 +25,7 @@ from backend.db_monitor.exceptions import AutofixException from backend.ticket.constants import ( EXCLUSIVE_TICKET_EXCEL_PATH, + TICKET_RUNNING_STATUS, FlowRetryType, FlowType, TicketFlowStatus, @@ -117,13 +118,13 @@ class Meta: def url(self): return f"{env.BK_SAAS_HOST}/{self.bk_biz_id}/ticket-manage/index?id={self.id}" - def set_terminated(self): - self.status = TicketStatus.TERMINATED + def set_status(self, status): + self.status = status self.save() def get_cost_time(self): # 计算耗时 - if self.status in [TicketStatus.PENDING, TicketStatus.RUNNING]: + if self.status in [TicketStatus.PENDING, *TICKET_RUNNING_STATUS]: return calculate_cost_time(timezone.now(), self.create_at) return calculate_cost_time(self.update_at, self.create_at) @@ -266,11 +267,17 @@ def get_cluster_configs(cls, ticket_type, bk_biz_id, cluster_ids): ] return cluster_configs + @classmethod + def get_config(cls, ticket_type): + """获取平台配置""" + global_cfg = cls.objects.get(bk_biz_id=PLAT_BIZ_ID, ticket_type=ticket_type) + return global_cfg + class ClusterOperateRecordManager(models.Manager): def filter_actives(self, cluster_id, *args, **kwargs): """获得集群正在运行的单据记录""" - return self.filter(cluster_id=cluster_id, ticket__status=TicketFlowStatus.RUNNING, *args, **kwargs) + return self.filter(cluster_id=cluster_id, ticket__status=TicketStatus.RUNNING, *args, **kwargs) def filter_inner_actives(self, cluster_id, *args, **kwargs): """获取集群正在 运行/失败 的inner flow的单据记录。此时认为集群会在互斥阶段""" @@ -355,7 +362,7 @@ def summary(self): def get_cluster_records_map(cls, cluster_ids: List[int]): """获取集群与操作记录之间的映射关系""" records = cls.objects.select_related("ticket", "flow").filter( - cluster_id__in=cluster_ids, ticket__status=TicketFlowStatus.RUNNING + cluster_id__in=cluster_ids, ticket__status__in=TICKET_RUNNING_STATUS ) cluster_operate_records_map: Dict[int, List] = defaultdict(list) for record in records: @@ -377,7 +384,7 @@ class InstanceOperateRecordManager(models.Manager): def filter_actives(self, instance_id, **kwargs): return self.filter( instance_id=instance_id, - ticket__status__in=[TicketStatus.RUNNING, TicketStatus.PENDING], + ticket__status=TicketStatus.RUNNING, **kwargs, ) @@ -419,9 +426,9 @@ def summary(self): @classmethod def get_instance_records_map(cls, instance_ids: List[Union[int, str]]): - """获取实例与操作记录之间的映射关系""" + """获取实例与操作记录之间的映射关系??????""" records = InstanceOperateRecord.objects.select_related("ticket").filter( - instance_id__in=instance_ids, ticket__status=TicketStatus.RUNNING + instance_id__in=instance_ids, ticket__status__in=TICKET_RUNNING_STATUS ) instance_operator_record_map: Dict[int, List] = defaultdict(list) for record in records: diff --git a/dbm-ui/backend/ticket/models/todo.py b/dbm-ui/backend/ticket/models/todo.py index 26f5e432dd..c2335e1574 100644 --- a/dbm-ui/backend/ticket/models/todo.py +++ b/dbm-ui/backend/ticket/models/todo.py @@ -19,7 +19,14 @@ from backend.bk_web.models import AuditedModel from backend.configuration.constants import BizSettingsEnum from backend.configuration.models import BizSettings -from backend.ticket.constants import FlowMsgStatus, FlowMsgType, TicketFlowStatus, TodoStatus, TodoType +from backend.ticket.constants import ( + TODO_RUNNING_STATUS, + FlowMsgStatus, + FlowMsgType, + TicketFlowStatus, + TodoStatus, + TodoType, +) from backend.ticket.tasks.ticket_tasks import send_msg_for_flow logger = logging.getLogger("root") @@ -27,25 +34,18 @@ class TodoManager(models.Manager): def exist_unfinished(self): - return self.filter(status__in=[TodoStatus.TODO, TodoStatus.RUNNING]).exists() + return self.filter(status__in=TODO_RUNNING_STATUS).exists() def create(self, **kwargs): - assistance_flag = ( - BizSettings.get_setting_value(kwargs["ticket"].bk_biz_id, BizSettingsEnum.BIZ_ASSISTANCE_SWITCH) or False - ) + bk_biz_id = kwargs["ticket"].bk_biz_id + assistance_flag = BizSettings.get_setting_value(bk_biz_id, key=BizSettingsEnum.BIZ_ASSISTANCE_SWITCH) or False if assistance_flag: # 获取业务协助人列表 - biz_facilitators: list = ( - BizSettings.get_setting_value( - bk_biz_id=kwargs["ticket"].bk_biz_id, key=BizSettingsEnum.BIZ_ASSISTANCE_VARS - ) - or [] - ) - operators = kwargs.get("operators", []) - # 过滤掉已经在 operators 中的协助人 - unique_biz_facilitators = [facilitator for facilitator in biz_facilitators if facilitator not in operators] - # 按顺序合并 - kwargs["operators"] = operators + unique_biz_facilitators + biz_helpers = BizSettings.get_setting_value(bk_biz_id, key=BizSettingsEnum.BIZ_ASSISTANCE_VARS, default=[]) + # 合并单据处理人 + 业务协助人 + kwargs["operators"] = kwargs.get("operators", []) + biz_helpers + # 对操作者去重 + kwargs["operators"] = list(set(kwargs["operators"])) todo = super().create(**kwargs) send_msg_for_flow.apply_async( kwargs={ @@ -109,7 +109,6 @@ def set_success(self, username, action): def set_terminated(self, username, action): self.set_status(username, TodoStatus.DONE_FAILED) - self.ticket.set_terminated() self.flow.update_status(TicketFlowStatus.TERMINATED) TodoHistory.objects.create(creator=username, todo=self, action=action) diff --git a/dbm-ui/backend/ticket/serializers.py b/dbm-ui/backend/ticket/serializers.py index 70a9686266..c958cee5d7 100644 --- a/dbm-ui/backend/ticket/serializers.py +++ b/dbm-ui/backend/ticket/serializers.py @@ -23,7 +23,15 @@ from backend.core.encrypt.handlers import AsymmetricHandler from backend.ticket import mock_data from backend.ticket.builders import BuilderFactory -from backend.ticket.constants import CountType, FlowType, TicketStatus, TicketType, TodoStatus +from backend.ticket.constants import ( + TICKET_RUNNING_STATUS, + TODO_RUNNING_STATUS, + FlowType, + TicketFlowStatus, + TicketStatus, + TicketType, + TodoStatus, +) from backend.ticket.flow_manager.manager import TicketFlowManager from backend.ticket.models import Flow, Ticket, Todo from backend.ticket.todos import ActionType @@ -82,14 +90,15 @@ class TicketSerializer(AuditedSerializer, serializers.ModelSerializer): ticket_type = serializers.ChoiceField( help_text=_("单据类型"), choices=TicketType.get_choices(), default=TicketType.MYSQL_SINGLE_APPLY ) - status = serializers.ChoiceField(help_text=_("状态"), choices=TicketStatus.get_choices(), read_only=True) remark = serializers.CharField(help_text=_("备注"), required=False, max_length=LEN_L_LONG, allow_blank=True) # 默认使用MySQL序列化器，不同单据类型不同字段序列化 group = serializers.CharField(help_text=_("单据分组类型"), required=False) details = TicketDetailsSerializer(help_text=_("单据详情")) # 额外补充展示字段 - ticket_type_display = serializers.SerializerMethodField(help_text=_("单据类型名称")) + todo_operators = serializers.SerializerMethodField(help_text=_("处理人列表")) + status = serializers.SerializerMethodField(help_text=_("状态"), read_only=True) status_display = serializers.SerializerMethodField(help_text=_("状态名称")) + ticket_type_display = serializers.SerializerMethodField(help_text=_("单据类型名称")) cost_time = serializers.SerializerMethodField(help_text=_("耗时")) bk_biz_name = serializers.SerializerMethodField(help_text=_("业务名")) db_app_abbr = serializers.SerializerMethodField(help_text=_("业务英文缩写")) @@ -111,14 +120,24 @@ def validate_ticket_type(self, value): raise serializers.ValidationError(_("不允许提交敏感单据类型{}").format(value)) return value + def get_todo_operators(self, obj): + # 任取一个运行中的todo，获取operators即可 + obj.running_todos = [todo for todo in obj.todo_of_ticket.all() if todo.status == TodoStatus.TODO] + return obj.running_todos[0].operators if obj.running_todos else [] + + def get_status(self, obj): + if obj.status == TicketStatus.RUNNING and obj.running_todos: + obj.status = TicketStatus.INNER_TODO + return obj.status + def get_ticket_type_display(self, obj): return obj.get_ticket_type_display() def get_status_display(self, obj): - return obj.get_status_display() + return TicketStatus.get_choice_label(obj.status) def get_cost_time(self, obj): - if obj.status in [TicketStatus.PENDING, TicketStatus.RUNNING]: + if obj.status in [TicketStatus.PENDING, *TICKET_RUNNING_STATUS]: return calculate_cost_time(timezone.now(), obj.create_at) return calculate_cost_time(obj.update_at, obj.create_at) @@ -164,7 +183,7 @@ def get_end_time(self, obj): def get_cost_time(self, obj): start_time = strptime(self.get_start_time(obj)) end_time = strptime(self.get_end_time(obj)) - if self.get_status(obj) in [TicketStatus.PENDING, TicketStatus.RUNNING]: + if self.get_status(obj) in [TicketFlowStatus.PENDING, TicketFlowStatus.RUNNING]: return calculate_cost_time(timezone.now(), start_time) return calculate_cost_time(end_time, start_time) @@ -215,7 +234,7 @@ class TodoSerializer(serializers.ModelSerializer): cost_time = serializers.SerializerMethodField(help_text=_("耗时")) def get_cost_time(self, obj): - if obj.status in [TodoStatus.TODO, TodoStatus.RUNNING]: + if obj.status in TODO_RUNNING_STATUS: return calculate_cost_time(timezone.now(), obj.create_at) return calculate_cost_time(obj.done_at, obj.create_at) @@ -260,30 +279,38 @@ class RevokeFlowSLZ(serializers.Serializer): flow_id = serializers.IntegerField(help_text=_("单据流程的ID")) +class RevokeTicketSLZ(serializers.Serializer): + ticket_ids = serializers.ListField(help_text=_("终止单据ID"), child=serializers.IntegerField()) + + class GetTodosSLZ(serializers.Serializer): todo_status = serializers.ChoiceField( help_text=_("状态"), choices=TodoStatus.get_choices(), required=False, allow_blank=True ) -class CountTicketSLZ(serializers.Serializer): - count_type = serializers.ChoiceField(help_text=_("类型"), choices=CountType.get_choices(), default=CountType.MY_TODO) - - -class ClusterModifyOpSerializer(serializers.Serializer): - cluster_id = serializers.IntegerField(help_text=_("集群ID")) +class OpRecordSerializer(serializers.Serializer): start_time = serializers.DateTimeField(help_text=_("查询起始时间"), required=False) end_time = serializers.DateTimeField(help_text=_("查询终止时间"), required=False) op_type = serializers.ChoiceField(help_text=_("操作类型"), choices=TicketType.get_choices(), required=False) op_status = serializers.ChoiceField(help_text=_("操作状态"), choices=TicketStatus.get_choices(), required=False) + def to_representation(self, instance): + return { + "create_at": instance.create_at, + "op_type": instance.ticket.ticket_type, + "op_status": instance.ticket.status, + "ticket_id": instance.ticket.id, + "creator": instance.creator, + } + -class InstanceModifyOpSerializer(serializers.Serializer): +class ClusterModifyOpSerializer(OpRecordSerializer): + cluster_id = serializers.IntegerField(help_text=_("集群ID")) + + +class InstanceModifyOpSerializer(OpRecordSerializer): instance_id = serializers.IntegerField(help_text=_("实例ID")) - start_time = serializers.DateTimeField(help_text=_("查询起始时间"), required=False) - end_time = serializers.DateTimeField(help_text=_("查询终止时间"), required=False) - op_type = serializers.ChoiceField(help_text=_("操作类型"), choices=TicketType.get_choices(), required=False) - op_status = serializers.ChoiceField(help_text=_("操作状态"), choices=TicketStatus.get_choices(), required=False) class QueryTicketFlowDescribeSerializer(serializers.Serializer): @@ -383,3 +410,15 @@ def validate(self, attrs): if todo_id not in existing_todo_ids: raise serializers.ValidationError(_("待办id{}不存在".format(attrs["todo_id"]))) return attrs + + +class BatchTicketOperateSerializer(serializers.Serializer): + action = serializers.ChoiceField( + choices=[ActionType.APPROVE.value, ActionType.TERMINATE.value], help_text=_("统一动作") + ) + ticket_ids = serializers.ListField(help_text=_("单据ID列表"), child=serializers.IntegerField()) + params = serializers.JSONField(help_text=_("动作参数"), required=False, default={}) + + +class GetInnerFlowSerializer(serializers.Serializer): + ticket_ids = serializers.CharField(help_text=_("单据ID(逗号分隔)")) diff --git a/dbm-ui/backend/ticket/tasks/ticket_tasks.py b/dbm-ui/backend/ticket/tasks/ticket_tasks.py index 9025ffaf21..4f45f0425a 100644 --- a/dbm-ui/backend/ticket/tasks/ticket_tasks.py +++ b/dbm-ui/backend/ticket/tasks/ticket_tasks.py @@ -45,7 +45,6 @@ TodoType, ) from backend.ticket.exceptions import TicketTaskTriggerException -from backend.ticket.flow_manager.inner import InnerFlow from backend.ticket.models.ticket import Flow, Ticket, TicketFlowsConfig from backend.utils.time import date2str, datetime2str @@ -69,6 +68,8 @@ def run_next_flow(self) -> None: @classmethod def retry_exclusive_inner_flow(cls) -> None: """重试互斥错误的inner flow""" + from backend.ticket.flow_manager.inner import InnerFlow + to_retry_flows = Flow.objects.filter(err_code=FlowErrCode.AUTO_EXCLUSIVE_ERROR) if not to_retry_flows: return diff --git a/dbm-ui/backend/ticket/todos/__init__.py b/dbm-ui/backend/ticket/todos/__init__.py index 701ce122b6..35412d6cc3 100644 --- a/dbm-ui/backend/ticket/todos/__init__.py +++ b/dbm-ui/backend/ticket/todos/__init__.py @@ -14,8 +14,12 @@ from dataclasses import asdict, dataclass from typing import Callable +from blueapps.account.models import User from django.utils.translation import ugettext_lazy as _ +from backend.constants import DEFAULT_SYSTEM_USER +from backend.ticket.constants import TODO_RUNNING_STATUS +from backend.ticket.exceptions import TodoWrongOperatorException from backend.ticket.models import Todo from blue_krill.data_types.enum import EnumField, StructuredEnum @@ -37,8 +41,32 @@ def __init__(self, todo: Todo): def name(cls): return cls.__name__ + def update_context(self, params): + # 更新上下文信息 + if "remark" in params: + self.todo.context.update(remark=params["remark"]) + self.todo.save(update_fields=["context"]) + def process(self, username, action, params): - """处理操作""" + # 当状态已经被确认，则不允许重复操作 + if self.todo.status not in TODO_RUNNING_STATUS: + raise TodoWrongOperatorException(_("当前代办操作已经处理，不能重复处理！")) + + # 允许系统内置用户确认 + if username == DEFAULT_SYSTEM_USER: + self._process(username, action, params) + return + # 允许超级用户和操作人确认 + is_superuser = User.objects.get(username=username).is_superuser + if not is_superuser and username not in self.todo.operators: + raise TodoWrongOperatorException(_("{}不在处理人: {}中，无法处理").format(username, self.todo.operators)) + + # 执行确认操作 + self._process(username, action, params) + self.update_context(params) + + def _process(self, username, action, params): + """处理操作的具体实现""" raise NotImplementedError diff --git a/dbm-ui/backend/ticket/todos/itsm_todo.py b/dbm-ui/backend/ticket/todos/itsm_todo.py new file mode 100644 index 0000000000..931565595b --- /dev/null +++ b/dbm-ui/backend/ticket/todos/itsm_todo.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import logging +from dataclasses import dataclass + +from backend.constants import DEFAULT_SYSTEM_USER +from backend.ticket import todos +from backend.ticket.constants import TODO_RUNNING_STATUS, OperateNodeActionType, TodoType +from backend.ticket.todos import ActionType, BaseTodoContext + +logger = logging.getLogger("root") + + +@dataclass +class ItsmTodoContext(BaseTodoContext): + pass + + +@todos.TodoActorFactory.register(TodoType.ITSM) +class ItsmTodo(todos.TodoActor): + """来自审批中的待办""" + + def process(self, username, action, params): + # itsm的todo允许本人操作 + if username == self.todo.ticket.creator and self.todo.status in TODO_RUNNING_STATUS: + self._process(username, action, params) + return + super().process(username, action, params) + + def _process(self, username, action, params): + from backend.ticket.handler import TicketHandler + + ticket_id = self.context.get("ticket_id") + own = self.todo.ticket.creator + message = params.get("remark", "") + + def approve_itsm_ticket(itsm_action, is_approved): + sn = TicketHandler.approve_itsm_ticket( + ticket_id, + action=itsm_action, + operator=username, + is_approved=is_approved, + action_message=message, + ) + return sn + + # 系统终止，认为是关单(调用itsm接口要用admin发起) + if action == ActionType.TERMINATE and username == DEFAULT_SYSTEM_USER: + username = "admin" + approve_itsm_ticket(OperateNodeActionType.TERMINATE, is_approved=False) + self.todo.set_terminated(username, action) + # 审批人终止，认为是拒单 + elif action == ActionType.TERMINATE and username != own: + approve_itsm_ticket(OperateNodeActionType.TRANSITION, is_approved=False) + self.todo.set_terminated(username, action) + # 自己终止，认为是撤单 + elif action == ActionType.TERMINATE and username == own: + approve_itsm_ticket(OperateNodeActionType.WITHDRAW, is_approved=False) + self.todo.set_terminated(username, action) + # 只允许审批人通过 + elif action == ActionType.APPROVE and username != own: + approve_itsm_ticket(OperateNodeActionType.TRANSITION, is_approved=True) + self.todo.set_success(username, action) diff --git a/dbm-ui/backend/ticket/todos/pause_todo.py b/dbm-ui/backend/ticket/todos/pause_todo.py index 10e8c412a8..9b44d8275a 100644 --- a/dbm-ui/backend/ticket/todos/pause_todo.py +++ b/dbm-ui/backend/ticket/todos/pause_todo.py @@ -10,13 +10,9 @@ """ from dataclasses import dataclass -from django.utils.translation import gettext as _ - -from backend.constants import DEFAULT_SYSTEM_USER from backend.db_meta.models.sqlserver_dts import DtsStatus, SqlserverDtsInfo from backend.ticket import todos from backend.ticket.constants import TicketFlowStatus, TicketType, TodoType -from backend.ticket.exceptions import TodoWrongOperatorException from backend.ticket.flow_manager import manager from backend.ticket.flow_manager.manager import TicketFlowManager from backend.ticket.todos import ActionType, BaseTodoContext @@ -37,11 +33,8 @@ class ResourceReplenishTodoContext(BaseTodoContext): class PauseTodo(todos.TodoActor): """来自主流程的待办""" - def process(self, username, action, params): + def _process(self, username, action, params): """确认/终止""" - if username not in self.todo.operators and username != DEFAULT_SYSTEM_USER: - raise TodoWrongOperatorException(_("{}不在处理人: {}中，无法处理").format(username, self.todo.operators)) - if action == ActionType.TERMINATE: self.todo.set_terminated(username, action) return @@ -63,11 +56,8 @@ def process(self, username, action, params): class ResourceReplenishTodo(todos.TodoActor): """资源补货的代办""" - def process(self, username, action, params): + def _process(self, username, action, params): """确认/终止""" - if username not in self.todo.operators and username != DEFAULT_SYSTEM_USER: - raise TodoWrongOperatorException(_("{}不在处理人: {}中，无法处理").format(username, self.todo.operators)) - # 终止单据 if action == ActionType.TERMINATE: self.todo.set_terminated(username, action) diff --git a/dbm-ui/backend/ticket/todos/pipeline_todo.py b/dbm-ui/backend/ticket/todos/pipeline_todo.py index d3fe379aee..ce68891bec 100644 --- a/dbm-ui/backend/ticket/todos/pipeline_todo.py +++ b/dbm-ui/backend/ticket/todos/pipeline_todo.py @@ -11,13 +11,12 @@ import logging from dataclasses import dataclass -from django.utils.translation import gettext as _ +from django.utils.translation import ugettext as _ -from backend.constants import DEFAULT_SYSTEM_USER from backend.flow.engine.bamboo.engine import BambooEngine from backend.ticket import todos -from backend.ticket.constants import TodoStatus, TodoType -from backend.ticket.exceptions import TodoWrongOperatorException +from backend.ticket.constants import TodoStatus +from backend.ticket.constants import TodoType from backend.ticket.models import TodoHistory from backend.ticket.todos import ActionType, BaseTodoContext @@ -34,11 +33,8 @@ class PipelineTodoContext(BaseTodoContext): class PipelineTodo(todos.TodoActor): """来自自动化流程中的待办""" - def process(self, username, action, params): + def _process(self, username, action, params): """确认/终止""" - if username not in self.todo.operators and username != DEFAULT_SYSTEM_USER: - raise TodoWrongOperatorException(_("{}不在处理人: {}中，无法处理").format(username, self.todo.operators)) - # 从todo的上下文获取pipeline节点信息 root_id, node_id = self.context.get("root_id"), self.context.get("node_id") engine = BambooEngine(root_id=root_id) @@ -66,3 +62,18 @@ def process(self, username, action, params): raise Exception(",".join(res.exc.args)) self.todo.set_success(username, action) + + @classmethod + def create(cls, ticket, flow, root_id, node_id): + from backend.ticket.models import Todo + + # 创建一条代办 + Todo.objects.create( + name=_("【{}】流程待确认,是否继续？").format(ticket.get_ticket_type_display()), + flow=flow, + ticket=ticket, + type=TodoType.INNER_APPROVE, + # todo: 待办人暂定为提单人 + operators=[ticket.creator], + context=PipelineTodoContext(flow.id, ticket.id, root_id, node_id).to_dict(), + ) diff --git a/dbm-ui/backend/ticket/views.py b/dbm-ui/backend/ticket/views.py index 9b66b6e7a4..b7d2f631d7 100644 --- a/dbm-ui/backend/ticket/views.py +++ b/dbm-ui/backend/ticket/views.py @@ -9,7 +9,9 @@ specific language governing permissions and limitations under the License. """ import operator +from collections import Counter from functools import reduce +from typing import Dict, List from django.db import transaction from django.db.models import Q @@ -22,7 +24,9 @@ from backend import env from backend.bk_web import viewsets +from backend.bk_web.pagination import AuditedLimitOffsetPagination from backend.bk_web.swagger import PaginatedResponseSwaggerAutoSchema, common_swagger_auto_schema +from backend.configuration.constants import DBType from backend.configuration.models import DBAdministrator from backend.db_services.ipchooser.query.resource import ResourceQueryHelper from backend.iam_app.dataclass import ResourceEnum @@ -38,34 +42,36 @@ from backend.ticket.builders import BuilderFactory from backend.ticket.builders.common.base import InfluxdbTicketFlowBuilderPatchMixin, fetch_cluster_ids from backend.ticket.constants import ( - TODO_DONE_STATUS, + FLOW_NOT_EXECUTE_STATUS, + TICKET_RUNNING_STATUS, + TICKET_TODO_STATUS, + TODO_RUNNING_STATUS, CountType, - OperateNodeActionType, - TicketStatus, + FlowType, TicketType, - TodoStatus, + TodoType, ) from backend.ticket.contexts import TicketContext from backend.ticket.exceptions import TicketDuplicationException -from backend.ticket.filters import TicketListFilter +from backend.ticket.filters import ClusterOpRecordListFilter, InstanceOpRecordListFilter, TicketListFilter from backend.ticket.flow_manager.manager import TicketFlowManager from backend.ticket.handler import TicketHandler -from backend.ticket.models import ClusterOperateRecord, InstanceOperateRecord, Ticket, TicketFlowsConfig, Todo +from backend.ticket.models import ClusterOperateRecord, Flow, InstanceOperateRecord, Ticket, TicketFlowsConfig from backend.ticket.serializers import ( - BatchApprovalSerializer, + BatchTicketOperateSerializer, BatchTodoOperateSerializer, ClusterModifyOpSerializer, - CountTicketSLZ, CreateTicketFlowConfigSerializer, DeleteTicketFlowConfigSerializer, FastCreateCloudComponentSerializer, + GetInnerFlowSerializer, GetNodesSLZ, - GetTodosSLZ, InstanceModifyOpSerializer, ListTicketStatusSerializer, QueryTicketFlowDescribeSerializer, RetryFlowSLZ, RevokeFlowSLZ, + RevokeTicketSLZ, SensitiveTicketSerializer, TicketFlowDescribeSerializer, TicketFlowSerializer, @@ -77,7 +83,6 @@ UpdateTicketFlowConfigSerializer, ) from backend.ticket.todos import TodoActorFactory -from backend.utils.batch_request import request_multi_thread TICKET_TAG = "ticket" @@ -87,9 +92,10 @@ class TicketViewSet(viewsets.AuditedModelViewSet): 单据视图 """ - queryset = Ticket.objects.all() + queryset = Ticket.objects.all().prefetch_related("todo_of_ticket") serializer_class = TicketSerializer filter_class = TicketListFilter + pagination_class = AuditedLimitOffsetPagination def _get_custom_permissions(self): # 创建单据，关联单据类型的动作 @@ -114,17 +120,18 @@ def _get_custom_permissions(self): elif self.action in ["update_ticket_flow_config", "create_ticket_flow_config", "delete_ticket_flow_config"]: return ticket_flows_config_permission(self.action, self.request) # 对于处理todo的接口，可以不用鉴权，todo本身会判断是否是确认人 - elif self.action in ["process_todo", "batch_process_todo"]: + elif self.action in ["process_todo", "batch_process_todo", "batch_process_ticket"]: return [] # 其他非敏感GET接口，不鉴权 elif self.action in [ "list", "flow_types", "get_nodes", - "get_todo_tickets", "get_tickets_count", "query_ticket_flow_describe", "list_ticket_status", + "get_inner_flow_infos", + "revoke_ticket", ]: return [] # 回调和处理无需鉴权 @@ -138,6 +145,19 @@ def _get_login_exempt_view_func(cls): # 需要豁免的接口方法与名字 return {"post": [cls.callback.__name__], "put": [], "get": [], "delete": []} + @classmethod + def _get_self_manage_tickets(cls, user): + # 超级管理员返回所有单据 + if user.username in env.ADMIN_USERS or user.is_superuser: + return Ticket.objects.all() + # 获取user管理的单据合集 + manage_filters = [ + Q(group=manage.db_type) & Q(bk_biz_id=manage.bk_biz_id) if manage.bk_biz_id else Q(group=manage.db_type) + for manage in DBAdministrator.objects.filter(users__contains=user.username) + ] + ticket_filter = Q(creator=user.username) | reduce(operator.or_, manage_filters or [Q()]) + return Ticket.objects.filter(ticket_filter) + def get_queryset(self): """ 单据queryset规则--针对list： @@ -151,17 +171,11 @@ def get_queryset(self): self_manage = int(self.request.query_params["self_manage"]) # 只返回自己创建的单据 if self_manage == 0: - return Ticket.objects.filter(creator=username) - # 超级管理员返回所有单据 - if username in env.ADMIN_USERS or self.request.user.is_superuser: - return Ticket.objects.all() + qs = Ticket.objects.filter(creator=username) # 返回自己管理的组件单据 - manage_filters = [ - Q(group=manage.db_type) & Q(bk_biz_id=manage.bk_biz_id) if manage.bk_biz_id else Q(group=manage.db_type) - for manage in DBAdministrator.objects.filter(users__contains=username) - ] - ticket_filter = Q(creator=username) | reduce(operator.or_, manage_filters or [Q()]) - return Ticket.objects.filter(ticket_filter) + else: + qs = self._get_self_manage_tickets(self.request.user) + return qs def get_serializer_context(self): context = super(TicketViewSet, self).get_serializer_context() @@ -171,28 +185,28 @@ def get_serializer_context(self): context["ticket_ctx"] = TicketContext() return context - def _verify_duplicate_ticket(self, ticket_type, details, user): - """校验是否重复提交""" + @staticmethod + def _verify_influxdb_duplicate_ticket(ticket_type, details, user, active_tickets): + current_instances = InfluxdbTicketFlowBuilderPatchMixin.get_instances(ticket_type, details) + for ticket in active_tickets: + active_instances = ticket.details["instances"] + duplicate_ids = list(set(active_instances).intersection(current_instances)) + if duplicate_ids: + raise TicketDuplicationException( + context=_("实例{}已存在相同类型的单据[{}]正在运行，请确认是否重复提交").format(duplicate_ids, ticket.id), + data={"duplicate_instance_ids": duplicate_ids, "duplicate_ticket_id": ticket.id}, + ) - active_tickets = self.get_queryset().filter(ticket_type=ticket_type, status=TicketStatus.RUNNING, creator=user) + def verify_duplicate_ticket(self, ticket_type, details, user): + """校验是否重复提交""" + active_tickets = self.get_queryset().filter( + ticket_type=ticket_type, status__in=TICKET_RUNNING_STATUS, creator=user + ) # influxdb 相关操作单独适配，这里暂时没有找到更好的写法，唯一的改进就是创建单据时，会提前提取出对比内容，比如instances - if ticket_type in [ - TicketType.INFLUXDB_ENABLE, - TicketType.INFLUXDB_DISABLE, - TicketType.INFLUXDB_REBOOT, - TicketType.INFLUXDB_DESTROY, - TicketType.INFLUXDB_REPLACE, - ]: - current_instances = InfluxdbTicketFlowBuilderPatchMixin.get_instances(ticket_type, details) - for ticket in active_tickets: - active_instances = ticket.details["instances"] - duplicate_ids = list(set(active_instances).intersection(current_instances)) - if duplicate_ids: - raise TicketDuplicationException( - context=_("实例{}已存在相同类型的单据[{}]正在运行，请确认是否重复提交").format(duplicate_ids, ticket.id), - data={"duplicate_instance_ids": duplicate_ids, "duplicate_ticket_id": ticket.id}, - ) + # TODO: 后续这段逻辑待删除，influxdb已经弃用 + if ticket_type in TicketType.get_ticket_type_by_db(DBType.InfluxDB): + self._verify_influxdb_duplicate_ticket(ticket_type, details, user, active_tickets) return cluster_ids = fetch_cluster_ids(details=details) @@ -210,7 +224,7 @@ def perform_create(self, serializer): ignore_duplication = self.request.data.get("ignore_duplication") or False # 如果不允许忽略重复提交，则进行校验 if not ignore_duplication: - self._verify_duplicate_ticket(ticket_type, self.request.data["details"], self.request.user.username) + self.verify_duplicate_ticket(ticket_type, self.request.data["details"], self.request.user.username) with transaction.atomic(): # 设置单据类别 TODO: 这里会请求两次数据库，是否考虑group参数让前端传递 @@ -338,6 +352,17 @@ def revoke_flow(self, request, pk): TicketHandler.operate_flow(ticket_id=pk, flow_id=validated_data["flow_id"], func="revoke", operator=user) return Response() + @common_swagger_auto_schema( + operation_summary=_("单据终止"), + request_body=RevokeTicketSLZ(), + tags=[TICKET_TAG], + ) + @action(methods=["POST"], detail=False, serializer_class=RevokeTicketSLZ) + def revoke_ticket(self, request, *args, **kwargs): + ticket_ids = self.params_validate(self.get_serializer_class())["ticket_ids"] + TicketHandler.revoke_ticket(ticket_ids, operator=request.user.username) + return Response() + @swagger_auto_schema( operation_summary=_("获取单据类型列表"), query_serializer=TicketTypeSLZ(), @@ -381,46 +406,6 @@ def get_nodes(self, request, *args, **kwargs): return Response(hosts) - @common_swagger_auto_schema( - operation_summary=_("待办单据列表"), - query_serializer=GetTodosSLZ(), - tags=[TICKET_TAG], - ) - @Permission.decorator_permission_field( - id_field=lambda d: d["id"], - data_field=lambda d: d["results"], - actions=[ActionEnum.TICKET_VIEW], - resource_meta=ResourceEnum.TICKET, - ) - @action(methods=["GET"], detail=False, serializer_class=GetTodosSLZ) - def get_todo_tickets(self, request, *args, **kwargs): - """待办视图单据列表""" - - # 获取我的待办 - validated_data = self.params_validate(self.get_serializer_class()) - todo_status = validated_data.get("todo_status") - my_todos = Todo.objects.filter(operators__contains=request.user.username) - - # 状态筛选：已处理/未处理 - if todo_status in TODO_DONE_STATUS: - my_todos = my_todos.filter(status__in=TODO_DONE_STATUS) - elif todo_status: - my_todos = my_todos.filter(status=todo_status) - - # 复用全局过滤器 - tickets = self.filter_queryset(self.get_queryset()) - - # 关联查询单据 - my_todo_tickets = tickets.filter(id__in=my_todos.values_list("ticket_id")) - context = self.get_serializer_context() - - # 分页处理 - page = self.paginate_queryset(my_todo_tickets) - serializer = TicketSerializer(page, many=True, context=context) - resp = self.get_paginated_response(serializer.data) - resp.data["results"] = TicketHandler.add_related_object(resp.data["results"]) - return resp - @swagger_auto_schema( operation_summary=_("待办处理"), request_body=TodoOperateSerializer(), @@ -444,96 +429,74 @@ def process_todo(self, request, *args, **kwargs): @common_swagger_auto_schema( operation_summary=_("待办单据数"), - query_serializer=CountTicketSLZ(), tags=[TICKET_TAG], ) - @action(methods=["GET"], detail=False, serializer_class=CountTicketSLZ) + @action(methods=["GET"], detail=False, filter_class=None, pagination_class=None) def get_tickets_count(self, request, *args, **kwargs): - validated_data = self.params_validate(self.get_serializer_class()) - count_type = validated_data.get("count_type") - - # 待办单数量 - if count_type == CountType.MY_TODO: - my_todos = Todo.objects.filter(status=TodoStatus.TODO, operators__contains=request.user.username) - tickets = self.filter_queryset(self.get_queryset()) - my_tickets = tickets.filter(id__in=my_todos.values_list("ticket_id")) - else: - # 申请单数量 - my_tickets = Ticket.objects.filter( - creator=request.user.username, status__in=[TicketStatus.RUNNING, TicketStatus.PENDING] + """ + 获取单据的数量，目前需要获取 + - 我的申请 + - (代办)待我审批、待我确认，待我补货 + - 我的已办 + - 我负责的业务 + """ + user = request.user.username + tickets = self._get_self_manage_tickets(request.user) + count_map = {count_type: 0 for count_type in CountType.get_values()} + + # 我负责的业务 + count_map[CountType.SELF_MANAGE] = tickets.count() + # 我的申请 + count_map[CountType.MY_APPROVE] = tickets.filter(creator=user).count() + # 我的代办 + todo_status = ( + tickets.filter( + status__in=TICKET_TODO_STATUS, + todo_of_ticket__operators__contains=user, + todo_of_ticket__status__in=TODO_RUNNING_STATUS, ) + .distinct() + .values_list("status", flat=True) + ) + for sts, count in Counter(todo_status).items(): + sts = CountType.INNER_TODO.value if sts == "RUNNING" else sts + count_map[sts] = count + # 我的已办 + count_map[CountType.DONE] = tickets.filter(todo_of_ticket__done_by=user).count() - return Response(my_tickets.count()) + return Response(count_map) @common_swagger_auto_schema( operation_summary=_("查询集群变更单据事件"), - query_serializer=ClusterModifyOpSerializer(), tags=[TICKET_TAG], ) - @action(methods=["GET"], detail=False, serializer_class=ClusterModifyOpSerializer) + @action( + methods=["GET"], + detail=False, + serializer_class=ClusterModifyOpSerializer, + queryset=ClusterOperateRecord.objects.select_related("ticket").order_by("-create_at"), + filter_class=ClusterOpRecordListFilter, + ) def get_cluster_operate_records(self, request, *args, **kwargs): - validated_data = self.params_validate(self.get_serializer_class()) - op_filters = Q(cluster_id=validated_data["cluster_id"]) - if validated_data.get("start_time"): - op_filters &= Q(create_at__gte=validated_data.get("start_time")) - - if validated_data.get("end_time"): - op_filters &= Q(create_at__lte=validated_data.get("end_time")) - - if validated_data.get("op_type"): - op_filters &= Q(ticket__ticket_type=validated_data.get("op_type")) - - if validated_data.get("op_status"): - op_filters &= Q(ticket__status=validated_data.get("op_status")) - - op_records = ClusterOperateRecord.objects.select_related("ticket").filter(op_filters).order_by("-create_at") - op_records_info = [ - { - "create_at": record.create_at, - "op_type": TicketType.get_choice_label(record.ticket.ticket_type), - "op_status": record.ticket.status, - "ticket_id": record.ticket.id, - "creator": record.creator, - } - for record in op_records - ] - op_records_page = self.paginate_queryset(op_records_info) - return self.get_paginated_response(op_records_page) + op_records_page_qs = self.paginate_queryset(self.filter_queryset(self.queryset)) + op_records_page_data = self.serializer_class(op_records_page_qs, many=True).data + return self.get_paginated_response(data=op_records_page_data) @common_swagger_auto_schema( operation_summary=_("查询集群实例变更单据事件"), - query_serializer=InstanceModifyOpSerializer(), tags=[TICKET_TAG], ) - @action(methods=["GET"], detail=False, serializer_class=InstanceModifyOpSerializer) + @action( + methods=["GET"], + detail=False, + serializer_class=InstanceModifyOpSerializer, + queryset=InstanceOperateRecord.objects.select_related("ticket").order_by("-create_at"), + filter_class=InstanceOpRecordListFilter, + ) def get_instance_operate_records(self, request, *args, **kwargs): - validated_data = self.params_validate(self.get_serializer_class()) - op_filters = Q(instance_id=validated_data["instance_id"]) - if validated_data.get("start_time"): - op_filters &= Q(create_at__gte=validated_data.get("start_time")) - - if validated_data.get("end_time"): - op_filters &= Q(create_at__lte=validated_data.get("end_time")) - - if validated_data.get("op_type"): - op_filters &= Q(ticket__ticket_type=validated_data.get("op_type")) - - if validated_data.get("op_status"): - op_filters &= Q(ticket__status=validated_data.get("op_status")) - - op_records = InstanceOperateRecord.objects.select_related("ticket").filter(op_filters).order_by("-create_at") - op_records_info = [ - { - "create_at": record.create_at, - "op_type": TicketType.get_choice_label(record.ticket.ticket_type), - "op_status": record.ticket.status, - "ticket_id": record.ticket.id, - "creator": record.creator, - } - for record in op_records - ] - op_records_page = self.paginate_queryset(op_records_info) - return self.get_paginated_response(op_records_page) + op_records_page_qs = self.paginate_queryset(self.filter_queryset(self.queryset)) + op_records_page_data = self.serializer_class(op_records_page_qs, many=True).data + return self.get_paginated_response(data=op_records_page_data) @swagger_auto_schema( operation_summary=_("查询可编辑单据流程描述"), @@ -610,49 +573,68 @@ def fast_create_cloud_component(self, request, *args, **kwargs): TicketHandler.fast_create_cloud_component_method(bk_biz_id, bk_cloud_id, ips, request.user.username) return Response() - @common_swagger_auto_schema( - operation_summary=_("批量审批"), - request_body=BatchApprovalSerializer(), + @swagger_auto_schema( + operation_summary=_("批量待办处理"), + request_body=BatchTodoOperateSerializer(), + responses={status.HTTP_200_OK: TodoSerializer(many=True)}, tags=[TICKET_TAG], ) - @action(methods=["POST"], detail=False, serializer_class=BatchApprovalSerializer) - def batch_approval(self, request, *args, **kwargs): + @action(methods=["POST"], detail=False, serializer_class=BatchTodoOperateSerializer) + def batch_process_todo(self, request, *args, **kwargs): """ - sns: 单号集合 - is_approved: 是否审批通过 + 批量处理待办: 返回处理后的待办列表 """ data = self.params_validate(self.get_serializer_class()) - ticket_ids, is_approved = data["ticket_ids"], data["is_approved"] - user, itsm_action = request.user.username, OperateNodeActionType.TRANSITION - params_list = [ - {"ticket_id": ticket, "action": itsm_action, "is_approved": is_approved, "operator": user} - for ticket in ticket_ids - ] - request_multi_thread(TicketHandler.approve_itsm_ticket, params_list) - return Response() + user = request.user.username + return Response(TicketHandler.batch_process_todo(user=user, **data)) @swagger_auto_schema( - operation_summary=_("批量待办处理"), - request_body=BatchTodoOperateSerializer(), + operation_summary=_("批量单据待办处理"), + request_body=BatchTicketOperateSerializer(), responses={status.HTTP_200_OK: TodoSerializer(many=True)}, tags=[TICKET_TAG], ) - @action(methods=["POST"], detail=False, serializer_class=BatchTodoOperateSerializer) - def batch_process_todo(self, request, *args, **kwargs): + @action(methods=["POST"], detail=False, serializer_class=BatchTicketOperateSerializer) + def batch_process_ticket(self, request, *args, **kwargs): """ - 批量处理待办: 返回处理后的待办列表 + 批量处理单据的待办，处理单据的第一个todo + 根据todo的类型可以触发不同的factor函数 """ - validated_data = self.params_validate(self.get_serializer_class()) - act = validated_data["action"] - - # 批量处理待办操作 - results = [] - for operation in validated_data["operations"]: - todo_id = operation["todo_id"] - params = operation["params"] - todo = Todo.objects.get(id=todo_id) - TodoActorFactory.actor(todo).process(request.user.username, act, params) - results.append(todo) - - # 使用 TodoSerializer 序列化响应数据 - return Response(TodoSerializer(results, many=True).data) + data = self.params_validate(self.get_serializer_class()) + user = request.user.username + + tickets = Ticket.objects.prefetch_related("todo_of_ticket").filter(id__in=data["ticket_ids"]) + # 找到单据第一个代办（排除INNER_APPROVE，这是任务流程的人工确认节点产生的，不允许在单据维度操作） + running_todos = [ + ticket.todo_of_ticket.exclude(type=TodoType.INNER_APPROVE).filter(status__in=TODO_RUNNING_STATUS).first() + for ticket in tickets + ] + operations = [{"todo_id": todo.id, "params": data["params"]} for todo in running_todos if todo] + + return Response(TicketHandler.batch_process_todo(user=user, action=data["action"], operations=operations)) + + @swagger_auto_schema( + operation_summary=_("获取单据关联任务流程信息"), + query_serializer=GetInnerFlowSerializer(), + tags=[TICKET_TAG], + ) + @action(methods=["GET"], detail=False, serializer_class=GetInnerFlowSerializer, filter_class=None) + def get_inner_flow_infos(self, request, *args, **kwargs): + """ + 获取单据关联后台任务的信息 + """ + ticket_ids = self.params_validate(self.get_serializer_class())["ticket_ids"].split(",") + inner_flows = Flow.objects.filter(flow_type=FlowType.INNER_FLOW, ticket_id__in=ticket_ids).values( + "ticket_id", "flow_obj_id", "flow_alias", "err_msg", "status" + ) + ticket__inner_flow_map: Dict[int, List] = {int(t_id): [] for t_id in ticket_ids} + for flow in inner_flows: + # 快速判断流程树是否存在：有root_id，不包含error_msg，并且流程状态是执行过 + has_tree = bool( + flow["flow_obj_id"] and not flow["err_msg"] and flow["status"] not in FLOW_NOT_EXECUTE_STATUS + ) + flow_info = {"flow_id": flow["flow_obj_id"], "flow_alias": flow["flow_alias"], "pipeline_tree": has_tree} + # 默认只将产生过pipeline tree的返回给前端 + if has_tree: + ticket__inner_flow_map[flow["ticket_id"]].append(flow_info) + return Response(ticket__inner_flow_map) From d669f9ecaa7aa246c17b1f16ef53aa440e8ecbd2 Mon Sep 17 00:00:00 2001 From: iSecloud <869820505@qq.com> Date: Tue, 10 Dec 2024 14:39:41 +0800 Subject: [PATCH 002/107] =?UTF-8?q?feat(backend):=20=E5=8D=95=E6=8D=AEtodo?= =?UTF-8?q?=E5=A4=84=E7=90=86=E4=BA=BA=E7=BB=9F=E4=B8=80=E5=8C=96=20#6755?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dbm-ui/backend/configuration/models/system.py | 11 +++ .../db_services/bigdata/resources/query.py | 4 +- .../db_services/mysql/dumper/handlers.py | 4 +- .../iam_app/handlers/drf_perm/storage.py | 9 +- dbm-ui/backend/ticket/builders/__init__.py | 2 +- .../builders/mysql/mysql_ha_full_backup.py | 67 --------------- .../redis/redis_toolbox_redis_scale_updown.py | 2 +- dbm-ui/backend/ticket/constants.py | 6 +- dbm-ui/backend/ticket/flow_manager/inner.py | 6 +- dbm-ui/backend/ticket/flow_manager/itsm.py | 3 - .../backend/ticket/flow_manager/resource.py | 7 +- dbm-ui/backend/ticket/flow_manager/timer.py | 1 - dbm-ui/backend/ticket/handler.py | 83 +++++++++---------- dbm-ui/backend/ticket/models/ticket.py | 15 ++-- dbm-ui/backend/ticket/models/todo.py | 37 ++++++--- dbm-ui/backend/ticket/serializers.py | 4 +- dbm-ui/backend/ticket/todos/pipeline_todo.py | 5 +- dbm-ui/backend/ticket/views.py | 8 +- 18 files changed, 110 insertions(+), 164 deletions(-) delete mode 100644 dbm-ui/backend/ticket/builders/mysql/mysql_ha_full_backup.py diff --git a/dbm-ui/backend/configuration/models/system.py b/dbm-ui/backend/configuration/models/system.py index 9698f1b66c..b92cb32bf7 100644 --- a/dbm-ui/backend/configuration/models/system.py +++ b/dbm-ui/backend/configuration/models/system.py @@ -19,6 +19,7 @@ from backend.bk_web.constants import LEN_LONG, LEN_NORMAL from backend.bk_web.models import AuditedModel from backend.configuration import constants +from backend.configuration.constants import BizSettingsEnum from backend.db_meta.enums import ClusterType logger = logging.getLogger("root") @@ -192,3 +193,13 @@ def get_exact_hosting_biz(cls, bk_biz_id: int, cluster_type: str) -> int: return bk_biz_id return env.DBA_APP_BK_BIZ_ID + + @classmethod + def get_assistance(cls, bk_biz_id: int): + """ + 获取业务协助人 + """ + assistance_flag = BizSettings.get_setting_value(bk_biz_id, key=BizSettingsEnum.BIZ_ASSISTANCE_SWITCH) or False + if not assistance_flag: + return [] + return BizSettings.get_setting_value(bk_biz_id, key=BizSettingsEnum.BIZ_ASSISTANCE_VARS, default=[]) diff --git a/dbm-ui/backend/db_services/bigdata/resources/query.py b/dbm-ui/backend/db_services/bigdata/resources/query.py index f6cbbcea13..252daf38c7 100644 --- a/dbm-ui/backend/db_services/bigdata/resources/query.py +++ b/dbm-ui/backend/db_services/bigdata/resources/query.py @@ -21,7 +21,7 @@ from backend.db_proxy.models import ClusterExtension from backend.db_services.dbbase.resources import query from backend.db_services.ipchooser.query.resource import ResourceQueryHelper -from backend.ticket.constants import TICKET_RUNNING_STATUS +from backend.ticket.constants import TICKET_RUNNING_STATUS_SET from backend.ticket.models import InstanceOperateRecord from backend.utils.time import datetime2str @@ -65,7 +65,7 @@ def _filter_instance_hook(cls, bk_biz_id, query_params, instances, **kwargs): # 获取实例的操作与实例记录 records = InstanceOperateRecord.objects.filter( - instance_id__in=instance_ids, ticket__status__in=TICKET_RUNNING_STATUS + instance_id__in=instance_ids, ticket__status__in=TICKET_RUNNING_STATUS_SET ) instance_operate_records_map: Dict[int, List] = defaultdict(list) for record in records: diff --git a/dbm-ui/backend/db_services/mysql/dumper/handlers.py b/dbm-ui/backend/db_services/mysql/dumper/handlers.py index 06db016c68..fd0ae9448c 100644 --- a/dbm-ui/backend/db_services/mysql/dumper/handlers.py +++ b/dbm-ui/backend/db_services/mysql/dumper/handlers.py @@ -15,7 +15,7 @@ from backend.db_meta.enums import InstanceInnerRole from backend.db_meta.models import Cluster from backend.db_services.mysql.dumper.models import DumperSubscribeConfig -from backend.ticket.constants import TICKET_RUNNING_STATUS, FlowType, TicketFlowStatus, TicketStatus, TicketType +from backend.ticket.constants import TICKET_RUNNING_STATUS_SET, FlowType, TicketFlowStatus, TicketStatus, TicketType from backend.ticket.models import Flow, Ticket @@ -66,7 +66,7 @@ def patch_dumper_list_info(cls, dumper_results: List[Dict], bk_biz_id: int = 0, dumper_ticket_types.remove(TicketType.TBINLOGDUMPER_INSTALL) dumper_ticket_types.extend([TicketType.MYSQL_MASTER_SLAVE_SWITCH, TicketType.MYSQL_MASTER_FAIL_OVER]) active_tickets = Ticket.objects.filter( - bk_biz_id=bk_biz_id, status__in=TICKET_RUNNING_STATUS, ticket_type__in=dumper_ticket_types + bk_biz_id=bk_biz_id, status__in=TICKET_RUNNING_STATUS_SET, ticket_type__in=dumper_ticket_types ) # 获取每个dumper单据状态与id的映射 dumper_inst_id__ticket: Dict[int, str] = {} diff --git a/dbm-ui/backend/iam_app/handlers/drf_perm/storage.py b/dbm-ui/backend/iam_app/handlers/drf_perm/storage.py index 5e0166ba8a..ff48a2a0f8 100644 --- a/dbm-ui/backend/iam_app/handlers/drf_perm/storage.py +++ b/dbm-ui/backend/iam_app/handlers/drf_perm/storage.py @@ -39,10 +39,15 @@ def __init__(self, actions: List[ActionMeta] = None, resource_meta: ResourceMeta def instance_id_getter(self, request, view): file_path = get_request_key_id(request, "file_path") file_path_list = get_request_key_id(request, "file_path_list") or [file_path] + medium_types = MediumEnum.get_values() + + # 解析路径下的文件类型 + try: + is_all_pkg = set([(path.strip("/").split("/")[1] in medium_types) for path in file_path_list]) + except IndexError: + raise PermissionDeniedError(_("文件操作路径{}不合法，请联系管理员").format(file_path_list)) # 保证文件列表都是同种类型，即不允许同时操作介质文件和业务文件(一般也无此需求) - medium_types = MediumEnum.get_values() - is_all_pkg = set([(path.strip("/").split("/")[1] in medium_types) for path in file_path_list]) if len(is_all_pkg) > 1: raise PermissionDeniedError(_("不允许同时操作业务临时文件和介质文件")) diff --git a/dbm-ui/backend/ticket/builders/__init__.py b/dbm-ui/backend/ticket/builders/__init__.py index a20de5af14..20b3bdd564 100644 --- a/dbm-ui/backend/ticket/builders/__init__.py +++ b/dbm-ui/backend/ticket/builders/__init__.py @@ -142,7 +142,7 @@ def get_params(self): {"key": "domain", "value": "\n".join(cluster_domains)}, {"key": "summary", "value": self.ticket.remark}, {"key": "approver", "value": self.get_approvers()}, - {"key": "ticket_url", "value": f"{self.ticket.url}&isFullscreen=true"}, + {"key": "ticket_url", "value": self.ticket.iframe_url}, ], "dynamic_fields": [], "meta": { diff --git a/dbm-ui/backend/ticket/builders/mysql/mysql_ha_full_backup.py b/dbm-ui/backend/ticket/builders/mysql/mysql_ha_full_backup.py deleted file mode 100644 index ae152a99dc..0000000000 --- a/dbm-ui/backend/ticket/builders/mysql/mysql_ha_full_backup.py +++ /dev/null @@ -1,67 +0,0 @@ -# -*- coding: utf-8 -*- -""" -TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. -Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. -Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. -You may obtain a copy of the License at https://opensource.org/licenses/MIT -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on -an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the -specific language governing permissions and limitations under the License. -""" - -from django.utils.translation import ugettext_lazy as _ -from rest_framework import serializers - -from backend.db_meta.enums import ClusterDBHAStatusFlags, InstanceInnerRole -from backend.db_meta.models import Cluster -from backend.flow.consts import MySQLBackupFileTagEnum, MySQLBackupTypeEnum -from backend.flow.engine.controller.mysql import MySQLController -from backend.ticket import builders -from backend.ticket.builders.common.base import fetch_cluster_ids -from backend.ticket.builders.mysql.base import BaseMySQLHATicketFlowBuilder, MySQLBaseOperateDetailSerializer -from backend.ticket.constants import FlowRetryType, TicketType - - -class MySQLHaFullBackupDetailSerializer(MySQLBaseOperateDetailSerializer): - class FullBackupDataInfoSerializer(serializers.Serializer): - class ClusterDetailSerializer(serializers.Serializer): - cluster_id = serializers.IntegerField(help_text=_("集群ID")) - backup_local = serializers.ChoiceField( - help_text=_("备份位置"), choices=InstanceInnerRole.get_choices(), default=InstanceInnerRole.SLAVE.value - ) - - # 废弃online，暂时不需要传递 - # online = serializers.BooleanField(help_text=_("是否在线备份"), required=False) - backup_type = serializers.ChoiceField(help_text=_("备份类型"), choices=MySQLBackupTypeEnum.get_choices()) - file_tag = serializers.ChoiceField(help_text=_("备份文件tag"), choices=MySQLBackupFileTagEnum.get_choices()) - clusters = serializers.ListSerializer(help_text=_("集群信息"), child=ClusterDetailSerializer()) - - infos = FullBackupDataInfoSerializer() - - def validate(self, attrs): - try: - self.validate_cluster_can_access(attrs) - except serializers.ValidationError as e: - clusters = Cluster.objects.filter(id__in=fetch_cluster_ids(details=attrs)) - id__cluster = {cluster.id: cluster for cluster in clusters} - # 如果备份位置选的是master，但是slave异常，则认为是可以的 - for info in attrs["infos"]["clusters"]: - if info["backup_local"] != InstanceInnerRole.MASTER: - raise serializers.ValidationError(e) - if id__cluster[info["cluster_id"]].status_flag & ClusterDBHAStatusFlags.BackendMasterUnavailable: - raise serializers.ValidationError(e) - - return attrs - - -class MySQLHaFullBackupFlowParamBuilder(builders.FlowParamBuilder): - """MySQL HA 备份执行单据参数""" - - controller = MySQLController.mysql_full_backup_scene - - -@builders.BuilderFactory.register(TicketType.MYSQL_HA_FULL_BACKUP) -class MySQLHaFullBackupFlowBuilder(BaseMySQLHATicketFlowBuilder): - serializer = MySQLHaFullBackupDetailSerializer - inner_flow_builder = MySQLHaFullBackupFlowParamBuilder - retry_type = FlowRetryType.AUTO_RETRY diff --git a/dbm-ui/backend/ticket/builders/redis/redis_toolbox_redis_scale_updown.py b/dbm-ui/backend/ticket/builders/redis/redis_toolbox_redis_scale_updown.py index 76aadb656a..9dc534a592 100644 --- a/dbm-ui/backend/ticket/builders/redis/redis_toolbox_redis_scale_updown.py +++ b/dbm-ui/backend/ticket/builders/redis/redis_toolbox_redis_scale_updown.py @@ -28,7 +28,7 @@ class RedisScaleUpDownDetailSerializer(SkipToRepresentationMixin, serializers.Serializer): """redis集群容量变更""" - class InfoSerializer(DisplayInfoSerializer, ClusterValidateMixin, serializers.Serializer): + class InfoSerializer(DisplayInfoSerializer, ClusterValidateMixin): class ResourceSpecSerializer(serializers.Serializer): class BackendGroupSerializer(serializers.Serializer): spec_id = serializers.IntegerField(help_text=_("规格ID")) diff --git a/dbm-ui/backend/ticket/constants.py b/dbm-ui/backend/ticket/constants.py index 407c632c9e..d1fa5b0eab 100644 --- a/dbm-ui/backend/ticket/constants.py +++ b/dbm-ui/backend/ticket/constants.py @@ -96,7 +96,7 @@ class TicketStatus(str, StructuredEnum): # 单据[正在进行]的状态合集 -TICKET_RUNNING_STATUS = [ +TICKET_RUNNING_STATUS_SET = [ TicketStatus.APPROVE, TicketStatus.TODO, TicketStatus.RESOURCE_REPLENISH, @@ -104,7 +104,7 @@ class TicketStatus(str, StructuredEnum): TicketStatus.TIMER, ] # 单据[包含TODO]的状态合集 -TICKET_TODO_STATUS = [ +TICKET_TODO_STATUS_SET = [ TicketStatus.APPROVE, TicketStatus.TODO, TicketStatus.RESOURCE_REPLENISH, @@ -112,7 +112,7 @@ class TicketStatus(str, StructuredEnum): TicketStatus.RUNNING, ] # 单据[失败]的状态合集 -TICKET_FAILED_STATUS = [TicketStatus.REVOKED, TicketStatus.TERMINATED, TicketStatus.FAILED] +TICKET_FAILED_STATUS_SET = [TicketStatus.REVOKED, TicketStatus.TERMINATED, TicketStatus.FAILED] class TicketFlowStatus(str, StructuredEnum): diff --git a/dbm-ui/backend/ticket/flow_manager/inner.py b/dbm-ui/backend/ticket/flow_manager/inner.py index b607e0009c..9773c200a1 100644 --- a/dbm-ui/backend/ticket/flow_manager/inner.py +++ b/dbm-ui/backend/ticket/flow_manager/inner.py @@ -27,8 +27,8 @@ from backend.ticket.constants import ( BAMBOO_STATE__TICKET_STATE_MAP, FlowCallbackType, - FlowMsgType, FlowErrCode, + FlowMsgType, TicketFlowStatus, TicketType, TodoStatus, @@ -122,13 +122,11 @@ def _url(self) -> str: return f"{env.BK_SAAS_HOST}/{self.ticket.bk_biz_id}/task-history/detail/{self.root_id}" def create_failed_todo(self): - # 创建一条todo失败记录，在失败时变更为TODO状态 Todo.objects.create( name=_("【{}】单据任务执行失败，待处理").format(self.ticket.get_ticket_type_display()), flow=self.flow_obj, ticket=self.ticket, type=TodoType.INNER_FAILED, - operators=[self.ticket.creator], context=BaseTodoContext(self.flow_obj.id, self.ticket.id).to_dict(), status=TodoStatus.DONE_SUCCESS, ) @@ -280,7 +278,7 @@ def _raw_status(self) -> str: @property def _status(self) -> str: status = self._raw_status - if status in [constants.TicketFlowStatus.SUCCEEDED, *constants.TICKET_FAILED_STATUS]: + if status in [constants.TicketFlowStatus.SUCCEEDED, *constants.TICKET_FAILED_STATUS_SET]: return constants.TicketFlowStatus.SUCCEEDED return status diff --git a/dbm-ui/backend/ticket/flow_manager/itsm.py b/dbm-ui/backend/ticket/flow_manager/itsm.py index d96c4b3f8f..c26587bc8f 100644 --- a/dbm-ui/backend/ticket/flow_manager/itsm.py +++ b/dbm-ui/backend/ticket/flow_manager/itsm.py @@ -113,14 +113,11 @@ def _url(self) -> str: def _run(self) -> str: itsm_fields = {f["key"]: f["value"] for f in self.flow_obj.details["fields"]} - # 创建审批todo - operators = itsm_fields["approver"].split(",") Todo.objects.create( name=_("【{}】单据等待审批").format(self.ticket.get_ticket_type_display()), flow=self.flow_obj, ticket=self.ticket, type=TodoType.ITSM, - operators=operators, context=ItsmTodoContext(self.flow_obj.id, self.ticket.id).to_dict(), ) # 创建单据 diff --git a/dbm-ui/backend/ticket/flow_manager/resource.py b/dbm-ui/backend/ticket/flow_manager/resource.py index bceb1d6651..4fa5166810 100644 --- a/dbm-ui/backend/ticket/flow_manager/resource.py +++ b/dbm-ui/backend/ticket/flow_manager/resource.py @@ -204,17 +204,14 @@ def create_replenish_todo(self): from backend.ticket.todos.pause_todo import ResourceReplenishTodoContext - user = self.ticket.creator - admins = DBAdministrator.get_biz_db_type_admins(self.ticket.bk_biz_id, self.ticket.group) + dba = DBAdministrator.get_biz_db_type_admins(self.ticket.bk_biz_id, self.ticket.group) Todo.objects.create( name=_("【{}】流程所需资源不足").format(self.ticket.get_ticket_type_display()), flow=self.flow_obj, ticket=self.ticket, type=TodoType.RESOURCE_REPLENISH, - # 需要提单人和管理人员共同关注todo单 - operators=[user, *admins], context=ResourceReplenishTodoContext( - flow_id=self.flow_obj.id, ticket_id=self.ticket.id, user=user, administrators=admins + flow_id=self.flow_obj.id, ticket_id=self.ticket.id, user=self.ticket.creator, administrators=dba ).to_dict(), ) diff --git a/dbm-ui/backend/ticket/flow_manager/timer.py b/dbm-ui/backend/ticket/flow_manager/timer.py index 8dfdae4d8d..287aa9bd0c 100644 --- a/dbm-ui/backend/ticket/flow_manager/timer.py +++ b/dbm-ui/backend/ticket/flow_manager/timer.py @@ -98,7 +98,6 @@ def _run(self) -> Union[int, str]: flow=self.flow_obj, ticket=self.ticket, type=TodoType.APPROVE, - operators=[self.ticket.creator], context=PauseTodoContext(self.flow_obj.id, self.ticket.id).to_dict(), ) else: diff --git a/dbm-ui/backend/ticket/handler.py b/dbm-ui/backend/ticket/handler.py index 9800c5fb0a..e60b776e60 100644 --- a/dbm-ui/backend/ticket/handler.py +++ b/dbm-ui/backend/ticket/handler.py @@ -11,7 +11,6 @@ import itertools import json import logging -import time from typing import Dict, List from django.db import transaction @@ -36,7 +35,6 @@ TicketFlowStatus, TicketStatus, TicketType, - TodoStatus, TodoType, ) from backend.ticket.exceptions import TicketFlowsConfigException @@ -240,11 +238,11 @@ def approve_itsm_ticket(cls, ticket_id, action, operator, **kwargs): {"key": itsm_fields[1], "value": act_msg}, ] params.update(sn=sn, state_id=state_id, action_type=action, operator=operator, fields=fields) - ItsmApi.operate_node(params, use_admin=True) + ItsmApi.operate_node(params) # 终止/撤销单据 elif action in [OperateNodeActionType.TERMINATE, OperateNodeActionType.WITHDRAW]: params.update(sn=sn, action_type=action, operator=operator) - ItsmApi.operate_ticket(params, use_admin=True) + ItsmApi.operate_ticket(params) return sn @@ -424,51 +422,44 @@ def ticket_status_standardization(cls): batch = 50 # 标准化只针对running的单据，其他状态单据不影响 - running_tickets = Ticket.objects.filter(status=TicketStatus.RUNNING) - count = running_tickets.count() - for current in range(0, count, batch): - for ticket in running_tickets[current : current + batch]: - raw_status = ticket.status - ticket.status = RUNNING_FLOW__TICKET_STATUS[ticket.current_flow().flow_type] - ticket.save() - print(f"ticket[{ticket.id}] status {raw_status} ---> {ticket.status}") - time.sleep(1) + running_tickets = list(Ticket.objects.filter(status=TicketStatus.RUNNING)) + for ticket in running_tickets: + raw_status = ticket.status + ticket.status = RUNNING_FLOW__TICKET_STATUS[ticket.current_flow().flow_type] + ticket.save() + print(f"ticket[{ticket.id}] status {raw_status} ---> {ticket.status}") # 失败的单据要增加一条todo关联 failed_tickets = Ticket.objects.prefetch_related("flows").filter(status=TicketStatus.FAILED) - for current in range(0, count, batch): - for ticket in failed_tickets[current : current + batch]: - inner_flow = ticket.flows.filter(flow_type=FlowType.INNER_FLOW, status=TicketFlowStatus.FAILED).first() - if not inner_flow or inner_flow.todo_of_flow.exists(): - continue - Todo.objects.create( - name=_("【{}】单据任务执行失败，待处理").format(ticket.get_ticket_type_display()), - flow=inner_flow, - ticket=ticket, - type=TodoType.INNER_FAILED, - operators=[ticket.creator], - context=BaseTodoContext(inner_flow.id, ticket.id).to_dict(), - status=TodoStatus.TODO, - ) - print(f"ticket[{ticket.id}] add a failed todo") - time.sleep(1) + todos = [] + for ticket in failed_tickets: + inner_flow = ticket.flows.filter(flow_type=FlowType.INNER_FLOW, status=TicketFlowStatus.FAILED).first() + if not inner_flow or inner_flow.todo_of_flow.exists(): + continue + todo = Todo( + name=_("【{}】单据任务执行失败，待处理").format(ticket.get_ticket_type_display()), + flow=inner_flow, + ticket=ticket, + type=TodoType.INNER_FAILED, + context=BaseTodoContext(inner_flow.id, ticket.id).to_dict(), + ) + todos.append(todo) + print(f"ticket[{ticket.id}] add a failed todo") # 待审批的单据要增加一条todo关联 itsm_tickets = Ticket.objects.prefetch_related("flows").filter(status=TicketStatus.FAILED) - for current in range(0, count, batch): - for ticket in itsm_tickets[current : current + batch]: - itsm_flow = ticket.flows.filter(flow_type=FlowType.BK_ITSM, status=TicketFlowStatus.RUNNING).first() - if not itsm_flow or itsm_flow.todo_of_flow.exists(): - continue - itsm_fields = {f["key"]: f["value"] for f in itsm_flow.details["fields"]} - operators = itsm_fields["approver"].split(",") - Todo.objects.create( - name=_("【{}】单据等待审批").format(ticket.get_ticket_type_display()), - flow=itsm_flow, - ticket=ticket, - type=TodoType.ITSM, - operators=operators, - context=ItsmTodoContext(itsm_flow.id, ticket.id).to_dict(), - ) - print(f"ticket[{ticket.id}] add a itsm todo") - time.sleep(1) + for ticket in itsm_tickets: + itsm_flow = ticket.flows.filter(flow_type=FlowType.BK_ITSM, status=TicketFlowStatus.RUNNING).first() + if not itsm_flow or itsm_flow.todo_of_flow.exists(): + continue + todo = Todo( + name=_("【{}】单据等待审批").format(ticket.get_ticket_type_display()), + flow=itsm_flow, + ticket=ticket, + type=TodoType.ITSM, + context=ItsmTodoContext(itsm_flow.id, ticket.id).to_dict(), + ) + todos.append(todo) + print(f"ticket[{ticket.id}] add a itsm todo") + + Todo.objects.bulk_create(todos, batch_size=batch) diff --git a/dbm-ui/backend/ticket/models/ticket.py b/dbm-ui/backend/ticket/models/ticket.py index 89844179b2..0df2ee00a1 100644 --- a/dbm-ui/backend/ticket/models/ticket.py +++ b/dbm-ui/backend/ticket/models/ticket.py @@ -25,7 +25,7 @@ from backend.db_monitor.exceptions import AutofixException from backend.ticket.constants import ( EXCLUSIVE_TICKET_EXCEL_PATH, - TICKET_RUNNING_STATUS, + TICKET_RUNNING_STATUS_SET, FlowRetryType, FlowType, TicketFlowStatus, @@ -116,7 +116,12 @@ class Meta: @property def url(self): - return f"{env.BK_SAAS_HOST}/{self.bk_biz_id}/ticket-manage/index?id={self.id}" + return f"{env.BK_SAAS_HOST}/ticket/{self.id}" + + @property + def iframe_url(self): + """iframe 单据链接，目前仅用在itsm表单""" + return f"{env.BK_SAAS_HOST}/sub/ticket/{self.id}" def set_status(self, status): self.status = status @@ -124,7 +129,7 @@ def set_status(self, status): def get_cost_time(self): # 计算耗时 - if self.status in [TicketStatus.PENDING, *TICKET_RUNNING_STATUS]: + if self.status in [TicketStatus.PENDING, *TICKET_RUNNING_STATUS_SET]: return calculate_cost_time(timezone.now(), self.create_at) return calculate_cost_time(self.update_at, self.create_at) @@ -362,7 +367,7 @@ def summary(self): def get_cluster_records_map(cls, cluster_ids: List[int]): """获取集群与操作记录之间的映射关系""" records = cls.objects.select_related("ticket", "flow").filter( - cluster_id__in=cluster_ids, ticket__status__in=TICKET_RUNNING_STATUS + cluster_id__in=cluster_ids, ticket__status__in=TICKET_RUNNING_STATUS_SET ) cluster_operate_records_map: Dict[int, List] = defaultdict(list) for record in records: @@ -428,7 +433,7 @@ def summary(self): def get_instance_records_map(cls, instance_ids: List[Union[int, str]]): """获取实例与操作记录之间的映射关系??????""" records = InstanceOperateRecord.objects.select_related("ticket").filter( - instance_id__in=instance_ids, ticket__status__in=TICKET_RUNNING_STATUS + instance_id__in=instance_ids, ticket__status__in=TICKET_RUNNING_STATUS_SET ) instance_operator_record_map: Dict[int, List] = defaultdict(list) for record in records: diff --git a/dbm-ui/backend/ticket/models/todo.py b/dbm-ui/backend/ticket/models/todo.py index c2335e1574..c437256aef 100644 --- a/dbm-ui/backend/ticket/models/todo.py +++ b/dbm-ui/backend/ticket/models/todo.py @@ -17,8 +17,7 @@ from backend import env from backend.bk_web.constants import LEN_MIDDLE, LEN_SHORT from backend.bk_web.models import AuditedModel -from backend.configuration.constants import BizSettingsEnum -from backend.configuration.models import BizSettings +from backend.configuration.models import BizSettings, DBAdministrator from backend.ticket.constants import ( TODO_RUNNING_STATUS, FlowMsgStatus, @@ -36,16 +35,31 @@ class TodoManager(models.Manager): def exist_unfinished(self): return self.filter(status__in=TODO_RUNNING_STATUS).exists() + def get_operators(self, todo_type, ticket, operators): + # 获得提单人，dba，业务协助人. TODO: 后续还会细分主、备、二线DBA，以及明确区分协助人角色 + creator = [ticket.creator] + dba = DBAdministrator.get_biz_db_type_admins(ticket.bk_biz_id, ticket.group) + biz_helpers = BizSettings.get_assistance(ticket.bk_biz_id) + + # 构造单据状态与处理人之间的对应关系 + # - 审批中：提单人可撤销，dba可处理 + # - 待执行：提单人 + 单据协助人 + # - 待继续：提单人 + dba + 单据协助人 + # - 待补货：提单人 + dba + 单据协助人 + # - 已失败：提单人 + dba + 单据协助人 + todo_operators_map = { + TodoType.ITSM: dba, + TodoType.APPROVE: creator + biz_helpers, + TodoType.INNER_APPROVE: creator + dba + biz_helpers, + TodoType.RESOURCE_REPLENISH: creator + dba + biz_helpers, + TodoType.INNER_FAILED: creator + dba + biz_helpers, + } + operators = list(set(operators + todo_operators_map.get(todo_type, []))) + return operators + def create(self, **kwargs): - bk_biz_id = kwargs["ticket"].bk_biz_id - assistance_flag = BizSettings.get_setting_value(bk_biz_id, key=BizSettingsEnum.BIZ_ASSISTANCE_SWITCH) or False - if assistance_flag: - # 获取业务协助人列表 - biz_helpers = BizSettings.get_setting_value(bk_biz_id, key=BizSettingsEnum.BIZ_ASSISTANCE_VARS, default=[]) - # 合并单据处理人 + 业务协助人 - kwargs["operators"] = kwargs.get("operators", []) + biz_helpers - # 对操作者去重 - kwargs["operators"] = list(set(kwargs["operators"])) + operators = self.get_operators(kwargs["type"], kwargs["ticket"], kwargs.get("operators", [])) + kwargs["operators"] = operators todo = super().create(**kwargs) send_msg_for_flow.apply_async( kwargs={ @@ -56,7 +70,6 @@ def create(self, **kwargs): "receiver": todo.creator, } ) - return todo diff --git a/dbm-ui/backend/ticket/serializers.py b/dbm-ui/backend/ticket/serializers.py index c958cee5d7..7e423b2698 100644 --- a/dbm-ui/backend/ticket/serializers.py +++ b/dbm-ui/backend/ticket/serializers.py @@ -24,7 +24,7 @@ from backend.ticket import mock_data from backend.ticket.builders import BuilderFactory from backend.ticket.constants import ( - TICKET_RUNNING_STATUS, + TICKET_RUNNING_STATUS_SET, TODO_RUNNING_STATUS, FlowType, TicketFlowStatus, @@ -137,7 +137,7 @@ def get_status_display(self, obj): return TicketStatus.get_choice_label(obj.status) def get_cost_time(self, obj): - if obj.status in [TicketStatus.PENDING, *TICKET_RUNNING_STATUS]: + if obj.status in [TicketStatus.PENDING, *TICKET_RUNNING_STATUS_SET]: return calculate_cost_time(timezone.now(), obj.create_at) return calculate_cost_time(obj.update_at, obj.create_at) diff --git a/dbm-ui/backend/ticket/todos/pipeline_todo.py b/dbm-ui/backend/ticket/todos/pipeline_todo.py index ce68891bec..cd9f0b019a 100644 --- a/dbm-ui/backend/ticket/todos/pipeline_todo.py +++ b/dbm-ui/backend/ticket/todos/pipeline_todo.py @@ -15,8 +15,7 @@ from backend.flow.engine.bamboo.engine import BambooEngine from backend.ticket import todos -from backend.ticket.constants import TodoStatus -from backend.ticket.constants import TodoType +from backend.ticket.constants import TodoStatus, TodoType from backend.ticket.models import TodoHistory from backend.ticket.todos import ActionType, BaseTodoContext @@ -73,7 +72,5 @@ def create(cls, ticket, flow, root_id, node_id): flow=flow, ticket=ticket, type=TodoType.INNER_APPROVE, - # todo: 待办人暂定为提单人 - operators=[ticket.creator], context=PipelineTodoContext(flow.id, ticket.id, root_id, node_id).to_dict(), ) diff --git a/dbm-ui/backend/ticket/views.py b/dbm-ui/backend/ticket/views.py index b7d2f631d7..ea244e4235 100644 --- a/dbm-ui/backend/ticket/views.py +++ b/dbm-ui/backend/ticket/views.py @@ -43,8 +43,8 @@ from backend.ticket.builders.common.base import InfluxdbTicketFlowBuilderPatchMixin, fetch_cluster_ids from backend.ticket.constants import ( FLOW_NOT_EXECUTE_STATUS, - TICKET_RUNNING_STATUS, - TICKET_TODO_STATUS, + TICKET_RUNNING_STATUS_SET, + TICKET_TODO_STATUS_SET, TODO_RUNNING_STATUS, CountType, FlowType, @@ -200,7 +200,7 @@ def _verify_influxdb_duplicate_ticket(ticket_type, details, user, active_tickets def verify_duplicate_ticket(self, ticket_type, details, user): """校验是否重复提交""" active_tickets = self.get_queryset().filter( - ticket_type=ticket_type, status__in=TICKET_RUNNING_STATUS, creator=user + ticket_type=ticket_type, status__in=TICKET_RUNNING_STATUS_SET, creator=user ) # influxdb 相关操作单独适配，这里暂时没有找到更好的写法，唯一的改进就是创建单据时，会提前提取出对比内容，比如instances @@ -451,7 +451,7 @@ def get_tickets_count(self, request, *args, **kwargs): # 我的代办 todo_status = ( tickets.filter( - status__in=TICKET_TODO_STATUS, + status__in=TICKET_TODO_STATUS_SET, todo_of_ticket__operators__contains=user, todo_of_ticket__status__in=TODO_RUNNING_STATUS, ) From b8ef8747f57c91632a925b521bb4f17c17c23028 Mon Sep 17 00:00:00 2001 From: xfwduke Date: Wed, 11 Dec 2024 15:03:20 +0800 Subject: [PATCH 003/107] =?UTF-8?q?fix(mysql):=20=E6=B8=85=E6=A1=A3?= =?UTF-8?q?=E5=9C=A8=E4=B8=AD=E6=8E=A7=E8=B6=85=E6=97=B6=20#8545?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../pkg/components/rename_dbs/pkg/drop_db.go | 2 +- .../components/truncate/pkg/safe_drop_tables.go | 2 +- .../dbactuator/pkg/components/truncate/via_ctl.go | 14 +++++++++++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/components/rename_dbs/pkg/drop_db.go b/dbm-services/mysql/db-tools/dbactuator/pkg/components/rename_dbs/pkg/drop_db.go index 4276cbd6ac..0b73efde26 100644 --- a/dbm-services/mysql/db-tools/dbactuator/pkg/components/rename_dbs/pkg/drop_db.go +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/components/rename_dbs/pkg/drop_db.go @@ -19,7 +19,7 @@ func DropDB(conn *sqlx.Conn, dbName, to string, onlyStageTable bool) error { return fmt.Errorf(`db "%s" is not trans clean`, dbName) } - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) defer cancel() _, err = conn.ExecContext( diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/components/truncate/pkg/safe_drop_tables.go b/dbm-services/mysql/db-tools/dbactuator/pkg/components/truncate/pkg/safe_drop_tables.go index a58d91e8de..01d6f36904 100644 --- a/dbm-services/mysql/db-tools/dbactuator/pkg/components/truncate/pkg/safe_drop_tables.go +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/components/truncate/pkg/safe_drop_tables.go @@ -29,7 +29,7 @@ func safeDropSourceTable(conn *sqlx.Conn, dbName, stageDBName, tableName string) return fmt.Errorf("table `%s` does not exist in `%s`", tableName, stageDBName) } - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() // 留着这里, drop table 不需要预处理触发器 diff --git a/dbm-services/mysql/db-tools/dbactuator/pkg/components/truncate/via_ctl.go b/dbm-services/mysql/db-tools/dbactuator/pkg/components/truncate/via_ctl.go index 90f5c40b07..49131ccb47 100644 --- a/dbm-services/mysql/db-tools/dbactuator/pkg/components/truncate/via_ctl.go +++ b/dbm-services/mysql/db-tools/dbactuator/pkg/components/truncate/via_ctl.go @@ -173,10 +173,22 @@ func (c *ViaCtlComponent) dropSourceTables() error { return nil } +// 如果库下面有很多表, 直接 drop database 很容易超时 +// 得循环起来先一个一个把表 drop 了 func (c *ViaCtlComponent) dropSourceDBs() error { for db := range c.dbTablesMap { stageDBName := generateStageDBName(c.Param.StageDBHeader, c.Param.FlowTimeStr, db) - err := rpkg.DropDB(c.dbConn, db, stageDBName, true) + + logger.Info(fmt.Sprintf("drop source dbs %v should drop it's tables", c.dbTablesMap[db])) + + err := tpkg.SafeDropSourceTables(c.dbConn, db, stageDBName, c.dbTablesMap[db]) + if err != nil { + logger.Error("drop source tables %v failed: ", c.dbTablesMap[db], err.Error()) + return err + } + logger.Info("drop source tables %v success", c.dbTablesMap[db]) + + err = rpkg.DropDB(c.dbConn, db, stageDBName, true) if err != nil { logger.Error( "drop db %s failed: %s", db, err.Error(), From 9e165b4f33a81dfb2337bb3c198effc932e44794 Mon Sep 17 00:00:00 2001 From: xfwduke Date: Tue, 10 Dec 2024 12:27:50 +0800 Subject: [PATCH 004/107] =?UTF-8?q?feat(mysql):=20mysql=E7=9B=91=E6=8E=A7?= =?UTF-8?q?=E6=8E=A2=E6=B5=8B=E5=90=8C=E6=AD=A5=E5=85=B3=E7=B3=BB=E6=98=AF?= =?UTF-8?q?=E5=90=A6=E7=AC=A6=E5=90=88meta=E5=90=8C=E6=AD=A5=E8=A1=A8?= =?UTF-8?q?=E6=8F=8F=E8=BF=B0=20#8481?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../mysql-crond/pkg/nginx_updater/init.go | 53 ++++++++++++ .../reverse_api/base_reverse_api_view.py | 4 +- .../db_proxy/reverse_api/common/__init__.py | 11 +++ .../reverse_api/common/impl/__init__.py | 11 +++ .../reverse_api/common/impl/list_nginx_ips.py | 19 +++++ .../reverse_api/{ => common}/views.py | 40 +++++---- .../reverse_api/mysql/impl/__init__.py | 12 +++ .../mysql/impl/get_instance_admin_password.py | 78 +++++++++++++++++ .../mysql/impl/list_instance_info.py | 84 +++++++++++++++++++ .../db_proxy/reverse_api/mysql/views.py | 21 +---- dbm-ui/backend/db_proxy/reverse_api/urls.py | 4 +- 11 files changed, 299 insertions(+), 38 deletions(-) create mode 100644 dbm-services/mysql/db-tools/mysql-crond/pkg/nginx_updater/init.go create mode 100644 dbm-ui/backend/db_proxy/reverse_api/common/__init__.py create mode 100644 dbm-ui/backend/db_proxy/reverse_api/common/impl/__init__.py create mode 100644 dbm-ui/backend/db_proxy/reverse_api/common/impl/list_nginx_ips.py rename dbm-ui/backend/db_proxy/reverse_api/{ => common}/views.py (56%) create mode 100644 dbm-ui/backend/db_proxy/reverse_api/mysql/impl/__init__.py create mode 100644 dbm-ui/backend/db_proxy/reverse_api/mysql/impl/get_instance_admin_password.py create mode 100644 dbm-ui/backend/db_proxy/reverse_api/mysql/impl/list_instance_info.py diff --git a/dbm-services/mysql/db-tools/mysql-crond/pkg/nginx_updater/init.go b/dbm-services/mysql/db-tools/mysql-crond/pkg/nginx_updater/init.go new file mode 100644 index 0000000000..d73d6a86d5 --- /dev/null +++ b/dbm-services/mysql/db-tools/mysql-crond/pkg/nginx_updater/init.go @@ -0,0 +1,53 @@ +package nginx_updater + +import ( + "bufio" + "os" +) + +const nginxAddrFile = "/home/mysql/nginx_conf/address.list" + +func Updater() { + addrs, err := readAddr() + if err != nil { + + } + + newAddrs, err := queryNewAddr(addrs) + + f, err := os.OpenFile(nginxAddrFile, os.O_CREATE|os.O_TRUNC, 0777) + if err != nil { + } + defer func() { + _ = f.Close() + }() + + for _, ad := range newAddrs { + _, _ = f.WriteString(ad + "\n") + } +} + +func readAddr() (res []string, err error) { + f, err := os.Open(nginxAddrFile) + if err != nil { + return nil, err + } + defer func() { + _ = f.Close() + }() + + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := scanner.Text() + res = append(res, line) + } + err = scanner.Err() + if err != nil { + return nil, err + } + return res, nil +} + +func queryNewAddr(addrs []string) (res []string, err error) { + return +} diff --git a/dbm-ui/backend/db_proxy/reverse_api/base_reverse_api_view.py b/dbm-ui/backend/db_proxy/reverse_api/base_reverse_api_view.py index 5108845d4b..70394ac0d7 100644 --- a/dbm-ui/backend/db_proxy/reverse_api/base_reverse_api_view.py +++ b/dbm-ui/backend/db_proxy/reverse_api/base_reverse_api_view.py @@ -11,7 +11,7 @@ import logging from types import FunctionType -from typing import Tuple +from typing import List, Tuple from django.utils.translation import ugettext as _ from rest_framework import permissions @@ -63,7 +63,7 @@ def _get_login_exempt_view_func(cls): def get_permissions(self): return [IPHasRegisteredPermission()] - def get_api_params(self) -> Tuple[int, str, int]: + def get_api_params(self) -> Tuple[int, str, List[int]]: """ return request bk_cloud_id, ip, port param """ diff --git a/dbm-ui/backend/db_proxy/reverse_api/common/__init__.py b/dbm-ui/backend/db_proxy/reverse_api/common/__init__.py new file mode 100644 index 0000000000..557cf21b01 --- /dev/null +++ b/dbm-ui/backend/db_proxy/reverse_api/common/__init__.py @@ -0,0 +1,11 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from .views import CommonReverseApiView diff --git a/dbm-ui/backend/db_proxy/reverse_api/common/impl/__init__.py b/dbm-ui/backend/db_proxy/reverse_api/common/impl/__init__.py new file mode 100644 index 0000000000..74292829f3 --- /dev/null +++ b/dbm-ui/backend/db_proxy/reverse_api/common/impl/__init__.py @@ -0,0 +1,11 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from .list_nginx_ips import list_nginx_ips diff --git a/dbm-ui/backend/db_proxy/reverse_api/common/impl/list_nginx_ips.py b/dbm-ui/backend/db_proxy/reverse_api/common/impl/list_nginx_ips.py new file mode 100644 index 0000000000..84fc0d6a35 --- /dev/null +++ b/dbm-ui/backend/db_proxy/reverse_api/common/impl/list_nginx_ips.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from typing import List + +from backend.db_proxy.constants import ExtensionType +from backend.db_proxy.models import DBExtension + + +def list_nginx_ips(bk_cloud_id: int) -> List[str]: + nginx = DBExtension.get_extension_in_cloud(bk_cloud_id=bk_cloud_id, extension_type=ExtensionType.NGINX.value) + return [n.details["ip"] for n in nginx] diff --git a/dbm-ui/backend/db_proxy/reverse_api/views.py b/dbm-ui/backend/db_proxy/reverse_api/common/views.py similarity index 56% rename from dbm-ui/backend/db_proxy/reverse_api/views.py rename to dbm-ui/backend/db_proxy/reverse_api/common/views.py index 3751a5e4d3..e5d38d8ba3 100644 --- a/dbm-ui/backend/db_proxy/reverse_api/views.py +++ b/dbm-ui/backend/db_proxy/reverse_api/common/views.py @@ -8,29 +8,39 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ +import logging from django.http import JsonResponse from django.utils.translation import ugettext_lazy as _ from backend.bk_web.swagger import common_swagger_auto_schema -from backend.db_proxy.models import DBCloudProxy from backend.db_proxy.reverse_api.base_reverse_api_view import BaseReverseApiView +from backend.db_proxy.reverse_api.common.impl import list_nginx_ips from backend.db_proxy.reverse_api.decorators import reverse_api +logger = logging.getLogger("root") -class CommonReverseApiView(BaseReverseApiView): - @common_swagger_auto_schema(operation_summary=_("云区域NGINX列表")) - @reverse_api(url_path="list_cloud_nginx") - def list_cloud_nginx(self, request): - bk_cloud_id, ip, port = self.get_api_params() - res = [] - for ele in DBCloudProxy.objects.filter(bk_cloud_id=bk_cloud_id): - res.append( - { - "internal-address": ele.internal_address, - "external-address": ele.external_address, - } - ) +class CommonReverseApiView(BaseReverseApiView): + @common_swagger_auto_schema(operation_summary=_("获取NGINX IP")) + @reverse_api(url_path="list_nginx_ips") + def list_nginx_ips(self, request, *args, **kwargs): + """ + 返回特定云区域的 NGINX IP 列表 + param: bk_cloud_id: int + return: ["ip1", "ip2", ...] + """ + bk_cloud_id, _, _ = self.get_api_params() + logger.info(f"bk_cloud_id: {bk_cloud_id}") + res = list_nginx_ips(bk_cloud_id=bk_cloud_id) + logger.info(f"res: {res}") - return JsonResponse({"result": True, "code": 0, "data": res, "message": "", "errors": None}) + return JsonResponse( + { + "result": True, + "code": 0, + "data": res, + "message": "", + "errors": None, + } + ) diff --git a/dbm-ui/backend/db_proxy/reverse_api/mysql/impl/__init__.py b/dbm-ui/backend/db_proxy/reverse_api/mysql/impl/__init__.py new file mode 100644 index 0000000000..73d80986e6 --- /dev/null +++ b/dbm-ui/backend/db_proxy/reverse_api/mysql/impl/__init__.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from .get_instance_admin_password import get_instance_admin_password +from .list_instance_info import list_instance_info diff --git a/dbm-ui/backend/db_proxy/reverse_api/mysql/impl/get_instance_admin_password.py b/dbm-ui/backend/db_proxy/reverse_api/mysql/impl/get_instance_admin_password.py new file mode 100644 index 0000000000..12bb9aa036 --- /dev/null +++ b/dbm-ui/backend/db_proxy/reverse_api/mysql/impl/get_instance_admin_password.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +import collections +from typing import List, Optional + +from backend.components import DBPrivManagerApi +from backend.configuration.constants import DB_ADMIN_USER_MAP, DBType +from backend.db_meta.enums import ClusterType, MachineType +from backend.db_meta.models import Machine, ProxyInstance, StorageInstance +from backend.utils.string import base64_decode + + +def get_instance_admin_password(bk_cloud_id: int, ip: str, port_list: Optional[List[int]] = None) -> dict: + """ + 目前不能正常工作 + """ + m = Machine.objects.get(bk_cloud_id=bk_cloud_id, ip=ip) + if m.cluster_type in [ClusterType.TenDBSingle, ClusterType.TenDBHA]: + dbtype = DBType.MySQL + elif m.cluster_type == ClusterType.TenDBCluster: + dbtype = DBType.TenDBCluster + else: + raise Exception(f"not support cluster type: {m.cluster_type}") # noqa + + if not port_list: + if m.machine_type in [MachineType.BACKEND, MachineType.REMOTE, MachineType.SINGLE]: + port_list = list( + StorageInstance.objects.filter( + machine__ip=ip, + machine__bk_cloud_id=bk_cloud_id, + ).values_list("port", flat=True) + ) + + elif m.machine_type == MachineType.SPIDER: + port_list = list( + ProxyInstance.objects.filter( + machine__ip=ip, + machine__bk_cloud_id=bk_cloud_id, + ).values_list("port", flat=True) + ) + else: + raise Exception(f"not support machine type: {m.machine_type}") # noqa + + instances = [] + for port in port_list: + instances.append( + { + "ip": ip, + "port": port, + } + ) + + filters = { + "bk_biz_id": m.bk_biz_id, + "db_type": dbtype.value, + "limit": 10, + "offset": 0, + "username": DB_ADMIN_USER_MAP[dbtype], + "instances": instances, + } + + admin_password_data = DBPrivManagerApi.get_mysql_admin_password(params=filters) + admin_password_data["results"] = admin_password_data.pop("items") + + res = collections.defaultdict(dict) + for data in admin_password_data["results"]: + res[data["port"]]["username"] = data["username"] + res[data["port"]]["password"] = base64_decode(data["password"]) + + return res diff --git a/dbm-ui/backend/db_proxy/reverse_api/mysql/impl/list_instance_info.py b/dbm-ui/backend/db_proxy/reverse_api/mysql/impl/list_instance_info.py new file mode 100644 index 0000000000..a6655519d4 --- /dev/null +++ b/dbm-ui/backend/db_proxy/reverse_api/mysql/impl/list_instance_info.py @@ -0,0 +1,84 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from typing import List, Optional + +from django.db.models import Q + +from backend.db_meta.enums import AccessLayer +from backend.db_meta.models import Machine, ProxyInstance, StorageInstance + + +def list_instance_info(bk_cloud_id: int, ip: str, port_list: Optional[List[int]] = None) -> List[dict]: + m = Machine.objects.get(ip=ip, bk_cloud_id=bk_cloud_id) + q = Q() + q |= Q(**{"machine": m}) + + if port_list: + q &= Q(**{"port__in": port_list}) + + if m.access_layer == AccessLayer.PROXY: + res = list_proxyinstance_info(q=q) + else: + res = list_storageinstance_info(q=q) + + return res + + +def list_storageinstance_info(q: Q) -> List: + res = [] + for i in StorageInstance.objects.filter(q): + receivers = [] + ejectors = [] + for t in i.as_ejector.all(): + receivers.append( + { + "ip": t.receiver.machine.ip, + "port": t.receiver.port, + } + ) + for t in i.as_receiver.all(): + ejectors.append( + { + "ip": t.ejector.machine.ip, + "port": t.ejector.port, + } + ) + + res.append( + { + "ip": i.machine.ip, + "port": i.port, + "phase": i.phase, + "status": i.status, + "is_standby": i.is_stand_by, + "instance_role": i.instance_role, + "instance_inner_role": i.instance_inner_role, + "receivers": receivers, + "ejectors": ejectors, + } + ) + + return res + + +def list_proxyinstance_info(q: Q) -> List: + res = [] + for i in ProxyInstance.objects.filter(q): + res.append( + { + "ip": i.machine.ip, + "port": i.port, + "phase": i.phase, + "status": i.status, + } + ) + + return res diff --git a/dbm-ui/backend/db_proxy/reverse_api/mysql/views.py b/dbm-ui/backend/db_proxy/reverse_api/mysql/views.py index 030118fda5..85e73383bb 100644 --- a/dbm-ui/backend/db_proxy/reverse_api/mysql/views.py +++ b/dbm-ui/backend/db_proxy/reverse_api/mysql/views.py @@ -10,15 +10,13 @@ """ import logging -from django.db.models import Q from django.http import JsonResponse from django.utils.translation import ugettext_lazy as _ from backend.bk_web.swagger import common_swagger_auto_schema -from backend.db_meta.enums import AccessLayer -from backend.db_meta.models import Machine, ProxyInstance, StorageInstance from backend.db_proxy.reverse_api.base_reverse_api_view import BaseReverseApiView from backend.db_proxy.reverse_api.decorators import reverse_api +from backend.db_proxy.reverse_api.mysql.impl import list_instance_info logger = logging.getLogger("root") @@ -29,22 +27,7 @@ class MySQLReverseApiView(BaseReverseApiView): def list_instance_info(self, request, *args, **kwargs): bk_cloud_id, ip, port_list = self.get_api_params() logger.info(f"bk_cloud_id: {bk_cloud_id}, ip: {ip}, port:{port_list}") - - m = Machine.objects.get(ip=ip, bk_cloud_id=bk_cloud_id) - q = Q() - q |= Q(**{"machine": m}) - - if port_list: - q &= Q(**{"port__in": port_list}) - - res = [] - if m.access_layer == AccessLayer.PROXY: - for i in ProxyInstance.objects.filter(q): - res.append({"ip": i.machine.ip, "port": i.port, "phase": i.phase, "status": i.status}) - else: - for i in StorageInstance.objects.filter(q): - res.append({"ip": i.machine.ip, "port": i.port, "phase": i.phase, "status": i.status}) - + res = list_instance_info(bk_cloud_id=bk_cloud_id, ip=ip, port_list=port_list) logger.info(f"instance info: {res}") return JsonResponse( { diff --git a/dbm-ui/backend/db_proxy/reverse_api/urls.py b/dbm-ui/backend/db_proxy/reverse_api/urls.py index ad62213c3e..12975d3c94 100644 --- a/dbm-ui/backend/db_proxy/reverse_api/urls.py +++ b/dbm-ui/backend/db_proxy/reverse_api/urls.py @@ -10,11 +10,11 @@ """ from rest_framework.routers import DefaultRouter +from backend.db_proxy.reverse_api.common import CommonReverseApiView from backend.db_proxy.reverse_api.mysql import MySQLReverseApiView -from backend.db_proxy.reverse_api.views import CommonReverseApiView routers = DefaultRouter(trailing_slash=True) routers.register("mysql", MySQLReverseApiView, basename="") -routers.register("", CommonReverseApiView, basename="") +routers.register("common", CommonReverseApiView, basename="") urlpatterns = routers.urls From 639e93b1384366413d969d57788d263927272ec6 Mon Sep 17 00:00:00 2001 From: yuanruji Date: Tue, 17 Dec 2024 15:28:32 +0800 Subject: [PATCH 005/107] =?UTF-8?q?fix(dbm-services):=20=E4=BF=AE=E6=AD=A3?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E6=A8=A1=E6=8B=9F=E6=89=A7=E8=A1=8C=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=E8=BF=94=E5=9B=9E=E7=9A=84=E7=8A=B6=E6=80=81=E7=A0=81?= =?UTF-8?q?=20#8660?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dbm-services/common/go-pubpkg/errno/code.go | 2 ++ dbm-services/mysql/db-simulation/handler/dbsimulation.go | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/dbm-services/common/go-pubpkg/errno/code.go b/dbm-services/common/go-pubpkg/errno/code.go index 95b7407e4d..a9f7ea44b4 100644 --- a/dbm-services/common/go-pubpkg/errno/code.go +++ b/dbm-services/common/go-pubpkg/errno/code.go @@ -13,6 +13,8 @@ package errno var ( // OK TODO OK = Errno{Code: 0, Message: "", CNMessage: ""} + // SimulationTaskFailed 语义检查失败 + SimulationTaskFailed = Errno{Code: 1, Message: "simulation failed", CNMessage: "模拟执行失败"} // InternalServerError TODO InternalServerError = Errno{Code: 10001, Message: "Internal server error", CNMessage: "服务器内部错误。"} diff --git a/dbm-services/mysql/db-simulation/handler/dbsimulation.go b/dbm-services/mysql/db-simulation/handler/dbsimulation.go index e2bdf50777..141a0caf2a 100644 --- a/dbm-services/mysql/db-simulation/handler/dbsimulation.go +++ b/dbm-services/mysql/db-simulation/handler/dbsimulation.go @@ -18,6 +18,7 @@ import ( "github.com/gin-gonic/gin" "dbm-services/common/go-pubpkg/cmutil" + "dbm-services/common/go-pubpkg/errno" "dbm-services/common/go-pubpkg/logger" "dbm-services/mysql/db-simulation/app/service" "dbm-services/mysql/db-simulation/model" @@ -121,7 +122,7 @@ func (s *SimulationHandler) QueryTask(c *gin.Context) { switch task.Status { case model.TaskFailed: allSuccessful = false - s.SendResponse(c, fmt.Errorf("%s", task.SysErrMsg), map[string]interface{}{ + s.SendResponse(c, errno.SimulationTaskFailed.Add(task.SysErrMsg), map[string]interface{}{ "simulation_version": task.MySQLVersion, "stdout": task.Stdout, "stderr": task.Stderr, @@ -131,7 +132,7 @@ func (s *SimulationHandler) QueryTask(c *gin.Context) { allSuccessful = true default: allSuccessful = false - s.SendResponse(c, fmt.Errorf("unknown transition state"), map[string]interface{}{ + s.SendResponse(c, errno.SimulationTaskFailed.Add("unknown transition state"), map[string]interface{}{ "stdout": task.Stdout, "stderr": task.Stderr, "errmsg": fmt.Sprintf("the program has been run with abnormal status:%s", task.Status)}) From 9760896a4b74a4b62a29c5d8cff25fff9a8f43de Mon Sep 17 00:00:00 2001 From: yksitu <1297650644@qq.com> Date: Wed, 4 Dec 2024 09:41:32 +0800 Subject: [PATCH 006/107] =?UTF-8?q?fix(sqlserver):=20=E4=BC=98=E5=8C=96sla?= =?UTF-8?q?ve=E9=87=8D=E5=BB=BA=E7=9A=84=E9=80=BB=E8=BE=91=20#8306?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../subcmd/checkcmd/check_mssql_service.go | 83 +++++++ .../internal/subcmd/checkcmd/checkcmd.go | 1 + .../components/check/check_mssql_service.go | 53 ++++ dbm-ui/backend/env/__init__.py | 2 + dbm-ui/backend/flow/consts.py | 1 + .../engine/bamboo/scene/common/builder.py | 55 +++- .../bamboo/scene/sqlserver/common_sub_flow.py | 25 +- .../sqlserver/sqlserver_slave_rebuild.py | 235 ++++++++++++++---- .../scene/sqlserver/sqlserver_sql_execute.py | 1 + .../collections/common/empty_node.py | 28 +++ .../sqlserver/check_mssql_service.py | 35 +++ .../sqlserver/check_slave_sync_status.py | 104 ++++++++ .../sqlserver/restore_for_do_dr.py | 8 +- .../sqlserver/update_window_gse_config.py | 83 +++++++ .../sqlserver/sqlserver_act_dataclass.py | 52 ++++ .../utils/sqlserver/sqlserver_act_payload.py | 40 ++- .../utils/sqlserver/sqlserver_db_function.py | 33 ++- 17 files changed, 779 insertions(+), 60 deletions(-) create mode 100644 dbm-services/sqlserver/db-tools/dbactuator/internal/subcmd/checkcmd/check_mssql_service.go create mode 100644 dbm-services/sqlserver/db-tools/dbactuator/pkg/components/check/check_mssql_service.go create mode 100644 dbm-ui/backend/flow/plugins/components/collections/common/empty_node.py create mode 100644 dbm-ui/backend/flow/plugins/components/collections/sqlserver/check_mssql_service.py create mode 100644 dbm-ui/backend/flow/plugins/components/collections/sqlserver/check_slave_sync_status.py create mode 100644 dbm-ui/backend/flow/plugins/components/collections/sqlserver/update_window_gse_config.py diff --git a/dbm-services/sqlserver/db-tools/dbactuator/internal/subcmd/checkcmd/check_mssql_service.go b/dbm-services/sqlserver/db-tools/dbactuator/internal/subcmd/checkcmd/check_mssql_service.go new file mode 100644 index 0000000000..1eaf0056a2 --- /dev/null +++ b/dbm-services/sqlserver/db-tools/dbactuator/internal/subcmd/checkcmd/check_mssql_service.go @@ -0,0 +1,83 @@ +/* + * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. + * Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at https://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package checkcmd + +import ( + "fmt" + + "dbm-services/common/go-pubpkg/logger" + "dbm-services/sqlserver/db-tools/dbactuator/internal/subcmd" + "dbm-services/sqlserver/db-tools/dbactuator/pkg/components/check" + "dbm-services/sqlserver/db-tools/dbactuator/pkg/util" + + "github.com/spf13/cobra" +) + +// MssqlServiceAct sqlserver 检查实例连接情况 +type MssqlServiceAct struct { + *subcmd.BaseOptions + BaseService check.MssqlServiceComp +} + +// MssqlServiceCommand godoc +// +// @Summary sqlserver 检查实例连接情况 +// @Description - +// @Tags sqlserver +// @Accept json +// @Param body body MssqlServiceComp true "short description" +func MssqlServiceCommand() *cobra.Command { + act := MssqlServiceAct{ + BaseOptions: subcmd.GBaseOptions, + } + cmd := &cobra.Command{ + Use: "MssqlServiceCheck", + Short: "检查机器注册Sqlserver进程情况", + Example: fmt.Sprintf(`dbactuator check MssqlServiceCheck %s `, subcmd.CmdBaseExampleStr), + Run: func(cmd *cobra.Command, args []string) { + util.CheckErr(act.Validate()) + if act.RollBack { + return + } + util.CheckErr(act.Init()) + util.CheckErr(act.Run()) + }, + } + return cmd +} + +// Init 初始化 +func (u *MssqlServiceAct) Init() (err error) { + logger.Info("MssqlServiceAct Init") + if err = u.Deserialize(&u.BaseService.Params); err != nil { + logger.Error("DeserializeAndValidate failed, %v", err) + return err + } + u.BaseService.GeneralParam = subcmd.GeneralRuntimeParam + + return nil +} + +// Run 执行 +func (u *MssqlServiceAct) Run() (err error) { + steps := subcmd.Steps{ + { + FunName: "检查机器注册Sqlserver进程情况", + Func: u.BaseService.CheckMssqlService, + }, + } + + if err := steps.Run(); err != nil { + return err + } + logger.Info("check-inst-processlists successfully") + return nil +} diff --git a/dbm-services/sqlserver/db-tools/dbactuator/internal/subcmd/checkcmd/checkcmd.go b/dbm-services/sqlserver/db-tools/dbactuator/internal/subcmd/checkcmd/checkcmd.go index e90c1320b2..a58be6219d 100644 --- a/dbm-services/sqlserver/db-tools/dbactuator/internal/subcmd/checkcmd/checkcmd.go +++ b/dbm-services/sqlserver/db-tools/dbactuator/internal/subcmd/checkcmd/checkcmd.go @@ -31,6 +31,7 @@ func CheckCommand() *cobra.Command { Commands: []*cobra.Command{ CheckAbnormalDBCommand(), CheckInstProcessCommand(), + MssqlServiceCommand(), }, }, } diff --git a/dbm-services/sqlserver/db-tools/dbactuator/pkg/components/check/check_mssql_service.go b/dbm-services/sqlserver/db-tools/dbactuator/pkg/components/check/check_mssql_service.go new file mode 100644 index 0000000000..c96cd9294e --- /dev/null +++ b/dbm-services/sqlserver/db-tools/dbactuator/pkg/components/check/check_mssql_service.go @@ -0,0 +1,53 @@ +/* + * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. + * Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. + * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at https://opensource.org/licenses/MIT + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package check + +import ( + "bk-dbconfig/pkg/core/logger" + "fmt" + + "dbm-services/sqlserver/db-tools/dbactuator/pkg/components" + "dbm-services/sqlserver/db-tools/dbactuator/pkg/util/osutil" + "dbm-services/sqlserver/db-tools/dbactuator/pkg/util/sqlserver" +) + +// MssqlServiceComp 检查db连接情况 +type MssqlServiceComp struct { + GeneralParam *components.GeneralParam + Params *MssqlServiceParam + DB *sqlserver.DbWorker +} + +// MssqlServiceParam 参数 +type MssqlServiceParam struct { + Host string `json:"host" validate:"ip" ` // 本地hostip +} + +// CheckMssqlService 检查机器注册Sqlserver进程情况 +func (c *MssqlServiceComp) CheckMssqlService() error { + var checkresult string + ret, err := osutil.StandardPowerShellCommand( + "GET-SERVICE -NAME MSSQL* | WHERE-OBJECT {$_.NAME -NOTLIKE \"*#*\"}", + ) + if err != nil { + return err + } + if ret != "" { + // 输出不为空则表示有部署进程 + logger.Info("there is a mssql process has been registered [%s]", osutil.CleanExecOutput(ret)) + checkresult = "1" + } + logger.Info("no mssql service registered") + checkresult = "0" + components.WrapperOutputString(fmt.Sprintf("{\"checkresult\": \"%s\"}", checkresult)) + return nil + +} diff --git a/dbm-ui/backend/env/__init__.py b/dbm-ui/backend/env/__init__.py index 870968c197..07cd33f1f0 100644 --- a/dbm-ui/backend/env/__init__.py +++ b/dbm-ui/backend/env/__init__.py @@ -120,6 +120,8 @@ SA_L5_AGENT_TEMPLATE_ID = get_type_env(key="SA_L5_AGENT_TEMPLATE_ID", _type=int) # 标准运维项目 ID BK_SOPS_PROJECT_ID = get_type_env(key="BK_SOPS_PROJECT_ID", _type=int, default=1) +# 标准运维更新window机器的模板ID +UPDATE_WINDOW_GSE_CONFIG = get_type_env(key="UPDATE_WINDOW_GSE_CONFIG", _type=int, default=1) # Bamboo ENABLE_CLEAN_EXPIRED_BAMBOO_TASK = get_type_env(key="ENABLE_CLEAN_EXPIRED_BAMBOO_TASK", _type=bool, default=False) diff --git a/dbm-ui/backend/flow/consts.py b/dbm-ui/backend/flow/consts.py index d07b40b755..4e198b3a8d 100644 --- a/dbm-ui/backend/flow/consts.py +++ b/dbm-ui/backend/flow/consts.py @@ -657,6 +657,7 @@ class SqlserverActuatorActionEnum(str, StructuredEnum): ClearConfig = EnumField("ClearConfig", _("清理实例周边配置。目前支持清理job、linkserver")) RemoteDr = EnumField("RemoteDr", _("将一些dr移除可用组")) Init = EnumField("init", _("部署后需要初始化实例的步骤")) + MssqlServiceCheck = EnumField("MssqlServiceCheck", _("检测进程是否注册")) class DorisActuatorActionEnum(str, StructuredEnum): diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/common/builder.py b/dbm-ui/backend/flow/engine/bamboo/scene/common/builder.py index 959a6fd421..514b3d8e22 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/common/builder.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/common/builder.py @@ -10,14 +10,17 @@ """ import copy import logging -from typing import Any, Dict, Optional +from dataclasses import dataclass +from typing import Any, Dict, List, Optional from bamboo_engine import api, builder from bamboo_engine.builder import ( + ConditionalParallelGateway, ConvergeGateway, Data, EmptyEndEvent, EmptyStartEvent, + NodeOutput, ParallelGateway, Params, RewritableNodeOutput, @@ -33,11 +36,18 @@ from backend.flow.models import FlowNode, FlowTree, StateType from backend.flow.plugins.components.collections.common.create_random_job_user import AddTempUserForClusterComponent from backend.flow.plugins.components.collections.common.drop_random_job_user import DropTempUserForClusterComponent +from backend.flow.plugins.components.collections.common.empty_node import EmptyNodeComponent from backend.ticket.constants import TicketType logger = logging.getLogger("json") +@dataclass +class Conditions: + act_object: Any + express: str + + class Builder(object): """ 构建bamboo流程的抽象类，解决开发人员在编排流程的学习成本，减少代码重复率 @@ -77,6 +87,9 @@ def __init__(self, root_id: str, data: Optional[Dict] = None, need_random_pass_c # 定义流程数据上下文参数trans_data self.rewritable_node_source_keys = [] + # 定义条件网关的上下文参数 + self.node_output_list = [] + # 判断是否添加临时账号的流程逻辑 if self.need_random_pass_cluster_ids: self.create_random_pass_act() @@ -203,6 +216,35 @@ def add_parallel_sub_pipeline(self, sub_flow_list: list): cg = ConvergeGateway() self.pipe = self.pipe.extend(pg).connect(*sub_flow_list).to(pg).converge(cg) + def add_conditional_subs(self, source_act, conditions: List[Conditions], name: str, conditions_param: str): + """ + add_conditional_subs：给流程添加条件分支节点，控制执行节点或者子流程 + @param source_act: 控制添加源节点 + @param conditions: 表达式 + @param name: 表达式名称 + @param conditions_param: 表达式变量名称 + """ + real_conditions = {} + connect_list = [] + for index, info in enumerate(conditions): + real_conditions[index] = f"${{{conditions_param}}} {info.express}" + connect_list.append(info.act_object) + + # 添你默认节点 + connect_list.append( + self.add_act(act_name="default_node", act_component_code=EmptyNodeComponent.code, kwargs={}, extend=False) + ) + real_conditions[len(connect_list) - 1] = "1==1" + + cpg = ConditionalParallelGateway( + conditions=real_conditions, name=name, default_condition_outgoing=len(connect_list) - 1 + ) + cg = ConvergeGateway() + self.pipe = self.pipe.extend(source_act).extend(cpg).connect(*connect_list).to(cpg).converge(cg) + + # 拼接有可能条件网关需要的上下文变量 + self.node_output_list.append({"conditions_param": conditions_param, "source_act_id": source_act.id}) + def run_pipeline(self, init_trans_data_class: Optional[Any] = None, is_drop_random_user: bool = True) -> bool: """ 开始运行 pipeline @@ -218,6 +260,11 @@ def run_pipeline(self, init_trans_data_class: Optional[Any] = None, is_drop_rand self.global_data.inputs["${trans_data}"] = RewritableNode( source_act=self.rewritable_node_source_keys, type=Var.SPLICE, value=init_trans_data_class ) + # 声明NodeOutput变量 + for i in self.node_output_list: + self.global_data.inputs[f"${{{i['conditions_param']}}}"] = NodeOutput( + type=Var.SPLICE, source_act=i["source_act_id"], source_key=f"{i['conditions_param']}" + ) self.pipe.extend(self.end_act) pipeline = builder.build_tree(self.start_act, id=self.root_id, data=self.global_data) pipeline_copy = copy.deepcopy(pipeline) @@ -272,7 +319,11 @@ def build_sub_process(self, sub_name) -> Optional[SubProcess]: sub_data.inputs["${trans_data}"] = RewritableNode( source_act=self.rewritable_node_source_keys, type=Var.SPLICE, value=None ) - # sub_data.inputs['${trans_data}'] = DataInput(type=Var.SPLICE, value='${trans_data}') + # 声明NodeOutput变量 + for i in self.node_output_list: + sub_data.inputs[f"${{{i['conditions_param']}}}"] = NodeOutput( + type=Var.SPLICE, source_act=i["source_act_id"], source_key=f"{i['conditions_param']}" + ) sub_params = Params({"${trans_data}": Var(type=Var.SPLICE, value="${trans_data}")}) self.pipe.extend(self.end_act) return SubProcess(start=self.start_act, data=sub_data, params=sub_params, name=sub_name) diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/sqlserver/common_sub_flow.py b/dbm-ui/backend/flow/engine/bamboo/scene/sqlserver/common_sub_flow.py index c48eef0fab..bd4cd99163 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/sqlserver/common_sub_flow.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/sqlserver/common_sub_flow.py @@ -54,6 +54,9 @@ SqlserverDownloadBackupFileComponent, ) from backend.flow.plugins.components.collections.sqlserver.trans_files import TransFileInWindowsComponent +from backend.flow.plugins.components.collections.sqlserver.update_window_gse_config import ( + UpdateWindowGseConfigComponent, +) from backend.flow.utils.common_act_dataclass import DownloadBackupClientKwargs, InstallNodemanPluginKwargs from backend.flow.utils.mysql.mysql_act_dataclass import InitCheckKwargs, UpdateDnsRecordKwargs from backend.flow.utils.sqlserver.sqlserver_act_dataclass import ( @@ -65,6 +68,7 @@ P2PFileForWindowKwargs, RestoreForDoDrKwargs, SqlserverBackupIDContext, + UpdateWindowGseConfigKwargs, ) from backend.flow.utils.sqlserver.sqlserver_act_payload import SqlserverActPayload from backend.flow.utils.sqlserver.sqlserver_db_function import get_backup_path @@ -137,6 +141,19 @@ def install_sqlserver_sub_flow( ) sub_pipeline.add_parallel_acts(acts_list=acts_list) + # 更新window机器的gse配置信息 + if env.UPDATE_WINDOW_GSE_CONFIG: + acts_list = [] + for host in target_hosts: + acts_list.append( + { + "act_name": _("更新gse配置信息[{}]".format(host.ip)), + "act_component_code": UpdateWindowGseConfigComponent.code, + "kwargs": asdict(UpdateWindowGseConfigKwargs(ips=[host.ip], bk_cloud_id=bk_cloud_id)), + } + ) + sub_pipeline.add_parallel_acts(acts_list=acts_list) + # 安装蓝鲸插件 acts_list = [] for plugin_name in DEPENDENCIES_PLUGINS: @@ -435,6 +452,8 @@ def sync_dbs_for_cluster_sub_flow( sync_dbs: list, clean_dbs: list = None, sub_flow_name: str = _("建立数据库同步子流程"), + is_recalc_sync_dbs: bool = False, + is_recalc_clean_dbs: bool = False, ): """ 数据库建立同步的子流程 @@ -445,6 +464,8 @@ def sync_dbs_for_cluster_sub_flow( @param sync_dbs: 待同步的db列表 @param clean_dbs: 这次清理的db列表，默认为空，则用sync_dbs列表作为清理db @param sub_flow_name: 子流程名称 + @param is_recalc_sync_dbs: 控制在流程运行是否在传输上下文获取sync_dbs,适配于原地重建slave场景 + @param is_recalc_clean_dbs: 控制在流程运行是否在传输上下文获取clean_dbs,适配于原地重建slave场景 """ # 获取当前master实例信息 master_instance = cluster.storageinstance_set.get(instance_role=InstanceRole.BACKEND_MASTER) @@ -463,7 +484,7 @@ def sync_dbs_for_cluster_sub_flow( SqlserverSyncMode.ALWAYS_ON: SqlserverActPayload.get_build_add_dbs_in_always_on.__name__, } # 判断必要参数 - if len(sync_slaves) == 0 or len(sync_dbs) == 0: + if len(sync_slaves) == 0 or (len(sync_dbs) == 0 and is_recalc_sync_dbs is False): raise Exception("sync_slaves or sync_dbs is null, check") # 做判断, cluster_sync_mode 如果是mirror，原则上不允许一主多从的架构, 所以判断传入的slave是否有多个 @@ -483,6 +504,8 @@ def sync_dbs_for_cluster_sub_flow( "ignore_clean_tables": [], "sync_mode": SqlserverSyncModeMaps[cluster_sync_mode], "slaves": [], + "is_recalc_sync_dbs": is_recalc_sync_dbs, # 判断标志位待入到全局上下文，获取payload进行判断 + "is_recalc_clean_dbs": is_recalc_clean_dbs, # 判断标志位待入到全局上下文，获取payload进行判断 } # 声明子流程 diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/sqlserver/sqlserver_slave_rebuild.py b/dbm-ui/backend/flow/engine/bamboo/scene/sqlserver/sqlserver_slave_rebuild.py index 7a8493f257..3b4163c4a2 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/sqlserver/sqlserver_slave_rebuild.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/sqlserver/sqlserver_slave_rebuild.py @@ -20,7 +20,7 @@ from backend.db_meta.models import Cluster, StorageInstance from backend.db_meta.models.storage_set_dtl import SqlserverClusterSyncMode from backend.flow.consts import SqlserverCleanMode, SqlserverLoginExecMode, SqlserverSyncMode, SqlserverSyncModeMaps -from backend.flow.engine.bamboo.scene.common.builder import Builder, SubBuilder +from backend.flow.engine.bamboo.scene.common.builder import Builder, Conditions, SubBuilder from backend.flow.engine.bamboo.scene.common.get_file_list import GetFileList from backend.flow.engine.bamboo.scene.sqlserver.base_flow import BaseFlow from backend.flow.engine.bamboo.scene.sqlserver.common_sub_flow import ( @@ -34,6 +34,7 @@ from backend.flow.plugins.components.collections.common.delete_cc_service_instance import DelCCServiceInstComponent from backend.flow.plugins.components.collections.common.pause import PauseComponent from backend.flow.plugins.components.collections.mysql.dns_manage import MySQLDnsManageComponent +from backend.flow.plugins.components.collections.sqlserver.check_slave_sync_status import CheckSlaveSyncStatusComponent from backend.flow.plugins.components.collections.sqlserver.create_random_job_user import SqlserverAddJobUserComponent from backend.flow.plugins.components.collections.sqlserver.drop_random_job_user import SqlserverDropJobUserComponent from backend.flow.plugins.components.collections.sqlserver.exec_actuator_script import SqlserverActuatorScriptComponent @@ -47,6 +48,7 @@ IpDnsRecordRecycleKwargs, ) from backend.flow.utils.sqlserver.sqlserver_act_dataclass import ( + CheckSlaveSyncStatusKwargs, CreateRandomJobUserKwargs, DBMetaOPKwargs, DownloadMediaKwargs, @@ -54,15 +56,13 @@ ExecActuatorKwargs, ExecLoginKwargs, SqlserverBackupIDContext, + SqlserverRebuildSlaveContext, ) from backend.flow.utils.sqlserver.sqlserver_act_payload import SqlserverActPayload from backend.flow.utils.sqlserver.sqlserver_db_function import ( - check_always_on_status, create_sqlserver_login_sid, get_dbs_for_drs, get_group_name, - get_no_sync_dbs, - get_restoring_dbs, get_sync_filter_dbs, ) from backend.flow.utils.sqlserver.sqlserver_db_meta import SqlserverDBMeta @@ -85,8 +85,13 @@ def slave_rebuild_in_local_flow(self): """ 原地重建子流程 流程逻辑： - 1: 清理slave实例的所有库 - 2: 建立数据库级别主从关系 + 1: 先判断slave处于什么级别的异常状态，这里分成4个等级，不同等级对应不同的流程过程 + 2: 根据CheckSlaveSyncStatusComponent类执行返回结果，通过条件分支网关处理不同流程 + 2.1: 如果集群尚未不部署Alwayson配置，且元数据记录是Alwayson同步类型，则走修复1：添加Alwayson可用组修复+数据同步修复 + 2.2: 如果待修复的slave可用组异常，则走修复2：重建slave可用组修复+数据同步修复 + 2.3: 如果集群中部分数据库尚未建立同步，则走修复3：部分数据库同步修复 + 2.4: 如果集群数据库同步正常，且待修复slave正常，则走修复4，默认不处理。 + """ # 定义主流程 @@ -96,9 +101,7 @@ def slave_rebuild_in_local_flow(self): for info in self.data["infos"]: cluster = Cluster.objects.get(id=info["cluster_id"]) master = cluster.storageinstance_set.get(instance_role=InstanceRole.BACKEND_MASTER) - cluster_sync_mode = SqlserverClusterSyncMode.objects.get(cluster_id=cluster.id).sync_mode rebuild_slave = cluster.storageinstance_set.get(machine__ip=info["slave_host"]["ip"]) - sync_dbs = get_no_sync_dbs(cluster_id=cluster.id) # 拼接子流程全局上下文 sub_flow_context = copy.deepcopy(self.data) @@ -108,9 +111,6 @@ def slave_rebuild_in_local_flow(self): sub_flow_context["sync_mode"] = SqlserverSyncModeMaps[ SqlserverClusterSyncMode.objects.get(cluster_id=cluster.id).sync_mode ] - sub_flow_context["clean_dbs"] = list( - set(sync_dbs) | set(get_restoring_dbs(rebuild_slave, cluster.bk_cloud_id)) - ) sub_flow_context["clean_mode"] = SqlserverCleanMode.DROP_DBS.value sub_flow_context["clean_tables"] = ["*"] sub_flow_context["ignore_clean_tables"] = [] @@ -129,45 +129,52 @@ def slave_rebuild_in_local_flow(self): ), ), ) - - if cluster_sync_mode == SqlserverSyncMode.ALWAYS_ON and not check_always_on_status(cluster, rebuild_slave): - # 表示这个从实例和可用组端口连接，需要重建可用组关系 - sub_pipeline.add_act( - act_name=_("[{}]重建可用组".format(info["slave_host"]["ip"])), - act_component_code=SqlserverActuatorScriptComponent.code, - kwargs=asdict( - ExecActuatorKwargs( - exec_ips=[Host(ip=master.machine.ip, bk_cloud_id=cluster.bk_cloud_id)], - get_payload_func=SqlserverActPayload.get_build_always_on.__name__, - custom_params={ - "port": master.port, - "add_slaves": [{"host": rebuild_slave.machine.ip, "port": rebuild_slave.port}], - "group_name": get_group_name(master_instance=master, bk_cloud_id=cluster.bk_cloud_id), - "is_first": False, - "is_use_sa": False, - }, - ) + source_act = sub_pipeline.add_act( + act_name=_("检测带重建slave状态[{}]".format(rebuild_slave.ip_port)), + act_component_code=CheckSlaveSyncStatusComponent.code, + kwargs=asdict( + CheckSlaveSyncStatusKwargs(cluster_id=cluster.id, fix_slave_host=info["slave_host"]["ip"]), + ), + extend=False, + ) + conditions = [ + # 如果集群尚未不部署Alwayson配置，且元数据记录是Alwayson同步类型，则走修复1：添加Alwayson可用组修复+数据同步修复 + Conditions( + act_object=self._create_always_on_fix_sub_flow( + sub_flow_context=sub_flow_context, + master=master, + rebuild_slave=rebuild_slave, + cluster=cluster, ), - ) - # 更新清理数据库列表 - sub_flow_context["clean_dbs"] = list( - set(get_dbs_for_drs(cluster_id=cluster.id, db_list=["*"], ignore_db_list=[])) - | set(get_restoring_dbs(rebuild_slave, cluster.bk_cloud_id)) - ) - sync_dbs = get_dbs_for_drs(cluster_id=cluster.id, db_list=["*"], ignore_db_list=[]) - - if len(sync_dbs) > 0: - # 在slave重新建立数据库级别主从关系 - sub_pipeline.add_sub_pipeline( - sub_flow=sync_dbs_for_cluster_sub_flow( - uid=self.data["uid"], - root_id=self.root_id, + express="==1", + ), + # 如果待修复的slave可用组异常，则走修复2：重建slave可用组修复 + 数据同步修复 + Conditions( + act_object=self._fix_always_on_status_sub_flow( + sub_flow_context=sub_flow_context, + master=master, + rebuild_slave=rebuild_slave, cluster=cluster, - sync_slaves=[Host(**info["slave_host"])], - sync_dbs=sync_dbs, - clean_dbs=sub_flow_context["clean_dbs"], - ) - ) + ), + express="==2", + ), + # 如果集群中部分数据库尚未建立同步，则走修复3：部分数据库同步修复 + Conditions( + act_object=self._fix_database_sync_sub_flow( + sub_flow_context=sub_flow_context, + rebuild_slave=rebuild_slave, + cluster=cluster, + ), + express="==3", + ), + ] + + sub_pipeline.add_conditional_subs( + source_act=source_act, + conditions=conditions, + name=_("判断待修复slave[{}]的状态".format(rebuild_slave.ip_port)), + conditions_param=SqlserverRebuildSlaveContext.conditions_var_name(), + ) # 先做克隆周边配置 sub_pipeline.add_sub_pipeline( @@ -242,7 +249,7 @@ def slave_rebuild_in_local_flow(self): ) main_pipeline.add_parallel_sub_pipeline(sub_flow_list=sub_pipelines) - main_pipeline.run_pipeline(init_trans_data_class=SqlserverBackupIDContext()) + main_pipeline.run_pipeline(init_trans_data_class=SqlserverRebuildSlaveContext()) def slave_rebuild_in_new_slave_flow(self): """ @@ -667,3 +674,133 @@ def remote_slave_in_cluster( ), ) return sub_pipeline.build_sub_process(sub_name=_("移除可用组[{}]".format(cluster.immute_domain))) + + def _create_always_on_fix_sub_flow( + self, sub_flow_context: dict, master: StorageInstance, rebuild_slave: StorageInstance, cluster: Cluster + ): + """ + 创建整个集群可用组，修复slave的子流程 + """ + # 声明子流程 + sub_pipeline = SubBuilder(root_id=self.root_id, data=copy.deepcopy(sub_flow_context)) + sub_pipeline.add_sub_pipeline( + sub_flow=build_always_on_sub_flow( + uid=self.data["uid"], + root_id=self.root_id, + master_instance=SqlserverInstance( + host=master.machine.ip, + port=master.port, + bk_cloud_id=cluster.bk_cloud_id, + is_new=True, + ), + slave_instances=[ + SqlserverInstance( + host=rebuild_slave.machine.ip, + port=rebuild_slave.port, + bk_cloud_id=cluster.bk_cloud_id, + is_new=True, + ) + ], + cluster_name=cluster.name, + group_name=cluster.immute_domain, + is_use_sa=True, + ) + ) + + sub_pipeline.add_sub_pipeline( + sub_flow=sync_dbs_for_cluster_sub_flow( + uid=self.data["uid"], + root_id=self.root_id, + cluster=cluster, + sync_slaves=[ + Host( + ip=rebuild_slave.machine.ip, + bk_cloud_id=rebuild_slave.machine.bk_cloud_id, + bk_host_id=rebuild_slave.machine.bk_host_id, + ) + ], + sync_dbs=[], + clean_dbs=[], + is_recalc_sync_dbs=True, + is_recalc_clean_dbs=True, + ) + ) + + return sub_pipeline.build_sub_process(sub_name=_("集群[{}]添加可用组修复流程".format(cluster.name))) + + def _fix_always_on_status_sub_flow( + self, sub_flow_context: dict, master: StorageInstance, rebuild_slave: StorageInstance, cluster: Cluster + ): + """ + 重建slave的可用组场景，修复slave的子流程 + """ + # 声明子流程 + + sub_pipeline = SubBuilder(root_id=self.root_id, data=copy.deepcopy(sub_flow_context)) + sub_pipeline.add_act( + act_name=_("[{}]重建可用组".format(rebuild_slave.ip_port)), + act_component_code=SqlserverActuatorScriptComponent.code, + kwargs=asdict( + ExecActuatorKwargs( + exec_ips=[Host(ip=master.machine.ip, bk_cloud_id=cluster.bk_cloud_id)], + get_payload_func=SqlserverActPayload.get_build_always_on.__name__, + custom_params={ + "port": master.port, + "add_slaves": [{"host": rebuild_slave.machine.ip, "port": rebuild_slave.port}], + "group_name": get_group_name( + master_instance=master, bk_cloud_id=cluster.bk_cloud_id, is_check_group=True + ), + "is_first": False, + "is_use_sa": False, + }, + ) + ), + ) + + sub_pipeline.add_sub_pipeline( + sub_flow=sync_dbs_for_cluster_sub_flow( + uid=self.data["uid"], + root_id=self.root_id, + cluster=cluster, + sync_slaves=[ + Host( + ip=rebuild_slave.machine.ip, + bk_cloud_id=rebuild_slave.machine.bk_cloud_id, + bk_host_id=rebuild_slave.machine.bk_host_id, + ) + ], + sync_dbs=[], + clean_dbs=[], + is_recalc_sync_dbs=True, + is_recalc_clean_dbs=True, + ) + ) + + return sub_pipeline.build_sub_process(sub_name=_("slave[{}]重建可用组修复流程".format(rebuild_slave.ip_port))) + + def _fix_database_sync_sub_flow(self, sub_flow_context: dict, rebuild_slave: StorageInstance, cluster: Cluster): + """ + 部分数据库未建立同步场景，修复slave的子流程 + """ + # 声明子流程 + sub_pipeline = SubBuilder(root_id=self.root_id, data=copy.deepcopy(sub_flow_context)) + sub_pipeline.add_sub_pipeline( + sub_flow=sync_dbs_for_cluster_sub_flow( + uid=self.data["uid"], + root_id=self.root_id, + cluster=cluster, + sync_slaves=[ + Host( + ip=rebuild_slave.machine.ip, + bk_cloud_id=rebuild_slave.machine.bk_cloud_id, + bk_host_id=rebuild_slave.machine.bk_host_id, + ) + ], + sync_dbs=[], + clean_dbs=[], + is_recalc_sync_dbs=True, + is_recalc_clean_dbs=True, + ) + ) + + return sub_pipeline.build_sub_process(sub_name=_("slave[{}]同步数据修复流程".format(rebuild_slave.ip_port))) diff --git a/dbm-ui/backend/flow/engine/bamboo/scene/sqlserver/sqlserver_sql_execute.py b/dbm-ui/backend/flow/engine/bamboo/scene/sqlserver/sqlserver_sql_execute.py index a9abfa79e8..3ea0b3ab79 100644 --- a/dbm-ui/backend/flow/engine/bamboo/scene/sqlserver/sqlserver_sql_execute.py +++ b/dbm-ui/backend/flow/engine/bamboo/scene/sqlserver/sqlserver_sql_execute.py @@ -8,6 +8,7 @@ specific language governing permissions and limitations under the License. """ + import copy import logging.config from dataclasses import asdict diff --git a/dbm-ui/backend/flow/plugins/components/collections/common/empty_node.py b/dbm-ui/backend/flow/plugins/components/collections/common/empty_node.py new file mode 100644 index 0000000000..95011ce6d8 --- /dev/null +++ b/dbm-ui/backend/flow/plugins/components/collections/common/empty_node.py @@ -0,0 +1,28 @@ +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import logging + +from pipeline.component_framework.component import Component + +from backend.flow.plugins.components.collections.common.base_service import BaseService + +logger = logging.getLogger("json") + + +class EmptyNodeService(BaseService): + def _execute(self, data, parent_data) -> bool: + return True + + +class EmptyNodeComponent(Component): + name = __name__ + code = "empty_node" + bound_service = EmptyNodeService diff --git a/dbm-ui/backend/flow/plugins/components/collections/sqlserver/check_mssql_service.py b/dbm-ui/backend/flow/plugins/components/collections/sqlserver/check_mssql_service.py new file mode 100644 index 0000000000..037fafb65b --- /dev/null +++ b/dbm-ui/backend/flow/plugins/components/collections/sqlserver/check_mssql_service.py @@ -0,0 +1,35 @@ +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import logging + +from pipeline.component_framework.component import Component + +from backend.flow.plugins.components.collections.sqlserver.exec_actuator_script import SqlserverActuatorScriptService + +logger = logging.getLogger("json") + + +class CheckSQLServerServiceService(SqlserverActuatorScriptService): + def _schedule(self, data, parent_data, callback_data=None) -> bool: + result = super()._schedule(data, parent_data) + if not result: + return False + # 处理判断变量 + trans_data = data.get_one_of_inputs("trans_data") + write_payload_var = data.get_one_of_inputs("write_payload_var") + data.outputs.is_registered = int(getattr(trans_data, write_payload_var)["is_registered"]) + return True + + +class CheckSQLServerServiceComponent(Component): + name = __name__ + code = "check_sqlserver_service" + bound_service = CheckSQLServerServiceService diff --git a/dbm-ui/backend/flow/plugins/components/collections/sqlserver/check_slave_sync_status.py b/dbm-ui/backend/flow/plugins/components/collections/sqlserver/check_slave_sync_status.py new file mode 100644 index 0000000000..a8cfee7d96 --- /dev/null +++ b/dbm-ui/backend/flow/plugins/components/collections/sqlserver/check_slave_sync_status.py @@ -0,0 +1,104 @@ +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from pipeline.component_framework.component import Component + +from backend.db_meta.enums import InstanceRole +from backend.db_meta.models import Cluster +from backend.db_meta.models.storage_set_dtl import SqlserverClusterSyncMode +from backend.flow.consts import SqlserverSyncMode +from backend.flow.plugins.components.collections.common.base_service import BaseService +from backend.flow.utils.sqlserver.sqlserver_db_function import ( + check_always_on_status, + exec_resume_sp, + get_dbs_for_drs, + get_group_name, + get_no_sync_dbs, + get_restoring_dbs, +) + + +class CheckSlaveSyncStatusService(BaseService): + """ + 判断带重建的slave处于什么状态，状态值都是用fix_number返回，不同fix_number代表不同修复流程： + 1: 可用组缺失 + 2: slave可用组状态异常 + 3: master部分库尚未建立同步 + 4: master所有库都建立同步，同步处于健康状态 + """ + + def _execute(self, data, parent_data) -> bool: + kwargs = data.get_one_of_inputs("kwargs") + trans_data = data.get_one_of_inputs("trans_data") + + # 获取集群的相关的信息 + cluster = Cluster.objects.get(id=kwargs["cluster_id"]) + fix_slave = cluster.storageinstance_set.get(machine__ip=kwargs["fix_slave_host"]) + master = cluster.storageinstance_set.get(instance_role=InstanceRole.BACKEND_MASTER) + cluster_sync_mode = SqlserverClusterSyncMode.objects.get(cluster_id=cluster.id).sync_mode + + # 首先确认集群同步类型 + if cluster_sync_mode == SqlserverSyncMode.ALWAYS_ON: + # 先确认可用组是否有配置 + sync_dbs = get_dbs_for_drs(cluster_id=cluster.id, db_list=["*"], ignore_db_list=[]) + clean_dbs = list(set(sync_dbs) | set(get_restoring_dbs(fix_slave, cluster.bk_cloud_id))) + if not get_group_name(master_instance=master, bk_cloud_id=cluster.bk_cloud_id, is_check_group=True): + # 如果可用组配置缺失，走建立可用组的流程 + self.log_info("group_name if null") + data.outputs.fix_number = 1 + trans_data.sync_dbs = sync_dbs + trans_data.clean_dbs = clean_dbs + data.outputs["trans_data"] = trans_data + return True + + elif not check_always_on_status(cluster, fix_slave): + # 如果可用组状态异常，走重建可用组流程 + self.log_info("always_on_status is abnormal") + data.outputs.fix_number = 2 + trans_data.sync_dbs = sync_dbs + trans_data.clean_dbs = clean_dbs + data.outputs["trans_data"] = trans_data + return True + + # 判断数据库同步状态 + if get_no_sync_dbs(cluster_id=cluster.id): + # 如果有数据库尚未同步，先修复同步，在判断同步是否正常 + exec_resume_sp( + slave_instances=[fix_slave], + master_host=master.machine.ip, + master_port=master.port, + bk_cloud_id=cluster.bk_cloud_id, + ) + self.log_info("exec exec_resume_sp finish, check the result...") + # 监测数据同步状态 + sync_dbs = get_no_sync_dbs(cluster_id=kwargs["cluster_id"]) + if sync_dbs: + # 表示修复失败 + self.log_warning("exec exec_resume_sp unsuccessfully") + trans_data.sync_dbs = sync_dbs + trans_data.clean_dbs = list(set(sync_dbs) | set(get_restoring_dbs(fix_slave, cluster.bk_cloud_id))) + data.outputs.fix_number = 3 + data.outputs["trans_data"] = trans_data + return True + else: + # 代表数据库重新建立成功 + self.log_info("exec exec_resume_sp successfully") + data.outputs.fix_number = 4 + return True + + self.log_info("no dbs fix sync") + data.outputs.fix_number = 4 + return True + + +class CheckSlaveSyncStatusComponent(Component): + name = __name__ + code = "sqlserver_check_rebuild_slave" + bound_service = CheckSlaveSyncStatusService diff --git a/dbm-ui/backend/flow/plugins/components/collections/sqlserver/restore_for_do_dr.py b/dbm-ui/backend/flow/plugins/components/collections/sqlserver/restore_for_do_dr.py index 3c36b6b5ff..2310b0c1e6 100644 --- a/dbm-ui/backend/flow/plugins/components/collections/sqlserver/restore_for_do_dr.py +++ b/dbm-ui/backend/flow/plugins/components/collections/sqlserver/restore_for_do_dr.py @@ -42,15 +42,21 @@ class RestoreForDoDrService(SqlserverActuatorScriptService): def _execute(self, data, parent_data) -> bool: kwargs = data.get_one_of_inputs("kwargs") + global_data = data.get_one_of_inputs("global_data") trans_data = data.get_one_of_inputs("trans_data") restore_dbs = [] restore_infos = [] + if global_data.get("is_recalc_sync_dbs", False): + check_restore_dbs = trans_data.sync_dbs + else: + check_restore_dbs = kwargs["restore_dbs"] + backup_id = getattr(trans_data, get_backup_id_map[kwargs["restore_mode"]])["id"] if not backup_id: raise Exception(f"backup id is null: backup_id:{backup_id}") - for db_name in kwargs["restore_dbs"]: + for db_name in check_restore_dbs: self.log_info(f"checking db:[{db_name}]") backup_info = get_backup_path_files( diff --git a/dbm-ui/backend/flow/plugins/components/collections/sqlserver/update_window_gse_config.py b/dbm-ui/backend/flow/plugins/components/collections/sqlserver/update_window_gse_config.py new file mode 100644 index 0000000000..c9fa93249b --- /dev/null +++ b/dbm-ui/backend/flow/plugins/components/collections/sqlserver/update_window_gse_config.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from django.utils.translation import ugettext as _ +from pipeline.component_framework.component import Component + +from backend import env +from backend.components import CCApi +from backend.components.sops.client import BkSopsApi +from backend.flow.consts import WINDOW_SYSTEM_JOB_USER +from backend.flow.plugins.components.collections.common.base_service import BkSopsService + + +class UpdateWindowGseConfig(BkSopsService): + def __get_bk_biz_id_for_ip(self, ip: str, bk_cloud_id: int): + + # 先通过ip获取对应的bk_host_id + res = CCApi.list_hosts_without_biz( + { + "fields": ["bk_host_id"], + "host_property_filter": { + "condition": "AND", + "rules": [ + {"field": "bk_host_innerip", "operator": "in", "value": [ip]}, + {"field": "bk_cloud_id", "operator": "equal", "value": bk_cloud_id}, + ], + }, + }, + use_admin=True, + ) + bk_host_id = res["info"][0]["bk_host_id"] + self.log_info(f"the bk_host_id of machine [{ip}] is {bk_host_id}") + + # 再通过bk_host_id获取 + res = CCApi.find_host_biz_relations({"bk_host_id": [bk_host_id]}) + return res[0]["bk_biz_id"] + + def _execute(self, data, parent_data) -> bool: + kwargs = data.get_one_of_inputs("kwargs") + ips = kwargs["ips"] + # 获取业务id + if kwargs.get("bk_biz_id"): + bk_biz_id = kwargs["bk_biz_id"] + else: + # 否则认为你这一批的机器都是来自于同一个业务上，则已其中一个ip获取的业务id为准 + bk_biz_id = self.__get_bk_biz_id_for_ip(ip=ips[0], bk_cloud_id=int(kwargs["bk_cloud_id"])) + + param = { + "template_id": env.UPDATE_WINDOW_GSE_CONFIG, + "bk_biz_id": bk_biz_id, + "template_source": "common", + "name": _("更新window的gse配置信息"), + "flow_type": "common", + "constants": { + "${biz_cc_id}": bk_biz_id, + "${bk_biz_id}": bk_biz_id, + "${job_ip_list}": "\n".join(ips), + "${job_account}": WINDOW_SYSTEM_JOB_USER, + }, + } + rpdata = BkSopsApi.create_task(param) + task_id = rpdata["task_id"] + # start task + self.log_info(f"job url:{env.BK_SOPS_URL}/taskflow/execute/{env.BK_SOPS_PROJECT_ID}/?instance_id={task_id}") + param = {"bk_biz_id": bk_biz_id, "task_id": task_id} + BkSopsApi.start_task(param) + data.outputs.task_id = task_id + data.inputs.kwargs["bk_biz_id"] = bk_biz_id + return True + + +class UpdateWindowGseConfigComponent(Component): + name = _("更新window的gse配置信息") + code = "update_window_gse_config_for_sop" + bound_service = UpdateWindowGseConfig diff --git a/dbm-ui/backend/flow/utils/sqlserver/sqlserver_act_dataclass.py b/dbm-ui/backend/flow/utils/sqlserver/sqlserver_act_dataclass.py index c6349aa3d3..851aaecca4 100644 --- a/dbm-ui/backend/flow/utils/sqlserver/sqlserver_act_dataclass.py +++ b/dbm-ui/backend/flow/utils/sqlserver/sqlserver_act_dataclass.py @@ -266,6 +266,38 @@ def log_backup_id_var_name() -> str: return "log_backup_id" +@dataclass() +class SqlserverRebuildSlaveContext: + """ + 定义重建slave的可交互上下文dataclass类 + """ + + sync_dbs: list = field(default_factory=list) + clean_dbs: list = field(default_factory=list) + full_backup_id: dict = field(default_factory=dict) + log_backup_id: dict = field(default_factory=dict) + + @staticmethod + def sync_dbs_var_name() -> str: + return "sync_dbs" + + @staticmethod + def clean_dbs_var_name() -> str: + return "clean_dbs" + + @staticmethod + def full_backup_id_var_name() -> str: + return "full_backup_id" + + @staticmethod + def log_backup_id_var_name() -> str: + return "log_backup_id" + + @staticmethod + def conditions_var_name() -> str: + return "fix_number" + + @dataclass() class CheckDBExistKwargs: """ @@ -277,3 +309,23 @@ class CheckDBExistKwargs: cluster_id: str check_dbs: list = field(default_factory=list) + + +@dataclass +class UpdateWindowGseConfigKwargs: + """ + 定义变更gse参数的配置私有变量结构体 + """ + + bk_cloud_id: int + ips: list + + +@dataclass +class CheckSlaveSyncStatusKwargs: + """ + 定义sqlserver_check_rebuild_slave私有变量结构体 + """ + + cluster_id: int + fix_slave_host: list diff --git a/dbm-ui/backend/flow/utils/sqlserver/sqlserver_act_payload.py b/dbm-ui/backend/flow/utils/sqlserver/sqlserver_act_payload.py index 6be84d4a86..d2e62aa0cb 100644 --- a/dbm-ui/backend/flow/utils/sqlserver/sqlserver_act_payload.py +++ b/dbm-ui/backend/flow/utils/sqlserver/sqlserver_act_payload.py @@ -38,6 +38,17 @@ def system_init_payload(self, **kwargs) -> dict: "payload": {**self.get_init_system_account(), **payload}, } + @staticmethod + def check_mssql_service_payload(self, **kwargs) -> dict: + """ + 测试实例是否注册存在的payload + """ + return { + "db_type": DBActuatorTypeEnum.Sqlserver_check.value, + "action": SqlserverActuatorActionEnum.MssqlServiceCheck.value, + "payload": {}, + } + def get_install_sqlserver_payload(self, **kwargs) -> dict: """ 拼接安装sqlserver的payload参数, 分别兼容集群申请、集群实例重建、集群实例添加单据的获取方式 @@ -159,6 +170,11 @@ def get_backup_dbs_payload(self, **kwargs) -> dict: """ 执行数据库备份的payload """ + if self.global_data.get("is_recalc_sync_dbs", False): + backup_dbs = kwargs["trans_data"]["sync_dbs"] + else: + backup_dbs = self.global_data["backup_dbs"] + return { "db_type": DBActuatorTypeEnum.Sqlserver.value, "action": SqlserverActuatorActionEnum.BackupDBS.value, @@ -167,7 +183,7 @@ def get_backup_dbs_payload(self, **kwargs) -> dict: "extend": { "host": kwargs["ips"][0]["ip"], "port": kwargs["custom_params"]["port"], - "backup_dbs": self.global_data["backup_dbs"], + "backup_dbs": backup_dbs, "backup_type": kwargs["custom_params"]["backup_type"], "job_id": self.global_data["job_id"], "file_tag": kwargs["custom_params"]["file_tag"], @@ -198,8 +214,12 @@ def get_rename_dbs_payload(self, **kwargs) -> dict: def get_clean_dbs_payload(self, **kwargs) -> dict: """ - 执行数据库重命名的payload + 执行数据库清档的payload """ + if self.global_data.get("is_recalc_clean_dbs", False): + clean_dbs = kwargs["trans_data"]["clean_dbs"] + else: + clean_dbs = self.global_data["clean_dbs"] return { "db_type": DBActuatorTypeEnum.Sqlserver.value, "action": SqlserverActuatorActionEnum.CleanDBS.value, @@ -208,7 +228,7 @@ def get_clean_dbs_payload(self, **kwargs) -> dict: "extend": { "host": kwargs["ips"][0]["ip"], "port": self.global_data["port"], - "clean_dbs": self.global_data["clean_dbs"], + "clean_dbs": clean_dbs, "sync_mode": self.global_data["sync_mode"], "clean_mode": self.global_data["clean_mode"], "slaves": self.global_data["slaves"], @@ -370,6 +390,11 @@ def get_build_database_mirroring(self, **kwargs) -> dict: """ 建立数据库级别镜像关系的payload """ + if self.global_data.get("is_recalc_sync_dbs", False): + sync_dbs = kwargs["trans_data"]["sync_dbs"] + else: + sync_dbs = kwargs["custom_params"]["dbs"] + return { "db_type": DBActuatorTypeEnum.Sqlserver.value, "action": SqlserverActuatorActionEnum.BuildDBMirroring.value, @@ -380,7 +405,7 @@ def get_build_database_mirroring(self, **kwargs) -> dict: "port": self.global_data["port"], "dr_host": kwargs["custom_params"]["dr_host"], "dr_port": kwargs["custom_params"]["dr_port"], - "dbs": kwargs["custom_params"]["dbs"], + "dbs": sync_dbs, }, }, } @@ -389,6 +414,11 @@ def get_build_add_dbs_in_always_on(self, **kwargs) -> dict: """ 建立数据库加入always_on可用组的payload """ + if self.global_data.get("is_recalc_sync_dbs", False): + sync_dbs = kwargs["trans_data"]["sync_dbs"] + else: + sync_dbs = kwargs["custom_params"]["dbs"] + return { "db_type": DBActuatorTypeEnum.Sqlserver.value, "action": SqlserverActuatorActionEnum.AddDBSInAlwaysOn.value, @@ -398,7 +428,7 @@ def get_build_add_dbs_in_always_on(self, **kwargs) -> dict: "host": kwargs["ips"][0]["ip"], "port": self.global_data["port"], "add_slaves": kwargs["custom_params"]["add_slaves"], - "dbs": kwargs["custom_params"]["dbs"], + "dbs": sync_dbs, }, }, } diff --git a/dbm-ui/backend/flow/utils/sqlserver/sqlserver_db_function.py b/dbm-ui/backend/flow/utils/sqlserver/sqlserver_db_function.py index 501a6db329..b56e3ff79e 100644 --- a/dbm-ui/backend/flow/utils/sqlserver/sqlserver_db_function.py +++ b/dbm-ui/backend/flow/utils/sqlserver/sqlserver_db_function.py @@ -8,7 +8,7 @@ specific language governing permissions and limitations under the License. """ import copy -import logging.config +import logging import re import secrets from collections import defaultdict @@ -360,11 +360,12 @@ def exec_instance_app_login(cluster: Cluster, exec_type: SqlserverLoginExecMode, return True -def get_group_name(master_instance: StorageInstance, bk_cloud_id: int): +def get_group_name(master_instance: StorageInstance, bk_cloud_id: int, is_check_group: bool = False): """ 获取集群group_name名称 @param master_instance: master实例 @param bk_cloud_id: 云区域id + @param is_check_group 默认False，表示如果查询group_name为空则异常，反之True为返回空，不报错 """ ret = DRSApi.sqlserver_rpc( { @@ -378,7 +379,11 @@ def get_group_name(master_instance: StorageInstance, bk_cloud_id: int): raise Exception(f"[{master_instance.ip_port}] get_group_name failed: {ret[0]['error_msg']}") if len(ret[0]["cmd_results"][0]["table_data"]) == 0: + if is_check_group: + # 如果设置True则正常返回空字符串 + return "" raise Exception(f"[{master_instance.ip_port}] get_group_name is null") + return ret[0]["cmd_results"][0]["table_data"][0]["name"] @@ -880,3 +885,27 @@ def check_ha_config( f"[{slave_instance.ip_port}]-{check_tag} configuration is not equal to master[{master_instance.ip_port}]", ) return True, "" + + +def exec_resume_sp(slave_instances: List[StorageInstance], master_host: str, master_port: int, bk_cloud_id: int): + """ + 执行尝试修复数据同步状态 + @param slave_instances: 待修复从库列表 + @param master_host: 待连接的master_host + @param master_port: 待连接的master_port + @param bk_cloud_id: 云区域ID + """ + cmd = f"use {SQLSERVER_CUSTOM_SYS_DB}; exec DBO.Sys_AutoSwitch_Resume '{master_host}','{master_port}', null" + logger.info(cmd) + ret = DRSApi.sqlserver_rpc( + { + "bk_cloud_id": bk_cloud_id, + "addresses": [storage.ip_port for storage in slave_instances], + "cmds": [cmd], + "force": False, + } + ) + + if ret[0]["error_msg"]: + raise Exception(f"Sys_AutoSwitch_Resume exec failed: {ret[0]['error_msg']}") + return True From 38c2fd4bc0d8c0615ccf393782c177daf25daf12 Mon Sep 17 00:00:00 2001 From: seanlook Date: Tue, 17 Dec 2024 18:08:57 +0800 Subject: [PATCH 007/107] =?UTF-8?q?fix(mysql):=20dbbackup=20=E7=89=A9?= =?UTF-8?q?=E7=90=86=E5=A4=87=E4=BB=BD=E4=B8=8D=E9=80=82=E7=94=A8lock-ddl?= =?UTF-8?q?=20#8676?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../mysql/db-tools/mysql-dbbackup/docs/readme.md | 8 +++++++- .../db-tools/mysql-dbbackup/pkg/config/physical.go | 4 ++++ .../pkg/src/backupexe/dumper_physical.go | 12 ++++++++++-- .../mysql/db-tools/mysql-monitor/items-config.sql | 2 +- .../mysql/db-tools/mysql-monitor/items-config.yaml | 1 + .../db-tools/mysql-monitor/pkg/internal/cst/const.go | 2 ++ .../pkg/itemscollect/mysqlerrlog/mysql_critical.go | 1 + .../itemscollect/mysqlprocesslist/mysql_inject.go | 4 +++- .../pkg/itemscollect/mysqlprocesslist/mysql_lock.go | 8 +++++--- 9 files changed, 34 insertions(+), 8 deletions(-) diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/docs/readme.md b/dbm-services/mysql/db-tools/mysql-dbbackup/docs/readme.md index 0d7287ae42..cd3b75bbd5 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/docs/readme.md +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/docs/readme.md @@ -358,4 +358,10 @@ DefaultsFile = /etc/my.cnf.3306 ### LogicalBackup - LogicalBackup.TrxConsistencyOnly mydumper `--trx-consistency-only`, 或者 mysqldump `--single-transaction`。默认 true - 对于多引擎混合的实例，如果想要保证整体数据的全局一致，需要设置为 false，会导致在整个备份期间持有 FTWRL，在主库上谨慎使用false。 \ No newline at end of file + 对于多引擎混合的实例，如果想要保证整体数据的全局一致，需要设置为 false，会导致在整个备份期间持有 FTWRL，在主库上谨慎使用false。 + +### PhysicalBackup +- PhysicalBackup.LockDDL + LockDDL 备份期间是否允许 ddl, >=5.7 参数有效 + - 默认 false，表示用户的 ddl 优先，备份无效。如果存在 Non-InnoDB 表，在拷贝这些非事务引擎表的时候，会阻塞对 Non-InnoDB dml + - 为 true 时，备份一开始就发送 lock tables for backup，全程不允许 ddl 和 Non-InnoDB dml \ No newline at end of file diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/physical.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/physical.go index 084b97ebcd..27cd7d5c2c 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/physical.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/config/physical.go @@ -14,6 +14,10 @@ type PhysicalBackup struct { Throttle int `ini:"Throttle"` // limits the number of chunks copied per second. The chunk size is 10 MB, 0 means no limit DefaultsFile string `ini:"DefaultsFile" validate:"required,file"` ExtraOpt string `ini:"ExtraOpt"` // other xtrabackup options string to be appended + // LockDDL 备份期间是否允许 ddl, >=5.7 参数有效 + // 默认 false，表示用户的 ddl 优先，备份无效。如果存在 Non-InnoDB 表，在拷贝这些非事务引擎表的时候，会阻塞对 Non-InnoDB dml + // 为 true 时，备份一开始就发送 lock tables for backup，全程不允许 ddl 和 Non-InnoDB dml + LockDDL bool `ini:"LockDDL"` // DisableSlaveMultiThread 在 slave并行多线程复制，且未开启 gtid 时，是否可临时关闭并行复制。默认值 false // 解决 The --slave-info option requires GTID enabled for a multi-threaded slave DisableSlaveMultiThread bool `ini:"DisableSlaveMultiThread"` diff --git a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_physical.go b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_physical.go index d8a5a1dd09..85fc6a52a0 100644 --- a/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_physical.go +++ b/dbm-services/mysql/db-tools/mysql-dbbackup/pkg/src/backupexe/dumper_physical.go @@ -103,7 +103,6 @@ func (p *PhysicalDumper) buildArgs() []string { args = append(args, fmt.Sprintf("--target-dir=%s", targetPath), "--backup") } if strings.Compare(p.mysqlVersion, "005007000") > 0 { - args = append(args, "--lock-ddl") if strings.Compare(p.mysqlVersion, "008000000") < 0 { // ver >=5.7 and ver < 8.0 args = append(args, "--binlog-info=ON") } @@ -131,12 +130,21 @@ func (p *PhysicalDumper) buildArgs() []string { if p.cnf.PhysicalBackup.Throttle > 0 { args = append(args, fmt.Sprintf("--throttle=%d", p.cnf.PhysicalBackup.Throttle)) } + if p.cnf.PhysicalBackup.LockDDL { + // will block all ddl and non-innodb dml from the backup beginning + if strings.Compare(p.mysqlVersion, "005007000") >= 0 { + args = append(args, "--lock-ddl") + } + } else { + if strings.Compare(p.mysqlVersion, "008000000") >= 0 { + args = append(args, "--skip-lock-ddl") + } + } if strings.ToLower(p.cnf.Public.MysqlRole) == cst.RoleSlave { // --safe-slave-backup args = append(args, "--slave-info") } - if strings.Compare(p.mysqlVersion, "008000000") >= 0 { if p.isOfficial { args = append(args, "--skip-strict") diff --git a/dbm-services/mysql/db-tools/mysql-monitor/items-config.sql b/dbm-services/mysql/db-tools/mysql-monitor/items-config.sql index edf3321f92..d38f9b48b1 100644 --- a/dbm-services/mysql/db-tools/mysql-monitor/items-config.sql +++ b/dbm-services/mysql/db-tools/mysql-monitor/items-config.sql @@ -16,7 +16,7 @@ REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, val REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'spider-err-warn', 'STRING', '{"role":[],"machine_type":["spider"],"schedule":"@every 1m","name":"spider-err-warn","enable":false}', '', 'MAP', 1, 0, 0, 0, 1); REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'spider-err-critical', 'STRING', '{"enable":false,"name":"spider-err-critical","machine_type":["spider"],"schedule":"@every 1m","role":[]}', '', 'MAP', 1, 0, 0, 0, 1); REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'mysqld-restarted', 'STRING', '{"role":[],"machine_type":["single","backend","remote","spider"],"schedule":"@every 1m","enable":true,"name":"mysqld-restarted"}', '', 'MAP', 1, 0, 0, 0, 1); -REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'mysql-lock', 'STRING', '{"role":["master","spider_master","orphan"],"machine_type":["single","backend","remote","spider"],"schedule":"@every 1m","enable":true,"name":"mysql-lock"}', '', 'MAP', 1, 0, 0, 0, 1); +REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'mysql-lock', 'STRING', '{"role":["master","spider_master","orphan","slave"],"machine_type":["single","backend","remote","spider"],"schedule":"@every 1m","enable":true,"name":"mysql-lock"}', '', 'MAP', 1, 0, 0, 0, 1); REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'mysql-inject', 'STRING', '{"schedule":"@every 1m","machine_type":["single","backend","spider"],"role":[],"name":"mysql-inject","enable":true}', '', 'MAP', 1, 0, 0, 0, 1); REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'proxy-backend', 'STRING', '{"role":[],"machine_type":["proxy"],"schedule":"@every 1m","name":"proxy-backend","enable":true}', '', 'MAP', 1, 0, 0, 0, 1); REPLACE INTO tb_config_name_def( namespace, conf_type, conf_file, conf_name, value_type, value_default, value_allowed, value_type_sub, flag_status, flag_disable, flag_locked, flag_encrypt, need_restart) VALUES( 'tendb', 'mysql_monitor', 'items-config.yaml', 'proxy-user-list', 'STRING', '{"enable":true,"name":"proxy-user-list","machine_type":["proxy"],"schedule":"0 55 23 * * *","role":[]}', '', 'MAP', 1, 0, 0, 0, 1); diff --git a/dbm-services/mysql/db-tools/mysql-monitor/items-config.yaml b/dbm-services/mysql/db-tools/mysql-monitor/items-config.yaml index cac3f2e770..92591bce3f 100644 --- a/dbm-services/mysql/db-tools/mysql-monitor/items-config.yaml +++ b/dbm-services/mysql/db-tools/mysql-monitor/items-config.yaml @@ -151,6 +151,7 @@ - spider role: - master + - slave - spider_master - orphan - name: mysql-inject diff --git a/dbm-services/mysql/db-tools/mysql-monitor/pkg/internal/cst/const.go b/dbm-services/mysql/db-tools/mysql-monitor/pkg/internal/cst/const.go index a43b69b973..f55bf760a2 100644 --- a/dbm-services/mysql/db-tools/mysql-monitor/pkg/internal/cst/const.go +++ b/dbm-services/mysql/db-tools/mysql-monitor/pkg/internal/cst/const.go @@ -5,4 +5,6 @@ const ( DBASchema = "infodba_schema" // DBAReportBase TODO DBAReportBase = "/home/mysql/dbareport" + // SystemUser replication thread user name + SystemUser = "system user" ) diff --git a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/mysqlerrlog/mysql_critical.go b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/mysqlerrlog/mysql_critical.go index 8886583259..83254e6f4a 100644 --- a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/mysqlerrlog/mysql_critical.go +++ b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/mysqlerrlog/mysql_critical.go @@ -34,6 +34,7 @@ func init() { "mysqld_safe mysqld restarted", "Failed to open log", "Could not open log file", + "Cannot open.*ib_buffer_pool.*No such file", // 物理备份恢复会产生这个日志 }, "|", ), diff --git a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/mysqlprocesslist/mysql_inject.go b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/mysqlprocesslist/mysql_inject.go index b8b0bcc237..c2907bdc1a 100644 --- a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/mysqlprocesslist/mysql_inject.go +++ b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/mysqlprocesslist/mysql_inject.go @@ -13,6 +13,8 @@ import ( "strings" "github.com/dlclark/regexp2" + + "dbm-services/mysql/db-tools/mysql-monitor/pkg/internal/cst" ) func mysqlInject() (string, error) { @@ -30,7 +32,7 @@ func mysqlInject() (string, error) { slog.Debug("mysql inject check process", slog.Any("process", pstr)) - if strings.ToLower(p.User.String) == "system user" { + if strings.ToLower(p.User.String) == cst.SystemUser { continue } diff --git a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/mysqlprocesslist/mysql_lock.go b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/mysqlprocesslist/mysql_lock.go index 2a7ee2aec9..1775e0a001 100644 --- a/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/mysqlprocesslist/mysql_lock.go +++ b/dbm-services/mysql/db-tools/mysql-monitor/pkg/itemscollect/mysqlprocesslist/mysql_lock.go @@ -1,10 +1,12 @@ package mysqlprocesslist import ( - "dbm-services/mysql/db-tools/mysql-monitor/pkg/utils" "log/slog" "strings" + "dbm-services/mysql/db-tools/mysql-monitor/pkg/internal/cst" + "dbm-services/mysql/db-tools/mysql-monitor/pkg/utils" + "github.com/dlclark/regexp2" ) @@ -48,7 +50,7 @@ func mysqlLock() (string, error) { slog.Debug("mysql lock check process", slog.Any("process", pstr)) - if strings.ToLower(p.User.String) == "system user" { + if strings.ToLower(p.User.String) == cst.SystemUser { continue } @@ -129,7 +131,7 @@ func hasNormalLock(p *mysqlProcess) (bool, error) { return false, err } - slog.Debug("check normal lock", slog.Bool("match status lock", match)) + slog.Debug("check system lock", slog.Bool("match status lock", match)) if match { return false, nil } From f62e23970bee0aaeceb139d2c04ff2b2e247d815 Mon Sep 17 00:00:00 2001 From: durant <826035498@qq.com> Date: Tue, 17 Dec 2024 16:30:37 +0800 Subject: [PATCH 008/107] =?UTF-8?q?feat(backend):=20=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E8=84=B1=E6=95=8F=20#8662?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dbm-ui/backend/bk_web/middleware.py | 5 -- dbm-ui/backend/bk_web/viewsets.py | 6 +-- dbm-ui/backend/components/__init__.py | 2 + dbm-ui/backend/components/bkbase/__init__.py | 10 ++++ dbm-ui/backend/components/bkbase/client.py | 55 ++++++++++++++++++++ dbm-ui/backend/components/domains.py | 2 + dbm-ui/backend/env/__init__.py | 4 ++ dbm-ui/backend/env/apigw_domains.py | 1 + dbm-ui/bin/build_frontend.sh | 2 +- dbm-ui/frontend/src/types/auto-imports.d.ts | 2 +- 10 files changed, 79 insertions(+), 10 deletions(-) create mode 100644 dbm-ui/backend/components/bkbase/__init__.py create mode 100644 dbm-ui/backend/components/bkbase/client.py diff --git a/dbm-ui/backend/bk_web/middleware.py b/dbm-ui/backend/bk_web/middleware.py index 6f03358786..08824c7024 100644 --- a/dbm-ui/backend/bk_web/middleware.py +++ b/dbm-ui/backend/bk_web/middleware.py @@ -26,7 +26,6 @@ from backend import env from backend.bk_web.constants import ( EXTERNAL_TICKET_TYPE_WHITELIST, - IP_RE, NON_EXTERNAL_PROXY_ROUTING, ROUTING_WHITELIST_PATTERNS, ) @@ -250,10 +249,6 @@ def __call__(self, request): return response - @staticmethod - def replace_ip(text): - return re.sub(IP_RE, "*.*.*.*", text) - class JWTUserModelBackend(UserModelBackend): """dbm jwt用户认证后端""" diff --git a/dbm-ui/backend/bk_web/viewsets.py b/dbm-ui/backend/bk_web/viewsets.py index 5e46290db5..15b27f0ee7 100644 --- a/dbm-ui/backend/bk_web/viewsets.py +++ b/dbm-ui/backend/bk_web/viewsets.py @@ -10,7 +10,6 @@ """ import copy import json -import re from typing import Any, Dict, List, Optional, Tuple, Union from blueapps.account.decorators import login_exempt @@ -22,7 +21,8 @@ from rest_framework.viewsets import ModelViewSet, ReadOnlyModelViewSet from backend import env -from backend.bk_web.constants import EXTERNAL_TICKET_TYPE_WHITELIST, IP_RE +from backend.bk_web.constants import EXTERNAL_TICKET_TYPE_WHITELIST +from backend.components import BKBaseApi from backend.components.dbconsole.client import DBConsoleApi from backend.iam_app.dataclass.actions import ActionEnum from backend.iam_app.handlers.drf_perm.base import RejectPermission @@ -233,7 +233,7 @@ def after_response(self, request, response, *args, **kwargs): if request.path.startswith("/external/apis/") and response.headers.get("Content-Type").startswith( "application/json" ): - data = re.sub(IP_RE, "*.*.*.*", response.content.decode("utf-8")) + data = BKBaseApi.data_desensitization(response.content.decode("utf-8")) return Response(json.loads(data)) # 按原样补充响应头 diff --git a/dbm-ui/backend/components/__init__.py b/dbm-ui/backend/components/__init__.py index 6c194e692f..ae9c218d08 100644 --- a/dbm-ui/backend/components/__init__.py +++ b/dbm-ui/backend/components/__init__.py @@ -11,6 +11,7 @@ from django.apps import AppConfig +from .bkbase.client import BKBaseApi from .bklog.client import BKLogApi from .bkmonitorv3.client import BKMonitorV3Api from .cc.client import CCApi @@ -44,6 +45,7 @@ "DRSApi", "BKMonitorV3Api", "NameServiceApi", + "BKBaseApi", ] diff --git a/dbm-ui/backend/components/bkbase/__init__.py b/dbm-ui/backend/components/bkbase/__init__.py new file mode 100644 index 0000000000..aa5085c628 --- /dev/null +++ b/dbm-ui/backend/components/bkbase/__init__.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" diff --git a/dbm-ui/backend/components/bkbase/client.py b/dbm-ui/backend/components/bkbase/client.py new file mode 100644 index 0000000000..c365a972cc --- /dev/null +++ b/dbm-ui/backend/components/bkbase/client.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +from django.utils.translation import ugettext_lazy as _ + +from ... import env +from ..base import BaseApi +from ..domains import BKBASE_APIGW_DOMAIN + + +class _BKBaseApi(BaseApi): + MODULE = _("基础计算平台") + BASE = BKBASE_APIGW_DOMAIN + + def __init__(self): + self.sensitive_text_classification_normal = self.generate_data_api( + method="POST", + url="v3/aiops/serving/processing/sensitive_text_classification_normal/execute/", + description=_("敏感信息识别"), + ) + + def data_desensitization(self, text): + """ + 敏感信息识别，并把敏感信息转为* + """ + detect_texts = self.sensitive_text_classification_normal( + { + "bkdata_authentication_method": "token", + "bkdata_data_token": env.BKDATA_DATA_TOKEN, + "data": {"inputs": [{"target_content": text}]}, + "config": { + # 心跳超时时间 + "timeout": 30, + # 返回结果不包含输入文本 + "passthrough_input": False, + "predict_args": { + # 填入可选参数，也可不填入，保持为空即按默认配置检测 + "input_config": "1,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22", + "is_masked": "yes", + }, + }, + } + ) + return detect_texts["data"]["data"][0]["output"][0]["masked_text"] + + +BKBaseApi = _BKBaseApi() diff --git a/dbm-ui/backend/components/domains.py b/dbm-ui/backend/components/domains.py index 9c965c29db..7e6b352b37 100644 --- a/dbm-ui/backend/components/domains.py +++ b/dbm-ui/backend/components/domains.py @@ -26,6 +26,8 @@ ITSM_APIGW_DOMAIN = env.ITSM_APIGW_DOMAIN or ESB_DOMAIN_TPL.format("itsm") BKLOG_APIGW_DOMAIN = env.BKLOG_APIGW_DOMAIN or ESB_DOMAIN_TPL.format("bk_log") BKNODEMAN_APIGW_DOMAIN = env.BKNODEMAN_APIGW_DOMAIN or ESB_DOMAIN_TPL.format("nodeman") +BKBASE_APIGW_DOMAIN = env.BKBASE_APIGW_DOMAIN or ESB_DOMAIN_TPL.format("bkbase") + DBCONFIG_APIGW_DOMAIN = env.DBCONFIG_APIGW_DOMAIN or ESB_DOMAIN_TPL.format("dbconfig") DNS_APIGW_DOMAIN = env.DNS_APIGW_DOMAIN or ESB_DOMAIN_TPL.format("dbdns") MYSQL_PRIV_MANAGER_APIGW_DOMAIN = env.MYSQL_PRIV_MANAGER_APIGW_DOMAIN or ESB_DOMAIN_TPL.format("mysql_priv_manager") diff --git a/dbm-ui/backend/env/__init__.py b/dbm-ui/backend/env/__init__.py index 07cd33f1f0..d15d3ac9f4 100644 --- a/dbm-ui/backend/env/__init__.py +++ b/dbm-ui/backend/env/__init__.py @@ -165,6 +165,10 @@ ASYMMETRIC_CIPHER_TYPE = get_type_env(key="ASYMMETRIC_CIPHER_TYPE", _type=str, default=AsymmetricCipherType.RSA.value) SYMMETRIC_CIPHER_TYPE = get_type_env(key="SYMMETRIC_CIPHER_TYPE", _type=str, default=SymmetricCipherType.AES.value) +# 数据平台应用 token +BKDATA_DATA_TOKEN = get_type_env(key="BKDATA_DATA_TOKEN", _type=str, default="") + + # gcs/scr平台 GCS_SCR_OPERATOR = get_type_env(key="GCS_SCR_OPERATOR", _type=str, default="scr-system") diff --git a/dbm-ui/backend/env/apigw_domains.py b/dbm-ui/backend/env/apigw_domains.py index 554219a330..5fab008ced 100644 --- a/dbm-ui/backend/env/apigw_domains.py +++ b/dbm-ui/backend/env/apigw_domains.py @@ -23,6 +23,7 @@ ITSM_APIGW_DOMAIN = get_type_env(key="ITSM_APIGW_DOMAIN", _type=str) BKLOG_APIGW_DOMAIN = get_type_env(key="BKLOG_APIGW_DOMAIN", _type=str) BKNODEMAN_APIGW_DOMAIN = get_type_env(key="BKNODEMAN_APIGW_DOMAIN", _type=str) +BKBASE_APIGW_DOMAIN = get_type_env(key="BKBASE_APIGW_DOMAIN", _type=str) BKMONITORV3_APIGW_DOMAIN = get_type_env(key="BKMONITORV3_APIGW_DOMAIN", _type=str) DRS_APIGW_DOMAIN = get_type_env(key="DRS_APIGW_DOMAIN", _type=str) diff --git a/dbm-ui/bin/build_frontend.sh b/dbm-ui/bin/build_frontend.sh index 9be139692c..b37673a4f2 100755 --- a/dbm-ui/bin/build_frontend.sh +++ b/dbm-ui/bin/build_frontend.sh @@ -4,7 +4,7 @@ SCRIPT_DIR=`dirname $0` cd $SCRIPT_DIR && cd ../frontend || exit 1 npm config set registry https://mirrors.tencent.com/npm/ export NODE_OPTIONS="--max_old_space_size=8192" -npm install . && npm run build +yarn install && yarn build mkdir -p ../static/ cp -rf dist/* ../static/ cd .. diff --git a/dbm-ui/frontend/src/types/auto-imports.d.ts b/dbm-ui/frontend/src/types/auto-imports.d.ts index 7021ac5101..d0c217f6f1 100644 --- a/dbm-ui/frontend/src/types/auto-imports.d.ts +++ b/dbm-ui/frontend/src/types/auto-imports.d.ts @@ -71,6 +71,6 @@ declare global { // for type re-export declare global { // @ts-ignore - export type { Component, ComponentPublicInstance, ComputedRef, ExtractDefaultPropTypes, ExtractPropTypes, ExtractPublicPropTypes, InjectionKey, PropType, Ref, VNode, WritableComputedRef } from 'vue' + export type { Component, ComponentPublicInstance, ComputedRef, DirectiveBinding, ExtractDefaultPropTypes, ExtractPropTypes, ExtractPublicPropTypes, InjectionKey, PropType, Ref, MaybeRef, MaybeRefOrGetter, VNode, WritableComputedRef } from 'vue' import('vue') } From cbac0cbec9498c3a924289231982dfb58b2a2dc9 Mon Sep 17 00:00:00 2001 From: iSecloud <869820505@qq.com> Date: Tue, 17 Dec 2024 15:19:55 +0800 Subject: [PATCH 009/107] =?UTF-8?q?fix(backend):=20=E5=8D=95=E6=8D=AEtodo?= =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=92=8C=E9=97=AE=E9=A2=98=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=20#8661?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dbm-ui/backend/db_meta/models/cluster.py | 10 +++++----- dbm-ui/backend/ticket/constants.py | 15 ++++++++++++--- dbm-ui/backend/ticket/flow_manager/inner.py | 4 +++- dbm-ui/backend/ticket/handler.py | 17 ++++++----------- dbm-ui/backend/ticket/models/todo.py | 15 ++++++++------- dbm-ui/backend/ticket/serializers.py | 2 +- dbm-ui/backend/ticket/todos/pause_todo.py | 19 ++++++++++++++++--- dbm-ui/backend/ticket/views.py | 7 ++++++- 8 files changed, 57 insertions(+), 32 deletions(-) diff --git a/dbm-ui/backend/db_meta/models/cluster.py b/dbm-ui/backend/db_meta/models/cluster.py index 5c380f687d..71ef4f0915 100644 --- a/dbm-ui/backend/db_meta/models/cluster.py +++ b/dbm-ui/backend/db_meta/models/cluster.py @@ -449,14 +449,14 @@ def get_cluster_id__primary_address_map(cls, cluster_ids: List[int]) -> Dict[int continue # 取一个状态正常的 spider-master 接入层 - spider_instance = cluster.proxyinstance_set.filter( + spider_inst = cluster.proxyinstance_set.filter( tendbclusterspiderext__spider_role=TenDBClusterSpiderRole.SPIDER_MASTER, status=InstanceStatus.RUNNING.value, ).first() - - ctl_address = "{}{}{}".format(spider_instance.machine.ip, IP_PORT_DIVIDER, spider_instance.port + 1000) - addresses.append(ctl_address) - ctl_address__cluster_id_map[ctl_address] = cluster.id + if spider_inst: + ctl_address = "{}{}{}".format(spider_inst.machine.ip, IP_PORT_DIVIDER, spider_inst.port + 1000) + addresses.append(ctl_address) + ctl_address__cluster_id_map[ctl_address] = cluster.id logger.info("addresses: {}".format(addresses)) diff --git a/dbm-ui/backend/ticket/constants.py b/dbm-ui/backend/ticket/constants.py index d1fa5b0eab..3fd03fac1d 100644 --- a/dbm-ui/backend/ticket/constants.py +++ b/dbm-ui/backend/ticket/constants.py @@ -127,9 +127,11 @@ class TicketFlowStatus(str, StructuredEnum): SKIPPED = EnumField("SKIPPED", _("跳过")) +# 流程成功状态 FLOW_FINISHED_STATUS = [TicketFlowStatus.SKIPPED, TicketFlowStatus.SUCCEEDED] +# 流程未执行状态 FLOW_NOT_EXECUTE_STATUS = [TicketFlowStatus.SKIPPED, TicketFlowStatus.PENDING] - +# 任务树和流程状态映射 BAMBOO_STATE__TICKET_STATE_MAP = { StateType.FINISHED.value: TicketFlowStatus.SUCCEEDED.value, StateType.FAILED.value: TicketFlowStatus.FAILED.value, @@ -140,6 +142,13 @@ class TicketFlowStatus(str, StructuredEnum): StateType.CREATED.value: TicketFlowStatus.RUNNING.value, StateType.READY.value: TicketFlowStatus.RUNNING.value, } +# 任务流程与todo的状态映射 +INNER_FLOW_TODO_STATUS_MAP = { + TicketFlowStatus.TERMINATED: TodoStatus.DONE_FAILED, + TicketFlowStatus.FAILED: TodoStatus.TODO, + TicketFlowStatus.SUCCEEDED: TodoStatus.DONE_SUCCESS, + TicketFlowStatus.RUNNING: TodoStatus.DONE_SUCCESS, +} EXCLUSIVE_TICKET_EXCEL_PATH = "backend/ticket/exclusive_ticket.xlsx" @@ -296,9 +305,9 @@ def get_approve_mode_by_ticket(cls, ticket_type): TENDBCLUSTER_AUTHORIZE_RULES = TicketEnumField("TENDBCLUSTER_AUTHORIZE_RULES", _("TenDB Cluster 授权"), _("权限管理")) TENDBCLUSTER_EXCEL_AUTHORIZE_RULES = TicketEnumField("TENDBCLUSTER_EXCEL_AUTHORIZE_RULES", _("TenDB Cluster EXCEL授权"), _("权限管理")) # noqa TENDBCLUSTER_STANDARDIZE = TicketEnumField("TENDBCLUSTER_STANDARDIZE", _("TenDB Cluster 集群标准化"), register_iam=False) - TENDBCLUSTER_METADATA_IMPORT = TicketEnumField("TENDBCLUSTER_METADATA_IMPORT", _("TenDB Cluster 元数据导入"),register_iam=False) # noqa + TENDBCLUSTER_METADATA_IMPORT = TicketEnumField("TENDBCLUSTER_METADATA_IMPORT", _("TenDB Cluster 元数据导入"), register_iam=False) # noqa TENDBCLUSTER_APPEND_DEPLOY_CTL = TicketEnumField("TENDBCLUSTER_APPEND_DEPLOY_CTL", _("TenDB Cluster 追加部署中控"), register_iam=False) # noqa - TENDBSINGLE_METADATA_IMPORT = TicketEnumField("TENDBSINGLE_METADATA_IMPORT", _("TenDB Single 元数据导入"), register_iam=False) # noqa + TENDBSINGLE_METADATA_IMPORT = TicketEnumField("TENDBSINGLE_METADATA_IMPORT", _("TenDB Single 元数据导入"), register_iam=False) # noqa TENDBSINGLE_STANDARDIZE = TicketEnumField("TENDBSINGLE_STANDARDIZE", _("TenDB Single 集群标准化"), register_iam=False) # noqa TENDBCLUSTER_DATA_MIGRATE = TicketEnumField("TENDBCLUSTER_DATA_MIGRATE", _("TenDB Cluster DB克隆"), _("数据处理")) TENDBCLUSTER_DUMP_DATA = TicketEnumField("TENDBCLUSTER_DUMP_DATA", _("TenDB Cluster 数据导出"), _("数据处理")) diff --git a/dbm-ui/backend/ticket/flow_manager/inner.py b/dbm-ui/backend/ticket/flow_manager/inner.py index 9773c200a1..3fccd85d86 100644 --- a/dbm-ui/backend/ticket/flow_manager/inner.py +++ b/dbm-ui/backend/ticket/flow_manager/inner.py @@ -26,6 +26,7 @@ from backend.ticket.builders.common.base import fetch_cluster_ids from backend.ticket.constants import ( BAMBOO_STATE__TICKET_STATE_MAP, + INNER_FLOW_TODO_STATUS_MAP, FlowCallbackType, FlowErrCode, FlowMsgType, @@ -106,7 +107,8 @@ def _status(self) -> str: else: status = BAMBOO_STATE__TICKET_STATE_MAP.get(self.flow_tree.status, constants.TicketFlowStatus.RUNNING) - todo_status = TodoStatus.TODO if status == TicketFlowStatus.FAILED else TodoStatus.DONE_SUCCESS + # 根据流程状态映射todo的状态 + todo_status = INNER_FLOW_TODO_STATUS_MAP.get(status, TodoStatus.TODO) fail_todo = self.flow_obj.todo_of_flow.filter(type=TodoType.INNER_FAILED).first() # 如果任务失败，且不存在todo，则创建一条 if not fail_todo and todo_status == TodoStatus.TODO: diff --git a/dbm-ui/backend/ticket/handler.py b/dbm-ui/backend/ticket/handler.py index e60b776e60..61152eeb40 100644 --- a/dbm-ui/backend/ticket/handler.py +++ b/dbm-ui/backend/ticket/handler.py @@ -68,7 +68,8 @@ def add_related_object(cls, ticket_data: List[Dict]) -> List[Dict]: {int(cluster_id): info["immute_domain"] for cluster_id, info in clusters.items()} ) instances = ticket.details.get("instances", {}) - instance_id_ip_port_map.update({int(inst_id): info["instance"] for inst_id, info in instances.items()}) + if isinstance(instances, dict): + instance_id_ip_port_map.update({int(inst_id): info["instance"] for inst_id, info in instances.items()}) ticket_id_obj_ids_map[ticket.id] = { "cluster_ids": fetch_cluster_ids(ticket.details), "instance_ids": fetch_instance_ids(ticket.details), @@ -419,7 +420,6 @@ def ticket_status_standardization(cls): """ 旧单据状态标准化。TODO: 迁移后此段代码可删除 """ - batch = 50 # 标准化只针对running的单据，其他状态单据不影响 running_tickets = list(Ticket.objects.filter(status=TicketStatus.RUNNING)) @@ -430,36 +430,31 @@ def ticket_status_standardization(cls): print(f"ticket[{ticket.id}] status {raw_status} ---> {ticket.status}") # 失败的单据要增加一条todo关联 - failed_tickets = Ticket.objects.prefetch_related("flows").filter(status=TicketStatus.FAILED) - todos = [] + failed_tickets = Ticket.objects.prefetch_related("flows__todo_of_flow").filter(status=TicketStatus.FAILED) for ticket in failed_tickets: inner_flow = ticket.flows.filter(flow_type=FlowType.INNER_FLOW, status=TicketFlowStatus.FAILED).first() if not inner_flow or inner_flow.todo_of_flow.exists(): continue - todo = Todo( + Todo.objects.create( name=_("【{}】单据任务执行失败，待处理").format(ticket.get_ticket_type_display()), flow=inner_flow, ticket=ticket, type=TodoType.INNER_FAILED, context=BaseTodoContext(inner_flow.id, ticket.id).to_dict(), ) - todos.append(todo) print(f"ticket[{ticket.id}] add a failed todo") # 待审批的单据要增加一条todo关联 - itsm_tickets = Ticket.objects.prefetch_related("flows").filter(status=TicketStatus.FAILED) + itsm_tickets = Ticket.objects.prefetch_related("flows").filter(status=TicketStatus.APPROVE) for ticket in itsm_tickets: itsm_flow = ticket.flows.filter(flow_type=FlowType.BK_ITSM, status=TicketFlowStatus.RUNNING).first() if not itsm_flow or itsm_flow.todo_of_flow.exists(): continue - todo = Todo( + Todo.objects.create( name=_("【{}】单据等待审批").format(ticket.get_ticket_type_display()), flow=itsm_flow, ticket=ticket, type=TodoType.ITSM, context=ItsmTodoContext(itsm_flow.id, ticket.id).to_dict(), ) - todos.append(todo) print(f"ticket[{ticket.id}] add a itsm todo") - - Todo.objects.bulk_create(todos, batch_size=batch) diff --git a/dbm-ui/backend/ticket/models/todo.py b/dbm-ui/backend/ticket/models/todo.py index c437256aef..b39fdfb2f0 100644 --- a/dbm-ui/backend/ticket/models/todo.py +++ b/dbm-ui/backend/ticket/models/todo.py @@ -44,17 +44,18 @@ def get_operators(self, todo_type, ticket, operators): # 构造单据状态与处理人之间的对应关系 # - 审批中：提单人可撤销，dba可处理 # - 待执行：提单人 + 单据协助人 - # - 待继续：提单人 + dba + 单据协助人 - # - 待补货：提单人 + dba + 单据协助人 - # - 已失败：提单人 + dba + 单据协助人 + # - 待继续：提单人 + 单据协助人 + dba + # - 待补货：提单人 + 单据协助人 + dba + # - 已失败：提单人 + 单据协助人 + dba todo_operators_map = { TodoType.ITSM: dba, TodoType.APPROVE: creator + biz_helpers, - TodoType.INNER_APPROVE: creator + dba + biz_helpers, - TodoType.RESOURCE_REPLENISH: creator + dba + biz_helpers, - TodoType.INNER_FAILED: creator + dba + biz_helpers, + TodoType.INNER_APPROVE: creator + biz_helpers + dba, + TodoType.RESOURCE_REPLENISH: creator + biz_helpers + dba, + TodoType.INNER_FAILED: creator + biz_helpers + dba, } - operators = list(set(operators + todo_operators_map.get(todo_type, []))) + # 按照顺序去重 + operators = list(dict.fromkeys(operators + todo_operators_map.get(todo_type, []))) return operators def create(self, **kwargs): diff --git a/dbm-ui/backend/ticket/serializers.py b/dbm-ui/backend/ticket/serializers.py index 7e423b2698..285e244d82 100644 --- a/dbm-ui/backend/ticket/serializers.py +++ b/dbm-ui/backend/ticket/serializers.py @@ -131,7 +131,7 @@ def get_status(self, obj): return obj.status def get_ticket_type_display(self, obj): - return obj.get_ticket_type_display() + return TicketType.get_choice_label(obj.ticket_type) def get_status_display(self, obj): return TicketStatus.get_choice_label(obj.status) diff --git a/dbm-ui/backend/ticket/todos/pause_todo.py b/dbm-ui/backend/ticket/todos/pause_todo.py index 9b44d8275a..2211bc6790 100644 --- a/dbm-ui/backend/ticket/todos/pause_todo.py +++ b/dbm-ui/backend/ticket/todos/pause_todo.py @@ -12,8 +12,7 @@ from backend.db_meta.models.sqlserver_dts import DtsStatus, SqlserverDtsInfo from backend.ticket import todos -from backend.ticket.constants import TicketFlowStatus, TicketType, TodoType -from backend.ticket.flow_manager import manager +from backend.ticket.constants import TicketFlowStatus, TicketType, TodoStatus, TodoType from backend.ticket.flow_manager.manager import TicketFlowManager from backend.ticket.todos import ActionType, BaseTodoContext @@ -43,7 +42,7 @@ def _process(self, username, action, params): # 所有待办完成后，执行后面的flow if not self.todo.ticket.todo_of_ticket.exist_unfinished(): - manager.TicketFlowManager(ticket=self.todo.ticket).run_next_flow() + TicketFlowManager(ticket=self.todo.ticket).run_next_flow() # 如果是数据迁移单据，更改迁移记录状态信息 if self.todo.ticket.ticket_type in [TicketType.SQLSERVER_INCR_MIGRATE, TicketType.SQLSERVER_FULL_MIGRATE]: @@ -73,3 +72,17 @@ def _process(self, username, action, params): self.todo.refresh_from_db(fields=["flow"]) if self.todo.flow.status == TicketFlowStatus.SUCCEEDED: self.todo.set_success(username, action) + + +@todos.TodoActorFactory.register(TodoType.INNER_FAILED) +class FailedTodo(todos.TodoActor): + """来自主流程-失败后待确认""" + + def _process(self, username, action, params): + # 终止-仅将todo进行终止(任务流程的终止)，确认-关联flow进行重试 + if action == ActionType.TERMINATE: + self.todo.set_status(username, TodoStatus.DONE_FAILED) + else: + manager = TicketFlowManager(ticket=self.todo.ticket) + fail_inner_flow = manager.get_ticket_flow_cls(self.todo.flow.flow_type)(self.todo.flow) + fail_inner_flow.retry() diff --git a/dbm-ui/backend/ticket/views.py b/dbm-ui/backend/ticket/views.py index ea244e4235..d2a11f8cae 100644 --- a/dbm-ui/backend/ticket/views.py +++ b/dbm-ui/backend/ticket/views.py @@ -92,7 +92,7 @@ class TicketViewSet(viewsets.AuditedModelViewSet): 单据视图 """ - queryset = Ticket.objects.all().prefetch_related("todo_of_ticket") + queryset = Ticket.objects.all() serializer_class = TicketSerializer filter_class = TicketListFilter pagination_class = AuditedLimitOffsetPagination @@ -158,6 +158,11 @@ def _get_self_manage_tickets(cls, user): ticket_filter = Q(creator=user.username) | reduce(operator.or_, manage_filters or [Q()]) return Ticket.objects.filter(ticket_filter) + def filter_queryset(self, queryset): + """filter_class可能导致预取的todo失效，这里重新取一次""" + queryset = super().filter_queryset(queryset) + return queryset.prefetch_related("todo_of_ticket") + def get_queryset(self): """ 单据queryset规则--针对list： From 1c5e39ed5028abab338ed691c8ab1a93c56be3e2 Mon Sep 17 00:00:00 2001 From: ycggyao Date: Wed, 18 Dec 2024 10:57:57 +0800 Subject: [PATCH 010/107] =?UTF-8?q?fix(backend):=20sqlserver=E6=89=B9?= =?UTF-8?q?=E9=87=8F=E9=83=A8=E7=BD=B2=E9=9B=86=E7=BE=A4=E9=97=AE=E9=A2=98?= =?UTF-8?q?=20#8681=20#=20Reviewed,=20transaction=20id:=2027114?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../builders/sqlserver/sqlserver_ha_apply.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/dbm-ui/backend/ticket/builders/sqlserver/sqlserver_ha_apply.py b/dbm-ui/backend/ticket/builders/sqlserver/sqlserver_ha_apply.py index a85dff3d2e..9db57848ec 100644 --- a/dbm-ui/backend/ticket/builders/sqlserver/sqlserver_ha_apply.py +++ b/dbm-ui/backend/ticket/builders/sqlserver/sqlserver_ha_apply.py @@ -42,7 +42,7 @@ def validate(self, attrs): attrs["cluster_count"] // attrs["inst_num"] + bool(attrs["cluster_count"] % attrs["inst_num"]) ) if attrs["ip_source"] == IpSource.RESOURCE_POOL: - machine_count = attrs["resource_spec"][MachineType.SQLSERVER_HA.value]["count"] + machine_count = attrs["resource_spec"]["backend_group"]["count"] * 2 else: machine_count = len(attrs["nodes"][MachineType.SQLSERVER_HA.value]) if machine_count != expected_count: @@ -85,12 +85,12 @@ def format_cluster_domains(self) -> List[Dict[str, str]]: @classmethod def insert_ip_into_apply_infos(cls, ticket_data, infos: List[Dict]): - backend_nodes = ticket_data["nodes"][MachineType.SQLSERVER_HA.value] + backend_nodes = ticket_data["nodes"]["backend_group"] for index, apply_info in enumerate(infos): - # 每组集群需要两个后端 IP 和两个 Proxy IP - start, end = index * 2, (index + 1) * 2 - apply_info["mssql_master_host"] = backend_nodes[start:end][0] - apply_info["mssql_slave_host"] = backend_nodes[start:end][1] + # # 每组集群需要两个后端 IP 和两个 Proxy IP + # start, end = index * 2, (index + 1) * 2 + apply_info["mssql_master_host"] = backend_nodes[index]["master"] + apply_info["mssql_slave_host"] = backend_nodes[index]["slave"] class SQLServerHaApplyResourceParamBuilder(SQLServerSingleApplyResourceParamBuilder): @@ -101,6 +101,9 @@ def post_callback(self): next_flow = self.ticket.next_flow() infos = next_flow.details["ticket_data"]["infos"] SQLServerHAApplyFlowParamBuilder.insert_ip_into_apply_infos(self.ticket.details, infos) + next_flow.details["ticket_data"]["resource_spec"]["sqlserver_ha"] = next_flow.details["ticket_data"][ + "resource_spec" + ]["master"] next_flow.details["ticket_data"].update(infos=infos) next_flow.save(update_fields=["details"]) From 0caa98114e09f83b94225c779679cb597db6af0f Mon Sep 17 00:00:00 2001 From: 3octaves <873551943@qq.com> Date: Tue, 17 Dec 2024 10:42:31 +0800 Subject: [PATCH 011/107] =?UTF-8?q?fix(frontend):=20=E4=BF=AE=E5=A4=8Dredi?= =?UTF-8?q?s=E8=87=AA=E5=AE=9A=E4=B9=89=E6=96=B9=E6=A1=88=E9=80=89?= =?UTF-8?q?=E4=B8=8D=E5=87=BA=E8=A7=84=E6=A0=BC=E7=9A=84=E9=97=AE=E9=A2=98?= =?UTF-8?q?=20#8654?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../frontend/src/views/db-manage/redis/apply/ApplyRedis.vue | 5 ++++- .../db-manage/redis/apply/components/backend-spec/Index.vue | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/dbm-ui/frontend/src/views/db-manage/redis/apply/ApplyRedis.vue b/dbm-ui/frontend/src/views/db-manage/redis/apply/ApplyRedis.vue index d95572687f..1d25a2bb14 100644 --- a/dbm-ui/frontend/src/views/db-manage/redis/apply/ApplyRedis.vue +++ b/dbm-ui/frontend/src/views/db-manage/redis/apply/ApplyRedis.vue @@ -300,6 +300,7 @@ - typeInfos.value.cluster_type === ClusterTypes.PREDIXY_REDIS_CLUSTER + typeInfos.value.cluster_type === ClusterTypes.PREDIXY_REDIS_CLUSTER && applySchema.value === APPLY_SCHEME.AUTO ? ClusterTypes.PREDIXY_REDIS_CLUSTER : specClusterMachineMap[typeInfos.value.cluster_type], ); diff --git a/dbm-ui/frontend/src/views/db-manage/redis/apply/components/backend-spec/Index.vue b/dbm-ui/frontend/src/views/db-manage/redis/apply/components/backend-spec/Index.vue index bf49dd86eb..0410e4796a 100644 --- a/dbm-ui/frontend/src/views/db-manage/redis/apply/components/backend-spec/Index.vue +++ b/dbm-ui/frontend/src/views/db-manage/redis/apply/components/backend-spec/Index.vue @@ -99,13 +99,13 @@ const props = defineProps(); const modelValue = defineModel({ required: true }); + const applySchema = defineModel('applySchema', {required: true}) const { t } = useI18n(); const specRef = ref(); const customSchemaRef = ref>() const isLoading = ref(false); - const applySchema = ref(APPLY_SCHEME.AUTO) const specs = shallowRef([]); const countMap = shallowRef({} as Record) @@ -192,7 +192,7 @@ if (capacityValue === '' || futureCapacityValue === '') { resetSlider(); } else { - modelValue.value.spec_id = -1; + modelValue.value.spec_id = ''; clearTimeout(timer); timer = setTimeout(() => { fetchFilterClusterSpec(); From dfb422fea5dfedae27f28b3acfd367b95773bbfa Mon Sep 17 00:00:00 2001 From: royalpioneer Date: Thu, 12 Dec 2024 17:39:36 +0800 Subject: [PATCH 012/107] =?UTF-8?q?feat(frontend):=20=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E4=BA=A7=E5=93=81=E6=96=87=E6=A1=A3=E5=85=A5=E5=8F=A3=20#8572?= =?UTF-8?q?=20#=20Reviewed,=20transaction=20id:=2026987?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dbm-ui/frontend/src/App.vue | 15 ++++++++++++++- dbm-ui/frontend/src/locales/zh-cn.json | 1 + 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/dbm-ui/frontend/src/App.vue b/dbm-ui/frontend/src/App.vue index 463954d91c..27dfa75059 100644 --- a/dbm-ui/frontend/src/App.vue +++ b/dbm-ui/frontend/src/App.vue @@ -22,6 +22,11 @@ @click="handleShowSystemVersionLog"> {{ t('版本日志') }} +
+ {{ t('产品文档') }} +
= { en: 'DBM | Tencent BlueKing', @@ -111,6 +117,13 @@ isShowSystemVersionLog.value = true; }; + const linkToDoc = () => { + const url = systemEnvironStore.urls.BK_HELPER_URL; + if (url) { + window.open(url); + } + }; + const handleSignOut = () => { InfoBox({ title: t('确认退出登录'), diff --git a/dbm-ui/frontend/src/locales/zh-cn.json b/dbm-ui/frontend/src/locales/zh-cn.json index 58cfd88607..49c3ec7889 100644 --- a/dbm-ui/frontend/src/locales/zh-cn.json +++ b/dbm-ui/frontend/src/locales/zh-cn.json @@ -3856,5 +3856,6 @@ "Proxy起始端口：": "Proxy起始端口：", "数量：": "数量：", "源 DB 名": "源 DB 名", + "产品文档": "产品文档", "这行勿动！新增翻译请在上一行添加！": "" } From 96dd56f7ca562bcb9d910125a9bd144c3102ba89 Mon Sep 17 00:00:00 2001 From: hLinx <327159425@qq.com> Date: Tue, 17 Dec 2024 14:37:20 +0800 Subject: [PATCH 013/107] =?UTF-8?q?fix(frontend):=20=E5=8D=95=E6=8D=AE?= =?UTF-8?q?=E5=89=8D=E7=AB=AF=E4=BA=A4=E4=BA=92=E9=97=AE=E9=A2=98=20#8656?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/components/tag-block/Index.vue | 11 ++- .../src/services/model/ticket/ticket.ts | 4 + .../ticket-center/common/TableModeTable.vue | 2 +- .../common/ticket-detail/Index.vue | 7 +- .../components/render-sqlfile/Index.vue | 18 +++- .../components/RenderFileList.vue | 15 +++- .../components/list/components/TableMode.vue | 1 + .../components/List.vue | 88 ++++++++++--------- 8 files changed, 95 insertions(+), 51 deletions(-) diff --git a/dbm-ui/frontend/src/components/tag-block/Index.vue b/dbm-ui/frontend/src/components/tag-block/Index.vue index c3313b2e9e..0800943f93 100644 --- a/dbm-ui/frontend/src/components/tag-block/Index.vue +++ b/dbm-ui/frontend/src/components/tag-block/Index.vue @@ -36,7 +36,7 @@
+ class="dbm-tag-block-more-panel"> @@ -151,6 +151,7 @@ theme: 'light', interactive: true, arrow: true, + maxWidth: 400, offset: [0, 8], zIndex: 999999, hideOnClick: true, @@ -228,4 +229,12 @@ } } } + + .dbm-tag-block-more-panel { + margin-top: -8px; + + .bk-tag { + margin-top: 8px; + } + } diff --git a/dbm-ui/frontend/src/services/model/ticket/ticket.ts b/dbm-ui/frontend/src/services/model/ticket/ticket.ts index 0cdd26bf33..de4e377e1f 100644 --- a/dbm-ui/frontend/src/services/model/ticket/ticket.ts +++ b/dbm-ui/frontend/src/services/model/ticket/ticket.ts @@ -139,4 +139,8 @@ export default class Ticket { Ticket.STATUS_RUNNING, ].includes(this.status); } + + get isFinished() { + return [Ticket.STATUS_SUCCEEDED, Ticket.STATUS_TERMINATED].includes(this.status); + } } diff --git a/dbm-ui/frontend/src/views/ticket-center/common/TableModeTable.vue b/dbm-ui/frontend/src/views/ticket-center/common/TableModeTable.vue index d5758ae701..5ad3c36377 100644 --- a/dbm-ui/frontend/src/views/ticket-center/common/TableModeTable.vue +++ b/dbm-ui/frontend/src/views/ticket-center/common/TableModeTable.vue @@ -162,7 +162,7 @@
- + props.ticketId, diff --git a/dbm-ui/frontend/src/views/ticket-center/common/ticket-detail/components/task-info/com-factory/common/mysql-import-sqlfile/components/render-sqlfile/Index.vue b/dbm-ui/frontend/src/views/ticket-center/common/ticket-detail/components/task-info/com-factory/common/mysql-import-sqlfile/components/render-sqlfile/Index.vue index 04f847aac9..10c036b738 100644 --- a/dbm-ui/frontend/src/views/ticket-center/common/ticket-detail/components/task-info/com-factory/common/mysql-import-sqlfile/components/render-sqlfile/Index.vue +++ b/dbm-ui/frontend/src/views/ticket-center/common/ticket-detail/components/task-info/com-factory/common/mysql-import-sqlfile/components/render-sqlfile/Index.vue @@ -13,19 +13,33 @@ {{ t('变更的 DB:') }} {{ item }} + + ... + {{ t('忽略的 DB:') }} {{ item }} + + ... + diff --git a/dbm-ui/frontend/src/views/ticket-center/common/ticket-detail/components/task-info/com-factory/common/mysql-import-sqlfile/components/render-sqlfile/components/RenderFileList.vue b/dbm-ui/frontend/src/views/ticket-center/common/ticket-detail/components/task-info/com-factory/common/mysql-import-sqlfile/components/render-sqlfile/components/RenderFileList.vue index 4ff22d31e6..30b3845047 100644 --- a/dbm-ui/frontend/src/views/ticket-center/common/ticket-detail/components/task-info/com-factory/common/mysql-import-sqlfile/components/render-sqlfile/components/RenderFileList.vue +++ b/dbm-ui/frontend/src/views/ticket-center/common/ticket-detail/components/task-info/com-factory/common/mysql-import-sqlfile/components/render-sqlfile/components/RenderFileList.vue @@ -28,7 +28,11 @@ active: fileItemData.name === modelValue, }" @click="handleClick(fileItemData.name)"> - {{ getSQLFilename(fileItemData.name) }} +
+ {{ getSQLFilename(fileItemData.name) }}asdasdasdasdasdasd +
@@ -105,12 +109,14 @@ .file-item { display: flex; - align-items: center; height: 36px; padding: 0 8px; + white-space: pre; cursor: pointer; background: rgb(255 255 255 / 8%); border-radius: 2px; + align-items: center; + user-select: none; &:hover { background: rgb(255 255 255 / 20%); @@ -132,6 +138,11 @@ margin-top: 8px; } } + + .file-item-text { + overflow: hidden; + text-overflow: ellipsis; + } } } diff --git a/dbm-ui/frontend/src/views/ticket-center/self-todo/components/list/components/TableMode.vue b/dbm-ui/frontend/src/views/ticket-center/self-todo/components/list/components/TableMode.vue index 9103093d83..b2ae0d5ea3 100644 --- a/dbm-ui/frontend/src/views/ticket-center/self-todo/components/list/components/TableMode.vue +++ b/dbm-ui/frontend/src/views/ticket-center/self-todo/components/list/components/TableMode.vue @@ -62,6 +62,7 @@ - + - + - + - From 7bbf09ea900776d52509357a4dc815b1a7aef4a5 Mon Sep 17 00:00:00 2001 From: hLinx <327159425@qq.com> Date: Fri, 20 Dec 2024 17:01:36 +0800 Subject: [PATCH 041/107] =?UTF-8?q?perf(frontend):=20sqlserver=20=E5=AE=9A?= =?UTF-8?q?=E7=82=B9=E6=9E=84=E9=80=A0=E5=88=B0=E5=85=B6=E5=AE=83=E9=9B=86?= =?UTF-8?q?=E7=BE=A4=E5=8E=BB=E6=8E=89=E9=9B=86=E7=BE=A4=E9=87=8D=E5=A4=8D?= =?UTF-8?q?=E7=9A=84=E6=A0=A1=E9=AA=8C=20#8747?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../other-cluster/components/DstCluster.vue | 28 +------------------ .../other-cluster/components/SrcCluster.vue | 28 +------------------ 2 files changed, 2 insertions(+), 54 deletions(-) diff --git a/dbm-ui/frontend/src/views/db-manage/sqlserver/rollback/pages/page1/components/other-cluster/components/DstCluster.vue b/dbm-ui/frontend/src/views/db-manage/sqlserver/rollback/pages/page1/components/other-cluster/components/DstCluster.vue index ad1e49edda..aad4289236 100644 --- a/dbm-ui/frontend/src/views/db-manage/sqlserver/rollback/pages/page1/components/other-cluster/components/DstCluster.vue +++ b/dbm-ui/frontend/src/views/db-manage/sqlserver/rollback/pages/page1/components/other-cluster/components/DstCluster.vue @@ -30,11 +30,8 @@ :tab-list-config="clusterSelectorTabConfig" @change="handelClusterChange" /> - + diff --git a/dbm-ui/frontend/src/services/model/tendbcluster/tendbcluster.ts b/dbm-ui/frontend/src/services/model/tendbcluster/tendbcluster.ts index 8ba032f26e..28486b1023 100644 --- a/dbm-ui/frontend/src/services/model/tendbcluster/tendbcluster.ts +++ b/dbm-ui/frontend/src/services/model/tendbcluster/tendbcluster.ts @@ -113,8 +113,8 @@ export default class TendbCluster { phase: 'online' | 'offline'; phase_name: string; region: string; - remote_db: ClusterListNode[]; - remote_dr: ClusterListNode[]; + remote_db: (ClusterListNode & { shard_id: number })[]; + remote_dr: (ClusterListNode & { shard_id: number })[]; remote_shard_num: number; slave_domain: string; spider_master: ClusterListNode[]; diff --git a/dbm-ui/frontend/src/services/types/db/clusterListNode.ts b/dbm-ui/frontend/src/services/types/db/clusterListNode.ts index e76a033b92..b8e114fe57 100644 --- a/dbm-ui/frontend/src/services/types/db/clusterListNode.ts +++ b/dbm-ui/frontend/src/services/types/db/clusterListNode.ts @@ -2,6 +2,7 @@ export interface ClusterListNode { bk_biz_id: number; bk_cloud_id: number; bk_host_id: number; + bk_sub_zone: string; bk_instance_id: number; instance: string; ip: string; diff --git a/dbm-ui/frontend/src/views/db-manage/common/render-instances/RenderInstances.vue b/dbm-ui/frontend/src/views/db-manage/common/render-instances/RenderInstances.vue index 4e0a5408ff..2cc9513b88 100644 --- a/dbm-ui/frontend/src/views/db-manage/common/render-instances/RenderInstances.vue +++ b/dbm-ui/frontend/src/views/db-manage/common/render-instances/RenderInstances.vue @@ -15,7 +15,7 @@

@@ -104,13 +104,24 @@

+ @request-finished="handleRequestFinished"> + + + + + {{ role }} + + + + + - - + diff --git a/dbm-ui/frontend/src/views/db-manage/redis/migrate/pages/page1/components/cluseter-instance/Index.vue b/dbm-ui/frontend/src/views/db-manage/redis/migrate/pages/page1/components/cluseter-instance/Index.vue new file mode 100644 index 0000000000..9fe95e5583 --- /dev/null +++ b/dbm-ui/frontend/src/views/db-manage/redis/migrate/pages/page1/components/cluseter-instance/Index.vue @@ -0,0 +1,472 @@ + + + + diff --git a/dbm-ui/frontend/src/views/db-manage/redis/migrate/pages/page1/components/cluseter-instance/RenderCurrentVersion.vue b/dbm-ui/frontend/src/views/db-manage/redis/migrate/pages/page1/components/cluseter-instance/RenderCurrentVersion.vue new file mode 100644 index 0000000000..319accfd70 --- /dev/null +++ b/dbm-ui/frontend/src/views/db-manage/redis/migrate/pages/page1/components/cluseter-instance/RenderCurrentVersion.vue @@ -0,0 +1,110 @@ + + + + + diff --git a/dbm-ui/frontend/src/views/db-manage/redis/migrate/pages/page1/components/cluseter-instance/Row.vue b/dbm-ui/frontend/src/views/db-manage/redis/migrate/pages/page1/components/cluseter-instance/Row.vue new file mode 100644 index 0000000000..4531f81c99 --- /dev/null +++ b/dbm-ui/frontend/src/views/db-manage/redis/migrate/pages/page1/components/cluseter-instance/Row.vue @@ -0,0 +1,194 @@ + + + + + + diff --git a/dbm-ui/frontend/src/views/db-manage/redis/migrate/pages/page1/components/common/RenderOldMasterSlaveHost.vue b/dbm-ui/frontend/src/views/db-manage/redis/migrate/pages/page1/components/common/RenderOldMasterSlaveHost.vue new file mode 100644 index 0000000000..799ec8a694 --- /dev/null +++ b/dbm-ui/frontend/src/views/db-manage/redis/migrate/pages/page1/components/common/RenderOldMasterSlaveHost.vue @@ -0,0 +1,228 @@ + + + + + + diff --git a/dbm-ui/frontend/src/views/db-manage/redis/migrate/pages/page1/components/common/render-spec/Index.vue b/dbm-ui/frontend/src/views/db-manage/redis/migrate/pages/page1/components/common/render-spec/Index.vue new file mode 100644 index 0000000000..1a69e1981a --- /dev/null +++ b/dbm-ui/frontend/src/views/db-manage/redis/migrate/pages/page1/components/common/render-spec/Index.vue @@ -0,0 +1,189 @@ + + + + + + + diff --git a/dbm-ui/frontend/src/views/db-manage/redis/migrate/pages/page1/components/common/render-spec/SpecPanel.vue b/dbm-ui/frontend/src/views/db-manage/redis/migrate/pages/page1/components/common/render-spec/SpecPanel.vue new file mode 100644 index 0000000000..9ed9a5a725 --- /dev/null +++ b/dbm-ui/frontend/src/views/db-manage/redis/migrate/pages/page1/components/common/render-spec/SpecPanel.vue @@ -0,0 +1,238 @@ + + + + + + diff --git a/dbm-ui/frontend/src/views/db-manage/redis/migrate/pages/page1/components/master-slave-instance/Index.vue b/dbm-ui/frontend/src/views/db-manage/redis/migrate/pages/page1/components/master-slave-instance/Index.vue new file mode 100644 index 0000000000..01e9825b1f --- /dev/null +++ b/dbm-ui/frontend/src/views/db-manage/redis/migrate/pages/page1/components/master-slave-instance/Index.vue @@ -0,0 +1,395 @@ + + + + diff --git a/dbm-ui/frontend/src/views/db-manage/redis/migrate/pages/page1/components/master-slave-instance/Row.vue b/dbm-ui/frontend/src/views/db-manage/redis/migrate/pages/page1/components/master-slave-instance/Row.vue new file mode 100644 index 0000000000..8efc6b97fb --- /dev/null +++ b/dbm-ui/frontend/src/views/db-manage/redis/migrate/pages/page1/components/master-slave-instance/Row.vue @@ -0,0 +1,223 @@ + + + + + + diff --git a/dbm-ui/frontend/src/views/db-manage/redis/migrate/pages/page1/components/master-slave-machine/Index.vue b/dbm-ui/frontend/src/views/db-manage/redis/migrate/pages/page1/components/master-slave-machine/Index.vue new file mode 100644 index 0000000000..5de90ed306 --- /dev/null +++ b/dbm-ui/frontend/src/views/db-manage/redis/migrate/pages/page1/components/master-slave-machine/Index.vue @@ -0,0 +1,423 @@ + + + + + + diff --git a/dbm-ui/frontend/src/views/db-manage/redis/migrate/pages/page1/components/master-slave-machine/Row.vue b/dbm-ui/frontend/src/views/db-manage/redis/migrate/pages/page1/components/master-slave-machine/Row.vue new file mode 100644 index 0000000000..35bd99a419 --- /dev/null +++ b/dbm-ui/frontend/src/views/db-manage/redis/migrate/pages/page1/components/master-slave-machine/Row.vue @@ -0,0 +1,221 @@ + + + + + + diff --git a/dbm-ui/frontend/src/views/db-manage/redis/migrate/pages/page2/Index.vue b/dbm-ui/frontend/src/views/db-manage/redis/migrate/pages/page2/Index.vue new file mode 100644 index 0000000000..7e80af2f5b --- /dev/null +++ b/dbm-ui/frontend/src/views/db-manage/redis/migrate/pages/page2/Index.vue @@ -0,0 +1,87 @@ + + + + diff --git a/dbm-ui/frontend/src/views/db-manage/redis/routes.ts b/dbm-ui/frontend/src/views/db-manage/redis/routes.ts index 51fc092d65..daeaaa301f 100644 --- a/dbm-ui/frontend/src/views/db-manage/redis/routes.ts +++ b/dbm-ui/frontend/src/views/db-manage/redis/routes.ts @@ -84,6 +84,15 @@ const redisDBReplaceRoute = { component: () => import('@views/db-manage/redis/db-replace/Index.vue'), }; +const redisMigrateRoute = { + name: 'RedisMigrate', + path: 'db-migrate/:page?', + meta: { + navName: t('迁移'), + }, + component: () => import('@views/db-manage/redis/migrate/Index.vue'), +}; + const redisClusterShardUpdateRoute = { name: 'RedisClusterShardUpdate', path: 'cluster-shard-update/:page?', @@ -175,6 +184,7 @@ const toolboxDbConsoleRouteMap = { 'redis.toolbox.slaveRebuild': redisDBCreateSlaveRoute, 'redis.toolbox.masterSlaveSwap': redisMasterFailoverRoute, 'redis.toolbox.dbReplace': redisDBReplaceRoute, + 'redis.toolbox.migrate': redisMigrateRoute, 'redis.toolbox.versionUpgrade': redisVersionUpgradeRoute, 'redis.toolbox.rollback': redisDBStructureRoute, 'redis.toolbox.rollbackRecord': redisStructureInstanceRoute, diff --git a/dbm-ui/frontend/src/views/db-manage/redis/toolbox-menu.ts b/dbm-ui/frontend/src/views/db-manage/redis/toolbox-menu.ts index 70909211bb..a41bf0a2a7 100644 --- a/dbm-ui/frontend/src/views/db-manage/redis/toolbox-menu.ts +++ b/dbm-ui/frontend/src/views/db-manage/redis/toolbox-menu.ts @@ -56,6 +56,12 @@ export default [ parentId: 'common-manage', dbConsoleValue: 'redis.toolbox.dbReplace', }, + { + name: t('迁移'), + id: 'RedisMigrate', + parentId: 'common-manage', + dbConsoleValue: 'redis.toolbox.migrate', + }, { name: t('版本升级'), id: 'RedisVersionUpgrade', diff --git a/dbm-ui/frontend/src/views/ticket-center/common/TicketClone.vue b/dbm-ui/frontend/src/views/ticket-center/common/TicketClone.vue index 7e00f05833..f1b367f2d4 100644 --- a/dbm-ui/frontend/src/views/ticket-center/common/TicketClone.vue +++ b/dbm-ui/frontend/src/views/ticket-center/common/TicketClone.vue @@ -76,6 +76,8 @@ [TicketTypes.REDIS_PLUGIN_DELETE_CLB]: 'DatabaseRedisList', // Redis 删除CLB [TicketTypes.REDIS_PLUGIN_CREATE_POLARIS]: 'DatabaseRedisList', // Redis 删除构造任务 [TicketTypes.REDIS_PLUGIN_DELETE_POLARIS]: 'DatabaseRedisList', // Redis 删除构造任务 + [TicketTypes.REDIS_CLUSTER_INS_MIGRATE]: 'RedisMigrate', // Redis 集群迁移 + [TicketTypes.REDIS_SINGLE_INS_MIGRATE]: 'RedisMigrate', // Redis 主从迁移 [TicketTypes.MYSQL_SINGLE_APPLY]: 'SelfServiceApplySingle', // Mysql 单节点部署 [TicketTypes.MYSQL_HA_APPLY]: 'SelfServiceApplyHa', // Mysql 主从部署 [TicketTypes.MYSQL_EXCEL_AUTHORIZE_RULES]: '', // Mysql excel 授权 diff --git a/dbm-ui/frontend/src/views/ticket-center/common/ticket-detail/components/task-info/com-factory/redis/MigrateCluster.vue b/dbm-ui/frontend/src/views/ticket-center/common/ticket-detail/components/task-info/com-factory/redis/MigrateCluster.vue new file mode 100644 index 0000000000..3a63f26e65 --- /dev/null +++ b/dbm-ui/frontend/src/views/ticket-center/common/ticket-detail/components/task-info/com-factory/redis/MigrateCluster.vue @@ -0,0 +1,93 @@ + + + + diff --git a/dbm-ui/frontend/src/views/ticket-center/common/ticket-detail/components/task-info/com-factory/redis/MigrateSingle.vue b/dbm-ui/frontend/src/views/ticket-center/common/ticket-detail/components/task-info/com-factory/redis/MigrateSingle.vue new file mode 100644 index 0000000000..fd8c405765 --- /dev/null +++ b/dbm-ui/frontend/src/views/ticket-center/common/ticket-detail/components/task-info/com-factory/redis/MigrateSingle.vue @@ -0,0 +1,127 @@ + + + + + + From 00821fbd612df2320d6cd931d78b513e60a68f77 Mon Sep 17 00:00:00 2001 From: hLinx <327159425@qq.com> Date: Mon, 23 Dec 2024 15:30:37 +0800 Subject: [PATCH 090/107] =?UTF-8?q?feat(frontend):=20mongo=E8=BF=99?= =?UTF-8?q?=E9=87=8C=E7=9A=84=E5=8F=98=E6=9B=B4=E8=84=9A=E6=9C=AC=E6=89=A7?= =?UTF-8?q?=E8=A1=8C=E6=8F=90=E4=BE=9B=E4=B8=8B=E8=BD=BD=E5=9C=B0=E5=9D=80?= =?UTF-8?q?=20#8771?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dbm-ui/frontend/src/locales/zh-cn.json | 27 +- dbm-ui/frontend/src/utils/downloadUrl.ts | 1 + .../db-manage/common/FlowRedisKeyExtract.vue | 1 - .../flow-type-inner-flow/StatusFailed.vue | 11 + .../flow-type-inner-flow/StatusSucceeded.vue | 14 +- .../flow-type-inner-flow/StatusTerminated.vue | 12 + .../MongodbExecScriptDownloadFile.vue | 74 ++++++ .../components/RedisKeysExtractFile.vue | 251 ++++++++++++++++++ .../common/mysql-dump-data/Index.vue | 179 +++---------- .../components/render-sqlfile/Index.vue | 4 +- .../com-factory/mongodb/ExecScriptApply.vue | 180 ++++--------- .../mongodb/components/RenderFileContent.vue | 174 ------------ .../mongodb/components/SqlFileList.vue | 139 ---------- 13 files changed, 467 insertions(+), 600 deletions(-) create mode 100644 dbm-ui/frontend/src/views/ticket-center/common/ticket-detail/components/flow-info/components/flow-type-inner-flow/components/MongodbExecScriptDownloadFile.vue create mode 100644 dbm-ui/frontend/src/views/ticket-center/common/ticket-detail/components/flow-info/components/flow-type-inner-flow/components/RedisKeysExtractFile.vue delete mode 100644 dbm-ui/frontend/src/views/ticket-center/common/ticket-detail/components/task-info/com-factory/mongodb/components/RenderFileContent.vue delete mode 100644 dbm-ui/frontend/src/views/ticket-center/common/ticket-detail/components/task-info/com-factory/mongodb/components/SqlFileList.vue diff --git a/dbm-ui/frontend/src/locales/zh-cn.json b/dbm-ui/frontend/src/locales/zh-cn.json index c0e13cd2fb..a9a70733a9 100644 --- a/dbm-ui/frontend/src/locales/zh-cn.json +++ b/dbm-ui/frontend/src/locales/zh-cn.json @@ -3864,26 +3864,13 @@ "后端存储规格：": "后端存储规格：", "SQLServer起始端口：": "SQLServer起始端口：", "高可用部署": "高可用部署", - "迁移提交成功": "迁移提交成功", - "集群架构：将集群的部分实例迁移到新机器，迁移保持规格、版本不变；主从架构：主从实例成对迁移到新机器上，可选择部分实例迁移，也可整机所有实例一起迁移。": "集群架构：将集群的部分实例迁移到新机器，迁移保持规格、版本不变；主从架构：主从实例成对迁移到新机器上，可选择部分实例迁移，也可整机所有实例一起迁移。", - "迁移": "迁移", - "实例迁移": "实例迁移", - "如 TendisCache 等，迁移过程保持规格、版本不变": "如 TendisCache 等，迁移过程保持规格、版本不变", - "支持部分或整机所有实例成对迁移至新主机，版本规格可变": "支持部分或整机所有实例成对迁移至新主机，版本规格可变", - "主从架构": "主从架构", - "只迁移目标实例": "只迁移目标实例", - "主机关联的所有实例一并迁移": "主机关联的所有实例一并迁移", - "整机迁移": "整机迁移", - "目标 Master 实例": "目标 Master 实例", - "实例选择": "实例选择", - "目标 Master 主机": "目标 Master 主机", - "关联的实例": "关联的实例", - "关联的主从实例": "关联的主从实例", - "目标实例输入格式有误": "目标实例输入格式有误", - "目标实例重复": "目标实例重复", - "Master 实例": "Master 实例", - "请先选择主机": "请先选择主机", - "集群或实例状态异常，不可选择": "集群或实例状态异常，不可选择", + "目标集群：": "目标集群：", + "目标 DB：": "目标 DB：", + "目标表名：": "目标表名：", + "where 条件：": "where 条件：", + "导出数据：": "导出数据：", + "脚本来源：": "脚本来源：", + "脚本执行内容：": "脚本执行内容：", "这行勿动！新增翻译请在上一行添加！": "" } diff --git a/dbm-ui/frontend/src/utils/downloadUrl.ts b/dbm-ui/frontend/src/utils/downloadUrl.ts index 5b1af01c2c..c06c380b8f 100644 --- a/dbm-ui/frontend/src/utils/downloadUrl.ts +++ b/dbm-ui/frontend/src/utils/downloadUrl.ts @@ -17,6 +17,7 @@ export const downloadUrl = (url: string) => { const eleLink = document.createElement('a'); eleLink.style.display = 'none'; eleLink.href = url; + eleLink.target = '_blank'; // 触发点击 document.body.appendChild(eleLink); diff --git a/dbm-ui/frontend/src/views/db-manage/common/FlowRedisKeyExtract.vue b/dbm-ui/frontend/src/views/db-manage/common/FlowRedisKeyExtract.vue index e2165eaa76..9fbf27b933 100644 --- a/dbm-ui/frontend/src/views/db-manage/common/FlowRedisKeyExtract.vue +++ b/dbm-ui/frontend/src/views/db-manage/common/FlowRedisKeyExtract.vue @@ -60,7 +60,6 @@ :data="state.data" :height="460" :is-anomalies="isAnomalies" - :row-height="56" @refresh="fetchKeyFiles" @selection-change="handleTableSelected"> {{ t('执行失败') }} + + diff --git a/dbm-ui/frontend/src/views/ticket-center/common/ticket-detail/components/flow-info/components/flow-type-inner-flow/components/RedisKeysExtractFile.vue b/dbm-ui/frontend/src/views/ticket-center/common/ticket-detail/components/flow-info/components/flow-type-inner-flow/components/RedisKeysExtractFile.vue new file mode 100644 index 0000000000..c663fea27d --- /dev/null +++ b/dbm-ui/frontend/src/views/ticket-center/common/ticket-detail/components/flow-info/components/flow-type-inner-flow/components/RedisKeysExtractFile.vue @@ -0,0 +1,251 @@ + + + + + + + + + diff --git a/dbm-ui/frontend/src/views/ticket-center/common/ticket-detail/components/task-info/com-factory/common/mysql-dump-data/Index.vue b/dbm-ui/frontend/src/views/ticket-center/common/ticket-detail/components/task-info/com-factory/common/mysql-dump-data/Index.vue index 1da8eb9f86..fadc3a0e56 100644 --- a/dbm-ui/frontend/src/views/ticket-center/common/ticket-detail/components/task-info/com-factory/common/mysql-dump-data/Index.vue +++ b/dbm-ui/frontend/src/views/ticket-center/common/ticket-detail/components/task-info/com-factory/common/mysql-dump-data/Index.vue @@ -12,9 +12,47 @@ --> diff --git a/dbm-ui/frontend/src/views/ticket-center/common/ticket-detail/components/task-info/com-factory/common/mysql-import-sqlfile/components/render-sqlfile/Index.vue b/dbm-ui/frontend/src/views/ticket-center/common/ticket-detail/components/task-info/com-factory/common/mysql-import-sqlfile/components/render-sqlfile/Index.vue index 10c036b738..a2cd342740 100644 --- a/dbm-ui/frontend/src/views/ticket-center/common/ticket-detail/components/task-info/com-factory/common/mysql-import-sqlfile/components/render-sqlfile/Index.vue +++ b/dbm-ui/frontend/src/views/ticket-center/common/ticket-detail/components/task-info/com-factory/common/mysql-import-sqlfile/components/render-sqlfile/Index.vue @@ -71,8 +71,8 @@ import { type Sqlserver } from '@services/model/ticket/ticket'; import { batchFetchFile } from '@services/source/storage'; - import RenderFileContent from './components/RenderFileContent.vue'; - import RenderFileList from './components/RenderFileList.vue'; + import RenderFileContent from '@views/ticket-center/common/ticket-detail/components/common/SqlFileContent.vue'; + import RenderFileList from '@views/ticket-center/common/ticket-detail/components/common/SqlFileList.vue'; interface Props { executeObject: Sqlserver.ImportSqlFile['execute_objects'][number]; diff --git a/dbm-ui/frontend/src/views/ticket-center/common/ticket-detail/components/task-info/com-factory/mongodb/ExecScriptApply.vue b/dbm-ui/frontend/src/views/ticket-center/common/ticket-detail/components/task-info/com-factory/mongodb/ExecScriptApply.vue index a62dc665f1..0218ddea48 100644 --- a/dbm-ui/frontend/src/views/ticket-center/common/ticket-detail/components/task-info/com-factory/mongodb/ExecScriptApply.vue +++ b/dbm-ui/frontend/src/views/ticket-center/common/ticket-detail/components/task-info/com-factory/mongodb/ExecScriptApply.vue @@ -12,37 +12,41 @@ --> @@ -57,16 +57,17 @@ diff --git a/dbm-ui/frontend/src/views/ticket-notice-setting/routes.ts b/dbm-ui/frontend/src/views/ticket-notice-setting/routes.ts new file mode 100644 index 0000000000..02a92c9cb9 --- /dev/null +++ b/dbm-ui/frontend/src/views/ticket-notice-setting/routes.ts @@ -0,0 +1,31 @@ +/* + * TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. + * + * Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. + * + * Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at https://opensource.org/licenses/MIT + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for + * the specific language governing permissions and limitations under the License. + */ +import { checkDbConsole } from '@utils'; + +import { t } from '@locales/index'; + +const routes = [ + { + name: 'TicketNoticeSetting', + path: 'ticket-notice-setting', + meta: { + navName: t('单据通知'), + fullscreen: true, + }, + component: () => import('@views/ticket-notice-setting/Index.vue'), + }, +]; + +export default function getRoutes() { + return checkDbConsole('bizConfigManage.ticketNoticeSetting') ? routes : []; +} From 6da45e40f49c8ebac47d614b8b92da3aca232bc6 Mon Sep 17 00:00:00 2001 From: hLinx <327159425@qq.com> Date: Mon, 16 Dec 2024 11:43:21 +0800 Subject: [PATCH 096/107] =?UTF-8?q?perf(frontend):=20=E5=85=A8=E7=AB=99?= =?UTF-8?q?=E6=90=9C=E7=B4=A2=E4=BC=98=E5=8C=96=20#8629?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../search-result/FilterOptions.vue | 29 +- .../components/search-result/Index.vue | 7 +- .../search-result/render-result/Index.vue | 6 +- .../{ClusterDomain.vue => entry.vue} | 0 dbm-ui/frontend/src/locales/zh-cn.json | 4 + ...luster-domain.ts => quick-search-entry.ts} | 52 ++- .../quiker-search/quick-search-instance.ts | 10 + .../src/services/source/quickSearch.ts | 9 +- .../frontend/src/views/quick-search/Index.vue | 87 ++--- .../src/views/quick-search/common/types.ts | 4 - .../src/views/quick-search/common/utils.ts | 27 +- .../quick-search/components/ClusterDomain.vue | 317 ------------------ .../quick-search/components/ClusterName.vue | 305 ----------------- .../views/quick-search/components/Entry.vue | 260 ++++++++++++++ .../quick-search/components/Instance.vue | 301 +++++++---------- 15 files changed, 462 insertions(+), 956 deletions(-) rename dbm-ui/frontend/src/components/system-search/components/search-result/render-result/{ClusterDomain.vue => entry.vue} (100%) rename dbm-ui/frontend/src/services/model/quiker-search/{quick-search-cluster-domain.ts => quick-search-entry.ts} (55%) delete mode 100644 dbm-ui/frontend/src/views/quick-search/common/types.ts delete mode 100644 dbm-ui/frontend/src/views/quick-search/components/ClusterDomain.vue delete mode 100644 dbm-ui/frontend/src/views/quick-search/components/ClusterName.vue create mode 100644 dbm-ui/frontend/src/views/quick-search/components/Entry.vue diff --git a/dbm-ui/frontend/src/components/system-search/components/search-result/FilterOptions.vue b/dbm-ui/frontend/src/components/system-search/components/search-result/FilterOptions.vue index 63d68d9d65..c0f0036835 100644 --- a/dbm-ui/frontend/src/components/system-search/components/search-result/FilterOptions.vue +++ b/dbm-ui/frontend/src/components/system-search/components/search-result/FilterOptions.vue @@ -85,19 +85,6 @@ - - - diff --git a/dbm-ui/frontend/src/views/quick-search/components/ClusterName.vue b/dbm-ui/frontend/src/views/quick-search/components/ClusterName.vue deleted file mode 100644 index 2c9371e168..0000000000 --- a/dbm-ui/frontend/src/views/quick-search/components/ClusterName.vue +++ /dev/null @@ -1,305 +0,0 @@ - - - - - diff --git a/dbm-ui/frontend/src/views/quick-search/components/Entry.vue b/dbm-ui/frontend/src/views/quick-search/components/Entry.vue new file mode 100644 index 0000000000..4bfac2402f --- /dev/null +++ b/dbm-ui/frontend/src/views/quick-search/components/Entry.vue @@ -0,0 +1,260 @@ + + + + + diff --git a/dbm-ui/frontend/src/views/quick-search/components/Instance.vue b/dbm-ui/frontend/src/views/quick-search/components/Instance.vue index a03bfe9083..d720543060 100644 --- a/dbm-ui/frontend/src/views/quick-search/components/Instance.vue +++ b/dbm-ui/frontend/src/views/quick-search/components/Instance.vue @@ -3,10 +3,10 @@ diff --git a/dbm-ui/frontend/src/views/db-manage/common/apply-items/ClusterName.vue b/dbm-ui/frontend/src/views/db-manage/common/apply-items/ClusterName.vue index 16180cdfa4..532c631afb 100644 --- a/dbm-ui/frontend/src/views/db-manage/common/apply-items/ClusterName.vue +++ b/dbm-ui/frontend/src/views/db-manage/common/apply-items/ClusterName.vue @@ -13,7 +13,7 @@ From 2a562d7929c5fd32ead171a362e4300a3b112f01 Mon Sep 17 00:00:00 2001 From: hLinx <327159425@qq.com> Date: Fri, 20 Dec 2024 10:27:40 +0800 Subject: [PATCH 098/107] =?UTF-8?q?feat(frontend):=20=E5=85=A8=E7=AB=99?= =?UTF-8?q?=E9=9B=86=E7=BE=A4=E5=92=8C=E5=AE=9E=E4=BE=8B=E8=A7=86=E5=9B=BE?= =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=95=B0=E9=87=8F=E6=98=BE=E7=A4=BA=20#8737?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../components/module-group/Doris.vue | 7 ++ .../components/module-group/Es.vue | 7 ++ .../components/module-group/Hdfs.vue | 7 ++ .../components/module-group/Influxdb.vue | 7 ++ .../components/module-group/Kafka.vue | 7 ++ .../components/module-group/MongoDB.vue | 47 ++++++++-- .../components/module-group/Mysql.vue | 22 ++++- .../components/module-group/Pulsar.vue | 7 ++ .../components/module-group/Redis.vue | 28 ++++-- .../components/module-group/Riak.vue | 7 ++ .../components/module-group/SqlServer.vue | 22 ++++- .../components/module-group/TendbCluster.vue | 19 +++- .../module-group/components/CountTag.vue | 64 ++++++++++++++ .../hooks/useClusterInstanceCount.ts | 18 ++++ dbm-ui/frontend/src/locales/zh-cn.json | 4 + dbm-ui/frontend/src/services/source/dbbase.ts | 15 ++++ .../detail/components/BaseInfo.vue | 0 .../detail/components/Config.vue | 0 .../detail/index.vue | 0 .../index.vue | 0 .../list/index.vue | 86 +++++-------------- .../src/views/db-manage/mongodb/routes.ts | 26 +++++- 22 files changed, 309 insertions(+), 91 deletions(-) create mode 100644 dbm-ui/frontend/src/layout/components/database-manage/components/module-group/components/CountTag.vue create mode 100644 dbm-ui/frontend/src/layout/components/database-manage/components/module-group/hooks/useClusterInstanceCount.ts rename dbm-ui/frontend/src/views/db-manage/mongodb/{mongodb-instance => instance-list}/detail/components/BaseInfo.vue (100%) rename dbm-ui/frontend/src/views/db-manage/mongodb/{mongodb-instance => instance-list}/detail/components/Config.vue (100%) rename dbm-ui/frontend/src/views/db-manage/mongodb/{mongodb-instance => instance-list}/detail/index.vue (100%) rename dbm-ui/frontend/src/views/db-manage/mongodb/{mongodb-instance => instance-list}/index.vue (100%) rename dbm-ui/frontend/src/views/db-manage/mongodb/{mongodb-instance => instance-list}/list/index.vue (84%) diff --git a/dbm-ui/frontend/src/layout/components/database-manage/components/module-group/Doris.vue b/dbm-ui/frontend/src/layout/components/database-manage/components/module-group/Doris.vue index 876a670522..2d0139f5cf 100644 --- a/dbm-ui/frontend/src/layout/components/database-manage/components/module-group/Doris.vue +++ b/dbm-ui/frontend/src/layout/components/database-manage/components/module-group/Doris.vue @@ -12,6 +12,9 @@ class="text-overflow"> {{ t('集群管理') }} + @@ -20,5 +23,9 @@ diff --git a/dbm-ui/frontend/src/layout/components/database-manage/components/module-group/Es.vue b/dbm-ui/frontend/src/layout/components/database-manage/components/module-group/Es.vue index b16c79dbe7..8480390872 100644 --- a/dbm-ui/frontend/src/layout/components/database-manage/components/module-group/Es.vue +++ b/dbm-ui/frontend/src/layout/components/database-manage/components/module-group/Es.vue @@ -12,6 +12,9 @@ class="text-overflow"> {{ t('集群管理') }} + @@ -19,5 +22,9 @@ diff --git a/dbm-ui/frontend/src/layout/components/database-manage/components/module-group/Hdfs.vue b/dbm-ui/frontend/src/layout/components/database-manage/components/module-group/Hdfs.vue index f51fac53b9..4f3ae725f9 100644 --- a/dbm-ui/frontend/src/layout/components/database-manage/components/module-group/Hdfs.vue +++ b/dbm-ui/frontend/src/layout/components/database-manage/components/module-group/Hdfs.vue @@ -12,6 +12,9 @@ class="text-overflow"> {{ t('集群管理') }} + @@ -19,5 +22,9 @@ diff --git a/dbm-ui/frontend/src/layout/components/database-manage/components/module-group/Influxdb.vue b/dbm-ui/frontend/src/layout/components/database-manage/components/module-group/Influxdb.vue index d067b71ca9..8259116ed8 100644 --- a/dbm-ui/frontend/src/layout/components/database-manage/components/module-group/Influxdb.vue +++ b/dbm-ui/frontend/src/layout/components/database-manage/components/module-group/Influxdb.vue @@ -12,6 +12,9 @@ class="text-overflow"> {{ t('实例管理') }} + @@ -19,5 +22,9 @@ diff --git a/dbm-ui/frontend/src/layout/components/database-manage/components/module-group/Kafka.vue b/dbm-ui/frontend/src/layout/components/database-manage/components/module-group/Kafka.vue index 2357936e2c..441e7ae4b3 100644 --- a/dbm-ui/frontend/src/layout/components/database-manage/components/module-group/Kafka.vue +++ b/dbm-ui/frontend/src/layout/components/database-manage/components/module-group/Kafka.vue @@ -12,6 +12,9 @@ class="text-overflow"> {{ t('集群管理') }} + @@ -19,5 +22,9 @@ diff --git a/dbm-ui/frontend/src/layout/components/database-manage/components/module-group/MongoDB.vue b/dbm-ui/frontend/src/layout/components/database-manage/components/module-group/MongoDB.vue index 53fda19ca4..5464cf203c 100644 --- a/dbm-ui/frontend/src/layout/components/database-manage/components/module-group/MongoDB.vue +++ b/dbm-ui/frontend/src/layout/components/database-manage/components/module-group/MongoDB.vue @@ -1,30 +1,66 @@ + diff --git a/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/ClusterNameColumn.vue b/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/ClusterNameColumn.vue new file mode 100644 index 0000000000..f552d4a051 --- /dev/null +++ b/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/ClusterNameColumn.vue @@ -0,0 +1,99 @@ + + diff --git a/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/ClusterStats.vue b/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/ClusterStats.vue new file mode 100644 index 0000000000..ded06dd7e2 --- /dev/null +++ b/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/ClusterStats.vue @@ -0,0 +1,28 @@ + + diff --git a/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/CommonColumn.vue b/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/CommonColumn.vue new file mode 100644 index 0000000000..4a79c56e3b --- /dev/null +++ b/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/CommonColumn.vue @@ -0,0 +1,119 @@ + + diff --git a/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/IdColumn.vue b/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/IdColumn.vue new file mode 100644 index 0000000000..5a30d045a0 --- /dev/null +++ b/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/IdColumn.vue @@ -0,0 +1,15 @@ + + diff --git a/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/MasterDomainColumn.vue b/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/MasterDomainColumn.vue new file mode 100644 index 0000000000..0dd52cbea3 --- /dev/null +++ b/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/MasterDomainColumn.vue @@ -0,0 +1,153 @@ + + diff --git a/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/RoleColumn.vue b/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/RoleColumn.vue new file mode 100644 index 0000000000..5af8fc1e38 --- /dev/null +++ b/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/RoleColumn.vue @@ -0,0 +1,111 @@ + + diff --git a/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/SlaveDomainColumn.vue b/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/SlaveDomainColumn.vue new file mode 100644 index 0000000000..49dbe27d2c --- /dev/null +++ b/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/SlaveDomainColumn.vue @@ -0,0 +1,113 @@ + + diff --git a/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/StatusColumn.vue b/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/StatusColumn.vue new file mode 100644 index 0000000000..8186378da6 --- /dev/null +++ b/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/StatusColumn.vue @@ -0,0 +1,28 @@ + + diff --git a/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/hooks/useColumnCopy.ts b/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/hooks/useColumnCopy.ts new file mode 100644 index 0000000000..cd87dcdbad --- /dev/null +++ b/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/hooks/useColumnCopy.ts @@ -0,0 +1,40 @@ +import { useI18n } from 'vue-i18n'; + +import DbTable from '@components/db-table/index.vue'; + +import { execCopy, messageWarn } from '@utils'; + +import type { ClusterModel, ISupportClusterType } from '../types'; + +export default (props: { + selectedList: ClusterModel[]; + getTableInstance: () => InstanceType | undefined; +}) => { + const { t } = useI18n(); + + const handleCopySelected = (field: keyof ClusterModel) => { + const copyList = props.selectedList.map((item) => item[field as keyof ClusterModel]); + + execCopy(copyList.join('\n')); + }; + + const handleCopyAll = (field: keyof ClusterModel) => { + props + .getTableInstance()! + .getAllData>() + .then((data) => { + if (data.length < 1) { + messageWarn(t('暂无数据可复制')); + return; + } + const copyList = data.map((item) => item[field as keyof ClusterModel]); + + execCopy(copyList.join('\n')); + }); + }; + + return { + handleCopySelected, + handleCopyAll, + }; +}; diff --git a/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/types.ts b/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/types.ts new file mode 100644 index 0000000000..44dfd755d7 --- /dev/null +++ b/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/types.ts @@ -0,0 +1,11 @@ +import { ClusterTypes } from '@common/const'; + +import TendbClusterModel from '@/services/model/tendbcluster/tendbcluster'; + +export type ISupportClusterType = ClusterTypes.TENDBCLUSTER; + +interface ClusterTypeRelateClusterModel { + [ClusterTypes.TENDBCLUSTER]: TendbClusterModel; +} + +export type ClusterModel = ClusterTypeRelateClusterModel[T]; diff --git a/dbm-ui/frontend/src/views/db-manage/common/render-instances/RenderInstances.vue b/dbm-ui/frontend/src/views/db-manage/common/render-instances/RenderInstances.vue index 2cc9513b88..5cc0b0a21e 100644 --- a/dbm-ui/frontend/src/views/db-manage/common/render-instances/RenderInstances.vue +++ b/dbm-ui/frontend/src/views/db-manage/common/render-instances/RenderInstances.vue @@ -135,6 +135,7 @@ ip: string; port: number; status: string; + shard_id?: string; }; - - diff --git a/dbm-ui/frontend/src/views/db-manage/tendb-cluster/list/components/list/components/ScaleUp.vue b/dbm-ui/frontend/src/views/db-manage/tendb-cluster/list/components/list/components/ScaleUp.vue deleted file mode 100644 index 99e20a1d3a..0000000000 --- a/dbm-ui/frontend/src/views/db-manage/tendb-cluster/list/components/list/components/ScaleUp.vue +++ /dev/null @@ -1,251 +0,0 @@ - - - - - diff --git a/dbm-ui/frontend/src/views/db-manage/tendb-cluster/list/components/list/components/Shrink.vue b/dbm-ui/frontend/src/views/db-manage/tendb-cluster/list/components/list/components/Shrink.vue deleted file mode 100644 index 0f490633b8..0000000000 --- a/dbm-ui/frontend/src/views/db-manage/tendb-cluster/list/components/list/components/Shrink.vue +++ /dev/null @@ -1,257 +0,0 @@ - - - - - diff --git a/dbm-ui/frontend/src/views/db-manage/tendb-cluster/list/components/list/components/SpecInfo.vue b/dbm-ui/frontend/src/views/db-manage/tendb-cluster/list/components/list/components/SpecInfo.vue deleted file mode 100644 index 3c06ec0528..0000000000 --- a/dbm-ui/frontend/src/views/db-manage/tendb-cluster/list/components/list/components/SpecInfo.vue +++ /dev/null @@ -1,99 +0,0 @@ - - - - - From 70f055610bc4cfa7d187c0ec79e0971d9b97976c Mon Sep 17 00:00:00 2001 From: hLinx <327159425@qq.com> Date: Mon, 23 Dec 2024 16:46:31 +0800 Subject: [PATCH 103/107] =?UTF-8?q?fix(frontend):=20=E5=BC=82=E5=B8=B8?= =?UTF-8?q?=E7=9A=84=E9=9B=86=E7=BE=A4=E4=B9=9F=E5=8F=AF=E4=BB=A5=E7=A6=81?= =?UTF-8?q?=E7=94=A8=E5=92=8C=E4=B8=8B=E6=9E=B6=20#8772?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../components/mongo-replica-set/Index.vue | 4 +- .../mongo-sharded-cluster/Index.vue | 4 +- .../components/redis-ha/Index.vue | 4 +- .../components/sqlserver-ha/Index.vue | 4 +- .../components/sqlserver-single/Index.vue | 4 +- .../components/tendb-cluster/Index.vue | 4 +- .../components/tendb-ha/Index.vue | 4 +- .../components/tendb-single/Index.vue | 4 +- .../doris/list/components/list/Index.vue | 32 +++++----- .../list/components/list/Index.vue | 2 +- .../hdfs/list/components/list/Index.vue | 30 +++++----- .../kafka/list/components/list/Index.vue | 32 +++++----- .../components/detail/Index.vue | 2 +- .../components/list/Index.vue | 2 +- .../components/detail/Index.vue | 2 +- .../components/list/Index.vue | 2 +- .../mysql/ha-cluster-list/components/List.vue | 2 +- .../single-cluster-list/components/List.vue | 2 +- .../pulsar/list/components/list/Index.vue | 59 +++++++------------ .../redis/list-ha/components/list/Index.vue | 2 +- .../redis/list/components/list/Index.vue | 2 +- .../src/views/db-manage/riak/list/Index.vue | 16 ++--- .../riak/list/components/list/Index.vue | 2 +- .../ha-cluster-list/components/List.vue | 4 +- .../single-cluster/components/List.vue | 2 +- .../tendb-cluster/partition-manage/Index.vue | 1 - 26 files changed, 118 insertions(+), 110 deletions(-) diff --git a/dbm-ui/frontend/src/views/db-manage/common/cluster-batch-opration/components/mongo-replica-set/Index.vue b/dbm-ui/frontend/src/views/db-manage/common/cluster-batch-opration/components/mongo-replica-set/Index.vue index 6fad9a3fd7..9438c3c2ef 100644 --- a/dbm-ui/frontend/src/views/db-manage/common/cluster-batch-opration/components/mongo-replica-set/Index.vue +++ b/dbm-ui/frontend/src/views/db-manage/common/cluster-batch-opration/components/mongo-replica-set/Index.vue @@ -106,7 +106,9 @@ const clusterAuthorizeShow = ref(false); const batchAuthorizeDisabled = computed(() => props.selected.some((data) => data.isOffline)); - const batchDisabledDisabled = computed(() => props.selected.some((data) => data.isOffline || data.operationDisabled)); + const batchDisabledDisabled = computed(() => + props.selected.some((data) => data.isOffline || Boolean(data.operationTicketId)), + ); const batchEnableDisabled = computed(() => props.selected.some((data) => data.isOnline || data.isStarting)); const batchDeleteDisabled = computed(() => props.selected.some((data) => data.isOnline || Boolean(data.operationTicketId)), diff --git a/dbm-ui/frontend/src/views/db-manage/common/cluster-batch-opration/components/mongo-sharded-cluster/Index.vue b/dbm-ui/frontend/src/views/db-manage/common/cluster-batch-opration/components/mongo-sharded-cluster/Index.vue index d5821dfc1d..b1b7ee9f92 100644 --- a/dbm-ui/frontend/src/views/db-manage/common/cluster-batch-opration/components/mongo-sharded-cluster/Index.vue +++ b/dbm-ui/frontend/src/views/db-manage/common/cluster-batch-opration/components/mongo-sharded-cluster/Index.vue @@ -106,7 +106,9 @@ const clusterAuthorizeShow = ref(false); const batchAuthorizeDisabled = computed(() => props.selected.some((data) => data.isOffline)); - const batchDisabledDisabled = computed(() => props.selected.some((data) => data.isOffline || data.operationDisabled)); + const batchDisabledDisabled = computed(() => + props.selected.some((data) => data.isOffline || Boolean(data.operationTicketId)), + ); const batchEnableDisabled = computed(() => props.selected.some((data) => data.isOnline || data.isStarting)); const batchDeleteDisabled = computed(() => props.selected.some((data) => data.isOnline || Boolean(data.operationTicketId)), diff --git a/dbm-ui/frontend/src/views/db-manage/common/cluster-batch-opration/components/redis-ha/Index.vue b/dbm-ui/frontend/src/views/db-manage/common/cluster-batch-opration/components/redis-ha/Index.vue index eacc24cbf4..db3047e652 100644 --- a/dbm-ui/frontend/src/views/db-manage/common/cluster-batch-opration/components/redis-ha/Index.vue +++ b/dbm-ui/frontend/src/views/db-manage/common/cluster-batch-opration/components/redis-ha/Index.vue @@ -190,7 +190,9 @@ }), ); - const batchDisabledDisabled = computed(() => props.selected.some((data) => data.isOffline || data.operationDisabled)); + const batchDisabledDisabled = computed(() => + props.selected.some((data) => data.isOffline || Boolean(data.operationTicketId)), + ); const batchEnableDisabled = computed(() => props.selected.some((data) => data.isOnline || data.isStarting)); const batchDeleteDisabled = computed(() => props.selected.some((data) => data.isOnline || Boolean(data.operationTicketId)), diff --git a/dbm-ui/frontend/src/views/db-manage/common/cluster-batch-opration/components/sqlserver-ha/Index.vue b/dbm-ui/frontend/src/views/db-manage/common/cluster-batch-opration/components/sqlserver-ha/Index.vue index 079aab313f..bb0299ab09 100644 --- a/dbm-ui/frontend/src/views/db-manage/common/cluster-batch-opration/components/sqlserver-ha/Index.vue +++ b/dbm-ui/frontend/src/views/db-manage/common/cluster-batch-opration/components/sqlserver-ha/Index.vue @@ -106,7 +106,9 @@ const clusterAuthorizeShow = ref(false); const batchAuthorizeDisabled = computed(() => props.selected.some((data) => data.isOffline)); - const batchDisabledDisabled = computed(() => props.selected.some((data) => data.isOffline || data.operationDisabled)); + const batchDisabledDisabled = computed(() => + props.selected.some((data) => data.isOffline || Boolean(data.operationTicketId)), + ); const batchEnableDisabled = computed(() => props.selected.some((data) => data.isOnline || data.isStarting)); const batchDeleteDisabled = computed(() => props.selected.some((data) => data.isOnline || Boolean(data.operationTicketId)), diff --git a/dbm-ui/frontend/src/views/db-manage/common/cluster-batch-opration/components/sqlserver-single/Index.vue b/dbm-ui/frontend/src/views/db-manage/common/cluster-batch-opration/components/sqlserver-single/Index.vue index 6b5b4a9f22..2adaabbd01 100644 --- a/dbm-ui/frontend/src/views/db-manage/common/cluster-batch-opration/components/sqlserver-single/Index.vue +++ b/dbm-ui/frontend/src/views/db-manage/common/cluster-batch-opration/components/sqlserver-single/Index.vue @@ -106,7 +106,9 @@ const clusterAuthorizeShow = ref(false); const batchAuthorizeDisabled = computed(() => props.selected.some((data) => data.isOffline)); - const batchDisabledDisabled = computed(() => props.selected.some((data) => data.isOffline || data.operationDisabled)); + const batchDisabledDisabled = computed(() => + props.selected.some((data) => data.isOffline || Boolean(data.operationTicketId)), + ); const batchEnableDisabled = computed(() => props.selected.some((data) => data.isOnline || data.isStarting)); const batchDeleteDisabled = computed(() => props.selected.some((data) => data.isOnline || Boolean(data.operationTicketId)), diff --git a/dbm-ui/frontend/src/views/db-manage/common/cluster-batch-opration/components/tendb-cluster/Index.vue b/dbm-ui/frontend/src/views/db-manage/common/cluster-batch-opration/components/tendb-cluster/Index.vue index e8aec1725b..052bbe7434 100644 --- a/dbm-ui/frontend/src/views/db-manage/common/cluster-batch-opration/components/tendb-cluster/Index.vue +++ b/dbm-ui/frontend/src/views/db-manage/common/cluster-batch-opration/components/tendb-cluster/Index.vue @@ -106,7 +106,9 @@ const clusterAuthorizeShow = ref(false); const batchAuthorizeDisabled = computed(() => props.selected.some((data) => data.isOffline)); - const batchDisabledDisabled = computed(() => props.selected.some((data) => data.isOffline || data.operationDisabled)); + const batchDisabledDisabled = computed(() => + props.selected.some((data) => data.isOffline || Boolean(data.operationTicketId)), + ); const batchEnableDisabled = computed(() => props.selected.some((data) => data.isOnline || data.isStarting)); const batchDeleteDisabled = computed(() => props.selected.some((data) => data.isOnline || Boolean(data.operationTicketId)), diff --git a/dbm-ui/frontend/src/views/db-manage/common/cluster-batch-opration/components/tendb-ha/Index.vue b/dbm-ui/frontend/src/views/db-manage/common/cluster-batch-opration/components/tendb-ha/Index.vue index e58ac8065a..a87246dea5 100644 --- a/dbm-ui/frontend/src/views/db-manage/common/cluster-batch-opration/components/tendb-ha/Index.vue +++ b/dbm-ui/frontend/src/views/db-manage/common/cluster-batch-opration/components/tendb-ha/Index.vue @@ -129,7 +129,9 @@ const batchSubscriptionDisabled = computed(() => props.selected.some((data) => data.isOffline)); const batchAuthorizeDisabled = computed(() => props.selected.some((data) => data.isOffline)); - const batchDisabledDisabled = computed(() => props.selected.some((data) => data.isOffline || data.operationDisabled)); + const batchDisabledDisabled = computed(() => + props.selected.some((data) => data.isOffline || Boolean(data.operationTicketId)), + ); const batchEnableDisabled = computed(() => props.selected.some((data) => data.isOnline || data.isStarting)); const batchDeleteDisabled = computed(() => props.selected.some((data) => data.isOnline || Boolean(data.operationTicketId)), diff --git a/dbm-ui/frontend/src/views/db-manage/common/cluster-batch-opration/components/tendb-single/Index.vue b/dbm-ui/frontend/src/views/db-manage/common/cluster-batch-opration/components/tendb-single/Index.vue index 68b0083f6d..b006b36798 100644 --- a/dbm-ui/frontend/src/views/db-manage/common/cluster-batch-opration/components/tendb-single/Index.vue +++ b/dbm-ui/frontend/src/views/db-manage/common/cluster-batch-opration/components/tendb-single/Index.vue @@ -106,7 +106,9 @@ const clusterAuthorizeShow = ref(false); const batchAuthorizeDisabled = computed(() => props.selected.some((data) => data.isOffline)); - const batchDisabledDisabled = computed(() => props.selected.some((data) => data.isOffline || data.operationDisabled)); + const batchDisabledDisabled = computed(() => + props.selected.some((data) => data.isOffline || Boolean(data.operationTicketId)), + ); const batchEnableDisabled = computed(() => props.selected.some((data) => data.isOnline || data.isStarting)); const batchDeleteDisabled = computed(() => props.selected.some((data) => data.isOnline || Boolean(data.operationTicketId)), diff --git a/dbm-ui/frontend/src/views/db-manage/doris/list/components/list/Index.vue b/dbm-ui/frontend/src/views/db-manage/doris/list/components/list/Index.vue index 70187cc2c0..dcdddc12b4 100644 --- a/dbm-ui/frontend/src/views/db-manage/doris/list/components/list/Index.vue +++ b/dbm-ui/frontend/src/views/db-manage/doris/list/components/list/Index.vue @@ -737,20 +737,22 @@ {t('管理')} , - - - handleDisableCluster([data])}> - { t('禁用') } - - - + { data.isOnline && ( + + + handleDisableCluster([data])}> + { t('禁用') } + + + + )} handleDeleteCluster([data])}> diff --git a/dbm-ui/frontend/src/views/db-manage/elastic-search/list/components/list/Index.vue b/dbm-ui/frontend/src/views/db-manage/elastic-search/list/components/list/Index.vue index 2981cab9ac..65fe8b60b6 100644 --- a/dbm-ui/frontend/src/views/db-manage/elastic-search/list/components/list/Index.vue +++ b/dbm-ui/frontend/src/views/db-manage/elastic-search/list/components/list/Index.vue @@ -757,7 +757,7 @@ action-id="es_enable_disable" permission={data.permission.es_enable_disable} resource={data.id} - disabled={data.operationDisabled} + disabled={Boolean(data.operationTicketId)} onClick={() => handleDisableCluster([data])}> { t('禁用') } diff --git a/dbm-ui/frontend/src/views/db-manage/hdfs/list/components/list/Index.vue b/dbm-ui/frontend/src/views/db-manage/hdfs/list/components/list/Index.vue index bc4154baee..74454b1f78 100644 --- a/dbm-ui/frontend/src/views/db-manage/hdfs/list/components/list/Index.vue +++ b/dbm-ui/frontend/src/views/db-manage/hdfs/list/components/list/Index.vue @@ -780,20 +780,22 @@ { t('缩容') } , - - handleDisableCluster([data])}> - { t('禁用') } - - , + data.isOnline && ( + + handleDisableCluster([data])}> + { t('禁用') } + + + ), diff --git a/dbm-ui/frontend/src/views/db-manage/kafka/list/components/list/Index.vue b/dbm-ui/frontend/src/views/db-manage/kafka/list/components/list/Index.vue index 805aad3f95..b26ce72e4d 100644 --- a/dbm-ui/frontend/src/views/db-manage/kafka/list/components/list/Index.vue +++ b/dbm-ui/frontend/src/views/db-manage/kafka/list/components/list/Index.vue @@ -688,21 +688,23 @@ { t('缩容') } , - - handleDisableCluster([data])}> - { t('禁用') } - - , + data.isOnline && ( + + handleDisableCluster([data])}> + { t('禁用') } + + + ), diff --git a/dbm-ui/frontend/src/views/db-manage/mongodb/replica-set-list/components/detail/Index.vue b/dbm-ui/frontend/src/views/db-manage/mongodb/replica-set-list/components/detail/Index.vue index 201c4a9e61..9d27ae1a5f 100644 --- a/dbm-ui/frontend/src/views/db-manage/mongodb/replica-set-list/components/detail/Index.vue +++ b/dbm-ui/frontend/src/views/db-manage/mongodb/replica-set-list/components/detail/Index.vue @@ -46,7 +46,7 @@ {{ t('禁用集群') }} diff --git a/dbm-ui/frontend/src/views/db-manage/mongodb/replica-set-list/components/list/Index.vue b/dbm-ui/frontend/src/views/db-manage/mongodb/replica-set-list/components/list/Index.vue index 3bb29d110a..cf9f5dce3f 100644 --- a/dbm-ui/frontend/src/views/db-manage/mongodb/replica-set-list/components/list/Index.vue +++ b/dbm-ui/frontend/src/views/db-manage/mongodb/replica-set-list/components/list/Index.vue @@ -605,7 +605,7 @@ text theme="primary" class="ml-16" - disabled={data.operationDisabled} + disabled={Boolean(data.operationTicketId)} onclick={() => handleDisableCluster([data])}> { t('禁用') } diff --git a/dbm-ui/frontend/src/views/db-manage/mongodb/shared-cluster-list/components/detail/Index.vue b/dbm-ui/frontend/src/views/db-manage/mongodb/shared-cluster-list/components/detail/Index.vue index 9699485910..fef022eb3d 100644 --- a/dbm-ui/frontend/src/views/db-manage/mongodb/shared-cluster-list/components/detail/Index.vue +++ b/dbm-ui/frontend/src/views/db-manage/mongodb/shared-cluster-list/components/detail/Index.vue @@ -46,7 +46,7 @@ {{ t('禁用集群') }} diff --git a/dbm-ui/frontend/src/views/db-manage/mongodb/shared-cluster-list/components/list/Index.vue b/dbm-ui/frontend/src/views/db-manage/mongodb/shared-cluster-list/components/list/Index.vue index e3f1245ec1..2e5c461d6a 100644 --- a/dbm-ui/frontend/src/views/db-manage/mongodb/shared-cluster-list/components/list/Index.vue +++ b/dbm-ui/frontend/src/views/db-manage/mongodb/shared-cluster-list/components/list/Index.vue @@ -628,7 +628,7 @@ text theme="primary" class="ml-16" - disabled={data.operationDisabled} + disabled={Boolean(data.operationTicketId)} onclick={() => handleDisableCluster([data])}> { t('禁用') } diff --git a/dbm-ui/frontend/src/views/db-manage/mysql/ha-cluster-list/components/List.vue b/dbm-ui/frontend/src/views/db-manage/mysql/ha-cluster-list/components/List.vue index 1cea9b41a8..c992a59b0a 100644 --- a/dbm-ui/frontend/src/views/db-manage/mysql/ha-cluster-list/components/List.vue +++ b/dbm-ui/frontend/src/views/db-manage/mysql/ha-cluster-list/components/List.vue @@ -817,7 +817,7 @@ handleDisableCluster([data])}> { t('禁用') } diff --git a/dbm-ui/frontend/src/views/db-manage/pulsar/list/components/list/Index.vue b/dbm-ui/frontend/src/views/db-manage/pulsar/list/components/list/Index.vue index f8abaa6920..d7e0b841e4 100644 --- a/dbm-ui/frontend/src/views/db-manage/pulsar/list/components/list/Index.vue +++ b/dbm-ui/frontend/src/views/db-manage/pulsar/list/components/list/Index.vue @@ -581,9 +581,9 @@ { t('获取访问方式') } , ]; - if (data.isOffline) { - return [ - handleEnableCluster([data])}> { t('启用') } - , - - handleDeleteCluster([data])}> - { t('删除') } - - , - ...baseAction, - ]; + + )) } return [ , - - handleDisableCluster([data])}> - { t('禁用') } - - , + data.isOnline && ( + + handleDisableCluster([data])}> + { t('禁用') } + + + ), diff --git a/dbm-ui/frontend/src/views/db-manage/redis/list-ha/components/list/Index.vue b/dbm-ui/frontend/src/views/db-manage/redis/list-ha/components/list/Index.vue index 7e10f56bb2..bd95fe1e80 100644 --- a/dbm-ui/frontend/src/views/db-manage/redis/list-ha/components/list/Index.vue +++ b/dbm-ui/frontend/src/views/db-manage/redis/list-ha/components/list/Index.vue @@ -885,7 +885,7 @@ resource={data.id} permission={data.permission.redis_open_close} style="width: 100%;height: 32px;" - disabled={data.operationDisabled} + disabled={Boolean(data.operationTicketId)} text onClick={() => handleDisableCluster([data])}> { t('禁用') } diff --git a/dbm-ui/frontend/src/views/db-manage/redis/list/components/list/Index.vue b/dbm-ui/frontend/src/views/db-manage/redis/list/components/list/Index.vue index 6161f1516a..7e66da04b2 100644 --- a/dbm-ui/frontend/src/views/db-manage/redis/list/components/list/Index.vue +++ b/dbm-ui/frontend/src/views/db-manage/redis/list/components/list/Index.vue @@ -966,7 +966,7 @@ resource={data.id} permission={data.permission.redis_open_close} style="width: 100%;height: 32px;" - disabled={data.operationDisabled} + disabled={Boolean(data.operationTicketId)} text onClick={() => handleDisableCluster([data])}> { t('禁用') } diff --git a/dbm-ui/frontend/src/views/db-manage/riak/list/Index.vue b/dbm-ui/frontend/src/views/db-manage/riak/list/Index.vue index d76f93d1e7..e7c01e85d3 100644 --- a/dbm-ui/frontend/src/views/db-manage/riak/list/Index.vue +++ b/dbm-ui/frontend/src/views/db-manage/riak/list/Index.vue @@ -14,7 +14,7 @@ + + + + @@ -65,6 +73,7 @@ import { execCopy } from '@utils'; + import UpdateClusterAliasName from './components/UpdateClusterAliasName.vue'; import useColumnCopy from './hooks/useColumnCopy'; import type { ClusterModel, ISupportClusterType } from './types'; @@ -76,7 +85,12 @@ getTableInstance: () => InstanceType | undefined; } + export interface Emits { + (e: 'refresh'): void; + } + const props = defineProps>(); + const emits = defineEmits(); const { t } = useI18n(); const router = useRouter(); @@ -96,4 +110,8 @@ const handleCopyClusterName = (clusterName: string) => { execCopy(clusterName); }; + + const handleUpdateAliasSuccess = () => { + emits('refresh'); + }; diff --git a/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/CommonColumn.vue b/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/CommonColumn.vue index 4a79c56e3b..62625984ea 100644 --- a/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/CommonColumn.vue +++ b/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/CommonColumn.vue @@ -6,7 +6,7 @@ checked: columnCheckedMap.major_version, }" :label="t('版本')" - :min-width="100"> + :min-width="150"> @@ -14,7 +14,7 @@ + :min-width="160"> diff --git a/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/IdColumn.vue b/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/IdColumn.vue index 5a30d045a0..3712bbc6c3 100644 --- a/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/IdColumn.vue +++ b/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/IdColumn.vue @@ -1,6 +1,7 @@ diff --git a/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/RoleColumn.vue b/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/RoleColumn.vue index 5af8fc1e38..8ea517bf4c 100644 --- a/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/RoleColumn.vue +++ b/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/RoleColumn.vue @@ -1,111 +1,27 @@ diff --git a/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/SlaveDomainColumn.vue b/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/SlaveDomainColumn.vue index 49dbe27d2c..c9d61f7491 100644 --- a/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/SlaveDomainColumn.vue +++ b/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/SlaveDomainColumn.vue @@ -2,7 +2,7 @@ + :min-width="280"> + diff --git a/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/components/base-role-column/Index.vue b/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/components/base-role-column/Index.vue new file mode 100644 index 0000000000..430db56db0 --- /dev/null +++ b/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/components/base-role-column/Index.vue @@ -0,0 +1,138 @@ + + + diff --git a/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/components/base-role-column/components/CellContent.vue b/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/components/base-role-column/components/CellContent.vue new file mode 100644 index 0000000000..40669aff8f --- /dev/null +++ b/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/components/base-role-column/components/CellContent.vue @@ -0,0 +1,249 @@ + + + + diff --git a/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/components/base-role-column/components/InstanceList.vue b/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/components/base-role-column/components/InstanceList.vue new file mode 100644 index 0000000000..f4afd1a009 --- /dev/null +++ b/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/components/base-role-column/components/InstanceList.vue @@ -0,0 +1,125 @@ + + diff --git a/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/types.ts b/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/types.ts index 44dfd755d7..b5ea840993 100644 --- a/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/types.ts +++ b/dbm-ui/frontend/src/views/db-manage/common/cluster-table-column/types.ts @@ -1,11 +1,52 @@ -import { ClusterTypes } from '@common/const'; +import DorisModel from '@services/model/doris/doris'; +import EsModel from '@services/model/es/es'; +import HdfsModel from '@services/model/hdfs/hdfs'; +import KafkaModel from '@services/model/kafka/kafka'; +import MongodbModel from '@services/model/mongodb/mongodb'; +import TendbhaModel from '@services/model/mysql/tendbha'; +import TendbsingleModel from '@services/model/mysql/tendbsingle'; +import PulsarModel from '@services/model/pulsar/pulsar'; +import RedisModel from '@services/model/redis/redis'; +import RiakModel from '@services/model/riak/riak'; +import SqlserverHaModel from '@services/model/sqlserver/sqlserver-ha'; +import SqlserverSingleModel from '@services/model/sqlserver/sqlserver-single'; +import TendbClusterModel from '@services/model/tendbcluster/tendbcluster'; -import TendbClusterModel from '@/services/model/tendbcluster/tendbcluster'; +import { ClusterTypes } from '@common/const'; -export type ISupportClusterType = ClusterTypes.TENDBCLUSTER; +export type ISupportClusterType = + | ClusterTypes.TENDBCLUSTER + | ClusterTypes.DORIS + | ClusterTypes.ES + | ClusterTypes.HDFS + | ClusterTypes.TENDBHA + | ClusterTypes.TENDBSINGLE + | ClusterTypes.PULSAR + | ClusterTypes.REDIS + | ClusterTypes.REDIS_INSTANCE + | ClusterTypes.RIAK + | ClusterTypes.KAFKA + | ClusterTypes.SQLSERVER_HA + | ClusterTypes.SQLSERVER_SINGLE + | ClusterTypes.MONGO_REPLICA_SET + | ClusterTypes.MONGO_SHARED_CLUSTER; interface ClusterTypeRelateClusterModel { [ClusterTypes.TENDBCLUSTER]: TendbClusterModel; + [ClusterTypes.DORIS]: DorisModel; + [ClusterTypes.ES]: EsModel; + [ClusterTypes.HDFS]: HdfsModel; + [ClusterTypes.TENDBHA]: TendbhaModel; + [ClusterTypes.TENDBSINGLE]: TendbsingleModel; + [ClusterTypes.PULSAR]: PulsarModel; + [ClusterTypes.REDIS]: RedisModel; + [ClusterTypes.REDIS_INSTANCE]: RedisModel; + [ClusterTypes.RIAK]: RiakModel; + [ClusterTypes.KAFKA]: KafkaModel; + [ClusterTypes.SQLSERVER_HA]: SqlserverHaModel; + [ClusterTypes.SQLSERVER_SINGLE]: SqlserverSingleModel; + [ClusterTypes.MONGO_REPLICA_SET]: MongodbModel; + [ClusterTypes.MONGO_SHARED_CLUSTER]: MongodbModel; } export type ClusterModel = ClusterTypeRelateClusterModel[T]; diff --git a/dbm-ui/frontend/src/views/db-manage/common/render-instances/Instacelist.vue b/dbm-ui/frontend/src/views/db-manage/common/render-instances/Instacelist.vue new file mode 100644 index 0000000000..43b062da58 --- /dev/null +++ b/dbm-ui/frontend/src/views/db-manage/common/render-instances/Instacelist.vue @@ -0,0 +1,142 @@ + + diff --git a/dbm-ui/frontend/src/views/db-manage/common/render-instances/RenderInstances.vue b/dbm-ui/frontend/src/views/db-manage/common/render-instances/RenderInstances.vue index 5cc0b0a21e..2703307211 100644 --- a/dbm-ui/frontend/src/views/db-manage/common/render-instances/RenderInstances.vue +++ b/dbm-ui/frontend/src/views/db-manage/common/render-instances/RenderInstances.vue @@ -12,11 +12,10 @@ --> -

- + + - -
-
- - {{ t('复制异常实例') }} - - - {{ t('复制全部实例') }} - - -
- - - - - - {{ role }} - - - - - -
- - + + + {{ t('复制实例') }} + + + + - - - diff --git a/dbm-ui/frontend/src/views/db-manage/riak/list/Index.vue b/dbm-ui/frontend/src/views/db-manage/riak/list/Index.vue index e7c01e85d3..39b8f5f8d6 100644 --- a/dbm-ui/frontend/src/views/db-manage/riak/list/Index.vue +++ b/dbm-ui/frontend/src/views/db-manage/riak/list/Index.vue @@ -79,7 +79,7 @@ + @setting-change="updateTableSettings"> + + + + + + + + + + + diff --git a/dbm-ui/frontend/src/views/db-manage/sqlserver/single-cluster/components/detail/BaseInfo.vue b/dbm-ui/frontend/src/views/db-manage/sqlserver/single-cluster-list/components/detail/BaseInfo.vue similarity index 100% rename from dbm-ui/frontend/src/views/db-manage/sqlserver/single-cluster/components/detail/BaseInfo.vue rename to dbm-ui/frontend/src/views/db-manage/sqlserver/single-cluster-list/components/detail/BaseInfo.vue diff --git a/dbm-ui/frontend/src/views/db-manage/sqlserver/single-cluster/components/detail/index.vue b/dbm-ui/frontend/src/views/db-manage/sqlserver/single-cluster-list/components/detail/index.vue similarity index 100% rename from dbm-ui/frontend/src/views/db-manage/sqlserver/single-cluster/components/detail/index.vue rename to dbm-ui/frontend/src/views/db-manage/sqlserver/single-cluster-list/components/detail/index.vue diff --git a/dbm-ui/frontend/src/views/db-manage/sqlserver/single-cluster/components/List.vue b/dbm-ui/frontend/src/views/db-manage/sqlserver/single-cluster/components/List.vue deleted file mode 100644 index 0b8cb0e6ec..0000000000 --- a/dbm-ui/frontend/src/views/db-manage/sqlserver/single-cluster/components/List.vue +++ /dev/null @@ -1,871 +0,0 @@ - - - - diff --git a/dbm-ui/frontend/src/views/db-manage/tendb-cluster/list/components/list/Index.vue b/dbm-ui/frontend/src/views/db-manage/tendb-cluster/list/components/list/Index.vue index ef3f5fe671..2c9412c656 100644 --- a/dbm-ui/frontend/src/views/db-manage/tendb-cluster/list/components/list/Index.vue +++ b/dbm-ui/frontend/src/views/db-manage/tendb-cluster/list/components/list/Index.vue @@ -79,26 +79,31 @@ + :label="t('主访问入口')" + :selected-list="selected" + @go-detail="handleToDetails" + @refresh="fetchTableData" /> + :selected-list="selected" + @refresh="fetchTableData" /> - -