feat(backend): 巡检通用框架 TencentBlueKing#9120

dbm-ui/backend/configuration/models/dba.py

@@ -8,7 +8,7 @@
 an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 specific language governing permissions and limitations under the License.
 """
-from typing import Dict, List, Union
+from typing import Dict, List, Tuple, Union
 
 from django.db import models
 from django.utils.translation import ugettext_lazy as _
@@ -62,9 +62,24 @@ def get_biz_db_type_admins(cls, bk_biz_id: int, db_type: str) -> List[str]:
         return DEFAULT_DB_ADMINISTRATORS
 
     @classmethod
-    def get_dba_for_db_type(cls, bk_biz_id: int, db_type: str) -> List[str]:
+    def get_dba_for_db_type(cls, bk_biz_id: int, db_type: str) -> Tuple[List[str], List[str], List[str]]:
         """获取主dba、备dba、二线dba人员"""
         dba_list = cls.list_biz_admins(bk_biz_id)
         dba_content = next((dba for dba in dba_list if dba["db_type"] == db_type), {"users": []})
         users = dba_content.get("users", [])
         return users[:1], users[1:2], users[2:]
+
+    @classmethod
+    def get_manage_bizs(cls, db_type: str, username: str) -> Tuple[List[str], List[str]]:
+        """获取待我处理，待我协助的业务"""
+        manage_biz = DBAdministrator.objects.filter(db_type=db_type, users__0=username).values_list(
+            "bk_biz_id", flat=True
+        )
+
+        assist_bizs = (
+            DBAdministrator.objects.filter(db_type=db_type, users__contains=username)
+            .exclude(users__0=username)
+            .values_list("bk_biz_id", flat=True)
+        )
+
+        return list(manage_biz), list(assist_bizs)

dbm-ui/backend/db_report/README.md

@@ -0,0 +1,4 @@
+1. 报告结果 _model_ 继承 `BaseReportABS`
+2. 报告结果视图继承 `ReportBaseViewSet`
+3. 视图
+3. 示例可以参考 `dbm-ui/backend/db_report/views/mysql/mysqlbackup_check_view.py`

dbm-ui/backend/db_report/apps.py

@@ -1,6 +1,21 @@
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+
 from django.apps import AppConfig
 
+from backend.db_report.register import register_all_reports
+
 
 class DbReportConfig(AppConfig):
     default_auto_field = "django.db.models.BigAutoField"
     name = "backend.db_report"
+
+    def ready(self):
+        register_all_reports()

dbm-ui/backend/db_report/enums/__init__.py

@@ -19,9 +19,23 @@
 
 SWAGGER_TAG = _("巡检报告")
 
+REPORT_COUNT_CACHE_KEY = "{user}_report_count_key"
+
 
 class ReportFieldFormat(str, StructuredEnum):
     TEXT = EnumField("text", _("文本渲染"))
     STATUS = EnumField("status", _("状态渲染"))
     # 数据校验失败详情字段
     FAIL_SLAVE_INSTANCE = EnumField("fail_slave_instance", _("数据校验失败详情渲染"))
+
+
+class ReportType(str, StructuredEnum):
+    CHECKSUM = EnumField("checksum", _("数据校验"))
+    FULL_BACKUP_CHECK = EnumField("full_backup_check", _("全备校验"))
+    BINLOG_BACKUP_CHECK = EnumField("binlog_backup_check", _("集群binlog检查"))
+
+    ALONE_INSTANCE_CHECK = EnumField("alone_instance_check", _("孤立实例检查"))
+    STATUS_ABNORMAL_CHECK = EnumField("status_abnormal_check", _("实例异常状态检查"))
+    META_CHECK = EnumField("meta_check", _("元数据检查"))
+
+    REDIS_DBMON_HEARTBEAT_CHECK = EnumField("dbmon_heartbeat_check", _("dbmon心跳超时检查"))

dbm-ui/backend/db_report/filters.py

@@ -0,0 +1,46 @@
+# -*- coding:utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+from django.utils.translation import ugettext_lazy as _
+from django_filters import rest_framework as filters
+from django_filters.rest_framework import DjangoFilterBackend
+
+from backend.configuration.models import DBAdministrator
+
+
+class ReportListFilter(filters.FilterSet):
+    manage = filters.CharFilter(
+        field_name="manage", method="filter_manage", label=_("处理类型"), help_text=_("todo待我处理/assist待我协助")
+    )
+    cluster = filters.CharFilter(field_name="cluster", method="filter_cluster", label=_("集群名"))
+
+    def filter_manage(self, queryset, name, value):
+        username = self.request.user.username
+        db_type = self.request.path.strip("/").split("/")[1]
+        manage_bizs, assist_bizs = DBAdministrator.get_manage_bizs(db_type, username)
+        # 待我处理
+        if value == "todo":
+            return queryset.filter(bk_biz_id__in=manage_bizs)
+        # 待我协助
+        elif value == "assist":
+            return queryset.filter(bk_biz_id__in=assist_bizs)
+        # 其他情况忽略
+        return queryset
+
+    def filter_cluster(self, queryset, name, value):
+        cluster = value.split(",")
+        if len(cluster) == 1:
+            return queryset.filter(cluster__icontains=cluster[0])
+        else:
+            return queryset.filter(cluster__in=cluster)
+
+
+class ReportFilterBackend(DjangoFilterBackend):
+    filterset_base = ReportListFilter

dbm-ui/backend/db_report/mock_data.py

@@ -144,3 +144,23 @@
         {"name": "msg", "display_name": "详情", "format": "text"},
     ],
 }
+
+REPORT_OVERVIEW_DATA = {
+    "redis": [
+        "dbmon_heartbeat_check",
+        "full_backup_check",
+        "binlog_backup_check",
+        "alone_instance_check",
+        "status_abnormal_check",
+    ],
+    "mysql": ["full_backup_check", "binlog_backup_check", "meta_check", "checksum"],
+}
+
+REPORT_COUNT_DATA = {
+    "redis": {
+        "dbmon_heartbeat_check": {"manage_count": 10896, "assist_count": 0},
+    },
+    "mysql": {
+        "full_backup_check": {"manage_count": 0, "assist_count": 26},
+    },
+}

dbm-ui/backend/db_report/register.py

@@ -0,0 +1,34 @@
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available.
+Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at https://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+import logging
+import os
+from collections import defaultdict
+from typing import Dict, List
+
+from backend.utils.register import re_import_modules
+from config import BASE_DIR
+
+logger = logging.getLogger("root")
+
+db_report_maps: Dict[str, List] = defaultdict(list)
+
+
+def register_report(db_type):
+    """巡检视图的注册器"""
+
+    def decorator(report_cls):
+        db_report_maps[db_type].append(report_cls)
+
+    return decorator
+
+
+def register_all_reports():
+    """递归注册当前目录下所有的巡检报告"""
+    re_import_modules(path=os.path.join(BASE_DIR, "backend/db_report/views"), module_path="backend.db_report.views")

dbm-ui/backend/db_report/report_baseview.py

@@ -8,29 +8,47 @@
 an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 specific language governing permissions and limitations under the License.
 """
-from rest_framework import mixins
+from collections import defaultdict
+from typing import Dict
+
+from django.core.cache import cache
+from django.utils.translation import gettext_lazy as _
+from rest_framework import mixins, status
+from rest_framework.decorators import action
+from rest_framework.response import Response
 from rest_framework.viewsets import GenericViewSet
 
+from backend.bk_web import viewsets
 from backend.bk_web.pagination import AuditedLimitOffsetPagination
+from backend.bk_web.swagger import common_swagger_auto_schema
+from backend.configuration.models import DBAdministrator
+from backend.db_report.enums import REPORT_COUNT_CACHE_KEY, SWAGGER_TAG, ReportType
+from backend.db_report.filters import ReportFilterBackend
+from backend.db_report.register import db_report_maps
+from backend.db_report.serializers import GetReportCountSerializer, GetReportOverviewSerializer
 from backend.iam_app.dataclass import ResourceEnum
 from backend.iam_app.dataclass.actions import ActionEnum
 from backend.iam_app.handlers.drf_perm.base import ResourceActionPermission, get_request_key_id
 
 
 class ReportBaseViewSet(GenericViewSet, mixins.ListModelMixin):
+    # 分页类
     pagination_class = AuditedLimitOffsetPagination
-
+    # 巡检类型
+    report_type = None
+    # 巡检名称
+    report_name = ""
+    # 巡检表头
+    report_title = []
+    # 巡检过滤类
+    filter_backends = [ReportFilterBackend]
     filter_fields = {
         "bk_biz_id": ["exact"],
-        "cluster": ["exact", "in"],
         "cluster_type": ["exact", "in"],
         "create_at": ["gte", "lte"],
         "status": ["exact", "in"],
     }
 
-    report_name = ""
-    report_title = []
-
     @staticmethod
     def instance_getter(request, view):
         return [get_request_key_id(request, "bk_biz_id")]
@@ -42,7 +60,48 @@ def get_permissions(self):
 
     def list(self, request, *args, **kwargs):
         response = super().list(request, *args, **kwargs)
-        response.data["name"] = self.report_name
+        response.data["name"] = self.report_name or ReportType.get_choice_label(self.report_type)
         response.data["title"] = self.report_title
-
         return response
+
+
+class ReportCommonViewSet(viewsets.SystemViewSet):
+    """巡检通用接口视图"""
+
+    @common_swagger_auto_schema(
+        operation_summary=_("获取巡检报告总览"),
+        responses={status.HTTP_200_OK: GetReportOverviewSerializer()},
+        tags=[SWAGGER_TAG],
+    )
+    @action(methods=["GET"], detail=False, serializer_class=GetReportOverviewSerializer)
+    def get_report_overview(self, request, *args, **kwargs):
+        db_report_types = defaultdict(list)
+        for db_type, report_cls_list in db_report_maps.items():
+            db_report_types[db_type] = [cls.report_type for cls in report_cls_list]
+        return Response(db_report_types)
+
+    @common_swagger_auto_schema(
+        operation_summary=_("获取巡检报告代办数量"),
+        responses={status.HTTP_200_OK: GetReportCountSerializer()},
+        tags=[SWAGGER_TAG],
+    )
+    @action(methods=["GET"], detail=False, serializer_class=GetReportCountSerializer)
+    def get_report_count(self, request, *args, **kwargs):
+        username = request.user.username
+        cache_key = REPORT_COUNT_CACHE_KEY.format(user=username)
+
+        report_count_cache = cache.get(cache_key)
+        if report_count_cache:
+            return Response(report_count_cache)
+        else:
+            report_count_map: Dict[str, Dict[str, Dict]] = defaultdict(lambda: defaultdict(dict))
+            for db_type, report_classes in db_report_maps.items():
+                manage_bizs, assist_bizs = DBAdministrator.get_manage_bizs(db_type, username)
+                for cls in report_classes:
+                    report_count_map[db_type][cls.report_type].update(
+                        manage_count=cls.queryset.filter(status=False, bk_biz_id__in=manage_bizs).count(),
+                        assist_count=cls.queryset.filter(status=False, bk_biz_id__in=assist_bizs).count(),
+                    )
+            # 数量精确性不高，可以做1h的缓存
+            cache.set(cache_key, report_count_map, 60 * 60)
+            return Response(report_count_map)

dbm-ui/backend/db_report/serializers/meta_check_report_serializer.py → dbm-ui/backend/db_report/serializers.py

@@ -10,10 +10,14 @@
 """
 from rest_framework import serializers
 
-from backend.db_report.models import MetaCheckReport
+from backend.db_report import mock_data
 
 
-class MetaCheckReportSerializer(serializers.ModelSerializer):
+class GetReportOverviewSerializer(serializers.Serializer):
     class Meta:
-        model = MetaCheckReport
-        fields = "__all__"
+        swagger_schema_fields = {"example": mock_data.REPORT_OVERVIEW_DATA}
+
+
+class GetReportCountSerializer(serializers.Serializer):
+    class Meta:
+        swagger_schema_fields = {"example": mock_data.REPORT_COUNT_DATA}

dbm-ui/backend/db_report/urls.py

@@ -8,19 +8,20 @@
 an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 specific language governing permissions and limitations under the License.
 """
-from django.conf.urls import url
+from rest_framework.routers import DefaultRouter
 
-from backend.db_report import views
+from backend.db_report.register import db_report_maps
+from backend.db_report.report_baseview import ReportCommonViewSet
+from backend.db_report.views.checksum_instance_view import ChecksumInstanceViewSet
 
-urlpatterns = [
-    url("^meta_check/instance_belong$", views.MetaCheckReportInstanceBelongViewSet.as_view({"get": "list"})),
-    url("^checksum_check/report$", views.ChecksumCheckReportViewSet.as_view({"get": "list"})),
-    url("^checksum_check/instance$", views.ChecksumInstanceViewSet.as_view({"get": "list"})),
-    url("^mysql_check/full_backup$", views.MysqlFullBackupCheckReportViewSet.as_view({"get": "list"})),
-    url("^mysql_check/binlog_backup$", views.MysqlBinlogBackupCheckReportViewSet.as_view({"get": "list"})),
-    url("^redis_check/full_backup$", views.RedisFullBackupCheckReportViewSet.as_view({"get": "list"})),
-    url("^redis_check/binlog_backup$", views.RedisBinlogBackupCheckReportViewSet.as_view({"get": "list"})),
-    url("^dbmon/heartbeat$", views.DbmonHeatbeartCheckReportBaseViewSet.as_view({"get": "list"})),
-    url("^redis_meta_check/status_abnormal$", views.RedisStatusAbnormalCheckReportViewSet.as_view({"get": "list"})),
-    url("^redis_meta_check/alone_instance$", views.RedisAloneInstanceCheckReportViewSet.as_view({"get": "list"})),
-]
+routers = DefaultRouter(trailing_slash=True)
+
+routers.register(r"", ReportCommonViewSet, basename="report_common")
+routers.register(r"checksum_instance", ChecksumInstanceViewSet, basename="checksum_instance")
+
+# 自动添加注册的巡检视图
+for db_type, reports in db_report_maps.items():
+    for report in reports:
+        routers.register(f"{db_type}/{report.report_type}", report, basename=f"{db_type}-{report.report_type}")
+
+urlpatterns = routers.urls

dbm-ui/backend/db_report/views/__init__.py

@@ -8,10 +8,5 @@
 an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 specific language governing permissions and limitations under the License.
 """
-from .checksum_check_report_view import ChecksumCheckReportViewSet
-from .checksum_instance_view import ChecksumInstanceViewSet
-from .dbmon_heartbeat_view import DbmonHeatbeartCheckReportBaseViewSet
-from .meta_check_view import MetaCheckReportInstanceBelongViewSet
-from .mysqlbackup_check_view import MysqlBinlogBackupCheckReportViewSet, MysqlFullBackupCheckReportViewSet
-from .redis_dbmeta_check_view import RedisAloneInstanceCheckReportViewSet, RedisStatusAbnormalCheckReportViewSet
-from .redisbackup_check_view import RedisBinlogBackupCheckReportViewSet, RedisFullBackupCheckReportViewSet
+from .mysql import *
+from .redis import *

dbm-ui/backend/db_report/views/checksum_check_report_view.py

@@ -12,7 +12,7 @@
 
 import logging
 
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext as _
 from rest_framework import serializers, status
 
 from backend.bk_web.swagger import common_swagger_auto_schema
@@ -31,16 +31,9 @@ class Meta:
         swagger_schema_fields = {"example": mock_data.CHECKSUM_CHECK_DATA}
 
 
-class ChecksumCheckReportViewSet(ReportBaseViewSet):
+class ChecksumCheckReportBaseViewSet(ReportBaseViewSet):
     queryset = ChecksumCheckReport.objects.all().order_by("-create_at")
     serializer_class = ChecksumCheckReportSerializer
-    filter_fields = {
-        "bk_biz_id": ["exact"],
-        "cluster_type": ["exact"],
-        "create_at": ["gte", "lte"],
-        "status": ["exact", "in"],
-        "cluster": ["exact", "in"],
-    }
     report_name = _("数据校验")
     report_title = [
         {