From 55b7547279f95fe8d77827e66a5fade6c612a929 Mon Sep 17 00:00:00 2001 From: yksitu <1297650644@qq.com> Date: Thu, 2 Jan 2025 16:42:40 +0800 Subject: [PATCH] =?UTF-8?q?fix(sqlserver):=20=E5=A2=9E=E5=8A=A0SQLserver?= =?UTF-8?q?=E5=A4=87=E4=BB=BD=E6=96=87=E4=BB=B6=E5=B7=A1=E6=A3=80=E5=92=8C?= =?UTF-8?q?=E4=BA=91=E6=95=B0=E6=8D=AE=E5=B7=A1=E6=A3=80=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=20#8825?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../backend/components/mysql_backup/client.py | 6 + .../local_tasks/db_meta/__init__.py | 2 +- .../db_meta/db_meta_check/__init__.py | 2 +- .../sqlserver_cluster_topo/__init__.py | 10 + .../sqlserver_cluster_topo/check.py | 77 ++++++++ .../local_tasks/db_meta/db_meta_check/task.py | 14 +- .../local_tasks/sqlserver/__init__.py | 2 +- .../sqlserver/backup_file_check.py | 176 ++++++++++++++++++ .../local_tasks/sqlserver/task.py | 12 +- ...inforeport_sqlserverlogbackupinforeport.py | 131 +++++++++++++ dbm-ui/backend/db_report/models/__init__.py | 2 + .../models/sqlserver_check_report.py | 22 +++ .../utils/sqlserver/sqlserver_db_function.py | 43 ++++- 13 files changed, 493 insertions(+), 6 deletions(-) create mode 100644 dbm-ui/backend/db_periodic_task/local_tasks/db_meta/db_meta_check/sqlserver_cluster_topo/__init__.py create mode 100644 dbm-ui/backend/db_periodic_task/local_tasks/db_meta/db_meta_check/sqlserver_cluster_topo/check.py create mode 100644 dbm-ui/backend/db_periodic_task/local_tasks/sqlserver/backup_file_check.py create mode 100644 dbm-ui/backend/db_report/migrations/0009_sqlserverfullbackupinforeport_sqlserverlogbackupinforeport.py diff --git a/dbm-ui/backend/components/mysql_backup/client.py b/dbm-ui/backend/components/mysql_backup/client.py index c2eacefb14..e81ca46b70 100644 --- a/dbm-ui/backend/components/mysql_backup/client.py +++ b/dbm-ui/backend/components/mysql_backup/client.py @@ -42,7 +42,13 @@ def __init__(self): default_timeout=300, max_retry_times=1, ) + self.query_for_task_ids = self.generate_data_api( + method="POST", + url="backupapi/queryTasks", + description=_("根据备份任务id列表，拉取备份文件信息"), + ) MysqlBackupApi = _BackupApi() RedisBackupApi = _BackupApi() +SQLServerBackupApi = _BackupApi() diff --git a/dbm-ui/backend/db_periodic_task/local_tasks/db_meta/__init__.py b/dbm-ui/backend/db_periodic_task/local_tasks/db_meta/__init__.py index cfe5a02af8..c46b174873 100644 --- a/dbm-ui/backend/db_periodic_task/local_tasks/db_meta/__init__.py +++ b/dbm-ui/backend/db_periodic_task/local_tasks/db_meta/__init__.py @@ -8,6 +8,6 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -from .db_meta_check import db_meta_check_task +from .db_meta_check import db_meta_check_task, sqlserver_topo_daily_check from .sync_cluster_stat import sync_cluster_stat_from_monitor from .update_app_cache import update_app_cache diff --git a/dbm-ui/backend/db_periodic_task/local_tasks/db_meta/db_meta_check/__init__.py b/dbm-ui/backend/db_periodic_task/local_tasks/db_meta/db_meta_check/__init__.py index c0e3d9b019..e57e3c1281 100644 --- a/dbm-ui/backend/db_periodic_task/local_tasks/db_meta/db_meta_check/__init__.py +++ b/dbm-ui/backend/db_periodic_task/local_tasks/db_meta/db_meta_check/__init__.py @@ -8,4 +8,4 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -from .task import db_meta_check_task +from .task import db_meta_check_task, sqlserver_topo_daily_check diff --git a/dbm-ui/backend/db_periodic_task/local_tasks/db_meta/db_meta_check/sqlserver_cluster_topo/__init__.py b/dbm-ui/backend/db_periodic_task/local_tasks/db_meta/db_meta_check/sqlserver_cluster_topo/__init__.py new file mode 100644 index 0000000000..3f343e621a --- /dev/null +++ b/dbm-ui/backend/db_periodic_task/local_tasks/db_meta/db_meta_check/sqlserver_cluster_topo/__init__.py @@ -0,0 +1,10 @@ +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from .check import sqlserver_dbmeta_check diff --git a/dbm-ui/backend/db_periodic_task/local_tasks/db_meta/db_meta_check/sqlserver_cluster_topo/check.py b/dbm-ui/backend/db_periodic_task/local_tasks/db_meta/db_meta_check/sqlserver_cluster_topo/check.py new file mode 100644 index 0000000000..e72aa44e43 --- /dev/null +++ b/dbm-ui/backend/db_periodic_task/local_tasks/db_meta/db_meta_check/sqlserver_cluster_topo/check.py @@ -0,0 +1,77 @@ +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from typing import List + +from backend.db_meta.enums import ClusterType +from backend.db_meta.models import Cluster +from backend.db_periodic_task.local_tasks.db_meta.db_meta_check.mysql_cluster_topo.check_response import CheckResponse +from backend.db_periodic_task.local_tasks.db_meta.db_meta_check.mysql_cluster_topo.tendbha.replicate import ( + _cluster_master_as_ejector, + _cluster_slave_as_receiver, +) +from backend.db_periodic_task.local_tasks.db_meta.db_meta_check.mysql_cluster_topo.tendbha.status import ( + _cluster_master_entry_count, + _cluster_master_status, + _cluster_one_master, + _cluster_one_standby_slave, + _cluster_standby_slave_status, + _cluster_status, +) +from backend.db_periodic_task.local_tasks.db_meta.db_meta_check.mysql_cluster_topo.tendbha.unique_cluster import ( + _cluster_instance_unique_cluster, +) + + +def sqlserver_dbmeta_check(cluster_id: int) -> List[CheckResponse]: + """ + SQLServer Ha集群检测项： + 1：集群状态正常 + 2：实例有且只有属于一个集群 + 3：主入口数 >= 1 + 4：唯一 master + 5：master 状态正常 + 6：唯一 standby slave + 7：standby slave 状态正常 + 8：master 只能作为 ejector + 9：slave 只能作为 receiver + """ + # 或者集群所有元信息 + clusters = Cluster.objects.filter(id=cluster_id).prefetch_related( + "clusterentry_set__storageinstance_set", + "storageinstance_set__as_receiver__ejector__cluster", + "storageinstance_set__as_ejector__receiver__cluster", + "storageinstance_set__cluster", + ) + + res = [] + for cluster_obj in clusters: + # 检查集群状态 + res.extend(_cluster_status(cluster_obj)) + # 实例有且只有属于一个集群 + res.extend(_cluster_instance_unique_cluster(cluster_obj)) + # 主入口数 >= 1 + res.extend(_cluster_master_entry_count(cluster_obj)) + + # 如果是ha架构，则需要检测下面子项 + if cluster_obj.cluster_type == ClusterType.SqlserverHA: + # 唯一 master + res.extend(_cluster_one_master(cluster_obj)) + # master 状态 + res.extend(_cluster_master_status(cluster_obj)) + # 唯一 standby slave + res.extend(_cluster_one_standby_slave(cluster_obj)) + # standby slave 状态正常 + res.extend(_cluster_standby_slave_status(cluster_obj)) + # master 只能作为 ejector + res.extend(_cluster_master_as_ejector(cluster_obj)) + # slave 只能作为 receiver + res.extend(_cluster_slave_as_receiver(cluster_obj)) + + return res diff --git a/dbm-ui/backend/db_periodic_task/local_tasks/db_meta/db_meta_check/task.py b/dbm-ui/backend/db_periodic_task/local_tasks/db_meta/db_meta_check/task.py index 466255c442..5f422894cb 100644 --- a/dbm-ui/backend/db_periodic_task/local_tasks/db_meta/db_meta_check/task.py +++ b/dbm-ui/backend/db_periodic_task/local_tasks/db_meta/db_meta_check/task.py @@ -12,13 +12,14 @@ from celery.schedules import crontab -from backend.db_meta.enums import ClusterType +from backend.db_meta.enums import ClusterPhase, ClusterType from backend.db_meta.models import Cluster from backend.db_periodic_task.local_tasks.register import register_periodic_task from backend.db_report.models import MetaCheckReport from .check_redis_instance import check_redis_instance from .mysql_cluster_topo.tendbha import health_check +from .sqlserver_cluster_topo.check import sqlserver_dbmeta_check logger = logging.getLogger("celery") @@ -37,3 +38,14 @@ def tendbha_topo_daily_check(): r: MetaCheckReport for r in health_check(c.id): r.save() + + +@register_periodic_task(run_every=crontab(hour=5, minute=30)) +def sqlserver_topo_daily_check(): + # 只检查online状态的集群 + for c in Cluster.objects.filter( + phase=ClusterPhase.ONLINE, cluster_type__in=[ClusterType.SqlserverHA, ClusterType.SqlserverSingle] + ): + r: MetaCheckReport + for r in sqlserver_dbmeta_check(c.id): + r.save() diff --git a/dbm-ui/backend/db_periodic_task/local_tasks/sqlserver/__init__.py b/dbm-ui/backend/db_periodic_task/local_tasks/sqlserver/__init__.py index 9e433aca64..a3ceea2bd5 100644 --- a/dbm-ui/backend/db_periodic_task/local_tasks/sqlserver/__init__.py +++ b/dbm-ui/backend/db_periodic_task/local_tasks/sqlserver/__init__.py @@ -7,4 +7,4 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -from .task import check_instance_app_setting +from .task import check_backup_info, check_instance_app_setting diff --git a/dbm-ui/backend/db_periodic_task/local_tasks/sqlserver/backup_file_check.py b/dbm-ui/backend/db_periodic_task/local_tasks/sqlserver/backup_file_check.py new file mode 100644 index 0000000000..50014bd9ea --- /dev/null +++ b/dbm-ui/backend/db_periodic_task/local_tasks/sqlserver/backup_file_check.py @@ -0,0 +1,176 @@ +""" +TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-DB管理系统(BlueKing-BK-DBM) available. +Copyright (C) 2017-2023 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at https://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import logging +from collections import defaultdict +from datetime import datetime, time, timedelta +from typing import Dict, List, Set + +from django.utils import timezone + +from backend.components.bklog.handler import BKLogHandler +from backend.components.mysql_backup.client import SQLServerBackupApi +from backend.db_meta.enums import ClusterPhase, ClusterType +from backend.db_meta.models import Cluster +from backend.db_report.models.sqlserver_check_report import SqlserverFullBackupInfoReport, SqlserverLogBackupInfoReport + +logger = logging.getLogger("root") + + +class CheckBackupInfo(object): + """ + 已dbm元数据为准 + 检查实例的app_setting表的信息是否符合预期，如果存在信息不一致，则需要已某种方式输出告知相关DBA + """ + + def __init__(self): + # 获取所有的online状态的cluster + self.clusters = Cluster.objects.prefetch_related( + "storageinstance_set", + "storageinstance_set__machine", + ).filter(phase=ClusterPhase.ONLINE, cluster_type__in=[ClusterType.SqlserverHA, ClusterType.SqlserverSingle]) + # 拼装查询的时间区间, 查找当前00点到前一天的00点 + today = datetime.now(timezone.utc).date() + midnight_utc = datetime.combine(today, time(), tzinfo=timezone.utc) + self.start_time = midnight_utc - timedelta(days=1) + self.end_time = midnight_utc + + def __query_log_bk_log(self, cluster: Cluster, collector: str): + return BKLogHandler.query_logs( + collector=collector, + start_time=self.start_time, + end_time=self.end_time, + query_string=f"cluster_id: {cluster.id}", + size=10000, + sorting_rule="asc", + ) + + def check_task(self): + for cluster in self.clusters: + self.check_full_backup_info_cluster(cluster) + self.check_log_backup_info_cluster(cluster) + + def check_full_backup_info_cluster(self, cluster: Cluster): + """ + 检查集群的全量备份文件周边信息是否存在 + 1：对应的备份文件是否在备份记录 + 2：对应的备份文件是否上传到备份系统 + """ + # 获取待巡检的全量备份信息 + backup_infos = self.__query_log_bk_log(cluster=cluster, collector="mssql_dbbackup_result") + # 判断每一次的备份任务是否缺失记录 + check_result, is_normal = self.check_backup_info_in_bk_log(backup_infos) + # 写入到巡检表 + SqlserverFullBackupInfoReport.objects.create( + bk_cloud_id=cluster.bk_cloud_id, + bk_biz_id=cluster.bk_biz_id, + cluster=cluster.name, + cluster_type=cluster.cluster_type, + status=is_normal, + msg=check_result, + ) + return + + def check_log_backup_info_cluster(self, cluster: Cluster): + """ + 检查集群的增量备份文件周边信息是否存在 + 1：对应的备份文件是否在备份记录 + 2：对应的备份文件是否上传到备份系统 + """ + # 获取待巡检的全量备份信息 + backup_infos = self.__query_log_bk_log(cluster=cluster, collector="mssql_binlog_result") + # 判断每一次的备份任务是否缺失记录 + check_result, is_normal = self.check_backup_info_in_bk_log(backup_infos) + # 写入到巡检表 + SqlserverLogBackupInfoReport.objects.create( + bk_cloud_id=cluster.bk_cloud_id, + bk_biz_id=cluster.bk_biz_id, + cluster=cluster.name, + cluster_type=cluster.cluster_type, + status=is_normal, + msg=check_result, + ) + return + + def check_backup_info_in_bk_log(self, backup_infos: list): + """ + 判断从bk_log拉取出来的备份信息，根据backup_id聚合，判断合法性 + """ + check_result = "" + is_normal = True + if not backup_infos: + # 如果查询到的备份文件为空， 怎么提前返回结果 + return f"backup-info is null , check [{self.start_time}-{self.end_time}]", False + + # 根据backup id聚合备份记录 + backup_id__logs: Dict[str, List] = defaultdict(list) + for log in backup_infos: + backup_id__logs[log["backup_id"]].append(log) + + # 对每一份备份记录去重，相同的backup id不能出现重复的dbname + backup_id__valid_logs: Dict[str, List] = defaultdict(list) + for backup_id, logs in backup_id__logs.items(): + dbname_set: Set[str] = set() + for log in logs: + if log["dbname"] not in dbname_set: + backup_id__valid_logs[backup_id].append(log) + dbname_set.add(log["dbname"]) + + # 遍历没有backup_id的备份任务 + for backup_id, logs in backup_id__valid_logs.items(): + # 按照备份任务，查询在备份系统上报情况 + task_ids = [i["task_id"] for i in logs] + result = self.check_backup_file_in_backup_system(task_ids=task_ids) + if result: + check_result += f"[{backup_id}] {result}\n" + is_normal = False + + # 判断每个备份任务的备份文件行数，跟bk_log上传的日志是否一致 + if len(logs) != logs[0]["file_cnt"]: + check_result += f"Backup tasks[{backup_id}] are missing backup records, check\n" + is_normal = False + + if not check_result: + # 代表正常返回结果 + return f"backup info check ok [{self.start_time}-{self.end_time}]", is_normal + + return check_result, is_normal + + @staticmethod + def check_backup_file_in_backup_system(task_ids: list): + """ + 根据传入的task_id列表，查询备份文件是否成功上传到备份系统 + """ + max_length = 100 + check_result = [] + if len(task_ids) > 100: + # 如果大于最大长度，进行切分 + split_lists = [task_ids[i : i + max_length] for i in range(0, len(task_ids), max_length)] + else: + # 如果不大于最大长度，直接返回原列表 + split_lists = [task_ids] + for task_list in split_lists: + # 分批请求 + check_result.extend(SQLServerBackupApi.query_for_task_ids({"task_ids": task_list})) + + # 判断长度 + if len(task_ids) != len(check_result): + # 如果传入的任务列表长度和返回的结果长度不一致，则必定是有缺漏，返回异常 + return "some backup files are not in the backup system, check" + + # 判断每个备份文件上传状态码，如果状态码不等于4（已上传完成），表示返回异常 + not_success_task_id_list = [] + for info in check_result: + if info["status"] != 4: + not_success_task_id_list.append(info["task_id"]) + if not_success_task_id_list: + return f"some backup files failed to upload, check:{not_success_task_id_list}" + + return "" diff --git a/dbm-ui/backend/db_periodic_task/local_tasks/sqlserver/task.py b/dbm-ui/backend/db_periodic_task/local_tasks/sqlserver/task.py index a11088b228..db733f0ce4 100644 --- a/dbm-ui/backend/db_periodic_task/local_tasks/sqlserver/task.py +++ b/dbm-ui/backend/db_periodic_task/local_tasks/sqlserver/task.py @@ -13,6 +13,7 @@ from celery.schedules import crontab from backend.db_periodic_task.local_tasks.register import register_periodic_task +from backend.db_periodic_task.local_tasks.sqlserver.backup_file_check import CheckBackupInfo from backend.db_periodic_task.local_tasks.sqlserver.check_app_setting_data import CheckAppSettingData logger = logging.getLogger("celery") @@ -22,6 +23,15 @@ def check_instance_app_setting(): """ 检查实例的元数据表(app_setting)是否正常 - 每条凌晨7点执行 + 每条凌晨6点30分执行 """ CheckAppSettingData().check_task() + + +@register_periodic_task(run_every=crontab(minute=00, hour=8)) +def check_backup_info(): + """ + 检查集群的备份信息的巡检报告 + 每条凌晨8点执行 + """ + CheckBackupInfo().check_task() diff --git a/dbm-ui/backend/db_report/migrations/0009_sqlserverfullbackupinforeport_sqlserverlogbackupinforeport.py b/dbm-ui/backend/db_report/migrations/0009_sqlserverfullbackupinforeport_sqlserverlogbackupinforeport.py new file mode 100644 index 0000000000..37c2117343 --- /dev/null +++ b/dbm-ui/backend/db_report/migrations/0009_sqlserverfullbackupinforeport_sqlserverlogbackupinforeport.py @@ -0,0 +1,131 @@ +# Generated by Django 3.2.25 on 2025-01-02 08:38 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("db_report", "0008_auto_20241129_1754"), + ] + + operations = [ + migrations.CreateModel( + name="SqlserverFullBackupInfoReport", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("creator", models.CharField(max_length=64, verbose_name="创建人")), + ("create_at", models.DateTimeField(auto_now_add=True, verbose_name="创建时间")), + ("updater", models.CharField(max_length=64, verbose_name="修改人")), + ("update_at", models.DateTimeField(auto_now=True, verbose_name="更新时间")), + ("bk_biz_id", models.IntegerField(default=0, help_text="业务的 cmdb id")), + ("bk_cloud_id", models.IntegerField(default=0, help_text="云区域 id")), + ("status", models.BooleanField(default=True, help_text="巡检结果状态, 默认正常")), + ("msg", models.TextField(default="", help_text="备注信息")), + ("cluster", models.CharField(default="", max_length=255, verbose_name="集群名称")), + ( + "cluster_type", + models.CharField( + choices=[ + ("tendbsingle", "MySQL单节点集群"), + ("tendbha", "MySQL高可用集群"), + ("tendbcluster", "TendbCluster集群"), + ("tbinlogdumper", "TBinlogDumper"), + ("redis", "Redis"), + ("PredixyRedisCluster", "RedisCluster集群"), + ("PredixyTendisplusCluster", "Tendisplus存储版集群"), + ("TwemproxyRedisInstance", "TendisCache集群"), + ("TwemproxyTendisSSDInstance", "TendisSSD集群"), + ("TwemproxyTendisplusInstance", "Tendis存储版集群"), + ("RedisInstance", "RedisCache主从版"), + ("TendisSSDInstance", "TendisSSD主从版"), + ("TendisplusInstance", "Tendisplus主从版"), + ("RedisCluster", "RedisCluster集群"), + ("TendisplusCluster", "TendisplusCluster集群"), + ("TendisplusInstance", "Tendisplus存储版集群"), + ("RedisInstance", "TendisCache集群"), + ("TendisSSDInstance", "TendisSSD集群"), + ("es", "ES集群"), + ("kafka", "Kafka集群"), + ("hdfs", "Hdfs集群"), + ("influxdb", "Influxdb实例"), + ("pulsar", "Pulsar集群"), + ("doris", "Doris集群"), + ("vm", "vm集群"), + ("dbmon", "redis监控"), + ("MongoReplicaSet", "Mongo副本集"), + ("MongoShardedCluster", "Mongo分片集群"), + ("riak", "Riak集群"), + ("sqlserver_single", "sqlserver单节点版"), + ("sqlserver_ha", "sqlserver主从版"), + ], + default="", + max_length=64, + verbose_name="集群类型", + ), + ), + ], + options={ + "abstract": False, + }, + ), + migrations.CreateModel( + name="SqlserverLogBackupInfoReport", + fields=[ + ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("creator", models.CharField(max_length=64, verbose_name="创建人")), + ("create_at", models.DateTimeField(auto_now_add=True, verbose_name="创建时间")), + ("updater", models.CharField(max_length=64, verbose_name="修改人")), + ("update_at", models.DateTimeField(auto_now=True, verbose_name="更新时间")), + ("bk_biz_id", models.IntegerField(default=0, help_text="业务的 cmdb id")), + ("bk_cloud_id", models.IntegerField(default=0, help_text="云区域 id")), + ("status", models.BooleanField(default=True, help_text="巡检结果状态, 默认正常")), + ("msg", models.TextField(default="", help_text="备注信息")), + ("cluster", models.CharField(default="", max_length=255, verbose_name="集群名称")), + ( + "cluster_type", + models.CharField( + choices=[ + ("tendbsingle", "MySQL单节点集群"), + ("tendbha", "MySQL高可用集群"), + ("tendbcluster", "TendbCluster集群"), + ("tbinlogdumper", "TBinlogDumper"), + ("redis", "Redis"), + ("PredixyRedisCluster", "RedisCluster集群"), + ("PredixyTendisplusCluster", "Tendisplus存储版集群"), + ("TwemproxyRedisInstance", "TendisCache集群"), + ("TwemproxyTendisSSDInstance", "TendisSSD集群"), + ("TwemproxyTendisplusInstance", "Tendis存储版集群"), + ("RedisInstance", "RedisCache主从版"), + ("TendisSSDInstance", "TendisSSD主从版"), + ("TendisplusInstance", "Tendisplus主从版"), + ("RedisCluster", "RedisCluster集群"), + ("TendisplusCluster", "TendisplusCluster集群"), + ("TendisplusInstance", "Tendisplus存储版集群"), + ("RedisInstance", "TendisCache集群"), + ("TendisSSDInstance", "TendisSSD集群"), + ("es", "ES集群"), + ("kafka", "Kafka集群"), + ("hdfs", "Hdfs集群"), + ("influxdb", "Influxdb实例"), + ("pulsar", "Pulsar集群"), + ("doris", "Doris集群"), + ("vm", "vm集群"), + ("dbmon", "redis监控"), + ("MongoReplicaSet", "Mongo副本集"), + ("MongoShardedCluster", "Mongo分片集群"), + ("riak", "Riak集群"), + ("sqlserver_single", "sqlserver单节点版"), + ("sqlserver_ha", "sqlserver主从版"), + ], + default="", + max_length=64, + verbose_name="集群类型", + ), + ), + ], + options={ + "abstract": False, + }, + ), + ] diff --git a/dbm-ui/backend/db_report/models/__init__.py b/dbm-ui/backend/db_report/models/__init__.py index e1d3032db8..c73918491c 100644 --- a/dbm-ui/backend/db_report/models/__init__.py +++ b/dbm-ui/backend/db_report/models/__init__.py @@ -19,4 +19,6 @@ SqlserverCheckLinkServerReport, SqlserverCheckSysJobStatuReport, SqlserverCheckUserSyncReport, + SqlserverFullBackupInfoReport, + SqlserverLogBackupInfoReport, ) diff --git a/dbm-ui/backend/db_report/models/sqlserver_check_report.py b/dbm-ui/backend/db_report/models/sqlserver_check_report.py index 424b810ad5..18ac43093a 100644 --- a/dbm-ui/backend/db_report/models/sqlserver_check_report.py +++ b/dbm-ui/backend/db_report/models/sqlserver_check_report.py @@ -65,3 +65,25 @@ class SqlserverCheckLinkServerReport(BaseSqlserverReportABS): """ is_link_server_inconsistent = models.BooleanField(default=False, verbose_name=_("业务LinkServer数量是否不一致")) + + +class SqlserverFullBackupInfoReport(BaseReportABS): + """ + 全量备份文件巡检表 + """ + + cluster = models.CharField(max_length=255, default="", verbose_name=_("集群名称")) + cluster_type = models.CharField( + max_length=64, choices=ClusterType.get_choices(), default="", verbose_name=_("集群类型") + ) + + +class SqlserverLogBackupInfoReport(BaseReportABS): + """ + 增量备份文件巡检表 + """ + + cluster = models.CharField(max_length=255, default="", verbose_name=_("集群名称")) + cluster_type = models.CharField( + max_length=64, choices=ClusterType.get_choices(), default="", verbose_name=_("集群类型") + ) diff --git a/dbm-ui/backend/flow/utils/sqlserver/sqlserver_db_function.py b/dbm-ui/backend/flow/utils/sqlserver/sqlserver_db_function.py index 3ed7bb2f4a..ed6ed2cc7d 100644 --- a/dbm-ui/backend/flow/utils/sqlserver/sqlserver_db_function.py +++ b/dbm-ui/backend/flow/utils/sqlserver/sqlserver_db_function.py @@ -17,7 +17,7 @@ from django.db.models import QuerySet from backend.components import DRSApi -from backend.db_meta.enums import ClusterType, InstanceRole +from backend.db_meta.enums import ClusterType, InstanceRole, InstanceStatus from backend.db_meta.models import Cluster, StorageInstance from backend.db_meta.models.storage_set_dtl import SqlserverClusterSyncMode from backend.flow.consts import ( @@ -912,3 +912,44 @@ def exec_resume_sp(slave_instances: List[StorageInstance], master_host: str, mas if ret[0]["error_msg"]: raise Exception(f"Sys_AutoSwitch_Resume exec failed: {ret[0]['error_msg']}") return True + + +def get_backup_info(cluster: Cluster, start_time: str, end_time: str, backup_type: int): + """ + 根据传入时间区间，返回集群所有实例的本地备份信息 + @param cluster: 集群 + @param start_time: 查询起始日期， 格式 xxxx-xx-xx + @param end_time: 查询截止日期， 格式 xxxx-xx-xx + @param backup_type: 备份类型 + """ + backup_infos = [] + err_msg = "" + instances = cluster.storageinstance_set.filter(status=InstanceStatus.RUNNING) + if not instances: + return [], "the cluster is not running instance" + + select_sql = f"""SELECT [BACKUP_ID] + [DBNAME] +,[FILENAME] +,[TYPE] +,[STARTTIME] +,[ENDTIME] +FROM [Monitor].[dbo].[BACKUP_TRACE](NOLOCK) where + STARTTIME between '{start_time}' and '{end_time}' and TYPE = {backup_type} +""" + + ret = DRSApi.sqlserver_rpc( + { + "bk_cloud_id": cluster.bk_cloud_id, + "addresses": [i.ip_port for i in instances], + "cmds": [select_sql], + "force": False, + } + ) + for info in ret: + if info["error_msg"]: + err_msg += f"{info['address']} error: {info['error_msg']}\n" + continue + backup_infos.extend(info["cmd_results"][0]["table_data"]) + + return backup_infos, err_msg