Python3.11.10版本及相关依赖包升级

.github/workflows/flake8_and_black.yml

@@ -16,10 +16,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
-    - name: Set up Python 3.6
+    - name: Set up Python 3.11
       uses: actions/setup-python@v2
       with:
-        python-version: 3.6
+        python-version: 3.11
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

.github/workflows/unittest.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       max-parallel: 4
       matrix:
-        python-version: [3.6]
+        python-version: [3.11]
 
     steps:
     - uses: actions/checkout@v2

.pre-commit-config.yaml

@@ -8,9 +8,9 @@ repos:
     rev: 22.8.0
     hooks:
     - id: black
-      language_version: python3.6
+      language_version: python3.11.10
   - repo: https://github.com/pycqa/isort
-    rev: 5.6.4
+    rev: 5.13.2
     hooks:
       - id: isort
         args: ["--profile", "black", "--filter-files"]

api/http.py

@@ -11,11 +11,11 @@
 specific language governing permissions and limitations under the License.
 """
 
-from django.utils.translation import ugettext_lazy as _
 import logging
 
-import requests
 import curlify
+import requests
+from django.utils.translation import gettext_lazy as _
 
 logger = logging.getLogger("component")
 
@@ -28,29 +28,67 @@ def _gen_header():
 
 
 def _http_request(
-    method, url, headers=None, data=None, verify=False, cert=None, timeout=None, cookies=None,
+    method,
+    url,
+    headers=None,
+    data=None,
+    verify=False,
+    cert=None,
+    timeout=None,
+    cookies=None,
 ):
     resp = requests.Response()
     request_id = None
 
     try:
         if method == "GET":
             resp = requests.get(
-                url=url, headers=headers, params=data, verify=verify, cert=cert, timeout=timeout, cookies=cookies,
+                url=url,
+                headers=headers,
+                params=data,
+                verify=verify,
+                cert=cert,
+                timeout=timeout,
+                cookies=cookies,
             )
         elif method == "HEAD":
-            resp = requests.head(url=url, headers=headers, verify=verify, cert=cert, timeout=timeout, cookies=cookies,)
+            resp = requests.head(
+                url=url,
+                headers=headers,
+                verify=verify,
+                cert=cert,
+                timeout=timeout,
+                cookies=cookies,
+            )
         elif method == "POST":
             resp = requests.post(
-                url=url, headers=headers, json=data, verify=verify, cert=cert, timeout=timeout, cookies=cookies,
+                url=url,
+                headers=headers,
+                json=data,
+                verify=verify,
+                cert=cert,
+                timeout=timeout,
+                cookies=cookies,
             )
         elif method == "DELETE":
             resp = requests.delete(
-                url=url, headers=headers, json=data, verify=verify, cert=cert, timeout=timeout, cookies=cookies,
+                url=url,
+                headers=headers,
+                json=data,
+                verify=verify,
+                cert=cert,
+                timeout=timeout,
+                cookies=cookies,
             )
         elif method == "PUT":
             resp = requests.put(
-                url=url, headers=headers, json=data, verify=verify, cert=cert, timeout=timeout, cookies=cookies,
+                url=url,
+                headers=headers,
+                json=data,
+                verify=verify,
+                cert=cert,
+                timeout=timeout,
+                cookies=cookies,
             )
         else:
             message = _(f"非法请求: 请求不是合法的HTTP Method: {method} | api http")
@@ -105,31 +143,54 @@ def _http_request(
             resp.request = requests.Request(method, url, headers=headers, data=data, cookies=cookies).prepare()
 
         logger.debug(
-            "the request_id: `%s`. curl: `%s`", request_id, curlify.to_curl(resp.request, verify=False),
+            "the request_id: `%s`. curl: `%s`",
+            request_id,
+            curlify.to_curl(resp.request, verify=False),
         )
 
 
 def get(url, data, headers=None, verify=False, cert=None, timeout=None, cookies=None):
     if not headers:
         headers = _gen_header()
     return _http_request(
-        method="GET", url=url, headers=headers, data=data, verify=verify, cert=cert, timeout=timeout, cookies=cookies,
+        method="GET",
+        url=url,
+        headers=headers,
+        data=data,
+        verify=verify,
+        cert=cert,
+        timeout=timeout,
+        cookies=cookies,
     )
 
 
 def post(url, data, headers=None, verify=False, cert=None, timeout=None, cookies=None):
     if not headers:
         headers = _gen_header()
     return _http_request(
-        method="POST", url=url, headers=headers, data=data, verify=verify, cert=cert, timeout=timeout, cookies=cookies,
+        method="POST",
+        url=url,
+        headers=headers,
+        data=data,
+        verify=verify,
+        cert=cert,
+        timeout=timeout,
+        cookies=cookies,
     )
 
 
 def put(url, data, headers=None, verify=False, cert=None, timeout=None, cookies=None):
     if not headers:
         headers = _gen_header()
     return _http_request(
-        method="PUT", url=url, headers=headers, data=data, verify=verify, cert=cert, timeout=timeout, cookies=cookies,
+        method="PUT",
+        url=url,
+        headers=headers,
+        data=data,
+        verify=verify,
+        cert=cert,
+        timeout=timeout,
+        cookies=cookies,
     )
 
 

api/utils/request.py

@@ -12,7 +12,7 @@
 """
 import logging
 
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 from gcloud.conf import settings
 from gcloud.exceptions import ApiRequestError

api/utils/thread.py

@@ -64,7 +64,7 @@ class ThreadPool(_ThreadPool):
 
     @staticmethod
     def get_func_with_local(func):
-        tz = timezone.get_current_timezone().zone
+        tz = timezone.get_current_timezone()
         lang = translation.get_language()
         items = [item for item in local]
         return partial(run_func_with_local, items, tz, lang, func)

config/default.py

@@ -21,7 +21,9 @@
 from blueapps.conf.default_settings import *  # noqa
 from blueapps.conf.log import get_logging_config_dict
 from blueapps.opentelemetry.utils import inject_logging_trace_info
-from django.utils.translation import ugettext_lazy as _
+from django.db.backends.mysql.features import DatabaseFeatures
+from django.utils.functional import cached_property
+from django.utils.translation import gettext_lazy as _
 from pipeline.celery.queues import ScalableQueues
 
 import env
@@ -893,3 +895,19 @@ def check_engine_admin_permission(request, *args, **kwargs):
 if "BKAPP_SOPS_BROKER_URL" in os.environ:
     BROKER_URL = os.getenv("BKAPP_SOPS_BROKER_URL")
     print(f"BROKER_URL: {BROKER_URL}")
+
+
+# 定义一个补丁来兼容 MySQL 5.7
+
+
+class PatchFeatures:
+    @cached_property
+    def minimum_database_version(self):
+        if self.connection.mysql_is_mariadb:
+            return (10, 4)
+        else:
+            return (5, 7)
+
+
+# 将补丁应用到 DatabaseFeatures 中
+DatabaseFeatures.minimum_database_version = PatchFeatures.minimum_database_version

config/urls_custom.py

@@ -10,37 +10,36 @@
 an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 specific language governing permissions and limitations under the License.
 """
-from rest_framework import permissions
-from drf_yasg.views import get_schema_view
-from drf_yasg import openapi
-from django.conf.urls import include, url
 from django.conf import settings
-
+from django.urls import include, re_path
+from drf_yasg import openapi
+from drf_yasg.views import get_schema_view
+from rest_framework import permissions
 
 # 用户自定义 urlconf
 urlpatterns_custom = [
-    url(r"^", include("gcloud.core.urls")),
-    url(r"^", include("gcloud.resources.urls")),
-    url(r"^apigw/", include("gcloud.apigw.urls")),
-    url(r"^common_template/", include("gcloud.common_template.urls")),
-    url(r"^template/", include("gcloud.tasktmpl3.urls")),
-    url(r"^template/", include("gcloud.template_base.urls")),
-    url(r"^taskflow/", include("gcloud.taskflow3.urls")),
-    url(r"^appmaker/", include("gcloud.contrib.appmaker.urls")),
-    url(r"^collection/", include("gcloud.contrib.collection.urls")),
-    url(r"^develop/", include("gcloud.contrib.develop.urls")),
-    url(r"^function/", include("gcloud.contrib.function.urls")),
-    url(r"^pipeline/", include("pipeline_plugins.base.urls")),
-    url(r"^pipeline/", include("pipeline_plugins.components.urls")),
-    url(r"^pipeline/", include("pipeline_plugins.variables.urls")),
-    url(r"^analysis/", include("gcloud.contrib.analysis.urls")),
-    url(r"^periodictask/", include("gcloud.periodictask.urls")),
-    url(r"^weixin/", include("weixin.urls")),
-    url(r"^weixin/login/", include("weixin.core.urls")),
-    url(r"^admin/", include("gcloud.contrib.admin.urls")),
-    url(r"^plugin_service/", include("plugin_service.urls")),
-    url(r"^mako_operations/", include("gcloud.mako_template_helper.urls")),
-    url(r"^engine_admin/", include("pipeline.contrib.engine_admin.urls")),
+    re_path(r"^", include("gcloud.core.urls")),
+    re_path(r"^", include("gcloud.resources.urls")),
+    re_path(r"^apigw/", include("gcloud.apigw.urls")),
+    re_path(r"^common_template/", include("gcloud.common_template.urls")),
+    re_path(r"^template/", include("gcloud.tasktmpl3.urls")),
+    re_path(r"^template/", include("gcloud.template_base.urls")),
+    re_path(r"^taskflow/", include("gcloud.taskflow3.urls")),
+    re_path(r"^appmaker/", include("gcloud.contrib.appmaker.urls")),
+    re_path(r"^collection/", include("gcloud.contrib.collection.urls")),
+    re_path(r"^develop/", include("gcloud.contrib.develop.urls")),
+    re_path(r"^function/", include("gcloud.contrib.function.urls")),
+    re_path(r"^pipeline/", include("pipeline_plugins.base.urls")),
+    re_path(r"^pipeline/", include("pipeline_plugins.components.urls")),
+    re_path(r"^pipeline/", include("pipeline_plugins.variables.urls")),
+    re_path(r"^analysis/", include("gcloud.contrib.analysis.urls")),
+    re_path(r"^periodictask/", include("gcloud.periodictask.urls")),
+    re_path(r"^weixin/", include("weixin.urls")),
+    re_path(r"^weixin/login/", include("weixin.core.urls")),
+    re_path(r"^admin/", include("gcloud.contrib.admin.urls")),
+    re_path(r"^plugin_service/", include("plugin_service.urls")),
+    re_path(r"^mako_operations/", include("gcloud.mako_template_helper.urls")),
+    re_path(r"^engine_admin/", include("pipeline.contrib.engine_admin.urls")),
 ]
 
 schema_view = get_schema_view(
@@ -56,7 +55,7 @@
 
 if settings.ENVIRONMENT != "production" or settings.ENABLE_SWAGGER_UI:
     urlpatterns_custom += [
-        url(r"^swagger(?P<format>\.json|\.yaml)$", schema_view.without_ui(cache_timeout=0), name="schema-json"),
-        url(r"^swagger/$", schema_view.with_ui("swagger", cache_timeout=0), name="schema-swagger-ui"),
-        url(r"^redoc/$", schema_view.with_ui("redoc", cache_timeout=0), name="schema-redoc"),
+        re_path(r"^swagger(?P<format>\.json|\.yaml)$", schema_view.without_ui(cache_timeout=0), name="schema-json"),
+        re_path(r"^swagger/$", schema_view.with_ui("swagger", cache_timeout=0), name="schema-swagger-ui"),
+        re_path(r"^redoc/$", schema_view.with_ui("redoc", cache_timeout=0), name="schema-redoc"),
     ]

docs/develop/dev_plugins.md

@@ -107,7 +107,7 @@ python manage.py create_new_plugin cc create_set v1.0 open pipeline_plugins site
 
 import logging
 
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 from pipeline.core.flow.activity import Service
 from pipeline.component_framework.component import Component

en_docs/develop/dev_plugins.md

@@ -107,7 +107,7 @@ Write plugin background logic in `plugin.py`. It mainly includes standard plugin
 
 import logging
 
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 from pipeline.core.flow.activity import Service
 from pipeline.component_framework.component import Component

files/bartenders/upload_module.py

@@ -10,11 +10,11 @@
 an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 specific language governing permissions and limitations under the License.
 """
-import re
 import logging
+import re
 import traceback
 
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 from .base import UploadRequestBartender
 
@@ -52,7 +52,9 @@ def process_request(self, request, *args, **kwargs):
         try:
             file_tag = self.manager.save(name=file_name, content=None, source_ip=source_ip, file_path=file_path)
         except Exception as e:
-            message = _(f"文件上传失败: 文件归档失败请重试, 如持续失败可联系管理员处理, {traceback.format_exc()} | process_request")
+            message = _(
+                f"文件上传失败: 文件归档失败请重试, 如持续失败可联系管理员处理, {traceback.format_exc()} | process_request"
+            )
             logger.error(message)
             return {"result": False, "message": message + f":{e}", "code": 500}
 

files/bartenders/utils.py

@@ -10,11 +10,11 @@
 an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 specific language governing permissions and limitations under the License.
 """
-import re
-import logging
 import hashlib
+import logging
+import re
 
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 logger = logging.getLogger("root")
 

files/factory.py

@@ -10,18 +10,19 @@
 an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 specific language governing permissions and limitations under the License.
 """
-from django.utils.translation import ugettext_lazy as _
+import logging
+
+from django.utils.translation import gettext_lazy as _
 
 from . import env
+from .bartenders.bk_repo import BKRepoBartender
 from .bartenders.job_repo import JobRepoBartender
+from .bartenders.nfs import HostNFSBartender
+from .bartenders.upload_module import UploadModuleBartender
 from .managers.bk_repo import BKRepoManager
 from .managers.job_repo import JobRepoManager
 from .managers.nfs import HostNFSManager
 from .managers.upload_module import UploadModuleManager
-from .bartenders.nfs import HostNFSBartender
-from .bartenders.upload_module import UploadModuleBartender
-from .bartenders.bk_repo import BKRepoBartender
-import logging
 
 logger = logging.getLogger("root")
 
@@ -31,7 +32,9 @@ class ManagerFactory(object):
     def get_manager(cls, manager_type):
         creator = getattr(cls, "_create_{}_manager".format(manager_type), None)
         if not creator or not callable(creator):
-            message = _(f"文件上传失败: 无法找到对应的FileManager: {manager_type}, 请重试, 如持续失败可联系管理员处理 | get_manager")
+            message = _(
+                f"文件上传失败: 无法找到对应的FileManager: {manager_type}, 请重试, 如持续失败可联系管理员处理 | get_manager"
+            )
             logger.error(message)
             raise LookupError(message)