Merge pull request #3338 from wuyzh39/issue_2986

perf: 优化local文件不存在时的报错信息 issue #2986

src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/constants/MetricsConstants.java

@@ -0,0 +1,48 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available.
+ *
+ * Copyright (C) 2021 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License.
+ *
+ * License for BK-JOB蓝鲸智云作业平台:
+ * --------------------------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+ * to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO
+ * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
+ * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+package com.tencent.bk.job.common.artifactory.constants;
+
+public class MetricsConstants {
+
+    /**
+     * 仅统计调用制品库 API的HTTP请求过程
+     */
+    public static final String METRICS_NAME_BKREPO_API_HTTP = "job.client.bkrepo.api.http";
+    /**
+     * 统计调用制品库 API整个过程，含反序列化
+     */
+    public static final String METRICS_NAME_BKREPO_API = "job.client.bkrepo.api";
+
+    public static final String TAG_KEY_API_NAME = "api_name";
+    public static final String TAG_KEY_STATUS = "status";
+
+    public static final String TAG_VALUE_OK = "ok";
+    public static final String TAG_VALUE_ERROR = "error";
+    public static final String TAG_VALUE_NONE = "none";
+    /**
+     * 用户请求参数错误：请求的节点路径在制品库中不存在
+     */
+    public static final String TAG_VALUE_CLIENT_ERROR_NODE_NOT_FOUND = "client.nodeNotFound";
+}

src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/sdk/ArtifactoryClient.java

@@ -26,6 +26,7 @@
 
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.tencent.bk.job.common.artifactory.constants.ArtifactoryInterfaceConsts;
+import com.tencent.bk.job.common.artifactory.constants.MetricsConstants;
 import com.tencent.bk.job.common.artifactory.model.dto.ArtifactoryResp;
 import com.tencent.bk.job.common.artifactory.model.dto.NodeDTO;
 import com.tencent.bk.job.common.artifactory.model.dto.PageData;
@@ -52,10 +53,10 @@
 import com.tencent.bk.job.common.artifactory.model.req.UploadGenericFileReq;
 import com.tencent.bk.job.common.constant.ErrorCode;
 import com.tencent.bk.job.common.constant.HttpMethodEnum;
+import com.tencent.bk.job.common.exception.HttpStatusException;
 import com.tencent.bk.job.common.exception.InternalException;
 import com.tencent.bk.job.common.exception.NotImplementedException;
 import com.tencent.bk.job.common.exception.ServiceException;
-import com.tencent.bk.job.common.metrics.CommonMetricNames;
 import com.tencent.bk.job.common.util.Base64Util;
 import com.tencent.bk.job.common.util.StringUtil;
 import com.tencent.bk.job.common.util.http.HttpHelper;
@@ -89,6 +90,8 @@
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
 
 @Slf4j
 public class ArtifactoryClient {
@@ -254,10 +257,10 @@ private <R> R getArtifactoryRespByReq(
         }
         String respStr;
         long start = System.nanoTime();
-        String status = "none";
+        AtomicReference<String> statusRef = new AtomicReference<>(MetricsConstants.TAG_VALUE_NONE);
         try {
-            HttpMetricUtil.setHttpMetricName(CommonMetricNames.BKREPO_API_HTTP);
-            HttpMetricUtil.addTagForCurrentMetric(Tag.of("api_name", urlTemplate));
+            HttpMetricUtil.setHttpMetricName(MetricsConstants.METRICS_NAME_BKREPO_API_HTTP);
+            HttpMetricUtil.addTagForCurrentMetric(Tag.of(MetricsConstants.TAG_KEY_API_NAME, urlTemplate));
             switch (method) {
                 case HttpGet.METHOD_NAME:
                     respStr = doHttpGet(url, reqBody, httpHelper);
@@ -284,13 +287,8 @@ private <R> R getArtifactoryRespByReq(
                 );
             }
             R result = JsonUtils.fromJson(respStr, typeReference);
-            try {
-                checkResult(result, method, url, reqStr, respStr);
-            } catch (Exception e) {
-                status = "error";
-                throw e;
-            }
-            status = "ok";
+            checkResult(result, method, url, reqStr, respStr);
+            statusRef.set(MetricsConstants.TAG_VALUE_OK);
             return result;
         } catch (Exception e) {
             String msg = MessageFormatter.arrayFormat(
@@ -302,13 +300,20 @@ private <R> R getArtifactoryRespByReq(
                 }
             ).getMessage();
             log.error(msg, e);
-            status = "error";
-            throw new InternalException("Fail to request ARTIFACTORY data", ErrorCode.ARTIFACTORY_API_DATA_ERROR);
+            statusRef.set(MetricsConstants.TAG_VALUE_ERROR);
+
+            // 特殊处理文件 NotFound 导致的 HttpStatusException
+            return convertException(statusRef, e);
+
         } finally {
             HttpMetricUtil.clearHttpMetric();
             long end = System.nanoTime();
             if (null != meterRegistry) {
-                meterRegistry.timer(CommonMetricNames.BKREPO_API, "api_name", urlTemplate, "status", status)
+                meterRegistry.timer(
+                        MetricsConstants.METRICS_NAME_BKREPO_API,
+                        MetricsConstants.TAG_KEY_API_NAME, urlTemplate,
+                        MetricsConstants.TAG_KEY_STATUS, statusRef.get()
+                    )
                     .record(end - start, TimeUnit.NANOSECONDS);
             }
         }
@@ -483,6 +488,7 @@ private List<String> parsePath(String filePath) {
     public NodeDTO getFileNode(String filePath) {
         List<String> pathList = parsePath(filePath);
         NodeDTO nodeDTO = queryNodeDetail(pathList.get(0), pathList.get(1), pathList.get(2));
+
         if (null == nodeDTO) {
             throw new InternalException(
                 "can not find node by filePath",
@@ -508,7 +514,7 @@ public Pair<InputStream, HttpRequestBase> getFileInputStream(String projectId, S
         url = getCompleteUrl(url);
         CloseableHttpResponse resp;
         try {
-            HttpMetricUtil.setHttpMetricName(CommonMetricNames.BKREPO_API_HTTP);
+            HttpMetricUtil.setHttpMetricName(MetricsConstants.METRICS_NAME_BKREPO_API_HTTP);
             HttpMetricUtil.addTagForCurrentMetric(Tag.of("api_name", "download:" + URL_DOWNLOAD_GENERIC_FILE));
             Pair<HttpRequestBase, CloseableHttpResponse> pair = longHttpHelper.getRawResp(false, url, getJsonHeaders());
             resp = pair.getRight();
@@ -556,7 +562,7 @@ private NodeDTO uploadGenericFileWithEntity(
         url = getCompleteUrl(url);
         String respStr;
         try {
-            HttpMetricUtil.setHttpMetricName(CommonMetricNames.BKREPO_API_HTTP);
+            HttpMetricUtil.setHttpMetricName(MetricsConstants.METRICS_NAME_BKREPO_API_HTTP);
             HttpMetricUtil.addTagForCurrentMetric(Tag.of("api_name", "upload:" + URL_UPLOAD_GENERIC_FILE));
 
             respStr = longHttpHelper.requestForSuccessResp(
@@ -662,4 +668,28 @@ public void shutdown() {
     private String getSimplifiedStrForLog(String rawStr) {
         return StringUtil.substring(rawStr, 20000);
     }
+
+    private <R> R convertException(AtomicReference<String> statusRef, Exception e) {
+        if (e instanceof HttpStatusException) {
+            String httpStatusExceptionRespStr = ((HttpStatusException) e).getRespBodyStr();
+            ArtifactoryResp<Object> artifactoryResp = JsonUtils.fromJson(httpStatusExceptionRespStr,
+                new TypeReference<ArtifactoryResp<Object>>() {
+                });
+            if (artifactoryResp != null
+                && artifactoryResp.getCode() == ArtifactoryInterfaceConsts.RESULT_CODE_NODE_NOT_FOUND) {
+                statusRef.set(MetricsConstants.TAG_VALUE_CLIENT_ERROR_NODE_NOT_FOUND);
+                throw new InternalException(
+                    artifactoryResp.getMessage(),
+                    ErrorCode.CAN_NOT_FIND_NODE_IN_ARTIFACTORY
+                );
+            } else {
+                throw new InternalException(
+                    "Fail to request ARTIFACTORY data",
+                    ErrorCode.ARTIFACTORY_API_DATA_ERROR
+                );
+            }
+        } else {
+            throw new InternalException("Fail to request ARTIFACTORY data", ErrorCode.ARTIFACTORY_API_DATA_ERROR);
+        }
+    }
 }

src/backend/commons/common-i18n/src/main/resources/i18n/exception/message.properties

@@ -177,7 +177,7 @@
 1244020=作业执行历史查询时间范围必须小于30天
 1244021=文件任务数量超过限制：{0}
 1244022=脚本任务目标服务器数量超过限制：{0}
-1244023=本地文件{0}在后台不存在
+1244023=本地文件{0}在后台不存在，可能过期被定时清除(默认过期时间为{1}天)，可重新上传后重试。若要持久化存储，请上传至业务服务器后，从服务器分发。
 1244024=保存文件到本地失败
 1244025=任务被丢弃
 1244026=非法的滚动策略

src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en.properties

@@ -177,7 +177,7 @@
 1244020=The job execution history query time range must be less than 30 days
 1244021=The number of file tasks exceeds the limit:{0}
 1244022=The number of script task target servers exceeds the limit: {0}
-1244023=The local file {0} not exists in backend
+1244023=The local file {0} not exists in backend, possibly because it was automatically deleted after expiration (default expiration time is {1} days), please upload it and try it again. To store it permanently, please upload it to the business server and distribute it from there.
 1244024=Fail to save file to local
 1244025=Task abandoned
 1244026=Invalid rolling strategy

src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en_US.properties

@@ -177,7 +177,7 @@
 1244020=The job execution history query time range must be less than 30 days
 1244021=The number of file tasks exceeds the limit:{0}
 1244022=The number of script task target servers exceeds the limit: {0}
-1244023=The local file {0} not exists in backend
+1244023=The local file {0} not exists in backend, possibly because it was automatically deleted after expiration (default expiration time is {1} days), please upload it and try it again. To store it permanently, please upload it to the business server and distribute it from there.
 1244024=Fail to save file to local
 1244025=Task abandoned
 1244026=Invalid rolling strategy

src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh.properties

@@ -177,7 +177,7 @@
 1244020=作业执行历史查询时间范围必须小于30天
 1244021=文件任务数量超过限制：{0}
 1244022=脚本任务目标服务器数量超过限制：{0}
-1244023=本地文件{0}在后台不存在
+1244023=本地文件{0}在后台不存在，可能过期被定时清除(默认过期时间为{1}天)，可重新上传后重试。若要持久化存储，请上传至业务服务器后，从服务器分发。
 1244024=保存文件到本地失败
 1244025=任务被丢弃
 1244026=非法的滚动策略

src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh_CN.properties

@@ -178,7 +178,7 @@
 1244020=作业执行历史查询时间范围必须小于30天
 1244021=文件任务数量超过限制：{0}
 1244022=脚本任务目标服务器数量超过限制：{0}
-1244023=本地文件{0}在后台不存在
+1244023=本地文件{0}在后台不存在，可能过期被定时清除(默认过期时间为{1}天)，可重新上传后重试。若要持久化存储，请上传至业务服务器后，从服务器分发。
 1244024=保存文件到本地失败
 1244025=任务被丢弃
 1244026=非法的滚动策略

src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/exception/handler/EsbExceptionControllerAdvice.java

@@ -110,7 +110,10 @@ ResponseEntity<?> handleServiceException(HttpServletRequest request, ServiceExce
     ResponseEntity<?> handleInternalException(HttpServletRequest request, InternalException ex) {
         String errorMsg = "Handle InternalException, uri: " + request.getRequestURI();
         log.error(errorMsg, ex);
-        return new ResponseEntity<>(EsbResp.buildCommonFailResp(ex.getErrorCode()), HttpStatus.OK);
+        return new ResponseEntity<>(
+            EsbResp.buildCommonFailResp(ex.getErrorCode(), ex.getErrorParams(), null),
+            HttpStatus.OK
+        );
     }
 
     @ExceptionHandler(PermissionDeniedException.class)

src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/HttpStatusException.java

@@ -36,11 +36,22 @@ public class HttpStatusException extends RuntimeException {
     private final String uri;
     private final int httpStatus;
     private final String reasonPhrase;
+    private final String respBodyStr;
 
     public HttpStatusException(String uri, int httpStatus, String reasonPhrase) {
         super("http status not ok, uri=" + uri + ", statusCode=" + httpStatus + ", reasonPhrase=" + reasonPhrase);
         this.uri = uri;
         this.httpStatus = httpStatus;
         this.reasonPhrase = reasonPhrase;
+        respBodyStr = null;
+    }
+
+    public HttpStatusException(String uri, int httpStatus, String reasonPhrase, String respBodyStr) {
+        super("http status not ok, uri=" + uri
+            + ", statusCode=" + httpStatus + ", reasonPhrase=" + reasonPhrase + ", respBodyStr=" + respBodyStr);
+        this.uri = uri;
+        this.httpStatus = httpStatus;
+        this.reasonPhrase = reasonPhrase;
+        this.respBodyStr = respBodyStr;
     }
 }

src/backend/commons/common/src/main/java/com/tencent/bk/job/common/metrics/CommonMetricNames.java

@@ -57,16 +57,6 @@ public class CommonMetricNames {
      */
     public static final String ESB_CMDB_API_HTTP = "job.client.cmdb.api.http";
 
-    /**
-     * 仅统计调用制品库 API的HTTP请求过程
-     */
-    public static final String BKREPO_API_HTTP = "job.client.bkrepo.api.http";
-    /**
-     * 统计调用制品库 API整个过程，含反序列化
-     */
-    public static final String BKREPO_API = "job.client.bkrepo.api";
-
-
     /**
      * 仅统计调用权限中心后台 API的HTTP请求过程
      */

src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/BaseHttpHelper.java

@@ -206,7 +206,12 @@ private HttpResponse execute(HttpRequestBase httpClientRequest,
                     respStr
                 );
                 if (throwExceptionWhenClientOrServerError) {
-                    throw new HttpStatusException(httpClientRequest.getURI().toString(), httpStatusCode, reasonPhrase);
+                    throw new HttpStatusException(
+                        httpClientRequest.getURI().toString(),
+                        httpStatusCode,
+                        reasonPhrase,
+                        respStr
+                    );
                 } else {
                     return new HttpResponse(httpStatusCode, respStr, httpResponse.getAllHeaders());
                 }

src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/LocalFileConfigForExecute.java

@@ -40,4 +40,7 @@ public class LocalFileConfigForExecute {
 
     @Value("${local-file.artifactory.download.concurrency:10}")
     private Integer downloadConcurrency;
+
+    @Value("${local-file.expire-days:7}")
+    private Integer expireDays;
 }

src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ArtifactoryLocalFileService.java

@@ -20,30 +20,29 @@ public class ArtifactoryLocalFileService {
     private final ArtifactoryConfig artifactoryConfig;
     private final LocalFileConfigForExecute localFileConfigForExecute;
     private final ArtifactoryClient artifactoryClient;
+    private final LocalFileConfigForExecute localFileConfig;
 
     @Autowired
     public ArtifactoryLocalFileService(
         ArtifactoryConfig artifactoryConfig,
         LocalFileConfigForExecute localFileConfigForExecute,
-        @Qualifier("jobArtifactoryClient") ArtifactoryClient artifactoryClient
+        @Qualifier("jobArtifactoryClient") ArtifactoryClient artifactoryClient,
+        LocalFileConfigForExecute localFileConfig
     ) {
         this.artifactoryConfig = artifactoryConfig;
         this.localFileConfigForExecute = localFileConfigForExecute;
         this.artifactoryClient = artifactoryClient;
+        this.localFileConfig = localFileConfig;
     }
 
     public FileDetailDTO getFileDetailFromArtifactory(String filePath) {
-        String fullPath = PathUtil.joinFilePath(
-            artifactoryConfig.getArtifactoryJobProject()
-                + "/" + localFileConfigForExecute.getLocalUploadRepo(),
-            filePath
-        );
-        NodeDTO nodeDTO = artifactoryClient.getFileNode(fullPath);
-        log.debug("nodeDTpwO={}", nodeDTO);
+        NodeDTO nodeDTO = getFileNodeAndHandleException(filePath);
+        log.debug("nodeDTO={}", nodeDTO);
         if (nodeDTO == null) {
             throw new InternalException(
+                "local file not found in artifactory",
                 ErrorCode.LOCAL_FILE_NOT_EXIST_IN_BACKEND,
-                new String[]{filePath}
+                new String[]{filePath, String.valueOf(localFileConfig.getExpireDays())}
             );
         }
         FileDetailDTO fileDetailDTO = new FileDetailDTO(filePath);
@@ -53,4 +52,28 @@ public FileDetailDTO getFileDetailFromArtifactory(String filePath) {
         return fileDetailDTO;
     }
 
+    private NodeDTO getFileNodeAndHandleException(String filePath) {
+        String fullPath = PathUtil.joinFilePath(
+            artifactoryConfig.getArtifactoryJobProject()
+                + "/" + localFileConfigForExecute.getLocalUploadRepo(),
+            filePath
+        );
+        NodeDTO nodeDTO;
+        try {
+            nodeDTO = artifactoryClient.getFileNode(fullPath);
+        } catch (InternalException e) {
+            if (e.getErrorCode() == ErrorCode.CAN_NOT_FIND_NODE_IN_ARTIFACTORY) {
+                log.error("[TransferLocalFile] transfer fail, local file {} not in artifactory", filePath);
+                throw new InternalException(
+                    "local file not found in artifactory",
+                    ErrorCode.LOCAL_FILE_NOT_EXIST_IN_BACKEND,
+                    new String[]{filePath, String.valueOf(localFileConfig.getExpireDays())}
+                );
+            } else {
+                throw e;
+            }
+        }
+        return nodeDTO;
+    }
+
 }