From d6e57d82d81bd185b8abe148ce29e726e993975a Mon Sep 17 00:00:00 2001 From: Gallardot Date: Wed, 31 Jan 2024 11:02:48 +0800 Subject: [PATCH] [Improvement][HTTP] support custom rendering of http body (#15531) Signed-off-by: Gallardot Co-authored-by: Eric Gao --- .../plugin/task/http/HttpParameters.java | 13 ++++++++ .../plugin/task/http/HttpTask.java | 27 ++++++++++----- .../plugin/task/http/HttpTaskTest.java | 33 +++++++++++++++---- .../src/locales/en_US/project.ts | 3 ++ .../src/locales/zh_CN/project.ts | 2 ++ .../task/components/node/fields/use-http.ts | 9 +++++ .../task/components/node/format-data.ts | 1 + .../task/components/node/tasks/use-http.ts | 1 + .../projects/task/components/node/types.ts | 1 + 9 files changed, 74 insertions(+), 16 deletions(-) diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpParameters.java index 73794819dc5f..83dcb78c1851 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpParameters.java @@ -45,6 +45,11 @@ public class HttpParameters extends AbstractParameters { */ private List httpParams; + /** + * httpBody + */ + private String httpBody; + /** * httpCheckCondition */ @@ -132,4 +137,12 @@ public int getSocketTimeout() { public void setSocketTimeout(int socketTimeout) { this.socketTimeout = socketTimeout; } + + public String getHttpBody() { + return httpBody; + } + + public void setHttpBody(String httpBody) { + this.httpBody = httpBody; + } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java index 8d696a19b104..bdbcd8d6a51f 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/main/java/org/apache/dolphinscheduler/plugin/task/http/HttpTask.java @@ -17,8 +17,6 @@ package org.apache.dolphinscheduler.plugin.task.http; -import static org.apache.dolphinscheduler.plugin.task.http.HttpTaskConstants.APPLICATION_JSON; - import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; @@ -32,7 +30,6 @@ import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.io.Charsets; import org.apache.commons.lang3.StringUtils; import org.apache.http.HttpEntity; import org.apache.http.ParseException; @@ -40,6 +37,7 @@ import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.client.methods.RequestBuilder; +import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; @@ -145,7 +143,9 @@ protected CloseableHttpResponse sendRequest(CloseableHttpClient client) throws I httpPropertyList.add(JSONUtils.parseObject(params, HttpProperty.class)); } } - addRequestParams(builder, httpPropertyList); + String httpBody = ParameterUtils.convertParameterPlaceholders(httpParameters.getHttpBody(), + ParameterUtils.convert(paramsMap)); + addRequestParams(builder, httpPropertyList, httpBody); String requestUrl = ParameterUtils.convertParameterPlaceholders(httpParameters.getUrl(), ParameterUtils.convert(paramsMap)); HttpUriRequest request = builder.setUri(requestUrl).build(); @@ -247,7 +247,14 @@ protected void appendMessage(String message) { * @param builder buidler * @param httpPropertyList http property list */ - protected void addRequestParams(RequestBuilder builder, List httpPropertyList) { + protected void addRequestParams(RequestBuilder builder, List httpPropertyList, String httpBody) { + if (StringUtils.isNotEmpty(httpBody)) { + builder.setEntity(new StringEntity( + httpBody, + ContentType.create(ContentType.APPLICATION_JSON.getMimeType(), + StandardCharsets.UTF_8))); + } + if (CollectionUtils.isNotEmpty(httpPropertyList)) { ObjectNode jsonParam = JSONUtils.createObjectNode(); for (HttpProperty property : httpPropertyList) { @@ -259,10 +266,12 @@ protected void addRequestParams(RequestBuilder builder, List httpP } } } - StringEntity postingString = new StringEntity(jsonParam.toString(), Charsets.UTF_8); - postingString.setContentEncoding(StandardCharsets.UTF_8.name()); - postingString.setContentType(APPLICATION_JSON); - builder.setEntity(postingString); + if (builder.getEntity() == null) { + builder.setEntity(new StringEntity( + jsonParam.toString(), + ContentType.create(ContentType.APPLICATION_JSON.getMimeType(), + StandardCharsets.UTF_8))); + } } } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/test/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/test/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskTest.java index ec908364ad78..4c8ace158c44 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/test/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/src/test/java/org/apache/dolphinscheduler/plugin/task/http/HttpTaskTest.java @@ -150,13 +150,28 @@ public void testHandleWithHttpBodyParams() throws Exception { prepareParamsMap.put("day", "20220812"); // The MockWebServer will return the request body as response body directly // So we just need to check if the response body contains string "20220812" - HttpTask httpTask = generateHttpTask(MOCK_DISPATCH_PATH_REQ_BODY_TO_RES_BODY, HttpMethod.POST, + HttpTask httpTask = generateHttpTask(MOCK_DISPATCH_PATH_REQ_BODY_TO_RES_BODY, HttpMethod.POST, null, httpParams, prepareParamsMap, HttpCheckCondition.BODY_CONTAINS, "20220812", HttpStatus.SC_OK, ""); httpTask.handle(null); Assertions.assertEquals(EXIT_CODE_SUCCESS, httpTask.getExitStatusCode()); } + @Test + public void testHandleWithHttpBody() throws Exception { + String httpBody = "{\"day\": ${day}}"; + + Map prepareParamsMap = new HashMap<>(); + prepareParamsMap.put("day", "20220812"); + // The MockWebServer will return the request body as response body directly + // So we just need to check if the response body contains string "20220812" + HttpTask httpTask = generateHttpTask(MOCK_DISPATCH_PATH_REQ_BODY_TO_RES_BODY, HttpMethod.POST, httpBody, + null, prepareParamsMap, HttpCheckCondition.BODY_CONTAINS, "20220812", + HttpStatus.SC_OK, ""); + httpTask.handle(null); + Assertions.assertEquals(EXIT_CODE_SUCCESS, httpTask.getExitStatusCode()); + } + @Test public void testHandleWithHttpParameterParams() throws Exception { List httpParams = new ArrayList<>(); @@ -170,7 +185,7 @@ public void testHandleWithHttpParameterParams() throws Exception { prepareParamsMap.put("day", "20220812"); // The MockWebServer will return the request parameter as response body directly // So we just need to check if the response body contains string "20220812" - HttpTask httpTask = generateHttpTask(MOCK_DISPATCH_PATH_REQ_PARAMS_TO_RES_BODY, HttpMethod.POST, + HttpTask httpTask = generateHttpTask(MOCK_DISPATCH_PATH_REQ_PARAMS_TO_RES_BODY, HttpMethod.POST, null, httpParams, prepareParamsMap, HttpCheckCondition.BODY_CONTAINS, "20220812", HttpStatus.SC_OK, ""); httpTask.handle(null); @@ -203,22 +218,24 @@ private String withMockWebServer(String path, int actualResponseCode, } private HttpTask generateHttpTask(HttpMethod httpMethod, int actualResponseCode) throws IOException { - return generateHttpTask("/test", httpMethod, null, null, + return generateHttpTask("/test", httpMethod, null, null, null, HttpCheckCondition.STATUS_CODE_DEFAULT, "", actualResponseCode, ""); } private HttpTask generateHttpTask(HttpMethod httpMethod, HttpCheckCondition httpCheckConditionType, String condition, int actualResponseCode, String actualResponseBody) throws IOException { - return generateHttpTask("/test", httpMethod, null, null, + return generateHttpTask("/test", httpMethod, null, null, null, httpCheckConditionType, condition, actualResponseCode, actualResponseBody); } - private HttpTask generateHttpTask(String mockPath, HttpMethod httpMethod, List httpParams, + private HttpTask generateHttpTask(String mockPath, HttpMethod httpMethod, String httpBody, + List httpParams, Map prepareParamsMap, HttpCheckCondition httpCheckConditionType, String condition, int actualResponseCode, String actualResponseBody) throws IOException { String url = withMockWebServer(mockPath, actualResponseCode, actualResponseBody); - String paramData = generateHttpParameters(url, httpMethod, httpParams, httpCheckConditionType, condition); + String paramData = + generateHttpParameters(url, httpMethod, httpBody, httpParams, httpCheckConditionType, condition); return generateHttpTaskFromParamData(paramData, prepareParamsMap); } @@ -240,13 +257,15 @@ private HttpTask generateHttpTaskFromParamData(String paramData, Map httpParams, + private String generateHttpParameters(String url, HttpMethod httpMethod, String httpBody, + List httpParams, HttpCheckCondition httpCheckConditionType, String condition) throws JsonProcessingException { ObjectMapper mapper = new ObjectMapper(); HttpParameters httpParameters = new HttpParameters(); httpParameters.setUrl(url); httpParameters.setHttpMethod(httpMethod); + httpParameters.setHttpBody(httpBody); httpParameters.setHttpCheckCondition(httpCheckConditionType); httpParameters.setCondition(condition); httpParameters.setConnectTimeout(10000); diff --git a/dolphinscheduler-ui/src/locales/en_US/project.ts b/dolphinscheduler-ui/src/locales/en_US/project.ts index ee33434ad2eb..d498a793005c 100644 --- a/dolphinscheduler-ui/src/locales/en_US/project.ts +++ b/dolphinscheduler-ui/src/locales/en_US/project.ts @@ -493,6 +493,9 @@ export default { http_url_validator: 'The request address must contain HTTP or HTTPS', http_method: 'Http Method', http_parameters: 'Http Parameters', + http_body: 'Http Body', + http_body_tips: + 'Please fill in the http body, if filled, http parameters in the body type will be ignored', http_check_condition: 'Http Check Condition', http_condition: 'Http Condition', http_condition_tips: 'Please Enter Http Condition', diff --git a/dolphinscheduler-ui/src/locales/zh_CN/project.ts b/dolphinscheduler-ui/src/locales/zh_CN/project.ts index 1470c6154ea6..ff525ed35bb8 100644 --- a/dolphinscheduler-ui/src/locales/zh_CN/project.ts +++ b/dolphinscheduler-ui/src/locales/zh_CN/project.ts @@ -483,6 +483,8 @@ export default { http_url_validator: '请求地址需包含http或者https', http_method: '请求类型', http_parameters: '请求参数', + http_body: '请求Body', + http_body_tips: '请填写http body,如若填写将忽略请求参数中的body类型参数', http_check_condition: '校验条件', http_condition: '校验内容', http_condition_tips: '请填写校验内容', diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-http.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-http.ts index 9e3a9af921d2..e01183dfe8fd 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-http.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-http.ts @@ -126,6 +126,15 @@ export function useHttp(model: { [field: string]: any }): IJsonItem[] { } ] }, + { + type: 'input', + field: 'httpBody', + name: t('project.node.http_body'), + props: { + type: 'textarea', + placeholder: t('project.node.http_body_tips') + } + }, { type: 'select', field: 'httpCheckCondition', diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts index 31398f98e6a2..e83dfa1fefaf 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts @@ -85,6 +85,7 @@ export function formatParams(data: INodeData): { } if (data.taskType === 'HTTP') { taskParams.httpMethod = data.httpMethod + taskParams.httpBody = data.httpBody taskParams.httpCheckCondition = data.httpCheckCondition taskParams.httpParams = data.httpParams taskParams.url = data.url diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-http.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-http.ts index a1fc52757aad..731929cb4507 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-http.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-http.ts @@ -44,6 +44,7 @@ export function useHttp({ delayTime: 0, timeout: 30, httpMethod: 'GET', + httpBody: '', httpCheckCondition: 'STATUS_CODE_DEFAULT', httpParams: [], url: '', diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts index 86eec11682a8..ebbe89bcf2c9 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts @@ -286,6 +286,7 @@ interface ITaskParams { mainArgs?: string others?: string httpMethod?: string + httpBody?: string httpCheckCondition?: string httpParams?: [] url?: string