diff --git a/Dockerfile b/Dockerfile index bc68382dc..24ef29dba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,19 +10,23 @@ COPY --chown=tomcat lib/webapp/* /usr/local/tomcat/webapps/ROOT/WEB-INF/lib/ # # Compile custom class # FROM eclipse-temurin:17-jdk-jammy AS custom-code-builder -# # Copy dependencies -# COPY --from=ff-base /usr/local/tomcat/lib/ /usr/local/tomcat/lib/ -# COPY --from=ff-base /usr/local/tomcat/webapps/ROOT /usr/local/tomcat/webapps/ROOT +# Compile custom class +FROM eclipse-temurin:17-jdk-jammy AS custom-code-builder -# # Copy custom class -# COPY src/main/java /tmp/java -# RUN mkdir /tmp/classes && \ -# javac \ -# /tmp/java/org/frankframework/parameters/Parameter.java \ -# -classpath "/usr/local/tomcat/webapps/ROOT/WEB-INF/lib/*:/usr/local/tomcat/lib/*" \ -# -verbose -d /tmp/classes +# Copy dependencies +COPY --from=ff-base /usr/local/tomcat/lib/ /usr/local/tomcat/lib/ +COPY --from=ff-base /usr/local/tomcat/webapps/ROOT /usr/local/tomcat/webapps/ROOT + +# Copy custom class +COPY src/main/java /tmp/java +RUN mkdir /tmp/classes && \ + javac \ + /tmp/java/org/frankframework/http/HttpSenderCached.java \ + /tmp/java/org/frankframework/http/HttpSenderBaseCached.java \ + -classpath "/usr/local/tomcat/webapps/ROOT/WEB-INF/lib/*:/usr/local/tomcat/lib/*" \ + -verbose -d /tmp/classes -# FROM ff-base +FROM ff-base # Copy custom entrypoint script with added options COPY --chown=tomcat docker/entrypoint.sh /scripts/entrypoint.sh @@ -44,7 +48,7 @@ COPY --chown=tomcat src/main/resources/ /opt/frank/resources/ COPY --chown=tomcat src/test/testtool/ /opt/frank/testtool/ # # Copy compiled custom class -# COPY --from=custom-code-builder --chown=tomcat /tmp/classes/ /usr/local/tomcat/webapps/ROOT/WEB-INF/classes +COPY --from=custom-code-builder --chown=tomcat /tmp/classes/ /usr/local/tomcat/webapps/ROOT/WEB-INF/classes # Check if Frank! is still healthy HEALTHCHECK --interval=15s --timeout=5s --start-period=30s --retries=60 \ diff --git a/src/main/configurations/Translate/Configuration_GetRolTypenByUrl.xml b/src/main/configurations/Translate/Configuration_GetRolTypenByUrl.xml index efd1ae995..6eaa17299 100644 --- a/src/main/configurations/Translate/Configuration_GetRolTypenByUrl.xml +++ b/src/main/configurations/Translate/Configuration_GetRolTypenByUrl.xml @@ -25,7 +25,7 @@ name="GetRolTypenByUrlSender" getInputFromSessionKey="originalMessage" > - - - - - - - - + @@ -82,7 +82,7 @@ - - + diff --git a/src/main/configurations/Translate/Configuration_Zaken_GetZgwRolTypeByUrl.xml b/src/main/configurations/Translate/Configuration_Zaken_GetZgwRolTypeByUrl.xml index 49fa3f360..d156a5adf 100644 --- a/src/main/configurations/Translate/Configuration_Zaken_GetZgwRolTypeByUrl.xml +++ b/src/main/configurations/Translate/Configuration_Zaken_GetZgwRolTypeByUrl.xml @@ -27,7 +27,7 @@ - Expected message format:

+ *

GET methods expect a message looking like this

+ * 
+ *   param_name=param_value&another_param_name=another_param_value
+ *
+ *

POST AND PUT methods expect a message similar as GET, or looking like this

+ * 
+ *   param_name=param_value
+ *   another_param_name=another_param_value
+ *
+ * + * @ff.parameters Any parameters present are appended to the request (when method is GET as request-parameters, when method POST as body part) except the headersParams list, which are added as HTTP headers, and the urlParam header + * @ff.forward "<statusCode of the HTTP response>" default + * + * @author Niels Meijer + * @since 7.0 + */ +//TODO: Fix javadoc! + +public abstract class HttpSenderBaseCached extends HttpSessionBase implements HasPhysicalDestination, ISenderWithParameters, CanUseSharedResource { + + private static final String CONTEXT_KEY_STATUS_CODE = "Http.StatusCode"; + private static final String CONTEXT_KEY_REASON_PHRASE = "Http.ReasonPhrase"; + public static final String MESSAGE_ID_HEADER = "Message-Id"; + public static final String CORRELATION_ID_HEADER = "Correlation-Id"; + + private final @Getter String domain = "Http"; + + private @Setter String sharedResourceRef; + + private @Getter String url; + private @Getter String urlParam = "url"; + + public enum HttpMethod { + GET,POST,PUT,PATCH,DELETE,HEAD,REPORT; + } + private @Getter HttpMethod httpMethod = HttpMethod.GET; + + private @Getter String charSet = StreamUtil.DEFAULT_INPUT_STREAM_ENCODING; + private @Getter ContentType fullContentType = null; + private @Getter String contentType = null; + + private @Getter String headersParams=""; + private @Getter boolean xhtml=false; + private @Getter String styleSheetName=null; + private @Getter String resultStatusCodeSessionKey; + private @Getter String parametersToSkipWhenEmpty; + + private final boolean APPEND_MESSAGEID_HEADER = AppConstants.getInstance(getConfigurationClassLoader()).getBoolean("http.headers.messageid", true); + private final boolean APPEND_CORRELATIONID_HEADER = AppConstants.getInstance(getConfigurationClassLoader()).getBoolean("http.headers.correlationid", true); + + private TransformerPool transformerPool=null; + + protected Parameter urlParameter; + + protected URI staticUri; + + protected Set requestOrBodyParamsSet=new HashSet<>(); + protected Set headerParamsSet=new LinkedHashSet<>(); + protected Set parametersToSkipWhenEmptySet=new HashSet<>(); + + protected ParameterList paramList = null; + + private IbisCacheManager ibisCacheManager = IbisCacheManager.getInstance(); + private Cache etagConfigCache = new Cache( + "eTagStore", + 512, + MemoryStoreEvictionPolicy.fromString("LRU"), + false, + null, + true, + 0, + 0, + false, + 600, + null, + null, + 10000 + ); + + private Cache messageConfigCache = new Cache( + "messageCacheStore", + 512, + MemoryStoreEvictionPolicy.fromString("LRU"), + false, + null, + true, + 0, + 0, + false, + 600, + null, + null, + 10000 + ); + + private Ehcache etagCache; + private Ehcache messageCache; + + public Ehcache getEtagCache(){ + return etagCache; + } + + public Ehcache getMessageCache(){ + return messageCache; + } + + public void setCache(){ + etagCache = ibisCacheManager.getCache("eTagStore"); + messageCache = ibisCacheManager.getCache("messageCacheStore"); + if (etagCache == null){ + ibisCacheManager.addCache(etagConfigCache); + etagCache = ibisCacheManager.getCache("eTagStore"); + } + if (messageCache == null){ + ibisCacheManager.addCache(messageConfigCache); + messageCache = ibisCacheManager.getCache("messageCacheStore"); + } + } + + @Override + public void addParameter(Parameter p) { + if (paramList==null) { + paramList=new ParameterList(); + } + paramList.add(p); + } + + /** + * return the Parameters + */ + @Override + public ParameterList getParameterList() { + return paramList; + } + + @Override + public void configure() throws ConfigurationException { + if(StringUtils.isBlank(sharedResourceRef)) { + log.debug("configuring local HttpSession"); + super.configure(); + } + + if (paramList!=null) { + paramList.configure(); + + headersParams.concat(",If-None-Match"); + + if (StringUtils.isNotEmpty(getHeadersParams())) { + headerParamsSet.addAll(StringUtil.split(getHeadersParams())); + } + for (Parameter p: paramList) { + String paramName = p.getName(); + if (!headerParamsSet.contains(paramName)) { + requestOrBodyParamsSet.add(paramName); + } + } + + if (StringUtils.isNotEmpty(getUrlParam())) { + headerParamsSet.remove(getUrlParam()); + requestOrBodyParamsSet.remove(getUrlParam()); + urlParameter = paramList.findParameter(getUrlParam()); + } + + if (StringUtils.isNotEmpty(getParametersToSkipWhenEmpty())) { + if (getParametersToSkipWhenEmpty().equals("*")) { + parametersToSkipWhenEmptySet.addAll(headerParamsSet); + parametersToSkipWhenEmptySet.addAll(requestOrBodyParamsSet); + } else { + parametersToSkipWhenEmptySet.addAll(StringUtil.split(getParametersToSkipWhenEmpty())); + } + } + } + + if(StringUtils.isNotEmpty(getContentType())) { + fullContentType = ContentType.parse(getContentType()); + if(fullContentType.getCharset() == null) { + fullContentType = fullContentType.withCharset(getCharSet()); + } + } + + try { + if (urlParameter == null) { + if (StringUtils.isEmpty(getUrl())) { + throw new ConfigurationException(getLogPrefix()+"url must be specified, either as attribute, or as parameter"); + } + staticUri = getURI(getUrl()); + } + } catch (URISyntaxException e) { + throw new ConfigurationException("cannot interpret url ["+getUrl()+"]", e); + } + + if (StringUtils.isNotEmpty(getStyleSheetName())) { + try { + Resource stylesheet = Resource.getResource(this, getStyleSheetName()); + if (stylesheet == null) { + throw new ConfigurationException("cannot find stylesheet ["+getStyleSheetName()+"]"); + } + transformerPool = TransformerPool.getInstance(stylesheet); + } catch (IOException e) { + throw new ConfigurationException("cannot retrieve ["+ getStyleSheetName() + "]", e); + } catch (TransformerConfigurationException te) { + throw new ConfigurationException("got error creating transformer from file [" + getStyleSheetName() + "]", te); + } + } + + setCache(); + } + + @Override + public void open() throws SenderException { + try { + start(); + } catch (Exception e) { + throw new SenderException(getLogPrefix()+"unable to create HttpClient", e); + } + + if (transformerPool!=null) { + try { + transformerPool.open(); + } catch (Exception e) { + throw new SenderException(getLogPrefix()+"cannot start TransformerPool", e); + } + } + } + + @Override + public Class getObjectType() { + return HttpSession.class; + } + + @Override + public void start() { + if(StringUtils.isNotBlank(sharedResourceRef)) { + HttpSession session = getSharedResource(sharedResourceRef); + setHttpClient(session.getHttpClient()); + setHttpContext(session.getDefaultHttpClientContext()); + } else { + log.debug("starting local HttpSession"); + super.start(); + } + } + + @Override + public void close() { + if(StringUtils.isBlank(sharedResourceRef)) { + super.stop(); + } + + if (transformerPool!=null) { + transformerPool.close(); + } + } + + protected boolean appendParameters(boolean parametersAppended, StringBuilder path, ParameterValueList parameters) throws SenderException { + if (parameters != null) { + log.debug("appending [{}] parameters", parameters::size); + for(ParameterValue pv : parameters) { + if (requestOrBodyParamsSet.contains(pv.getName())) { + String value = pv.asStringValue(""); + if (StringUtils.isNotEmpty(value) || !parametersToSkipWhenEmptySet.contains(pv.getName())) { + try { + if (parametersAppended) { + path.append("&"); + } else { + path.append("?"); + parametersAppended = true; + } + + String parameterToAppend = pv.getDefinition().getName() +"="+ URLEncoder.encode(value, getCharSet()); + log.debug("appending parameter [{}]", parameterToAppend); + path.append(parameterToAppend); + } catch (UnsupportedEncodingException e) { + throw new SenderException(getLogPrefix()+"["+getCharSet()+"] encoding error. Failed to add parameter ["+pv.getDefinition().getName()+"]", e); + } + } + } + } + } + return parametersAppended; + } + + + /** + * Returns the true name of the class and not XsltPipe$$EnhancerBySpringCGLIB$$563e6b5d. + * {@link ClassUtils#nameOf(Object)} makes sure the original class will be used. + * + * @return className + name of the ISender + */ + protected String getLogPrefix() { + return ClassUtils.nameOf(this) + " "; + } + + /** + * Custom implementation to create a {@link HttpRequestBase HttpRequest} object. + * @param uri endpoint to send the message to + * @param message to be sent + * @param parameters ParameterValueList that contains all the senders parameters + * @param session PipeLineSession to retrieve or store data from, or NULL when not set + * @return a {@link HttpRequestBase HttpRequest} object + * @throws SenderException + */ + protected abstract HttpRequestBase getMethod(URI uri, Message message, ParameterValueList parameters, PipeLineSession session) throws SenderException; + + /** + * Custom implementation to extract the response and format it to a String result.
+ * It is important that the {@link HttpResponseHandler#getResponse() response} + * will be read or will be {@link HttpResponseHandler#close() closed}. + * @param responseHandler {@link HttpResponseHandler} that contains the response information + * @param session {@link PipeLineSession} which may be null + * @return a string that will be passed to the pipeline + */ + protected abstract Message extractResult(URI targetURI, HttpResponseHandler responseHandler, PipeLineSession session) throws SenderException, IOException; + + protected boolean validateResponseCode(int statusCode) { + boolean ok = false; + if (StringUtils.isNotEmpty(getResultStatusCodeSessionKey())) { + ok = true; + } else { + if (statusCode==200 || statusCode==201 || statusCode==202 || statusCode==204 || statusCode==206 || statusCode==304) { + ok = true; + } else { + if (isIgnoreRedirects() && (statusCode==HttpServletResponse.SC_MOVED_PERMANENTLY || statusCode==HttpServletResponse.SC_MOVED_TEMPORARILY || statusCode==HttpServletResponse.SC_TEMPORARY_REDIRECT)) { + ok = true; + } + } + } + return ok; + } + + @Override + public SenderResult sendMessage(Message message, PipeLineSession session) throws SenderException, TimeoutException { + ParameterValueList pvl = null; + try { + if (paramList !=null) { + pvl=paramList.getValues(message, session); + } + } catch (ParameterException e) { + throw new SenderException(getLogPrefix()+"Sender ["+getName()+"] caught exception evaluating parameters",e); + } + + URI targetUri; + final HttpRequestBase httpRequestBase; + try { + if (urlParameter != null) { + String url = pvl.get(getUrlParam()).asStringValue(); + targetUri = getURI(url); + } else { + targetUri = staticUri; + } + + // Resolve HeaderParameters + Map headersParamsMap = new HashMap<>(); + if (!headerParamsSet.isEmpty() && pvl!=null) { + log.debug("appending header parameters "+headersParams); + for (String paramName:headerParamsSet) { + ParameterValue paramValue = pvl.get(paramName); + if(paramValue != null) { + String value = paramValue.asStringValue(null); + if (StringUtils.isNotEmpty(value) || !parametersToSkipWhenEmptySet.contains(paramName)) { + headersParamsMap.put(paramName, value); + } + } + } + } + + net.sf.ehcache.Element cacheval = etagCache.get(targetUri.toString()); + + if (cacheval != null){ + headersParamsMap.put("If-None-Match", cacheval.getObjectValue().toString()); + } + + httpRequestBase = getMethod(targetUri, message, pvl, session); + if(httpRequestBase == null) + throw new MethodNotSupportedException("could not find implementation for method ["+getHttpMethod()+"]"); + + //Set all headers + if(session != null) { + if (APPEND_MESSAGEID_HEADER && StringUtils.isNotEmpty(session.getMessageId())) { + httpRequestBase.setHeader(MESSAGE_ID_HEADER, session.getMessageId()); + } + if (APPEND_CORRELATIONID_HEADER && StringUtils.isNotEmpty(session.getCorrelationId())) { + httpRequestBase.setHeader(CORRELATION_ID_HEADER, session.getCorrelationId()); + } + } + + for (String param: headersParamsMap.keySet()) { + httpRequestBase.setHeader(param, headersParamsMap.get(param)); + } + + log.info("configured httpclient for host [{}]", targetUri::getHost); + + } catch (Exception e) { + throw new SenderException(e); + } + + Message result; + int statusCode; + boolean success; + String reasonPhrase; + + TimeoutGuard tg = new TimeoutGuard(1+getTimeout()/1000, getName()) { + + @Override + protected void abort() { + httpRequestBase.abort(); + } + + }; + try { + log.debug("executing method [{}]", httpRequestBase::getRequestLine); + HttpResponse httpResponse = execute(targetUri, httpRequestBase, session); + log.debug("executed method"); + + HttpResponseHandler responseHandler = new HttpResponseHandler(httpResponse); + StatusLine statusline = httpResponse.getStatusLine(); + statusCode = statusline.getStatusCode(); + success = validateResponseCode(statusCode); + reasonPhrase = StringUtils.isNotEmpty(statusline.getReasonPhrase()) ? statusline.getReasonPhrase() : "HTTP status-code ["+statusCode+"]"; + + if (StringUtils.isNotEmpty(getResultStatusCodeSessionKey()) && session != null) { + session.put(getResultStatusCodeSessionKey(), Integer.toString(statusCode)); + } + + // Only give warnings for 4xx (client errors) and 5xx (server errors) + if (statusCode >= 400 && statusCode < 600) { + log.warn("status [{}]", statusline); + } else { + log.debug("status [{}]", statusCode); + } + + result = extractResult(targetUri, responseHandler, session); + + log.debug("retrieved result [{}]", result); + } catch (IOException e) { + httpRequestBase.abort(); + if (e instanceof SocketTimeoutException) { + throw new TimeoutException(e); + } + throw new SenderException(e); + } finally { + // By forcing the use of the HttpResponseHandler the resultStream + // will automatically be closed when it has been read. + // See HttpResponseHandler and ReleaseConnectionAfterReadInputStream. + // We cannot close the connection as the response might be kept + // in a sessionKey for later use in the pipeline. + // + // IMPORTANT: It is possible that poorly written implementations + // won't read or close the response. + // This will cause the connection to become stale. + + if (tg.cancel()) { + throw new TimeoutException(getLogPrefix()+"timeout of ["+getTimeout()+"] ms exceeded"); + } + } + + if (statusCode == -1){ + throw new SenderException("Failed to recover from exception"); + } + + if (isXhtml() && !Message.isEmpty(result)) { + // TODO: Streaming XHTML conversion for better performance with large result message? + String xhtml; + try(Message m = result) { + xhtml = XmlUtils.toXhtml(m); + } catch (IOException e) { + throw new SenderException("error reading http response as String", e); + } + + if (transformerPool != null && xhtml != null) { + log.debug("transforming result [{}]", xhtml); + try { + xhtml = transformerPool.transform(XmlUtils.stringToSourceForSingleUse(xhtml)); + } catch (Exception e) { + throw new SenderException("Exception on transforming input", e); + } + } + + result = Message.asMessage(xhtml); + } + + if (result == null) { + result = Message.nullMessage(); + } + log.debug("Storing [{}]=[{}], [{}]=[{}]", CONTEXT_KEY_STATUS_CODE, statusCode, CONTEXT_KEY_REASON_PHRASE, reasonPhrase); + result.getContext().put(CONTEXT_KEY_STATUS_CODE, statusCode); + result.getContext().put(CONTEXT_KEY_REASON_PHRASE, reasonPhrase); + return new SenderResult(success, result, reasonPhrase, Integer.toString(statusCode)); + } + + @Override + public String getPhysicalDestinationName() { + if (urlParameter!=null) { + return "dynamic url"; + } + return getUrl(); + } + + /** URL or base of URL to be used */ + public void setUrl(String string) { + url = string; + } + + /** + * Parameter that is used to obtain URL; overrides url-attribute. + * @ff.default url + */ + public void setUrlParam(String urlParam) { + this.urlParam = urlParam; + } + + /** + * The HTTP Method used to execute the request + * @ff.default GET + */ + public void setMethodType(HttpMethod method) { + this.httpMethod = method; + } + + /** + * Content-Type (superset of mimetype + charset) of the request, for POST, PUT and PATCH methods + * @ff.default text/html, when postType=RAW + */ + public void setContentType(String string) { + contentType = string; + } + + /** + * Charset of the request. Typically only used on PUT and POST requests. + * @ff.default UTF-8 + */ + public void setCharSet(String string) { + charSet = string; + } + + @Deprecated + @ConfigurationWarning("Please use attribute keystore instead") + public void setCertificate(String string) { + setKeystore(string); + } + @Deprecated + @ConfigurationWarning("has been replaced with keystoreType") + public void setCertificateType(KeystoreType value) { + setKeystoreType(value); + } + @Deprecated + @ConfigurationWarning("Please use attribute keystoreAuthAlias instead") + public void setCertificateAuthAlias(String string) { + setKeystoreAuthAlias(string); + } + @Deprecated + @ConfigurationWarning("Please use attribute keystorePassword instead") + public void setCertificatePassword(String string) { + setKeystorePassword(string); + } + + /** Comma separated list of parameter names which should be set as HTTP headers */ + public void setHeadersParams(String headersParams) { + this.headersParams = headersParams; + } + + /** Comma separated list of parameter names that should not be added as request or body parameter, or as HTTP header, if they are empty. Set to '*' for this behaviour for all parameters */ + public void setParametersToSkipWhenEmpty(String parametersToSkipWhenEmpty) { + this.parametersToSkipWhenEmpty = parametersToSkipWhenEmpty; + } + + /** + * If true, the HTML response is transformed to XHTML + * @ff.default false + */ + public void setXhtml(boolean xHtml) { + xhtml = xHtml; + } + + /** (Only used when xHtml=true) stylesheet to apply to the HTML response */ + public void setStyleSheetName(String stylesheetName){ + this.styleSheetName=stylesheetName; + } + + /** If set, the status code of the HTTP response is put in the specified sessionKey and the (error or okay) response message is returned. + * Setting this property has a side effect. If a 4xx or 5xx result code is returned and if the configuration does not implement + * the specific forward for the returned HTTP result code, then the success forward is followed instead of the exception forward. + */ + public void setResultStatusCodeSessionKey(String resultStatusCodeSessionKey) { + this.resultStatusCodeSessionKey = resultStatusCodeSessionKey; + } +} diff --git a/src/main/java/org/frankframework/http/HttpSenderBaseCached.java-orig b/src/main/java/org/frankframework/http/HttpSenderBaseCached.java-orig new file mode 100644 index 000000000..281b2a563 --- /dev/null +++ b/src/main/java/org/frankframework/http/HttpSenderBaseCached.java-orig @@ -0,0 +1,588 @@ +/* + Copyright 2017-2024 WeAreFrank! + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package org.frankframework.http; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.SocketTimeoutException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URLEncoder; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; + +import javax.servlet.http.HttpServletResponse; +import javax.xml.transform.TransformerConfigurationException; + +import org.apache.commons.lang3.StringUtils; +import org.apache.http.HttpResponse; +import org.apache.http.MethodNotSupportedException; +import org.apache.http.StatusLine; +import org.apache.http.client.methods.HttpRequestBase; +import org.apache.http.entity.ContentType; +import org.frankframework.configuration.ConfigurationException; +import org.frankframework.configuration.ConfigurationWarning; +import org.frankframework.core.CanUseSharedResource; +import org.frankframework.core.HasPhysicalDestination; +import org.frankframework.core.ISenderWithParameters; +import org.frankframework.core.ParameterException; +import org.frankframework.core.PipeLineSession; +import org.frankframework.core.Resource; +import org.frankframework.core.SenderException; +import org.frankframework.core.SenderResult; +import org.frankframework.core.TimeoutException; +import org.frankframework.encryption.KeystoreType; +import org.frankframework.parameters.Parameter; +import org.frankframework.parameters.ParameterList; +import org.frankframework.parameters.ParameterValue; +import org.frankframework.parameters.ParameterValueList; +import org.frankframework.stream.Message; +import org.frankframework.task.TimeoutGuard; +import org.frankframework.util.AppConstants; +import org.frankframework.util.ClassUtils; +import org.frankframework.util.StreamUtil; +import org.frankframework.util.StringUtil; +import org.frankframework.util.TransformerPool; +import org.frankframework.util.XmlUtils; + +import lombok.Getter; +import lombok.Setter; + +/** + * Sender for the HTTP protocol using GET, POST, PUT or DELETE using httpclient 4+ + * + *

Expected message format:

+ *

GET methods expect a message looking like this

+ * 
+ *   param_name=param_value&another_param_name=another_param_value
+ *
+ *

POST AND PUT methods expect a message similar as GET, or looking like this

+ * 
+ *   param_name=param_value
+ *   another_param_name=another_param_value
+ *
+ * + * @ff.parameters Any parameters present are appended to the request (when method is GET as request-parameters, when method POST as body part) except the headersParams list, which are added as HTTP headers, and the urlParam header + * @ff.forward "<statusCode of the HTTP response>" default + * + * @author Niels Meijer + * @since 7.0 + */ +//TODO: Fix javadoc! + +public abstract class HttpSenderBase extends HttpSessionBase implements HasPhysicalDestination, ISenderWithParameters, CanUseSharedResource { + + private static final String CONTEXT_KEY_STATUS_CODE = "Http.StatusCode"; + private static final String CONTEXT_KEY_REASON_PHRASE = "Http.ReasonPhrase"; + public static final String MESSAGE_ID_HEADER = "Message-Id"; + public static final String CORRELATION_ID_HEADER = "Correlation-Id"; + + private final @Getter String domain = "Http"; + + private @Setter String sharedResourceRef; + + private @Getter String url; + private @Getter String urlParam = "url"; + + public enum HttpMethod { + GET,POST,PUT,PATCH,DELETE,HEAD,REPORT; + } + private @Getter HttpMethod httpMethod = HttpMethod.GET; + + private @Getter String charSet = StreamUtil.DEFAULT_INPUT_STREAM_ENCODING; + private @Getter ContentType fullContentType = null; + private @Getter String contentType = null; + + private @Getter String headersParams=""; + private @Getter boolean xhtml=false; + private @Getter String styleSheetName=null; + private @Getter String resultStatusCodeSessionKey; + private @Getter String parametersToSkipWhenEmpty; + + private final boolean APPEND_MESSAGEID_HEADER = AppConstants.getInstance(getConfigurationClassLoader()).getBoolean("http.headers.messageid", true); + private final boolean APPEND_CORRELATIONID_HEADER = AppConstants.getInstance(getConfigurationClassLoader()).getBoolean("http.headers.correlationid", true); + + private TransformerPool transformerPool=null; + + protected Parameter urlParameter; + + protected URI staticUri; + + protected Set requestOrBodyParamsSet=new HashSet<>(); + protected Set headerParamsSet=new LinkedHashSet<>(); + protected Set parametersToSkipWhenEmptySet=new HashSet<>(); + + protected ParameterList paramList = null; + + @Override + public void addParameter(Parameter p) { + if (paramList==null) { + paramList=new ParameterList(); + } + paramList.add(p); + } + + /** + * return the Parameters + */ + @Override + public ParameterList getParameterList() { + return paramList; + } + + @Override + public void configure() throws ConfigurationException { + if(StringUtils.isBlank(sharedResourceRef)) { + log.debug("configuring local HttpSession"); + super.configure(); + } + + if (paramList!=null) { + paramList.configure(); + + if (StringUtils.isNotEmpty(getHeadersParams())) { + headerParamsSet.addAll(StringUtil.split(getHeadersParams())); + } + for (Parameter p: paramList) { + String paramName = p.getName(); + if (!headerParamsSet.contains(paramName)) { + requestOrBodyParamsSet.add(paramName); + } + } + + if (StringUtils.isNotEmpty(getUrlParam())) { + headerParamsSet.remove(getUrlParam()); + requestOrBodyParamsSet.remove(getUrlParam()); + urlParameter = paramList.findParameter(getUrlParam()); + } + + if (StringUtils.isNotEmpty(getParametersToSkipWhenEmpty())) { + if (getParametersToSkipWhenEmpty().equals("*")) { + parametersToSkipWhenEmptySet.addAll(headerParamsSet); + parametersToSkipWhenEmptySet.addAll(requestOrBodyParamsSet); + } else { + parametersToSkipWhenEmptySet.addAll(StringUtil.split(getParametersToSkipWhenEmpty())); + } + } + } + + if(StringUtils.isNotEmpty(getContentType())) { + fullContentType = ContentType.parse(getContentType()); + if(fullContentType.getCharset() == null) { + fullContentType = fullContentType.withCharset(getCharSet()); + } + } + + try { + if (urlParameter == null) { + if (StringUtils.isEmpty(getUrl())) { + throw new ConfigurationException(getLogPrefix()+"url must be specified, either as attribute, or as parameter"); + } + staticUri = getURI(getUrl()); + } + } catch (URISyntaxException e) { + throw new ConfigurationException("cannot interpret url ["+getUrl()+"]", e); + } + + if (StringUtils.isNotEmpty(getStyleSheetName())) { + try { + Resource stylesheet = Resource.getResource(this, getStyleSheetName()); + if (stylesheet == null) { + throw new ConfigurationException("cannot find stylesheet ["+getStyleSheetName()+"]"); + } + transformerPool = TransformerPool.getInstance(stylesheet); + } catch (IOException e) { + throw new ConfigurationException("cannot retrieve ["+ getStyleSheetName() + "]", e); + } catch (TransformerConfigurationException te) { + throw new ConfigurationException("got error creating transformer from file [" + getStyleSheetName() + "]", te); + } + } + } + + @Override + public void open() throws SenderException { + try { + start(); + } catch (Exception e) { + throw new SenderException(getLogPrefix()+"unable to create HttpClient", e); + } + + if (transformerPool!=null) { + try { + transformerPool.open(); + } catch (Exception e) { + throw new SenderException(getLogPrefix()+"cannot start TransformerPool", e); + } + } + } + + @Override + public Class getObjectType() { + return HttpSession.class; + } + + @Override + public void start() { + if(StringUtils.isNotBlank(sharedResourceRef)) { + HttpSession session = getSharedResource(sharedResourceRef); + setHttpClient(session.getHttpClient()); + setHttpContext(session.getDefaultHttpClientContext()); + } else { + log.debug("starting local HttpSession"); + super.start(); + } + } + + @Override + public void close() { + if(StringUtils.isBlank(sharedResourceRef)) { + super.stop(); + } + + if (transformerPool!=null) { + transformerPool.close(); + } + } + + protected boolean appendParameters(boolean parametersAppended, StringBuilder path, ParameterValueList parameters) throws SenderException { + if (parameters != null) { + log.debug("appending [{}] parameters", parameters::size); + for(ParameterValue pv : parameters) { + if (requestOrBodyParamsSet.contains(pv.getName())) { + String value = pv.asStringValue(""); + if (StringUtils.isNotEmpty(value) || !parametersToSkipWhenEmptySet.contains(pv.getName())) { + try { + if (parametersAppended) { + path.append("&"); + } else { + path.append("?"); + parametersAppended = true; + } + + String parameterToAppend = pv.getDefinition().getName() +"="+ URLEncoder.encode(value, getCharSet()); + log.debug("appending parameter [{}]", parameterToAppend); + path.append(parameterToAppend); + } catch (UnsupportedEncodingException e) { + throw new SenderException(getLogPrefix()+"["+getCharSet()+"] encoding error. Failed to add parameter ["+pv.getDefinition().getName()+"]", e); + } + } + } + } + } + return parametersAppended; + } + + + /** + * Returns the true name of the class and not XsltPipe$$EnhancerBySpringCGLIB$$563e6b5d. + * {@link ClassUtils#nameOf(Object)} makes sure the original class will be used. + * + * @return className + name of the ISender + */ + protected String getLogPrefix() { + return ClassUtils.nameOf(this) + " "; + } + + /** + * Custom implementation to create a {@link HttpRequestBase HttpRequest} object. + * @param uri endpoint to send the message to + * @param message to be sent + * @param parameters ParameterValueList that contains all the senders parameters + * @param session PipeLineSession to retrieve or store data from, or NULL when not set + * @return a {@link HttpRequestBase HttpRequest} object + * @throws SenderException + */ + protected abstract HttpRequestBase getMethod(URI uri, Message message, ParameterValueList parameters, PipeLineSession session) throws SenderException; + + /** + * Custom implementation to extract the response and format it to a String result.
+ * It is important that the {@link HttpResponseHandler#getResponse() response} + * will be read or will be {@link HttpResponseHandler#close() closed}. + * @param responseHandler {@link HttpResponseHandler} that contains the response information + * @param session {@link PipeLineSession} which may be null + * @return a string that will be passed to the pipeline + */ + protected abstract Message extractResult(HttpResponseHandler responseHandler, PipeLineSession session) throws SenderException, IOException; + + protected boolean validateResponseCode(int statusCode) { + boolean ok = false; + if (StringUtils.isNotEmpty(getResultStatusCodeSessionKey())) { + ok = true; + } else { + if (statusCode==200 || statusCode==201 || statusCode==202 || statusCode==204 || statusCode==206) { + ok = true; + } else { + if (isIgnoreRedirects() && (statusCode==HttpServletResponse.SC_MOVED_PERMANENTLY || statusCode==HttpServletResponse.SC_MOVED_TEMPORARILY || statusCode==HttpServletResponse.SC_TEMPORARY_REDIRECT)) { + ok = true; + } + } + } + return ok; + } + + @Override + public SenderResult sendMessage(Message message, PipeLineSession session) throws SenderException, TimeoutException { + ParameterValueList pvl = null; + try { + if (paramList !=null) { + pvl=paramList.getValues(message, session); + } + } catch (ParameterException e) { + throw new SenderException(getLogPrefix()+"Sender ["+getName()+"] caught exception evaluating parameters",e); + } + + URI targetUri; + final HttpRequestBase httpRequestBase; + try { + if (urlParameter != null) { + String url = pvl.get(getUrlParam()).asStringValue(); + targetUri = getURI(url); + } else { + targetUri = staticUri; + } + + // Resolve HeaderParameters + Map headersParamsMap = new HashMap<>(); + if (!headerParamsSet.isEmpty() && pvl!=null) { + log.debug("appending header parameters "+headersParams); + for (String paramName:headerParamsSet) { + ParameterValue paramValue = pvl.get(paramName); + if(paramValue != null) { + String value = paramValue.asStringValue(null); + if (StringUtils.isNotEmpty(value) || !parametersToSkipWhenEmptySet.contains(paramName)) { + headersParamsMap.put(paramName, value); + } + } + } + } + + httpRequestBase = getMethod(targetUri, message, pvl, session); + if(httpRequestBase == null) + throw new MethodNotSupportedException("could not find implementation for method ["+getHttpMethod()+"]"); + + //Set all headers + if(session != null) { + if (APPEND_MESSAGEID_HEADER && StringUtils.isNotEmpty(session.getMessageId())) { + httpRequestBase.setHeader(MESSAGE_ID_HEADER, session.getMessageId()); + } + if (APPEND_CORRELATIONID_HEADER && StringUtils.isNotEmpty(session.getCorrelationId())) { + httpRequestBase.setHeader(CORRELATION_ID_HEADER, session.getCorrelationId()); + } + } + for (String param: headersParamsMap.keySet()) { + httpRequestBase.setHeader(param, headersParamsMap.get(param)); + } + + log.info("configured httpclient for host [{}]", targetUri::getHost); + + } catch (Exception e) { + throw new SenderException(e); + } + + Message result; + int statusCode; + boolean success; + String reasonPhrase; + + TimeoutGuard tg = new TimeoutGuard(1+getTimeout()/1000, getName()) { + + @Override + protected void abort() { + httpRequestBase.abort(); + } + + }; + try { + log.debug("executing method [{}]", httpRequestBase::getRequestLine); + HttpResponse httpResponse = execute(targetUri, httpRequestBase, session); + log.debug("executed method"); + + HttpResponseHandler responseHandler = new HttpResponseHandler(httpResponse); + StatusLine statusline = httpResponse.getStatusLine(); + statusCode = statusline.getStatusCode(); + success = validateResponseCode(statusCode); + reasonPhrase = StringUtils.isNotEmpty(statusline.getReasonPhrase()) ? statusline.getReasonPhrase() : "HTTP status-code ["+statusCode+"]"; + + if (StringUtils.isNotEmpty(getResultStatusCodeSessionKey()) && session != null) { + session.put(getResultStatusCodeSessionKey(), Integer.toString(statusCode)); + } + + // Only give warnings for 4xx (client errors) and 5xx (server errors) + if (statusCode >= 400 && statusCode < 600) { + log.warn("status [{}]", statusline); + } else { + log.debug("status [{}]", statusCode); + } + + result = extractResult(responseHandler, session); + + log.debug("retrieved result [{}]", result); + } catch (IOException e) { + httpRequestBase.abort(); + if (e instanceof SocketTimeoutException) { + throw new TimeoutException(e); + } + throw new SenderException(e); + } finally { + // By forcing the use of the HttpResponseHandler the resultStream + // will automatically be closed when it has been read. + // See HttpResponseHandler and ReleaseConnectionAfterReadInputStream. + // We cannot close the connection as the response might be kept + // in a sessionKey for later use in the pipeline. + // + // IMPORTANT: It is possible that poorly written implementations + // won't read or close the response. + // This will cause the connection to become stale. + + if (tg.cancel()) { + throw new TimeoutException(getLogPrefix()+"timeout of ["+getTimeout()+"] ms exceeded"); + } + } + + if (statusCode == -1){ + throw new SenderException("Failed to recover from exception"); + } + + if (isXhtml() && !Message.isEmpty(result)) { + // TODO: Streaming XHTML conversion for better performance with large result message? + String xhtml; + try(Message m = result) { + xhtml = XmlUtils.toXhtml(m); + } catch (IOException e) { + throw new SenderException("error reading http response as String", e); + } + + if (transformerPool != null && xhtml != null) { + log.debug("transforming result [{}]", xhtml); + try { + xhtml = transformerPool.transform(XmlUtils.stringToSourceForSingleUse(xhtml)); + } catch (Exception e) { + throw new SenderException("Exception on transforming input", e); + } + } + + result = Message.asMessage(xhtml); + } + + if (result == null) { + result = Message.nullMessage(); + } + log.debug("Storing [{}]=[{}], [{}]=[{}]", CONTEXT_KEY_STATUS_CODE, statusCode, CONTEXT_KEY_REASON_PHRASE, reasonPhrase); + result.getContext().put(CONTEXT_KEY_STATUS_CODE, statusCode); + result.getContext().put(CONTEXT_KEY_REASON_PHRASE, reasonPhrase); + return new SenderResult(success, result, reasonPhrase, Integer.toString(statusCode)); + } + + @Override + public String getPhysicalDestinationName() { + if (urlParameter!=null) { + return "dynamic url"; + } + return getUrl(); + } + + /** URL or base of URL to be used */ + public void setUrl(String string) { + url = string; + } + + /** + * Parameter that is used to obtain URL; overrides url-attribute. + * @ff.default url + */ + public void setUrlParam(String urlParam) { + this.urlParam = urlParam; + } + + /** + * The HTTP Method used to execute the request + * @ff.default GET + */ + public void setMethodType(HttpMethod method) { + this.httpMethod = method; + } + + /** + * Content-Type (superset of mimetype + charset) of the request, for POST, PUT and PATCH methods + * @ff.default text/html, when postType=RAW + */ + public void setContentType(String string) { + contentType = string; + } + + /** + * Charset of the request. Typically only used on PUT and POST requests. + * @ff.default UTF-8 + */ + public void setCharSet(String string) { + charSet = string; + } + + @Deprecated + @ConfigurationWarning("Please use attribute keystore instead") + public void setCertificate(String string) { + setKeystore(string); + } + @Deprecated + @ConfigurationWarning("has been replaced with keystoreType") + public void setCertificateType(KeystoreType value) { + setKeystoreType(value); + } + @Deprecated + @ConfigurationWarning("Please use attribute keystoreAuthAlias instead") + public void setCertificateAuthAlias(String string) { + setKeystoreAuthAlias(string); + } + @Deprecated + @ConfigurationWarning("Please use attribute keystorePassword instead") + public void setCertificatePassword(String string) { + setKeystorePassword(string); + } + + /** Comma separated list of parameter names which should be set as HTTP headers */ + public void setHeadersParams(String headersParams) { + this.headersParams = headersParams; + } + + /** Comma separated list of parameter names that should not be added as request or body parameter, or as HTTP header, if they are empty. Set to '*' for this behaviour for all parameters */ + public void setParametersToSkipWhenEmpty(String parametersToSkipWhenEmpty) { + this.parametersToSkipWhenEmpty = parametersToSkipWhenEmpty; + } + + /** + * If true, the HTML response is transformed to XHTML + * @ff.default false + */ + public void setXhtml(boolean xHtml) { + xhtml = xHtml; + } + + /** (Only used when xHtml=true) stylesheet to apply to the HTML response */ + public void setStyleSheetName(String stylesheetName){ + this.styleSheetName=stylesheetName; + } + + /** If set, the status code of the HTTP response is put in the specified sessionKey and the (error or okay) response message is returned. + * Setting this property has a side effect. If a 4xx or 5xx result code is returned and if the configuration does not implement + * the specific forward for the returned HTTP result code, then the success forward is followed instead of the exception forward. + */ + public void setResultStatusCodeSessionKey(String resultStatusCodeSessionKey) { + this.resultStatusCodeSessionKey = resultStatusCodeSessionKey; + } +} \ No newline at end of file diff --git a/src/main/java/org/frankframework/http/HttpSenderCached.java b/src/main/java/org/frankframework/http/HttpSenderCached.java new file mode 100644 index 000000000..22a48e569 --- /dev/null +++ b/src/main/java/org/frankframework/http/HttpSenderCached.java @@ -0,0 +1,619 @@ +/* + Copyright 2013, 2016-2020 Nationale-Nederlanden, 2020-2023 WeAreFrank! + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package org.frankframework.http; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URLEncoder; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import javax.servlet.http.HttpServletResponse; + +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.http.Header; +import org.apache.http.HttpEntity; +import org.apache.http.NameValuePair; +import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpEntityEnclosingRequestBase; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpHead; +import org.apache.http.client.methods.HttpPatch; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpPut; +import org.apache.http.client.methods.HttpRequestBase; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.entity.ByteArrayEntity; +import org.apache.http.entity.mime.FormBodyPart; +import org.apache.http.entity.mime.FormBodyPartBuilder; +import org.apache.http.entity.mime.MIME; +import org.apache.http.message.BasicNameValuePair; +import org.apache.logging.log4j.Logger; +import org.frankframework.configuration.ConfigurationException; +import org.frankframework.configuration.ConfigurationWarnings; +import org.frankframework.configuration.SuppressKeys; +import org.frankframework.core.PipeLineSession; +import org.frankframework.core.SenderException; +import org.frankframework.http.mime.MessageContentBody; +import org.frankframework.http.mime.MultipartEntityBuilder; +import org.frankframework.parameters.ParameterValue; +import org.frankframework.parameters.ParameterValueList; +import org.frankframework.stream.Message; +import org.frankframework.util.ClassUtils; +import org.frankframework.util.DomBuilderException; +import org.frankframework.util.StreamUtil; +import org.frankframework.util.XmlBuilder; +import org.frankframework.util.XmlUtils; +import org.springframework.http.MediaType; +import org.springframework.util.MimeType; +import org.w3c.dom.Element; +import org.w3c.dom.Node; + +import jakarta.mail.BodyPart; +import jakarta.mail.MessagingException; +import jakarta.mail.internet.MimeMultipart; +import lombok.Getter; +import net.sf.ehcache.Ehcache; + +/** + * Sender for the HTTP protocol using {@link HttpMethod HttpMethod}. By default, any response code outside the 2xx or 3xx range + * is considered an error and the exception forward of the SenderPipe is followed if present and if there + * is no forward for the specific HTTP status code. Forwards for specific HTTP codes (e.g. "200", "201", ...) + * are returned by this sender, so they are available to the SenderPipe. + * + *

Expected message format:

+ *

GET methods expect a message looking like this: + * 

+ *    param_name=param_value&another_param_name=another_param_value
+ *
+ *

POST AND PUT methods expect a message similar as GET, or looking like this: + * 

+ *   param_name=param_value
+ *   another_param_name=another_param_value
+ *
+ * + * Note: + * When used as MTOM sender and MTOM receiver doesn't support Content-Transfer-Encoding "base64", messages without line feeds will give an error. + * This can be fixed by setting the Content-Transfer-Encoding in the MTOM sender. + *

+ * + * @author Niels Meijer + * @since 7.0 + * @version 2.0 + */ +public class HttpSenderCached extends HttpSenderBaseCached { + + private @Getter boolean paramsInUrl=true; + private @Getter String firstBodyPartName=null; + + private @Getter String multipartXmlSessionKey; + private @Getter String mtomContentTransferEncoding = null; //Defaults to 8-bit for normal String messages, 7-bit for e-mails and binary for streams + private @Getter boolean encodeMessages = false; + private @Getter Boolean treatInputMessageAsParameters = null; + + private @Getter PostType postType = PostType.RAW; + + private static final MimeType APPLICATION_XOP_XML = MimeType.valueOf("application/xop+xml"); + + private Ehcache etagCache; + private Ehcache messageCache; + + public enum PostType { + /** The input message is sent unchanged as character data, like text, XML or JSON, with possibly parameter data appended */ + RAW, // text/html;charset=UTF8 + /** The input message is sent unchanged as binary data */ + BINARY, //application/octet-stream +// SWA("Soap with Attachments"), // text/xml + /** Yields a x-www-form-urlencoded form entity */ + URLENCODED, + /** Yields a multipart/form-data form entity */ + FORMDATA, + /** Yields a MTOM multipart/related form entity */ + MTOM; + } + + @Override + public void configure() throws ConfigurationException { + //For backwards compatibility we have to set the contentType to text/html on POST and PUT requests + if(StringUtils.isEmpty(getContentType()) && postType == PostType.RAW && (getHttpMethod() == HttpMethod.POST || getHttpMethod() == HttpMethod.PUT || getHttpMethod() == HttpMethod.PATCH)) { + setContentType("text/html"); + } + + super.configure(); + super.setCache(); + + if (getTreatInputMessageAsParameters()==null && getHttpMethod()!=HttpMethod.GET) { + setTreatInputMessageAsParameters(Boolean.TRUE); + } + + if (getHttpMethod() != HttpMethod.POST) { + if (!isParamsInUrl()) { + throw new ConfigurationException(getLogPrefix()+"paramsInUrl can only be set to false for methodType POST"); + } + if (StringUtils.isNotEmpty(getFirstBodyPartName())) { + throw new ConfigurationException(getLogPrefix()+"firstBodyPartName can only be set for methodType POST"); + } + } + } + + @Override + protected HttpRequestBase getMethod(URI url, Message message, ParameterValueList parameters, PipeLineSession session) throws SenderException { + if (isEncodeMessages() && !Message.isEmpty(message)) { + try { + message = new Message(URLEncoder.encode(message.asString(), getCharSet())); + } catch (IOException e) { + throw new SenderException(getLogPrefix()+"unable to encode message",e); + } + } + + URI uri; + try { + uri = encodeQueryParameters(url); + } catch (UnsupportedEncodingException | URISyntaxException e) { + throw new SenderException("error encoding queryparameters in url ["+url.toString()+"]", e); + } + + if(postType==PostType.URLENCODED || postType==PostType.FORMDATA || postType==PostType.MTOM) { + try { + return getMultipartPostMethodWithParamsInBody(uri, message, parameters, session); + } catch (IOException e) { + throw new SenderException(getLogPrefix()+"unable to read message", e); + } + } + // RAW + BINARY + return getMethod(uri, message, parameters); + } + + // Encode query parameter values. + private URI encodeQueryParameters(URI url) throws UnsupportedEncodingException, URISyntaxException { + URIBuilder uri = new URIBuilder(url); + ArrayList pairs = new ArrayList<>(uri.getQueryParams().size()); + for(NameValuePair pair : uri.getQueryParams()) { + String paramValue = pair.getValue(); //May be NULL + if(StringUtils.isNotEmpty(paramValue)) { + paramValue = URLEncoder.encode(paramValue, getCharSet()); //Only encode if the value is not null + } + pairs.add(new BasicNameValuePair(pair.getName(), paramValue)); + } + if(!pairs.isEmpty()) { + uri.clearParameters(); + uri.addParameters(pairs); + } + return uri.build(); + } + + /** + * Returns HttpRequestBase, with (optional) RAW or as BINAIRY content + */ + protected HttpRequestBase getMethod(URI uri, Message message, ParameterValueList parameters) throws SenderException { + try { + boolean queryParametersAppended = false; + StringBuilder relativePath = new StringBuilder(uri.getRawPath()); + if (!StringUtils.isEmpty(uri.getQuery())) { + relativePath.append("?").append(uri.getQuery()); + queryParametersAppended = true; + } + + switch (getHttpMethod()) { + case GET: + if (parameters!=null) { + queryParametersAppended = appendParameters(queryParametersAppended,relativePath,parameters); + if (log.isDebugEnabled()) log.debug(getLogPrefix()+"path after appending of parameters ["+relativePath+"]"); + } + + HttpGet getMethod = new HttpGet(relativePath+(parameters==null && BooleanUtils.isTrue(getTreatInputMessageAsParameters()) && !Message.isEmpty(message)? message.asString():"")); + + if (log.isDebugEnabled()) log.debug(getLogPrefix()+"HttpSender constructed GET-method ["+getMethod.getURI().getQuery()+"]"); + if (null != getFullContentType()) { //Manually set Content-Type header + getMethod.setHeader("Content-Type", getFullContentType().toString()); + } + return getMethod; + + case POST: + case PUT: + case PATCH: + HttpEntity entity; + if(postType == PostType.RAW) { + String messageString = BooleanUtils.isTrue(getTreatInputMessageAsParameters()) && !Message.isEmpty(message) ? message.asString() : ""; + if (parameters!=null) { + StringBuilder msg = new StringBuilder(messageString); + appendParameters(true,msg,parameters); + if (StringUtils.isEmpty(messageString) && msg.length()>1) { + messageString=msg.substring(1); + } else { + messageString=msg.toString(); + } + } + entity = new ByteArrayEntity(messageString.getBytes(StreamUtil.DEFAULT_INPUT_STREAM_ENCODING), getFullContentType()); + } else if(postType == PostType.BINARY) { + entity = new HttpMessageEntity(message, getFullContentType()); + } else { + throw new SenderException("PostType ["+postType.name()+"] not allowed!"); + } + + HttpEntityEnclosingRequestBase method; + if (getHttpMethod() == HttpMethod.POST) { + method = new HttpPost(relativePath.toString()); + } else if (getHttpMethod() == HttpMethod.PATCH) { + method = new HttpPatch(relativePath.toString()); + } else { + method = new HttpPut(relativePath.toString()); + } + + method.setEntity(entity); + return method; + + case DELETE: + HttpDelete deleteMethod = new HttpDelete(relativePath.toString()); + if (null != getFullContentType()) { //Manually set Content-Type header + deleteMethod.setHeader("Content-Type", getFullContentType().toString()); + } + return deleteMethod; + + case HEAD: + return new HttpHead(relativePath.toString()); + + case REPORT: + Element element = XmlUtils.buildElement(message.asString(), true); + HttpReport reportMethod = new HttpReport(relativePath.toString(), element); + if (null != getFullContentType()) { //Manually set Content-Type header + reportMethod.setHeader("Content-Type", getFullContentType().toString()); + } + return reportMethod; + + default: + return null; + } + } catch (Exception e) { + //Catch all exceptions and throw them as SenderException + throw new SenderException(e); + } + } + + /** + * Returns a multi-parted message, either as X-WWW-FORM-URLENCODED, FORM-DATA or MTOM + */ + private HttpPost getMultipartPostMethodWithParamsInBody(URI uri, Message message, ParameterValueList parameters, PipeLineSession session) throws SenderException, IOException { + HttpPost hmethod = new HttpPost(uri); + + if (postType==PostType.URLENCODED && StringUtils.isEmpty(getMultipartXmlSessionKey())) { // x-www-form-urlencoded + List requestFormElements = new ArrayList<>(); + + if (StringUtils.isNotEmpty(getFirstBodyPartName())) { + requestFormElements.add(new BasicNameValuePair(getFirstBodyPartName(), message.asString())); + log.debug(getLogPrefix()+"appended parameter ["+getFirstBodyPartName()+"] with value ["+message+"]"); + } + if (parameters!=null) { + for(ParameterValue pv : parameters) { + String name = pv.getDefinition().getName(); + String value = pv.asStringValue(""); + + if (requestOrBodyParamsSet.contains(name) && (StringUtils.isNotEmpty(value) || !parametersToSkipWhenEmptySet.contains(name))) { + requestFormElements.add(new BasicNameValuePair(name,value)); + if (log.isDebugEnabled()) log.debug(getLogPrefix()+"appended parameter ["+name+"] with value ["+value+"]"); + } + } + } + try { + hmethod.setEntity(new UrlEncodedFormEntity(requestFormElements, getCharSet())); + } catch (UnsupportedEncodingException e) { + throw new SenderException(getLogPrefix()+"unsupported encoding for one or more POST parameters", e); + } + } + else { //formdata and mtom + HttpEntity requestEntity = createMultiPartEntity(message, parameters, session); + hmethod.setEntity(requestEntity); + } + + return hmethod; + } + + private FormBodyPart createStringBodypart(Message message) { + MimeType mimeType = (postType == PostType.MTOM) ? APPLICATION_XOP_XML : MediaType.TEXT_PLAIN; // only the first part is XOP+XML, other parts should use their own content-type + FormBodyPartBuilder bodyPart = FormBodyPartBuilder.create(getFirstBodyPartName(), new MessageContentBody(message, mimeType)); + + // Should only be set when request is MTOM and it's the first BodyPart + if (postType == PostType.MTOM && StringUtils.isNotEmpty(getMtomContentTransferEncoding())) { + bodyPart.setField(MIME.CONTENT_TRANSFER_ENC, getMtomContentTransferEncoding()); + } + + return bodyPart.build(); + } + + private HttpEntity createMultiPartEntity(Message message, ParameterValueList parameters, PipeLineSession session) throws SenderException, IOException { + MultipartEntityBuilder entity = MultipartEntityBuilder.create(); + + entity.setCharset(Charset.forName(getCharSet())); + if(postType == PostType.MTOM) + entity.setMtomMultipart(); + + if (StringUtils.isNotEmpty(getFirstBodyPartName())) { + entity.addPart(createStringBodypart(message)); + if (log.isDebugEnabled()) log.debug(getLogPrefix()+"appended stringpart ["+getFirstBodyPartName()+"] with value ["+message+"]"); + } + if (parameters!=null) { + for(ParameterValue pv : parameters) { + String name = pv.getDefinition().getName(); + if (requestOrBodyParamsSet.contains(name)) { + Message msg = pv.asMessage(); + if (!msg.isEmpty() || !parametersToSkipWhenEmptySet.contains(name)) { + + String fileName = null; + String sessionKey = pv.getDefinition().getSessionKey(); + if (sessionKey != null) { + fileName = session.getString(sessionKey + "Name"); + } + if(fileName != null) { + log.warn("setting filename using [{}Name] for bodypart [{}]. Consider using a MultipartXml with the attribute [name] instead.", sessionKey, fileName, name); + } + + entity.addPart(name, new MessageContentBody(msg, null, fileName)); + if (log.isDebugEnabled()) log.debug("{}appended bodypart [{}] with message [{}]", getLogPrefix(), name, msg); + } + } + } + } + + if (StringUtils.isNotEmpty(getMultipartXmlSessionKey())) { + String multipartXml = session.getString(getMultipartXmlSessionKey()); + log.debug(getLogPrefix()+"building multipart message with MultipartXmlSessionKey ["+multipartXml+"]"); + if (StringUtils.isEmpty(multipartXml)) { + log.warn(getLogPrefix()+"sessionKey [" +getMultipartXmlSessionKey()+"] is empty"); + } else { + Element partsElement; + try { + partsElement = XmlUtils.buildElement(multipartXml); + } catch (DomBuilderException e) { + throw new SenderException(getLogPrefix()+"error building multipart xml", e); + } + Collection parts = XmlUtils.getChildTags(partsElement, "part"); + if (parts.isEmpty()) { + log.warn(getLogPrefix()+"no part(s) in multipart xml [" + multipartXml + "]"); + } else { + for (final Node part : parts) { + Element partElement = (Element) part; + entity.addPart(elementToFormBodyPart(partElement, session)); + } + } + } + } + return entity.build(); + } + + protected FormBodyPart elementToFormBodyPart(Element element, PipeLineSession session) throws IOException { + String part = element.getAttribute("name"); //Name of the part + boolean isFile = "file".equals(element.getAttribute("type")); //text of file, empty == text + String filename = element.getAttribute("filename"); //if type == file, the filename + String partSessionKey = element.getAttribute("sessionKey"); //SessionKey to retrieve data from + String partMimeType = element.getAttribute("mimeType"); //MimeType of the part + Message partObject = session.getMessage(partSessionKey); + MimeType mimeType = null; + if(StringUtils.isNotEmpty(partMimeType)) { + mimeType = MimeType.valueOf(partMimeType); + } + + final String filenameToUse; + if(isFile || StringUtils.isNotBlank(filename)) { + String filenamebackup = StringUtils.isBlank(part) ? partSessionKey : part; + filenameToUse = StringUtils.isNotBlank(filename) ? filename : filenamebackup; + } else { + filenameToUse = null; + } + + String partname = isFile || StringUtils.isBlank(part) ? partSessionKey : part; + return FormBodyPartBuilder.create(partname, new MessageContentBody(partObject, mimeType, filenameToUse)).build(); + } + + @Override + protected Message extractResult(URI targetURI, HttpResponseHandler responseHandler, PipeLineSession session) throws SenderException, IOException { + int statusCode = responseHandler.getStatusLine().getStatusCode(); + + if (!validateResponseCode(statusCode)) { + Message responseBody = responseHandler.getResponseMessage(); + String body = ""; + if(responseBody != null) { + responseBody.preserve(); + try { + body = responseBody.asString(); + } catch(IOException e) { + body = "(" + ClassUtils.nameOf(e) + "): " + e.getMessage(); + } + } + log.warn(getLogPrefix() + "httpstatus [" + statusCode + "] reason [" + responseHandler.getStatusLine().getReasonPhrase() + "]"); + return new Message(body); + } + + Message responseMessage = responseHandler.getResponseMessage(); + if (!Message.isEmpty(responseMessage)) { + responseMessage.closeOnCloseOf(session, this); + } + + if (responseHandler.isMultipart()) { + return handleMultipartResponse(responseHandler, session); + } else { + return getResponseBody(targetURI, responseHandler, statusCode); + } + } + + public Message getResponseBody(URI targetURI, HttpResponseHandler responseHandler, int statusCode) throws IOException { + Header[] headers = responseHandler.getAllHeaders(); + if (getHttpMethod() == HttpMethod.HEAD) { + XmlBuilder headersXml = new XmlBuilder("headers"); + for (Header header : headers) { + XmlBuilder headerXml = new XmlBuilder("header"); + headerXml.addAttribute("name", header.getName()); + headerXml.setCdataValue(header.getValue()); + headersXml.addSubElement(headerXml); + } + return Message.asMessage(headersXml.toXML()); + } + + etagCache = super.getEtagCache(); + messageCache = super.getMessageCache(); + Message responseMessage = responseHandler.getResponseMessage(); + + // Begin Etag handling + Header etagHeader = null; + + for (Header header : headers) { + String key = header.getName().toLowerCase().strip(); + if (new String(key).toLowerCase().equals("etag")) { + etagHeader = header; + } + } + + // If the statusCode is 304 and Etag is present + if (statusCode == 304 && etagHeader != null) { + responseMessage = new Message(messageCache.get(etagHeader.getValue()).getObjectValue().toString()); + log.debug(getLogPrefix() + "Cached response with ETag: " + etagHeader.getValue() + " unmodified. Returning cached response."); + } + // Etag is present but no 304 + else if (etagHeader != null) { + log.debug(getLogPrefix() + "Caching response with ETag: " + etagHeader.getValue() + " for url: [{targetURI.toString()}]."); + + // Remove out-of-date stored message + if (etagCache.get(targetURI.toString()) != null) { + messageCache.remove(etagCache.get(targetURI.toString()).getObjectValue()); + } + String etagHeaderValue = etagHeader.getValue(); + net.sf.ehcache.Element etagToStore = new net.sf.ehcache.Element(targetURI.toString(), etagHeaderValue); + net.sf.ehcache.Element messageToStore = new net.sf.ehcache.Element(etagHeaderValue, responseMessage.asString()); + etagCache.put(etagToStore); + messageCache.put(messageToStore); + } + + return responseMessage; + } + + /** + * return the first part as Message and put the other parts as InputStream in the PipeLineSession + */ + private static Message handleMultipartResponse(HttpResponseHandler httpHandler, PipeLineSession session) throws IOException { + return handleMultipartResponse(httpHandler.getContentType().getMimeType(), httpHandler.getResponse(), session); + } + + /** + * return the first part as Message and put the other parts as InputStream in the PipeLineSession + */ + public static Message handleMultipartResponse(String mimeType, InputStream inputStream, PipeLineSession session) throws IOException { + Message result = null; + try { + InputStreamDataSource dataSource = new InputStreamDataSource(mimeType, inputStream); //the entire InputStream will be read here! + MimeMultipart mimeMultipart = new MimeMultipart(dataSource); + for (int i = 0; i < mimeMultipart.getCount(); i++) { + BodyPart bodyPart = mimeMultipart.getBodyPart(i); + if (i == 0) { + result = new PartMessage(bodyPart); + } else { + session.put("multipart" + i, new PartMessage(bodyPart)); + } + } + } catch(MessagingException e) { + throw new IOException("Could not read mime multipart response", e); + } + return result; + } + + public static void streamResponseBody(InputStream is, String contentType, String contentDisposition, HttpServletResponse response, Logger log, String logPrefix, String redirectLocation) throws IOException { + if (StringUtils.isNotEmpty(contentType)) { + response.setHeader("Content-Type", contentType); + } + if (StringUtils.isNotEmpty(contentDisposition)) { + response.setHeader("Content-Disposition", contentDisposition); + } + if (StringUtils.isNotEmpty(redirectLocation)) { + response.sendRedirect(redirectLocation); + } + if (is != null) { + try (OutputStream outputStream = response.getOutputStream()) { + StreamUtil.streamToStream(is, outputStream); + log.debug(logPrefix + "copied response body input stream [" + is + "] to output stream [" + outputStream + "]"); + } + } + } + + /** + * If methodType=POST, PUT or PATCH, the type of post request + * @ff.default RAW + */ + public void setPostType(PostType type) { + this.postType = type; + } + + /** + * If false and methodType=POST, request parameters are put in the request body instead of in the url + * @ff.default true + */ + @Deprecated + public void setParamsInUrl(boolean b) { + if(!b) { + if(postType != PostType.MTOM && postType != PostType.FORMDATA) { //Don't override if another type has explicitly been set + postType = PostType.URLENCODED; + ConfigurationWarnings.add(this, log, "attribute [paramsInUrl] is deprecated: please use postType='URLENCODED' instead", SuppressKeys.DEPRECATION_SUPPRESS_KEY, null); + } else { + ConfigurationWarnings.add(this, log, "attribute [paramsInUrl] is deprecated: no longer required when using FORMDATA or MTOM requests", SuppressKeys.DEPRECATION_SUPPRESS_KEY, null); + } + } + paramsInUrl = b; + } + + /** (Only used when methodType=POST and postType=URLENCODED, FORM-DATA or MTOM) Prepends a new BodyPart using the specified name and uses the input of the Sender as content */ + public void setFirstBodyPartName(String firstBodyPartName) { + this.firstBodyPartName = firstBodyPartName; + } + + /** + * If set and methodType=POST and paramsInUrl=false, a multipart/form-data entity is created instead of a request body. + * For each part element in the session key a part in the multipart entity is created. Part elements can contain the following attributes: + *
    + *
  • name: optional, used as 'filename' in Content-Disposition
    • + *
  • sessionKey: mandatory, refers to contents of part
    • + *
  • mimeType: optional MIME type
    • + *
+ * The name of the part is determined by the name attribute, unless that is empty, or the contents is binary. In those cases the sessionKey name is used as name of the part. + */ + public void setMultipartXmlSessionKey(String multipartXmlSessionKey) { + this.multipartXmlSessionKey = multipartXmlSessionKey; + } + + public void setMtomContentTransferEncoding(String mtomContentTransferEncoding) { + this.mtomContentTransferEncoding = mtomContentTransferEncoding; + } + + /** + * Specifies whether messages will encoded, e.g. spaces will be replaced by '+' etc. + * @ff.default false + */ + public void setEncodeMessages(boolean b) { + encodeMessages = b; + } + + /** + * If true, the input will be added to the URL for methodType=GET, or for methodType=POST, PUT or PATCH if postType=RAW. This used to be the default behaviour in framework version 7.7 and earlier + * @ff.default for methodType=GET: false,
for methodTypes POST, PUT, PATCH: true + */ + public void setTreatInputMessageAsParameters(Boolean b) { + treatInputMessageAsParameters = b; + } +} \ No newline at end of file diff --git a/src/main/java/org/frankframework/http/HttpSenderCached.java-orig b/src/main/java/org/frankframework/http/HttpSenderCached.java-orig new file mode 100644 index 000000000..df1ba4bd5 --- /dev/null +++ b/src/main/java/org/frankframework/http/HttpSenderCached.java-orig @@ -0,0 +1,580 @@ +/* + Copyright 2013, 2016-2020 Nationale-Nederlanden, 2020-2023 WeAreFrank! + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package org.frankframework.http; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URLEncoder; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import javax.servlet.http.HttpServletResponse; + +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.http.Header; +import org.apache.http.HttpEntity; +import org.apache.http.NameValuePair; +import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpEntityEnclosingRequestBase; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpHead; +import org.apache.http.client.methods.HttpPatch; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpPut; +import org.apache.http.client.methods.HttpRequestBase; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.entity.ByteArrayEntity; +import org.apache.http.entity.mime.FormBodyPart; +import org.apache.http.entity.mime.FormBodyPartBuilder; +import org.apache.http.entity.mime.MIME; +import org.apache.http.message.BasicNameValuePair; +import org.apache.logging.log4j.Logger; +import org.frankframework.configuration.ConfigurationException; +import org.frankframework.configuration.ConfigurationWarnings; +import org.frankframework.configuration.SuppressKeys; +import org.frankframework.core.PipeLineSession; +import org.frankframework.core.SenderException; +import org.frankframework.http.mime.MessageContentBody; +import org.frankframework.http.mime.MultipartEntityBuilder; +import org.frankframework.parameters.ParameterValue; +import org.frankframework.parameters.ParameterValueList; +import org.frankframework.stream.Message; +import org.frankframework.util.ClassUtils; +import org.frankframework.util.DomBuilderException; +import org.frankframework.util.StreamUtil; +import org.frankframework.util.XmlBuilder; +import org.frankframework.util.XmlUtils; +import org.springframework.http.MediaType; +import org.springframework.util.MimeType; +import org.w3c.dom.Element; +import org.w3c.dom.Node; + +import jakarta.mail.BodyPart; +import jakarta.mail.MessagingException; +import jakarta.mail.internet.MimeMultipart; +import lombok.Getter; + +/** + * Sender for the HTTP protocol using {@link HttpMethod HttpMethod}. By default, any response code outside the 2xx or 3xx range + * is considered an error and the exception forward of the SenderPipe is followed if present and if there + * is no forward for the specific HTTP status code. Forwards for specific HTTP codes (e.g. "200", "201", ...) + * are returned by this sender, so they are available to the SenderPipe. + * + *

Expected message format:

+ *

GET methods expect a message looking like this: + * 

+ *    param_name=param_value&another_param_name=another_param_value
+ *
+ *

POST AND PUT methods expect a message similar as GET, or looking like this: + * 

+ *   param_name=param_value
+ *   another_param_name=another_param_value
+ *
+ * + * Note: + * When used as MTOM sender and MTOM receiver doesn't support Content-Transfer-Encoding "base64", messages without line feeds will give an error. + * This can be fixed by setting the Content-Transfer-Encoding in the MTOM sender. + *

+ * + * @author Niels Meijer + * @since 7.0 + * @version 2.0 + */ +public class HttpSender extends HttpSenderBase { + + private @Getter boolean paramsInUrl=true; + private @Getter String firstBodyPartName=null; + + private @Getter String multipartXmlSessionKey; + private @Getter String mtomContentTransferEncoding = null; //Defaults to 8-bit for normal String messages, 7-bit for e-mails and binary for streams + private @Getter boolean encodeMessages = false; + private @Getter Boolean treatInputMessageAsParameters = null; + + private @Getter PostType postType = PostType.RAW; + + private static final MimeType APPLICATION_XOP_XML = MimeType.valueOf("application/xop+xml"); + + public enum PostType { + /** The input message is sent unchanged as character data, like text, XML or JSON, with possibly parameter data appended */ + RAW, // text/html;charset=UTF8 + /** The input message is sent unchanged as binary data */ + BINARY, //application/octet-stream +// SWA("Soap with Attachments"), // text/xml + /** Yields a x-www-form-urlencoded form entity */ + URLENCODED, + /** Yields a multipart/form-data form entity */ + FORMDATA, + /** Yields a MTOM multipart/related form entity */ + MTOM; + } + + @Override + public void configure() throws ConfigurationException { + //For backwards compatibility we have to set the contentType to text/html on POST and PUT requests + if(StringUtils.isEmpty(getContentType()) && postType == PostType.RAW && (getHttpMethod() == HttpMethod.POST || getHttpMethod() == HttpMethod.PUT || getHttpMethod() == HttpMethod.PATCH)) { + setContentType("text/html"); + } + + super.configure(); + + if (getTreatInputMessageAsParameters()==null && getHttpMethod()!=HttpMethod.GET) { + setTreatInputMessageAsParameters(Boolean.TRUE); + } + + if (getHttpMethod() != HttpMethod.POST) { + if (!isParamsInUrl()) { + throw new ConfigurationException(getLogPrefix()+"paramsInUrl can only be set to false for methodType POST"); + } + if (StringUtils.isNotEmpty(getFirstBodyPartName())) { + throw new ConfigurationException(getLogPrefix()+"firstBodyPartName can only be set for methodType POST"); + } + } + } + + @Override + protected HttpRequestBase getMethod(URI url, Message message, ParameterValueList parameters, PipeLineSession session) throws SenderException { + if (isEncodeMessages() && !Message.isEmpty(message)) { + try { + message = new Message(URLEncoder.encode(message.asString(), getCharSet())); + } catch (IOException e) { + throw new SenderException(getLogPrefix()+"unable to encode message",e); + } + } + + URI uri; + try { + uri = encodeQueryParameters(url); + } catch (UnsupportedEncodingException | URISyntaxException e) { + throw new SenderException("error encoding queryparameters in url ["+url.toString()+"]", e); + } + + if(postType==PostType.URLENCODED || postType==PostType.FORMDATA || postType==PostType.MTOM) { + try { + return getMultipartPostMethodWithParamsInBody(uri, message, parameters, session); + } catch (IOException e) { + throw new SenderException(getLogPrefix()+"unable to read message", e); + } + } + // RAW + BINARY + return getMethod(uri, message, parameters); + } + + // Encode query parameter values. + private URI encodeQueryParameters(URI url) throws UnsupportedEncodingException, URISyntaxException { + URIBuilder uri = new URIBuilder(url); + ArrayList pairs = new ArrayList<>(uri.getQueryParams().size()); + for(NameValuePair pair : uri.getQueryParams()) { + String paramValue = pair.getValue(); //May be NULL + if(StringUtils.isNotEmpty(paramValue)) { + paramValue = URLEncoder.encode(paramValue, getCharSet()); //Only encode if the value is not null + } + pairs.add(new BasicNameValuePair(pair.getName(), paramValue)); + } + if(!pairs.isEmpty()) { + uri.clearParameters(); + uri.addParameters(pairs); + } + return uri.build(); + } + + /** + * Returns HttpRequestBase, with (optional) RAW or as BINAIRY content + */ + protected HttpRequestBase getMethod(URI uri, Message message, ParameterValueList parameters) throws SenderException { + try { + boolean queryParametersAppended = false; + StringBuilder relativePath = new StringBuilder(uri.getRawPath()); + if (!StringUtils.isEmpty(uri.getQuery())) { + relativePath.append("?").append(uri.getQuery()); + queryParametersAppended = true; + } + + switch (getHttpMethod()) { + case GET: + if (parameters!=null) { + queryParametersAppended = appendParameters(queryParametersAppended,relativePath,parameters); + if (log.isDebugEnabled()) log.debug(getLogPrefix()+"path after appending of parameters ["+relativePath+"]"); + } + + HttpGet getMethod = new HttpGet(relativePath+(parameters==null && BooleanUtils.isTrue(getTreatInputMessageAsParameters()) && !Message.isEmpty(message)? message.asString():"")); + + if (log.isDebugEnabled()) log.debug(getLogPrefix()+"HttpSender constructed GET-method ["+getMethod.getURI().getQuery()+"]"); + if (null != getFullContentType()) { //Manually set Content-Type header + getMethod.setHeader("Content-Type", getFullContentType().toString()); + } + return getMethod; + + case POST: + case PUT: + case PATCH: + HttpEntity entity; + if(postType == PostType.RAW) { + String messageString = BooleanUtils.isTrue(getTreatInputMessageAsParameters()) && !Message.isEmpty(message) ? message.asString() : ""; + if (parameters!=null) { + StringBuilder msg = new StringBuilder(messageString); + appendParameters(true,msg,parameters); + if (StringUtils.isEmpty(messageString) && msg.length()>1) { + messageString=msg.substring(1); + } else { + messageString=msg.toString(); + } + } + entity = new ByteArrayEntity(messageString.getBytes(StreamUtil.DEFAULT_INPUT_STREAM_ENCODING), getFullContentType()); + } else if(postType == PostType.BINARY) { + entity = new HttpMessageEntity(message, getFullContentType()); + } else { + throw new SenderException("PostType ["+postType.name()+"] not allowed!"); + } + + HttpEntityEnclosingRequestBase method; + if (getHttpMethod() == HttpMethod.POST) { + method = new HttpPost(relativePath.toString()); + } else if (getHttpMethod() == HttpMethod.PATCH) { + method = new HttpPatch(relativePath.toString()); + } else { + method = new HttpPut(relativePath.toString()); + } + + method.setEntity(entity); + return method; + + case DELETE: + HttpDelete deleteMethod = new HttpDelete(relativePath.toString()); + if (null != getFullContentType()) { //Manually set Content-Type header + deleteMethod.setHeader("Content-Type", getFullContentType().toString()); + } + return deleteMethod; + + case HEAD: + return new HttpHead(relativePath.toString()); + + case REPORT: + Element element = XmlUtils.buildElement(message.asString(), true); + HttpReport reportMethod = new HttpReport(relativePath.toString(), element); + if (null != getFullContentType()) { //Manually set Content-Type header + reportMethod.setHeader("Content-Type", getFullContentType().toString()); + } + return reportMethod; + + default: + return null; + } + } catch (Exception e) { + //Catch all exceptions and throw them as SenderException + throw new SenderException(e); + } + } + + /** + * Returns a multi-parted message, either as X-WWW-FORM-URLENCODED, FORM-DATA or MTOM + */ + private HttpPost getMultipartPostMethodWithParamsInBody(URI uri, Message message, ParameterValueList parameters, PipeLineSession session) throws SenderException, IOException { + HttpPost hmethod = new HttpPost(uri); + + if (postType==PostType.URLENCODED && StringUtils.isEmpty(getMultipartXmlSessionKey())) { // x-www-form-urlencoded + List requestFormElements = new ArrayList<>(); + + if (StringUtils.isNotEmpty(getFirstBodyPartName())) { + requestFormElements.add(new BasicNameValuePair(getFirstBodyPartName(), message.asString())); + log.debug(getLogPrefix()+"appended parameter ["+getFirstBodyPartName()+"] with value ["+message+"]"); + } + if (parameters!=null) { + for(ParameterValue pv : parameters) { + String name = pv.getDefinition().getName(); + String value = pv.asStringValue(""); + + if (requestOrBodyParamsSet.contains(name) && (StringUtils.isNotEmpty(value) || !parametersToSkipWhenEmptySet.contains(name))) { + requestFormElements.add(new BasicNameValuePair(name,value)); + if (log.isDebugEnabled()) log.debug(getLogPrefix()+"appended parameter ["+name+"] with value ["+value+"]"); + } + } + } + try { + hmethod.setEntity(new UrlEncodedFormEntity(requestFormElements, getCharSet())); + } catch (UnsupportedEncodingException e) { + throw new SenderException(getLogPrefix()+"unsupported encoding for one or more POST parameters", e); + } + } + else { //formdata and mtom + HttpEntity requestEntity = createMultiPartEntity(message, parameters, session); + hmethod.setEntity(requestEntity); + } + + return hmethod; + } + + private FormBodyPart createStringBodypart(Message message) { + MimeType mimeType = (postType == PostType.MTOM) ? APPLICATION_XOP_XML : MediaType.TEXT_PLAIN; // only the first part is XOP+XML, other parts should use their own content-type + FormBodyPartBuilder bodyPart = FormBodyPartBuilder.create(getFirstBodyPartName(), new MessageContentBody(message, mimeType)); + + // Should only be set when request is MTOM and it's the first BodyPart + if (postType == PostType.MTOM && StringUtils.isNotEmpty(getMtomContentTransferEncoding())) { + bodyPart.setField(MIME.CONTENT_TRANSFER_ENC, getMtomContentTransferEncoding()); + } + + return bodyPart.build(); + } + + private HttpEntity createMultiPartEntity(Message message, ParameterValueList parameters, PipeLineSession session) throws SenderException, IOException { + MultipartEntityBuilder entity = MultipartEntityBuilder.create(); + + entity.setCharset(Charset.forName(getCharSet())); + if(postType == PostType.MTOM) + entity.setMtomMultipart(); + + if (StringUtils.isNotEmpty(getFirstBodyPartName())) { + entity.addPart(createStringBodypart(message)); + if (log.isDebugEnabled()) log.debug(getLogPrefix()+"appended stringpart ["+getFirstBodyPartName()+"] with value ["+message+"]"); + } + if (parameters!=null) { + for(ParameterValue pv : parameters) { + String name = pv.getDefinition().getName(); + if (requestOrBodyParamsSet.contains(name)) { + Message msg = pv.asMessage(); + if (!msg.isEmpty() || !parametersToSkipWhenEmptySet.contains(name)) { + + String fileName = null; + String sessionKey = pv.getDefinition().getSessionKey(); + if (sessionKey != null) { + fileName = session.getString(sessionKey + "Name"); + } + if(fileName != null) { + log.warn("setting filename using [{}Name] for bodypart [{}]. Consider using a MultipartXml with the attribute [name] instead.", sessionKey, fileName, name); + } + + entity.addPart(name, new MessageContentBody(msg, null, fileName)); + if (log.isDebugEnabled()) log.debug("{}appended bodypart [{}] with message [{}]", getLogPrefix(), name, msg); + } + } + } + } + + if (StringUtils.isNotEmpty(getMultipartXmlSessionKey())) { + String multipartXml = session.getString(getMultipartXmlSessionKey()); + log.debug(getLogPrefix()+"building multipart message with MultipartXmlSessionKey ["+multipartXml+"]"); + if (StringUtils.isEmpty(multipartXml)) { + log.warn(getLogPrefix()+"sessionKey [" +getMultipartXmlSessionKey()+"] is empty"); + } else { + Element partsElement; + try { + partsElement = XmlUtils.buildElement(multipartXml); + } catch (DomBuilderException e) { + throw new SenderException(getLogPrefix()+"error building multipart xml", e); + } + Collection parts = XmlUtils.getChildTags(partsElement, "part"); + if (parts.isEmpty()) { + log.warn(getLogPrefix()+"no part(s) in multipart xml [" + multipartXml + "]"); + } else { + for (final Node part : parts) { + Element partElement = (Element) part; + entity.addPart(elementToFormBodyPart(partElement, session)); + } + } + } + } + return entity.build(); + } + + protected FormBodyPart elementToFormBodyPart(Element element, PipeLineSession session) throws IOException { + String part = element.getAttribute("name"); //Name of the part + boolean isFile = "file".equals(element.getAttribute("type")); //text of file, empty == text + String filename = element.getAttribute("filename"); //if type == file, the filename + String partSessionKey = element.getAttribute("sessionKey"); //SessionKey to retrieve data from + String partMimeType = element.getAttribute("mimeType"); //MimeType of the part + Message partObject = session.getMessage(partSessionKey); + MimeType mimeType = null; + if(StringUtils.isNotEmpty(partMimeType)) { + mimeType = MimeType.valueOf(partMimeType); + } + + final String filenameToUse; + if(isFile || StringUtils.isNotBlank(filename)) { + String filenamebackup = StringUtils.isBlank(part) ? partSessionKey : part; + filenameToUse = StringUtils.isNotBlank(filename) ? filename : filenamebackup; + } else { + filenameToUse = null; + } + + String partname = isFile || StringUtils.isBlank(part) ? partSessionKey : part; + return FormBodyPartBuilder.create(partname, new MessageContentBody(partObject, mimeType, filenameToUse)).build(); + } + + @Override + protected Message extractResult(HttpResponseHandler responseHandler, PipeLineSession session) throws SenderException, IOException { + int statusCode = responseHandler.getStatusLine().getStatusCode(); + + if (!validateResponseCode(statusCode)) { + Message responseBody = responseHandler.getResponseMessage(); + String body = ""; + if(responseBody != null) { + responseBody.preserve(); + try { + body = responseBody.asString(); + } catch(IOException e) { + body = "(" + ClassUtils.nameOf(e) + "): " + e.getMessage(); + } + } + log.warn(getLogPrefix() + "httpstatus [" + statusCode + "] reason [" + responseHandler.getStatusLine().getReasonPhrase() + "]"); + return new Message(body); + } + + Message responseMessage = responseHandler.getResponseMessage(); + if (!Message.isEmpty(responseMessage)) { + responseMessage.closeOnCloseOf(session, this); + } + + if (responseHandler.isMultipart()) { + return handleMultipartResponse(responseHandler, session); + } else { + return getResponseBody(responseHandler); + } + } + + public Message getResponseBody(HttpResponseHandler responseHandler) { + if (getHttpMethod() == HttpMethod.HEAD) { + XmlBuilder headersXml = new XmlBuilder("headers"); + Header[] headers = responseHandler.getAllHeaders(); + for (Header header : headers) { + XmlBuilder headerXml = new XmlBuilder("header"); + headerXml.addAttribute("name", header.getName()); + headerXml.setCdataValue(header.getValue()); + headersXml.addSubElement(headerXml); + } + return Message.asMessage(headersXml.toXML()); + } + + return responseHandler.getResponseMessage(); + } + + /** + * return the first part as Message and put the other parts as InputStream in the PipeLineSession + */ + private static Message handleMultipartResponse(HttpResponseHandler httpHandler, PipeLineSession session) throws IOException { + return handleMultipartResponse(httpHandler.getContentType().getMimeType(), httpHandler.getResponse(), session); + } + + /** + * return the first part as Message and put the other parts as InputStream in the PipeLineSession + */ + public static Message handleMultipartResponse(String mimeType, InputStream inputStream, PipeLineSession session) throws IOException { + Message result = null; + try { + InputStreamDataSource dataSource = new InputStreamDataSource(mimeType, inputStream); //the entire InputStream will be read here! + MimeMultipart mimeMultipart = new MimeMultipart(dataSource); + for (int i = 0; i < mimeMultipart.getCount(); i++) { + BodyPart bodyPart = mimeMultipart.getBodyPart(i); + if (i == 0) { + result = new PartMessage(bodyPart); + } else { + session.put("multipart" + i, new PartMessage(bodyPart)); + } + } + } catch(MessagingException e) { + throw new IOException("Could not read mime multipart response", e); + } + return result; + } + + public static void streamResponseBody(InputStream is, String contentType, String contentDisposition, HttpServletResponse response, Logger log, String logPrefix, String redirectLocation) throws IOException { + if (StringUtils.isNotEmpty(contentType)) { + response.setHeader("Content-Type", contentType); + } + if (StringUtils.isNotEmpty(contentDisposition)) { + response.setHeader("Content-Disposition", contentDisposition); + } + if (StringUtils.isNotEmpty(redirectLocation)) { + response.sendRedirect(redirectLocation); + } + if (is != null) { + try (OutputStream outputStream = response.getOutputStream()) { + StreamUtil.streamToStream(is, outputStream); + log.debug(logPrefix + "copied response body input stream [" + is + "] to output stream [" + outputStream + "]"); + } + } + } + + /** + * If methodType=POST, PUT or PATCH, the type of post request + * @ff.default RAW + */ + public void setPostType(PostType type) { + this.postType = type; + } + + /** + * If false and methodType=POST, request parameters are put in the request body instead of in the url + * @ff.default true + */ + @Deprecated + public void setParamsInUrl(boolean b) { + if(!b) { + if(postType != PostType.MTOM && postType != PostType.FORMDATA) { //Don't override if another type has explicitly been set + postType = PostType.URLENCODED; + ConfigurationWarnings.add(this, log, "attribute [paramsInUrl] is deprecated: please use postType='URLENCODED' instead", SuppressKeys.DEPRECATION_SUPPRESS_KEY, null); + } else { + ConfigurationWarnings.add(this, log, "attribute [paramsInUrl] is deprecated: no longer required when using FORMDATA or MTOM requests", SuppressKeys.DEPRECATION_SUPPRESS_KEY, null); + } + } + paramsInUrl = b; + } + + /** (Only used when methodType=POST and postType=URLENCODED, FORM-DATA or MTOM) Prepends a new BodyPart using the specified name and uses the input of the Sender as content */ + public void setFirstBodyPartName(String firstBodyPartName) { + this.firstBodyPartName = firstBodyPartName; + } + + /** + * If set and methodType=POST and paramsInUrl=false, a multipart/form-data entity is created instead of a request body. + * For each part element in the session key a part in the multipart entity is created. Part elements can contain the following attributes: + *
    + *
  • name: optional, used as 'filename' in Content-Disposition
    • + *
  • sessionKey: mandatory, refers to contents of part
    • + *
  • mimeType: optional MIME type
    • + *
+ * The name of the part is determined by the name attribute, unless that is empty, or the contents is binary. In those cases the sessionKey name is used as name of the part. + */ + public void setMultipartXmlSessionKey(String multipartXmlSessionKey) { + this.multipartXmlSessionKey = multipartXmlSessionKey; + } + + public void setMtomContentTransferEncoding(String mtomContentTransferEncoding) { + this.mtomContentTransferEncoding = mtomContentTransferEncoding; + } + + /** + * Specifies whether messages will encoded, e.g. spaces will be replaced by '+' etc. + * @ff.default false + */ + public void setEncodeMessages(boolean b) { + encodeMessages = b; + } + + /** + * If true, the input will be added to the URL for methodType=GET, or for methodType=POST, PUT or PATCH if postType=RAW. This used to be the default behaviour in framework version 7.7 and earlier + * @ff.default for methodType=GET: false,
for methodTypes POST, PUT, PATCH: true + */ + public void setTreatInputMessageAsParameters(Boolean b) { + treatInputMessageAsParameters = b; + } +} \ No newline at end of file