From f9e250f44e834c06628f2f95d3509a6fdbcb1d38 Mon Sep 17 00:00:00 2001 From: Nikolas Falco Date: Mon, 6 Jan 2025 04:11:23 +0100 Subject: [PATCH] Update Apache HTTP Client to version 5.4 WIP --- pom.xml | 20 +-- .../bitbucket/api/BitbucketAuthenticator.java | 8 +- .../client/BitbucketCloudApiClient.java | 42 +++-- .../client/ClosingConnectionInputStream.java | 16 +- .../impl/client/AbstractBitbucketApi.java | 106 ++++++------ .../ExponentialBackOffRetryStrategy.java | 159 ++++++++++++++++++ ...ackOffServiceUnavailableRetryStrategy.java | 114 ------------- .../BitbucketAccessTokenAuthenticator.java | 5 +- ...tbucketClientCertificateAuthenticator.java | 26 ++- .../BitbucketOAuthAuthenticator.java | 2 +- ...itbucketUsernamePasswordAuthenticator.java | 26 +-- .../impl/util/BitbucketApiUtils.java | 20 +++ .../client/BitbucketServerAPIClient.java | 28 +-- ...ffServiceUnavailableRetryStrategyTest.java | 2 +- 14 files changed, 321 insertions(+), 253 deletions(-) create mode 100644 src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/client/ExponentialBackOffRetryStrategy.java delete mode 100644 src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/client/ExponentialBackOffServiceUnavailableRetryStrategy.java diff --git a/pom.xml b/pom.xml index e2f9e4e99..16f9d1a84 100644 --- a/pom.xml +++ b/pom.xml @@ -76,8 +76,8 @@ - org.jenkins-ci.plugins - apache-httpcomponents-client-4-api + io.jenkins.plugins + apache-httpcomponents-client-5-api org.jenkins-ci.plugins @@ -92,6 +92,12 @@ git 5.7.0 + + + org.jenkins-ci.plugins + apache-httpcomponents-client-4-api + + org.jenkins-ci.plugins @@ -233,16 +239,6 @@ - - org.kohsuke - access-modifier-checker - - - default-enforce - - - - com.infradna.tool bridge-method-injector diff --git a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/api/BitbucketAuthenticator.java b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/api/BitbucketAuthenticator.java index 80c9af0f8..1aac369b1 100644 --- a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/api/BitbucketAuthenticator.java +++ b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/api/BitbucketAuthenticator.java @@ -28,10 +28,10 @@ import com.cloudbees.plugins.credentials.common.StandardUsernameCredentials; import hudson.plugins.git.GitSCM; import jenkins.authentication.tokens.api.AuthenticationTokenContext; -import org.apache.http.HttpHost; -import org.apache.http.HttpRequest; -import org.apache.http.client.protocol.HttpClientContext; -import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.hc.client5.http.impl.classic.HttpClientBuilder; +import org.apache.hc.client5.http.protocol.HttpClientContext; +import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.http.HttpRequest; /** * Support for various different methods of authenticating with Bitbucket diff --git a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/client/BitbucketCloudApiClient.java b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/client/BitbucketCloudApiClient.java index d1d5a7752..de3bcf07c 100644 --- a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/client/BitbucketCloudApiClient.java +++ b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/client/BitbucketCloudApiClient.java @@ -49,6 +49,7 @@ import com.cloudbees.jenkins.plugins.bitbucket.filesystem.BitbucketSCMFile; import com.cloudbees.jenkins.plugins.bitbucket.impl.client.AbstractBitbucketApi; import com.cloudbees.jenkins.plugins.bitbucket.impl.credentials.BitbucketUsernamePasswordAuthenticator; +import com.cloudbees.jenkins.plugins.bitbucket.impl.util.BitbucketApiUtils; import com.cloudbees.jenkins.plugins.bitbucket.impl.util.JsonParser; import com.cloudbees.plugins.credentials.common.StandardUsernamePasswordCredentials; import com.damnhandy.uri.template.UriTemplate; @@ -69,24 +70,26 @@ import java.util.List; import java.util.Map; import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; import java.util.logging.Level; import javax.imageio.ImageIO; import jenkins.scm.api.SCMFile; import org.apache.commons.lang.StringUtils; -import org.apache.http.HttpHost; -import org.apache.http.HttpStatus; -import org.apache.http.config.SocketConfig; -import org.apache.http.conn.HttpClientConnectionManager; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; -import org.apache.http.message.BasicNameValuePair; +import org.apache.hc.client5.http.config.ConnectionConfig; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder; +import org.apache.hc.client5.http.io.HttpClientConnectionManager; +import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.http.HttpStatus; +import org.apache.hc.core5.http.io.SocketConfig; +import org.apache.hc.core5.http.message.BasicNameValuePair; import static java.util.concurrent.TimeUnit.HOURS; import static java.util.concurrent.TimeUnit.MINUTES; public class BitbucketCloudApiClient extends AbstractBitbucketApi implements BitbucketApi { - private static final HttpHost API_HOST = HttpHost.create("https://api.bitbucket.org"); + private static final HttpHost API_HOST = BitbucketApiUtils.toHttpHost("https://api.bitbucket.org"); private static final String V2_API_BASE_URL = "https://api.bitbucket.org/2.0/repositories"; private static final String V2_WORKSPACES_API_BASE_URL = "https://api.bitbucket.org/2.0/workspaces"; private static final String REPO_URL_TEMPLATE = V2_API_BASE_URL + "{/owner,repo}"; @@ -109,11 +112,24 @@ public class BitbucketCloudApiClient extends AbstractBitbucketApi implements Bit private static HttpClientConnectionManager connectionManager() { try { - PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(); // NOSONAR - connManager.setDefaultMaxPerRoute(20); - connManager.setMaxTotal(22); - connManager.setSocketConfig(API_HOST, SocketConfig.custom().setSoTimeout(60 * 1000).build()); - return connManager; + int connectTimeout = Integer.getInteger("http.connect.timeout", 10); + int socketTimeout = Integer.getInteger("http.socket.timeout", 60); + + ConnectionConfig connCfg = ConnectionConfig.custom() + .setConnectTimeout(connectTimeout, TimeUnit.SECONDS) + .setSocketTimeout(socketTimeout, TimeUnit.SECONDS) + .build(); + + SocketConfig socketConfig = SocketConfig.custom() + .setSoTimeout(60, TimeUnit.SECONDS) + .build(); + + return PoolingHttpClientConnectionManagerBuilder.create() + .setMaxConnPerRoute(20) + .setMaxConnTotal(22) + .setDefaultConnectionConfig(connCfg) + .setSocketConfigResolver(host -> host.getTargetHost().equals(API_HOST) ? socketConfig : SocketConfig.DEFAULT) + .build(); } catch (Exception e) { // in case of exception this avoids ClassNotFoundError which prevents the classloader from loading this class again return null; diff --git a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/client/ClosingConnectionInputStream.java b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/client/ClosingConnectionInputStream.java index 2d028532e..62de5626b 100644 --- a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/client/ClosingConnectionInputStream.java +++ b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/client/ClosingConnectionInputStream.java @@ -2,22 +2,22 @@ import java.io.IOException; import java.io.InputStream; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpRequestBase; -import org.apache.http.conn.HttpClientConnectionManager; -import org.apache.http.util.EntityUtils; +import org.apache.hc.client5.http.classic.methods.HttpUriRequest; +import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; +import org.apache.hc.client5.http.io.HttpClientConnectionManager; +import org.apache.hc.core5.http.io.entity.EntityUtils; public class ClosingConnectionInputStream extends InputStream { private final CloseableHttpResponse response; - private final HttpRequestBase method; + private final HttpUriRequest method; private final HttpClientConnectionManager connectionManager; private final InputStream delegate; - public ClosingConnectionInputStream(final CloseableHttpResponse response, final HttpRequestBase method, + public ClosingConnectionInputStream(final CloseableHttpResponse response, final HttpUriRequest method, final HttpClientConnectionManager connectionmanager) throws UnsupportedOperationException, IOException { this.response = response; @@ -35,8 +35,8 @@ public int available() throws IOException { public void close() throws IOException { EntityUtils.consume(response.getEntity()); delegate.close(); - method.releaseConnection(); - connectionManager.closeExpiredConnections(); +//TODO method.releaseConnection(); +//TODO connectionManager.closeExpiredConnections(); } @Override diff --git a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/client/AbstractBitbucketApi.java b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/client/AbstractBitbucketApi.java index 4787827af..43f429a87 100644 --- a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/client/AbstractBitbucketApi.java +++ b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/client/AbstractBitbucketApi.java @@ -45,35 +45,35 @@ import jenkins.model.Jenkins; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; -import org.apache.http.Header; -import org.apache.http.HttpHost; -import org.apache.http.HttpStatus; -import org.apache.http.NameValuePair; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.AuthCache; -import org.apache.http.client.CredentialsProvider; -import org.apache.http.client.ServiceUnavailableRetryStrategy; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.entity.UrlEncodedFormEntity; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpDelete; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpHead; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.methods.HttpPut; -import org.apache.http.client.methods.HttpRequestBase; -import org.apache.http.client.protocol.HttpClientContext; -import org.apache.http.conn.HttpClientConnectionManager; -import org.apache.http.entity.ContentType; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.auth.BasicScheme; -import org.apache.http.impl.client.BasicAuthCache; -import org.apache.http.impl.client.BasicCredentialsProvider; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.impl.client.StandardHttpRequestRetryHandler; -import org.apache.http.util.EntityUtils; +import org.apache.hc.client5.http.auth.AuthCache; +import org.apache.hc.client5.http.auth.AuthScope; +import org.apache.hc.client5.http.auth.CredentialsProvider; +import org.apache.hc.client5.http.auth.CredentialsStore; +import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; +import org.apache.hc.client5.http.classic.methods.HttpDelete; +import org.apache.hc.client5.http.classic.methods.HttpGet; +import org.apache.hc.client5.http.classic.methods.HttpHead; +import org.apache.hc.client5.http.classic.methods.HttpPost; +import org.apache.hc.client5.http.classic.methods.HttpPut; +import org.apache.hc.client5.http.classic.methods.HttpUriRequest; +import org.apache.hc.client5.http.config.RequestConfig; +import org.apache.hc.client5.http.entity.UrlEncodedFormEntity; +import org.apache.hc.client5.http.impl.auth.BasicAuthCache; +import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider; +import org.apache.hc.client5.http.impl.auth.BasicScheme; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; +import org.apache.hc.client5.http.impl.classic.HttpClientBuilder; +import org.apache.hc.client5.http.io.HttpClientConnectionManager; +import org.apache.hc.client5.http.protocol.HttpClientContext; +import org.apache.hc.core5.http.ContentType; +import org.apache.hc.core5.http.Header; +import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.http.HttpStatus; +import org.apache.hc.core5.http.NameValuePair; +import org.apache.hc.core5.http.io.entity.EntityUtils; +import org.apache.hc.core5.http.io.entity.StringEntity; +import org.apache.hc.core5.util.TimeValue; import org.kohsuke.accmod.Restricted; import org.kohsuke.accmod.restrictions.ProtectedExternally; @@ -98,9 +98,9 @@ protected String truncateMiddle(@CheckForNull String value, int maxLength) { } protected BitbucketRequestException buildResponseException(CloseableHttpResponse response, String errorMessage) { - String headers = StringUtils.join(response.getAllHeaders(), "\n"); - String message = String.format("HTTP request error.%nStatus: %s%nResponse: %s%n%s", response.getStatusLine(), errorMessage, headers); - return new BitbucketRequestException(response.getStatusLine().getStatusCode(), message); + String headers = StringUtils.join(response.getHeaders(), "\n"); + String message = String.format("HTTP request error.%nStatus: %s%nResponse: %s%n%s", response.getReasonPhrase(), errorMessage, headers); + return new BitbucketRequestException(response.getCode(), message); } protected String getResponseContent(CloseableHttpResponse response) throws IOException { @@ -139,25 +139,21 @@ private long getLenghtFromHeader(CloseableHttpResponse response) { } protected HttpClientBuilder setupClientBuilder(@Nullable String host) { - int connectTimeout = Integer.getInteger("http.connect.timeout", 10); int connectionRequestTimeout = Integer.getInteger("http.connect.request.timeout", 60); - int socketTimeout = Integer.getInteger("http.socket.timeout", 60); RequestConfig config = RequestConfig.custom() - .setConnectTimeout(connectTimeout * 1000) - .setConnectionRequestTimeout(connectionRequestTimeout * 1000) - .setSocketTimeout(socketTimeout * 1000) + .setConnectionRequestTimeout(connectionRequestTimeout, TimeUnit.SECONDS) .build(); HttpClientConnectionManager connectionManager = getConnectionManager(); - ServiceUnavailableRetryStrategy serviceUnavailableStrategy = new ExponentialBackOffServiceUnavailableRetryStrategy(2, TimeUnit.SECONDS.toMillis(5), TimeUnit.HOURS.toMillis(1)); HttpClientBuilder httpClientBuilder = HttpClientBuilder.create() .useSystemProperties() .setConnectionManager(connectionManager) .setConnectionManagerShared(connectionManager != null) - .setServiceUnavailableRetryStrategy(serviceUnavailableStrategy) - .setRetryHandler(new StandardHttpRequestRetryHandler()) + .setRetryStrategy(new ExponentialBackOffRetryStrategy(2, TimeUnit.SECONDS.toMillis(5), TimeUnit.HOURS.toMillis(1))) .setDefaultRequestConfig(config) + .evictExpiredConnections() + .evictIdleConnections(TimeValue.ofSeconds(2)) .disableCookieManagement(); if (authenticator != null) { @@ -200,7 +196,9 @@ private void setClientProxyParams(String host, HttpClientBuilder builder) { // may have been already set in com.cloudbees.jenkins.plugins.bitbucket.api.credentials.BitbucketUsernamePasswordAuthenticator.configureContext(HttpClientContext, HttpHost) context.setCredentialsProvider(credentialsProvider); } - credentialsProvider.setCredentials(new AuthScope(proxyHttpHost), new UsernamePasswordCredentials(username, password)); + if (credentialsProvider instanceof CredentialsStore credentialsStore) { + credentialsStore.setCredentials(new AuthScope(proxyHttpHost), new UsernamePasswordCredentials(username, password.toCharArray())); + } AuthCache authCache = context.getAuthCache(); if (authCache == null) { authCache = new BasicAuthCache(); @@ -221,7 +219,7 @@ private void setClientProxyParams(String host, HttpClientBuilder builder) { protected abstract CloseableHttpClient getClient(); protected CloseableHttpResponse executeMethod(HttpHost host, - HttpRequestBase httpMethod, + HttpUriRequest httpMethod, boolean requireAuthentication) throws IOException { if (requireAuthentication && authenticator != null) { authenticator.configureRequest(httpMethod); @@ -229,15 +227,15 @@ protected CloseableHttpResponse executeMethod(HttpHost host, return getClient().execute(host, httpMethod, context); } - protected CloseableHttpResponse executeMethod(HttpHost host, HttpRequestBase httpMethod) throws IOException { + protected CloseableHttpResponse executeMethod(HttpHost host, HttpUriRequest httpMethod) throws IOException { return executeMethod(host, httpMethod, true); } - protected String doRequest(HttpRequestBase request, boolean requireAuthentication) throws IOException { + protected String doRequest(HttpUriRequest request, boolean requireAuthentication) throws IOException { try (CloseableHttpResponse response = executeMethod(getHost(), request, requireAuthentication)) { - int statusCode = response.getStatusLine().getStatusCode(); + int statusCode = response.getCode(); if (statusCode == HttpStatus.SC_NOT_FOUND) { - throw new FileNotFoundException("URL: " + request.getURI()); + throw new FileNotFoundException("URL: " + request.getRequestUri()); } if (statusCode == HttpStatus.SC_NO_CONTENT) { EntityUtils.consume(response.getEntity()); @@ -254,23 +252,13 @@ protected String doRequest(HttpRequestBase request, boolean requireAuthenticatio throw e; } catch (IOException e) { throw new IOException("Communication error for url: " + request, e); - } finally { - release(request); } } - protected String doRequest(HttpRequestBase request) throws IOException { + protected String doRequest(HttpUriRequest request) throws IOException { return doRequest(request, true); } - private void release(HttpRequestBase method) { - method.releaseConnection(); - HttpClientConnectionManager connectionManager = getConnectionManager(); - if (connectionManager != null) { - connectionManager.closeExpiredConnections(); - } - } - /* * Caller's responsible to close the InputStream. */ @@ -289,7 +277,7 @@ protected InputStream getRequestAsInputStream(String path) throws IOException { } CloseableHttpResponse response = executeMethod(host, httpget); - int statusCode = response.getStatusLine().getStatusCode(); + int statusCode = response.getCode(); if (statusCode == HttpStatus.SC_NOT_FOUND) { EntityUtils.consume(response.getEntity()); throw new FileNotFoundException("URL: " + path); @@ -305,7 +293,7 @@ protected int headRequestStatus(String path) throws IOException { HttpHead httpHead = new HttpHead(path); try (CloseableHttpResponse response = executeMethod(getHost(), httpHead)) { EntityUtils.consume(response.getEntity()); - return response.getStatusLine().getStatusCode(); + return response.getCode(); } catch (IOException e) { throw new IOException("Communication error for url: " + path, e); } finally { diff --git a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/client/ExponentialBackOffRetryStrategy.java b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/client/ExponentialBackOffRetryStrategy.java new file mode 100644 index 000000000..7134dab08 --- /dev/null +++ b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/client/ExponentialBackOffRetryStrategy.java @@ -0,0 +1,159 @@ +package com.cloudbees.jenkins.plugins.bitbucket.impl.client; + +import java.io.IOException; +import java.io.InterruptedIOException; +import java.net.ConnectException; +import java.net.NoRouteToHostException; +import java.net.UnknownHostException; +import java.time.Instant; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import javax.net.ssl.SSLException; +import org.apache.hc.client5.http.impl.DefaultHttpRequestRetryStrategy; +import org.apache.hc.client5.http.utils.DateUtils; +import org.apache.hc.core5.annotation.Contract; +import org.apache.hc.core5.annotation.ThreadingBehavior; +import org.apache.hc.core5.concurrent.CancellableDependency; +import org.apache.hc.core5.http.ConnectionClosedException; +import org.apache.hc.core5.http.Header; +import org.apache.hc.core5.http.HttpHeaders; +import org.apache.hc.core5.http.HttpRequest; +import org.apache.hc.core5.http.HttpResponse; +import org.apache.hc.core5.http.HttpStatus; +import org.apache.hc.core5.http.protocol.HttpContext; +import org.apache.hc.core5.util.Args; +import org.apache.hc.core5.util.TimeValue; + +/** + * An implementation that backs off exponentially based on the number of + * consecutive failed attempts. It uses the following defaults: + * 
+ *         no delay in case it was never tried or didn't fail so far
+ *     6 secs delay for one failed attempt (= {@link #getInitialExpiryInMillis()})
+ *    60 secs delay for two failed attempts
+ *    10 mins delay for three failed attempts
+ *   100 mins delay for four failed attempts
+ *  ~16 hours delay for five failed attempts
+ *   24 hours delay for six or more failed attempts (= {@link #getMaxExpiryInMillis()})
+ *
+ * + * The following equation is used to calculate the delay for a specific revalidation request: + * 
+ *     delay = {@link #getInitialExpiryInMillis()} * Math.pow({@link #getBackOffRate()},
+ *     {@link AsynchronousValidationRequest#getConsecutiveFailedAttempts()} - 1))
+ *
+ * The resulting delay won't exceed {@link #getMaxExpiryInMillis()}. + */ +@Contract(threading = ThreadingBehavior.SAFE) +public class ExponentialBackOffRetryStrategy extends DefaultHttpRequestRetryStrategy { + + public static final long DEFAULT_BACK_OFF_RATE = 10; + public static final long DEFAULT_INITIAL_EXPIRY_IN_MILLIS = TimeUnit.SECONDS.toMillis(6); + public static final long DEFAULT_MAX_EXPIRY_IN_MILLIS = TimeUnit.SECONDS.toMillis(86400); + + private final long backOffRate; + private final long initialExpiryInMillis; + private final long maxExpiryInMillis; + /** + * Derived {@code IOExceptions} which shall not be retried + */ + private final Set> nonRetriableIOExceptionClasses; + + /** + * Create a new strategy using a fixed pool of worker threads. + */ + public ExponentialBackOffRetryStrategy() { + this(DEFAULT_BACK_OFF_RATE, + DEFAULT_INITIAL_EXPIRY_IN_MILLIS, + DEFAULT_MAX_EXPIRY_IN_MILLIS); + } + + /** + * Create a new strategy by using a fixed pool of worker threads and the + * given parameters to calculated the delay. + * + * @param backOffRate the back off rate to be used; not negative + * @param initialExpiryInMillis the initial expiry in milli seconds; not negative + * @param maxExpiryInMillis the upper limit of the delay in milli seconds; not negative + */ + public ExponentialBackOffRetryStrategy( + final long backOffRate, + final long initialExpiryInMillis, + final long maxExpiryInMillis) { + this.backOffRate = Args.notNegative(backOffRate, "BackOffRate"); + this.initialExpiryInMillis = Args.notNegative(initialExpiryInMillis, "InitialExpiryInMillis"); + this.maxExpiryInMillis = Args.notNegative(maxExpiryInMillis, "MaxExpiryInMillis"); + this.nonRetriableIOExceptionClasses = Set.of( + InterruptedIOException.class, + UnknownHostException.class, + ConnectException.class, + ConnectionClosedException.class, + NoRouteToHostException.class, + SSLException.class); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean retryRequest(HttpResponse response, int executionCount, HttpContext context) { + int statusCode = response.getCode(); + return getRetryInterval(executionCount) < maxExpiryInMillis + && (statusCode == HttpStatus.SC_TOO_MANY_REQUESTS + || statusCode == HttpStatus.SC_SERVICE_UNAVAILABLE); + } + + private long getRetryInterval(int failedAttempts) { + if (failedAttempts > 0) { + final long delayInSeconds = (long) (initialExpiryInMillis * Math.pow(backOffRate, failedAttempts - 1)); + return Math.min(delayInSeconds, maxExpiryInMillis); + } else { + return 0; + } + } + + @Override + public boolean retryRequest(HttpRequest request, IOException exception, int execCount, HttpContext context) { + if (this.nonRetriableIOExceptionClasses.contains(exception.getClass())) { + return false; + } else { + for (final Class rejectException : this.nonRetriableIOExceptionClasses) { + if (rejectException.isInstance(exception)) { + return false; + } + } + } + if (request instanceof CancellableDependency cancellable && cancellable.isCancelled()) { + return false; + } + + // Retry if the request is considered idempotent + return handleAsIdempotent(request); + } + + /** + * {@inheritDoc} + */ + @Override + public TimeValue getRetryInterval(HttpResponse response, int execCount, HttpContext context) { + final Header header = response.getFirstHeader(HttpHeaders.RETRY_AFTER); + TimeValue retryAfter = null; + if (header != null) { + final String value = header.getValue(); + try { + retryAfter = TimeValue.ofSeconds(Long.parseLong(value)); + } catch (final NumberFormatException ignore) { + final Instant retryAfterDate = DateUtils.parseStandardDate(value); + if (retryAfterDate != null) { + retryAfter = TimeValue.ofMilliseconds(retryAfterDate.toEpochMilli() - System.currentTimeMillis()); + } + } + + if (TimeValue.isPositive(retryAfter)) { + return retryAfter; + } + } + return TimeValue.ofMilliseconds(getRetryInterval(execCount)); + } + +} diff --git a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/client/ExponentialBackOffServiceUnavailableRetryStrategy.java b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/client/ExponentialBackOffServiceUnavailableRetryStrategy.java deleted file mode 100644 index 78dfb868c..000000000 --- a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/client/ExponentialBackOffServiceUnavailableRetryStrategy.java +++ /dev/null @@ -1,114 +0,0 @@ -package com.cloudbees.jenkins.plugins.bitbucket.impl.client; - -import java.util.concurrent.TimeUnit; -import org.apache.http.HttpResponse; -import org.apache.http.HttpStatus; -import org.apache.http.annotation.Contract; -import org.apache.http.annotation.ThreadingBehavior; -import org.apache.http.client.ServiceUnavailableRetryStrategy; -import org.apache.http.impl.client.cache.AsynchronousValidationRequest; -import org.apache.http.protocol.HttpContext; -import org.apache.http.util.Args; - -/** - * An implementation that backs off exponentially based on the number of - * consecutive failed attempts. It uses the following defaults: - * 
- *         no delay in case it was never tried or didn't fail so far
- *     6 secs delay for one failed attempt (= {@link #getInitialExpiryInMillis()})
- *    60 secs delay for two failed attempts
- *    10 mins delay for three failed attempts
- *   100 mins delay for four failed attempts
- *  ~16 hours delay for five failed attempts
- *   24 hours delay for six or more failed attempts (= {@link #getMaxExpiryInMillis()})
- *
- * - * The following equation is used to calculate the delay for a specific revalidation request: - * 
- *     delay = {@link #getInitialExpiryInMillis()} * Math.pow({@link #getBackOffRate()},
- *     {@link AsynchronousValidationRequest#getConsecutiveFailedAttempts()} - 1))
- *
- * The resulting delay won't exceed {@link #getMaxExpiryInMillis()}. - */ -@Contract(threading = ThreadingBehavior.SAFE) -public class ExponentialBackOffServiceUnavailableRetryStrategy implements ServiceUnavailableRetryStrategy { - - public static final long DEFAULT_BACK_OFF_RATE = 10; - public static final long DEFAULT_INITIAL_EXPIRY_IN_MILLIS = TimeUnit.SECONDS.toMillis(6); - public static final long DEFAULT_MAX_EXPIRY_IN_MILLIS = TimeUnit.SECONDS.toMillis(86400); - - private final long backOffRate; - private final long initialExpiryInMillis; - private final long maxExpiryInMillis; - private ThreadLocal consecutiveFailedAttempts; // TODO call ThreadLocal#remove method is not possible using the lifecycle of apache client 4.x. Move to http client 5.x ASAP - - /** - * Create a new strategy using a fixed pool of worker threads. - */ - public ExponentialBackOffServiceUnavailableRetryStrategy() { - this(DEFAULT_BACK_OFF_RATE, - DEFAULT_INITIAL_EXPIRY_IN_MILLIS, - DEFAULT_MAX_EXPIRY_IN_MILLIS); - } - - /** - * Create a new strategy by using a fixed pool of worker threads and the - * given parameters to calculated the delay. - * - * @param backOffRate the back off rate to be used; not negative - * @param initialExpiryInMillis the initial expiry in milli seconds; not negative - * @param maxExpiryInMillis the upper limit of the delay in milli seconds; not negative - */ - public ExponentialBackOffServiceUnavailableRetryStrategy( - final long backOffRate, - final long initialExpiryInMillis, - final long maxExpiryInMillis) { - this.backOffRate = Args.notNegative(backOffRate, "BackOffRate"); - this.initialExpiryInMillis = Args.notNegative(initialExpiryInMillis, "InitialExpiryInMillis"); - this.maxExpiryInMillis = Args.notNegative(maxExpiryInMillis, "MaxExpiryInMillis"); - this.consecutiveFailedAttempts = new ThreadLocal<>(); - } - - public long getBackOffRate() { - return backOffRate; - } - - public long getInitialExpiryInMillis() { - return initialExpiryInMillis; - } - - public long getMaxExpiryInMillis() { - return maxExpiryInMillis; - } - - /** - * {@inheritDoc} - */ - @Override - public boolean retryRequest(HttpResponse response, int executionCount, HttpContext context) { - int statusCode = response.getStatusLine().getStatusCode(); - consecutiveFailedAttempts.set(executionCount); - return getRetryInterval(executionCount) < maxExpiryInMillis - && (statusCode == HttpStatus.SC_TOO_MANY_REQUESTS - || statusCode == HttpStatus.SC_SERVICE_UNAVAILABLE); - } - - private long getRetryInterval(int failedAttempts) { - if (failedAttempts > 0) { - final long delayInSeconds = (long) (initialExpiryInMillis * Math.pow(backOffRate, failedAttempts - 1)); - return Math.min(delayInSeconds, maxExpiryInMillis); - } else { - return 0; - } - } - - /** - * {@inheritDoc} - */ - @Override - public long getRetryInterval() { - Integer attempts = consecutiveFailedAttempts.get(); - return getRetryInterval(attempts == null ? 0 : attempts.intValue()); - } - -} diff --git a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/credentials/BitbucketAccessTokenAuthenticator.java b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/credentials/BitbucketAccessTokenAuthenticator.java index c0c606cc3..6859e0442 100644 --- a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/credentials/BitbucketAccessTokenAuthenticator.java +++ b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/credentials/BitbucketAccessTokenAuthenticator.java @@ -30,8 +30,8 @@ import com.cloudbees.plugins.credentials.impl.UsernamePasswordCredentialsImpl; import hudson.model.Descriptor.FormException; import hudson.util.Secret; -import org.apache.http.HttpHeaders; -import org.apache.http.HttpRequest; +import org.apache.hc.core5.http.HttpHeaders; +import org.apache.hc.core5.http.HttpRequest; import org.jenkinsci.plugins.plaincredentials.StringCredentials; /** @@ -57,6 +57,7 @@ public BitbucketAccessTokenAuthenticator(StringCredentials credentials) { * * @param request to configure with the access token */ + @Override public void configureRequest(HttpRequest request) { request.setHeader(HttpHeaders.AUTHORIZATION, "Bearer " + Secret.toString(token)); } diff --git a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/credentials/BitbucketClientCertificateAuthenticator.java b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/credentials/BitbucketClientCertificateAuthenticator.java index e8199eb32..8fbb23a89 100644 --- a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/credentials/BitbucketClientCertificateAuthenticator.java +++ b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/credentials/BitbucketClientCertificateAuthenticator.java @@ -35,21 +35,29 @@ import java.util.logging.Level; import java.util.logging.Logger; import javax.net.ssl.SSLContext; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.ssl.SSLContextBuilder; -import org.apache.http.ssl.SSLContexts; +import org.apache.hc.client5.http.protocol.HttpClientContext; +import org.apache.hc.client5.http.socket.ConnectionSocketFactory; +import org.apache.hc.client5.http.socket.PlainConnectionSocketFactory; +import org.apache.hc.client5.http.ssl.HttpsSupport; +import org.apache.hc.client5.http.ssl.SSLConnectionSocketFactory; +import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.http.URIScheme; +import org.apache.hc.core5.http.config.Registry; +import org.apache.hc.core5.http.config.RegistryBuilder; +import org.apache.hc.core5.ssl.SSLContextBuilder; +import org.apache.hc.core5.ssl.SSLContexts; /** * Authenticates against Bitbucket using a TLS client certificate */ public class BitbucketClientCertificateAuthenticator implements BitbucketAuthenticator { + private static final Logger LOGGER = Logger.getLogger(BitbucketClientCertificateAuthenticator.class.getName()); + private static final String SOCKET_FACTORY_REGISTRY = "http.socket-factory-registry"; private final String credentialsId; private final KeyStore keyStore; private final Secret password; - private static final Logger LOGGER = Logger.getLogger(BitbucketClientCertificateAuthenticator.class.getName()); - public BitbucketClientCertificateAuthenticator(StandardCertificateCredentials credentials) { this.credentialsId = credentials.getId(); keyStore = credentials.getKeyStore(); @@ -61,9 +69,13 @@ public BitbucketClientCertificateAuthenticator(StandardCertificateCredentials cr * @param builder The client builder. */ @Override - public void configureBuilder(HttpClientBuilder builder) { + public void configureContext(HttpClientContext context, HttpHost host) { try { - builder.setSSLContext(buildSSLContext()); + Registry registry = RegistryBuilder.create() + .register(URIScheme.HTTP.id, PlainConnectionSocketFactory.getSocketFactory()) + .register(URIScheme.HTTPS.id, new SSLConnectionSocketFactory(buildSSLContext(), HttpsSupport.getDefaultHostnameVerifier())) + .build(); + context.setAttribute(SOCKET_FACTORY_REGISTRY, registry); // override SSL registry for this context } catch (NoSuchAlgorithmException | UnrecoverableKeyException | KeyStoreException | KeyManagementException e) { LOGGER.log(Level.WARNING, "Failed to set up SSL context from provided client certificate: " + e.getMessage()); // TODO: handle this error in a way that provides feedback to the user diff --git a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/credentials/BitbucketOAuthAuthenticator.java b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/credentials/BitbucketOAuthAuthenticator.java index 188b4636b..305e0e3e1 100644 --- a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/credentials/BitbucketOAuthAuthenticator.java +++ b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/credentials/BitbucketOAuthAuthenticator.java @@ -39,7 +39,7 @@ import java.io.IOException; import java.util.concurrent.ExecutionException; import jenkins.util.SetContextClassLoader; -import org.apache.http.HttpRequest; +import org.apache.hc.core5.http.HttpRequest; public class BitbucketOAuthAuthenticator implements BitbucketAuthenticator { diff --git a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/credentials/BitbucketUsernamePasswordAuthenticator.java b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/credentials/BitbucketUsernamePasswordAuthenticator.java index cc4fafbf3..a69376bf5 100644 --- a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/credentials/BitbucketUsernamePasswordAuthenticator.java +++ b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/credentials/BitbucketUsernamePasswordAuthenticator.java @@ -30,15 +30,15 @@ import hudson.util.Secret; import java.util.logging.Level; import java.util.logging.Logger; -import org.apache.http.HttpHost; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.AuthCache; -import org.apache.http.client.CredentialsProvider; -import org.apache.http.client.protocol.HttpClientContext; -import org.apache.http.impl.auth.BasicScheme; -import org.apache.http.impl.client.BasicAuthCache; -import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.hc.client5.http.auth.AuthCache; +import org.apache.hc.client5.http.auth.AuthScope; +import org.apache.hc.client5.http.auth.CredentialsStore; +import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; +import org.apache.hc.client5.http.impl.auth.BasicAuthCache; +import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider; +import org.apache.hc.client5.http.impl.auth.BasicScheme; +import org.apache.hc.client5.http.protocol.HttpClientContext; +import org.apache.hc.core5.http.HttpHost; /** * Authenticator that uses a username and password (probably the default) @@ -57,7 +57,7 @@ public class BitbucketUsernamePasswordAuthenticator implements BitbucketAuthenti public BitbucketUsernamePasswordAuthenticator(StandardUsernamePasswordCredentials credentials) { credentialsId = credentials.getId(); String password = Secret.toString(credentials.getPassword()); - httpCredentials = new UsernamePasswordCredentials(credentials.getUsername(), password); + httpCredentials = new UsernamePasswordCredentials(credentials.getUsername(), password.toCharArray()); } /** @@ -68,12 +68,12 @@ public BitbucketUsernamePasswordAuthenticator(StandardUsernamePasswordCredential */ @Override public void configureContext(HttpClientContext context, HttpHost host) { - CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials(new AuthScope(host), httpCredentials); + CredentialsStore credentialsStore = new BasicCredentialsProvider(); + credentialsStore.setCredentials(new AuthScope(host), httpCredentials); AuthCache authCache = new BasicAuthCache(); LOGGER.log(Level.FINE,"Add host={0} to authCache.", host); authCache.put(host, new BasicScheme()); - context.setCredentialsProvider(credentialsProvider); + context.setCredentialsProvider(credentialsStore); context.setAuthCache(authCache); } diff --git a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/util/BitbucketApiUtils.java b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/util/BitbucketApiUtils.java index 74f605397..4573d8d83 100644 --- a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/util/BitbucketApiUtils.java +++ b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/impl/util/BitbucketApiUtils.java @@ -16,12 +16,16 @@ import hudson.util.FormFillFailure; import hudson.util.ListBoxModel; import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; import java.util.logging.Level; import java.util.logging.Logger; import jenkins.authentication.tokens.api.AuthenticationTokens; import jenkins.model.Jenkins; import jenkins.scm.api.SCMSourceOwner; import org.apache.commons.lang.StringUtils; +import org.apache.hc.core5.http.HttpHost; import org.kohsuke.accmod.Restricted; import org.kohsuke.accmod.restrictions.NoExternalUse; @@ -104,4 +108,20 @@ public interface BitbucketSupplier { T get(BitbucketApi bitbucketApi) throws IOException, InterruptedException; } + public static HttpHost toHttpHost(String url) { + String checkedURL = url; + try { + URL tmp = new URL(url); + if (tmp.getProtocol() == null) { + checkedURL = new URL("http", tmp.getHost(), tmp.getPort(), tmp.getFile()).toString(); + } + } catch (MalformedURLException e) { + } + try { + return HttpHost.create(checkedURL); + } catch (URISyntaxException e) { + throw new RuntimeException("Invalid URL " + checkedURL, e); + } + } + } diff --git a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/server/client/BitbucketServerAPIClient.java b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/server/client/BitbucketServerAPIClient.java index 06addc5fb..280ac380d 100644 --- a/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/server/client/BitbucketServerAPIClient.java +++ b/src/main/java/com/cloudbees/jenkins/plugins/bitbucket/server/client/BitbucketServerAPIClient.java @@ -42,6 +42,7 @@ import com.cloudbees.jenkins.plugins.bitbucket.filesystem.BitbucketSCMFile; import com.cloudbees.jenkins.plugins.bitbucket.impl.client.AbstractBitbucketApi; import com.cloudbees.jenkins.plugins.bitbucket.impl.credentials.BitbucketUsernamePasswordAuthenticator; +import com.cloudbees.jenkins.plugins.bitbucket.impl.util.BitbucketApiUtils; import com.cloudbees.jenkins.plugins.bitbucket.impl.util.JsonParser; import com.cloudbees.jenkins.plugins.bitbucket.server.BitbucketServerVersion; import com.cloudbees.jenkins.plugins.bitbucket.server.BitbucketServerWebhookImplementation; @@ -72,8 +73,6 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; -import java.net.MalformedURLException; -import java.net.URL; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; @@ -87,13 +86,13 @@ import jenkins.scm.api.SCMFile; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; -import org.apache.http.HttpHost; -import org.apache.http.HttpStatus; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.conn.HttpClientConnectionManager; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; -import org.apache.http.message.BasicNameValuePair; +import org.apache.hc.client5.http.classic.methods.HttpGet; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager; +import org.apache.hc.client5.http.io.HttpClientConnectionManager; +import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.http.HttpStatus; +import org.apache.hc.core5.http.message.BasicNameValuePair; import static java.util.Objects.requireNonNull; @@ -946,16 +945,7 @@ protected CloseableHttpClient getClient() { @NonNull @Override protected HttpHost getHost() { - String url = baseURL; - try { - // it's really needed? - URL tmp = new URL(baseURL); - if (tmp.getProtocol() == null) { - url = new URL("http", tmp.getHost(), tmp.getPort(), tmp.getFile()).toString(); - } - } catch (MalformedURLException e) { - } - return HttpHost.create(url); + return BitbucketApiUtils.toHttpHost(this.baseURL); } @Override diff --git a/src/test/java/com/cloudbees/jenkins/plugins/bitbucket/impl/client/ExponentialBackOffServiceUnavailableRetryStrategyTest.java b/src/test/java/com/cloudbees/jenkins/plugins/bitbucket/impl/client/ExponentialBackOffServiceUnavailableRetryStrategyTest.java index 8766175ab..5f3374ae0 100644 --- a/src/test/java/com/cloudbees/jenkins/plugins/bitbucket/impl/client/ExponentialBackOffServiceUnavailableRetryStrategyTest.java +++ b/src/test/java/com/cloudbees/jenkins/plugins/bitbucket/impl/client/ExponentialBackOffServiceUnavailableRetryStrategyTest.java @@ -70,7 +70,7 @@ void test_retry(ClientAndServer mockServer) throws Exception { protected HttpClientBuilder setupClientBuilder(String host) { return super.setupClientBuilder(host) .disableAutomaticRetries() - .setServiceUnavailableRetryStrategy(new ExponentialBackOffServiceUnavailableRetryStrategy(2, 5, 100)) + .setServiceUnavailableRetryStrategy(new ExponentialBackOffRetryStrategy(2, 5, 100)) .addInterceptorFirst(counterInterceptor); } }) {