From 1618d8ffa9bce8bf7f68eb1e0d223308ff2c20cc Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Thu, 23 Jan 2025 12:17:11 -0800 Subject: [PATCH 01/11] no stack trace for known errors Signed-off-by: Maxwell Brown --- .../plugins/source/jira/rest/JiraRestClient.java | 8 ++++---- .../plugins/source/jira/rest/auth/JiraOauthConfig.java | 2 +- .../plugins/source/jira/utils/AddressValidation.java | 2 +- .../coordination/partition/LeaderPartition.java | 2 +- .../coordination/scheduler/WorkerScheduler.java | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java index 92420ac319..c743251221 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java @@ -119,19 +119,19 @@ private ResponseEntity invokeRestApi(URI uri, Class responseType) thro } catch (HttpClientErrorException ex) { HttpStatus statusCode = ex.getStatusCode(); String statusMessage = ex.getMessage(); - log.error("An exception has occurred while getting response from Jira search API {}", ex.getMessage()); + log.error("An exception has occurred while getting response from Jira search API with statusCode {} and error message {}.", statusCode, statusMessage); if (statusCode == HttpStatus.FORBIDDEN) { throw new UnAuthorizedException(statusMessage); } else if (statusCode == HttpStatus.UNAUTHORIZED) { - log.error(NOISY, "Token expired. We will try to renew the tokens now", ex); + log.error(NOISY, "Token expired. We will try to renew the tokens now."); authConfig.renewCredentials(); } else if (statusCode == HttpStatus.TOO_MANY_REQUESTS) { - log.error(NOISY, "Hitting API rate limit. Backing off with sleep timer.", ex); + log.error(NOISY, "Hitting API rate limit. Backing off with sleep timer."); } try { Thread.sleep((long) RETRY_ATTEMPT_SLEEP_TIME.get(retryCount) * sleepTimeMultiplier); } catch (InterruptedException e) { - throw new RuntimeException("Sleep in the retry attempt got interrupted", e); + throw new RuntimeException("Sleep in the retry attempt got interrupted."); } } retryCount++; diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraOauthConfig.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraOauthConfig.java index ddcf1c8468..15ecf5e2fa 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraOauthConfig.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraOauthConfig.java @@ -100,7 +100,7 @@ public String getJiraAccountCloudId() { if (e.getRawStatusCode() == HttpStatus.UNAUTHORIZED.value()) { renewCredentials(); } - log.error("Error occurred while accessing resources: ", e); + log.error("Error occurred while accessing resources. Status code: {}. Error message: {}", e.getStatusCode(), e.getMessage()); } } throw new UnAuthorizedException(String.format("Access token expired. Unable to renew even after %s attempts", RETRY_ATTEMPT)); diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/AddressValidation.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/AddressValidation.java index d6cc166226..7572723df9 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/AddressValidation.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/AddressValidation.java @@ -38,7 +38,7 @@ public static InetAddress getInetAddress(String url) { try { return InetAddress.getByName(new URL(url).getHost()); } catch (UnknownHostException | MalformedURLException e) { - log.error(INVALID_URL, e); + log.error(INVALID_URL + " : {}", url); throw new BadRequestException(e.getMessage(), e); } } diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/coordination/partition/LeaderPartition.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/coordination/partition/LeaderPartition.java index a54e50d36f..5a0397c3b1 100644 --- a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/coordination/partition/LeaderPartition.java +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/coordination/partition/LeaderPartition.java @@ -75,7 +75,7 @@ public LeaderProgressState convertToPartitionState(final String serializedPartit try { return objectMapper.readValue(serializedPartitionProgressState, LeaderProgressState.class); } catch (final JsonProcessingException e) { - LOG.error("Unable to convert string to partition progress state class ", e); + LOG.error("Unable to convert string to partition progress state class due to {}", e.getMessage()); return null; } } diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/coordination/scheduler/WorkerScheduler.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/coordination/scheduler/WorkerScheduler.java index f2fc7e4b40..e738c0e19c 100644 --- a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/coordination/scheduler/WorkerScheduler.java +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/coordination/scheduler/WorkerScheduler.java @@ -89,7 +89,7 @@ public void run() { try { Thread.sleep(RETRY_BACKOFF_ON_EXCEPTION_MILLIS); } catch (InterruptedException ex) { - log.warn("Thread interrupted while waiting to retry", ex); + log.warn("Thread interrupted while waiting to retry due to {}", ex.getMessage()); } } } From f7fa34839910b59b056c96736fdaea3218c24051 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Thu, 23 Jan 2025 12:29:40 -0800 Subject: [PATCH 02/11] add else block for error handling in jirarestclient Signed-off-by: Maxwell Brown --- .../plugins/source/jira/rest/JiraRestClient.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java index c743251221..b78a58f1db 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java @@ -119,14 +119,16 @@ private ResponseEntity invokeRestApi(URI uri, Class responseType) thro } catch (HttpClientErrorException ex) { HttpStatus statusCode = ex.getStatusCode(); String statusMessage = ex.getMessage(); - log.error("An exception has occurred while getting response from Jira search API with statusCode {} and error message {}.", statusCode, statusMessage); + log.error("An exception has occurred while getting response from Jira search API with statusCode {} and error message: {}", statusCode, statusMessage); if (statusCode == HttpStatus.FORBIDDEN) { throw new UnAuthorizedException(statusMessage); } else if (statusCode == HttpStatus.UNAUTHORIZED) { - log.error(NOISY, "Token expired. We will try to renew the tokens now."); + log.error("Token expired. We will try to renew the tokens now."); authConfig.renewCredentials(); } else if (statusCode == HttpStatus.TOO_MANY_REQUESTS) { - log.error(NOISY, "Hitting API rate limit. Backing off with sleep timer."); + log.error("Hitting API rate limit. Backing off with sleep timer."); + } else { + log.error(NOISY, "Exception: ", ex); } try { Thread.sleep((long) RETRY_ATTEMPT_SLEEP_TIME.get(retryCount) * sleepTimeMultiplier); From 813d53ec8b55e04f18225c622ef9d52e563f0f5b Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Thu, 23 Jan 2025 13:29:24 -0800 Subject: [PATCH 03/11] print partition progress streate as well Signed-off-by: Maxwell Brown --- .../source_crawler/coordination/partition/LeaderPartition.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/coordination/partition/LeaderPartition.java b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/coordination/partition/LeaderPartition.java index 5a0397c3b1..4a013e9373 100644 --- a/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/coordination/partition/LeaderPartition.java +++ b/data-prepper-plugins/saas-source-plugins/source-crawler/src/main/java/org/opensearch/dataprepper/plugins/source/source_crawler/coordination/partition/LeaderPartition.java @@ -75,7 +75,7 @@ public LeaderProgressState convertToPartitionState(final String serializedPartit try { return objectMapper.readValue(serializedPartitionProgressState, LeaderProgressState.class); } catch (final JsonProcessingException e) { - LOG.error("Unable to convert string to partition progress state class due to {}", e.getMessage()); + LOG.error("Unable to convert string to partition progress state class due to {}. Partition progress state string: {}.", e.getMessage(), serializedPartitionProgressState); return null; } } From 7edf388cb0da6c53a4cfbb4f502868238910c316 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Thu, 23 Jan 2025 17:21:32 -0800 Subject: [PATCH 04/11] me when i address comments or something Signed-off-by: Maxwell Brown --- .../plugins/source/jira/rest/JiraRestClient.java | 8 +++++++- .../plugins/source/jira/utils/AddressValidation.java | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java index b78a58f1db..fc2369117f 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java @@ -127,14 +127,20 @@ private ResponseEntity invokeRestApi(URI uri, Class responseType) thro authConfig.renewCredentials(); } else if (statusCode == HttpStatus.TOO_MANY_REQUESTS) { log.error("Hitting API rate limit. Backing off with sleep timer."); + } else if (statusCode == HttpStatus.SERVICE_UNAVAILABLE) { + log.error("Service unavailable. Will retry after backing off with sleep timer."); + } else if (statusCode == HttpStatus.GATEWAY_TIMEOUT) { + log.error("Gateway timeout. Will retry after backing off with sleep timer."); } else { - log.error(NOISY, "Exception: ", ex); + log.error(NOISY, "Received an unexpected status code {} response from Jira.", statusCode, ex); } try { Thread.sleep((long) RETRY_ATTEMPT_SLEEP_TIME.get(retryCount) * sleepTimeMultiplier); } catch (InterruptedException e) { throw new RuntimeException("Sleep in the retry attempt got interrupted."); } + } catch (Exception ex) { + log.error(NOISY, "An exception has occurred while getting a response from the Jira search API", ex); } retryCount++; } diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/AddressValidation.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/AddressValidation.java index 7572723df9..e82acb2a07 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/AddressValidation.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/AddressValidation.java @@ -38,7 +38,7 @@ public static InetAddress getInetAddress(String url) { try { return InetAddress.getByName(new URL(url).getHost()); } catch (UnknownHostException | MalformedURLException e) { - log.error(INVALID_URL + " : {}", url); + log.error("{}: {}", INVALID_URL, url); throw new BadRequestException(e.getMessage(), e); } } From e9f389ecdda06d11dd9c3b40e1f71aaf6a0a598a Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Fri, 24 Jan 2025 16:30:38 -0800 Subject: [PATCH 05/11] remove unused metric Signed-off-by: Maxwell Brown --- .../dataprepper/plugins/source/jira/rest/JiraRestClient.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java index fc2369117f..3db208a50a 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java @@ -51,13 +51,11 @@ public class JiraRestClient { public static final List RETRY_ATTEMPT_SLEEP_TIME = List.of(1, 2, 5, 10, 20, 40); private static final String TICKET_FETCH_LATENCY_TIMER = "ticketFetchLatency"; private static final String SEARCH_CALL_LATENCY_TIMER = "searchCallLatency"; - private static final String PROJECTS_FETCH_LATENCY_TIMER = "projectFetchLatency"; private static final String ISSUES_REQUESTED = "issuesRequested"; private final RestTemplate restTemplate; private final JiraAuthConfig authConfig; private final Timer ticketFetchLatencyTimer; private final Timer searchCallLatencyTimer; - private final Timer projectFetchLatencyTimer; private final Counter issuesRequestedCounter; private final PluginMetrics jiraPluginMetrics = PluginMetrics.fromNames("jiraRestClient", "aws"); private int sleepTimeMultiplier = 1000; @@ -68,8 +66,6 @@ public JiraRestClient(RestTemplate restTemplate, JiraAuthConfig authConfig) { ticketFetchLatencyTimer = jiraPluginMetrics.timer(TICKET_FETCH_LATENCY_TIMER); searchCallLatencyTimer = jiraPluginMetrics.timer(SEARCH_CALL_LATENCY_TIMER); - projectFetchLatencyTimer = jiraPluginMetrics.timer(PROJECTS_FETCH_LATENCY_TIMER); - issuesRequestedCounter = jiraPluginMetrics.counter(ISSUES_REQUESTED); } From a8b00f163d732e0fbf66ea483fec2de7a9e4756b Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Fri, 24 Jan 2025 16:32:32 -0800 Subject: [PATCH 06/11] ensure hosts is list of length 1 Signed-off-by: Maxwell Brown --- .../dataprepper/plugins/source/jira/JiraSourceConfig.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraSourceConfig.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraSourceConfig.java index ed873fc49b..ba456e1e5e 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraSourceConfig.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraSourceConfig.java @@ -12,6 +12,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import jakarta.validation.Valid; +import jakarta.validation.constraints.AssertTrue; import lombok.Getter; import org.opensearch.dataprepper.plugins.source.jira.configuration.AuthenticationConfig; import org.opensearch.dataprepper.plugins.source.jira.configuration.FilterConfig; @@ -31,6 +32,11 @@ public class JiraSourceConfig implements CrawlerSourceConfig { @JsonProperty("hosts") private List hosts; + @AssertTrue(message = "Jira hosts must be a list of length 1") + boolean isValidHosts() { + return hosts != null && hosts.size() == 1; + } + /** * Authentication Config to Access Jira */ From 9a7c8f093233cb7546f729bdc321d9ba2e7c668f Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Wed, 29 Jan 2025 14:15:01 -0800 Subject: [PATCH 07/11] change error to info log Signed-off-by: Maxwell Brown --- .../opensearch/dataprepper/plugins/source/jira/JiraService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraService.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraService.java index 4600d1bdeb..415c4d54c8 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraService.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraService.java @@ -166,7 +166,7 @@ private StringBuilder createIssueFilterCriteria(JiraSourceConfig configuration, .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX))) .append(CLOSING_ROUND_BRACKET); } - log.error("Created issue filter criteria JiraQl query: {}", jiraQl); + log.info("Created issue filter criteria JiraQl query: {}", jiraQl); return jiraQl; } From 275309a05b030c84921b60b4df9008323ad159b2 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Wed, 29 Jan 2025 14:53:43 -0800 Subject: [PATCH 08/11] NOISY flag for logs Signed-off-by: Maxwell Brown --- .../plugins/source/jira/rest/JiraRestClient.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java index 3db208a50a..b2fbb91a27 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java @@ -115,18 +115,18 @@ private ResponseEntity invokeRestApi(URI uri, Class responseType) thro } catch (HttpClientErrorException ex) { HttpStatus statusCode = ex.getStatusCode(); String statusMessage = ex.getMessage(); - log.error("An exception has occurred while getting response from Jira search API with statusCode {} and error message: {}", statusCode, statusMessage); + log.error(NOISY, "An exception has occurred while getting response from Jira search API with statusCode {} and error message: {}", statusCode, statusMessage); if (statusCode == HttpStatus.FORBIDDEN) { throw new UnAuthorizedException(statusMessage); } else if (statusCode == HttpStatus.UNAUTHORIZED) { - log.error("Token expired. We will try to renew the tokens now."); + log.error(NOISY, "Token expired. We will try to renew the tokens now."); authConfig.renewCredentials(); } else if (statusCode == HttpStatus.TOO_MANY_REQUESTS) { - log.error("Hitting API rate limit. Backing off with sleep timer."); + log.error(NOISY, "Hitting API rate limit. Backing off with sleep timer."); } else if (statusCode == HttpStatus.SERVICE_UNAVAILABLE) { - log.error("Service unavailable. Will retry after backing off with sleep timer."); + log.error(NOISY, "Service unavailable. Will retry after backing off with sleep timer."); } else if (statusCode == HttpStatus.GATEWAY_TIMEOUT) { - log.error("Gateway timeout. Will retry after backing off with sleep timer."); + log.error(NOISY, "Gateway timeout. Will retry after backing off with sleep timer."); } else { log.error(NOISY, "Received an unexpected status code {} response from Jira.", statusCode, ex); } From ff1dce82ebebb439ea585dd54238fa916586c35c Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Wed, 29 Jan 2025 16:26:11 -0800 Subject: [PATCH 09/11] sleep timer logging and secret refresh error and logging changes Signed-off-by: Maxwell Brown --- .../plugins/source/jira/rest/JiraRestClient.java | 6 +++--- .../plugins/source/jira/rest/auth/JiraOauthConfig.java | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java index b2fbb91a27..daa5babbb6 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java @@ -122,11 +122,11 @@ private ResponseEntity invokeRestApi(URI uri, Class responseType) thro log.error(NOISY, "Token expired. We will try to renew the tokens now."); authConfig.renewCredentials(); } else if (statusCode == HttpStatus.TOO_MANY_REQUESTS) { - log.error(NOISY, "Hitting API rate limit. Backing off with sleep timer."); + log.error(NOISY, "Hitting API rate limit. Backing off with sleep timer for {} seconds.", RETRY_ATTEMPT_SLEEP_TIME.get(retryCount)); } else if (statusCode == HttpStatus.SERVICE_UNAVAILABLE) { - log.error(NOISY, "Service unavailable. Will retry after backing off with sleep timer."); + log.error(NOISY, "Service unavailable. Will retry after backing off with sleep timer for {} seconds.", RETRY_ATTEMPT_SLEEP_TIME.get(retryCount)); } else if (statusCode == HttpStatus.GATEWAY_TIMEOUT) { - log.error(NOISY, "Gateway timeout. Will retry after backing off with sleep timer."); + log.error(NOISY, "Gateway timeout. Will retry after backing off with sleep timer for {} seconds.", RETRY_ATTEMPT_SLEEP_TIME.get(retryCount)); } else { log.error(NOISY, "Received an unexpected status code {} response from Jira.", statusCode, ex); } diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraOauthConfig.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraOauthConfig.java index 15ecf5e2fa..514066afd2 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraOauthConfig.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraOauthConfig.java @@ -153,6 +153,8 @@ public void renewCredentials() { this.accessToken = (String) oauth2Config.getAccessToken().getValue(); this.refreshToken = (String) oauth2Config.getRefreshToken().getValue(); this.expireTime = Instant.now().plusSeconds(10); + log.info("Access Token and Refresh Token pair is now refreshed. Corresponding Secret store key updated."); + return; } throw new RuntimeException("Failed to renew access token message:" + ex.getMessage(), ex); } From 70411df1d18353351235ce2d0bae44b8af31b8ff Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Thu, 30 Jan 2025 09:26:35 -0800 Subject: [PATCH 10/11] address comments Signed-off-by: Maxwell Brown --- .../plugins/source/jira/rest/JiraRestClient.java | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java index daa5babbb6..28a71f55ee 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java @@ -119,14 +119,10 @@ private ResponseEntity invokeRestApi(URI uri, Class responseType) thro if (statusCode == HttpStatus.FORBIDDEN) { throw new UnAuthorizedException(statusMessage); } else if (statusCode == HttpStatus.UNAUTHORIZED) { - log.error(NOISY, "Token expired. We will try to renew the tokens now."); + log.warn(NOISY, "Token expired. We will try to renew the tokens now."); authConfig.renewCredentials(); - } else if (statusCode == HttpStatus.TOO_MANY_REQUESTS) { - log.error(NOISY, "Hitting API rate limit. Backing off with sleep timer for {} seconds.", RETRY_ATTEMPT_SLEEP_TIME.get(retryCount)); - } else if (statusCode == HttpStatus.SERVICE_UNAVAILABLE) { - log.error(NOISY, "Service unavailable. Will retry after backing off with sleep timer for {} seconds.", RETRY_ATTEMPT_SLEEP_TIME.get(retryCount)); - } else if (statusCode == HttpStatus.GATEWAY_TIMEOUT) { - log.error(NOISY, "Gateway timeout. Will retry after backing off with sleep timer for {} seconds.", RETRY_ATTEMPT_SLEEP_TIME.get(retryCount)); + } else if (statusCode == HttpStatus.TOO_MANY_REQUESTS || statusCode == HttpStatus.SERVICE_UNAVAILABLE || statusCode == HttpStatus.GATEWAY_TIMEOUT) { + log.error(NOISY, "Received {}. Backing off with sleep timer for {} seconds.", statusCode, RETRY_ATTEMPT_SLEEP_TIME.get(retryCount)); } else { log.error(NOISY, "Received an unexpected status code {} response from Jira.", statusCode, ex); } From aa7947f82e885a0438feb1ff643064f902a46db7 Mon Sep 17 00:00:00 2001 From: Maxwell Brown Date: Thu, 30 Jan 2025 10:02:21 -0800 Subject: [PATCH 11/11] fix refresh because test failing Signed-off-by: Maxwell Brown --- .../plugins/source/jira/rest/auth/JiraOauthConfig.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraOauthConfig.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraOauthConfig.java index 514066afd2..9640ba3818 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraOauthConfig.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraOauthConfig.java @@ -153,8 +153,7 @@ public void renewCredentials() { this.accessToken = (String) oauth2Config.getAccessToken().getValue(); this.refreshToken = (String) oauth2Config.getRefreshToken().getValue(); this.expireTime = Instant.now().plusSeconds(10); - log.info("Access Token and Refresh Token pair is now refreshed. Corresponding Secret store key updated."); - return; + log.info("Access Token and Refresh Token pair is now refreshed."); } throw new RuntimeException("Failed to renew access token message:" + ex.getMessage(), ex); }